create-windy 0.5.1 → 0.6.0

package/template/apps/server/src/report-staging/drizzle-repository.ts

@@ -0,0 +1,255 @@
+import {
+  reportStagingClaims,
+  reportStagingEntries,
+  reportStagingGroups,
+  reportStagingSources,
+} from "@southwind-ai/database";
+import { and, asc, eq, gt } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import type {
+  ReportStagingGroupRecord,
+  ReportStagingReadLeaseRecord,
+  ReportStagingSourceRecord,
+  SealedReportStagingGroup,
+} from "./contracts.js";
+import {
+  mapReportClaim,
+  mapReportEntry,
+  mapReportGroup,
+  mapReportSource,
+  reportGroupValues,
+  reportSourceValues,
+} from "./drizzle-mapping.js";
+import {
+  acquireDrizzleReportLease,
+  closeDrizzleReportLease,
+  failDrizzleReportGroup,
+  getDrizzleReportLease,
+  listDrizzleReportPurgeCandidates,
+  markDrizzleReportDiscarded,
+  markDrizzleReportDiscarding,
+} from "./drizzle-lifecycle.js";
+import { sealDrizzleReportGroup } from "./drizzle-seal.js";
+import type { ReportStagingRepository } from "./repository.js";
+
+export class DrizzleReportStagingRepository implements ReportStagingRepository {
+  constructor(private readonly db: NodePgDatabase) {}
+
+  async createOrGetGroup(
+    group: ReportStagingGroupRecord,
+    sources: readonly ReportStagingSourceRecord[],
+  ) {
+    return this.db.transaction(async (tx) => {
+      const [created] = await tx
+        .insert(reportStagingGroups)
+        .values(reportGroupValues(group))
+        .onConflictDoNothing()
+        .returning();
+      if (created) {
+        for (let offset = 0; offset < sources.length; offset += 1_000) {
+          await tx
+            .insert(reportStagingSources)
+            .values(
+              sources.slice(offset, offset + 1_000).map(reportSourceValues),
+            );
+        }
+        return { group: mapReportGroup(created), created: true };
+      }
+      const [existing] = await tx
+        .select()
+        .from(reportStagingGroups)
+        .where(
+          and(
+            eq(reportStagingGroups.moduleName, group.moduleName),
+            eq(reportStagingGroups.ownerId, group.ownerId),
+            eq(reportStagingGroups.scopeHash, group.scopeHash),
+            eq(reportStagingGroups.purpose, group.purpose),
+            eq(reportStagingGroups.idempotencyHash, group.idempotencyHash),
+          ),
+        )
+        .limit(1);
+      if (!existing) throw new Error("报告暂存并发创建后无法读取");
+      return { group: mapReportGroup(existing), created: false };
+    });
+  }
+
+  async getGroup(groupId: string) {
+    const [row] = await this.db
+      .select()
+      .from(reportStagingGroups)
+      .where(eq(reportStagingGroups.groupId, groupId))
+      .limit(1);
+    return row ? mapReportGroup(row) : undefined;
+  }
+
+  async getSources(groupId: string) {
+    return (
+      await this.db
+        .select()
+        .from(reportStagingSources)
+        .where(eq(reportStagingSources.groupId, groupId))
+        .orderBy(asc(reportStagingSources.ordinal))
+    ).map(mapReportSource);
+  }
+
+  async getEntry(groupId: string, entryId: string) {
+    const [row] = await this.db
+      .select()
+      .from(reportStagingEntries)
+      .where(
+        and(
+          eq(reportStagingEntries.groupId, groupId),
+          eq(reportStagingEntries.entryId, entryId),
+        ),
+      )
+      .limit(1);
+    return row ? mapReportEntry(row) : undefined;
+  }
+
+  async listEntries(
+    groupId: string,
+    afterOrdinal: number | undefined,
+    limit: number,
+  ) {
+    return (
+      await this.db
+        .select()
+        .from(reportStagingEntries)
+        .where(
+          and(
+            eq(reportStagingEntries.groupId, groupId),
+            afterOrdinal === undefined
+              ? undefined
+              : gt(reportStagingEntries.ordinal, afterOrdinal),
+          ),
+        )
+        .orderBy(asc(reportStagingEntries.ordinal))
+        .limit(limit)
+    ).map(mapReportEntry);
+  }
+
+  async acquireClaim(
+    groupId: string,
+    tokenHash: string,
+    now: string,
+    expiresAt: string,
+  ) {
+    return this.db.transaction(async (tx) => {
+      const [group] = await tx
+        .select()
+        .from(reportStagingGroups)
+        .where(eq(reportStagingGroups.groupId, groupId))
+        .limit(1)
+        .for("update");
+      if (
+        !group ||
+        group.status !== "indexing" ||
+        group.expiresAt <= new Date(now)
+      ) {
+        return undefined;
+      }
+      const [existing] = await tx
+        .select()
+        .from(reportStagingClaims)
+        .where(eq(reportStagingClaims.groupId, groupId))
+        .limit(1)
+        .for("update");
+      if (existing && existing.expiresAt > new Date(now)) return undefined;
+      const values = {
+        groupId,
+        tokenHash,
+        generation: (existing?.generation ?? 0) + 1,
+        expiresAt: new Date(expiresAt),
+        createdAt: existing?.createdAt ?? new Date(now),
+        updatedAt: new Date(now),
+      };
+      const [claimed] = await tx
+        .insert(reportStagingClaims)
+        .values(values)
+        .onConflictDoUpdate({
+          target: reportStagingClaims.groupId,
+          set: values,
+        })
+        .returning();
+      return claimed ? mapReportClaim(claimed) : undefined;
+    });
+  }
+
+  async renewClaim(
+    groupId: string,
+    tokenHash: string,
+    now: string,
+    expiresAt: string,
+  ) {
+    const rows = await this.db
+      .update(reportStagingClaims)
+      .set({ expiresAt: new Date(expiresAt), updatedAt: new Date(now) })
+      .where(
+        and(
+          eq(reportStagingClaims.groupId, groupId),
+          eq(reportStagingClaims.tokenHash, tokenHash),
+          gt(reportStagingClaims.expiresAt, new Date(now)),
+        ),
+      )
+      .returning({ groupId: reportStagingClaims.groupId });
+    return rows.length === 1;
+  }
+
+  async releaseClaim(groupId: string, tokenHash: string) {
+    await this.db
+      .delete(reportStagingClaims)
+      .where(
+        and(
+          eq(reportStagingClaims.groupId, groupId),
+          eq(reportStagingClaims.tokenHash, tokenHash),
+        ),
+      );
+  }
+
+  async sealGroup(
+    groupId: string,
+    tokenHash: string,
+    sealed: SealedReportStagingGroup,
+  ) {
+    return sealDrizzleReportGroup(this.db, groupId, tokenHash, sealed);
+  }
+
+  async failGroup(
+    groupId: string,
+    tokenHash: string,
+    failureCode: string,
+    updatedAt: string,
+  ) {
+    return failDrizzleReportGroup(
+      this.db,
+      groupId,
+      tokenHash,
+      failureCode,
+      updatedAt,
+    );
+  }
+
+  async acquireReadLease(lease: ReportStagingReadLeaseRecord, now: string) {
+    return acquireDrizzleReportLease(this.db, lease, now);
+  }
+
+  async getReadLease(groupId: string, leaseId: string) {
+    return getDrizzleReportLease(this.db, groupId, leaseId);
+  }
+
+  async closeReadLease(groupId: string, leaseId: string, closedAt: string) {
+    return closeDrizzleReportLease(this.db, groupId, leaseId, closedAt);
+  }
+
+  async markDiscarding(groupId: string, now: string) {
+    return markDrizzleReportDiscarding(this.db, groupId, now);
+  }
+
+  async markDiscarded(groupId: string, now: string) {
+    return markDrizzleReportDiscarded(this.db, groupId, now);
+  }
+
+  async listPurgeCandidates(now: string, limit: number) {
+    return listDrizzleReportPurgeCandidates(this.db, now, limit);
+  }
+}

package/template/apps/server/src/report-staging/drizzle-seal.integration.test.ts

@@ -0,0 +1,267 @@
+import { createHash } from "node:crypto";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { migrate } from "drizzle-orm/node-postgres/migrator";
+import { Pool } from "pg";
+import { DrizzleReportStagingRepository } from "./drizzle-repository.js";
+import { ReportStagingEntryCatalog } from "./metadata-spool.js";
+import { computeManifestSha256 } from "./service-input.js";
+
+const databaseUrl = process.env.STORAGE_TEST_DATABASE_URL;
+const describePostgres = databaseUrl ? describe : describe.skip;
+const groupIds = [opaqueId("5"), opaqueId("6"), opaqueId("7")];
+const roots: string[] = [];
+const catalogs: ReportStagingEntryCatalog[] = [];
+const emptySha = createHash("sha256").update("").digest("hex");
+
+describePostgres("Report Staging PostgreSQL catalog batch seal", () => {
+  let pool: Pool;
+  let repository: DrizzleReportStagingRepository;
+
+  beforeAll(async () => {
+    pool = new Pool({ connectionString: databaseUrl });
+    await migrate(drizzle(pool), {
+      migrationsFolder: resolve(
+        import.meta.dir,
+        "../../../../packages/database/drizzle",
+      ),
+      migrationsSchema: "public",
+      migrationsTable: "__storage_test_migrations",
+    });
+    await cleanupGroups(pool);
+    repository = new DrizzleReportStagingRepository(drizzle(pool));
+  });
+
+  afterAll(async () => {
+    catalogs.forEach((catalog) => catalog.close());
+    await cleanupGroups(pool);
+    await Promise.all(
+      roots.map((root) => rm(root, { recursive: true, force: true })),
+    );
+    await pool?.query(
+      "drop trigger if exists report_staging_test_delay on report_staging_entries",
+    );
+    await pool?.query("drop function if exists report_staging_test_delay() ");
+    await pool?.end();
+  });
+
+  test("2501 entries 以三批提交且不物化全量数组", async () => {
+    const id = groupIds[0]!;
+    const sealed = await prepare(id, "5", 2_501);
+    await createAndClaim(repository, id, "5");
+    await expect(
+      repository.sealGroup(id, token("5"), sealed),
+    ).resolves.toMatchObject({ status: "ready", entryCount: 2_501 });
+    expect(await entryCount(pool, id)).toBe(2_501);
+  });
+
+  test("第二批唯一键失败会回滚第一批与 source receipt", async () => {
+    const id = groupIds[1]!;
+    const sealed = await prepare(id, "6", 2_501);
+    await createAndClaim(repository, id, "6");
+    const conflict = sealed.catalog.listEntries(1_499, 1)[0]!;
+    expect(conflict.ordinal).toBe(1_500);
+    await pool.query(
+      "insert into report_staging_entries(entry_id, group_id, ordinal, relative_path, path_key, byte_size, sha256, created_at) values ($1, $2, 999999, 'existing', $3, 0, $4, now())",
+      [conflict.entryId, id, hash("a"), emptySha],
+    );
+    await expect(
+      repository.sealGroup(id, token("6"), sealed),
+    ).rejects.toMatchObject({
+      cause: {
+        code: "23505",
+        constraint: "report_staging_entries_pkey",
+      },
+    });
+    expect(await entryCount(pool, id)).toBe(1);
+    expect(await repository.getGroup(id)).toMatchObject({ status: "indexing" });
+    expect((await repository.getSources(id))[0]?.byteSize).toBeUndefined();
+  });
+
+  test("clock_timestamp 在三批插入后发现 claim 到期并回滚", async () => {
+    const id = groupIds[2]!;
+    const sealed = await prepare(id, "7", 2_501);
+    await createAndClaim(repository, id, "7");
+    await pool.query(`
+      create or replace function report_staging_test_delay()
+      returns trigger language plpgsql as $$
+      begin perform pg_sleep(0.1); return null; end $$;
+      create trigger report_staging_test_delay
+      before insert on report_staging_entries
+      for each statement execute function report_staging_test_delay();
+    `);
+    try {
+      await pool.query(
+        "update report_staging_claims set expires_at = clock_timestamp() + interval '250 milliseconds' where group_id = $1",
+        [id],
+      );
+      await expect(
+        repository.sealGroup(id, token("7"), sealed),
+      ).resolves.toBeUndefined();
+      expect(await entryCount(pool, id)).toBe(0);
+      expect(await repository.getGroup(id)).toMatchObject({
+        status: "indexing",
+      });
+      expect((await repository.getSources(id))[0]?.byteSize).toBeUndefined();
+    } finally {
+      await pool.query(
+        "drop trigger if exists report_staging_test_delay on report_staging_entries",
+      );
+      await pool.query("drop function if exists report_staging_test_delay() ");
+    }
+  });
+});
+
+async function prepare(groupId: string, discriminator: string, count: number) {
+  const root = await mkdtemp(join(tmpdir(), "windy-seal-catalog-"));
+  roots.push(root);
+  const catalog = await ReportStagingEntryCatalog.create({
+    path: join(root, "index.sqlite"),
+    groupId,
+    policy: { ...reportPolicy(), maxEntries: count },
+  });
+  catalogs.push(catalog);
+  for (let ordinal = 0; ordinal < count; ordinal++) {
+    catalog.declareArchiveEntry(`f${ordinal}`, false, 0);
+  }
+  catalog.finishArchivePass(count, 0);
+  for (let ordinal = 0; ordinal < count; ordinal++) {
+    catalog.markScanned({
+      relativePath: `f${ordinal}`,
+      directory: false,
+      byteSize: 0,
+      sha256: emptySha,
+      ordinal,
+    });
+  }
+  catalog.finishScan(instant(1));
+  const sources = [
+    { ...source(groupId, discriminator), byteSize: 13, sha256: hash("b") },
+  ];
+  return {
+    sealId: token(discriminator),
+    sources,
+    catalog,
+    archiveByteSize: 13,
+    expandedByteSize: 0,
+    manifestSha256: computeManifestSha256(
+      group(groupId, discriminator),
+      sources,
+      catalog,
+    ),
+    updatedAt: instant(1),
+  };
+}
+
+async function createAndClaim(
+  repository: DrizzleReportStagingRepository,
+  id: string,
+  discriminator: string,
+) {
+  await repository.createOrGetGroup(group(id, discriminator), [
+    source(id, discriminator),
+  ]);
+  await repository.acquireClaim(
+    id,
+    token(discriminator),
+    instant(0),
+    instant(120),
+  );
+}
+
+function group(groupId: string, discriminator: string) {
+  return {
+    groupId,
+    moduleName: "evidence",
+    ownerId: "user-1",
+    scopeHash: hash("1"),
+    purpose: "evidence.report-staging",
+    sourceMode: "archive" as const,
+    idempotencyHash: hash(discriminator),
+    fingerprint: hash("2"),
+    policyDigest: hash("3"),
+    entrypoint: "report.rar",
+    status: "indexing" as const,
+    sourceCount: 1,
+    entryCount: 0,
+    archiveByteSize: 0,
+    expandedByteSize: 0,
+    revision: 1,
+    expiresAt: instant(300),
+    createdAt: instant(0),
+    updatedAt: instant(0),
+  };
+}
+
+function source(groupId: string, discriminator: string) {
+  return {
+    sourceId: `rss_${discriminator.repeat(32)}`,
+    groupId,
+    ordinal: 0,
+    fileId: "file-1",
+    filePurpose: "evidence.original",
+    resourceType: "evidence.file",
+    resourceId: "evidence-1",
+    relativePath: "report.rar",
+    createdAt: instant(0),
+  };
+}
+
+function reportPolicy() {
+  return {
+    purpose: "evidence.report-staging",
+    sourcePurposes: ["evidence.original"],
+    retentionMs: 300_000,
+    claimTtlMs: 120_000,
+    maxReadLeaseMs: 10_000,
+    maxProcessMs: 10_000,
+    maxSources: 1,
+    maxArchiveBytes: 1_024,
+    maxExpandedBytes: 1,
+    maxEntryBytes: 1,
+    maxEntries: 2_501,
+    maxCompressionRatio: 100,
+    maxPathDepth: 1,
+  };
+}
+
+async function entryCount(pool: Pool, groupId: string) {
+  const result = await pool.query<{ count: string }>(
+    "select count(*)::text as count from report_staging_entries where group_id = $1",
+    [groupId],
+  );
+  return Number(result.rows[0]?.count ?? 0);
+}
+
+async function cleanupGroups(pool: Pool) {
+  for (const table of [
+    "report_staging_read_leases",
+    "report_staging_claims",
+    "report_staging_entries",
+    "report_staging_sources",
+    "report_staging_groups",
+  ]) {
+    await pool.query(
+      `delete from ${table} where group_id = any($1::varchar[])`,
+      [groupIds],
+    );
+  }
+}
+
+function opaqueId(value: string) {
+  return `rsg_${value.repeat(32)}`;
+}
+function hash(value: string) {
+  return value.repeat(64);
+}
+function token(value: string) {
+  return hash(value);
+}
+function instant(seconds: number) {
+  return new Date(
+    Date.parse("2036-08-05T00:00:00.000Z") + seconds * 1_000,
+  ).toISOString();
+}

package/template/apps/server/src/report-staging/drizzle-seal.ts

@@ -0,0 +1,177 @@
+import {
+  reportStagingClaims,
+  reportStagingEntries,
+  reportStagingGroups,
+  reportStagingSources,
+} from "@southwind-ai/database";
+import { and, eq, sql } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import type { SealedReportStagingGroup } from "./contracts.js";
+import { mapReportGroup, reportEntryValues } from "./drizzle-mapping.js";
+import { createManifestHash } from "./manifest-stream.js";
+
+class SealFenceLost extends Error {}
+
+export async function sealDrizzleReportGroup(
+  db: NodePgDatabase,
+  groupId: string,
+  tokenHash: string,
+  sealed: SealedReportStagingGroup,
+) {
+  try {
+    return await db.transaction(async (tx) => {
+      const [group] = await tx
+        .select()
+        .from(reportStagingGroups)
+        .where(eq(reportStagingGroups.groupId, groupId))
+        .limit(1)
+        .for("update");
+      const [claim] = await tx
+        .select()
+        .from(reportStagingClaims)
+        .where(eq(reportStagingClaims.groupId, groupId))
+        .limit(1)
+        .for("update");
+      const startedAt = await databaseNow(tx);
+      const sourceBytes = sealedSourceBytes(sealed, groupId);
+      if (
+        !group ||
+        group.status !== "indexing" ||
+        group.expiresAt <= startedAt ||
+        claim?.tokenHash !== tokenHash ||
+        claim.expiresAt <= startedAt ||
+        sealed.sealId !== tokenHash ||
+        sourceBytes === undefined ||
+        sourceBytes !== sealed.archiveByteSize ||
+        sealed.catalog.expandedByteSize !== sealed.expandedByteSize ||
+        !/^[0-9a-f]{64}$/.test(sealed.manifestSha256) ||
+        sealed.sources.length !== group.sourceCount ||
+        sealed.sources.some((source) => source.groupId !== groupId)
+      ) {
+        return undefined;
+      }
+
+      for (const source of sealed.sources) {
+        const updated = await tx
+          .update(reportStagingSources)
+          .set({ byteSize: source.byteSize, sha256: source.sha256 })
+          .where(
+            and(
+              eq(reportStagingSources.groupId, groupId),
+              eq(reportStagingSources.sourceId, source.sourceId),
+            ),
+          )
+          .returning({ sourceId: reportStagingSources.sourceId });
+        if (updated.length !== 1) throw new SealFenceLost();
+      }
+      const manifest = createManifestHash(
+        {
+          groupId: group.groupId,
+          purpose: group.purpose,
+          sourceMode: group.sourceMode as "archive" | "expanded",
+          entrypoint: group.entrypoint,
+        },
+        sealed.sources,
+      );
+      let count = 0;
+      let bytes = 0;
+      let ordinal = 0;
+      for (const batch of sealed.catalog.batches(1_000)) {
+        if (batch.length > 1_000) throw new SealFenceLost();
+        for (const entry of batch) {
+          if (entry.groupId !== groupId || entry.ordinal !== ordinal) {
+            throw new SealFenceLost();
+          }
+          manifest.add(entry);
+          count++;
+          ordinal++;
+          bytes += entry.byteSize;
+          if (!Number.isSafeInteger(bytes)) throw new SealFenceLost();
+        }
+        await tx
+          .insert(reportStagingEntries)
+          .values(batch.map(reportEntryValues));
+      }
+      if (
+        count !== sealed.catalog.entryCount ||
+        bytes !== sealed.expandedByteSize ||
+        manifest.digest() !== sealed.manifestSha256
+      ) {
+        throw new SealFenceLost();
+      }
+
+      const readyAt = await databaseNow(tx);
+      if (group.expiresAt <= readyAt || claim.expiresAt <= readyAt) {
+        throw new SealFenceLost();
+      }
+      const [ready] = await tx
+        .update(reportStagingGroups)
+        .set({
+          status: "ready",
+          entryCount: sealed.catalog.entryCount,
+          archiveByteSize: sealed.archiveByteSize,
+          expandedByteSize: sealed.expandedByteSize,
+          sealId: sealed.sealId,
+          manifestSha256: sealed.manifestSha256,
+          failureCode: null,
+          revision: sql`${reportStagingGroups.revision} + 1`,
+          updatedAt: readyAt,
+        })
+        .where(
+          and(
+            eq(reportStagingGroups.groupId, groupId),
+            eq(reportStagingGroups.status, "indexing"),
+          ),
+        )
+        .returning();
+      if (!ready) throw new SealFenceLost();
+      await tx
+        .delete(reportStagingClaims)
+        .where(
+          and(
+            eq(reportStagingClaims.groupId, groupId),
+            eq(reportStagingClaims.tokenHash, tokenHash),
+          ),
+        );
+      return mapReportGroup(ready);
+    });
+  } catch (error) {
+    if (error instanceof SealFenceLost) return undefined;
+    throw error;
+  }
+}
+
+function sealedSourceBytes(sealed: SealedReportStagingGroup, groupId: string) {
+  const ids = new Set<string>();
+  let bytes = 0;
+  for (const [ordinal, source] of sealed.sources.entries()) {
+    if (
+      source.groupId !== groupId ||
+      source.ordinal !== ordinal ||
+      ids.has(source.sourceId) ||
+      source.byteSize === undefined ||
+      !Number.isSafeInteger(source.byteSize) ||
+      source.byteSize < 0 ||
+      !source.sha256?.match(/^[0-9a-f]{64}$/)
+    ) {
+      return undefined;
+    }
+    ids.add(source.sourceId);
+    bytes += source.byteSize;
+    if (!Number.isSafeInteger(bytes)) return undefined;
+  }
+  return bytes;
+}
+
+async function databaseNow(
+  tx: Parameters<Parameters<NodePgDatabase["transaction"]>[0]>[0],
+) {
+  const [row] = await tx
+    .select({
+      now: sql<Date>`clock_timestamp()`.mapWith(reportStagingGroups.updatedAt),
+    })
+    .from(sql`(values (1)) as clock_source(value)`)
+    .limit(1);
+  if (!row) throw new SealFenceLost();
+  return row.now;
+}