create-windy 0.5.2 → 0.6.0

package/template/apps/server/src/report-staging/metadata-spool.test.ts

@@ -0,0 +1,214 @@
+import { createHash } from "node:crypto";
+import { stat } from "node:fs/promises";
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  cleanupReportStagingFixtures,
+  createFixture,
+  createInput,
+  policy,
+} from "./service.test-fixture.js";
+import { ReportStagingEntryCatalog } from "./metadata-spool.js";
+import { InMemoryReportStagingRepository } from "./memory-repository.js";
+import {
+  computeManifestSha256,
+  createGroupRecord,
+  sourceRecords,
+} from "./service-input.js";
+
+const emptySha256 = createHash("sha256").update("").digest("hex");
+
+afterEach(cleanupReportStagingFixtures);
+
+describe("claim-owned SQLite metadata spool", () => {
+  test("200k flat entries 保持 lazy batch<=1000、4096 path contract 与 page cap", async () => {
+    const fixture = await createFixture();
+    const path = `${fixture.root}/index.sqlite`;
+    const stagingPolicy = {
+      ...policy(),
+      maxEntries: 200_001,
+      maxExpandedBytes: 1,
+      maxEntryBytes: 1,
+      maxPathDepth: 1,
+    };
+    const catalog = await ReportStagingEntryCatalog.create({
+      path,
+      groupId: "rsg_11111111111111111111111111111111",
+      policy: stagingPolicy,
+    });
+    expect((await stat(path)).mode & 0o777).toBe(0o600);
+    for (let ordinal = 0; ordinal < 200_000; ordinal++) {
+      const relativePath = `f${ordinal.toString().padStart(6, "0")}`;
+      catalog.declareArchiveEntry(relativePath, false, 0);
+    }
+    expect(() =>
+      catalog.declareArchiveEntry("host-limit-overflow", false, 0),
+    ).toThrow("metadata spool 超过 Host 上限");
+    catalog.finishArchivePass(200_000, 0);
+    for (let ordinal = 0; ordinal < 200_000; ordinal++) {
+      const relativePath = `f${ordinal.toString().padStart(6, "0")}`;
+      catalog.markScanned({
+        relativePath,
+        directory: false,
+        byteSize: 0,
+        sha256: emptySha256,
+        ordinal,
+      });
+    }
+    catalog.finishScan("2026-08-05T00:00:00.000Z");
+    expect((await stat(path)).mode & 0o777).toBe(0o400);
+    let count = 0;
+    let batches = 0;
+    for (const batch of catalog.batches()) {
+      expect(batch.length).toBeLessThanOrEqual(1_000);
+      count += batch.length;
+      batches++;
+    }
+    expect({ count, batches, nodes: catalog.nodeCount }).toEqual({
+      count: 200_000,
+      batches: 200,
+      nodes: 200_000,
+    });
+    expect(catalog.maxPageCount * 4_096).toBeLessThanOrEqual(4 * 1_024 ** 3);
+    catalog.close();
+  }, 60_000);
+
+  test("远距 casefold 与双向 file/directory prefix 冲突均由 DB 拒绝", async () => {
+    const fixture = await createFixture();
+    const catalog = await ReportStagingEntryCatalog.create({
+      path: `${fixture.root}/collision.sqlite`,
+      groupId: "rsg_22222222222222222222222222222222",
+      policy: { ...policy(), maxEntries: 5_000 },
+    });
+    catalog.declareArchiveEntry("Alpha/one", false, 0);
+    for (let index = 0; index < 1_000; index++) {
+      catalog.declareArchiveEntry(`f${index}`, false, 0);
+    }
+    expect(() => catalog.declareArchiveEntry("alpha/two", false, 0)).toThrow(
+      "大小写折叠",
+    );
+    expect(() => catalog.declareArchiveEntry("Alpha", false, 0)).toThrow(
+      "冲突",
+    );
+    catalog.declareArchiveEntry("plain", false, 0);
+    expect(() => catalog.declareArchiveEntry("plain/child", false, 0)).toThrow(
+      "冲突",
+    );
+    catalog.close();
+  });
+
+  test("增量 manifest hash 与旧 JSON.stringify 小样本完全一致", async () => {
+    const fixture = await createFixture();
+    const catalog = await ReportStagingEntryCatalog.create({
+      path: `${fixture.root}/manifest.sqlite`,
+      groupId: "rsg_33333333333333333333333333333333",
+      policy: policy(),
+    });
+    for (const relativePath of ["a.txt", "b.txt"]) {
+      catalog.declareArchiveEntry(relativePath, false, 0);
+    }
+    catalog.finishArchivePass(2, 0);
+    for (const [ordinal, relativePath] of ["a.txt", "b.txt"].entries()) {
+      catalog.markScanned({
+        relativePath,
+        directory: false,
+        byteSize: 0,
+        sha256: emptySha256,
+        ordinal,
+      });
+    }
+    catalog.finishScan("2026-08-05T00:00:00.000Z");
+    const group = {
+      groupId: "rsg_33333333333333333333333333333333",
+      purpose: "evidence.report-staging",
+      sourceMode: "archive" as const,
+      entrypoint: "report.rar",
+    };
+    const sources: [] = [];
+    const entries = [...catalog.batches()]
+      .flat()
+      .map(({ entryId, relativePath, byteSize, sha256, ordinal }) => ({
+        entryId,
+        relativePath,
+        byteSize,
+        sha256,
+        ordinal,
+      }));
+    const legacy = createHash("sha256")
+      .update(
+        JSON.stringify({
+          version: "1",
+          ...group,
+          sources,
+          entries,
+        }),
+      )
+      .digest("hex");
+    expect(computeManifestSha256(group, sources, catalog)).toBe(legacy);
+    catalog.close();
+  });
+
+  test("memory repository 只 fork 只读 catalog，不调用全量 batch 物化", async () => {
+    const fixture = await createFixture();
+    const groupId = "rsg_44444444444444444444444444444444";
+    const now = "2026-08-05T00:00:00.000Z";
+    const catalog = await ReportStagingEntryCatalog.create({
+      path: `${fixture.root}/memory.sqlite`,
+      groupId,
+      policy: policy(),
+    });
+    catalog.declareArchiveEntry("a.txt", false, 0);
+    catalog.finishArchivePass(1, 0);
+    catalog.markScanned({
+      relativePath: "a.txt",
+      directory: false,
+      byteSize: 0,
+      sha256: emptySha256,
+      ordinal: 0,
+    });
+    catalog.finishScan(now);
+    const repository = new InMemoryReportStagingRepository();
+    const group = createGroupRecord({
+      groupId,
+      moduleName: "evidence",
+      ownerId: "user-1",
+      scopeHash: "a".repeat(64),
+      create: createInput(),
+      policy: policy(),
+      now,
+    });
+    const sources = sourceRecords(groupId, createInput(), now).map(
+      (source) => ({
+        ...source,
+        byteSize: 0,
+        sha256: emptySha256,
+      }),
+    );
+    await repository.createOrGetGroup(group, sources);
+    const token = "b".repeat(64);
+    await repository.acquireClaim(
+      groupId,
+      token,
+      now,
+      "2026-08-05T00:00:30.000Z",
+    );
+    catalog.batches = function forbiddenMaterialization() {
+      throw new Error("memory repository must not iterate batches");
+    };
+    await expect(
+      repository.sealGroup(groupId, token, {
+        sealId: token,
+        sources,
+        catalog,
+        archiveByteSize: 0,
+        expandedByteSize: 0,
+        manifestSha256: "c".repeat(64),
+        updatedAt: "2026-08-05T00:00:01.000Z",
+      }),
+    ).resolves.toMatchObject({ status: "ready", entryCount: 1 });
+    expect(await repository.listEntries(groupId, undefined, 10)).toHaveLength(
+      1,
+    );
+    await repository.markDiscarding(groupId, "2026-08-05T00:00:02.000Z");
+    catalog.close();
+  });
+});

package/template/apps/server/src/report-staging/metadata-spool.ts

@@ -0,0 +1,305 @@
+import { chmodSync } from "node:fs";
+import { chmod, mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import { Database } from "bun:sqlite";
+import {
+  ModuleReportStagingError,
+  type ModuleReportStagingPolicyV1,
+} from "@southwind-ai/server-sdk";
+import {
+  metadataEntryId,
+  readCatalogBatches,
+  readCatalogEntry,
+  readCatalogPage,
+} from "./metadata-catalog.js";
+import {
+  initializeMetadataSpool,
+  isSqliteFull,
+} from "./metadata-spool-schema.js";
+import {
+  declareMetadataEntry,
+  declaredMetadataEntry,
+  expectedMetadataEntry,
+  lookupMetadataNode,
+  type MetadataPathState,
+} from "./metadata-spool-paths.js";
+import { validateArchiveEntryPath } from "./path-policy.js";
+
+export class ReportStagingEntryCatalog {
+  #db: Database;
+  #path: string;
+  readonly #policy: ModuleReportStagingPolicyV1;
+  readonly #groupId: string;
+  readonly #state: MetadataPathState = {
+    archiveEntryCount: 0,
+    nodeCount: 0,
+    expandedByteSize: 0,
+  };
+  #createdAt = new Date().toISOString();
+  #phase: "declaring" | "scanning" | "readonly" | "closed";
+  #maxPageCount = 0;
+
+  private constructor(
+    path: string,
+    groupId: string,
+    policy: ModuleReportStagingPolicyV1,
+    readonly = false,
+  ) {
+    this.#path = path;
+    this.#groupId = groupId;
+    this.#policy = policy;
+    this.#phase = readonly ? "readonly" : "declaring";
+    this.#db = new Database(path, {
+      ...(readonly ? { readonly: true } : { create: true }),
+      strict: true,
+    });
+  }
+
+  static async create(input: {
+    readonly path: string;
+    readonly groupId: string;
+    readonly policy: ModuleReportStagingPolicyV1;
+  }) {
+    await mkdir(dirname(input.path), { recursive: true, mode: 0o700 });
+    const catalog = new ReportStagingEntryCatalog(
+      input.path,
+      input.groupId,
+      input.policy,
+    );
+    try {
+      catalog.#maxPageCount = initializeMetadataSpool(
+        catalog.#db,
+        input.policy,
+      );
+      await chmod(input.path, 0o600);
+      return catalog;
+    } catch (error) {
+      catalog.close();
+      if (isSqliteFull(error)) throw limit();
+      throw error;
+    }
+  }
+
+  declareArchiveEntry(raw: string, directory: boolean, byteSize: number) {
+    return declareMetadataEntry({
+      db: this.#db,
+      policy: this.#policy,
+      state: this.#state,
+      raw,
+      directory,
+      byteSize,
+    });
+  }
+
+  expectedArchiveEntry(
+    ordinal: number,
+    raw: string,
+    directory: boolean,
+    byteSize: number,
+  ) {
+    return expectedMetadataEntry({
+      db: this.#db,
+      policy: this.#policy,
+      ordinal,
+      raw,
+      directory,
+      byteSize,
+    });
+  }
+
+  declaredArchiveEntry(ordinal: number) {
+    return declaredMetadataEntry(this.#db, ordinal);
+  }
+
+  finishArchivePass(entryCount: number, expandedByteSize: number) {
+    if (
+      entryCount !== this.#state.archiveEntryCount ||
+      expandedByteSize !== this.#state.expandedByteSize
+    ) {
+      throw unsafe("报告实际展开大小或条目数与 preflight 不一致");
+    }
+    if (this.#phase === "declaring") {
+      this.writeBounded(() => this.#db.exec("commit; begin immediate;"));
+      this.#phase = "scanning";
+    }
+  }
+
+  markScanned(input: {
+    readonly relativePath: string;
+    readonly directory: boolean;
+    readonly byteSize: number;
+    readonly sha256?: string;
+    readonly ordinal?: number;
+  }) {
+    if (this.#phase !== "scanning") throw integrity();
+    if (
+      (!input.directory &&
+        (input.ordinal === undefined ||
+          !input.sha256?.match(/^[0-9a-f]{64}$/))) ||
+      (input.directory &&
+        (input.ordinal !== undefined || input.sha256 !== undefined))
+    ) {
+      throw integrity();
+    }
+    const path = validateArchiveEntryPath(
+      input.relativePath,
+      this.#policy,
+      input.directory,
+      input.byteSize,
+    );
+    const row = lookupMetadataNode(this.#db, path.pathKey);
+    if (
+      !row ||
+      row.relativePath !== path.relativePath ||
+      row.kind !== (input.directory ? "directory" : "file") ||
+      row.declaredSize !== input.byteSize
+    ) {
+      throw integrity();
+    }
+    const entryId = input.directory
+      ? null
+      : metadataEntryId(this.#groupId, path.relativePath);
+    let changed: unknown;
+    this.writeBounded(() => {
+      changed = this.#db
+        .query<unknown, [string | null, number | null, string | null, string]>(
+          "update nodes set scan_seen = 1, sha256 = ?, entry_ordinal = ?, entry_id = ? where path_key = ? and scan_seen = 0 returning path_key",
+        )
+        .get(
+          input.sha256 ?? null,
+          input.ordinal ?? null,
+          entryId,
+          path.pathKey,
+        );
+    });
+    if (!changed) throw integrity();
+  }
+
+  finishScan(createdAt: string) {
+    const summary = this.#db
+      .query<{ missing: number; entries: number; bytes: number }, []>(
+        "select coalesce(sum(case when scan_seen = 0 then 1 else 0 end), 0) as missing, coalesce(sum(case when kind = 'file' then 1 else 0 end), 0) as entries, coalesce(sum(case when kind = 'file' then declared_size else 0 end), 0) as bytes from nodes",
+      )
+      .get();
+    if (
+      !summary ||
+      summary.missing !== 0 ||
+      summary.entries !== this.entryCount ||
+      summary.bytes !== this.#state.expandedByteSize
+    ) {
+      throw integrity();
+    }
+    this.#createdAt = createdAt;
+    this.writeBounded(() => this.#db.exec("commit;"));
+    this.#db.close(false);
+    chmodSync(this.#path, 0o400);
+    this.#db = new Database(this.#path, { readonly: true, strict: true });
+    this.#phase = "readonly";
+  }
+
+  *batches(size = 1_000) {
+    if (!Number.isSafeInteger(size) || size < 1 || size > 1_000) throw limit();
+    yield* readCatalogBatches({
+      db: this.#db,
+      groupId: this.#groupId,
+      createdAt: this.#createdAt,
+      size,
+    });
+  }
+
+  getEntry(entryId: string) {
+    return readCatalogEntry({
+      db: this.#db,
+      groupId: this.#groupId,
+      createdAt: this.#createdAt,
+      entryId,
+    });
+  }
+
+  listEntries(afterOrdinal: number | undefined, limit: number) {
+    return readCatalogPage({
+      db: this.#db,
+      groupId: this.#groupId,
+      createdAt: this.#createdAt,
+      afterOrdinal,
+      limit,
+    });
+  }
+
+  get entryCount() {
+    return this.#db
+      .query<{ count: number }, []>(
+        "select count(*) as count from nodes where kind = 'file'",
+      )
+      .get()!.count;
+  }
+
+  get nodeCount() {
+    return this.#state.nodeCount;
+  }
+
+  get expandedByteSize() {
+    return this.#state.expandedByteSize;
+  }
+
+  get maxPageCount() {
+    return this.#maxPageCount;
+  }
+
+  close() {
+    if (this.#phase === "closed") return;
+    if (this.#phase === "declaring" || this.#phase === "scanning") {
+      try {
+        this.#db.exec("rollback;");
+      } catch {
+        // Closing the claim-owned handle remains the terminal cleanup fence.
+      }
+    }
+    this.#db.close(false);
+    this.#phase = "closed";
+  }
+
+  relocate(path: string) {
+    this.#path = path;
+  }
+
+  forkReadOnly() {
+    const catalog = new ReportStagingEntryCatalog(
+      this.#path,
+      this.#groupId,
+      this.#policy,
+      true,
+    );
+    Object.assign(catalog.#state, this.#state);
+    catalog.#createdAt = this.#createdAt;
+    catalog.#maxPageCount = this.#maxPageCount;
+    return catalog;
+  }
+
+  private writeBounded(operation: () => void) {
+    try {
+      operation();
+    } catch (error) {
+      if (isSqliteFull(error)) throw limit();
+      throw error;
+    }
+  }
+}
+
+function unsafe(message: string) {
+  return new ModuleReportStagingError("REPORT_STAGING_UNSAFE_ARCHIVE", message);
+}
+
+function limit() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_LIMIT",
+    "报告 metadata spool 超过 Host 上限",
+  );
+}
+
+function integrity() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INTEGRITY_MISMATCH",
+    "报告 metadata spool 与完整展开结果不一致",
+  );
+}