create-windy 0.5.1 → 0.6.0

package/template/apps/server/src/report-staging/repository-fence.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, test } from "bun:test";
+import type { ReportStagingGroupRecord } from "./contracts.js";
+import { InMemoryReportStagingRepository } from "./memory-repository.js";
+
+describe("Report Staging failure terminal fence", () => {
+  test("stale token 与 expired claim 都不能把 indexing 永久写 failed", async () => {
+    const repository = new InMemoryReportStagingRepository();
+    const id = groupId("1");
+    await repository.createOrGetGroup(group(id, instant(120), "1"), []);
+    await repository.acquireClaim(id, token("a"), instant(0), instant(10));
+    await expect(
+      repository.acquireClaim(id, token("b"), instant(11), instant(30)),
+    ).resolves.toMatchObject({ generation: 2 });
+    await expect(
+      repository.failGroup(id, token("a"), "PROCESS_FAILED", instant(12)),
+    ).resolves.toBeUndefined();
+    await expect(
+      repository.failGroup(id, token("b"), "PROCESS_FAILED", instant(31)),
+    ).resolves.toBeUndefined();
+    expect(await repository.getGroup(id)).toMatchObject({
+      status: "indexing",
+      revision: 1,
+    });
+  });
+
+  test("group TTL 与 discarding 状态都 fence 旧 worker failure replay", async () => {
+    const repository = new InMemoryReportStagingRepository();
+    const expiredId = groupId("2");
+    await repository.createOrGetGroup(group(expiredId, instant(20), "2"), []);
+    await repository.acquireClaim(
+      expiredId,
+      token("c"),
+      instant(0),
+      instant(30),
+    );
+    await expect(
+      repository.failGroup(
+        expiredId,
+        token("c"),
+        "PROCESS_FAILED",
+        instant(21),
+      ),
+    ).resolves.toBeUndefined();
+
+    const discardingId = groupId("3");
+    await repository.createOrGetGroup(
+      group(discardingId, instant(120), "3"),
+      [],
+    );
+    await repository.acquireClaim(
+      discardingId,
+      token("d"),
+      instant(0),
+      instant(10),
+    );
+    await expect(
+      repository.markDiscarding(discardingId, instant(11)),
+    ).resolves.toMatchObject({ status: "discarding" });
+    await expect(
+      repository.failGroup(
+        discardingId,
+        token("d"),
+        "PROCESS_FAILED",
+        instant(12),
+      ),
+    ).resolves.toBeUndefined();
+    expect(await repository.getGroup(discardingId)).toMatchObject({
+      status: "discarding",
+      revision: 2,
+    });
+  });
+});
+
+function group(
+  groupId: string,
+  expiresAt: string,
+  discriminator: string,
+): ReportStagingGroupRecord {
+  return {
+    groupId,
+    moduleName: "evidence",
+    ownerId: "user-1",
+    scopeHash: token("s"),
+    purpose: "evidence.report-staging",
+    sourceMode: "archive",
+    idempotencyHash: token(discriminator),
+    fingerprint: token("f"),
+    policyDigest: token("p"),
+    entrypoint: "report.rar",
+    status: "indexing",
+    sourceCount: 0,
+    entryCount: 0,
+    archiveByteSize: 0,
+    expandedByteSize: 0,
+    revision: 1,
+    expiresAt,
+    createdAt: instant(0),
+    updatedAt: instant(0),
+  };
+}
+
+function groupId(value: string) {
+  return `rsg_${value.repeat(32)}`;
+}
+
+function token(value: string) {
+  return value.repeat(64);
+}
+
+function instant(seconds: number) {
+  return new Date(
+    Date.parse("2026-07-31T00:00:00.000Z") + seconds * 1_000,
+  ).toISOString();
+}

package/template/apps/server/src/report-staging/repository.ts

@@ -0,0 +1,75 @@
+import type {
+  ReportStagingClaimRecord,
+  ReportStagingEntryRecord,
+  ReportStagingGroupRecord,
+  ReportStagingReadLeaseRecord,
+  ReportStagingSourceRecord,
+  SealedReportStagingGroup,
+} from "./contracts.js";
+
+export interface ReportStagingRepository {
+  createOrGetGroup(
+    group: ReportStagingGroupRecord,
+    sources: readonly ReportStagingSourceRecord[],
+  ): Promise<{ group: ReportStagingGroupRecord; created: boolean }>;
+  getGroup(groupId: string): Promise<ReportStagingGroupRecord | undefined>;
+  getSources(groupId: string): Promise<ReportStagingSourceRecord[]>;
+  getEntry(
+    groupId: string,
+    entryId: string,
+  ): Promise<ReportStagingEntryRecord | undefined>;
+  listEntries(
+    groupId: string,
+    afterOrdinal: number | undefined,
+    limit: number,
+  ): Promise<ReportStagingEntryRecord[]>;
+  acquireClaim(
+    groupId: string,
+    tokenHash: string,
+    now: string,
+    expiresAt: string,
+  ): Promise<ReportStagingClaimRecord | undefined>;
+  renewClaim(
+    groupId: string,
+    tokenHash: string,
+    now: string,
+    expiresAt: string,
+  ): Promise<boolean>;
+  releaseClaim(groupId: string, tokenHash: string): Promise<void>;
+  sealGroup(
+    groupId: string,
+    tokenHash: string,
+    sealed: SealedReportStagingGroup,
+  ): Promise<ReportStagingGroupRecord | undefined>;
+  failGroup(
+    groupId: string,
+    tokenHash: string,
+    failureCode: string,
+    updatedAt: string,
+  ): Promise<ReportStagingGroupRecord | undefined>;
+  acquireReadLease(
+    lease: ReportStagingReadLeaseRecord,
+    now: string,
+  ): Promise<ReportStagingReadLeaseRecord | undefined>;
+  getReadLease(
+    groupId: string,
+    leaseId: string,
+  ): Promise<ReportStagingReadLeaseRecord | undefined>;
+  closeReadLease(
+    groupId: string,
+    leaseId: string,
+    closedAt: string,
+  ): Promise<boolean>;
+  markDiscarding(
+    groupId: string,
+    now: string,
+  ): Promise<ReportStagingGroupRecord | undefined>;
+  markDiscarded(
+    groupId: string,
+    now: string,
+  ): Promise<ReportStagingGroupRecord | undefined>;
+  listPurgeCandidates(
+    now: string,
+    limit: number,
+  ): Promise<ReportStagingGroupRecord[]>;
+}

package/template/apps/server/src/report-staging/runtime.ts

@@ -0,0 +1,31 @@
+import { join, resolve } from "node:path";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import {
+  SupervisedLibarchiveProcess,
+  type ArchiveProcess,
+} from "./archive-process.js";
+import { DrizzleReportStagingRepository } from "./drizzle-repository.js";
+import { ReportStagingFilesystemStore } from "./filesystem-store.js";
+import { InMemoryReportStagingRepository } from "./memory-repository.js";
+import { ReportStagingService } from "./service.js";
+
+export function createReportStagingRuntime(
+  env: Record<string, string | undefined>,
+  db?: NodePgDatabase,
+  archive: ArchiveProcess = new SupervisedLibarchiveProcess(),
+) {
+  if (env.NODE_ENV === "production" && !db) return undefined;
+  const storageRoot = resolve(env.WINDY_STORAGE_ROOT || ".data/storage");
+  const root = resolve(
+    env.WINDY_REPORT_STAGING_ROOT ||
+      join(
+        storageRoot,
+        db ? "report-staging" : `report-staging-dev-${process.pid}`,
+      ),
+  );
+  const repository = db
+    ? new DrizzleReportStagingRepository(db)
+    : new InMemoryReportStagingRepository();
+  const files = new ReportStagingFilesystemStore(root, archive);
+  return new ReportStagingService(repository, files);
+}

package/template/apps/server/src/report-staging/sealed-file.test.ts

@@ -0,0 +1,34 @@
+import { createHash } from "node:crypto";
+import { mkdtemp, rename, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "bun:test";
+import { openVerifiedSealedFile } from "./sealed-file.js";
+
+const roots: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+  );
+});
+
+test("验证与返回流复用同一 inode，路径替换不能切换读取对象", async () => {
+  const root = await mkdtemp(join(tmpdir(), "windy-sealed-file-"));
+  roots.push(root);
+  const path = join(root, "entry.txt");
+  const original = Buffer.from("trusted");
+  await writeFile(path, original);
+  const opened = await openVerifiedSealedFile({
+    path,
+    expectedByteSize: original.byteLength,
+    expectedSha256: createHash("sha256").update(original).digest("hex"),
+    signal: new AbortController().signal,
+  });
+
+  await rename(path, join(root, "original.txt"));
+  await writeFile(path, "hostile");
+
+  expect(await Bun.readableStreamToText(opened.body)).toBe("trusted");
+  await expect(opened.close()).resolves.toBeUndefined();
+});

package/template/apps/server/src/report-staging/sealed-file.ts

@@ -0,0 +1,137 @@
+import { createHash } from "node:crypto";
+import { constants, type Stats } from "node:fs";
+import { open, type FileHandle } from "node:fs/promises";
+import { ModuleReportStagingError } from "@southwind-ai/server-sdk";
+
+const READ_CHUNK_BYTES = 64 * 1_024;
+
+export interface OpenedVerifiedSealedFile {
+  readonly body: ReadableStream<Uint8Array>;
+  close(): Promise<void>;
+}
+
+export async function openVerifiedSealedFile(input: {
+  readonly path: string;
+  readonly expectedByteSize: number;
+  readonly expectedSha256: string;
+  readonly signal: AbortSignal;
+}): Promise<OpenedVerifiedSealedFile> {
+  const handle = await open(
+    input.path,
+    constants.O_RDONLY | constants.O_NOFOLLOW,
+  ).catch(() => undefined);
+  if (!handle) throw integrity();
+  let closed = false;
+  let abortListener = () => undefined;
+  const close = async () => {
+    if (closed) return;
+    closed = true;
+    input.signal.removeEventListener("abort", abortListener);
+    await handle.close().catch(() => undefined);
+  };
+  try {
+    const metadata = await handle.stat();
+    requireExpectedFile(metadata, input.expectedByteSize);
+    const sha256 = await hashHandle(
+      handle,
+      input.expectedByteSize,
+      input.signal,
+    );
+    requireExpectedFile(await handle.stat(), input.expectedByteSize);
+    if (sha256 !== input.expectedSha256) throw integrity();
+  } catch (error) {
+    await close();
+    throw error;
+  }
+  if (input.signal.aborted) {
+    await close();
+    throw cancelled();
+  }
+  abortListener = () => void close();
+  input.signal.addEventListener("abort", abortListener, { once: true });
+
+  let position = 0;
+  const body = new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        if (input.signal.aborted) throw cancelled();
+        const remaining = input.expectedByteSize - position;
+        if (remaining === 0) {
+          controller.close();
+          await close();
+          return;
+        }
+        const buffer = Buffer.allocUnsafe(
+          Math.min(READ_CHUNK_BYTES, remaining),
+        );
+        const { bytesRead } = await handle.read(
+          buffer,
+          0,
+          buffer.byteLength,
+          position,
+        );
+        if (bytesRead <= 0 || bytesRead > remaining) throw integrity();
+        position += bytesRead;
+        controller.enqueue(buffer.subarray(0, bytesRead));
+      } catch (error) {
+        controller.error(error);
+        await close();
+      }
+    },
+    async cancel() {
+      await close();
+    },
+  });
+  return { body, close };
+}
+
+async function hashHandle(
+  handle: FileHandle,
+  expectedByteSize: number,
+  signal: AbortSignal,
+) {
+  const hash = createHash("sha256");
+  let position = 0;
+  while (position < expectedByteSize) {
+    if (signal.aborted) throw cancelled();
+    const buffer = Buffer.allocUnsafe(
+      Math.min(READ_CHUNK_BYTES, expectedByteSize - position),
+    );
+    const { bytesRead } = await handle.read(
+      buffer,
+      0,
+      buffer.byteLength,
+      position,
+    );
+    if (bytesRead <= 0) throw integrity();
+    position += bytesRead;
+    hash.update(buffer.subarray(0, bytesRead));
+  }
+  if (signal.aborted) throw cancelled();
+  return hash.digest("hex");
+}
+
+function requireExpectedFile(metadata: Stats, expectedByteSize: number) {
+  if (
+    !metadata.isFile() ||
+    metadata.nlink !== 1 ||
+    metadata.size !== expectedByteSize
+  ) {
+    throw integrity();
+  }
+}
+
+function cancelled() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_CANCELLED",
+    "报告暂存已取消",
+    true,
+  );
+}
+
+function integrity() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INTEGRITY_MISMATCH",
+    "已封存报告 entry 发生漂移",
+  );
+}

package/template/apps/server/src/report-staging/service-input-limits.test.ts

@@ -0,0 +1,62 @@
+import { expect, test } from "bun:test";
+import { validateCreateInput } from "./service-input.js";
+import { policy } from "./service.test-fixture.js";
+
+test("expanded 最多 10k sources，而 archive volume set 独立限制 256", () => {
+  const expandedSources = Array.from({ length: 10_000 }, (_, index) =>
+    source(`f${index}`),
+  );
+  expect(() =>
+    validateCreateInput(
+      {
+        purpose: "evidence.report-staging",
+        idempotencyKey: "expanded-10k",
+        sourceMode: "expanded",
+        entrypoint: "f0",
+        sources: expandedSources,
+      },
+      { ...policy(), maxSources: 10_000, maxEntries: 10_000 },
+    ),
+  ).not.toThrow();
+
+  expect(() =>
+    validateCreateInput(
+      {
+        purpose: "evidence.report-staging",
+        idempotencyKey: "expanded-10k-plus-one",
+        sourceMode: "expanded",
+        entrypoint: "f0",
+        sources: [...expandedSources, source("overflow")],
+      },
+      { ...policy(), maxSources: 20_000, maxEntries: 20_000 },
+    ),
+  ).toThrow("输入不符合 Host 策略");
+
+  const archiveSources = Array.from({ length: 257 }, (_, index) =>
+    source(`part${index}`),
+  );
+  expect(() =>
+    validateCreateInput(
+      {
+        purpose: "evidence.report-staging",
+        idempotencyKey: "archive-257",
+        sourceMode: "archive",
+        entrypoint: "part0",
+        sources: archiveSources,
+      },
+      { ...policy(), maxSources: 10_000, maxEntries: 10_000 },
+    ),
+  ).toThrow("输入不符合 Host 策略");
+});
+
+function source(relativePath: string) {
+  return {
+    relativePath,
+    file: {
+      fileId: `file-${relativePath}`,
+      purpose: "evidence.original",
+      resourceType: "evidence.file",
+      resourceId: `evidence-${relativePath}`,
+    },
+  };
+}

package/template/apps/server/src/report-staging/service-input.ts

@@ -0,0 +1,204 @@
+import { createHash, randomBytes } from "node:crypto";
+import {
+  MODULE_REPORT_STAGING_MAX_ARCHIVE_VOLUMES,
+  MODULE_REPORT_STAGING_MAX_SOURCES,
+  ModuleReportStagingError,
+  type ModuleReportStagingCreateInput,
+  type ModuleReportStagingManifestV1,
+  type ModuleReportStagingPolicyV1,
+} from "@southwind-ai/server-sdk";
+import type {
+  ReportStagingGroupRecord,
+  ReportStagingSourceRecord,
+} from "./contracts.js";
+import type { ReportStagingEntryCatalog } from "./metadata-spool.js";
+import { createManifestHash } from "./manifest-stream.js";
+import {
+  validateExpandedSourceRelativePaths,
+  validateSourceRelativePaths,
+} from "./path-policy.js";
+
+const SAFE_IDEMPOTENCY_KEY = /^[\u0021-\u007e]{1,160}$/;
+
+export function validateCreateInput(
+  input: ModuleReportStagingCreateInput,
+  policy: ModuleReportStagingPolicyV1,
+) {
+  if (
+    !SAFE_IDEMPOTENCY_KEY.test(input.idempotencyKey) ||
+    (input.sourceMode !== "archive" && input.sourceMode !== "expanded") ||
+    input.sources.length === 0 ||
+    input.sources.length > policy.maxSources ||
+    input.sources.length > MODULE_REPORT_STAGING_MAX_SOURCES ||
+    (input.sourceMode === "archive" &&
+      input.sources.length > MODULE_REPORT_STAGING_MAX_ARCHIVE_VOLUMES)
+  ) {
+    throw invalid();
+  }
+  const sourcePaths = input.sources.map(({ relativePath }) => relativePath);
+  const paths =
+    input.sourceMode === "expanded"
+      ? validateExpandedSourceRelativePaths(sourcePaths, policy)
+      : validateSourceRelativePaths(sourcePaths, policy.maxPathDepth);
+  if (
+    input.sourceMode === "archive"
+      ? paths[0] !== input.entrypoint
+      : !paths.includes(input.entrypoint)
+  ) {
+    throw invalid();
+  }
+  for (const source of input.sources) {
+    if (!policy.sourcePurposes.includes(source.file.purpose)) throw invalid();
+  }
+  return paths;
+}
+
+export function groupFingerprint(input: ModuleReportStagingCreateInput) {
+  return digest({
+    sourceMode: input.sourceMode,
+    entrypoint: input.entrypoint,
+    sources: input.sources.map((source) => ({
+      relativePath: source.relativePath,
+      fileId: source.file.fileId,
+      purpose: source.file.purpose,
+      resourceType: source.file.resourceType,
+      resourceId: source.file.resourceId,
+    })),
+  });
+}
+
+export function policyDigest(policy: ModuleReportStagingPolicyV1) {
+  return digest(policy);
+}
+
+export function idempotencyHash(value: string) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+export function opaqueId(prefix: "rsg" | "rsl") {
+  return `${prefix}_${randomBytes(16).toString("hex")}`;
+}
+
+export function sourceRecords(
+  groupId: string,
+  input: ModuleReportStagingCreateInput,
+  now: string,
+): ReportStagingSourceRecord[] {
+  return input.sources.map((source, ordinal) => ({
+    sourceId: stableId("rss", groupId, source.relativePath),
+    groupId,
+    ordinal,
+    fileId: source.file.fileId,
+    filePurpose: source.file.purpose,
+    resourceType: source.file.resourceType,
+    resourceId: source.file.resourceId,
+    relativePath: source.relativePath,
+    createdAt: now,
+  }));
+}
+
+export function createGroupRecord(input: {
+  readonly groupId: string;
+  readonly moduleName: string;
+  readonly ownerId: string;
+  readonly scopeHash: string;
+  readonly create: ModuleReportStagingCreateInput;
+  readonly policy: ModuleReportStagingPolicyV1;
+  readonly now: string;
+}): ReportStagingGroupRecord {
+  return {
+    groupId: input.groupId,
+    moduleName: input.moduleName,
+    ownerId: input.ownerId,
+    scopeHash: input.scopeHash,
+    purpose: input.create.purpose,
+    sourceMode: input.create.sourceMode,
+    idempotencyHash: idempotencyHash(input.create.idempotencyKey),
+    fingerprint: groupFingerprint(input.create),
+    policyDigest: policyDigest(input.policy),
+    entrypoint: input.create.entrypoint,
+    status: "indexing",
+    sourceCount: input.create.sources.length,
+    entryCount: 0,
+    archiveByteSize: 0,
+    expandedByteSize: 0,
+    revision: 1,
+    expiresAt: new Date(
+      new Date(input.now).getTime() + input.policy.retentionMs,
+    ).toISOString(),
+    createdAt: input.now,
+    updatedAt: input.now,
+  };
+}
+
+export function manifestFor(
+  group: ReportStagingGroupRecord,
+  sources: readonly ReportStagingSourceRecord[],
+): ModuleReportStagingManifestV1 {
+  if (!group.manifestSha256) throw integrity();
+  return {
+    version: "1",
+    groupId: group.groupId,
+    purpose: group.purpose,
+    sourceMode: group.sourceMode,
+    entrypoint: group.entrypoint,
+    sources:
+      group.sourceMode === "expanded"
+        ? []
+        : sources.map((source) => {
+            if (source.byteSize === undefined || source.sha256 === undefined) {
+              throw integrity();
+            }
+            return {
+              sourceId: source.sourceId,
+              relativePath: source.relativePath,
+              byteSize: source.byteSize,
+              sha256: source.sha256,
+            };
+          }),
+    entryCount: group.entryCount,
+    archiveByteSize: group.archiveByteSize,
+    expandedByteSize: group.expandedByteSize,
+    sha256: group.manifestSha256,
+  };
+}
+
+export function computeManifestSha256(
+  group: Pick<
+    ReportStagingGroupRecord,
+    "groupId" | "purpose" | "sourceMode" | "entrypoint"
+  >,
+  sources: readonly ReportStagingSourceRecord[],
+  catalog: ReportStagingEntryCatalog,
+) {
+  const manifest = createManifestHash(group, sources);
+  for (const batch of catalog.batches()) {
+    batch.forEach((entry) => manifest.add(entry));
+  }
+  return manifest.digest();
+}
+
+function stableId(prefix: string, groupId: string, value: string) {
+  return `${prefix}_${createHash("sha256")
+    .update(`${groupId}\0${value}`)
+    .digest("hex")
+    .slice(0, 32)}`;
+}
+
+function digest(value: unknown) {
+  return createHash("sha256").update(JSON.stringify(value)).digest("hex");
+}
+
+function invalid() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INVALID_INPUT",
+    "报告暂存输入不符合 Host 策略",
+  );
+}
+
+function integrity() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INTEGRITY_MISMATCH",
+    "报告暂存清单不完整",
+  );
+}