create-windy 0.5.1 → 0.6.0

package/template/apps/server/src/report-staging/filesystem-cleanup.test.ts

@@ -0,0 +1,68 @@
+import { lstat, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { afterEach, expect, test } from "bun:test";
+import {
+  discardFilesystemPaths,
+  snapshotFilesystemOrphans,
+} from "./filesystem-cleanup.js";
+import {
+  cleanupReportStagingFixtures,
+  createFixture,
+} from "./service.test-fixture.js";
+
+afterEach(cleanupReportStagingFixtures);
+
+test("orphan 快照后的新 worker attempt 不会被旧 worker 删除", async () => {
+  const fixture = await createFixture();
+  const { work, sealed } = groupPaths(fixture.root);
+  const oldSeal = "a".repeat(64);
+  const newSeal = "b".repeat(64);
+  await createAttempts(work, sealed, oldSeal);
+  const signal = new AbortController().signal;
+  const snapshot = await snapshotFilesystemOrphans(work, sealed, signal);
+  await createAttempts(work, sealed, newSeal);
+  await discardFilesystemPaths(snapshot, signal);
+  await expect(exists(join(work, oldSeal))).resolves.toBe(false);
+  await expect(exists(join(sealed, oldSeal))).resolves.toBe(false);
+  await expect(exists(join(work, newSeal))).resolves.toBe(true);
+  await expect(exists(join(sealed, newSeal))).resolves.toBe(true);
+});
+
+test("claim 续租失败后旧 worker 不再删除快照路径", async () => {
+  const fixture = await createFixture();
+  const { work, sealed } = groupPaths(fixture.root);
+  const oldSeal = "c".repeat(64);
+  await createAttempts(work, sealed, oldSeal);
+  const controller = new AbortController();
+  const snapshot = await snapshotFilesystemOrphans(
+    work,
+    sealed,
+    controller.signal,
+  );
+  controller.abort(new Error("claim lost"));
+  await expect(
+    discardFilesystemPaths(snapshot, controller.signal),
+  ).rejects.toThrow("claim lost");
+  await expect(exists(join(work, oldSeal))).resolves.toBe(true);
+  await expect(exists(join(sealed, oldSeal))).resolves.toBe(true);
+});
+
+async function createAttempts(work: string, sealed: string, sealId: string) {
+  await Promise.all([
+    mkdir(join(work, sealId), { recursive: true }),
+    mkdir(join(sealed, sealId), { recursive: true }),
+  ]);
+}
+
+async function exists(path: string) {
+  return Boolean(await lstat(path).catch(() => undefined));
+}
+
+function groupPaths(root: string) {
+  return {
+    work: join(root, ".work", groupId),
+    sealed: join(root, "groups", groupId.slice(4, 6), groupId),
+  };
+}
+
+const groupId = "rsg_11111111111111111111111111111111";

package/template/apps/server/src/report-staging/filesystem-cleanup.ts

@@ -0,0 +1,102 @@
+import { chmod, readdir, rm, rmdir } from "node:fs/promises";
+import { resolve } from "node:path";
+import { assertInside, makeTreeRemovable } from "./filesystem-io.js";
+
+export async function discardFilesystemGroup(
+  workGroup: string,
+  sealedGroup: string,
+) {
+  await removeTree(workGroup);
+  await removeTree(sealedGroup);
+}
+
+export async function discardFilesystemOrphans(
+  workGroup: string,
+  sealedGroup: string,
+  signal: AbortSignal,
+) {
+  const snapshot = await snapshotFilesystemOrphans(
+    workGroup,
+    sealedGroup,
+    signal,
+  );
+  await discardFilesystemPaths(snapshot, signal);
+  await removeEmptyDirectory(workGroup);
+  await removeEmptyDirectory(sealedGroup);
+}
+
+export async function snapshotFilesystemOrphans(
+  workGroup: string,
+  sealedGroup: string,
+  signal: AbortSignal,
+) {
+  const paths = (
+    await Promise.all([
+      snapshotChildren(workGroup, signal),
+      snapshotChildren(sealedGroup, signal),
+    ])
+  ).flat();
+  requireActive(signal);
+  return paths;
+}
+
+export async function discardFilesystemPaths(
+  paths: readonly string[],
+  signal: AbortSignal,
+) {
+  for (const path of paths) {
+    requireActive(signal);
+    await removeTree(path);
+    requireActive(signal);
+  }
+}
+
+export async function discardSealSiblings(
+  workGroup: string,
+  sealedGroup: string,
+  keepSealId: string,
+) {
+  await removeTree(workGroup);
+  const names = await readdir(sealedGroup).catch(
+    (error: NodeJS.ErrnoException) => {
+      if (error.code === "ENOENT") return [];
+      throw error;
+    },
+  );
+  await chmod(sealedGroup, 0o700);
+  for (const name of names) {
+    if (name === keepSealId) continue;
+    const sibling = resolve(sealedGroup, name);
+    assertInside(sealedGroup, sibling);
+    await removeTree(sibling);
+  }
+}
+
+async function removeTree(path: string) {
+  await makeTreeRemovable(path);
+  await rm(path, { recursive: true, force: true });
+}
+
+async function snapshotChildren(parent: string, signal: AbortSignal) {
+  requireActive(signal);
+  const names = await readdir(parent).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  });
+  requireActive(signal);
+  return names.map((name) => {
+    const child = resolve(parent, name);
+    assertInside(parent, child);
+    return child;
+  });
+}
+
+async function removeEmptyDirectory(path: string) {
+  await rmdir(path).catch((error: NodeJS.ErrnoException) => {
+    if (error.code !== "ENOENT" && error.code !== "ENOTEMPTY") throw error;
+  });
+}
+
+function requireActive(signal: AbortSignal) {
+  if (signal.aborted) throw signal.reason;
+}

package/template/apps/server/src/report-staging/filesystem-errors.ts

@@ -0,0 +1,15 @@
+import { ModuleReportStagingError } from "@southwind-ai/server-sdk";
+
+export function forbidden() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_FORBIDDEN",
+    "报告源文件访问未授权",
+  );
+}
+
+export function integrityMismatch() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INTEGRITY_MISMATCH",
+    "报告索引不一致",
+  );
+}

package/template/apps/server/src/report-staging/filesystem-identifiers.ts

@@ -0,0 +1,19 @@
+import { ModuleReportStagingError } from "@southwind-ai/server-sdk";
+
+export function requireOpaqueGroupId(groupId: string) {
+  if (!/^rsg_[0-9a-f]{32}$/.test(groupId)) {
+    throw new ModuleReportStagingError(
+      "REPORT_STAGING_INVALID_INPUT",
+      "报告暂存组 ID 无效",
+    );
+  }
+}
+
+export function requireOpaqueSealId(sealId: string) {
+  if (!/^[0-9a-f]{64}$/.test(sealId)) {
+    throw new ModuleReportStagingError(
+      "REPORT_STAGING_INVALID_INPUT",
+      "报告封存 attempt ID 无效",
+    );
+  }
+}

package/template/apps/server/src/report-staging/filesystem-io.test.ts

@@ -0,0 +1,85 @@
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "bun:test";
+import { copyAndHashStream, writeBufferFully } from "./filesystem-io.js";
+
+const roots: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+  );
+});
+
+test("源 stream 卡住且 cancel 不返回时 AbortSignal 仍有硬边界", async () => {
+  const root = await mkdtemp(join(tmpdir(), "windy-report-stream-"));
+  roots.push(root);
+  const controller = new AbortController();
+  const body = new ReadableStream<Uint8Array>({
+    pull: () => new Promise<void>(() => undefined),
+    cancel: () => new Promise<void>(() => undefined),
+  });
+  setTimeout(() => controller.abort(), 20);
+  const startedAt = Date.now();
+  await expect(
+    copyAndHashStream({
+      body,
+      destination: join(root, "source.rar"),
+      expectedByteSize: 1,
+      expectedSha256: "0".repeat(64),
+      maxByteSize: 1,
+      signal: controller.signal,
+      heartbeat: async () => undefined,
+    }),
+  ).rejects.toMatchObject({ code: "REPORT_STAGING_CANCELLED" });
+  expect(Date.now() - startedAt).toBeLessThan(1_000);
+});
+
+test("partial write 按实际 bytesWritten 推进并完整写入 chunk", async () => {
+  const input = Buffer.from("partial-write-source");
+  const output = Buffer.alloc(input.byteLength);
+  let calls = 0;
+  await expect(
+    writeBufferFully(input, async (value, offset, length) => {
+      const bytesWritten = Math.min(length, 3);
+      Buffer.from(value).copy(output, offset, offset, offset + bytesWritten);
+      calls++;
+      return { bytesWritten };
+    }),
+  ).resolves.toBe(input.byteLength);
+  expect(calls).toBeGreaterThan(1);
+  expect(output).toEqual(input);
+});
+
+test.each([0, 9])(
+  "无进展或越界 bytesWritten=%d 时 fail closed",
+  async (count) => {
+    await expect(
+      writeBufferFully(Buffer.from("1234"), async () => ({
+        bytesWritten: count,
+      })),
+    ).rejects.toMatchObject({ code: "REPORT_STAGING_INTEGRITY_MISMATCH" });
+  },
+);
+
+test("source receipt 的 size/hash 与实际落盘字节一致", async () => {
+  const root = await mkdtemp(join(tmpdir(), "windy-report-stream-"));
+  roots.push(root);
+  const destination = join(root, "source.rar");
+  const content = Buffer.from("source-byte-integrity");
+  const sha256 = createHash("sha256").update(content).digest("hex");
+  await expect(
+    copyAndHashStream({
+      body: new Blob([content]).stream(),
+      destination,
+      expectedByteSize: content.byteLength,
+      expectedSha256: sha256,
+      maxByteSize: content.byteLength,
+      signal: new AbortController().signal,
+      heartbeat: async () => undefined,
+    }),
+  ).resolves.toEqual({ byteSize: content.byteLength, sha256 });
+  expect(await readFile(destination)).toEqual(content);
+});

package/template/apps/server/src/report-staging/filesystem-io.ts

@@ -0,0 +1,206 @@
+import { createHash } from "node:crypto";
+import { chmod, lstat, mkdir, open, readdir } from "node:fs/promises";
+import { resolve, sep } from "node:path";
+import { ModuleReportStagingError } from "@southwind-ai/server-sdk";
+
+export async function copyAndHashStream(input: {
+  readonly body: ReadableStream<Uint8Array>;
+  readonly destination: string;
+  readonly expectedByteSize: number;
+  readonly expectedSha256: string;
+  readonly maxByteSize: number;
+  readonly signal: AbortSignal;
+  readonly heartbeat: () => Promise<void>;
+}) {
+  await mkdir(resolve(input.destination, ".."), {
+    recursive: true,
+    mode: 0o700,
+  });
+  const file = await open(input.destination, "wx", 0o600);
+  const reader = input.body.getReader();
+  const abort = abortPromise(input.signal);
+  const hash = createHash("sha256");
+  let byteSize = 0;
+  let heartbeatAt = Date.now();
+  try {
+    while (true) {
+      if (input.signal.aborted) throw cancelled();
+      const { value, done } = await Promise.race([
+        reader.read(),
+        abort.promise,
+      ]);
+      if (done) break;
+      const nextByteSize = byteSize + value.byteLength;
+      if (
+        !Number.isSafeInteger(nextByteSize) ||
+        nextByteSize > input.maxByteSize
+      ) {
+        throw limit();
+      }
+      await writeBufferFully(
+        value,
+        (buffer, offset, length) => file.write(buffer, offset, length, null),
+        () => {
+          if (input.signal.aborted) throw cancelled();
+        },
+      );
+      byteSize = nextByteSize;
+      hash.update(value);
+      if (Date.now() - heartbeatAt >= 5_000) {
+        await input.heartbeat();
+        heartbeatAt = Date.now();
+      }
+    }
+    await file.sync();
+    if (input.signal.aborted) throw cancelled();
+  } catch (error) {
+    await cancelReader(reader, error);
+    throw error;
+  } finally {
+    abort.dispose();
+    await file.close();
+    try {
+      reader.releaseLock();
+    } catch {
+      // A bounded cancellation may leave a hostile reader pending.
+    }
+  }
+  const sha256 = hash.digest("hex");
+  if (byteSize !== input.expectedByteSize || sha256 !== input.expectedSha256) {
+    throw new ModuleReportStagingError(
+      "REPORT_STAGING_INTEGRITY_MISMATCH",
+      "报告源文件大小或摘要发生漂移",
+    );
+  }
+  return { byteSize, sha256 };
+}
+
+export async function writeBufferFully(
+  value: Uint8Array,
+  write: (
+    value: Uint8Array,
+    offset: number,
+    length: number,
+  ) => Promise<{ readonly bytesWritten: number }>,
+  requireActive: () => void = () => undefined,
+) {
+  let offset = 0;
+  while (offset < value.byteLength) {
+    requireActive();
+    const requested = value.byteLength - offset;
+    const { bytesWritten } = await write(value, offset, requested);
+    if (bytesWritten <= 0 || bytesWritten > requested) throw writeFailure();
+    offset += bytesWritten;
+    requireActive();
+  }
+  return offset;
+}
+
+export async function makeTreeRemovable(path: string): Promise<void> {
+  const metadata = await lstat(path).catch(() => undefined);
+  if (!metadata?.isDirectory()) return;
+  await chmod(path, 0o700);
+  for (const name of await readdir(path)) {
+    const child = resolve(path, name);
+    const childMetadata = await lstat(child).catch(() => undefined);
+    if (childMetadata?.isDirectory()) await makeTreeRemovable(child);
+  }
+}
+
+export async function hashFile(
+  path: string,
+  signal: AbortSignal,
+  heartbeat: () => Promise<void>,
+) {
+  const hash = createHash("sha256");
+  const file = Bun.file(path);
+  const reader = file.stream().getReader();
+  const abort = abortPromise(signal);
+  let heartbeatAt = Date.now();
+  try {
+    while (true) {
+      if (signal.aborted) throw cancelled();
+      const { value, done } = await Promise.race([
+        reader.read(),
+        abort.promise,
+      ]);
+      if (done) break;
+      hash.update(value);
+      if (Date.now() - heartbeatAt >= 5_000) {
+        await heartbeat();
+        heartbeatAt = Date.now();
+      }
+    }
+    if (signal.aborted) throw cancelled();
+  } catch (error) {
+    await cancelReader(reader, error);
+    throw error;
+  } finally {
+    abort.dispose();
+    try {
+      reader.releaseLock();
+    } catch {
+      // A bounded cancellation may leave a hostile reader pending.
+    }
+  }
+  return hash.digest("hex");
+}
+
+async function cancelReader(
+  reader: ReadableStreamDefaultReader<Uint8Array>,
+  reason: unknown,
+) {
+  await Promise.race([
+    reader.cancel(reason).catch(() => undefined),
+    new Promise<void>((resolve) => setTimeout(resolve, 250)),
+  ]);
+}
+
+function abortPromise(signal: AbortSignal) {
+  let listener: () => void = () => {};
+  const promise = new Promise<never>((_resolve, reject) => {
+    listener = () => reject(cancelled());
+    if (signal.aborted) listener();
+    else signal.addEventListener("abort", listener, { once: true });
+  });
+  return {
+    promise,
+    dispose: () => signal.removeEventListener("abort", listener),
+  };
+}
+
+export function assertInside(root: string, path: string) {
+  const normalizedRoot = resolve(root);
+  const normalizedPath = resolve(path);
+  if (
+    normalizedPath !== normalizedRoot &&
+    !normalizedPath.startsWith(`${normalizedRoot}${sep}`)
+  ) {
+    throw new ModuleReportStagingError(
+      "REPORT_STAGING_UNSAFE_ARCHIVE",
+      "报告路径越过 Host 暂存根目录",
+    );
+  }
+}
+
+function cancelled() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_CANCELLED",
+    "报告暂存已取消",
+    true,
+  );
+}
+
+function limit() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_LIMIT",
+    "报告超过 Host 暂存资源上限",
+  );
+}
+
+function writeFailure() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INTEGRITY_MISMATCH",
+    "Host 未完整写入报告源文件",
+  );
+}

package/template/apps/server/src/report-staging/filesystem-scan.ts

@@ -0,0 +1,85 @@
+import { chmod, lstat, readdir } from "node:fs/promises";
+import { relative, resolve, sep } from "node:path";
+import { ModuleReportStagingError } from "@southwind-ai/server-sdk";
+import { assertInside, hashFile } from "./filesystem-io.js";
+import { createReportStagingHeartbeat } from "./heartbeat.js";
+import type { ReportStagingEntryCatalog } from "./metadata-spool.js";
+
+export async function scanReportStagingTree(input: {
+  readonly root: string;
+  readonly catalog: ReportStagingEntryCatalog;
+  readonly signal: AbortSignal;
+  readonly heartbeat: () => Promise<void>;
+}) {
+  let entryOrdinal = 0;
+  let expandedByteSize = 0;
+  const heartbeat = createReportStagingHeartbeat(input.heartbeat);
+  await visit(input.root);
+  input.catalog.finishScan(new Date().toISOString());
+  if (
+    input.catalog.entryCount !== entryOrdinal ||
+    input.catalog.expandedByteSize !== expandedByteSize
+  ) {
+    throw integrity();
+  }
+  return { entryCount: entryOrdinal, expandedByteSize };
+
+  async function visit(directory: string): Promise<void> {
+    for (const name of (await readdir(directory)).sort()) {
+      if (input.signal.aborted) throw cancelled();
+      await heartbeat();
+      const path = resolve(directory, name);
+      assertInside(input.root, path);
+      const metadata = await lstat(path);
+      const relativePath = relative(input.root, path).split(sep).join("/");
+      if (metadata.isDirectory()) {
+        input.catalog.markScanned({
+          relativePath,
+          directory: true,
+          byteSize: 0,
+        });
+        await chmod(path, 0o700);
+        await visit(path);
+        await chmod(path, 0o500);
+        continue;
+      }
+      rejectSpecialType(metadata);
+      const sha256 = await hashFile(path, input.signal, heartbeat);
+      input.catalog.markScanned({
+        relativePath,
+        directory: false,
+        byteSize: metadata.size,
+        sha256,
+        ordinal: entryOrdinal,
+      });
+      expandedByteSize += metadata.size;
+      entryOrdinal++;
+      await chmod(path, 0o400);
+      await heartbeat();
+    }
+  }
+}
+
+function rejectSpecialType(metadata: Awaited<ReturnType<typeof lstat>>) {
+  if (!metadata.isFile() || metadata.nlink !== 1) {
+    throw new ModuleReportStagingError(
+      "REPORT_STAGING_UNSAFE_ARCHIVE",
+      "报告包含 symlink、hardlink 或特殊文件",
+    );
+  }
+}
+
+function cancelled() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_CANCELLED",
+    "报告暂存已取消",
+    true,
+  );
+}
+
+function integrity() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_INTEGRITY_MISMATCH",
+    "报告 metadata spool 与完整展开结果不一致",
+  );
+}