create-windy 0.5.2 → 0.6.1

package/template/apps/server/src/report-staging/archive-frame-stream.ts

@@ -0,0 +1,341 @@
+import { mkdir, open, type FileHandle } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import type { Readable } from "node:stream";
+import {
+  MODULE_REPORT_STAGING_PATH_MAX_BYTES,
+  ModuleReportStagingError,
+  type ModuleReportStagingPolicyV1,
+} from "@southwind-ai/server-sdk";
+import type { ArchiveListing } from "./archive-process.js";
+import type { ReportStagingEntryCatalog } from "./metadata-spool.js";
+import {
+  reportStagingCancelled,
+  requireReportStagingActive as requireActive,
+} from "./cancellation.js";
+import { assertInside } from "./filesystem-io.js";
+import type { ValidatedArchivePath } from "./path-policy.js";
+
+const MAGIC = Buffer.from([0x57, 0x52, 0x53, 0x46, 1, 0, 0, 0]);
+const ENTRY_FILE = 1;
+const ENTRY_DIRECTORY = 2;
+const DATA = 3;
+const ENTRY_END = 4;
+const ARCHIVE_END = 5;
+const MAX_DATA_BYTES = 64 * 1_024;
+
+export async function consumeArchiveFrames(input: {
+  readonly stdout: Readable;
+  readonly policy: ModuleReportStagingPolicyV1;
+  readonly catalog: ReportStagingEntryCatalog;
+  readonly destination?: string;
+  readonly expected?: ArchiveListing;
+  readonly signal?: AbortSignal;
+}): Promise<ArchiveListing> {
+  requireActive(input.signal);
+  const reader = new ExactReader(
+    input.stdout,
+    maximumFrameBytes(input.policy),
+    input.signal,
+  );
+  if (!(await reader.read(MAGIC.byteLength)).equals(MAGIC)) {
+    throw unsafe("Host 收到无效的报告解包协议头");
+  }
+  let current: CurrentEntry | undefined;
+  let entryIndex = 0;
+  let actualExpandedBytes = 0;
+  let dataFrameCount = 0;
+  let archiveEnded = false;
+  try {
+    while (!archiveEnded) {
+      requireActive(input.signal);
+      const header = await reader.read(5);
+      const type = header[0]!;
+      const length = header.readUInt32BE(1);
+      if (type === ENTRY_FILE || type === ENTRY_DIRECTORY) {
+        if (
+          current ||
+          length < 12 ||
+          length > 12 + MODULE_REPORT_STAGING_PATH_MAX_BYTES
+        ) {
+          throw unsafe("报告 entry framing 无效");
+        }
+        const payload = await reader.read(length);
+        const pathLength = payload.readUInt32BE(0);
+        if (pathLength === 0 || pathLength !== length - 12) {
+          throw unsafe("报告 entry 路径 framing 无效");
+        }
+        const declared = safeSize(payload.readBigUInt64BE(4));
+        const path = decodePath(payload.subarray(12));
+        const directory = type === ENTRY_DIRECTORY;
+        const validated = input.expected
+          ? input.catalog.expectedArchiveEntry(
+              entryIndex,
+              path,
+              directory,
+              declared,
+            )
+          : input.catalog.declareArchiveEntry(path, directory, declared);
+        current = {
+          remaining: declared,
+          directory,
+          file: await openDestination(
+            input.destination,
+            validated,
+            input.signal,
+          ),
+        };
+        entryIndex += 1;
+        continue;
+      }
+      if (type === DATA) {
+        if (
+          !current ||
+          current.directory ||
+          length === 0 ||
+          length > MAX_DATA_BYTES
+        ) {
+          throw unsafe("报告 data framing 无效");
+        }
+        dataFrameCount += 1;
+        if (
+          dataFrameCount > maximumDataFrames(input.policy) ||
+          length > current.remaining ||
+          length > input.policy.maxExpandedBytes - actualExpandedBytes
+        ) {
+          throw limit("报告实际展开数据超过 Host 上限");
+        }
+        const payload = await reader.read(length);
+        const written = current.file
+          ? await writeBounded(
+              current.file,
+              payload,
+              current,
+              input.policy,
+              actualExpandedBytes,
+              input.signal,
+            )
+          : payload.byteLength;
+        current.remaining -= written;
+        actualExpandedBytes += written;
+        continue;
+      }
+      if (type === ENTRY_END) {
+        requireEmpty(length);
+        if (!current || current.remaining !== 0) {
+          throw unsafe("报告 entry 实际大小与声明不一致");
+        }
+        await current.file?.close();
+        requireActive(input.signal);
+        current = undefined;
+        continue;
+      }
+      if (type === ARCHIVE_END) {
+        requireEmpty(length);
+        if (
+          current ||
+          (input.expected && entryIndex !== input.expected.entryCount)
+        ) {
+          throw unsafe("报告解包结束标记与 preflight 不一致");
+        }
+        archiveEnded = true;
+        continue;
+      }
+      throw unsafe("Host 收到未知的报告解包 frame");
+    }
+    if (await reader.hasTrailingByte()) {
+      throw unsafe("报告解包协议包含尾随数据");
+    }
+    requireActive(input.signal);
+    input.catalog.finishArchivePass(entryIndex, actualExpandedBytes);
+    const listing = {
+      catalog: input.catalog,
+      entryCount: entryIndex,
+      expandedByteSize: input.catalog.expandedByteSize,
+    };
+    if (
+      input.expected &&
+      actualExpandedBytes !== input.expected.expandedByteSize
+    ) {
+      throw unsafe("报告实际展开大小与 preflight 不一致");
+    }
+    return listing;
+  } finally {
+    await current?.file?.close().catch(() => undefined);
+  }
+}
+
+interface CurrentEntry {
+  remaining: number;
+  readonly directory: boolean;
+  readonly file?: FileHandle;
+}
+
+async function openDestination(
+  destination: string | undefined,
+  entry: ValidatedArchivePath,
+  signal?: AbortSignal,
+) {
+  requireActive(signal);
+  if (!destination) return undefined;
+  const path = resolve(destination, entry.relativePath);
+  assertInside(destination, path);
+  if (entry.directory) {
+    await mkdir(path, { recursive: true, mode: 0o700 });
+    requireActive(signal);
+    return undefined;
+  }
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  requireActive(signal);
+  const file = await open(path, "wx", 0o600);
+  if (signal?.aborted) {
+    await file.close().catch(() => undefined);
+    throw reportStagingCancelled();
+  }
+  return file;
+}
+
+async function writeBounded(
+  file: FileHandle,
+  payload: Buffer,
+  entry: CurrentEntry,
+  policy: ModuleReportStagingPolicyV1,
+  aggregateWritten: number,
+  signal?: AbortSignal,
+) {
+  let offset = 0;
+  while (offset < payload.byteLength) {
+    requireActive(signal);
+    const requested = payload.byteLength - offset;
+    if (
+      requested > entry.remaining - offset ||
+      requested > policy.maxExpandedBytes - aggregateWritten - offset
+    ) {
+      throw limit("报告实际展开数据超过 Host 上限");
+    }
+    const { bytesWritten } = await file.write(payload, offset, requested, null);
+    if (bytesWritten <= 0 || bytesWritten > requested) {
+      throw unsafe("Host 无法完整写入报告 entry");
+    }
+    requireActive(signal);
+    offset += bytesWritten;
+  }
+  return offset;
+}
+
+class ExactReader {
+  readonly #iterator: AsyncIterator<unknown>;
+  readonly #chunks: Buffer[] = [];
+  #offset = 0;
+  #buffered = 0;
+  #received = 0;
+  #done = false;
+
+  constructor(
+    stream: Readable,
+    private readonly maximumBytes: number,
+    private readonly signal?: AbortSignal,
+  ) {
+    this.#iterator = stream[Symbol.asyncIterator]();
+  }
+
+  async read(length: number): Promise<Buffer> {
+    requireActive(this.signal);
+    while (this.#buffered < length && !this.#done) await this.#pull();
+    if (this.#buffered < length) throw unsafe("报告解包协议被截断");
+    const output = Buffer.allocUnsafe(length);
+    let written = 0;
+    while (written < length) {
+      const chunk = this.#chunks[0]!;
+      const available = chunk.byteLength - this.#offset;
+      const count = Math.min(length - written, available);
+      chunk.copy(output, written, this.#offset, this.#offset + count);
+      written += count;
+      this.#offset += count;
+      this.#buffered -= count;
+      if (this.#offset === chunk.byteLength) {
+        this.#chunks.shift();
+        this.#offset = 0;
+      }
+    }
+    requireActive(this.signal);
+    return output;
+  }
+
+  async hasTrailingByte() {
+    requireActive(this.signal);
+    while (this.#buffered === 0 && !this.#done) await this.#pull();
+    requireActive(this.signal);
+    return this.#buffered > 0;
+  }
+
+  async #pull() {
+    requireActive(this.signal);
+    let next: IteratorResult<unknown>;
+    try {
+      next = await this.#iterator.next();
+    } catch (error) {
+      if (this.signal?.aborted) throw reportStagingCancelled();
+      throw error;
+    }
+    requireActive(this.signal);
+    if (next.done) {
+      this.#done = true;
+      return;
+    }
+    if (!(next.value instanceof Uint8Array)) {
+      throw unsafe("报告解包协议输出类型无效");
+    }
+    const value = Buffer.from(next.value);
+    this.#received += value.byteLength;
+    if (
+      !Number.isSafeInteger(this.#received) ||
+      this.#received > this.maximumBytes
+    ) {
+      throw limit("报告解包 frame 总量超过 Host 上限");
+    }
+    this.#chunks.push(value);
+    this.#buffered += value.byteLength;
+  }
+}
+
+function safeSize(value: bigint) {
+  if (value > BigInt(Number.MAX_SAFE_INTEGER)) {
+    throw unsafe("报告 entry 声明大小无效");
+  }
+  return Number(value);
+}
+
+function decodePath(value: Buffer) {
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(value);
+  } catch {
+    throw unsafe("报告 entry 路径不是有效 UTF-8");
+  }
+}
+
+function requireEmpty(length: number) {
+  if (length !== 0) throw unsafe("报告结束 frame 包含无效 payload");
+}
+
+function maximumDataFrames(policy: ModuleReportStagingPolicyV1) {
+  return (
+    Math.ceil(policy.maxExpandedBytes / MAX_DATA_BYTES) + policy.maxEntries
+  );
+}
+
+function maximumFrameBytes(policy: ModuleReportStagingPolicyV1) {
+  const metadata =
+    policy.maxEntries * (5 + 12 + MODULE_REPORT_STAGING_PATH_MAX_BYTES + 5);
+  const framing = maximumDataFrames(policy) * 5;
+  const maximum =
+    MAGIC.byteLength + policy.maxExpandedBytes + metadata + framing + 5;
+  return Number.isSafeInteger(maximum) ? maximum : Number.MAX_SAFE_INTEGER;
+}
+
+function unsafe(message: string) {
+  return new ModuleReportStagingError("REPORT_STAGING_UNSAFE_ARCHIVE", message);
+}
+
+function limit(message: string) {
+  return new ModuleReportStagingError("REPORT_STAGING_LIMIT", message);
+}

package/template/apps/server/src/report-staging/archive-process.test.ts

@@ -0,0 +1,302 @@
+import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, test } from "bun:test";
+import type { ModuleReportStagingPolicyV1 } from "@southwind-ai/server-sdk";
+import { SupervisedLibarchiveProcess } from "./archive-process.js";
+import { ReportStagingEntryCatalog } from "./metadata-spool.js";
+import { runSupervisedProcess } from "./supervised-process.js";
+
+const roots: string[] = [];
+const catalogs: ReportStagingEntryCatalog[] = [];
+const helper = Bun.which("windy-report-archive-stream");
+const testWithCanonicalHelper = helper ? test : test.skip;
+const rar5Fixture = Buffer.from(
+  "UmFyIRoHAQAzkrXlCgEFBgAFAQGAgAA4MAZjLAIDC50ABJ0ApIMCtEOglYAAAQ5oZWxsb3dvcmxkLnR4dAoDE34Oq1tW6Q4aaGVsbG8gbGliYXJjaGl2ZSB0ZXN0IHN1aXRlIQodd1ZRAwUEAA==",
+  "base64",
+);
+
+afterEach(async () => {
+  catalogs.splice(0).forEach((catalog) => catalog.close());
+  await Promise.all(
+    roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+  );
+});
+
+describe("受监督 libarchive framing helper", () => {
+  testWithCanonicalHelper(
+    "真实 RAR5 probe 与同序 synthetic volume 的 list/extract 闭环",
+    async () => {
+      const root = await temporaryRoot();
+      const archive = join(root, "report.rar");
+      const first = join(root, "report.part1");
+      const second = join(root, "report.part2");
+      const unrelated = join(root, "unrelated.bin");
+      const destination = join(root, "destination");
+      await writeFile(archive, rar5Fixture);
+      const split = Math.floor(rar5Fixture.byteLength / 2);
+      await writeFile(first, rar5Fixture.subarray(0, split));
+      await writeFile(second, rar5Fixture.subarray(split));
+      await writeFile(unrelated, "not-a-volume");
+
+      const process = new SupervisedLibarchiveProcess(helper!);
+      const signal = new AbortController().signal;
+      await process.probeRar5(signal);
+      const listing = await process.list(
+        [first, second],
+        await createCatalog(),
+        policy(),
+        signal,
+        heartbeat,
+      );
+      expect(listing.expandedByteSize).toBe(29);
+      expect(listing.catalog.declaredArchiveEntry(0)).toMatchObject({
+        relativePath: "helloworld.txt",
+        kind: "file",
+        declaredSize: 29,
+      });
+      await process.extract(
+        [first, second],
+        destination,
+        listing,
+        policy(),
+        signal,
+        heartbeat,
+      );
+      expect(await Bun.file(join(destination, "helloworld.txt")).text()).toBe(
+        "hello libarchive test suite!\n",
+      );
+      await expect(
+        process.list(
+          [first],
+          await createCatalog(),
+          policy(),
+          signal,
+          heartbeat,
+        ),
+      ).rejects.toBeDefined();
+      await expect(
+        process.list(
+          [second, first],
+          await createCatalog(),
+          policy(),
+          signal,
+          heartbeat,
+        ),
+      ).rejects.toBeDefined();
+      await expect(
+        process.list(
+          [archive, unrelated],
+          await createCatalog(),
+          policy(),
+          signal,
+          heartbeat,
+        ),
+      ).rejects.toBeDefined();
+    },
+  );
+
+  test("缺少固定 helper 时如实报告 unavailable，不回退 bsdtar", async () => {
+    await expect(
+      new SupervisedLibarchiveProcess("windy-helper-does-not-exist").probeRar5(
+        new AbortController().signal,
+      ),
+    ).rejects.toMatchObject({
+      code: "REPORT_STAGING_UNSUPPORTED_ARCHIVE",
+    });
+  });
+
+  test("RAR5 probe 必须通过 helper 的内容读取闭环", async () => {
+    const valid = await framedExecutable([
+      {
+        path: "helloworld.txt",
+        body: Buffer.from("hello libarchive test suite!\n"),
+      },
+    ]);
+    await expect(
+      new SupervisedLibarchiveProcess(valid).probeRar5(
+        new AbortController().signal,
+      ),
+    ).resolves.toBeUndefined();
+
+    const wrong = await framedExecutable([
+      { path: "helloworld.txt", body: Buffer.from("wrong") },
+    ]);
+    await expect(
+      new SupervisedLibarchiveProcess(wrong).probeRar5(
+        new AbortController().signal,
+      ),
+    ).rejects.toMatchObject({
+      code: "REPORT_STAGING_UNSUPPORTED_ARCHIVE",
+    });
+  });
+
+  test("timeout 有硬边界并在 grace 后 SIGKILL", async () => {
+    const binary = await hangingExecutable();
+    const startedAt = Date.now();
+    await expect(
+      new SupervisedLibarchiveProcess(binary).list(
+        ["ignored.rar"],
+        await createCatalog(),
+        { ...policy(), maxProcessMs: 20 },
+        new AbortController().signal,
+        heartbeat,
+      ),
+    ).rejects.toMatchObject({
+      code: "REPORT_STAGING_PROCESS_TIMEOUT",
+    });
+    expect(Date.now() - startedAt).toBeLessThan(1_000);
+  });
+
+  test("AbortSignal 取消同样受 grace 硬边界约束", async () => {
+    const binary = await hangingExecutable();
+    const controller = new AbortController();
+    setTimeout(() => controller.abort(), 20);
+    const startedAt = Date.now();
+    await expect(
+      new SupervisedLibarchiveProcess(binary).list(
+        ["ignored.rar"],
+        await createCatalog(),
+        policy(),
+        controller.signal,
+        heartbeat,
+      ),
+    ).rejects.toMatchObject({ code: "REPORT_STAGING_CANCELLED" });
+    expect(Date.now() - startedAt).toBeLessThan(1_000);
+  });
+
+  test("helper 已退出但 Host consumer 卡住时 deadline 仍生效", async () => {
+    const binary = await framedExecutable([]);
+    const startedAt = Date.now();
+    await expect(
+      runSupervisedProcess(binary, ["ignored.rar"], {
+        signal: new AbortController().signal,
+        timeoutMs: 20,
+        maxStdoutBytes: 0,
+        consumeStdout: () => new Promise(() => {}),
+      }),
+    ).rejects.toMatchObject({ code: "REPORT_STAGING_PROCESS_TIMEOUT" });
+    expect(Date.now() - startedAt).toBeLessThan(1_000);
+  });
+
+  test("deadline 会协作取消 Host consumer 并 destroy stdout", async () => {
+    const binary = await framedExecutable([]);
+    let observedAbort = false;
+    await expect(
+      runSupervisedProcess(binary, ["ignored.rar"], {
+        signal: new AbortController().signal,
+        timeoutMs: 20,
+        maxStdoutBytes: 0,
+        consumeStdout: (_stdout, signal) =>
+          new Promise<void>((resolve) => {
+            signal.addEventListener(
+              "abort",
+              () => {
+                observedAbort = true;
+                resolve();
+              },
+              { once: true },
+            );
+          }),
+      }),
+    ).rejects.toMatchObject({ code: "REPORT_STAGING_PROCESS_TIMEOUT" });
+    expect(observedAbort).toBe(true);
+  });
+
+  test("native helper 对 entry/archive 加密状态 unknown 也 fail closed", async () => {
+    const source = await readFile(
+      join(import.meta.dir, "native/archive-frame-helper.c"),
+      "utf8",
+    );
+    expect(source).toContain("archive_entry_is_encrypted(entry) != 0");
+    expect(source).toContain("archive_read_has_encrypted_entries(reader) != 0");
+    expect(source).toContain(
+      "!all_volumes_consumed(reader, volumes, volume_count)",
+    );
+    expect(source).toContain("archive_filter_bytes(reader, -1)");
+    expect(source).toContain("archive_read_set_switch_callback");
+    expect(source).toContain("#define MAX_PATH_BYTES 4096U");
+  });
+});
+
+async function temporaryRoot() {
+  const root = await mkdtemp(join(tmpdir(), "windy-archive-process-"));
+  roots.push(root);
+  return root;
+}
+
+async function createCatalog() {
+  const root = await temporaryRoot();
+  const catalog = await ReportStagingEntryCatalog.create({
+    path: join(root, "metadata", "index.sqlite"),
+    groupId: "rsg_11111111111111111111111111111111",
+    policy: policy(),
+  });
+  catalogs.push(catalog);
+  return catalog;
+}
+
+async function framedExecutable(
+  entries: readonly { readonly path: string; readonly body: Buffer }[],
+) {
+  const root = await temporaryRoot();
+  const path = join(root, "fake-helper");
+  const output = archiveFrames(entries).toString("base64");
+  await writeFile(
+    path,
+    `#!${process.execPath}\nprocess.stdout.write(Buffer.from(${JSON.stringify(output)}, "base64"));\n`,
+  );
+  await chmod(path, 0o700);
+  return path;
+}
+
+async function hangingExecutable() {
+  const root = await temporaryRoot();
+  const path = join(root, "hanging-helper");
+  await writeFile(path, "#!/bin/sh\ntrap '' TERM\nwhile :; do :; done\n");
+  await chmod(path, 0o700);
+  return path;
+}
+
+function archiveFrames(
+  entries: readonly { readonly path: string; readonly body: Buffer }[],
+) {
+  const frames: Buffer[] = [Buffer.from([0x57, 0x52, 0x53, 0x46, 1, 0, 0, 0])];
+  for (const entry of entries) {
+    const path = Buffer.from(entry.path);
+    const metadata = Buffer.alloc(12 + path.byteLength);
+    metadata.writeUInt32BE(path.byteLength, 0);
+    metadata.writeBigUInt64BE(BigInt(entry.body.byteLength), 4);
+    path.copy(metadata, 12);
+    frames.push(frame(1, metadata), frame(3, entry.body), frame(4));
+  }
+  frames.push(frame(5));
+  return Buffer.concat(frames);
+}
+
+function frame(type: number, payload: Buffer = Buffer.alloc(0)): Buffer {
+  const header = Buffer.alloc(5);
+  header[0] = type;
+  header.writeUInt32BE(payload.byteLength, 1);
+  return Buffer.concat([header, payload]);
+}
+
+async function heartbeat() {}
+
+function policy(): ModuleReportStagingPolicyV1 {
+  return {
+    purpose: "evidence.report-staging",
+    sourcePurposes: ["evidence.original"],
+    retentionMs: 60_000,
+    claimTtlMs: 30_000,
+    maxReadLeaseMs: 10_000,
+    maxProcessMs: 10_000,
+    maxSources: 4,
+    maxArchiveBytes: 1_024 * 1_024,
+    maxExpandedBytes: 4_096,
+    maxEntryBytes: 2_048,
+    maxEntries: 20,
+    maxCompressionRatio: 100,
+    maxPathDepth: 8,
+  };
+}