create-windy 0.5.2 → 0.6.0

package/template/apps/server/src/report-staging/filesystem-store.ts

@@ -0,0 +1,346 @@
+import { chmod, mkdir, rename, rm } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import {
+  ModuleReportStagingError,
+  type ModuleReportStagingPolicyV1,
+  type ModuleReportStagingSourceMode,
+  type ModuleReportStagingSourceReader,
+} from "@southwind-ai/server-sdk";
+import type {
+  PreparedReportStagingGroup,
+  ReportStagingSourceRecord,
+} from "./contracts.js";
+import type { ArchiveProcess } from "./archive-process.js";
+import { resolveArchiveVolumes } from "./archive-volumes.js";
+import {
+  discardFilesystemGroup,
+  discardFilesystemOrphans,
+  discardSealSiblings,
+} from "./filesystem-cleanup.js";
+import { forbidden, integrityMismatch } from "./filesystem-errors.js";
+import {
+  assertInside,
+  copyAndHashStream,
+  makeTreeRemovable,
+} from "./filesystem-io.js";
+import {
+  requireOpaqueGroupId,
+  requireOpaqueSealId,
+} from "./filesystem-identifiers.js";
+import { scanReportStagingTree } from "./filesystem-scan.js";
+import { createReportStagingHeartbeat } from "./heartbeat.js";
+import { ReportStagingEntryCatalog } from "./metadata-spool.js";
+import { openVerifiedSealedFile } from "./sealed-file.js";
+
+export class ReportStagingFilesystemStore {
+  readonly #root: string;
+
+  constructor(
+    root: string,
+    private readonly archive: ArchiveProcess,
+  ) {
+    this.#root = resolve(root);
+  }
+
+  async prepare(input: {
+    readonly groupId: string;
+    readonly sealId: string;
+    readonly sourceMode: ModuleReportStagingSourceMode;
+    readonly entrypoint: string;
+    readonly sources: readonly ReportStagingSourceRecord[];
+    readonly policy: ModuleReportStagingPolicyV1;
+    readonly reader: ModuleReportStagingSourceReader;
+    readonly signal: AbortSignal;
+    readonly heartbeat: () => Promise<void>;
+  }): Promise<PreparedReportStagingGroup> {
+    await this.ensureRoot();
+    const work = this.workPath(input.groupId, input.sealId);
+    const sealed = this.sealPath(input.groupId, input.sealId);
+    await makeTreeRemovable(work);
+    await makeTreeRemovable(sealed);
+    await rm(work, { recursive: true, force: true });
+    await rm(sealed, { recursive: true, force: true });
+    const sourceRoot = resolve(work, "sources");
+    const entryRoot = resolve(work, "entries");
+    const catalog = await ReportStagingEntryCatalog.create({
+      path: resolve(work, ".metadata", "index.sqlite"),
+      groupId: input.groupId,
+      policy: input.policy,
+    });
+    await mkdir(sourceRoot, { recursive: true, mode: 0o700 });
+    await mkdir(entryRoot, { recursive: true, mode: 0o700 });
+    try {
+      const destinationRoot =
+        input.sourceMode === "archive" ? sourceRoot : entryRoot;
+      const sources = await this.copySources(input, destinationRoot);
+      const archiveByteSize = sources.reduce(
+        (total, source) => total + source.byteSize!,
+        0,
+      );
+      const listed =
+        input.sourceMode === "archive"
+          ? await this.extractArchive(
+              input,
+              sourceRoot,
+              entryRoot,
+              archiveByteSize,
+              sources,
+              catalog,
+            )
+          : undefined;
+      if (!listed) {
+        for (const source of sources) {
+          catalog.declareArchiveEntry(
+            source.relativePath,
+            false,
+            source.byteSize!,
+          );
+        }
+        catalog.finishArchivePass(sources.length, archiveByteSize);
+      }
+      const scanned = await scanReportStagingTree({
+        root: entryRoot,
+        catalog,
+        signal: input.signal,
+        heartbeat: input.heartbeat,
+      });
+      if (!listed) {
+        for (const source of sources) {
+          const entry = catalog.listEntries(source.ordinal - 1, 1)[0];
+          if (
+            entry?.relativePath !== source.relativePath ||
+            entry.byteSize !== source.byteSize ||
+            entry.sha256 !== source.sha256
+          ) {
+            throw integrityMismatch();
+          }
+        }
+        if (archiveByteSize !== scanned.expandedByteSize) {
+          throw integrityMismatch();
+        }
+      }
+      await rm(sourceRoot, { recursive: true, force: true });
+      await chmod(entryRoot, 0o500);
+      await input.heartbeat();
+      await mkdir(dirname(sealed), { recursive: true, mode: 0o700 });
+      await rename(work, sealed);
+      catalog.relocate(resolve(sealed, ".metadata", "index.sqlite"));
+      await chmod(sealed, 0o500);
+      return {
+        sealId: input.sealId,
+        sources,
+        catalog,
+        archiveByteSize,
+        expandedByteSize: scanned.expandedByteSize,
+      };
+    } catch (error) {
+      catalog.close();
+      await makeTreeRemovable(work);
+      await makeTreeRemovable(sealed);
+      await rm(work, { recursive: true, force: true }).catch(() => undefined);
+      await rm(sealed, { recursive: true, force: true }).catch(() => undefined);
+      throw error;
+    }
+  }
+
+  async openEntry(input: {
+    readonly groupId: string;
+    readonly sealId: string;
+    readonly relativePath: string;
+    readonly byteSize: number;
+    readonly sha256: string;
+    readonly signal: AbortSignal;
+  }) {
+    const entryRoot = resolve(
+      this.sealPath(input.groupId, input.sealId),
+      "entries",
+    );
+    const path = resolve(entryRoot, input.relativePath);
+    assertInside(entryRoot, path);
+    return openVerifiedSealedFile({
+      path,
+      expectedByteSize: input.byteSize,
+      expectedSha256: input.sha256,
+      signal: input.signal,
+    });
+  }
+
+  async discard(groupId: string): Promise<void> {
+    const work = this.workGroupPath(groupId);
+    const sealed = this.groupPath(groupId);
+    await discardFilesystemGroup(work, sealed);
+  }
+
+  async discardOrphans(groupId: string, signal: AbortSignal): Promise<void> {
+    await discardFilesystemOrphans(
+      this.workGroupPath(groupId),
+      this.groupPath(groupId),
+      signal,
+    );
+  }
+
+  async discardPreparedSiblings(
+    groupId: string,
+    keepSealId: string,
+  ): Promise<void> {
+    requireOpaqueSealId(keepSealId);
+    await discardSealSiblings(
+      this.workGroupPath(groupId),
+      this.groupPath(groupId),
+      keepSealId,
+    );
+  }
+
+  async discardPrepared(groupId: string, sealId: string): Promise<void> {
+    const work = this.workPath(groupId, sealId);
+    const sealed = this.sealPath(groupId, sealId);
+    await makeTreeRemovable(work);
+    await makeTreeRemovable(sealed);
+    await rm(work, { recursive: true, force: true });
+    await rm(sealed, { recursive: true, force: true });
+  }
+
+  private async copySources(
+    input: Parameters<ReportStagingFilesystemStore["prepare"]>[0],
+    destinationRoot: string,
+  ) {
+    const totalByteLimit =
+      input.sourceMode === "archive"
+        ? input.policy.maxArchiveBytes
+        : input.policy.maxExpandedBytes;
+    let declaredBytes = 0;
+    let copiedBytes = 0;
+    const copied: ReportStagingSourceRecord[] = [];
+    const heartbeat = createReportStagingHeartbeat(input.heartbeat);
+    for (const source of input.sources) {
+      await heartbeat();
+      const opened = await input.reader.open({
+        fileId: source.fileId,
+        purpose: source.filePurpose,
+        resourceType: source.resourceType,
+        resourceId: source.resourceId,
+      });
+      if (
+        opened.id !== source.fileId ||
+        opened.purpose !== source.filePurpose ||
+        !Number.isSafeInteger(opened.byteSize) ||
+        opened.byteSize < 0 ||
+        !/^[0-9a-f]{64}$/.test(opened.sha256)
+      ) {
+        throw forbidden();
+      }
+      declaredBytes += opened.byteSize;
+      if (
+        !Number.isSafeInteger(declaredBytes) ||
+        declaredBytes > totalByteLimit
+      ) {
+        throw new ModuleReportStagingError(
+          "REPORT_STAGING_LIMIT",
+          "报告源文件总量超过 Host 上限",
+        );
+      }
+      if (
+        input.sourceMode === "expanded" &&
+        opened.byteSize > input.policy.maxEntryBytes
+      ) {
+        throw new ModuleReportStagingError(
+          "REPORT_STAGING_LIMIT",
+          "报告 entry 超过 Host 上限",
+        );
+      }
+      const destination = resolve(destinationRoot, source.relativePath);
+      assertInside(destinationRoot, destination);
+      const actual = await copyAndHashStream({
+        body: opened.body,
+        destination,
+        expectedByteSize: opened.byteSize,
+        expectedSha256: opened.sha256,
+        maxByteSize: Math.min(
+          totalByteLimit - copiedBytes,
+          input.sourceMode === "archive"
+            ? totalByteLimit
+            : input.policy.maxEntryBytes,
+        ),
+        signal: input.signal,
+        heartbeat,
+      });
+      copiedBytes += actual.byteSize;
+      copied.push({ ...source, ...actual });
+      await heartbeat();
+    }
+    return copied;
+  }
+
+  private async extractArchive(
+    input: Parameters<ReportStagingFilesystemStore["prepare"]>[0],
+    sourceRoot: string,
+    entryRoot: string,
+    archiveByteSize: number,
+    sources: readonly ReportStagingSourceRecord[],
+    catalog: ReportStagingEntryCatalog,
+  ) {
+    const archivePaths = await resolveArchiveVolumes(
+      sourceRoot,
+      input.entrypoint,
+      sources,
+    );
+    await this.archive.probeRar5(input.signal);
+    const listing = await this.archive.list(
+      archivePaths,
+      catalog,
+      input.policy,
+      input.signal,
+      input.heartbeat,
+    );
+    if (
+      listing.expandedByteSize / Math.max(archiveByteSize, 1) >
+      input.policy.maxCompressionRatio
+    ) {
+      throw new ModuleReportStagingError(
+        "REPORT_STAGING_LIMIT",
+        "报告声明展开压缩比超过 Host 上限",
+      );
+    }
+    await this.archive.extract(
+      archivePaths,
+      entryRoot,
+      listing,
+      input.policy,
+      input.signal,
+      input.heartbeat,
+    );
+    return listing;
+  }
+
+  private async ensureRoot() {
+    await mkdir(resolve(this.#root, ".work"), {
+      recursive: true,
+      mode: 0o700,
+    });
+    await mkdir(resolve(this.#root, "groups"), {
+      recursive: true,
+      mode: 0o700,
+    });
+    await chmod(this.#root, 0o700);
+  }
+
+  private workGroupPath(groupId: string) {
+    requireOpaqueGroupId(groupId);
+    return resolve(this.#root, ".work", groupId);
+  }
+
+  private workPath(groupId: string, sealId: string) {
+    requireOpaqueSealId(sealId);
+    return resolve(this.workGroupPath(groupId), sealId);
+  }
+
+  private groupPath(groupId: string) {
+    requireOpaqueGroupId(groupId);
+    return resolve(this.#root, "groups", groupId.slice(4, 6), groupId);
+  }
+
+  private sealPath(groupId: string, sealId: string) {
+    requireOpaqueSealId(sealId);
+    return resolve(this.groupPath(groupId), sealId);
+  }
+}

package/template/apps/server/src/report-staging/heartbeat.test.ts

@@ -0,0 +1,47 @@
+import { expect, test } from "bun:test";
+import {
+  createReportStagingHeartbeat,
+  withPeriodicReportStagingHeartbeat,
+} from "./heartbeat.js";
+
+test("共享 heartbeat 不会被大量短 entry 重置", async () => {
+  let now = 0;
+  let renewals = 0;
+  const heartbeat = createReportStagingHeartbeat(
+    async () => {
+      renewals++;
+    },
+    () => now,
+  );
+  for (let entry = 0; entry < 20_000; entry++) {
+    now++;
+    await heartbeat();
+  }
+  expect(renewals).toBe(4);
+});
+
+test("整组 orphan 删除完成后强制续租并传播失败", async () => {
+  let renewals = 0;
+  await expect(
+    withPeriodicReportStagingHeartbeat(
+      async () => "removed",
+      async () => {
+        renewals++;
+      },
+    ),
+  ).resolves.toBe("removed");
+  expect(renewals).toBe(1);
+
+  await expect(
+    withPeriodicReportStagingHeartbeat(
+      async (signal) => {
+        while (!signal.aborted) await Bun.sleep(1);
+        throw signal.reason;
+      },
+      async () => {
+        throw new Error("claim lost");
+      },
+      1,
+    ),
+  ).rejects.toThrow("claim lost");
+});

package/template/apps/server/src/report-staging/heartbeat.ts

@@ -0,0 +1,53 @@
+const HEARTBEAT_INTERVAL_MS = 5_000;
+
+export function createReportStagingHeartbeat(
+  renew: () => Promise<void>,
+  now: () => number = Date.now,
+) {
+  let renewedAt = now();
+  return async () => {
+    if (now() - renewedAt < HEARTBEAT_INTERVAL_MS) return;
+    await renew();
+    renewedAt = now();
+  };
+}
+
+export async function withPeriodicReportStagingHeartbeat<T>(
+  operation: (signal: AbortSignal) => Promise<T>,
+  renew: () => Promise<void>,
+  intervalMs = HEARTBEAT_INTERVAL_MS,
+): Promise<T> {
+  let heartbeatFailure: unknown;
+  let heartbeatFailed = false;
+  let heartbeatInFlight: Promise<void> | undefined;
+  const controller = new AbortController();
+  const pulse = () => {
+    if (heartbeatInFlight || heartbeatFailed) return;
+    heartbeatInFlight = renew()
+      .catch((error) => {
+        heartbeatFailure = error;
+        heartbeatFailed = true;
+        controller.abort(error);
+      })
+      .finally(() => {
+        heartbeatInFlight = undefined;
+      });
+  };
+  const interval = setInterval(pulse, intervalMs);
+  let result: T | undefined;
+  let operationFailure: unknown;
+  let operationFailed = false;
+  try {
+    result = await operation(controller.signal);
+  } catch (error) {
+    operationFailure = error;
+    operationFailed = true;
+  } finally {
+    clearInterval(interval);
+  }
+  await heartbeatInFlight;
+  if (heartbeatFailed) throw heartbeatFailure;
+  if (operationFailed) throw operationFailure;
+  await renew();
+  return result!;
+}

package/template/apps/server/src/report-staging/manifest-stream.ts

@@ -0,0 +1,46 @@
+import { createHash } from "node:crypto";
+import type {
+  ReportStagingEntryRecord,
+  ReportStagingGroupRecord,
+  ReportStagingSourceRecord,
+} from "./contracts.js";
+
+export function createManifestHash(
+  group: Pick<
+    ReportStagingGroupRecord,
+    "groupId" | "purpose" | "sourceMode" | "entrypoint"
+  >,
+  sources: readonly ReportStagingSourceRecord[],
+) {
+  const hash = createHash("sha256");
+  hash.update(
+    `{"version":"1","groupId":${JSON.stringify(group.groupId)},"purpose":${JSON.stringify(group.purpose)},"sourceMode":${JSON.stringify(group.sourceMode)},"entrypoint":${JSON.stringify(group.entrypoint)},"sources":[`,
+  );
+  let separator = "";
+  for (const { sourceId, relativePath, byteSize, sha256 } of sources) {
+    hash.update(separator);
+    hash.update(JSON.stringify({ sourceId, relativePath, byteSize, sha256 }));
+    separator = ",";
+  }
+  hash.update('],"entries":[');
+  separator = "";
+  return {
+    add({
+      entryId,
+      relativePath,
+      byteSize,
+      sha256,
+      ordinal,
+    }: ReportStagingEntryRecord) {
+      hash.update(separator);
+      hash.update(
+        JSON.stringify({ entryId, relativePath, byteSize, sha256, ordinal }),
+      );
+      separator = ",";
+    },
+    digest() {
+      hash.update("]}");
+      return hash.digest("hex");
+    },
+  };
+}