create-windy 0.5.1 → 0.6.0

package/template/apps/server/src/report-staging/memory-repository.ts

@@ -0,0 +1,256 @@
+import type {
+  ReportStagingClaimRecord,
+  ReportStagingGroupRecord,
+  ReportStagingReadLeaseRecord,
+  ReportStagingSourceRecord,
+  SealedReportStagingGroup,
+} from "./contracts.js";
+import type { ReportStagingRepository } from "./repository.js";
+import type { ReportStagingEntryCatalog } from "./metadata-spool.js";
+
+export class InMemoryReportStagingRepository implements ReportStagingRepository {
+  readonly #groups = new Map<string, ReportStagingGroupRecord>();
+  readonly #sources = new Map<string, ReportStagingSourceRecord[]>();
+  readonly #entries = new Map<string, ReportStagingEntryCatalog>();
+  readonly #claims = new Map<string, ReportStagingClaimRecord>();
+  readonly #leases = new Map<string, ReportStagingReadLeaseRecord>();
+
+  async createOrGetGroup(
+    group: ReportStagingGroupRecord,
+    sources: readonly ReportStagingSourceRecord[],
+  ) {
+    const existing = [...this.#groups.values()].find(
+      (candidate) =>
+        candidate.moduleName === group.moduleName &&
+        candidate.ownerId === group.ownerId &&
+        candidate.scopeHash === group.scopeHash &&
+        candidate.purpose === group.purpose &&
+        candidate.idempotencyHash === group.idempotencyHash,
+    );
+    if (existing) return { group: existing, created: false };
+    this.#groups.set(group.groupId, group);
+    this.#sources.set(group.groupId, [...sources]);
+    return { group, created: true };
+  }
+
+  async getGroup(groupId: string) {
+    return this.#groups.get(groupId);
+  }
+
+  async getSources(groupId: string) {
+    return [...(this.#sources.get(groupId) ?? [])];
+  }
+
+  async getEntry(groupId: string, entryId: string) {
+    return this.#entries.get(groupId)?.getEntry(entryId);
+  }
+
+  async listEntries(
+    groupId: string,
+    afterOrdinal: number | undefined,
+    limit: number,
+  ) {
+    return this.#entries.get(groupId)?.listEntries(afterOrdinal, limit) ?? [];
+  }
+
+  async acquireClaim(
+    groupId: string,
+    tokenHash: string,
+    now: string,
+    expiresAt: string,
+  ) {
+    const group = this.#groups.get(groupId);
+    if (!group || group.status !== "indexing" || group.expiresAt <= now) {
+      return undefined;
+    }
+    const existing = this.#claims.get(groupId);
+    if (existing && existing.expiresAt > now) return undefined;
+    const claim: ReportStagingClaimRecord = {
+      groupId,
+      tokenHash,
+      generation: (existing?.generation ?? 0) + 1,
+      expiresAt,
+      createdAt: existing?.createdAt ?? now,
+      updatedAt: now,
+    };
+    this.#claims.set(groupId, claim);
+    return claim;
+  }
+
+  async renewClaim(
+    groupId: string,
+    tokenHash: string,
+    now: string,
+    expiresAt: string,
+  ) {
+    const existing = this.#claims.get(groupId);
+    if (
+      !existing ||
+      existing.tokenHash !== tokenHash ||
+      existing.expiresAt <= now
+    ) {
+      return false;
+    }
+    this.#claims.set(groupId, { ...existing, expiresAt, updatedAt: now });
+    return true;
+  }
+
+  async releaseClaim(groupId: string, tokenHash: string) {
+    if (this.#claims.get(groupId)?.tokenHash === tokenHash) {
+      this.#claims.delete(groupId);
+    }
+  }
+
+  async sealGroup(
+    groupId: string,
+    tokenHash: string,
+    sealed: SealedReportStagingGroup,
+  ) {
+    const group = this.#groups.get(groupId);
+    const claim = this.#claims.get(groupId);
+    if (
+      !group ||
+      group.status !== "indexing" ||
+      group.expiresAt <= sealed.updatedAt ||
+      claim?.tokenHash !== tokenHash ||
+      claim.expiresAt <= sealed.updatedAt ||
+      sealed.sealId !== tokenHash ||
+      sealed.catalog.expandedByteSize !== sealed.expandedByteSize
+    ) {
+      return undefined;
+    }
+    const ready: ReportStagingGroupRecord = {
+      ...group,
+      status: "ready",
+      entryCount: sealed.catalog.entryCount,
+      archiveByteSize: sealed.archiveByteSize,
+      expandedByteSize: sealed.expandedByteSize,
+      sealId: sealed.sealId,
+      manifestSha256: sealed.manifestSha256,
+      failureCode: undefined,
+      revision: group.revision + 1,
+      updatedAt: sealed.updatedAt,
+    };
+    this.#groups.set(groupId, ready);
+    this.#sources.set(groupId, [...sealed.sources]);
+    this.#entries.set(groupId, sealed.catalog.forkReadOnly());
+    this.#claims.delete(groupId);
+    return ready;
+  }
+
+  async failGroup(
+    groupId: string,
+    tokenHash: string,
+    failureCode: string,
+    updatedAt: string,
+  ) {
+    const group = this.#groups.get(groupId);
+    const claim = this.#claims.get(groupId);
+    if (
+      !group ||
+      group.status !== "indexing" ||
+      group.expiresAt <= updatedAt ||
+      claim?.tokenHash !== tokenHash ||
+      claim.expiresAt <= updatedAt
+    ) {
+      return undefined;
+    }
+    const failed: ReportStagingGroupRecord = {
+      ...group,
+      status: "failed",
+      failureCode: failureCode as ReportStagingGroupRecord["failureCode"],
+      revision: group.revision + 1,
+      updatedAt,
+    };
+    this.#groups.set(groupId, failed);
+    this.#claims.delete(groupId);
+    return failed;
+  }
+
+  async acquireReadLease(lease: ReportStagingReadLeaseRecord, now: string) {
+    const group = this.#groups.get(lease.groupId);
+    if (!group || group.status !== "ready" || group.expiresAt <= now) {
+      return undefined;
+    }
+    this.#leases.set(lease.leaseId, lease);
+    return lease;
+  }
+
+  async getReadLease(groupId: string, leaseId: string) {
+    const lease = this.#leases.get(leaseId);
+    return lease?.groupId === groupId ? lease : undefined;
+  }
+
+  async closeReadLease(groupId: string, leaseId: string, closedAt: string) {
+    const lease = this.#leases.get(leaseId);
+    if (!lease || lease.groupId !== groupId || lease.closedAt) return false;
+    this.#leases.set(leaseId, { ...lease, closedAt });
+    return true;
+  }
+
+  async markDiscarding(groupId: string, now: string) {
+    const group = this.#groups.get(groupId);
+    if (!group || group.status === "discarded") return group;
+    const claim = this.#claims.get(groupId);
+    if (claim && claim.expiresAt > now) return undefined;
+    if (
+      [...this.#leases.values()].some(
+        (lease) =>
+          lease.groupId === groupId && !lease.closedAt && lease.expiresAt > now,
+      )
+    ) {
+      return undefined;
+    }
+    const discarding: ReportStagingGroupRecord = {
+      ...group,
+      status: "discarding",
+      revision: group.revision + 1,
+      updatedAt: now,
+    };
+    this.#groups.set(groupId, discarding);
+    this.#claims.delete(groupId);
+    this.#entries.get(groupId)?.close();
+    this.#entries.delete(groupId);
+    return discarding;
+  }
+
+  async markDiscarded(groupId: string, now: string) {
+    const group = this.#groups.get(groupId);
+    if (!group || group.status !== "discarding") return undefined;
+    const discarded: ReportStagingGroupRecord = {
+      ...group,
+      status: "discarded",
+      revision: group.revision + 1,
+      updatedAt: now,
+    };
+    this.#groups.set(groupId, discarded);
+    return discarded;
+  }
+
+  async listPurgeCandidates(now: string, limit: number) {
+    const groups: ReportStagingGroupRecord[] = [];
+    for (const group of this.#groups.values()) {
+      if (
+        group.status !== "discarding" &&
+        (group.status === "discarded" || group.expiresAt > now)
+      ) {
+        continue;
+      }
+      const claim = this.#claims.get(group.groupId);
+      if (claim && claim.expiresAt > now) continue;
+      if (
+        [...this.#leases.values()].some(
+          (lease) =>
+            lease.groupId === group.groupId &&
+            !lease.closedAt &&
+            lease.expiresAt > now,
+        )
+      ) {
+        continue;
+      }
+      groups.push(group);
+      if (groups.length === limit) break;
+    }
+    return groups;
+  }
+}

package/template/apps/server/src/report-staging/metadata-catalog.ts

@@ -0,0 +1,89 @@
+import { createHash } from "node:crypto";
+import type { Database } from "bun:sqlite";
+import type { ReportStagingEntryRecord } from "./contracts.js";
+import { unicodePathCasefold } from "./path-policy.js";
+
+type EntryRow = {
+  pathKey: string;
+  relativePath: string;
+  byteSize: number;
+  sha256: string;
+  ordinal: number;
+  entryId: string;
+};
+
+const ENTRY_SELECT =
+  "select path_key as pathKey, relative_path as relativePath, declared_size as byteSize, sha256, entry_ordinal as ordinal, entry_id as entryId from nodes";
+const ENTRY_PAGE_SQL = `${ENTRY_SELECT} where entry_ordinal > ? order by entry_ordinal limit ?`;
+
+export function* readCatalogBatches(input: {
+  readonly db: Database;
+  readonly groupId: string;
+  readonly createdAt: string;
+  readonly size: number;
+}): Generator<readonly ReportStagingEntryRecord[]> {
+  let after = -1;
+  while (true) {
+    const rows = input.db
+      .query<EntryRow, [number, number]>(ENTRY_PAGE_SQL)
+      .all(after, input.size);
+    if (rows.length === 0) return;
+    const entries = rows.map((row) => mapEntry(row, input));
+    yield entries;
+    after = entries.at(-1)!.ordinal;
+  }
+}
+
+export function readCatalogEntry(input: {
+  readonly db: Database;
+  readonly groupId: string;
+  readonly createdAt: string;
+  readonly entryId: string;
+}) {
+  const row = input.db
+    .query<EntryRow, [string]>(`${ENTRY_SELECT} where entry_id = ?`)
+    .get(input.entryId);
+  return row ? mapEntry(row, input) : undefined;
+}
+
+export function readCatalogPage(input: {
+  readonly db: Database;
+  readonly groupId: string;
+  readonly createdAt: string;
+  readonly afterOrdinal: number | undefined;
+  readonly limit: number;
+}) {
+  return input.db
+    .query<EntryRow, [number, number]>(ENTRY_PAGE_SQL)
+    .all(input.afterOrdinal ?? -1, input.limit)
+    .map((row) => mapEntry(row, input));
+}
+
+export function metadataPathKey(relativePath: string) {
+  return createHash("sha256")
+    .update(unicodePathCasefold(relativePath))
+    .digest("hex");
+}
+
+export function metadataEntryId(groupId: string, relativePath: string) {
+  return `rse_${createHash("sha256")
+    .update(`${groupId}\0${relativePath}`)
+    .digest("hex")
+    .slice(0, 32)}`;
+}
+
+function mapEntry(
+  row: EntryRow,
+  input: { readonly groupId: string; readonly createdAt: string },
+): ReportStagingEntryRecord {
+  return {
+    groupId: input.groupId,
+    entryId: row.entryId,
+    ordinal: row.ordinal,
+    relativePath: row.relativePath,
+    pathKey: row.pathKey,
+    byteSize: row.byteSize,
+    sha256: row.sha256,
+    createdAt: input.createdAt,
+  };
+}

package/template/apps/server/src/report-staging/metadata-spool-paths.ts

@@ -0,0 +1,195 @@
+import type { Database } from "bun:sqlite";
+import {
+  MODULE_REPORT_STAGING_MAX_ENTRIES,
+  ModuleReportStagingError,
+  type ModuleReportStagingPolicyV1,
+} from "@southwind-ai/server-sdk";
+import { metadataPathKey } from "./metadata-catalog.js";
+import { isSqliteFull } from "./metadata-spool-schema.js";
+import { validateArchiveEntryPath } from "./path-policy.js";
+
+type NodeKind = "file" | "directory";
+export type MetadataNodeRow = {
+  relativePath: string;
+  kind: NodeKind;
+  explicit: number;
+  declaredSize: number;
+};
+
+export interface MetadataPathState {
+  archiveEntryCount: number;
+  nodeCount: number;
+  expandedByteSize: number;
+}
+
+export function declareMetadataEntry(input: {
+  readonly db: Database;
+  readonly policy: ModuleReportStagingPolicyV1;
+  readonly state: MetadataPathState;
+  readonly raw: string;
+  readonly directory: boolean;
+  readonly byteSize: number;
+}) {
+  const { db, policy, state, raw, directory, byteSize } = input;
+  if (
+    !Number.isSafeInteger(byteSize) ||
+    byteSize < 0 ||
+    (directory && byteSize !== 0) ||
+    (!directory && byteSize > policy.maxEntryBytes)
+  ) {
+    throw limit();
+  }
+  const entry = validateArchiveEntryPath(raw, policy, directory, byteSize);
+  const nextExpanded = state.expandedByteSize + byteSize;
+  if (
+    state.archiveEntryCount >= policy.maxEntries ||
+    state.archiveEntryCount >= MODULE_REPORT_STAGING_MAX_ENTRIES ||
+    !Number.isSafeInteger(nextExpanded) ||
+    nextExpanded > policy.maxExpandedBytes
+  ) {
+    throw limit();
+  }
+  let inserted = 0;
+  writeBounded(db, () => {
+    const segments = entry.relativePath.split("/");
+    for (let index = 1; index < segments.length; index++) {
+      inserted += ensureDirectory(db, segments.slice(0, index).join("/"));
+    }
+    inserted += insertExplicit(
+      db,
+      entry.relativePath,
+      directory ? "directory" : "file",
+      byteSize,
+      state.archiveEntryCount,
+    );
+    if (
+      state.nodeCount + inserted > policy.maxEntries ||
+      state.nodeCount + inserted > MODULE_REPORT_STAGING_MAX_ENTRIES
+    ) {
+      throw limit();
+    }
+  });
+  state.nodeCount += inserted;
+  state.archiveEntryCount++;
+  state.expandedByteSize = nextExpanded;
+  return entry;
+}
+
+export function expectedMetadataEntry(input: {
+  readonly db: Database;
+  readonly policy: ModuleReportStagingPolicyV1;
+  readonly ordinal: number;
+  readonly raw: string;
+  readonly directory: boolean;
+  readonly byteSize: number;
+}) {
+  const actual = validateArchiveEntryPath(
+    input.raw,
+    input.policy,
+    input.directory,
+    input.byteSize,
+  );
+  const expected = declaredMetadataEntry(input.db, input.ordinal);
+  if (
+    !expected ||
+    expected.relativePath !== actual.relativePath ||
+    expected.kind !== (input.directory ? "directory" : "file") ||
+    expected.declaredSize !== input.byteSize
+  ) {
+    throw unsafe("报告二次读取的 entry 顺序或元数据发生漂移");
+  }
+  return actual;
+}
+
+export function declaredMetadataEntry(db: Database, ordinal: number) {
+  return db
+    .query<MetadataNodeRow, [number]>(
+      "select relative_path as relativePath, kind, explicit, declared_size as declaredSize from nodes where archive_ordinal = ?",
+    )
+    .get(ordinal);
+}
+
+export function lookupMetadataNode(db: Database, key: string) {
+  return db
+    .query<MetadataNodeRow, [string]>(
+      "select relative_path as relativePath, kind, explicit, declared_size as declaredSize from nodes where path_key = ?",
+    )
+    .get(key);
+}
+
+function ensureDirectory(db: Database, relativePath: string) {
+  const key = metadataPathKey(relativePath);
+  const existing = lookupMetadataNode(db, key);
+  if (existing) {
+    if (
+      existing.relativePath !== relativePath ||
+      existing.kind !== "directory"
+    ) {
+      throw unsafe("报告路径存在大小写折叠或文件目录冲突");
+    }
+    return 0;
+  }
+  insertNode(db, key, relativePath, "directory", 0, null, 0);
+  return 1;
+}
+
+function insertExplicit(
+  db: Database,
+  relativePath: string,
+  kind: NodeKind,
+  declaredSize: number,
+  ordinal: number,
+) {
+  const key = metadataPathKey(relativePath);
+  const existing = lookupMetadataNode(db, key);
+  if (existing) {
+    if (
+      existing.relativePath !== relativePath ||
+      existing.kind !== "directory" ||
+      kind !== "directory" ||
+      existing.explicit !== 0
+    ) {
+      throw unsafe("报告包含重复、大小写折叠或文件目录冲突路径");
+    }
+    db.query(
+      "update nodes set explicit = 1, archive_ordinal = ? where path_key = ?",
+    ).run(ordinal, key);
+    return 0;
+  }
+  insertNode(db, key, relativePath, kind, 1, ordinal, declaredSize);
+  return 1;
+}
+
+function insertNode(
+  db: Database,
+  key: string,
+  relativePath: string,
+  kind: NodeKind,
+  explicit: number,
+  ordinal: number | null,
+  declaredSize: number,
+) {
+  db.query(
+    "insert into nodes(path_key, relative_path, kind, explicit, archive_ordinal, declared_size) values (?, ?, ?, ?, ?, ?)",
+  ).run(key, relativePath, kind, explicit, ordinal, declaredSize);
+}
+
+function writeBounded(db: Database, operation: () => void) {
+  try {
+    operation();
+  } catch (error) {
+    if (isSqliteFull(error)) throw limit();
+    throw error;
+  }
+}
+
+function unsafe(message: string) {
+  return new ModuleReportStagingError("REPORT_STAGING_UNSAFE_ARCHIVE", message);
+}
+
+function limit() {
+  return new ModuleReportStagingError(
+    "REPORT_STAGING_LIMIT",
+    "报告 metadata spool 超过 Host 上限",
+  );
+}

package/template/apps/server/src/report-staging/metadata-spool-schema.ts

@@ -0,0 +1,54 @@
+import type { Database } from "bun:sqlite";
+import {
+  MODULE_REPORT_STAGING_MAX_ENTRIES,
+  MODULE_REPORT_STAGING_PATH_MAX_BYTES,
+  type ModuleReportStagingPolicyV1,
+} from "@southwind-ai/server-sdk";
+
+export function initializeMetadataSpool(
+  db: Database,
+  policy: ModuleReportStagingPolicyV1,
+) {
+  const maximumBytes = Math.min(
+    4 * 1_024 ** 3,
+    16 * 1_024 ** 2 +
+      Math.min(policy.maxEntries, MODULE_REPORT_STAGING_MAX_ENTRIES) *
+        (MODULE_REPORT_STAGING_PATH_MAX_BYTES + 768) *
+        4,
+  );
+  const maxPages = Math.ceil(maximumBytes / 4_096);
+  db.exec(`
+    pragma journal_mode = off;
+    pragma synchronous = off;
+    pragma mmap_size = 0;
+    pragma temp_store = file;
+    pragma cache_size = -8192;
+    pragma page_size = 4096;
+    pragma max_page_count = ${maxPages};
+    create table nodes (
+      path_key text primary key,
+      relative_path text not null,
+      kind text not null check(kind in ('file', 'directory')),
+      explicit integer not null check(explicit in (0, 1)),
+      archive_ordinal integer unique,
+      declared_size integer not null,
+      scan_seen integer not null default 0 check(scan_seen in (0, 1)),
+      sha256 text,
+      entry_ordinal integer unique,
+      entry_id text unique
+    );
+    create index idx_nodes_archive_ordinal on nodes(archive_ordinal);
+    create index idx_nodes_entry_ordinal on nodes(entry_ordinal);
+    create index idx_nodes_entry_id on nodes(entry_id);
+    begin immediate;
+  `);
+  return maxPages;
+}
+
+export function isSqliteFull(error: unknown) {
+  return (
+    error instanceof Error &&
+    (("code" in error && error.code === "SQLITE_FULL") ||
+      error.message.includes("database or disk is full"))
+  );
+}