create-windy 0.5.1 → 0.6.0

package/template/apps/server/src/ai-output-artifact/memory-repository.ts

@@ -0,0 +1,348 @@
+import type {
+  AiOutputArtifactRecord,
+  AiOutputArtifactScope,
+} from "./contracts.js";
+import type {
+  AiOutputArtifactRepository,
+  NewAiOutputArtifact,
+  NewAiOutputScope,
+} from "./repository.js";
+import { AiOutputArtifactRepositoryError } from "./repository.js";
+import {
+  prepareMemoryOutputGrant,
+  promoteMemoryOutputGrant,
+  rollbackMemoryOutputGrant,
+} from "./memory-grants.js";
+import {
+  clearExpiredMemoryGrantHashes,
+  listMemoryCleanupScopes,
+} from "./memory-cleanup.js";
+import { deleteMemoryOutputReadinessProbe } from "./memory-readiness.js";
+
+export class InMemoryAiOutputArtifactRepository implements AiOutputArtifactRepository {
+  readonly #scopes = new Map<string, AiOutputArtifactScope>();
+  readonly #artifacts = new Map<string, AiOutputArtifactRecord>();
+
+  async createOrGetScope(input: NewAiOutputScope) {
+    const existing = [...this.#scopes.values()].find(
+      ({ idempotencyHash }) => idempotencyHash === input.idempotencyHash,
+    );
+    if (existing) return { scope: clone(existing), created: false };
+    const sameRunKind = [...this.#scopes.values()].find(
+      (scope) =>
+        scope.runId === input.runId && scope.outputKind === input.outputKind,
+    );
+    if (sameRunKind) {
+      return { scope: clone(sameRunKind), created: false };
+    }
+    const scope = { ...clone(input), artifactCount: 0, completedByteSize: 0 };
+    this.#scopes.set(scope.scopeId, scope);
+    return { scope: clone(scope), created: true };
+  }
+
+  async getScope(scopeId: string) {
+    return cloned(this.#scopes.get(scopeId));
+  }
+
+  async listScopesByRun(runId: string) {
+    return [...this.#scopes.values()]
+      .filter((scope) => scope.runId === runId)
+      .map(clone);
+  }
+
+  async prepareGrant(
+    scopeId: string,
+    expectedRevision: number,
+    pendingGrantHash: string,
+    pendingGrantExpiresAt: string,
+    updatedAt: string,
+  ) {
+    return prepareMemoryOutputGrant(
+      this.#scopes,
+      scopeId,
+      expectedRevision,
+      pendingGrantHash,
+      pendingGrantExpiresAt,
+      updatedAt,
+    );
+  }
+
+  async promoteGrant(
+    scopeId: string,
+    expectedRevision: number,
+    pendingGrantGeneration: number,
+    pendingGrantHash: string,
+    updatedAt: string,
+  ) {
+    return promoteMemoryOutputGrant(
+      this.#scopes,
+      scopeId,
+      expectedRevision,
+      pendingGrantGeneration,
+      pendingGrantHash,
+      updatedAt,
+    );
+  }
+
+  async rollbackGrant(
+    scopeId: string,
+    expectedRevision: number,
+    pendingGrantGeneration: number,
+    pendingGrantHash: string,
+    updatedAt: string,
+  ) {
+    return rollbackMemoryOutputGrant(
+      this.#scopes,
+      scopeId,
+      expectedRevision,
+      pendingGrantGeneration,
+      pendingGrantHash,
+      updatedAt,
+    );
+  }
+
+  async transitionScope(
+    scopeId: string,
+    expectedRevision: number,
+    expectedStatuses: readonly AiOutputArtifactScope["status"][],
+    status: AiOutputArtifactScope["status"],
+    updatedAt: string,
+  ) {
+    const scope = this.#scopes.get(scopeId);
+    if (
+      !scope ||
+      scope.revision !== expectedRevision ||
+      !expectedStatuses.includes(scope.status)
+    ) {
+      return undefined;
+    }
+    const updated = {
+      ...scope,
+      status,
+      grantHash: status === "cleaned" ? undefined : scope.grantHash,
+      activeGrantGeneration:
+        status === "cleaned" ? undefined : scope.activeGrantGeneration,
+      pendingGrantHash: status === "open" ? scope.pendingGrantHash : undefined,
+      pendingGrantGeneration:
+        status === "open" ? scope.pendingGrantGeneration : undefined,
+      pendingGrantExpiresAt:
+        status === "open" ? scope.pendingGrantExpiresAt : undefined,
+      revision: scope.revision + 1,
+      updatedAt,
+    };
+    this.#scopes.set(scopeId, updated);
+    return clone(updated);
+  }
+
+  async sealScope(
+    scopeId: string,
+    expectedRevision: number,
+    refExpiresAt: string,
+    updatedAt: string,
+  ) {
+    const scope = this.#scopes.get(scopeId);
+    if (
+      !scope ||
+      scope.status !== "open" ||
+      scope.revision !== expectedRevision ||
+      refExpiresAt <= scope.scopeExpiresAt ||
+      refExpiresAt > scope.fileExpiresAt ||
+      [...this.#artifacts.values()].some(
+        (artifact) =>
+          artifact.scopeId === scopeId && artifact.status !== "completed",
+      )
+    ) {
+      return undefined;
+    }
+    const sealed: AiOutputArtifactScope = {
+      ...scope,
+      status: "sealed",
+      revision: scope.revision + 1,
+      pendingGrantHash: undefined,
+      pendingGrantGeneration: undefined,
+      pendingGrantExpiresAt: undefined,
+      refExpiresAt,
+      updatedAt,
+    };
+    this.#scopes.set(scopeId, sealed);
+    return clone(sealed);
+  }
+
+  async listCleanupScopes(expiredBefore: string, limit: number) {
+    return listMemoryCleanupScopes(this.#scopes, expiredBefore, limit);
+  }
+
+  async clearExpiredGrantHashes(expiredBefore: string, limit: number) {
+    return clearExpiredMemoryGrantHashes(this.#scopes, expiredBefore, limit);
+  }
+
+  async claimArtifact(input: NewAiOutputArtifact) {
+    const scope = this.#scopes.get(input.scopeId);
+    if (!scope || scope.status !== "open") {
+      throw new AiOutputArtifactRepositoryError("CONFLICT");
+    }
+    const existing = [...this.#artifacts.values()].find(
+      (artifact) =>
+        artifact.scopeId === input.scopeId &&
+        artifact.outputKey === input.outputKey,
+    );
+    if (existing) {
+      if (existing.fingerprint !== input.fingerprint) {
+        return { artifact: clone(existing), claimed: false };
+      }
+      if (existing.status !== "creating") {
+        return { artifact: clone(existing), claimed: false };
+      }
+      const claimed = {
+        ...existing,
+        generation: existing.generation + 1,
+        updatedAt: input.createdAt,
+      };
+      this.#artifacts.set(claimed.artifactId, claimed);
+      return { artifact: clone(claimed), claimed: true };
+    }
+    if (
+      scope.artifactCount >= scope.maxArtifacts ||
+      this.expectedBytes(scope.scopeId) + input.expectedByteSize >
+        scope.maxTotalBytes
+    ) {
+      throw new AiOutputArtifactRepositoryError("LIMIT");
+    }
+    const artifact: AiOutputArtifactRecord = {
+      ...clone(input),
+      generation: 1,
+      status: "creating",
+      updatedAt: input.createdAt,
+    };
+    this.#artifacts.set(artifact.artifactId, artifact);
+    this.#scopes.set(scope.scopeId, {
+      ...scope,
+      artifactCount: scope.artifactCount + 1,
+      updatedAt: input.createdAt,
+    });
+    return { artifact: clone(artifact), claimed: true };
+  }
+
+  async bindUploadSession(
+    artifactId: string,
+    generation: number,
+    uploadSessionId: string,
+    updatedAt: string,
+  ) {
+    const artifact = this.#artifacts.get(artifactId);
+    if (
+      !artifact ||
+      artifact.status !== "creating" ||
+      artifact.generation !== generation
+    ) {
+      return undefined;
+    }
+    return this.save({
+      ...artifact,
+      status: "uploading",
+      uploadSessionId,
+      updatedAt,
+    });
+  }
+
+  async completeArtifact(
+    artifactId: string,
+    generation: number,
+    file: { id: string; byteSize: number; sha256: string },
+    updatedAt: string,
+  ) {
+    const artifact = this.#artifacts.get(artifactId);
+    if (
+      !artifact ||
+      artifact.status !== "uploading" ||
+      artifact.generation !== generation ||
+      artifact.expectedByteSize !== file.byteSize ||
+      artifact.expectedSha256 !== file.sha256
+    ) {
+      return undefined;
+    }
+    const scope = this.#scopes.get(artifact.scopeId);
+    if (
+      !scope ||
+      scope.status !== "open" ||
+      scope.completedByteSize + file.byteSize > scope.maxTotalBytes
+    ) {
+      return undefined;
+    }
+    this.#scopes.set(scope.scopeId, {
+      ...scope,
+      completedByteSize: scope.completedByteSize + file.byteSize,
+      updatedAt,
+    });
+    const completed = this.save({
+      ...artifact,
+      status: "completed",
+      fileId: file.id,
+      byteSize: file.byteSize,
+      sha256: file.sha256,
+      updatedAt,
+    });
+    return completed;
+  }
+
+  async transitionArtifact(
+    artifactId: string,
+    expectedGeneration: number,
+    expectedStatuses: readonly AiOutputArtifactRecord["status"][],
+    status: AiOutputArtifactRecord["status"],
+    updatedAt: string,
+  ) {
+    const artifact = this.#artifacts.get(artifactId);
+    if (
+      !artifact ||
+      artifact.generation !== expectedGeneration ||
+      !expectedStatuses.includes(artifact.status)
+    ) {
+      return undefined;
+    }
+    return this.save({
+      ...artifact,
+      generation: artifact.generation + 1,
+      status,
+      updatedAt,
+    });
+  }
+
+  async getArtifact(artifactId: string) {
+    return cloned(this.#artifacts.get(artifactId));
+  }
+
+  async listArtifacts(scopeId: string) {
+    return [...this.#artifacts.values()]
+      .filter((artifact) => artifact.scopeId === scopeId)
+      .map(clone);
+  }
+
+  async deleteReadinessProbe(scopeId: string, purpose: string) {
+    return deleteMemoryOutputReadinessProbe(
+      this.#scopes,
+      this.#artifacts,
+      scopeId,
+      purpose,
+    );
+  }
+
+  private save(artifact: AiOutputArtifactRecord) {
+    this.#artifacts.set(artifact.artifactId, artifact);
+    return clone(artifact);
+  }
+
+  private expectedBytes(scopeId: string) {
+    return [...this.#artifacts.values()]
+      .filter((artifact) => artifact.scopeId === scopeId)
+      .reduce((total, artifact) => total + artifact.expectedByteSize, 0);
+  }
+}
+
+function cloned<T>(value: T | undefined): T | undefined {
+  return value === undefined ? undefined : clone(value);
+}
+
+function clone<T>(value: T): T {
+  return structuredClone(value);
+}

package/template/apps/server/src/ai-output-artifact/policy.test.ts

@@ -0,0 +1,67 @@
+import { describe, expect, test } from "bun:test";
+import {
+  AiOutputArtifactPolicyRegistry,
+  type AiOutputArtifactLimitSet,
+} from "./policy.js";
+import { createOutputArtifactFixture, outputLimits } from "./test-support.js";
+
+describe("AI Output Artifact policy", () => {
+  test("Operation/readiness/runtime/Host 四方上限逐项取最小值", () => {
+    const policy = new AiOutputArtifactPolicyRegistry().resolve({
+      runtimeKind: "document",
+      outputKind: "searchable-pdf",
+      dataClassification: "internal",
+      operationLimits: limits(20_000, 5, 30_000, 900),
+      readinessLimits: limits(18_000, 4, 25_000, 800),
+      runtimeLimits: limits(16_000, 3, 20_000, 700),
+    });
+
+    expect(policy).toMatchObject({
+      maxArtifactBytes: 16_000,
+      maxArtifacts: 3,
+      maxTotalBytes: 20_000,
+      scopeTtlMs: 760_000,
+    });
+  });
+
+  test.each([Number.NaN, Number.POSITIVE_INFINITY, 1.5, 0])(
+    "非有限、非安全或非正限额 %p fail closed",
+    (invalid) => {
+      expect(() =>
+        new AiOutputArtifactPolicyRegistry().resolve({
+          runtimeKind: "document",
+          outputKind: "searchable-pdf",
+          dataClassification: "internal",
+          operationLimits: {
+            ...outputLimits(),
+            maxTotalBytes: invalid,
+          },
+          readinessLimits: outputLimits(),
+          runtimeLimits: outputLimits(),
+        }),
+      ).toThrow("限额无效");
+    },
+  );
+
+  test("idempotency key 仅接受有界稳定 token", async () => {
+    const fixture = createOutputArtifactFixture();
+    await expect(fixture.begin(" has-space")).rejects.toThrow("治理上下文无效");
+    await expect(fixture.begin("x".repeat(161))).rejects.toThrow(
+      "治理上下文无效",
+    );
+  });
+});
+
+function limits(
+  maxArtifactBytes: number,
+  maxArtifacts: number,
+  maxTotalBytes: number,
+  maxDurationSeconds: number,
+): AiOutputArtifactLimitSet {
+  return {
+    maxArtifactBytes,
+    maxArtifacts,
+    maxTotalBytes,
+    maxDurationSeconds,
+  };
+}

package/template/apps/server/src/ai-output-artifact/policy.ts

@@ -0,0 +1,125 @@
+import { createHash } from "node:crypto";
+import type { AiOutputRuntimeKind } from "./contracts.js";
+
+export const aiOutputArtifactReadinessPurpose =
+  "ai-runtime-output-readiness" as const;
+
+export interface AiOutputArtifactPolicy {
+  readonly runtimeKind: AiOutputRuntimeKind;
+  readonly outputKind: string;
+  readonly dataClassification: "public" | "internal" | "sensitive";
+  readonly purpose:
+    | "ai-runtime-output"
+    | typeof aiOutputArtifactReadinessPurpose;
+  readonly allowedMediaTypes: readonly string[];
+  readonly maxArtifactBytes: number;
+  readonly maxArtifacts: number;
+  readonly maxTotalBytes: number;
+  readonly scopeTtlMs: number;
+  readonly fileRetentionMs: number;
+  readonly referenceTtlMs: number;
+  readonly digest: string;
+}
+
+export interface AiOutputArtifactLimitSet {
+  readonly maxArtifactBytes: number;
+  readonly maxArtifacts: number;
+  readonly maxTotalBytes: number;
+  readonly maxDurationSeconds: number;
+}
+
+export class AiOutputArtifactPolicyRegistry {
+  resolve(input: {
+    runtimeKind: AiOutputRuntimeKind;
+    outputKind: string;
+    dataClassification: AiOutputArtifactPolicy["dataClassification"];
+    operationLimits: AiOutputArtifactLimitSet;
+    readinessLimits: AiOutputArtifactLimitSet;
+    runtimeLimits: AiOutputArtifactLimitSet;
+  }): AiOutputArtifactPolicy {
+    const base = basePolicy(input.runtimeKind, input.outputKind);
+    const sensitive = input.dataClassification === "sensitive";
+    const limits = minimumLimits(
+      base,
+      input.operationLimits,
+      input.readinessLimits,
+      input.runtimeLimits,
+    );
+    const values = {
+      runtimeKind: input.runtimeKind,
+      outputKind: input.outputKind,
+      dataClassification: input.dataClassification,
+      purpose: "ai-runtime-output" as const,
+      allowedMediaTypes: base.mediaTypes,
+      maxArtifactBytes: limits.maxArtifactBytes,
+      maxArtifacts: limits.maxArtifacts,
+      maxTotalBytes: limits.maxTotalBytes,
+      scopeTtlMs: limits.maxDurationSeconds * 1_000 + 60_000,
+      fileRetentionMs: sensitive ? 24 * 60 * 60_000 : 7 * 24 * 60 * 60_000,
+      referenceTtlMs: sensitive ? 15 * 60_000 : 60 * 60_000,
+    };
+    return { ...values, digest: policyDigest(values) };
+  }
+}
+
+function basePolicy(runtimeKind: AiOutputRuntimeKind, outputKind: string) {
+  if (runtimeKind === "document" && outputKind === "searchable-pdf") {
+    return {
+      mediaTypes: ["application/pdf"] as const,
+      maxArtifactBytes: 64 * 1024 * 1024,
+      maxArtifacts: 16,
+      maxTotalBytes: 64 * 1024 * 1024,
+      maxDurationSeconds: 3_600,
+    };
+  }
+  if (runtimeKind === "speech" && outputKind === "normalized-audio") {
+    return {
+      mediaTypes: ["audio/wav"] as const,
+      maxArtifactBytes: 512 * 1024 * 1024,
+      maxArtifacts: 1,
+      maxTotalBytes: 512 * 1024 * 1024,
+      maxDurationSeconds: 3_600,
+    };
+  }
+  throw new Error(
+    `未注册 AI Output Artifact 策略：${runtimeKind}/${outputKind}`,
+  );
+}
+
+function minimumLimits(
+  ...sets: readonly AiOutputArtifactLimitSet[]
+): AiOutputArtifactLimitSet {
+  sets.forEach(validateLimits);
+  return {
+    maxArtifactBytes: Math.min(
+      ...sets.map(({ maxArtifactBytes }) => maxArtifactBytes),
+    ),
+    maxArtifacts: Math.min(...sets.map(({ maxArtifacts }) => maxArtifacts)),
+    maxTotalBytes: Math.min(...sets.map(({ maxTotalBytes }) => maxTotalBytes)),
+    maxDurationSeconds: Math.min(
+      ...sets.map(({ maxDurationSeconds }) => maxDurationSeconds),
+    ),
+  };
+}
+
+function validateLimits(value: AiOutputArtifactLimitSet) {
+  for (const [field, limit, maximum] of [
+    ["maxArtifactBytes", value.maxArtifactBytes, 512 * 1024 * 1024],
+    ["maxArtifacts", value.maxArtifacts, 100],
+    ["maxTotalBytes", value.maxTotalBytes, 512 * 1024 * 1024],
+    ["maxDurationSeconds", value.maxDurationSeconds, 3_600],
+  ] as const) {
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > maximum) {
+      throw new Error(`AI Output Artifact ${field} 限额无效`);
+    }
+  }
+  if (value.maxArtifactBytes > value.maxTotalBytes) {
+    throw new Error("AI Output Artifact 单件上限超过总量上限");
+  }
+}
+
+function policyDigest(value: object) {
+  return createHash("sha256")
+    .update(JSON.stringify(value, Object.keys(value).sort()))
+    .digest("hex");
+}

package/template/apps/server/src/ai-output-artifact/postgres-test-support.ts

@@ -0,0 +1,147 @@
+import { drizzle } from "drizzle-orm/node-postgres";
+import { Pool } from "pg";
+import { buildScopeContext } from "../data-access/context.js";
+import type { RequestGuardContext } from "../guards.js";
+import { createStorageRuntime } from "../storage/runtime.js";
+import { AiOutputArtifactCleanupService } from "./cleanup.js";
+import { DrizzleAiOutputArtifactRepository } from "./drizzle-repository.js";
+import { AiOutputArtifactPolicyRegistry } from "./policy.js";
+import { AiOutputArtifactReferenceCodec } from "./reference-codec.js";
+import { AiOutputArtifactService } from "./service.js";
+import { AiOutputArtifactStorageWriterFactory } from "./storage-writer.js";
+import { outputLimits } from "./test-support.js";
+
+export function createPostgresArtifactStack(
+  databaseUrl: string,
+  root: string,
+  now: () => Date = () => new Date(),
+) {
+  const pool = new Pool({ connectionString: databaseUrl });
+  const database = drizzle(pool);
+  const repository = new DrizzleAiOutputArtifactRepository(database);
+  const storage = createStorageRuntime(
+    { WINDY_BLOB_DRIVER: "local", WINDY_STORAGE_ROOT: root },
+    database,
+    [],
+    now,
+  );
+  const writers = new AiOutputArtifactStorageWriterFactory(
+    storage,
+    { record: () => undefined },
+    now,
+  );
+  const service = new AiOutputArtifactService({
+    repository,
+    storage,
+    writers,
+    references: new AiOutputArtifactReferenceCodec("p".repeat(32)),
+    authorizeRead: async (scope) => ({
+      actorId: scope.actorId,
+      scopeHash: scope.scopeHash,
+      requestId: "request-read-postgres",
+    }),
+    recordRead: () => undefined,
+    now,
+  });
+  return {
+    pool,
+    repository,
+    storage,
+    writers,
+    service,
+    cleanup: new AiOutputArtifactCleanupService(
+      repository,
+      storage,
+      writers,
+      now,
+    ),
+  };
+}
+
+export async function beginPostgresArtifactScope(
+  value: ReturnType<typeof createPostgresArtifactStack>,
+  idempotencyKey: string,
+  runId: string,
+) {
+  const prepared = await preparePostgresArtifactScope(
+    value,
+    idempotencyKey,
+    runId,
+  );
+  if (!prepared.grant || !prepared.descriptor) return prepared;
+  const accepted = await value.service.acceptGrant(
+    prepared.scope.scopeId,
+    prepared.descriptor.grantGeneration,
+    prepared.grant,
+  );
+  return { ...prepared, scope: accepted };
+}
+
+export async function preparePostgresArtifactScope(
+  value: ReturnType<typeof createPostgresArtifactStack>,
+  idempotencyKey: string,
+  runId: string,
+) {
+  const limits = outputLimits();
+  const policy = new AiOutputArtifactPolicyRegistry().resolve({
+    runtimeKind: "document",
+    outputKind: "searchable-pdf",
+    dataClassification: "sensitive",
+    operationLimits: limits,
+    readinessLimits: limits,
+    runtimeLimits: limits,
+  });
+  return value.service.beginScope({
+    runId,
+    moduleName: "evidence",
+    operationKey: "evidence.document.ocr",
+    runtimeKind: "document",
+    idempotencyKey,
+    invocationDigest: "c".repeat(64),
+    dataClassification: "sensitive",
+    deploymentIdentity:
+      "windy-deployment-v1_7db61d8e-e50d-4f8a-9f3f-9b7a1d932e01",
+    releaseDigest: `sha256:${"a".repeat(64)}`,
+    offersDigest: `sha256:${"b".repeat(64)}`,
+    policy,
+    context: guardContext(),
+  });
+}
+
+export async function clearPostgresArtifactTables(databaseUrl: string) {
+  const pool = new Pool({ connectionString: databaseUrl });
+  try {
+    await pool.query(`
+      truncate table
+        ai_output_artifacts,
+        ai_output_artifact_scopes,
+        file_upload_parts,
+        file_metadata,
+        file_upload_sessions
+    `);
+  } finally {
+    await pool.end();
+  }
+}
+
+function guardContext(): RequestGuardContext {
+  return {
+    requestId: "pg-output-artifact",
+    actor: {
+      id: "user-1",
+      type: "user",
+      name: "PG User",
+      roleCodes: ["analyst"],
+      permissionKeys: ["evidence.document.ocr.execute"],
+      scopeContext: buildScopeContext("user-1", undefined, [
+        {
+          id: "analyst",
+          code: "analyst",
+          dataScope: "self",
+          departmentIds: [],
+        },
+      ]),
+    },
+    startedAt: new Date().toISOString(),
+  };
+}