create-windy 0.5.2 → 0.6.0

package/template/packages/storage/src/upload-cleanup-race.test.ts

@@ -0,0 +1,207 @@
+import { createHash } from "node:crypto";
+import { describe, expect, test } from "bun:test";
+import type { PutBlobInput, StoredBlob } from "./blob-store.js";
+import { InMemoryUploadRepository } from "./in-memory-upload-repository.js";
+import { MemoryBlobStore } from "./memory-blob-store.js";
+import { UploadService } from "./upload-service.js";
+
+const policy = {
+  purpose: "module.output",
+  maxByteSize: 1024,
+  maxPartByteSize: 512,
+  sessionTtlMs: 60_000,
+  allowedTypes: { "text/plain": ["txt"] },
+  signature: "none" as const,
+};
+
+describe("Upload cleanup 持久化 claim 竞态", () => {
+  test("complete 先提交时，稍后的 cleanup claim 不删除会话或文件", async () => {
+    let now = new Date("2026-07-31T00:00:00.000Z");
+    const repository = new ClaimGateRepository();
+    const blobs = new FinalPutGateBlobStore();
+    const service = createService(repository, blobs, () => now);
+    const session = await seed(service);
+
+    blobs.pauseNextPut();
+    const completion = service.complete(session.id, "owner");
+    await blobs.started;
+    now = new Date("2026-07-31T00:02:00.000Z");
+    repository.pauseNextClaim();
+    const cleanup = service.cleanupExpired();
+    await repository.claimStarted;
+
+    blobs.resume();
+    const file = await completion;
+    repository.resumeClaim();
+
+    await expect(cleanup).resolves.toMatchObject({ sessions: 0 });
+    await expect(repository.getSession(session.id)).resolves.toMatchObject({
+      status: "completed",
+      fileId: file.id,
+    });
+    expect(await repository.getFile(file.id)).toBeDefined();
+    expect(blobs.size).toBe(1);
+  });
+
+  test("cleanup 先 claim 时，已开始的 complete 不能发布文件", async () => {
+    let now = new Date("2026-07-31T00:00:00.000Z");
+    const repository = new InMemoryUploadRepository();
+    const blobs = new FinalPutGateBlobStore();
+    const service = createService(repository, blobs, () => now);
+    const session = await seed(service);
+
+    blobs.pauseNextPut();
+    const completion = service.complete(session.id, "owner");
+    await blobs.started;
+    now = new Date("2026-07-31T00:02:00.000Z");
+    await expect(service.cleanupExpired()).resolves.toMatchObject({
+      sessions: 1,
+    });
+    blobs.resume();
+
+    await expect(completion).rejects.toMatchObject({
+      code: "UPLOAD_NOT_FOUND",
+    });
+    expect(await repository.getSession(session.id)).toBeUndefined();
+    expect(blobs.size).toBe(0);
+  });
+
+  test("claim 或 claimed-row 删除失败时都不删除 Blob 引用", async () => {
+    for (const repository of [
+      new FailingClaimRepository(),
+      new FailingClaimedDeleteRepository(),
+    ]) {
+      let now = new Date("2026-07-31T00:00:00.000Z");
+      const blobs = new DeleteTrackingBlobStore();
+      const service = createService(repository, blobs, () => now);
+      const session = await seed(service);
+      now = new Date("2026-07-31T00:02:00.000Z");
+
+      await expect(service.cleanupExpired()).resolves.toMatchObject({
+        sessions: 0,
+        blobs: 0,
+      });
+      expect(blobs.deleteCalls).toBe(0);
+      expect(await repository.listParts(session.id)).toHaveLength(1);
+    }
+  });
+});
+
+class ClaimGateRepository extends InMemoryUploadRepository {
+  private pause = false;
+  private markClaimStarted: () => void = () => undefined;
+  private releaseClaim: () => void = () => undefined;
+  claimStarted = Promise.resolve();
+
+  pauseNextClaim() {
+    this.pause = true;
+    this.claimStarted = new Promise((resolve) => {
+      this.markClaimStarted = resolve;
+    });
+  }
+
+  resumeClaim() {
+    this.releaseClaim();
+  }
+
+  override async claimExpiredSession(id: string, now: string) {
+    if (this.pause) {
+      this.pause = false;
+      this.markClaimStarted();
+      await new Promise<void>((resolve) => {
+        this.releaseClaim = resolve;
+      });
+    }
+    return super.claimExpiredSession(id, now);
+  }
+}
+
+class FailingClaimRepository extends InMemoryUploadRepository {
+  override async claimExpiredSession() {
+    return false;
+  }
+}
+
+class FailingClaimedDeleteRepository extends InMemoryUploadRepository {
+  override async deleteClaimedSession() {
+    return false;
+  }
+}
+
+class FinalPutGateBlobStore extends MemoryBlobStore {
+  private pause = false;
+  private markStarted: () => void = () => undefined;
+  private release: () => void = () => undefined;
+  started = Promise.resolve();
+
+  pauseNextPut() {
+    this.pause = true;
+    this.started = new Promise((resolve) => {
+      this.markStarted = resolve;
+    });
+  }
+
+  resume() {
+    this.release();
+  }
+
+  override async put(input: PutBlobInput): Promise<StoredBlob> {
+    const stored = await super.put(input);
+    if (this.pause) {
+      this.pause = false;
+      this.markStarted();
+      await new Promise<void>((resolve) => {
+        this.release = resolve;
+      });
+    }
+    return stored;
+  }
+}
+
+class DeleteTrackingBlobStore extends MemoryBlobStore {
+  deleteCalls = 0;
+
+  override async delete(ref: Parameters<MemoryBlobStore["delete"]>[0]) {
+    this.deleteCalls += 1;
+    return super.delete(ref);
+  }
+}
+
+function createService(
+  repository: InMemoryUploadRepository,
+  blobStore: MemoryBlobStore,
+  now: () => Date,
+) {
+  return new UploadService({
+    repository,
+    blobStore,
+    policies: [policy],
+    now,
+  });
+}
+
+async function seed(service: UploadService) {
+  const value = new TextEncoder().encode("abc");
+  const session = await service.create({
+    ownerId: "owner",
+    purpose: policy.purpose,
+    filename: "result.txt",
+    mimeType: "text/plain",
+    expectedByteSize: value.byteLength,
+    expectedSha256: digest(value),
+  });
+  await service.appendPart({
+    sessionId: session.id,
+    ownerId: "owner",
+    partNumber: 0,
+    byteOffset: 0,
+    byteSize: value.byteLength,
+    sha256: digest(value),
+    body: new Blob([value]).stream(),
+  });
+  return session;
+}
+
+function digest(value: Uint8Array) {
+  return createHash("sha256").update(value).digest("hex");
+}

package/template/packages/storage/src/upload-cleanup.ts

@@ -0,0 +1,85 @@
+import type {
+  ArtifactRef,
+  BlobStore,
+  BlobStoreMaintenance,
+} from "./blob-store.js";
+import {
+  recoverableUploadRepository,
+  type UploadRepository,
+} from "./upload-repository.js";
+
+export interface UploadCleanupDependencies {
+  readonly repository: UploadRepository;
+  readonly blobStore: BlobStore;
+  readonly maintenance?: BlobStoreMaintenance;
+  readonly orphanGraceMs?: number;
+}
+
+export interface CleanupResult {
+  sessions: number;
+  files: number;
+  blobs: number;
+  orphans: number;
+}
+
+export async function cleanupExpiredUploads(
+  dependencies: UploadCleanupDependencies,
+  now: Date,
+  limit: number,
+  additionalReferencedRefs: Iterable<ArtifactRef>,
+): Promise<CleanupResult> {
+  const timestamp = now.toISOString();
+  const sessions = await dependencies.repository.listExpiredSessions(
+    timestamp,
+    limit,
+  );
+  const files = await dependencies.repository.listExpiredFiles(
+    timestamp,
+    limit,
+  );
+  const recovery = recoverableUploadRepository(dependencies.repository);
+  let blobs = 0;
+  let deletedSessions = 0;
+  for (const session of sessions) {
+    if (
+      !recovery ||
+      !(await recovery.claimExpiredSession(session.id, timestamp))
+    ) {
+      continue;
+    }
+    const parts = await dependencies.repository.listParts(session.id);
+    if (!(await recovery.deleteClaimedSession(session.id))) continue;
+    deletedSessions += 1;
+    blobs += await deleteBlobRefs(
+      dependencies.blobStore,
+      parts.map((part) => part.artifactRef),
+    );
+  }
+  for (const file of files) {
+    await dependencies.repository.deleteFile(file.id);
+    blobs += Number(await dependencies.blobStore.delete(file.artifactRef));
+  }
+  const referencedRefs = new Set([
+    ...(await dependencies.repository.listArtifactRefs()),
+    ...additionalReferencedRefs,
+  ]);
+  const orphans = dependencies.maintenance
+    ? await dependencies.maintenance.cleanupOrphans({
+        referencedRefs,
+        olderThan: new Date(
+          now.getTime() - (dependencies.orphanGraceMs ?? 60 * 60_000),
+        ),
+        limit,
+      })
+    : 0;
+  return { sessions: deletedSessions, files: files.length, blobs, orphans };
+}
+
+async function deleteBlobRefs(
+  blobStore: BlobStore,
+  refs: readonly ArtifactRef[],
+): Promise<number> {
+  let deleted = 0;
+  for (const ref of refs) deleted += Number(await blobStore.delete(ref));
+  return deleted;
+}

package/template/packages/storage/src/upload-completion.ts

@@ -0,0 +1,52 @@
+import type { BlobStore } from "./blob-store.js";
+import type { UploadRepository } from "./upload-repository.js";
+import {
+  UploadError,
+  type FileMetadata,
+  type UploadPart,
+  type UploadSession,
+} from "./upload-types.js";
+
+interface CompletionDependencies {
+  readonly repository: UploadRepository;
+  readonly blobStore: BlobStore;
+}
+
+export async function replayCompletedUpload(
+  dependencies: CompletionDependencies,
+  session: UploadSession,
+): Promise<FileMetadata> {
+  const file = session.fileId
+    ? await dependencies.repository.getFile(session.fileId)
+    : undefined;
+  if (
+    !file ||
+    file.ownerId !== session.ownerId ||
+    file.purpose !== session.purpose ||
+    file.byteSize !== session.expectedByteSize ||
+    (session.expectedContentHash &&
+      file.contentHash.value !== session.expectedContentHash.value)
+  ) {
+    throw new UploadError("FILE_CONTENT_MISSING", "已完成上传的文件不可用");
+  }
+  const parts = await dependencies.repository.listParts(session.id);
+  if (parts.length > 0) {
+    await cleanupCompletedUploadParts(dependencies, session.id, parts);
+  }
+  return file;
+}
+
+export async function cleanupCompletedUploadParts(
+  dependencies: CompletionDependencies,
+  sessionId: string,
+  parts: readonly UploadPart[],
+): Promise<void> {
+  try {
+    await dependencies.repository.deleteParts(sessionId);
+  } catch {
+    return;
+  }
+  for (const part of parts) {
+    await dependencies.blobStore.delete(part.artifactRef).catch(() => false);
+  }
+}

package/template/packages/storage/src/upload-purge.test.ts

@@ -0,0 +1,163 @@
+import { createHash } from "node:crypto";
+import { expect, test } from "bun:test";
+import type { ArtifactRef, BlobStore, PutBlobInput } from "./blob-store.js";
+import { InMemoryUploadRepository } from "./in-memory-upload-repository.js";
+import { MemoryBlobStore } from "./memory-blob-store.js";
+import { UploadService } from "./upload-service.js";
+
+test("completed purge 先删 Blob 后删 metadata，失败可从持久 claim 重启恢复", async () => {
+  const repository = new InMemoryUploadRepository();
+  const blobStore = new FailingDeleteBlobStore();
+  const service = uploadService(repository, blobStore);
+  const bytes = new TextEncoder().encode("{}");
+  const session = await service.create({
+    ownerId: "owner",
+    purpose: "module.output",
+    authorizationBinding: {
+      moduleName: "module",
+      scopeHash: "a".repeat(64),
+    },
+    filename: "output.json",
+    mimeType: "application/json",
+    expectedByteSize: bytes.byteLength,
+    expectedSha256: digest(bytes),
+    fileRetentionMs: 120_000,
+  });
+  await service.appendPart({
+    sessionId: session.id,
+    ownerId: "owner",
+    partNumber: 0,
+    byteOffset: 0,
+    byteSize: bytes.byteLength,
+    sha256: digest(bytes),
+    body: new Blob([bytes]).stream(),
+  });
+  const file = await service.complete(session.id, "owner");
+  const bound = {
+    sessionId: session.id,
+    ownerId: "owner",
+    purpose: "module.output",
+    authorizationBinding: {
+      moduleName: "module",
+      scopeHash: "a".repeat(64),
+    },
+    filename: "output.json",
+    mimeType: "application/json",
+    expectedByteSize: bytes.byteLength,
+    expectedSha256: digest(bytes),
+    fileExpiresAfter: new Date(
+      Date.parse(session.fileExpiresAt!) - 60_000,
+    ).toISOString(),
+    fileExpiresAt: session.fileExpiresAt!,
+    expectedFileId: file.id,
+  };
+
+  await expect(
+    service.purgeCompletedBound({
+      ...bound,
+      authorizationBinding: {
+        ...bound.authorizationBinding,
+        scopeHash: "b".repeat(64),
+      },
+    }),
+  ).rejects.toMatchObject({ code: "FILE_CONTENT_MISSING" });
+  await expect(
+    service.purgeCompletedBound({ ...bound, filename: " output.json " }),
+  ).rejects.toMatchObject({ code: "FILE_CONTENT_MISSING" });
+  expect(await repository.getSession(session.id)).toMatchObject({
+    status: "completed",
+    fileId: file.id,
+  });
+  blobStore.failNextDelete();
+
+  await expect(
+    service.purgeCompletedBound({ ...bound, ownerId: "other" }),
+  ).rejects.toMatchObject({ code: "FILE_CONTENT_MISSING" });
+  await expect(service.purgeCompletedBound(bound)).rejects.toThrow(
+    "simulated blob delete",
+  );
+  expect(await repository.getSession(session.id)).toMatchObject({
+    status: "aborting",
+    fileId: file.id,
+  });
+  expect(await repository.getFile(file.id)).toBeDefined();
+  const afterSessionExpiry = new Date(
+    Date.parse(session.expiresAt) + 1,
+  ).toISOString();
+  expect(await repository.listExpiredSessions(afterSessionExpiry, 100)).toEqual(
+    [],
+  );
+  expect(
+    await repository.claimExpiredSession(session.id, afterSessionExpiry),
+  ).toBe(false);
+  await expect(service.abort(session.id, "owner")).rejects.toMatchObject({
+    code: "UPLOAD_NOT_ACTIVE",
+  });
+  expect(await repository.getSession(session.id)).toMatchObject({
+    status: "aborting",
+    fileId: file.id,
+  });
+  expect(await repository.getFile(file.id)).toBeDefined();
+
+  await uploadService(repository, blobStore).purgeCompletedBound(bound);
+  expect(await repository.getSession(session.id)).toBeUndefined();
+  expect(await repository.getFile(file.id)).toBeUndefined();
+  expect(blobStore.size).toBe(0);
+});
+
+class FailingDeleteBlobStore implements BlobStore {
+  readonly delegate = new MemoryBlobStore();
+  private failNext = false;
+
+  put(input: PutBlobInput) {
+    return this.delegate.put(input);
+  }
+
+  inspect(ref: ArtifactRef) {
+    return this.delegate.inspect(ref);
+  }
+
+  open(ref: ArtifactRef) {
+    return this.delegate.open(ref);
+  }
+
+  delete(ref: ArtifactRef) {
+    if (this.failNext) {
+      this.failNext = false;
+      throw new Error("simulated blob delete");
+    }
+    return this.delegate.delete(ref);
+  }
+
+  get size() {
+    return this.delegate.size;
+  }
+
+  failNextDelete() {
+    this.failNext = true;
+  }
+}
+
+function uploadService(
+  repository: InMemoryUploadRepository,
+  blobStore: BlobStore,
+) {
+  return new UploadService({
+    repository,
+    blobStore,
+    policies: [
+      {
+        purpose: "module.output",
+        maxByteSize: 1024,
+        maxPartByteSize: 512,
+        sessionTtlMs: 60_000,
+        allowedTypes: { "application/json": ["json"] },
+        signature: "none",
+      },
+    ],
+  });
+}
+
+function digest(bytes: Uint8Array) {
+  return createHash("sha256").update(bytes).digest("hex");
+}