create-windy 0.5.2 → 0.6.0

package/template/packages/storage/src/upload-purge.ts

@@ -0,0 +1,261 @@
+import type { BlobStore } from "./blob-store.js";
+import {
+  purgeRecoverableUploadRepository,
+  type UploadRepository,
+} from "./upload-repository.js";
+import { throwIfUploadCancelled } from "./upload-streams.js";
+import { UploadError, type UploadSession } from "./upload-types.js";
+import type { UploadAuthorizationBinding } from "./upload-types.js";
+import {
+  canonicalizeUploadIdentity,
+  validateAuthorizationBinding,
+} from "./upload-validation.js";
+
+interface PurgeDependencies {
+  readonly repository: UploadRepository;
+  readonly blobStore: BlobStore;
+}
+
+export interface BoundCompletedUploadPurge {
+  readonly sessionId: string;
+  readonly ownerId: string;
+  readonly purpose: string;
+  readonly authorizationBinding: UploadAuthorizationBinding;
+  readonly filename: string;
+  readonly mimeType: string;
+  readonly expectedByteSize: number;
+  readonly expectedSha256: string;
+  readonly fileExpiresAfter: string;
+  readonly fileExpiresAt: string;
+  readonly expectedFileId?: string;
+}
+
+export async function purgeBoundCompletedUpload(
+  dependencies: PurgeDependencies,
+  input: BoundCompletedUploadPurge,
+  allowedTypes: Readonly<Record<string, readonly string[]>>,
+  updatedAt: string,
+  signal?: AbortSignal,
+): Promise<boolean> {
+  validateBoundInput(input, allowedTypes);
+  const session = await dependencies.repository.getSession(input.sessionId);
+  if (!session) {
+    if (
+      input.expectedFileId &&
+      (await dependencies.repository.getFile(input.expectedFileId))
+    ) {
+      throw bindingDrift();
+    }
+    return true;
+  }
+  requireBoundSession(session, input);
+  if (!session.fileId || !["completed", "aborting"].includes(session.status)) {
+    return false;
+  }
+  const file = await dependencies.repository.getFile(session.fileId);
+  if (file) requireBoundFile(file, input, session.fileId);
+  await purgeCompletedUpload(
+    dependencies,
+    session,
+    input.ownerId,
+    updatedAt,
+    signal,
+  );
+  return true;
+}
+
+/**
+ * 对已完成 Upload 做先 Blob、后 metadata 的持久化 fenced 清理。
+ * completed -> aborting 是持久 claim；进程重启后可从 aborting 继续。
+ */
+export async function purgeCompletedUpload(
+  dependencies: PurgeDependencies,
+  initial: UploadSession,
+  ownerId: string,
+  updatedAt: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const repository = purgeRecoverableUploadRepository(dependencies.repository);
+  if (!repository) {
+    throw new UploadError(
+      "CONCURRENT_UPLOAD",
+      "当前 Storage Repository 不支持已完成会话恢复清理",
+    );
+  }
+  requireOwner(initial, ownerId);
+  const fileId = initial.fileId;
+  if (!fileId) {
+    throw new UploadError("FILE_CONTENT_MISSING", "上传会话缺少文件引用");
+  }
+  let session = initial;
+  if (session.status === "completed") {
+    throwIfUploadCancelled(signal);
+    const claimed = await repository.beginPurgeCompletedSession(
+      session.id,
+      ownerId,
+      fileId,
+      updatedAt,
+    );
+    session = claimed
+      ? { ...session, status: "aborting", updatedAt }
+      : await reload(repository, session.id, ownerId);
+  }
+  if (session.status !== "aborting" || session.fileId !== fileId) {
+    throw new UploadError("CONCURRENT_UPLOAD", "上传会话清理 claim 已漂移");
+  }
+  await purgeParts(dependencies, session.id, signal);
+  await purgeFile(dependencies, fileId, session, signal);
+  throwIfUploadCancelled(signal);
+  if (!(await repository.deletePurgedSession(session.id, ownerId, fileId))) {
+    const current = await repository.getSession(session.id);
+    if (current) {
+      throw new UploadError("CONCURRENT_UPLOAD", "上传会话清理尚未完成");
+    }
+  }
+}
+
+async function purgeParts(
+  dependencies: PurgeDependencies,
+  sessionId: string,
+  signal?: AbortSignal,
+) {
+  const parts = await dependencies.repository.listParts(sessionId);
+  for (const part of parts) {
+    throwIfUploadCancelled(signal);
+    await dependencies.blobStore.delete(part.artifactRef);
+  }
+  throwIfUploadCancelled(signal);
+  await dependencies.repository.deleteParts(sessionId);
+}
+
+async function purgeFile(
+  dependencies: PurgeDependencies,
+  fileId: string,
+  session: UploadSession,
+  signal?: AbortSignal,
+) {
+  const file = await dependencies.repository.getFile(fileId);
+  if (!file) return;
+  if (
+    file.ownerId !== session.ownerId ||
+    file.purpose !== session.purpose ||
+    file.byteSize !== session.expectedByteSize ||
+    (session.expectedContentHash &&
+      file.contentHash.value !== session.expectedContentHash.value)
+  ) {
+    throw new UploadError("FILE_CONTENT_MISSING", "已完成上传文件已漂移");
+  }
+  throwIfUploadCancelled(signal);
+  await dependencies.blobStore.delete(file.artifactRef);
+  throwIfUploadCancelled(signal);
+  await dependencies.repository.deleteFile(fileId);
+}
+
+async function reload(
+  repository: UploadRepository,
+  id: string,
+  ownerId: string,
+) {
+  const session = await repository.getSession(id);
+  if (!session) throw new UploadError("UPLOAD_NOT_FOUND", "上传会话不存在");
+  requireOwner(session, ownerId);
+  return session;
+}
+
+function requireOwner(session: UploadSession, ownerId: string) {
+  if (session.ownerId !== ownerId) {
+    throw new UploadError("UPLOAD_FORBIDDEN", "无权清理该上传会话");
+  }
+}
+
+function requireBoundSession(
+  session: UploadSession,
+  input: BoundCompletedUploadPurge,
+) {
+  if (
+    session.ownerId !== input.ownerId ||
+    session.purpose !== input.purpose ||
+    session.authorizationBinding?.moduleName !==
+      input.authorizationBinding.moduleName ||
+    session.authorizationBinding.scopeHash !==
+      input.authorizationBinding.scopeHash ||
+    session.filename !== input.filename ||
+    session.mimeType !== input.mimeType ||
+    session.expectedByteSize !== input.expectedByteSize ||
+    session.expectedContentHash?.value !== input.expectedSha256 ||
+    !validExpiry(session.fileExpiresAt, input) ||
+    (input.expectedFileId && session.fileId !== input.expectedFileId)
+  ) {
+    throw bindingDrift();
+  }
+}
+
+function requireBoundFile(
+  file: Awaited<ReturnType<UploadRepository["getFile"]>>,
+  input: BoundCompletedUploadPurge,
+  fileId: string,
+) {
+  if (
+    !file ||
+    file.id !== fileId ||
+    file.ownerId !== input.ownerId ||
+    file.purpose !== input.purpose ||
+    file.filename !== input.filename ||
+    file.mimeType !== input.mimeType ||
+    file.byteSize !== input.expectedByteSize ||
+    file.contentHash.value !== input.expectedSha256 ||
+    !validExpiry(file.expiresAt, input)
+  ) {
+    throw bindingDrift();
+  }
+}
+
+function validExpiry(
+  value: string | undefined,
+  input: BoundCompletedUploadPurge,
+) {
+  return (
+    value !== undefined &&
+    value > input.fileExpiresAfter &&
+    value <= input.fileExpiresAt
+  );
+}
+
+function bindingDrift() {
+  return new UploadError(
+    "FILE_CONTENT_MISSING",
+    "已完成上传的授权或文件绑定已漂移",
+  );
+}
+
+function validateBoundInput(
+  input: BoundCompletedUploadPurge,
+  allowedTypes: Readonly<Record<string, readonly string[]>>,
+) {
+  let canonical: ReturnType<typeof canonicalizeUploadIdentity>;
+  try {
+    canonical = canonicalizeUploadIdentity(input, allowedTypes);
+    validateAuthorizationBinding(input.authorizationBinding);
+  } catch {
+    throw bindingDrift();
+  }
+  const after = Date.parse(input.fileExpiresAfter);
+  const expires = Date.parse(input.fileExpiresAt);
+  if (
+    !input.sessionId ||
+    input.sessionId.length > 80 ||
+    !input.ownerId ||
+    !input.purpose ||
+    canonical.filename !== input.filename ||
+    canonical.mimeType !== input.mimeType ||
+    !Number.isSafeInteger(input.expectedByteSize) ||
+    input.expectedByteSize < 0 ||
+    !/^[a-f0-9]{64}$/.test(input.expectedSha256) ||
+    !Number.isFinite(after) ||
+    !Number.isFinite(expires) ||
+    after >= expires ||
+    input.expectedFileId === ""
+  ) {
+    throw bindingDrift();
+  }
+}

package/template/packages/storage/src/upload-repository-v052-compatibility.test.ts

@@ -0,0 +1,31 @@
+import { expect, test } from "bun:test";
+import {
+  recoverableUploadRepository,
+  type UploadRepository,
+} from "./upload-repository.js";
+
+const unsupported = async (): Promise<never> => {
+  throw new Error("fixed 0.5.2 mock");
+};
+
+const fixed052RepositoryMock: UploadRepository = {
+  createSession: unsupported,
+  getSession: unsupported,
+  getPart: unsupported,
+  listParts: unsupported,
+  commitPart: unsupported,
+  completeSession: unsupported,
+  abortSession: unsupported,
+  getFile: unsupported,
+  listExpiredSessions: unsupported,
+  listExpiredFiles: unsupported,
+  listArtifactRefs: unsupported,
+  deleteParts: unsupported,
+  deleteSession: unsupported,
+  deleteFile: unsupported,
+};
+
+test("0.5.2 UploadRepository mock remains source compatible", () => {
+  expect(recoverableUploadRepository(fixed052RepositoryMock)).toBeUndefined();
+  expect(typeof fixed052RepositoryMock.abortSession).toBe("function");
+});

package/template/packages/storage/src/upload-repository.ts

@@ -41,3 +41,64 @@ export interface UploadRepository {
   deleteSession(id: string): Promise<void>;
   deleteFile(id: string): Promise<void>;
 }
+
+/** 可恢复终止与过期清理所需的附加 CAS 能力。 */
+export interface UploadRepositoryRecovery {
+  beginAbortSession(
+    id: string,
+    ownerId: string,
+    updatedAt: string,
+  ): Promise<boolean>;
+  finalizeAbortSession(
+    id: string,
+    ownerId: string,
+    updatedAt: string,
+  ): Promise<boolean>;
+  /** 把已过期且未完成的会话持久化标记为 expired。 */
+  claimExpiredSession(id: string, now: string): Promise<boolean>;
+  /** 仅删除已经持久化 claim 的会话及其分片行。 */
+  deleteClaimedSession(id: string): Promise<boolean>;
+}
+
+export interface RecoverableUploadRepository
+  extends UploadRepository, UploadRepositoryRecovery {}
+
+/** 已完成会话的崩溃可恢复清理；与旧 Recovery 能力分开以保持 v0.5.2 兼容。 */
+export interface UploadRepositoryPurgeRecovery {
+  beginPurgeCompletedSession(
+    id: string,
+    ownerId: string,
+    fileId: string,
+    updatedAt: string,
+  ): Promise<boolean>;
+  deletePurgedSession(
+    id: string,
+    ownerId: string,
+    fileId: string,
+  ): Promise<boolean>;
+}
+
+/** 兼容旧 Repository mock：能力缺失时返回 undefined，由服务 fail closed。 */
+export function recoverableUploadRepository(
+  repository: UploadRepository,
+): RecoverableUploadRepository | undefined {
+  const candidate = repository as UploadRepository &
+    Partial<UploadRepositoryRecovery>;
+  return typeof candidate.beginAbortSession === "function" &&
+    typeof candidate.finalizeAbortSession === "function" &&
+    typeof candidate.claimExpiredSession === "function" &&
+    typeof candidate.deleteClaimedSession === "function"
+    ? (candidate as RecoverableUploadRepository)
+    : undefined;
+}
+
+export function purgeRecoverableUploadRepository(
+  repository: UploadRepository,
+): (UploadRepository & UploadRepositoryPurgeRecovery) | undefined {
+  const candidate = repository as UploadRepository &
+    Partial<UploadRepositoryPurgeRecovery>;
+  return typeof candidate.beginPurgeCompletedSession === "function" &&
+    typeof candidate.deletePurgedSession === "function"
+    ? (candidate as UploadRepository & UploadRepositoryPurgeRecovery)
+    : undefined;
+}

package/template/packages/storage/src/upload-service-recovery.test.ts

@@ -0,0 +1,296 @@
+import { createHash } from "node:crypto";
+import { describe, expect, test } from "bun:test";
+import type { ArtifactRef, BlobStore } from "./blob-store.js";
+import { InMemoryUploadRepository } from "./in-memory-upload-repository.js";
+import { MemoryBlobStore } from "./memory-blob-store.js";
+import { UploadService } from "./upload-service.js";
+import type { UploadPolicy } from "./upload-types.js";
+
+const policy: UploadPolicy = {
+  purpose: "module.output",
+  maxByteSize: 1024,
+  maxPartByteSize: 512,
+  sessionTtlMs: 60_000,
+  allowedTypes: { "application/json": ["json"] },
+  signature: "none",
+};
+
+describe("UploadService 流式恢复", () => {
+  test("顺序、并发和服务重建后的 complete 都返回同一文件", async () => {
+    const repository = new InMemoryUploadRepository();
+    const blobStore = new MemoryBlobStore();
+    const service = createService(repository, blobStore);
+    const session = await seedPart(service);
+
+    const [first, concurrent] = await Promise.all([
+      service.complete(session.id, "owner"),
+      service.complete(session.id, "owner"),
+    ]);
+    const restarted = createService(repository, blobStore);
+
+    expect(concurrent).toEqual(first);
+    expect(await restarted.complete(session.id, "owner")).toEqual(first);
+    expect(await restarted.inspectSession(session.id, "owner")).toMatchObject({
+      status: "completed",
+      fileId: first.id,
+    });
+    await expect(
+      restarted.inspectSession(session.id, "other"),
+    ).rejects.toMatchObject({ code: "UPLOAD_FORBIDDEN" });
+    expect(await repository.listParts(session.id)).toEqual([]);
+    expect(blobStore.size).toBe(1);
+  });
+
+  test("重复 append 会取消未消费 body，流中取消不会提交分片", async () => {
+    const repository = new InMemoryUploadRepository();
+    const blobStore = new MemoryBlobStore();
+    const service = createService(repository, blobStore);
+    const bytes = new TextEncoder().encode("{}");
+    const session = await createSession(service, bytes);
+    const first = await service.appendPart(partInput(session.id, bytes));
+    let duplicateCancelled = false;
+
+    const duplicate = await service.appendPart({
+      ...partInput(session.id, bytes),
+      body: new ReadableStream({
+        cancel() {
+          duplicateCancelled = true;
+        },
+      }),
+    });
+    expect(duplicate.artifactRef).toBe(first.artifactRef);
+    expect(duplicateCancelled).toBe(true);
+
+    const pendingSession = await createSession(service, bytes);
+    const cancellation = cancellableBody();
+    const controller = new AbortController();
+    const append = service.appendPart({
+      ...partInput(pendingSession.id, bytes),
+      body: cancellation.body,
+      signal: controller.signal,
+    });
+    await cancellation.started;
+    controller.abort();
+
+    await expect(append).rejects.toMatchObject({ code: "UPLOAD_CANCELLED" });
+    expect(cancellation.cancelled()).toBe(true);
+    expect(await repository.listParts(pendingSession.id)).toEqual([]);
+    expect(
+      (await repository.getSession(pendingSession.id))?.nextPartNumber,
+    ).toBe(0);
+    expect(blobStore.size).toBe(1);
+
+    const abortController = new AbortController();
+    abortController.abort();
+    await expect(
+      service.abort(pendingSession.id, "owner", abortController.signal),
+    ).rejects.toMatchObject({ code: "UPLOAD_CANCELLED" });
+    expect((await repository.getSession(pendingSession.id))?.status).toBe(
+      "uploading",
+    );
+  });
+
+  test("complete 取消保留可续传分片且不发布最终文件", async () => {
+    const repository = new InMemoryUploadRepository();
+    const base = new MemoryBlobStore();
+    let pauseFinal = false;
+    let markStarted: () => void = () => undefined;
+    let resume: () => void = () => undefined;
+    const started = new Promise<void>((resolve) => {
+      markStarted = resolve;
+    });
+    const gate = new Promise<void>((resolve) => {
+      resume = resolve;
+    });
+    const blobStore: BlobStore = {
+      async put(input) {
+        if (!pauseFinal) return base.put(input);
+        pauseFinal = false;
+        const reader = input.body.getReader();
+        try {
+          await reader.read();
+          markStarted();
+          await gate;
+          await reader.read();
+          throw new Error("取消后的最终写入不应完成");
+        } finally {
+          reader.releaseLock();
+        }
+      },
+      inspect: (ref) => base.inspect(ref),
+      open: (ref) => base.open(ref),
+      delete: (ref) => base.delete(ref),
+    };
+    const service = createService(repository, blobStore);
+    const session = await seedPart(service);
+    pauseFinal = true;
+    const controller = new AbortController();
+    const completion = service.complete(session.id, "owner", controller.signal);
+    await started;
+    controller.abort();
+    resume();
+
+    await expect(completion).rejects.toMatchObject({
+      code: "UPLOAD_CANCELLED",
+    });
+    expect(await repository.listParts(session.id)).toHaveLength(1);
+    expect((await repository.getSession(session.id))?.status).toBe("uploading");
+    expect(base.size).toBe(1);
+
+    const file = await service.complete(session.id, "owner");
+    expect(file.id).toStartWith("file_");
+    expect(base.size).toBe(1);
+  });
+
+  test("完成提交后的清理失败可重放，并由 orphan cleanup 收口", async () => {
+    const repository = new InMemoryUploadRepository();
+    const base = new MemoryBlobStore();
+    const orphan: { ref?: ArtifactRef } = {};
+    let failPartDelete = true;
+    const blobStore: BlobStore = {
+      put: (input) => base.put(input),
+      inspect: (ref) => base.inspect(ref),
+      open: (ref) => base.open(ref),
+      async delete(ref) {
+        if (failPartDelete && ref === orphan.ref) {
+          failPartDelete = false;
+          throw new Error("part delete failed");
+        }
+        return base.delete(ref);
+      },
+    };
+    const service = new UploadService({
+      repository,
+      blobStore,
+      policies: [policy],
+      maintenance: {
+        async cleanupOrphans({ referencedRefs }) {
+          if (!orphan.ref || referencedRefs.has(orphan.ref)) return 0;
+          return Number(await base.delete(orphan.ref));
+        },
+      },
+    });
+    const session = await seedPart(service);
+    orphan.ref = (await repository.listParts(session.id))[0]!.artifactRef;
+
+    const file = await service.complete(session.id, "owner");
+    expect(await repository.listParts(session.id)).toEqual([]);
+    const completed = await service.inspectSession(session.id, "owner");
+    expect(completed.status).toBe("completed");
+
+    const replayed = await service.complete(session.id, "owner");
+    expect(replayed).toEqual(file);
+    expect(replayed.id).toBe(completed.fileId!);
+    expect(await service.cleanupExpired()).toMatchObject({ orphans: 1 });
+    expect(base.size).toBe(1);
+  });
+
+  test("分片行清理失败不遮蔽完成结果，重复 complete 会继续收口", async () => {
+    const repository = new FailingPartCleanupRepository();
+    const blobStore = new MemoryBlobStore();
+    const service = createService(repository, blobStore);
+    const session = await seedPart(service);
+
+    repository.failNextPartCleanup();
+    const first = await service.complete(session.id, "owner");
+    expect(await repository.listParts(session.id)).toHaveLength(1);
+    const completed = await service.inspectSession(session.id, "owner");
+    expect(completed.status).toBe("completed");
+    expect(blobStore.size).toBe(2);
+    expect(await repository.listArtifactRefs()).toEqual([first.artifactRef]);
+
+    const replayed = await service.complete(session.id, "owner");
+    expect(replayed).toEqual(first);
+    expect(replayed.id).toBe(completed.fileId!);
+    expect(await repository.listParts(session.id)).toEqual([]);
+    expect(blobStore.size).toBe(1);
+  });
+});
+
+class FailingPartCleanupRepository extends InMemoryUploadRepository {
+  private shouldFailPartCleanup = false;
+
+  failNextPartCleanup() {
+    this.shouldFailPartCleanup = true;
+  }
+
+  override async deleteParts(sessionId: string) {
+    if (this.shouldFailPartCleanup) {
+      this.shouldFailPartCleanup = false;
+      throw new Error("part row cleanup failed");
+    }
+    await super.deleteParts(sessionId);
+  }
+}
+
+function createService(
+  repository: InMemoryUploadRepository,
+  blobStore: BlobStore,
+) {
+  return new UploadService({
+    repository,
+    blobStore,
+    policies: [policy],
+  });
+}
+
+async function seedPart(service: UploadService) {
+  const bytes = new TextEncoder().encode('{"ok":true}');
+  const session = await createSession(service, bytes);
+  await service.appendPart(partInput(session.id, bytes));
+  return session;
+}
+
+function createSession(service: UploadService, value: Uint8Array) {
+  return service.create({
+    ownerId: "owner",
+    purpose: policy.purpose,
+    filename: "result.json",
+    mimeType: "application/json",
+    expectedByteSize: value.byteLength,
+    expectedSha256: digest(value),
+  });
+}
+
+function partInput(sessionId: string, value: Uint8Array) {
+  return {
+    sessionId,
+    ownerId: "owner",
+    partNumber: 0,
+    byteOffset: 0,
+    byteSize: value.byteLength,
+    sha256: digest(value),
+    body: new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(value);
+        controller.close();
+      },
+    }),
+  };
+}
+
+function cancellableBody() {
+  let markStarted: () => void = () => undefined;
+  let cancelled = false;
+  const started = new Promise<void>((resolve) => {
+    markStarted = resolve;
+  });
+  let pulled = false;
+  const body = new ReadableStream<Uint8Array>({
+    pull(controller) {
+      if (pulled) return;
+      pulled = true;
+      controller.enqueue(Uint8Array.of(123));
+      markStarted();
+      return new Promise<void>((resolve) => setTimeout(resolve, 10));
+    },
+    cancel() {
+      cancelled = true;
+    },
+  });
+  return { body, started, cancelled: () => cancelled };
+}
+
+function digest(value: Uint8Array) {
+  return createHash("sha256").update(value).digest("hex");
+}