create-windy 0.1.0

package/template/packages/storage/src/upload-service.test.ts

@@ -0,0 +1,246 @@
+import { describe, expect, test } from "bun:test";
+import { createHash } from "node:crypto";
+import { artifactRef, BlobStoreError } from "./blob-store.js";
+import { InMemoryUploadRepository } from "./in-memory-upload-repository.js";
+import { MemoryBlobStore } from "./memory-blob-store.js";
+import { UploadService } from "./upload-service.js";
+import { UploadError, type UploadPolicy } from "./upload-types.js";
+
+const policies: UploadPolicy[] = [
+  {
+    purpose: "bulk-data",
+    maxByteSize: 64 * 1024 * 1024,
+    maxPartByteSize: 8 * 1024 * 1024,
+    sessionTtlMs: 60_000,
+    allowedTypes: {
+      "application/x-ndjson": ["ndjson"],
+      "application/json": ["json"],
+    },
+    signature: "none",
+  },
+  {
+    purpose: "branding-logo",
+    maxByteSize: 5 * 1024 * 1024,
+    maxPartByteSize: 1024 * 1024,
+    sessionTtlMs: 60_000,
+    allowedTypes: {
+      "image/png": ["png"],
+      "image/jpeg": ["jpg", "jpeg"],
+      "image/webp": ["webp"],
+    },
+    signature: "image",
+  },
+];
+
+describe("UploadService", () => {
+  test("分片可续传、重复请求幂等，完成时按顺序流式合并并清理分片", async () => {
+    const repository = new InMemoryUploadRepository();
+    const blobStore = new MemoryBlobStore();
+    const service = new UploadService({ repository, blobStore, policies });
+    const chunks = [bytes('{"a":'), bytes("1}\n")];
+    const whole = join(chunks);
+    const session = await service.create({
+      ownerId: "user_1",
+      purpose: "bulk-data",
+      filename: "data.ndjson",
+      mimeType: "application/x-ndjson; charset=utf-8",
+      expectedByteSize: whole.byteLength,
+      expectedSha256: digest(whole),
+    });
+
+    const first = await service.appendPart(
+      partInput(session.id, 0, 0, chunks[0]!),
+    );
+    const duplicate = await service.appendPart(
+      partInput(session.id, 0, 0, chunks[0]!),
+    );
+    expect(duplicate.artifactRef).toBe(first.artifactRef);
+    expect(blobStore.size).toBe(1);
+    await service.appendPart(
+      partInput(session.id, 1, chunks[0]!.byteLength, chunks[1]!),
+    );
+    expect((await repository.getSession(session.id))?.receivedByteSize).toBe(
+      whole.byteLength,
+    );
+
+    const file = await service.complete(session.id, "user_1");
+    expect(file.contentHash.value).toBe(digest(whole));
+    expect(blobStore.size).toBe(1);
+    expect(await repository.listParts(session.id)).toEqual([]);
+    const stored = await blobStore.open(file.artifactRef);
+    expect(await new Response(stored!.body).text()).toBe('{"a":1}\n');
+  });
+
+  test("校验用途、大小、MIME 与扩展名，并嗅探图片签名", async () => {
+    const { service, blobStore } = fixture();
+    await expect(
+      service.create({
+        ownerId: "user_1",
+        purpose: "branding-logo",
+        filename: "logo.exe",
+        mimeType: "image/png",
+        expectedByteSize: 8,
+      }),
+    ).rejects.toMatchObject({ code: "UNSUPPORTED_FILE_TYPE" });
+    const session = await service.create({
+      ownerId: "user_1",
+      purpose: "branding-logo",
+      filename: "logo.png",
+      mimeType: "image/png",
+      expectedByteSize: 8,
+    });
+    await expect(
+      service.appendPart(partInput(session.id, 0, 0, bytes("not-a-png"))),
+    ).rejects.toBeInstanceOf(UploadError);
+    expect(blobStore.size).toBe(0);
+  });
+
+  test("错误总摘要不会产生最终孤儿，分片仍可供诊断或重试", async () => {
+    const { service, blobStore, repository } = fixture();
+    const body = bytes("{}\n");
+    const session = await service.create({
+      ownerId: "user_1",
+      purpose: "bulk-data",
+      filename: "data.ndjson",
+      mimeType: "application/x-ndjson",
+      expectedByteSize: body.byteLength,
+      expectedSha256: "0".repeat(64),
+    });
+    await service.appendPart(partInput(session.id, 0, 0, body));
+    await expect(service.complete(session.id, "user_1")).rejects.toMatchObject({
+      code: "HASH_MISMATCH",
+    } satisfies Partial<BlobStoreError>);
+    expect(blobStore.size).toBe(1);
+    expect(await repository.listParts(session.id)).toHaveLength(1);
+  });
+
+  test("过期会话和文件清理底层 blob 及元数据", async () => {
+    let now = new Date("2026-07-16T00:00:00.000Z");
+    const repository = new InMemoryUploadRepository();
+    const blobStore = new MemoryBlobStore();
+    const service = new UploadService({
+      repository,
+      blobStore,
+      policies,
+      now: () => now,
+    });
+    const body = bytes("{}\n");
+    const completed = await service.create({
+      ownerId: "user_1",
+      purpose: "bulk-data",
+      filename: "result.ndjson",
+      mimeType: "application/x-ndjson",
+      expectedByteSize: body.byteLength,
+      fileExpiresAt: new Date("2026-07-16T00:00:30.000Z"),
+    });
+    await service.appendPart(partInput(completed.id, 0, 0, body));
+    const file = await service.complete(completed.id, "user_1");
+    const incomplete = await service.create({
+      ownerId: "user_1",
+      purpose: "bulk-data",
+      filename: "source.ndjson",
+      mimeType: "application/x-ndjson",
+      expectedByteSize: body.byteLength,
+    });
+    await service.appendPart(partInput(incomplete.id, 0, 0, body));
+    now = new Date("2026-07-16T00:01:00.000Z");
+
+    expect(await service.cleanupExpired()).toEqual({
+      sessions: 1,
+      files: 1,
+      blobs: 2,
+      orphans: 0,
+    });
+    expect(await repository.getSession(incomplete.id)).toBeUndefined();
+    expect(await repository.getFile(file.id)).toBeUndefined();
+    expect(blobStore.size).toBe(0);
+  });
+
+  test("孤儿清理合并应用层登记的其他 Blob 引用", async () => {
+    const external = artifactRef("blob_11111111111111111111111111111111");
+    let captured = new Set<string>();
+    const service = new UploadService({
+      repository: new InMemoryUploadRepository(),
+      blobStore: new MemoryBlobStore(),
+      policies,
+      maintenance: {
+        async cleanupOrphans({ referencedRefs }) {
+          captured = new Set<string>(referencedRefs);
+          return 0;
+        },
+      },
+    });
+
+    await service.cleanupExpired(100, [external]);
+    expect(captured).toEqual(new Set([external]));
+  });
+
+  test("所有者和续传偏移均为服务端约束", async () => {
+    const { service } = fixture();
+    const session = await service.create({
+      ownerId: "user_1",
+      purpose: "bulk-data",
+      filename: "data.json",
+      mimeType: "application/json",
+      expectedByteSize: 2,
+    });
+    await expect(service.complete(session.id, "user_2")).rejects.toMatchObject({
+      code: "UPLOAD_FORBIDDEN",
+    });
+    await expect(
+      service.appendPart(partInput(session.id, 1, 1, bytes("{}"))),
+    ).rejects.toMatchObject({ code: "UPLOAD_OFFSET_CONFLICT" });
+  });
+});
+
+function fixture() {
+  const repository = new InMemoryUploadRepository();
+  const blobStore = new MemoryBlobStore();
+  const service = new UploadService({ repository, blobStore, policies });
+  return { service, blobStore, repository };
+}
+
+function partInput(
+  sessionId: string,
+  partNumber: number,
+  byteOffset: number,
+  value: Uint8Array,
+) {
+  return {
+    sessionId,
+    ownerId: "user_1",
+    partNumber,
+    byteOffset,
+    byteSize: value.byteLength,
+    sha256: digest(value),
+    body: stream(value),
+  };
+}
+
+function stream(value: Uint8Array) {
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(value);
+      controller.close();
+    },
+  });
+}
+
+function bytes(value: string) {
+  return new TextEncoder().encode(value);
+}
+
+function digest(value: Uint8Array) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function join(values: Uint8Array[]) {
+  const size = values.reduce((sum, value) => sum + value.byteLength, 0);
+  const result = new Uint8Array(size);
+  let offset = 0;
+  for (const value of values) {
+    result.set(value, offset);
+    offset += value.byteLength;
+  }
+  return result;
+}

package/template/packages/storage/src/upload-service.ts

@@ -0,0 +1,334 @@
+import { randomUUID } from "node:crypto";
+import type {
+  ArtifactRef,
+  BlobStore,
+  BlobStoreMaintenance,
+} from "./blob-store.js";
+import { sha256ContentHash } from "./content-hash.js";
+import type { UploadRepository } from "./upload-repository.js";
+import { concatBlobStreams, guardFileSignature } from "./upload-streams.js";
+import {
+  UploadError,
+  type CreateUploadInput,
+  type FileMetadata,
+  type UploadPart,
+  type UploadPartInput,
+  type UploadPolicy,
+  type UploadSession,
+} from "./upload-types.js";
+
+export interface UploadServiceOptions {
+  repository: UploadRepository;
+  blobStore: BlobStore;
+  policies: readonly UploadPolicy[];
+  maintenance?: BlobStoreMaintenance;
+  orphanGraceMs?: number;
+  now?: () => Date;
+  createId?: (prefix: "upload" | "file") => string;
+}
+
+export interface CleanupResult {
+  sessions: number;
+  files: number;
+  blobs: number;
+  orphans: number;
+}
+
+export class UploadService {
+  private readonly now: () => Date;
+  private readonly createId: (prefix: "upload" | "file") => string;
+  private readonly policies: ReadonlyMap<string, UploadPolicy>;
+
+  constructor(private readonly options: UploadServiceOptions) {
+    this.now = options.now ?? (() => new Date());
+    this.createId = options.createId ?? randomOpaqueId;
+    this.policies = new Map(
+      options.policies.map((policy) => [policy.purpose, policy]),
+    );
+  }
+
+  async create(input: CreateUploadInput): Promise<UploadSession> {
+    const policy = this.requirePolicy(input.purpose);
+    const filename = validFilename(input.filename);
+    const extension = extensionOf(filename);
+    const mimeType = normalizeMime(input.mimeType);
+    validateType(policy, mimeType, extension);
+    if (
+      !Number.isSafeInteger(input.expectedByteSize) ||
+      input.expectedByteSize < 0
+    ) {
+      throw new UploadError("INVALID_UPLOAD", "文件大小无效");
+    }
+    if (input.expectedByteSize > policy.maxByteSize) {
+      throw new UploadError("UPLOAD_TOO_LARGE", "文件超过用途允许的大小");
+    }
+    const now = this.now();
+    if (input.fileExpiresAt && input.fileExpiresAt <= now) {
+      throw new UploadError("INVALID_UPLOAD", "文件过期时间必须晚于当前时间");
+    }
+    return this.options.repository.createSession({
+      id: this.createId("upload"),
+      ownerId: input.ownerId,
+      purpose: input.purpose,
+      filename,
+      extension,
+      mimeType,
+      expectedByteSize: input.expectedByteSize,
+      expectedContentHash: input.expectedSha256
+        ? sha256ContentHash(input.expectedSha256)
+        : undefined,
+      expiresAt: new Date(now.getTime() + policy.sessionTtlMs).toISOString(),
+      fileExpiresAt: input.fileExpiresAt?.toISOString(),
+      createdAt: now.toISOString(),
+    });
+  }
+
+  async appendPart(input: UploadPartInput): Promise<UploadPart> {
+    const session = await this.requireOwnedActive(
+      input.sessionId,
+      input.ownerId,
+    );
+    const duplicate = await this.options.repository.getPart(
+      session.id,
+      input.partNumber,
+    );
+    const hash = sha256ContentHash(input.sha256);
+    if (duplicate && samePart(duplicate, input, hash.value)) return duplicate;
+    if (
+      input.partNumber !== session.nextPartNumber ||
+      input.byteOffset !== session.receivedByteSize
+    ) {
+      throw new UploadError(
+        "UPLOAD_OFFSET_CONFLICT",
+        `续传位置冲突，服务端偏移量为 ${session.receivedByteSize}`,
+      );
+    }
+    const policy = this.requirePolicy(session.purpose);
+    validatePartSize(input.byteSize, policy, session);
+    const body =
+      input.partNumber === 0 && policy.signature === "image"
+        ? guardFileSignature(input.body, session.mimeType)
+        : input.body;
+    const stored = await this.options.blobStore.put({
+      body,
+      expectedByteSize: input.byteSize,
+      expectedContentHash: hash,
+    });
+    const part: UploadPart = {
+      sessionId: session.id,
+      partNumber: input.partNumber,
+      byteOffset: input.byteOffset,
+      artifactRef: stored.ref,
+      byteSize: stored.byteSize,
+      contentHash: stored.contentHash,
+      createdAt: this.now().toISOString(),
+    };
+    const committed = await this.options.repository.commitPart(
+      part,
+      session.receivedByteSize,
+    );
+    if (!committed) {
+      await this.options.blobStore.delete(stored.ref);
+      throw new UploadError(
+        "CONCURRENT_UPLOAD",
+        "上传会话已被并发更新，请续传",
+      );
+    }
+    return part;
+  }
+
+  getSession(sessionId: string, ownerId: string): Promise<UploadSession> {
+    return this.requireOwnedActive(sessionId, ownerId);
+  }
+
+  async complete(sessionId: string, ownerId: string): Promise<FileMetadata> {
+    const session = await this.requireOwnedActive(sessionId, ownerId);
+    if (session.receivedByteSize !== session.expectedByteSize) {
+      throw new UploadError("UPLOAD_INCOMPLETE", "上传内容尚未完整接收");
+    }
+    const parts = await this.options.repository.listParts(session.id);
+    ensureContiguous(parts, session);
+    const stored = await this.options.blobStore.put({
+      body: concatBlobStreams(
+        this.options.blobStore,
+        parts.map((part) => part.artifactRef),
+      ),
+      expectedByteSize: session.expectedByteSize,
+      expectedContentHash: session.expectedContentHash,
+    });
+    const file: FileMetadata = {
+      id: this.createId("file"),
+      ownerId: session.ownerId,
+      purpose: session.purpose,
+      filename: session.filename,
+      extension: session.extension,
+      mimeType: session.mimeType,
+      artifactRef: stored.ref,
+      byteSize: stored.byteSize,
+      contentHash: stored.contentHash,
+      expiresAt: session.fileExpiresAt,
+      createdAt: this.now().toISOString(),
+    };
+    const committed = await this.options.repository.completeSession(
+      session.id,
+      session.receivedByteSize,
+      file,
+      this.now().toISOString(),
+    );
+    if (!committed) {
+      await this.options.blobStore.delete(stored.ref);
+      throw new UploadError("CONCURRENT_UPLOAD", "上传会话已被并发更新");
+    }
+    await this.deleteParts(session.id, parts);
+    return file;
+  }
+
+  async abort(sessionId: string, ownerId: string): Promise<void> {
+    const session = await this.requireOwnedActive(sessionId, ownerId);
+    const parts = await this.options.repository.listParts(session.id);
+    if (
+      !(await this.options.repository.abortSession(
+        session.id,
+        ownerId,
+        this.now().toISOString(),
+      ))
+    ) {
+      throw new UploadError("CONCURRENT_UPLOAD", "上传会话已被并发更新");
+    }
+    await this.deleteParts(session.id, parts);
+  }
+
+  async cleanupExpired(
+    limit = 100,
+    additionalReferencedRefs: Iterable<ArtifactRef> = [],
+  ): Promise<CleanupResult> {
+    const now = this.now().toISOString();
+    const sessions = await this.options.repository.listExpiredSessions(
+      now,
+      limit,
+    );
+    const files = await this.options.repository.listExpiredFiles(now, limit);
+    let blobs = 0;
+    for (const session of sessions) {
+      const parts = await this.options.repository.listParts(session.id);
+      await this.options.repository.deleteSession(session.id);
+      blobs += await this.deleteBlobRefs(parts.map((part) => part.artifactRef));
+    }
+    for (const file of files) {
+      await this.options.repository.deleteFile(file.id);
+      blobs += Number(await this.options.blobStore.delete(file.artifactRef));
+    }
+    const referencedRefs = new Set([
+      ...(await this.options.repository.listArtifactRefs()),
+      ...additionalReferencedRefs,
+    ]);
+    const orphans = this.options.maintenance
+      ? await this.options.maintenance.cleanupOrphans({
+          referencedRefs,
+          olderThan: new Date(
+            this.now().getTime() - (this.options.orphanGraceMs ?? 60 * 60_000),
+          ),
+          limit,
+        })
+      : 0;
+    return { sessions: sessions.length, files: files.length, blobs, orphans };
+  }
+
+  private async requireOwnedActive(id: string, ownerId: string) {
+    const session = await this.options.repository.getSession(id);
+    if (!session) throw new UploadError("UPLOAD_NOT_FOUND", "上传会话不存在");
+    if (session.ownerId !== ownerId)
+      throw new UploadError("UPLOAD_FORBIDDEN", "无权访问该上传会话");
+    if (session.expiresAt <= this.now().toISOString())
+      throw new UploadError("UPLOAD_EXPIRED", "上传会话已过期");
+    if (session.status !== "uploading")
+      throw new UploadError("UPLOAD_NOT_ACTIVE", "上传会话不可继续操作");
+    return session;
+  }
+
+  private requirePolicy(purpose: string) {
+    const policy = this.policies.get(purpose);
+    if (!policy) throw new UploadError("INVALID_UPLOAD", "上传用途无效");
+    return policy;
+  }
+
+  private async deleteParts(sessionId: string, parts: readonly UploadPart[]) {
+    await this.deleteBlobRefs(parts.map((part) => part.artifactRef));
+    await this.options.repository.deleteParts(sessionId);
+  }
+
+  private async deleteBlobRefs(refs: readonly UploadPart["artifactRef"][]) {
+    let deleted = 0;
+    for (const ref of refs)
+      deleted += Number(await this.options.blobStore.delete(ref));
+    return deleted;
+  }
+}
+
+function randomOpaqueId(prefix: "upload" | "file") {
+  return `${prefix}_${randomUUID().replaceAll("-", "")}`;
+}
+
+function validFilename(value: string) {
+  const filename = value.trim();
+  if (
+    !filename ||
+    filename.length > 240 ||
+    filename.includes("/") ||
+    filename.includes("\\") ||
+    filename.includes(String.fromCharCode(0))
+  )
+    throw new UploadError("INVALID_UPLOAD", "文件名无效");
+  return filename;
+}
+
+function extensionOf(filename: string) {
+  const dot = filename.lastIndexOf(".");
+  if (dot < 1 || dot === filename.length - 1)
+    throw new UploadError("UNSUPPORTED_FILE_TYPE", "文件扩展名无效");
+  return filename.slice(dot + 1).toLowerCase();
+}
+
+function normalizeMime(value: string) {
+  return value.split(";", 1)[0]!.trim().toLowerCase();
+}
+
+function validateType(policy: UploadPolicy, mime: string, extension: string) {
+  if (!policy.allowedTypes[mime]?.includes(extension))
+    throw new UploadError(
+      "UNSUPPORTED_FILE_TYPE",
+      "MIME 类型与文件扩展名不匹配或不受支持",
+    );
+}
+
+function validatePartSize(
+  size: number,
+  policy: UploadPolicy,
+  session: UploadSession,
+) {
+  if (!Number.isSafeInteger(size) || size <= 0)
+    throw new UploadError("INVALID_UPLOAD", "分片大小无效");
+  if (size > policy.maxPartByteSize)
+    throw new UploadError("PART_TOO_LARGE", "分片超过允许大小");
+  if (session.receivedByteSize + size > session.expectedByteSize)
+    throw new UploadError("UPLOAD_TOO_LARGE", "分片超过文件声明大小");
+}
+
+function samePart(part: UploadPart, input: UploadPartInput, hash: string) {
+  return (
+    part.byteOffset === input.byteOffset &&
+    part.byteSize === input.byteSize &&
+    part.contentHash.value === hash
+  );
+}
+
+function ensureContiguous(parts: UploadPart[], session: UploadSession) {
+  let offset = 0;
+  for (const [index, part] of parts.entries()) {
+    if (part.partNumber !== index || part.byteOffset !== offset)
+      throw new UploadError("UPLOAD_INCOMPLETE", "上传分片不连续");
+    offset += part.byteSize;
+  }
+  if (offset !== session.expectedByteSize)
+    throw new UploadError("UPLOAD_INCOMPLETE", "上传分片大小不完整");
+}

package/template/packages/storage/src/upload-streams.ts

@@ -0,0 +1,123 @@
+import type { ArtifactRef, BlobStore } from "./blob-store.js";
+import { UploadError } from "./upload-types.js";
+
+export function concatBlobStreams(
+  store: BlobStore,
+  refs: readonly ArtifactRef[],
+): ReadableStream<Uint8Array> {
+  let current: ReadableStreamDefaultReader<Uint8Array> | undefined;
+  let index = 0;
+  return new ReadableStream({
+    async pull(controller) {
+      while (true) {
+        if (!current) {
+          const ref = refs[index++];
+          if (!ref) return controller.close();
+          const blob = await store.open(ref);
+          if (!blob) {
+            return controller.error(
+              new UploadError("UPLOAD_INCOMPLETE", "上传分片内容不存在"),
+            );
+          }
+          current = blob.body.getReader();
+        }
+        const result = await current.read();
+        if (result.done) {
+          current.releaseLock();
+          current = undefined;
+          continue;
+        }
+        controller.enqueue(result.value);
+        return;
+      }
+    },
+    async cancel(reason) {
+      await current?.cancel(reason);
+    },
+  });
+}
+
+export function guardFileSignature(
+  body: ReadableStream<Uint8Array>,
+  mimeType: string,
+): ReadableStream<Uint8Array> {
+  const reader = body.getReader();
+  const requiredBytes = signatureByteCount(mimeType);
+  const pending: Uint8Array[] = [];
+  const prefix = new Uint8Array(requiredBytes);
+  let prefixSize = 0;
+  let checked = false;
+  return new ReadableStream({
+    async pull(controller) {
+      if (!checked) {
+        while (prefixSize < requiredBytes) {
+          const result = await reader.read();
+          if (result.done) {
+            controller.error(signatureError());
+            return;
+          }
+          pending.push(result.value);
+          const needed = requiredBytes - prefixSize;
+          const slice = result.value.subarray(0, needed);
+          prefix.set(slice, prefixSize);
+          prefixSize += slice.byteLength;
+        }
+        checked = true;
+        if (!matchesSignature(prefix, mimeType)) {
+          await reader.cancel();
+          controller.error(signatureError());
+          return;
+        }
+      }
+      const buffered = pending.shift();
+      if (buffered) {
+        controller.enqueue(buffered);
+        return;
+      }
+      const result = await reader.read();
+      if (result.done) controller.close();
+      else controller.enqueue(result.value);
+    },
+    async cancel(reason) {
+      await reader.cancel(reason);
+    },
+  });
+}
+
+function signatureByteCount(mimeType: string) {
+  if (mimeType === "image/png") return 8;
+  if (mimeType === "image/jpeg") return 3;
+  if (mimeType === "image/webp") return 12;
+  return 1;
+}
+
+function matchesSignature(prefix: Uint8Array, mimeType: string): boolean {
+  if (mimeType === "image/png") {
+    return startsWith(prefix, [137, 80, 78, 71, 13, 10, 26, 10]);
+  }
+  if (mimeType === "image/jpeg") {
+    return startsWith(prefix, [255, 216, 255]);
+  }
+  if (mimeType === "image/webp") {
+    return text(prefix, 0, 4) === "RIFF" && text(prefix, 8, 12) === "WEBP";
+  }
+  return true;
+}
+
+function startsWith(value: Uint8Array, expected: number[]) {
+  return (
+    value.length >= expected.length &&
+    expected.every((byte, index) => value[index] === byte)
+  );
+}
+
+function text(value: Uint8Array, start: number, end: number) {
+  return new TextDecoder().decode(value.subarray(start, end));
+}
+
+function signatureError() {
+  return new UploadError(
+    "FILE_SIGNATURE_MISMATCH",
+    "文件内容与声明的 MIME 类型不一致",
+  );
+}