create-windy 0.5.1 → 0.6.0

package/template/packages/storage/src/upload-service.ts

@@ -1,12 +1,26 @@
-import { randomUUID } from "node:crypto";
 import type {
   ArtifactRef,
   BlobStore,
   BlobStoreMaintenance,
 } from "./blob-store.js";
 import { sha256ContentHash } from "./content-hash.js";
+import { abortUploadSession } from "./upload-abort.js";
+import { cleanupExpiredUploads, type CleanupResult } from "./upload-cleanup.js";
 import type { UploadRepository } from "./upload-repository.js";
-import { concatBlobStreams, guardFileSignature } from "./upload-streams.js";
+import {
+  cleanupCompletedUploadParts,
+  replayCompletedUpload,
+} from "./upload-completion.js";
+import {
+  purgeBoundCompletedUpload,
+  type BoundCompletedUploadPurge,
+} from "./upload-purge.js";
+import {
+  abortAwareStream,
+  concatBlobStreams,
+  guardFileSignature,
+  throwIfUploadCancelled,
+} from "./upload-streams.js";
 import {
   UploadError,
   type CreateUploadInput,
@@ -16,6 +30,14 @@ import {
   type UploadPolicy,
   type UploadSession,
 } from "./upload-types.js";
+import {
+  canonicalizeUploadIdentity,
+  ensureContiguous,
+  randomOpaqueId,
+  samePart,
+  validateAuthorizationBinding,
+  validatePartSize,
+} from "./upload-validation.js";
 
 export interface UploadServiceOptions {
   repository: UploadRepository;
@@ -27,12 +49,7 @@ export interface UploadServiceOptions {
   createId?: (prefix: "upload" | "file") => string;
 }
 
-export interface CleanupResult {
-  sessions: number;
-  files: number;
-  blobs: number;
-  orphans: number;
-}
+export type { CleanupResult } from "./upload-cleanup.js";
 
 export class UploadService {
   private readonly now: () => Date;
@@ -53,10 +70,10 @@ export class UploadService {
 
   async create(input: CreateUploadInput): Promise<UploadSession> {
     const policy = this.requirePolicy(input.purpose);
-    const filename = validFilename(input.filename);
-    const extension = extensionOf(filename);
-    const mimeType = normalizeMime(input.mimeType);
-    validateType(policy, mimeType, extension);
+    const { filename, extension, mimeType } = canonicalizeUploadIdentity(
+      input,
+      policy.allowedTypes,
+    );
     if (
       !Number.isSafeInteger(input.expectedByteSize) ||
       input.expectedByteSize < 0
@@ -66,14 +83,33 @@ export class UploadService {
     if (input.expectedByteSize > policy.maxByteSize) {
       throw new UploadError("UPLOAD_TOO_LARGE", "文件超过用途允许的大小");
     }
+    validateAuthorizationBinding(input.authorizationBinding);
     const now = this.now();
-    if (input.fileExpiresAt && input.fileExpiresAt <= now) {
+    if (input.fileExpiresAt && input.fileRetentionMs !== undefined) {
+      throw new UploadError("INVALID_UPLOAD", "文件保留策略不能重复指定");
+    }
+    if (
+      input.fileRetentionMs !== undefined &&
+      (!Number.isSafeInteger(input.fileRetentionMs) ||
+        input.fileRetentionMs <= 0)
+    ) {
+      throw new UploadError("INVALID_UPLOAD", "文件保留期无效");
+    }
+    const fileExpiresAt =
+      input.fileRetentionMs === undefined
+        ? input.fileExpiresAt
+        : new Date(now.getTime() + input.fileRetentionMs);
+    if (
+      fileExpiresAt &&
+      (!Number.isFinite(fileExpiresAt.getTime()) || fileExpiresAt <= now)
+    ) {
       throw new UploadError("INVALID_UPLOAD", "文件过期时间必须晚于当前时间");
     }
     return this.options.repository.createSession({
       id: this.createId("upload"),
       ownerId: input.ownerId,
       purpose: input.purpose,
+      authorizationBinding: input.authorizationBinding,
       filename,
       extension,
       mimeType,
@@ -82,12 +118,13 @@ export class UploadService {
         ? sha256ContentHash(input.expectedSha256)
         : undefined,
       expiresAt: new Date(now.getTime() + policy.sessionTtlMs).toISOString(),
-      fileExpiresAt: input.fileExpiresAt?.toISOString(),
+      fileExpiresAt: fileExpiresAt?.toISOString(),
       createdAt: now.toISOString(),
     });
   }
 
   async appendPart(input: UploadPartInput): Promise<UploadPart> {
+    throwIfUploadCancelled(input.signal);
     const session = await this.requireOwnedActive(
       input.sessionId,
       input.ownerId,
@@ -97,7 +134,10 @@ export class UploadService {
       input.partNumber,
     );
     const hash = sha256ContentHash(input.sha256);
-    if (duplicate && samePart(duplicate, input, hash.value)) return duplicate;
+    if (duplicate && samePart(duplicate, input, hash.value)) {
+      await input.body.cancel("duplicate-upload-part").catch(() => undefined);
+      return duplicate;
+    }
     if (
       input.partNumber !== session.nextPartNumber ||
       input.byteOffset !== session.receivedByteSize
@@ -114,7 +154,7 @@ export class UploadService {
         ? guardFileSignature(input.body, session.mimeType)
         : input.body;
     const stored = await this.options.blobStore.put({
-      body,
+      body: abortAwareStream(body, input.signal),
       expectedByteSize: input.byteSize,
       expectedContentHash: hash,
     });
@@ -127,12 +167,18 @@ export class UploadService {
       contentHash: stored.contentHash,
       createdAt: this.now().toISOString(),
     };
+    try {
+      throwIfUploadCancelled(input.signal);
+    } catch (error) {
+      await this.options.blobStore.delete(stored.ref).catch(() => false);
+      throw error;
+    }
     const committed = await this.options.repository.commitPart(
       part,
       session.receivedByteSize,
     );
     if (!committed) {
-      await this.options.blobStore.delete(stored.ref);
+      await this.options.blobStore.delete(stored.ref).catch(() => false);
       throw new UploadError(
         "CONCURRENT_UPLOAD",
         "上传会话已被并发更新，请续传",
@@ -145,17 +191,41 @@ export class UploadService {
     return this.requireOwnedActive(sessionId, ownerId);
   }
 
-  async complete(sessionId: string, ownerId: string): Promise<FileMetadata> {
-    const session = await this.requireOwnedActive(sessionId, ownerId);
+  async inspectSession(
+    sessionId: string,
+    ownerId: string,
+  ): Promise<UploadSession> {
+    const session = await this.options.repository.getSession(sessionId);
+    if (!session) throw new UploadError("UPLOAD_NOT_FOUND", "上传会话不存在");
+    if (session.ownerId !== ownerId) {
+      throw new UploadError("UPLOAD_FORBIDDEN", "无权访问该上传会话");
+    }
+    return session;
+  }
+
+  async complete(
+    sessionId: string,
+    ownerId: string,
+    signal?: AbortSignal,
+  ): Promise<FileMetadata> {
+    throwIfUploadCancelled(signal);
+    const session = await this.inspectSession(sessionId, ownerId);
+    if (session.status === "completed") {
+      return replayCompletedUpload(this.options, session);
+    }
+    this.requireActive(session);
     if (session.receivedByteSize !== session.expectedByteSize) {
       throw new UploadError("UPLOAD_INCOMPLETE", "上传内容尚未完整接收");
     }
     const parts = await this.options.repository.listParts(session.id);
     ensureContiguous(parts, session);
     const stored = await this.options.blobStore.put({
-      body: concatBlobStreams(
-        this.options.blobStore,
-        parts.map((part) => part.artifactRef),
+      body: abortAwareStream(
+        concatBlobStreams(
+          this.options.blobStore,
+          parts.map((part) => part.artifactRef),
+        ),
+        signal,
       ),
       expectedByteSize: session.expectedByteSize,
       expectedContentHash: session.expectedContentHash,
@@ -173,6 +243,12 @@ export class UploadService {
       expiresAt: session.fileExpiresAt,
       createdAt: this.now().toISOString(),
     };
+    try {
+      throwIfUploadCancelled(signal);
+    } catch (error) {
+      await this.options.blobStore.delete(stored.ref).catch(() => false);
+      throw error;
+    }
     const committed = await this.options.repository.completeSession(
       session.id,
       session.receivedByteSize,
@@ -180,74 +256,66 @@ export class UploadService {
       this.now().toISOString(),
     );
     if (!committed) {
-      await this.options.blobStore.delete(stored.ref);
+      await this.options.blobStore.delete(stored.ref).catch(() => false);
+      const current = await this.inspectSession(session.id, ownerId);
+      if (current.status === "completed") {
+        return replayCompletedUpload(this.options, current);
+      }
       throw new UploadError("CONCURRENT_UPLOAD", "上传会话已被并发更新");
     }
-    await this.deleteParts(session.id, parts);
+    await cleanupCompletedUploadParts(this.options, session.id, parts);
     return file;
   }
 
-  async abort(sessionId: string, ownerId: string): Promise<void> {
-    const session = await this.requireOwnedActive(sessionId, ownerId);
-    const parts = await this.options.repository.listParts(session.id);
-    if (
-      !(await this.options.repository.abortSession(
-        session.id,
-        ownerId,
-        this.now().toISOString(),
-      ))
-    ) {
-      throw new UploadError("CONCURRENT_UPLOAD", "上传会话已被并发更新");
-    }
-    await this.deleteParts(session.id, parts);
+  async abort(
+    sessionId: string,
+    ownerId: string,
+    signal?: AbortSignal,
+  ): Promise<void> {
+    throwIfUploadCancelled(signal);
+    const session = await this.inspectSession(sessionId, ownerId);
+    await abortUploadSession(
+      this.options,
+      session,
+      ownerId,
+      this.now().toISOString(),
+      signal,
+    );
+  }
+
+  purgeCompletedBound(input: BoundCompletedUploadPurge, signal?: AbortSignal) {
+    return purgeBoundCompletedUpload(
+      this.options,
+      input,
+      this.requirePolicy(input.purpose).allowedTypes,
+      this.now().toISOString(),
+      signal,
+    );
   }
 
   async cleanupExpired(
     limit = 100,
     additionalReferencedRefs: Iterable<ArtifactRef> = [],
   ): Promise<CleanupResult> {
-    const now = this.now().toISOString();
-    const sessions = await this.options.repository.listExpiredSessions(
-      now,
+    return cleanupExpiredUploads(
+      this.options,
+      this.now(),
       limit,
+      additionalReferencedRefs,
     );
-    const files = await this.options.repository.listExpiredFiles(now, limit);
-    let blobs = 0;
-    for (const session of sessions) {
-      const parts = await this.options.repository.listParts(session.id);
-      await this.options.repository.deleteSession(session.id);
-      blobs += await this.deleteBlobRefs(parts.map((part) => part.artifactRef));
-    }
-    for (const file of files) {
-      await this.options.repository.deleteFile(file.id);
-      blobs += Number(await this.options.blobStore.delete(file.artifactRef));
-    }
-    const referencedRefs = new Set([
-      ...(await this.options.repository.listArtifactRefs()),
-      ...additionalReferencedRefs,
-    ]);
-    const orphans = this.options.maintenance
-      ? await this.options.maintenance.cleanupOrphans({
-          referencedRefs,
-          olderThan: new Date(
-            this.now().getTime() - (this.options.orphanGraceMs ?? 60 * 60_000),
-          ),
-          limit,
-        })
-      : 0;
-    return { sessions: sessions.length, files: files.length, blobs, orphans };
   }
 
   private async requireOwnedActive(id: string, ownerId: string) {
-    const session = await this.options.repository.getSession(id);
-    if (!session) throw new UploadError("UPLOAD_NOT_FOUND", "上传会话不存在");
-    if (session.ownerId !== ownerId)
-      throw new UploadError("UPLOAD_FORBIDDEN", "无权访问该上传会话");
+    const session = await this.inspectSession(id, ownerId);
+    this.requireActive(session);
+    return session;
+  }
+
+  private requireActive(session: UploadSession): void {
     if (session.expiresAt <= this.now().toISOString())
       throw new UploadError("UPLOAD_EXPIRED", "上传会话已过期");
     if (session.status !== "uploading")
       throw new UploadError("UPLOAD_NOT_ACTIVE", "上传会话不可继续操作");
-    return session;
   }
 
   private requirePolicy(purpose: string) {
@@ -255,84 +323,4 @@ export class UploadService {
     if (!policy) throw new UploadError("INVALID_UPLOAD", "上传用途无效");
     return policy;
   }
-
-  private async deleteParts(sessionId: string, parts: readonly UploadPart[]) {
-    await this.deleteBlobRefs(parts.map((part) => part.artifactRef));
-    await this.options.repository.deleteParts(sessionId);
-  }
-
-  private async deleteBlobRefs(refs: readonly UploadPart["artifactRef"][]) {
-    let deleted = 0;
-    for (const ref of refs)
-      deleted += Number(await this.options.blobStore.delete(ref));
-    return deleted;
-  }
-}
-
-function randomOpaqueId(prefix: "upload" | "file") {
-  return `${prefix}_${randomUUID().replaceAll("-", "")}`;
-}
-
-function validFilename(value: string) {
-  const filename = value.trim();
-  if (
-    !filename ||
-    filename.length > 240 ||
-    filename.includes("/") ||
-    filename.includes("\\") ||
-    filename.includes(String.fromCharCode(0))
-  )
-    throw new UploadError("INVALID_UPLOAD", "文件名无效");
-  return filename;
-}
-
-function extensionOf(filename: string) {
-  const dot = filename.lastIndexOf(".");
-  if (dot < 1 || dot === filename.length - 1)
-    throw new UploadError("UNSUPPORTED_FILE_TYPE", "文件扩展名无效");
-  return filename.slice(dot + 1).toLowerCase();
-}
-
-function normalizeMime(value: string) {
-  return value.split(";", 1)[0]!.trim().toLowerCase();
-}
-
-function validateType(policy: UploadPolicy, mime: string, extension: string) {
-  if (!policy.allowedTypes[mime]?.includes(extension))
-    throw new UploadError(
-      "UNSUPPORTED_FILE_TYPE",
-      "MIME 类型与文件扩展名不匹配或不受支持",
-    );
-}
-
-function validatePartSize(
-  size: number,
-  policy: UploadPolicy,
-  session: UploadSession,
-) {
-  if (!Number.isSafeInteger(size) || size <= 0)
-    throw new UploadError("INVALID_UPLOAD", "分片大小无效");
-  if (size > policy.maxPartByteSize)
-    throw new UploadError("PART_TOO_LARGE", "分片超过允许大小");
-  if (session.receivedByteSize + size > session.expectedByteSize)
-    throw new UploadError("UPLOAD_TOO_LARGE", "分片超过文件声明大小");
-}
-
-function samePart(part: UploadPart, input: UploadPartInput, hash: string) {
-  return (
-    part.byteOffset === input.byteOffset &&
-    part.byteSize === input.byteSize &&
-    part.contentHash.value === hash
-  );
-}
-
-function ensureContiguous(parts: UploadPart[], session: UploadSession) {
-  let offset = 0;
-  for (const [index, part] of parts.entries()) {
-    if (part.partNumber !== index || part.byteOffset !== offset)
-      throw new UploadError("UPLOAD_INCOMPLETE", "上传分片不连续");
-    offset += part.byteSize;
-  }
-  if (offset !== session.expectedByteSize)
-    throw new UploadError("UPLOAD_INCOMPLETE", "上传分片大小不完整");
 }

package/template/packages/storage/src/upload-streams.ts

@@ -1,6 +1,52 @@
 import type { ArtifactRef, BlobStore } from "./blob-store.js";
 import { UploadError } from "./upload-types.js";
 
+export function throwIfUploadCancelled(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw new UploadError("UPLOAD_CANCELLED", "上传已取消");
+  }
+}
+
+export function abortAwareStream(
+  body: ReadableStream<Uint8Array>,
+  signal?: AbortSignal,
+): ReadableStream<Uint8Array> {
+  if (!signal) return body;
+  let reader: ReadableStreamDefaultReader<Uint8Array> | undefined;
+  let finished = false;
+  return new ReadableStream<Uint8Array>(
+    {
+      async pull(controller) {
+        const source = (reader ??= body.getReader());
+        try {
+          const result = await raceAbort(source.read(), signal);
+          if (result.done) {
+            finished = true;
+            source.releaseLock();
+            controller.close();
+            return;
+          }
+          controller.enqueue(result.value);
+        } catch (error) {
+          finished = true;
+          await cancelReader(source, error);
+          controller.error(error);
+        }
+      },
+      async cancel(reason?: unknown) {
+        if (finished) return;
+        finished = true;
+        if (reader) {
+          await cancelReader(reader, reason);
+          return;
+        }
+        await body.cancel(reason).catch(() => undefined);
+      },
+    },
+    { highWaterMark: 0 },
+  );
+}
+
 export function concatBlobStreams(
   store: BlobStore,
   refs: readonly ArtifactRef[],
@@ -41,47 +87,61 @@ export function guardFileSignature(
   body: ReadableStream<Uint8Array>,
   mimeType: string,
 ): ReadableStream<Uint8Array> {
-  const reader = body.getReader();
+  let reader: ReadableStreamDefaultReader<Uint8Array> | undefined;
   const requiredBytes = signatureByteCount(mimeType);
   const pending: Uint8Array[] = [];
   const prefix = new Uint8Array(requiredBytes);
   let prefixSize = 0;
   let checked = false;
-  return new ReadableStream({
-    async pull(controller) {
-      if (!checked) {
-        while (prefixSize < requiredBytes) {
-          const result = await reader.read();
-          if (result.done) {
-            controller.error(signatureError());
+  return new ReadableStream(
+    {
+      async pull(controller) {
+        const source = (reader ??= body.getReader());
+        if (!checked) {
+          while (prefixSize < requiredBytes) {
+            const result = await source.read();
+            if (result.done) {
+              source.releaseLock();
+              controller.error(signatureError());
+              return;
+            }
+            pending.push(result.value);
+            const needed = requiredBytes - prefixSize;
+            const slice = result.value.subarray(0, needed);
+            prefix.set(slice, prefixSize);
+            prefixSize += slice.byteLength;
+          }
+          checked = true;
+          if (!matchesSignature(prefix, mimeType)) {
+            const error = signatureError();
+            await cancelReader(source, error);
+            controller.error(error);
             return;
           }
-          pending.push(result.value);
-          const needed = requiredBytes - prefixSize;
-          const slice = result.value.subarray(0, needed);
-          prefix.set(slice, prefixSize);
-          prefixSize += slice.byteLength;
         }
-        checked = true;
-        if (!matchesSignature(prefix, mimeType)) {
-          await reader.cancel();
-          controller.error(signatureError());
+        const buffered = pending.shift();
+        if (buffered) {
+          controller.enqueue(buffered);
           return;
         }
-      }
-      const buffered = pending.shift();
-      if (buffered) {
-        controller.enqueue(buffered);
-        return;
-      }
-      const result = await reader.read();
-      if (result.done) controller.close();
-      else controller.enqueue(result.value);
-    },
-    async cancel(reason) {
-      await reader.cancel(reason);
+        const result = await source.read();
+        if (result.done) {
+          source.releaseLock();
+          controller.close();
+        } else {
+          controller.enqueue(result.value);
+        }
+      },
+      async cancel(reason) {
+        if (reader) {
+          await cancelReader(reader, reason);
+          return;
+        }
+        await body.cancel(reason).catch(() => undefined);
+      },
     },
-  });
+    { highWaterMark: 0 },
+  );
 }
 
 function signatureByteCount(mimeType: string) {
@@ -121,3 +181,39 @@ function signatureError() {
     "文件内容与声明的 MIME 类型不一致",
   );
 }
+
+function raceAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
+  throwIfUploadCancelled(signal);
+  return new Promise<T>((resolve, reject) => {
+    const onAbort = () =>
+      reject(new UploadError("UPLOAD_CANCELLED", "上传已取消"));
+    signal.addEventListener("abort", onAbort, { once: true });
+    promise.then(
+      (value) => {
+        signal.removeEventListener("abort", onAbort);
+        resolve(value);
+      },
+      (error: unknown) => {
+        signal.removeEventListener("abort", onAbort);
+        reject(error);
+      },
+    );
+  });
+}
+
+async function cancelReader(
+  reader: ReadableStreamDefaultReader<Uint8Array>,
+  reason?: unknown,
+): Promise<void> {
+  try {
+    await reader.cancel(reason);
+  } catch {
+    // Cancellation is best effort; preserve the original stream error.
+  } finally {
+    try {
+      reader.releaseLock();
+    } catch {
+      // A rejected cancel can leave a read pending; the errored stream owns it.
+    }
+  }
+}

package/template/packages/storage/src/upload-types.ts

@@ -2,14 +2,21 @@ import type { ArtifactRef, ContentHash } from "./blob-store.js";
 
 export type UploadSessionStatus =
   | "uploading"
+  | "aborting"
   | "completed"
   | "aborted"
   | "expired";
 
+export interface UploadAuthorizationBinding {
+  readonly moduleName: string;
+  readonly scopeHash: string;
+}
+
 export interface UploadSession {
   id: string;
   ownerId: string;
   purpose: string;
+  authorizationBinding?: UploadAuthorizationBinding;
   filename: string;
   extension: string;
   mimeType: string;
@@ -58,6 +65,7 @@ export interface UploadPolicy {
   maxByteSize: number;
   maxPartByteSize: number;
   sessionTtlMs: number;
+  fileRetentionMs?: number;
   allowedTypes: Readonly<Record<string, readonly string[]>>;
   signature?: "image" | "none";
 }
@@ -65,11 +73,14 @@ export interface UploadPolicy {
 export interface CreateUploadInput {
   ownerId: string;
   purpose: string;
+  authorizationBinding?: UploadAuthorizationBinding;
   filename: string;
   mimeType: string;
   expectedByteSize: number;
   expectedSha256?: string;
   fileExpiresAt?: Date;
+  /** 平台内部按用途策略派生的相对保留期。 */
+  fileRetentionMs?: number;
 }
 
 export interface UploadPartInput {
@@ -80,6 +91,7 @@ export interface UploadPartInput {
   byteSize: number;
   sha256: string;
   body: ReadableStream<Uint8Array>;
+  signal?: AbortSignal;
 }
 
 export type UploadErrorCode =
@@ -93,6 +105,7 @@ export type UploadErrorCode =
   | "UPLOAD_EXPIRED"
   | "UPLOAD_OFFSET_CONFLICT"
   | "UPLOAD_INCOMPLETE"
+  | "UPLOAD_CANCELLED"
   | "FILE_SIGNATURE_MISMATCH"
   | "CONCURRENT_UPLOAD"
   | "FILE_NOT_FOUND"