@southwind-ai/storage 0.1.0

package/src/local-file-maintenance.ts

@@ -0,0 +1,133 @@
+import { readdir, rm, stat } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  artifactRef,
+  type ArtifactRef,
+  type BlobStoreMaintenance,
+} from "./blob-store.js";
+
+export class LocalFileBlobMaintenance implements BlobStoreMaintenance {
+  constructor(private readonly root: string) {}
+
+  async cleanupOrphans(input: {
+    referencedRefs: ReadonlySet<ArtifactRef>;
+    olderThan: Date;
+    limit: number;
+  }) {
+    let cleaned = await cleanupTemporaryFiles(
+      join(this.root, ".tmp"),
+      input.olderThan,
+      input.limit,
+    );
+    const root = join(this.root, "blobs");
+    for (const directory of await safeReadDirectories(root)) {
+      if (cleaned >= input.limit) break;
+      cleaned += await cleanupBlobDirectory(
+        join(root, directory),
+        input,
+        input.limit - cleaned,
+      );
+    }
+    return cleaned;
+  }
+}
+
+async function cleanupTemporaryFiles(
+  directory: string,
+  olderThan: Date,
+  limit: number,
+) {
+  let cleaned = 0;
+  for (const name of await safeReadFiles(directory)) {
+    if (cleaned >= limit) break;
+    const path = join(directory, name);
+    if (await isOlder(path, olderThan)) {
+      await rm(path, { force: true });
+      cleaned += 1;
+    }
+  }
+  return cleaned;
+}
+
+async function cleanupBlobDirectory(
+  directory: string,
+  input: {
+    referencedRefs: ReadonlySet<ArtifactRef>;
+    olderThan: Date;
+  },
+  limit: number,
+) {
+  const names = new Set(await safeReadFiles(directory));
+  const refs = new Set<string>();
+  for (const name of names) {
+    if (name.endsWith(".blob")) refs.add(name.slice(0, -5));
+    if (name.endsWith(".json")) refs.add(name.slice(0, -5));
+  }
+  let cleaned = 0;
+  for (const rawRef of refs) {
+    if (cleaned >= limit) break;
+    let ref: ArtifactRef;
+    try {
+      ref = artifactRef(rawRef);
+    } catch {
+      continue;
+    }
+    if (input.referencedRefs.has(ref)) continue;
+    const blob = join(directory, `${ref}.blob`);
+    const metadata = join(directory, `${ref}.json`);
+    if (
+      (await isOlderOrMissing(blob, input.olderThan)) &&
+      (await isOlderOrMissing(metadata, input.olderThan))
+    ) {
+      await Promise.all([
+        rm(blob, { force: true }),
+        rm(metadata, { force: true }),
+      ]);
+      cleaned += 1;
+    }
+  }
+  return cleaned;
+}
+
+async function safeReadDirectories(path: string) {
+  try {
+    return (await readdir(path, { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name);
+  } catch (error) {
+    if (isMissing(error)) return [];
+    throw error;
+  }
+}
+
+async function safeReadFiles(path: string) {
+  try {
+    return (await readdir(path, { withFileTypes: true }))
+      .filter((entry) => entry.isFile())
+      .map((entry) => entry.name);
+  } catch (error) {
+    if (isMissing(error)) return [];
+    throw error;
+  }
+}
+
+async function isOlder(path: string, cutoff: Date) {
+  return (await stat(path)).mtime <= cutoff;
+}
+
+async function isOlderOrMissing(path: string, cutoff: Date) {
+  try {
+    return await isOlder(path, cutoff);
+  } catch (error) {
+    if (isMissing(error)) return true;
+    throw error;
+  }
+}
+
+function isMissing(error: unknown) {
+  return (
+    error instanceof Error &&
+    "code" in error &&
+    (error as NodeJS.ErrnoException).code === "ENOENT"
+  );
+}

package/src/memory-blob-store.ts

@@ -0,0 +1,100 @@
+import { randomUUID } from "node:crypto";
+import {
+  artifactRef,
+  BlobStoreError,
+  type ArtifactRef,
+  type BlobStore,
+  type PutBlobInput,
+  type ReadableBlob,
+  type StoredBlob,
+} from "./blob-store.js";
+import { createSha256Accumulator } from "./content-hash.js";
+
+interface MemoryBlob extends StoredBlob {
+  bytes: Uint8Array;
+}
+
+/** Small development/test adapter. Production large files use a streaming adapter. */
+export class MemoryBlobStore implements BlobStore {
+  private readonly blobs = new Map<ArtifactRef, MemoryBlob>();
+
+  async put(input: PutBlobInput): Promise<StoredBlob> {
+    const chunks: Uint8Array[] = [];
+    const hash = createSha256Accumulator();
+    let byteSize = 0;
+    const reader = input.body.getReader();
+    while (true) {
+      const result = await reader.read();
+      if (result.done) break;
+      const copy = Uint8Array.from(result.value);
+      chunks.push(copy);
+      hash.update(copy);
+      byteSize += copy.byteLength;
+    }
+    if (
+      input.expectedByteSize !== undefined &&
+      input.expectedByteSize !== byteSize
+    ) {
+      throw new BlobStoreError("SIZE_MISMATCH", "Blob 大小与声明不一致");
+    }
+    const contentHash = hash.digest();
+    if (
+      input.expectedContentHash &&
+      input.expectedContentHash.value !== contentHash.value
+    ) {
+      throw new BlobStoreError("HASH_MISMATCH", "Blob SHA-256 与声明不一致");
+    }
+    const ref = artifactRef(`blob_${randomUUID().replaceAll("-", "")}`);
+    const blob: MemoryBlob = {
+      ref,
+      byteSize,
+      contentHash,
+      createdAt: new Date().toISOString(),
+      bytes: join(chunks, byteSize),
+    };
+    this.blobs.set(ref, blob);
+    return metadata(blob);
+  }
+
+  async open(ref: ArtifactRef): Promise<ReadableBlob | undefined> {
+    const blob = this.blobs.get(artifactRef(ref));
+    if (!blob) return undefined;
+    return {
+      ...metadata(blob),
+      body: new ReadableStream({
+        start(controller) {
+          controller.enqueue(Uint8Array.from(blob.bytes));
+          controller.close();
+        },
+      }),
+    };
+  }
+
+  async inspect(ref: ArtifactRef): Promise<StoredBlob | undefined> {
+    const blob = this.blobs.get(artifactRef(ref));
+    return blob ? metadata(blob) : undefined;
+  }
+
+  async delete(ref: ArtifactRef) {
+    return this.blobs.delete(artifactRef(ref));
+  }
+
+  get size() {
+    return this.blobs.size;
+  }
+}
+
+function metadata(blob: MemoryBlob): StoredBlob {
+  const { bytes: _bytes, ...value } = blob;
+  return { ...value, contentHash: { ...value.contentHash } };
+}
+
+function join(chunks: readonly Uint8Array[], byteSize: number) {
+  const output = new Uint8Array(byteSize);
+  let offset = 0;
+  for (const chunk of chunks) {
+    output.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return output;
+}

package/src/upload-repository.ts

@@ -0,0 +1,43 @@
+import type { ArtifactRef } from "./blob-store.js";
+import type {
+  FileMetadata,
+  UploadPart,
+  UploadSession,
+} from "./upload-types.js";
+
+export type NewUploadSession = Omit<
+  UploadSession,
+  "receivedByteSize" | "nextPartNumber" | "status" | "updatedAt"
+>;
+
+export interface UploadRepository {
+  createSession(input: NewUploadSession): Promise<UploadSession>;
+  getSession(id: string): Promise<UploadSession | undefined>;
+  getPart(
+    sessionId: string,
+    partNumber: number,
+  ): Promise<UploadPart | undefined>;
+  listParts(sessionId: string): Promise<UploadPart[]>;
+  commitPart(
+    part: UploadPart,
+    expectedReceivedByteSize: number,
+  ): Promise<boolean>;
+  completeSession(
+    sessionId: string,
+    expectedReceivedByteSize: number,
+    file: FileMetadata,
+    updatedAt: string,
+  ): Promise<boolean>;
+  abortSession(
+    id: string,
+    ownerId: string,
+    updatedAt: string,
+  ): Promise<boolean>;
+  getFile(id: string): Promise<FileMetadata | undefined>;
+  listExpiredSessions(now: string, limit: number): Promise<UploadSession[]>;
+  listExpiredFiles(now: string, limit: number): Promise<FileMetadata[]>;
+  listArtifactRefs(): Promise<ArtifactRef[]>;
+  deleteParts(sessionId: string): Promise<void>;
+  deleteSession(id: string): Promise<void>;
+  deleteFile(id: string): Promise<void>;
+}

package/src/upload-service.ts

@@ -0,0 +1,334 @@
+import { randomUUID } from "node:crypto";
+import type {
+  ArtifactRef,
+  BlobStore,
+  BlobStoreMaintenance,
+} from "./blob-store.js";
+import { sha256ContentHash } from "./content-hash.js";
+import type { UploadRepository } from "./upload-repository.js";
+import { concatBlobStreams, guardFileSignature } from "./upload-streams.js";
+import {
+  UploadError,
+  type CreateUploadInput,
+  type FileMetadata,
+  type UploadPart,
+  type UploadPartInput,
+  type UploadPolicy,
+  type UploadSession,
+} from "./upload-types.js";
+
+export interface UploadServiceOptions {
+  repository: UploadRepository;
+  blobStore: BlobStore;
+  policies: readonly UploadPolicy[];
+  maintenance?: BlobStoreMaintenance;
+  orphanGraceMs?: number;
+  now?: () => Date;
+  createId?: (prefix: "upload" | "file") => string;
+}
+
+export interface CleanupResult {
+  sessions: number;
+  files: number;
+  blobs: number;
+  orphans: number;
+}
+
+export class UploadService {
+  private readonly now: () => Date;
+  private readonly createId: (prefix: "upload" | "file") => string;
+  private readonly policies: ReadonlyMap<string, UploadPolicy>;
+
+  constructor(private readonly options: UploadServiceOptions) {
+    this.now = options.now ?? (() => new Date());
+    this.createId = options.createId ?? randomOpaqueId;
+    this.policies = new Map(
+      options.policies.map((policy) => [policy.purpose, policy]),
+    );
+  }
+
+  async create(input: CreateUploadInput): Promise<UploadSession> {
+    const policy = this.requirePolicy(input.purpose);
+    const filename = validFilename(input.filename);
+    const extension = extensionOf(filename);
+    const mimeType = normalizeMime(input.mimeType);
+    validateType(policy, mimeType, extension);
+    if (
+      !Number.isSafeInteger(input.expectedByteSize) ||
+      input.expectedByteSize < 0
+    ) {
+      throw new UploadError("INVALID_UPLOAD", "文件大小无效");
+    }
+    if (input.expectedByteSize > policy.maxByteSize) {
+      throw new UploadError("UPLOAD_TOO_LARGE", "文件超过用途允许的大小");
+    }
+    const now = this.now();
+    if (input.fileExpiresAt && input.fileExpiresAt <= now) {
+      throw new UploadError("INVALID_UPLOAD", "文件过期时间必须晚于当前时间");
+    }
+    return this.options.repository.createSession({
+      id: this.createId("upload"),
+      ownerId: input.ownerId,
+      purpose: input.purpose,
+      filename,
+      extension,
+      mimeType,
+      expectedByteSize: input.expectedByteSize,
+      expectedContentHash: input.expectedSha256
+        ? sha256ContentHash(input.expectedSha256)
+        : undefined,
+      expiresAt: new Date(now.getTime() + policy.sessionTtlMs).toISOString(),
+      fileExpiresAt: input.fileExpiresAt?.toISOString(),
+      createdAt: now.toISOString(),
+    });
+  }
+
+  async appendPart(input: UploadPartInput): Promise<UploadPart> {
+    const session = await this.requireOwnedActive(
+      input.sessionId,
+      input.ownerId,
+    );
+    const duplicate = await this.options.repository.getPart(
+      session.id,
+      input.partNumber,
+    );
+    const hash = sha256ContentHash(input.sha256);
+    if (duplicate && samePart(duplicate, input, hash.value)) return duplicate;
+    if (
+      input.partNumber !== session.nextPartNumber ||
+      input.byteOffset !== session.receivedByteSize
+    ) {
+      throw new UploadError(
+        "UPLOAD_OFFSET_CONFLICT",
+        `续传位置冲突，服务端偏移量为 ${session.receivedByteSize}`,
+      );
+    }
+    const policy = this.requirePolicy(session.purpose);
+    validatePartSize(input.byteSize, policy, session);
+    const body =
+      input.partNumber === 0 && policy.signature === "image"
+        ? guardFileSignature(input.body, session.mimeType)
+        : input.body;
+    const stored = await this.options.blobStore.put({
+      body,
+      expectedByteSize: input.byteSize,
+      expectedContentHash: hash,
+    });
+    const part: UploadPart = {
+      sessionId: session.id,
+      partNumber: input.partNumber,
+      byteOffset: input.byteOffset,
+      artifactRef: stored.ref,
+      byteSize: stored.byteSize,
+      contentHash: stored.contentHash,
+      createdAt: this.now().toISOString(),
+    };
+    const committed = await this.options.repository.commitPart(
+      part,
+      session.receivedByteSize,
+    );
+    if (!committed) {
+      await this.options.blobStore.delete(stored.ref);
+      throw new UploadError(
+        "CONCURRENT_UPLOAD",
+        "上传会话已被并发更新，请续传",
+      );
+    }
+    return part;
+  }
+
+  getSession(sessionId: string, ownerId: string): Promise<UploadSession> {
+    return this.requireOwnedActive(sessionId, ownerId);
+  }
+
+  async complete(sessionId: string, ownerId: string): Promise<FileMetadata> {
+    const session = await this.requireOwnedActive(sessionId, ownerId);
+    if (session.receivedByteSize !== session.expectedByteSize) {
+      throw new UploadError("UPLOAD_INCOMPLETE", "上传内容尚未完整接收");
+    }
+    const parts = await this.options.repository.listParts(session.id);
+    ensureContiguous(parts, session);
+    const stored = await this.options.blobStore.put({
+      body: concatBlobStreams(
+        this.options.blobStore,
+        parts.map((part) => part.artifactRef),
+      ),
+      expectedByteSize: session.expectedByteSize,
+      expectedContentHash: session.expectedContentHash,
+    });
+    const file: FileMetadata = {
+      id: this.createId("file"),
+      ownerId: session.ownerId,
+      purpose: session.purpose,
+      filename: session.filename,
+      extension: session.extension,
+      mimeType: session.mimeType,
+      artifactRef: stored.ref,
+      byteSize: stored.byteSize,
+      contentHash: stored.contentHash,
+      expiresAt: session.fileExpiresAt,
+      createdAt: this.now().toISOString(),
+    };
+    const committed = await this.options.repository.completeSession(
+      session.id,
+      session.receivedByteSize,
+      file,
+      this.now().toISOString(),
+    );
+    if (!committed) {
+      await this.options.blobStore.delete(stored.ref);
+      throw new UploadError("CONCURRENT_UPLOAD", "上传会话已被并发更新");
+    }
+    await this.deleteParts(session.id, parts);
+    return file;
+  }
+
+  async abort(sessionId: string, ownerId: string): Promise<void> {
+    const session = await this.requireOwnedActive(sessionId, ownerId);
+    const parts = await this.options.repository.listParts(session.id);
+    if (
+      !(await this.options.repository.abortSession(
+        session.id,
+        ownerId,
+        this.now().toISOString(),
+      ))
+    ) {
+      throw new UploadError("CONCURRENT_UPLOAD", "上传会话已被并发更新");
+    }
+    await this.deleteParts(session.id, parts);
+  }
+
+  async cleanupExpired(
+    limit = 100,
+    additionalReferencedRefs: Iterable<ArtifactRef> = [],
+  ): Promise<CleanupResult> {
+    const now = this.now().toISOString();
+    const sessions = await this.options.repository.listExpiredSessions(
+      now,
+      limit,
+    );
+    const files = await this.options.repository.listExpiredFiles(now, limit);
+    let blobs = 0;
+    for (const session of sessions) {
+      const parts = await this.options.repository.listParts(session.id);
+      await this.options.repository.deleteSession(session.id);
+      blobs += await this.deleteBlobRefs(parts.map((part) => part.artifactRef));
+    }
+    for (const file of files) {
+      await this.options.repository.deleteFile(file.id);
+      blobs += Number(await this.options.blobStore.delete(file.artifactRef));
+    }
+    const referencedRefs = new Set([
+      ...(await this.options.repository.listArtifactRefs()),
+      ...additionalReferencedRefs,
+    ]);
+    const orphans = this.options.maintenance
+      ? await this.options.maintenance.cleanupOrphans({
+          referencedRefs,
+          olderThan: new Date(
+            this.now().getTime() - (this.options.orphanGraceMs ?? 60 * 60_000),
+          ),
+          limit,
+        })
+      : 0;
+    return { sessions: sessions.length, files: files.length, blobs, orphans };
+  }
+
+  private async requireOwnedActive(id: string, ownerId: string) {
+    const session = await this.options.repository.getSession(id);
+    if (!session) throw new UploadError("UPLOAD_NOT_FOUND", "上传会话不存在");
+    if (session.ownerId !== ownerId)
+      throw new UploadError("UPLOAD_FORBIDDEN", "无权访问该上传会话");
+    if (session.expiresAt <= this.now().toISOString())
+      throw new UploadError("UPLOAD_EXPIRED", "上传会话已过期");
+    if (session.status !== "uploading")
+      throw new UploadError("UPLOAD_NOT_ACTIVE", "上传会话不可继续操作");
+    return session;
+  }
+
+  private requirePolicy(purpose: string) {
+    const policy = this.policies.get(purpose);
+    if (!policy) throw new UploadError("INVALID_UPLOAD", "上传用途无效");
+    return policy;
+  }
+
+  private async deleteParts(sessionId: string, parts: readonly UploadPart[]) {
+    await this.deleteBlobRefs(parts.map((part) => part.artifactRef));
+    await this.options.repository.deleteParts(sessionId);
+  }
+
+  private async deleteBlobRefs(refs: readonly UploadPart["artifactRef"][]) {
+    let deleted = 0;
+    for (const ref of refs)
+      deleted += Number(await this.options.blobStore.delete(ref));
+    return deleted;
+  }
+}
+
+function randomOpaqueId(prefix: "upload" | "file") {
+  return `${prefix}_${randomUUID().replaceAll("-", "")}`;
+}
+
+function validFilename(value: string) {
+  const filename = value.trim();
+  if (
+    !filename ||
+    filename.length > 240 ||
+    filename.includes("/") ||
+    filename.includes("\\") ||
+    filename.includes(String.fromCharCode(0))
+  )
+    throw new UploadError("INVALID_UPLOAD", "文件名无效");
+  return filename;
+}
+
+function extensionOf(filename: string) {
+  const dot = filename.lastIndexOf(".");
+  if (dot < 1 || dot === filename.length - 1)
+    throw new UploadError("UNSUPPORTED_FILE_TYPE", "文件扩展名无效");
+  return filename.slice(dot + 1).toLowerCase();
+}
+
+function normalizeMime(value: string) {
+  return value.split(";", 1)[0]!.trim().toLowerCase();
+}
+
+function validateType(policy: UploadPolicy, mime: string, extension: string) {
+  if (!policy.allowedTypes[mime]?.includes(extension))
+    throw new UploadError(
+      "UNSUPPORTED_FILE_TYPE",
+      "MIME 类型与文件扩展名不匹配或不受支持",
+    );
+}
+
+function validatePartSize(
+  size: number,
+  policy: UploadPolicy,
+  session: UploadSession,
+) {
+  if (!Number.isSafeInteger(size) || size <= 0)
+    throw new UploadError("INVALID_UPLOAD", "分片大小无效");
+  if (size > policy.maxPartByteSize)
+    throw new UploadError("PART_TOO_LARGE", "分片超过允许大小");
+  if (session.receivedByteSize + size > session.expectedByteSize)
+    throw new UploadError("UPLOAD_TOO_LARGE", "分片超过文件声明大小");
+}
+
+function samePart(part: UploadPart, input: UploadPartInput, hash: string) {
+  return (
+    part.byteOffset === input.byteOffset &&
+    part.byteSize === input.byteSize &&
+    part.contentHash.value === hash
+  );
+}
+
+function ensureContiguous(parts: UploadPart[], session: UploadSession) {
+  let offset = 0;
+  for (const [index, part] of parts.entries()) {
+    if (part.partNumber !== index || part.byteOffset !== offset)
+      throw new UploadError("UPLOAD_INCOMPLETE", "上传分片不连续");
+    offset += part.byteSize;
+  }
+  if (offset !== session.expectedByteSize)
+    throw new UploadError("UPLOAD_INCOMPLETE", "上传分片大小不完整");
+}