@southwind-ai/storage 0.1.0

package/index.ts

@@ -0,0 +1,11 @@
+export * from "./src/blob-store.js";
+export * from "./src/content-hash.js";
+export * from "./src/file-service.js";
+export * from "./src/local-file-blob-store.js";
+export * from "./src/local-file-maintenance.js";
+export * from "./src/memory-blob-store.js";
+export * from "./src/in-memory-upload-repository.js";
+export * from "./src/upload-repository.js";
+export * from "./src/upload-service.js";
+export * from "./src/upload-streams.js";
+export * from "./src/upload-types.js";

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@southwind-ai/storage",
+  "version": "0.1.0",
+  "license": "UNLICENSED",
+  "type": "module",
+  "engines": {
+    "bun": ">=1.3.0"
+  },
+  "files": [
+    "index.ts",
+    "src/**/*.ts",
+    "!src/**/*.test.ts"
+  ],
+  "scripts": {
+    "typecheck": "tsc --noEmit --project tsconfig.json"
+  },
+  "exports": {
+    ".": "./index.ts"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/blob-store.ts

@@ -0,0 +1,64 @@
+export type ArtifactRef = string & { readonly __artifactRef: unique symbol };
+
+export interface ContentHash {
+  algorithm: "sha256";
+  value: string;
+}
+
+export interface StoredBlob {
+  ref: ArtifactRef;
+  byteSize: number;
+  contentHash: ContentHash;
+  createdAt: string;
+}
+
+export interface ReadableBlob extends StoredBlob {
+  body: ReadableStream<Uint8Array>;
+}
+
+export interface PutBlobInput {
+  body: ReadableStream<Uint8Array>;
+  expectedByteSize?: number;
+  expectedContentHash?: ContentHash;
+}
+
+export interface BlobStore {
+  put(input: PutBlobInput): Promise<StoredBlob>;
+  inspect(ref: ArtifactRef): Promise<StoredBlob | undefined>;
+  open(ref: ArtifactRef): Promise<ReadableBlob | undefined>;
+  delete(ref: ArtifactRef): Promise<boolean>;
+}
+
+export interface BlobStoreMaintenance {
+  cleanupOrphans(input: {
+    referencedRefs: ReadonlySet<ArtifactRef>;
+    olderThan: Date;
+    limit: number;
+  }): Promise<number>;
+}
+
+export type BlobStoreErrorCode =
+  | "INVALID_ARTIFACT_REF"
+  | "INVALID_CONTENT_HASH"
+  | "SIZE_MISMATCH"
+  | "HASH_MISMATCH"
+  | "CORRUPT_BLOB_METADATA";
+
+export class BlobStoreError extends Error {
+  constructor(
+    readonly code: BlobStoreErrorCode,
+    message: string,
+  ) {
+    super(message);
+    this.name = "BlobStoreError";
+  }
+}
+
+const artifactRefPattern = /^blob_[a-f0-9]{32}$/;
+
+export function artifactRef(value: string): ArtifactRef {
+  if (!artifactRefPattern.test(value)) {
+    throw new BlobStoreError("INVALID_ARTIFACT_REF", "Artifact 引用格式无效");
+  }
+  return value as ArtifactRef;
+}

package/src/content-hash.ts

@@ -0,0 +1,24 @@
+import { createHash } from "node:crypto";
+import { BlobStoreError, type ContentHash } from "./blob-store.js";
+
+const sha256Pattern = /^[a-f0-9]{64}$/;
+
+export function sha256ContentHash(value: string): ContentHash {
+  const normalized = value.toLowerCase();
+  if (!sha256Pattern.test(normalized)) {
+    throw new BlobStoreError("INVALID_CONTENT_HASH", "SHA-256 摘要格式无效");
+  }
+  return { algorithm: "sha256", value: normalized };
+}
+
+export function createSha256Accumulator() {
+  const hash = createHash("sha256");
+  return {
+    update(chunk: Uint8Array): void {
+      hash.update(chunk);
+    },
+    digest(): ContentHash {
+      return sha256ContentHash(hash.digest("hex"));
+    },
+  };
+}

package/src/file-service.ts

@@ -0,0 +1,59 @@
+import type { BlobStore } from "./blob-store.js";
+import type { UploadRepository } from "./upload-repository.js";
+import {
+  UploadError,
+  type FileMetadata,
+  type OpenedFile,
+} from "./upload-types.js";
+
+export class FileService {
+  constructor(
+    private readonly repository: UploadRepository,
+    private readonly blobStore: BlobStore,
+    private readonly now: () => Date = () => new Date(),
+  ) {}
+
+  async open(
+    fileId: string,
+    ownerId: string,
+    allowManage = false,
+  ): Promise<OpenedFile> {
+    const file = await this.requireAvailable(fileId);
+    if (file.ownerId !== ownerId && !allowManage) {
+      throw new UploadError("UPLOAD_FORBIDDEN", "无权访问该文件");
+    }
+    return this.openStored(file);
+  }
+
+  async openPublic(
+    fileId: string,
+    requiredPurpose: string,
+  ): Promise<OpenedFile> {
+    const file = await this.requireAvailable(fileId);
+    if (file.purpose !== requiredPurpose) {
+      throw new UploadError("FILE_NOT_FOUND", "文件不存在");
+    }
+    return this.openStored(file);
+  }
+
+  private async requireAvailable(id: string) {
+    const file = await this.repository.getFile(id);
+    if (!file) throw new UploadError("FILE_NOT_FOUND", "文件不存在");
+    if (file.expiresAt && file.expiresAt <= this.now().toISOString()) {
+      throw new UploadError("FILE_EXPIRED", "文件已过期");
+    }
+    return file;
+  }
+
+  private async openStored(file: FileMetadata): Promise<OpenedFile> {
+    const blob = await this.blobStore.open(file.artifactRef);
+    if (
+      !blob ||
+      blob.byteSize !== file.byteSize ||
+      blob.contentHash.value !== file.contentHash.value
+    ) {
+      throw new UploadError("FILE_CONTENT_MISSING", "文件内容不可用");
+    }
+    return { ...file, body: blob.body };
+  }
+}

package/src/in-memory-upload-repository.ts

@@ -0,0 +1,154 @@
+import type { ArtifactRef } from "./blob-store.js";
+import type {
+  NewUploadSession,
+  UploadRepository,
+} from "./upload-repository.js";
+import type {
+  FileMetadata,
+  UploadPart,
+  UploadSession,
+} from "./upload-types.js";
+
+export class InMemoryUploadRepository implements UploadRepository {
+  private readonly sessions = new Map<string, UploadSession>();
+  private readonly parts = new Map<string, UploadPart>();
+  private readonly files = new Map<string, FileMetadata>();
+
+  async createSession(input: NewUploadSession) {
+    const session: UploadSession = {
+      ...input,
+      receivedByteSize: 0,
+      nextPartNumber: 0,
+      status: "uploading",
+      updatedAt: input.createdAt,
+    };
+    this.sessions.set(session.id, session);
+    return cloneSession(session);
+  }
+
+  async getSession(id: string) {
+    const value = this.sessions.get(id);
+    return value ? cloneSession(value) : undefined;
+  }
+
+  async getPart(sessionId: string, partNumber: number) {
+    const value = this.parts.get(partKey(sessionId, partNumber));
+    return value ? { ...value } : undefined;
+  }
+
+  async listParts(sessionId: string) {
+    return [...this.parts.values()]
+      .filter((part) => part.sessionId === sessionId)
+      .sort((a, b) => a.partNumber - b.partNumber)
+      .map((part) => ({ ...part }));
+  }
+
+  async commitPart(part: UploadPart, expectedReceivedByteSize: number) {
+    const session = this.sessions.get(part.sessionId);
+    if (
+      !session ||
+      session.status !== "uploading" ||
+      session.receivedByteSize !== expectedReceivedByteSize ||
+      session.nextPartNumber !== part.partNumber
+    ) {
+      return false;
+    }
+    this.parts.set(partKey(part.sessionId, part.partNumber), { ...part });
+    session.receivedByteSize += part.byteSize;
+    session.nextPartNumber += 1;
+    session.updatedAt = part.createdAt;
+    return true;
+  }
+
+  async completeSession(
+    sessionId: string,
+    expectedReceivedByteSize: number,
+    file: FileMetadata,
+    updatedAt: string,
+  ) {
+    const session = this.sessions.get(sessionId);
+    if (
+      !session ||
+      session.status !== "uploading" ||
+      session.receivedByteSize !== expectedReceivedByteSize
+    ) {
+      return false;
+    }
+    this.files.set(file.id, { ...file });
+    Object.assign(session, {
+      status: "completed",
+      fileId: file.id,
+      updatedAt,
+    });
+    return true;
+  }
+
+  async abortSession(id: string, ownerId: string, updatedAt: string) {
+    const session = this.sessions.get(id);
+    if (
+      !session ||
+      session.ownerId !== ownerId ||
+      session.status !== "uploading"
+    )
+      return false;
+    session.status = "aborted";
+    session.updatedAt = updatedAt;
+    return true;
+  }
+
+  async getFile(id: string) {
+    const value = this.files.get(id);
+    return value ? { ...value } : undefined;
+  }
+
+  async listExpiredSessions(now: string, limit: number) {
+    return [...this.sessions.values()]
+      .filter(
+        (session) => session.status !== "completed" && session.expiresAt <= now,
+      )
+      .slice(0, limit)
+      .map(cloneSession);
+  }
+
+  async listExpiredFiles(now: string, limit: number) {
+    return [...this.files.values()]
+      .filter((file) => file.expiresAt && file.expiresAt <= now)
+      .slice(0, limit)
+      .map((file) => ({ ...file }));
+  }
+
+  async listArtifactRefs() {
+    return [
+      ...[...this.parts.values()].map((part) => part.artifactRef),
+      ...[...this.files.values()].map((file) => file.artifactRef),
+    ] as ArtifactRef[];
+  }
+
+  async deleteSession(id: string) {
+    this.sessions.delete(id);
+    await this.deleteParts(id);
+  }
+
+  async deleteParts(id: string) {
+    for (const [key, part] of this.parts) {
+      if (part.sessionId === id) this.parts.delete(key);
+    }
+  }
+
+  async deleteFile(id: string) {
+    this.files.delete(id);
+  }
+}
+
+function partKey(sessionId: string, partNumber: number) {
+  return `${sessionId}:${partNumber}`;
+}
+
+function cloneSession(value: UploadSession): UploadSession {
+  return {
+    ...value,
+    expectedContentHash: value.expectedContentHash
+      ? { ...value.expectedContentHash }
+      : undefined,
+  };
+}

package/src/local-file-blob-store.ts

@@ -0,0 +1,344 @@
+import { randomUUID } from "node:crypto";
+import {
+  mkdir,
+  open,
+  readFile,
+  rename,
+  rm,
+  stat,
+  type FileHandle,
+} from "node:fs/promises";
+import { join } from "node:path";
+import {
+  artifactRef,
+  BlobStoreError,
+  type ArtifactRef,
+  type BlobStore,
+  type PutBlobInput,
+  type ReadableBlob,
+  type StoredBlob,
+} from "./blob-store.js";
+import { createSha256Accumulator, sha256ContentHash } from "./content-hash.js";
+
+export interface LocalFileBlobStoreOptions {
+  root: string;
+  now?: () => Date;
+  createRef?: () => ArtifactRef;
+}
+
+export class LocalFileBlobStore implements BlobStore {
+  private readonly now: () => Date;
+  private readonly createRef: () => ArtifactRef;
+
+  constructor(private readonly options: LocalFileBlobStoreOptions) {
+    this.now = options.now ?? (() => new Date());
+    this.createRef = options.createRef ?? randomArtifactRef;
+  }
+
+  async put(input: PutBlobInput): Promise<StoredBlob> {
+    validateExpectedSize(input.expectedByteSize);
+    const ref = this.createRef();
+    const paths = this.paths(ref);
+    await mkdir(paths.directory, { recursive: true });
+    await mkdir(paths.temporaryDirectory, { recursive: true });
+    const temporaryBlob = join(paths.temporaryDirectory, `${ref}.partial`);
+    const temporaryMetadata = join(
+      paths.temporaryDirectory,
+      `${ref}.json.partial`,
+    );
+
+    try {
+      const stored = await this.writeBody(temporaryBlob, ref, input);
+      await rename(temporaryBlob, paths.blob);
+      await writeDurably(temporaryMetadata, `${JSON.stringify(stored)}\n`);
+      await rename(temporaryMetadata, paths.metadata);
+      await syncDirectory(paths.directory);
+      return stored;
+    } catch (error) {
+      await Promise.all([
+        rm(temporaryBlob, { force: true }),
+        rm(temporaryMetadata, { force: true }),
+      ]);
+      throw error;
+    }
+  }
+
+  async open(ref: ArtifactRef): Promise<ReadableBlob | undefined> {
+    const paths = this.paths(ref);
+    const metadata = await readMetadata(paths.metadata, ref);
+    if (!metadata) return undefined;
+    let handle: FileHandle;
+    try {
+      handle = await open(paths.blob, "r");
+    } catch (error) {
+      if (isMissing(error)) throw corruptMetadata(ref);
+      throw error;
+    }
+    try {
+      await validateHandle(handle, metadata);
+      return {
+        ...metadata,
+        body: streamHandle(handle),
+      };
+    } catch (error) {
+      await handle.close();
+      throw error;
+    }
+  }
+
+  async inspect(ref: ArtifactRef): Promise<StoredBlob | undefined> {
+    const paths = this.paths(ref);
+    const metadata = await readMetadata(paths.metadata, ref);
+    if (!metadata) return undefined;
+    let handle: FileHandle;
+    try {
+      handle = await open(paths.blob, "r");
+    } catch (error) {
+      if (isMissing(error)) throw corruptMetadata(ref);
+      throw error;
+    }
+    try {
+      await validateHandle(handle, metadata);
+      return metadata;
+    } finally {
+      await handle.close();
+    }
+  }
+
+  async delete(ref: ArtifactRef): Promise<boolean> {
+    const paths = this.paths(ref);
+    const existed = await pathExists(paths.blob);
+    await Promise.all([
+      rm(paths.blob, { force: true }),
+      rm(paths.metadata, { force: true }),
+    ]);
+    return existed;
+  }
+
+  private async writeBody(
+    path: string,
+    ref: ArtifactRef,
+    input: PutBlobInput,
+  ): Promise<StoredBlob> {
+    const handle = await open(path, "wx", 0o600);
+    const reader = input.body.getReader();
+    const hash = createSha256Accumulator();
+    let byteSize = 0;
+    try {
+      while (true) {
+        const result = await reader.read();
+        if (result.done) break;
+        byteSize += result.value.byteLength;
+        if (
+          input.expectedByteSize !== undefined &&
+          byteSize > input.expectedByteSize
+        ) {
+          throw sizeMismatch(input.expectedByteSize, byteSize);
+        }
+        hash.update(result.value);
+        await writeAll(handle, result.value);
+      }
+      await handle.sync();
+    } finally {
+      reader.releaseLock();
+      await handle.close();
+    }
+    if (
+      input.expectedByteSize !== undefined &&
+      byteSize !== input.expectedByteSize
+    ) {
+      throw sizeMismatch(input.expectedByteSize, byteSize);
+    }
+    const contentHash = hash.digest();
+    if (
+      input.expectedContentHash &&
+      input.expectedContentHash.value !== contentHash.value
+    ) {
+      throw new BlobStoreError("HASH_MISMATCH", "Blob SHA-256 与声明不一致");
+    }
+    return {
+      ref,
+      byteSize,
+      contentHash,
+      createdAt: this.now().toISOString(),
+    };
+  }
+
+  private paths(ref: ArtifactRef) {
+    const safeRef = artifactRef(ref);
+    const directory = join(this.options.root, "blobs", safeRef.slice(5, 7));
+    return {
+      directory,
+      temporaryDirectory: join(this.options.root, ".tmp"),
+      blob: join(directory, `${safeRef}.blob`),
+      metadata: join(directory, `${safeRef}.json`),
+    };
+  }
+}
+
+async function hashHandle(handle: FileHandle): Promise<string> {
+  const hash = createSha256Accumulator();
+  const chunk = new Uint8Array(64 * 1024);
+  let position = 0;
+  while (true) {
+    const { bytesRead } = await handle.read(
+      chunk,
+      0,
+      chunk.byteLength,
+      position,
+    );
+    if (bytesRead === 0) break;
+    hash.update(chunk.subarray(0, bytesRead));
+    position += bytesRead;
+  }
+  return hash.digest().value;
+}
+
+function streamHandle(handle: FileHandle): ReadableStream<Uint8Array> {
+  let position = 0;
+  let closed = false;
+  const close = async () => {
+    if (closed) return;
+    closed = true;
+    await handle.close();
+  };
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      const chunk = new Uint8Array(64 * 1024);
+      try {
+        const { bytesRead } = await handle.read(
+          chunk,
+          0,
+          chunk.byteLength,
+          position,
+        );
+        if (bytesRead === 0) {
+          await close();
+          controller.close();
+          return;
+        }
+        position += bytesRead;
+        controller.enqueue(chunk.subarray(0, bytesRead));
+      } catch (error) {
+        await close();
+        controller.error(error);
+      }
+    },
+    cancel: close,
+  });
+}
+
+async function readMetadata(path: string, ref: ArtifactRef) {
+  try {
+    return parseMetadata(await readFile(path, "utf8"), ref);
+  } catch (error) {
+    if (isMissing(error)) return undefined;
+    throw error;
+  }
+}
+
+async function validateHandle(
+  handle: FileHandle,
+  metadata: StoredBlob,
+): Promise<void> {
+  if ((await handle.stat()).size !== metadata.byteSize) {
+    throw corruptMetadata(metadata.ref);
+  }
+  if ((await hashHandle(handle)) !== metadata.contentHash.value) {
+    throw corruptMetadata(metadata.ref);
+  }
+}
+
+function randomArtifactRef(): ArtifactRef {
+  return artifactRef(`blob_${randomUUID().replaceAll("-", "")}`);
+}
+
+async function writeAll(handle: FileHandle, chunk: Uint8Array): Promise<void> {
+  let offset = 0;
+  while (offset < chunk.byteLength) {
+    const result = await handle.write(chunk, offset, chunk.byteLength - offset);
+    offset += result.bytesWritten;
+  }
+}
+
+async function writeDurably(path: string, content: string): Promise<void> {
+  const handle = await open(path, "wx", 0o600);
+  try {
+    await handle.writeFile(content, "utf8");
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+}
+
+async function syncDirectory(path: string): Promise<void> {
+  const handle = await open(path, "r");
+  try {
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch (error) {
+    if (isMissing(error)) return false;
+    throw error;
+  }
+}
+
+function parseMetadata(raw: string, expectedRef: ArtifactRef): StoredBlob {
+  try {
+    const value = JSON.parse(raw) as Record<string, unknown>;
+    const ref = artifactRef(String(value.ref));
+    const byteSize = Number(value.byteSize);
+    const createdAt = String(value.createdAt);
+    const hash = value.contentHash as Record<string, unknown>;
+    if (
+      ref !== expectedRef ||
+      !Number.isSafeInteger(byteSize) ||
+      byteSize < 0 ||
+      Number.isNaN(Date.parse(createdAt)) ||
+      hash.algorithm !== "sha256"
+    ) {
+      throw corruptMetadata(expectedRef);
+    }
+    return {
+      ref,
+      byteSize,
+      createdAt,
+      contentHash: sha256ContentHash(String(hash.value)),
+    };
+  } catch (error) {
+    if (error instanceof BlobStoreError) throw error;
+    throw corruptMetadata(expectedRef);
+  }
+}
+
+function validateExpectedSize(value: number | undefined): void {
+  if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) {
+    throw new BlobStoreError("SIZE_MISMATCH", "Blob 声明大小无效");
+  }
+}
+
+function sizeMismatch(expected: number, actual: number): BlobStoreError {
+  return new BlobStoreError(
+    "SIZE_MISMATCH",
+    `Blob 大小与声明不一致：expected=${expected}, actual=${actual}`,
+  );
+}
+
+function corruptMetadata(ref: ArtifactRef): BlobStoreError {
+  return new BlobStoreError("CORRUPT_BLOB_METADATA", `Blob 元数据损坏：${ref}`);
+}
+
+function isMissing(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    "code" in error &&
+    (error as NodeJS.ErrnoException).code === "ENOENT"
+  );
+}