@podosoft/podokit 1.2.0 → 1.4.0

package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.spec.ts

@@ -1,18 +1,20 @@
 import { describe, expect, it } from "bun:test";
 import { AppException } from "@podosoft/podokit-contracts";
-import type { ObjectBody, ObjectInfo, ObjectStore } from "@podosoft/podokit-runtime";
+import { objectBodyChunks, type ObjectBody, type ObjectInfo, type ObjectStore } from "@podosoft/podokit-runtime";
 import { Elysia } from "elysia";
 import type { RequestIdentity } from "../core/services";
 import { filesRoutes } from "./files.module";
 
 async function bytes(body: ObjectBody): Promise<Uint8Array> {
-  if (typeof body === "string") return Buffer.from(body);
-  if (body instanceof Uint8Array) return body;
   const chunks: Uint8Array[] = [];
-  for await (const chunk of body) chunks.push(chunk);
+  for await (const chunk of objectBodyChunks(body)) chunks.push(chunk);
   return Buffer.concat(chunks);
 }
 
+async function unused(): Promise<never> {
+  throw new Error("The files routes do not use multipart uploads");
+}
+
 function streamOf(value: Uint8Array): ReadableStream<Uint8Array> {
   return new ReadableStream({
     start(controller): void {
@@ -48,6 +50,10 @@ function memoryStore(): ObjectStore {
       objects.delete(key);
     },
     getDownloadUrl: async (key) => `https://storage.example.com/${key}`,
+    createMultipartUpload: unused,
+    uploadPart: unused,
+    completeMultipartUpload: unused,
+    abortMultipartUpload: unused,
     close: () => undefined,
   };
 }

package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.ts

@@ -1,24 +1,29 @@
 import { AppException } from "@podosoft/podokit-contracts";
 import { Elysia, t } from "elysia";
-import { OBJECT_STORAGE, type ObjectStore } from "@podosoft/podokit-runtime";
+import { OBJECT_STORAGE, RATE_LIMIT_ROUTES, type ObjectStore } from "@podosoft/podokit-runtime";
+import { apiServerOptions } from "../config/server-options";
 import {
+  DATABASE,
+  LOGGER,
   REQUEST_IDENTITY,
   type AppPlugin,
   type ElysiaPlugin,
   type PodokitModule,
   type RequestIdentity,
+  type ServiceKey,
 } from "../core/services";
+import { ChunkedUploads } from "./chunked-uploads";
+import { allowedOrigins, chunkedUploadRoutes } from "./chunked-uploads.routes";
+import { uploadConfig } from "./upload-config";
 import { uploadAccess, uploadKey } from "./upload-keys";
 
+export const CHUNKED_UPLOADS = Symbol("chunked-uploads") as ServiceKey<ChunkedUploads>;
+
 export function filesRoutes(storage: ObjectStore, identity: RequestIdentity): ElysiaPlugin {
   return new Elysia({ name: "podokit.files" })
     .post("/files", async ({ request, body, set }) => {
       const key = uploadKey(await identity.userId(request), body.file.name);
-      await storage.put(
-        key,
-        new Uint8Array(await body.file.arrayBuffer()),
-        { contentType: body.file.type },
-      );
+      await storage.put(key, body.file, { contentType: body.file.type });
       set.status = 201;
       return { key, url: await storage.getDownloadUrl(key) };
     }, {
@@ -45,10 +50,34 @@ export function filesRoutes(storage: ObjectStore, identity: RequestIdentity): El
     });
 }
 
-const filesPlugin: AppPlugin = ({ services }) =>
-  filesRoutes(services.resolve(OBJECT_STORAGE), services.resolve(REQUEST_IDENTITY));
+const filesPlugin: AppPlugin = ({ env, services }) => {
+  const uploads = services.resolve(CHUNKED_UPLOADS);
+  const identity = services.resolve(REQUEST_IDENTITY);
+  // Registered here rather than in configure: rate-limit may configure later.
+  services.tryResolve(RATE_LIMIT_ROUTES)?.register({
+    method: "PUT",
+    pathPrefix: "/files/uploads",
+    profile: { name: "file-uploads", limit: uploads.settings.rateLimitMax },
+  });
+  return new Elysia({ name: "podokit.files.module" })
+    .use(filesRoutes(services.resolve(OBJECT_STORAGE), identity))
+    .use(chunkedUploadRoutes(uploads, identity, allowedOrigins(env.CORS_ORIGIN)));
+};
 
 export const filesModule: PodokitModule = {
   name: "file-upload",
+  configure: (env, services): void => {
+    const database = services.resolve(DATABASE);
+    const uploads = new ChunkedUploads({
+      sql: database.sql,
+      dialect: database.provider,
+      config: uploadConfig(process.env, apiServerOptions(env).maxRequestBodySize),
+      // Resolved on use: an application may replace these after this point.
+      storage: () => services.resolve(OBJECT_STORAGE),
+      logger: { warn: (details, message) => services.resolve(LOGGER).warn(details, message) },
+    });
+    services.register(CHUNKED_UPLOADS, uploads, () => uploads.close());
+    services.onStart(() => uploads.start());
+  },
   plugin: filesPlugin,
 };

package/dist/templates/modules/file-upload/files/apps/api/src/files/upload-config.ts

@@ -0,0 +1,99 @@
+import { MAX_MULTIPART_PARTS, MIN_MULTIPART_PART_BYTES } from "@podosoft/podokit-runtime";
+
+const MIB = 1024 * 1024;
+
+export interface UploadConfig {
+  /** The largest file one chunked upload may hold. */
+  maxBytes: number;
+  /** Part size for uploads small enough to stay within MAX_MULTIPART_PARTS. */
+  partBytes: number;
+  /** Seconds an open upload may sit idle; each stored part restarts the clock. */
+  sessionTtlSeconds: number;
+  /** Seconds between sweeps of expired uploads. 0 turns the sweep off. */
+  sweepIntervalSeconds: number;
+  /** Part uploads per caller in each rate-limit window, when rate-limit is installed. */
+  rateLimitMax: number;
+}
+
+export const DEFAULT_UPLOAD_CONFIG: UploadConfig = {
+  maxBytes: 1024 * MIB,
+  partBytes: 8 * MIB,
+  sessionTtlSeconds: 24 * 60 * 60,
+  sweepIntervalSeconds: 5 * 60,
+  rateLimitMax: 3000,
+};
+
+export interface PartLayout {
+  partSize: number;
+  partCount: number;
+}
+
+/**
+ * Splits `size` bytes into parts of at least `partBytes`. Files that would need
+ * more than MAX_MULTIPART_PARTS parts get larger parts, rounded up to a whole
+ * MiB, instead.
+ */
+export function partLayout(size: number, partBytes: number): PartLayout {
+  const needed = Math.ceil(size / MAX_MULTIPART_PARTS);
+  const partSize = Math.max(partBytes, Math.ceil(needed / MIB) * MIB);
+  return { partSize, partCount: Math.max(1, Math.ceil(size / partSize)) };
+}
+
+/** The byte count part `partNumber` must have: `partSize`, except the last part. */
+export function expectedPartSize(size: number, layout: PartLayout, partNumber: number): number {
+  return partNumber < layout.partCount ? layout.partSize : size - layout.partSize * (layout.partCount - 1);
+}
+
+function wholeNumber(
+  source: Record<string, string | undefined>,
+  name: string,
+  fallback: number,
+  minimum: number,
+): number {
+  const text = source[name]?.trim() ?? "";
+  if (text === "") return fallback;
+  const value = Number(text);
+  if (!Number.isSafeInteger(value) || value < minimum) {
+    throw new Error(`${name} must be a whole number of at least ${minimum}`);
+  }
+  return value;
+}
+
+/**
+ * Reads the FILE_UPLOAD_* settings. `maxRequestBodySize` is the API server's
+ * body limit: the part size of the largest allowed upload must fit in it.
+ */
+export function uploadConfig(
+  source: Record<string, string | undefined>,
+  maxRequestBodySize: number,
+): UploadConfig {
+  const config: UploadConfig = {
+    maxBytes: wholeNumber(source, "FILE_UPLOAD_MAX_BYTES", DEFAULT_UPLOAD_CONFIG.maxBytes, 1),
+    partBytes: wholeNumber(
+      source,
+      "FILE_UPLOAD_PART_BYTES",
+      DEFAULT_UPLOAD_CONFIG.partBytes,
+      MIN_MULTIPART_PART_BYTES,
+    ),
+    sessionTtlSeconds: wholeNumber(
+      source,
+      "FILE_UPLOAD_SESSION_TTL",
+      DEFAULT_UPLOAD_CONFIG.sessionTtlSeconds,
+      60,
+    ),
+    sweepIntervalSeconds: wholeNumber(
+      source,
+      "FILE_UPLOAD_SWEEP_INTERVAL",
+      DEFAULT_UPLOAD_CONFIG.sweepIntervalSeconds,
+      0,
+    ),
+    rateLimitMax: wholeNumber(source, "FILE_UPLOAD_RATE_LIMIT_MAX", DEFAULT_UPLOAD_CONFIG.rateLimitMax, 1),
+  };
+  const largestPart = partLayout(config.maxBytes, config.partBytes).partSize;
+  if (largestPart > maxRequestBodySize) {
+    throw new Error(
+      `Chunked uploads of FILE_UPLOAD_MAX_BYTES need ${largestPart}-byte parts, more than the API body limit of ${maxRequestBodySize} bytes; raise API_MAX_REQUEST_BODY_SIZE or lower FILE_UPLOAD_PART_BYTES or FILE_UPLOAD_MAX_BYTES`,
+    );
+  }
+  return config;
+}

package/dist/templates/modules/file-upload/files/apps/api/src/migrations/1792100000000-InitFileUploadSessions.ts

@@ -0,0 +1,49 @@
+import type { MigrationInterface, QueryRunner } from "typeorm";
+
+export class InitFileUploadSessions1792100000000 implements MigrationInterface {
+  name = "InitFileUploadSessions1792100000000";
+
+  async up(queryRunner: QueryRunner): Promise<void> {
+    const sqlite = queryRunner.connection.options.type === "better-sqlite3";
+    const id = sqlite ? "text" : "uuid";
+    const timestamp = sqlite ? "text" : "TIMESTAMP WITH TIME ZONE";
+    await queryRunner.query(`
+      CREATE TABLE "file_upload_sessions" (
+        "id" ${id} NOT NULL,
+        "ownerId" character varying(128),
+        "objectKey" character varying(1024) NOT NULL,
+        "multipartUploadId" character varying(1024),
+        "filename" character varying(255) NOT NULL,
+        "contentType" character varying(255),
+        "size" bigint NOT NULL,
+        "partSize" bigint NOT NULL,
+        "partCount" integer NOT NULL,
+        "status" character varying(16) NOT NULL,
+        "expiresAt" ${timestamp} NOT NULL,
+        "createdAt" ${timestamp} NOT NULL,
+        "updatedAt" ${timestamp} NOT NULL,
+        CONSTRAINT "PK_file_upload_sessions_id" PRIMARY KEY ("id")
+      )
+    `);
+    await queryRunner.query(
+      `CREATE INDEX "IDX_file_upload_sessions_status_expiresAt" ON "file_upload_sessions" ("status", "expiresAt")`,
+    );
+    await queryRunner.query(`
+      CREATE TABLE "file_upload_parts" (
+        "sessionId" ${id} NOT NULL,
+        "partNumber" integer NOT NULL,
+        "etag" character varying(1024) NOT NULL,
+        "size" bigint NOT NULL,
+        "updatedAt" ${timestamp} NOT NULL,
+        CONSTRAINT "PK_file_upload_parts" PRIMARY KEY ("sessionId", "partNumber"),
+        CONSTRAINT "FK_file_upload_parts_sessionId" FOREIGN KEY ("sessionId")
+          REFERENCES "file_upload_sessions" ("id") ON DELETE CASCADE
+      )
+    `);
+  }
+
+  async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE "file_upload_parts"`);
+    await queryRunner.query(`DROP TABLE "file_upload_sessions"`);
+  }
+}