ocel 0.0.0 → 0.0.1-alpha.1

package/src/blob/uploader.ts

@@ -0,0 +1,426 @@
+import slugify from "@sindresorhus/slugify";
+import type {
+  CallbackBody,
+  HandleUploadBody,
+  PathConfig,
+  PathGenerator,
+  PresignUploadBody,
+  TUploaderConfig,
+  UploadFileBody,
+} from "./types";
+import { UploadError } from "./upload-error";
+import type { IncomingMessage } from "node:http";
+import path from "node:path";
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { getNanoid } from "../utils/nanoid";
+import { UploadSession } from "../internal/db";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import type { ZodType } from "zod";
+import { parseFileSize, parseReq } from "./parse";
+import type { Bucket } from "./bucket";
+
+export class Uploader<TMeta = unknown> {
+  private pathGenerator: PathGenerator<TMeta> | null = null;
+
+  constructor(
+    private bucket: Bucket<any>,
+    private name: string,
+    private config: TUploaderConfig<any, TMeta>
+  ) {
+    const path = this.config.path;
+    if (path) {
+      if (typeof path === "function") {
+        this.pathGenerator = path;
+      } else {
+        this.pathGenerator = this.createConfigBasedGenerator(path);
+      }
+    }
+  }
+
+  async handleUpload({
+    action,
+    body,
+    request,
+    sessionId,
+  }: {
+    request: Request | IncomingMessage;
+    body: HandleUploadBody;
+    action: "presign" | "callback" | "poll";
+    sessionId?: string;
+  }) {
+    try {
+      const req = parseReq(request);
+
+      switch (action) {
+        case "presign": {
+          if (!("files" in body)) {
+            throw UploadError.badRequest("Invalid presign body");
+          }
+
+          const presignBody = body as PresignUploadBody;
+          const rawInput = presignBody.input || {};
+          const parser = this.config.input as ZodType | undefined;
+
+          let parsedInput: any;
+
+          // TODO: error handling
+          if (parser) {
+            const parsed = parser.safeParse(rawInput);
+            if (!parsed.success) {
+              throw UploadError.badRequest(parsed.error.issues[0]?.message);
+            }
+
+            parsedInput = parsed.data;
+          }
+
+          const middleware = this.config.onBeforeUpload;
+          const customMeta = await middleware?.({
+            req: request,
+            input: parsedInput,
+            files: presignBody.files,
+          });
+
+          const metadata = {
+            "x-ocel-path": req.path,
+            "x-ocel-uploader": this.name,
+            ...(customMeta
+              ? { "x-ocel-metadata": JSON.stringify(customMeta) }
+              : {}),
+          };
+
+          // TODO: validate files against uploader config (size, count, types, etc)
+          this.validateFiles(presignBody.files, customMeta);
+
+          const files = body.files.map((f) => {
+            const name = slugify(f.name, { preserveCharacters: ["."] });
+
+            return {
+              ...f,
+              key:
+                this.pathGenerator?.({
+                  fileNameWithExt: name,
+                  metadata: customMeta as any,
+                  fileNameWithoutExt: name.replace(path.extname(name), ""),
+                  ext: path.extname(name),
+                } as any) || name,
+              disposition: "inline",
+              metadata: {
+                ...metadata,
+                "x-ocel-original-filename": f.name,
+              },
+            };
+          }) satisfies UploadFileBody[];
+
+          const provider = this.getProvider();
+
+          return provider.presign(files);
+        }
+        case "poll": {
+          if (!sessionId) {
+            throw UploadError.badRequest("sessionId is required for poll");
+          }
+
+          const result = await UploadSession.query
+            .sessions({ sessionId })
+            .go();
+
+          const files = result.data.map((record) => {
+            let parsedMeta: TMeta = {} as TMeta;
+            try {
+              if (record.metadata) {
+                parsedMeta = JSON.parse(record.metadata) as TMeta;
+              }
+            } catch {
+              // ignore parse errors
+            }
+            return {
+              fileKey: record.fileKey,
+              status: record.status,
+              file:
+                record.status === "SUCCESS"
+                  ? {
+                      path: record.fileKey,
+                      contentDisposition:
+                        record.contentDisposition || "inline",
+                      contentType: record.contentType,
+                    }
+                  : undefined,
+              metadata: parsedMeta,
+            };
+          });
+
+          const allComplete = files.every(
+            (f) => f.status === "SUCCESS" || f.status === "FAILED"
+          );
+
+          return {
+            files,
+            completed: allComplete,
+          };
+        }
+        case "callback": {
+          // TODO: validate callback body
+          if (!("metadata" in body)) {
+            throw UploadError.badRequest("Invalid callback body");
+          }
+
+          const data = body as CallbackBody<TMeta>;
+
+          await this.config.onUploadComplete?.(data);
+
+          return {
+            type: "UPLOAD_COMPLETED" as const,
+            response: "ok" as const,
+          };
+        }
+        default:
+          throw UploadError.badRequest(`Unhandled action type: ${action}`);
+      }
+    } catch (err) {
+      if (err instanceof UploadError) {
+        this.config.onUploadError?.({ error: err });
+
+        return {
+          error: {
+            message: err.message,
+            code: err.code, // e.g. "FORBIDDEN"
+            status: err.statusCode, // e.g. 403
+          },
+        };
+      }
+
+      return {
+        error: {
+          message: "Internal Server Error",
+          code: "INTERNAL_SERVER_ERROR",
+          status: 500,
+        },
+      };
+    }
+  }
+
+  private startPolling(sessionId: string): void {
+    const POLL_INTERVAL_MS = 1000;
+    const MAX_DURATION_MS = 10 * 60 * 1000; // 10 minutes
+    const startTime = Date.now();
+
+    const tick = async (): Promise<boolean> => {
+      if (Date.now() - startTime > MAX_DURATION_MS) {
+        return true;
+      }
+
+      const result = await UploadSession.query
+        .sessions({ sessionId })
+        .go();
+
+      const allComplete = result.data.every(
+        (r) => r.status === "SUCCESS" || r.status === "FAILED"
+      );
+
+      if (!allComplete) {
+        return false;
+      }
+
+      // Session complete - invoke callbacks
+      for (const record of result.data) {
+        let parsedMeta: TMeta = {} as TMeta;
+        try {
+          if (record.metadata) {
+            parsedMeta = JSON.parse(record.metadata) as TMeta;
+          }
+        } catch {
+          // ignore
+        }
+
+        if (record.status === "SUCCESS") {
+          await this.config.onUploadComplete?.({
+            metadata: parsedMeta,
+            file: {
+              path: record.fileKey,
+              contentDisposition: record.contentDisposition || "inline",
+              contentType: record.contentType,
+            },
+          });
+        } else if (record.status === "FAILED" && this.config.onUploadError) {
+          this.config.onUploadError({
+            error: new UploadError(
+              `Upload failed for ${record.fileKey}`,
+              500,
+              "UPLOAD_FAILED"
+            ),
+          });
+        }
+      }
+
+      return true;
+    };
+
+    const intervalId = setInterval(async () => {
+      try {
+        const done = await tick();
+        if (done) {
+          clearInterval(intervalId);
+        }
+      } catch {
+        clearInterval(intervalId);
+      }
+    }, POLL_INTERVAL_MS);
+  }
+
+  getProvider() {
+    const bucket = this.bucket;
+    const bucketId = bucket.__name();
+
+    return {
+      presign: async (files: UploadFileBody[]) => {
+        const s3Client = new S3Client();
+        const sessionId = getNanoid(8);
+
+        const presignedFiles = await Promise.all(
+          files.map(async (file) => {
+            const metadataWithSession = {
+              ...file.metadata,
+              "x-ocel-session-id": sessionId,
+            };
+
+            await UploadSession.create({
+              sessionId,
+              bucketName: bucketId,
+              fileKey: file.key,
+              createdAt: new Date().toISOString(),
+              status: "PENDING",
+              contentType: file.mimeType,
+              contentDisposition: file.disposition,
+              fileSize: file.size,
+              metadata: file.metadata
+                ? JSON.stringify(file.metadata)
+                : undefined,
+            }).go();
+
+            const url = await getSignedUrl(
+              s3Client,
+              new PutObjectCommand({
+                Key: file.key,
+                Bucket: bucketId,
+                ContentType: file.mimeType,
+                Metadata: metadataWithSession,
+              }),
+              { expiresIn: 3600 }
+            );
+
+            return {
+              url,
+              key: file.key,
+              name: file.name,
+            };
+          })
+        );
+
+        this.startPolling(sessionId);
+
+        return {
+          files: presignedFiles,
+          sessionId,
+        };
+      },
+    };
+  }
+
+  createConfigBasedGenerator(config: PathConfig): PathGenerator<any> {
+    return (ctx) => {
+      let key = ctx.fileNameWithoutExt;
+
+      // random suffix
+      if (config.randomSuffix) {
+        key += `-${Math.random().toString(36).slice(2, 10)}`;
+      }
+
+      // static suffix
+      else if (config.suffix) {
+        key += `-${config.suffix}`;
+      }
+
+      // add extension
+      key += ctx.ext;
+
+      // prefix
+      if (config.prefix) {
+        return `${config.prefix.replace(/\/$/, "")}/${key}`;
+      }
+
+      return key;
+    };
+  }
+
+  private validateFiles(
+    files: PresignUploadBody["files"],
+    metadata?: TMeta
+  ): void {
+    const { accept, limits } = this.config;
+
+    if (limits?.maxFileCount !== undefined) {
+      const max =
+        typeof limits.maxFileCount === "function"
+          ? limits.maxFileCount({ metadata })
+          : limits.maxFileCount;
+
+      if (files.length > max) {
+        throw UploadError.badRequest(
+          `File limit exceeded. Maximum files allowed: ${max}.`
+        );
+      }
+    }
+    if (limits?.minFileCount !== undefined) {
+      const min =
+        typeof limits.minFileCount === "function"
+          ? limits.minFileCount({ metadata })
+          : limits.minFileCount;
+
+      if (files.length < min) {
+        throw UploadError.badRequest(
+          `File limit not met. Minimum files required: ${min}.`
+        );
+      }
+    }
+
+    // 2. Per-File Validation (Size and MIME Type)
+    for (const file of files) {
+      // MIME Type Validation
+      if (accept && accept.length > 0) {
+        const isAccepted = accept.some((allowedType) => {
+          // Simple check for wildcards (*/*) and direct matches (image/png)
+          if (allowedType.endsWith("/*")) {
+            const typeGroup = allowedType.slice(0, -2);
+            return file.mimeType.startsWith(typeGroup);
+          }
+          return file.mimeType === allowedType;
+        });
+
+        if (!isAccepted) {
+          throw UploadError.badRequest(
+            `File type not allowed: ${
+              file.mimeType
+            }. Must be one of: ${accept.join(", ")}.`
+          );
+        }
+      }
+
+      // Max File Size Validation
+      if (limits?.maxFileSize !== undefined) {
+        const sizeValue =
+          typeof limits.maxFileSize === "function"
+            ? limits.maxFileSize({ metadata })
+            : limits.maxFileSize;
+
+        const maxSizeInBytes = parseFileSize(sizeValue);
+
+        if (file.size > maxSizeInBytes) {
+          // You might want to convert maxSize from bytes to KB/MB for a better error message
+          throw UploadError.badRequest(
+            `File size too large. Maximum size: ${maxSizeInBytes} bytes.`
+          );
+        }
+      }
+    }
+  }
+}

package/src/internal/db/index.ts

@@ -0,0 +1,78 @@
+import { Entity } from "electrodb";
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+
+const client = new DynamoDBClient({});
+
+export const UploadSession = new Entity(
+  {
+    model: {
+      entity: "upload-session",
+      version: "1",
+      service: "blob",
+    },
+    attributes: {
+      sessionId: {
+        type: "string",
+        required: true,
+      },
+      bucketName: {
+        type: "string",
+        required: true,
+      },
+      fileKey: {
+        type: "string",
+        required: true,
+      },
+      contentType: {
+        type: "string",
+        required: true,
+      },
+      contentDisposition: {
+        type: "string",
+        required: false,
+      },
+      fileSize: {
+        type: "number",
+        required: true,
+      },
+      createdAt: {
+        type: "string",
+        required: true,
+        default: () => new Date().toISOString(),
+      },
+      updatedAt: {
+        type: "string",
+        required: true,
+        default: () => new Date().toISOString(),
+        set: () => new Date().toISOString(),
+      },
+      status: {
+        type: ["PENDING", "SUCCESS", "FAILED"] as const,
+        required: true,
+        default: "PENDING",
+      },
+      metadata: {
+        type: "string",
+        required: false,
+      },
+    },
+    indexes: {
+      sessions: {
+        pk: {
+          field: "pk",
+          composite: ["sessionId"],
+          template: "SESSION#${sessionId}",
+        },
+        sk: {
+          field: "sk",
+          composite: ["bucketName", "fileKey"],
+          template: "FILE#${bucketName}#${fileKey}",
+        },
+      },
+    },
+  },
+  {
+    client,
+    table: "OcelTable", // required for all ocel logic
+  }
+);

package/src/lambda/index.ts

@@ -0,0 +1,8 @@
+import { Lambda, type LambdaConfig, type TriggerMap } from "./lambda";
+
+export function lambda<T extends keyof TriggerMap>(
+  id: string,
+  config: LambdaConfig<T>
+) {
+  return new Lambda(id, config);
+}

package/src/lambda/integrations/express.ts

@@ -0,0 +1,25 @@
+import {
+  Lambda,
+  type LambdaConfig,
+  type TriggerMap,
+  type TriggerUnion,
+} from "../lambda";
+import sls from "serverless-http";
+import type { Express } from "express";
+
+export function lambda<T extends keyof TriggerMap>(
+  id: string,
+  app: Express,
+  config?: Omit<LambdaConfig<T>, "trigger" | "handler">,
+) {
+  const handler = sls(app);
+
+  return new Lambda(id, {
+    ...config,
+    handler: handler as any,
+    trigger: {
+      type: "url",
+      config: {},
+    } as Extract<TriggerUnion, { type: T }>,
+  });
+}

package/src/lambda/integrations/hono.ts

@@ -0,0 +1,25 @@
+import {
+  Lambda,
+  type LambdaConfig,
+  type TriggerMap,
+  type TriggerUnion,
+} from "../lambda";
+import type { Hono } from "hono";
+import { handle } from "hono/aws-lambda";
+
+export function lambda<T extends keyof TriggerMap>(
+  id: string,
+  app: Hono,
+  config?: Omit<LambdaConfig<T>, "trigger" | "handler">,
+) {
+  const handler = handle(app);
+
+  return new Lambda(id, {
+    ...config,
+    handler: handler as any,
+    trigger: {
+      type: "url",
+      config: {},
+    } as Extract<TriggerUnion, { type: T }>,
+  });
+}

package/src/lambda/lambda.ts

@@ -0,0 +1,94 @@
+import type { Component } from "../utils/component";
+import { rpc } from "../utils/rpc";
+import { getCallerFile } from "../utils/stack";
+import type {
+  Context,
+  APIGatewayEvent,
+  S3Event,
+  LambdaFunctionURLEvent,
+} from "aws-lambda";
+
+export type TriggerMap = {
+  s3: {
+    bucket: string;
+    events: string[];
+  };
+  cron: {
+    schedule: string;
+  };
+  url: {
+    streaming?: boolean;
+  };
+  api: {
+    id?: string;
+    path: string;
+    method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "OPTIONS" | "*";
+  };
+};
+
+export type TriggerUnion = {
+  [K in keyof TriggerMap]: { type: K; config: TriggerMap[K] };
+}[keyof TriggerMap];
+
+export type LambdaHandlerOutput<T extends keyof TriggerMap> = T extends "api"
+  ? {
+      statusCode: number;
+      headers: Record<string, string>;
+      body: string;
+    }
+  : any;
+
+export type LambdaEvent<T extends keyof TriggerMap> = T extends "s3"
+  ? S3Event
+  : T extends "api"
+    ? APIGatewayEvent
+    : T extends "url"
+      ? LambdaFunctionURLEvent
+      : any;
+
+export interface LambdaConfig<T extends keyof TriggerMap> {
+  handler: (
+    event: LambdaEvent<T>,
+    context: Context,
+  ) => Promise<LambdaHandlerOutput<T>> | LambdaHandlerOutput<T>;
+
+  trigger: Extract<TriggerUnion, { type: T }>;
+  link?: Component[];
+}
+
+export class Lambda<T extends keyof TriggerMap> implements Component {
+  constructor(
+    public id: string,
+    config: LambdaConfig<T>,
+  ) {
+    // remove non-serializable stuff
+    const { handler, link, ...rest } = config;
+    const links = link?.map((component) => component.__id()) || [];
+
+    // @ts-expect-error - accessed by cli during build
+    this.__handler = handler;
+    // @ts-expect-error - accessed by cli during build
+    this[Symbol.for("ocel:lambda:id")] = id;
+
+    if (process.env.OCEL_PHASE === "discovery") {
+      const source = getCallerFile();
+
+      if (!source) {
+        throw new Error(
+          "Could not determine caller file for lambda registration",
+        );
+      }
+
+      rpc.register({
+        id: this.id,
+        type: "lambda",
+        source,
+        config: { ...rest, links },
+      });
+    }
+  }
+
+  __id(): string {
+    return this.id;
+  }
+}

package/src/postgres/index.ts

@@ -0,0 +1,5 @@
+import { Postgres, type PostgresConfig } from "./pg";
+
+export function postgres(id: string, config: PostgresConfig) {
+  return new Postgres(id, config);
+}

package/src/postgres/pg.ts

@@ -0,0 +1,33 @@
+import type { Component } from "../utils/component";
+import { rpc } from "../utils/rpc";
+import { getCallerFile } from "../utils/stack";
+
+export interface PostgresConfig {
+  version?: string;
+  migrations?: string;
+}
+
+export class Postgres implements Component {
+  constructor(public id: string, config: PostgresConfig) {
+    // remove non-serializable stuff
+
+    if (process.env.OCEL_PHASE === "discovery") {
+      const source = getCallerFile();
+
+      rpc.register({
+        id: this.id,
+        type: "postgres",
+        source: source || "unknown",
+        config: {},
+      });
+    }
+  }
+
+  __id(): string {
+    return this.id;
+  }
+
+  async sql(query: string) {
+    return {};
+  }
+}

package/src/utils/component.ts

@@ -0,0 +1,3 @@
+export interface Component {
+  __id(): string;
+}

package/src/utils/constants.ts

@@ -0,0 +1,2 @@
+export const IS_DISCOVERY_PHASE = process.env.OCEL_PHASE === "discovery";
+export const IS_DEV_MODE = process.env.OCEL_DEV === "1";