@uploadista/core 0.0.2

package/dist/flow/nodes/storage-node.d.ts

@@ -0,0 +1,67 @@
+import { Effect } from "effect";
+import { z } from "zod";
+import { UploadistaError } from "../../errors";
+import { type UploadFile } from "../../types";
+import { UploadServer } from "../../upload";
+/**
+ * Schema for storage node parameters.
+ * Currently empty but can be extended for storage-specific configuration.
+ */
+export declare const storageParamsSchema: z.ZodObject<{}, z.core.$strip>;
+/**
+ * Parameters for the storage node.
+ * Currently no parameters are required, but the schema is available for future extensions.
+ */
+export type StorageParams = z.infer<typeof storageParamsSchema>;
+/**
+ * Creates a storage node for storing files in the specified storage.
+ *
+ * The storage node handles the process of:
+ * 1. Reading the input file from the upload server
+ * 2. Checking if the file is already in the target storage
+ * 3. If not, transferring the file to the target storage
+ * 4. Applying optional post-processing
+ * 5. Returning the final stored file
+ *
+ * @param id - Unique identifier for the node
+ * @param postProcessFile - Optional function to process the file after storage
+ * @returns An Effect that creates a flow node configured for file storage
+ *
+ * @example
+ * ```typescript
+ * // Create basic storage node
+ * const storageNode = yield* createStorageNode("store-file");
+ *
+ * // Create storage node with post-processing
+ * const storageWithProcessing = yield* createStorageNode("store-and-process", (file) => {
+ *   return Effect.succeed({
+ *     ...file,
+ *     metadata: { ...file.metadata, processed: true }
+ *   });
+ * });
+ * ```
+ */
+export declare function createStorageNode(id: string, postProcessFile?: (file: UploadFile) => Effect.Effect<UploadFile>): Effect.Effect<import("..").FlowNode<{
+    id: string;
+    offset: number;
+    storage: {
+        id: string;
+        type: string;
+        path?: string | undefined;
+        uploadId?: string | undefined;
+        bucket?: string | undefined;
+    };
+    size?: number | undefined;
+    metadata?: Record<string, string | number | boolean> | undefined;
+    creationDate?: string | undefined;
+    url?: string | undefined;
+    sizeIsDeferred?: boolean | undefined;
+    checksum?: string | undefined;
+    checksumAlgorithm?: string | undefined;
+    flow?: {
+        flowId: string;
+        nodeId: string;
+        jobId: string;
+    } | undefined;
+}, UploadFile, UploadistaError>, never, UploadServer>;
+//# sourceMappingURL=storage-node.d.ts.map

package/dist/flow/nodes/storage-node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-node.d.ts","sourceRoot":"","sources":["../../../src/flow/nodes/storage-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,KAAK,UAAU,EAAoB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C;;;GAGG;AACH,eAAO,MAAM,mBAAmB,gCAAe,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,EAAE,EAAE,MAAM,EACV,eAAe,GAAE,CAAC,IAAI,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,UAAU,CACzC;;;;;;;;;;;;;;;;;;;;;;sDA2EvB"}

package/dist/flow/nodes/storage-node.js

@@ -0,0 +1,94 @@
+import { Effect } from "effect";
+import { z } from "zod";
+import { UploadistaError } from "../../errors";
+import { uploadFileSchema } from "../../types";
+import { UploadServer } from "../../upload";
+import { createFlowNode, NodeType } from "../node";
+import { completeNodeExecution } from "../types";
+import { resolveUploadMetadata } from "../utils/resolve-upload-metadata";
+/**
+ * Schema for storage node parameters.
+ * Currently empty but can be extended for storage-specific configuration.
+ */
+export const storageParamsSchema = z.object({});
+/**
+ * Creates a storage node for storing files in the specified storage.
+ *
+ * The storage node handles the process of:
+ * 1. Reading the input file from the upload server
+ * 2. Checking if the file is already in the target storage
+ * 3. If not, transferring the file to the target storage
+ * 4. Applying optional post-processing
+ * 5. Returning the final stored file
+ *
+ * @param id - Unique identifier for the node
+ * @param postProcessFile - Optional function to process the file after storage
+ * @returns An Effect that creates a flow node configured for file storage
+ *
+ * @example
+ * ```typescript
+ * // Create basic storage node
+ * const storageNode = yield* createStorageNode("store-file");
+ *
+ * // Create storage node with post-processing
+ * const storageWithProcessing = yield* createStorageNode("store-and-process", (file) => {
+ *   return Effect.succeed({
+ *     ...file,
+ *     metadata: { ...file.metadata, processed: true }
+ *   });
+ * });
+ * ```
+ */
+export function createStorageNode(id, postProcessFile = (file) => Effect.succeed(file)) {
+    return Effect.gen(function* () {
+        const uploadServer = yield* UploadServer;
+        return yield* createFlowNode({
+            id,
+            name: "Storage",
+            description: "Stores a file in the storage",
+            type: NodeType.output,
+            inputSchema: uploadFileSchema,
+            outputSchema: uploadFileSchema,
+            run: ({ data: file, storageId, flowId, jobId, clientId }) => {
+                return Effect.gen(function* () {
+                    const { type, fileName, metadata, metadataJson } = resolveUploadMetadata(file.metadata);
+                    const flow = {
+                        flowId,
+                        nodeId: id,
+                        jobId,
+                    };
+                    const normalizedFile = metadata ? { ...file, metadata } : file;
+                    const upload = yield* uploadServer.getUpload(file.id);
+                    if (!upload.id) {
+                        return yield* Effect.fail(UploadistaError.fromCode("FILE_READ_ERROR", new Error("Upload Key is undefined")));
+                    }
+                    // If the upload is already in the correct storage, return the file, just update the flow
+                    if (upload.storage.id === storageId) {
+                        return completeNodeExecution(yield* postProcessFile({ ...normalizedFile, flow }));
+                    }
+                    const inputBytes = yield* uploadServer.read(file.id, clientId);
+                    const stream = new ReadableStream({
+                        start(controller) {
+                            controller.enqueue(inputBytes);
+                            controller.close();
+                        },
+                    });
+                    const uploadResult = yield* uploadServer.upload({
+                        storageId,
+                        size: inputBytes.byteLength,
+                        type,
+                        fileName,
+                        lastModified: 0,
+                        metadata: metadataJson,
+                        flow,
+                    }, clientId, stream);
+                    const resolvedUploadResult = resolveUploadMetadata(uploadResult.metadata);
+                    const postProcessed = yield* postProcessFile(resolvedUploadResult.metadata
+                        ? { ...uploadResult, metadata: resolvedUploadResult.metadata }
+                        : uploadResult);
+                    return completeNodeExecution(postProcessed);
+                });
+            },
+        });
+    });
+}

package/dist/flow/nodes/streaming-input-node.d.ts

@@ -0,0 +1,69 @@
+import { Effect } from "effect";
+import { z } from "zod";
+import { UploadistaError } from "../../errors";
+import { UploadServer } from "../../upload";
+export declare const streamingInputDataSchema: z.ZodUnion<readonly [z.ZodObject<{
+    operation: z.ZodLiteral<"init">;
+    storageId: z.ZodString;
+    metadata: z.ZodOptional<z.ZodObject<{
+        originalName: z.ZodOptional<z.ZodString>;
+        mimeType: z.ZodOptional<z.ZodString>;
+        size: z.ZodOptional<z.ZodNumber>;
+        extension: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>, z.ZodObject<{
+    operation: z.ZodLiteral<"finalize">;
+    uploadId: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    operation: z.ZodLiteral<"url">;
+    url: z.ZodString;
+    storageId: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodObject<{
+        originalName: z.ZodOptional<z.ZodString>;
+        mimeType: z.ZodOptional<z.ZodString>;
+        size: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+}, z.core.$strip>]>;
+export declare function createStreamingInputNode(id: string): Effect.Effect<import("..").FlowNode<{
+    operation: "init";
+    storageId: string;
+    metadata?: {
+        originalName?: string | undefined;
+        mimeType?: string | undefined;
+        size?: number | undefined;
+        extension?: string | undefined;
+    } | undefined;
+} | {
+    operation: "finalize";
+    uploadId: string;
+} | {
+    operation: "url";
+    url: string;
+    storageId?: string | undefined;
+    metadata?: {
+        originalName?: string | undefined;
+        mimeType?: string | undefined;
+        size?: number | undefined;
+    } | undefined;
+}, {
+    id: string;
+    offset: number;
+    storage: {
+        id: string;
+        type: string;
+        path?: string | undefined;
+        uploadId?: string | undefined;
+        bucket?: string | undefined;
+    };
+    size?: number | undefined;
+    metadata?: Record<string, string> | undefined;
+    creationDate?: string | undefined;
+    url?: string | undefined;
+    sizeIsDeferred?: boolean | undefined;
+    flow?: {
+        flowId: string;
+        nodeId: string;
+        jobId: string;
+    } | undefined;
+}, UploadistaError>, never, UploadServer>;
+//# sourceMappingURL=streaming-input-node.d.ts.map

package/dist/flow/nodes/streaming-input-node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming-input-node.d.ts","sourceRoot":"","sources":["../../../src/flow/nodes/streaming-input-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA+D5C,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;mBAInC,CAAC;AAIH,wBAAgB,wBAAwB,CAAC,EAAE,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;0CAmHlD"}

package/dist/flow/nodes/streaming-input-node.js

@@ -0,0 +1,156 @@
+import { Effect } from "effect";
+import { z } from "zod";
+import { UploadistaError } from "../../errors";
+import { uploadFileSchema } from "../../types";
+import { UploadServer } from "../../upload";
+import { createFlowNode, NodeType } from "../node";
+import { completeNodeExecution, waitingNodeExecution } from "../types";
+// Helper functions for URL-based file fetching
+const fetchFile = (url) => {
+    return Effect.tryPromise({
+        try: async () => {
+            return await fetch(url);
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", {
+                cause: error,
+            });
+        },
+    });
+};
+const arrayBuffer = (response) => {
+    return Effect.tryPromise({
+        try: async () => {
+            return await response.arrayBuffer();
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", {
+                cause: error,
+            });
+        },
+    });
+};
+// Input schemas for different operations
+const initStreamingInputSchema = z.object({
+    operation: z.literal("init"),
+    storageId: z.string(),
+    metadata: z
+        .object({
+        originalName: z.string().optional(),
+        mimeType: z.string().optional(),
+        size: z.number().optional(),
+        extension: z.string().optional(),
+    })
+        .optional(),
+});
+const finalizeStreamingInputSchema = z.object({
+    operation: z.literal("finalize"),
+    uploadId: z.string(),
+});
+const urlInputSchema = z.object({
+    operation: z.literal("url"),
+    url: z.string(),
+    storageId: z.string().optional(),
+    metadata: z
+        .object({
+        originalName: z.string().optional(),
+        mimeType: z.string().optional(),
+        size: z.number().optional(),
+    })
+        .optional(),
+});
+export const streamingInputDataSchema = z.union([
+    initStreamingInputSchema,
+    finalizeStreamingInputSchema,
+    urlInputSchema,
+]);
+export function createStreamingInputNode(id) {
+    return Effect.gen(function* () {
+        const uploadServer = yield* UploadServer;
+        return yield* createFlowNode({
+            id,
+            name: "Input",
+            description: "Handles file input through multiple methods - streaming upload (init/finalize) or direct URL fetch",
+            type: NodeType.input,
+            inputSchema: streamingInputDataSchema,
+            outputSchema: uploadFileSchema,
+            run: ({ data, flowId, jobId, }) => {
+                return Effect.gen(function* () {
+                    switch (data.operation) {
+                        case "init": {
+                            // Create upload using upload server - it handles all state management
+                            const inputFile = {
+                                storageId: data.storageId,
+                                size: data.metadata?.size || 0,
+                                type: data.metadata?.mimeType || "application/octet-stream",
+                                fileName: data.metadata?.originalName,
+                                lastModified: data.metadata?.size ? Date.now() : undefined,
+                            };
+                            const uploadFile = yield* uploadServer.createUpload(inputFile);
+                            // Return waiting state with the upload file
+                            // Client will upload chunks directly to the upload API
+                            return waitingNodeExecution(uploadFile);
+                        }
+                        case "finalize": {
+                            // Get final upload file from upload server's KV store
+                            const finalUploadFile = yield* uploadServer.getUpload(data.uploadId);
+                            // Complete the node execution with the final upload file
+                            // Flow can now continue to next nodes (e.g., save to storage, optimize)
+                            return completeNodeExecution({
+                                ...finalUploadFile,
+                                flow: {
+                                    flowId,
+                                    nodeId: id,
+                                    jobId,
+                                },
+                            });
+                        }
+                        case "url": {
+                            // Fetch file from URL directly
+                            const response = yield* fetchFile(data.url);
+                            const buffer = yield* arrayBuffer(response);
+                            // Extract metadata from response or use provided metadata
+                            const mimeType = data.metadata?.mimeType ||
+                                response.headers.get("content-type") ||
+                                "application/octet-stream";
+                            const size = data.metadata?.size ||
+                                Number(response.headers.get("content-length") || 0);
+                            const fileName = data.metadata?.originalName ||
+                                data.url.split("/").pop() ||
+                                "file";
+                            // Create a readable stream from the buffer
+                            const stream = new ReadableStream({
+                                start(controller) {
+                                    controller.enqueue(new Uint8Array(buffer));
+                                    controller.close();
+                                },
+                            });
+                            // Use upload server to create and store the file
+                            const inputFile = {
+                                storageId: data.storageId || "buffer",
+                                size,
+                                type: mimeType,
+                                fileName,
+                                lastModified: Date.now(),
+                            };
+                            const uploadFile = yield* uploadServer.upload(inputFile, stream);
+                            // Complete the node execution with the upload file
+                            return completeNodeExecution({
+                                ...uploadFile,
+                                flow: {
+                                    flowId,
+                                    nodeId: id,
+                                    jobId,
+                                },
+                            });
+                        }
+                        default:
+                            throw yield* UploadistaError.fromCode("VALIDATION_ERROR", {
+                                cause: new Error("Invalid operation"),
+                            }).toEffect();
+                    }
+                });
+            },
+        });
+    });
+}

package/dist/flow/nodes/transform-node.d.ts

@@ -0,0 +1,85 @@
+import { Effect } from "effect";
+import type { UploadistaError } from "../../errors";
+import type { UploadFile } from "../../types";
+import { UploadServer } from "../../upload";
+/**
+ * Configuration object for creating a transform node.
+ */
+export interface TransformNodeConfig {
+    /** Unique identifier for the node */
+    id: string;
+    /** Human-readable name for the node */
+    name: string;
+    /** Description of what the node does */
+    description: string;
+    /** Function that transforms file bytes */
+    transform: (bytes: Uint8Array, file: UploadFile) => Effect.Effect<Uint8Array | {
+        bytes: Uint8Array;
+        type?: string;
+        fileName?: string;
+    }, UploadistaError>;
+}
+/**
+ * Creates a transform node that handles the common pattern of:
+ * 1. Reading bytes from an UploadFile
+ * 2. Transforming the bytes
+ * 3. Uploading the result as a new UploadFile
+ *
+ * This simplifies nodes that just need to transform file bytes without
+ * worrying about upload server interactions.
+ *
+ * @param config - Configuration object for the transform node
+ * @returns An Effect that creates a flow node configured for file transformation
+ *
+ * @example
+ * ```typescript
+ * // Create an image resize transform node
+ * const resizeNode = yield* createTransformNode({
+ *   id: "resize-image",
+ *   name: "Resize Image",
+ *   description: "Resizes images to specified dimensions",
+ *   transform: (bytes, file) => {
+ *     // Your transformation logic here
+ *     return Effect.succeed(transformedBytes);
+ *   }
+ * });
+ *
+ * // Create a transform node that changes file metadata
+ * const metadataTransformNode = yield* createTransformNode({
+ *   id: "add-metadata",
+ *   name: "Add Metadata",
+ *   description: "Adds custom metadata to files",
+ *   transform: (bytes, file) => {
+ *     return Effect.succeed({
+ *       bytes,
+ *       type: "application/custom",
+ *       fileName: `processed-${file.fileName}`
+ *     });
+ *   }
+ * });
+ * ```
+ */
+export declare function createTransformNode({ id, name, description, transform, }: TransformNodeConfig): Effect.Effect<import("..").FlowNode<{
+    id: string;
+    offset: number;
+    storage: {
+        id: string;
+        type: string;
+        path?: string | undefined;
+        uploadId?: string | undefined;
+        bucket?: string | undefined;
+    };
+    size?: number | undefined;
+    metadata?: Record<string, string | number | boolean> | undefined;
+    creationDate?: string | undefined;
+    url?: string | undefined;
+    sizeIsDeferred?: boolean | undefined;
+    checksum?: string | undefined;
+    checksumAlgorithm?: string | undefined;
+    flow?: {
+        flowId: string;
+        nodeId: string;
+        jobId: string;
+    } | undefined;
+}, UploadFile, UploadistaError>, never, UploadServer>;
+//# sourceMappingURL=transform-node.d.ts.map

package/dist/flow/nodes/transform-node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform-node.d.ts","sourceRoot":"","sources":["../../../src/flow/nodes/transform-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,SAAS,EAAE,CACT,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,UAAU,KACb,MAAM,CAAC,MAAM,CAChB,UAAU,GAAG;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EACpE,eAAe,CAChB,CAAC;CACH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAgB,mBAAmB,CAAC,EAClC,EAAE,EACF,IAAI,EACJ,WAAW,EACX,SAAS,GACV,EAAE,mBAAmB;;;;;;;;;;;;;;;;;;;;;;sDA+ErB"}

package/dist/flow/nodes/transform-node.js

@@ -0,0 +1,107 @@
+import { Effect } from "effect";
+import { uploadFileSchema } from "../../types";
+import { UploadServer } from "../../upload";
+import { createFlowNode, NodeType } from "../node";
+import { completeNodeExecution } from "../types";
+import { resolveUploadMetadata } from "../utils/resolve-upload-metadata";
+/**
+ * Creates a transform node that handles the common pattern of:
+ * 1. Reading bytes from an UploadFile
+ * 2. Transforming the bytes
+ * 3. Uploading the result as a new UploadFile
+ *
+ * This simplifies nodes that just need to transform file bytes without
+ * worrying about upload server interactions.
+ *
+ * @param config - Configuration object for the transform node
+ * @returns An Effect that creates a flow node configured for file transformation
+ *
+ * @example
+ * ```typescript
+ * // Create an image resize transform node
+ * const resizeNode = yield* createTransformNode({
+ *   id: "resize-image",
+ *   name: "Resize Image",
+ *   description: "Resizes images to specified dimensions",
+ *   transform: (bytes, file) => {
+ *     // Your transformation logic here
+ *     return Effect.succeed(transformedBytes);
+ *   }
+ * });
+ *
+ * // Create a transform node that changes file metadata
+ * const metadataTransformNode = yield* createTransformNode({
+ *   id: "add-metadata",
+ *   name: "Add Metadata",
+ *   description: "Adds custom metadata to files",
+ *   transform: (bytes, file) => {
+ *     return Effect.succeed({
+ *       bytes,
+ *       type: "application/custom",
+ *       fileName: `processed-${file.fileName}`
+ *     });
+ *   }
+ * });
+ * ```
+ */
+export function createTransformNode({ id, name, description, transform, }) {
+    return Effect.gen(function* () {
+        const uploadServer = yield* UploadServer;
+        return yield* createFlowNode({
+            id,
+            name,
+            description,
+            type: NodeType.process,
+            inputSchema: uploadFileSchema,
+            outputSchema: uploadFileSchema,
+            run: ({ data: file, storageId, flowId, jobId, clientId }) => {
+                return Effect.gen(function* () {
+                    const flow = {
+                        flowId,
+                        nodeId: id,
+                        jobId,
+                    };
+                    // Read input bytes from upload server
+                    const inputBytes = yield* uploadServer.read(file.id, clientId);
+                    // Transform the bytes using the provided function
+                    const transformResult = yield* transform(inputBytes, file);
+                    // Handle both simple Uint8Array and object with metadata
+                    const outputBytes = transformResult instanceof Uint8Array
+                        ? transformResult
+                        : transformResult.bytes;
+                    const outputType = transformResult instanceof Uint8Array
+                        ? undefined
+                        : transformResult.type;
+                    const outputFileName = transformResult instanceof Uint8Array
+                        ? undefined
+                        : transformResult.fileName;
+                    // Create a stream from the output bytes
+                    const stream = new ReadableStream({
+                        start(controller) {
+                            controller.enqueue(outputBytes);
+                            controller.close();
+                        },
+                    });
+                    const { type, fileName, metadata, metadataJson } = resolveUploadMetadata(file.metadata);
+                    // Upload the transformed bytes back to the upload server
+                    // Use output metadata if provided, otherwise fall back to original
+                    const result = yield* uploadServer.upload({
+                        storageId,
+                        size: outputBytes.byteLength,
+                        type: outputType ?? type,
+                        fileName: outputFileName ?? fileName,
+                        lastModified: 0,
+                        metadata: metadataJson,
+                        flow,
+                    }, clientId, stream);
+                    return completeNodeExecution(metadata
+                        ? {
+                            ...result,
+                            metadata,
+                        }
+                        : result);
+                });
+            },
+        });
+    });
+}