@uploadista/core 0.0.2

package/dist/upload/convert-to-stream.d.ts

@@ -0,0 +1,38 @@
+import { Stream } from "effect";
+import { UploadistaError } from "../errors";
+/**
+ * Converts a ReadableStream to an Effect Stream.
+ *
+ * This utility function wraps a ReadableStream in an Effect Stream, providing
+ * proper error handling and integration with the Effect ecosystem. It's used
+ * throughout the upload system to convert raw streams into Effect-compatible
+ * streams for processing.
+ *
+ * The function handles:
+ * - Stream conversion with proper error mapping
+ * - UploadistaError creation for stream errors
+ * - Integration with Effect Stream processing
+ *
+ * @param data - The ReadableStream to convert
+ * @returns Effect Stream that can be processed with Effect operations
+ *
+ * @example
+ * ```typescript
+ * // Convert a file stream to Effect Stream
+ * const fileStream = new ReadableStream(...);
+ * const effectStream = convertToStream(fileStream);
+ *
+ * // Process with Effect operations
+ * const processedStream = effectStream.pipe(
+ *   Stream.map((chunk) => processChunk(chunk)),
+ *   Stream.filter((chunk) => chunk.length > 0)
+ * );
+ *
+ * // Run the stream
+ * await Stream.runForEach(processedStream, (chunk) =>
+ *   Effect.logInfo(`Processed chunk: ${chunk.length} bytes`)
+ * );
+ * ```
+ */
+export declare function convertToStream<T>(data: ReadableStream<T>): Stream.Stream<T, UploadistaError, never>;
+//# sourceMappingURL=convert-to-stream.d.ts.map

package/dist/upload/convert-to-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-to-stream.d.ts","sourceRoot":"","sources":["../../src/upload/convert-to-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,4CAUzD"}

package/dist/upload/convert-to-stream.js

@@ -0,0 +1,43 @@
+import { Stream } from "effect";
+import { UploadistaError } from "../errors";
+/**
+ * Converts a ReadableStream to an Effect Stream.
+ *
+ * This utility function wraps a ReadableStream in an Effect Stream, providing
+ * proper error handling and integration with the Effect ecosystem. It's used
+ * throughout the upload system to convert raw streams into Effect-compatible
+ * streams for processing.
+ *
+ * The function handles:
+ * - Stream conversion with proper error mapping
+ * - UploadistaError creation for stream errors
+ * - Integration with Effect Stream processing
+ *
+ * @param data - The ReadableStream to convert
+ * @returns Effect Stream that can be processed with Effect operations
+ *
+ * @example
+ * ```typescript
+ * // Convert a file stream to Effect Stream
+ * const fileStream = new ReadableStream(...);
+ * const effectStream = convertToStream(fileStream);
+ *
+ * // Process with Effect operations
+ * const processedStream = effectStream.pipe(
+ *   Stream.map((chunk) => processChunk(chunk)),
+ *   Stream.filter((chunk) => chunk.length > 0)
+ * );
+ *
+ * // Run the stream
+ * await Stream.runForEach(processedStream, (chunk) =>
+ *   Effect.logInfo(`Processed chunk: ${chunk.length} bytes`)
+ * );
+ * ```
+ */
+export function convertToStream(data) {
+    return Stream.fromReadableStream(() => data, (error) => new UploadistaError({
+        code: "UNKNOWN_ERROR",
+        status: 500,
+        body: String(error),
+    }));
+}

package/dist/upload/convert-upload-to-flow-file.d.ts

@@ -0,0 +1,14 @@
+import { Effect } from "effect";
+import type { UploadFile } from "../types";
+import type { UploadServerShape } from "./upload-server";
+export declare const uploadFileToFlowFile: (uploadFile: UploadFile, uploadServer: UploadServerShape) => Effect.Effect<{
+    path: string;
+    inputBytes: Uint8Array<ArrayBufferLike>;
+    metadata: {
+        mimeType: string;
+        size: number;
+        originalName: string;
+        extension: string;
+    };
+}, import("..").UploadistaError, never>;
+//# sourceMappingURL=convert-upload-to-flow-file.d.ts.map

package/dist/upload/convert-upload-to-flow-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-upload-to-flow-file.d.ts","sourceRoot":"","sources":["../../src/upload/convert-upload-to-flow-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,eAAO,MAAM,oBAAoB,GAC/B,YAAY,UAAU,EACtB,cAAc,iBAAiB;;;;;;;;;uCAqBhC,CAAC"}

package/dist/upload/convert-upload-to-flow-file.js

@@ -0,0 +1,21 @@
+import { Effect } from "effect";
+import { detectMimeType } from "./mime";
+// Helper to convert UploadFile to FlowFile
+export const uploadFileToFlowFile = (uploadFile, uploadServer) => {
+    return Effect.gen(function* () {
+        const metadata = uploadFile.metadata || {};
+        const originalName = metadata.fileName || `upload-${uploadFile.id}`;
+        const mimeType = metadata.type || detectMimeType(new Uint8Array(0), originalName);
+        const fileBytes = yield* uploadServer.read(uploadFile.id);
+        return {
+            path: originalName,
+            inputBytes: fileBytes,
+            metadata: {
+                mimeType,
+                size: uploadFile.size || 0,
+                originalName,
+                extension: originalName.split(".").pop() || "",
+            },
+        };
+    });
+};

package/dist/upload/create-upload.d.ts

@@ -0,0 +1,68 @@
+import { Effect } from "effect";
+import { type EventEmitter, type InputFile, type KvStore, type UploadEvent, type UploadFile, type UploadFileDataStoresShape } from "../types";
+import type { GenerateIdShape } from "../utils/generate-id";
+/**
+ * Creates a new upload and initializes it in the storage system.
+ *
+ * This function handles the initial upload creation process including:
+ * - Generating a unique upload ID
+ * - Routing to appropriate data store based on storage ID
+ * - Creating the upload record in the data store
+ * - Storing upload metadata in KV store
+ * - Emitting upload started events
+ * - Parsing and validating metadata
+ *
+ * The function includes comprehensive observability with:
+ * - Effect tracing spans for performance monitoring
+ * - Metrics tracking for upload creation, file sizes, and success rates
+ * - Structured logging for debugging and monitoring
+ * - Error handling with proper UploadistaError types
+ *
+ * @param inputFile - Input file configuration including storage, size, type, etc.
+ * @param clientId - Client identifier (null for anonymous uploads)
+ * @param dataStoreService - Service for routing to appropriate data stores
+ * @param kvStore - KV store for upload metadata persistence
+ * @param eventEmitter - Event emitter for upload lifecycle events
+ * @param generateId - ID generator for creating unique upload identifiers
+ * @returns Effect that yields the created UploadFile
+ *
+ * @example
+ * ```typescript
+ * // Create a new upload
+ * const inputFile: InputFile = {
+ *   storageId: "s3-production",
+ *   size: 1024000,
+ *   type: "image/jpeg",
+ *   fileName: "photo.jpg",
+ *   metadata: JSON.stringify({ category: "photos" })
+ * };
+ *
+ * const createEffect = createUpload(
+ *   inputFile,
+ *   "client-123",
+ *   {
+ *     dataStoreService,
+ *     kvStore,
+ *     eventEmitter,
+ *     generateId
+ *   }
+ * );
+ *
+ * // Run with dependencies
+ * const upload = await Effect.runPromise(
+ *   createEffect.pipe(
+ *     Effect.provide(dataStoreLayer),
+ *     Effect.provide(kvStoreLayer),
+ *     Effect.provide(eventEmitterLayer),
+ *     Effect.provide(generateIdLayer)
+ *   )
+ * );
+ * ```
+ */
+export declare const createUpload: (inputFile: InputFile, clientId: string | null, { dataStoreService, kvStore, eventEmitter, generateId, }: {
+    dataStoreService: UploadFileDataStoresShape;
+    kvStore: KvStore<UploadFile>;
+    eventEmitter: EventEmitter<UploadEvent>;
+    generateId: GenerateIdShape;
+}) => Effect.Effect<UploadFile, import("..").UploadistaError, never>;
+//# sourceMappingURL=create-upload.d.ts.map

package/dist/upload/create-upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-upload.d.ts","sourceRoot":"","sources":["../../src/upload/create-upload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA4B,MAAM,QAAQ,CAAC;AAC1D,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,OAAO,EACZ,KAAK,WAAW,EAEhB,KAAK,UAAU,EACf,KAAK,yBAAyB,EAC/B,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,eAAO,MAAM,YAAY,GACvB,WAAW,SAAS,EACpB,UAAU,MAAM,GAAG,IAAI,EACvB,0DAKG;IACD,gBAAgB,EAAE,yBAAyB,CAAC;IAC5C,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7B,YAAY,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IACxC,UAAU,EAAE,eAAe,CAAC;CAC7B,mEAkIA,CAAC"}

package/dist/upload/create-upload.js

@@ -0,0 +1,157 @@
+import { Effect, Metric, MetricBoundaries } from "effect";
+import { UploadEventType, } from "../types";
+/**
+ * Creates a new upload and initializes it in the storage system.
+ *
+ * This function handles the initial upload creation process including:
+ * - Generating a unique upload ID
+ * - Routing to appropriate data store based on storage ID
+ * - Creating the upload record in the data store
+ * - Storing upload metadata in KV store
+ * - Emitting upload started events
+ * - Parsing and validating metadata
+ *
+ * The function includes comprehensive observability with:
+ * - Effect tracing spans for performance monitoring
+ * - Metrics tracking for upload creation, file sizes, and success rates
+ * - Structured logging for debugging and monitoring
+ * - Error handling with proper UploadistaError types
+ *
+ * @param inputFile - Input file configuration including storage, size, type, etc.
+ * @param clientId - Client identifier (null for anonymous uploads)
+ * @param dataStoreService - Service for routing to appropriate data stores
+ * @param kvStore - KV store for upload metadata persistence
+ * @param eventEmitter - Event emitter for upload lifecycle events
+ * @param generateId - ID generator for creating unique upload identifiers
+ * @returns Effect that yields the created UploadFile
+ *
+ * @example
+ * ```typescript
+ * // Create a new upload
+ * const inputFile: InputFile = {
+ *   storageId: "s3-production",
+ *   size: 1024000,
+ *   type: "image/jpeg",
+ *   fileName: "photo.jpg",
+ *   metadata: JSON.stringify({ category: "photos" })
+ * };
+ *
+ * const createEffect = createUpload(
+ *   inputFile,
+ *   "client-123",
+ *   {
+ *     dataStoreService,
+ *     kvStore,
+ *     eventEmitter,
+ *     generateId
+ *   }
+ * );
+ *
+ * // Run with dependencies
+ * const upload = await Effect.runPromise(
+ *   createEffect.pipe(
+ *     Effect.provide(dataStoreLayer),
+ *     Effect.provide(kvStoreLayer),
+ *     Effect.provide(eventEmitterLayer),
+ *     Effect.provide(generateIdLayer)
+ *   )
+ * );
+ * ```
+ */
+export const createUpload = (inputFile, clientId, { dataStoreService, kvStore, eventEmitter, generateId, }) => Effect.gen(function* () {
+    // Get datastore using Effect
+    const dataStore = yield* dataStoreService.getDataStore(inputFile.storageId, clientId);
+    const id = yield* generateId.generateId();
+    const { size, type, fileName, lastModified, metadata, flow } = inputFile;
+    let parsedMetadata = {};
+    if (metadata) {
+        try {
+            parsedMetadata = JSON.parse(metadata);
+        }
+        catch {
+            parsedMetadata = {};
+        }
+    }
+    const metadataObject = {
+        ...parsedMetadata,
+        type,
+        fileName: fileName ?? "",
+    };
+    if (lastModified) {
+        metadataObject.lastModified = lastModified.toString();
+    }
+    const file = {
+        id,
+        size,
+        metadata: metadataObject,
+        offset: 0,
+        creationDate: new Date().toISOString(),
+        storage: {
+            id: inputFile.storageId,
+            type,
+            path: "",
+            bucket: dataStore.bucket,
+        },
+        flow,
+    };
+    // Create file using Effect
+    const fileCreated = yield* dataStore.create(file);
+    // Store in KV store
+    yield* kvStore.set(id, fileCreated);
+    // Emit event
+    yield* eventEmitter.emit(id, {
+        type: UploadEventType.UPLOAD_STARTED,
+        data: fileCreated,
+        flow: fileCreated.flow,
+    });
+    return fileCreated;
+}).pipe(
+// Add tracing span for the entire create operation
+Effect.withSpan("upload-create", {
+    attributes: {
+        "upload.file_name": inputFile.fileName ?? "unknown",
+        "upload.file_size": inputFile.size?.toString() ?? "0",
+        "upload.storage_id": inputFile.storageId,
+        "upload.mime_type": inputFile.type,
+        "upload.has_flow": inputFile.flow ? "true" : "false",
+    },
+}), 
+// Track upload creation metrics
+Effect.tap((file) => Effect.gen(function* () {
+    // Increment upload created counter
+    yield* Metric.increment(Metric.counter("upload_created_total", {
+        description: "Total number of uploads created",
+    }));
+    // Record file size
+    if (file.size) {
+        const fileSizeHistogram = Metric.histogram("upload_file_size_bytes", MetricBoundaries.exponential({
+            start: 1024,
+            factor: 2,
+            count: 25,
+        }));
+        yield* Metric.update(fileSizeHistogram, file.size);
+    }
+    // Track active uploads gauge
+    const activeUploadsGauge = Metric.gauge("active_uploads");
+    yield* Metric.increment(activeUploadsGauge);
+})), 
+// Add structured logging
+Effect.tap((file) => Effect.logInfo("Upload created").pipe(Effect.annotateLogs({
+    "upload.id": file.id,
+    "upload.file_name": inputFile.fileName ?? "unknown",
+    "upload.file_size": inputFile.size?.toString() ?? "0",
+    "upload.storage_id": inputFile.storageId,
+}))), 
+// Handle errors with logging and metrics
+Effect.tapError((error) => Effect.gen(function* () {
+    // Log error
+    yield* Effect.logError("Upload creation failed").pipe(Effect.annotateLogs({
+        "upload.file_name": inputFile.fileName ?? "unknown",
+        "upload.storage_id": inputFile.storageId,
+        error: String(error),
+    }));
+    // Track failed upload metric
+    yield* Metric.increment(Metric.counter("upload_failed_total", {
+        description: "Total number of uploads that failed",
+    }));
+})));

package/dist/upload/index.cjs

@@ -0,0 +1 @@
+require(`../uploadista-error-BB-Wdiz9.cjs`),require(`../errors-CRm1FHHT.cjs`),require(`../types-BT-cvi7T.cjs`);const e=require(`../upload-DLuICjpP.cjs`);require(`../generate-id-xHp_Z7Cl.cjs`),require(`../stream-limiter-JgOwmkMa.cjs`),exports.UploadServer=e.UploadServer,exports.UploadStrategyNegotiator=e.UploadStrategyNegotiator,exports.createUploadServer=e.createUploadServer,exports.detectMimeType=e.detectMimeType,exports.uploadServer=e.uploadServer;

package/dist/upload/index.d.cts

@@ -0,0 +1,6 @@
+import "../uploadista-error-kKlhLRhY.cjs";
+import "../index-BtBZHVmz.cjs";
+import { NegotiatedStrategy, UploadServer, UploadServerOptions, UploadServerShape, UploadStrategyNegotiator, UploadStrategyOptions, createUploadServer, detectMimeType, uploadServer } from "../index-BO8GZlbD.cjs";
+import "../generate-id-LjJRLD6N.cjs";
+import "../semaphore-BHprIjFI.cjs";
+export { NegotiatedStrategy, UploadServer, UploadServerOptions, UploadServerShape, UploadStrategyNegotiator, UploadStrategyOptions, createUploadServer, detectMimeType, uploadServer };

package/dist/upload/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./mime";
+export * from "./upload-server";
+export * from "./upload-strategy-negotiator";
+//# sourceMappingURL=index.d.ts.map

package/dist/upload/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/upload/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,iBAAiB,CAAC;AAChC,cAAc,8BAA8B,CAAC"}

package/dist/upload/index.js

@@ -0,0 +1,3 @@
+export * from "./mime";
+export * from "./upload-server";
+export * from "./upload-strategy-negotiator";

package/dist/upload/mime.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Detect MIME type from buffer using magic bytes (file signatures).
+ * Supports a wide range of common file types including images, videos, audio, documents, and archives.
+ *
+ * @param buffer - File content as Uint8Array
+ * @param filename - Optional filename for extension-based fallback
+ * @returns Detected MIME type or "application/octet-stream" if unknown
+ */
+export declare const detectMimeType: (buffer: Uint8Array, filename?: string) => string;
+/**
+ * Compare two MIME types with lenient matching.
+ * Matches on major type (e.g., "image/*") to allow for minor variations.
+ *
+ * @param declared - MIME type provided by client
+ * @param detected - MIME type detected from file content
+ * @returns true if MIME types are compatible
+ *
+ * @example
+ * compareMimeTypes("image/png", "image/apng") // true
+ * compareMimeTypes("image/jpeg", "image/png") // true (both images)
+ * compareMimeTypes("image/png", "application/pdf") // false
+ */
+export declare function compareMimeTypes(declared: string, detected: string): boolean;
+//# sourceMappingURL=mime.d.ts.map

package/dist/upload/mime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mime.d.ts","sourceRoot":"","sources":["../../src/upload/mime.ts"],"names":[],"mappings":"AA2BA;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,GACzB,QAAQ,UAAU,EAClB,WAAW,MAAM,KAChB,MAkXF,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAY5E"}