@uploadista/data-store-s3 0.0.3

package/dist/observability.js

@@ -0,0 +1,155 @@
+import { Context, Effect, Layer, Metric, MetricBoundaries } from "effect";
+// ============================================================================
+// Metrics Definition
+// ============================================================================
+// Counter metrics
+export const uploadRequestsTotal = Metric.counter("s3_upload_requests_total", {
+    description: "Total number of upload requests",
+});
+export const uploadPartsTotal = Metric.counter("s3_upload_parts_total", {
+    description: "Total number of individual parts uploaded",
+});
+export const uploadSuccessTotal = Metric.counter("s3_upload_success_total", {
+    description: "Total number of successful uploads",
+});
+export const uploadErrorsTotal = Metric.counter("s3_upload_errors_total", {
+    description: "Total number of upload errors",
+});
+export const s3ApiCallsTotal = Metric.counter("s3_api_calls_total", {
+    description: "Total number of S3 API calls",
+});
+// Histogram metrics for timing and sizes
+export const uploadDurationHistogram = Metric.histogram("s3_upload_duration_seconds", MetricBoundaries.exponential({
+    start: 0.01, // 10ms
+    factor: 2,
+    count: 20, // Up to ~10 seconds
+}), "Duration of upload operations in seconds");
+export const partUploadDurationHistogram = Metric.histogram("s3_part_upload_duration_seconds", MetricBoundaries.exponential({
+    start: 0.001, // 1ms
+    factor: 2,
+    count: 15, // Up to ~32 seconds
+}), "Duration of individual part uploads in seconds");
+export const fileSizeHistogram = Metric.histogram("s3_file_size_bytes", MetricBoundaries.exponential({
+    start: 1024, // 1KB
+    factor: 2,
+    count: 25, // Up to ~33GB
+}), "Size of uploaded files in bytes");
+export const partSizeHistogram = Metric.histogram("s3_part_size_bytes", MetricBoundaries.linear({
+    start: 5_242_880, // 5MB (minimum part size)
+    width: 1_048_576, // 1MB increments
+    count: 20, // Up to ~25MB
+}), "Size of upload parts in bytes");
+// Gauge metrics for current state
+export const activeUploadsGauge = Metric.gauge("s3_active_uploads", {
+    description: "Number of currently active uploads",
+});
+export const uploadThroughputGauge = Metric.gauge("s3_upload_throughput_bytes_per_second", {
+    description: "Current upload throughput in bytes per second",
+});
+// Summary metrics for percentiles
+export const uploadLatencySummary = Metric.summary({
+    name: "s3_upload_latency_seconds",
+    maxAge: "10 minutes",
+    maxSize: 1000,
+    error: 0.01,
+    quantiles: [0.5, 0.9, 0.95, 0.99],
+    description: "Upload latency percentiles",
+});
+// ============================================================================
+// Tracing Configuration
+// ============================================================================
+// Create a service tag for tracing context
+export const TracingService = Context.GenericTag("TracingService");
+// Create a simple tracing layer using Effect's native tracing (environment-agnostic)
+export const createTracingLayer = (options) => {
+    const serviceName = options?.serviceName ?? "uploadista-s3-store";
+    // Return a layer that provides tracing service context
+    return Layer.succeed(TracingService, { serviceName });
+};
+// Default tracing layer for development
+export const TracingLayerLive = createTracingLayer({
+    serviceName: "uploadista-s3-store",
+});
+// ============================================================================
+// Observability Layer
+// ============================================================================
+export const ObservabilityLayer = Layer.mergeAll(TracingLayerLive
+// Effect's native metrics are automatically available
+);
+// ============================================================================
+// Utility Functions
+// ============================================================================
+export const withUploadMetrics = (uploadId, effect) => effect.pipe(Effect.tap(() => uploadRequestsTotal.pipe(Metric.tagged("upload_id", uploadId))(Effect.succeed(1))), Effect.tapError(() => uploadErrorsTotal.pipe(Metric.tagged("upload_id", uploadId))(Effect.succeed(1))), Effect.tap(() => uploadSuccessTotal.pipe(Metric.tagged("upload_id", uploadId))(Effect.succeed(1))));
+export const withS3ApiMetrics = (operation, effect) => effect.pipe(Effect.tap(() => s3ApiCallsTotal.pipe(Metric.tagged("operation", operation))(Effect.succeed(1))));
+export const withTimingMetrics = (metric, effect) => Effect.gen(function* () {
+    const startTime = yield* Effect.sync(() => Date.now());
+    const result = yield* effect;
+    const endTime = yield* Effect.sync(() => Date.now());
+    const duration = (endTime - startTime) / 1000; // Convert to seconds
+    yield* metric(Effect.succeed(duration));
+    return result;
+});
+export const classifyS3Error = (error) => {
+    if (!error || typeof error !== 'object')
+        return "unknown_error";
+    const errorCode = 'code' in error ? error.code : undefined;
+    if (!errorCode)
+        return "unknown_error";
+    switch (errorCode) {
+        case "NetworkError":
+        case "ECONNRESET":
+        case "ENOTFOUND":
+        case "ETIMEDOUT":
+            return "network_error";
+        case "InvalidAccessKeyId":
+        case "SignatureDoesNotMatch":
+        case "TokenRefreshRequired":
+            return "authentication_error";
+        case "AccessDenied":
+        case "AccountProblem":
+            return "authorization_error";
+        case "SlowDown":
+        case "RequestTimeTooSkewed":
+            return "throttling_error";
+        case "InternalError":
+        case "ServiceUnavailable":
+            return "server_error";
+        case "InvalidRequest":
+        case "MalformedXML":
+        case "RequestEntityTooLarge":
+            return "client_error";
+        default:
+            return "unknown_error";
+    }
+};
+export const trackS3Error = (operation, error, context = {}) => Effect.gen(function* () {
+    const errorCategory = classifyS3Error(error);
+    yield* uploadErrorsTotal.pipe(Metric.tagged("operation", operation), Metric.tagged("error_category", errorCategory))(Effect.succeed(1));
+    const errorDetails = {
+        operation,
+        error_category: errorCategory,
+        error_type: typeof error,
+        error_message: error instanceof Error ? error.message : String(error),
+        error_code: error && typeof error === 'object' && 'code' in error ? error.code : undefined,
+        ...context,
+    };
+    yield* Effect.logError(`S3 ${operation} failed`).pipe(Effect.annotateLogs(errorDetails));
+});
+// ============================================================================
+// Enhanced Logging Helpers
+// ============================================================================
+export const logWithContext = (message, context) => Effect.log(message).pipe(Effect.annotateLogs(context));
+export const logUploadProgress = (uploadId, progress) => logWithContext("Upload progress", {
+    upload_id: uploadId,
+    uploaded_bytes: progress.uploadedBytes,
+    total_bytes: progress.totalBytes,
+    progress_percentage: Math.round((progress.uploadedBytes / progress.totalBytes) * 100),
+    ...(progress.partNumber && { part_number: progress.partNumber }),
+    ...(progress.speed && { upload_speed_bps: progress.speed }),
+});
+export const logS3Operation = (operation, uploadId, metadata) => logWithContext(`S3 ${operation}`, {
+    operation,
+    upload_id: uploadId,
+    ...metadata,
+});
+export const logS3Error = (operation, error, context = {}) => trackS3Error(operation, error, context);

package/dist/s3-store-old.d.ts

@@ -0,0 +1,51 @@
+import type { S3ClientConfig } from "@aws-sdk/client-s3";
+import { UploadistaError } from "@uploadista/core/errors";
+import type { DataStore, KvStore, UploadFile } from "@uploadista/core/types";
+import { UploadFileDataStore, UploadFileKVStore } from "@uploadista/core/types";
+import { Effect, Layer } from "effect";
+export type S3StoreOptions = {
+    deliveryUrl: string;
+    /**
+     * The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
+     * The server calculates the optimal part size, which takes this size into account,
+     * but may increase it to not exceed the S3 10K parts limit.
+     */
+    partSize?: number;
+    /**
+     * The minimal part size for parts.
+     * Can be used to ensure that all non-trailing parts are exactly the same size.
+     * Can not be lower than 5MiB or more than 5GiB.
+     */
+    minPartSize?: number;
+    /**
+     * The maximum number of parts allowed in a multipart upload. Defaults to 10,000.
+     */
+    maxMultipartParts?: number;
+    useTags?: boolean;
+    maxConcurrentPartUploads?: number;
+    expirationPeriodInMilliseconds?: number;
+    s3ClientConfig: S3ClientConfig & {
+        bucket: string;
+    };
+};
+export type S3StoreFactoryOptions = S3StoreOptions;
+export type S3Store = DataStore<UploadFile> & {
+    getUpload: (id: string) => Effect.Effect<UploadFile, UploadistaError>;
+    read: (id: string) => Effect.Effect<ReadableStream, UploadistaError>;
+    getChunkerConstraints: () => {
+        minChunkSize: number;
+        maxChunkSize: number;
+        optimalChunkSize: number;
+        requiresOrderedChunks: boolean;
+    };
+};
+export declare function createS3Storexx(options: Omit<S3StoreOptions, "kvStore">): Effect.Effect<S3Store, never, UploadFileKVStore>;
+declare function createS3StoreImplementation({ deliveryUrl, partSize, minPartSize, useTags, maxMultipartParts, kvStore, maxConcurrentPartUploads, expirationPeriodInMilliseconds, // 1 week
+s3ClientConfig: { bucket, ...restS3ClientConfig }, }: S3StoreOptions & {
+    kvStore: KvStore<UploadFile>;
+}): S3Store;
+export declare const createS3Store: (options: S3StoreFactoryOptions) => Effect.Effect<S3Store, never, UploadFileKVStore>;
+export declare const S3StoreLayer: (options: Omit<S3StoreOptions, "kvStore">) => Layer.Layer<UploadFileDataStore, never, UploadFileKVStore>;
+export declare const s3Store: typeof createS3StoreImplementation;
+export {};
+//# sourceMappingURL=s3-store-old.d.ts.map

package/dist/s3-store-old.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3-store-old.d.ts","sourceRoot":"","sources":["../src/s3-store-old.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEzD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,KAAK,EACV,SAAS,EAGT,OAAO,EACP,UAAU,EAEX,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAgBhF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAyB,MAAM,QAAQ,CAAC;AAiF9D,MAAM,MAAM,cAAc,GAAG;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,8BAA8B,CAAC,EAAE,MAAM,CAAC;IAExC,cAAc,EAAE,cAAc,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,cAAc,CAAC;AAQnD,MAAM,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,GAAG;IAC5C,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IACtE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;IACrE,qBAAqB,EAAE,MAAM;QAC3B,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,qBAAqB,EAAE,OAAO,CAAC;KAChC,CAAC;CACH,CAAC;AAEF,wBAAgB,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,oDAKvE;AAGD,iBAAS,2BAA2B,CAAC,EACnC,WAAW,EACX,QAAQ,EACR,WAAuB,EACvB,OAAc,EACd,iBAA0B,EAC1B,OAAO,EACP,wBAA6B,EAC7B,8BAAwD,EAAE,SAAS;AACnE,cAAc,EAAE,EAAE,MAAM,EAAE,GAAG,kBAAkB,EAAE,GAClD,EAAE,cAAc,GAAG;IAClB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAC9B,GAAG,OAAO,CAw9BV;AAGD,eAAO,MAAM,aAAa,GAAI,SAAS,qBAAqB,qDAQxD,CAAC;AAGL,eAAO,MAAM,YAAY,GAAI,SAAS,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,+DACV,CAAC;AAG5D,eAAO,MAAM,OAAO,oCAA8B,CAAC"}