@uploadista/observability 0.0.3

package/src/upload/errors.ts

@@ -0,0 +1,166 @@
+import { Effect, Metric } from "effect";
+import type { UploadServerMetrics } from "./metrics.js";
+
+// ============================================================================
+// Upload Error Classification and Tracking
+// ============================================================================
+
+export type UploadErrorCategory =
+  | "network_error"
+  | "authentication_error"
+  | "authorization_error"
+  | "validation_error"
+  | "size_limit_error"
+  | "storage_error"
+  | "abort_error"
+  | "unknown_error";
+
+/**
+ * Classify upload errors into standard categories
+ */
+export const classifyUploadError = (error: unknown): UploadErrorCategory => {
+  if (!error || typeof error !== "object") return "unknown_error";
+
+  const errorCode = "code" in error ? error.code : undefined;
+  const errorName = "name" in error ? error.name : undefined;
+  const errorMessage =
+    error instanceof Error ? error.message.toLowerCase() : "";
+
+  // Abort errors
+  if (
+    errorCode === "ABORTED" ||
+    errorName === "AbortError" ||
+    errorMessage.includes("abort")
+  ) {
+    return "abort_error";
+  }
+
+  // Size limit errors
+  if (
+    errorCode === "FILE_TOO_LARGE" ||
+    errorCode === "LIMIT_FILE_SIZE" ||
+    errorCode === "RequestEntityTooLarge" ||
+    errorMessage.includes("too large") ||
+    errorMessage.includes("size limit") ||
+    errorMessage.includes("max size")
+  ) {
+    return "size_limit_error";
+  }
+
+  // Validation errors
+  if (
+    errorCode === "INVALID_FILE" ||
+    errorCode === "INVALID_METADATA" ||
+    errorCode === "VALIDATION_ERROR" ||
+    errorMessage.includes("validation") ||
+    errorMessage.includes("invalid")
+  ) {
+    return "validation_error";
+  }
+
+  // Network errors
+  if (
+    errorCode === "NetworkError" ||
+    errorCode === "ECONNRESET" ||
+    errorCode === "ENOTFOUND" ||
+    errorCode === "ETIMEDOUT" ||
+    errorMessage.includes("network") ||
+    errorMessage.includes("timeout")
+  ) {
+    return "network_error";
+  }
+
+  // Authentication errors
+  if (
+    errorCode === "UNAUTHORIZED" ||
+    errorCode === "AuthenticationFailed" ||
+    errorName === "AuthenticationError" ||
+    errorMessage.includes("authentication") ||
+    errorMessage.includes("unauthorized")
+  ) {
+    return "authentication_error";
+  }
+
+  // Authorization errors
+  if (
+    errorCode === "FORBIDDEN" ||
+    errorCode === "AccessDenied" ||
+    errorName === "AuthorizationError" ||
+    errorMessage.includes("forbidden") ||
+    errorMessage.includes("permission")
+  ) {
+    return "authorization_error";
+  }
+
+  // Storage errors
+  if (
+    errorCode === "FILE_WRITE_ERROR" ||
+    errorCode === "STORAGE_ERROR" ||
+    errorMessage.includes("storage") ||
+    errorMessage.includes("write error")
+  ) {
+    return "storage_error";
+  }
+
+  return "unknown_error";
+};
+
+/**
+ * Track upload errors with metrics and structured logging
+ */
+export const trackUploadError = (
+  metrics: UploadServerMetrics,
+  operation: string,
+  error: unknown,
+  context: Record<string, unknown> = {},
+) =>
+  Effect.gen(function* () {
+    const errorCategory = classifyUploadError(error);
+
+    // Record error metrics
+    const errorMetric = metrics.uploadFailedTotal.pipe(
+      Metric.tagged("operation", operation),
+      Metric.tagged("error_category", errorCategory),
+    );
+    yield* errorMetric(Effect.succeed(1));
+
+    // Create detailed error context
+    const errorDetails = {
+      operation,
+      error_category: errorCategory,
+      error_type: typeof error,
+      error_message: error instanceof Error ? error.message : String(error),
+      error_code:
+        error && typeof error === "object" && "code" in error
+          ? String(error.code)
+          : undefined,
+      error_name:
+        error && typeof error === "object" && "name" in error
+          ? String(error.name)
+          : undefined,
+      ...context,
+    };
+
+    // Log structured error
+    yield* Effect.logError(`Upload ${operation} failed`).pipe(
+      Effect.annotateLogs(errorDetails),
+    );
+  });
+
+/**
+ * Create a custom error classifier for upload operations
+ */
+export const createUploadErrorClassifier = (
+  customErrorMapping?: (error: unknown) => UploadErrorCategory | null,
+) => {
+  return (error: unknown): UploadErrorCategory => {
+    // Try custom mapping first
+    if (customErrorMapping) {
+      const customResult = customErrorMapping(error);
+      if (customResult !== null) return customResult;
+    }
+
+    // Fall back to generic classification
+    return classifyUploadError(error);
+  };
+};

package/src/upload/index.ts

@@ -0,0 +1,12 @@
+// Upload observability exports
+export * from "./metrics.js";
+export * from "./tracing.js";
+export {
+  makeUploadObservabilityLive,
+  UploadObservabilityLive,
+  getUploadMetrics,
+  withUploadDuration,
+  withChunkDuration,
+} from "./layers.js";
+export * from "./errors.js";
+export * from "./testing.js";

package/src/upload/layers.ts

@@ -0,0 +1,88 @@
+import { Effect, Layer, Metric } from "effect";
+import {
+  makeUploadObservabilityLayer,
+  UploadObservability,
+} from "../core/layers.js";
+import { createUploadServerMetrics } from "./metrics.js";
+
+// ============================================================================
+// Upload Observability Layer Implementation
+// ============================================================================
+
+/**
+ * Create a live upload observability layer with full metrics
+ */
+export const makeUploadObservabilityLive = (
+  serviceName = "uploadista-upload-server",
+): Layer.Layer<UploadObservability> => {
+  const metrics = createUploadServerMetrics();
+
+  return Layer.succeed(UploadObservability, {
+    serviceName,
+    enabled: true,
+    metrics: {
+      uploadCreated: Effect.succeed(metrics.uploadCreatedTotal).pipe(
+        Effect.flatMap((metric) => Metric.increment(metric)),
+      ),
+      uploadCompleted: Effect.succeed(metrics.uploadCompletedTotal).pipe(
+        Effect.flatMap((metric) => Metric.increment(metric)),
+      ),
+      uploadFailed: Effect.succeed(metrics.uploadFailedTotal).pipe(
+        Effect.flatMap((metric) => Metric.increment(metric)),
+      ),
+      chunkUploaded: Effect.succeed(metrics.chunkUploadedTotal).pipe(
+        Effect.flatMap((metric) => Metric.increment(metric)),
+      ),
+    },
+  });
+};
+
+/**
+ * Default live upload observability layer
+ */
+export const UploadObservabilityLive = makeUploadObservabilityLive();
+
+/**
+ * No-op upload observability layer (for testing or disabled observability)
+ */
+export const UploadObservabilityDisabled = makeUploadObservabilityLayer(false);
+
+/**
+ * Helper to get upload metrics from context
+ */
+export const getUploadMetrics = Effect.gen(function* () {
+  const obs = yield* UploadObservability;
+  return obs.metrics;
+});
+
+/**
+ * Helper to track upload duration
+ */
+export const withUploadDuration = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R | UploadObservability> => {
+  const metrics = createUploadServerMetrics();
+  return Effect.gen(function* () {
+    const startTime = Date.now();
+    const result = yield* effect;
+    const duration = (Date.now() - startTime) / 1000; // Convert to seconds
+    yield* Metric.update(metrics.uploadDurationHistogram, duration);
+    return result;
+  }).pipe(Effect.withSpan("upload-operation"));
+};
+
+/**
+ * Helper to track chunk upload duration
+ */
+export const withChunkDuration = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> => {
+  const metrics = createUploadServerMetrics();
+  return Effect.gen(function* () {
+    const startTime = Date.now();
+    const result = yield* effect;
+    const duration = (Date.now() - startTime) / 1000; // Convert to seconds
+    yield* Metric.update(metrics.chunkUploadDurationHistogram, duration);
+    return result;
+  }).pipe(Effect.withSpan("chunk-upload"));
+};

package/src/upload/metrics.ts

@@ -0,0 +1,118 @@
+import { Metric, MetricBoundaries } from "effect";
+
+// ============================================================================
+// Upload Server Metrics
+// ============================================================================
+
+/**
+ * Upload server metrics for tracking upload operations
+ */
+export const createUploadServerMetrics = () => ({
+  // Counter metrics
+  uploadCreatedTotal: Metric.counter("upload_created_total", {
+    description: "Total number of uploads created",
+  }),
+
+  uploadCompletedTotal: Metric.counter("upload_completed_total", {
+    description: "Total number of uploads completed successfully",
+  }),
+
+  uploadFailedTotal: Metric.counter("upload_failed_total", {
+    description: "Total number of uploads that failed",
+  }),
+
+  chunkUploadedTotal: Metric.counter("chunk_uploaded_total", {
+    description: "Total number of chunks uploaded",
+  }),
+
+  uploadFromUrlTotal: Metric.counter("upload_from_url_total", {
+    description: "Total number of URL-based uploads",
+  }),
+
+  uploadFromUrlSuccessTotal: Metric.counter("upload_from_url_success_total", {
+    description: "Total number of successful URL-based uploads",
+  }),
+
+  uploadFromUrlFailedTotal: Metric.counter("upload_from_url_failed_total", {
+    description: "Total number of failed URL-based uploads",
+  }),
+
+  // Histogram metrics
+  uploadDurationHistogram: Metric.histogram(
+    "upload_duration_seconds",
+    MetricBoundaries.exponential({
+      start: 0.01, // 10ms
+      factor: 2,
+      count: 20, // Up to ~10 seconds
+    }),
+    "Duration of complete upload operations in seconds",
+  ),
+
+  chunkUploadDurationHistogram: Metric.histogram(
+    "chunk_upload_duration_seconds",
+    MetricBoundaries.exponential({
+      start: 0.001, // 1ms
+      factor: 2,
+      count: 15, // Up to ~32 seconds
+    }),
+    "Duration of individual chunk uploads in seconds",
+  ),
+
+  uploadFileSizeHistogram: Metric.histogram(
+    "upload_file_size_bytes",
+    MetricBoundaries.exponential({
+      start: 1024, // 1KB
+      factor: 2,
+      count: 25, // Up to ~33GB
+    }),
+    "Size of uploaded files in bytes",
+  ),
+
+  chunkSizeHistogram: Metric.histogram(
+    "chunk_size_bytes",
+    MetricBoundaries.linear({
+      start: 262_144, // 256KB
+      width: 262_144, // 256KB increments
+      count: 20, // Up to ~5MB
+    }),
+    "Size of uploaded chunks in bytes",
+  ),
+
+  // Gauge metrics
+  activeUploadsGauge: Metric.gauge("active_uploads", {
+    description: "Number of currently active uploads",
+  }),
+
+  uploadThroughputGauge: Metric.gauge("upload_throughput_bytes_per_second", {
+    description: "Current upload throughput in bytes per second",
+  }),
+
+  // Summary metrics for latency percentiles
+  uploadLatencySummary: Metric.summary({
+    name: "upload_latency_seconds",
+    maxAge: "10 minutes",
+    maxSize: 1000,
+    error: 0.01,
+    quantiles: [0.5, 0.9, 0.95, 0.99],
+    description: "Upload operation latency percentiles",
+  }),
+
+  chunkLatencySummary: Metric.summary({
+    name: "chunk_latency_seconds",
+    maxAge: "10 minutes",
+    maxSize: 1000,
+    error: 0.01,
+    quantiles: [0.5, 0.9, 0.95, 0.99],
+    description: "Chunk upload latency percentiles",
+  }),
+});
+
+/**
+ * Type for upload server metrics
+ */
+export type UploadServerMetrics = ReturnType<typeof createUploadServerMetrics>;
+
+/**
+ * Default upload server metrics instance
+ */
+export const uploadServerMetrics = createUploadServerMetrics();

package/src/upload/testing.ts

@@ -0,0 +1,60 @@
+import { Layer } from "effect";
+import { UploadObservability } from "../core/layers.js";
+import { createUploadServerMetrics } from "./metrics.js";
+
+// ============================================================================
+// Upload Observability Testing Utilities
+// ============================================================================
+
+/**
+ * Create a test upload observability layer that doesn't actually emit metrics
+ * but validates that the observability system is wired correctly
+ */
+export const UploadObservabilityTest = Layer.succeed(UploadObservability, {
+  serviceName: "uploadista-upload-server-test",
+  enabled: true,
+  metrics: {
+    uploadCreated: () => Promise.resolve(),
+    uploadCompleted: () => Promise.resolve(),
+    uploadFailed: () => Promise.resolve(),
+    chunkUploaded: () => Promise.resolve(),
+  } as any,
+});
+
+/**
+ * Get metrics for validation (useful for testing metric definitions)
+ */
+export const getTestMetrics = () => createUploadServerMetrics();
+
+/**
+ * Validate that all required metrics exist
+ */
+export const validateMetricsExist = () => {
+  const metrics = getTestMetrics();
+
+  const requiredMetrics = [
+    "uploadCreatedTotal",
+    "uploadCompletedTotal",
+    "uploadFailedTotal",
+    "chunkUploadedTotal",
+    "uploadFromUrlTotal",
+    "uploadFromUrlSuccessTotal",
+    "uploadFromUrlFailedTotal",
+    "uploadDurationHistogram",
+    "chunkUploadDurationHistogram",
+    "uploadFileSizeHistogram",
+    "chunkSizeHistogram",
+    "activeUploadsGauge",
+    "uploadThroughputGauge",
+    "uploadLatencySummary",
+    "chunkLatencySummary",
+  ];
+
+  const missingMetrics = requiredMetrics.filter((name) => !(name in metrics));
+
+  if (missingMetrics.length > 0) {
+    throw new Error(`Missing required metrics: ${missingMetrics.join(", ")}`);
+  }
+
+  return true;
+};

package/src/upload/tracing.ts

@@ -0,0 +1,58 @@
+import { Effect } from "effect";
+
+// ============================================================================
+// Upload Tracing Utilities
+// ============================================================================
+
+/**
+ * Wrap an Effect with an upload operation span
+ */
+export const withUploadSpan =
+  <A, E, R>(operation: string, attributes?: Record<string, unknown>) =>
+  (effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    effect.pipe(
+      Effect.withSpan(`upload-${operation}`, {
+        attributes: {
+          "upload.operation": operation,
+          ...attributes,
+        },
+      }),
+    );
+
+/**
+ * Add upload context to the current span
+ */
+export const withUploadContext = (context: {
+  uploadId?: string;
+  fileName?: string;
+  fileSize?: number;
+  storageId?: string;
+  mimeType?: string;
+}) =>
+  Effect.annotateCurrentSpan({
+    "upload.id": context.uploadId ?? "unknown",
+    "upload.file_name": context.fileName ?? "unknown",
+    "upload.file_size": context.fileSize?.toString() ?? "0",
+    "upload.storage_id": context.storageId ?? "unknown",
+    "upload.mime_type": context.mimeType ?? "unknown",
+  });
+
+/**
+ * Add chunk context to the current span
+ */
+export const withChunkContext = (context: {
+  uploadId: string;
+  chunkSize: number;
+  offset: number;
+  totalSize?: number;
+}) =>
+  Effect.annotateCurrentSpan({
+    "chunk.upload_id": context.uploadId,
+    "chunk.size": context.chunkSize.toString(),
+    "chunk.offset": context.offset.toString(),
+    "chunk.total_size": context.totalSize?.toString() ?? "0",
+    "chunk.progress":
+      context.totalSize && context.totalSize > 0
+        ? ((context.offset / context.totalSize) * 100).toFixed(2)
+        : "0",
+  });

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "@uploadista/typescript-config/base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "types": ["node"]
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"root":["./src/index.ts","./src/test-observability.ts","./src/core/errors.ts","./src/core/index.ts","./src/core/layers.ts","./src/core/logging.ts","./src/core/metrics.ts","./src/core/testing.ts","./src/core/tracing.ts","./src/core/utilities.ts","./src/flow/errors.ts","./src/flow/index.ts","./src/flow/layers.ts","./src/flow/metrics.ts","./src/flow/testing.ts","./src/flow/tracing.ts","./src/service/metrics.ts","./src/storage/azure.ts","./src/storage/filesystem.ts","./src/storage/gcs.ts","./src/storage/index.ts","./src/storage/s3.ts","./src/upload/errors.ts","./src/upload/index.ts","./src/upload/layers.ts","./src/upload/metrics.ts","./src/upload/testing.ts","./src/upload/tracing.ts"],"version":"5.9.3"}