@uploadista/observability 0.0.3

package/src/core/logging.ts

@@ -0,0 +1,81 @@
+import { Effect } from "effect";
+
+// ============================================================================
+// Enhanced Logging Helpers (Storage-agnostic)
+// ============================================================================
+
+export const logWithContext = (
+  message: string,
+  context: Record<string, unknown>,
+) => Effect.log(message).pipe(Effect.annotateLogs(context));
+
+export const logUploadProgress = (
+  storageType: string,
+  uploadId: string,
+  progress: {
+    uploadedBytes: number;
+    totalBytes: number;
+    partNumber?: number;
+    speed?: number;
+  },
+) =>
+  logWithContext("Upload progress", {
+    storage_type: storageType,
+    upload_id: uploadId,
+    uploaded_bytes: progress.uploadedBytes,
+    total_bytes: progress.totalBytes,
+    progress_percentage: Math.round(
+      (progress.uploadedBytes / progress.totalBytes) * 100,
+    ),
+    ...(progress.partNumber && { part_number: progress.partNumber }),
+    ...(progress.speed && { upload_speed_bps: progress.speed }),
+  });
+
+export const logStorageOperation = (
+  storageType: string,
+  operation: string,
+  uploadId: string,
+  metadata?: Record<string, unknown>,
+) =>
+  logWithContext(`${storageType.toUpperCase()} ${operation}`, {
+    storage_type: storageType,
+    operation,
+    upload_id: uploadId,
+    ...metadata,
+  });
+
+export const logUploadCompletion = (
+  storageType: string,
+  uploadId: string,
+  metrics: {
+    fileSize: number;
+    totalDurationMs: number;
+    partsCount?: number;
+    averagePartSize?: number;
+    throughputBps?: number;
+    retryCount?: number;
+  },
+) => {
+  const throughputMBps = metrics.throughputBps
+    ? metrics.throughputBps / (1024 * 1024)
+    : 0;
+
+  return logWithContext(`${storageType.toUpperCase()} upload completed`, {
+    storage_type: storageType,
+    upload_id: uploadId,
+    file_size_bytes: metrics.fileSize,
+    file_size_mb: Math.round((metrics.fileSize / (1024 * 1024)) * 100) / 100,
+    total_duration_ms: metrics.totalDurationMs,
+    total_duration_seconds:
+      Math.round((metrics.totalDurationMs / 1000) * 100) / 100,
+    throughput_bps: metrics.throughputBps,
+    throughput_mbps: Math.round(throughputMBps * 100) / 100,
+    ...(metrics.partsCount && { parts_count: metrics.partsCount }),
+    ...(metrics.averagePartSize && {
+      average_part_size_bytes: metrics.averagePartSize,
+      average_part_size_mb:
+        Math.round((metrics.averagePartSize / (1024 * 1024)) * 100) / 100,
+    }),
+    ...(metrics.retryCount && { retry_count: metrics.retryCount }),
+  });
+};

package/src/core/metrics.ts

@@ -0,0 +1,108 @@
+import { Metric, MetricBoundaries } from "effect";
+
+// ============================================================================
+// Core Storage Metrics (reusable across all storage types)
+// ============================================================================
+
+// Counter metrics
+export const createUploadMetrics = (storageType: string) => ({
+  uploadRequestsTotal: Metric.counter(`${storageType}_upload_requests_total`, {
+    description: `Total number of upload requests for ${storageType}`,
+  }),
+
+  uploadPartsTotal: Metric.counter(`${storageType}_upload_parts_total`, {
+    description: `Total number of individual parts uploaded for ${storageType}`,
+  }),
+
+  uploadSuccessTotal: Metric.counter(`${storageType}_upload_success_total`, {
+    description: `Total number of successful uploads for ${storageType}`,
+  }),
+
+  uploadErrorsTotal: Metric.counter(`${storageType}_upload_errors_total`, {
+    description: `Total number of upload errors for ${storageType}`,
+  }),
+
+  apiCallsTotal: Metric.counter(`${storageType}_api_calls_total`, {
+    description: `Total number of API calls for ${storageType}`,
+  }),
+});
+
+// Histogram metrics for timing and sizes (reusable)
+export const createUploadHistograms = (storageType: string) => ({
+  uploadDurationHistogram: Metric.histogram(
+    `${storageType}_upload_duration_seconds`,
+    MetricBoundaries.exponential({
+      start: 0.01, // 10ms
+      factor: 2,
+      count: 20, // Up to ~10 seconds
+    }),
+    `Duration of upload operations in seconds for ${storageType}`,
+  ),
+
+  partUploadDurationHistogram: Metric.histogram(
+    `${storageType}_part_upload_duration_seconds`,
+    MetricBoundaries.exponential({
+      start: 0.001, // 1ms
+      factor: 2,
+      count: 15, // Up to ~32 seconds
+    }),
+    `Duration of individual part uploads in seconds for ${storageType}`,
+  ),
+
+  fileSizeHistogram: Metric.histogram(
+    `${storageType}_file_size_bytes`,
+    MetricBoundaries.exponential({
+      start: 1024, // 1KB
+      factor: 2,
+      count: 25, // Up to ~33GB
+    }),
+    `Size of uploaded files in bytes for ${storageType}`,
+  ),
+
+  partSizeHistogram: Metric.histogram(
+    `${storageType}_part_size_bytes`,
+    MetricBoundaries.linear({
+      start: 5_242_880, // 5MB (minimum part size)
+      width: 1_048_576, // 1MB increments
+      count: 20, // Up to ~25MB
+    }),
+    `Size of upload parts in bytes for ${storageType}`,
+  ),
+});
+
+// Gauge metrics for current state (reusable)
+export const createUploadGauges = (storageType: string) => ({
+  activeUploadsGauge: Metric.gauge(`${storageType}_active_uploads`, {
+    description: `Number of currently active uploads for ${storageType}`,
+  }),
+
+  uploadThroughputGauge: Metric.gauge(
+    `${storageType}_upload_throughput_bytes_per_second`,
+    {
+      description: `Current upload throughput in bytes per second for ${storageType}`,
+    },
+  ),
+});
+
+// Summary metrics for percentiles (reusable)
+export const createUploadSummaries = (storageType: string) => ({
+  uploadLatencySummary: Metric.summary({
+    name: `${storageType}_upload_latency_seconds`,
+    maxAge: "10 minutes",
+    maxSize: 1000,
+    error: 0.01,
+    quantiles: [0.5, 0.9, 0.95, 0.99],
+    description: `Upload latency percentiles for ${storageType}`,
+  }),
+});
+
+// Combined metrics factory
+export const createStorageMetrics = (storageType: string) => ({
+  ...createUploadMetrics(storageType),
+  ...createUploadHistograms(storageType),
+  ...createUploadGauges(storageType),
+  ...createUploadSummaries(storageType),
+});
+
+// Type for storage metrics
+export type StorageMetrics = ReturnType<typeof createStorageMetrics>;

package/src/core/testing.ts

@@ -0,0 +1,142 @@
+import { Effect, Layer, Metric } from "effect";
+import type {
+  FlowObservabilityService,
+  StorageObservabilityService,
+  UploadObservabilityService,
+} from "./layers.js";
+import {
+  FlowObservability,
+  StorageObservability,
+  UploadObservability,
+} from "./layers.js";
+import { createStorageMetrics } from "./metrics.js";
+
+// ============================================================================
+// Test Observability Layers
+// ============================================================================
+
+/**
+ * Mock storage observability for testing
+ */
+export const makeTestStorageObservability = (
+  storageType: string,
+): Layer.Layer<StorageObservability> => {
+  const metrics = createStorageMetrics(storageType);
+  const service: StorageObservabilityService = {
+    serviceName: `test-${storageType}-store`,
+    storageType,
+    metrics,
+    enabled: true,
+  };
+  return Layer.succeed(StorageObservability, service);
+};
+
+/**
+ * Mock upload observability for testing
+ */
+export const makeTestUploadObservability =
+  (): Layer.Layer<UploadObservability> => {
+    const service: UploadObservabilityService = {
+      serviceName: "test-upload-server",
+      enabled: true,
+      metrics: {
+        uploadCreated: Effect.void,
+        uploadCompleted: Effect.void,
+        uploadFailed: Effect.void,
+        chunkUploaded: Effect.void,
+      },
+    };
+    return Layer.succeed(UploadObservability, service);
+  };
+
+/**
+ * Mock flow observability for testing
+ */
+export const makeTestFlowObservability = (): Layer.Layer<FlowObservability> => {
+  const service: FlowObservabilityService = {
+    serviceName: "test-flow-engine",
+    enabled: true,
+    metrics: {
+      flowStarted: Effect.void,
+      flowCompleted: Effect.void,
+      flowFailed: Effect.void,
+      nodeExecuted: Effect.void,
+    },
+  };
+  return Layer.succeed(FlowObservability, service);
+};
+
+// ============================================================================
+// Test Utilities
+// ============================================================================
+
+/**
+ * Capture metrics snapshot from an effect for testing
+ * Note: Metric snapshots are simplified - for full metric testing,
+ * use Effect's built-in metric testing utilities
+ */
+export const captureMetrics = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  Effect.gen(function* () {
+    const result = yield* effect;
+    // Metrics are automatically captured by Effect runtime
+    yield* Metric.snapshot;
+    return result;
+  });
+
+/**
+ * Test helper to capture metrics around effect execution
+ * This is a simplified version - for production testing,
+ * use Effect's metric testing utilities
+ */
+export const withMetricTracking = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  Effect.gen(function* () {
+    // Track metric before execution
+    yield* Metric.snapshot;
+    const result = yield* effect;
+    // Track metric after execution
+    yield* Metric.snapshot;
+    return result;
+  });
+
+/**
+ * Test fixture for observability testing
+ */
+export interface ObservabilityTestFixture {
+  readonly storageObservability: Layer.Layer<StorageObservability>;
+  readonly uploadObservability: Layer.Layer<UploadObservability>;
+  readonly flowObservability: Layer.Layer<FlowObservability>;
+}
+
+/**
+ * Create a complete test fixture with all observability layers
+ */
+export const createTestFixture = (
+  storageType = "test-storage",
+): ObservabilityTestFixture => ({
+  storageObservability: makeTestStorageObservability(storageType),
+  uploadObservability: makeTestUploadObservability(),
+  flowObservability: makeTestFlowObservability(),
+});
+
+/**
+ * Run an effect with test observability layers
+ */
+export const runWithTestObservability = <A, E>(
+  effect: Effect.Effect<
+    A,
+    E,
+    StorageObservability | UploadObservability | FlowObservability
+  >,
+  storageType = "test-storage",
+): Effect.Effect<A, E> => {
+  const fixture = createTestFixture(storageType);
+  return effect.pipe(
+    Effect.provide(fixture.storageObservability),
+    Effect.provide(fixture.uploadObservability),
+    Effect.provide(fixture.flowObservability),
+  );
+};

package/src/core/tracing.ts

@@ -0,0 +1,67 @@
+import { NodeSdk, WebSdk } from "@effect/opentelemetry";
+import {
+  BatchSpanProcessor,
+  ConsoleSpanExporter,
+} from "@opentelemetry/sdk-trace-base";
+import { Context, Effect, Layer } from "effect";
+
+// ============================================================================
+// Universal Tracing (Environment-agnostic)
+// ============================================================================
+
+// Generic service tag for tracing context
+export const TracingService = Context.GenericTag<{ serviceName: string }>(
+  "TracingService",
+);
+
+// Create a tracing layer using Effect's native tracing (works in all environments)
+export const createTracingLayer = (options?: { serviceName?: string }) => {
+  const serviceName = options?.serviceName ?? "uploadista-storage";
+
+  // Return a layer that provides tracing service context
+  return Layer.succeed(TracingService, { serviceName });
+};
+
+// Storage-specific tracing layers
+export const createStorageTracingLayer = (storageType: string) =>
+  createTracingLayer({
+    serviceName: `uploadista-${storageType}-store`,
+  });
+
+// Utility to add storage context to spans
+export const withStorageSpan =
+  <A, E, R>(
+    operation: string,
+    storageType: string,
+    attributes?: Record<string, unknown>,
+  ) =>
+  (effect: Effect.Effect<A, E, R>) =>
+    effect.pipe(
+      Effect.withSpan(`${storageType}-${operation}`, {
+        attributes: {
+          "storage.type": storageType,
+          operation: operation,
+          ...attributes,
+        },
+      }),
+    );
+
+// Set up tracing with the OpenTelemetry SDK
+export const WebSdkLive = WebSdk.layer(() => ({
+  resource: { serviceName: "uploadista-storage" },
+  // Export span data to the console
+  spanProcessor: new BatchSpanProcessor(new ConsoleSpanExporter()),
+}));
+
+export const NodeSdkLive = NodeSdk.layer(() => ({
+  resource: { serviceName: "uploadista-storage" },
+  // Export span data to the console
+  spanProcessor: new BatchSpanProcessor(new ConsoleSpanExporter()),
+}));
+
+// Cloudflare Workers SDK (uses WebSdk as base)
+export const WorkersSdkLive = WebSdk.layer(() => ({
+  resource: { serviceName: "uploadista-storage-workers" },
+  // Export span data to the console in Workers environment
+  spanProcessor: new BatchSpanProcessor(new ConsoleSpanExporter()),
+}));

package/src/core/utilities.ts

@@ -0,0 +1,133 @@
+import { Effect, Metric } from "effect";
+import type { StorageMetrics } from "./metrics.js";
+
+// ============================================================================
+// Storage Observability Utility Functions
+// ============================================================================
+
+// Generic upload metrics wrapper
+export const withUploadMetrics = <A, E, R>(
+  metrics: StorageMetrics,
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  effect.pipe(
+    Effect.tap(() =>
+      metrics.uploadRequestsTotal.pipe(Metric.tagged("upload_id", uploadId))(
+        Effect.succeed(1),
+      ),
+    ),
+    Effect.tapError(() =>
+      metrics.uploadErrorsTotal.pipe(Metric.tagged("upload_id", uploadId))(
+        Effect.succeed(1),
+      ),
+    ),
+    Effect.tap(() =>
+      metrics.uploadSuccessTotal.pipe(Metric.tagged("upload_id", uploadId))(
+        Effect.succeed(1),
+      ),
+    ),
+  );
+
+// Generic API call metrics wrapper
+export const withApiMetrics = <A, E, R>(
+  metrics: StorageMetrics,
+  operation: string,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  effect.pipe(
+    Effect.tap(() =>
+      metrics.apiCallsTotal.pipe(Metric.tagged("operation", operation))(
+        Effect.succeed(1),
+      ),
+    ),
+  );
+
+// Generic timing metrics wrapper
+export const withTimingMetrics = <A, E, R>(
+  metric: Metric.Metric.Histogram<number>,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  Effect.gen(function* () {
+    const startTime = yield* Effect.sync(() => Date.now());
+    const result = yield* effect;
+    const endTime = yield* Effect.sync(() => Date.now());
+    const duration = (endTime - startTime) / 1000; // Convert to seconds
+
+    yield* metric(Effect.succeed(duration));
+
+    return result;
+  });
+
+// File size tracking
+export const trackFileSize = <A, E, R>(
+  metrics: StorageMetrics,
+  fileSize: number,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  effect.pipe(
+    Effect.tap(() => metrics.fileSizeHistogram(Effect.succeed(fileSize))),
+  );
+
+// Part size tracking
+export const trackPartSize = <A, E, R>(
+  metrics: StorageMetrics,
+  partSize: number,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  effect.pipe(
+    Effect.tap(() => metrics.partSizeHistogram(Effect.succeed(partSize))),
+  );
+
+// Active uploads tracking
+export const withActiveUploadTracking = <A, E, R>(
+  metrics: StorageMetrics,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  effect.pipe(
+    Effect.tap(() => metrics.activeUploadsGauge(Effect.succeed(1))),
+    Effect.ensuring(metrics.activeUploadsGauge(Effect.succeed(-1))),
+  );
+
+// Throughput calculation and tracking
+export const withThroughputTracking = <A, E, R>(
+  metrics: StorageMetrics,
+  bytes: number,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> =>
+  Effect.gen(function* () {
+    const startTime = yield* Effect.sync(() => Date.now());
+    const result = yield* effect;
+    const endTime = yield* Effect.sync(() => Date.now());
+    const durationSeconds = (endTime - startTime) / 1000;
+    const throughputBps = durationSeconds > 0 ? bytes / durationSeconds : 0;
+
+    yield* metrics.uploadThroughputGauge(Effect.succeed(throughputBps));
+
+    return result;
+  });
+
+// Combined metrics wrapper for common upload operations
+export const withStorageOperationMetrics = <A, E, R>(
+  metrics: StorageMetrics,
+  operation: string,
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+  fileSize?: number,
+): Effect.Effect<A, E, R> => {
+  let wrappedEffect = effect.pipe(
+    (eff) => withApiMetrics(metrics, operation, eff),
+    (eff) => withUploadMetrics(metrics, uploadId, eff),
+    (eff) => withTimingMetrics(metrics.uploadDurationHistogram, eff),
+    (eff) => withActiveUploadTracking(metrics, eff),
+  );
+
+  if (fileSize !== undefined) {
+    wrappedEffect = wrappedEffect.pipe(
+      (eff) => trackFileSize(metrics, fileSize, eff),
+      (eff) => withThroughputTracking(metrics, fileSize, eff),
+    );
+  }
+
+  return wrappedEffect;
+};

package/src/flow/errors.ts

@@ -0,0 +1,95 @@
+import { Effect, Metric } from "effect";
+import { createFlowMetrics } from "./metrics.js";
+
+// ============================================================================
+// Flow Error Classification
+// ============================================================================
+
+export type FlowErrorCategory =
+  | "flow_validation_error"
+  | "node_execution_error"
+  | "node_not_found_error"
+  | "flow_timeout_error"
+  | "flow_cancelled_error"
+  | "unknown_flow_error";
+
+/**
+ * Classify flow execution errors
+ */
+export const classifyFlowError = (error: unknown): FlowErrorCategory => {
+  if (!error || typeof error !== "object") return "unknown_flow_error";
+
+  const errorCode = "code" in error ? error.code : undefined;
+  if (!errorCode) return "unknown_flow_error";
+
+  // Flow-specific error codes
+  switch (errorCode) {
+    case "FLOW_VALIDATION_ERROR":
+    case "FLOW_INVALID_INPUT":
+    case "FLOW_INVALID_OUTPUT":
+      return "flow_validation_error";
+    case "FLOW_NODE_NOT_FOUND":
+    case "FLOW_EDGE_INVALID":
+      return "node_not_found_error";
+    case "FLOW_NODE_EXECUTION_FAILED":
+    case "FLOW_NODE_ERROR":
+      return "node_execution_error";
+    case "FLOW_TIMEOUT":
+      return "flow_timeout_error";
+    case "FLOW_CANCELLED":
+    case "ABORTED":
+      return "flow_cancelled_error";
+    default:
+      return "unknown_flow_error";
+  }
+};
+
+/**
+ * Track flow errors with classification
+ */
+export const trackFlowError = <E>(
+  error: E,
+): Effect.Effect<void, never, never> => {
+  const metrics = createFlowMetrics();
+  const category = classifyFlowError(error);
+
+  return Effect.gen(function* () {
+    // Increment total failed flows
+    yield* Metric.increment(metrics.flowFailedTotal);
+
+    // Log error with classification
+    yield* Effect.logError("Flow execution failed").pipe(
+      Effect.annotateLogs({
+        "error.category": category,
+        "error.message": String(error),
+      }),
+    );
+  });
+};
+
+/**
+ * Track node errors with classification
+ */
+export const trackNodeError = <E>(
+  nodeId: string,
+  nodeType: string,
+  error: E,
+): Effect.Effect<void, never, never> => {
+  const metrics = createFlowMetrics();
+  const category = classifyFlowError(error);
+
+  return Effect.gen(function* () {
+    // Increment node failed counter
+    yield* Metric.increment(metrics.nodeFailedTotal);
+
+    // Log error with node context
+    yield* Effect.logError("Node execution failed").pipe(
+      Effect.annotateLogs({
+        "node.id": nodeId,
+        "node.type": nodeType,
+        "error.category": category,
+        "error.message": String(error),
+      }),
+    );
+  });
+};

package/src/flow/index.ts

@@ -0,0 +1,17 @@
+// Flow observability exports
+export * from "./metrics.js";
+export * from "./tracing.js";
+export {
+  makeFlowObservabilityLive,
+  FlowObservabilityLive,
+  getFlowMetrics,
+  withFlowDuration,
+  withNodeDuration,
+  trackActiveFlow,
+  trackActiveNode,
+} from "./layers.js";
+export * from "./errors.js";
+export {
+  makeTestFlowObservability as makeTestFlowObservabilityUtil,
+  runWithTestFlowObservability,
+} from "./testing.js";