@uploadista/observability 0.0.3

package/src/flow/layers.ts

@@ -0,0 +1,131 @@
+import { Effect, Layer, Metric } from "effect";
+import {
+  FlowObservability,
+  makeFlowObservabilityLayer,
+} from "../core/layers.js";
+import { createFlowMetrics, type FlowMetrics } from "./metrics.js";
+
+// ============================================================================
+// Flow Observability Layer Implementation
+// ============================================================================
+
+/**
+ * Create a live flow observability layer with full metrics
+ */
+export const makeFlowObservabilityLive = (
+  serviceName = "uploadista-flow-engine",
+): Layer.Layer<FlowObservability> => {
+  const metrics = createFlowMetrics();
+
+  return Layer.succeed(FlowObservability, {
+    serviceName,
+    enabled: true,
+    metrics: {
+      flowStarted: Metric.increment(metrics.flowStartedTotal),
+      flowCompleted: Metric.increment(metrics.flowCompletedTotal),
+      flowFailed: Metric.increment(metrics.flowFailedTotal),
+      nodeExecuted: Metric.increment(metrics.nodeExecutedTotal),
+    },
+  });
+};
+
+/**
+ * Default live flow observability layer
+ */
+export const FlowObservabilityLive = makeFlowObservabilityLive();
+
+/**
+ * No-op flow observability layer (for testing or disabled observability)
+ */
+export const FlowObservabilityDisabled = makeFlowObservabilityLayer(false);
+
+/**
+ * Helper to get flow metrics from context
+ */
+export const getFlowMetrics = Effect.gen(function* () {
+  const obs = yield* FlowObservability;
+  return obs.metrics;
+});
+
+/**
+ * Helper to track flow duration
+ */
+export const withFlowDuration = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> => {
+  const metrics = createFlowMetrics();
+  return Effect.gen(function* () {
+    const startTime = Date.now();
+    const result = yield* effect;
+    const duration = (Date.now() - startTime) / 1000; // Convert to seconds
+    yield* Metric.update(metrics.flowDurationHistogram, duration);
+    yield* Metric.update(metrics.flowLatencySummary, duration);
+    return result;
+  }).pipe(Effect.withSpan("flow-execution"));
+};
+
+/**
+ * Helper to track node duration
+ */
+export const withNodeDuration = <A, E, R>(
+  nodeId: string,
+  nodeType: string,
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> => {
+  const metrics = createFlowMetrics();
+  return Effect.gen(function* () {
+    const startTime = Date.now();
+    const result = yield* effect;
+    const duration = (Date.now() - startTime) / 1000; // Convert to seconds
+    yield* Metric.update(metrics.nodeDurationHistogram, duration);
+    yield* Metric.update(metrics.nodeLatencySummary, duration);
+    return result;
+  }).pipe(
+    Effect.withSpan(`node-${nodeType}`, {
+      attributes: {
+        "node.id": nodeId,
+        "node.type": nodeType,
+      },
+    }),
+  );
+};
+
+/**
+ * Helper to track active flows
+ */
+export const trackActiveFlow = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> => {
+  const metrics = createFlowMetrics();
+  return Effect.gen(function* () {
+    // Increment active flows
+    yield* Metric.increment(metrics.activeFlowsGauge);
+
+    // Use acquireUseRelease for proper cleanup
+    return yield* Effect.acquireUseRelease(
+      Effect.void,
+      () => effect,
+      () => Metric.set(metrics.activeFlowsGauge, -1),
+    );
+  });
+};
+
+/**
+ * Helper to track active nodes
+ */
+export const trackActiveNode = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, E, R> => {
+  const metrics = createFlowMetrics();
+  return Effect.gen(function* () {
+    // Increment active nodes
+    yield* Metric.increment(metrics.activeNodesGauge);
+
+    // Use acquireUseRelease for proper cleanup
+    return yield* Effect.acquireUseRelease(
+      Effect.void,
+      () => effect,
+      () => Metric.set(metrics.activeNodesGauge, -1),
+    );
+  });
+};

package/src/flow/metrics.ts

@@ -0,0 +1,130 @@
+import { Metric, MetricBoundaries } from "effect";
+
+// ============================================================================
+// Flow Engine Metrics
+// ============================================================================
+
+/**
+ * Flow engine metrics for tracking flow execution operations
+ */
+export const createFlowMetrics = () => ({
+  // Counter metrics
+  flowStartedTotal: Metric.counter("flow_started_total", {
+    description: "Total number of flows started",
+  }),
+
+  flowCompletedTotal: Metric.counter("flow_completed_total", {
+    description: "Total number of flows completed successfully",
+  }),
+
+  flowFailedTotal: Metric.counter("flow_failed_total", {
+    description: "Total number of flows that failed",
+  }),
+
+  flowPausedTotal: Metric.counter("flow_paused_total", {
+    description: "Total number of flows that were paused",
+  }),
+
+  flowResumedTotal: Metric.counter("flow_resumed_total", {
+    description: "Total number of flows that were resumed",
+  }),
+
+  nodeExecutedTotal: Metric.counter("node_executed_total", {
+    description: "Total number of nodes executed",
+  }),
+
+  nodeSuccessTotal: Metric.counter("node_success_total", {
+    description: "Total number of nodes executed successfully",
+  }),
+
+  nodeFailedTotal: Metric.counter("node_failed_total", {
+    description: "Total number of nodes that failed",
+  }),
+
+  nodeSkippedTotal: Metric.counter("node_skipped_total", {
+    description: "Total number of nodes skipped (conditional)",
+  }),
+
+  // Histogram metrics
+  flowDurationHistogram: Metric.histogram(
+    "flow_duration_seconds",
+    MetricBoundaries.exponential({
+      start: 0.1, // 100ms
+      factor: 2,
+      count: 20, // Up to ~100 seconds
+    }),
+    "Duration of complete flow execution in seconds",
+  ),
+
+  nodeDurationHistogram: Metric.histogram(
+    "node_duration_seconds",
+    MetricBoundaries.exponential({
+      start: 0.01, // 10ms
+      factor: 2,
+      count: 18, // Up to ~26 seconds
+    }),
+    "Duration of individual node execution in seconds",
+  ),
+
+  flowNodeCountHistogram: Metric.histogram(
+    "flow_node_count",
+    MetricBoundaries.linear({
+      start: 1,
+      width: 5,
+      count: 20, // Up to 100 nodes
+    }),
+    "Number of nodes in a flow",
+  ),
+
+  parallelNodesHistogram: Metric.histogram(
+    "parallel_nodes_count",
+    MetricBoundaries.linear({
+      start: 1,
+      width: 2,
+      count: 15, // Up to 30 parallel nodes
+    }),
+    "Number of nodes executed in parallel",
+  ),
+
+  // Gauge metrics
+  activeFlowsGauge: Metric.gauge("active_flows", {
+    description: "Number of currently active flows",
+  }),
+
+  activeNodesGauge: Metric.gauge("active_nodes", {
+    description: "Number of currently executing nodes",
+  }),
+
+  pausedFlowsGauge: Metric.gauge("paused_flows", {
+    description: "Number of currently paused flows",
+  }),
+
+  // Summary metrics for latency percentiles
+  flowLatencySummary: Metric.summary({
+    name: "flow_latency_seconds",
+    maxAge: "10 minutes",
+    maxSize: 1000,
+    error: 0.01,
+    quantiles: [0.5, 0.9, 0.95, 0.99],
+    description: "Flow execution latency percentiles",
+  }),
+
+  nodeLatencySummary: Metric.summary({
+    name: "node_latency_seconds",
+    maxAge: "10 minutes",
+    maxSize: 1000,
+    error: 0.01,
+    quantiles: [0.5, 0.9, 0.95, 0.99],
+    description: "Node execution latency percentiles",
+  }),
+});
+
+/**
+ * Type for flow metrics
+ */
+export type FlowMetrics = ReturnType<typeof createFlowMetrics>;
+
+/**
+ * Default flow metrics instance
+ */
+export const flowMetrics = createFlowMetrics();

package/src/flow/testing.ts

@@ -0,0 +1,33 @@
+import { Effect, Layer } from "effect";
+import { FlowObservability } from "../core/layers.js";
+import type { FlowObservabilityService } from "../core/layers.js";
+
+// ============================================================================
+// Test Flow Observability Layers
+// ============================================================================
+
+/**
+ * Mock flow observability for testing
+ */
+export const makeTestFlowObservability = (): Layer.Layer<FlowObservability> => {
+  const service: FlowObservabilityService = {
+    serviceName: "test-flow-engine",
+    enabled: true,
+    metrics: {
+      flowStarted: Effect.void,
+      flowCompleted: Effect.void,
+      flowFailed: Effect.void,
+      nodeExecuted: Effect.void,
+    },
+  };
+  return Layer.succeed(FlowObservability, service);
+};
+
+/**
+ * Run an effect with test flow observability
+ */
+export const runWithTestFlowObservability = <A, E>(
+  effect: Effect.Effect<A, E, FlowObservability>,
+): Effect.Effect<A, E> => {
+  return effect.pipe(Effect.provide(makeTestFlowObservability()));
+};

package/src/flow/tracing.ts

@@ -0,0 +1,72 @@
+import { Effect } from "effect";
+
+// ============================================================================
+// Flow Tracing Utilities
+// ============================================================================
+
+/**
+ * Wrap an Effect with a flow operation span
+ */
+export const withFlowSpan =
+  <A, E, R>(operation: string, attributes?: Record<string, unknown>) =>
+  (effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    effect.pipe(
+      Effect.withSpan(`flow-${operation}`, {
+        attributes: {
+          "flow.operation": operation,
+          ...attributes,
+        },
+      }),
+    );
+
+/**
+ * Add flow context to the current span
+ */
+export const withFlowContext = (context: {
+  flowId?: string;
+  flowName?: string;
+  jobId?: string;
+  nodeCount?: number;
+  storageId?: string;
+}) =>
+  Effect.annotateCurrentSpan({
+    "flow.id": context.flowId ?? "unknown",
+    "flow.name": context.flowName ?? "unknown",
+    "flow.job_id": context.jobId ?? "unknown",
+    "flow.node_count": context.nodeCount?.toString() ?? "0",
+    "flow.storage_id": context.storageId ?? "unknown",
+  });
+
+/**
+ * Add node context to the current span
+ */
+export const withNodeContext = (context: {
+  nodeId: string;
+  nodeType: string;
+  nodeName?: string;
+  flowId?: string;
+  jobId?: string;
+}) =>
+  Effect.annotateCurrentSpan({
+    "node.id": context.nodeId,
+    "node.type": context.nodeType,
+    "node.name": context.nodeName ?? "unknown",
+    "node.flow_id": context.flowId ?? "unknown",
+    "node.job_id": context.jobId ?? "unknown",
+  });
+
+/**
+ * Add execution state context to the current span
+ */
+export const withExecutionContext = (context: {
+  executionOrder?: string[];
+  currentIndex?: number;
+  totalNodes?: number;
+  parallelCount?: number;
+}) =>
+  Effect.annotateCurrentSpan({
+    "execution.order": context.executionOrder?.join(",") ?? "",
+    "execution.current_index": context.currentIndex?.toString() ?? "0",
+    "execution.total_nodes": context.totalNodes?.toString() ?? "0",
+    "execution.parallel_count": context.parallelCount?.toString() ?? "0",
+  });

package/src/index.ts

@@ -0,0 +1,31 @@
+// Main observability package exports
+export * from "./core/index.js";
+// Tracing specific exports
+export { NodeSdkLive, WebSdkLive, WorkersSdkLive } from "./core/tracing.js";
+export * from "./flow/index.js";
+export * from "./service/metrics.js";
+export * from "./storage/index.js";
+// Convenience re-exports for common use cases
+export {
+  logS3Context,
+  logS3Operation,
+  logS3UploadCompletion,
+  logS3UploadProgress,
+  S3ObservabilityLayer,
+  S3TracingLayer,
+  s3ActiveUploadsGauge,
+  s3ApiCallsTotal,
+  s3FileSizeHistogram,
+  // S3 specific
+  s3Metrics,
+  s3UploadDurationHistogram,
+  // Individual S3 metrics
+  s3UploadRequestsTotal,
+  trackS3Error,
+  withS3ApiMetrics,
+  withS3OperationMetrics,
+  withS3Span,
+  withS3TimingMetrics,
+  withS3UploadMetrics,
+} from "./storage/s3.js";
+export * from "./upload/index.js";

package/src/service/metrics.ts

@@ -0,0 +1,31 @@
+import { Context, Effect, Layer } from "effect";
+
+/**
+ * Metrics Recording Service
+ *
+ * Provides access to metrics recording functionality throughout
+ * the upload and flow processing pipeline. The service is provided
+ * via Effect Layer and can be accessed using Effect.service().
+ */
+export class MetricsService extends Context.Tag("MetricsService")<
+  MetricsService,
+  {
+    /**
+     * Record upload metrics for an organization
+     */
+    readonly recordUpload: (
+      clientId: string,
+      bytes: number,
+      metadata?: Record<string, unknown>,
+    ) => Effect.Effect<void, never>;
+  }
+>() {}
+
+/**
+ * No-op implementation of MetricsService that does nothing.
+ * Used when metrics are disabled or database is not available.
+ */
+export const NoOpMetricsServiceLive: Layer.Layer<MetricsService> =
+  Layer.succeed(MetricsService, {
+    recordUpload: (_organizationId: string, _bytes: number) => Effect.void,
+  });

package/src/storage/azure.ts

@@ -0,0 +1,163 @@
+import { type Effect, Layer } from "effect";
+import {
+  createStorageErrorTracker,
+  type StorageErrorCategory,
+} from "../core/errors.js";
+import {
+  logStorageOperation,
+  logUploadCompletion,
+  logUploadProgress,
+  logWithContext,
+} from "../core/logging.js";
+import { createStorageMetrics, type StorageMetrics } from "../core/metrics.js";
+import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
+import {
+  withApiMetrics,
+  withStorageOperationMetrics,
+  withTimingMetrics,
+  withUploadMetrics,
+} from "../core/utilities.js";
+
+// ============================================================================
+// Azure Blob Storage-Specific Observability
+// ============================================================================
+
+const STORAGE_TYPE = "azure";
+
+// Azure-specific metrics
+export const azureMetrics = createStorageMetrics(STORAGE_TYPE);
+
+// Azure-specific tracing layer
+export const AzureTracingLayer = createStorageTracingLayer(STORAGE_TYPE);
+
+// Azure-specific error classification
+const classifyAzureError = (error: unknown): StorageErrorCategory | null => {
+  if (!error || typeof error !== "object") return null;
+
+  const errorCode =
+    "code" in error
+      ? error.code
+      : "statusCode" in error
+        ? error.statusCode
+        : undefined;
+  if (!errorCode) return null;
+
+  // Azure-specific error codes
+  switch (errorCode) {
+    case "BlobNotFound":
+    case "ContainerNotFound":
+    case "InvalidBlobOrBlock":
+      return "client_error";
+    case "ContainerAlreadyExists":
+    case "BlobAlreadyExists":
+      return "client_error";
+    case "InvalidBlockId":
+    case "InvalidBlockList":
+    case "InvalidBlobType":
+      return "client_error";
+    case "RequestBodyTooLarge":
+    case "InvalidHeaderValue":
+      return "client_error";
+    case "AuthenticationFailed":
+    case "InvalidAuthenticationInfo":
+      return "authentication_error";
+    case "AccountIsDisabled":
+      return "authorization_error";
+    case "InsufficientAccountPermissions":
+      return "authorization_error";
+    case "OperationTimedOut":
+    case "ServerBusy":
+    case "InternalError":
+      return "server_error";
+    default:
+      // Check for HTTP status codes
+      if (typeof errorCode === "number") {
+        if (errorCode >= 500) return "server_error";
+        if (errorCode === 429) return "throttling_error";
+        if (errorCode === 403) return "authorization_error";
+        if (errorCode === 401) return "authentication_error";
+        if (errorCode >= 400) return "client_error";
+      }
+      return null; // Fall back to generic classification
+  }
+};
+
+// Azure-specific error tracker
+export const trackAzureError = createStorageErrorTracker(
+  STORAGE_TYPE,
+  azureMetrics,
+  classifyAzureError,
+);
+
+// Azure-specific observability layer
+export const AzureObservabilityLayer = Layer.mergeAll(
+  AzureTracingLayer,
+  // Metrics are automatically available through Effect
+);
+
+// ============================================================================
+// Azure Utility Functions
+// ============================================================================
+
+export const withAzureUploadMetrics = <A, E, R>(
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+) => withUploadMetrics(azureMetrics, uploadId, effect);
+
+export const withAzureApiMetrics = <A, E, R>(
+  operation: string,
+  effect: Effect.Effect<A, E, R>,
+) => withApiMetrics(azureMetrics, operation, effect);
+
+export const withAzureTimingMetrics = withTimingMetrics;
+
+export const withAzureOperationMetrics = <A, E, R>(
+  operation: string,
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+  fileSize?: number,
+) =>
+  withStorageOperationMetrics(
+    azureMetrics,
+    operation,
+    uploadId,
+    effect,
+    fileSize,
+  );
+
+// Azure-specific span wrapper
+export const withAzureSpan =
+  <A, E, R>(operation: string, attributes?: Record<string, unknown>) =>
+  (effect: Effect.Effect<A, E, R>) =>
+    withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
+
+// Azure-specific logging functions
+export const logAzureOperation = logStorageOperation.bind(null, STORAGE_TYPE);
+export const logAzureUploadProgress = logUploadProgress.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logAzureUploadCompletion = logUploadCompletion.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logAzureContext = logWithContext;
+
+// Export metrics for external access
+export const {
+  uploadRequestsTotal: azureUploadRequestsTotal,
+  uploadPartsTotal: azureUploadPartsTotal,
+  uploadSuccessTotal: azureUploadSuccessTotal,
+  uploadErrorsTotal: azureUploadErrorsTotal,
+  apiCallsTotal: azureApiCallsTotal,
+  uploadDurationHistogram: azureUploadDurationHistogram,
+  partUploadDurationHistogram: azurePartUploadDurationHistogram,
+  fileSizeHistogram: azureFileSizeHistogram,
+  partSizeHistogram: azurePartSizeHistogram,
+  activeUploadsGauge: azureActiveUploadsGauge,
+  uploadThroughputGauge: azureUploadThroughputGauge,
+  uploadLatencySummary: azureUploadLatencySummary,
+} = azureMetrics;
+
+// Type exports
+export type AzureMetrics = StorageMetrics;