@uploadista/observability 0.0.18-beta.8 → 0.0.18

package/src/core/tracing.ts

@@ -1,9 +1,18 @@
 import { NodeSdk, WebSdk } from "@effect/opentelemetry";
+import { trace } from "@opentelemetry/api";
 import {
   BatchSpanProcessor,
   ConsoleSpanExporter,
+  type SpanProcessor,
 } from "@opentelemetry/sdk-trace-base";
-import { Context, Effect, Layer } from "effect";
+import { Context, Effect, Layer, Option, Tracer } from "effect";
+import {
+  createOtlpTraceExporter,
+  getServiceName,
+  isOtlpExportEnabled,
+  parseResourceAttributes,
+} from "./exporters.js";
+import type { TraceContext } from "./types.js";
 
 // ============================================================================
 // Universal Tracing (Environment-agnostic)
@@ -65,3 +74,410 @@ export const WorkersSdkLive = WebSdk.layer(() => ({
   // Export span data to the console in Workers environment
   spanProcessor: new BatchSpanProcessor(new ConsoleSpanExporter()),
 }));
+
+// ============================================================================
+// OTLP Export Layers (Production)
+// ============================================================================
+
+/**
+ * Configuration options for OTLP SDK layers.
+ */
+export interface OtlpSdkConfig {
+  /** Service name for traces. Defaults to OTEL_SERVICE_NAME or "uploadista" */
+  serviceName?: string;
+  /** Additional resource attributes to include in all spans */
+  resourceAttributes?: Record<string, string>;
+  /** Maximum queue size for batch processor. Defaults to 512 */
+  maxQueueSize?: number;
+  /** Maximum export batch size. Defaults to 512 */
+  maxExportBatchSize?: number;
+  /** Schedule delay in milliseconds. Defaults to 5000 */
+  scheduledDelayMillis?: number;
+  /** Export timeout in milliseconds. Defaults to 5000 */
+  exportTimeoutMillis?: number;
+}
+
+/**
+ * Creates a BatchSpanProcessor with OTLP exporter and graceful degradation.
+ *
+ * The processor is configured with:
+ * - Configurable queue limits to prevent memory issues
+ * - Export timeouts to prevent blocking
+ * - Error handling that drops data rather than failing requests
+ *
+ * @param config - Optional configuration
+ * @returns Configured BatchSpanProcessor
+ */
+function createOtlpSpanProcessor(config: OtlpSdkConfig = {}): SpanProcessor {
+  const exporter = createOtlpTraceExporter();
+
+  return new BatchSpanProcessor(exporter, {
+    maxQueueSize: config.maxQueueSize ?? 512,
+    maxExportBatchSize: config.maxExportBatchSize ?? 512,
+    scheduledDelayMillis: config.scheduledDelayMillis ?? 5000,
+    // Default to 30 seconds to accommodate cloud endpoints like Grafana Cloud
+    exportTimeoutMillis: config.exportTimeoutMillis ?? 30000,
+  });
+}
+
+/**
+ * Creates resource configuration from environment and config.
+ */
+function createResourceConfig(config: OtlpSdkConfig = {}): {
+  serviceName: string;
+  [key: string]: string;
+} {
+  const serviceName = config.serviceName ?? getServiceName("uploadista");
+  const envAttributes = parseResourceAttributes();
+  const configAttributes = config.resourceAttributes ?? {};
+
+  return {
+    serviceName,
+    ...envAttributes,
+    ...configAttributes,
+  };
+}
+
+/**
+ * Node.js OTLP SDK Layer for production use.
+ *
+ * Exports traces to an OTLP-compatible endpoint configured via environment variables:
+ * - OTEL_EXPORTER_OTLP_ENDPOINT: Base endpoint (default: http://localhost:4318)
+ * - OTEL_EXPORTER_OTLP_HEADERS: Authentication headers
+ * - OTEL_SERVICE_NAME: Service name (default: uploadista)
+ * - OTEL_RESOURCE_ATTRIBUTES: Additional resource attributes
+ * - UPLOADISTA_OBSERVABILITY_ENABLED: Set to "false" to disable (default: true)
+ *
+ * @example
+ * ```typescript
+ * import { OtlpNodeSdkLive } from "@uploadista/observability";
+ * import { Effect } from "effect";
+ *
+ * // With default environment configuration
+ * const program = myEffect.pipe(Effect.provide(OtlpNodeSdkLive));
+ *
+ * // Run with:
+ * // OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+ * // OTEL_SERVICE_NAME=my-upload-service
+ * ```
+ */
+export const OtlpNodeSdkLive = NodeSdk.layer(() => {
+  // Check if observability is disabled
+  if (!isOtlpExportEnabled()) {
+    // Return no-op configuration (no span processor means no export)
+    return {
+      resource: createResourceConfig(),
+    };
+  }
+
+  return {
+    resource: createResourceConfig(),
+    spanProcessor: createOtlpSpanProcessor(),
+  };
+});
+
+/**
+ * Creates a customized OTLP Node.js SDK Layer.
+ *
+ * Use this when you need to customize the SDK configuration beyond
+ * what environment variables provide.
+ *
+ * @param config - Custom configuration options
+ * @returns Configured Effect Layer
+ *
+ * @example
+ * ```typescript
+ * const customSdk = createOtlpNodeSdkLayer({
+ *   serviceName: "my-custom-service",
+ *   resourceAttributes: {
+ *     "tenant.id": "abc123",
+ *     "deployment.environment": "production"
+ *   },
+ *   maxQueueSize: 1024,
+ * });
+ *
+ * const program = myEffect.pipe(Effect.provide(customSdk));
+ * ```
+ */
+export function createOtlpNodeSdkLayer(config: OtlpSdkConfig = {}) {
+  return NodeSdk.layer(() => {
+    if (!isOtlpExportEnabled()) {
+      return {
+        resource: createResourceConfig(config),
+      };
+    }
+
+    return {
+      resource: createResourceConfig(config),
+      spanProcessor: createOtlpSpanProcessor(config),
+    };
+  });
+}
+
+/**
+ * Browser OTLP SDK Layer for production use.
+ *
+ * Similar to OtlpNodeSdkLive but uses fetch API for browser compatibility.
+ * Note: Browser environments may have CORS restrictions.
+ *
+ * @example
+ * ```typescript
+ * import { OtlpWebSdkLive } from "@uploadista/observability";
+ *
+ * const program = myEffect.pipe(Effect.provide(OtlpWebSdkLive));
+ * ```
+ */
+export const OtlpWebSdkLive = WebSdk.layer(() => {
+  if (!isOtlpExportEnabled()) {
+    return {
+      resource: createResourceConfig(),
+    };
+  }
+
+  return {
+    resource: createResourceConfig(),
+    spanProcessor: createOtlpSpanProcessor(),
+  };
+});
+
+/**
+ * Creates a customized OTLP Web SDK Layer.
+ *
+ * @param config - Custom configuration options
+ * @returns Configured Effect Layer for browser environments
+ */
+export function createOtlpWebSdkLayer(config: OtlpSdkConfig = {}) {
+  return WebSdk.layer(() => {
+    if (!isOtlpExportEnabled()) {
+      return {
+        resource: createResourceConfig(config),
+      };
+    }
+
+    return {
+      resource: createResourceConfig(config),
+      spanProcessor: createOtlpSpanProcessor(config),
+    };
+  });
+}
+
+/**
+ * Cloudflare Workers OTLP SDK Layer for production use.
+ *
+ * Uses the Web SDK under the hood with fetch-based export.
+ * Suitable for edge computing environments.
+ *
+ * @example
+ * ```typescript
+ * import { OtlpWorkersSdkLive } from "@uploadista/observability";
+ *
+ * export default {
+ *   async fetch(request, env) {
+ *     const program = handleRequest(request).pipe(
+ *       Effect.provide(OtlpWorkersSdkLive)
+ *     );
+ *     return Effect.runPromise(program);
+ *   }
+ * };
+ * ```
+ */
+export const OtlpWorkersSdkLive = WebSdk.layer(() => {
+  const config: OtlpSdkConfig = {
+    serviceName: getServiceName("uploadista-workers"),
+  };
+
+  if (!isOtlpExportEnabled()) {
+    return {
+      resource: createResourceConfig(config),
+    };
+  }
+
+  return {
+    resource: createResourceConfig(config),
+    spanProcessor: createOtlpSpanProcessor(config),
+  };
+});
+
+/**
+ * Creates a customized OTLP Workers SDK Layer.
+ *
+ * @param config - Custom configuration options
+ * @returns Configured Effect Layer for Cloudflare Workers
+ */
+export function createOtlpWorkersSdkLayer(config: OtlpSdkConfig = {}) {
+  return WebSdk.layer(() => {
+    const effectiveConfig = {
+      serviceName: getServiceName("uploadista-workers"),
+      ...config,
+    };
+
+    if (!isOtlpExportEnabled()) {
+      return {
+        resource: createResourceConfig(effectiveConfig),
+      };
+    }
+
+    return {
+      resource: createResourceConfig(effectiveConfig),
+      spanProcessor: createOtlpSpanProcessor(effectiveConfig),
+    };
+  });
+}
+
+// ============================================================================
+// Distributed Tracing Context Utilities
+// ============================================================================
+
+/**
+ * @deprecated Use `captureTraceContextEffect` instead. This synchronous function
+ * uses OpenTelemetry's `trace.getActiveSpan()` which may not be synchronized
+ * with Effect's span context when using @effect/opentelemetry.
+ *
+ * @returns TraceContext if there's an active OpenTelemetry span, undefined otherwise
+ */
+export function captureTraceContext(): TraceContext | undefined {
+  const currentSpan = trace.getActiveSpan();
+  if (!currentSpan) {
+    return undefined;
+  }
+
+  const spanContext = currentSpan.spanContext();
+  return {
+    traceId: spanContext.traceId,
+    spanId: spanContext.spanId,
+    traceFlags: spanContext.traceFlags,
+  };
+}
+
+/**
+ * Captures the current Effect trace context for distributed tracing.
+ *
+ * Uses Effect's `currentSpan` to get the active span, which is more reliable
+ * than OpenTelemetry's `trace.getActiveSpan()` when using @effect/opentelemetry
+ * because Effect manages its own span context that may not be synchronized
+ * with OpenTelemetry's global context.
+ *
+ * Use this to save the trace context (traceId, spanId, traceFlags) for later
+ * use in distributed tracing. The captured context can be stored alongside
+ * data (e.g., in KV store with upload metadata) and restored later using
+ * `createExternalSpan` and passing it to `Effect.withSpan`'s `parent` option.
+ *
+ * @returns Effect yielding TraceContext if there's an active span, undefined otherwise
+ *
+ * @example
+ * ```typescript
+ * // Capture context during upload creation
+ * const createUpload = Effect.gen(function* () {
+ *   const traceContext = yield* captureTraceContextEffect;
+ *
+ *   const file: UploadFile = {
+ *     id: uploadId,
+ *     traceContext, // Store for later
+ *     // ...
+ *   };
+ *   yield* kvStore.set(uploadId, file);
+ * }).pipe(Effect.withSpan("upload-create", { ... }));
+ * ```
+ */
+export const captureTraceContextEffect: Effect.Effect<
+  TraceContext | undefined
+> = Effect.gen(function* () {
+  const spanOption = yield* Effect.currentSpan.pipe(Effect.option);
+  return Option.match(spanOption, {
+    onNone: () => undefined,
+    onSome: (span) => ({
+      traceId: span.traceId,
+      spanId: span.spanId,
+      traceFlags: span.sampled ? 1 : 0,
+    }),
+  });
+});
+
+/**
+ * Creates an ExternalSpan from a stored trace context.
+ *
+ * Use this to create a parent span reference that can be passed to
+ * `Effect.withSpan`'s `parent` option for distributed tracing.
+ *
+ * **Important:** The parent must be passed directly to `Effect.withSpan`'s
+ * options, not provided as a service afterward.
+ *
+ * @param traceContext - Previously captured trace context
+ * @returns ExternalSpan that can be used as a parent in Effect.withSpan
+ *
+ * @example
+ * ```typescript
+ * // Create parent span from stored trace context
+ * const parentSpan = file.traceContext
+ *   ? createExternalSpan(file.traceContext)
+ *   : undefined;
+ *
+ * // Pass parent directly to withSpan
+ * const chunkEffect = Effect.gen(function* () {
+ *   // ... chunk upload logic
+ * }).pipe(
+ *   Effect.withSpan("upload-chunk", {
+ *     attributes: { ... },
+ *     parent: parentSpan,  // Link to original trace
+ *   })
+ * );
+ * ```
+ */
+export function createExternalSpan(traceContext: TraceContext) {
+  return Tracer.externalSpan({
+    traceId: traceContext.traceId,
+    spanId: traceContext.spanId,
+    sampled: traceContext.traceFlags === 1,
+  });
+}
+
+/**
+ * @deprecated Use `createExternalSpan` instead and pass the result to
+ * `Effect.withSpan`'s `parent` option directly. This function doesn't
+ * work correctly because Effect.withSpan reads the parent at construction
+ * time, not from the provided service.
+ *
+ * @example
+ * ```typescript
+ * // Instead of:
+ * withParentContext(traceContext)(effect.pipe(Effect.withSpan(...)))
+ *
+ * // Do this:
+ * const parent = createExternalSpan(traceContext);
+ * effect.pipe(Effect.withSpan("name", { parent }))
+ * ```
+ */
+export function withParentContext(traceContext: TraceContext) {
+  return <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> => {
+    const externalSpan = Tracer.externalSpan({
+      traceId: traceContext.traceId,
+      spanId: traceContext.spanId,
+      sampled: traceContext.traceFlags === 1,
+    });
+
+    return effect.pipe(Effect.provideService(Tracer.ParentSpan, externalSpan));
+  };
+}
+
+/**
+ * Checks if there's an active trace context.
+ *
+ * Useful for conditional logic based on whether tracing is active.
+ *
+ * @returns true if there's an active span with valid trace context
+ *
+ * @example
+ * ```typescript
+ * if (hasActiveTraceContext()) {
+ *   console.log("Tracing is active");
+ * }
+ * ```
+ */
+export function hasActiveTraceContext(): boolean {
+  const span = trace.getActiveSpan();
+  if (!span) return false;
+
+  const ctx = span.spanContext();
+  // Check if the trace ID is valid (not all zeros)
+  return ctx.traceId !== "00000000000000000000000000000000";
+}

package/src/core/types.ts

@@ -0,0 +1,44 @@
+/**
+ * OpenTelemetry trace context types for distributed tracing.
+ *
+ * These types enable trace context propagation across HTTP requests,
+ * allowing spans from multiple requests to be grouped under a single trace.
+ *
+ * @module observability/core/types
+ */
+
+/**
+ * Trace context for distributed tracing.
+ *
+ * This structure holds the essential OpenTelemetry trace context that needs
+ * to be persisted and propagated across requests to maintain trace continuity.
+ *
+ * @property traceId - 128-bit unique identifier for the entire trace (32 hex chars)
+ * @property spanId - 64-bit unique identifier for the parent span (16 hex chars)
+ * @property traceFlags - Sampling decision (1 = sampled, 0 = not sampled)
+ *
+ * @example
+ * ```typescript
+ * // Store trace context with upload metadata
+ * const traceContext: TraceContext = {
+ *   traceId: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4",
+ *   spanId: "a1b2c3d4e5f6a1b2",
+ *   traceFlags: 1
+ * };
+ *
+ * // Later, restore context to link spans
+ * if (uploadFile.traceContext) {
+ *   yield* withParentContext(uploadFile.traceContext)(
+ *     Effect.withSpan("upload-chunk", { ... })(chunkEffect)
+ *   );
+ * }
+ * ```
+ */
+export type TraceContext = {
+  /** 128-bit trace identifier (32 hex characters) */
+  traceId: string;
+  /** 64-bit span identifier (16 hex characters) */
+  spanId: string;
+  /** Trace flags (1 = sampled) */
+  traceFlags: number;
+};

package/src/flow/layers.ts

@@ -66,13 +66,20 @@ export const withFlowDuration = <A, E, R>(
 
 /**
  * Helper to track node duration
+ * @param nodeId - Unique node identifier
+ * @param nodeType - Generic node type (e.g., "optimize", "resize")
+ * @param effect - The effect to track
+ * @param nodeTypeId - Optional specific node type ID (e.g., "optimize-image", "resize-video")
  */
 export const withNodeDuration = <A, E, R>(
   nodeId: string,
   nodeType: string,
   effect: Effect.Effect<A, E, R>,
+  nodeTypeId?: string,
 ): Effect.Effect<A, E, R> => {
   const metrics = createFlowMetrics();
+  // Use nodeTypeId for span name if available, fallback to nodeType
+  const spanName = nodeTypeId ?? nodeType;
   return Effect.gen(function* () {
     const startTime = Date.now();
     const result = yield* effect;
@@ -81,10 +88,11 @@ export const withNodeDuration = <A, E, R>(
     yield* Metric.update(metrics.nodeLatencySummary, duration);
     return result;
   }).pipe(
-    Effect.withSpan(`node-${nodeType}`, {
+    Effect.withSpan(`node-${spanName}`, {
       attributes: {
         "node.id": nodeId,
         "node.type": nodeType,
+        "node.type_id": nodeTypeId ?? nodeType,
       },
     }),
   );

package/src/flow/metrics.ts

@@ -124,32 +124,44 @@ export const createFlowMetrics = () => ({
 
   /** Total number of times circuit breakers opened */
   circuitBreakerOpenTotal: Metric.counter("circuit_breaker_open_total", {
-    description: "Total number of times circuit breakers transitioned to open state",
+    description:
+      "Total number of times circuit breakers transitioned to open state",
   }),
 
   /** Total number of times circuit breakers closed */
   circuitBreakerCloseTotal: Metric.counter("circuit_breaker_close_total", {
-    description: "Total number of times circuit breakers transitioned to closed state",
+    description:
+      "Total number of times circuit breakers transitioned to closed state",
   }),
 
   /** Total number of requests rejected by open circuit breakers */
-  circuitBreakerRejectedTotal: Metric.counter("circuit_breaker_rejected_total", {
-    description: "Total number of requests rejected because circuit breaker is open",
-  }),
+  circuitBreakerRejectedTotal: Metric.counter(
+    "circuit_breaker_rejected_total",
+    {
+      description:
+        "Total number of requests rejected because circuit breaker is open",
+    },
+  ),
 
   /** Total number of times circuit breakers transitioned to half-open */
-  circuitBreakerHalfOpenTotal: Metric.counter("circuit_breaker_half_open_total", {
-    description: "Total number of times circuit breakers transitioned to half-open state",
-  }),
+  circuitBreakerHalfOpenTotal: Metric.counter(
+    "circuit_breaker_half_open_total",
+    {
+      description:
+        "Total number of times circuit breakers transitioned to half-open state",
+    },
+  ),
 
   /** Current state of circuit breakers (0=closed, 1=open, 2=half-open) */
   circuitBreakerStateGauge: Metric.gauge("circuit_breaker_state", {
-    description: "Current circuit breaker state (0=closed, 1=open, 2=half-open)",
+    description:
+      "Current circuit breaker state (0=closed, 1=open, 2=half-open)",
   }),
 
   /** Number of failures in circuit breaker sliding window */
   circuitBreakerFailuresGauge: Metric.gauge("circuit_breaker_failures", {
-    description: "Number of failures currently in the circuit breaker sliding window",
+    description:
+      "Number of failures currently in the circuit breaker sliding window",
   }),
 });
 

package/src/flow/tracing.ts

@@ -71,6 +71,76 @@ export const withExecutionContext = (context: {
     "execution.parallel_count": context.parallelCount?.toString() ?? "0",
   });
 
+// ============================================================================
+// Plugin Operation Tracing
+// ============================================================================
+
+/**
+ * Operation domains for plugin-level tracing
+ */
+export type OperationDomain =
+  | "image"
+  | "video"
+  | "document"
+  | "ai"
+  | "virus-scan"
+  | "zip";
+
+/**
+ * Wrap an Effect with a plugin operation span
+ *
+ * @param domain - The operation domain (e.g., "image", "video", "document")
+ * @param operation - The specific operation (e.g., "optimize", "transcode", "extract-text")
+ * @param attributes - Optional span attributes with operation-specific details
+ *
+ * @example
+ * ```typescript
+ * // Image optimization span
+ * withOperationSpan("image", "optimize", {
+ *   "image.format": "webp",
+ *   "image.quality": 80,
+ * })(imageService.optimize(inputBytes, params))
+ *
+ * // Video transcoding span
+ * withOperationSpan("video", "transcode", {
+ *   "video.format": "mp4",
+ *   "video.codec": "h264",
+ * })(videoService.transcode(inputBytes, params))
+ * ```
+ */
+export const withOperationSpan =
+  <A, E, R>(
+    domain: OperationDomain,
+    operation: string,
+    attributes?: Record<string, unknown>,
+  ) =>
+  (effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    effect.pipe(
+      Effect.withSpan(`${domain}-${operation}`, {
+        attributes: {
+          "operation.domain": domain,
+          "operation.name": operation,
+          ...attributes,
+        },
+      }),
+    );
+
+/**
+ * Add operation context to the current span
+ */
+export const withOperationContext = (context: {
+  domain: OperationDomain;
+  operation: string;
+  inputSize?: number;
+  outputSize?: number;
+}) =>
+  Effect.annotateCurrentSpan({
+    "operation.domain": context.domain,
+    "operation.name": context.operation,
+    "operation.input_size": context.inputSize?.toString() ?? "unknown",
+    "operation.output_size": context.outputSize?.toString() ?? "unknown",
+  });
+
 // ============================================================================
 // Circuit Breaker Tracing
 // ============================================================================
@@ -117,7 +187,8 @@ export const withCircuitBreakerContext = (context: {
     "circuit_breaker.failure_count": context.failureCount?.toString() ?? "0",
     "circuit_breaker.failure_threshold":
       context.failureThreshold?.toString() ?? "5",
-    "circuit_breaker.reset_timeout": context.resetTimeout?.toString() ?? "30000",
+    "circuit_breaker.reset_timeout":
+      context.resetTimeout?.toString() ?? "30000",
     "circuit_breaker.decision": context.decision ?? "unknown",
   });
 
@@ -137,5 +208,6 @@ export const annotateCircuitBreakerStateChange = (event: {
     "circuit_breaker.previous_state": event.previousState,
     "circuit_breaker.new_state": event.newState,
     "circuit_breaker.failure_count": event.failureCount?.toString() ?? "0",
-    "circuit_breaker.timestamp": event.timestamp?.toString() ?? Date.now().toString(),
+    "circuit_breaker.timestamp":
+      event.timestamp?.toString() ?? Date.now().toString(),
   });

package/src/index.ts

@@ -1,7 +1,5 @@
 // Main observability package exports
 export * from "./core/index.js";
-// Tracing specific exports
-export { NodeSdkLive, WebSdkLive, WorkersSdkLive } from "./core/tracing.js";
 export * from "./flow/index.js";
 export * from "./service/metrics.js";
 export * from "./storage/index.js";