@uploadista/observability 0.0.3

package/src/storage/filesystem.ts

@@ -0,0 +1,153 @@
+import { type Effect, Layer } from "effect";
+import {
+  createStorageErrorTracker,
+  type StorageErrorCategory,
+} from "../core/errors.js";
+import {
+  logStorageOperation,
+  logUploadCompletion,
+  logUploadProgress,
+  logWithContext,
+} from "../core/logging.js";
+import { createStorageMetrics, type StorageMetrics } from "../core/metrics.js";
+import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
+import {
+  withApiMetrics,
+  withStorageOperationMetrics,
+  withTimingMetrics,
+  withUploadMetrics,
+} from "../core/utilities.js";
+
+// ============================================================================
+// Filesystem Storage-Specific Observability
+// ============================================================================
+
+const STORAGE_TYPE = "filesystem";
+
+// Filesystem-specific metrics
+export const filesystemMetrics = createStorageMetrics(STORAGE_TYPE);
+
+// Filesystem-specific tracing layer
+export const FilesystemTracingLayer = createStorageTracingLayer(STORAGE_TYPE);
+
+// Filesystem-specific error classification
+const classifyFilesystemError = (
+  error: unknown,
+): StorageErrorCategory | null => {
+  if (!error || typeof error !== "object") return null;
+
+  const errorCode = "code" in error ? error.code : undefined;
+  if (!errorCode) return null;
+
+  // Node.js filesystem error codes
+  switch (errorCode) {
+    case "ENOENT": // File/directory not found
+    case "ENOTDIR": // Not a directory
+      return "client_error";
+    case "EEXIST": // File/directory already exists
+      return "client_error";
+    case "EISDIR": // Is a directory
+      return "client_error";
+    case "EINVAL": // Invalid argument
+    case "ENAMETOOLONG": // Filename too long
+      return "client_error";
+    case "EACCES": // Permission denied
+    case "EPERM": // Operation not permitted
+      return "authorization_error";
+    case "ENOSPC": // No space left on device
+    case "EDQUOT": // Disk quota exceeded
+      return "server_error";
+    case "EIO": // I/O error
+    case "EROFS": // Read-only filesystem
+    case "EMFILE": // Too many open files
+    case "ENFILE": // File table overflow
+      return "server_error";
+    case "EBUSY": // Device or resource busy
+      return "throttling_error";
+    default:
+      return null; // Fall back to generic classification
+  }
+};
+
+// Filesystem-specific error tracker
+export const trackFilesystemError = createStorageErrorTracker(
+  STORAGE_TYPE,
+  filesystemMetrics,
+  classifyFilesystemError,
+);
+
+// Filesystem-specific observability layer
+export const FilesystemObservabilityLayer = Layer.mergeAll(
+  FilesystemTracingLayer,
+  // Metrics are automatically available through Effect
+);
+
+// ============================================================================
+// Filesystem Utility Functions
+// ============================================================================
+
+export const withFilesystemUploadMetrics = <A, E, R>(
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+) => withUploadMetrics(filesystemMetrics, uploadId, effect);
+
+export const withFilesystemApiMetrics = <A, E, R>(
+  operation: string,
+  effect: Effect.Effect<A, E, R>,
+) => withApiMetrics(filesystemMetrics, operation, effect);
+
+export const withFilesystemTimingMetrics = withTimingMetrics;
+
+export const withFilesystemOperationMetrics = <A, E, R>(
+  operation: string,
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+  fileSize?: number,
+) =>
+  withStorageOperationMetrics(
+    filesystemMetrics,
+    operation,
+    uploadId,
+    effect,
+    fileSize,
+  );
+
+// Filesystem-specific span wrapper
+export const withFilesystemSpan =
+  <A, E, R>(operation: string, attributes?: Record<string, unknown>) =>
+  (effect: Effect.Effect<A, E, R>) =>
+    withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
+
+// Filesystem-specific logging functions
+export const logFilesystemOperation = logStorageOperation.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logFilesystemUploadProgress = logUploadProgress.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logFilesystemUploadCompletion = logUploadCompletion.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logFilesystemContext = logWithContext;
+
+// Export metrics for external access
+export const {
+  uploadRequestsTotal: filesystemUploadRequestsTotal,
+  uploadPartsTotal: filesystemUploadPartsTotal,
+  uploadSuccessTotal: filesystemUploadSuccessTotal,
+  uploadErrorsTotal: filesystemUploadErrorsTotal,
+  apiCallsTotal: filesystemApiCallsTotal,
+  uploadDurationHistogram: filesystemUploadDurationHistogram,
+  partUploadDurationHistogram: filesystemPartUploadDurationHistogram,
+  fileSizeHistogram: filesystemFileSizeHistogram,
+  partSizeHistogram: filesystemPartSizeHistogram,
+  activeUploadsGauge: filesystemActiveUploadsGauge,
+  uploadThroughputGauge: filesystemUploadThroughputGauge,
+  uploadLatencySummary: filesystemUploadLatencySummary,
+} = filesystemMetrics;
+
+// Type exports
+export type FilesystemMetrics = StorageMetrics;

package/src/storage/gcs.ts

@@ -0,0 +1,161 @@
+import { type Effect, Layer } from "effect";
+import {
+  createStorageErrorTracker,
+  type StorageErrorCategory,
+} from "../core/errors.js";
+import {
+  logStorageOperation,
+  logUploadCompletion,
+  logUploadProgress,
+  logWithContext,
+} from "../core/logging.js";
+import { createStorageMetrics, type StorageMetrics } from "../core/metrics.js";
+import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
+import {
+  withApiMetrics,
+  withStorageOperationMetrics,
+  withTimingMetrics,
+  withUploadMetrics,
+} from "../core/utilities.js";
+
+// ============================================================================
+// Google Cloud Storage-Specific Observability
+// ============================================================================
+
+const STORAGE_TYPE = "gcs";
+
+// GCS-specific metrics
+export const gcsMetrics = createStorageMetrics(STORAGE_TYPE);
+
+// GCS-specific tracing layer
+export const GCSTracingLayer = createStorageTracingLayer(STORAGE_TYPE);
+
+// GCS-specific error classification
+const classifyGCSError = (error: unknown): StorageErrorCategory | null => {
+  if (!error || typeof error !== "object") return null;
+
+  const errorCode =
+    "code" in error ? error.code : "status" in error ? error.status : undefined;
+  if (!errorCode) return null;
+
+  // GCS-specific error codes
+  switch (errorCode) {
+    case "NoSuchBucket":
+    case "NoSuchKey":
+    case "NoSuchUpload":
+      return "client_error";
+    case "BucketAlreadyOwnedByYou":
+    case "BucketNotEmpty":
+      return "client_error";
+    case "InvalidBucketName":
+    case "InvalidArgument":
+    case "InvalidPart":
+    case "InvalidPartOrder":
+      return "client_error";
+    case "EntityTooSmall":
+    case "EntityTooLarge":
+      return "client_error";
+    case "MalformedPolicy":
+      return "client_error";
+    case "Unauthorized":
+    case "AuthenticationRequired":
+      return "authentication_error";
+    case "Forbidden":
+    case "AccessDenied":
+      return "authorization_error";
+    case "TooManyRequests":
+    case "RateLimitExceeded":
+      return "throttling_error";
+    case "InternalError":
+    case "ServiceUnavailable":
+    case "BackendError":
+      return "server_error";
+    default:
+      // Check for HTTP status codes
+      if (typeof errorCode === "number") {
+        if (errorCode >= 500) return "server_error";
+        if (errorCode === 429) return "throttling_error";
+        if (errorCode === 403) return "authorization_error";
+        if (errorCode === 401) return "authentication_error";
+        if (errorCode >= 400) return "client_error";
+      }
+      return null; // Fall back to generic classification
+  }
+};
+
+// GCS-specific error tracker
+export const trackGCSError = createStorageErrorTracker(
+  STORAGE_TYPE,
+  gcsMetrics,
+  classifyGCSError,
+);
+
+// GCS-specific observability layer
+export const GCSObservabilityLayer = Layer.mergeAll(
+  GCSTracingLayer,
+  // Metrics are automatically available through Effect
+);
+
+// ============================================================================
+// GCS Utility Functions
+// ============================================================================
+
+export const withGCSUploadMetrics = <A, E, R>(
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+) => withUploadMetrics(gcsMetrics, uploadId, effect);
+
+export const withGCSApiMetrics = <A, E, R>(
+  operation: string,
+  effect: Effect.Effect<A, E, R>,
+) => withApiMetrics(gcsMetrics, operation, effect);
+
+export const withGCSTimingMetrics = withTimingMetrics;
+
+export const withGCSOperationMetrics = <A, E, R>(
+  operation: string,
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+  fileSize?: number,
+) =>
+  withStorageOperationMetrics(
+    gcsMetrics,
+    operation,
+    uploadId,
+    effect,
+    fileSize,
+  );
+
+// GCS-specific span wrapper
+export const withGCSSpan =
+  <A, E, R>(operation: string, attributes?: Record<string, unknown>) =>
+  (effect: Effect.Effect<A, E, R>) =>
+    withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
+
+// GCS-specific logging functions
+export const logGCSOperation = logStorageOperation.bind(null, STORAGE_TYPE);
+export const logGCSUploadProgress = logUploadProgress.bind(null, STORAGE_TYPE);
+export const logGCSUploadCompletion = logUploadCompletion.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logGCSContext = logWithContext;
+
+// Export metrics for external access
+export const {
+  uploadRequestsTotal: gcsUploadRequestsTotal,
+  uploadPartsTotal: gcsUploadPartsTotal,
+  uploadSuccessTotal: gcsUploadSuccessTotal,
+  uploadErrorsTotal: gcsUploadErrorsTotal,
+  apiCallsTotal: gcsApiCallsTotal,
+  uploadDurationHistogram: gcsUploadDurationHistogram,
+  partUploadDurationHistogram: gcsPartUploadDurationHistogram,
+  fileSizeHistogram: gcsFileSizeHistogram,
+  partSizeHistogram: gcsPartSizeHistogram,
+  activeUploadsGauge: gcsActiveUploadsGauge,
+  uploadThroughputGauge: gcsUploadThroughputGauge,
+  uploadLatencySummary: gcsUploadLatencySummary,
+} = gcsMetrics;
+
+// Type exports
+export type GCSMetrics = StorageMetrics;

package/src/storage/index.ts

@@ -0,0 +1,5 @@
+// Storage-specific observability exports
+export * from "./s3.js";
+export * from "./azure.js";
+export * from "./gcs.js";
+export * from "./filesystem.js";

package/src/storage/s3.ts

@@ -0,0 +1,136 @@
+import { type Effect, Layer } from "effect";
+import {
+  createStorageErrorTracker,
+  type StorageErrorCategory,
+} from "../core/errors.js";
+import {
+  logStorageOperation,
+  logUploadCompletion,
+  logUploadProgress,
+  logWithContext,
+} from "../core/logging.js";
+import { createStorageMetrics, type StorageMetrics } from "../core/metrics.js";
+import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
+import {
+  withApiMetrics,
+  withStorageOperationMetrics,
+  withTimingMetrics,
+  withUploadMetrics,
+} from "../core/utilities.js";
+
+// ============================================================================
+// S3-Specific Observability
+// ============================================================================
+
+const STORAGE_TYPE = "s3";
+
+// S3-specific metrics
+export const s3Metrics = createStorageMetrics(STORAGE_TYPE);
+
+// S3-specific tracing layer
+export const S3TracingLayer = createStorageTracingLayer(STORAGE_TYPE);
+
+// S3-specific error classification
+const classifyS3Error = (error: unknown): StorageErrorCategory | null => {
+  if (!error || typeof error !== "object") return null;
+
+  const errorCode = "code" in error ? error.code : undefined;
+  if (!errorCode) return null;
+
+  // S3-specific error codes
+  switch (errorCode) {
+    case "NoSuchKey":
+    case "NoSuchBucket":
+    case "NoSuchUpload":
+      return "client_error";
+    case "BucketAlreadyExists":
+    case "BucketNotEmpty":
+      return "client_error";
+    case "InvalidBucketName":
+    case "InvalidPart":
+    case "InvalidPartOrder":
+      return "client_error";
+    case "EntityTooSmall":
+    case "EntityTooLarge":
+      return "client_error";
+    case "ExpiredToken":
+    case "TokenRefreshRequired":
+      return "authentication_error";
+    case "RequestTimeTooSkewed":
+    case "SlowDown":
+      return "throttling_error";
+    default:
+      return null; // Fall back to generic classification
+  }
+};
+
+// S3-specific error tracker
+export const trackS3Error = createStorageErrorTracker(
+  STORAGE_TYPE,
+  s3Metrics,
+  classifyS3Error,
+);
+
+// S3-specific observability layer
+export const S3ObservabilityLayer = Layer.mergeAll(
+  S3TracingLayer,
+  // Metrics are automatically available through Effect
+);
+
+// ============================================================================
+// S3 Utility Functions
+// ============================================================================
+
+export const withS3UploadMetrics = <A, E, R>(
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+) => withUploadMetrics(s3Metrics, uploadId, effect);
+
+export const withS3ApiMetrics = <A, E, R>(
+  operation: string,
+  effect: Effect.Effect<A, E, R>,
+) => withApiMetrics(s3Metrics, operation, effect);
+
+export const withS3TimingMetrics = withTimingMetrics;
+
+export const withS3OperationMetrics = <A, E, R>(
+  operation: string,
+  uploadId: string,
+  effect: Effect.Effect<A, E, R>,
+  fileSize?: number,
+) =>
+  withStorageOperationMetrics(s3Metrics, operation, uploadId, effect, fileSize);
+
+// S3-specific span wrapper
+export const withS3Span =
+  <A, E, R>(operation: string, attributes?: Record<string, unknown>) =>
+  (effect: Effect.Effect<A, E, R>) =>
+    withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
+
+// S3-specific logging functions
+export const logS3Operation = logStorageOperation.bind(null, STORAGE_TYPE);
+export const logS3UploadProgress = logUploadProgress.bind(null, STORAGE_TYPE);
+export const logS3UploadCompletion = logUploadCompletion.bind(
+  null,
+  STORAGE_TYPE,
+);
+export const logS3Context = logWithContext;
+
+// Export metrics for external access
+export const {
+  uploadRequestsTotal: s3UploadRequestsTotal,
+  uploadPartsTotal: s3UploadPartsTotal,
+  uploadSuccessTotal: s3UploadSuccessTotal,
+  uploadErrorsTotal: s3UploadErrorsTotal,
+  apiCallsTotal: s3ApiCallsTotal,
+  uploadDurationHistogram: s3UploadDurationHistogram,
+  partUploadDurationHistogram: s3PartUploadDurationHistogram,
+  fileSizeHistogram: s3FileSizeHistogram,
+  partSizeHistogram: s3PartSizeHistogram,
+  activeUploadsGauge: s3ActiveUploadsGauge,
+  uploadThroughputGauge: s3UploadThroughputGauge,
+  uploadLatencySummary: s3UploadLatencySummary,
+} = s3Metrics;
+
+// Type exports
+export type S3Metrics = StorageMetrics;

package/src/test-observability.ts

@@ -0,0 +1,234 @@
+/**
+ * Validation test for core observability infrastructure
+ *
+ * This test validates:
+ * 1. Layer creation and composition
+ * 2. Storage observability with metrics
+ * 3. Error classification
+ * 4. Tracing with spans
+ * 5. Structured logging
+ */
+
+import { Effect, Layer, Metric } from "effect";
+import {
+  classifyStorageError,
+  createStorageMetrics,
+  makeStorageObservabilityLayer,
+  StorageObservability,
+  trackStorageError,
+  withStorageSpan,
+  logStorageOperation,
+  logUploadCompletion,
+} from "./index.js";
+
+// ============================================================================
+// Test: Create Storage Observability Layer
+// ============================================================================
+
+const testStorageType = "test-s3";
+const testMetrics = createStorageMetrics(testStorageType);
+const TestStorageObservabilityLayer = makeStorageObservabilityLayer(
+  testStorageType,
+  testMetrics,
+  true,
+);
+
+// ============================================================================
+// Test: Simulate Upload Operation with Full Observability
+// ============================================================================
+
+const simulateUpload = (fileSize: number, shouldFail: boolean) =>
+  Effect.gen(function* () {
+    const obs = yield* StorageObservability;
+
+    console.log(`\n📊 Testing with storage type: ${obs.storageType}`);
+    console.log(`Service: ${obs.serviceName}`);
+    console.log(`Enabled: ${obs.enabled}`);
+
+    // Log operation start
+    yield* logStorageOperation(
+      testStorageType,
+      "uploadFile",
+      "test-upload-123",
+      { file_size: fileSize }
+    );
+
+    // Simulate upload with tracing
+    const uploadEffect = Effect.gen(function* () {
+      // Track upload request
+      yield* obs.metrics.uploadRequestsTotal.pipe(
+        Metric.tagged("upload_id", "test-upload-123")
+      )(Effect.succeed(1));
+
+      // Track file size
+      yield* obs.metrics.fileSizeHistogram(Effect.succeed(fileSize));
+
+      // Simulate upload work
+      yield* Effect.sleep("100 millis");
+
+      if (shouldFail) {
+        return yield* Effect.fail(new Error("NetworkError: Connection timeout"));
+      }
+
+      // Track success
+      yield* obs.metrics.uploadSuccessTotal.pipe(
+        Metric.tagged("upload_id", "test-upload-123")
+      )(Effect.succeed(1));
+
+      return { uploadId: "test-upload-123", key: "test-file.jpg" };
+    });
+
+    // Wrap with span
+    const result = yield* uploadEffect.pipe(
+      withStorageSpan("uploadFile", testStorageType, {
+        "file.size": fileSize,
+        "upload.id": "test-upload-123",
+      })
+    );
+
+    // Log completion
+    yield* logUploadCompletion(testStorageType, "test-upload-123", {
+      fileSize,
+      totalDurationMs: 100,
+      throughputBps: (fileSize / 100) * 1000,
+    });
+
+    return result;
+  });
+
+// ============================================================================
+// Test: Error Classification
+// ============================================================================
+
+const testErrorClassification = Effect.gen(function* () {
+  console.log("\n🔍 Testing Error Classification:");
+
+  const testErrors = [
+    { error: { code: "NetworkError" }, expected: "network_error" },
+    { error: { code: "InvalidAccessKeyId" }, expected: "authentication_error" },
+    { error: { code: "AccessDenied" }, expected: "authorization_error" },
+    { error: { code: "SlowDown" }, expected: "throttling_error" },
+    { error: { code: "InternalError" }, expected: "server_error" },
+    { error: { code: "InvalidRequest" }, expected: "client_error" },
+    { error: { code: "UnknownError" }, expected: "unknown_error" },
+  ];
+
+  for (const { error, expected } of testErrors) {
+    const category = classifyStorageError(error);
+    const status = category === expected ? "✅" : "❌";
+    console.log(`  ${status} ${error.code} -> ${category} (expected: ${expected})`);
+  }
+});
+
+// ============================================================================
+// Test: Error Tracking with Metrics
+// ============================================================================
+
+const testErrorTracking = Effect.gen(function* () {
+  const obs = yield* StorageObservability;
+
+  console.log("\n⚠️  Testing Error Tracking:");
+
+  const error = new Error("ECONNRESET: Connection reset by peer");
+  (error as any).code = "ECONNRESET";
+
+  yield* trackStorageError(
+    testStorageType,
+    obs.metrics,
+    "uploadPart",
+    error,
+    { upload_id: "test-upload-456", part_number: 1 }
+  );
+
+  console.log("  ✅ Error tracked with metrics and logs");
+});
+
+// ============================================================================
+// Test: Successful Upload Flow
+// ============================================================================
+
+const testSuccessfulUpload = Effect.gen(function* () {
+  console.log("\n✅ Testing Successful Upload:");
+
+  const result = yield* simulateUpload(1024 * 1024 * 10, false); // 10MB file
+  console.log(`  ✅ Upload completed: ${result.uploadId} -> ${result.key}`);
+});
+
+// ============================================================================
+// Test: Failed Upload Flow
+// ============================================================================
+
+const testFailedUpload = Effect.gen(function* () {
+  console.log("\n❌ Testing Failed Upload:");
+
+  const obs = yield* StorageObservability;
+
+  // Use either to handle success and failure cases
+  const result = yield* Effect.either(simulateUpload(1024 * 1024 * 5, true));
+
+  if (result._tag === "Left") {
+    const error: unknown = result.left;
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    console.log(`  ❌ Upload failed as expected: ${errorMsg}`);
+
+    // Track the error
+    yield* trackStorageError(
+      testStorageType,
+      obs.metrics,
+      "uploadFile",
+      error,
+      { upload_id: "test-upload-789" }
+    );
+
+    console.log("  ✅ Error handled and tracked successfully");
+  } else {
+    console.log("  ❌ Upload should have failed but succeeded");
+  }
+});
+
+// ============================================================================
+// Test: Metrics Snapshot
+// ============================================================================
+
+const testMetricsSnapshot = Effect.gen(function* () {
+  console.log("\n📈 Testing Metrics Snapshot:");
+
+  const snapshot = yield* Metric.snapshot;
+  console.log(`  ✅ Captured ${snapshot.length} metric(s)`);
+
+  // Display some metrics
+  for (const metric of snapshot.slice(0, 5)) {
+    console.log(`     - ${metric.metricKey.name}: ${JSON.stringify(metric.metricState)}`);
+  }
+});
+
+// ============================================================================
+// Run All Tests
+// ============================================================================
+
+const runTests = Effect.gen(function* () {
+  console.log("🧪 Observability Infrastructure Validation Test");
+  console.log("=".repeat(50));
+
+  // Run all tests
+  yield* testErrorClassification;
+  yield* testSuccessfulUpload;
+  yield* testFailedUpload;
+  yield* testErrorTracking;
+  yield* testMetricsSnapshot;
+
+  console.log("\n" + "=".repeat(50));
+  console.log("✅ All tests completed successfully!");
+  console.log("\nCore observability infrastructure is working correctly:");
+  console.log("  ✅ Layer creation and composition");
+  console.log("  ✅ Metrics tracking (counters, histograms, gauges)");
+  console.log("  ✅ Error classification and tracking");
+  console.log("  ✅ Tracing with spans");
+  console.log("  ✅ Structured logging");
+}).pipe(Effect.provide(TestStorageObservabilityLayer));
+
+// Run the tests
+Effect.runPromise(runTests).catch((error) => {
+  console.error("\n❌ Test failed:", error);
+  process.exit(1);
+});