@uploadista/observability 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +0 -0
- package/LICENSE +21 -0
- package/dist/core/errors.d.ts +8 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +108 -0
- package/dist/core/index.d.ts +8 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +8 -0
- package/dist/core/layers.d.ts +104 -0
- package/dist/core/layers.d.ts.map +1 -0
- package/dist/core/layers.js +110 -0
- package/dist/core/logging.d.ts +18 -0
- package/dist/core/logging.d.ts.map +1 -0
- package/dist/core/logging.js +41 -0
- package/dist/core/metrics.d.ts +37 -0
- package/dist/core/metrics.d.ts.map +1 -0
- package/dist/core/metrics.js +72 -0
- package/dist/core/testing.d.ts +43 -0
- package/dist/core/testing.d.ts.map +1 -0
- package/dist/core/testing.js +93 -0
- package/dist/core/tracing.d.ts +19 -0
- package/dist/core/tracing.d.ts.map +1 -0
- package/dist/core/tracing.js +43 -0
- package/dist/core/utilities.d.ts +11 -0
- package/dist/core/utilities.d.ts.map +1 -0
- package/dist/core/utilities.js +41 -0
- package/dist/flow/errors.d.ts +15 -0
- package/dist/flow/errors.d.ts.map +1 -0
- package/dist/flow/errors.js +66 -0
- package/dist/flow/index.d.ts +6 -0
- package/dist/flow/index.d.ts.map +1 -0
- package/dist/flow/index.js +6 -0
- package/dist/flow/layers.d.ts +40 -0
- package/dist/flow/layers.d.ts.map +1 -0
- package/dist/flow/layers.js +94 -0
- package/dist/flow/metrics.d.ts +52 -0
- package/dist/flow/metrics.d.ts.map +1 -0
- package/dist/flow/metrics.js +89 -0
- package/dist/flow/testing.d.ts +11 -0
- package/dist/flow/testing.d.ts.map +1 -0
- package/dist/flow/testing.js +27 -0
- package/dist/flow/tracing.d.ts +35 -0
- package/dist/flow/tracing.d.ts.map +1 -0
- package/dist/flow/tracing.js +42 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/service/metrics.d.ts +23 -0
- package/dist/service/metrics.d.ts.map +1 -0
- package/dist/service/metrics.js +17 -0
- package/dist/storage/azure.d.ts +47 -0
- package/dist/storage/azure.d.ts.map +1 -0
- package/dist/storage/azure.js +89 -0
- package/dist/storage/filesystem.d.ts +47 -0
- package/dist/storage/filesystem.d.ts.map +1 -0
- package/dist/storage/filesystem.js +70 -0
- package/dist/storage/gcs.d.ts +47 -0
- package/dist/storage/gcs.d.ts.map +1 -0
- package/dist/storage/gcs.js +90 -0
- package/dist/storage/index.d.ts +5 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/s3.d.ts +47 -0
- package/dist/storage/s3.d.ts.map +1 -0
- package/dist/storage/s3.js +67 -0
- package/dist/test-observability.d.ts +12 -0
- package/dist/test-observability.d.ts.map +1 -0
- package/dist/test-observability.js +153 -0
- package/dist/upload/errors.d.ts +16 -0
- package/dist/upload/errors.d.ts.map +1 -0
- package/dist/upload/errors.js +107 -0
- package/dist/upload/index.d.ts +6 -0
- package/dist/upload/index.d.ts.map +1 -0
- package/dist/upload/index.js +6 -0
- package/dist/upload/layers.d.ts +32 -0
- package/dist/upload/layers.d.ts.map +1 -0
- package/dist/upload/layers.js +63 -0
- package/dist/upload/metrics.d.ts +46 -0
- package/dist/upload/metrics.d.ts.map +1 -0
- package/dist/upload/metrics.js +80 -0
- package/dist/upload/testing.d.ts +32 -0
- package/dist/upload/testing.d.ts.map +1 -0
- package/dist/upload/testing.js +52 -0
- package/dist/upload/tracing.d.ts +25 -0
- package/dist/upload/tracing.d.ts.map +1 -0
- package/dist/upload/tracing.js +35 -0
- package/package.json +37 -0
- package/src/core/errors.ts +187 -0
- package/src/core/index.ts +9 -0
- package/src/core/layers.ts +205 -0
- package/src/core/logging.ts +81 -0
- package/src/core/metrics.ts +108 -0
- package/src/core/testing.ts +142 -0
- package/src/core/tracing.ts +67 -0
- package/src/core/utilities.ts +133 -0
- package/src/flow/errors.ts +95 -0
- package/src/flow/index.ts +17 -0
- package/src/flow/layers.ts +131 -0
- package/src/flow/metrics.ts +130 -0
- package/src/flow/testing.ts +33 -0
- package/src/flow/tracing.ts +72 -0
- package/src/index.ts +31 -0
- package/src/service/metrics.ts +31 -0
- package/src/storage/azure.ts +163 -0
- package/src/storage/filesystem.ts +153 -0
- package/src/storage/gcs.ts +161 -0
- package/src/storage/index.ts +5 -0
- package/src/storage/s3.ts +136 -0
- package/src/test-observability.ts +234 -0
- package/src/upload/errors.ts +166 -0
- package/src/upload/index.ts +12 -0
- package/src/upload/layers.ts +88 -0
- package/src/upload/metrics.ts +118 -0
- package/src/upload/testing.ts +60 -0
- package/src/upload/tracing.ts +58 -0
- package/tsconfig.json +10 -0
- package/tsconfig.tsbuildinfo +1 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Effect, Layer } from "effect";
|
|
2
|
+
import { FlowObservability } from "../core/layers.js";
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// Test Flow Observability Layers
|
|
5
|
+
// ============================================================================
|
|
6
|
+
/**
|
|
7
|
+
* Mock flow observability for testing
|
|
8
|
+
*/
|
|
9
|
+
export const makeTestFlowObservability = () => {
|
|
10
|
+
const service = {
|
|
11
|
+
serviceName: "test-flow-engine",
|
|
12
|
+
enabled: true,
|
|
13
|
+
metrics: {
|
|
14
|
+
flowStarted: Effect.void,
|
|
15
|
+
flowCompleted: Effect.void,
|
|
16
|
+
flowFailed: Effect.void,
|
|
17
|
+
nodeExecuted: Effect.void,
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
return Layer.succeed(FlowObservability, service);
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Run an effect with test flow observability
|
|
24
|
+
*/
|
|
25
|
+
export const runWithTestFlowObservability = (effect) => {
|
|
26
|
+
return effect.pipe(Effect.provide(makeTestFlowObservability()));
|
|
27
|
+
};
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Wrap an Effect with a flow operation span
|
|
4
|
+
*/
|
|
5
|
+
export declare const withFlowSpan: <A, E, R>(operation: string, attributes?: Record<string, unknown>) => (effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
6
|
+
/**
|
|
7
|
+
* Add flow context to the current span
|
|
8
|
+
*/
|
|
9
|
+
export declare const withFlowContext: (context: {
|
|
10
|
+
flowId?: string;
|
|
11
|
+
flowName?: string;
|
|
12
|
+
jobId?: string;
|
|
13
|
+
nodeCount?: number;
|
|
14
|
+
storageId?: string;
|
|
15
|
+
}) => Effect.Effect<void, never, never>;
|
|
16
|
+
/**
|
|
17
|
+
* Add node context to the current span
|
|
18
|
+
*/
|
|
19
|
+
export declare const withNodeContext: (context: {
|
|
20
|
+
nodeId: string;
|
|
21
|
+
nodeType: string;
|
|
22
|
+
nodeName?: string;
|
|
23
|
+
flowId?: string;
|
|
24
|
+
jobId?: string;
|
|
25
|
+
}) => Effect.Effect<void, never, never>;
|
|
26
|
+
/**
|
|
27
|
+
* Add execution state context to the current span
|
|
28
|
+
*/
|
|
29
|
+
export declare const withExecutionContext: (context: {
|
|
30
|
+
executionOrder?: string[];
|
|
31
|
+
currentIndex?: number;
|
|
32
|
+
totalNodes?: number;
|
|
33
|
+
parallelCount?: number;
|
|
34
|
+
}) => Effect.Effect<void, never, never>;
|
|
35
|
+
//# sourceMappingURL=tracing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tracing.d.ts","sourceRoot":"","sources":["../../src/flow/tracing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAMhC;;GAEG;AACH,eAAO,MAAM,YAAY,GACtB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,MAAM,EAAE,aAAa,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAChE,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAQpD,CAAC;AAEN;;GAEG;AACH,eAAO,MAAM,eAAe,GAAI,SAAS;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,sCAOG,CAAC;AAEL;;GAEG;AACH,eAAO,MAAM,eAAe,GAAI,SAAS;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,sCAOG,CAAC;AAEL;;GAEG;AACH,eAAO,MAAM,oBAAoB,GAAI,SAAS;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,sCAMG,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
// ============================================================================
|
|
3
|
+
// Flow Tracing Utilities
|
|
4
|
+
// ============================================================================
|
|
5
|
+
/**
|
|
6
|
+
* Wrap an Effect with a flow operation span
|
|
7
|
+
*/
|
|
8
|
+
export const withFlowSpan = (operation, attributes) => (effect) => effect.pipe(Effect.withSpan(`flow-${operation}`, {
|
|
9
|
+
attributes: {
|
|
10
|
+
"flow.operation": operation,
|
|
11
|
+
...attributes,
|
|
12
|
+
},
|
|
13
|
+
}));
|
|
14
|
+
/**
|
|
15
|
+
* Add flow context to the current span
|
|
16
|
+
*/
|
|
17
|
+
export const withFlowContext = (context) => Effect.annotateCurrentSpan({
|
|
18
|
+
"flow.id": context.flowId ?? "unknown",
|
|
19
|
+
"flow.name": context.flowName ?? "unknown",
|
|
20
|
+
"flow.job_id": context.jobId ?? "unknown",
|
|
21
|
+
"flow.node_count": context.nodeCount?.toString() ?? "0",
|
|
22
|
+
"flow.storage_id": context.storageId ?? "unknown",
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* Add node context to the current span
|
|
26
|
+
*/
|
|
27
|
+
export const withNodeContext = (context) => Effect.annotateCurrentSpan({
|
|
28
|
+
"node.id": context.nodeId,
|
|
29
|
+
"node.type": context.nodeType,
|
|
30
|
+
"node.name": context.nodeName ?? "unknown",
|
|
31
|
+
"node.flow_id": context.flowId ?? "unknown",
|
|
32
|
+
"node.job_id": context.jobId ?? "unknown",
|
|
33
|
+
});
|
|
34
|
+
/**
|
|
35
|
+
* Add execution state context to the current span
|
|
36
|
+
*/
|
|
37
|
+
export const withExecutionContext = (context) => Effect.annotateCurrentSpan({
|
|
38
|
+
"execution.order": context.executionOrder?.join(",") ?? "",
|
|
39
|
+
"execution.current_index": context.currentIndex?.toString() ?? "0",
|
|
40
|
+
"execution.total_nodes": context.totalNodes?.toString() ?? "0",
|
|
41
|
+
"execution.parallel_count": context.parallelCount?.toString() ?? "0",
|
|
42
|
+
});
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * from "./core/index.js";
|
|
2
|
+
export { NodeSdkLive, WebSdkLive, WorkersSdkLive } from "./core/tracing.js";
|
|
3
|
+
export * from "./flow/index.js";
|
|
4
|
+
export * from "./service/metrics.js";
|
|
5
|
+
export * from "./storage/index.js";
|
|
6
|
+
export { logS3Context, logS3Operation, logS3UploadCompletion, logS3UploadProgress, S3ObservabilityLayer, S3TracingLayer, s3ActiveUploadsGauge, s3ApiCallsTotal, s3FileSizeHistogram, s3Metrics, s3UploadDurationHistogram, s3UploadRequestsTotal, trackS3Error, withS3ApiMetrics, withS3OperationMetrics, withS3Span, withS3TimingMetrics, withS3UploadMetrics, } from "./storage/s3.js";
|
|
7
|
+
export * from "./upload/index.js";
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,iBAAiB,CAAC;AAEhC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC5E,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EACL,YAAY,EACZ,cAAc,EACd,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,mBAAmB,EAEnB,SAAS,EACT,yBAAyB,EAEzB,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,sBAAsB,EACtB,UAAU,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AACzB,cAAc,mBAAmB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// Main observability package exports
|
|
2
|
+
export * from "./core/index.js";
|
|
3
|
+
// Tracing specific exports
|
|
4
|
+
export { NodeSdkLive, WebSdkLive, WorkersSdkLive } from "./core/tracing.js";
|
|
5
|
+
export * from "./flow/index.js";
|
|
6
|
+
export * from "./service/metrics.js";
|
|
7
|
+
export * from "./storage/index.js";
|
|
8
|
+
// Convenience re-exports for common use cases
|
|
9
|
+
export { logS3Context, logS3Operation, logS3UploadCompletion, logS3UploadProgress, S3ObservabilityLayer, S3TracingLayer, s3ActiveUploadsGauge, s3ApiCallsTotal, s3FileSizeHistogram,
|
|
10
|
+
// S3 specific
|
|
11
|
+
s3Metrics, s3UploadDurationHistogram,
|
|
12
|
+
// Individual S3 metrics
|
|
13
|
+
s3UploadRequestsTotal, trackS3Error, withS3ApiMetrics, withS3OperationMetrics, withS3Span, withS3TimingMetrics, withS3UploadMetrics, } from "./storage/s3.js";
|
|
14
|
+
export * from "./upload/index.js";
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
declare const MetricsService_base: Context.TagClass<MetricsService, "MetricsService", {
|
|
3
|
+
/**
|
|
4
|
+
* Record upload metrics for an organization
|
|
5
|
+
*/
|
|
6
|
+
readonly recordUpload: (clientId: string, bytes: number, metadata?: Record<string, unknown>) => Effect.Effect<void, never>;
|
|
7
|
+
}>;
|
|
8
|
+
/**
|
|
9
|
+
* Metrics Recording Service
|
|
10
|
+
*
|
|
11
|
+
* Provides access to metrics recording functionality throughout
|
|
12
|
+
* the upload and flow processing pipeline. The service is provided
|
|
13
|
+
* via Effect Layer and can be accessed using Effect.service().
|
|
14
|
+
*/
|
|
15
|
+
export declare class MetricsService extends MetricsService_base {
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* No-op implementation of MetricsService that does nothing.
|
|
19
|
+
* Used when metrics are disabled or database is not available.
|
|
20
|
+
*/
|
|
21
|
+
export declare const NoOpMetricsServiceLive: Layer.Layer<MetricsService>;
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/service/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;;IAY5C;;OAEG;2BACoB,CACrB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC/B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;;AAjBnC;;;;;;GAMG;AACH,qBAAa,cAAe,SAAQ,mBAYjC;CAAG;AAEN;;;GAGG;AACH,eAAO,MAAM,sBAAsB,EAAE,KAAK,CAAC,KAAK,CAAC,cAAc,CAG3D,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Metrics Recording Service
|
|
4
|
+
*
|
|
5
|
+
* Provides access to metrics recording functionality throughout
|
|
6
|
+
* the upload and flow processing pipeline. The service is provided
|
|
7
|
+
* via Effect Layer and can be accessed using Effect.service().
|
|
8
|
+
*/
|
|
9
|
+
export class MetricsService extends Context.Tag("MetricsService")() {
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* No-op implementation of MetricsService that does nothing.
|
|
13
|
+
* Used when metrics are disabled or database is not available.
|
|
14
|
+
*/
|
|
15
|
+
export const NoOpMetricsServiceLive = Layer.succeed(MetricsService, {
|
|
16
|
+
recordUpload: (_organizationId, _bytes) => Effect.void,
|
|
17
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { type Effect, Layer } from "effect";
|
|
2
|
+
import { type StorageMetrics } from "../core/metrics.js";
|
|
3
|
+
export declare const azureMetrics: {
|
|
4
|
+
uploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
5
|
+
activeUploadsGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
6
|
+
uploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
7
|
+
uploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
8
|
+
partUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
9
|
+
fileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
10
|
+
partSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
11
|
+
uploadRequestsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
12
|
+
uploadPartsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
13
|
+
uploadSuccessTotal: import("effect/Metric").Metric.Counter<number>;
|
|
14
|
+
uploadErrorsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
15
|
+
apiCallsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
16
|
+
};
|
|
17
|
+
export declare const AzureTracingLayer: Layer.Layer<{
|
|
18
|
+
serviceName: string;
|
|
19
|
+
}, never, never>;
|
|
20
|
+
export declare const trackAzureError: (operation: string, error: unknown, context?: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
21
|
+
export declare const AzureObservabilityLayer: Layer.Layer<{
|
|
22
|
+
serviceName: string;
|
|
23
|
+
}, never, never>;
|
|
24
|
+
export declare const withAzureUploadMetrics: <A, E, R>(uploadId: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
25
|
+
export declare const withAzureApiMetrics: <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
26
|
+
export declare const withAzureTimingMetrics: <A, E, R>(metric: import("effect/Metric").Metric.Histogram<number>, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
27
|
+
export declare const withAzureOperationMetrics: <A, E, R>(operation: string, uploadId: string, effect: Effect.Effect<A, E, R>, fileSize?: number) => Effect.Effect<A, E, R>;
|
|
28
|
+
export declare const withAzureSpan: <A, E, R>(operation: string, attributes?: Record<string, unknown>) => (effect: Effect.Effect<A, E, R>) => Effect.Effect<unknown, unknown, unknown>;
|
|
29
|
+
export declare const logAzureOperation: (operation: string, uploadId: string, metadata?: Record<string, unknown> | undefined) => Effect.Effect<void, never, never>;
|
|
30
|
+
export declare const logAzureUploadProgress: (uploadId: string, progress: {
|
|
31
|
+
uploadedBytes: number;
|
|
32
|
+
totalBytes: number;
|
|
33
|
+
partNumber?: number;
|
|
34
|
+
speed?: number;
|
|
35
|
+
}) => Effect.Effect<void, never, never>;
|
|
36
|
+
export declare const logAzureUploadCompletion: (uploadId: string, metrics: {
|
|
37
|
+
fileSize: number;
|
|
38
|
+
totalDurationMs: number;
|
|
39
|
+
partsCount?: number;
|
|
40
|
+
averagePartSize?: number;
|
|
41
|
+
throughputBps?: number;
|
|
42
|
+
retryCount?: number;
|
|
43
|
+
}) => Effect.Effect<void, never, never>;
|
|
44
|
+
export declare const logAzureContext: (message: string, context: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
45
|
+
export declare const azureUploadRequestsTotal: import("effect/Metric").Metric.Counter<number>, azureUploadPartsTotal: import("effect/Metric").Metric.Counter<number>, azureUploadSuccessTotal: import("effect/Metric").Metric.Counter<number>, azureUploadErrorsTotal: import("effect/Metric").Metric.Counter<number>, azureApiCallsTotal: import("effect/Metric").Metric.Counter<number>, azureUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, azurePartUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, azureFileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, azurePartSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, azureActiveUploadsGauge: import("effect/Metric").Metric.Gauge<number>, azureUploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>, azureUploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
46
|
+
export type AzureMetrics = StorageMetrics;
|
|
47
|
+
//# sourceMappingURL=azure.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"azure.d.ts","sourceRoot":"","sources":["../../src/storage/azure.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAW5C,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAgB/E,eAAO,MAAM,YAAY;;;;;;;;;;;;;CAAqC,CAAC;AAG/D,eAAO,MAAM,iBAAiB;;gBAA0C,CAAC;AAuDzE,eAAO,MAAM,eAAe,6GAI3B,CAAC;AAGF,eAAO,MAAM,uBAAuB;;gBAGnC,CAAC;AAMF,eAAO,MAAM,sBAAsB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAC5C,UAAU,MAAM,EAChB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,2BACsB,CAAC;AAEvD,eAAO,MAAM,mBAAmB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EACzC,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,2BACoB,CAAC;AAErD,eAAO,MAAM,sBAAsB,+HAAoB,CAAC;AAExD,eAAO,MAAM,yBAAyB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAC/C,WAAW,MAAM,EACjB,UAAU,MAAM,EAChB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,WAAW,MAAM,2BAQhB,CAAC;AAGJ,eAAO,MAAM,aAAa,GACvB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,MAAM,EAAE,aAAa,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAChE,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,6CAC+B,CAAC;AAGjE,eAAO,MAAM,iBAAiB,4HAA+C,CAAC;AAC9E,eAAO,MAAM,sBAAsB;;;;;uCAGlC,CAAC;AACF,eAAO,MAAM,wBAAwB;;;;;;;uCAGpC,CAAC;AACF,eAAO,MAAM,eAAe,0FAAiB,CAAC;AAG9C,eAAO,MACgB,wBAAwB,kDAC3B,qBAAqB,kDACnB,uBAAuB,kDACxB,sBAAsB,kDAC1B,kBAAkB,kDACR,4BAA4B,sJACxB,gCAAgC,sJAC1C,sBAAsB,sJACtB,sBAAsB,sJACrB,uBAAuB,gDACpB,0BAA0B,gDAC3B,yBAAyB,gDACjC,CAAC;AAGjB,MAAM,MAAM,YAAY,GAAG,cAAc,CAAC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Layer } from "effect";
|
|
2
|
+
import { createStorageErrorTracker, } from "../core/errors.js";
|
|
3
|
+
import { logStorageOperation, logUploadCompletion, logUploadProgress, logWithContext, } from "../core/logging.js";
|
|
4
|
+
import { createStorageMetrics } from "../core/metrics.js";
|
|
5
|
+
import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
|
|
6
|
+
import { withApiMetrics, withStorageOperationMetrics, withTimingMetrics, withUploadMetrics, } from "../core/utilities.js";
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// Azure Blob Storage-Specific Observability
|
|
9
|
+
// ============================================================================
|
|
10
|
+
const STORAGE_TYPE = "azure";
|
|
11
|
+
// Azure-specific metrics
|
|
12
|
+
export const azureMetrics = createStorageMetrics(STORAGE_TYPE);
|
|
13
|
+
// Azure-specific tracing layer
|
|
14
|
+
export const AzureTracingLayer = createStorageTracingLayer(STORAGE_TYPE);
|
|
15
|
+
// Azure-specific error classification
|
|
16
|
+
const classifyAzureError = (error) => {
|
|
17
|
+
if (!error || typeof error !== "object")
|
|
18
|
+
return null;
|
|
19
|
+
const errorCode = "code" in error
|
|
20
|
+
? error.code
|
|
21
|
+
: "statusCode" in error
|
|
22
|
+
? error.statusCode
|
|
23
|
+
: undefined;
|
|
24
|
+
if (!errorCode)
|
|
25
|
+
return null;
|
|
26
|
+
// Azure-specific error codes
|
|
27
|
+
switch (errorCode) {
|
|
28
|
+
case "BlobNotFound":
|
|
29
|
+
case "ContainerNotFound":
|
|
30
|
+
case "InvalidBlobOrBlock":
|
|
31
|
+
return "client_error";
|
|
32
|
+
case "ContainerAlreadyExists":
|
|
33
|
+
case "BlobAlreadyExists":
|
|
34
|
+
return "client_error";
|
|
35
|
+
case "InvalidBlockId":
|
|
36
|
+
case "InvalidBlockList":
|
|
37
|
+
case "InvalidBlobType":
|
|
38
|
+
return "client_error";
|
|
39
|
+
case "RequestBodyTooLarge":
|
|
40
|
+
case "InvalidHeaderValue":
|
|
41
|
+
return "client_error";
|
|
42
|
+
case "AuthenticationFailed":
|
|
43
|
+
case "InvalidAuthenticationInfo":
|
|
44
|
+
return "authentication_error";
|
|
45
|
+
case "AccountIsDisabled":
|
|
46
|
+
return "authorization_error";
|
|
47
|
+
case "InsufficientAccountPermissions":
|
|
48
|
+
return "authorization_error";
|
|
49
|
+
case "OperationTimedOut":
|
|
50
|
+
case "ServerBusy":
|
|
51
|
+
case "InternalError":
|
|
52
|
+
return "server_error";
|
|
53
|
+
default:
|
|
54
|
+
// Check for HTTP status codes
|
|
55
|
+
if (typeof errorCode === "number") {
|
|
56
|
+
if (errorCode >= 500)
|
|
57
|
+
return "server_error";
|
|
58
|
+
if (errorCode === 429)
|
|
59
|
+
return "throttling_error";
|
|
60
|
+
if (errorCode === 403)
|
|
61
|
+
return "authorization_error";
|
|
62
|
+
if (errorCode === 401)
|
|
63
|
+
return "authentication_error";
|
|
64
|
+
if (errorCode >= 400)
|
|
65
|
+
return "client_error";
|
|
66
|
+
}
|
|
67
|
+
return null; // Fall back to generic classification
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
// Azure-specific error tracker
|
|
71
|
+
export const trackAzureError = createStorageErrorTracker(STORAGE_TYPE, azureMetrics, classifyAzureError);
|
|
72
|
+
// Azure-specific observability layer
|
|
73
|
+
export const AzureObservabilityLayer = Layer.mergeAll(AzureTracingLayer);
|
|
74
|
+
// ============================================================================
|
|
75
|
+
// Azure Utility Functions
|
|
76
|
+
// ============================================================================
|
|
77
|
+
export const withAzureUploadMetrics = (uploadId, effect) => withUploadMetrics(azureMetrics, uploadId, effect);
|
|
78
|
+
export const withAzureApiMetrics = (operation, effect) => withApiMetrics(azureMetrics, operation, effect);
|
|
79
|
+
export const withAzureTimingMetrics = withTimingMetrics;
|
|
80
|
+
export const withAzureOperationMetrics = (operation, uploadId, effect, fileSize) => withStorageOperationMetrics(azureMetrics, operation, uploadId, effect, fileSize);
|
|
81
|
+
// Azure-specific span wrapper
|
|
82
|
+
export const withAzureSpan = (operation, attributes) => (effect) => withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
|
|
83
|
+
// Azure-specific logging functions
|
|
84
|
+
export const logAzureOperation = logStorageOperation.bind(null, STORAGE_TYPE);
|
|
85
|
+
export const logAzureUploadProgress = logUploadProgress.bind(null, STORAGE_TYPE);
|
|
86
|
+
export const logAzureUploadCompletion = logUploadCompletion.bind(null, STORAGE_TYPE);
|
|
87
|
+
export const logAzureContext = logWithContext;
|
|
88
|
+
// Export metrics for external access
|
|
89
|
+
export const { uploadRequestsTotal: azureUploadRequestsTotal, uploadPartsTotal: azureUploadPartsTotal, uploadSuccessTotal: azureUploadSuccessTotal, uploadErrorsTotal: azureUploadErrorsTotal, apiCallsTotal: azureApiCallsTotal, uploadDurationHistogram: azureUploadDurationHistogram, partUploadDurationHistogram: azurePartUploadDurationHistogram, fileSizeHistogram: azureFileSizeHistogram, partSizeHistogram: azurePartSizeHistogram, activeUploadsGauge: azureActiveUploadsGauge, uploadThroughputGauge: azureUploadThroughputGauge, uploadLatencySummary: azureUploadLatencySummary, } = azureMetrics;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { type Effect, Layer } from "effect";
|
|
2
|
+
import { type StorageMetrics } from "../core/metrics.js";
|
|
3
|
+
export declare const filesystemMetrics: {
|
|
4
|
+
uploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
5
|
+
activeUploadsGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
6
|
+
uploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
7
|
+
uploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
8
|
+
partUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
9
|
+
fileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
10
|
+
partSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
11
|
+
uploadRequestsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
12
|
+
uploadPartsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
13
|
+
uploadSuccessTotal: import("effect/Metric").Metric.Counter<number>;
|
|
14
|
+
uploadErrorsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
15
|
+
apiCallsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
16
|
+
};
|
|
17
|
+
export declare const FilesystemTracingLayer: Layer.Layer<{
|
|
18
|
+
serviceName: string;
|
|
19
|
+
}, never, never>;
|
|
20
|
+
export declare const trackFilesystemError: (operation: string, error: unknown, context?: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
21
|
+
export declare const FilesystemObservabilityLayer: Layer.Layer<{
|
|
22
|
+
serviceName: string;
|
|
23
|
+
}, never, never>;
|
|
24
|
+
export declare const withFilesystemUploadMetrics: <A, E, R>(uploadId: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
25
|
+
export declare const withFilesystemApiMetrics: <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
26
|
+
export declare const withFilesystemTimingMetrics: <A, E, R>(metric: import("effect/Metric").Metric.Histogram<number>, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
27
|
+
export declare const withFilesystemOperationMetrics: <A, E, R>(operation: string, uploadId: string, effect: Effect.Effect<A, E, R>, fileSize?: number) => Effect.Effect<A, E, R>;
|
|
28
|
+
export declare const withFilesystemSpan: <A, E, R>(operation: string, attributes?: Record<string, unknown>) => (effect: Effect.Effect<A, E, R>) => Effect.Effect<unknown, unknown, unknown>;
|
|
29
|
+
export declare const logFilesystemOperation: (operation: string, uploadId: string, metadata?: Record<string, unknown> | undefined) => Effect.Effect<void, never, never>;
|
|
30
|
+
export declare const logFilesystemUploadProgress: (uploadId: string, progress: {
|
|
31
|
+
uploadedBytes: number;
|
|
32
|
+
totalBytes: number;
|
|
33
|
+
partNumber?: number;
|
|
34
|
+
speed?: number;
|
|
35
|
+
}) => Effect.Effect<void, never, never>;
|
|
36
|
+
export declare const logFilesystemUploadCompletion: (uploadId: string, metrics: {
|
|
37
|
+
fileSize: number;
|
|
38
|
+
totalDurationMs: number;
|
|
39
|
+
partsCount?: number;
|
|
40
|
+
averagePartSize?: number;
|
|
41
|
+
throughputBps?: number;
|
|
42
|
+
retryCount?: number;
|
|
43
|
+
}) => Effect.Effect<void, never, never>;
|
|
44
|
+
export declare const logFilesystemContext: (message: string, context: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
45
|
+
export declare const filesystemUploadRequestsTotal: import("effect/Metric").Metric.Counter<number>, filesystemUploadPartsTotal: import("effect/Metric").Metric.Counter<number>, filesystemUploadSuccessTotal: import("effect/Metric").Metric.Counter<number>, filesystemUploadErrorsTotal: import("effect/Metric").Metric.Counter<number>, filesystemApiCallsTotal: import("effect/Metric").Metric.Counter<number>, filesystemUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, filesystemPartUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, filesystemFileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, filesystemPartSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, filesystemActiveUploadsGauge: import("effect/Metric").Metric.Gauge<number>, filesystemUploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>, filesystemUploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
46
|
+
export type FilesystemMetrics = StorageMetrics;
|
|
47
|
+
//# sourceMappingURL=filesystem.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../src/storage/filesystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAW5C,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAgB/E,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;CAAqC,CAAC;AAGpE,eAAO,MAAM,sBAAsB;;gBAA0C,CAAC;AA0C9E,eAAO,MAAM,oBAAoB,6GAIhC,CAAC;AAGF,eAAO,MAAM,4BAA4B;;gBAGxC,CAAC;AAMF,eAAO,MAAM,2BAA2B,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EACjD,UAAU,MAAM,EAChB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,2BAC2B,CAAC;AAE5D,eAAO,MAAM,wBAAwB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAC9C,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,2BACyB,CAAC;AAE1D,eAAO,MAAM,2BAA2B,+HAAoB,CAAC;AAE7D,eAAO,MAAM,8BAA8B,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EACpD,WAAW,MAAM,EACjB,UAAU,MAAM,EAChB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,WAAW,MAAM,2BAQhB,CAAC;AAGJ,eAAO,MAAM,kBAAkB,GAC5B,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,MAAM,EAAE,aAAa,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAChE,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,6CAC+B,CAAC;AAGjE,eAAO,MAAM,sBAAsB,4HAGlC,CAAC;AACF,eAAO,MAAM,2BAA2B;;;;;uCAGvC,CAAC;AACF,eAAO,MAAM,6BAA6B;;;;;;;uCAGzC,CAAC;AACF,eAAO,MAAM,oBAAoB,0FAAiB,CAAC;AAGnD,eAAO,MACgB,6BAA6B,kDAChC,0BAA0B,kDACxB,4BAA4B,kDAC7B,2BAA2B,kDAC/B,uBAAuB,kDACb,iCAAiC,sJAC7B,qCAAqC,sJAC/C,2BAA2B,sJAC3B,2BAA2B,sJAC1B,4BAA4B,gDACzB,+BAA+B,gDAChC,8BAA8B,gDACjC,CAAC;AAGtB,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { Layer } from "effect";
|
|
2
|
+
import { createStorageErrorTracker, } from "../core/errors.js";
|
|
3
|
+
import { logStorageOperation, logUploadCompletion, logUploadProgress, logWithContext, } from "../core/logging.js";
|
|
4
|
+
import { createStorageMetrics } from "../core/metrics.js";
|
|
5
|
+
import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
|
|
6
|
+
import { withApiMetrics, withStorageOperationMetrics, withTimingMetrics, withUploadMetrics, } from "../core/utilities.js";
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// Filesystem Storage-Specific Observability
|
|
9
|
+
// ============================================================================
|
|
10
|
+
const STORAGE_TYPE = "filesystem";
|
|
11
|
+
// Filesystem-specific metrics
|
|
12
|
+
export const filesystemMetrics = createStorageMetrics(STORAGE_TYPE);
|
|
13
|
+
// Filesystem-specific tracing layer
|
|
14
|
+
export const FilesystemTracingLayer = createStorageTracingLayer(STORAGE_TYPE);
|
|
15
|
+
// Filesystem-specific error classification
|
|
16
|
+
const classifyFilesystemError = (error) => {
|
|
17
|
+
if (!error || typeof error !== "object")
|
|
18
|
+
return null;
|
|
19
|
+
const errorCode = "code" in error ? error.code : undefined;
|
|
20
|
+
if (!errorCode)
|
|
21
|
+
return null;
|
|
22
|
+
// Node.js filesystem error codes
|
|
23
|
+
switch (errorCode) {
|
|
24
|
+
case "ENOENT": // File/directory not found
|
|
25
|
+
case "ENOTDIR": // Not a directory
|
|
26
|
+
return "client_error";
|
|
27
|
+
case "EEXIST": // File/directory already exists
|
|
28
|
+
return "client_error";
|
|
29
|
+
case "EISDIR": // Is a directory
|
|
30
|
+
return "client_error";
|
|
31
|
+
case "EINVAL": // Invalid argument
|
|
32
|
+
case "ENAMETOOLONG": // Filename too long
|
|
33
|
+
return "client_error";
|
|
34
|
+
case "EACCES": // Permission denied
|
|
35
|
+
case "EPERM": // Operation not permitted
|
|
36
|
+
return "authorization_error";
|
|
37
|
+
case "ENOSPC": // No space left on device
|
|
38
|
+
case "EDQUOT": // Disk quota exceeded
|
|
39
|
+
return "server_error";
|
|
40
|
+
case "EIO": // I/O error
|
|
41
|
+
case "EROFS": // Read-only filesystem
|
|
42
|
+
case "EMFILE": // Too many open files
|
|
43
|
+
case "ENFILE": // File table overflow
|
|
44
|
+
return "server_error";
|
|
45
|
+
case "EBUSY": // Device or resource busy
|
|
46
|
+
return "throttling_error";
|
|
47
|
+
default:
|
|
48
|
+
return null; // Fall back to generic classification
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
// Filesystem-specific error tracker
|
|
52
|
+
export const trackFilesystemError = createStorageErrorTracker(STORAGE_TYPE, filesystemMetrics, classifyFilesystemError);
|
|
53
|
+
// Filesystem-specific observability layer
|
|
54
|
+
export const FilesystemObservabilityLayer = Layer.mergeAll(FilesystemTracingLayer);
|
|
55
|
+
// ============================================================================
|
|
56
|
+
// Filesystem Utility Functions
|
|
57
|
+
// ============================================================================
|
|
58
|
+
export const withFilesystemUploadMetrics = (uploadId, effect) => withUploadMetrics(filesystemMetrics, uploadId, effect);
|
|
59
|
+
export const withFilesystemApiMetrics = (operation, effect) => withApiMetrics(filesystemMetrics, operation, effect);
|
|
60
|
+
export const withFilesystemTimingMetrics = withTimingMetrics;
|
|
61
|
+
export const withFilesystemOperationMetrics = (operation, uploadId, effect, fileSize) => withStorageOperationMetrics(filesystemMetrics, operation, uploadId, effect, fileSize);
|
|
62
|
+
// Filesystem-specific span wrapper
|
|
63
|
+
export const withFilesystemSpan = (operation, attributes) => (effect) => withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
|
|
64
|
+
// Filesystem-specific logging functions
|
|
65
|
+
export const logFilesystemOperation = logStorageOperation.bind(null, STORAGE_TYPE);
|
|
66
|
+
export const logFilesystemUploadProgress = logUploadProgress.bind(null, STORAGE_TYPE);
|
|
67
|
+
export const logFilesystemUploadCompletion = logUploadCompletion.bind(null, STORAGE_TYPE);
|
|
68
|
+
export const logFilesystemContext = logWithContext;
|
|
69
|
+
// Export metrics for external access
|
|
70
|
+
export const { uploadRequestsTotal: filesystemUploadRequestsTotal, uploadPartsTotal: filesystemUploadPartsTotal, uploadSuccessTotal: filesystemUploadSuccessTotal, uploadErrorsTotal: filesystemUploadErrorsTotal, apiCallsTotal: filesystemApiCallsTotal, uploadDurationHistogram: filesystemUploadDurationHistogram, partUploadDurationHistogram: filesystemPartUploadDurationHistogram, fileSizeHistogram: filesystemFileSizeHistogram, partSizeHistogram: filesystemPartSizeHistogram, activeUploadsGauge: filesystemActiveUploadsGauge, uploadThroughputGauge: filesystemUploadThroughputGauge, uploadLatencySummary: filesystemUploadLatencySummary, } = filesystemMetrics;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { type Effect, Layer } from "effect";
|
|
2
|
+
import { type StorageMetrics } from "../core/metrics.js";
|
|
3
|
+
export declare const gcsMetrics: {
|
|
4
|
+
uploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
5
|
+
activeUploadsGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
6
|
+
uploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
7
|
+
uploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
8
|
+
partUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
9
|
+
fileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
10
|
+
partSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
11
|
+
uploadRequestsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
12
|
+
uploadPartsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
13
|
+
uploadSuccessTotal: import("effect/Metric").Metric.Counter<number>;
|
|
14
|
+
uploadErrorsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
15
|
+
apiCallsTotal: import("effect/Metric").Metric.Counter<number>;
|
|
16
|
+
};
|
|
17
|
+
export declare const GCSTracingLayer: Layer.Layer<{
|
|
18
|
+
serviceName: string;
|
|
19
|
+
}, never, never>;
|
|
20
|
+
export declare const trackGCSError: (operation: string, error: unknown, context?: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
21
|
+
export declare const GCSObservabilityLayer: Layer.Layer<{
|
|
22
|
+
serviceName: string;
|
|
23
|
+
}, never, never>;
|
|
24
|
+
export declare const withGCSUploadMetrics: <A, E, R>(uploadId: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
25
|
+
export declare const withGCSApiMetrics: <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
26
|
+
export declare const withGCSTimingMetrics: <A, E, R>(metric: import("effect/Metric").Metric.Histogram<number>, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
27
|
+
export declare const withGCSOperationMetrics: <A, E, R>(operation: string, uploadId: string, effect: Effect.Effect<A, E, R>, fileSize?: number) => Effect.Effect<A, E, R>;
|
|
28
|
+
export declare const withGCSSpan: <A, E, R>(operation: string, attributes?: Record<string, unknown>) => (effect: Effect.Effect<A, E, R>) => Effect.Effect<unknown, unknown, unknown>;
|
|
29
|
+
export declare const logGCSOperation: (operation: string, uploadId: string, metadata?: Record<string, unknown> | undefined) => Effect.Effect<void, never, never>;
|
|
30
|
+
export declare const logGCSUploadProgress: (uploadId: string, progress: {
|
|
31
|
+
uploadedBytes: number;
|
|
32
|
+
totalBytes: number;
|
|
33
|
+
partNumber?: number;
|
|
34
|
+
speed?: number;
|
|
35
|
+
}) => Effect.Effect<void, never, never>;
|
|
36
|
+
export declare const logGCSUploadCompletion: (uploadId: string, metrics: {
|
|
37
|
+
fileSize: number;
|
|
38
|
+
totalDurationMs: number;
|
|
39
|
+
partsCount?: number;
|
|
40
|
+
averagePartSize?: number;
|
|
41
|
+
throughputBps?: number;
|
|
42
|
+
retryCount?: number;
|
|
43
|
+
}) => Effect.Effect<void, never, never>;
|
|
44
|
+
export declare const logGCSContext: (message: string, context: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
45
|
+
export declare const gcsUploadRequestsTotal: import("effect/Metric").Metric.Counter<number>, gcsUploadPartsTotal: import("effect/Metric").Metric.Counter<number>, gcsUploadSuccessTotal: import("effect/Metric").Metric.Counter<number>, gcsUploadErrorsTotal: import("effect/Metric").Metric.Counter<number>, gcsApiCallsTotal: import("effect/Metric").Metric.Counter<number>, gcsUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, gcsPartUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, gcsFileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, gcsPartSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>, gcsActiveUploadsGauge: import("effect/Metric").Metric.Gauge<number>, gcsUploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>, gcsUploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
46
|
+
export type GCSMetrics = StorageMetrics;
|
|
47
|
+
//# sourceMappingURL=gcs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gcs.d.ts","sourceRoot":"","sources":["../../src/storage/gcs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAW5C,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAgB/E,eAAO,MAAM,UAAU;;;;;;;;;;;;;CAAqC,CAAC;AAG7D,eAAO,MAAM,eAAe;;gBAA0C,CAAC;AAwDvE,eAAO,MAAM,aAAa,6GAIzB,CAAC;AAGF,eAAO,MAAM,qBAAqB;;gBAGjC,CAAC;AAMF,eAAO,MAAM,oBAAoB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAC1C,UAAU,MAAM,EAChB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,2BACoB,CAAC;AAErD,eAAO,MAAM,iBAAiB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EACvC,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,2BACkB,CAAC;AAEnD,eAAO,MAAM,oBAAoB,+HAAoB,CAAC;AAEtD,eAAO,MAAM,uBAAuB,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAC7C,WAAW,MAAM,EACjB,UAAU,MAAM,EAChB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,WAAW,MAAM,2BAQhB,CAAC;AAGJ,eAAO,MAAM,WAAW,GACrB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,MAAM,EAAE,aAAa,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAChE,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,6CAC+B,CAAC;AAGjE,eAAO,MAAM,eAAe,4HAA+C,CAAC;AAC5E,eAAO,MAAM,oBAAoB;;;;;uCAA6C,CAAC;AAC/E,eAAO,MAAM,sBAAsB;;;;;;;uCAGlC,CAAC;AACF,eAAO,MAAM,aAAa,0FAAiB,CAAC;AAG5C,eAAO,MACgB,sBAAsB,kDACzB,mBAAmB,kDACjB,qBAAqB,kDACtB,oBAAoB,kDACxB,gBAAgB,kDACN,0BAA0B,sJACtB,8BAA8B,sJACxC,oBAAoB,sJACpB,oBAAoB,sJACnB,qBAAqB,gDAClB,wBAAwB,gDACzB,uBAAuB,gDACjC,CAAC;AAGf,MAAM,MAAM,UAAU,GAAG,cAAc,CAAC"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { Layer } from "effect";
|
|
2
|
+
import { createStorageErrorTracker, } from "../core/errors.js";
|
|
3
|
+
import { logStorageOperation, logUploadCompletion, logUploadProgress, logWithContext, } from "../core/logging.js";
|
|
4
|
+
import { createStorageMetrics } from "../core/metrics.js";
|
|
5
|
+
import { createStorageTracingLayer, withStorageSpan } from "../core/tracing.js";
|
|
6
|
+
import { withApiMetrics, withStorageOperationMetrics, withTimingMetrics, withUploadMetrics, } from "../core/utilities.js";
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// Google Cloud Storage-Specific Observability
|
|
9
|
+
// ============================================================================
|
|
10
|
+
const STORAGE_TYPE = "gcs";
|
|
11
|
+
// GCS-specific metrics
|
|
12
|
+
export const gcsMetrics = createStorageMetrics(STORAGE_TYPE);
|
|
13
|
+
// GCS-specific tracing layer
|
|
14
|
+
export const GCSTracingLayer = createStorageTracingLayer(STORAGE_TYPE);
|
|
15
|
+
// GCS-specific error classification
|
|
16
|
+
const classifyGCSError = (error) => {
|
|
17
|
+
if (!error || typeof error !== "object")
|
|
18
|
+
return null;
|
|
19
|
+
const errorCode = "code" in error ? error.code : "status" in error ? error.status : undefined;
|
|
20
|
+
if (!errorCode)
|
|
21
|
+
return null;
|
|
22
|
+
// GCS-specific error codes
|
|
23
|
+
switch (errorCode) {
|
|
24
|
+
case "NoSuchBucket":
|
|
25
|
+
case "NoSuchKey":
|
|
26
|
+
case "NoSuchUpload":
|
|
27
|
+
return "client_error";
|
|
28
|
+
case "BucketAlreadyOwnedByYou":
|
|
29
|
+
case "BucketNotEmpty":
|
|
30
|
+
return "client_error";
|
|
31
|
+
case "InvalidBucketName":
|
|
32
|
+
case "InvalidArgument":
|
|
33
|
+
case "InvalidPart":
|
|
34
|
+
case "InvalidPartOrder":
|
|
35
|
+
return "client_error";
|
|
36
|
+
case "EntityTooSmall":
|
|
37
|
+
case "EntityTooLarge":
|
|
38
|
+
return "client_error";
|
|
39
|
+
case "MalformedPolicy":
|
|
40
|
+
return "client_error";
|
|
41
|
+
case "Unauthorized":
|
|
42
|
+
case "AuthenticationRequired":
|
|
43
|
+
return "authentication_error";
|
|
44
|
+
case "Forbidden":
|
|
45
|
+
case "AccessDenied":
|
|
46
|
+
return "authorization_error";
|
|
47
|
+
case "TooManyRequests":
|
|
48
|
+
case "RateLimitExceeded":
|
|
49
|
+
return "throttling_error";
|
|
50
|
+
case "InternalError":
|
|
51
|
+
case "ServiceUnavailable":
|
|
52
|
+
case "BackendError":
|
|
53
|
+
return "server_error";
|
|
54
|
+
default:
|
|
55
|
+
// Check for HTTP status codes
|
|
56
|
+
if (typeof errorCode === "number") {
|
|
57
|
+
if (errorCode >= 500)
|
|
58
|
+
return "server_error";
|
|
59
|
+
if (errorCode === 429)
|
|
60
|
+
return "throttling_error";
|
|
61
|
+
if (errorCode === 403)
|
|
62
|
+
return "authorization_error";
|
|
63
|
+
if (errorCode === 401)
|
|
64
|
+
return "authentication_error";
|
|
65
|
+
if (errorCode >= 400)
|
|
66
|
+
return "client_error";
|
|
67
|
+
}
|
|
68
|
+
return null; // Fall back to generic classification
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
// GCS-specific error tracker
|
|
72
|
+
export const trackGCSError = createStorageErrorTracker(STORAGE_TYPE, gcsMetrics, classifyGCSError);
|
|
73
|
+
// GCS-specific observability layer
|
|
74
|
+
export const GCSObservabilityLayer = Layer.mergeAll(GCSTracingLayer);
|
|
75
|
+
// ============================================================================
|
|
76
|
+
// GCS Utility Functions
|
|
77
|
+
// ============================================================================
|
|
78
|
+
export const withGCSUploadMetrics = (uploadId, effect) => withUploadMetrics(gcsMetrics, uploadId, effect);
|
|
79
|
+
export const withGCSApiMetrics = (operation, effect) => withApiMetrics(gcsMetrics, operation, effect);
|
|
80
|
+
export const withGCSTimingMetrics = withTimingMetrics;
|
|
81
|
+
export const withGCSOperationMetrics = (operation, uploadId, effect, fileSize) => withStorageOperationMetrics(gcsMetrics, operation, uploadId, effect, fileSize);
|
|
82
|
+
// GCS-specific span wrapper
|
|
83
|
+
export const withGCSSpan = (operation, attributes) => (effect) => withStorageSpan(operation, STORAGE_TYPE, attributes)(effect);
|
|
84
|
+
// GCS-specific logging functions
|
|
85
|
+
export const logGCSOperation = logStorageOperation.bind(null, STORAGE_TYPE);
|
|
86
|
+
export const logGCSUploadProgress = logUploadProgress.bind(null, STORAGE_TYPE);
|
|
87
|
+
export const logGCSUploadCompletion = logUploadCompletion.bind(null, STORAGE_TYPE);
|
|
88
|
+
export const logGCSContext = logWithContext;
|
|
89
|
+
// Export metrics for external access
|
|
90
|
+
export const { uploadRequestsTotal: gcsUploadRequestsTotal, uploadPartsTotal: gcsUploadPartsTotal, uploadSuccessTotal: gcsUploadSuccessTotal, uploadErrorsTotal: gcsUploadErrorsTotal, apiCallsTotal: gcsApiCallsTotal, uploadDurationHistogram: gcsUploadDurationHistogram, partUploadDurationHistogram: gcsPartUploadDurationHistogram, fileSizeHistogram: gcsFileSizeHistogram, partSizeHistogram: gcsPartSizeHistogram, activeUploadsGauge: gcsActiveUploadsGauge, uploadThroughputGauge: gcsUploadThroughputGauge, uploadLatencySummary: gcsUploadLatencySummary, } = gcsMetrics;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AACA,cAAc,SAAS,CAAC;AACxB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC"}
|