@uploadista/observability 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +0 -0
- package/LICENSE +21 -0
- package/dist/core/errors.d.ts +8 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +108 -0
- package/dist/core/index.d.ts +8 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +8 -0
- package/dist/core/layers.d.ts +104 -0
- package/dist/core/layers.d.ts.map +1 -0
- package/dist/core/layers.js +110 -0
- package/dist/core/logging.d.ts +18 -0
- package/dist/core/logging.d.ts.map +1 -0
- package/dist/core/logging.js +41 -0
- package/dist/core/metrics.d.ts +37 -0
- package/dist/core/metrics.d.ts.map +1 -0
- package/dist/core/metrics.js +72 -0
- package/dist/core/testing.d.ts +43 -0
- package/dist/core/testing.d.ts.map +1 -0
- package/dist/core/testing.js +93 -0
- package/dist/core/tracing.d.ts +19 -0
- package/dist/core/tracing.d.ts.map +1 -0
- package/dist/core/tracing.js +43 -0
- package/dist/core/utilities.d.ts +11 -0
- package/dist/core/utilities.d.ts.map +1 -0
- package/dist/core/utilities.js +41 -0
- package/dist/flow/errors.d.ts +15 -0
- package/dist/flow/errors.d.ts.map +1 -0
- package/dist/flow/errors.js +66 -0
- package/dist/flow/index.d.ts +6 -0
- package/dist/flow/index.d.ts.map +1 -0
- package/dist/flow/index.js +6 -0
- package/dist/flow/layers.d.ts +40 -0
- package/dist/flow/layers.d.ts.map +1 -0
- package/dist/flow/layers.js +94 -0
- package/dist/flow/metrics.d.ts +52 -0
- package/dist/flow/metrics.d.ts.map +1 -0
- package/dist/flow/metrics.js +89 -0
- package/dist/flow/testing.d.ts +11 -0
- package/dist/flow/testing.d.ts.map +1 -0
- package/dist/flow/testing.js +27 -0
- package/dist/flow/tracing.d.ts +35 -0
- package/dist/flow/tracing.d.ts.map +1 -0
- package/dist/flow/tracing.js +42 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/service/metrics.d.ts +23 -0
- package/dist/service/metrics.d.ts.map +1 -0
- package/dist/service/metrics.js +17 -0
- package/dist/storage/azure.d.ts +47 -0
- package/dist/storage/azure.d.ts.map +1 -0
- package/dist/storage/azure.js +89 -0
- package/dist/storage/filesystem.d.ts +47 -0
- package/dist/storage/filesystem.d.ts.map +1 -0
- package/dist/storage/filesystem.js +70 -0
- package/dist/storage/gcs.d.ts +47 -0
- package/dist/storage/gcs.d.ts.map +1 -0
- package/dist/storage/gcs.js +90 -0
- package/dist/storage/index.d.ts +5 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/s3.d.ts +47 -0
- package/dist/storage/s3.d.ts.map +1 -0
- package/dist/storage/s3.js +67 -0
- package/dist/test-observability.d.ts +12 -0
- package/dist/test-observability.d.ts.map +1 -0
- package/dist/test-observability.js +153 -0
- package/dist/upload/errors.d.ts +16 -0
- package/dist/upload/errors.d.ts.map +1 -0
- package/dist/upload/errors.js +107 -0
- package/dist/upload/index.d.ts +6 -0
- package/dist/upload/index.d.ts.map +1 -0
- package/dist/upload/index.js +6 -0
- package/dist/upload/layers.d.ts +32 -0
- package/dist/upload/layers.d.ts.map +1 -0
- package/dist/upload/layers.js +63 -0
- package/dist/upload/metrics.d.ts +46 -0
- package/dist/upload/metrics.d.ts.map +1 -0
- package/dist/upload/metrics.js +80 -0
- package/dist/upload/testing.d.ts +32 -0
- package/dist/upload/testing.d.ts.map +1 -0
- package/dist/upload/testing.js +52 -0
- package/dist/upload/tracing.d.ts +25 -0
- package/dist/upload/tracing.d.ts.map +1 -0
- package/dist/upload/tracing.js +35 -0
- package/package.json +37 -0
- package/src/core/errors.ts +187 -0
- package/src/core/index.ts +9 -0
- package/src/core/layers.ts +205 -0
- package/src/core/logging.ts +81 -0
- package/src/core/metrics.ts +108 -0
- package/src/core/testing.ts +142 -0
- package/src/core/tracing.ts +67 -0
- package/src/core/utilities.ts +133 -0
- package/src/flow/errors.ts +95 -0
- package/src/flow/index.ts +17 -0
- package/src/flow/layers.ts +131 -0
- package/src/flow/metrics.ts +130 -0
- package/src/flow/testing.ts +33 -0
- package/src/flow/tracing.ts +72 -0
- package/src/index.ts +31 -0
- package/src/service/metrics.ts +31 -0
- package/src/storage/azure.ts +163 -0
- package/src/storage/filesystem.ts +153 -0
- package/src/storage/gcs.ts +161 -0
- package/src/storage/index.ts +5 -0
- package/src/storage/s3.ts +136 -0
- package/src/test-observability.ts +234 -0
- package/src/upload/errors.ts +166 -0
- package/src/upload/index.ts +12 -0
- package/src/upload/layers.ts +88 -0
- package/src/upload/metrics.ts +118 -0
- package/src/upload/testing.ts +60 -0
- package/src/upload/tracing.ts +58 -0
- package/tsconfig.json +10 -0
- package/tsconfig.tsbuildinfo +1 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { Metric, MetricBoundaries } from "effect";
|
|
2
|
+
// ============================================================================
|
|
3
|
+
// Upload Server Metrics
|
|
4
|
+
// ============================================================================
|
|
5
|
+
/**
|
|
6
|
+
* Upload server metrics for tracking upload operations
|
|
7
|
+
*/
|
|
8
|
+
export const createUploadServerMetrics = () => ({
|
|
9
|
+
// Counter metrics
|
|
10
|
+
uploadCreatedTotal: Metric.counter("upload_created_total", {
|
|
11
|
+
description: "Total number of uploads created",
|
|
12
|
+
}),
|
|
13
|
+
uploadCompletedTotal: Metric.counter("upload_completed_total", {
|
|
14
|
+
description: "Total number of uploads completed successfully",
|
|
15
|
+
}),
|
|
16
|
+
uploadFailedTotal: Metric.counter("upload_failed_total", {
|
|
17
|
+
description: "Total number of uploads that failed",
|
|
18
|
+
}),
|
|
19
|
+
chunkUploadedTotal: Metric.counter("chunk_uploaded_total", {
|
|
20
|
+
description: "Total number of chunks uploaded",
|
|
21
|
+
}),
|
|
22
|
+
uploadFromUrlTotal: Metric.counter("upload_from_url_total", {
|
|
23
|
+
description: "Total number of URL-based uploads",
|
|
24
|
+
}),
|
|
25
|
+
uploadFromUrlSuccessTotal: Metric.counter("upload_from_url_success_total", {
|
|
26
|
+
description: "Total number of successful URL-based uploads",
|
|
27
|
+
}),
|
|
28
|
+
uploadFromUrlFailedTotal: Metric.counter("upload_from_url_failed_total", {
|
|
29
|
+
description: "Total number of failed URL-based uploads",
|
|
30
|
+
}),
|
|
31
|
+
// Histogram metrics
|
|
32
|
+
uploadDurationHistogram: Metric.histogram("upload_duration_seconds", MetricBoundaries.exponential({
|
|
33
|
+
start: 0.01, // 10ms
|
|
34
|
+
factor: 2,
|
|
35
|
+
count: 20, // Up to ~10 seconds
|
|
36
|
+
}), "Duration of complete upload operations in seconds"),
|
|
37
|
+
chunkUploadDurationHistogram: Metric.histogram("chunk_upload_duration_seconds", MetricBoundaries.exponential({
|
|
38
|
+
start: 0.001, // 1ms
|
|
39
|
+
factor: 2,
|
|
40
|
+
count: 15, // Up to ~32 seconds
|
|
41
|
+
}), "Duration of individual chunk uploads in seconds"),
|
|
42
|
+
uploadFileSizeHistogram: Metric.histogram("upload_file_size_bytes", MetricBoundaries.exponential({
|
|
43
|
+
start: 1024, // 1KB
|
|
44
|
+
factor: 2,
|
|
45
|
+
count: 25, // Up to ~33GB
|
|
46
|
+
}), "Size of uploaded files in bytes"),
|
|
47
|
+
chunkSizeHistogram: Metric.histogram("chunk_size_bytes", MetricBoundaries.linear({
|
|
48
|
+
start: 262_144, // 256KB
|
|
49
|
+
width: 262_144, // 256KB increments
|
|
50
|
+
count: 20, // Up to ~5MB
|
|
51
|
+
}), "Size of uploaded chunks in bytes"),
|
|
52
|
+
// Gauge metrics
|
|
53
|
+
activeUploadsGauge: Metric.gauge("active_uploads", {
|
|
54
|
+
description: "Number of currently active uploads",
|
|
55
|
+
}),
|
|
56
|
+
uploadThroughputGauge: Metric.gauge("upload_throughput_bytes_per_second", {
|
|
57
|
+
description: "Current upload throughput in bytes per second",
|
|
58
|
+
}),
|
|
59
|
+
// Summary metrics for latency percentiles
|
|
60
|
+
uploadLatencySummary: Metric.summary({
|
|
61
|
+
name: "upload_latency_seconds",
|
|
62
|
+
maxAge: "10 minutes",
|
|
63
|
+
maxSize: 1000,
|
|
64
|
+
error: 0.01,
|
|
65
|
+
quantiles: [0.5, 0.9, 0.95, 0.99],
|
|
66
|
+
description: "Upload operation latency percentiles",
|
|
67
|
+
}),
|
|
68
|
+
chunkLatencySummary: Metric.summary({
|
|
69
|
+
name: "chunk_latency_seconds",
|
|
70
|
+
maxAge: "10 minutes",
|
|
71
|
+
maxSize: 1000,
|
|
72
|
+
error: 0.01,
|
|
73
|
+
quantiles: [0.5, 0.9, 0.95, 0.99],
|
|
74
|
+
description: "Chunk upload latency percentiles",
|
|
75
|
+
}),
|
|
76
|
+
});
|
|
77
|
+
/**
|
|
78
|
+
* Default upload server metrics instance
|
|
79
|
+
*/
|
|
80
|
+
export const uploadServerMetrics = createUploadServerMetrics();
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { Layer } from "effect";
|
|
2
|
+
import { UploadObservability } from "../core/layers.js";
|
|
3
|
+
/**
|
|
4
|
+
* Create a test upload observability layer that doesn't actually emit metrics
|
|
5
|
+
* but validates that the observability system is wired correctly
|
|
6
|
+
*/
|
|
7
|
+
export declare const UploadObservabilityTest: Layer.Layer<UploadObservability, never, never>;
|
|
8
|
+
/**
|
|
9
|
+
* Get metrics for validation (useful for testing metric definitions)
|
|
10
|
+
*/
|
|
11
|
+
export declare const getTestMetrics: () => {
|
|
12
|
+
uploadCreatedTotal: import("effect/Metric").Metric.Counter<number>;
|
|
13
|
+
uploadCompletedTotal: import("effect/Metric").Metric.Counter<number>;
|
|
14
|
+
uploadFailedTotal: import("effect/Metric").Metric.Counter<number>;
|
|
15
|
+
chunkUploadedTotal: import("effect/Metric").Metric.Counter<number>;
|
|
16
|
+
uploadFromUrlTotal: import("effect/Metric").Metric.Counter<number>;
|
|
17
|
+
uploadFromUrlSuccessTotal: import("effect/Metric").Metric.Counter<number>;
|
|
18
|
+
uploadFromUrlFailedTotal: import("effect/Metric").Metric.Counter<number>;
|
|
19
|
+
uploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
20
|
+
chunkUploadDurationHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
21
|
+
uploadFileSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
22
|
+
chunkSizeHistogram: import("effect/Metric").Metric<import("effect/MetricKeyType").MetricKeyType.Histogram, number, import("effect/MetricState").MetricState.Histogram>;
|
|
23
|
+
activeUploadsGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
24
|
+
uploadThroughputGauge: import("effect/Metric").Metric.Gauge<number>;
|
|
25
|
+
uploadLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
26
|
+
chunkLatencySummary: import("effect/Metric").Metric.Summary<number>;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Validate that all required metrics exist
|
|
30
|
+
*/
|
|
31
|
+
export declare const validateMetricsExist: () => boolean;
|
|
32
|
+
//# sourceMappingURL=testing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../src/upload/testing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAOxD;;;GAGG;AACH,eAAO,MAAM,uBAAuB,gDASlC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;CAAoC,CAAC;AAEhE;;GAEG;AACH,eAAO,MAAM,oBAAoB,eA4BhC,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { Layer } from "effect";
|
|
2
|
+
import { UploadObservability } from "../core/layers.js";
|
|
3
|
+
import { createUploadServerMetrics } from "./metrics.js";
|
|
4
|
+
// ============================================================================
|
|
5
|
+
// Upload Observability Testing Utilities
|
|
6
|
+
// ============================================================================
|
|
7
|
+
/**
|
|
8
|
+
* Create a test upload observability layer that doesn't actually emit metrics
|
|
9
|
+
* but validates that the observability system is wired correctly
|
|
10
|
+
*/
|
|
11
|
+
export const UploadObservabilityTest = Layer.succeed(UploadObservability, {
|
|
12
|
+
serviceName: "uploadista-upload-server-test",
|
|
13
|
+
enabled: true,
|
|
14
|
+
metrics: {
|
|
15
|
+
uploadCreated: () => Promise.resolve(),
|
|
16
|
+
uploadCompleted: () => Promise.resolve(),
|
|
17
|
+
uploadFailed: () => Promise.resolve(),
|
|
18
|
+
chunkUploaded: () => Promise.resolve(),
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
/**
|
|
22
|
+
* Get metrics for validation (useful for testing metric definitions)
|
|
23
|
+
*/
|
|
24
|
+
export const getTestMetrics = () => createUploadServerMetrics();
|
|
25
|
+
/**
|
|
26
|
+
* Validate that all required metrics exist
|
|
27
|
+
*/
|
|
28
|
+
export const validateMetricsExist = () => {
|
|
29
|
+
const metrics = getTestMetrics();
|
|
30
|
+
const requiredMetrics = [
|
|
31
|
+
"uploadCreatedTotal",
|
|
32
|
+
"uploadCompletedTotal",
|
|
33
|
+
"uploadFailedTotal",
|
|
34
|
+
"chunkUploadedTotal",
|
|
35
|
+
"uploadFromUrlTotal",
|
|
36
|
+
"uploadFromUrlSuccessTotal",
|
|
37
|
+
"uploadFromUrlFailedTotal",
|
|
38
|
+
"uploadDurationHistogram",
|
|
39
|
+
"chunkUploadDurationHistogram",
|
|
40
|
+
"uploadFileSizeHistogram",
|
|
41
|
+
"chunkSizeHistogram",
|
|
42
|
+
"activeUploadsGauge",
|
|
43
|
+
"uploadThroughputGauge",
|
|
44
|
+
"uploadLatencySummary",
|
|
45
|
+
"chunkLatencySummary",
|
|
46
|
+
];
|
|
47
|
+
const missingMetrics = requiredMetrics.filter((name) => !(name in metrics));
|
|
48
|
+
if (missingMetrics.length > 0) {
|
|
49
|
+
throw new Error(`Missing required metrics: ${missingMetrics.join(", ")}`);
|
|
50
|
+
}
|
|
51
|
+
return true;
|
|
52
|
+
};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Wrap an Effect with an upload operation span
|
|
4
|
+
*/
|
|
5
|
+
export declare const withUploadSpan: <A, E, R>(operation: string, attributes?: Record<string, unknown>) => (effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
6
|
+
/**
|
|
7
|
+
* Add upload context to the current span
|
|
8
|
+
*/
|
|
9
|
+
export declare const withUploadContext: (context: {
|
|
10
|
+
uploadId?: string;
|
|
11
|
+
fileName?: string;
|
|
12
|
+
fileSize?: number;
|
|
13
|
+
storageId?: string;
|
|
14
|
+
mimeType?: string;
|
|
15
|
+
}) => Effect.Effect<void, never, never>;
|
|
16
|
+
/**
|
|
17
|
+
* Add chunk context to the current span
|
|
18
|
+
*/
|
|
19
|
+
export declare const withChunkContext: (context: {
|
|
20
|
+
uploadId: string;
|
|
21
|
+
chunkSize: number;
|
|
22
|
+
offset: number;
|
|
23
|
+
totalSize?: number;
|
|
24
|
+
}) => Effect.Effect<void, never, never>;
|
|
25
|
+
//# sourceMappingURL=tracing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tracing.d.ts","sourceRoot":"","sources":["../../src/upload/tracing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAMhC;;GAEG;AACH,eAAO,MAAM,cAAc,GACxB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,MAAM,EAAE,aAAa,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAChE,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAQpD,CAAC;AAEN;;GAEG;AACH,eAAO,MAAM,iBAAiB,GAAI,SAAS;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,sCAOG,CAAC;AAEL;;GAEG;AACH,eAAO,MAAM,gBAAgB,GAAI,SAAS;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,sCAUG,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
// ============================================================================
|
|
3
|
+
// Upload Tracing Utilities
|
|
4
|
+
// ============================================================================
|
|
5
|
+
/**
|
|
6
|
+
* Wrap an Effect with an upload operation span
|
|
7
|
+
*/
|
|
8
|
+
export const withUploadSpan = (operation, attributes) => (effect) => effect.pipe(Effect.withSpan(`upload-${operation}`, {
|
|
9
|
+
attributes: {
|
|
10
|
+
"upload.operation": operation,
|
|
11
|
+
...attributes,
|
|
12
|
+
},
|
|
13
|
+
}));
|
|
14
|
+
/**
|
|
15
|
+
* Add upload context to the current span
|
|
16
|
+
*/
|
|
17
|
+
export const withUploadContext = (context) => Effect.annotateCurrentSpan({
|
|
18
|
+
"upload.id": context.uploadId ?? "unknown",
|
|
19
|
+
"upload.file_name": context.fileName ?? "unknown",
|
|
20
|
+
"upload.file_size": context.fileSize?.toString() ?? "0",
|
|
21
|
+
"upload.storage_id": context.storageId ?? "unknown",
|
|
22
|
+
"upload.mime_type": context.mimeType ?? "unknown",
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* Add chunk context to the current span
|
|
26
|
+
*/
|
|
27
|
+
export const withChunkContext = (context) => Effect.annotateCurrentSpan({
|
|
28
|
+
"chunk.upload_id": context.uploadId,
|
|
29
|
+
"chunk.size": context.chunkSize.toString(),
|
|
30
|
+
"chunk.offset": context.offset.toString(),
|
|
31
|
+
"chunk.total_size": context.totalSize?.toString() ?? "0",
|
|
32
|
+
"chunk.progress": context.totalSize && context.totalSize > 0
|
|
33
|
+
? ((context.offset / context.totalSize) * 100).toFixed(2)
|
|
34
|
+
: "0",
|
|
35
|
+
});
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@uploadista/observability",
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.0.3",
|
|
5
|
+
"description": "Observability package for Uploadista",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Uploadista",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dependencies": {
|
|
16
|
+
"@effect/opentelemetry": "0.58.0",
|
|
17
|
+
"@opentelemetry/sdk-logs": "0.206.0",
|
|
18
|
+
"@opentelemetry/sdk-metrics": "2.1.0",
|
|
19
|
+
"@opentelemetry/sdk-trace-base": "2.1.0",
|
|
20
|
+
"@opentelemetry/sdk-trace-node": "2.1.0",
|
|
21
|
+
"@opentelemetry/sdk-trace-web": "2.1.0",
|
|
22
|
+
"effect": "3.18.4"
|
|
23
|
+
},
|
|
24
|
+
"devDependencies": {
|
|
25
|
+
"@cloudflare/workers-types": "4.20251011.0",
|
|
26
|
+
"@types/node": "24.8.1",
|
|
27
|
+
"typescript": "5.9.3",
|
|
28
|
+
"@uploadista/typescript-config": "0.0.3"
|
|
29
|
+
},
|
|
30
|
+
"scripts": {
|
|
31
|
+
"build": "tsc -b",
|
|
32
|
+
"format": "biome format --write ./src",
|
|
33
|
+
"lint": "biome lint --write ./src",
|
|
34
|
+
"check": "biome check --write ./src",
|
|
35
|
+
"clean": "rimraf -rf dist && rimraf -rf .turbo && rimraf tsconfig.tsbuildinfo"
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { Effect, Metric } from "effect";
|
|
2
|
+
import type { StorageMetrics } from "./metrics.js";
|
|
3
|
+
|
|
4
|
+
// ============================================================================
|
|
5
|
+
// Generic Storage Error Classification and Tracking
|
|
6
|
+
// ============================================================================
|
|
7
|
+
|
|
8
|
+
export type StorageErrorCategory =
|
|
9
|
+
| "network_error"
|
|
10
|
+
| "authentication_error"
|
|
11
|
+
| "authorization_error"
|
|
12
|
+
| "throttling_error"
|
|
13
|
+
| "server_error"
|
|
14
|
+
| "client_error"
|
|
15
|
+
| "unknown_error";
|
|
16
|
+
|
|
17
|
+
// Generic error classifier - can be extended per storage type
|
|
18
|
+
export const classifyStorageError = (error: unknown): StorageErrorCategory => {
|
|
19
|
+
if (!error || typeof error !== "object") return "unknown_error";
|
|
20
|
+
|
|
21
|
+
const errorCode = "code" in error ? error.code : undefined;
|
|
22
|
+
const errorName = "name" in error ? error.name : undefined;
|
|
23
|
+
const errorMessage =
|
|
24
|
+
error instanceof Error ? error.message.toLowerCase() : "";
|
|
25
|
+
|
|
26
|
+
// Network errors (common across all storage types)
|
|
27
|
+
if (
|
|
28
|
+
errorCode === "NetworkError" ||
|
|
29
|
+
errorCode === "ECONNRESET" ||
|
|
30
|
+
errorCode === "ENOTFOUND" ||
|
|
31
|
+
errorCode === "ETIMEDOUT" ||
|
|
32
|
+
errorMessage.indexOf("network") >= 0 ||
|
|
33
|
+
errorMessage.indexOf("timeout") >= 0
|
|
34
|
+
) {
|
|
35
|
+
return "network_error";
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Authentication errors (common patterns)
|
|
39
|
+
if (
|
|
40
|
+
errorCode === "InvalidAccessKeyId" ||
|
|
41
|
+
errorCode === "SignatureDoesNotMatch" ||
|
|
42
|
+
errorCode === "TokenRefreshRequired" ||
|
|
43
|
+
errorCode === "AuthenticationFailed" ||
|
|
44
|
+
errorName === "AuthenticationError" ||
|
|
45
|
+
errorMessage.indexOf("authentication") >= 0 ||
|
|
46
|
+
errorMessage.indexOf("unauthorized") >= 0
|
|
47
|
+
) {
|
|
48
|
+
return "authentication_error";
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Authorization errors
|
|
52
|
+
if (
|
|
53
|
+
errorCode === "AccessDenied" ||
|
|
54
|
+
errorCode === "AccountProblem" ||
|
|
55
|
+
errorCode === "Forbidden" ||
|
|
56
|
+
errorName === "AuthorizationError" ||
|
|
57
|
+
errorMessage.indexOf("forbidden") >= 0 ||
|
|
58
|
+
errorMessage.indexOf("permission") >= 0
|
|
59
|
+
) {
|
|
60
|
+
return "authorization_error";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Throttling errors
|
|
64
|
+
if (
|
|
65
|
+
errorCode === "SlowDown" ||
|
|
66
|
+
errorCode === "RequestTimeTooSkewed" ||
|
|
67
|
+
errorCode === "TooManyRequests" ||
|
|
68
|
+
errorName === "ThrottlingError" ||
|
|
69
|
+
errorMessage.indexOf("throttl") >= 0 ||
|
|
70
|
+
errorMessage.indexOf("rate limit") >= 0
|
|
71
|
+
) {
|
|
72
|
+
return "throttling_error";
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Server errors
|
|
76
|
+
if (
|
|
77
|
+
errorCode === "InternalError" ||
|
|
78
|
+
errorCode === "ServiceUnavailable" ||
|
|
79
|
+
errorCode === "InternalServerError" ||
|
|
80
|
+
errorName === "ServerError" ||
|
|
81
|
+
errorMessage.indexOf("server error") >= 0 ||
|
|
82
|
+
errorMessage.indexOf("service unavailable") >= 0
|
|
83
|
+
) {
|
|
84
|
+
return "server_error";
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Client errors
|
|
88
|
+
if (
|
|
89
|
+
errorCode === "InvalidRequest" ||
|
|
90
|
+
errorCode === "MalformedXML" ||
|
|
91
|
+
errorCode === "RequestEntityTooLarge" ||
|
|
92
|
+
errorCode === "BadRequest" ||
|
|
93
|
+
errorName === "ClientError" ||
|
|
94
|
+
errorMessage.indexOf("bad request") >= 0 ||
|
|
95
|
+
errorMessage.indexOf("invalid") >= 0
|
|
96
|
+
) {
|
|
97
|
+
return "client_error";
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return "unknown_error";
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
// Storage-specific error classifier factory
|
|
104
|
+
export const createStorageErrorClassifier = (
|
|
105
|
+
storageType: string,
|
|
106
|
+
customErrorMapping?: (error: unknown) => StorageErrorCategory | null,
|
|
107
|
+
) => {
|
|
108
|
+
return (error: unknown): StorageErrorCategory => {
|
|
109
|
+
// Try custom mapping first
|
|
110
|
+
if (customErrorMapping) {
|
|
111
|
+
const customResult = customErrorMapping(error);
|
|
112
|
+
if (customResult !== null) return customResult;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Fall back to generic classification
|
|
116
|
+
return classifyStorageError(error);
|
|
117
|
+
};
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
// Generic error tracking function
|
|
121
|
+
export const trackStorageError = (
|
|
122
|
+
storageType: string,
|
|
123
|
+
metrics: StorageMetrics,
|
|
124
|
+
operation: string,
|
|
125
|
+
error: unknown,
|
|
126
|
+
context: Record<string, unknown> = {},
|
|
127
|
+
errorClassifier = classifyStorageError,
|
|
128
|
+
) =>
|
|
129
|
+
Effect.gen(function* () {
|
|
130
|
+
const errorCategory = errorClassifier(error);
|
|
131
|
+
|
|
132
|
+
// Record error metrics
|
|
133
|
+
const errorMetric = metrics.uploadErrorsTotal.pipe(
|
|
134
|
+
Metric.tagged("operation", operation),
|
|
135
|
+
Metric.tagged("error_category", errorCategory),
|
|
136
|
+
);
|
|
137
|
+
yield* errorMetric(Effect.succeed(1));
|
|
138
|
+
|
|
139
|
+
// Create detailed error context
|
|
140
|
+
const errorDetails = {
|
|
141
|
+
storage_type: storageType,
|
|
142
|
+
operation,
|
|
143
|
+
error_category: errorCategory,
|
|
144
|
+
error_type: typeof error,
|
|
145
|
+
error_message: error instanceof Error ? error.message : String(error),
|
|
146
|
+
error_code:
|
|
147
|
+
error && typeof error === "object" && "code" in error
|
|
148
|
+
? error.code
|
|
149
|
+
: undefined,
|
|
150
|
+
error_name:
|
|
151
|
+
error && typeof error === "object" && "name" in error
|
|
152
|
+
? error.name
|
|
153
|
+
: undefined,
|
|
154
|
+
...context,
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
// Log structured error
|
|
158
|
+
yield* Effect.logError(
|
|
159
|
+
`${storageType.toUpperCase()} ${operation} failed`,
|
|
160
|
+
).pipe(Effect.annotateLogs(errorDetails));
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
// Factory for storage-specific error tracking
|
|
164
|
+
export const createStorageErrorTracker = (
|
|
165
|
+
storageType: string,
|
|
166
|
+
metrics: StorageMetrics,
|
|
167
|
+
customErrorClassifier?: (error: unknown) => StorageErrorCategory | null,
|
|
168
|
+
) => {
|
|
169
|
+
const errorClassifier = createStorageErrorClassifier(
|
|
170
|
+
storageType,
|
|
171
|
+
customErrorClassifier,
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
return (
|
|
175
|
+
operation: string,
|
|
176
|
+
error: unknown,
|
|
177
|
+
context: Record<string, unknown> = {},
|
|
178
|
+
) =>
|
|
179
|
+
trackStorageError(
|
|
180
|
+
storageType,
|
|
181
|
+
metrics,
|
|
182
|
+
operation,
|
|
183
|
+
error,
|
|
184
|
+
context,
|
|
185
|
+
errorClassifier,
|
|
186
|
+
);
|
|
187
|
+
};
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { Context, Effect, Layer, Option } from "effect";
|
|
2
|
+
import type { StorageMetrics } from "./metrics.js";
|
|
3
|
+
|
|
4
|
+
// ============================================================================
|
|
5
|
+
// Observability Layer Interfaces
|
|
6
|
+
// ============================================================================
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Core observability service providing tracing, metrics, and logging capabilities
|
|
10
|
+
*/
|
|
11
|
+
export interface ObservabilityService {
|
|
12
|
+
readonly serviceName: string;
|
|
13
|
+
readonly enabled: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Observability service tag for Effect Context
|
|
18
|
+
*/
|
|
19
|
+
export class Observability extends Context.Tag("Observability")<
|
|
20
|
+
Observability,
|
|
21
|
+
ObservabilityService
|
|
22
|
+
>() {}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Storage observability service extending base observability with storage-specific metrics
|
|
26
|
+
*/
|
|
27
|
+
export interface StorageObservabilityService extends ObservabilityService {
|
|
28
|
+
readonly storageType: string;
|
|
29
|
+
readonly metrics: StorageMetrics;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Storage observability service tag
|
|
34
|
+
*/
|
|
35
|
+
export class StorageObservability extends Context.Tag("StorageObservability")<
|
|
36
|
+
StorageObservability,
|
|
37
|
+
StorageObservabilityService
|
|
38
|
+
>() {}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Upload observability service for upload-specific operations
|
|
42
|
+
*/
|
|
43
|
+
export interface UploadObservabilityService extends ObservabilityService {
|
|
44
|
+
readonly metrics: {
|
|
45
|
+
uploadCreated: Effect.Effect<void>;
|
|
46
|
+
uploadCompleted: Effect.Effect<void>;
|
|
47
|
+
uploadFailed: Effect.Effect<void>;
|
|
48
|
+
chunkUploaded: Effect.Effect<void>;
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Upload observability service tag
|
|
54
|
+
*/
|
|
55
|
+
export class UploadObservability extends Context.Tag("UploadObservability")<
|
|
56
|
+
UploadObservability,
|
|
57
|
+
UploadObservabilityService
|
|
58
|
+
>() {}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Flow observability service for flow execution operations
|
|
62
|
+
*/
|
|
63
|
+
export interface FlowObservabilityService extends ObservabilityService {
|
|
64
|
+
readonly metrics: {
|
|
65
|
+
flowStarted: Effect.Effect<void>;
|
|
66
|
+
flowCompleted: Effect.Effect<void>;
|
|
67
|
+
flowFailed: Effect.Effect<void>;
|
|
68
|
+
nodeExecuted: Effect.Effect<void>;
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Flow observability service tag
|
|
74
|
+
*/
|
|
75
|
+
export class FlowObservability extends Context.Tag("FlowObservability")<
|
|
76
|
+
FlowObservability,
|
|
77
|
+
FlowObservabilityService
|
|
78
|
+
>() {}
|
|
79
|
+
|
|
80
|
+
// ============================================================================
|
|
81
|
+
// Layer Factories
|
|
82
|
+
// ============================================================================
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Create a base observability layer
|
|
86
|
+
*/
|
|
87
|
+
export const makeObservabilityLayer = (
|
|
88
|
+
serviceName: string,
|
|
89
|
+
enabled = true,
|
|
90
|
+
): Layer.Layer<Observability> =>
|
|
91
|
+
Layer.succeed(Observability, {
|
|
92
|
+
serviceName,
|
|
93
|
+
enabled,
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Create a storage observability layer
|
|
98
|
+
*/
|
|
99
|
+
export const makeStorageObservabilityLayer = (
|
|
100
|
+
storageType: string,
|
|
101
|
+
metrics: StorageMetrics,
|
|
102
|
+
enabled = true,
|
|
103
|
+
): Layer.Layer<StorageObservability> =>
|
|
104
|
+
Layer.succeed(StorageObservability, {
|
|
105
|
+
serviceName: `uploadista-${storageType}-store`,
|
|
106
|
+
storageType,
|
|
107
|
+
metrics,
|
|
108
|
+
enabled,
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Create an upload observability layer
|
|
113
|
+
*/
|
|
114
|
+
export const makeUploadObservabilityLayer = (
|
|
115
|
+
enabled = true,
|
|
116
|
+
): Layer.Layer<UploadObservability> =>
|
|
117
|
+
Layer.succeed(UploadObservability, {
|
|
118
|
+
serviceName: "uploadista-upload-server",
|
|
119
|
+
enabled,
|
|
120
|
+
metrics: {
|
|
121
|
+
uploadCreated: Effect.void,
|
|
122
|
+
uploadCompleted: Effect.void,
|
|
123
|
+
uploadFailed: Effect.void,
|
|
124
|
+
chunkUploaded: Effect.void,
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Create a flow observability layer
|
|
130
|
+
*/
|
|
131
|
+
export const makeFlowObservabilityLayer = (
|
|
132
|
+
enabled = true,
|
|
133
|
+
): Layer.Layer<FlowObservability> =>
|
|
134
|
+
Layer.succeed(FlowObservability, {
|
|
135
|
+
serviceName: "uploadista-flow-engine",
|
|
136
|
+
enabled,
|
|
137
|
+
metrics: {
|
|
138
|
+
flowStarted: Effect.void,
|
|
139
|
+
flowCompleted: Effect.void,
|
|
140
|
+
flowFailed: Effect.void,
|
|
141
|
+
nodeExecuted: Effect.void,
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
// ============================================================================
|
|
146
|
+
// No-op Layers (for testing and opt-out)
|
|
147
|
+
// ============================================================================
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* No-op observability layer (disabled)
|
|
151
|
+
*/
|
|
152
|
+
export const ObservabilityDisabled = makeObservabilityLayer(
|
|
153
|
+
"uploadista-disabled",
|
|
154
|
+
false,
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* No-op storage observability layer
|
|
159
|
+
*/
|
|
160
|
+
export const StorageObservabilityDisabled = (storageType: string) =>
|
|
161
|
+
makeStorageObservabilityLayer(
|
|
162
|
+
storageType,
|
|
163
|
+
{} as StorageMetrics, // No-op metrics
|
|
164
|
+
false,
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* No-op upload observability layer
|
|
169
|
+
*/
|
|
170
|
+
export const UploadObservabilityDisabled = makeUploadObservabilityLayer(false);
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* No-op flow observability layer
|
|
174
|
+
*/
|
|
175
|
+
export const FlowObservabilityDisabled = makeFlowObservabilityLayer(false);
|
|
176
|
+
|
|
177
|
+
// ============================================================================
|
|
178
|
+
// Helper Functions
|
|
179
|
+
// ============================================================================
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Check if observability is enabled in the current context
|
|
183
|
+
*/
|
|
184
|
+
export const isObservabilityEnabled = Effect.gen(function* () {
|
|
185
|
+
const observability = yield* Effect.serviceOption(Observability);
|
|
186
|
+
return Option.match(observability, {
|
|
187
|
+
onNone: () => false,
|
|
188
|
+
onSome: (obs) => obs.enabled,
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Execute an effect only if observability is enabled
|
|
194
|
+
*/
|
|
195
|
+
export const whenObservabilityEnabled = <A, E, R>(
|
|
196
|
+
effect: Effect.Effect<A, E, R>,
|
|
197
|
+
): Effect.Effect<Option.Option<A>, E, R | Observability> =>
|
|
198
|
+
Effect.gen(function* () {
|
|
199
|
+
const enabled = yield* isObservabilityEnabled;
|
|
200
|
+
if (enabled) {
|
|
201
|
+
const result = yield* effect;
|
|
202
|
+
return Option.some(result);
|
|
203
|
+
}
|
|
204
|
+
return Option.none();
|
|
205
|
+
});
|