@powerhousedao/opentelemetry-instrumentation-reactor 6.0.0-dev.99 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -38,8 +38,12 @@ import { NodeSDK } from "@opentelemetry/sdk-node";
38
38
 
39
39
  const sdk = new NodeSDK({
40
40
  metricReader: new PeriodicExportingMetricReader({
41
- exporter: new OTLPMetricExporter({ url: process.env.METRICS_ENDPOINT }),
42
- exportIntervalMillis: 60_000,
41
+ exporter: new OTLPMetricExporter({
42
+ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
43
+ }),
44
+ exportIntervalMillis: Number(
45
+ process.env.OTEL_METRIC_EXPORT_INTERVAL ?? 60_000,
46
+ ),
43
47
  }),
44
48
  });
45
49
 
@@ -0,0 +1,46 @@
1
+ import { ReactorModule } from "@powerhousedao/reactor";
2
+ import * as _opentelemetry_api0 from "@opentelemetry/api";
3
+
4
+ //#region src/instrumentation.d.ts
5
+ declare class ReactorInstrumentation {
6
+ private readonly module;
7
+ private metrics;
8
+ private unsubscribes;
9
+ private observableCallbacks;
10
+ private pendingTimestamps;
11
+ private runningTimestamps;
12
+ private writeReadyTimestamps;
13
+ constructor(module: ReactorModule);
14
+ start(): void;
15
+ stop(): void;
16
+ private subscribeJobPending;
17
+ private subscribeJobRunning;
18
+ private subscribeJobWriteReady;
19
+ private subscribeJobReadReady;
20
+ private subscribeJobFailed;
21
+ private subscribeDeadLetterAdded;
22
+ private registerObservableGauges;
23
+ private cleanup;
24
+ }
25
+ //#endregion
26
+ //#region src/metrics.d.ts
27
+ declare function createMetrics(): {
28
+ queueJobsEnqueued: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
29
+ queueJobsDequeued: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
30
+ queueJobsCompleted: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
31
+ queueJobsFailed: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
32
+ queueDepth: _opentelemetry_api0.ObservableGauge<_opentelemetry_api0.Attributes>;
33
+ executorJobDuration: _opentelemetry_api0.Histogram<_opentelemetry_api0.Attributes>;
34
+ executorActiveJobs: _opentelemetry_api0.ObservableGauge<_opentelemetry_api0.Attributes>;
35
+ executorTotalProcessed: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
36
+ executorOperationsGenerated: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
37
+ jobTotalDuration: _opentelemetry_api0.Histogram<_opentelemetry_api0.Attributes>;
38
+ readmodelIndexDuration: _opentelemetry_api0.Histogram<_opentelemetry_api0.Attributes>;
39
+ eventbusEventsEmitted: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
40
+ syncRemotes: _opentelemetry_api0.ObservableGauge<_opentelemetry_api0.Attributes>;
41
+ syncDeadLettersAdded: _opentelemetry_api0.Counter<_opentelemetry_api0.Attributes>;
42
+ };
43
+ type ReactorMetrics = ReturnType<typeof createMetrics>;
44
+ //#endregion
45
+ export { ReactorInstrumentation, type ReactorMetrics };
46
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","names":[],"sources":["../src/instrumentation.ts","../src/metrics.ts"],"mappings":";;;;cAkBa,sBAAA;EAAA,iBACM,MAAA;EAAA,QACT,OAAA;EAAA,QACA,YAAA;EAAA,QACA,mBAAA;EAAA,QAGA,iBAAA;EAAA,QACA,iBAAA;EAAA,QACA,oBAAA;cAEI,MAAA,EAAQ,aAAA;EAIpB,KAAA,CAAA;EAaA,IAAA,CAAA;EAAA,QAeQ,mBAAA;EAAA,QAgBA,mBAAA;EAAA,QAgBA,sBAAA;EAAA,QA2BA,qBAAA;EAAA,QA6BA,kBAAA;EAAA,QA0BA,wBAAA;EAAA,QAiBA,wBAAA;EAAA,QAyDA,OAAA;AAAA;;;iBCrPM,aAAA,CAAA;iDAAa,mBAAA,CAAA,UAAA;;;;;;;;;;;;;;;KA4FjB,cAAA,GAAiB,UAAA,QAAkB,aAAA"}
package/dist/index.js ADDED
@@ -0,0 +1,201 @@
1
+ import { ReactorEventTypes, SyncEventTypes } from "@powerhousedao/reactor";
2
+ import { metrics } from "@opentelemetry/api";
3
+ //#region src/metrics.ts
4
+ const METER_NAME = "@powerhousedao/reactor";
5
+ function createMetrics() {
6
+ const meter = metrics.getMeter(METER_NAME);
7
+ return {
8
+ queueJobsEnqueued: meter.createCounter("reactor.queue.jobs.enqueued", {
9
+ description: "Jobs enqueued",
10
+ unit: "{job}"
11
+ }),
12
+ queueJobsDequeued: meter.createCounter("reactor.queue.jobs.dequeued", {
13
+ description: "Jobs dequeued for execution",
14
+ unit: "{job}"
15
+ }),
16
+ queueJobsCompleted: meter.createCounter("reactor.queue.jobs.completed", {
17
+ description: "Jobs completed (READ_READY)",
18
+ unit: "{job}"
19
+ }),
20
+ queueJobsFailed: meter.createCounter("reactor.queue.jobs.failed", {
21
+ description: "Jobs permanently failed",
22
+ unit: "{job}"
23
+ }),
24
+ queueDepth: meter.createObservableGauge("reactor.queue.depth", {
25
+ description: "Pending jobs across all queues",
26
+ unit: "{job}"
27
+ }),
28
+ executorJobDuration: meter.createHistogram("reactor.executor.job.duration", {
29
+ description: "Job execution time (RUNNING to WRITE_READY)",
30
+ unit: "ms"
31
+ }),
32
+ executorActiveJobs: meter.createObservableGauge("reactor.executor.active_jobs", {
33
+ description: "Currently executing jobs",
34
+ unit: "{job}"
35
+ }),
36
+ executorTotalProcessed: meter.createCounter("reactor.executor.processed", {
37
+ description: "Total jobs processed",
38
+ unit: "{job}"
39
+ }),
40
+ executorOperationsGenerated: meter.createCounter("reactor.executor.operations_generated", {
41
+ description: "Operations produced by executors",
42
+ unit: "{operation}"
43
+ }),
44
+ jobTotalDuration: meter.createHistogram("reactor.job.total.duration", {
45
+ description: "Full job lifecycle (PENDING to READ_READY/FAILED)",
46
+ unit: "ms"
47
+ }),
48
+ readmodelIndexDuration: meter.createHistogram("reactor.readmodel.index.duration", {
49
+ description: "Read model indexing time (WRITE_READY to READ_READY)",
50
+ unit: "ms"
51
+ }),
52
+ eventbusEventsEmitted: meter.createCounter("reactor.eventbus.events.emitted", {
53
+ description: "Events emitted",
54
+ unit: "{event}"
55
+ }),
56
+ syncRemotes: meter.createObservableGauge("reactor.sync.remotes", {
57
+ description: "Active remote count",
58
+ unit: "{remote}"
59
+ }),
60
+ syncDeadLettersAdded: meter.createCounter("reactor.sync.dead_letters.added", {
61
+ description: "Sync operations moved to dead letter storage",
62
+ unit: "{operation}"
63
+ })
64
+ };
65
+ }
66
+ //#endregion
67
+ //#region src/instrumentation.ts
68
+ var ReactorInstrumentation = class {
69
+ module;
70
+ metrics;
71
+ unsubscribes = [];
72
+ observableCallbacks = [];
73
+ pendingTimestamps = /* @__PURE__ */ new Map();
74
+ runningTimestamps = /* @__PURE__ */ new Map();
75
+ writeReadyTimestamps = /* @__PURE__ */ new Map();
76
+ constructor(module) {
77
+ this.module = module;
78
+ }
79
+ start() {
80
+ this.metrics = createMetrics();
81
+ const { eventBus, queue, executorManager, syncModule } = this.module;
82
+ this.subscribeJobPending(eventBus);
83
+ this.subscribeJobRunning(eventBus);
84
+ this.subscribeJobWriteReady(eventBus);
85
+ this.subscribeJobReadReady(eventBus);
86
+ this.subscribeJobFailed(eventBus);
87
+ this.subscribeDeadLetterAdded(eventBus);
88
+ this.registerObservableGauges(queue, executorManager, syncModule);
89
+ }
90
+ stop() {
91
+ for (const unsub of this.unsubscribes) unsub();
92
+ this.unsubscribes = [];
93
+ for (const [gauge, cb] of this.observableCallbacks) gauge.removeCallback(cb);
94
+ this.observableCallbacks = [];
95
+ this.pendingTimestamps.clear();
96
+ this.runningTimestamps.clear();
97
+ this.writeReadyTimestamps.clear();
98
+ this.metrics = void 0;
99
+ }
100
+ subscribeJobPending(eventBus) {
101
+ this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_PENDING, (_type, event) => {
102
+ if (!this.metrics) return;
103
+ this.metrics.queueJobsEnqueued.add(1);
104
+ this.metrics.eventbusEventsEmitted.add(1, { "event.type": "JOB_PENDING" });
105
+ this.pendingTimestamps.set(event.jobId, performance.now());
106
+ }));
107
+ }
108
+ subscribeJobRunning(eventBus) {
109
+ this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_RUNNING, (_type, event) => {
110
+ if (!this.metrics) return;
111
+ this.metrics.queueJobsDequeued.add(1);
112
+ this.metrics.eventbusEventsEmitted.add(1, { "event.type": "JOB_RUNNING" });
113
+ this.runningTimestamps.set(event.jobId, performance.now());
114
+ }));
115
+ }
116
+ subscribeJobWriteReady(eventBus) {
117
+ this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, (_type, event) => {
118
+ if (!this.metrics) return;
119
+ const runningTs = this.runningTimestamps.get(event.jobId);
120
+ if (runningTs !== void 0) this.metrics.executorJobDuration.record(performance.now() - runningTs, { "job.success": "true" });
121
+ this.metrics.executorTotalProcessed.add(1, { "job.success": "true" });
122
+ this.metrics.executorOperationsGenerated.add(event.operations.length);
123
+ this.metrics.eventbusEventsEmitted.add(1, { "event.type": "JOB_WRITE_READY" });
124
+ this.writeReadyTimestamps.set(event.jobId, performance.now());
125
+ this.runningTimestamps.delete(event.jobId);
126
+ }));
127
+ }
128
+ subscribeJobReadReady(eventBus) {
129
+ this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_READ_READY, (_type, event) => {
130
+ if (!this.metrics) return;
131
+ const writeReadyTs = this.writeReadyTimestamps.get(event.jobId);
132
+ if (writeReadyTs !== void 0) this.metrics.readmodelIndexDuration.record(performance.now() - writeReadyTs);
133
+ const pendingTs = this.pendingTimestamps.get(event.jobId);
134
+ if (pendingTs !== void 0) this.metrics.jobTotalDuration.record(performance.now() - pendingTs, { "job.success": "true" });
135
+ this.metrics.queueJobsCompleted.add(1);
136
+ this.metrics.eventbusEventsEmitted.add(1, { "event.type": "JOB_READ_READY" });
137
+ this.cleanup(event.jobId);
138
+ }));
139
+ }
140
+ subscribeJobFailed(eventBus) {
141
+ this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_FAILED, (_type, event) => {
142
+ if (!this.metrics) return;
143
+ this.metrics.queueJobsFailed.add(1);
144
+ this.metrics.executorTotalProcessed.add(1, { "job.success": "false" });
145
+ const pendingTs = this.pendingTimestamps.get(event.jobId);
146
+ if (pendingTs !== void 0) this.metrics.jobTotalDuration.record(performance.now() - pendingTs, { "job.success": "false" });
147
+ this.metrics.eventbusEventsEmitted.add(1, { "event.type": "JOB_FAILED" });
148
+ this.cleanup(event.jobId);
149
+ }));
150
+ }
151
+ subscribeDeadLetterAdded(eventBus) {
152
+ this.unsubscribes.push(eventBus.subscribe(SyncEventTypes.DEAD_LETTER_ADDED, (_type, event) => {
153
+ if (!this.metrics) return;
154
+ this.metrics.syncDeadLettersAdded.add(1, { "remote.name": event.remoteName });
155
+ this.metrics.eventbusEventsEmitted.add(1, { "event.type": "DEAD_LETTER_ADDED" });
156
+ }));
157
+ }
158
+ registerObservableGauges(queue, executorManager, syncModule) {
159
+ if (!this.metrics) return;
160
+ const depthCb = async (result) => {
161
+ if (!this.metrics) return;
162
+ const TIMEOUT_MS = 2e3;
163
+ let timeoutId;
164
+ try {
165
+ const depth = await Promise.race([queue.totalSize(), new Promise((_, reject) => {
166
+ timeoutId = setTimeout(() => reject(/* @__PURE__ */ new Error("queue.totalSize() timed out")), TIMEOUT_MS);
167
+ })]);
168
+ result.observe(depth);
169
+ } catch (err) {
170
+ console.warn("[ReactorInstrumentation] queueDepth observation failed:", err);
171
+ } finally {
172
+ clearTimeout(timeoutId);
173
+ }
174
+ };
175
+ this.metrics.queueDepth.addCallback(depthCb);
176
+ this.observableCallbacks.push([this.metrics.queueDepth, depthCb]);
177
+ const activeJobsCb = (result) => {
178
+ if (!this.metrics) return;
179
+ const status = executorManager.getStatus();
180
+ result.observe(status.activeJobs);
181
+ };
182
+ this.metrics.executorActiveJobs.addCallback(activeJobsCb);
183
+ this.observableCallbacks.push([this.metrics.executorActiveJobs, activeJobsCb]);
184
+ const remotesCb = (result) => {
185
+ if (!this.metrics) return;
186
+ const count = syncModule?.syncManager.list().length ?? 0;
187
+ result.observe(count);
188
+ };
189
+ this.metrics.syncRemotes.addCallback(remotesCb);
190
+ this.observableCallbacks.push([this.metrics.syncRemotes, remotesCb]);
191
+ }
192
+ cleanup(jobId) {
193
+ this.pendingTimestamps.delete(jobId);
194
+ this.runningTimestamps.delete(jobId);
195
+ this.writeReadyTimestamps.delete(jobId);
196
+ }
197
+ };
198
+ //#endregion
199
+ export { ReactorInstrumentation };
200
+
201
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","names":[],"sources":["../src/metrics.ts","../src/instrumentation.ts"],"sourcesContent":["import { metrics } from \"@opentelemetry/api\";\n\nconst METER_NAME = \"@powerhousedao/reactor\";\n\nexport function createMetrics() {\n const meter = metrics.getMeter(METER_NAME);\n\n return {\n // Queue metrics\n queueJobsEnqueued: meter.createCounter(\"reactor.queue.jobs.enqueued\", {\n description: \"Jobs enqueued\",\n unit: \"{job}\",\n }),\n queueJobsDequeued: meter.createCounter(\"reactor.queue.jobs.dequeued\", {\n description: \"Jobs dequeued for execution\",\n unit: \"{job}\",\n }),\n queueJobsCompleted: meter.createCounter(\"reactor.queue.jobs.completed\", {\n description: \"Jobs completed (READ_READY)\",\n unit: \"{job}\",\n }),\n queueJobsFailed: meter.createCounter(\"reactor.queue.jobs.failed\", {\n description: \"Jobs permanently failed\",\n unit: \"{job}\",\n }),\n queueDepth: meter.createObservableGauge(\"reactor.queue.depth\", {\n description: \"Pending jobs across all queues\",\n unit: \"{job}\",\n }),\n\n // Executor metrics\n executorJobDuration: meter.createHistogram(\n \"reactor.executor.job.duration\",\n {\n description: \"Job execution time (RUNNING to WRITE_READY)\",\n unit: \"ms\",\n },\n ),\n executorActiveJobs: meter.createObservableGauge(\n \"reactor.executor.active_jobs\",\n {\n description: \"Currently executing jobs\",\n unit: \"{job}\",\n },\n ),\n executorTotalProcessed: meter.createCounter(\"reactor.executor.processed\", {\n description: \"Total jobs processed\",\n unit: \"{job}\",\n }),\n executorOperationsGenerated: meter.createCounter(\n \"reactor.executor.operations_generated\",\n {\n description: \"Operations produced by executors\",\n unit: \"{operation}\",\n },\n ),\n\n // Job lifecycle metrics\n jobTotalDuration: meter.createHistogram(\"reactor.job.total.duration\", {\n description: \"Full job lifecycle (PENDING to READ_READY/FAILED)\",\n unit: \"ms\",\n }),\n\n // Read model metrics\n readmodelIndexDuration: meter.createHistogram(\n \"reactor.readmodel.index.duration\",\n {\n description: \"Read model indexing time (WRITE_READY to READ_READY)\",\n unit: \"ms\",\n },\n ),\n\n // Event bus metrics\n eventbusEventsEmitted: meter.createCounter(\n \"reactor.eventbus.events.emitted\",\n {\n description: \"Events emitted\",\n unit: \"{event}\",\n },\n ),\n\n // Sync metrics\n syncRemotes: meter.createObservableGauge(\"reactor.sync.remotes\", {\n description: \"Active remote count\",\n unit: \"{remote}\",\n }),\n syncDeadLettersAdded: meter.createCounter(\n \"reactor.sync.dead_letters.added\",\n {\n description: \"Sync operations moved to dead letter storage\",\n unit: \"{operation}\",\n },\n ),\n };\n}\n\nexport type ReactorMetrics = ReturnType<typeof createMetrics>;\n","import { ReactorEventTypes, SyncEventTypes } from \"@powerhousedao/reactor\";\nimport type {\n DeadLetterAddedEvent,\n IEventBus,\n IJobExecutorManager,\n IQueue,\n JobPendingEvent,\n JobReadReadyEvent,\n JobRunningEvent,\n JobWriteReadyEvent,\n ReactorJobFailedEvent,\n ReactorModule,\n SyncModule,\n Unsubscribe,\n} from \"@powerhousedao/reactor\";\nimport type { ObservableCallback, ObservableGauge } from \"@opentelemetry/api\";\nimport { createMetrics, type ReactorMetrics } from \"./metrics.js\";\n\nexport class ReactorInstrumentation {\n private readonly module: ReactorModule;\n private metrics: ReactorMetrics | undefined;\n private unsubscribes: Unsubscribe[] = [];\n private observableCallbacks: Array<[ObservableGauge, ObservableCallback]> =\n [];\n\n private pendingTimestamps = new Map<string, number>();\n private runningTimestamps = new Map<string, number>();\n private writeReadyTimestamps = new Map<string, number>();\n\n constructor(module: ReactorModule) {\n this.module = module;\n }\n\n start(): void {\n this.metrics = createMetrics();\n const { eventBus, queue, executorManager, syncModule } = this.module;\n\n this.subscribeJobPending(eventBus);\n this.subscribeJobRunning(eventBus);\n this.subscribeJobWriteReady(eventBus);\n this.subscribeJobReadReady(eventBus);\n this.subscribeJobFailed(eventBus);\n this.subscribeDeadLetterAdded(eventBus);\n this.registerObservableGauges(queue, executorManager, syncModule);\n }\n\n stop(): void {\n for (const unsub of this.unsubscribes) {\n unsub();\n }\n this.unsubscribes = [];\n for (const [gauge, cb] of this.observableCallbacks) {\n gauge.removeCallback(cb);\n }\n this.observableCallbacks = [];\n this.pendingTimestamps.clear();\n this.runningTimestamps.clear();\n this.writeReadyTimestamps.clear();\n this.metrics = undefined;\n }\n\n private subscribeJobPending(eventBus: IEventBus): void {\n this.unsubscribes.push(\n eventBus.subscribe<JobPendingEvent>(\n ReactorEventTypes.JOB_PENDING,\n (_type, event) => {\n if (!this.metrics) return;\n this.metrics.queueJobsEnqueued.add(1);\n this.metrics.eventbusEventsEmitted.add(1, {\n \"event.type\": \"JOB_PENDING\",\n });\n this.pendingTimestamps.set(event.jobId, performance.now());\n },\n ),\n );\n }\n\n private subscribeJobRunning(eventBus: IEventBus): void {\n this.unsubscribes.push(\n eventBus.subscribe<JobRunningEvent>(\n ReactorEventTypes.JOB_RUNNING,\n (_type, event) => {\n if (!this.metrics) return;\n this.metrics.queueJobsDequeued.add(1);\n this.metrics.eventbusEventsEmitted.add(1, {\n \"event.type\": \"JOB_RUNNING\",\n });\n this.runningTimestamps.set(event.jobId, performance.now());\n },\n ),\n );\n }\n\n private subscribeJobWriteReady(eventBus: IEventBus): void {\n this.unsubscribes.push(\n eventBus.subscribe<JobWriteReadyEvent>(\n ReactorEventTypes.JOB_WRITE_READY,\n (_type, event) => {\n if (!this.metrics) return;\n const runningTs = this.runningTimestamps.get(event.jobId);\n if (runningTs !== undefined) {\n this.metrics.executorJobDuration.record(\n performance.now() - runningTs,\n { \"job.success\": \"true\" },\n );\n }\n this.metrics.executorTotalProcessed.add(1, {\n \"job.success\": \"true\",\n });\n this.metrics.executorOperationsGenerated.add(event.operations.length);\n this.metrics.eventbusEventsEmitted.add(1, {\n \"event.type\": \"JOB_WRITE_READY\",\n });\n this.writeReadyTimestamps.set(event.jobId, performance.now());\n this.runningTimestamps.delete(event.jobId);\n },\n ),\n );\n }\n\n private subscribeJobReadReady(eventBus: IEventBus): void {\n this.unsubscribes.push(\n eventBus.subscribe<JobReadReadyEvent>(\n ReactorEventTypes.JOB_READ_READY,\n (_type, event) => {\n if (!this.metrics) return;\n const writeReadyTs = this.writeReadyTimestamps.get(event.jobId);\n if (writeReadyTs !== undefined) {\n this.metrics.readmodelIndexDuration.record(\n performance.now() - writeReadyTs,\n );\n }\n const pendingTs = this.pendingTimestamps.get(event.jobId);\n if (pendingTs !== undefined) {\n this.metrics.jobTotalDuration.record(\n performance.now() - pendingTs,\n { \"job.success\": \"true\" },\n );\n }\n this.metrics.queueJobsCompleted.add(1);\n this.metrics.eventbusEventsEmitted.add(1, {\n \"event.type\": \"JOB_READ_READY\",\n });\n this.cleanup(event.jobId);\n },\n ),\n );\n }\n\n private subscribeJobFailed(eventBus: IEventBus): void {\n this.unsubscribes.push(\n eventBus.subscribe<ReactorJobFailedEvent>(\n ReactorEventTypes.JOB_FAILED,\n (_type, event) => {\n if (!this.metrics) return;\n this.metrics.queueJobsFailed.add(1);\n this.metrics.executorTotalProcessed.add(1, {\n \"job.success\": \"false\",\n });\n const pendingTs = this.pendingTimestamps.get(event.jobId);\n if (pendingTs !== undefined) {\n this.metrics.jobTotalDuration.record(\n performance.now() - pendingTs,\n { \"job.success\": \"false\" },\n );\n }\n this.metrics.eventbusEventsEmitted.add(1, {\n \"event.type\": \"JOB_FAILED\",\n });\n this.cleanup(event.jobId);\n },\n ),\n );\n }\n\n private subscribeDeadLetterAdded(eventBus: IEventBus): void {\n this.unsubscribes.push(\n eventBus.subscribe<DeadLetterAddedEvent>(\n SyncEventTypes.DEAD_LETTER_ADDED,\n (_type, event) => {\n if (!this.metrics) return;\n this.metrics.syncDeadLettersAdded.add(1, {\n \"remote.name\": event.remoteName,\n });\n this.metrics.eventbusEventsEmitted.add(1, {\n \"event.type\": \"DEAD_LETTER_ADDED\",\n });\n },\n ),\n );\n }\n\n private registerObservableGauges(\n queue: IQueue,\n executorManager: IJobExecutorManager,\n syncModule: SyncModule | undefined,\n ): void {\n if (!this.metrics) return;\n\n const depthCb: ObservableCallback = async (result) => {\n if (!this.metrics) return;\n // queue.totalSize() is a DB query. If it exceeds the OTel collection\n // window the observation is silently dropped, making the gauge appear\n // to drop to zero under load. The timeout makes the failure explicit.\n const TIMEOUT_MS = 2_000;\n let timeoutId: ReturnType<typeof setTimeout> | undefined;\n try {\n const depth = await Promise.race([\n queue.totalSize(),\n new Promise<never>((_, reject) => {\n timeoutId = setTimeout(\n () => reject(new Error(\"queue.totalSize() timed out\")),\n TIMEOUT_MS,\n );\n }),\n ]);\n result.observe(depth);\n } catch (err) {\n console.warn(\n \"[ReactorInstrumentation] queueDepth observation failed:\",\n err,\n );\n } finally {\n clearTimeout(timeoutId);\n }\n };\n this.metrics.queueDepth.addCallback(depthCb);\n this.observableCallbacks.push([this.metrics.queueDepth, depthCb]);\n\n const activeJobsCb: ObservableCallback = (result) => {\n if (!this.metrics) return;\n const status = executorManager.getStatus();\n result.observe(status.activeJobs);\n };\n this.metrics.executorActiveJobs.addCallback(activeJobsCb);\n this.observableCallbacks.push([\n this.metrics.executorActiveJobs,\n activeJobsCb,\n ]);\n\n const remotesCb: ObservableCallback = (result) => {\n if (!this.metrics) return;\n const count = syncModule?.syncManager.list().length ?? 0;\n result.observe(count);\n };\n this.metrics.syncRemotes.addCallback(remotesCb);\n this.observableCallbacks.push([this.metrics.syncRemotes, remotesCb]);\n }\n\n private cleanup(jobId: string): void {\n this.pendingTimestamps.delete(jobId);\n this.runningTimestamps.delete(jobId);\n this.writeReadyTimestamps.delete(jobId);\n }\n}\n"],"mappings":";;;AAEA,MAAM,aAAa;AAEnB,SAAgB,gBAAgB;CAC9B,MAAM,QAAQ,QAAQ,SAAS,WAAW;AAE1C,QAAO;EAEL,mBAAmB,MAAM,cAAc,+BAA+B;GACpE,aAAa;GACb,MAAM;GACP,CAAC;EACF,mBAAmB,MAAM,cAAc,+BAA+B;GACpE,aAAa;GACb,MAAM;GACP,CAAC;EACF,oBAAoB,MAAM,cAAc,gCAAgC;GACtE,aAAa;GACb,MAAM;GACP,CAAC;EACF,iBAAiB,MAAM,cAAc,6BAA6B;GAChE,aAAa;GACb,MAAM;GACP,CAAC;EACF,YAAY,MAAM,sBAAsB,uBAAuB;GAC7D,aAAa;GACb,MAAM;GACP,CAAC;EAGF,qBAAqB,MAAM,gBACzB,iCACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EACD,oBAAoB,MAAM,sBACxB,gCACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EACD,wBAAwB,MAAM,cAAc,8BAA8B;GACxE,aAAa;GACb,MAAM;GACP,CAAC;EACF,6BAA6B,MAAM,cACjC,yCACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EAGD,kBAAkB,MAAM,gBAAgB,8BAA8B;GACpE,aAAa;GACb,MAAM;GACP,CAAC;EAGF,wBAAwB,MAAM,gBAC5B,oCACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EAGD,uBAAuB,MAAM,cAC3B,mCACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EAGD,aAAa,MAAM,sBAAsB,wBAAwB;GAC/D,aAAa;GACb,MAAM;GACP,CAAC;EACF,sBAAsB,MAAM,cAC1B,mCACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EACF;;;;AC3EH,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA,eAAsC,EAAE;CACxC,sBACE,EAAE;CAEJ,oCAA4B,IAAI,KAAqB;CACrD,oCAA4B,IAAI,KAAqB;CACrD,uCAA+B,IAAI,KAAqB;CAExD,YAAY,QAAuB;AACjC,OAAK,SAAS;;CAGhB,QAAc;AACZ,OAAK,UAAU,eAAe;EAC9B,MAAM,EAAE,UAAU,OAAO,iBAAiB,eAAe,KAAK;AAE9D,OAAK,oBAAoB,SAAS;AAClC,OAAK,oBAAoB,SAAS;AAClC,OAAK,uBAAuB,SAAS;AACrC,OAAK,sBAAsB,SAAS;AACpC,OAAK,mBAAmB,SAAS;AACjC,OAAK,yBAAyB,SAAS;AACvC,OAAK,yBAAyB,OAAO,iBAAiB,WAAW;;CAGnE,OAAa;AACX,OAAK,MAAM,SAAS,KAAK,aACvB,QAAO;AAET,OAAK,eAAe,EAAE;AACtB,OAAK,MAAM,CAAC,OAAO,OAAO,KAAK,oBAC7B,OAAM,eAAe,GAAG;AAE1B,OAAK,sBAAsB,EAAE;AAC7B,OAAK,kBAAkB,OAAO;AAC9B,OAAK,kBAAkB,OAAO;AAC9B,OAAK,qBAAqB,OAAO;AACjC,OAAK,UAAU,KAAA;;CAGjB,oBAA4B,UAA2B;AACrD,OAAK,aAAa,KAChB,SAAS,UACP,kBAAkB,cACjB,OAAO,UAAU;AAChB,OAAI,CAAC,KAAK,QAAS;AACnB,QAAK,QAAQ,kBAAkB,IAAI,EAAE;AACrC,QAAK,QAAQ,sBAAsB,IAAI,GAAG,EACxC,cAAc,eACf,CAAC;AACF,QAAK,kBAAkB,IAAI,MAAM,OAAO,YAAY,KAAK,CAAC;IAE7D,CACF;;CAGH,oBAA4B,UAA2B;AACrD,OAAK,aAAa,KAChB,SAAS,UACP,kBAAkB,cACjB,OAAO,UAAU;AAChB,OAAI,CAAC,KAAK,QAAS;AACnB,QAAK,QAAQ,kBAAkB,IAAI,EAAE;AACrC,QAAK,QAAQ,sBAAsB,IAAI,GAAG,EACxC,cAAc,eACf,CAAC;AACF,QAAK,kBAAkB,IAAI,MAAM,OAAO,YAAY,KAAK,CAAC;IAE7D,CACF;;CAGH,uBAA+B,UAA2B;AACxD,OAAK,aAAa,KAChB,SAAS,UACP,kBAAkB,kBACjB,OAAO,UAAU;AAChB,OAAI,CAAC,KAAK,QAAS;GACnB,MAAM,YAAY,KAAK,kBAAkB,IAAI,MAAM,MAAM;AACzD,OAAI,cAAc,KAAA,EAChB,MAAK,QAAQ,oBAAoB,OAC/B,YAAY,KAAK,GAAG,WACpB,EAAE,eAAe,QAAQ,CAC1B;AAEH,QAAK,QAAQ,uBAAuB,IAAI,GAAG,EACzC,eAAe,QAChB,CAAC;AACF,QAAK,QAAQ,4BAA4B,IAAI,MAAM,WAAW,OAAO;AACrE,QAAK,QAAQ,sBAAsB,IAAI,GAAG,EACxC,cAAc,mBACf,CAAC;AACF,QAAK,qBAAqB,IAAI,MAAM,OAAO,YAAY,KAAK,CAAC;AAC7D,QAAK,kBAAkB,OAAO,MAAM,MAAM;IAE7C,CACF;;CAGH,sBAA8B,UAA2B;AACvD,OAAK,aAAa,KAChB,SAAS,UACP,kBAAkB,iBACjB,OAAO,UAAU;AAChB,OAAI,CAAC,KAAK,QAAS;GACnB,MAAM,eAAe,KAAK,qBAAqB,IAAI,MAAM,MAAM;AAC/D,OAAI,iBAAiB,KAAA,EACnB,MAAK,QAAQ,uBAAuB,OAClC,YAAY,KAAK,GAAG,aACrB;GAEH,MAAM,YAAY,KAAK,kBAAkB,IAAI,MAAM,MAAM;AACzD,OAAI,cAAc,KAAA,EAChB,MAAK,QAAQ,iBAAiB,OAC5B,YAAY,KAAK,GAAG,WACpB,EAAE,eAAe,QAAQ,CAC1B;AAEH,QAAK,QAAQ,mBAAmB,IAAI,EAAE;AACtC,QAAK,QAAQ,sBAAsB,IAAI,GAAG,EACxC,cAAc,kBACf,CAAC;AACF,QAAK,QAAQ,MAAM,MAAM;IAE5B,CACF;;CAGH,mBAA2B,UAA2B;AACpD,OAAK,aAAa,KAChB,SAAS,UACP,kBAAkB,aACjB,OAAO,UAAU;AAChB,OAAI,CAAC,KAAK,QAAS;AACnB,QAAK,QAAQ,gBAAgB,IAAI,EAAE;AACnC,QAAK,QAAQ,uBAAuB,IAAI,GAAG,EACzC,eAAe,SAChB,CAAC;GACF,MAAM,YAAY,KAAK,kBAAkB,IAAI,MAAM,MAAM;AACzD,OAAI,cAAc,KAAA,EAChB,MAAK,QAAQ,iBAAiB,OAC5B,YAAY,KAAK,GAAG,WACpB,EAAE,eAAe,SAAS,CAC3B;AAEH,QAAK,QAAQ,sBAAsB,IAAI,GAAG,EACxC,cAAc,cACf,CAAC;AACF,QAAK,QAAQ,MAAM,MAAM;IAE5B,CACF;;CAGH,yBAAiC,UAA2B;AAC1D,OAAK,aAAa,KAChB,SAAS,UACP,eAAe,oBACd,OAAO,UAAU;AAChB,OAAI,CAAC,KAAK,QAAS;AACnB,QAAK,QAAQ,qBAAqB,IAAI,GAAG,EACvC,eAAe,MAAM,YACtB,CAAC;AACF,QAAK,QAAQ,sBAAsB,IAAI,GAAG,EACxC,cAAc,qBACf,CAAC;IAEL,CACF;;CAGH,yBACE,OACA,iBACA,YACM;AACN,MAAI,CAAC,KAAK,QAAS;EAEnB,MAAM,UAA8B,OAAO,WAAW;AACpD,OAAI,CAAC,KAAK,QAAS;GAInB,MAAM,aAAa;GACnB,IAAI;AACJ,OAAI;IACF,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAC/B,MAAM,WAAW,EACjB,IAAI,SAAgB,GAAG,WAAW;AAChC,iBAAY,iBACJ,uBAAO,IAAI,MAAM,8BAA8B,CAAC,EACtD,WACD;MACD,CACH,CAAC;AACF,WAAO,QAAQ,MAAM;YACd,KAAK;AACZ,YAAQ,KACN,2DACA,IACD;aACO;AACR,iBAAa,UAAU;;;AAG3B,OAAK,QAAQ,WAAW,YAAY,QAAQ;AAC5C,OAAK,oBAAoB,KAAK,CAAC,KAAK,QAAQ,YAAY,QAAQ,CAAC;EAEjE,MAAM,gBAAoC,WAAW;AACnD,OAAI,CAAC,KAAK,QAAS;GACnB,MAAM,SAAS,gBAAgB,WAAW;AAC1C,UAAO,QAAQ,OAAO,WAAW;;AAEnC,OAAK,QAAQ,mBAAmB,YAAY,aAAa;AACzD,OAAK,oBAAoB,KAAK,CAC5B,KAAK,QAAQ,oBACb,aACD,CAAC;EAEF,MAAM,aAAiC,WAAW;AAChD,OAAI,CAAC,KAAK,QAAS;GACnB,MAAM,QAAQ,YAAY,YAAY,MAAM,CAAC,UAAU;AACvD,UAAO,QAAQ,MAAM;;AAEvB,OAAK,QAAQ,YAAY,YAAY,UAAU;AAC/C,OAAK,oBAAoB,KAAK,CAAC,KAAK,QAAQ,aAAa,UAAU,CAAC;;CAGtE,QAAgB,OAAqB;AACnC,OAAK,kBAAkB,OAAO,MAAM;AACpC,OAAK,kBAAkB,OAAO,MAAM;AACpC,OAAK,qBAAqB,OAAO,MAAM"}
package/package.json CHANGED
@@ -1,8 +1,9 @@
1
1
  {
2
2
  "name": "@powerhousedao/opentelemetry-instrumentation-reactor",
3
- "version": "6.0.0-dev.99",
3
+ "version": "6.0.0",
4
4
  "description": "OpenTelemetry instrumentation for @powerhousedao/reactor",
5
5
  "type": "module",
6
+ "sideEffects": false,
6
7
  "repository": {
7
8
  "type": "git",
8
9
  "url": "https://github.com/powerhouse-inc/powerhouse",
@@ -10,12 +11,13 @@
10
11
  },
11
12
  "exports": {
12
13
  ".": {
13
- "types": "./dist/src/index.d.ts",
14
- "import": "./dist/src/index.js"
14
+ "source": "./src/index.ts",
15
+ "types": "./dist/index.d.ts",
16
+ "import": "./dist/index.js"
15
17
  }
16
18
  },
17
19
  "files": [
18
- "./dist/src"
20
+ "./dist"
19
21
  ],
20
22
  "publishConfig": {
21
23
  "access": "public"
@@ -31,12 +33,15 @@
31
33
  "dependencies": {
32
34
  "@opentelemetry/api": "^1.9.0"
33
35
  },
36
+ "devDependencies": {
37
+ "tsdown": "0.21.1"
38
+ },
34
39
  "peerDependencies": {
35
- "@powerhousedao/reactor": "6.0.0-dev.99"
40
+ "@powerhousedao/reactor": "6.0.0"
36
41
  },
37
42
  "scripts": {
38
43
  "tsc": "tsc",
39
- "build": "tsc",
44
+ "build": "tsdown",
40
45
  "lint": "eslint"
41
46
  }
42
47
  }
@@ -1,3 +0,0 @@
1
- export { ReactorInstrumentation } from "./instrumentation.js";
2
- export type { ReactorMetrics } from "./metrics.js";
3
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC"}
package/dist/src/index.js DELETED
@@ -1,2 +0,0 @@
1
- export { ReactorInstrumentation } from "./instrumentation.js";
2
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC"}
@@ -1,22 +0,0 @@
1
- import type { ReactorModule } from "@powerhousedao/reactor";
2
- export declare class ReactorInstrumentation {
3
- private readonly module;
4
- private metrics;
5
- private unsubscribes;
6
- private observableCallbacks;
7
- private pendingTimestamps;
8
- private runningTimestamps;
9
- private writeReadyTimestamps;
10
- constructor(module: ReactorModule);
11
- start(): void;
12
- stop(): void;
13
- private subscribeJobPending;
14
- private subscribeJobRunning;
15
- private subscribeJobWriteReady;
16
- private subscribeJobReadReady;
17
- private subscribeJobFailed;
18
- private subscribeDeadLetterAdded;
19
- private registerObservableGauges;
20
- private cleanup;
21
- }
22
- //# sourceMappingURL=instrumentation.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"instrumentation.d.ts","sourceRoot":"","sources":["../../src/instrumentation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAUV,aAAa,EAGd,MAAM,wBAAwB,CAAC;AAIhC,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAAqB;IACzC,OAAO,CAAC,mBAAmB,CACtB;IAEL,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,oBAAoB,CAA6B;gBAE7C,MAAM,EAAE,aAAa;IAIjC,KAAK,IAAI,IAAI;IAab,IAAI,IAAI,IAAI;IAeZ,OAAO,CAAC,mBAAmB;IAgB3B,OAAO,CAAC,mBAAmB;IAgB3B,OAAO,CAAC,sBAAsB;IA2B9B,OAAO,CAAC,qBAAqB;IA6B7B,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,wBAAwB;IAiBhC,OAAO,CAAC,wBAAwB;IAyDhC,OAAO,CAAC,OAAO;CAKhB"}
@@ -1,184 +0,0 @@
1
- import { ReactorEventTypes, SyncEventTypes } from "@powerhousedao/reactor";
2
- import { createMetrics } from "./metrics.js";
3
- export class ReactorInstrumentation {
4
- module;
5
- metrics;
6
- unsubscribes = [];
7
- observableCallbacks = [];
8
- pendingTimestamps = new Map();
9
- runningTimestamps = new Map();
10
- writeReadyTimestamps = new Map();
11
- constructor(module) {
12
- this.module = module;
13
- }
14
- start() {
15
- this.metrics = createMetrics();
16
- const { eventBus, queue, executorManager, syncModule } = this.module;
17
- this.subscribeJobPending(eventBus);
18
- this.subscribeJobRunning(eventBus);
19
- this.subscribeJobWriteReady(eventBus);
20
- this.subscribeJobReadReady(eventBus);
21
- this.subscribeJobFailed(eventBus);
22
- this.subscribeDeadLetterAdded(eventBus);
23
- this.registerObservableGauges(queue, executorManager, syncModule);
24
- }
25
- stop() {
26
- for (const unsub of this.unsubscribes) {
27
- unsub();
28
- }
29
- this.unsubscribes = [];
30
- for (const [gauge, cb] of this.observableCallbacks) {
31
- gauge.removeCallback(cb);
32
- }
33
- this.observableCallbacks = [];
34
- this.pendingTimestamps.clear();
35
- this.runningTimestamps.clear();
36
- this.writeReadyTimestamps.clear();
37
- this.metrics = undefined;
38
- }
39
- subscribeJobPending(eventBus) {
40
- this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_PENDING, (_type, event) => {
41
- if (!this.metrics)
42
- return;
43
- this.metrics.queueJobsEnqueued.add(1);
44
- this.metrics.eventbusEventsEmitted.add(1, {
45
- "event.type": "JOB_PENDING",
46
- });
47
- this.pendingTimestamps.set(event.jobId, performance.now());
48
- }));
49
- }
50
- subscribeJobRunning(eventBus) {
51
- this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_RUNNING, (_type, event) => {
52
- if (!this.metrics)
53
- return;
54
- this.metrics.queueJobsDequeued.add(1);
55
- this.metrics.eventbusEventsEmitted.add(1, {
56
- "event.type": "JOB_RUNNING",
57
- });
58
- this.runningTimestamps.set(event.jobId, performance.now());
59
- }));
60
- }
61
- subscribeJobWriteReady(eventBus) {
62
- this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, (_type, event) => {
63
- if (!this.metrics)
64
- return;
65
- const runningTs = this.runningTimestamps.get(event.jobId);
66
- if (runningTs !== undefined) {
67
- this.metrics.executorJobDuration.record(performance.now() - runningTs, { "job.success": "true" });
68
- }
69
- this.metrics.executorTotalProcessed.add(1, {
70
- "job.success": "true",
71
- });
72
- this.metrics.executorOperationsGenerated.add(event.operations.length);
73
- this.metrics.eventbusEventsEmitted.add(1, {
74
- "event.type": "JOB_WRITE_READY",
75
- });
76
- this.writeReadyTimestamps.set(event.jobId, performance.now());
77
- this.runningTimestamps.delete(event.jobId);
78
- }));
79
- }
80
- subscribeJobReadReady(eventBus) {
81
- this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_READ_READY, (_type, event) => {
82
- if (!this.metrics)
83
- return;
84
- const writeReadyTs = this.writeReadyTimestamps.get(event.jobId);
85
- if (writeReadyTs !== undefined) {
86
- this.metrics.readmodelIndexDuration.record(performance.now() - writeReadyTs);
87
- }
88
- const pendingTs = this.pendingTimestamps.get(event.jobId);
89
- if (pendingTs !== undefined) {
90
- this.metrics.jobTotalDuration.record(performance.now() - pendingTs, { "job.success": "true" });
91
- }
92
- this.metrics.queueJobsCompleted.add(1);
93
- this.metrics.eventbusEventsEmitted.add(1, {
94
- "event.type": "JOB_READ_READY",
95
- });
96
- this.cleanup(event.jobId);
97
- }));
98
- }
99
- subscribeJobFailed(eventBus) {
100
- this.unsubscribes.push(eventBus.subscribe(ReactorEventTypes.JOB_FAILED, (_type, event) => {
101
- if (!this.metrics)
102
- return;
103
- this.metrics.queueJobsFailed.add(1);
104
- this.metrics.executorTotalProcessed.add(1, {
105
- "job.success": "false",
106
- });
107
- const pendingTs = this.pendingTimestamps.get(event.jobId);
108
- if (pendingTs !== undefined) {
109
- this.metrics.jobTotalDuration.record(performance.now() - pendingTs, { "job.success": "false" });
110
- }
111
- this.metrics.eventbusEventsEmitted.add(1, {
112
- "event.type": "JOB_FAILED",
113
- });
114
- this.cleanup(event.jobId);
115
- }));
116
- }
117
- subscribeDeadLetterAdded(eventBus) {
118
- this.unsubscribes.push(eventBus.subscribe(SyncEventTypes.DEAD_LETTER_ADDED, (_type, event) => {
119
- if (!this.metrics)
120
- return;
121
- this.metrics.syncDeadLettersAdded.add(1, {
122
- "remote.name": event.remoteName,
123
- });
124
- this.metrics.eventbusEventsEmitted.add(1, {
125
- "event.type": "DEAD_LETTER_ADDED",
126
- });
127
- }));
128
- }
129
- registerObservableGauges(queue, executorManager, syncModule) {
130
- if (!this.metrics)
131
- return;
132
- const depthCb = async (result) => {
133
- if (!this.metrics)
134
- return;
135
- // queue.totalSize() is a DB query. If it exceeds the OTel collection
136
- // window the observation is silently dropped, making the gauge appear
137
- // to drop to zero under load. The timeout makes the failure explicit.
138
- const TIMEOUT_MS = 2_000;
139
- let timeoutId;
140
- try {
141
- const depth = await Promise.race([
142
- queue.totalSize(),
143
- new Promise((_, reject) => {
144
- timeoutId = setTimeout(() => reject(new Error("queue.totalSize() timed out")), TIMEOUT_MS);
145
- }),
146
- ]);
147
- result.observe(depth);
148
- }
149
- catch (err) {
150
- console.warn("[ReactorInstrumentation] queueDepth observation failed:", err);
151
- }
152
- finally {
153
- clearTimeout(timeoutId);
154
- }
155
- };
156
- this.metrics.queueDepth.addCallback(depthCb);
157
- this.observableCallbacks.push([this.metrics.queueDepth, depthCb]);
158
- const activeJobsCb = (result) => {
159
- if (!this.metrics)
160
- return;
161
- const status = executorManager.getStatus();
162
- result.observe(status.activeJobs);
163
- };
164
- this.metrics.executorActiveJobs.addCallback(activeJobsCb);
165
- this.observableCallbacks.push([
166
- this.metrics.executorActiveJobs,
167
- activeJobsCb,
168
- ]);
169
- const remotesCb = (result) => {
170
- if (!this.metrics)
171
- return;
172
- const count = syncModule?.syncManager.list().length ?? 0;
173
- result.observe(count);
174
- };
175
- this.metrics.syncRemotes.addCallback(remotesCb);
176
- this.observableCallbacks.push([this.metrics.syncRemotes, remotesCb]);
177
- }
178
- cleanup(jobId) {
179
- this.pendingTimestamps.delete(jobId);
180
- this.runningTimestamps.delete(jobId);
181
- this.writeReadyTimestamps.delete(jobId);
182
- }
183
- }
184
- //# sourceMappingURL=instrumentation.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"instrumentation.js","sourceRoot":"","sources":["../../src/instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAgB3E,OAAO,EAAE,aAAa,EAAuB,MAAM,cAAc,CAAC;AAElE,MAAM,OAAO,sBAAsB;IAChB,MAAM,CAAgB;IAC/B,OAAO,CAA6B;IACpC,YAAY,GAAkB,EAAE,CAAC;IACjC,mBAAmB,GACzB,EAAE,CAAC;IAEG,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,oBAAoB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzD,YAAY,MAAqB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAErE,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;IACpE,CAAC;IAED,IAAI;QACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,KAAK,EAAE,CAAC;QACV,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACnD,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;IAC3B,CAAC;IAEO,mBAAmB,CAAC,QAAmB;QAC7C,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,QAAQ,CAAC,SAAS,CAChB,iBAAiB,CAAC,WAAW,EAC7B,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACxC,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAC;YACH,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7D,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEO,mBAAmB,CAAC,QAAmB;QAC7C,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,QAAQ,CAAC,SAAS,CAChB,iBAAiB,CAAC,WAAW,EAC7B,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACxC,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAC;YACH,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7D,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEO,sBAAsB,CAAC,QAAmB;QAChD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,QAAQ,CAAC,SAAS,CAChB,iBAAiB,CAAC,eAAe,EACjC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,MAAM,CACrC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,EAAE,aAAa,EAAE,MAAM,EAAE,CAC1B,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzC,aAAa,EAAE,MAAM;aACtB,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,2BAA2B,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACtE,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACxC,YAAY,EAAE,iBAAiB;aAChC,CAAC,CAAC;YACH,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAAC,QAAmB;QAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,QAAQ,CAAC,SAAS,CAChB,iBAAiB,CAAC,cAAc,EAChC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChE,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC/B,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,MAAM,CACxC,WAAW,CAAC,GAAG,EAAE,GAAG,YAAY,CACjC,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAClC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,EAAE,aAAa,EAAE,MAAM,EAAE,CAC1B,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACxC,YAAY,EAAE,gBAAgB;aAC/B,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEO,kBAAkB,CAAC,QAAmB;QAC5C,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,QAAQ,CAAC,SAAS,CAChB,iBAAiB,CAAC,UAAU,EAC5B,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzC,aAAa,EAAE,OAAO;aACvB,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAClC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAC7B,EAAE,aAAa,EAAE,OAAO,EAAE,CAC3B,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACxC,YAAY,EAAE,YAAY;aAC3B,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEO,wBAAwB,CAAC,QAAmB;QAClD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,QAAQ,CAAC,SAAS,CAChB,cAAc,CAAC,iBAAiB,EAChC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACvC,aAAa,EAAE,KAAK,CAAC,UAAU;aAChC,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE;gBACxC,YAAY,EAAE,mBAAmB;aAClC,CAAC,CAAC;QACL,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEO,wBAAwB,CAC9B,KAAa,EACb,eAAoC,EACpC,UAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,MAAM,OAAO,GAAuB,KAAK,EAAE,MAAM,EAAE,EAAE;YACnD,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,qEAAqE;YACrE,sEAAsE;YACtE,sEAAsE;YACtE,MAAM,UAAU,GAAG,KAAK,CAAC;YACzB,IAAI,SAAoD,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;oBAC/B,KAAK,CAAC,SAAS,EAAE;oBACjB,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;wBAC/B,SAAS,GAAG,UAAU,CACpB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,EACtD,UAAU,CACX,CAAC;oBACJ,CAAC,CAAC;iBACH,CAAC,CAAC;gBACH,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CACV,yDAAyD,EACzD,GAAG,CACJ,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,SAAS,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QAElE,MAAM,YAAY,GAAuB,CAAC,MAAM,EAAE,EAAE;YAClD,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,EAAE,CAAC;YAC3C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACpC,CAAC,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,kBAAkB;YAC/B,YAAY;SACb,CAAC,CAAC;QAEH,MAAM,SAAS,GAAuB,CAAC,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1B,MAAM,KAAK,GAAG,UAAU,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,OAAO,CAAC,KAAa;QAC3B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF"}
@@ -1,18 +0,0 @@
1
- export declare function createMetrics(): {
2
- queueJobsEnqueued: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
3
- queueJobsDequeued: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
4
- queueJobsCompleted: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
5
- queueJobsFailed: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
6
- queueDepth: import("@opentelemetry/api").ObservableGauge<import("@opentelemetry/api").Attributes>;
7
- executorJobDuration: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
8
- executorActiveJobs: import("@opentelemetry/api").ObservableGauge<import("@opentelemetry/api").Attributes>;
9
- executorTotalProcessed: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
10
- executorOperationsGenerated: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
11
- jobTotalDuration: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
12
- readmodelIndexDuration: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
13
- eventbusEventsEmitted: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
14
- syncRemotes: import("@opentelemetry/api").ObservableGauge<import("@opentelemetry/api").Attributes>;
15
- syncDeadLettersAdded: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
16
- };
17
- export type ReactorMetrics = ReturnType<typeof createMetrics>;
18
- //# sourceMappingURL=metrics.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/metrics.ts"],"names":[],"mappings":"AAIA,wBAAgB,aAAa;;;;;;;;;;;;;;;EA0F5B;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC"}
@@ -1,70 +0,0 @@
1
- import { metrics } from "@opentelemetry/api";
2
- const METER_NAME = "@powerhousedao/reactor";
3
- export function createMetrics() {
4
- const meter = metrics.getMeter(METER_NAME);
5
- return {
6
- // Queue metrics
7
- queueJobsEnqueued: meter.createCounter("reactor.queue.jobs.enqueued", {
8
- description: "Jobs enqueued",
9
- unit: "{job}",
10
- }),
11
- queueJobsDequeued: meter.createCounter("reactor.queue.jobs.dequeued", {
12
- description: "Jobs dequeued for execution",
13
- unit: "{job}",
14
- }),
15
- queueJobsCompleted: meter.createCounter("reactor.queue.jobs.completed", {
16
- description: "Jobs completed (READ_READY)",
17
- unit: "{job}",
18
- }),
19
- queueJobsFailed: meter.createCounter("reactor.queue.jobs.failed", {
20
- description: "Jobs permanently failed",
21
- unit: "{job}",
22
- }),
23
- queueDepth: meter.createObservableGauge("reactor.queue.depth", {
24
- description: "Pending jobs across all queues",
25
- unit: "{job}",
26
- }),
27
- // Executor metrics
28
- executorJobDuration: meter.createHistogram("reactor.executor.job.duration", {
29
- description: "Job execution time (RUNNING to WRITE_READY)",
30
- unit: "ms",
31
- }),
32
- executorActiveJobs: meter.createObservableGauge("reactor.executor.active_jobs", {
33
- description: "Currently executing jobs",
34
- unit: "{job}",
35
- }),
36
- executorTotalProcessed: meter.createCounter("reactor.executor.processed", {
37
- description: "Total jobs processed",
38
- unit: "{job}",
39
- }),
40
- executorOperationsGenerated: meter.createCounter("reactor.executor.operations_generated", {
41
- description: "Operations produced by executors",
42
- unit: "{operation}",
43
- }),
44
- // Job lifecycle metrics
45
- jobTotalDuration: meter.createHistogram("reactor.job.total.duration", {
46
- description: "Full job lifecycle (PENDING to READ_READY/FAILED)",
47
- unit: "ms",
48
- }),
49
- // Read model metrics
50
- readmodelIndexDuration: meter.createHistogram("reactor.readmodel.index.duration", {
51
- description: "Read model indexing time (WRITE_READY to READ_READY)",
52
- unit: "ms",
53
- }),
54
- // Event bus metrics
55
- eventbusEventsEmitted: meter.createCounter("reactor.eventbus.events.emitted", {
56
- description: "Events emitted",
57
- unit: "{event}",
58
- }),
59
- // Sync metrics
60
- syncRemotes: meter.createObservableGauge("reactor.sync.remotes", {
61
- description: "Active remote count",
62
- unit: "{remote}",
63
- }),
64
- syncDeadLettersAdded: meter.createCounter("reactor.sync.dead_letters.added", {
65
- description: "Sync operations moved to dead letter storage",
66
- unit: "{operation}",
67
- }),
68
- };
69
- }
70
- //# sourceMappingURL=metrics.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,MAAM,UAAU,GAAG,wBAAwB,CAAC;AAE5C,MAAM,UAAU,aAAa;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAE3C,OAAO;QACL,gBAAgB;QAChB,iBAAiB,EAAE,KAAK,CAAC,aAAa,CAAC,6BAA6B,EAAE;YACpE,WAAW,EAAE,eAAe;YAC5B,IAAI,EAAE,OAAO;SACd,CAAC;QACF,iBAAiB,EAAE,KAAK,CAAC,aAAa,CAAC,6BAA6B,EAAE;YACpE,WAAW,EAAE,6BAA6B;YAC1C,IAAI,EAAE,OAAO;SACd,CAAC;QACF,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,8BAA8B,EAAE;YACtE,WAAW,EAAE,6BAA6B;YAC1C,IAAI,EAAE,OAAO;SACd,CAAC;QACF,eAAe,EAAE,KAAK,CAAC,aAAa,CAAC,2BAA2B,EAAE;YAChE,WAAW,EAAE,yBAAyB;YACtC,IAAI,EAAE,OAAO;SACd,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,qBAAqB,CAAC,qBAAqB,EAAE;YAC7D,WAAW,EAAE,gCAAgC;YAC7C,IAAI,EAAE,OAAO;SACd,CAAC;QAEF,mBAAmB;QACnB,mBAAmB,EAAE,KAAK,CAAC,eAAe,CACxC,+BAA+B,EAC/B;YACE,WAAW,EAAE,6CAA6C;YAC1D,IAAI,EAAE,IAAI;SACX,CACF;QACD,kBAAkB,EAAE,KAAK,CAAC,qBAAqB,CAC7C,8BAA8B,EAC9B;YACE,WAAW,EAAE,0BAA0B;YACvC,IAAI,EAAE,OAAO;SACd,CACF;QACD,sBAAsB,EAAE,KAAK,CAAC,aAAa,CAAC,4BAA4B,EAAE;YACxE,WAAW,EAAE,sBAAsB;YACnC,IAAI,EAAE,OAAO;SACd,CAAC;QACF,2BAA2B,EAAE,KAAK,CAAC,aAAa,CAC9C,uCAAuC,EACvC;YACE,WAAW,EAAE,kCAAkC;YAC/C,IAAI,EAAE,aAAa;SACpB,CACF;QAED,wBAAwB;QACxB,gBAAgB,EAAE,KAAK,CAAC,eAAe,CAAC,4BAA4B,EAAE;YACpE,WAAW,EAAE,mDAAmD;YAChE,IAAI,EAAE,IAAI;SACX,CAAC;QAEF,qBAAqB;QACrB,sBAAsB,EAAE,KAAK,CAAC,eAAe,CAC3C,kCAAkC,EAClC;YACE,WAAW,EAAE,sDAAsD;YACnE,IAAI,EAAE,IAAI;SACX,CACF;QAED,oBAAoB;QACpB,qBAAqB,EAAE,KAAK,CAAC,aAAa,CACxC,iCAAiC,EACjC;YACE,WAAW,EAAE,gBAAgB;YAC7B,IAAI,EAAE,SAAS;SAChB,CACF;QAED,eAAe;QACf,WAAW,EAAE,KAAK,CAAC,qBAAqB,CAAC,sBAAsB,EAAE;YAC/D,WAAW,EAAE,qBAAqB;YAClC,IAAI,EAAE,UAAU;SACjB,CAAC;QACF,oBAAoB,EAAE,KAAK,CAAC,aAAa,CACvC,iCAAiC,EACjC;YACE,WAAW,EAAE,8CAA8C;YAC3D,IAAI,EAAE,aAAa;SACpB,CACF;KACF,CAAC;AACJ,CAAC"}