@dudousxd/nestjs-durable-telescope 0.7.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +150 -31
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +165 -8
- package/dist/index.d.ts +165 -8
- package/dist/index.js +146 -32
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.d.cts
CHANGED
|
@@ -1,28 +1,185 @@
|
|
|
1
1
|
import * as _dudousxd_nestjs_telescope from '@dudousxd/nestjs-telescope';
|
|
2
2
|
import { Watcher, WatcherContext, DataProvider, DashboardSpec } from '@dudousxd/nestjs-telescope';
|
|
3
|
+
import { DurableExecutionWrapper } from '@dudousxd/nestjs-durable-core';
|
|
4
|
+
import { Context } from '@opentelemetry/api';
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* A Telescope watcher that turns durable-workflow lifecycle events into Telescope entries, so
|
|
6
8
|
* runs and steps (including remote/Python steps) show up alongside the app's requests, queries
|
|
7
9
|
* and jobs in the Telescope UI — under the **Workflows** tab.
|
|
8
10
|
*
|
|
9
|
-
* It
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* It does two things, and the second is the one that makes a failing workflow debuggable:
|
|
12
|
+
*
|
|
13
|
+
* 1. **Records an entry per engine event**, stamped with the run's trace (see
|
|
14
|
+
* {@link runTraceContext}), so a whole run — across all its turns, all its steps and every
|
|
15
|
+
* process that executed them — reads as a single trace at `/telescope#/traces/<id>`.
|
|
16
|
+
*
|
|
17
|
+
* 2. **Opens a Telescope batch around the execution itself** (see {@link durableExecutionScope}),
|
|
18
|
+
* through the engine's `useDurableExecution` seam. Without this, everything a workflow turn or a
|
|
19
|
+
* step handler *did* — its queries, its logs, the exception it threw — was recorded outside any
|
|
20
|
+
* batch and outside any span: traceless, and so impossible to correlate back to the step that
|
|
21
|
+
* caused it. Recording an event after the fact cannot fix that; the scope has to be open while
|
|
22
|
+
* the body runs.
|
|
14
23
|
*
|
|
15
24
|
* Add it to `TelescopeModule.forRoot({ watchers: [new DurableTelescopeWatcher()] })`. It resolves
|
|
16
25
|
* the engine from the (global) durable providers and subscribes to its events.
|
|
26
|
+
*
|
|
27
|
+
* ## What is NOT covered
|
|
28
|
+
*
|
|
29
|
+
* A **remote step** — one whose handler lives in another runtime, e.g. the Python worker — executes
|
|
30
|
+
* out of process, so nothing here can wrap it. Its lifecycle still shows up (the engine emits
|
|
31
|
+
* `step.started`/`step.completed`/`step.failed` when it dispatches and when the result lands, and
|
|
32
|
+
* those entries carry the run's trace id), but the queries and exceptions *inside* that handler are
|
|
33
|
+
* that runtime's to record. Because the trace id is derived from the run id rather than held in this
|
|
34
|
+
* process, a remote runtime that derives it the same way joins the same trace — but that is a change
|
|
35
|
+
* on its side, not something this watcher can do on its behalf.
|
|
36
|
+
*
|
|
37
|
+
* A **remote workflow** (a run whose body itself lives in another runtime) is likewise unwrapped:
|
|
38
|
+
* the engine dispatches a workflow task and awaits a decision rather than executing a body, and a
|
|
39
|
+
* scope around a dispatch would describe the wrong thing.
|
|
17
40
|
*/
|
|
18
41
|
declare class DurableTelescopeWatcher implements Watcher {
|
|
19
42
|
readonly type = "durable";
|
|
20
|
-
/**
|
|
21
|
-
private
|
|
43
|
+
/** Removes the execution wrapper from the engine's process-level registry. */
|
|
44
|
+
private disposeScope;
|
|
22
45
|
register(ctx: WatcherContext): void;
|
|
46
|
+
/**
|
|
47
|
+
* Detach the execution wrapper. The `Watcher` SPI has no teardown hook, so nothing calls this in a
|
|
48
|
+
* running app — where the wrapper should live exactly as long as the process does. It exists so a
|
|
49
|
+
* test that registers a watcher does not leave a wrapper behind in the process-level registry for
|
|
50
|
+
* the next test to trip over.
|
|
51
|
+
*/
|
|
52
|
+
unregister(): void;
|
|
23
53
|
private tags;
|
|
24
54
|
}
|
|
25
55
|
|
|
56
|
+
/** What the scope publishes to anything recording on this async path. */
|
|
57
|
+
interface DurableScope {
|
|
58
|
+
/** The run whose unit of work is executing. */
|
|
59
|
+
readonly runId: string;
|
|
60
|
+
/** The run's trace — see {@link runTraceId}. */
|
|
61
|
+
readonly traceId: string;
|
|
62
|
+
/** This scope's span within that trace. Stable for every entry recorded inside it. */
|
|
63
|
+
readonly spanId: string;
|
|
64
|
+
/**
|
|
65
|
+
* Identity of the wrapper that opened this scope. Reuse is only ever offered to the wrapper that
|
|
66
|
+
* owns the scope: two Telescope instances in one process (a torn-down test module whose wrapper
|
|
67
|
+
* outlived it, an app that mounts two consoles) would otherwise have the first one's scope
|
|
68
|
+
* silently suppress the second one's batch, and the second console would record everything
|
|
69
|
+
* batchless while looking correctly wired.
|
|
70
|
+
*/
|
|
71
|
+
readonly owner: object;
|
|
72
|
+
}
|
|
73
|
+
/** The durable execution scope active on this async path, or undefined outside one. */
|
|
74
|
+
declare function currentDurableScope(): DurableScope | undefined;
|
|
75
|
+
/**
|
|
76
|
+
* Builds the {@link DurableExecutionWrapper} that puts every workflow turn and every step handler
|
|
77
|
+
* body inside a Telescope batch and an OTel span, so the queries, logs and exceptions those bodies
|
|
78
|
+
* produce are correlated to the unit of durable work that caused them instead of landing traceless.
|
|
79
|
+
*
|
|
80
|
+
* ## Why a batch around execution and not around the event
|
|
81
|
+
*
|
|
82
|
+
* The watcher's engine-event subscription records what *happened*, after the fact. A batch opened
|
|
83
|
+
* there would contain exactly one entry — the durable entry itself — and nothing the step actually
|
|
84
|
+
* did, because by then the handler has returned and the exception it threw has long since been
|
|
85
|
+
* recorded with no batch at all. Wrapping the execution is the only shape that puts the failing code
|
|
86
|
+
* inside the scope.
|
|
87
|
+
*
|
|
88
|
+
* ## Nesting: reuse the batch when the same run already has one open on this path
|
|
89
|
+
*
|
|
90
|
+
* With an in-process transport (event-emitter, in-memory, the DB transport's inline path) a
|
|
91
|
+
* dispatched step's handler starts synchronously inside the turn that dispatched it, and the result
|
|
92
|
+
* it produces resumes the next turn on that same async path — so the whole run is one unbroken
|
|
93
|
+
* causal chain. Opening a fresh batch at each hop would split that chain across many batch ids for
|
|
94
|
+
* no gain: Telescope's entry model has no parent-batch pointer, so the split is not recoverable in
|
|
95
|
+
* the UI — the step's queries would appear in a batch with nothing visible causing them.
|
|
96
|
+
*
|
|
97
|
+
* So the rule is the causal one, not the syntactic one: a unit reuses the batch when THIS wrapper
|
|
98
|
+
* already has one open for THIS run on this async path. The consequence is worth stating plainly,
|
|
99
|
+
* because it differs by transport and looks like an inconsistency if you meet it unprepared:
|
|
100
|
+
*
|
|
101
|
+
* - **in-process transport** — one batch for the whole run (every turn, every step), because it is
|
|
102
|
+
* genuinely one chain;
|
|
103
|
+
* - **queue-backed transport** (BullMQ, SQS, a thin worker) — one batch per turn and one per step,
|
|
104
|
+
* because each really is a separate entry point, in a separate process, with no ambient scope to
|
|
105
|
+
* inherit.
|
|
106
|
+
*
|
|
107
|
+
* What holds either way is the trace: every entry of a run carries {@link runTraceId}, so the run is
|
|
108
|
+
* one trace whether it executed in one process or twenty. That invariant is the thing to rely on;
|
|
109
|
+
* batch granularity follows the execution shape.
|
|
110
|
+
*
|
|
111
|
+
* A different run is a different chain and always gets its own batch. That includes a child
|
|
112
|
+
* workflow: it has its own run id, its own timeline and its own dashboard row, and its trace is
|
|
113
|
+
* anchored on ITS run id (see {@link runTraceContext}), so it reads as a related-but-distinct trace
|
|
114
|
+
* instead of being folded into the parent's. The parent/child relationship is already carried by the
|
|
115
|
+
* run graph; flattening the batches to restate it would erase the child's own boundaries and gain
|
|
116
|
+
* nothing.
|
|
117
|
+
*
|
|
118
|
+
* ## Origin
|
|
119
|
+
*
|
|
120
|
+
* `BatchOrigin` in `@dudousxd/nestjs-telescope` is the closed union
|
|
121
|
+
* `'http' | 'queue' | 'schedule' | 'cli' | 'manual'` — it has no durable/workflow member. Widening
|
|
122
|
+
* it belongs to that repo and a coordinated release, so this uses `'queue'`, which is also the
|
|
123
|
+
* honest description: durable work reaches an executor by being dispatched over a transport
|
|
124
|
+
* (BullMQ, SQS, an event emitter), exactly like the jobs the BullMQ watcher marks `'queue'`.
|
|
125
|
+
*/
|
|
126
|
+
declare function durableExecutionScope(ctx: WatcherContext): DurableExecutionWrapper;
|
|
127
|
+
/** Minimal structural view of Telescope's `TraceContextProvider` (it is one method, and typing
|
|
128
|
+
* against the interface would drag a value import in for nothing). */
|
|
129
|
+
interface TraceContextLike {
|
|
130
|
+
current(): {
|
|
131
|
+
traceId: string;
|
|
132
|
+
spanId: string;
|
|
133
|
+
} | null;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* A Telescope `traceContext` provider that reports the active durable scope, so entries recorded by
|
|
137
|
+
* OTHER watchers (queries, cache, outbound HTTP) during a workflow turn or a step body carry the
|
|
138
|
+
* run's trace id.
|
|
139
|
+
*
|
|
140
|
+
* Wire it as `TelescopeModule.forRoot({ traceContext: durableTraceContext() })`.
|
|
141
|
+
*
|
|
142
|
+
* This exists because Telescope resolves an entry's trace id from an ambient OTel span, and
|
|
143
|
+
* `@opentelemetry/api` on its own propagates nothing: without a registered context manager
|
|
144
|
+
* `context.with(...)` is a no-op and `getActiveSpan()` is always undefined. An app that runs a full
|
|
145
|
+
* OTel SDK needs none of this — it should pass `OtelTraceContextProvider` (optionally as the
|
|
146
|
+
* `delegate` below, which is then consulted first so the app's real spans win), and the scope's
|
|
147
|
+
* spans hang off the run's trace anyway. An app that runs no OTel at all — the common case — would
|
|
148
|
+
* otherwise get correlated batches and no trace ids, which is half the feature.
|
|
149
|
+
*/
|
|
150
|
+
declare function durableTraceContext(delegate?: TraceContextLike): TraceContextLike;
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The trace a run's spans hang under, DERIVED FROM THE RUN ID rather than held as a live root span.
|
|
154
|
+
*
|
|
155
|
+
* The obvious implementation — open a root span on `run.started`, keep it in a map, end it when the
|
|
156
|
+
* run settles — cannot work for a durable workflow. A run suspends: `ctx.step` dispatches and the
|
|
157
|
+
* turn unwinds, and the step body then executes minutes (or days) later, on a different turn, very
|
|
158
|
+
* often in a different process. A live root span would either have to be ended at every suspension —
|
|
159
|
+
* fragmenting one run into a trace per turn, so the step that failed is in a different trace from
|
|
160
|
+
* the turn that dispatched it — or held open across the suspension, which pins a span object for the
|
|
161
|
+
* lifetime of a workflow that is explicitly designed to wait for days.
|
|
162
|
+
*
|
|
163
|
+
* Hashing the run id sidesteps both. The trace id is a pure function of the run, so every turn and
|
|
164
|
+
* every step of a run lands in ONE trace with no state held anywhere, and a worker in another
|
|
165
|
+
* process computes the same id from the same run id — which is what makes a remotely-executed step
|
|
166
|
+
* correlate with the run that dispatched it at all.
|
|
167
|
+
*
|
|
168
|
+
* The parent span context is marked `isRemote` because that is what it is: a reference to a trace
|
|
169
|
+
* whose "root" is the run itself, not a span this process ever recorded.
|
|
170
|
+
*/
|
|
171
|
+
declare function runTraceId(runId: string): string;
|
|
172
|
+
/**
|
|
173
|
+
* An OTel {@link Context} whose active span context is the run's synthetic root. Spans started
|
|
174
|
+
* against it are children of the run, and Telescope's trace-context provider stamps every entry
|
|
175
|
+
* recorded inside it with {@link runTraceId}.
|
|
176
|
+
*
|
|
177
|
+
* `TraceFlags.SAMPLED` is set deliberately: a non-sampled parent leaves the derived span
|
|
178
|
+
* non-recording, and its span context would then never reach an exporter — the run would be
|
|
179
|
+
* traceable in Telescope but invisible in any OTel backend, which is the confusing half-state.
|
|
180
|
+
*/
|
|
181
|
+
declare function runTraceContext(runId: string, parent?: Context): Context;
|
|
182
|
+
|
|
26
183
|
/** The first-class Telescope extension for nestjs-durable: watcher + Workflows dashboard. */
|
|
27
184
|
declare function durableTelescopeExtension(opts?: {
|
|
28
185
|
runHref?: string;
|
|
@@ -103,4 +260,4 @@ declare function durableDashboard(opts?: {
|
|
|
103
260
|
recentFailuresWindowMs?: number;
|
|
104
261
|
}): DashboardSpec;
|
|
105
262
|
|
|
106
|
-
export { DurableTelescopeWatcher, durableDashboard, durableDurationProvider, durableRecentFailuresProvider, durableRunsOverTimeProvider, durableStateBreakdownProvider, durableStateProvider, durableSuccessRateProvider, durableTelescopeExtension, durableThroughputProvider, durableTimeseriesProvider, durableWorkerHealthProvider, durableWorkerStatusProvider };
|
|
263
|
+
export { type DurableScope, DurableTelescopeWatcher, currentDurableScope, durableDashboard, durableDurationProvider, durableExecutionScope, durableRecentFailuresProvider, durableRunsOverTimeProvider, durableStateBreakdownProvider, durableStateProvider, durableSuccessRateProvider, durableTelescopeExtension, durableThroughputProvider, durableTimeseriesProvider, durableTraceContext, durableWorkerHealthProvider, durableWorkerStatusProvider, runTraceContext, runTraceId };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,28 +1,185 @@
|
|
|
1
1
|
import * as _dudousxd_nestjs_telescope from '@dudousxd/nestjs-telescope';
|
|
2
2
|
import { Watcher, WatcherContext, DataProvider, DashboardSpec } from '@dudousxd/nestjs-telescope';
|
|
3
|
+
import { DurableExecutionWrapper } from '@dudousxd/nestjs-durable-core';
|
|
4
|
+
import { Context } from '@opentelemetry/api';
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* A Telescope watcher that turns durable-workflow lifecycle events into Telescope entries, so
|
|
6
8
|
* runs and steps (including remote/Python steps) show up alongside the app's requests, queries
|
|
7
9
|
* and jobs in the Telescope UI — under the **Workflows** tab.
|
|
8
10
|
*
|
|
9
|
-
* It
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* It does two things, and the second is the one that makes a failing workflow debuggable:
|
|
12
|
+
*
|
|
13
|
+
* 1. **Records an entry per engine event**, stamped with the run's trace (see
|
|
14
|
+
* {@link runTraceContext}), so a whole run — across all its turns, all its steps and every
|
|
15
|
+
* process that executed them — reads as a single trace at `/telescope#/traces/<id>`.
|
|
16
|
+
*
|
|
17
|
+
* 2. **Opens a Telescope batch around the execution itself** (see {@link durableExecutionScope}),
|
|
18
|
+
* through the engine's `useDurableExecution` seam. Without this, everything a workflow turn or a
|
|
19
|
+
* step handler *did* — its queries, its logs, the exception it threw — was recorded outside any
|
|
20
|
+
* batch and outside any span: traceless, and so impossible to correlate back to the step that
|
|
21
|
+
* caused it. Recording an event after the fact cannot fix that; the scope has to be open while
|
|
22
|
+
* the body runs.
|
|
14
23
|
*
|
|
15
24
|
* Add it to `TelescopeModule.forRoot({ watchers: [new DurableTelescopeWatcher()] })`. It resolves
|
|
16
25
|
* the engine from the (global) durable providers and subscribes to its events.
|
|
26
|
+
*
|
|
27
|
+
* ## What is NOT covered
|
|
28
|
+
*
|
|
29
|
+
* A **remote step** — one whose handler lives in another runtime, e.g. the Python worker — executes
|
|
30
|
+
* out of process, so nothing here can wrap it. Its lifecycle still shows up (the engine emits
|
|
31
|
+
* `step.started`/`step.completed`/`step.failed` when it dispatches and when the result lands, and
|
|
32
|
+
* those entries carry the run's trace id), but the queries and exceptions *inside* that handler are
|
|
33
|
+
* that runtime's to record. Because the trace id is derived from the run id rather than held in this
|
|
34
|
+
* process, a remote runtime that derives it the same way joins the same trace — but that is a change
|
|
35
|
+
* on its side, not something this watcher can do on its behalf.
|
|
36
|
+
*
|
|
37
|
+
* A **remote workflow** (a run whose body itself lives in another runtime) is likewise unwrapped:
|
|
38
|
+
* the engine dispatches a workflow task and awaits a decision rather than executing a body, and a
|
|
39
|
+
* scope around a dispatch would describe the wrong thing.
|
|
17
40
|
*/
|
|
18
41
|
declare class DurableTelescopeWatcher implements Watcher {
|
|
19
42
|
readonly type = "durable";
|
|
20
|
-
/**
|
|
21
|
-
private
|
|
43
|
+
/** Removes the execution wrapper from the engine's process-level registry. */
|
|
44
|
+
private disposeScope;
|
|
22
45
|
register(ctx: WatcherContext): void;
|
|
46
|
+
/**
|
|
47
|
+
* Detach the execution wrapper. The `Watcher` SPI has no teardown hook, so nothing calls this in a
|
|
48
|
+
* running app — where the wrapper should live exactly as long as the process does. It exists so a
|
|
49
|
+
* test that registers a watcher does not leave a wrapper behind in the process-level registry for
|
|
50
|
+
* the next test to trip over.
|
|
51
|
+
*/
|
|
52
|
+
unregister(): void;
|
|
23
53
|
private tags;
|
|
24
54
|
}
|
|
25
55
|
|
|
56
|
+
/** What the scope publishes to anything recording on this async path. */
|
|
57
|
+
interface DurableScope {
|
|
58
|
+
/** The run whose unit of work is executing. */
|
|
59
|
+
readonly runId: string;
|
|
60
|
+
/** The run's trace — see {@link runTraceId}. */
|
|
61
|
+
readonly traceId: string;
|
|
62
|
+
/** This scope's span within that trace. Stable for every entry recorded inside it. */
|
|
63
|
+
readonly spanId: string;
|
|
64
|
+
/**
|
|
65
|
+
* Identity of the wrapper that opened this scope. Reuse is only ever offered to the wrapper that
|
|
66
|
+
* owns the scope: two Telescope instances in one process (a torn-down test module whose wrapper
|
|
67
|
+
* outlived it, an app that mounts two consoles) would otherwise have the first one's scope
|
|
68
|
+
* silently suppress the second one's batch, and the second console would record everything
|
|
69
|
+
* batchless while looking correctly wired.
|
|
70
|
+
*/
|
|
71
|
+
readonly owner: object;
|
|
72
|
+
}
|
|
73
|
+
/** The durable execution scope active on this async path, or undefined outside one. */
|
|
74
|
+
declare function currentDurableScope(): DurableScope | undefined;
|
|
75
|
+
/**
|
|
76
|
+
* Builds the {@link DurableExecutionWrapper} that puts every workflow turn and every step handler
|
|
77
|
+
* body inside a Telescope batch and an OTel span, so the queries, logs and exceptions those bodies
|
|
78
|
+
* produce are correlated to the unit of durable work that caused them instead of landing traceless.
|
|
79
|
+
*
|
|
80
|
+
* ## Why a batch around execution and not around the event
|
|
81
|
+
*
|
|
82
|
+
* The watcher's engine-event subscription records what *happened*, after the fact. A batch opened
|
|
83
|
+
* there would contain exactly one entry — the durable entry itself — and nothing the step actually
|
|
84
|
+
* did, because by then the handler has returned and the exception it threw has long since been
|
|
85
|
+
* recorded with no batch at all. Wrapping the execution is the only shape that puts the failing code
|
|
86
|
+
* inside the scope.
|
|
87
|
+
*
|
|
88
|
+
* ## Nesting: reuse the batch when the same run already has one open on this path
|
|
89
|
+
*
|
|
90
|
+
* With an in-process transport (event-emitter, in-memory, the DB transport's inline path) a
|
|
91
|
+
* dispatched step's handler starts synchronously inside the turn that dispatched it, and the result
|
|
92
|
+
* it produces resumes the next turn on that same async path — so the whole run is one unbroken
|
|
93
|
+
* causal chain. Opening a fresh batch at each hop would split that chain across many batch ids for
|
|
94
|
+
* no gain: Telescope's entry model has no parent-batch pointer, so the split is not recoverable in
|
|
95
|
+
* the UI — the step's queries would appear in a batch with nothing visible causing them.
|
|
96
|
+
*
|
|
97
|
+
* So the rule is the causal one, not the syntactic one: a unit reuses the batch when THIS wrapper
|
|
98
|
+
* already has one open for THIS run on this async path. The consequence is worth stating plainly,
|
|
99
|
+
* because it differs by transport and looks like an inconsistency if you meet it unprepared:
|
|
100
|
+
*
|
|
101
|
+
* - **in-process transport** — one batch for the whole run (every turn, every step), because it is
|
|
102
|
+
* genuinely one chain;
|
|
103
|
+
* - **queue-backed transport** (BullMQ, SQS, a thin worker) — one batch per turn and one per step,
|
|
104
|
+
* because each really is a separate entry point, in a separate process, with no ambient scope to
|
|
105
|
+
* inherit.
|
|
106
|
+
*
|
|
107
|
+
* What holds either way is the trace: every entry of a run carries {@link runTraceId}, so the run is
|
|
108
|
+
* one trace whether it executed in one process or twenty. That invariant is the thing to rely on;
|
|
109
|
+
* batch granularity follows the execution shape.
|
|
110
|
+
*
|
|
111
|
+
* A different run is a different chain and always gets its own batch. That includes a child
|
|
112
|
+
* workflow: it has its own run id, its own timeline and its own dashboard row, and its trace is
|
|
113
|
+
* anchored on ITS run id (see {@link runTraceContext}), so it reads as a related-but-distinct trace
|
|
114
|
+
* instead of being folded into the parent's. The parent/child relationship is already carried by the
|
|
115
|
+
* run graph; flattening the batches to restate it would erase the child's own boundaries and gain
|
|
116
|
+
* nothing.
|
|
117
|
+
*
|
|
118
|
+
* ## Origin
|
|
119
|
+
*
|
|
120
|
+
* `BatchOrigin` in `@dudousxd/nestjs-telescope` is the closed union
|
|
121
|
+
* `'http' | 'queue' | 'schedule' | 'cli' | 'manual'` — it has no durable/workflow member. Widening
|
|
122
|
+
* it belongs to that repo and a coordinated release, so this uses `'queue'`, which is also the
|
|
123
|
+
* honest description: durable work reaches an executor by being dispatched over a transport
|
|
124
|
+
* (BullMQ, SQS, an event emitter), exactly like the jobs the BullMQ watcher marks `'queue'`.
|
|
125
|
+
*/
|
|
126
|
+
declare function durableExecutionScope(ctx: WatcherContext): DurableExecutionWrapper;
|
|
127
|
+
/** Minimal structural view of Telescope's `TraceContextProvider` (it is one method, and typing
|
|
128
|
+
* against the interface would drag a value import in for nothing). */
|
|
129
|
+
interface TraceContextLike {
|
|
130
|
+
current(): {
|
|
131
|
+
traceId: string;
|
|
132
|
+
spanId: string;
|
|
133
|
+
} | null;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* A Telescope `traceContext` provider that reports the active durable scope, so entries recorded by
|
|
137
|
+
* OTHER watchers (queries, cache, outbound HTTP) during a workflow turn or a step body carry the
|
|
138
|
+
* run's trace id.
|
|
139
|
+
*
|
|
140
|
+
* Wire it as `TelescopeModule.forRoot({ traceContext: durableTraceContext() })`.
|
|
141
|
+
*
|
|
142
|
+
* This exists because Telescope resolves an entry's trace id from an ambient OTel span, and
|
|
143
|
+
* `@opentelemetry/api` on its own propagates nothing: without a registered context manager
|
|
144
|
+
* `context.with(...)` is a no-op and `getActiveSpan()` is always undefined. An app that runs a full
|
|
145
|
+
* OTel SDK needs none of this — it should pass `OtelTraceContextProvider` (optionally as the
|
|
146
|
+
* `delegate` below, which is then consulted first so the app's real spans win), and the scope's
|
|
147
|
+
* spans hang off the run's trace anyway. An app that runs no OTel at all — the common case — would
|
|
148
|
+
* otherwise get correlated batches and no trace ids, which is half the feature.
|
|
149
|
+
*/
|
|
150
|
+
declare function durableTraceContext(delegate?: TraceContextLike): TraceContextLike;
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The trace a run's spans hang under, DERIVED FROM THE RUN ID rather than held as a live root span.
|
|
154
|
+
*
|
|
155
|
+
* The obvious implementation — open a root span on `run.started`, keep it in a map, end it when the
|
|
156
|
+
* run settles — cannot work for a durable workflow. A run suspends: `ctx.step` dispatches and the
|
|
157
|
+
* turn unwinds, and the step body then executes minutes (or days) later, on a different turn, very
|
|
158
|
+
* often in a different process. A live root span would either have to be ended at every suspension —
|
|
159
|
+
* fragmenting one run into a trace per turn, so the step that failed is in a different trace from
|
|
160
|
+
* the turn that dispatched it — or held open across the suspension, which pins a span object for the
|
|
161
|
+
* lifetime of a workflow that is explicitly designed to wait for days.
|
|
162
|
+
*
|
|
163
|
+
* Hashing the run id sidesteps both. The trace id is a pure function of the run, so every turn and
|
|
164
|
+
* every step of a run lands in ONE trace with no state held anywhere, and a worker in another
|
|
165
|
+
* process computes the same id from the same run id — which is what makes a remotely-executed step
|
|
166
|
+
* correlate with the run that dispatched it at all.
|
|
167
|
+
*
|
|
168
|
+
* The parent span context is marked `isRemote` because that is what it is: a reference to a trace
|
|
169
|
+
* whose "root" is the run itself, not a span this process ever recorded.
|
|
170
|
+
*/
|
|
171
|
+
declare function runTraceId(runId: string): string;
|
|
172
|
+
/**
|
|
173
|
+
* An OTel {@link Context} whose active span context is the run's synthetic root. Spans started
|
|
174
|
+
* against it are children of the run, and Telescope's trace-context provider stamps every entry
|
|
175
|
+
* recorded inside it with {@link runTraceId}.
|
|
176
|
+
*
|
|
177
|
+
* `TraceFlags.SAMPLED` is set deliberately: a non-sampled parent leaves the derived span
|
|
178
|
+
* non-recording, and its span context would then never reach an exporter — the run would be
|
|
179
|
+
* traceable in Telescope but invisible in any OTel backend, which is the confusing half-state.
|
|
180
|
+
*/
|
|
181
|
+
declare function runTraceContext(runId: string, parent?: Context): Context;
|
|
182
|
+
|
|
26
183
|
/** The first-class Telescope extension for nestjs-durable: watcher + Workflows dashboard. */
|
|
27
184
|
declare function durableTelescopeExtension(opts?: {
|
|
28
185
|
runHref?: string;
|
|
@@ -103,4 +260,4 @@ declare function durableDashboard(opts?: {
|
|
|
103
260
|
recentFailuresWindowMs?: number;
|
|
104
261
|
}): DashboardSpec;
|
|
105
262
|
|
|
106
|
-
export { DurableTelescopeWatcher, durableDashboard, durableDurationProvider, durableRecentFailuresProvider, durableRunsOverTimeProvider, durableStateBreakdownProvider, durableStateProvider, durableSuccessRateProvider, durableTelescopeExtension, durableThroughputProvider, durableTimeseriesProvider, durableWorkerHealthProvider, durableWorkerStatusProvider };
|
|
263
|
+
export { type DurableScope, DurableTelescopeWatcher, currentDurableScope, durableDashboard, durableDurationProvider, durableExecutionScope, durableRecentFailuresProvider, durableRunsOverTimeProvider, durableStateBreakdownProvider, durableStateProvider, durableSuccessRateProvider, durableTelescopeExtension, durableThroughputProvider, durableTimeseriesProvider, durableTraceContext, durableWorkerHealthProvider, durableWorkerStatusProvider, runTraceContext, runTraceId };
|
package/dist/index.js
CHANGED
|
@@ -2,24 +2,137 @@ var __defProp = Object.defineProperty;
|
|
|
2
2
|
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
3
|
|
|
4
4
|
// src/durable-telescope.watcher.ts
|
|
5
|
-
import { WorkflowEngine } from "@dudousxd/nestjs-durable-core";
|
|
6
|
-
import { SpanStatusCode, context, trace } from "@opentelemetry/api";
|
|
5
|
+
import { WorkflowEngine, useDurableExecution } from "@dudousxd/nestjs-durable-core";
|
|
6
|
+
import { SpanStatusCode as SpanStatusCode2, context as context3, trace as trace3 } from "@opentelemetry/api";
|
|
7
|
+
|
|
8
|
+
// src/durable-execution-scope.ts
|
|
9
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
10
|
+
import { randomBytes } from "node:crypto";
|
|
11
|
+
import { SpanStatusCode, context as context2, trace as trace2 } from "@opentelemetry/api";
|
|
12
|
+
|
|
13
|
+
// src/durable-run-trace.ts
|
|
14
|
+
import { createHash } from "node:crypto";
|
|
15
|
+
import { TraceFlags, context, trace } from "@opentelemetry/api";
|
|
16
|
+
function runTraceId(runId) {
|
|
17
|
+
return createHash("sha256").update(`durable:run:${runId}`).digest("hex").slice(0, 32);
|
|
18
|
+
}
|
|
19
|
+
__name(runTraceId, "runTraceId");
|
|
20
|
+
function runSpanId(runId) {
|
|
21
|
+
return createHash("sha256").update(`durable:run-span:${runId}`).digest("hex").slice(0, 16);
|
|
22
|
+
}
|
|
23
|
+
__name(runSpanId, "runSpanId");
|
|
24
|
+
function runTraceContext(runId, parent = context.active()) {
|
|
25
|
+
return trace.setSpanContext(parent, {
|
|
26
|
+
traceId: runTraceId(runId),
|
|
27
|
+
spanId: runSpanId(runId),
|
|
28
|
+
traceFlags: TraceFlags.SAMPLED,
|
|
29
|
+
isRemote: true
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
__name(runTraceContext, "runTraceContext");
|
|
33
|
+
|
|
34
|
+
// src/durable-execution-scope.ts
|
|
35
|
+
var activeScope = new AsyncLocalStorage();
|
|
36
|
+
function currentDurableScope() {
|
|
37
|
+
return activeScope.getStore();
|
|
38
|
+
}
|
|
39
|
+
__name(currentDurableScope, "currentDurableScope");
|
|
40
|
+
var DURABLE_BATCH_ORIGIN = "queue";
|
|
41
|
+
function spanName(execution) {
|
|
42
|
+
if (execution.unit === "step") return `step ${execution.name ?? "?"}`;
|
|
43
|
+
return `workflow ${execution.workflow ?? "?"}`;
|
|
44
|
+
}
|
|
45
|
+
__name(spanName, "spanName");
|
|
46
|
+
function spanAttributes(execution) {
|
|
47
|
+
const attributes = {
|
|
48
|
+
"durable.run_id": execution.runId
|
|
49
|
+
};
|
|
50
|
+
if (execution.workflow !== void 0) attributes["durable.workflow"] = execution.workflow;
|
|
51
|
+
if (execution.name !== void 0) attributes["durable.step.name"] = execution.name;
|
|
52
|
+
if (execution.seq !== void 0) attributes["durable.step.seq"] = execution.seq;
|
|
53
|
+
if (execution.attempt !== void 0) attributes["durable.step.attempt"] = execution.attempt;
|
|
54
|
+
return attributes;
|
|
55
|
+
}
|
|
56
|
+
__name(spanAttributes, "spanAttributes");
|
|
57
|
+
function durableExecutionScope(ctx) {
|
|
58
|
+
const tracer = trace2.getTracer("@dudousxd/nestjs-durable-telescope");
|
|
59
|
+
const owner = {};
|
|
60
|
+
return (execution, next) => {
|
|
61
|
+
const open = activeScope.getStore();
|
|
62
|
+
if (open?.owner === owner && open.runId === execution.runId) return next();
|
|
63
|
+
if (ctx.config.enabled === false) return next();
|
|
64
|
+
try {
|
|
65
|
+
const parent = runTraceContext(execution.runId);
|
|
66
|
+
const span = tracer.startSpan(spanName(execution), {
|
|
67
|
+
attributes: spanAttributes(execution)
|
|
68
|
+
}, parent);
|
|
69
|
+
const spanContext = span.spanContext();
|
|
70
|
+
const scope = {
|
|
71
|
+
runId: execution.runId,
|
|
72
|
+
traceId: runTraceId(execution.runId),
|
|
73
|
+
spanId: span.isRecording() ? spanContext.spanId : randomBytes(8).toString("hex"),
|
|
74
|
+
owner
|
|
75
|
+
};
|
|
76
|
+
return context2.with(trace2.setSpan(parent, span), () => activeScope.run(scope, () => ctx.runInBatch(DURABLE_BATCH_ORIGIN, next).then((value) => {
|
|
77
|
+
span.end();
|
|
78
|
+
return value;
|
|
79
|
+
}, (error) => {
|
|
80
|
+
span.setStatus({
|
|
81
|
+
code: SpanStatusCode.ERROR,
|
|
82
|
+
...error instanceof Error ? {
|
|
83
|
+
message: error.message
|
|
84
|
+
} : {}
|
|
85
|
+
});
|
|
86
|
+
span.end();
|
|
87
|
+
throw error;
|
|
88
|
+
})));
|
|
89
|
+
} catch {
|
|
90
|
+
return next();
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
__name(durableExecutionScope, "durableExecutionScope");
|
|
95
|
+
function durableTraceContext(delegate) {
|
|
96
|
+
return {
|
|
97
|
+
current() {
|
|
98
|
+
const fromDelegate = delegate?.current() ?? null;
|
|
99
|
+
if (fromDelegate) return fromDelegate;
|
|
100
|
+
const scope = activeScope.getStore();
|
|
101
|
+
return scope ? {
|
|
102
|
+
traceId: scope.traceId,
|
|
103
|
+
spanId: scope.spanId
|
|
104
|
+
} : null;
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
__name(durableTraceContext, "durableTraceContext");
|
|
109
|
+
|
|
110
|
+
// src/durable-telescope.watcher.ts
|
|
7
111
|
var DurableTelescopeWatcher = class {
|
|
8
112
|
static {
|
|
9
113
|
__name(this, "DurableTelescopeWatcher");
|
|
10
114
|
}
|
|
11
115
|
type = "durable";
|
|
12
|
-
/**
|
|
13
|
-
|
|
116
|
+
/** Removes the execution wrapper from the engine's process-level registry. */
|
|
117
|
+
disposeScope;
|
|
14
118
|
register(ctx) {
|
|
119
|
+
this.disposeScope?.();
|
|
120
|
+
this.disposeScope = useDurableExecution(durableExecutionScope(ctx));
|
|
15
121
|
const engine = ctx.moduleRef.get(WorkflowEngine, {
|
|
16
122
|
strict: false
|
|
17
123
|
});
|
|
18
124
|
if (typeof engine?.subscribe !== "function") return;
|
|
19
|
-
const tracer =
|
|
125
|
+
const tracer = trace3.getTracer("@dudousxd/nestjs-durable-telescope");
|
|
20
126
|
engine.subscribe((event) => {
|
|
21
127
|
const record = /* @__PURE__ */ __name(() => ctx.record({
|
|
22
128
|
type: this.type,
|
|
129
|
+
// Stated explicitly rather than left to the ambient span, and the Recorder honours an
|
|
130
|
+
// explicit `traceId` above everything else. A lifecycle event is very often emitted
|
|
131
|
+
// OUTSIDE any execution scope — a remote step's result lands in a transport callback, a
|
|
132
|
+
// recovery sweep emits for a run this process never executed — and those entries are
|
|
133
|
+
// exactly the ones an operator follows into the trace. We know the run, so we know the
|
|
134
|
+
// trace; there is no reason to let it depend on where the emit happened to land.
|
|
135
|
+
traceId: runTraceId(event.runId),
|
|
23
136
|
content: {
|
|
24
137
|
event: event.type,
|
|
25
138
|
workflow: event.workflow,
|
|
@@ -33,16 +146,7 @@ var DurableTelescopeWatcher = class {
|
|
|
33
146
|
},
|
|
34
147
|
tags: this.tags(event)
|
|
35
148
|
}), "record");
|
|
36
|
-
|
|
37
|
-
this.roots.set(event.runId, tracer.startSpan(`workflow ${event.workflow ?? "?"}`, {
|
|
38
|
-
attributes: {
|
|
39
|
-
"durable.run_id": event.runId,
|
|
40
|
-
"durable.workflow": event.workflow
|
|
41
|
-
}
|
|
42
|
-
}));
|
|
43
|
-
}
|
|
44
|
-
const root = this.roots.get(event.runId);
|
|
45
|
-
const base = root ? trace.setSpan(context.active(), root) : context.active();
|
|
149
|
+
const base = runTraceContext(event.runId);
|
|
46
150
|
if (event.name != null && event.seq != null) {
|
|
47
151
|
const step = tracer.startSpan(`step ${event.name}`, {
|
|
48
152
|
attributes: {
|
|
@@ -51,27 +155,32 @@ var DurableTelescopeWatcher = class {
|
|
|
51
155
|
"durable.step.kind": event.kind
|
|
52
156
|
}
|
|
53
157
|
}, base);
|
|
54
|
-
|
|
158
|
+
if (event.type === "step.failed") {
|
|
159
|
+
const message = event.error?.message;
|
|
160
|
+
step.setStatus(message !== void 0 ? {
|
|
161
|
+
code: SpanStatusCode2.ERROR,
|
|
162
|
+
message
|
|
163
|
+
} : {
|
|
164
|
+
code: SpanStatusCode2.ERROR
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
context3.with(trace3.setSpan(base, step), record);
|
|
55
168
|
step.end();
|
|
56
169
|
} else {
|
|
57
|
-
|
|
58
|
-
}
|
|
59
|
-
if (event.type === "run.completed" || event.type === "run.suspended") {
|
|
60
|
-
root?.end();
|
|
61
|
-
this.roots.delete(event.runId);
|
|
62
|
-
} else if (event.type === "run.failed") {
|
|
63
|
-
const message = event.error?.message;
|
|
64
|
-
root?.setStatus(message !== void 0 ? {
|
|
65
|
-
code: SpanStatusCode.ERROR,
|
|
66
|
-
message
|
|
67
|
-
} : {
|
|
68
|
-
code: SpanStatusCode.ERROR
|
|
69
|
-
});
|
|
70
|
-
root?.end();
|
|
71
|
-
this.roots.delete(event.runId);
|
|
170
|
+
context3.with(base, record);
|
|
72
171
|
}
|
|
73
172
|
});
|
|
74
173
|
}
|
|
174
|
+
/**
|
|
175
|
+
* Detach the execution wrapper. The `Watcher` SPI has no teardown hook, so nothing calls this in a
|
|
176
|
+
* running app — where the wrapper should live exactly as long as the process does. It exists so a
|
|
177
|
+
* test that registers a watcher does not leave a wrapper behind in the process-level registry for
|
|
178
|
+
* the next test to trip over.
|
|
179
|
+
*/
|
|
180
|
+
unregister() {
|
|
181
|
+
this.disposeScope?.();
|
|
182
|
+
this.disposeScope = void 0;
|
|
183
|
+
}
|
|
75
184
|
tags(event) {
|
|
76
185
|
const tags = [
|
|
77
186
|
"durable",
|
|
@@ -878,8 +987,10 @@ function durableTelescopeExtension(opts = {}) {
|
|
|
878
987
|
__name(durableTelescopeExtension, "durableTelescopeExtension");
|
|
879
988
|
export {
|
|
880
989
|
DurableTelescopeWatcher,
|
|
990
|
+
currentDurableScope,
|
|
881
991
|
durableDashboard,
|
|
882
992
|
durableDurationProvider,
|
|
993
|
+
durableExecutionScope,
|
|
883
994
|
durableRecentFailuresProvider,
|
|
884
995
|
durableRunsOverTimeProvider,
|
|
885
996
|
durableStateBreakdownProvider,
|
|
@@ -888,7 +999,10 @@ export {
|
|
|
888
999
|
durableTelescopeExtension,
|
|
889
1000
|
durableThroughputProvider,
|
|
890
1001
|
durableTimeseriesProvider,
|
|
1002
|
+
durableTraceContext,
|
|
891
1003
|
durableWorkerHealthProvider,
|
|
892
|
-
durableWorkerStatusProvider
|
|
1004
|
+
durableWorkerStatusProvider,
|
|
1005
|
+
runTraceContext,
|
|
1006
|
+
runTraceId
|
|
893
1007
|
};
|
|
894
1008
|
//# sourceMappingURL=index.js.map
|