@xeonr/upload-pool-sdk 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/job-context.d.ts +19 -1
- package/dist/job-context.d.ts.map +1 -1
- package/dist/job-context.js +229 -140
- package/dist/job-context.js.map +1 -1
- package/dist/pool.d.ts +1 -0
- package/dist/pool.d.ts.map +1 -1
- package/dist/pool.js +120 -40
- package/dist/pool.js.map +1 -1
- package/dist/rpc-clients.d.ts.map +1 -1
- package/dist/rpc-clients.js +122 -28
- package/dist/rpc-clients.js.map +1 -1
- package/dist/tracing.d.ts +128 -0
- package/dist/tracing.d.ts.map +1 -0
- package/dist/tracing.js +299 -0
- package/dist/tracing.js.map +1 -0
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +8 -1
- package/src/job-context.ts +328 -150
- package/src/pool.ts +160 -49
- package/src/rpc-clients.ts +150 -28
- package/src/tracing.ts +363 -0
- package/src/types.ts +8 -0
package/dist/pool.js
CHANGED
|
@@ -12,17 +12,20 @@
|
|
|
12
12
|
import { hostname } from "node:os";
|
|
13
13
|
import { randomBytes } from "node:crypto";
|
|
14
14
|
import { create } from "@bufbuild/protobuf";
|
|
15
|
+
import { SpanStatusCode } from "@opentelemetry/api";
|
|
15
16
|
import { AcceptJobRequestSchema, CompleteJobRequestSchema, ReportErrorRequestSchema, } from "./protocol/uplim/workflow/v1/integration_queue_pb.js";
|
|
16
17
|
import { createRpcClients } from "./rpc-clients.js";
|
|
17
18
|
import { SseClient } from "./sse-client.js";
|
|
18
19
|
import { createJobContext } from "./job-context.js";
|
|
19
20
|
import { NonRetryableError } from "./errors.js";
|
|
20
21
|
import { JsonLogger } from "./logger.js";
|
|
22
|
+
import { initTracing, shutdownTracing, recordSpanError, stampJobAttributes, withJobContext, } from "./tracing.js";
|
|
21
23
|
export class Pool {
|
|
22
24
|
config;
|
|
23
25
|
rpc;
|
|
24
26
|
sse;
|
|
25
27
|
logger;
|
|
28
|
+
tracing;
|
|
26
29
|
inFlight = 0;
|
|
27
30
|
workerId;
|
|
28
31
|
capabilities;
|
|
@@ -39,6 +42,17 @@ export class Pool {
|
|
|
39
42
|
this.logger = (config.logger ?? new JsonLogger()).child({
|
|
40
43
|
workerId: this.workerId,
|
|
41
44
|
});
|
|
45
|
+
// Initialise tracing *before* RPC clients so the interceptor in
|
|
46
|
+
// rpc-clients picks up an active TracerProvider on its first call.
|
|
47
|
+
// Spans go to pipeline-api's /v1/traces receiver; see tracing.ts
|
|
48
|
+
// for the isolation rationale wrt the host app's OTel pipeline.
|
|
49
|
+
this.tracing = initTracing({
|
|
50
|
+
pipelineEndpoint: config.pipelineEndpoint,
|
|
51
|
+
poolToken: config.token,
|
|
52
|
+
workerId: this.workerId,
|
|
53
|
+
sdkVersion: SDK_VERSION,
|
|
54
|
+
enabled: config.tracing?.enabled,
|
|
55
|
+
});
|
|
42
56
|
this.rpc = createRpcClients({
|
|
43
57
|
apiEndpoint: config.apiEndpoint,
|
|
44
58
|
pipelineEndpoint: config.pipelineEndpoint,
|
|
@@ -114,6 +128,9 @@ export class Pool {
|
|
|
114
128
|
clearInterval(this.keepAliveInterval);
|
|
115
129
|
this.keepAliveInterval = null;
|
|
116
130
|
}
|
|
131
|
+
// Flush any buffered spans before exit so the final job's traces
|
|
132
|
+
// land in the admin UI even on a clean SIGTERM.
|
|
133
|
+
await shutdownTracing();
|
|
117
134
|
if (this.runningResolve) {
|
|
118
135
|
this.runningResolve();
|
|
119
136
|
this.runningResolve = null;
|
|
@@ -137,49 +154,112 @@ export class Pool {
|
|
|
137
154
|
return;
|
|
138
155
|
}
|
|
139
156
|
this.inFlight++;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
|
|
157
|
+
// Root the entire job under iq.job, parented to the dispatching
|
|
158
|
+
// activity's span via the traceparent the pipeline-worker stamped
|
|
159
|
+
// onto the envelope. All ctx.* + rpc.* spans created inside the
|
|
160
|
+
// handler will nest under this via OTel's AsyncLocalStorage-backed
|
|
161
|
+
// context manager.
|
|
162
|
+
const parentCtx = this.tracing.contextFromEnvelope(envelope.traceContext ?? "", envelope.wfRunId ?? "");
|
|
163
|
+
// Carry the job's identifiers in the active OTel context so
|
|
164
|
+
// stampJobAttributes() called by child spans (interceptors,
|
|
165
|
+
// ctx.* spans) can read them. OTel deliberately doesn't expose
|
|
166
|
+
// parent-span attributes back to child-creation sites — context
|
|
167
|
+
// values are the canonical cross-span carrier.
|
|
168
|
+
const jobAttrs = {
|
|
169
|
+
runId: envelope.wfRunId ?? "",
|
|
170
|
+
jobId: envelope.jobId,
|
|
171
|
+
uploadId: envelope.uploadId,
|
|
172
|
+
urn: envelope.contentTypeContext.urn,
|
|
173
|
+
workerId: this.workerId,
|
|
174
|
+
};
|
|
175
|
+
await withJobContext(jobAttrs, () => this.tracing.tracer.startActiveSpan("iq.job", {
|
|
176
|
+
attributes: {
|
|
177
|
+
"iq.filename": envelope.filename,
|
|
178
|
+
"iq.mime_type": envelope.mimeType,
|
|
179
|
+
},
|
|
180
|
+
}, parentCtx, async (rootSpan) => {
|
|
181
|
+
stampJobAttributes(rootSpan, {
|
|
182
|
+
runId: envelope.wfRunId ?? "",
|
|
165
183
|
jobId: envelope.jobId,
|
|
184
|
+
uploadId: envelope.uploadId,
|
|
185
|
+
urn: envelope.contentTypeContext.urn,
|
|
166
186
|
workerId: this.workerId,
|
|
167
|
-
queueToken: this.config.token,
|
|
168
|
-
}));
|
|
169
|
-
jobLogger.info("job.completed", {
|
|
170
|
-
durationMs: Date.now() - startedAt,
|
|
171
187
|
});
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
durationMs: Date.now() - startedAt,
|
|
188
|
+
jobLogger.info("job.dispatched", {
|
|
189
|
+
filename: envelope.filename,
|
|
190
|
+
mimeType: envelope.mimeType,
|
|
191
|
+
inFlight: this.inFlight,
|
|
177
192
|
});
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
193
|
+
const handler = this.resolveHandler(envelope.contentTypeContext.urn);
|
|
194
|
+
const ctx = createJobContext(envelope, this.rpc, jobLogger, {
|
|
195
|
+
workerId: this.workerId,
|
|
196
|
+
});
|
|
197
|
+
const startedAt = Date.now();
|
|
198
|
+
try {
|
|
199
|
+
// AcceptJob — clears the pipeline's accept-timeout.
|
|
200
|
+
await this.rpc.integrationQueue.acceptJob(create(AcceptJobRequestSchema, {
|
|
201
|
+
jobId: envelope.jobId,
|
|
202
|
+
workerId: this.workerId,
|
|
203
|
+
queueToken: this.config.token,
|
|
204
|
+
}));
|
|
205
|
+
jobLogger.info("job.accepted");
|
|
206
|
+
if (!handler) {
|
|
207
|
+
jobLogger.warn("job.unhandled", {
|
|
208
|
+
availableHandlers: this.capabilities,
|
|
209
|
+
});
|
|
210
|
+
await this.reportError(envelope.jobId, ctx, jobLogger, new NonRetryableError(`no handler for URN ${envelope.contentTypeContext.urn}`));
|
|
211
|
+
rootSpan.setStatus({
|
|
212
|
+
code: SpanStatusCode.ERROR,
|
|
213
|
+
message: "no handler",
|
|
214
|
+
});
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
// iq.handler wraps the customer's handler function so its
|
|
218
|
+
// own latency is visible as a single row (excluding the
|
|
219
|
+
// accept/complete bookends). Children appear nested.
|
|
220
|
+
await this.tracing.tracer.startActiveSpan("iq.handler", async (handlerSpan) => {
|
|
221
|
+
stampJobAttributes(handlerSpan, {
|
|
222
|
+
runId: envelope.wfRunId ?? "",
|
|
223
|
+
jobId: envelope.jobId,
|
|
224
|
+
uploadId: envelope.uploadId,
|
|
225
|
+
urn: envelope.contentTypeContext.urn,
|
|
226
|
+
workerId: this.workerId,
|
|
227
|
+
});
|
|
228
|
+
try {
|
|
229
|
+
await handler(ctx);
|
|
230
|
+
handlerSpan.setStatus({ code: SpanStatusCode.OK });
|
|
231
|
+
}
|
|
232
|
+
catch (err) {
|
|
233
|
+
recordSpanError(handlerSpan, err);
|
|
234
|
+
throw err;
|
|
235
|
+
}
|
|
236
|
+
finally {
|
|
237
|
+
handlerSpan.end();
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
await this.rpc.integrationQueue.completeJob(create(CompleteJobRequestSchema, {
|
|
241
|
+
jobId: envelope.jobId,
|
|
242
|
+
workerId: this.workerId,
|
|
243
|
+
queueToken: this.config.token,
|
|
244
|
+
}));
|
|
245
|
+
jobLogger.info("job.completed", {
|
|
246
|
+
durationMs: Date.now() - startedAt,
|
|
247
|
+
});
|
|
248
|
+
rootSpan.setStatus({ code: SpanStatusCode.OK });
|
|
249
|
+
}
|
|
250
|
+
catch (err) {
|
|
251
|
+
jobLogger.error("job.failed", {
|
|
252
|
+
err,
|
|
253
|
+
durationMs: Date.now() - startedAt,
|
|
254
|
+
});
|
|
255
|
+
recordSpanError(rootSpan, err);
|
|
256
|
+
await this.reportError(envelope.jobId, ctx, jobLogger, err);
|
|
257
|
+
}
|
|
258
|
+
finally {
|
|
259
|
+
rootSpan.end();
|
|
260
|
+
this.inFlight--;
|
|
261
|
+
}
|
|
262
|
+
}));
|
|
183
263
|
}
|
|
184
264
|
resolveHandler(urn) {
|
|
185
265
|
return this.config.handlers[urn] ?? this.config.onUnhandled;
|
|
@@ -203,5 +283,5 @@ export class Pool {
|
|
|
203
283
|
}
|
|
204
284
|
}
|
|
205
285
|
}
|
|
206
|
-
const SDK_VERSION = "1.
|
|
286
|
+
const SDK_VERSION = "1.4.1";
|
|
207
287
|
//# sourceMappingURL=pool.js.map
|
package/dist/pool.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pool.js","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EACN,sBAAsB,EACtB,wBAAwB,EACxB,wBAAwB,GACxB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAmB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAe,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pool.js","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EACN,sBAAsB,EACtB,wBAAwB,EACxB,wBAAwB,GACxB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAmB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAe,MAAM,aAAa,CAAC;AACtD,OAAO,EACN,WAAW,EACX,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,cAAc,GAEd,MAAM,cAAc,CAAC;AAOtB,MAAM,OAAO,IAAI;IACC,MAAM,CAAa;IACnB,GAAG,CAAa;IAChB,GAAG,CAAY;IACf,MAAM,CAAS;IACf,OAAO,CAAgB;IAChC,QAAQ,GAAG,CAAC,CAAC;IACJ,QAAQ,CAAS;IACjB,YAAY,CAAW;IAChC,cAAc,GAAyB,IAAI,CAAC;IAC5C,cAAc,GAAwB,IAAI,CAAC;IAC3C,iBAAiB,GAA0B,IAAI,CAAC;IAExD,YAAY,MAAkB;QAC7B,IAAI,CAAC,MAAM,GAAG;YACb,WAAW,EAAE,CAAC;YACd,GAAG,MAAM;SACT,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,GAAG,QAAQ,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrF,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC;YACvD,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC,CAAC;QAEH,gEAAgE;QAChE,mEAAmE;QACnE,iEAAiE;QACjE,gEAAgE;QAChE,IAAI,CAAC,OAAO,GAAG,WAAW,CAAC;YAC1B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,SAAS,EAAE,MAAM,CAAC,KAAK;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,WAAW;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,GAAG,gBAAgB,CAAC;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,MAAM,EAAE,IAAI,CAAC,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,GAAG,IAAI,SAAS,CAAC;YACxB,QAAQ,EAAE,MAAM,CAAC,gBAAgB;YACjC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;YAC/C,WAAW,EAAE,GAAG,EAAE;gBACjB,qEAAqE;YACtE,CAAC;YACD,cAAc,EAAE,GAAG,EAAE;gBACpB,8BAA8B;YAC/B,CAAC;YACD,aAAa,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAsB,CAAC;SACvE,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE;YAC5B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;YACpC,OAAO,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBACxC,IAAI,EAAE,2DAA2D;aACjE,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC,cAAc,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,CAAC,cAAc,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,iEAAiE;QACjE,4BAA4B;QAC5B,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,EAAE,MAAM,CAAC,CAAC;QACX,OAAO,IAAI,CAAC,cAAc,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACtC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,iEAAiE;QACjE,gDAAgD;QAChD,MAAM,eAAe,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,QAAqB;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,GAAG,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG;SACpC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC;YACrD,gEAAgE;YAChE,0DAA0D;YAC1D,0DAA0D;YAC1D,SAAS,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACpD,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;aACpC,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,gEAAgE;QAChE,kEAAkE;QAClE,gEAAgE;QAChE,mEAAmE;QACnE,mBAAmB;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACjD,QAAQ,CAAC,YAAY,IAAI,EAAE,EAC3B,QAAQ,CAAC,OAAO,IAAI,EAAE,CACtB,CAAC;QAEF,4DAA4D;QAC5D,4DAA4D;QAC5D,+DAA+D;QAC/D,gEAAgE;QAChE,+CAA+C;QAC/C,MAAM,QAAQ,GAAG;YAChB,KAAK,EAAE,QAAQ,CAAC,OAAO,IAAI,EAAE;YAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,GAAG,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG;YACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC;QAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,CACpC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAClC,QAAQ,EACR;YACC,UAAU,EAAE;gBACX,aAAa,EAAE,QAAQ,CAAC,QAAQ;gBAChC,cAAc,EAAE,QAAQ,CAAC,QAAQ;aACjC;SACD,EACD,SAAS,EACT,KAAK,EAAE,QAAQ,EAAE,EAAE;YAClB,kBAAkB,CAAC,QAAQ,EAAE;gBAC5B,KAAK,EAAE,QAAQ,CAAC,OAAO,IAAI,EAAE;gBAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,GAAG,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG;gBACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACvB,CAAC,CAAC;YAEH,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAChC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACvB,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACrE,MAAM,GAAG,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE;gBAC3D,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACvB,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,IAAI,CAAC;gBACJ,oDAAoD;gBACpD,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,SAAS,CACxC,MAAM,CAAC,sBAAsB,EAAE;oBAC9B,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;iBAC7B,CAAC,CACF,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAE/B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACd,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE;wBAC/B,iBAAiB,EAAE,IAAI,CAAC,YAAY;qBACpC,CAAC,CAAC;oBACH,MAAM,IAAI,CAAC,WAAW,CACrB,QAAQ,CAAC,KAAK,EACd,GAAG,EACH,SAAS,EACT,IAAI,iBAAiB,CACpB,sBAAsB,QAAQ,CAAC,kBAAkB,CAAC,GAAG,EAAE,CACvD,CACD,CAAC;oBACF,QAAQ,CAAC,SAAS,CAAC;wBAClB,IAAI,EAAE,cAAc,CAAC,KAAK;wBAC1B,OAAO,EAAE,YAAY;qBACrB,CAAC,CAAC;oBACH,OAAO;gBACR,CAAC;gBAED,0DAA0D;gBAC1D,wDAAwD;gBACxD,qDAAqD;gBACrD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CACxC,YAAY,EACZ,KAAK,EAAE,WAAW,EAAE,EAAE;oBACrB,kBAAkB,CAAC,WAAW,EAAE;wBAC/B,KAAK,EAAE,QAAQ,CAAC,OAAO,IAAI,EAAE;wBAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;wBACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,GAAG,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG;wBACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;qBACvB,CAAC,CAAC;oBACH,IAAI,CAAC;wBACJ,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBACnB,WAAW,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;oBACpD,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACd,eAAe,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;wBAClC,MAAM,GAAG,CAAC;oBACX,CAAC;4BAAS,CAAC;wBACV,WAAW,CAAC,GAAG,EAAE,CAAC;oBACnB,CAAC;gBACF,CAAC,CACD,CAAC;gBAEF,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,WAAW,CAC1C,MAAM,CAAC,wBAAwB,EAAE;oBAChC,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;iBAC7B,CAAC,CACF,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE;oBAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBAClC,CAAC,CAAC;gBACH,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE;oBAC7B,GAAG;oBACH,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBAClC,CAAC,CAAC;gBACH,eAAe,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC/B,MAAM,IAAI,CAAC,WAAW,CACrB,QAAQ,CAAC,KAAK,EACd,GAAG,EACH,SAAS,EACT,GAAY,CACZ,CAAC;YACH,CAAC;oBAAS,CAAC;gBACV,QAAQ,CAAC,GAAG,EAAE,CAAC;gBACf,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,CAAC;QACF,CAAC,CACD,CACA,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,WAAW,CACxB,KAAa,EACb,GAAe,EACf,SAAiB,EACjB,GAAU;QAEV,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,YAAY,iBAAiB,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,WAAW,CAC1C,MAAM,CAAC,wBAAwB,EAAE;gBAChC,KAAK;gBACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBAC7B,KAAK,EAAE,GAAG,CAAC,OAAO;gBAClB,KAAK;aACL,CAAC,CACF,CAAC;YACF,SAAS,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YACjB,SAAS,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAe,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;CACD;AAED,MAAM,WAAW,GAAG,OAAO,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rpc-clients.d.ts","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rpc-clients.d.ts","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AA+BA,OAAO,EAEN,KAAK,MAAM,EAIX,MAAM,qBAAqB,CAAC;AAM7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,sDAAsD,CAAC;AAC/F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAU1C,MAAM,WAAW,UAAU;IAC1B,eAAe,EAAE,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;IACvD,gBAAgB,EAAE,MAAM,CAAC,OAAO,uBAAuB,CAAC,CAAC;CACzD;AA8HD,MAAM,WAAW,gBAAgB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,UAAU,CAiBrE"}
|
package/dist/rpc-clients.js
CHANGED
|
@@ -13,41 +13,132 @@
|
|
|
13
13
|
* pipeline-api) so we keep them on separate transports and don't try to
|
|
14
14
|
* share a baseUrl.
|
|
15
15
|
*
|
|
16
|
-
* Each request is wrapped in a logging interceptor
|
|
17
|
-
* `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* Each request is wrapped in a tracing-and-logging interceptor:
|
|
17
|
+
* - emits one `iq.callback.<Method>` span per call, child of the
|
|
18
|
+
* active job span (so the admin UI sees a row per RPC under the
|
|
19
|
+
* dispatching parseUpload run)
|
|
20
|
+
* - injects W3C `traceparent` on outbound headers *for the pipeline
|
|
21
|
+
* transport only* — calls to `apiEndpoint` ship without traceparent
|
|
22
|
+
* so upl-im-api's own OTel pipeline doesn't pick up our trace_id
|
|
23
|
+
* and conflate the two backends
|
|
24
|
+
* - captures sanitised request/response bodies as span events (4 KB
|
|
25
|
+
* cap, keys matching token/secret/auth/... redacted), so debugging
|
|
26
|
+
* a worker callback failure doesn't require correlating with
|
|
27
|
+
* server-side logs
|
|
28
|
+
* - logs `rpc.request` / `rpc.response` / `rpc.error` at info/error
|
|
29
|
+
* for the legacy log-only consumer surface that predates tracing
|
|
20
30
|
*/
|
|
21
31
|
import { createConnectTransport } from "@connectrpc/connect-node";
|
|
22
32
|
import { createClient, ConnectError, Code, } from "@connectrpc/connect";
|
|
33
|
+
import { context as otelContext, SpanStatusCode, trace, } from "@opentelemetry/api";
|
|
23
34
|
import { InternalUploadsService } from "./protocol/uplim/api/v1/uploads_pb.js";
|
|
24
35
|
import { IntegrationQueueService } from "./protocol/uplim/workflow/v1/integration_queue_pb.js";
|
|
25
|
-
|
|
36
|
+
import { SPAN_ATTR, getTracingHandle, recordSpanError, sanitizeRpcBody, spanContextToTraceparent, stampJobAttributes, } from "./tracing.js";
|
|
37
|
+
function tracingLoggingInterceptor(logger, target) {
|
|
26
38
|
return (next) => async (req) => {
|
|
27
|
-
const
|
|
39
|
+
const handle = getTracingHandle();
|
|
28
40
|
const method = `${req.service.typeName}/${req.method.name}`;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
41
|
+
// Inject W3C traceparent on the pipeline target *only*. upl-im-api
|
|
42
|
+
// runs its own OTel pipeline against a different backend; passing
|
|
43
|
+
// our trace_id over would graft our worker activity into its trees,
|
|
44
|
+
// muddying its admin dashboards. The pool token in the request
|
|
45
|
+
// body / queue token in headers remain the trust boundary either
|
|
46
|
+
// way.
|
|
47
|
+
if (target === "pipeline") {
|
|
48
|
+
const activeSpan = trace.getActiveSpan();
|
|
49
|
+
if (activeSpan) {
|
|
50
|
+
req.header.set("traceparent", spanContextToTraceparent(activeSpan.spanContext()));
|
|
51
|
+
}
|
|
38
52
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
const startedAt = Date.now();
|
|
54
|
+
logger.debug("rpc.request", { method, target });
|
|
55
|
+
// When tracing is uninitialised (e.g. tests, dry-runs), fall through
|
|
56
|
+
// to a logging-only path so we don't crash on a null tracer.
|
|
57
|
+
if (!handle) {
|
|
58
|
+
try {
|
|
59
|
+
const res = await next(req);
|
|
60
|
+
logger.debug("rpc.response", {
|
|
61
|
+
method,
|
|
62
|
+
target,
|
|
63
|
+
durationMs: Date.now() - startedAt,
|
|
64
|
+
});
|
|
65
|
+
return res;
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
const code = err instanceof ConnectError ? Code[err.code] : undefined;
|
|
69
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
70
|
+
logger.error("rpc.error", {
|
|
71
|
+
method,
|
|
72
|
+
target,
|
|
73
|
+
code,
|
|
74
|
+
message,
|
|
75
|
+
durationMs: Date.now() - startedAt,
|
|
76
|
+
});
|
|
77
|
+
throw err;
|
|
78
|
+
}
|
|
50
79
|
}
|
|
80
|
+
return handle.tracer.startActiveSpan(`iq.callback.${req.method.name}`, async (span) => {
|
|
81
|
+
span.setAttribute(SPAN_ATTR.RPC_METHOD, method);
|
|
82
|
+
span.setAttribute(SPAN_ATTR.RPC_SERVICE, req.service.typeName);
|
|
83
|
+
span.setAttribute(SPAN_ATTR.RPC_TARGET, target);
|
|
84
|
+
// Job-scoped attrs (runId, jobId, ...) pulled from the
|
|
85
|
+
// active OTel context — set once by Pool.handleDispatch
|
|
86
|
+
// via withJobContext. The receiver requires pipeline.run_id
|
|
87
|
+
// or the span is dropped, so this stamp is load-bearing.
|
|
88
|
+
stampJobAttributes(span);
|
|
89
|
+
// Body capture: request first, response on success, error
|
|
90
|
+
// payload on failure. Sanitiser truncates to 4 KB and redacts
|
|
91
|
+
// token-shaped keys (see tracing.ts).
|
|
92
|
+
try {
|
|
93
|
+
span.addEvent("rpc.request", {
|
|
94
|
+
body: sanitizeRpcBody(req.message),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
/* never let body capture crash the RPC */
|
|
99
|
+
}
|
|
100
|
+
try {
|
|
101
|
+
const res = await next(req);
|
|
102
|
+
const durationMs = Date.now() - startedAt;
|
|
103
|
+
span.setAttribute(SPAN_ATTR.RPC_DURATION_MS, durationMs);
|
|
104
|
+
span.setAttribute(SPAN_ATTR.RPC_CODE, "ok");
|
|
105
|
+
try {
|
|
106
|
+
span.addEvent("rpc.response", {
|
|
107
|
+
body: sanitizeRpcBody(res.message),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
/* body capture is best-effort */
|
|
112
|
+
}
|
|
113
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
114
|
+
logger.debug("rpc.response", {
|
|
115
|
+
method,
|
|
116
|
+
target,
|
|
117
|
+
durationMs,
|
|
118
|
+
});
|
|
119
|
+
return res;
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
const durationMs = Date.now() - startedAt;
|
|
123
|
+
const code = err instanceof ConnectError ? Code[err.code] : "internal";
|
|
124
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
125
|
+
span.setAttribute(SPAN_ATTR.RPC_DURATION_MS, durationMs);
|
|
126
|
+
span.setAttribute(SPAN_ATTR.RPC_CODE, String(code));
|
|
127
|
+
span.addEvent("rpc.error", { code: String(code), message });
|
|
128
|
+
recordSpanError(span, err);
|
|
129
|
+
logger.error("rpc.error", {
|
|
130
|
+
method,
|
|
131
|
+
target,
|
|
132
|
+
code,
|
|
133
|
+
message,
|
|
134
|
+
durationMs,
|
|
135
|
+
});
|
|
136
|
+
throw err;
|
|
137
|
+
}
|
|
138
|
+
finally {
|
|
139
|
+
span.end();
|
|
140
|
+
}
|
|
141
|
+
});
|
|
51
142
|
};
|
|
52
143
|
}
|
|
53
144
|
export function createRpcClients(config) {
|
|
@@ -55,16 +146,19 @@ export function createRpcClients(config) {
|
|
|
55
146
|
const apiTransport = createConnectTransport({
|
|
56
147
|
baseUrl: config.apiEndpoint,
|
|
57
148
|
httpVersion: "1.1",
|
|
58
|
-
interceptors: [
|
|
149
|
+
interceptors: [tracingLoggingInterceptor(rpcLogger, "api")],
|
|
59
150
|
});
|
|
60
151
|
const pipelineTransport = createConnectTransport({
|
|
61
152
|
baseUrl: config.pipelineEndpoint,
|
|
62
153
|
httpVersion: "1.1",
|
|
63
|
-
interceptors: [
|
|
154
|
+
interceptors: [tracingLoggingInterceptor(rpcLogger, "pipeline")],
|
|
64
155
|
});
|
|
65
156
|
return {
|
|
66
157
|
internalUploads: createClient(InternalUploadsService, apiTransport),
|
|
67
158
|
integrationQueue: createClient(IntegrationQueueService, pipelineTransport),
|
|
68
159
|
};
|
|
69
160
|
}
|
|
161
|
+
// keep otelContext import alive for future extensions that may want to
|
|
162
|
+
// detach span scopes around streaming responses.
|
|
163
|
+
void otelContext;
|
|
70
164
|
//# sourceMappingURL=rpc-clients.js.map
|
package/dist/rpc-clients.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rpc-clients.js","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"rpc-clients.js","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACN,YAAY,EAGZ,YAAY,EACZ,IAAI,GACJ,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACN,OAAO,IAAI,WAAW,EACtB,cAAc,EACd,KAAK,GACL,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,sDAAsD,CAAC;AAE/F,OAAO,EACN,SAAS,EACT,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,wBAAwB,EACxB,kBAAkB,GAClB,MAAM,cAAc,CAAC;AAStB,SAAS,yBAAyB,CACjC,MAAc,EACd,MAAiB;IAEjB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5D,mEAAmE;QACnE,kEAAkE;QAClE,oEAAoE;QACpE,+DAA+D;QAC/D,iEAAiE;QACjE,OAAO;QACP,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;YACzC,IAAI,UAAU,EAAE,CAAC;gBAChB,GAAG,CAAC,MAAM,CAAC,GAAG,CACb,aAAa,EACb,wBAAwB,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAClD,CAAC;YACH,CAAC;QACF,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAEhD,qEAAqE;QACrE,6DAA6D;QAC7D,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,IAAI,CAAC;gBACJ,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE;oBAC5B,MAAM;oBACN,MAAM;oBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBAClC,CAAC,CAAC;gBACH,OAAO,GAAG,CAAC;YACZ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,IAAI,GACT,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1D,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE;oBACzB,MAAM;oBACN,MAAM;oBACN,IAAI;oBACJ,OAAO;oBACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBAClC,CAAC,CAAC;gBACH,MAAM,GAAG,CAAC;YACX,CAAC;QACF,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CACnC,eAAe,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAChC,KAAK,EAAE,IAAI,EAAE,EAAE;YACd,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAChD,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAEhD,uDAAuD;YACvD,wDAAwD;YACxD,4DAA4D;YAC5D,yDAAyD;YACzD,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAEzB,0DAA0D;YAC1D,8DAA8D;YAC9D,sCAAsC;YACtC,IAAI,CAAC;gBACJ,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE;oBAC5B,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC;iBAClC,CAAC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACR,0CAA0C;YAC3C,CAAC;YAED,IAAI,CAAC;gBACJ,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBACzD,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC5C,IAAI,CAAC;oBACJ,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE;wBAC7B,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC;qBAClC,CAAC,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACR,iCAAiC;gBAClC,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC5C,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE;oBAC5B,MAAM;oBACN,MAAM;oBACN,UAAU;iBACV,CAAC,CAAC;gBACH,OAAO,GAAG,CAAC;YACZ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAC1C,MAAM,IAAI,GACT,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;gBAC3D,MAAM,OAAO,GACZ,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAClD,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBACzD,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACpD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5D,eAAe,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC3B,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE;oBACzB,MAAM;oBACN,MAAM;oBACN,IAAI;oBACJ,OAAO;oBACP,UAAU;iBACV,CAAC,CAAC;gBACH,MAAM,GAAG,CAAC;YACX,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,GAAG,EAAE,CAAC;YACZ,CAAC;QACF,CAAC,CACD,CAAC;IACH,CAAC,CAAC;AACH,CAAC;AAQD,MAAM,UAAU,gBAAgB,CAAC,MAAwB;IACxD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAE5D,MAAM,YAAY,GAAG,sBAAsB,CAAC;QAC3C,OAAO,EAAE,MAAM,CAAC,WAAW;QAC3B,WAAW,EAAE,KAAK;QAClB,YAAY,EAAE,CAAC,yBAAyB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;KAC3D,CAAC,CAAC;IACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;QAChD,OAAO,EAAE,MAAM,CAAC,gBAAgB;QAChC,WAAW,EAAE,KAAK;QAClB,YAAY,EAAE,CAAC,yBAAyB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;KAChE,CAAC,CAAC;IACH,OAAO;QACN,eAAe,EAAE,YAAY,CAAC,sBAAsB,EAAE,YAAY,CAAC;QACnE,gBAAgB,EAAE,YAAY,CAAC,uBAAuB,EAAE,iBAAiB,CAAC;KAC1E,CAAC;AACH,CAAC;AAED,uEAAuE;AACvE,iDAAiD;AACjD,KAAK,WAAW,CAAC"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenTelemetry tracing setup for the worker SDK.
|
|
3
|
+
*
|
|
4
|
+
* Spans are exported via OTLP/HTTP-JSON to pipeline-api's `/v1/traces`
|
|
5
|
+
* receiver, authenticated with the pool's queue token. The pipeline-api
|
|
6
|
+
* receiver decodes the OTLP envelope and writes spans to its MySQL trace
|
|
7
|
+
* store, which backs the `/admin/workflow-run` admin UI — so worker
|
|
8
|
+
* callbacks nest naturally under the dispatching parseUpload trace.
|
|
9
|
+
*
|
|
10
|
+
* Design notes (see also: discussion in epic plan):
|
|
11
|
+
* - The TracerProvider is private to the SDK. It does *not* register
|
|
12
|
+
* globally — calling `trace.getTracer(...)` outside the SDK won't pick
|
|
13
|
+
* it up. This avoids interfering with any OTel setup the host
|
|
14
|
+
* application already has (e.g. its own collector → Grafana
|
|
15
|
+
* pipeline). The exporter ships SDK-emitted spans only.
|
|
16
|
+
* - `wf.run_id` is stamped on every emitted span as an attribute,
|
|
17
|
+
* mirrored from the job envelope's `wf_run_id` field. The pipeline-api
|
|
18
|
+
* receiver uses this attribute to associate the span with the right
|
|
19
|
+
* workflow run in `workflow_run_spans`.
|
|
20
|
+
* - The W3C traceparent propagator runs on outbound RPC calls *only when
|
|
21
|
+
* targeting the pipeline endpoint*. The uploads-api side has its own
|
|
22
|
+
* OTel pipeline and we don't want to inject our trace_id into its
|
|
23
|
+
* trees. See the interceptor in rpc-clients.ts.
|
|
24
|
+
*
|
|
25
|
+
* Body sanitization for `rpc.request` / `rpc.response` span events lives
|
|
26
|
+
* in the interceptor — keys matching /(token|secret|password|authorization
|
|
27
|
+
* |api[_-]?key|signature|sig|cookie)/i are redacted; payloads truncated
|
|
28
|
+
* to MAX_BODY_EVENT_BYTES.
|
|
29
|
+
*/
|
|
30
|
+
import { type Context, type Span, type SpanContext, type Tracer, SpanKind, SpanStatusCode } from "@opentelemetry/api";
|
|
31
|
+
/**
|
|
32
|
+
* OTel context key carrying per-job attributes. Set once at the top of
|
|
33
|
+
* Pool.handleDispatch via `withJobContext()` and read by helpers that
|
|
34
|
+
* create child spans (interceptors, ctx.* spans) so every emitted span
|
|
35
|
+
* carries `pipeline.run_id` + identifiers — without it the trace-store
|
|
36
|
+
* receiver drops the span as orphan.
|
|
37
|
+
*
|
|
38
|
+
* The OTel `Span` API surface deliberately doesn't expose attributes
|
|
39
|
+
* back to read sites (only set/record/event), so we can't inherit them
|
|
40
|
+
* from a parent span at child-creation time. Carrying them in the
|
|
41
|
+
* Context object is the canonical OTel pattern.
|
|
42
|
+
*/
|
|
43
|
+
export declare const JOB_ATTRS_KEY: symbol;
|
|
44
|
+
export interface JobAttrs {
|
|
45
|
+
runId: string;
|
|
46
|
+
jobId: string;
|
|
47
|
+
uploadId: string;
|
|
48
|
+
urn: string;
|
|
49
|
+
workerId: string;
|
|
50
|
+
}
|
|
51
|
+
export declare function getJobAttrs(): JobAttrs | undefined;
|
|
52
|
+
export declare function withJobContext<T>(attrs: JobAttrs, fn: () => T): T;
|
|
53
|
+
export declare const SPAN_ATTR: {
|
|
54
|
+
readonly JOB_ID: "iq.job_id";
|
|
55
|
+
readonly UPLOAD_ID: "iq.upload_id";
|
|
56
|
+
readonly URN: "iq.content_type_urn";
|
|
57
|
+
readonly WORKER_ID: "iq.worker_id";
|
|
58
|
+
readonly POOL_TOKEN_PREFIX: "iq.pool_token_prefix";
|
|
59
|
+
readonly RUN_ID: "pipeline.run_id";
|
|
60
|
+
readonly RPC_METHOD: "rpc.method";
|
|
61
|
+
readonly RPC_SERVICE: "rpc.service";
|
|
62
|
+
readonly RPC_TARGET: "rpc.target";
|
|
63
|
+
readonly RPC_CODE: "rpc.code";
|
|
64
|
+
readonly RPC_DURATION_MS: "rpc.duration_ms";
|
|
65
|
+
readonly S3_BYTES: "s3.bytes";
|
|
66
|
+
readonly S3_URL_ORIGIN: "s3.url_origin";
|
|
67
|
+
readonly S3_OUTCOME: "s3.outcome";
|
|
68
|
+
};
|
|
69
|
+
export interface TracingInitConfig {
|
|
70
|
+
pipelineEndpoint: string;
|
|
71
|
+
poolToken: string;
|
|
72
|
+
workerId: string;
|
|
73
|
+
sdkVersion: string;
|
|
74
|
+
/**
|
|
75
|
+
* If true (default), the SDK initializes its own TracerProvider and
|
|
76
|
+
* exports spans to pipeline-api. Set false to disable all telemetry
|
|
77
|
+
* — handlers that read `trace.getTracer(...)` will still work via the
|
|
78
|
+
* global no-op provider, but nothing flushes anywhere.
|
|
79
|
+
*/
|
|
80
|
+
enabled?: boolean;
|
|
81
|
+
}
|
|
82
|
+
export interface TracingHandle {
|
|
83
|
+
tracer: Tracer;
|
|
84
|
+
shutdown: () => Promise<void>;
|
|
85
|
+
/**
|
|
86
|
+
* Apply a job envelope's trace_context + wf_run_id to the active context.
|
|
87
|
+
* The returned Context, passed to `tracer.startActiveSpan(..., ctx, fn)`,
|
|
88
|
+
* roots a new span under the dispatching parseUpload activity.
|
|
89
|
+
*/
|
|
90
|
+
contextFromEnvelope: (traceparent: string, runId: string) => Context;
|
|
91
|
+
}
|
|
92
|
+
export declare function initTracing(cfg: TracingInitConfig): TracingHandle;
|
|
93
|
+
export declare function getTracingHandle(): TracingHandle | null;
|
|
94
|
+
export declare function shutdownTracing(): Promise<void>;
|
|
95
|
+
/**
|
|
96
|
+
* Parse a W3C `traceparent` header into an OTel SpanContext.
|
|
97
|
+
*
|
|
98
|
+
* Format: `00-<32 hex trace id>-<16 hex span id>-<2 hex flags>`. Returns
|
|
99
|
+
* null on malformed input. Used by the SDK to root iq.job spans under
|
|
100
|
+
* whatever pipeline activity dispatched the job.
|
|
101
|
+
*/
|
|
102
|
+
export declare function parseTraceparent(value: string): SpanContext | null;
|
|
103
|
+
/**
|
|
104
|
+
* Serialise a SpanContext as a W3C traceparent header value.
|
|
105
|
+
*/
|
|
106
|
+
export declare function spanContextToTraceparent(sc: SpanContext): string;
|
|
107
|
+
/**
|
|
108
|
+
* Sanitize an arbitrary RPC request/response object for inclusion as a
|
|
109
|
+
* span event attribute. Same shape as the pipeline-api's tracing/sanitize
|
|
110
|
+
* (4 KB cap, key-pattern redaction) so the two ends produce consistent
|
|
111
|
+
* payloads in the admin UI.
|
|
112
|
+
*/
|
|
113
|
+
export declare function sanitizeRpcBody(value: unknown): string;
|
|
114
|
+
/**
|
|
115
|
+
* Set common job-scoped attributes on a span. Called by every site that
|
|
116
|
+
* creates a per-job span so RUN_ID / JOB_ID / UPLOAD_ID / URN are
|
|
117
|
+
* consistently present (the trace-store receiver requires
|
|
118
|
+
* `pipeline.run_id` to route the span into the right workflow run).
|
|
119
|
+
*
|
|
120
|
+
* Pulls the attribute values from the active OTel context if not
|
|
121
|
+
* passed explicitly, so RPC / ctx.* spans created mid-handler inherit
|
|
122
|
+
* them without needing to know about the envelope. The Pool root sets
|
|
123
|
+
* the context via `withJobContext` at the start of handleDispatch.
|
|
124
|
+
*/
|
|
125
|
+
export declare function stampJobAttributes(span: Span, attrs?: Partial<JobAttrs>): void;
|
|
126
|
+
export declare function recordSpanError(span: Span, err: unknown): void;
|
|
127
|
+
export { SpanKind, SpanStatusCode };
|
|
128
|
+
//# sourceMappingURL=tracing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tracing.d.ts","sourceRoot":"","sources":["../src/tracing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,EAIN,KAAK,OAAO,EACZ,KAAK,IAAI,EACT,KAAK,WAAW,EAChB,KAAK,MAAM,EAEX,QAAQ,EACR,cAAc,EACd,MAAM,oBAAoB,CAAC;AAc5B;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,aAAa,QAAsC,CAAC;AAEjE,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,WAAW,IAAI,QAAQ,GAAG,SAAS,CAElD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAEjE;AAED,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;CAoBZ,CAAC;AAEX,MAAM,WAAW,iBAAiB;IACjC,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B;;;;OAIG;IACH,mBAAmB,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;CACrE;AAID,wBAAgB,WAAW,CAAC,GAAG,EAAE,iBAAiB,GAAG,aAAa,CAuFjE;AAED,wBAAgB,gBAAgB,IAAI,aAAa,GAAG,IAAI,CAEvD;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAKrD;AAID;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAgBlE;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,EAAE,EAAE,WAAW,GAAG,MAAM,CAGhE;AASD;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAWtD;AAiCD;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CACjC,IAAI,EAAE,IAAI,EACV,KAAK,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,GACvB,IAAI,CAYN;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAO9D;AAED,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC"}
|