@xeonr/upload-pool-sdk 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/job-context.d.ts +2 -1
- package/dist/job-context.d.ts.map +1 -1
- package/dist/job-context.js +33 -1
- package/dist/job-context.js.map +1 -1
- package/dist/logger.d.ts +45 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +132 -0
- package/dist/logger.js.map +1 -0
- package/dist/pool.d.ts +1 -0
- package/dist/pool.d.ts.map +1 -1
- package/dist/pool.js +60 -13
- package/dist/pool.js.map +1 -1
- package/dist/rpc-clients.d.ts +2 -1
- package/dist/rpc-clients.d.ts.map +1 -1
- package/dist/rpc-clients.js +34 -2
- package/dist/rpc-clients.js.map +1 -1
- package/dist/sse-client.d.ts +4 -0
- package/dist/sse-client.d.ts.map +1 -1
- package/dist/sse-client.js +58 -4
- package/dist/sse-client.js.map +1 -1
- package/dist/types.d.ts +14 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +2 -0
- package/src/job-context.ts +34 -0
- package/src/logger.ts +159 -0
- package/src/pool.ts +63 -11
- package/src/rpc-clients.ts +45 -2
- package/src/sse-client.ts +68 -4
- package/src/types.ts +14 -1
package/dist/rpc-clients.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rpc-clients.js","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"rpc-clients.js","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACN,YAAY,EAGZ,YAAY,EACZ,IAAI,GACJ,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,sDAAsD,CAAC;AAQ/F,SAAS,kBAAkB,CAAC,MAAc;IACzC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE;gBAC5B,MAAM;gBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACZ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,GACT,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1D,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE;gBACzB,MAAM;gBACN,IAAI;gBACJ,OAAO;gBACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC,CAAC;YACH,MAAM,GAAG,CAAC;QACX,CAAC;IACF,CAAC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC/B,QAAgB,EAChB,MAAc;IAEd,MAAM,SAAS,GAAG,sBAAsB,CAAC;QACxC,OAAO,EAAE,QAAQ;QACjB,WAAW,EAAE,KAAK;QAClB,YAAY,EAAE,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;KACtE,CAAC,CAAC;IACH,OAAO;QACN,eAAe,EAAE,YAAY,CAAC,sBAAsB,EAAE,SAAS,CAAC;QAChE,gBAAgB,EAAE,YAAY,CAAC,uBAAuB,EAAE,SAAS,CAAC;KAClE,CAAC;AACH,CAAC"}
|
package/dist/sse-client.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import type { Logger } from "./logger.js";
|
|
1
2
|
export interface SseClientConfig {
|
|
2
3
|
endpoint: string;
|
|
3
4
|
token: string;
|
|
4
5
|
workerId: string;
|
|
5
6
|
capabilities: string[];
|
|
7
|
+
logger: Logger;
|
|
6
8
|
onConnected: () => void;
|
|
7
9
|
onDisconnected: (reason: string) => void;
|
|
8
10
|
onJobDispatch: (payload: unknown) => void;
|
|
@@ -12,6 +14,8 @@ export declare class SseClient {
|
|
|
12
14
|
private es;
|
|
13
15
|
private stopped;
|
|
14
16
|
private reconnectDelay;
|
|
17
|
+
private connectAttempt;
|
|
18
|
+
private connectedAt;
|
|
15
19
|
constructor(config: SseClientConfig);
|
|
16
20
|
start(): void;
|
|
17
21
|
stop(): void;
|
package/dist/sse-client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sse-client.d.ts","sourceRoot":"","sources":["../src/sse-client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sse-client.d.ts","sourceRoot":"","sources":["../src/sse-client.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,IAAI,CAAC;IACxB,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC,aAAa,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;CAC1C;AAKD,qBAAa,SAAS;IAOT,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC,OAAO,CAAC,EAAE,CAA4B;IACtC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAwB;IAC9C,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,WAAW,CAAuB;gBAEb,MAAM,EAAE,eAAe;IAEpD,KAAK,IAAI,IAAI;IAKb,IAAI,IAAI,IAAI;IASZ,OAAO,CAAC,OAAO;CA6Ff"}
|
package/dist/sse-client.js
CHANGED
|
@@ -5,6 +5,11 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Reconnect strategy: exponential backoff (1s, 2s, 4s, 8s, capped at 30s).
|
|
7
7
|
* On `job:dispatch` events the SDK invokes the registered onJob callback.
|
|
8
|
+
*
|
|
9
|
+
* All lifecycle transitions emit structured logs via the injected
|
|
10
|
+
* `Logger` so connection failures are visible in the worker's k8s logs
|
|
11
|
+
* (otherwise a worker that can't connect at all looks identical to one
|
|
12
|
+
* that's idle).
|
|
8
13
|
*/
|
|
9
14
|
import { EventSource } from "eventsource";
|
|
10
15
|
const RECONNECT_INITIAL_MS = 1_000;
|
|
@@ -14,6 +19,8 @@ export class SseClient {
|
|
|
14
19
|
es = null;
|
|
15
20
|
stopped = false;
|
|
16
21
|
reconnectDelay = RECONNECT_INITIAL_MS;
|
|
22
|
+
connectAttempt = 0;
|
|
23
|
+
connectedAt = null;
|
|
17
24
|
constructor(config) {
|
|
18
25
|
this.config = config;
|
|
19
26
|
}
|
|
@@ -27,35 +34,82 @@ export class SseClient {
|
|
|
27
34
|
this.es.close();
|
|
28
35
|
this.es = null;
|
|
29
36
|
}
|
|
37
|
+
this.config.logger.info("sse.stopped");
|
|
30
38
|
}
|
|
31
39
|
connect() {
|
|
32
40
|
if (this.stopped)
|
|
33
41
|
return;
|
|
42
|
+
this.connectAttempt++;
|
|
34
43
|
const url = new URL(`${this.config.endpoint}/queue/connect`);
|
|
35
44
|
url.searchParams.set("queueToken", this.config.token);
|
|
36
45
|
url.searchParams.set("workerId", this.config.workerId);
|
|
37
46
|
if (this.config.capabilities.length > 0) {
|
|
38
47
|
url.searchParams.set("capabilities", this.config.capabilities.join(","));
|
|
39
48
|
}
|
|
49
|
+
// Strip the token from the logged URL — the queueToken param is the
|
|
50
|
+
// pool secret, no reason to bake it into telemetry.
|
|
51
|
+
const loggedUrl = new URL(url.toString());
|
|
52
|
+
loggedUrl.searchParams.set("queueToken", "[redacted]");
|
|
53
|
+
this.config.logger.info("sse.connect.start", {
|
|
54
|
+
url: loggedUrl.toString(),
|
|
55
|
+
attempt: this.connectAttempt,
|
|
56
|
+
capabilities: this.config.capabilities,
|
|
57
|
+
});
|
|
40
58
|
this.es = new EventSource(url.toString());
|
|
59
|
+
this.es.addEventListener("open", () => {
|
|
60
|
+
this.config.logger.debug("sse.open");
|
|
61
|
+
});
|
|
41
62
|
this.es.addEventListener("connected", () => {
|
|
42
63
|
this.reconnectDelay = RECONNECT_INITIAL_MS;
|
|
64
|
+
this.connectedAt = Date.now();
|
|
65
|
+
this.config.logger.info("sse.connect.opened", {
|
|
66
|
+
attempt: this.connectAttempt,
|
|
67
|
+
});
|
|
43
68
|
this.config.onConnected();
|
|
44
69
|
});
|
|
45
70
|
this.es.addEventListener("job:dispatch", (event) => {
|
|
46
71
|
try {
|
|
47
72
|
const data = JSON.parse(event.data);
|
|
73
|
+
this.config.logger.debug("sse.event.job_dispatch", {
|
|
74
|
+
jobId: data.jobId,
|
|
75
|
+
});
|
|
48
76
|
this.config.onJobDispatch(data);
|
|
49
77
|
}
|
|
50
78
|
catch (err) {
|
|
51
|
-
|
|
79
|
+
const reason = `bad job payload: ${err.message}`;
|
|
80
|
+
this.config.logger.error("sse.event.parse_error", {
|
|
81
|
+
err,
|
|
82
|
+
rawData: event.data,
|
|
83
|
+
});
|
|
84
|
+
this.config.onDisconnected(reason);
|
|
52
85
|
}
|
|
53
86
|
});
|
|
54
|
-
// heartbeat events are no-ops
|
|
55
|
-
|
|
87
|
+
// heartbeat events are no-ops at the dispatch layer — receipt alone
|
|
88
|
+
// keeps the connection alive. We log at debug so noisy keep-alive
|
|
89
|
+
// traffic doesn't drown the info stream.
|
|
90
|
+
this.es.addEventListener("heartbeat", () => {
|
|
91
|
+
this.config.logger.debug("sse.event.heartbeat");
|
|
92
|
+
});
|
|
93
|
+
this.es.onerror = (err) => {
|
|
56
94
|
if (this.stopped)
|
|
57
95
|
return;
|
|
58
|
-
|
|
96
|
+
const errMessage = err?.message ??
|
|
97
|
+
err?.type ??
|
|
98
|
+
"unknown";
|
|
99
|
+
const status = err?.status ??
|
|
100
|
+
err?.code;
|
|
101
|
+
const wasConnected = this.connectedAt !== null;
|
|
102
|
+
const uptimeMs = wasConnected ? Date.now() - this.connectedAt : null;
|
|
103
|
+
this.config.logger.warn("sse.disconnected", {
|
|
104
|
+
message: errMessage,
|
|
105
|
+
status,
|
|
106
|
+
wasConnected,
|
|
107
|
+
uptimeMs,
|
|
108
|
+
attempt: this.connectAttempt,
|
|
109
|
+
nextReconnectMs: this.reconnectDelay,
|
|
110
|
+
});
|
|
111
|
+
this.config.onDisconnected(`sse error: ${errMessage}`);
|
|
112
|
+
this.connectedAt = null;
|
|
59
113
|
if (this.es) {
|
|
60
114
|
this.es.close();
|
|
61
115
|
this.es = null;
|
package/dist/sse-client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sse-client.js","sourceRoot":"","sources":["../src/sse-client.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"sse-client.js","sourceRoot":"","sources":["../src/sse-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAc1C,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,OAAO,SAAS;IAOQ;IANrB,EAAE,GAAuB,IAAI,CAAC;IAC9B,OAAO,GAAG,KAAK,CAAC;IAChB,cAAc,GAAG,oBAAoB,CAAC;IACtC,cAAc,GAAG,CAAC,CAAC;IACnB,WAAW,GAAkB,IAAI,CAAC;IAE1C,YAA6B,MAAuB;QAAvB,WAAM,GAAN,MAAM,CAAiB;IAAG,CAAC;IAExD,KAAK;QACJ,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,OAAO,EAAE,CAAC;IAChB,CAAC;IAED,IAAI;QACH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACxC,CAAC;IAEO,OAAO;QACd,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAEzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,gBAAgB,CAAC,CAAC;QAC7D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,CAAC;QAED,oEAAoE;QACpE,oDAAoD;QACpD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1C,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC5C,GAAG,EAAE,SAAS,CAAC,QAAQ,EAAE;YACzB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1C,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE;YACrC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,WAAW,EAAE,GAAG,EAAE;YAC1C,IAAI,CAAC,cAAc,GAAG,oBAAoB,CAAC;YAC3C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;gBAC7C,OAAO,EAAE,IAAI,CAAC,cAAc;aAC5B,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;YAClD,IAAI,CAAC;gBACJ,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAE,KAAsB,CAAC,IAAI,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;oBAClD,KAAK,EAAG,IAA2B,CAAC,KAAK;iBACzC,CAAC,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,oBAAqB,GAAa,CAAC,OAAO,EAAE,CAAC;gBAC5D,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE;oBACjD,GAAG;oBACH,OAAO,EAAG,KAAsB,CAAC,IAAI;iBACrC,CAAC,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACpC,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,oEAAoE;QACpE,kEAAkE;QAClE,yCAAyC;QACzC,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,WAAW,EAAE,GAAG,EAAE;YAC1C,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,GAAG,CAAC,GAAY,EAAE,EAAE;YAClC,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YACzB,MAAM,UAAU,GACd,GAA4D,EAAE,OAAO;gBACrE,GAAyB,EAAE,IAAI;gBAChC,SAAS,CAAC;YACX,MAAM,MAAM,GAAI,GAA0C,EAAE,MAAM;gBAChE,GAAyB,EAAE,IAAI,CAAC;YAClC,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC;YAC/C,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAY,CAAC,CAAC,CAAC,IAAI,CAAC;YAEtE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC3C,OAAO,EAAE,UAAU;gBACnB,MAAM;gBACN,YAAY;gBACZ,QAAQ;gBACR,OAAO,EAAE,IAAI,CAAC,cAAc;gBAC5B,eAAe,EAAE,IAAI,CAAC,cAAc;aACpC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,cAAc,UAAU,EAAE,CAAC,CAAC;YACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;gBACb,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YAChB,CAAC;YAED,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YACtD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAC3E,CAAC,CAAC;IACH,CAAC;CACD"}
|
package/dist/types.d.ts
CHANGED
|
@@ -24,8 +24,21 @@ export interface PoolConfig {
|
|
|
24
24
|
handlers: Record<string, JobHandler>;
|
|
25
25
|
/** Fallback handler if URN has no specific entry in `handlers`. Optional. */
|
|
26
26
|
onUnhandled?: JobHandler;
|
|
27
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* Lifecycle / error hook. Receives every handler exception alongside
|
|
29
|
+
* the job context. The SDK also logs every failure via `logger` —
|
|
30
|
+
* `onError` is for hooks that need to fan out to external systems
|
|
31
|
+
* (sentry, pagerduty). Optional.
|
|
32
|
+
*/
|
|
28
33
|
onError?: (err: Error, ctx?: JobContext) => void;
|
|
34
|
+
/**
|
|
35
|
+
* Structured logger. Defaults to a JSON-line logger that writes to
|
|
36
|
+
* stdout (`info`/`debug`) and stderr (`warn`/`error`), honoring
|
|
37
|
+
* `UPL_LOG_LEVEL`. Pass `noopLogger` from this package for silence,
|
|
38
|
+
* or supply your own implementing the `Logger` interface to bridge
|
|
39
|
+
* to pino/winston.
|
|
40
|
+
*/
|
|
41
|
+
logger?: import("./logger.js").Logger;
|
|
29
42
|
/** Max concurrent in-flight jobs. Defaults to 1. */
|
|
30
43
|
concurrency?: number;
|
|
31
44
|
}
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,UAAU;IAC1B,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrC,6EAA6E;IAC7E,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,UAAU;IAC1B,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrC,6EAA6E;IAC7E,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,UAAU,KAAK,IAAI,CAAC;IACjD;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,aAAa,EAAE,MAAM,CAAC;IACtC,oDAAoD;IACpD,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,EAAE,EAAE,MAAM,CAAC;QACX,8DAA8D;QAC9D,MAAM,EAAE,UAAU,CAAC;KACnB,CAAC;IACF,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,QAAQ,IAAI,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IAChD,0EAA0E;IAC1E,cAAc,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,mCAAmC;IACnC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5C,gFAAgF;IAChF,UAAU,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD;;;;;;OAMG;IACH,WAAW,CAAC,QAAQ,EAAE,qBAAqB,GAAG,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,mEAAmE;IACnE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,0EAA0E;IAC1E,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD;;;;;;;;;;;;;;;;OAgBG;IACH,oBAAoB,CAAC,CAAC,EACrB,IAAI,EAAE,sBAAsB,EAAE,EAC9B,QAAQ,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,GACnD,OAAO,CAAC,CAAC,CAAC,CAAC;CACd;AAED,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAChC,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,2GAA2G;IAC3G,WAAW,EAAE,MAAM,CAAC;IACpB,6FAA6F;IAC7F,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,iBAAiB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,cAAc,GAAG,cAAc,GAAG,YAAY,GAAG,iBAAiB,CAAC;IACnI,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAE9E;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,CAAC,EAAE;QACP,UAAU,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,GAAG,CAAC,EAAE;QACL,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,mBAAmB;IACnC,SAAS,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,YAAY,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACnD,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAA;KAAE,CAAC;CACxD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xeonr/upload-pool-sdk",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Self-hosted worker SDK for the upl.im content type pipeline. Implement a handler per content type URN; the SDK handles SSE connection, job acceptance, presigned thumbnail upload, and metadata callbacks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
package/src/index.ts
CHANGED
|
@@ -23,6 +23,8 @@ import type { PoolConfig } from "./types.js";
|
|
|
23
23
|
|
|
24
24
|
export { Pool } from "./pool.js";
|
|
25
25
|
export { NonRetryableError } from "./errors.js";
|
|
26
|
+
export { JsonLogger, noopLogger } from "./logger.js";
|
|
27
|
+
export type { Logger, LogLevel } from "./logger.js";
|
|
26
28
|
export type {
|
|
27
29
|
PoolConfig,
|
|
28
30
|
JobContext,
|
package/src/job-context.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
} from "./protocol/uplim/api/v1/uploads_pb.js";
|
|
18
18
|
import { UploadMetadataSchema } from "./protocol/uplim/api/v1/metadata_pb.js";
|
|
19
19
|
import type { RpcClients } from "./rpc-clients.js";
|
|
20
|
+
import type { Logger } from "./logger.js";
|
|
20
21
|
import type {
|
|
21
22
|
FolderContext,
|
|
22
23
|
JobContext,
|
|
@@ -54,6 +55,7 @@ export interface JobEnvelope {
|
|
|
54
55
|
export function createJobContext(
|
|
55
56
|
envelope: JobEnvelope,
|
|
56
57
|
rpc: RpcClients,
|
|
58
|
+
logger: Logger,
|
|
57
59
|
): JobContext {
|
|
58
60
|
const ctx: JobContext = {
|
|
59
61
|
jobId: envelope.jobId,
|
|
@@ -71,28 +73,45 @@ export function createJobContext(
|
|
|
71
73
|
sourceUrl: envelope.sourceUrl,
|
|
72
74
|
|
|
73
75
|
async download() {
|
|
76
|
+
logger.debug("source.download.start");
|
|
74
77
|
const resp = await fetch(envelope.sourceUrl);
|
|
75
78
|
if (!resp.ok || !resp.body) {
|
|
79
|
+
logger.error("source.download.failed", {
|
|
80
|
+
status: resp.status,
|
|
81
|
+
statusText: resp.statusText,
|
|
82
|
+
});
|
|
76
83
|
throw new Error(`source download failed: ${resp.status} ${resp.statusText}`);
|
|
77
84
|
}
|
|
78
85
|
return resp.body;
|
|
79
86
|
},
|
|
80
87
|
|
|
81
88
|
async downloadBuffer() {
|
|
89
|
+
logger.debug("source.download.start", { mode: "buffer" });
|
|
82
90
|
const resp = await fetch(envelope.sourceUrl);
|
|
83
91
|
if (!resp.ok) {
|
|
92
|
+
logger.error("source.download.failed", {
|
|
93
|
+
status: resp.status,
|
|
94
|
+
statusText: resp.statusText,
|
|
95
|
+
});
|
|
84
96
|
throw new Error(`source download failed: ${resp.status} ${resp.statusText}`);
|
|
85
97
|
}
|
|
86
98
|
const buf = await resp.arrayBuffer();
|
|
99
|
+
logger.debug("source.download.complete", { sizeBytes: buf.byteLength });
|
|
87
100
|
return new Uint8Array(buf);
|
|
88
101
|
},
|
|
89
102
|
|
|
90
103
|
async downloadToFile(path: string) {
|
|
91
104
|
const data = await ctx.downloadBuffer();
|
|
92
105
|
await writeFile(path, data);
|
|
106
|
+
logger.debug("source.download.to_file", { path, sizeBytes: data.byteLength });
|
|
93
107
|
},
|
|
94
108
|
|
|
95
109
|
async uploadMeta(opts: UploadMetaOpts) {
|
|
110
|
+
logger.debug("meta.upload.start", {
|
|
111
|
+
type: opts.type,
|
|
112
|
+
instance: opts.instance,
|
|
113
|
+
filename: opts.filename,
|
|
114
|
+
});
|
|
96
115
|
const protoType = META_TYPE_MAP[opts.type];
|
|
97
116
|
const metaType = create(MetaTypeSchema, {
|
|
98
117
|
type: protoType,
|
|
@@ -135,6 +154,10 @@ export function createJobContext(
|
|
|
135
154
|
metadataId: [meta.metadataId],
|
|
136
155
|
}),
|
|
137
156
|
);
|
|
157
|
+
logger.info("meta.upload.complete", {
|
|
158
|
+
type: opts.type,
|
|
159
|
+
metadataId: meta.metadataId,
|
|
160
|
+
});
|
|
138
161
|
},
|
|
139
162
|
|
|
140
163
|
async setMetadata(metadata: UploadMetadataPartial | UploadMetadataProto) {
|
|
@@ -163,6 +186,7 @@ export function createJobContext(
|
|
|
163
186
|
thumbnailGenerationVersion: thumbnailVersion,
|
|
164
187
|
}),
|
|
165
188
|
);
|
|
189
|
+
logger.debug("metadata.set", { thumbnailVersion });
|
|
166
190
|
},
|
|
167
191
|
|
|
168
192
|
async markHasThumbnail(version: number) {
|
|
@@ -173,6 +197,7 @@ export function createJobContext(
|
|
|
173
197
|
thumbnailGenerationVersion: version,
|
|
174
198
|
}),
|
|
175
199
|
);
|
|
200
|
+
logger.info("thumbnail.marked", { version });
|
|
176
201
|
},
|
|
177
202
|
|
|
178
203
|
async setDescription(text: string, tags?: string[]) {
|
|
@@ -185,12 +210,20 @@ export function createJobContext(
|
|
|
185
210
|
}),
|
|
186
211
|
}),
|
|
187
212
|
);
|
|
213
|
+
logger.info("description.set", {
|
|
214
|
+
length: text.length,
|
|
215
|
+
tags: tags?.length ?? 0,
|
|
216
|
+
});
|
|
188
217
|
},
|
|
189
218
|
|
|
190
219
|
async withPresignedUploads<T>(
|
|
191
220
|
opts: PresignedUploadRequest[],
|
|
192
221
|
callback: (handles: import("./types.js").MetaUploadHandle[]) => Promise<T>,
|
|
193
222
|
): Promise<T> {
|
|
223
|
+
logger.debug("presigned.upload.request", {
|
|
224
|
+
count: opts.length,
|
|
225
|
+
types: opts.map((o) => o.type),
|
|
226
|
+
});
|
|
194
227
|
const metaTypes = opts.map((opt) => create(MetaTypeSchema, {
|
|
195
228
|
type: META_TYPE_MAP[opt.type],
|
|
196
229
|
n: opt.instance,
|
|
@@ -222,6 +255,7 @@ export function createJobContext(
|
|
|
222
255
|
metadataId: requested.metaUploads.map((m) => m.metadataId),
|
|
223
256
|
}),
|
|
224
257
|
);
|
|
258
|
+
logger.info("presigned.upload.complete", { count: handles.length });
|
|
225
259
|
return result;
|
|
226
260
|
},
|
|
227
261
|
};
|
package/src/logger.ts
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logging interface for the SDK. Customers can plug in their own
|
|
3
|
+
* logger (pino, winston, console) by passing `logger` to createPool;
|
|
4
|
+
* by default we emit structured JSON lines to stdout/stderr so logs
|
|
5
|
+
* are grep-friendly in k8s.
|
|
6
|
+
*
|
|
7
|
+
* The shape mirrors the pipeline-api's logger contract for consistency
|
|
8
|
+
* across the upl.im stack.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export type LogLevel = "debug" | "info" | "warn" | "error";
|
|
12
|
+
|
|
13
|
+
export interface Logger {
|
|
14
|
+
debug(event: string, context?: Record<string, unknown>): void;
|
|
15
|
+
info(event: string, context?: Record<string, unknown>): void;
|
|
16
|
+
warn(event: string, context?: Record<string, unknown>): void;
|
|
17
|
+
error(event: string, context?: Record<string, unknown>): void;
|
|
18
|
+
/**
|
|
19
|
+
* Return a derived logger that merges these context fields into every
|
|
20
|
+
* subsequent log call. Used by Pool.handleDispatch to attach jobId /
|
|
21
|
+
* uploadId / urn to every line emitted during the job.
|
|
22
|
+
*/
|
|
23
|
+
child(context: Record<string, unknown>): Logger;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const SENSITIVE_KEYS = new Set([
|
|
27
|
+
"token",
|
|
28
|
+
"queueToken",
|
|
29
|
+
"queue_token",
|
|
30
|
+
"workerToken",
|
|
31
|
+
"worker_token",
|
|
32
|
+
"updateToken",
|
|
33
|
+
"update_token",
|
|
34
|
+
"authorization",
|
|
35
|
+
"Authorization",
|
|
36
|
+
]);
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Redact obvious secrets so a careless `logger.info("rpc.request", { headers })`
|
|
40
|
+
* doesn't leak the pool token to log aggregators. Best-effort; not a
|
|
41
|
+
* substitute for thinking about what you're logging.
|
|
42
|
+
*/
|
|
43
|
+
function redact(value: unknown): unknown {
|
|
44
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
45
|
+
const out: Record<string, unknown> = {};
|
|
46
|
+
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
47
|
+
if (SENSITIVE_KEYS.has(k)) {
|
|
48
|
+
out[k] = typeof v === "string" && v.length > 8
|
|
49
|
+
? `${v.slice(0, 4)}…(${v.length - 4} redacted)`
|
|
50
|
+
: "[redacted]";
|
|
51
|
+
} else if (v && typeof v === "object") {
|
|
52
|
+
out[k] = redact(v);
|
|
53
|
+
} else {
|
|
54
|
+
out[k] = v;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
if (Array.isArray(value)) return value.map(redact);
|
|
60
|
+
return value;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function serializeError(err: unknown): Record<string, unknown> {
|
|
64
|
+
if (err instanceof Error) {
|
|
65
|
+
return {
|
|
66
|
+
name: err.name,
|
|
67
|
+
message: err.message,
|
|
68
|
+
stack: err.stack,
|
|
69
|
+
...(err as unknown as Record<string, unknown>),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
return { value: String(err) };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Default JSON logger. Emits one line per call. `error` and `warn` go to
|
|
77
|
+
* stderr; `info` and `debug` go to stdout. Honors `UPL_LOG_LEVEL` env var
|
|
78
|
+
* (`debug` | `info` | `warn` | `error`) to filter out noise in prod.
|
|
79
|
+
*/
|
|
80
|
+
export class JsonLogger implements Logger {
|
|
81
|
+
private readonly base: Record<string, unknown>;
|
|
82
|
+
private readonly minLevel: LogLevel;
|
|
83
|
+
|
|
84
|
+
constructor(base: Record<string, unknown> = {}, minLevel?: LogLevel) {
|
|
85
|
+
this.base = base;
|
|
86
|
+
this.minLevel = minLevel ?? (process.env.UPL_LOG_LEVEL as LogLevel) ?? "info";
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
child(context: Record<string, unknown>): Logger {
|
|
90
|
+
return new JsonLogger({ ...this.base, ...context }, this.minLevel);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
debug(event: string, context?: Record<string, unknown>): void {
|
|
94
|
+
this.emit("debug", event, context);
|
|
95
|
+
}
|
|
96
|
+
info(event: string, context?: Record<string, unknown>): void {
|
|
97
|
+
this.emit("info", event, context);
|
|
98
|
+
}
|
|
99
|
+
warn(event: string, context?: Record<string, unknown>): void {
|
|
100
|
+
this.emit("warn", event, context);
|
|
101
|
+
}
|
|
102
|
+
error(event: string, context?: Record<string, unknown>): void {
|
|
103
|
+
this.emit("error", event, context);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
private shouldEmit(level: LogLevel): boolean {
|
|
107
|
+
const order: Record<LogLevel, number> = {
|
|
108
|
+
debug: 10,
|
|
109
|
+
info: 20,
|
|
110
|
+
warn: 30,
|
|
111
|
+
error: 40,
|
|
112
|
+
};
|
|
113
|
+
return order[level] >= order[this.minLevel];
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private emit(
|
|
117
|
+
level: LogLevel,
|
|
118
|
+
event: string,
|
|
119
|
+
context?: Record<string, unknown>,
|
|
120
|
+
): void {
|
|
121
|
+
if (!this.shouldEmit(level)) return;
|
|
122
|
+
|
|
123
|
+
const merged: Record<string, unknown> = {
|
|
124
|
+
...this.base,
|
|
125
|
+
...(context ? (redact(context) as Record<string, unknown>) : {}),
|
|
126
|
+
};
|
|
127
|
+
if (merged.err !== undefined) {
|
|
128
|
+
merged.err = serializeError(merged.err);
|
|
129
|
+
}
|
|
130
|
+
if (merged.error !== undefined && merged.error instanceof Error) {
|
|
131
|
+
merged.error = serializeError(merged.error);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const line = JSON.stringify({
|
|
135
|
+
ts: new Date().toISOString(),
|
|
136
|
+
level,
|
|
137
|
+
logger: "@xeonr/upload-pool-sdk",
|
|
138
|
+
event,
|
|
139
|
+
...merged,
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
// eslint-disable-next-line no-console
|
|
143
|
+
(level === "error" || level === "warn" ? console.error : console.log)(line);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* No-op logger for tests or callers who want silence. createPool ignores
|
|
149
|
+
* this if `logger` is unset and uses JsonLogger by default.
|
|
150
|
+
*/
|
|
151
|
+
export const noopLogger: Logger = {
|
|
152
|
+
debug() {},
|
|
153
|
+
info() {},
|
|
154
|
+
warn() {},
|
|
155
|
+
error() {},
|
|
156
|
+
child() {
|
|
157
|
+
return noopLogger;
|
|
158
|
+
},
|
|
159
|
+
};
|
package/src/pool.ts
CHANGED
|
@@ -3,6 +3,11 @@
|
|
|
3
3
|
* to the pool's job stream, dispatches job:dispatch events to the right
|
|
4
4
|
* handler keyed by content type URN, and orchestrates accept/complete/
|
|
5
5
|
* report-error against IntegrationQueueService.
|
|
6
|
+
*
|
|
7
|
+
* Every transition emits a structured log line via the injected Logger
|
|
8
|
+
* so a worker's k8s logs surface SSE connection state, dispatched jobs,
|
|
9
|
+
* handler outcomes, and RPC errors without callers needing to wire up
|
|
10
|
+
* their own onError hook.
|
|
6
11
|
*/
|
|
7
12
|
import { hostname } from "node:os";
|
|
8
13
|
import { randomBytes } from "node:crypto";
|
|
@@ -16,6 +21,7 @@ import { createRpcClients, type RpcClients } from "./rpc-clients.js";
|
|
|
16
21
|
import { SseClient } from "./sse-client.js";
|
|
17
22
|
import { createJobContext, type JobEnvelope } from "./job-context.js";
|
|
18
23
|
import { NonRetryableError } from "./errors.js";
|
|
24
|
+
import { JsonLogger, type Logger } from "./logger.js";
|
|
19
25
|
import type {
|
|
20
26
|
JobContext,
|
|
21
27
|
JobHandler,
|
|
@@ -26,6 +32,7 @@ export class Pool {
|
|
|
26
32
|
private readonly config: PoolConfig;
|
|
27
33
|
private readonly rpc: RpcClients;
|
|
28
34
|
private readonly sse: SseClient;
|
|
35
|
+
private readonly logger: Logger;
|
|
29
36
|
private inFlight = 0;
|
|
30
37
|
private readonly workerId: string;
|
|
31
38
|
private readonly capabilities: string[];
|
|
@@ -33,46 +40,80 @@ export class Pool {
|
|
|
33
40
|
constructor(config: PoolConfig) {
|
|
34
41
|
this.config = {
|
|
35
42
|
concurrency: 1,
|
|
36
|
-
onError: defaultErrorHandler,
|
|
37
43
|
...config,
|
|
38
44
|
};
|
|
39
45
|
this.workerId = config.workerId ?? `${hostname()}-${randomBytes(4).toString("hex")}`;
|
|
40
46
|
this.capabilities = Object.keys(config.handlers);
|
|
41
|
-
this.
|
|
47
|
+
this.logger = (config.logger ?? new JsonLogger()).child({
|
|
48
|
+
workerId: this.workerId,
|
|
49
|
+
});
|
|
50
|
+
this.rpc = createRpcClients(config.endpoint, this.logger);
|
|
42
51
|
this.sse = new SseClient({
|
|
43
52
|
endpoint: config.endpoint,
|
|
44
53
|
token: config.token,
|
|
45
54
|
workerId: this.workerId,
|
|
46
55
|
capabilities: this.capabilities,
|
|
56
|
+
logger: this.logger.child({ component: "sse" }),
|
|
47
57
|
onConnected: () => {
|
|
48
|
-
/*
|
|
58
|
+
/* logged inside sse-client; callers can hook lifecycle here later */
|
|
49
59
|
},
|
|
50
60
|
onDisconnected: () => {
|
|
51
|
-
/*
|
|
61
|
+
/* logged inside sse-client */
|
|
52
62
|
},
|
|
53
63
|
onJobDispatch: (payload) => this.handleDispatch(payload as JobEnvelope),
|
|
54
64
|
});
|
|
65
|
+
|
|
66
|
+
this.logger.info("sdk.boot", {
|
|
67
|
+
endpoint: config.endpoint,
|
|
68
|
+
capabilities: this.capabilities,
|
|
69
|
+
concurrency: this.config.concurrency,
|
|
70
|
+
version: SDK_VERSION,
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
if (this.capabilities.length === 0) {
|
|
74
|
+
this.logger.warn("sdk.boot.no_handlers", {
|
|
75
|
+
note: "no handlers registered — worker will not receive any jobs",
|
|
76
|
+
});
|
|
77
|
+
}
|
|
55
78
|
}
|
|
56
79
|
|
|
57
80
|
async start(): Promise<void> {
|
|
81
|
+
this.logger.info("sdk.start");
|
|
58
82
|
this.sse.start();
|
|
59
83
|
}
|
|
60
84
|
|
|
61
85
|
async stop(): Promise<void> {
|
|
86
|
+
this.logger.info("sdk.stop", { inFlight: this.inFlight });
|
|
62
87
|
this.sse.stop();
|
|
63
88
|
}
|
|
64
89
|
|
|
65
90
|
private async handleDispatch(envelope: JobEnvelope): Promise<void> {
|
|
91
|
+
const jobLogger = this.logger.child({
|
|
92
|
+
jobId: envelope.jobId,
|
|
93
|
+
uploadId: envelope.uploadId,
|
|
94
|
+
urn: envelope.contentTypeContext.urn,
|
|
95
|
+
});
|
|
96
|
+
|
|
66
97
|
if (this.inFlight >= (this.config.concurrency ?? 1)) {
|
|
67
98
|
// Pipeline only dispatches to idle workers (zero in-flight), so
|
|
68
99
|
// this branch is defensive — if it ever fires we silently
|
|
69
100
|
// drop the dispatch and let the pipeline timeout requeue.
|
|
101
|
+
jobLogger.warn("job.dispatched.dropped_at_capacity", {
|
|
102
|
+
inFlight: this.inFlight,
|
|
103
|
+
concurrency: this.config.concurrency,
|
|
104
|
+
});
|
|
70
105
|
return;
|
|
71
106
|
}
|
|
72
107
|
this.inFlight++;
|
|
108
|
+
jobLogger.info("job.dispatched", {
|
|
109
|
+
filename: envelope.filename,
|
|
110
|
+
mimeType: envelope.mimeType,
|
|
111
|
+
inFlight: this.inFlight,
|
|
112
|
+
});
|
|
73
113
|
|
|
74
114
|
const handler = this.resolveHandler(envelope.contentTypeContext.urn);
|
|
75
|
-
const ctx = createJobContext(envelope, this.rpc);
|
|
115
|
+
const ctx = createJobContext(envelope, this.rpc, jobLogger);
|
|
116
|
+
const startedAt = Date.now();
|
|
76
117
|
|
|
77
118
|
try {
|
|
78
119
|
// AcceptJob — clears the pipeline's accept-timeout.
|
|
@@ -83,11 +124,16 @@ export class Pool {
|
|
|
83
124
|
queueToken: this.config.token,
|
|
84
125
|
}),
|
|
85
126
|
);
|
|
127
|
+
jobLogger.info("job.accepted");
|
|
86
128
|
|
|
87
129
|
if (!handler) {
|
|
130
|
+
jobLogger.warn("job.unhandled", {
|
|
131
|
+
availableHandlers: this.capabilities,
|
|
132
|
+
});
|
|
88
133
|
await this.reportError(
|
|
89
134
|
envelope.jobId,
|
|
90
135
|
ctx,
|
|
136
|
+
jobLogger,
|
|
91
137
|
new NonRetryableError(`no handler for URN ${envelope.contentTypeContext.urn}`),
|
|
92
138
|
);
|
|
93
139
|
return;
|
|
@@ -102,8 +148,15 @@ export class Pool {
|
|
|
102
148
|
queueToken: this.config.token,
|
|
103
149
|
}),
|
|
104
150
|
);
|
|
151
|
+
jobLogger.info("job.completed", {
|
|
152
|
+
durationMs: Date.now() - startedAt,
|
|
153
|
+
});
|
|
105
154
|
} catch (err) {
|
|
106
|
-
|
|
155
|
+
jobLogger.error("job.failed", {
|
|
156
|
+
err,
|
|
157
|
+
durationMs: Date.now() - startedAt,
|
|
158
|
+
});
|
|
159
|
+
await this.reportError(envelope.jobId, ctx, jobLogger, err as Error);
|
|
107
160
|
} finally {
|
|
108
161
|
this.inFlight--;
|
|
109
162
|
}
|
|
@@ -116,6 +169,7 @@ export class Pool {
|
|
|
116
169
|
private async reportError(
|
|
117
170
|
jobId: string,
|
|
118
171
|
ctx: JobContext,
|
|
172
|
+
jobLogger: Logger,
|
|
119
173
|
err: Error,
|
|
120
174
|
): Promise<void> {
|
|
121
175
|
const retry = !(err instanceof NonRetryableError);
|
|
@@ -130,14 +184,12 @@ export class Pool {
|
|
|
130
184
|
retry,
|
|
131
185
|
}),
|
|
132
186
|
);
|
|
187
|
+
jobLogger.info("job.error_reported", { retry });
|
|
133
188
|
} catch (rptErr) {
|
|
189
|
+
jobLogger.error("job.error_report_failed", { err: rptErr });
|
|
134
190
|
this.config.onError?.(rptErr as Error, ctx);
|
|
135
191
|
}
|
|
136
192
|
}
|
|
137
193
|
}
|
|
138
194
|
|
|
139
|
-
|
|
140
|
-
const where = ctx ? `job ${ctx.jobId} (${ctx.contentType.urn})` : "pool";
|
|
141
|
-
// eslint-disable-next-line no-console
|
|
142
|
-
console.error(`[@xeonr/upload-pool-sdk] ${where}: ${err.message}`);
|
|
143
|
-
}
|
|
195
|
+
const SDK_VERSION = "1.1.0";
|