@xeonr/upload-pool-sdk 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/pool.d.ts +22 -0
- package/dist/pool.d.ts.map +1 -1
- package/dist/pool.js +53 -4
- package/dist/pool.js.map +1 -1
- package/dist/rpc-clients.d.ts +6 -1
- package/dist/rpc-clients.d.ts.map +1 -1
- package/dist/rpc-clients.js +28 -14
- package/dist/rpc-clients.js.map +1 -1
- package/dist/types.d.ts +20 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +2 -1
- package/src/pool.ts +53 -4
- package/src/rpc-clients.ts +35 -17
- package/src/types.ts +20 -2
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
*
|
|
9
9
|
* await createPool({
|
|
10
10
|
* token: process.env.UPL_POOL_TOKEN!,
|
|
11
|
-
*
|
|
11
|
+
* apiEndpoint: process.env.UPL_API_URL!,
|
|
12
|
+
* pipelineEndpoint: process.env.UPL_PIPELINE_API_URL!,
|
|
12
13
|
* handlers: {
|
|
13
14
|
* "my-integration:invoice": async (ctx) => {
|
|
14
15
|
* const buf = await ctx.downloadBuffer();
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACpD,YAAY,EACX,UAAU,EACV,UAAU,EACV,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,EACb,mBAAmB,EACnB,sBAAsB,EACtB,gBAAgB,GAChB,MAAM,YAAY,CAAC;AAEpB,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAEnD"}
|
package/dist/index.js
CHANGED
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
*
|
|
9
9
|
* await createPool({
|
|
10
10
|
* token: process.env.UPL_POOL_TOKEN!,
|
|
11
|
-
*
|
|
11
|
+
* apiEndpoint: process.env.UPL_API_URL!,
|
|
12
|
+
* pipelineEndpoint: process.env.UPL_PIPELINE_API_URL!,
|
|
12
13
|
* handlers: {
|
|
13
14
|
* "my-integration:invoice": async (ctx) => {
|
|
14
15
|
* const buf = await ctx.downloadBuffer();
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAerD,MAAM,UAAU,UAAU,CAAC,MAAkB;IAC5C,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;AACzB,CAAC"}
|
package/dist/pool.d.ts
CHANGED
|
@@ -7,7 +7,29 @@ export declare class Pool {
|
|
|
7
7
|
private inFlight;
|
|
8
8
|
private readonly workerId;
|
|
9
9
|
private readonly capabilities;
|
|
10
|
+
private runningPromise;
|
|
11
|
+
private runningResolve;
|
|
12
|
+
private keepAliveInterval;
|
|
10
13
|
constructor(config: PoolConfig);
|
|
14
|
+
/**
|
|
15
|
+
* Start the pool and keep the worker process alive until `stop()` is
|
|
16
|
+
* called.
|
|
17
|
+
*
|
|
18
|
+
* We deliberately return a never-resolving promise (resolved only by
|
|
19
|
+
* stop()) for two reasons:
|
|
20
|
+
*
|
|
21
|
+
* 1. ESM top-level `await pool.start()` is the documented worker
|
|
22
|
+
* entrypoint. If start() resolved as soon as the SSE client was
|
|
23
|
+
* kicked off, the script would end and Node would exit — the
|
|
24
|
+
* EventSource's underlying fetch socket isn't enough to hold the
|
|
25
|
+
* event loop open by itself in Node 20+ (undici unrefs idle
|
|
26
|
+
* connections under some conditions).
|
|
27
|
+
* 2. A keep-alive interval lives next to the unresolved promise so
|
|
28
|
+
* even if every socket is unrefed, the loop stays open.
|
|
29
|
+
*
|
|
30
|
+
* Callers that integrate the pool into a larger app (where some other
|
|
31
|
+
* piece holds the event loop) can ignore the returned promise.
|
|
32
|
+
*/
|
|
11
33
|
start(): Promise<void>;
|
|
12
34
|
stop(): Promise<void>;
|
|
13
35
|
private handleDispatch;
|
package/dist/pool.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pool.d.ts","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAGX,UAAU,EACV,MAAM,YAAY,CAAC;AAEpB,qBAAa,IAAI;IAChB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IACjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAY;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAW;
|
|
1
|
+
{"version":3,"file":"pool.d.ts","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAGX,UAAU,EACV,MAAM,YAAY,CAAC;AAEpB,qBAAa,IAAI;IAChB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IACjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAY;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAW;IACxC,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,iBAAiB,CAA+B;gBAE5C,MAAM,EAAE,UAAU;IA6C9B;;;;;;;;;;;;;;;;;;OAkBG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAcb,cAAc;IA2E5B,OAAO,CAAC,cAAc;YAIR,WAAW;CAwBzB"}
|
package/dist/pool.js
CHANGED
|
@@ -26,6 +26,9 @@ export class Pool {
|
|
|
26
26
|
inFlight = 0;
|
|
27
27
|
workerId;
|
|
28
28
|
capabilities;
|
|
29
|
+
runningPromise = null;
|
|
30
|
+
runningResolve = null;
|
|
31
|
+
keepAliveInterval = null;
|
|
29
32
|
constructor(config) {
|
|
30
33
|
this.config = {
|
|
31
34
|
concurrency: 1,
|
|
@@ -36,9 +39,13 @@ export class Pool {
|
|
|
36
39
|
this.logger = (config.logger ?? new JsonLogger()).child({
|
|
37
40
|
workerId: this.workerId,
|
|
38
41
|
});
|
|
39
|
-
this.rpc = createRpcClients(
|
|
42
|
+
this.rpc = createRpcClients({
|
|
43
|
+
apiEndpoint: config.apiEndpoint,
|
|
44
|
+
pipelineEndpoint: config.pipelineEndpoint,
|
|
45
|
+
logger: this.logger,
|
|
46
|
+
});
|
|
40
47
|
this.sse = new SseClient({
|
|
41
|
-
endpoint: config.
|
|
48
|
+
endpoint: config.pipelineEndpoint,
|
|
42
49
|
token: config.token,
|
|
43
50
|
workerId: this.workerId,
|
|
44
51
|
capabilities: this.capabilities,
|
|
@@ -52,7 +59,8 @@ export class Pool {
|
|
|
52
59
|
onJobDispatch: (payload) => this.handleDispatch(payload),
|
|
53
60
|
});
|
|
54
61
|
this.logger.info("sdk.boot", {
|
|
55
|
-
|
|
62
|
+
apiEndpoint: config.apiEndpoint,
|
|
63
|
+
pipelineEndpoint: config.pipelineEndpoint,
|
|
56
64
|
capabilities: this.capabilities,
|
|
57
65
|
concurrency: this.config.concurrency,
|
|
58
66
|
version: SDK_VERSION,
|
|
@@ -63,13 +71,54 @@ export class Pool {
|
|
|
63
71
|
});
|
|
64
72
|
}
|
|
65
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Start the pool and keep the worker process alive until `stop()` is
|
|
76
|
+
* called.
|
|
77
|
+
*
|
|
78
|
+
* We deliberately return a never-resolving promise (resolved only by
|
|
79
|
+
* stop()) for two reasons:
|
|
80
|
+
*
|
|
81
|
+
* 1. ESM top-level `await pool.start()` is the documented worker
|
|
82
|
+
* entrypoint. If start() resolved as soon as the SSE client was
|
|
83
|
+
* kicked off, the script would end and Node would exit — the
|
|
84
|
+
* EventSource's underlying fetch socket isn't enough to hold the
|
|
85
|
+
* event loop open by itself in Node 20+ (undici unrefs idle
|
|
86
|
+
* connections under some conditions).
|
|
87
|
+
* 2. A keep-alive interval lives next to the unresolved promise so
|
|
88
|
+
* even if every socket is unrefed, the loop stays open.
|
|
89
|
+
*
|
|
90
|
+
* Callers that integrate the pool into a larger app (where some other
|
|
91
|
+
* piece holds the event loop) can ignore the returned promise.
|
|
92
|
+
*/
|
|
66
93
|
async start() {
|
|
94
|
+
if (this.runningPromise) {
|
|
95
|
+
this.logger.warn("sdk.start.already_running");
|
|
96
|
+
return this.runningPromise;
|
|
97
|
+
}
|
|
67
98
|
this.logger.info("sdk.start");
|
|
68
99
|
this.sse.start();
|
|
100
|
+
this.runningPromise = new Promise((resolve) => {
|
|
101
|
+
this.runningResolve = resolve;
|
|
102
|
+
});
|
|
103
|
+
// 60s heartbeat — also doubles as a "still alive" telemetry tick
|
|
104
|
+
// for log-based dashboards.
|
|
105
|
+
this.keepAliveInterval = setInterval(() => {
|
|
106
|
+
this.logger.debug("sdk.heartbeat", { inFlight: this.inFlight });
|
|
107
|
+
}, 60_000);
|
|
108
|
+
return this.runningPromise;
|
|
69
109
|
}
|
|
70
110
|
async stop() {
|
|
71
111
|
this.logger.info("sdk.stop", { inFlight: this.inFlight });
|
|
72
112
|
this.sse.stop();
|
|
113
|
+
if (this.keepAliveInterval) {
|
|
114
|
+
clearInterval(this.keepAliveInterval);
|
|
115
|
+
this.keepAliveInterval = null;
|
|
116
|
+
}
|
|
117
|
+
if (this.runningResolve) {
|
|
118
|
+
this.runningResolve();
|
|
119
|
+
this.runningResolve = null;
|
|
120
|
+
}
|
|
121
|
+
this.runningPromise = null;
|
|
73
122
|
}
|
|
74
123
|
async handleDispatch(envelope) {
|
|
75
124
|
const jobLogger = this.logger.child({
|
|
@@ -154,5 +203,5 @@ export class Pool {
|
|
|
154
203
|
}
|
|
155
204
|
}
|
|
156
205
|
}
|
|
157
|
-
const SDK_VERSION = "1.
|
|
206
|
+
const SDK_VERSION = "1.3.0";
|
|
158
207
|
//# sourceMappingURL=pool.js.map
|
package/dist/pool.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pool.js","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EACN,sBAAsB,EACtB,wBAAwB,EACxB,wBAAwB,GACxB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAmB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAe,MAAM,aAAa,CAAC;AAOtD,MAAM,OAAO,IAAI;IACC,MAAM,CAAa;IACnB,GAAG,CAAa;IAChB,GAAG,CAAY;IACf,MAAM,CAAS;IACxB,QAAQ,GAAG,CAAC,CAAC;IACJ,QAAQ,CAAS;IACjB,YAAY,CAAW;
|
|
1
|
+
{"version":3,"file":"pool.js","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EACN,sBAAsB,EACtB,wBAAwB,EACxB,wBAAwB,GACxB,MAAM,sDAAsD,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAmB,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,UAAU,EAAe,MAAM,aAAa,CAAC;AAOtD,MAAM,OAAO,IAAI;IACC,MAAM,CAAa;IACnB,GAAG,CAAa;IAChB,GAAG,CAAY;IACf,MAAM,CAAS;IACxB,QAAQ,GAAG,CAAC,CAAC;IACJ,QAAQ,CAAS;IACjB,YAAY,CAAW;IAChC,cAAc,GAAyB,IAAI,CAAC;IAC5C,cAAc,GAAwB,IAAI,CAAC;IAC3C,iBAAiB,GAA0B,IAAI,CAAC;IAExD,YAAY,MAAkB;QAC7B,IAAI,CAAC,MAAM,GAAG;YACb,WAAW,EAAE,CAAC;YACd,GAAG,MAAM;SACT,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,GAAG,QAAQ,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrF,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC;YACvD,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,GAAG,gBAAgB,CAAC;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,MAAM,EAAE,IAAI,CAAC,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,GAAG,IAAI,SAAS,CAAC;YACxB,QAAQ,EAAE,MAAM,CAAC,gBAAgB;YACjC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;YAC/C,WAAW,EAAE,GAAG,EAAE;gBACjB,qEAAqE;YACtE,CAAC;YACD,cAAc,EAAE,GAAG,EAAE;gBACpB,8BAA8B;YAC/B,CAAC;YACD,aAAa,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAsB,CAAC;SACvE,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE;YAC5B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;YACpC,OAAO,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBACxC,IAAI,EAAE,2DAA2D;aACjE,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC,cAAc,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,CAAC,cAAc,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,iEAAiE;QACjE,4BAA4B;QAC5B,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,EAAE,MAAM,CAAC,CAAC;QACX,OAAO,IAAI,CAAC,cAAc,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACtC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,QAAqB;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,GAAG,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG;SACpC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC;YACrD,gEAAgE;YAChE,0DAA0D;YAC1D,0DAA0D;YAC1D,SAAS,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACpD,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;aACpC,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAChC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,CAAC;YACJ,oDAAoD;YACpD,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,SAAS,CACxC,MAAM,CAAC,sBAAsB,EAAE;gBAC9B,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;aAC7B,CAAC,CACF,CAAC;YACF,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAE/B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE;oBAC/B,iBAAiB,EAAE,IAAI,CAAC,YAAY;iBACpC,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,CACrB,QAAQ,CAAC,KAAK,EACd,GAAG,EACH,SAAS,EACT,IAAI,iBAAiB,CAAC,sBAAsB,QAAQ,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAC9E,CAAC;gBACF,OAAO;YACR,CAAC;YAED,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAEnB,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,WAAW,CAC1C,MAAM,CAAC,wBAAwB,EAAE;gBAChC,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;aAC7B,CAAC,CACF,CAAC;YACF,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE;gBAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE;gBAC7B,GAAG;gBACH,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,GAAY,CAAC,CAAC;QACtE,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjB,CAAC;IACF,CAAC;IAEO,cAAc,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,WAAW,CACxB,KAAa,EACb,GAAe,EACf,SAAiB,EACjB,GAAU;QAEV,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,YAAY,iBAAiB,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,WAAW,CAC1C,MAAM,CAAC,wBAAwB,EAAE;gBAChC,KAAK;gBACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBAC7B,KAAK,EAAE,GAAG,CAAC,OAAO;gBAClB,KAAK;aACL,CAAC,CACF,CAAC;YACF,SAAS,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YACjB,SAAS,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAe,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;CACD;AAED,MAAM,WAAW,GAAG,OAAO,CAAC"}
|
package/dist/rpc-clients.d.ts
CHANGED
|
@@ -6,5 +6,10 @@ export interface RpcClients {
|
|
|
6
6
|
internalUploads: Client<typeof InternalUploadsService>;
|
|
7
7
|
integrationQueue: Client<typeof IntegrationQueueService>;
|
|
8
8
|
}
|
|
9
|
-
export
|
|
9
|
+
export interface RpcClientsConfig {
|
|
10
|
+
apiEndpoint: string;
|
|
11
|
+
pipelineEndpoint: string;
|
|
12
|
+
logger: Logger;
|
|
13
|
+
}
|
|
14
|
+
export declare function createRpcClients(config: RpcClientsConfig): RpcClients;
|
|
10
15
|
//# sourceMappingURL=rpc-clients.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rpc-clients.d.ts","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rpc-clients.d.ts","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAqBA,OAAO,EAEN,KAAK,MAAM,EAIX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,sDAAsD,CAAC;AAC/F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,WAAW,UAAU;IAC1B,eAAe,EAAE,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;IACvD,gBAAgB,EAAE,MAAM,CAAC,OAAO,uBAAuB,CAAC,CAAC;CACzD;AA+BD,MAAM,WAAW,gBAAgB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,UAAU,CAiBrE"}
|
package/dist/rpc-clients.js
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* ConnectRPC clients for the two services the SDK talks to:
|
|
3
|
-
* - InternalUploadsService — per-upload callbacks
|
|
4
|
-
* RequestMetaUpload, ConfirmMetaUpload,
|
|
5
|
-
* Auth: the per-job update_token from the
|
|
6
|
-
* as the in-band `updateToken` field on each
|
|
7
|
-
*
|
|
8
|
-
*
|
|
3
|
+
* - InternalUploadsService (`apiEndpoint`) — per-upload callbacks
|
|
4
|
+
* (UpdateUpload, RequestMetaUpload, ConfirmMetaUpload,
|
|
5
|
+
* GetProcessingContext). Auth: the per-job `update_token` from the
|
|
6
|
+
* job envelope, passed as the in-band `updateToken` field on each
|
|
7
|
+
* request.
|
|
8
|
+
* - IntegrationQueueService (`pipelineEndpoint`) — accept / complete /
|
|
9
|
+
* report-error. Auth: the pool token, passed as `queueToken` on each
|
|
10
|
+
* request.
|
|
11
|
+
*
|
|
12
|
+
* The two services live on different processes (Go uploads-api vs Node
|
|
13
|
+
* pipeline-api) so we keep them on separate transports and don't try to
|
|
14
|
+
* share a baseUrl.
|
|
9
15
|
*
|
|
10
16
|
* Each request is wrapped in a logging interceptor that emits one
|
|
11
17
|
* `rpc.request` line on dispatch and one of `rpc.response` /
|
|
@@ -16,15 +22,16 @@ import { createConnectTransport } from "@connectrpc/connect-node";
|
|
|
16
22
|
import { createClient, ConnectError, Code, } from "@connectrpc/connect";
|
|
17
23
|
import { InternalUploadsService } from "./protocol/uplim/api/v1/uploads_pb.js";
|
|
18
24
|
import { IntegrationQueueService } from "./protocol/uplim/workflow/v1/integration_queue_pb.js";
|
|
19
|
-
function loggingInterceptor(logger) {
|
|
25
|
+
function loggingInterceptor(logger, target) {
|
|
20
26
|
return (next) => async (req) => {
|
|
21
27
|
const startedAt = Date.now();
|
|
22
28
|
const method = `${req.service.typeName}/${req.method.name}`;
|
|
23
|
-
logger.debug("rpc.request", { method });
|
|
29
|
+
logger.debug("rpc.request", { method, target });
|
|
24
30
|
try {
|
|
25
31
|
const res = await next(req);
|
|
26
32
|
logger.debug("rpc.response", {
|
|
27
33
|
method,
|
|
34
|
+
target,
|
|
28
35
|
durationMs: Date.now() - startedAt,
|
|
29
36
|
});
|
|
30
37
|
return res;
|
|
@@ -34,6 +41,7 @@ function loggingInterceptor(logger) {
|
|
|
34
41
|
const message = err instanceof Error ? err.message : String(err);
|
|
35
42
|
logger.error("rpc.error", {
|
|
36
43
|
method,
|
|
44
|
+
target,
|
|
37
45
|
code,
|
|
38
46
|
message,
|
|
39
47
|
durationMs: Date.now() - startedAt,
|
|
@@ -42,15 +50,21 @@ function loggingInterceptor(logger) {
|
|
|
42
50
|
}
|
|
43
51
|
};
|
|
44
52
|
}
|
|
45
|
-
export function createRpcClients(
|
|
46
|
-
const
|
|
47
|
-
|
|
53
|
+
export function createRpcClients(config) {
|
|
54
|
+
const rpcLogger = config.logger.child({ component: "rpc" });
|
|
55
|
+
const apiTransport = createConnectTransport({
|
|
56
|
+
baseUrl: config.apiEndpoint,
|
|
57
|
+
httpVersion: "1.1",
|
|
58
|
+
interceptors: [loggingInterceptor(rpcLogger, "api")],
|
|
59
|
+
});
|
|
60
|
+
const pipelineTransport = createConnectTransport({
|
|
61
|
+
baseUrl: config.pipelineEndpoint,
|
|
48
62
|
httpVersion: "1.1",
|
|
49
|
-
interceptors: [loggingInterceptor(
|
|
63
|
+
interceptors: [loggingInterceptor(rpcLogger, "pipeline")],
|
|
50
64
|
});
|
|
51
65
|
return {
|
|
52
|
-
internalUploads: createClient(InternalUploadsService,
|
|
53
|
-
integrationQueue: createClient(IntegrationQueueService,
|
|
66
|
+
internalUploads: createClient(InternalUploadsService, apiTransport),
|
|
67
|
+
integrationQueue: createClient(IntegrationQueueService, pipelineTransport),
|
|
54
68
|
};
|
|
55
69
|
}
|
|
56
70
|
//# sourceMappingURL=rpc-clients.js.map
|
package/dist/rpc-clients.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rpc-clients.js","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"rpc-clients.js","sourceRoot":"","sources":["../src/rpc-clients.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACN,YAAY,EAGZ,YAAY,EACZ,IAAI,GACJ,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,sDAAsD,CAAC;AAQ/F,SAAS,kBAAkB,CAAC,MAAc,EAAE,MAAc;IACzD,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE;gBAC5B,MAAM;gBACN,MAAM;gBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACZ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,GACT,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1D,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE;gBACzB,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,OAAO;gBACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAClC,CAAC,CAAC;YACH,MAAM,GAAG,CAAC;QACX,CAAC;IACF,CAAC,CAAC;AACH,CAAC;AAQD,MAAM,UAAU,gBAAgB,CAAC,MAAwB;IACxD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAE5D,MAAM,YAAY,GAAG,sBAAsB,CAAC;QAC3C,OAAO,EAAE,MAAM,CAAC,WAAW;QAC3B,WAAW,EAAE,KAAK;QAClB,YAAY,EAAE,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;KACpD,CAAC,CAAC;IACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;QAChD,OAAO,EAAE,MAAM,CAAC,gBAAgB;QAChC,WAAW,EAAE,KAAK;QAClB,YAAY,EAAE,CAAC,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;KACzD,CAAC,CAAC;IACH,OAAO;QACN,eAAe,EAAE,YAAY,CAAC,sBAAsB,EAAE,YAAY,CAAC;QACnE,gBAAgB,EAAE,YAAY,CAAC,uBAAuB,EAAE,iBAAiB,CAAC;KAC1E,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -16,8 +16,26 @@
|
|
|
16
16
|
export interface PoolConfig {
|
|
17
17
|
/** Pool token, prefixed with "tpq_". From the pool's creation response. */
|
|
18
18
|
token: string;
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Public uploads-api base URL, e.g. "https://uploads-api.xeonr.dev".
|
|
21
|
+
* Hosts `InternalUploadsService` (RequestMetaUpload, ConfirmMetaUpload,
|
|
22
|
+
* UpdateUpload, GetProcessingContext).
|
|
23
|
+
*
|
|
24
|
+
* Use the public https URL even when running inside the same k8s cluster
|
|
25
|
+
* as the API — keeps third-party workers and first-party workers on a
|
|
26
|
+
* single code path (TLS, normal ingress, no service-mesh assumptions).
|
|
27
|
+
*/
|
|
28
|
+
apiEndpoint: string;
|
|
29
|
+
/**
|
|
30
|
+
* Public pipeline-api base URL, e.g. "https://uploads-pipeline-api.xeonr.dev".
|
|
31
|
+
* Hosts the SSE `/queue/connect` endpoint and `IntegrationQueueService`
|
|
32
|
+
* (AcceptJob, CompleteJob, ReportError).
|
|
33
|
+
*
|
|
34
|
+
* Distinct from `apiEndpoint` because the pipeline runs as a separate
|
|
35
|
+
* Node service; conflating the two would silently route worker traffic
|
|
36
|
+
* to the wrong process.
|
|
37
|
+
*/
|
|
38
|
+
pipelineEndpoint: string;
|
|
21
39
|
/** Optional worker identifier. Defaults to hostname + random suffix. */
|
|
22
40
|
workerId?: string;
|
|
23
41
|
/** Handlers keyed by content type URN (e.g. "default:image"). */
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,UAAU;IAC1B,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,UAAU;IAC1B,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;;;OAQG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrC,6EAA6E;IAC7E,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,UAAU,KAAK,IAAI,CAAC;IACjD;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,aAAa,EAAE,MAAM,CAAC;IACtC,oDAAoD;IACpD,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,EAAE,EAAE,MAAM,CAAC;QACX,8DAA8D;QAC9D,MAAM,EAAE,UAAU,CAAC;KACnB,CAAC;IACF,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,QAAQ,IAAI,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IAChD,0EAA0E;IAC1E,cAAc,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,mCAAmC;IACnC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5C,gFAAgF;IAChF,UAAU,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD;;;;;;OAMG;IACH,WAAW,CAAC,QAAQ,EAAE,qBAAqB,GAAG,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,mEAAmE;IACnE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,0EAA0E;IAC1E,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD;;;;;;;;;;;;;;;;OAgBG;IACH,oBAAoB,CAAC,CAAC,EACrB,IAAI,EAAE,sBAAsB,EAAE,EAC9B,QAAQ,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,GACnD,OAAO,CAAC,CAAC,CAAC,CAAC;CACd;AAED,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAChC,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,2GAA2G;IAC3G,WAAW,EAAE,MAAM,CAAC;IACpB,6FAA6F;IAC7F,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,iBAAiB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,cAAc,GAAG,cAAc,GAAG,YAAY,GAAG,iBAAiB,CAAC;IACnI,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAE9E;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,CAAC,EAAE;QACP,UAAU,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,GAAG,CAAC,EAAE;QACL,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,mBAAmB;IACnC,SAAS,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,YAAY,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACnD,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAA;KAAE,CAAC;CACxD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xeonr/upload-pool-sdk",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Self-hosted worker SDK for the upl.im content type pipeline. Implement a handler per content type URN; the SDK handles SSE connection, job acceptance, presigned thumbnail upload, and metadata callbacks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
package/src/index.ts
CHANGED
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
*
|
|
9
9
|
* await createPool({
|
|
10
10
|
* token: process.env.UPL_POOL_TOKEN!,
|
|
11
|
-
*
|
|
11
|
+
* apiEndpoint: process.env.UPL_API_URL!,
|
|
12
|
+
* pipelineEndpoint: process.env.UPL_PIPELINE_API_URL!,
|
|
12
13
|
* handlers: {
|
|
13
14
|
* "my-integration:invoice": async (ctx) => {
|
|
14
15
|
* const buf = await ctx.downloadBuffer();
|
package/src/pool.ts
CHANGED
|
@@ -36,6 +36,9 @@ export class Pool {
|
|
|
36
36
|
private inFlight = 0;
|
|
37
37
|
private readonly workerId: string;
|
|
38
38
|
private readonly capabilities: string[];
|
|
39
|
+
private runningPromise: Promise<void> | null = null;
|
|
40
|
+
private runningResolve: (() => void) | null = null;
|
|
41
|
+
private keepAliveInterval: NodeJS.Timeout | null = null;
|
|
39
42
|
|
|
40
43
|
constructor(config: PoolConfig) {
|
|
41
44
|
this.config = {
|
|
@@ -47,9 +50,13 @@ export class Pool {
|
|
|
47
50
|
this.logger = (config.logger ?? new JsonLogger()).child({
|
|
48
51
|
workerId: this.workerId,
|
|
49
52
|
});
|
|
50
|
-
this.rpc = createRpcClients(
|
|
53
|
+
this.rpc = createRpcClients({
|
|
54
|
+
apiEndpoint: config.apiEndpoint,
|
|
55
|
+
pipelineEndpoint: config.pipelineEndpoint,
|
|
56
|
+
logger: this.logger,
|
|
57
|
+
});
|
|
51
58
|
this.sse = new SseClient({
|
|
52
|
-
endpoint: config.
|
|
59
|
+
endpoint: config.pipelineEndpoint,
|
|
53
60
|
token: config.token,
|
|
54
61
|
workerId: this.workerId,
|
|
55
62
|
capabilities: this.capabilities,
|
|
@@ -64,7 +71,8 @@ export class Pool {
|
|
|
64
71
|
});
|
|
65
72
|
|
|
66
73
|
this.logger.info("sdk.boot", {
|
|
67
|
-
|
|
74
|
+
apiEndpoint: config.apiEndpoint,
|
|
75
|
+
pipelineEndpoint: config.pipelineEndpoint,
|
|
68
76
|
capabilities: this.capabilities,
|
|
69
77
|
concurrency: this.config.concurrency,
|
|
70
78
|
version: SDK_VERSION,
|
|
@@ -77,14 +85,55 @@ export class Pool {
|
|
|
77
85
|
}
|
|
78
86
|
}
|
|
79
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Start the pool and keep the worker process alive until `stop()` is
|
|
90
|
+
* called.
|
|
91
|
+
*
|
|
92
|
+
* We deliberately return a never-resolving promise (resolved only by
|
|
93
|
+
* stop()) for two reasons:
|
|
94
|
+
*
|
|
95
|
+
* 1. ESM top-level `await pool.start()` is the documented worker
|
|
96
|
+
* entrypoint. If start() resolved as soon as the SSE client was
|
|
97
|
+
* kicked off, the script would end and Node would exit — the
|
|
98
|
+
* EventSource's underlying fetch socket isn't enough to hold the
|
|
99
|
+
* event loop open by itself in Node 20+ (undici unrefs idle
|
|
100
|
+
* connections under some conditions).
|
|
101
|
+
* 2. A keep-alive interval lives next to the unresolved promise so
|
|
102
|
+
* even if every socket is unrefed, the loop stays open.
|
|
103
|
+
*
|
|
104
|
+
* Callers that integrate the pool into a larger app (where some other
|
|
105
|
+
* piece holds the event loop) can ignore the returned promise.
|
|
106
|
+
*/
|
|
80
107
|
async start(): Promise<void> {
|
|
108
|
+
if (this.runningPromise) {
|
|
109
|
+
this.logger.warn("sdk.start.already_running");
|
|
110
|
+
return this.runningPromise;
|
|
111
|
+
}
|
|
81
112
|
this.logger.info("sdk.start");
|
|
82
113
|
this.sse.start();
|
|
114
|
+
this.runningPromise = new Promise<void>((resolve) => {
|
|
115
|
+
this.runningResolve = resolve;
|
|
116
|
+
});
|
|
117
|
+
// 60s heartbeat — also doubles as a "still alive" telemetry tick
|
|
118
|
+
// for log-based dashboards.
|
|
119
|
+
this.keepAliveInterval = setInterval(() => {
|
|
120
|
+
this.logger.debug("sdk.heartbeat", { inFlight: this.inFlight });
|
|
121
|
+
}, 60_000);
|
|
122
|
+
return this.runningPromise;
|
|
83
123
|
}
|
|
84
124
|
|
|
85
125
|
async stop(): Promise<void> {
|
|
86
126
|
this.logger.info("sdk.stop", { inFlight: this.inFlight });
|
|
87
127
|
this.sse.stop();
|
|
128
|
+
if (this.keepAliveInterval) {
|
|
129
|
+
clearInterval(this.keepAliveInterval);
|
|
130
|
+
this.keepAliveInterval = null;
|
|
131
|
+
}
|
|
132
|
+
if (this.runningResolve) {
|
|
133
|
+
this.runningResolve();
|
|
134
|
+
this.runningResolve = null;
|
|
135
|
+
}
|
|
136
|
+
this.runningPromise = null;
|
|
88
137
|
}
|
|
89
138
|
|
|
90
139
|
private async handleDispatch(envelope: JobEnvelope): Promise<void> {
|
|
@@ -192,4 +241,4 @@ export class Pool {
|
|
|
192
241
|
}
|
|
193
242
|
}
|
|
194
243
|
|
|
195
|
-
const SDK_VERSION = "1.
|
|
244
|
+
const SDK_VERSION = "1.3.0";
|
package/src/rpc-clients.ts
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* ConnectRPC clients for the two services the SDK talks to:
|
|
3
|
-
* - InternalUploadsService — per-upload callbacks
|
|
4
|
-
* RequestMetaUpload, ConfirmMetaUpload,
|
|
5
|
-
* Auth: the per-job update_token from the
|
|
6
|
-
* as the in-band `updateToken` field on each
|
|
7
|
-
*
|
|
8
|
-
*
|
|
3
|
+
* - InternalUploadsService (`apiEndpoint`) — per-upload callbacks
|
|
4
|
+
* (UpdateUpload, RequestMetaUpload, ConfirmMetaUpload,
|
|
5
|
+
* GetProcessingContext). Auth: the per-job `update_token` from the
|
|
6
|
+
* job envelope, passed as the in-band `updateToken` field on each
|
|
7
|
+
* request.
|
|
8
|
+
* - IntegrationQueueService (`pipelineEndpoint`) — accept / complete /
|
|
9
|
+
* report-error. Auth: the pool token, passed as `queueToken` on each
|
|
10
|
+
* request.
|
|
11
|
+
*
|
|
12
|
+
* The two services live on different processes (Go uploads-api vs Node
|
|
13
|
+
* pipeline-api) so we keep them on separate transports and don't try to
|
|
14
|
+
* share a baseUrl.
|
|
9
15
|
*
|
|
10
16
|
* Each request is wrapped in a logging interceptor that emits one
|
|
11
17
|
* `rpc.request` line on dispatch and one of `rpc.response` /
|
|
@@ -29,15 +35,16 @@ export interface RpcClients {
|
|
|
29
35
|
integrationQueue: Client<typeof IntegrationQueueService>;
|
|
30
36
|
}
|
|
31
37
|
|
|
32
|
-
function loggingInterceptor(logger: Logger): Interceptor {
|
|
38
|
+
function loggingInterceptor(logger: Logger, target: string): Interceptor {
|
|
33
39
|
return (next) => async (req) => {
|
|
34
40
|
const startedAt = Date.now();
|
|
35
41
|
const method = `${req.service.typeName}/${req.method.name}`;
|
|
36
|
-
logger.debug("rpc.request", { method });
|
|
42
|
+
logger.debug("rpc.request", { method, target });
|
|
37
43
|
try {
|
|
38
44
|
const res = await next(req);
|
|
39
45
|
logger.debug("rpc.response", {
|
|
40
46
|
method,
|
|
47
|
+
target,
|
|
41
48
|
durationMs: Date.now() - startedAt,
|
|
42
49
|
});
|
|
43
50
|
return res;
|
|
@@ -47,6 +54,7 @@ function loggingInterceptor(logger: Logger): Interceptor {
|
|
|
47
54
|
const message = err instanceof Error ? err.message : String(err);
|
|
48
55
|
logger.error("rpc.error", {
|
|
49
56
|
method,
|
|
57
|
+
target,
|
|
50
58
|
code,
|
|
51
59
|
message,
|
|
52
60
|
durationMs: Date.now() - startedAt,
|
|
@@ -56,17 +64,27 @@ function loggingInterceptor(logger: Logger): Interceptor {
|
|
|
56
64
|
};
|
|
57
65
|
}
|
|
58
66
|
|
|
59
|
-
export
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
67
|
+
export interface RpcClientsConfig {
|
|
68
|
+
apiEndpoint: string;
|
|
69
|
+
pipelineEndpoint: string;
|
|
70
|
+
logger: Logger;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function createRpcClients(config: RpcClientsConfig): RpcClients {
|
|
74
|
+
const rpcLogger = config.logger.child({ component: "rpc" });
|
|
75
|
+
|
|
76
|
+
const apiTransport = createConnectTransport({
|
|
77
|
+
baseUrl: config.apiEndpoint,
|
|
78
|
+
httpVersion: "1.1",
|
|
79
|
+
interceptors: [loggingInterceptor(rpcLogger, "api")],
|
|
80
|
+
});
|
|
81
|
+
const pipelineTransport = createConnectTransport({
|
|
82
|
+
baseUrl: config.pipelineEndpoint,
|
|
65
83
|
httpVersion: "1.1",
|
|
66
|
-
interceptors: [loggingInterceptor(
|
|
84
|
+
interceptors: [loggingInterceptor(rpcLogger, "pipeline")],
|
|
67
85
|
});
|
|
68
86
|
return {
|
|
69
|
-
internalUploads: createClient(InternalUploadsService,
|
|
70
|
-
integrationQueue: createClient(IntegrationQueueService,
|
|
87
|
+
internalUploads: createClient(InternalUploadsService, apiTransport),
|
|
88
|
+
integrationQueue: createClient(IntegrationQueueService, pipelineTransport),
|
|
71
89
|
};
|
|
72
90
|
}
|
package/src/types.ts
CHANGED
|
@@ -17,8 +17,26 @@
|
|
|
17
17
|
export interface PoolConfig {
|
|
18
18
|
/** Pool token, prefixed with "tpq_". From the pool's creation response. */
|
|
19
19
|
token: string;
|
|
20
|
-
/**
|
|
21
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Public uploads-api base URL, e.g. "https://uploads-api.xeonr.dev".
|
|
22
|
+
* Hosts `InternalUploadsService` (RequestMetaUpload, ConfirmMetaUpload,
|
|
23
|
+
* UpdateUpload, GetProcessingContext).
|
|
24
|
+
*
|
|
25
|
+
* Use the public https URL even when running inside the same k8s cluster
|
|
26
|
+
* as the API — keeps third-party workers and first-party workers on a
|
|
27
|
+
* single code path (TLS, normal ingress, no service-mesh assumptions).
|
|
28
|
+
*/
|
|
29
|
+
apiEndpoint: string;
|
|
30
|
+
/**
|
|
31
|
+
* Public pipeline-api base URL, e.g. "https://uploads-pipeline-api.xeonr.dev".
|
|
32
|
+
* Hosts the SSE `/queue/connect` endpoint and `IntegrationQueueService`
|
|
33
|
+
* (AcceptJob, CompleteJob, ReportError).
|
|
34
|
+
*
|
|
35
|
+
* Distinct from `apiEndpoint` because the pipeline runs as a separate
|
|
36
|
+
* Node service; conflating the two would silently route worker traffic
|
|
37
|
+
* to the wrong process.
|
|
38
|
+
*/
|
|
39
|
+
pipelineEndpoint: string;
|
|
22
40
|
/** Optional worker identifier. Defaults to hostname + random suffix. */
|
|
23
41
|
workerId?: string;
|
|
24
42
|
/** Handlers keyed by content type URN (e.g. "default:image"). */
|