@xeonr/pipelines-sdk 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +334 -2
  2. package/dist/auth.d.ts +72 -0
  3. package/dist/auth.d.ts.map +1 -0
  4. package/dist/auth.js +139 -0
  5. package/dist/auth.js.map +1 -0
  6. package/dist/client.d.ts +129 -0
  7. package/dist/client.d.ts.map +1 -0
  8. package/dist/client.js +106 -0
  9. package/dist/client.js.map +1 -0
  10. package/dist/errors.d.ts +113 -0
  11. package/dist/errors.d.ts.map +1 -0
  12. package/dist/errors.js +115 -0
  13. package/dist/errors.js.map +1 -0
  14. package/dist/gen/buf/validate/validate_pb.d.ts +4828 -0
  15. package/dist/gen/buf/validate/validate_pb.d.ts.map +1 -0
  16. package/dist/gen/buf/validate/validate_pb.js +357 -0
  17. package/dist/gen/buf/validate/validate_pb.js.map +1 -0
  18. package/dist/gen/gnostic/openapi/v3/annotations_pb.d.ts +24 -0
  19. package/dist/gen/gnostic/openapi/v3/annotations_pb.d.ts.map +1 -0
  20. package/dist/gen/gnostic/openapi/v3/annotations_pb.js +37 -0
  21. package/dist/gen/gnostic/openapi/v3/annotations_pb.js.map +1 -0
  22. package/dist/gen/gnostic/openapi/v3/openapiv3_pb.d.ts +2245 -0
  23. package/dist/gen/gnostic/openapi/v3/openapiv3_pb.d.ts.map +1 -0
  24. package/dist/gen/gnostic/openapi/v3/openapiv3_pb.js +410 -0
  25. package/dist/gen/gnostic/openapi/v3/openapiv3_pb.js.map +1 -0
  26. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +1031 -0
  27. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -0
  28. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +219 -0
  29. package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -0
  30. package/dist/gen/xeonr/pipelines/private/v1/runners_pb.d.ts +190 -0
  31. package/dist/gen/xeonr/pipelines/private/v1/runners_pb.d.ts.map +1 -0
  32. package/dist/gen/xeonr/pipelines/private/v1/runners_pb.js +53 -0
  33. package/dist/gen/xeonr/pipelines/private/v1/runners_pb.js.map +1 -0
  34. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +1161 -0
  35. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -0
  36. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +269 -0
  37. package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -0
  38. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +1061 -0
  39. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -0
  40. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +221 -0
  41. package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -0
  42. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +580 -0
  43. package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -0
  44. package/dist/gen/xeonr/pipelines/v1/common_pb.js +483 -0
  45. package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -0
  46. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +762 -0
  47. package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -0
  48. package/dist/gen/xeonr/pipelines/v1/definition_pb.js +190 -0
  49. package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -0
  50. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +919 -0
  51. package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -0
  52. package/dist/gen/xeonr/pipelines/v1/pools_pb.js +272 -0
  53. package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -0
  54. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +895 -0
  55. package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -0
  56. package/dist/gen/xeonr/pipelines/v1/registry_pb.js +277 -0
  57. package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -0
  58. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +1237 -0
  59. package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -0
  60. package/dist/gen/xeonr/pipelines/v1/runs_pb.js +204 -0
  61. package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -0
  62. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +647 -0
  63. package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -0
  64. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +181 -0
  65. package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -0
  66. package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +381 -0
  67. package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -0
  68. package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +145 -0
  69. package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -0
  70. package/dist/gen/xeonr/platform/v1/options_pb.d.ts +313 -0
  71. package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -0
  72. package/dist/gen/xeonr/platform/v1/options_pb.js +235 -0
  73. package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -0
  74. package/dist/handler.d.ts +94 -0
  75. package/dist/handler.d.ts.map +1 -0
  76. package/dist/handler.js +18 -0
  77. package/dist/handler.js.map +1 -0
  78. package/dist/index.d.ts +30 -0
  79. package/dist/index.d.ts.map +1 -0
  80. package/dist/index.js +24 -0
  81. package/dist/index.js.map +1 -0
  82. package/dist/internal/time.d.ts +8 -0
  83. package/dist/internal/time.d.ts.map +1 -0
  84. package/dist/internal/time.js +25 -0
  85. package/dist/internal/time.js.map +1 -0
  86. package/dist/logship.d.ts +117 -0
  87. package/dist/logship.d.ts.map +1 -0
  88. package/dist/logship.js +319 -0
  89. package/dist/logship.js.map +1 -0
  90. package/dist/testing/index.d.ts +215 -0
  91. package/dist/testing/index.d.ts.map +1 -0
  92. package/dist/testing/index.js +1015 -0
  93. package/dist/testing/index.js.map +1 -0
  94. package/dist/transport.d.ts +43 -0
  95. package/dist/transport.d.ts.map +1 -0
  96. package/dist/transport.js +44 -0
  97. package/dist/transport.js.map +1 -0
  98. package/dist/worker.d.ts +236 -0
  99. package/dist/worker.d.ts.map +1 -0
  100. package/dist/worker.js +733 -0
  101. package/dist/worker.js.map +1 -0
  102. package/package.json +70 -3
package/dist/worker.js ADDED
@@ -0,0 +1,733 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { create, fromJson } from "@bufbuild/protobuf";
3
+ import { durationFromMs, durationMs, timestampMs, ValueSchema } from "@bufbuild/protobuf/wkt";
4
+ import { Code, ConnectError, createClient } from "@connectrpc/connect";
5
+ import { AttemptAbortedError, errorKind, handlerFailureKind, WorkerStartError } from "./errors.js";
6
+ import { AttemptState, Termination } from "./gen/xeonr/pipelines/v1/common_pb.js";
7
+ import { PoolKind, PoolOwnerKind, PoolsService } from "./gen/xeonr/pipelines/v1/pools_pb.js";
8
+ import { PackageKind, RegistryService, Visibility } from "./gen/xeonr/pipelines/v1/registry_pb.js";
9
+ import { AttemptRefSchema, HandlerFailureKind, WorkerService } from "./gen/xeonr/pipelines/private/v1/worker_pb.js";
10
+ import { LogShipper } from "./logship.js";
11
+ import { backoff, sleep } from "./internal/time.js";
12
+ import { transportOf } from "./transport.js";
13
+ /** Writes warnings and errors to stderr; drops debug and info. */
14
+ export const consoleLogger = {
15
+ debug: () => undefined,
16
+ info: () => undefined,
17
+ warn: (m, f) => console.warn(`[pipelines-sdk] ${m}`, f ?? ""),
18
+ error: (m, f) => console.error(`[pipelines-sdk] ${m}`, f ?? ""),
19
+ };
20
+ const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
21
+ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
22
+ const MAX_MESSAGE = 2000;
23
+ /**
24
+ * An SDK worker: runs a product's handlers inside its own Node.js process.
25
+ *
26
+ * It authenticates as the product's Auth service account (`jwt-bearer`,
27
+ * scope `pipelines:worker:edit`) and may claim only its owner's handlers.
28
+ * It long-polls `ClaimAttempts` for its handlers within its concurrency
29
+ * limits, starts each attempt, renews the lease by heartbeat while the
30
+ * handler runs, ships the handler's log, and completes the attempt with its
31
+ * kv outputs or the right termination. Retries, delays, timeouts and leases
32
+ * are the platform's.
33
+ *
34
+ * ```ts
35
+ * const worker = new Worker({ baseUrl, token, tenant, pool: { name: "app" } });
36
+ * worker.handle(resizeImage);
37
+ * await worker.start();
38
+ * process.on("SIGTERM", () => void worker.stop());
39
+ * ```
40
+ */
41
+ export class Worker {
42
+ /** This worker instance's id (see {@link WorkerOptions.workerId}). */
43
+ workerId;
44
+ opts;
45
+ log;
46
+ api;
47
+ registry;
48
+ pools;
49
+ handlers = new Map();
50
+ limit;
51
+ active = 0;
52
+ reserved = 0;
53
+ poolId = "";
54
+ _state = "idle";
55
+ claims = new Set();
56
+ attempts = new Map();
57
+ claimDelay = 0;
58
+ claimRetryAt = 0;
59
+ claimRetryTimer;
60
+ fatal;
61
+ stopped;
62
+ constructor(opts) {
63
+ this.opts = opts;
64
+ this.log = opts.logger ?? consoleLogger;
65
+ this.workerId = opts.workerId ?? randomUUID();
66
+ if (!UUID.test(this.workerId))
67
+ throw new TypeError("Worker: workerId must be a UUID");
68
+ this.limit = opts.concurrency ?? 10;
69
+ if (!Number.isInteger(this.limit) || this.limit < 1 || this.limit > 256) {
70
+ throw new RangeError("Worker: concurrency must be an integer from 1 to 256");
71
+ }
72
+ const transport = transportOf(opts);
73
+ this.api = createClient(WorkerService, transport);
74
+ this.registry = createClient(RegistryService, transport);
75
+ this.pools = createClient(PoolsService, transport);
76
+ this.stopped = deferred();
77
+ }
78
+ /** Where the worker is in its life. */
79
+ get state() {
80
+ return this._state;
81
+ }
82
+ /** The `sdk` pool's id, once started. */
83
+ get pool() {
84
+ return this.poolId;
85
+ }
86
+ /**
87
+ * Registers a handler. Call before {@link start}. A handler is claimed by
88
+ * name; its version and manifest are published at boot.
89
+ */
90
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- any handler shape may be registered
91
+ handle(def) {
92
+ if (this._state !== "idle")
93
+ throw new Error("Worker.handle: register handlers before start()");
94
+ if (this.handlers.has(def.name))
95
+ throw new Error(`Worker.handle: ${def.name} is registered twice`);
96
+ if (this.handlers.size >= 256)
97
+ throw new Error("Worker.handle: at most 256 handlers per worker");
98
+ this.handlers.set(def.name, { def, limit: Math.min(def.concurrency ?? this.limit, this.limit), active: 0, reserved: 0 });
99
+ return this;
100
+ }
101
+ /**
102
+ * Boots the worker: publishes the handlers and declares the pool (unless
103
+ * `register` is false), then starts claiming. Resolves once claiming has
104
+ * begun; rejects with a {@link WorkerStartError} when registration fails.
105
+ */
106
+ async start() {
107
+ if (this._state !== "idle")
108
+ throw new Error(`Worker.start: the worker is ${this._state}`);
109
+ if (this.handlers.size === 0)
110
+ throw new WorkerStartError("Worker.start: no handlers are registered");
111
+ this._state = "starting";
112
+ try {
113
+ this.poolId = this.opts.register === false ? this.requirePoolId() : await this.register();
114
+ }
115
+ catch (err) {
116
+ this._state = "idle";
117
+ throw err instanceof WorkerStartError ? err : new WorkerStartError(`Worker.start: ${message(err)}`, { cause: err });
118
+ }
119
+ this._state = "running";
120
+ this.log.info("Worker started", { workerId: this.workerId, pool: this.poolId, handlers: [...this.handlers.keys()] });
121
+ this.pump();
122
+ }
123
+ /**
124
+ * Stops gracefully: claims nothing more, hands back every attempt claimed
125
+ * but not started (`ReleaseAttempt`: offered again at once, nothing
126
+ * spent), lets running handlers finish within `timeoutMs`, then aborts the
127
+ * rest (reason `shutdown`) and, after `graceMs`, completes them with
128
+ * `TERMINATION_INTERRUPTED` (`FAILURE_CLASS_INTERRUPTED`: transient, and
129
+ * free of the retry budget on an `interruptible` node).
130
+ */
131
+ async stop(opts = {}) {
132
+ if (this._state === "stopped")
133
+ return;
134
+ if (this._state === "idle" || this._state === "starting") {
135
+ this._state = "stopped";
136
+ this.stopped.resolve();
137
+ return;
138
+ }
139
+ const timeoutMs = opts.timeoutMs ?? 30_000;
140
+ const graceMs = opts.graceMs ?? 5_000;
141
+ this._state = "stopping";
142
+ this.haltClaims();
143
+ const settled = () => Promise.all([...this.attempts.values()].map((a) => a.done));
144
+ await Promise.race([settled(), sleep(timeoutMs)]);
145
+ if (this.attempts.size > 0) {
146
+ this.log.warn("Shutdown timeout passed; aborting in-flight handlers", { attempts: this.attempts.size });
147
+ for (const a of this.attempts.values())
148
+ a.shutdown(graceMs);
149
+ await Promise.race([settled(), sleep(graceMs + 15_000)]);
150
+ }
151
+ this.finish();
152
+ }
153
+ /**
154
+ * Settles when the worker has stopped: resolves after {@link stop}, and
155
+ * rejects with the error that stopped it claiming (see `onError`).
156
+ */
157
+ wait() {
158
+ return this.stopped.promise;
159
+ }
160
+ /** What the worker is doing now. */
161
+ stats() {
162
+ const handlers = {};
163
+ for (const [name, h] of this.handlers)
164
+ handlers[name] = h.active;
165
+ return { state: this._state, active: this.active, handlers, claims: this.claims.size };
166
+ }
167
+ requirePoolId() {
168
+ if (!this.opts.pool.id)
169
+ throw new WorkerStartError("Worker.start: pool.id is required when register is false");
170
+ return this.opts.pool.id;
171
+ }
172
+ async register() {
173
+ for (const { def } of this.handlers.values()) {
174
+ try {
175
+ await this.registry.publishVersion({
176
+ name: def.name,
177
+ version: def.version,
178
+ tenant: this.opts.tenant,
179
+ visibility: this.opts.visibility ?? Visibility.PRIVATE,
180
+ manifest: {
181
+ kind: PackageKind.WORKER,
182
+ description: def.description ?? "",
183
+ inputs: def.inputs ?? [],
184
+ outputs: def.outputs ?? [],
185
+ resourceKinds: def.resourceKinds ?? [],
186
+ readmeMd: def.readmeMd ?? "",
187
+ },
188
+ }, { timeoutMs: 30_000 });
189
+ }
190
+ catch (err) {
191
+ if (ConnectError.from(err).code === Code.AlreadyExists) {
192
+ throw new WorkerStartError(`Worker.start: ${def.name}@${def.version} is already published with a different manifest; bump the handler's version`, { cause: err });
193
+ }
194
+ throw new WorkerStartError(`Worker.start: publishing ${def.name}@${def.version} failed: ${message(err)}`, { cause: err });
195
+ }
196
+ }
197
+ const res = await this.pools.setPool({
198
+ pool: {
199
+ ...(this.opts.pool.id ? { id: this.opts.pool.id } : {}),
200
+ owner: { kind: PoolOwnerKind.TENANT, tenant: this.opts.tenant },
201
+ name: this.opts.pool.name,
202
+ kind: PoolKind.SDK,
203
+ tier: this.opts.pool.tier ?? "",
204
+ handlers: [...this.handlers.keys()],
205
+ capacityPerInstance: this.limit,
206
+ },
207
+ }, { timeoutMs: 30_000 });
208
+ if (!res.pool?.id)
209
+ throw new WorkerStartError("Worker.start: SetPool answered without a pool id");
210
+ return res.pool.id;
211
+ }
212
+ /**
213
+ * Starts claim long-polls while there is unreserved capacity. Each poll
214
+ * reserves its `free_slots` from the overall limit and from every handler
215
+ * it names, so no answer can take the worker past a limit: a poll asks for
216
+ * at most the smallest free count among the handlers it names, and a
217
+ * handler with no free slot is left out of new polls until one frees.
218
+ */
219
+ pump() {
220
+ while (this._state === "running" && this.fatal === undefined && this.claims.size < (this.opts.maxConcurrentClaims ?? 4)) {
221
+ const wait = this.claimRetryAt - Date.now();
222
+ if (wait > 0) {
223
+ this.claimRetryTimer ??= setTimeout(() => {
224
+ this.claimRetryTimer = undefined;
225
+ this.pump();
226
+ }, wait);
227
+ return;
228
+ }
229
+ const overall = this.limit - this.active - this.reserved;
230
+ if (overall <= 0)
231
+ return;
232
+ const eligible = [...this.handlers.values()].filter((h) => h.limit - h.active - h.reserved > 0);
233
+ if (eligible.length === 0)
234
+ return;
235
+ const slots = Math.min(overall, 256, ...eligible.map((h) => h.limit - h.active - h.reserved));
236
+ this.reserved += slots;
237
+ for (const h of eligible)
238
+ h.reserved += slots;
239
+ void this.claim(eligible, slots);
240
+ }
241
+ }
242
+ async claim(eligible, slots) {
243
+ const ac = new AbortController();
244
+ this.claims.add(ac);
245
+ let taken = 0;
246
+ const takenBy = new Map();
247
+ try {
248
+ const res = await this.api.claimAttempts({
249
+ freeSlots: slots,
250
+ selector: { case: "handlers", value: { handlers: eligible.map((h) => h.def.name), workerId: this.workerId, poolId: this.poolId } },
251
+ wait: durationFromMs(Math.min(this.opts.claimWaitMs ?? 30_000, 30_000)),
252
+ }, { signal: ac.signal, timeoutMs: Math.min(this.opts.claimWaitMs ?? 30_000, 30_000) + 15_000 });
253
+ this.claimDelay = 0;
254
+ for (const c of res.attempts) {
255
+ const h = this.handlers.get(c.package?.name ?? "");
256
+ const n = h ? (takenBy.get(h) ?? 0) : 0;
257
+ if (!c.ref || !h || !eligible.includes(h) || taken >= slots || n >= slots || this._state !== "running") {
258
+ // More than was asked for, for a handler not asked for, or claimed
259
+ // as the worker began to stop: handed straight back, unstarted.
260
+ if (this._state === "running") {
261
+ this.log.warn("Claim answered an attempt the worker did not ask for; releasing it", {
262
+ attempt: c.ref?.attemptId,
263
+ handler: c.package?.name,
264
+ });
265
+ }
266
+ if (c.ref)
267
+ void this.release(this.refOf(c), "not taken by this worker");
268
+ continue;
269
+ }
270
+ taken++;
271
+ takenBy.set(h, n + 1);
272
+ this.reserved--;
273
+ h.reserved--;
274
+ this.active++;
275
+ h.active++;
276
+ this.run(h, c);
277
+ }
278
+ }
279
+ catch (err) {
280
+ const kind = errorKind(err);
281
+ if (ac.signal.aborted || this._state !== "running") {
282
+ // Stopping: a claim reply lost here costs nothing (its lease lapses).
283
+ }
284
+ else if (kind === "auth" || kind === "final") {
285
+ this.fail(err);
286
+ }
287
+ else {
288
+ this.claimDelay = backoff(this.claimDelay, 500, 30_000);
289
+ this.claimRetryAt = Date.now() + this.claimDelay;
290
+ this.log.warn("ClaimAttempts failed; retrying", { error: message(err), retryInMs: Math.round(this.claimDelay) });
291
+ }
292
+ }
293
+ finally {
294
+ this.reserved -= slots - taken;
295
+ for (const h of eligible)
296
+ h.reserved -= slots - (takenBy.get(h) ?? 0);
297
+ this.claims.delete(ac);
298
+ this.pump();
299
+ }
300
+ }
301
+ run(h, c) {
302
+ const run = new AttemptRun(this, h, c);
303
+ this.attempts.set(run.id, run);
304
+ void run.done.finally(() => {
305
+ this.attempts.delete(run.id);
306
+ this.active--;
307
+ h.active--;
308
+ if (this._state === "running")
309
+ this.pump();
310
+ else if (this.fatal !== undefined && this.attempts.size === 0)
311
+ this.finish();
312
+ });
313
+ }
314
+ /** The claim's ref as this worker echoes it: with its worker id. */
315
+ refOf(c) {
316
+ return create(AttemptRefSchema, {
317
+ attemptId: c.ref?.attemptId ?? "",
318
+ claimEpoch: c.ref?.claimEpoch ?? 0n,
319
+ workerId: c.ref?.workerId || this.workerId,
320
+ });
321
+ }
322
+ /**
323
+ * Hands back an attempt claimed and not started (`ReleaseAttempt`).
324
+ * `failed_precondition` means it is no longer this worker's: done.
325
+ * Returns the state the server answered, or undefined when the release
326
+ * could not be delivered (the lease then lapses, which spends nothing
327
+ * either, a lease later).
328
+ */
329
+ async release(ref, reason) {
330
+ let delay = 0;
331
+ const giveUpAt = Date.now() + 10_000;
332
+ for (;;) {
333
+ try {
334
+ return (await this.api.releaseAttempt({ ref, reason }, { timeoutMs: 10_000 })).state;
335
+ }
336
+ catch (err) {
337
+ const kind = errorKind(err);
338
+ if (kind === "lost")
339
+ return AttemptState.OFFERED;
340
+ if (kind !== "transient" || Date.now() >= giveUpAt) {
341
+ this.log.warn("ReleaseAttempt failed; the lease will lapse instead", { attempt: ref.attemptId, error: message(err) });
342
+ return undefined;
343
+ }
344
+ delay = backoff(delay, 200, 2_000);
345
+ await sleep(delay);
346
+ }
347
+ }
348
+ }
349
+ /** Stops claiming for good after an error that will not pass. */
350
+ fail(err) {
351
+ if (this.fatal !== undefined)
352
+ return;
353
+ this.fatal = err ?? new Error("unknown worker failure");
354
+ this.log.error("Worker stopped claiming", { error: message(err) });
355
+ this.haltClaims();
356
+ try {
357
+ this.opts.onError?.(err);
358
+ }
359
+ catch (e) {
360
+ this.log.error("onError threw", { error: message(e) });
361
+ }
362
+ if (this._state === "running")
363
+ this._state = "stopping";
364
+ if (this.attempts.size === 0)
365
+ this.finish();
366
+ }
367
+ haltClaims() {
368
+ for (const ac of this.claims)
369
+ ac.abort();
370
+ if (this.claimRetryTimer)
371
+ clearTimeout(this.claimRetryTimer);
372
+ this.claimRetryTimer = undefined;
373
+ }
374
+ finish() {
375
+ if (this._state === "stopped")
376
+ return;
377
+ this._state = "stopped";
378
+ if (this.fatal !== undefined)
379
+ this.stopped.reject(this.fatal);
380
+ else
381
+ this.stopped.resolve();
382
+ }
383
+ /** @internal */
384
+ get internals() {
385
+ return {
386
+ api: this.api,
387
+ log: this.log,
388
+ completeTimeoutMs: this.opts.completeTimeoutMs ?? 60_000,
389
+ running: () => this._state === "running",
390
+ };
391
+ }
392
+ }
393
+ /** One attempt, from claim to completion. */
394
+ class AttemptRun {
395
+ worker;
396
+ h;
397
+ c;
398
+ id;
399
+ done;
400
+ ref;
401
+ ac = new AbortController();
402
+ hbStop = new AbortController();
403
+ cause;
404
+ graceMs;
405
+ graceElapsed;
406
+ /** Why the last {@link call} returned. */
407
+ ended = "ok";
408
+ constructor(worker, h, c) {
409
+ this.worker = worker;
410
+ this.h = h;
411
+ this.c = c;
412
+ this.ref = worker.refOf(c);
413
+ this.id = this.ref.attemptId;
414
+ this.graceMs = c.cancelGrace ? durationMs(c.cancelGrace) : 10_000;
415
+ this.graceElapsed = deferred();
416
+ this.graceElapsed.promise.catch(() => undefined);
417
+ this.done = this.execute().catch((err) => {
418
+ this.worker.internals.log.error("Attempt failed inside the SDK", { attempt: this.id, error: message(err) });
419
+ });
420
+ }
421
+ /** Aborts the handler with `cause`, once; the grace starts now. */
422
+ abort(cause, why, graceMs = this.graceMs) {
423
+ if (this.cause) {
424
+ // A lost lease outranks a cancel or a deadline: nothing is reported.
425
+ if (cause === "lost")
426
+ this.cause = "lost";
427
+ return;
428
+ }
429
+ this.cause = cause;
430
+ this.graceMs = graceMs;
431
+ this.ac.abort(new AttemptAbortedError(cause, why));
432
+ void sleep(graceMs).then(() => this.graceElapsed.resolve());
433
+ }
434
+ /** Whether the attempt is no longer this worker's. */
435
+ lost() {
436
+ return this.cause === "lost";
437
+ }
438
+ shutdown(graceMs) {
439
+ this.abort("shutdown", "the worker is shutting down", graceMs);
440
+ }
441
+ get api() {
442
+ return this.worker.internals.api;
443
+ }
444
+ async execute() {
445
+ const heartbeats = this.heartbeat();
446
+ try {
447
+ await this.attempt();
448
+ }
449
+ finally {
450
+ this.hbStop.abort();
451
+ await heartbeats;
452
+ }
453
+ }
454
+ async attempt() {
455
+ const log = this.worker.internals.log;
456
+ // 1. Start: CLAIMED -> RUNNING. User code starts only after this.
457
+ const start = await this.call("StartAttempt", (signal) => this.api.startAttempt({ ref: this.ref }, { signal, timeoutMs: 15_000 }), {
458
+ until: () => !this.worker.internals.running() || this.cause !== undefined,
459
+ });
460
+ if (start === undefined) {
461
+ // Lost, refused or cancelled: report nothing. Stopping: hand it back.
462
+ if (this.ended === "until" && this.cause !== "lost" && this.cause !== "cancelled")
463
+ await this.handBack();
464
+ return;
465
+ }
466
+ const timeoutMs = this.c.timeout ? durationMs(this.c.timeout) : 0;
467
+ const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : start.deadline ? timestampMs(start.deadline) : undefined;
468
+ let deadlineTimer;
469
+ if (deadline !== undefined) {
470
+ deadlineTimer = setTimeout(() => this.abort("deadline", "the attempt's deadline passed"), Math.max(0, deadline - Date.now()));
471
+ }
472
+ // 2. Logs.
473
+ const shipper = new LogShipper((seq, data, final, signal) => this.api.appendLogs({ ref: this.ref, seq, data, final }, { signal, timeoutMs: 15_000 }), {
474
+ maxBytes: Number(this.c.logMaxBytes),
475
+ onLost: () => this.abort("lost", "the attempt is no longer this worker's"),
476
+ onRetry: (err) => log.debug("AppendLogs failed", { attempt: this.id, error: message(err) }),
477
+ });
478
+ // 3. Run the handler.
479
+ let outcome;
480
+ try {
481
+ outcome = await this.invoke(shipper, deadline);
482
+ }
483
+ finally {
484
+ if (deadlineTimer)
485
+ clearTimeout(deadlineTimer);
486
+ }
487
+ if (outcome.kind === "lost" || this.lost()) {
488
+ shipper.stop();
489
+ return;
490
+ }
491
+ // 4. Close the log, then complete.
492
+ await shipper.close(outcome.termination === Termination.EXITED ? "" : `[pipelines] ${outcome.message}\n`, 10_000);
493
+ if (this.lost())
494
+ return;
495
+ const done = await this.call("CompleteAttempt", (signal) => this.api.completeAttempt({
496
+ ref: this.ref,
497
+ termination: outcome.termination,
498
+ ...(outcome.exitCode !== undefined ? { exitCode: outcome.exitCode } : {}),
499
+ kv: outcome.kv ?? {},
500
+ message: outcome.message,
501
+ handlerFailureKind: outcome.termination === Termination.HANDLER_ERROR ? (outcome.failureKind ?? HandlerFailureKind.UNSPECIFIED) : HandlerFailureKind.UNSPECIFIED,
502
+ }, { signal, timeoutMs: 15_000 }), { timeoutMs: this.cause === "shutdown" ? 10_000 : this.worker.internals.completeTimeoutMs });
503
+ if (done === undefined && !this.lost()) {
504
+ log.warn("CompleteAttempt was not delivered; the lease will lapse and the platform will abandon the attempt", { attempt: this.id });
505
+ }
506
+ }
507
+ async invoke(shipper, deadline) {
508
+ const def = this.h.def;
509
+ const logger = attemptLogger(shipper);
510
+ const info = {
511
+ id: this.id,
512
+ runId: this.c.runId,
513
+ nodeId: this.c.nodeId,
514
+ number: this.c.number,
515
+ claimEpoch: this.ref.claimEpoch,
516
+ tenant: this.c.tenant,
517
+ deadline: deadline !== undefined ? new Date(deadline) : undefined,
518
+ resource: this.c.resource ? { kind: this.c.resource.kind, id: this.c.resource.id, json: this.c.resource.json } : undefined,
519
+ attestedFacts: this.c.attestedFacts,
520
+ variables: this.c.variables,
521
+ };
522
+ const settled = (async () => {
523
+ const input = def.parseInput ? def.parseInput(this.c.inputs ?? {}) : (this.c.inputs ?? {});
524
+ const ctx = { input, signal: this.ac.signal, attempt: info, log: logger };
525
+ return (await def.run(ctx));
526
+ })().then((value) => ({ ok: true, value }), (error) => ({ ok: false, error }));
527
+ const result = await Promise.race([settled, this.graceElapsed.promise.then(() => undefined)]);
528
+ const cause = this.cause;
529
+ if (cause === "lost")
530
+ return { kind: "lost" };
531
+ if (result === undefined) {
532
+ // Aborted, and the handler did not return within the grace. JavaScript
533
+ // cannot stop it: the slot is freed and anything it does later is
534
+ // ignored.
535
+ this.worker.internals.log.warn("Handler did not stop within its grace after an abort", { attempt: this.id, cause });
536
+ return byCause(cause, "the handler did not stop within its grace");
537
+ }
538
+ if (result.ok) {
539
+ if (cause === "deadline")
540
+ return byCause(cause, "the handler returned after its deadline");
541
+ const kv = outputs(result.value);
542
+ // Outputs that cannot be sent will not improve on a retry.
543
+ if (typeof kv === "string") {
544
+ return { kind: "complete", termination: Termination.HANDLER_ERROR, message: clip(kv), failureKind: HandlerFailureKind.PERMANENT };
545
+ }
546
+ return { kind: "complete", termination: Termination.EXITED, exitCode: 0, kv, message: "" };
547
+ }
548
+ if (cause)
549
+ return byCause(cause, `the handler stopped: ${message(result.error)}`);
550
+ const failureKind = handlerFailureKind(result.error);
551
+ const label = failureKind === HandlerFailureKind.PERMANENT ? " (permanent)" : failureKind === HandlerFailureKind.TRANSIENT ? " (transient)" : "";
552
+ logger.error(`handler failed${label}: ${message(result.error)}`);
553
+ return { kind: "complete", termination: Termination.HANDLER_ERROR, message: clip(message(result.error)), failureKind };
554
+ }
555
+ /**
556
+ * Hands an unstarted attempt back as the worker stops. If the server says
557
+ * it is already RUNNING (a StartAttempt that landed but whose reply was
558
+ * lost), no handler ran, and it is completed as interrupted.
559
+ */
560
+ async handBack() {
561
+ const state = await this.worker.release(this.ref, "worker shutting down");
562
+ if (state !== AttemptState.RUNNING)
563
+ return;
564
+ await this.call("CompleteAttempt", (signal) => this.api.completeAttempt({ ref: this.ref, termination: Termination.INTERRUPTED, message: "the worker shut down before the handler started" }, { signal, timeoutMs: 10_000 }), { timeoutMs: 10_000 });
565
+ }
566
+ /**
567
+ * Renews the lease every heartbeat interval from the claim until the
568
+ * attempt completes. A cancel aborts the handler; `failed_precondition`,
569
+ * an auth error, or the lease running out locally without a renewal loses
570
+ * the attempt.
571
+ */
572
+ async heartbeat() {
573
+ const intervalMs = this.c.heartbeatInterval ? Math.max(durationMs(this.c.heartbeatInterval), 50) : 15_000;
574
+ const received = Date.now();
575
+ const fromServer = this.c.leaseExpiresAt ? timestampMs(this.c.leaseExpiresAt) - received : 0;
576
+ const leaseMs = fromServer > intervalMs ? fromServer : intervalMs * 4;
577
+ let renewedAt = received;
578
+ let failing = false;
579
+ const signal = this.hbStop.signal;
580
+ for (;;) {
581
+ if (!(await sleep(failing ? Math.min(intervalMs, 2_000) : intervalMs, signal)))
582
+ return;
583
+ const sentAt = Date.now();
584
+ try {
585
+ const res = await this.api.heartbeat({ ref: this.ref }, { signal, timeoutMs: Math.max(intervalMs, 5_000) });
586
+ renewedAt = sentAt;
587
+ failing = false;
588
+ if (res.cancel)
589
+ this.abort("cancelled", res.cancelReason || "the run was cancelled");
590
+ }
591
+ catch (err) {
592
+ if (signal.aborted)
593
+ return;
594
+ const kind = errorKind(err);
595
+ if (kind === "lost") {
596
+ this.abort("lost", "the attempt is no longer this worker's");
597
+ return;
598
+ }
599
+ if (kind === "auth") {
600
+ this.worker.fail(err);
601
+ this.abort("lost", "the worker's credential was refused");
602
+ return;
603
+ }
604
+ failing = true;
605
+ this.worker.internals.log.warn("Heartbeat failed", { attempt: this.id, error: message(err) });
606
+ if (Date.now() >= renewedAt + leaseMs) {
607
+ this.abort("lost", "the lease ran out without a renewal");
608
+ return;
609
+ }
610
+ }
611
+ }
612
+ }
613
+ /**
614
+ * Makes a call until it succeeds. Returns undefined when it cannot:
615
+ * `failed_precondition` (the attempt is lost: report nothing), a final
616
+ * code (`invalid_argument`: not retried), an auth error (the worker stops
617
+ * claiming), `until()` turning true, or `timeoutMs` passing.
618
+ */
619
+ async call(what, fn, opts = {}) {
620
+ const log = this.worker.internals.log;
621
+ const giveUpAt = opts.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : Infinity;
622
+ let delay = 0;
623
+ for (;;) {
624
+ if (opts.until?.()) {
625
+ this.ended = "until";
626
+ return undefined;
627
+ }
628
+ try {
629
+ const res = await fn(this.hbStop.signal);
630
+ this.ended = "ok";
631
+ return res;
632
+ }
633
+ catch (err) {
634
+ const kind = errorKind(err);
635
+ this.ended = kind === "transient" ? "timeout" : kind;
636
+ if (kind === "lost") {
637
+ this.abort("lost", `${what} answered that the attempt is no longer this worker's`);
638
+ return undefined;
639
+ }
640
+ if (kind === "auth") {
641
+ this.worker.fail(err);
642
+ return undefined;
643
+ }
644
+ if (kind === "final") {
645
+ log.error(`${what} was refused; not retrying`, { attempt: this.id, error: message(err) });
646
+ return undefined;
647
+ }
648
+ if (this.hbStop.signal.aborted && kind === "aborted")
649
+ return undefined;
650
+ delay = backoff(delay, 200, 5_000);
651
+ if (Date.now() + delay >= giveUpAt)
652
+ return undefined;
653
+ log.warn(`${what} failed; retrying`, { attempt: this.id, error: message(err) });
654
+ await sleep(delay);
655
+ }
656
+ }
657
+ }
658
+ }
659
+ /** The completion for a handler stopped by an abort. */
660
+ function byCause(cause, why) {
661
+ switch (cause) {
662
+ case "lost":
663
+ return { kind: "lost" };
664
+ case "cancelled":
665
+ return { kind: "complete", termination: Termination.CANCELLED, message: clip(`cancelled: ${why}`) };
666
+ case "deadline":
667
+ return { kind: "complete", termination: Termination.TIMED_OUT, message: clip(`timed out: ${why}`) };
668
+ case "shutdown":
669
+ return { kind: "complete", termination: Termination.INTERRUPTED, message: clip(`the worker shut down: ${why}`) };
670
+ }
671
+ }
672
+ /** Converts a handler's return value to kv outputs, or says why it cannot. */
673
+ function outputs(value) {
674
+ if (value === undefined || value === null)
675
+ return {};
676
+ if (typeof value !== "object" || Array.isArray(value))
677
+ return "the handler returned something other than an object of outputs";
678
+ const entries = Object.entries(value);
679
+ if (entries.length > 100)
680
+ return "the handler returned more than 100 outputs";
681
+ const kv = {};
682
+ for (const [k, v] of entries) {
683
+ if (k.length > 64 || !KV_KEY.test(k))
684
+ return `output ${JSON.stringify(k)} is not a valid output name`;
685
+ try {
686
+ kv[k] = fromJson(ValueSchema, v);
687
+ }
688
+ catch (err) {
689
+ return `output ${JSON.stringify(k)} is not JSON: ${message(err)}`;
690
+ }
691
+ }
692
+ return kv;
693
+ }
694
+ function attemptLogger(shipper) {
695
+ const line = (level, msg, fields) => {
696
+ let suffix = "";
697
+ if (fields && Object.keys(fields).length > 0) {
698
+ try {
699
+ suffix = ` ${JSON.stringify(fields)}`;
700
+ }
701
+ catch {
702
+ suffix = " [unserialisable fields]";
703
+ }
704
+ }
705
+ shipper.write(`${new Date().toISOString()} ${level} ${msg}${suffix}\n`);
706
+ };
707
+ return {
708
+ info: (m, f) => line("INFO", m, f),
709
+ warn: (m, f) => line("WARN", m, f),
710
+ error: (m, f) => line("ERROR", m, f),
711
+ write: (t) => shipper.write(t),
712
+ };
713
+ }
714
+ function message(err) {
715
+ if (err instanceof Error)
716
+ return err.message;
717
+ return String(err);
718
+ }
719
+ function clip(s) {
720
+ return s.length > MAX_MESSAGE ? `${s.slice(0, MAX_MESSAGE - 1)}…` : s;
721
+ }
722
+ function deferred() {
723
+ let resolve;
724
+ let reject;
725
+ const promise = new Promise((res, rej) => {
726
+ resolve = res;
727
+ reject = rej;
728
+ });
729
+ // A rejection nobody awaits must not crash the process.
730
+ promise.catch(() => undefined);
731
+ return { promise, resolve, reject };
732
+ }
733
+ //# sourceMappingURL=worker.js.map