@xeonr/pipelines-sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +267 -2
- package/dist/auth.d.ts +72 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +139 -0
- package/dist/auth.js.map +1 -0
- package/dist/client.d.ts +118 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +101 -0
- package/dist/client.js.map +1 -0
- package/dist/errors.d.ts +77 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +72 -0
- package/dist/errors.js.map +1 -0
- package/dist/gen/buf/validate/validate_pb.d.ts +4828 -0
- package/dist/gen/buf/validate/validate_pb.d.ts.map +1 -0
- package/dist/gen/buf/validate/validate_pb.js +357 -0
- package/dist/gen/buf/validate/validate_pb.js.map +1 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.d.ts +24 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.d.ts.map +1 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.js +37 -0
- package/dist/gen/gnostic/openapi/v3/annotations_pb.js.map +1 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.d.ts +2245 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.d.ts.map +1 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.js +410 -0
- package/dist/gen/gnostic/openapi/v3/openapiv3_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts +1031 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js +219 -0
- package/dist/gen/xeonr/pipelines/private/v1/attempt_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.d.ts +190 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.js +53 -0
- package/dist/gen/xeonr/pipelines/private/v1/runners_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts +1161 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js +269 -0
- package/dist/gen/xeonr/pipelines/private/v1/step_local_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts +915 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js +159 -0
- package/dist/gen/xeonr/pipelines/private/v1/worker_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts +550 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.js +453 -0
- package/dist/gen/xeonr/pipelines/v1/common_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts +760 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js +190 -0
- package/dist/gen/xeonr/pipelines/v1/definition_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts +800 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js +247 -0
- package/dist/gen/xeonr/pipelines/v1/pools_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts +895 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js +277 -0
- package/dist/gen/xeonr/pipelines/v1/registry_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts +1217 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js +204 -0
- package/dist/gen/xeonr/pipelines/v1/runs_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts +642 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js +181 -0
- package/dist/gen/xeonr/pipelines/v1/secrets_pb.js.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts +381 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js +145 -0
- package/dist/gen/xeonr/pipelines/v1/sinks_pb.js.map +1 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts +313 -0
- package/dist/gen/xeonr/platform/v1/options_pb.d.ts.map +1 -0
- package/dist/gen/xeonr/platform/v1/options_pb.js +235 -0
- package/dist/gen/xeonr/platform/v1/options_pb.js.map +1 -0
- package/dist/handler.d.ts +94 -0
- package/dist/handler.d.ts.map +1 -0
- package/dist/handler.js +18 -0
- package/dist/handler.js.map +1 -0
- package/dist/index.d.ts +29 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/time.d.ts +8 -0
- package/dist/internal/time.d.ts.map +1 -0
- package/dist/internal/time.js +25 -0
- package/dist/internal/time.js.map +1 -0
- package/dist/logship.d.ts +67 -0
- package/dist/logship.d.ts.map +1 -0
- package/dist/logship.js +176 -0
- package/dist/logship.js.map +1 -0
- package/dist/testing/index.d.ts +202 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +909 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/transport.d.ts +43 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/transport.js +44 -0
- package/dist/transport.js.map +1 -0
- package/dist/worker.d.ts +213 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +664 -0
- package/dist/worker.js.map +1 -0
- package/package.json +70 -3
package/dist/worker.js
ADDED
|
@@ -0,0 +1,664 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { fromJson } from "@bufbuild/protobuf";
|
|
3
|
+
import { durationFromMs, durationMs, timestampMs, ValueSchema } from "@bufbuild/protobuf/wkt";
|
|
4
|
+
import { Code, ConnectError, createClient } from "@connectrpc/connect";
|
|
5
|
+
import { AttemptAbortedError, errorKind, WorkerStartError } from "./errors.js";
|
|
6
|
+
import { Termination } from "./gen/xeonr/pipelines/v1/common_pb.js";
|
|
7
|
+
import { PoolKind, PoolOwnerKind, PoolsService } from "./gen/xeonr/pipelines/v1/pools_pb.js";
|
|
8
|
+
import { PackageKind, RegistryService, Visibility } from "./gen/xeonr/pipelines/v1/registry_pb.js";
|
|
9
|
+
import { WorkerService } from "./gen/xeonr/pipelines/private/v1/worker_pb.js";
|
|
10
|
+
import { LogShipper } from "./logship.js";
|
|
11
|
+
import { backoff, sleep } from "./internal/time.js";
|
|
12
|
+
import { transportOf } from "./transport.js";
|
|
13
|
+
/** Writes warnings and errors to stderr; drops debug and info. */
|
|
14
|
+
export const consoleLogger = {
|
|
15
|
+
debug: () => undefined,
|
|
16
|
+
info: () => undefined,
|
|
17
|
+
warn: (m, f) => console.warn(`[pipelines-sdk] ${m}`, f ?? ""),
|
|
18
|
+
error: (m, f) => console.error(`[pipelines-sdk] ${m}`, f ?? ""),
|
|
19
|
+
};
|
|
20
|
+
const KV_KEY = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
21
|
+
const MAX_MESSAGE = 2000;
|
|
22
|
+
/**
|
|
23
|
+
* An SDK worker: runs a product's handlers inside its own Node.js process.
|
|
24
|
+
*
|
|
25
|
+
* It authenticates as the product's Auth service account (`jwt-bearer`,
|
|
26
|
+
* scope `pipelines:worker:edit`) and may claim only its owner's handlers.
|
|
27
|
+
* It long-polls `ClaimAttempts` for its handlers within its concurrency
|
|
28
|
+
* limits, starts each attempt, renews the lease by heartbeat while the
|
|
29
|
+
* handler runs, ships the handler's log, and completes the attempt with its
|
|
30
|
+
* kv outputs or the right termination. Retries, delays, timeouts and leases
|
|
31
|
+
* are the platform's.
|
|
32
|
+
*
|
|
33
|
+
* ```ts
|
|
34
|
+
* const worker = new Worker({ baseUrl, token, tenant, pool: { name: "app" } });
|
|
35
|
+
* worker.handle(resizeImage);
|
|
36
|
+
* await worker.start();
|
|
37
|
+
* process.on("SIGTERM", () => void worker.stop());
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export class Worker {
|
|
41
|
+
/** This process's worker id. */
|
|
42
|
+
workerId;
|
|
43
|
+
opts;
|
|
44
|
+
log;
|
|
45
|
+
api;
|
|
46
|
+
registry;
|
|
47
|
+
pools;
|
|
48
|
+
handlers = new Map();
|
|
49
|
+
limit;
|
|
50
|
+
active = 0;
|
|
51
|
+
reserved = 0;
|
|
52
|
+
poolId = "";
|
|
53
|
+
_state = "idle";
|
|
54
|
+
claims = new Set();
|
|
55
|
+
attempts = new Map();
|
|
56
|
+
claimDelay = 0;
|
|
57
|
+
claimRetryAt = 0;
|
|
58
|
+
claimRetryTimer;
|
|
59
|
+
fatal;
|
|
60
|
+
stopped;
|
|
61
|
+
constructor(opts) {
|
|
62
|
+
this.opts = opts;
|
|
63
|
+
this.log = opts.logger ?? consoleLogger;
|
|
64
|
+
this.workerId = opts.workerId ?? randomUUID();
|
|
65
|
+
this.limit = opts.concurrency ?? 10;
|
|
66
|
+
if (!Number.isInteger(this.limit) || this.limit < 1 || this.limit > 256) {
|
|
67
|
+
throw new RangeError("Worker: concurrency must be an integer from 1 to 256");
|
|
68
|
+
}
|
|
69
|
+
const transport = transportOf(opts);
|
|
70
|
+
this.api = createClient(WorkerService, transport);
|
|
71
|
+
this.registry = createClient(RegistryService, transport);
|
|
72
|
+
this.pools = createClient(PoolsService, transport);
|
|
73
|
+
this.stopped = deferred();
|
|
74
|
+
}
|
|
75
|
+
/** Where the worker is in its life. */
|
|
76
|
+
get state() {
|
|
77
|
+
return this._state;
|
|
78
|
+
}
|
|
79
|
+
/** The `sdk` pool's id, once started. */
|
|
80
|
+
get pool() {
|
|
81
|
+
return this.poolId;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Registers a handler. Call before {@link start}. A handler is claimed by
|
|
85
|
+
* name; its version and manifest are published at boot.
|
|
86
|
+
*/
|
|
87
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any handler shape may be registered
|
|
88
|
+
handle(def) {
|
|
89
|
+
if (this._state !== "idle")
|
|
90
|
+
throw new Error("Worker.handle: register handlers before start()");
|
|
91
|
+
if (this.handlers.has(def.name))
|
|
92
|
+
throw new Error(`Worker.handle: ${def.name} is registered twice`);
|
|
93
|
+
if (this.handlers.size >= 256)
|
|
94
|
+
throw new Error("Worker.handle: at most 256 handlers per worker");
|
|
95
|
+
this.handlers.set(def.name, { def, limit: Math.min(def.concurrency ?? this.limit, this.limit), active: 0, reserved: 0 });
|
|
96
|
+
return this;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Boots the worker: publishes the handlers and declares the pool (unless
|
|
100
|
+
* `register` is false), then starts claiming. Resolves once claiming has
|
|
101
|
+
* begun; rejects with a {@link WorkerStartError} when registration fails.
|
|
102
|
+
*/
|
|
103
|
+
async start() {
|
|
104
|
+
if (this._state !== "idle")
|
|
105
|
+
throw new Error(`Worker.start: the worker is ${this._state}`);
|
|
106
|
+
if (this.handlers.size === 0)
|
|
107
|
+
throw new WorkerStartError("Worker.start: no handlers are registered");
|
|
108
|
+
this._state = "starting";
|
|
109
|
+
try {
|
|
110
|
+
this.poolId = this.opts.register === false ? this.requirePoolId() : await this.register();
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
this._state = "idle";
|
|
114
|
+
throw err instanceof WorkerStartError ? err : new WorkerStartError(`Worker.start: ${message(err)}`, { cause: err });
|
|
115
|
+
}
|
|
116
|
+
this._state = "running";
|
|
117
|
+
this.log.info("Worker started", { workerId: this.workerId, pool: this.poolId, handlers: [...this.handlers.keys()] });
|
|
118
|
+
this.pump();
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Stops gracefully: claims nothing more, lets in-flight attempts finish
|
|
122
|
+
* within `timeoutMs`, then aborts the rest (reason `shutdown`) and
|
|
123
|
+
* completes them as a worker failure after `graceMs`. Attempts claimed but
|
|
124
|
+
* not yet started are not started; their leases lapse and they are offered
|
|
125
|
+
* again with no retry spent.
|
|
126
|
+
*/
|
|
127
|
+
async stop(opts = {}) {
|
|
128
|
+
if (this._state === "stopped")
|
|
129
|
+
return;
|
|
130
|
+
if (this._state === "idle" || this._state === "starting") {
|
|
131
|
+
this._state = "stopped";
|
|
132
|
+
this.stopped.resolve();
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
const timeoutMs = opts.timeoutMs ?? 30_000;
|
|
136
|
+
const graceMs = opts.graceMs ?? 5_000;
|
|
137
|
+
this._state = "stopping";
|
|
138
|
+
this.haltClaims();
|
|
139
|
+
const settled = () => Promise.all([...this.attempts.values()].map((a) => a.done));
|
|
140
|
+
await Promise.race([settled(), sleep(timeoutMs)]);
|
|
141
|
+
if (this.attempts.size > 0) {
|
|
142
|
+
this.log.warn("Shutdown timeout passed; aborting in-flight handlers", { attempts: this.attempts.size });
|
|
143
|
+
for (const a of this.attempts.values())
|
|
144
|
+
a.shutdown(graceMs);
|
|
145
|
+
await Promise.race([settled(), sleep(graceMs + 15_000)]);
|
|
146
|
+
}
|
|
147
|
+
this.finish();
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Settles when the worker has stopped: resolves after {@link stop}, and
|
|
151
|
+
* rejects with the error that stopped it claiming (see `onError`).
|
|
152
|
+
*/
|
|
153
|
+
wait() {
|
|
154
|
+
return this.stopped.promise;
|
|
155
|
+
}
|
|
156
|
+
/** What the worker is doing now. */
|
|
157
|
+
stats() {
|
|
158
|
+
const handlers = {};
|
|
159
|
+
for (const [name, h] of this.handlers)
|
|
160
|
+
handlers[name] = h.active;
|
|
161
|
+
return { state: this._state, active: this.active, handlers, claims: this.claims.size };
|
|
162
|
+
}
|
|
163
|
+
requirePoolId() {
|
|
164
|
+
if (!this.opts.pool.id)
|
|
165
|
+
throw new WorkerStartError("Worker.start: pool.id is required when register is false");
|
|
166
|
+
return this.opts.pool.id;
|
|
167
|
+
}
|
|
168
|
+
async register() {
|
|
169
|
+
for (const { def } of this.handlers.values()) {
|
|
170
|
+
try {
|
|
171
|
+
await this.registry.publishVersion({
|
|
172
|
+
name: def.name,
|
|
173
|
+
version: def.version,
|
|
174
|
+
tenant: this.opts.tenant,
|
|
175
|
+
visibility: this.opts.visibility ?? Visibility.PRIVATE,
|
|
176
|
+
manifest: {
|
|
177
|
+
kind: PackageKind.WORKER,
|
|
178
|
+
description: def.description ?? "",
|
|
179
|
+
inputs: def.inputs ?? [],
|
|
180
|
+
outputs: def.outputs ?? [],
|
|
181
|
+
resourceKinds: def.resourceKinds ?? [],
|
|
182
|
+
readmeMd: def.readmeMd ?? "",
|
|
183
|
+
},
|
|
184
|
+
}, { timeoutMs: 30_000 });
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
if (ConnectError.from(err).code === Code.AlreadyExists) {
|
|
188
|
+
throw new WorkerStartError(`Worker.start: ${def.name}@${def.version} is already published with a different manifest; bump the handler's version`, { cause: err });
|
|
189
|
+
}
|
|
190
|
+
throw new WorkerStartError(`Worker.start: publishing ${def.name}@${def.version} failed: ${message(err)}`, { cause: err });
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
const res = await this.pools.setPool({
|
|
194
|
+
pool: {
|
|
195
|
+
...(this.opts.pool.id ? { id: this.opts.pool.id } : {}),
|
|
196
|
+
owner: { kind: PoolOwnerKind.TENANT, tenant: this.opts.tenant },
|
|
197
|
+
name: this.opts.pool.name,
|
|
198
|
+
kind: PoolKind.SDK,
|
|
199
|
+
tier: this.opts.pool.tier ?? "",
|
|
200
|
+
handlers: [...this.handlers.keys()],
|
|
201
|
+
capacityPerInstance: this.limit,
|
|
202
|
+
},
|
|
203
|
+
}, { timeoutMs: 30_000 });
|
|
204
|
+
if (!res.pool?.id)
|
|
205
|
+
throw new WorkerStartError("Worker.start: SetPool answered without a pool id");
|
|
206
|
+
return res.pool.id;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Starts claim long-polls while there is unreserved capacity. Each poll
|
|
210
|
+
* reserves its `free_slots` from the overall limit and from every handler
|
|
211
|
+
* it names, so no answer can take the worker past a limit: a poll asks for
|
|
212
|
+
* at most the smallest free count among the handlers it names, and a
|
|
213
|
+
* handler with no free slot is left out of new polls until one frees.
|
|
214
|
+
*/
|
|
215
|
+
pump() {
|
|
216
|
+
while (this._state === "running" && this.fatal === undefined && this.claims.size < (this.opts.maxConcurrentClaims ?? 4)) {
|
|
217
|
+
const wait = this.claimRetryAt - Date.now();
|
|
218
|
+
if (wait > 0) {
|
|
219
|
+
this.claimRetryTimer ??= setTimeout(() => {
|
|
220
|
+
this.claimRetryTimer = undefined;
|
|
221
|
+
this.pump();
|
|
222
|
+
}, wait);
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
const overall = this.limit - this.active - this.reserved;
|
|
226
|
+
if (overall <= 0)
|
|
227
|
+
return;
|
|
228
|
+
const eligible = [...this.handlers.values()].filter((h) => h.limit - h.active - h.reserved > 0);
|
|
229
|
+
if (eligible.length === 0)
|
|
230
|
+
return;
|
|
231
|
+
const slots = Math.min(overall, 256, ...eligible.map((h) => h.limit - h.active - h.reserved));
|
|
232
|
+
this.reserved += slots;
|
|
233
|
+
for (const h of eligible)
|
|
234
|
+
h.reserved += slots;
|
|
235
|
+
void this.claim(eligible, slots);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
async claim(eligible, slots) {
|
|
239
|
+
const ac = new AbortController();
|
|
240
|
+
this.claims.add(ac);
|
|
241
|
+
let taken = 0;
|
|
242
|
+
const takenBy = new Map();
|
|
243
|
+
try {
|
|
244
|
+
const res = await this.api.claimAttempts({
|
|
245
|
+
freeSlots: slots,
|
|
246
|
+
selector: { case: "handlers", value: { handlers: eligible.map((h) => h.def.name), workerId: this.workerId, poolId: this.poolId } },
|
|
247
|
+
wait: durationFromMs(Math.min(this.opts.claimWaitMs ?? 30_000, 30_000)),
|
|
248
|
+
}, { signal: ac.signal, timeoutMs: Math.min(this.opts.claimWaitMs ?? 30_000, 30_000) + 15_000 });
|
|
249
|
+
this.claimDelay = 0;
|
|
250
|
+
for (const c of res.attempts) {
|
|
251
|
+
const h = this.handlers.get(c.package?.name ?? "");
|
|
252
|
+
const n = h ? (takenBy.get(h) ?? 0) : 0;
|
|
253
|
+
if (!c.ref || !h || !eligible.includes(h) || taken >= slots || n >= slots || this._state !== "running") {
|
|
254
|
+
// More than was asked for, or for a handler not asked for: not
|
|
255
|
+
// started, so its lease lapses and it is offered again with no
|
|
256
|
+
// retry spent.
|
|
257
|
+
this.log.warn("Claim answered an attempt the worker did not ask for; leaving it to lapse", {
|
|
258
|
+
attempt: c.ref?.attemptId,
|
|
259
|
+
handler: c.package?.name,
|
|
260
|
+
});
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
taken++;
|
|
264
|
+
takenBy.set(h, n + 1);
|
|
265
|
+
this.reserved--;
|
|
266
|
+
h.reserved--;
|
|
267
|
+
this.active++;
|
|
268
|
+
h.active++;
|
|
269
|
+
this.run(h, c);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
catch (err) {
|
|
273
|
+
const kind = errorKind(err);
|
|
274
|
+
if (ac.signal.aborted || this._state !== "running") {
|
|
275
|
+
// Stopping: a claim reply lost here costs nothing (its lease lapses).
|
|
276
|
+
}
|
|
277
|
+
else if (kind === "auth" || kind === "final") {
|
|
278
|
+
this.fail(err);
|
|
279
|
+
}
|
|
280
|
+
else {
|
|
281
|
+
this.claimDelay = backoff(this.claimDelay, 500, 30_000);
|
|
282
|
+
this.claimRetryAt = Date.now() + this.claimDelay;
|
|
283
|
+
this.log.warn("ClaimAttempts failed; retrying", { error: message(err), retryInMs: Math.round(this.claimDelay) });
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
finally {
|
|
287
|
+
this.reserved -= slots - taken;
|
|
288
|
+
for (const h of eligible)
|
|
289
|
+
h.reserved -= slots - (takenBy.get(h) ?? 0);
|
|
290
|
+
this.claims.delete(ac);
|
|
291
|
+
this.pump();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
run(h, c) {
|
|
295
|
+
const run = new AttemptRun(this, h, c);
|
|
296
|
+
this.attempts.set(run.id, run);
|
|
297
|
+
void run.done.finally(() => {
|
|
298
|
+
this.attempts.delete(run.id);
|
|
299
|
+
this.active--;
|
|
300
|
+
h.active--;
|
|
301
|
+
if (this._state === "running")
|
|
302
|
+
this.pump();
|
|
303
|
+
else if (this.fatal !== undefined && this.attempts.size === 0)
|
|
304
|
+
this.finish();
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
/** Stops claiming for good after an error that will not pass. */
|
|
308
|
+
fail(err) {
|
|
309
|
+
if (this.fatal !== undefined)
|
|
310
|
+
return;
|
|
311
|
+
this.fatal = err ?? new Error("unknown worker failure");
|
|
312
|
+
this.log.error("Worker stopped claiming", { error: message(err) });
|
|
313
|
+
this.haltClaims();
|
|
314
|
+
try {
|
|
315
|
+
this.opts.onError?.(err);
|
|
316
|
+
}
|
|
317
|
+
catch (e) {
|
|
318
|
+
this.log.error("onError threw", { error: message(e) });
|
|
319
|
+
}
|
|
320
|
+
if (this._state === "running")
|
|
321
|
+
this._state = "stopping";
|
|
322
|
+
if (this.attempts.size === 0)
|
|
323
|
+
this.finish();
|
|
324
|
+
}
|
|
325
|
+
haltClaims() {
|
|
326
|
+
for (const ac of this.claims)
|
|
327
|
+
ac.abort();
|
|
328
|
+
if (this.claimRetryTimer)
|
|
329
|
+
clearTimeout(this.claimRetryTimer);
|
|
330
|
+
this.claimRetryTimer = undefined;
|
|
331
|
+
}
|
|
332
|
+
finish() {
|
|
333
|
+
if (this._state === "stopped")
|
|
334
|
+
return;
|
|
335
|
+
this._state = "stopped";
|
|
336
|
+
if (this.fatal !== undefined)
|
|
337
|
+
this.stopped.reject(this.fatal);
|
|
338
|
+
else
|
|
339
|
+
this.stopped.resolve();
|
|
340
|
+
}
|
|
341
|
+
/** @internal */
|
|
342
|
+
get internals() {
|
|
343
|
+
return {
|
|
344
|
+
api: this.api,
|
|
345
|
+
log: this.log,
|
|
346
|
+
completeTimeoutMs: this.opts.completeTimeoutMs ?? 60_000,
|
|
347
|
+
running: () => this._state === "running",
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
/** One attempt, from claim to completion. */
|
|
352
|
+
class AttemptRun {
|
|
353
|
+
worker;
|
|
354
|
+
h;
|
|
355
|
+
c;
|
|
356
|
+
id;
|
|
357
|
+
done;
|
|
358
|
+
ref;
|
|
359
|
+
ac = new AbortController();
|
|
360
|
+
hbStop = new AbortController();
|
|
361
|
+
cause;
|
|
362
|
+
graceMs;
|
|
363
|
+
graceElapsed;
|
|
364
|
+
constructor(worker, h, c) {
|
|
365
|
+
this.worker = worker;
|
|
366
|
+
this.h = h;
|
|
367
|
+
this.c = c;
|
|
368
|
+
this.ref = c.ref;
|
|
369
|
+
this.id = this.ref.attemptId;
|
|
370
|
+
this.graceMs = c.cancelGrace ? durationMs(c.cancelGrace) : 10_000;
|
|
371
|
+
this.graceElapsed = deferred();
|
|
372
|
+
this.graceElapsed.promise.catch(() => undefined);
|
|
373
|
+
this.done = this.execute().catch((err) => {
|
|
374
|
+
this.worker.internals.log.error("Attempt failed inside the SDK", { attempt: this.id, error: message(err) });
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
/** Aborts the handler with `cause`, once; the grace starts now. */
|
|
378
|
+
abort(cause, why, graceMs = this.graceMs) {
|
|
379
|
+
if (this.cause) {
|
|
380
|
+
// A lost lease outranks a cancel or a deadline: nothing is reported.
|
|
381
|
+
if (cause === "lost")
|
|
382
|
+
this.cause = "lost";
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
this.cause = cause;
|
|
386
|
+
this.graceMs = graceMs;
|
|
387
|
+
this.ac.abort(new AttemptAbortedError(cause, why));
|
|
388
|
+
void sleep(graceMs).then(() => this.graceElapsed.resolve());
|
|
389
|
+
}
|
|
390
|
+
/** Whether the attempt is no longer this worker's. */
|
|
391
|
+
lost() {
|
|
392
|
+
return this.cause === "lost";
|
|
393
|
+
}
|
|
394
|
+
shutdown(graceMs) {
|
|
395
|
+
this.abort("shutdown", "the worker is shutting down", graceMs);
|
|
396
|
+
}
|
|
397
|
+
get api() {
|
|
398
|
+
return this.worker.internals.api;
|
|
399
|
+
}
|
|
400
|
+
async execute() {
|
|
401
|
+
const heartbeats = this.heartbeat();
|
|
402
|
+
try {
|
|
403
|
+
await this.attempt();
|
|
404
|
+
}
|
|
405
|
+
finally {
|
|
406
|
+
this.hbStop.abort();
|
|
407
|
+
await heartbeats;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
async attempt() {
|
|
411
|
+
const log = this.worker.internals.log;
|
|
412
|
+
// 1. Start: CLAIMED -> RUNNING. User code starts only after this.
|
|
413
|
+
const start = await this.call("StartAttempt", (signal) => this.api.startAttempt({ ref: this.ref }, { signal, timeoutMs: 15_000 }), {
|
|
414
|
+
until: () => !this.worker.internals.running() || this.cause !== undefined,
|
|
415
|
+
});
|
|
416
|
+
if (start === undefined)
|
|
417
|
+
return; // lost, refused, or stopping: report nothing
|
|
418
|
+
const timeoutMs = this.c.timeout ? durationMs(this.c.timeout) : 0;
|
|
419
|
+
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : start.deadline ? timestampMs(start.deadline) : undefined;
|
|
420
|
+
let deadlineTimer;
|
|
421
|
+
if (deadline !== undefined) {
|
|
422
|
+
deadlineTimer = setTimeout(() => this.abort("deadline", "the attempt's deadline passed"), Math.max(0, deadline - Date.now()));
|
|
423
|
+
}
|
|
424
|
+
// 2. Logs.
|
|
425
|
+
const shipper = new LogShipper((seq, data, final, signal) => this.api.appendLogs({ ref: this.ref, seq, data, final }, { signal, timeoutMs: 15_000 }), {
|
|
426
|
+
maxBytes: Number(this.c.logMaxBytes),
|
|
427
|
+
onLost: () => this.abort("lost", "the attempt is no longer this worker's"),
|
|
428
|
+
onRetry: (err) => log.debug("AppendLogs failed", { attempt: this.id, error: message(err) }),
|
|
429
|
+
});
|
|
430
|
+
// 3. Run the handler.
|
|
431
|
+
let outcome;
|
|
432
|
+
try {
|
|
433
|
+
outcome = await this.invoke(shipper, deadline);
|
|
434
|
+
}
|
|
435
|
+
finally {
|
|
436
|
+
if (deadlineTimer)
|
|
437
|
+
clearTimeout(deadlineTimer);
|
|
438
|
+
}
|
|
439
|
+
if (outcome.kind === "lost" || this.lost()) {
|
|
440
|
+
shipper.stop();
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
// 4. Close the log, then complete.
|
|
444
|
+
await shipper.close(outcome.termination === Termination.EXITED ? "" : `[pipelines] ${outcome.message}\n`, 10_000);
|
|
445
|
+
if (this.lost())
|
|
446
|
+
return;
|
|
447
|
+
const done = await this.call("CompleteAttempt", (signal) => this.api.completeAttempt({
|
|
448
|
+
ref: this.ref,
|
|
449
|
+
termination: outcome.termination,
|
|
450
|
+
...(outcome.exitCode !== undefined ? { exitCode: outcome.exitCode } : {}),
|
|
451
|
+
kv: outcome.kv ?? {},
|
|
452
|
+
message: outcome.message,
|
|
453
|
+
}, { signal, timeoutMs: 15_000 }), { timeoutMs: this.cause === "shutdown" ? 10_000 : this.worker.internals.completeTimeoutMs });
|
|
454
|
+
if (done === undefined && !this.lost()) {
|
|
455
|
+
log.warn("CompleteAttempt was not delivered; the lease will lapse and the platform will abandon the attempt", { attempt: this.id });
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
async invoke(shipper, deadline) {
|
|
459
|
+
const def = this.h.def;
|
|
460
|
+
const logger = attemptLogger(shipper);
|
|
461
|
+
const info = {
|
|
462
|
+
id: this.id,
|
|
463
|
+
runId: this.c.runId,
|
|
464
|
+
nodeId: this.c.nodeId,
|
|
465
|
+
number: this.c.number,
|
|
466
|
+
claimEpoch: this.ref.claimEpoch,
|
|
467
|
+
tenant: this.c.tenant,
|
|
468
|
+
deadline: deadline !== undefined ? new Date(deadline) : undefined,
|
|
469
|
+
resource: this.c.resource ? { kind: this.c.resource.kind, id: this.c.resource.id, json: this.c.resource.json } : undefined,
|
|
470
|
+
attestedFacts: this.c.attestedFacts,
|
|
471
|
+
variables: this.c.variables,
|
|
472
|
+
};
|
|
473
|
+
const settled = (async () => {
|
|
474
|
+
const input = def.parseInput ? def.parseInput(this.c.inputs ?? {}) : (this.c.inputs ?? {});
|
|
475
|
+
const ctx = { input, signal: this.ac.signal, attempt: info, log: logger };
|
|
476
|
+
return (await def.run(ctx));
|
|
477
|
+
})().then((value) => ({ ok: true, value }), (error) => ({ ok: false, error }));
|
|
478
|
+
const result = await Promise.race([settled, this.graceElapsed.promise.then(() => undefined)]);
|
|
479
|
+
const cause = this.cause;
|
|
480
|
+
if (cause === "lost")
|
|
481
|
+
return { kind: "lost" };
|
|
482
|
+
if (result === undefined) {
|
|
483
|
+
// Aborted, and the handler did not return within the grace. JavaScript
|
|
484
|
+
// cannot stop it: the slot is freed and anything it does later is
|
|
485
|
+
// ignored.
|
|
486
|
+
this.worker.internals.log.warn("Handler did not stop within its grace after an abort", { attempt: this.id, cause });
|
|
487
|
+
return byCause(cause, "the handler did not stop within its grace");
|
|
488
|
+
}
|
|
489
|
+
if (result.ok) {
|
|
490
|
+
if (cause === "deadline")
|
|
491
|
+
return byCause(cause, "the handler returned after its deadline");
|
|
492
|
+
const kv = outputs(result.value);
|
|
493
|
+
if (typeof kv === "string")
|
|
494
|
+
return { kind: "complete", termination: Termination.HANDLER_ERROR, message: clip(kv) };
|
|
495
|
+
return { kind: "complete", termination: Termination.EXITED, exitCode: 0, kv, message: "" };
|
|
496
|
+
}
|
|
497
|
+
if (cause)
|
|
498
|
+
return byCause(cause, `the handler stopped: ${message(result.error)}`);
|
|
499
|
+
logger.error(`handler failed: ${message(result.error)}`);
|
|
500
|
+
return { kind: "complete", termination: Termination.HANDLER_ERROR, message: clip(message(result.error)) };
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Renews the lease every heartbeat interval from the claim until the
|
|
504
|
+
* attempt completes. A cancel aborts the handler; `failed_precondition`,
|
|
505
|
+
* an auth error, or the lease running out locally without a renewal loses
|
|
506
|
+
* the attempt.
|
|
507
|
+
*/
|
|
508
|
+
async heartbeat() {
|
|
509
|
+
const intervalMs = this.c.heartbeatInterval ? Math.max(durationMs(this.c.heartbeatInterval), 50) : 15_000;
|
|
510
|
+
const received = Date.now();
|
|
511
|
+
const fromServer = this.c.leaseExpiresAt ? timestampMs(this.c.leaseExpiresAt) - received : 0;
|
|
512
|
+
const leaseMs = fromServer > intervalMs ? fromServer : intervalMs * 4;
|
|
513
|
+
let renewedAt = received;
|
|
514
|
+
let failing = false;
|
|
515
|
+
const signal = this.hbStop.signal;
|
|
516
|
+
for (;;) {
|
|
517
|
+
if (!(await sleep(failing ? Math.min(intervalMs, 2_000) : intervalMs, signal)))
|
|
518
|
+
return;
|
|
519
|
+
const sentAt = Date.now();
|
|
520
|
+
try {
|
|
521
|
+
const res = await this.api.heartbeat({ ref: this.ref }, { signal, timeoutMs: Math.max(intervalMs, 5_000) });
|
|
522
|
+
renewedAt = sentAt;
|
|
523
|
+
failing = false;
|
|
524
|
+
if (res.cancel)
|
|
525
|
+
this.abort("cancelled", res.cancelReason || "the run was cancelled");
|
|
526
|
+
}
|
|
527
|
+
catch (err) {
|
|
528
|
+
if (signal.aborted)
|
|
529
|
+
return;
|
|
530
|
+
const kind = errorKind(err);
|
|
531
|
+
if (kind === "lost") {
|
|
532
|
+
this.abort("lost", "the attempt is no longer this worker's");
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
if (kind === "auth") {
|
|
536
|
+
this.worker.fail(err);
|
|
537
|
+
this.abort("lost", "the worker's credential was refused");
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
failing = true;
|
|
541
|
+
this.worker.internals.log.warn("Heartbeat failed", { attempt: this.id, error: message(err) });
|
|
542
|
+
if (Date.now() >= renewedAt + leaseMs) {
|
|
543
|
+
this.abort("lost", "the lease ran out without a renewal");
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Makes a call until it succeeds. Returns undefined when it cannot:
|
|
551
|
+
* `failed_precondition` (the attempt is lost: report nothing), a final
|
|
552
|
+
* code (`invalid_argument`: not retried), an auth error (the worker stops
|
|
553
|
+
* claiming), `until()` turning true, or `timeoutMs` passing.
|
|
554
|
+
*/
|
|
555
|
+
async call(what, fn, opts = {}) {
|
|
556
|
+
const log = this.worker.internals.log;
|
|
557
|
+
const giveUpAt = opts.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : Infinity;
|
|
558
|
+
let delay = 0;
|
|
559
|
+
for (;;) {
|
|
560
|
+
if (opts.until?.())
|
|
561
|
+
return undefined;
|
|
562
|
+
try {
|
|
563
|
+
return await fn(this.hbStop.signal);
|
|
564
|
+
}
|
|
565
|
+
catch (err) {
|
|
566
|
+
const kind = errorKind(err);
|
|
567
|
+
if (kind === "lost") {
|
|
568
|
+
this.abort("lost", `${what} answered that the attempt is no longer this worker's`);
|
|
569
|
+
return undefined;
|
|
570
|
+
}
|
|
571
|
+
if (kind === "auth") {
|
|
572
|
+
this.worker.fail(err);
|
|
573
|
+
return undefined;
|
|
574
|
+
}
|
|
575
|
+
if (kind === "final") {
|
|
576
|
+
log.error(`${what} was refused; not retrying`, { attempt: this.id, error: message(err) });
|
|
577
|
+
return undefined;
|
|
578
|
+
}
|
|
579
|
+
if (this.hbStop.signal.aborted && kind === "aborted")
|
|
580
|
+
return undefined;
|
|
581
|
+
delay = backoff(delay, 200, 5_000);
|
|
582
|
+
if (Date.now() + delay >= giveUpAt)
|
|
583
|
+
return undefined;
|
|
584
|
+
log.warn(`${what} failed; retrying`, { attempt: this.id, error: message(err) });
|
|
585
|
+
await sleep(delay);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
/** The completion for a handler stopped by an abort. */
|
|
591
|
+
function byCause(cause, why) {
|
|
592
|
+
switch (cause) {
|
|
593
|
+
case "lost":
|
|
594
|
+
return { kind: "lost" };
|
|
595
|
+
case "cancelled":
|
|
596
|
+
return { kind: "complete", termination: Termination.CANCELLED, message: clip(`cancelled: ${why}`) };
|
|
597
|
+
case "deadline":
|
|
598
|
+
return { kind: "complete", termination: Termination.TIMED_OUT, message: clip(`timed out: ${why}`) };
|
|
599
|
+
case "shutdown":
|
|
600
|
+
return { kind: "complete", termination: Termination.RUNNER_ERROR, message: clip(`the worker shut down: ${why}`) };
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
/** Converts a handler's return value to kv outputs, or says why it cannot. */
|
|
604
|
+
function outputs(value) {
|
|
605
|
+
if (value === undefined || value === null)
|
|
606
|
+
return {};
|
|
607
|
+
if (typeof value !== "object" || Array.isArray(value))
|
|
608
|
+
return "the handler returned something other than an object of outputs";
|
|
609
|
+
const entries = Object.entries(value);
|
|
610
|
+
if (entries.length > 100)
|
|
611
|
+
return "the handler returned more than 100 outputs";
|
|
612
|
+
const kv = {};
|
|
613
|
+
for (const [k, v] of entries) {
|
|
614
|
+
if (k.length > 64 || !KV_KEY.test(k))
|
|
615
|
+
return `output ${JSON.stringify(k)} is not a valid output name`;
|
|
616
|
+
try {
|
|
617
|
+
kv[k] = fromJson(ValueSchema, v);
|
|
618
|
+
}
|
|
619
|
+
catch (err) {
|
|
620
|
+
return `output ${JSON.stringify(k)} is not JSON: ${message(err)}`;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
return kv;
|
|
624
|
+
}
|
|
625
|
+
function attemptLogger(shipper) {
|
|
626
|
+
const line = (level, msg, fields) => {
|
|
627
|
+
let suffix = "";
|
|
628
|
+
if (fields && Object.keys(fields).length > 0) {
|
|
629
|
+
try {
|
|
630
|
+
suffix = ` ${JSON.stringify(fields)}`;
|
|
631
|
+
}
|
|
632
|
+
catch {
|
|
633
|
+
suffix = " [unserialisable fields]";
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
shipper.write(`${new Date().toISOString()} ${level} ${msg}${suffix}\n`);
|
|
637
|
+
};
|
|
638
|
+
return {
|
|
639
|
+
info: (m, f) => line("INFO", m, f),
|
|
640
|
+
warn: (m, f) => line("WARN", m, f),
|
|
641
|
+
error: (m, f) => line("ERROR", m, f),
|
|
642
|
+
write: (t) => shipper.write(t),
|
|
643
|
+
};
|
|
644
|
+
}
|
|
645
|
+
function message(err) {
|
|
646
|
+
if (err instanceof Error)
|
|
647
|
+
return err.message;
|
|
648
|
+
return String(err);
|
|
649
|
+
}
|
|
650
|
+
function clip(s) {
|
|
651
|
+
return s.length > MAX_MESSAGE ? `${s.slice(0, MAX_MESSAGE - 1)}…` : s;
|
|
652
|
+
}
|
|
653
|
+
function deferred() {
|
|
654
|
+
let resolve;
|
|
655
|
+
let reject;
|
|
656
|
+
const promise = new Promise((res, rej) => {
|
|
657
|
+
resolve = res;
|
|
658
|
+
reject = rej;
|
|
659
|
+
});
|
|
660
|
+
// A rejection nobody awaits must not crash the process.
|
|
661
|
+
promise.catch(() => undefined);
|
|
662
|
+
return { promise, resolve, reject };
|
|
663
|
+
}
|
|
664
|
+
//# sourceMappingURL=worker.js.map
|