@deepnoodle/mobius 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -45
- package/dist/api/index.d.ts +117 -336
- package/dist/api/index.d.ts.map +1 -1
- package/dist/api/schema.d.ts +4620 -12987
- package/dist/api/schema.d.ts.map +1 -1
- package/dist/client.d.ts +64 -151
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +164 -368
- package/dist/client.js.map +1 -1
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/signing.d.ts +43 -0
- package/dist/signing.d.ts.map +1 -0
- package/dist/signing.js +101 -0
- package/dist/signing.js.map +1 -0
- package/dist/webhook.d.ts +5 -17
- package/dist/webhook.d.ts.map +1 -1
- package/dist/webhook.js +38 -50
- package/dist/webhook.js.map +1 -1
- package/dist/worker.d.ts +41 -138
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +343 -463
- package/dist/worker.js.map +1 -1
- package/package.json +4 -4
package/dist/worker.js
CHANGED
|
@@ -1,37 +1,11 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { hostname as systemHostname } from "node:os";
|
|
3
|
-
import { AuthRevokedError,
|
|
3
|
+
import { AuthRevokedError, WorkerInstanceConflictError, } from "./client.js";
|
|
4
4
|
const CLOUD_RUN_METADATA_TIMEOUT_MS = 1000;
|
|
5
|
-
// Generated once per process and reused by both the system_hostname
|
|
6
|
-
// rung (as an 8-char suffix) and the generated_uuid rung (full value).
|
|
7
|
-
// worker_instance_id is process identity and must be stable across
|
|
8
|
-
// calls within the same boot — without the cache, a caller that
|
|
9
|
-
// resolved twice would observe two different IDs.
|
|
10
5
|
const BOOT_INSTANCE_ID = randomUUID();
|
|
11
|
-
/**
|
|
12
|
-
* Resolve a per-process `worker_instance_id` from the runtime
|
|
13
|
-
* environment. Order: explicit → Cloud Run K_REVISION + metadata →
|
|
14
|
-
* HOSTNAME env → FLY_MACHINE_ID → RAILWAY_REPLICA_ID →
|
|
15
|
-
* RENDER_INSTANCE_ID → `os.hostname()` suffixed with a per-boot
|
|
16
|
-
* random tag (laptops and bare metal) → generated UUID.
|
|
17
|
-
*
|
|
18
|
-
* The system_hostname rung carries a random suffix because
|
|
19
|
-
* `os.hostname()` identifies the host, not the process — two
|
|
20
|
-
* processes started on the same machine would otherwise auto-detect
|
|
21
|
-
* the same identifier and collide on the server's conflict detector.
|
|
22
|
-
* Set {@link WorkerConfig.workerInstanceId} explicitly only when a
|
|
23
|
-
* stable identity across restarts is desired (named singleton
|
|
24
|
-
* workers).
|
|
25
|
-
*
|
|
26
|
-
* The returned source is informational only — workers log it once at
|
|
27
|
-
* startup so operators can confirm the right platform was picked up.
|
|
28
|
-
*/
|
|
29
6
|
export async function resolveInstanceID(explicit) {
|
|
30
|
-
if (explicit)
|
|
31
|
-
|
|
32
|
-
if (trimmed)
|
|
33
|
-
return { id: trimmed, source: "configured" };
|
|
34
|
-
}
|
|
7
|
+
if (explicit?.trim())
|
|
8
|
+
return { id: explicit.trim(), source: "configured" };
|
|
35
9
|
const cloudRun = await cloudRunInstanceID();
|
|
36
10
|
if (cloudRun)
|
|
37
11
|
return { id: cloudRun, source: "cloud_run_revision_instance" };
|
|
@@ -50,21 +24,18 @@ export async function resolveInstanceID(explicit) {
|
|
|
50
24
|
try {
|
|
51
25
|
const host = systemHostname().trim();
|
|
52
26
|
if (host) {
|
|
53
|
-
|
|
54
|
-
|
|
27
|
+
return {
|
|
28
|
+
id: `${host}-${BOOT_INSTANCE_ID.replace(/-/g, "").slice(0, 8)}`,
|
|
29
|
+
source: "system_hostname",
|
|
30
|
+
};
|
|
55
31
|
}
|
|
56
32
|
}
|
|
57
33
|
catch {
|
|
58
|
-
// fall through
|
|
34
|
+
// fall through
|
|
59
35
|
}
|
|
60
36
|
return { id: BOOT_INSTANCE_ID, source: "generated_uuid" };
|
|
61
37
|
}
|
|
62
38
|
async function cloudRunInstanceID() {
|
|
63
|
-
// Returns null on any metadata-server failure rather than the bare
|
|
64
|
-
// revision — every replica sharing the same revision string would
|
|
65
|
-
// otherwise collapse onto one row and trip the conflict detector.
|
|
66
|
-
// The next strategy (HOSTNAME, which Cloud Run sets per-instance)
|
|
67
|
-
// takes over.
|
|
68
39
|
const revision = (process.env.K_REVISION ?? "").trim();
|
|
69
40
|
if (!revision)
|
|
70
41
|
return null;
|
|
@@ -86,495 +57,404 @@ async function cloudRunInstanceID() {
|
|
|
86
57
|
return null;
|
|
87
58
|
}
|
|
88
59
|
}
|
|
89
|
-
const silentLogger = {
|
|
90
|
-
info: () => { },
|
|
91
|
-
warn: () => { },
|
|
92
|
-
error: () => { },
|
|
93
|
-
};
|
|
60
|
+
const silentLogger = { info: () => { }, warn: () => { }, error: () => { } };
|
|
94
61
|
const defaultLogger = {
|
|
95
62
|
info: (...args) => console.log(...args),
|
|
96
63
|
warn: (...args) => console.warn(...args),
|
|
97
64
|
error: (...args) => console.error(...args),
|
|
98
65
|
};
|
|
99
|
-
/**
|
|
100
|
-
* Worker claims jobs from Mobius and dispatches each to the corresponding
|
|
101
|
-
* registered action function. A *job* is a single action invocation on
|
|
102
|
-
* behalf of a workflow run; the backend owns the workflow engine.
|
|
103
|
-
*
|
|
104
|
-
* With `concurrency > 1` the worker holds up to N jobs in flight
|
|
105
|
-
* simultaneously, all reporting the same `worker_instance_id` and
|
|
106
|
-
* session token; the admin UI shows one row with a saturation bar
|
|
107
|
-
* rather than N independent rows.
|
|
108
|
-
*/
|
|
109
66
|
export class Worker {
|
|
110
|
-
constructor(client, config, actions) {
|
|
67
|
+
constructor(client, config = {}, actions) {
|
|
111
68
|
this.client = client;
|
|
112
|
-
this.
|
|
113
|
-
this.
|
|
114
|
-
this.
|
|
115
|
-
|
|
116
|
-
this.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
heartbeatIntervalMs: config.heartbeatIntervalMs,
|
|
125
|
-
eventQueueSize: config.eventQueueSize ?? 256,
|
|
126
|
-
eventBatchSize: config.eventBatchSize ?? 20,
|
|
127
|
-
};
|
|
128
|
-
this.logger = logger;
|
|
129
|
-
this.actions = actions ?? new Map();
|
|
130
|
-
this.sessionToken = randomUUID();
|
|
69
|
+
this.config = config;
|
|
70
|
+
this.actions = new Map();
|
|
71
|
+
this.generators = new Map();
|
|
72
|
+
this.sessionToken = "";
|
|
73
|
+
this.stopping = false;
|
|
74
|
+
this.stopController = new AbortController();
|
|
75
|
+
this.logger =
|
|
76
|
+
config.logger === null ? silentLogger : (config.logger ?? defaultLogger);
|
|
77
|
+
if (actions) {
|
|
78
|
+
for (const [name, fn] of actions)
|
|
79
|
+
this.actions.set(name, fn);
|
|
80
|
+
}
|
|
131
81
|
}
|
|
132
|
-
/** Register an action function under the given name. */
|
|
133
82
|
register(name, fn) {
|
|
134
83
|
this.actions.set(name, fn);
|
|
135
84
|
return this;
|
|
136
85
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
*
|
|
146
|
-
* - {@link stop} aborts the claim loop only — in-flight jobs continue
|
|
147
|
-
* to run to natural completion and the returned promise resolves
|
|
148
|
-
* once they all settle. This is graceful drain.
|
|
149
|
-
* - The caller's `signal`, when aborted, propagates into both the
|
|
150
|
-
* claim loop and every running action. This is the emergency-stop
|
|
151
|
-
* path (e.g. SIGTERM with a hard deadline).
|
|
152
|
-
*/
|
|
86
|
+
registerGenerator(provider, model, fn) {
|
|
87
|
+
this.generators.set(`${provider}/${model}`, fn);
|
|
88
|
+
return this;
|
|
89
|
+
}
|
|
90
|
+
stop() {
|
|
91
|
+
this.stopping = true;
|
|
92
|
+
this.stopController.abort();
|
|
93
|
+
}
|
|
153
94
|
async run(signal) {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
const callerSignal = signal;
|
|
160
|
-
const combined = callerSignal
|
|
161
|
-
? anySignal(callerSignal, this.abortController.signal)
|
|
162
|
-
: this.abortController.signal;
|
|
163
|
-
if (!this.instanceIDResolved) {
|
|
164
|
-
const { id, source } = await resolveInstanceID(this.config.workerInstanceId || undefined);
|
|
165
|
-
this.config.workerInstanceId = id;
|
|
166
|
-
this.instanceIDResolved = true;
|
|
167
|
-
this.logger.info(`[mobius] worker instance id ${id} (source: ${source})`);
|
|
168
|
-
}
|
|
169
|
-
this.logger.info(`[mobius] worker ${this.config.workerInstanceId} started (concurrency=${this.config.concurrency})`);
|
|
170
|
-
const inflight = new Set();
|
|
171
|
-
while (!combined.aborted) {
|
|
172
|
-
if (this.authRevoked)
|
|
173
|
-
break;
|
|
174
|
-
while (inflight.size >= this.config.concurrency && !combined.aborted) {
|
|
175
|
-
const aborted = abortAsPromise(combined);
|
|
176
|
-
try {
|
|
177
|
-
await Promise.race([...inflight, aborted.promise]);
|
|
178
|
-
}
|
|
179
|
-
finally {
|
|
180
|
-
aborted.cancel();
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
if (combined.aborted)
|
|
184
|
-
break;
|
|
185
|
-
let job;
|
|
95
|
+
const { id, source } = await resolveInstanceID(this.config.workerInstanceId);
|
|
96
|
+
this.config.workerInstanceId = id;
|
|
97
|
+
this.logger.info(`[mobius] worker instance id ${id} (source: ${source})`);
|
|
98
|
+
const runSignal = mergeSignals(this.stopController.signal, signal);
|
|
99
|
+
while (!this.stopping && !runSignal.aborted) {
|
|
186
100
|
try {
|
|
187
|
-
|
|
188
|
-
worker_instance_id: this.config.workerInstanceId,
|
|
189
|
-
worker_session_token: this.sessionToken,
|
|
190
|
-
concurrency_limit: this.config.concurrency,
|
|
191
|
-
wait_seconds: this.config.pollWaitSeconds,
|
|
192
|
-
};
|
|
193
|
-
if (this.config.name)
|
|
194
|
-
claimReq.worker_name = this.config.name;
|
|
195
|
-
if (this.config.version)
|
|
196
|
-
claimReq.worker_version = this.config.version;
|
|
197
|
-
if (this.config.queues.length > 0)
|
|
198
|
-
claimReq.queues = this.config.queues;
|
|
199
|
-
if (this.config.actions.length > 0)
|
|
200
|
-
claimReq.actions = this.config.actions;
|
|
201
|
-
job = await this.client.claimJob(claimReq, combined);
|
|
101
|
+
await this.runSocket(runSignal);
|
|
202
102
|
}
|
|
203
103
|
catch (err) {
|
|
204
|
-
if (
|
|
205
|
-
|
|
206
|
-
if (err instanceof AuthRevokedError
|
|
207
|
-
|
|
104
|
+
if (runSignal.aborted || this.stopping)
|
|
105
|
+
return;
|
|
106
|
+
if (err instanceof AuthRevokedError ||
|
|
107
|
+
err instanceof WorkerInstanceConflictError) {
|
|
108
|
+
// Terminal protocol failures must not reconnect: a revoked
|
|
109
|
+
// credential needs a fresh process, and a duplicate
|
|
110
|
+
// worker_instance_id means another live process owns this
|
|
111
|
+
// identity. Rethrow so a supervisor restarts (rotated credential)
|
|
112
|
+
// or an operator fixes the duplicate ID, instead of spinning in a
|
|
113
|
+
// reconnect loop.
|
|
208
114
|
throw err;
|
|
209
115
|
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
116
|
+
this.logger.warn("[mobius] worker socket disconnected; reconnecting", err);
|
|
117
|
+
await sleep(this.config.reconnectDelayMs ?? 2000, runSignal);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async runSocket(signal) {
|
|
122
|
+
const ws = new WebSocket(this.client.workerSocketURL());
|
|
123
|
+
const concurrency = this.config.concurrency && this.config.concurrency > 0
|
|
124
|
+
? this.config.concurrency
|
|
125
|
+
: 1;
|
|
126
|
+
const running = new Map();
|
|
127
|
+
let claimOutstanding = false;
|
|
128
|
+
await waitForOpen(ws, signal);
|
|
129
|
+
ws.send(JSON.stringify(this.registerFrame(concurrency)));
|
|
130
|
+
for await (const frame of socketFrames(ws, signal)) {
|
|
131
|
+
switch (frame.type) {
|
|
132
|
+
case "worker.registered":
|
|
133
|
+
this.sessionToken = String(frame.worker_session_token ?? "");
|
|
134
|
+
this.claim(ws, concurrency, running.size, claimOutstanding);
|
|
135
|
+
claimOutstanding = true;
|
|
136
|
+
break;
|
|
137
|
+
case "jobs.claimed":
|
|
138
|
+
claimOutstanding = false;
|
|
139
|
+
for (const job of (frame.jobs ?? [])) {
|
|
140
|
+
if (running.size >= concurrency)
|
|
141
|
+
break;
|
|
142
|
+
const ctrl = new AbortController();
|
|
143
|
+
running.set(job.id, ctrl);
|
|
144
|
+
void this.executeJob(ws, job, ctrl.signal).finally(() => {
|
|
145
|
+
running.delete(job.id);
|
|
146
|
+
this.claim(ws, concurrency, running.size, claimOutstanding);
|
|
147
|
+
claimOutstanding = true;
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
break;
|
|
151
|
+
case "work.available":
|
|
152
|
+
this.claim(ws, concurrency, running.size, claimOutstanding);
|
|
153
|
+
claimOutstanding = true;
|
|
154
|
+
break;
|
|
155
|
+
case "job.cancel": {
|
|
156
|
+
const ctrl = running.get(String(frame.job_id));
|
|
157
|
+
ctrl?.abort();
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
case "job.heartbeat.ack": {
|
|
161
|
+
const cancel = frame.cancel;
|
|
162
|
+
if (cancel?.job_id)
|
|
163
|
+
running.get(cancel.job_id)?.abort();
|
|
164
|
+
break;
|
|
165
|
+
}
|
|
166
|
+
case "worker.drain":
|
|
167
|
+
this.stopping = true;
|
|
168
|
+
ws.send(JSON.stringify({ type: "worker.draining", message_id: msgID() }));
|
|
169
|
+
break;
|
|
170
|
+
case "keepalive":
|
|
171
|
+
ws.send(JSON.stringify({ type: "keepalive", message_id: msgID() }));
|
|
172
|
+
break;
|
|
173
|
+
case "error": {
|
|
174
|
+
const terminal = this.terminalProtocolError(frame.error);
|
|
175
|
+
if (terminal)
|
|
176
|
+
throw terminal;
|
|
177
|
+
this.logger.error("[mobius] worker socket protocol error", frame.error);
|
|
178
|
+
break;
|
|
213
179
|
}
|
|
214
|
-
this.logger.error("[mobius] claim error:", err);
|
|
215
|
-
await sleep(2000, combined);
|
|
216
|
-
continue;
|
|
217
180
|
}
|
|
218
|
-
if (
|
|
219
|
-
|
|
220
|
-
|
|
181
|
+
if (this.stopping && running.size === 0) {
|
|
182
|
+
ws.close();
|
|
183
|
+
return;
|
|
221
184
|
}
|
|
222
|
-
const task = this.executeJob(job, callerSignal).finally(() => {
|
|
223
|
-
inflight.delete(task);
|
|
224
|
-
});
|
|
225
|
-
inflight.add(task);
|
|
226
185
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
186
|
+
}
|
|
187
|
+
// terminalProtocolError maps a worker-socket protocol error frame to a
|
|
188
|
+
// terminal error the worker must not reconnect through, or undefined for
|
|
189
|
+
// protocol errors it can keep running past (logged by the caller).
|
|
190
|
+
// invalid_actor means the credential was revoked; worker_instance_conflict
|
|
191
|
+
// means another live process already owns this worker_instance_id.
|
|
192
|
+
terminalProtocolError(error) {
|
|
193
|
+
switch (error?.code) {
|
|
194
|
+
case "invalid_actor":
|
|
195
|
+
return new AuthRevokedError();
|
|
196
|
+
case "worker_instance_conflict":
|
|
197
|
+
return new WorkerInstanceConflictError(this.config.workerInstanceId, this.client.project, error.message);
|
|
198
|
+
default:
|
|
199
|
+
return undefined;
|
|
231
200
|
}
|
|
232
201
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
202
|
+
registerFrame(concurrency) {
|
|
203
|
+
const actions = this.config.actions?.length
|
|
204
|
+
? this.config.actions
|
|
205
|
+
: [...this.actions.keys()].sort();
|
|
206
|
+
return removeUndefined({
|
|
207
|
+
type: "worker.register",
|
|
208
|
+
message_id: msgID(),
|
|
209
|
+
worker_instance_id: this.config.workerInstanceId ?? "",
|
|
210
|
+
worker_session_token: this.sessionToken || undefined,
|
|
211
|
+
concurrency_limit: concurrency,
|
|
212
|
+
available_slots: concurrency,
|
|
213
|
+
name: this.config.name,
|
|
214
|
+
version: this.config.version,
|
|
215
|
+
queues: this.config.queues?.length ? this.config.queues : undefined,
|
|
216
|
+
action_names: actions.length ? actions : undefined,
|
|
217
|
+
models: this.modelCapabilities(),
|
|
218
|
+
});
|
|
236
219
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const jobId = job.job_id;
|
|
240
|
-
const workerInstanceId = this.config.workerInstanceId;
|
|
241
|
-
const leaseToken = job.lease_token;
|
|
242
|
-
const attempt = job.attempt_number ?? 1;
|
|
243
|
-
if (job.spec.kind !== "action") {
|
|
244
|
-
const msg = `unsupported job spec kind ${JSON.stringify(job.spec.kind)} (this SDK only handles action jobs)`;
|
|
245
|
-
this.logger.error(`[mobius] ${msg}`);
|
|
246
|
-
await this.failJob(jobId, workerInstanceId, leaseToken, "UnsupportedSpec", msg);
|
|
220
|
+
claim(ws, concurrency, running, outstanding) {
|
|
221
|
+
if (outstanding || this.stopping)
|
|
247
222
|
return;
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
const actionName = actionSpec.name;
|
|
251
|
-
const parameters = actionSpec.parameters ?? {};
|
|
252
|
-
this.logger.info(`[mobius] job ${jobId} claimed (workflow=${job.workflow_name}, step=${job.step_name}, action=${actionName}, attempt=${attempt})`);
|
|
253
|
-
const fn = this.actions.get(actionName);
|
|
254
|
-
if (!fn) {
|
|
255
|
-
const msg = `action ${JSON.stringify(actionName)} not registered on this worker`;
|
|
256
|
-
this.logger.error(`[mobius] ${msg}`);
|
|
257
|
-
await this.failJob(jobId, workerInstanceId, leaseToken, "ActionNotRegistered", msg);
|
|
223
|
+
const available = concurrency - running;
|
|
224
|
+
if (available <= 0)
|
|
258
225
|
return;
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
batchSize: this.config.eventBatchSize,
|
|
270
|
-
logger: this.logger,
|
|
226
|
+
const actions = this.config.actions?.length
|
|
227
|
+
? this.config.actions
|
|
228
|
+
: [...this.actions.keys()].sort();
|
|
229
|
+
const frame = removeUndefined({
|
|
230
|
+
type: "jobs.claim",
|
|
231
|
+
message_id: msgID(),
|
|
232
|
+
available_slots: available,
|
|
233
|
+
queues: this.config.queues?.length ? this.config.queues : undefined,
|
|
234
|
+
action_names: actions.length ? actions : undefined,
|
|
235
|
+
models: this.modelCapabilities(),
|
|
271
236
|
});
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
237
|
+
ws.send(JSON.stringify(frame));
|
|
238
|
+
}
|
|
239
|
+
modelCapabilities() {
|
|
240
|
+
const models = this.config.models
|
|
241
|
+
?.filter((m) => m.provider && m.model)
|
|
242
|
+
.map((m) => ({ provider: m.provider, model: m.model }));
|
|
243
|
+
return models && models.length > 0 ? models : undefined;
|
|
244
|
+
}
|
|
245
|
+
async executeJob(ws, job, signal) {
|
|
246
|
+
const heartbeat = setInterval(() => {
|
|
247
|
+
const frame = {
|
|
248
|
+
type: "job.heartbeat",
|
|
249
|
+
message_id: msgID(),
|
|
250
|
+
job_id: job.id,
|
|
251
|
+
lease_token: job.lease_token,
|
|
252
|
+
};
|
|
253
|
+
ws.send(JSON.stringify(frame));
|
|
254
|
+
}, this.config.heartbeatIntervalMs ?? job.heartbeat_cadence_seconds * 1000);
|
|
255
|
+
const ctx = this.actionContext(job);
|
|
256
|
+
try {
|
|
257
|
+
let result;
|
|
258
|
+
if (job.kind === "action_execution") {
|
|
259
|
+
const actionName = job.action_name ?? String(job.spec.action_name ?? "");
|
|
260
|
+
const fn = this.actions.get(actionName);
|
|
261
|
+
if (!fn)
|
|
262
|
+
throw new Error(`action ${JSON.stringify(actionName)} is not registered`);
|
|
263
|
+
result = await fn(parameters(job), signal, ctx);
|
|
297
264
|
}
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
265
|
+
else if (job.kind === "llm_generation") {
|
|
266
|
+
const fn = this.generator(job.provider, job.model);
|
|
267
|
+
if (!fn)
|
|
268
|
+
throw new Error(`generation ${job.provider}/${job.model} is not registered`);
|
|
269
|
+
let seq = 0;
|
|
270
|
+
result = await fn(ctx, {
|
|
271
|
+
jobId: job.id,
|
|
272
|
+
runId: job.run_id,
|
|
273
|
+
sessionId: job.session_id,
|
|
274
|
+
agentTurnId: job.agent_turn_id,
|
|
275
|
+
toolCallId: job.tool_call_id,
|
|
276
|
+
provider: job.provider,
|
|
277
|
+
model: job.model,
|
|
278
|
+
spec: job.spec,
|
|
279
|
+
}, (delta) => {
|
|
280
|
+
seq += 1;
|
|
281
|
+
const frame = {
|
|
282
|
+
type: "generation.delta",
|
|
283
|
+
message_id: msgID(),
|
|
284
|
+
job_id: job.id,
|
|
285
|
+
lease_token: job.lease_token,
|
|
286
|
+
sequence: seq,
|
|
287
|
+
delta,
|
|
288
|
+
};
|
|
289
|
+
ws.send(JSON.stringify(frame));
|
|
290
|
+
});
|
|
312
291
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
const result = await fn(parameters, actionController.signal, actionContext);
|
|
316
|
-
clearInterval(heartbeatTimer);
|
|
317
|
-
signal?.removeEventListener("abort", onAbort);
|
|
318
|
-
eventer.stop();
|
|
319
|
-
await eventerPromise;
|
|
320
|
-
if (hbLost)
|
|
321
|
-
return;
|
|
322
|
-
const complete = { kind: "complete" };
|
|
323
|
-
if (result != null) {
|
|
324
|
-
complete.result_b64 = Buffer.from(JSON.stringify(result)).toString("base64");
|
|
292
|
+
else {
|
|
293
|
+
throw new Error(`unsupported job kind ${job.kind}`);
|
|
325
294
|
}
|
|
326
|
-
|
|
327
|
-
worker_instance_id: workerInstanceId,
|
|
328
|
-
lease_token: leaseToken,
|
|
329
|
-
outcomes: [complete],
|
|
330
|
-
};
|
|
331
|
-
await this.client.reportJob(jobId, reportReq);
|
|
332
|
-
this.logger.info(`[mobius] job ${jobId} completed`);
|
|
295
|
+
this.report(ws, job, "completed", result);
|
|
333
296
|
}
|
|
334
297
|
catch (err) {
|
|
335
|
-
|
|
336
|
-
signal?.removeEventListener("abort", onAbort);
|
|
337
|
-
eventer.stop();
|
|
338
|
-
await eventerPromise;
|
|
339
|
-
if (err instanceof AuthRevokedError) {
|
|
340
|
-
this.logger.warn(`[mobius] job ${jobId}: credential revoked during complete; worker will exit`);
|
|
341
|
-
this.authRevoked = true;
|
|
342
|
-
return;
|
|
343
|
-
}
|
|
344
|
-
if (err instanceof LeaseLostError || hbLost) {
|
|
345
|
-
this.logger.warn(`[mobius] job ${jobId}: lease lost — will be retried`);
|
|
346
|
-
return;
|
|
347
|
-
}
|
|
348
|
-
this.logger.error(`[mobius] job ${jobId} failed:`, err);
|
|
349
|
-
const errType = actionController.signal.aborted ? "Timeout" : "Error";
|
|
350
|
-
await this.failJob(jobId, workerInstanceId, leaseToken, errType, String(err));
|
|
298
|
+
this.report(ws, job, "failed", undefined, signal.aborted ? "Cancelled" : "Error", String(err));
|
|
351
299
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
if (this.config.heartbeatIntervalMs != null)
|
|
355
|
-
return this.config.heartbeatIntervalMs;
|
|
356
|
-
if (job.heartbeat_interval_seconds != null) {
|
|
357
|
-
return job.heartbeat_interval_seconds * 1000;
|
|
300
|
+
finally {
|
|
301
|
+
clearInterval(heartbeat);
|
|
358
302
|
}
|
|
359
|
-
return 10000;
|
|
360
303
|
}
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
304
|
+
generator(provider, model) {
|
|
305
|
+
if (!provider || !model)
|
|
306
|
+
return undefined;
|
|
307
|
+
return this.generators.get(`${provider}/${model}`) ?? this.generators.get(`${provider}/*`);
|
|
308
|
+
}
|
|
309
|
+
actionContext(job) {
|
|
310
|
+
return {
|
|
311
|
+
jobId: job.id,
|
|
312
|
+
runId: job.run_id,
|
|
313
|
+
sessionId: job.session_id,
|
|
314
|
+
agentTurnId: job.agent_turn_id,
|
|
315
|
+
toolCallId: job.tool_call_id,
|
|
316
|
+
projectId: this.client.project,
|
|
317
|
+
workerInstanceId: this.config.workerInstanceId ?? "",
|
|
318
|
+
attempt: job.claim_attempt,
|
|
319
|
+
queue: job.queue,
|
|
320
|
+
stepId: job.step_id,
|
|
321
|
+
action: job.action_name,
|
|
322
|
+
emitEvent: () => {
|
|
323
|
+
this.logger.warn("[mobius] custom worker events are not supported by the WebSocket protocol yet");
|
|
324
|
+
},
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
report(ws, job, status, result, errorType, errorMessage) {
|
|
328
|
+
const frame = removeUndefined({
|
|
329
|
+
type: "job.report",
|
|
330
|
+
message_id: msgID(),
|
|
331
|
+
job_id: job.id,
|
|
332
|
+
lease_token: job.lease_token,
|
|
333
|
+
status,
|
|
334
|
+
result: status === "completed" ? resultMap(result) : undefined,
|
|
335
|
+
error_type: errorType,
|
|
336
|
+
error_message: errorMessage,
|
|
337
|
+
});
|
|
338
|
+
ws.send(JSON.stringify(frame));
|
|
377
339
|
}
|
|
378
340
|
}
|
|
379
|
-
/**
|
|
380
|
-
* Runs multiple worker instances in one process. Most callers do not
|
|
381
|
-
* need a pool — to run several jobs from one process, set
|
|
382
|
-
* {@link WorkerConfig.concurrency} on a single {@link Worker} and the
|
|
383
|
-
* admin UI will show one row with a saturation bar. Reach for a pool
|
|
384
|
-
* only when each child should surface as its own row on the workers
|
|
385
|
-
* page (independent draining, in-flight isolation).
|
|
386
|
-
*/
|
|
387
341
|
export class WorkerPool {
|
|
388
342
|
constructor(client, config) {
|
|
389
343
|
this.client = client;
|
|
344
|
+
this.config = config;
|
|
390
345
|
this.actions = new Map();
|
|
391
|
-
this.
|
|
392
|
-
const prefix = config.workerInstanceIdPrefix && config.workerInstanceIdPrefix !== ""
|
|
393
|
-
? config.workerInstanceIdPrefix
|
|
394
|
-
: `worker-${randomUUID()}`;
|
|
395
|
-
this.config = {
|
|
396
|
-
workerInstanceIdPrefix: prefix,
|
|
397
|
-
concurrency: config.concurrency && config.concurrency > 0 ? config.concurrency : 1,
|
|
398
|
-
name: config.name ?? "",
|
|
399
|
-
version: config.version ?? "",
|
|
400
|
-
queues: config.queues ?? [],
|
|
401
|
-
actions: config.actions ?? [],
|
|
402
|
-
count: config.count && config.count > 0 ? config.count : 1,
|
|
403
|
-
pollWaitSeconds: config.pollWaitSeconds ?? 20,
|
|
404
|
-
heartbeatIntervalMs: config.heartbeatIntervalMs,
|
|
405
|
-
logger: config.logger,
|
|
406
|
-
eventQueueSize: config.eventQueueSize ?? 256,
|
|
407
|
-
eventBatchSize: config.eventBatchSize ?? 20,
|
|
408
|
-
};
|
|
346
|
+
this.generators = new Map();
|
|
409
347
|
}
|
|
410
|
-
/** Register an action function under the given name for every pool worker. */
|
|
411
348
|
register(name, fn) {
|
|
412
349
|
this.actions.set(name, fn);
|
|
413
350
|
return this;
|
|
414
351
|
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
352
|
+
registerGenerator(provider, model, fn) {
|
|
353
|
+
this.generators.set(`${provider}/${model}`, fn);
|
|
354
|
+
return this;
|
|
355
|
+
}
|
|
419
356
|
async run(signal) {
|
|
420
|
-
this.
|
|
421
|
-
const
|
|
422
|
-
|
|
423
|
-
: this.
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
return new Worker(this.client, {
|
|
428
|
-
workerInstanceId: `${this.config.workerInstanceIdPrefix}-${i + 1}`,
|
|
429
|
-
concurrency: this.config.concurrency,
|
|
430
|
-
name: this.config.name,
|
|
431
|
-
version: this.config.version,
|
|
432
|
-
queues: this.config.queues,
|
|
433
|
-
actions: this.config.actions,
|
|
434
|
-
pollWaitSeconds: this.config.pollWaitSeconds,
|
|
435
|
-
heartbeatIntervalMs: this.config.heartbeatIntervalMs,
|
|
436
|
-
logger: this.config.logger,
|
|
437
|
-
eventQueueSize: this.config.eventQueueSize,
|
|
438
|
-
eventBatchSize: this.config.eventBatchSize,
|
|
439
|
-
}, this.actions);
|
|
440
|
-
});
|
|
441
|
-
await Promise.allSettled(workers.map(async (worker) => {
|
|
442
|
-
try {
|
|
443
|
-
await worker.run(combined);
|
|
444
|
-
}
|
|
445
|
-
catch (err) {
|
|
446
|
-
if (err instanceof AuthRevokedError) {
|
|
447
|
-
authRevoked = true;
|
|
448
|
-
this.abortController.abort();
|
|
449
|
-
}
|
|
450
|
-
else if (!combined.aborted) {
|
|
451
|
-
firstError ?? (firstError = err);
|
|
452
|
-
this.abortController.abort();
|
|
453
|
-
}
|
|
357
|
+
const count = this.config.count && this.config.count > 0 ? this.config.count : 1;
|
|
358
|
+
const prefix = this.config.workerInstanceIdPrefix ?? `worker-${randomUUID()}`;
|
|
359
|
+
const workers = Array.from({ length: count }, (_, i) => {
|
|
360
|
+
const worker = new Worker(this.client, { ...this.config, workerInstanceId: `${prefix}-${i + 1}` }, this.actions);
|
|
361
|
+
for (const [key, fn] of this.generators) {
|
|
362
|
+
const [provider, model] = key.split("/", 2);
|
|
363
|
+
worker.registerGenerator(provider, model, fn);
|
|
454
364
|
}
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
}
|
|
459
|
-
if (firstError != null) {
|
|
460
|
-
throw firstError;
|
|
461
|
-
}
|
|
365
|
+
return worker.run(signal);
|
|
366
|
+
});
|
|
367
|
+
await Promise.all(workers);
|
|
462
368
|
}
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
369
|
+
}
|
|
370
|
+
function parameters(job) {
|
|
371
|
+
const raw = job.spec.parameters;
|
|
372
|
+
return raw && typeof raw === "object" && !Array.isArray(raw)
|
|
373
|
+
? raw
|
|
374
|
+
: {};
|
|
375
|
+
}
|
|
376
|
+
function resultMap(result) {
|
|
377
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
378
|
+
return result;
|
|
466
379
|
}
|
|
380
|
+
return { output: result };
|
|
467
381
|
}
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
// so it doesn't accumulate on a long-lived signal across thousands of
|
|
471
|
-
// poll cycles.
|
|
472
|
-
function sleep(ms, signal) {
|
|
473
|
-
// AbortSignal does not retroactively dispatch abort to listeners
|
|
474
|
-
// added after the signal already fired, so an already-aborted signal
|
|
475
|
-
// would otherwise wait the full ms before resolving.
|
|
476
|
-
if (signal.aborted)
|
|
477
|
-
return Promise.resolve();
|
|
478
|
-
return new Promise((resolve) => {
|
|
479
|
-
const onAbort = () => {
|
|
480
|
-
clearTimeout(t);
|
|
481
|
-
resolve();
|
|
482
|
-
};
|
|
483
|
-
const t = setTimeout(() => {
|
|
484
|
-
signal.removeEventListener("abort", onAbort);
|
|
485
|
-
resolve();
|
|
486
|
-
}, ms);
|
|
487
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
488
|
-
});
|
|
382
|
+
function msgID() {
|
|
383
|
+
return `msg_${randomUUID()}`;
|
|
489
384
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
// is no longer needed (e.g. when something else won a Promise.race).
|
|
493
|
-
// Without the cancel handle, every Promise.race in the concurrency
|
|
494
|
-
// loop would leak a listener on the long-lived worker signal.
|
|
495
|
-
function abortAsPromise(signal) {
|
|
496
|
-
if (signal.aborted)
|
|
497
|
-
return { promise: Promise.resolve(), cancel: () => { } };
|
|
498
|
-
let resolveFn;
|
|
499
|
-
const promise = new Promise((resolve) => {
|
|
500
|
-
resolveFn = resolve;
|
|
501
|
-
});
|
|
502
|
-
const onAbort = () => resolveFn();
|
|
503
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
504
|
-
return {
|
|
505
|
-
promise,
|
|
506
|
-
cancel: () => signal.removeEventListener("abort", onAbort),
|
|
507
|
-
};
|
|
385
|
+
function removeUndefined(obj) {
|
|
386
|
+
return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
|
|
508
387
|
}
|
|
509
|
-
function
|
|
388
|
+
function mergeSignals(...signals) {
|
|
510
389
|
const controller = new AbortController();
|
|
511
|
-
for (const
|
|
512
|
-
if (
|
|
513
|
-
|
|
390
|
+
for (const signal of signals) {
|
|
391
|
+
if (!signal)
|
|
392
|
+
continue;
|
|
393
|
+
if (signal.aborted) {
|
|
394
|
+
controller.abort(signal.reason);
|
|
514
395
|
break;
|
|
515
396
|
}
|
|
516
|
-
|
|
397
|
+
signal.addEventListener("abort", () => controller.abort(signal.reason), {
|
|
398
|
+
once: true,
|
|
399
|
+
});
|
|
517
400
|
}
|
|
518
401
|
return controller.signal;
|
|
519
402
|
}
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
this.options = options;
|
|
525
|
-
this.queue = [];
|
|
526
|
-
this.closed = false;
|
|
527
|
-
this.wake = null;
|
|
528
|
-
}
|
|
529
|
-
emit(type, payload) {
|
|
530
|
-
if (!type || this.closed)
|
|
403
|
+
function waitForOpen(ws, signal) {
|
|
404
|
+
return new Promise((resolve, reject) => {
|
|
405
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
406
|
+
resolve();
|
|
531
407
|
return;
|
|
532
|
-
if (this.queue.length >= this.options.queueSize) {
|
|
533
|
-
this.queue.shift();
|
|
534
|
-
this.options.logger.warn("[mobius] custom event queue full; dropping oldest event");
|
|
535
408
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
events: batch,
|
|
560
|
-
});
|
|
561
|
-
}
|
|
562
|
-
catch (err) {
|
|
563
|
-
if (err instanceof LeaseLostError) {
|
|
564
|
-
this.options.logger.warn(`[mobius] job ${this.job.job_id}: lease lost during custom event emit`);
|
|
565
|
-
return;
|
|
566
|
-
}
|
|
567
|
-
if (err instanceof PayloadTooLargeError) {
|
|
568
|
-
this.options.logger.warn(`[mobius] job ${this.job.job_id}: custom event payload too large`);
|
|
569
|
-
continue;
|
|
570
|
-
}
|
|
571
|
-
if (err instanceof RateLimitedError) {
|
|
572
|
-
this.options.logger.warn(`[mobius] job ${this.job.job_id}: custom event rate limited`);
|
|
573
|
-
continue;
|
|
574
|
-
}
|
|
575
|
-
this.options.logger.error(`[mobius] job ${this.job.job_id}: custom event emit failed:`, err);
|
|
576
|
-
}
|
|
409
|
+
const onOpen = () => resolve();
|
|
410
|
+
const onError = () => reject(new Error("worker socket failed to open"));
|
|
411
|
+
ws.addEventListener("open", onOpen, { once: true });
|
|
412
|
+
ws.addEventListener("error", onError, { once: true });
|
|
413
|
+
signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
async function* socketFrames(ws, signal) {
|
|
417
|
+
const queue = [];
|
|
418
|
+
let wake;
|
|
419
|
+
let done = false;
|
|
420
|
+
ws.addEventListener("message", (event) => {
|
|
421
|
+
queue.push(JSON.parse(String(event.data)));
|
|
422
|
+
wake?.();
|
|
423
|
+
});
|
|
424
|
+
ws.addEventListener("close", () => {
|
|
425
|
+
done = true;
|
|
426
|
+
wake?.();
|
|
427
|
+
});
|
|
428
|
+
signal?.addEventListener("abort", () => {
|
|
429
|
+
done = true;
|
|
430
|
+
try {
|
|
431
|
+
ws.close();
|
|
577
432
|
}
|
|
433
|
+
catch {
|
|
434
|
+
// ignore
|
|
435
|
+
}
|
|
436
|
+
wake?.();
|
|
437
|
+
});
|
|
438
|
+
while (!done || queue.length > 0) {
|
|
439
|
+
if (queue.length === 0) {
|
|
440
|
+
await new Promise((resolve) => {
|
|
441
|
+
wake = resolve;
|
|
442
|
+
});
|
|
443
|
+
wake = undefined;
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
yield queue.shift();
|
|
578
447
|
}
|
|
579
448
|
}
|
|
449
|
+
async function sleep(ms, signal) {
|
|
450
|
+
if (ms <= 0)
|
|
451
|
+
return;
|
|
452
|
+
await new Promise((resolve, reject) => {
|
|
453
|
+
const timer = setTimeout(resolve, ms);
|
|
454
|
+
signal?.addEventListener("abort", () => {
|
|
455
|
+
clearTimeout(timer);
|
|
456
|
+
reject(signal.reason ?? new Error("aborted"));
|
|
457
|
+
}, { once: true });
|
|
458
|
+
});
|
|
459
|
+
}
|
|
580
460
|
//# sourceMappingURL=worker.js.map
|