@bastani/atomic 0.9.26-alpha.5 → 0.9.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/dist/builtin/intercom/CHANGELOG.md +6 -0
- package/dist/builtin/intercom/package.json +2 -2
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +3 -3
- package/dist/builtin/web-access/package.json +2 -2
- package/dist/builtin/workflows/CHANGELOG.md +21 -0
- package/dist/builtin/workflows/builtin/{chunk-613yaa53.js → chunk-9bhs02gm.js} +195 -10
- package/dist/builtin/workflows/builtin/{chunk-6kqrk0pf.js → chunk-gts8wgjr.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-ea6nrbn3.js → chunk-gw7rghda.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +2 -2
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1082 -220
- package/dist/builtin/workflows/src/index.js +967 -186
- package/dist/core/extensions/workflow-run-control.d.ts +3 -0
- package/dist/core/extensions/workflow-run-control.d.ts.map +1 -1
- package/dist/core/extensions/workflow-run-control.js.map +1 -1
- package/dist/core/fast-model-routing.d.ts.map +1 -1
- package/dist/core/fast-model-routing.js +1 -5
- package/dist/core/fast-model-routing.js.map +1 -1
- package/dist/core/model-config.d.ts +36 -0
- package/dist/core/model-config.d.ts.map +1 -1
- package/dist/core/model-config.js +14 -2
- package/dist/core/model-config.js.map +1 -1
- package/dist/core/provider-composer-internal.d.ts +2 -1
- package/dist/core/provider-composer-internal.d.ts.map +1 -1
- package/dist/core/provider-composer-internal.js +13 -0
- package/dist/core/provider-composer-internal.js.map +1 -1
- package/docs/models/reference.md +28 -1
- package/docs/models.md +4 -0
- package/docs/sdk.md +26 -2
- package/docs/workflows/operations.md +15 -1
- package/npm-shrinkwrap.json +67 -67
- package/package.json +8 -8
|
@@ -206,6 +206,602 @@ var init_dbos_admission = __esm(() => {
|
|
|
206
206
|
dbosAdmissionContext = bag[key2];
|
|
207
207
|
});
|
|
208
208
|
|
|
209
|
+
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
210
|
+
function fenceDbosAdmissionPool(pool) {
|
|
211
|
+
const connect = pool.connect.bind(pool);
|
|
212
|
+
async function acquire(signal) {
|
|
213
|
+
if (signal.aborted)
|
|
214
|
+
throw sdkDependencyError();
|
|
215
|
+
const pending = connect().then((client) => {
|
|
216
|
+
if (signal.aborted) {
|
|
217
|
+
client.release(true);
|
|
218
|
+
throw sdkDependencyError();
|
|
219
|
+
}
|
|
220
|
+
return fencedClient(client, signal);
|
|
221
|
+
});
|
|
222
|
+
try {
|
|
223
|
+
return await raceAbort(pending, signal);
|
|
224
|
+
} catch (error) {
|
|
225
|
+
throw databaseError(error, signal);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
function scopedConnect(callback) {
|
|
229
|
+
const signal = dbosAdmissionContext.getStore();
|
|
230
|
+
if (signal === undefined) {
|
|
231
|
+
if (callback === undefined)
|
|
232
|
+
return connect();
|
|
233
|
+
connect(callback);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
const pending = acquire(signal);
|
|
237
|
+
if (callback === undefined)
|
|
238
|
+
return pending;
|
|
239
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
240
|
+
}
|
|
241
|
+
pool.connect = scopedConnect;
|
|
242
|
+
return pool;
|
|
243
|
+
}
|
|
244
|
+
function fencedClient(client, signal) {
|
|
245
|
+
let released = false;
|
|
246
|
+
const release = (error) => {
|
|
247
|
+
if (released)
|
|
248
|
+
return;
|
|
249
|
+
released = true;
|
|
250
|
+
signal.removeEventListener("abort", abort);
|
|
251
|
+
client.release(error);
|
|
252
|
+
};
|
|
253
|
+
const abort = () => release(true);
|
|
254
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
255
|
+
const query = new Proxy(client.query, {
|
|
256
|
+
apply(target, _receiver, args) {
|
|
257
|
+
const values = [...args];
|
|
258
|
+
const last = values.at(-1);
|
|
259
|
+
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
260
|
+
const pending = (async () => {
|
|
261
|
+
if (signal.aborted || released)
|
|
262
|
+
throw sdkDependencyError();
|
|
263
|
+
try {
|
|
264
|
+
return await raceAbort(new Promise((resolve, reject) => {
|
|
265
|
+
Reflect.apply(target, client, [
|
|
266
|
+
...values,
|
|
267
|
+
(error, result) => {
|
|
268
|
+
if (error)
|
|
269
|
+
reject(error);
|
|
270
|
+
else
|
|
271
|
+
resolve(result);
|
|
272
|
+
}
|
|
273
|
+
]);
|
|
274
|
+
}), signal);
|
|
275
|
+
} catch (error) {
|
|
276
|
+
throw databaseError(error, signal);
|
|
277
|
+
}
|
|
278
|
+
})();
|
|
279
|
+
if (callback === undefined)
|
|
280
|
+
return pending;
|
|
281
|
+
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
282
|
+
}
|
|
283
|
+
});
|
|
284
|
+
return new Proxy(client, {
|
|
285
|
+
get(target, property, receiver) {
|
|
286
|
+
if (property === "query")
|
|
287
|
+
return query;
|
|
288
|
+
if (property === "release")
|
|
289
|
+
return release;
|
|
290
|
+
return Reflect.get(target, property, receiver);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
function sdkDependencyError(message) {
|
|
295
|
+
const reported = message === undefined ? new DbosDependencyError : new DbosDependencyError(message, null);
|
|
296
|
+
return new DbosDependencyError(undefined, null, reported.message);
|
|
297
|
+
}
|
|
298
|
+
function databaseError(error, signal) {
|
|
299
|
+
if (signal?.aborted)
|
|
300
|
+
return sdkDependencyError();
|
|
301
|
+
if (isDbosDependencyError(error))
|
|
302
|
+
return sdkDependencyError(error.admissionDetail ?? error.message);
|
|
303
|
+
const pending = [error];
|
|
304
|
+
const seen = new Set;
|
|
305
|
+
while (pending.length > 0) {
|
|
306
|
+
const value2 = pending.pop();
|
|
307
|
+
if (typeof value2 !== "object" || value2 === null || seen.has(value2))
|
|
308
|
+
continue;
|
|
309
|
+
seen.add(value2);
|
|
310
|
+
if ("cause" in value2)
|
|
311
|
+
pending.push(value2.cause);
|
|
312
|
+
if ("error" in value2)
|
|
313
|
+
pending.push(value2.error);
|
|
314
|
+
if ("errors" in value2 && Array.isArray(value2.errors))
|
|
315
|
+
pending.push(...value2.errors);
|
|
316
|
+
const code = "code" in value2 && typeof value2.code === "string" ? value2.code : undefined;
|
|
317
|
+
const message = value2 instanceof Error ? value2.message : "";
|
|
318
|
+
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
319
|
+
return sdkDependencyError();
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
return error;
|
|
323
|
+
}
|
|
324
|
+
var connectionErrnos, connectionMessages;
|
|
325
|
+
var init_dbos_admission_pool = __esm(() => {
|
|
326
|
+
init_dbos_admission();
|
|
327
|
+
connectionErrnos = new Set([
|
|
328
|
+
"ECONNRESET",
|
|
329
|
+
"ECONNREFUSED",
|
|
330
|
+
"ECONNABORTED",
|
|
331
|
+
"EHOSTUNREACH",
|
|
332
|
+
"ENETUNREACH",
|
|
333
|
+
"ETIMEDOUT",
|
|
334
|
+
"ENOTFOUND",
|
|
335
|
+
"EAI_AGAIN",
|
|
336
|
+
"EPIPE"
|
|
337
|
+
]);
|
|
338
|
+
connectionMessages = new Set([
|
|
339
|
+
"Connection terminated unexpectedly",
|
|
340
|
+
"Connection terminated due to connection timeout",
|
|
341
|
+
"Client has encountered a connection error and is not queryable",
|
|
342
|
+
"timeout exceeded when trying to connect",
|
|
343
|
+
"Client was closed and is not queryable",
|
|
344
|
+
"Postgres connection pool changed while acquiring a connection"
|
|
345
|
+
]);
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
// dist/builtin/workflows/src/durable/dbos-row-guard.ts
|
|
349
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
350
|
+
async function installDbosRowGuard(client) {
|
|
351
|
+
const ready = await client.query("SELECT to_regclass('dbos.workflow_status') IS NOT NULL AS status, to_regclass('dbos.operation_outputs') IS NOT NULL AS operations");
|
|
352
|
+
if (!ready.rows[0]?.status && !ready.rows[0]?.operations)
|
|
353
|
+
return;
|
|
354
|
+
if (!ready.rows[0]?.status || !ready.rows[0]?.operations)
|
|
355
|
+
throw new Error("Atomic workflow database schema is incomplete");
|
|
356
|
+
if (await validateInstalledGuard(client))
|
|
357
|
+
return;
|
|
358
|
+
await client.query("BEGIN");
|
|
359
|
+
try {
|
|
360
|
+
await client.query("SELECT pg_advisory_xact_lock(3419, 9003)");
|
|
361
|
+
if (await validateInstalledGuard(client)) {
|
|
362
|
+
await client.query("COMMIT");
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
await client.query(guardSql);
|
|
366
|
+
await client.query("CREATE TABLE IF NOT EXISTS dbos.atomic_guard_protocol (singleton boolean PRIMARY KEY CHECK(singleton), fingerprint text NOT NULL)");
|
|
367
|
+
await client.query("INSERT INTO dbos.atomic_guard_protocol VALUES (true, $1) ON CONFLICT DO NOTHING", [
|
|
368
|
+
fingerprint
|
|
369
|
+
]);
|
|
370
|
+
await client.query("COMMIT");
|
|
371
|
+
} catch (error) {
|
|
372
|
+
await client.query("ROLLBACK");
|
|
373
|
+
throw error;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
async function validateInstalledGuard(client) {
|
|
377
|
+
const installed = await client.query("SELECT to_regclass('dbos.atomic_guard_protocol') IS NOT NULL AS present");
|
|
378
|
+
if (!installed.rows[0]?.present)
|
|
379
|
+
return false;
|
|
380
|
+
const validated = await client.query("SELECT fingerprint, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid IN ('dbos.workflow_status'::regclass,'dbos.operation_outputs'::regclass) AND tgname = 'atomic_generation_guard' AND tgenabled = 'O' AND tgtype = 31 AND tgfoid = 'dbos.atomic_guard_row()'::regprocedure) AS triggers, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection' AND tgenabled = 'O' AND tgtype = 21 AND tgfoid = 'dbos.atomic_project_metadata()'::regprocedure) AS projections FROM dbos.atomic_guard_protocol WHERE singleton");
|
|
381
|
+
if (validated.rows.length !== 1 || validated.rows[0].fingerprint !== fingerprint || validated.rows[0].triggers !== 2 || validated.rows[0].projections !== 1)
|
|
382
|
+
throw new Error("Atomic database ownership guard schema is incompatible or incomplete");
|
|
383
|
+
return true;
|
|
384
|
+
}
|
|
385
|
+
var guardSql = `
|
|
386
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_owner_generation (
|
|
387
|
+
root text PRIMARY KEY, owner text, generation bigint,
|
|
388
|
+
terminal boolean NOT NULL DEFAULT false, claim text, claimant text, protocol integer NOT NULL DEFAULT 1 CHECK(protocol = 1)
|
|
389
|
+
);
|
|
390
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_guard_rows (workflow_uuid text PRIMARY KEY, root text NOT NULL);
|
|
391
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_metadata(value text) RETURNS jsonb
|
|
392
|
+
LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog AS $$
|
|
393
|
+
DECLARE decoded jsonb;
|
|
394
|
+
BEGIN
|
|
395
|
+
IF value IS NULL THEN RETURN NULL; END IF;
|
|
396
|
+
BEGIN decoded := value::jsonb; EXCEPTION WHEN invalid_text_representation THEN RETURN NULL; END;
|
|
397
|
+
IF decoded->>'__dbos_serializer' = 'superjson' THEN decoded := decoded->'json'; END IF;
|
|
398
|
+
IF decoded->>'__atomicDurableMetadata' IS DISTINCT FROM 'true' OR decoded->>'version' IS DISTINCT FROM '3' THEN RETURN NULL; END IF;
|
|
399
|
+
RETURN decoded->'metadata';
|
|
400
|
+
END $$;
|
|
401
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_guard_row() RETURNS trigger
|
|
402
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
403
|
+
DECLARE
|
|
404
|
+
row_value jsonb; id text; root_id text; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE;
|
|
405
|
+
meta jsonb; prior jsonb; prior_generation bigint; next_generation bigint; absorbing boolean;
|
|
406
|
+
allowed boolean; cap_owner text; cap_generation bigint; claim_token text;
|
|
407
|
+
BEGIN
|
|
408
|
+
row_value := CASE WHEN TG_OP = 'DELETE' THEN to_jsonb(OLD) ELSE to_jsonb(NEW) END;
|
|
409
|
+
id := row_value->>'workflow_uuid';
|
|
410
|
+
root_id := split_part(id, ':checkpoint:', 1);
|
|
411
|
+
IF TG_OP = 'UPDATE' AND to_jsonb(OLD)->>'workflow_uuid' IS DISTINCT FROM id AND (
|
|
412
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid') OR
|
|
413
|
+
(TG_TABLE_NAME = 'workflow_status' AND to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')) OR
|
|
414
|
+
(TG_TABLE_NAME = 'operation_outputs' AND EXISTS(SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid' AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')))
|
|
415
|
+
) THEN
|
|
416
|
+
IF NULLIF(current_setting('atomic.row_authority',true),'') IS NULL OR current_setting('atomic.row_authority',true) = 'null' THEN RETURN NULL; END IF;
|
|
417
|
+
RAISE EXCEPTION 'Atomic workflow identities are immutable';
|
|
418
|
+
END IF;
|
|
419
|
+
IF TG_TABLE_NAME = 'workflow_status' AND NOT (
|
|
420
|
+
COALESCE(row_value->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
421
|
+
COALESCE(to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
422
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id)
|
|
423
|
+
) THEN
|
|
424
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
425
|
+
END IF;
|
|
426
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
427
|
+
IF TG_TABLE_NAME = 'operation_outputs' AND NOT EXISTS (
|
|
428
|
+
SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = id AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')
|
|
429
|
+
) AND NOT EXISTS (SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id) THEN
|
|
430
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
431
|
+
END IF;
|
|
432
|
+
INSERT INTO dbos.atomic_guard_rows VALUES(id, root_id) ON CONFLICT DO NOTHING;
|
|
433
|
+
INSERT INTO dbos.atomic_guard_rows SELECT workflow_uuid, root_id FROM dbos.workflow_status
|
|
434
|
+
WHERE workflow_uuid = root_id AND name = 'atomicWorkflowHandle' ON CONFLICT DO NOTHING;
|
|
435
|
+
INSERT INTO dbos.atomic_owner_generation(root) VALUES(root_id)
|
|
436
|
+
ON CONFLICT(root) DO UPDATE SET root = EXCLUDED.root RETURNING * INTO state;
|
|
437
|
+
IF state.owner IS NULL THEN
|
|
438
|
+
IF TG_TABLE_NAME <> 'workflow_status' OR TG_OP = 'DELETE' OR row_value->>'status' <> 'SUCCESS'
|
|
439
|
+
OR id NOT LIKE root_id || ':checkpoint:__atomic_metadata:%'
|
|
440
|
+
OR authority->>'root' IS DISTINCT FROM root_id OR authority->>'enroll' IS DISTINCT FROM 'true' THEN
|
|
441
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
442
|
+
END IF;
|
|
443
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
444
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id THEN
|
|
445
|
+
RAISE EXCEPTION 'Atomic metadata encoding is unavailable for enrollment';
|
|
446
|
+
END IF;
|
|
447
|
+
SELECT candidate.metadata, candidate.generation INTO prior, prior_generation FROM (
|
|
448
|
+
SELECT dbos.atomic_metadata(output) AS metadata,
|
|
449
|
+
split_part(substring(workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint AS generation
|
|
450
|
+
FROM dbos.workflow_status
|
|
451
|
+
WHERE workflow_uuid LIKE root_id || ':checkpoint:__atomic_metadata:%' AND status = 'SUCCESS'
|
|
452
|
+
) candidate WHERE candidate.metadata IS NOT NULL
|
|
453
|
+
ORDER BY COALESCE(candidate.metadata->>'status' IN ('completed', 'cancelled') OR
|
|
454
|
+
(candidate.metadata->>'status' IN ('failed','blocked') AND candidate.metadata->>'resumable' = 'false'), false) DESC,
|
|
455
|
+
candidate.generation DESC LIMIT 1;
|
|
456
|
+
IF prior_generation IS DISTINCT FROM (authority->>'generation')::bigint
|
|
457
|
+
OR prior->>'ownerExecutorId' IS DISTINCT FROM authority->>'owner' THEN
|
|
458
|
+
RAISE EXCEPTION 'Atomic ownership generation changed before enrollment';
|
|
459
|
+
END IF;
|
|
460
|
+
ELSE
|
|
461
|
+
IF authority->>'root' IS DISTINCT FROM root_id THEN RETURN NULL; END IF;
|
|
462
|
+
cap_owner := authority->>'owner'; cap_generation := (authority->>'generation')::bigint;
|
|
463
|
+
allowed := cap_owner = state.owner AND cap_generation = state.generation AND (
|
|
464
|
+
authority->>'actor' = state.owner OR
|
|
465
|
+
(authority->>'claim' = state.claim AND authority->>'actor' = state.claimant) OR
|
|
466
|
+
(authority->>'claim' IS NOT NULL AND authority->>'checkpoint' = id));
|
|
467
|
+
IF NOT COALESCE(allowed, false) THEN RAISE EXCEPTION 'Atomic ownership generation changed; stale database writes are refused'; END IF;
|
|
468
|
+
END IF;
|
|
469
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP = 'UPDATE' THEN
|
|
470
|
+
IF OLD.status = 'SUCCESS' AND OLD.name = 'atomicWorkflowCheckpoint' THEN
|
|
471
|
+
IF NEW.output IS DISTINCT FROM OLD.output OR NEW.status <> 'SUCCESS' THEN RAISE EXCEPTION 'Completed Atomic checkpoints are immutable'; END IF;
|
|
472
|
+
RETURN NEW;
|
|
473
|
+
END IF;
|
|
474
|
+
END IF;
|
|
475
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP <> 'DELETE' AND row_value->>'status' = 'SUCCESS'
|
|
476
|
+
AND id LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN
|
|
477
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
478
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN RAISE EXCEPTION 'Atomic metadata encoding or owner is unavailable'; END IF;
|
|
479
|
+
next_generation := split_part(substring(id FROM length(root_id) + 13), ':', 2)::bigint;
|
|
480
|
+
absorbing := COALESCE(meta->>'status' IN ('completed', 'cancelled') OR
|
|
481
|
+
(meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
482
|
+
claim_token := meta->>'transitionClaimId';
|
|
483
|
+
IF state.root IS NOT NULL AND (next_generation < state.generation OR (state.terminal AND NOT absorbing)) THEN
|
|
484
|
+
RAISE EXCEPTION 'Atomic terminal or newer generation cannot be overwritten';
|
|
485
|
+
END IF;
|
|
486
|
+
IF state.owner IS NOT NULL AND authority->>'actor' <> state.owner THEN
|
|
487
|
+
IF id NOT LIKE '%:claim' OR next_generation <> state.generation + 1 OR claim_token IS DISTINCT FROM authority->>'claim' OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN
|
|
488
|
+
RAISE EXCEPTION 'Atomic ownership transfer requires an exact generation claim';
|
|
489
|
+
END IF;
|
|
490
|
+
END IF;
|
|
491
|
+
END IF;
|
|
492
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
493
|
+
END $$;
|
|
494
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_project_metadata() RETURNS trigger
|
|
495
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
496
|
+
DECLARE root_id text; meta jsonb; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE; projected_generation bigint; absorbing boolean;
|
|
497
|
+
BEGIN
|
|
498
|
+
IF NEW.name <> 'atomicWorkflowCheckpoint' OR NEW.status <> 'SUCCESS' THEN RETURN NEW; END IF;
|
|
499
|
+
root_id := split_part(NEW.workflow_uuid, ':checkpoint:', 1);
|
|
500
|
+
IF NEW.workflow_uuid NOT LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN RETURN NEW; END IF;
|
|
501
|
+
IF TG_OP = 'UPDATE' THEN IF OLD.status = 'SUCCESS' THEN RETURN NEW; END IF; END IF;
|
|
502
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
503
|
+
SELECT * INTO state FROM dbos.atomic_owner_generation WHERE root = root_id FOR UPDATE;
|
|
504
|
+
IF state.owner IS NULL AND authority->>'enroll' IS DISTINCT FROM 'true' THEN RETURN NEW; END IF;
|
|
505
|
+
meta := dbos.atomic_metadata(NEW.output);
|
|
506
|
+
projected_generation := split_part(substring(NEW.workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint;
|
|
507
|
+
absorbing := COALESCE(meta->>'status' IN ('completed','cancelled') OR (meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
508
|
+
UPDATE dbos.atomic_owner_generation SET owner = meta->>'ownerExecutorId', generation = projected_generation,
|
|
509
|
+
terminal = absorbing, claim = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN meta->>'transitionClaimId' END,
|
|
510
|
+
claimant = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN authority->>'actor' END WHERE root = root_id;
|
|
511
|
+
RETURN NEW;
|
|
512
|
+
END $$;
|
|
513
|
+
DO $$ BEGIN
|
|
514
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
515
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.workflow_status
|
|
516
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
517
|
+
END IF;
|
|
518
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.operation_outputs'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
519
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.operation_outputs
|
|
520
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
521
|
+
END IF;
|
|
522
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection') THEN
|
|
523
|
+
CREATE TRIGGER atomic_generation_projection AFTER INSERT OR UPDATE ON dbos.workflow_status
|
|
524
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_project_metadata();
|
|
525
|
+
END IF;
|
|
526
|
+
END $$;
|
|
527
|
+
`, fingerprint;
|
|
528
|
+
var init_dbos_row_guard = __esm(() => {
|
|
529
|
+
fingerprint = createHash2("sha256").update(guardSql).digest("hex");
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
// dist/builtin/workflows/src/durable/dbos-owner-fence.ts
|
|
533
|
+
import { AsyncLocalStorage as AsyncLocalStorage2 } from "node:async_hooks";
|
|
534
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
535
|
+
function lockKey(identity) {
|
|
536
|
+
const digest = createHash3("sha256").update(`atomic-workflow-fence:${identity}`).digest();
|
|
537
|
+
return [digest.readInt32BE(0), digest.readInt32BE(4)];
|
|
538
|
+
}
|
|
539
|
+
function isDatabaseExecutor(executorId) {
|
|
540
|
+
return /^atomic-db-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(executorId ?? "");
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
class DbosOwnerFence {
|
|
544
|
+
pool;
|
|
545
|
+
executorId;
|
|
546
|
+
endPool;
|
|
547
|
+
lifetime;
|
|
548
|
+
lost = false;
|
|
549
|
+
closing;
|
|
550
|
+
closed = false;
|
|
551
|
+
get invalidated() {
|
|
552
|
+
return this.lost;
|
|
553
|
+
}
|
|
554
|
+
constructor(pool, executorId, endPool) {
|
|
555
|
+
this.pool = pool;
|
|
556
|
+
this.executorId = executorId;
|
|
557
|
+
this.endPool = endPool;
|
|
558
|
+
}
|
|
559
|
+
fail() {
|
|
560
|
+
this.lost = true;
|
|
561
|
+
return new DbosDependencyError("Workflow database unavailable: ownership connection was lost; this execution generation cannot write again");
|
|
562
|
+
}
|
|
563
|
+
owner() {
|
|
564
|
+
if (this.lost)
|
|
565
|
+
return Promise.reject(this.fail());
|
|
566
|
+
this.lifetime ??= (async () => {
|
|
567
|
+
const client = await this.pool().connect();
|
|
568
|
+
client.on("error", () => {
|
|
569
|
+
this.fail();
|
|
570
|
+
});
|
|
571
|
+
client.on("end", () => {
|
|
572
|
+
this.fail();
|
|
573
|
+
});
|
|
574
|
+
try {
|
|
575
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-owner:${this.executorId}`]);
|
|
576
|
+
const held = await client.query("SELECT pg_try_advisory_lock($1, $2) AS held", [
|
|
577
|
+
...lockKey(this.executorId)
|
|
578
|
+
]);
|
|
579
|
+
if (!held.rows[0]?.held || this.lost)
|
|
580
|
+
throw this.fail();
|
|
581
|
+
await client.query("SELECT pg_advisory_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
582
|
+
await client.query("SELECT pg_advisory_unlock($1, $2)", [...lockKey(this.executorId)]);
|
|
583
|
+
return client;
|
|
584
|
+
} catch (error) {
|
|
585
|
+
this.fail();
|
|
586
|
+
client.release(true);
|
|
587
|
+
throw error;
|
|
588
|
+
}
|
|
589
|
+
})().catch((error) => {
|
|
590
|
+
this.fail();
|
|
591
|
+
throw error;
|
|
592
|
+
});
|
|
593
|
+
return this.lifetime;
|
|
594
|
+
}
|
|
595
|
+
async liveness(executorId) {
|
|
596
|
+
if (!isDatabaseExecutor(executorId))
|
|
597
|
+
return "unknown";
|
|
598
|
+
const client = await this.pool().connect();
|
|
599
|
+
try {
|
|
600
|
+
await client.query("BEGIN");
|
|
601
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
602
|
+
...lockKey(executorId)
|
|
603
|
+
]);
|
|
604
|
+
return result.rows[0]?.held ? "dead" : "alive";
|
|
605
|
+
} finally {
|
|
606
|
+
try {
|
|
607
|
+
await client.query("ROLLBACK");
|
|
608
|
+
} finally {
|
|
609
|
+
client.release();
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
async recover(executorId, callback) {
|
|
614
|
+
if (!isDatabaseExecutor(executorId))
|
|
615
|
+
return;
|
|
616
|
+
const client = await this.pool().connect();
|
|
617
|
+
const fail = () => {
|
|
618
|
+
this.fail();
|
|
619
|
+
};
|
|
620
|
+
client.on("error", fail);
|
|
621
|
+
client.on("end", fail);
|
|
622
|
+
try {
|
|
623
|
+
await client.query("BEGIN");
|
|
624
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
625
|
+
...lockKey(executorId)
|
|
626
|
+
]);
|
|
627
|
+
if (!result.rows[0]?.held)
|
|
628
|
+
return;
|
|
629
|
+
return await callback();
|
|
630
|
+
} finally {
|
|
631
|
+
try {
|
|
632
|
+
await client.query("ROLLBACK");
|
|
633
|
+
} finally {
|
|
634
|
+
client.removeListener("error", fail);
|
|
635
|
+
client.removeListener("end", fail);
|
|
636
|
+
client.release();
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
async write(workflowId, callback, claim = false) {
|
|
641
|
+
try {
|
|
642
|
+
return await this.writeLocked(workflowId, callback, claim);
|
|
643
|
+
} catch (error) {
|
|
644
|
+
const mapped = databaseError(error);
|
|
645
|
+
throw this.lost && isDbosDependencyError(mapped) ? this.fail() : mapped;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
async writeLocked(workflowId, callback, claim) {
|
|
649
|
+
const context = writeContext.getStore();
|
|
650
|
+
const active = context?.fence === this ? context.workflows : undefined;
|
|
651
|
+
if (active?.has(workflowId)) {
|
|
652
|
+
if (this.lost)
|
|
653
|
+
throw this.fail();
|
|
654
|
+
return await callback();
|
|
655
|
+
}
|
|
656
|
+
const owner = await this.owner();
|
|
657
|
+
const client = await this.pool().connect();
|
|
658
|
+
const fail = () => {
|
|
659
|
+
this.fail();
|
|
660
|
+
};
|
|
661
|
+
client.on("error", fail);
|
|
662
|
+
client.on("end", fail);
|
|
663
|
+
try {
|
|
664
|
+
await client.query("BEGIN");
|
|
665
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-guard:${workflowId}`]);
|
|
666
|
+
await client.query("SET LOCAL statement_timeout = '10000ms'");
|
|
667
|
+
await client.query("SELECT pg_advisory_xact_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
668
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`workflow:${workflowId}`)]);
|
|
669
|
+
if (claim)
|
|
670
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`sql:${workflowId}`)]);
|
|
671
|
+
if (this.lost)
|
|
672
|
+
throw this.fail();
|
|
673
|
+
try {
|
|
674
|
+
await owner.query("SELECT 1");
|
|
675
|
+
} catch {
|
|
676
|
+
throw this.fail();
|
|
677
|
+
}
|
|
678
|
+
if (this.lost)
|
|
679
|
+
throw this.fail();
|
|
680
|
+
return await writeContext.run({ fence: this, workflows: new Map([...active ?? [], [workflowId, claim]]) }, callback);
|
|
681
|
+
} finally {
|
|
682
|
+
try {
|
|
683
|
+
await client.query("ROLLBACK");
|
|
684
|
+
} finally {
|
|
685
|
+
client.removeListener("error", fail);
|
|
686
|
+
client.removeListener("end", fail);
|
|
687
|
+
client.release();
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
async withRowAuthority(authority, callback) {
|
|
692
|
+
return await rowContext.run(authority, callback);
|
|
693
|
+
}
|
|
694
|
+
protectPool(pool) {
|
|
695
|
+
const connect = pool.connect.bind(pool);
|
|
696
|
+
const acquire = async () => {
|
|
697
|
+
const active = writeContext.getStore();
|
|
698
|
+
const client = await connect();
|
|
699
|
+
try {
|
|
700
|
+
await installDbosRowGuard(client);
|
|
701
|
+
await client.query("SELECT set_config('atomic.row_authority', $1, false)", [
|
|
702
|
+
JSON.stringify(rowContext.getStore() ?? null)
|
|
703
|
+
]);
|
|
704
|
+
} catch (error) {
|
|
705
|
+
client.release(true);
|
|
706
|
+
throw error;
|
|
707
|
+
}
|
|
708
|
+
if (active === undefined)
|
|
709
|
+
return client;
|
|
710
|
+
const fence = active.fence;
|
|
711
|
+
try {
|
|
712
|
+
if (fence.lost)
|
|
713
|
+
throw fence.fail();
|
|
714
|
+
await client.query("SELECT set_config('application_name', $1, false)", [
|
|
715
|
+
`atomic-sql:${[...active.workflows.keys()][0]}`
|
|
716
|
+
]);
|
|
717
|
+
for (const identity of [
|
|
718
|
+
fence.executorId,
|
|
719
|
+
...[...active.workflows].filter(([, claim]) => !claim).map(([id]) => `sql:${id}`)
|
|
720
|
+
]) {
|
|
721
|
+
const held = await client.query("SELECT pg_try_advisory_lock_shared($1, $2) AS held", [...lockKey(identity)]);
|
|
722
|
+
if (!held.rows[0]?.held)
|
|
723
|
+
throw fence.fail();
|
|
724
|
+
}
|
|
725
|
+
try {
|
|
726
|
+
await (await fence.owner()).query("SELECT 1");
|
|
727
|
+
} catch {
|
|
728
|
+
throw fence.fail();
|
|
729
|
+
}
|
|
730
|
+
if (fence.lost)
|
|
731
|
+
throw fence.fail();
|
|
732
|
+
} catch (error) {
|
|
733
|
+
client.release(true);
|
|
734
|
+
throw error;
|
|
735
|
+
}
|
|
736
|
+
let released = false;
|
|
737
|
+
const release = () => {
|
|
738
|
+
if (released)
|
|
739
|
+
return;
|
|
740
|
+
released = true;
|
|
741
|
+
client.release(true);
|
|
742
|
+
};
|
|
743
|
+
const query = new Proxy(client.query, {
|
|
744
|
+
apply: (target, _receiver, args) => {
|
|
745
|
+
if (!fence.lost && !released)
|
|
746
|
+
return Reflect.apply(target, client, args);
|
|
747
|
+
const error = fence.fail();
|
|
748
|
+
const callback = args.at(-1);
|
|
749
|
+
if (typeof callback === "function") {
|
|
750
|
+
queueMicrotask(() => callback(error));
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
return Promise.reject(error);
|
|
754
|
+
}
|
|
755
|
+
});
|
|
756
|
+
return new Proxy(client, {
|
|
757
|
+
get(target, property, receiver) {
|
|
758
|
+
if (property === "release")
|
|
759
|
+
return release;
|
|
760
|
+
if (property === "query")
|
|
761
|
+
return query;
|
|
762
|
+
return Reflect.get(target, property, receiver);
|
|
763
|
+
}
|
|
764
|
+
});
|
|
765
|
+
};
|
|
766
|
+
function scopedConnect(callback) {
|
|
767
|
+
const pending = acquire();
|
|
768
|
+
if (callback === undefined)
|
|
769
|
+
return pending;
|
|
770
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
771
|
+
}
|
|
772
|
+
pool.connect = scopedConnect;
|
|
773
|
+
return pool;
|
|
774
|
+
}
|
|
775
|
+
close() {
|
|
776
|
+
this.closed = true;
|
|
777
|
+
this.lost = true;
|
|
778
|
+
this.closing ??= (async () => {
|
|
779
|
+
if (this.lifetime !== undefined) {
|
|
780
|
+
const client = await this.lifetime.catch(() => {
|
|
781
|
+
return;
|
|
782
|
+
});
|
|
783
|
+
client?.release(true);
|
|
784
|
+
}
|
|
785
|
+
await this.endPool();
|
|
786
|
+
})();
|
|
787
|
+
return this.closing;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
var contextKey, contextBag, writeContext, rowContextKey, rowContextBag, rowContext;
|
|
791
|
+
var init_dbos_owner_fence = __esm(() => {
|
|
792
|
+
init_dbos_admission();
|
|
793
|
+
init_dbos_admission_pool();
|
|
794
|
+
init_dbos_row_guard();
|
|
795
|
+
contextKey = Symbol.for("atomic-workflows/dbos-owner-write-context@1");
|
|
796
|
+
contextBag = globalThis;
|
|
797
|
+
contextBag[contextKey] ??= new AsyncLocalStorage2;
|
|
798
|
+
writeContext = contextBag[contextKey];
|
|
799
|
+
rowContextKey = Symbol.for("atomic-workflows/dbos-row-authority@1");
|
|
800
|
+
rowContextBag = globalThis;
|
|
801
|
+
rowContextBag[rowContextKey] ??= new AsyncLocalStorage2;
|
|
802
|
+
rowContext = rowContextBag[rowContextKey];
|
|
803
|
+
});
|
|
804
|
+
|
|
209
805
|
// ../../node_modules/@dbos-inc/dbos-sdk/dist/src/error.js
|
|
210
806
|
var require_error = __commonJS(function(exports) {
|
|
211
807
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -30362,145 +30958,6 @@ SELECT
|
|
|
30362
30958
|
} });
|
|
30363
30959
|
});
|
|
30364
30960
|
|
|
30365
|
-
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
30366
|
-
function fenceDbosAdmissionPool(pool) {
|
|
30367
|
-
const connect = pool.connect.bind(pool);
|
|
30368
|
-
async function acquire(signal) {
|
|
30369
|
-
if (signal.aborted)
|
|
30370
|
-
throw sdkDependencyError();
|
|
30371
|
-
const pending = connect().then((client) => {
|
|
30372
|
-
if (signal.aborted) {
|
|
30373
|
-
client.release(true);
|
|
30374
|
-
throw sdkDependencyError();
|
|
30375
|
-
}
|
|
30376
|
-
return fencedClient(client, signal);
|
|
30377
|
-
});
|
|
30378
|
-
try {
|
|
30379
|
-
return await raceAbort(pending, signal);
|
|
30380
|
-
} catch (error) {
|
|
30381
|
-
throw databaseError(error, signal);
|
|
30382
|
-
}
|
|
30383
|
-
}
|
|
30384
|
-
function scopedConnect(callback) {
|
|
30385
|
-
const signal = dbosAdmissionContext.getStore();
|
|
30386
|
-
if (signal === undefined) {
|
|
30387
|
-
if (callback === undefined)
|
|
30388
|
-
return connect();
|
|
30389
|
-
connect(callback);
|
|
30390
|
-
return;
|
|
30391
|
-
}
|
|
30392
|
-
const pending = acquire(signal);
|
|
30393
|
-
if (callback === undefined)
|
|
30394
|
-
return pending;
|
|
30395
|
-
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
30396
|
-
}
|
|
30397
|
-
pool.connect = scopedConnect;
|
|
30398
|
-
return pool;
|
|
30399
|
-
}
|
|
30400
|
-
function fencedClient(client, signal) {
|
|
30401
|
-
let released = false;
|
|
30402
|
-
const release = (error) => {
|
|
30403
|
-
if (released)
|
|
30404
|
-
return;
|
|
30405
|
-
released = true;
|
|
30406
|
-
signal.removeEventListener("abort", abort);
|
|
30407
|
-
client.release(error);
|
|
30408
|
-
};
|
|
30409
|
-
const abort = () => release(true);
|
|
30410
|
-
signal.addEventListener("abort", abort, { once: true });
|
|
30411
|
-
const query = new Proxy(client.query, {
|
|
30412
|
-
apply(target, _receiver, args) {
|
|
30413
|
-
const values = [...args];
|
|
30414
|
-
const last = values.at(-1);
|
|
30415
|
-
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
30416
|
-
const pending = (async () => {
|
|
30417
|
-
if (signal.aborted || released)
|
|
30418
|
-
throw sdkDependencyError();
|
|
30419
|
-
try {
|
|
30420
|
-
return await raceAbort(new Promise((resolve, reject) => {
|
|
30421
|
-
Reflect.apply(target, client, [
|
|
30422
|
-
...values,
|
|
30423
|
-
(error, result) => {
|
|
30424
|
-
if (error)
|
|
30425
|
-
reject(error);
|
|
30426
|
-
else
|
|
30427
|
-
resolve(result);
|
|
30428
|
-
}
|
|
30429
|
-
]);
|
|
30430
|
-
}), signal);
|
|
30431
|
-
} catch (error) {
|
|
30432
|
-
throw databaseError(error, signal);
|
|
30433
|
-
}
|
|
30434
|
-
})();
|
|
30435
|
-
if (callback === undefined)
|
|
30436
|
-
return pending;
|
|
30437
|
-
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
30438
|
-
}
|
|
30439
|
-
});
|
|
30440
|
-
return new Proxy(client, {
|
|
30441
|
-
get(target, property, receiver) {
|
|
30442
|
-
if (property === "query")
|
|
30443
|
-
return query;
|
|
30444
|
-
if (property === "release")
|
|
30445
|
-
return release;
|
|
30446
|
-
return Reflect.get(target, property, receiver);
|
|
30447
|
-
}
|
|
30448
|
-
});
|
|
30449
|
-
}
|
|
30450
|
-
function sdkDependencyError(message) {
|
|
30451
|
-
const reported = message === undefined ? new DbosDependencyError : new DbosDependencyError(message, null);
|
|
30452
|
-
return new DbosDependencyError(undefined, null, reported.message);
|
|
30453
|
-
}
|
|
30454
|
-
function databaseError(error, signal) {
|
|
30455
|
-
if (signal.aborted)
|
|
30456
|
-
return sdkDependencyError();
|
|
30457
|
-
if (isDbosDependencyError(error))
|
|
30458
|
-
return sdkDependencyError(error.admissionDetail ?? error.message);
|
|
30459
|
-
const pending = [error];
|
|
30460
|
-
const seen = new Set;
|
|
30461
|
-
while (pending.length > 0) {
|
|
30462
|
-
const value2 = pending.pop();
|
|
30463
|
-
if (typeof value2 !== "object" || value2 === null || seen.has(value2))
|
|
30464
|
-
continue;
|
|
30465
|
-
seen.add(value2);
|
|
30466
|
-
if ("cause" in value2)
|
|
30467
|
-
pending.push(value2.cause);
|
|
30468
|
-
if ("error" in value2)
|
|
30469
|
-
pending.push(value2.error);
|
|
30470
|
-
if ("errors" in value2 && Array.isArray(value2.errors))
|
|
30471
|
-
pending.push(...value2.errors);
|
|
30472
|
-
const code = "code" in value2 && typeof value2.code === "string" ? value2.code : undefined;
|
|
30473
|
-
const message = value2 instanceof Error ? value2.message : "";
|
|
30474
|
-
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
30475
|
-
return sdkDependencyError();
|
|
30476
|
-
}
|
|
30477
|
-
}
|
|
30478
|
-
return error;
|
|
30479
|
-
}
|
|
30480
|
-
var connectionErrnos, connectionMessages;
|
|
30481
|
-
var init_dbos_admission_pool = __esm(() => {
|
|
30482
|
-
init_dbos_admission();
|
|
30483
|
-
connectionErrnos = new Set([
|
|
30484
|
-
"ECONNRESET",
|
|
30485
|
-
"ECONNREFUSED",
|
|
30486
|
-
"ECONNABORTED",
|
|
30487
|
-
"EHOSTUNREACH",
|
|
30488
|
-
"ENETUNREACH",
|
|
30489
|
-
"ETIMEDOUT",
|
|
30490
|
-
"ENOTFOUND",
|
|
30491
|
-
"EAI_AGAIN",
|
|
30492
|
-
"EPIPE"
|
|
30493
|
-
]);
|
|
30494
|
-
connectionMessages = new Set([
|
|
30495
|
-
"Connection terminated unexpectedly",
|
|
30496
|
-
"Connection terminated due to connection timeout",
|
|
30497
|
-
"Client has encountered a connection error and is not queryable",
|
|
30498
|
-
"timeout exceeded when trying to connect",
|
|
30499
|
-
"Client was closed and is not queryable",
|
|
30500
|
-
"Postgres connection pool changed while acquiring a connection"
|
|
30501
|
-
]);
|
|
30502
|
-
});
|
|
30503
|
-
|
|
30504
30961
|
// dist/builtin/workflows/src/durable/dbos-default-postgres-url.ts
|
|
30505
30962
|
function defaultPostgresUrl(database, host = process.env.PGHOST || "localhost", port = process.env.PGPORT || "5432") {
|
|
30506
30963
|
const url = new URL("postgresql://localhost");
|
|
@@ -30790,7 +31247,7 @@ function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock
|
|
|
30790
31247
|
url: health?.endpoint ?? systemDatabaseUrl,
|
|
30791
31248
|
failure: health?.lastFailure?.message
|
|
30792
31249
|
});
|
|
30793
|
-
const createPool = () => {
|
|
31250
|
+
const createPool = (fenced = true) => {
|
|
30794
31251
|
let unsubscribe;
|
|
30795
31252
|
const managed = createRecoverablePostgresPool(systemDatabaseUrl, {
|
|
30796
31253
|
beforeConnect: health === undefined ? undefined : () => health.check(),
|
|
@@ -30800,14 +31257,38 @@ function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock
|
|
|
30800
31257
|
});
|
|
30801
31258
|
unsubscribe = health?.subscribe(managed.invalidate);
|
|
30802
31259
|
health?.start();
|
|
30803
|
-
return fenceDbosAdmissionPool(managed.pool);
|
|
31260
|
+
return fenced ? fenceDbosAdmissionPool(managed.pool) : managed.pool;
|
|
30804
31261
|
};
|
|
30805
31262
|
let pool = createPool();
|
|
31263
|
+
const createFence = (executorId) => {
|
|
31264
|
+
const fencePool = createPool(false);
|
|
31265
|
+
const created = new DbosOwnerFence(() => fencePool, executorId, () => fencePool.end());
|
|
31266
|
+
const owner = getDbosProcessOwner();
|
|
31267
|
+
owner.executorFences ??= new Set;
|
|
31268
|
+
owner.executorFences.add(created);
|
|
31269
|
+
return created;
|
|
31270
|
+
};
|
|
31271
|
+
let fence = createFence(config.executorID);
|
|
31272
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31273
|
+
getDbosProcessOwner().createExecutorFence = () => {
|
|
31274
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31275
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31276
|
+
const next = createFence(executorId);
|
|
31277
|
+
getDbosProcessOwner().executorFence = next;
|
|
31278
|
+
return next;
|
|
31279
|
+
};
|
|
31280
|
+
pool = fence.protectPool(pool);
|
|
30806
31281
|
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30807
31282
|
const launch = async () => {
|
|
31283
|
+
if (fence.closed) {
|
|
31284
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31285
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31286
|
+
fence = createFence(executorId);
|
|
31287
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31288
|
+
}
|
|
30808
31289
|
if (pool.ended) {
|
|
30809
|
-
pool = createPool();
|
|
30810
|
-
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
31290
|
+
pool = fence.protectPool(createPool());
|
|
31291
|
+
sdk.setConfig({ ...config, executorID: fence.executorId, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30811
31292
|
}
|
|
30812
31293
|
const urlToEnsure = health === undefined ? systemDatabaseUrl : await health.check();
|
|
30813
31294
|
const result = await import_datasource2.ensurePGDatabase({ urlToEnsure, logger: () => {} });
|
|
@@ -30827,6 +31308,7 @@ var import_datasource2;
|
|
|
30827
31308
|
var init_dbos_admission_config = __esm(() => {
|
|
30828
31309
|
init_dbos_admission_pool();
|
|
30829
31310
|
init_dbos_launch_lock();
|
|
31311
|
+
init_dbos_owner_fence();
|
|
30830
31312
|
init_dbos_process_owner();
|
|
30831
31313
|
init_dbos_recoverable_pool();
|
|
30832
31314
|
import_datasource2 = __toESM(require_datasource(), 1);
|
|
@@ -81463,7 +81945,7 @@ function getDurableBackendProcessOwner() {
|
|
|
81463
81945
|
}
|
|
81464
81946
|
|
|
81465
81947
|
// dist/builtin/workflows/src/durable/dbos-backend.ts
|
|
81466
|
-
import { isDeepStrictEqual as
|
|
81948
|
+
import { isDeepStrictEqual as isDeepStrictEqual3 } from "node:util";
|
|
81467
81949
|
|
|
81468
81950
|
// dist/builtin/workflows/src/durable/format-version.ts
|
|
81469
81951
|
var DURABLE_FORMAT_VERSION = 3;
|
|
@@ -82326,10 +82808,10 @@ function isForeignLiveWorkflow(candidate, localExecutorId, now = Date.now()) {
|
|
|
82326
82808
|
return false;
|
|
82327
82809
|
if (candidate.ownerExecutorId === undefined || candidate.ownerExecutorId === localExecutorId)
|
|
82328
82810
|
return false;
|
|
82329
|
-
return now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
82811
|
+
return candidate.ownerLiveness === "alive" || now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
82330
82812
|
}
|
|
82331
82813
|
function isLiveRunningWorkflow(candidate, now = Date.now()) {
|
|
82332
|
-
return candidate.status === "running" && now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
82814
|
+
return candidate.status === "running" && (candidate.ownerLiveness === "alive" || now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS);
|
|
82333
82815
|
}
|
|
82334
82816
|
|
|
82335
82817
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
@@ -82379,7 +82861,7 @@ class InMemoryDurableBackend {
|
|
|
82379
82861
|
workflows = new Map;
|
|
82380
82862
|
promptReservations = new Map;
|
|
82381
82863
|
deletedWorkflowIds = new Set;
|
|
82382
|
-
registerWorkflow(handle) {
|
|
82864
|
+
registerWorkflow(handle, authoritativeOwner = false) {
|
|
82383
82865
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
82384
82866
|
const existing = this.workflows.get(handle.workflowId);
|
|
82385
82867
|
const completedCheckpoints = handle.completedCheckpoints ?? existing?.handle.completedCheckpoints ?? 0;
|
|
@@ -82393,7 +82875,7 @@ class InMemoryDurableBackend {
|
|
|
82393
82875
|
status: handle.status,
|
|
82394
82876
|
...handle.invocationCwd !== undefined ? { invocationCwd: handle.invocationCwd } : existing?.handle.invocationCwd !== undefined ? { invocationCwd: existing.handle.invocationCwd } : {},
|
|
82395
82877
|
...handle.origin !== undefined ? { origin: handle.origin } : existing?.handle.origin !== undefined ? { origin: existing.handle.origin } : {},
|
|
82396
|
-
modelOwner: existing?.handle.modelOwner ?? handle.modelOwner,
|
|
82878
|
+
modelOwner: authoritativeOwner ? handle.modelOwner : existing?.handle.modelOwner ?? handle.modelOwner,
|
|
82397
82879
|
...handle.workflowCwd !== undefined ? { workflowCwd: handle.workflowCwd } : existing?.handle.workflowCwd !== undefined ? { workflowCwd: existing.handle.workflowCwd } : {},
|
|
82398
82880
|
...handle.repositoryRoot !== undefined ? { repositoryRoot: handle.repositoryRoot } : existing?.handle.repositoryRoot !== undefined ? { repositoryRoot: existing.handle.repositoryRoot } : {},
|
|
82399
82881
|
...handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: handle.gitWorktreeRoot } : existing?.handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: existing.handle.gitWorktreeRoot } : {},
|
|
@@ -82407,7 +82889,9 @@ class InMemoryDurableBackend {
|
|
|
82407
82889
|
...handle.rootWorkflowId !== undefined ? { rootWorkflowId: handle.rootWorkflowId } : {},
|
|
82408
82890
|
...handle.resumable !== undefined ? { resumable: handle.resumable } : {},
|
|
82409
82891
|
...workflowFailureFields(hasWorkflowFailureMetadata(handle) ? handle : handle.status === "running" ? undefined : existing?.handle),
|
|
82410
|
-
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {}
|
|
82892
|
+
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {},
|
|
82893
|
+
...handle.ownerLiveness !== undefined ? { ownerLiveness: handle.ownerLiveness } : {},
|
|
82894
|
+
...handle.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {}
|
|
82411
82895
|
};
|
|
82412
82896
|
if (existing)
|
|
82413
82897
|
existing.handle = full;
|
|
@@ -82557,11 +83041,14 @@ class InMemoryDurableBackend {
|
|
|
82557
83041
|
if (pendingPrompts !== undefined)
|
|
82558
83042
|
this.promptReservations.delete(workflowId);
|
|
82559
83043
|
}
|
|
82560
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
82561
|
-
const
|
|
83044
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
83045
|
+
const record = this.workflows.get(workflowId);
|
|
83046
|
+
const current = record?.handle;
|
|
82562
83047
|
if (current === undefined || !expected.includes(current.status) || expectedUpdatedAt !== undefined && current.updatedAt !== expectedUpdatedAt)
|
|
82563
83048
|
return false;
|
|
82564
83049
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
83050
|
+
if (modelOwner !== undefined && record !== undefined)
|
|
83051
|
+
record.handle = { ...record.handle, modelOwner };
|
|
82565
83052
|
return true;
|
|
82566
83053
|
}
|
|
82567
83054
|
adjustPendingPrompts(workflowId, delta) {
|
|
@@ -82634,6 +83121,7 @@ class InMemoryDurableBackend {
|
|
|
82634
83121
|
pendingStageMessages: h.pendingStageMessages ?? [],
|
|
82635
83122
|
...h.possibleStages !== undefined ? { possibleStages: h.possibleStages } : {},
|
|
82636
83123
|
...h.ownerExecutorId !== undefined ? { ownerExecutorId: h.ownerExecutorId } : {},
|
|
83124
|
+
...h.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
82637
83125
|
...h.sessionFile !== undefined ? { sessionFile: h.sessionFile } : {},
|
|
82638
83126
|
...h.label !== undefined ? { label: h.label } : {},
|
|
82639
83127
|
...h.rootWorkflowId !== undefined ? { rootWorkflowId: h.rootWorkflowId } : {},
|
|
@@ -85464,6 +85952,7 @@ function encodeMetadata(metadata) {
|
|
|
85464
85952
|
} : {},
|
|
85465
85953
|
...metadata.ownerExecutorId !== undefined ? { ownerExecutorId: metadata.ownerExecutorId } : {},
|
|
85466
85954
|
...metadata.transitionClaimId !== undefined ? { transitionClaimId: metadata.transitionClaimId } : {},
|
|
85955
|
+
...metadata.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
85467
85956
|
...metadata.sessionFile !== undefined ? { sessionFile: metadata.sessionFile } : {},
|
|
85468
85957
|
...metadata.label !== undefined ? { label: metadata.label } : {},
|
|
85469
85958
|
...metadata.rootWorkflowId !== undefined ? { rootWorkflowId: metadata.rootWorkflowId } : {},
|
|
@@ -85526,7 +86015,7 @@ function parseDurableWorkflowMetadata(value2, workflowId) {
|
|
|
85526
86015
|
return;
|
|
85527
86016
|
const possibleStages = coercePossibleStages(serialized.possibleStages);
|
|
85528
86017
|
const metadata = value2;
|
|
85529
|
-
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
86018
|
+
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.legacyRecoveryPending !== undefined && metadata.legacyRecoveryPending !== true || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
85530
86019
|
return;
|
|
85531
86020
|
const { origin, possibleStages: rawPossibleStages, ...metadataWithoutOrigin } = metadata;
|
|
85532
86021
|
return {
|
|
@@ -86005,11 +86494,16 @@ function classifyDbosDeletionTombstone(records, workflowId) {
|
|
|
86005
86494
|
init_dbos_process_owner();
|
|
86006
86495
|
|
|
86007
86496
|
// dist/builtin/workflows/src/durable/dbos-sdk-handle.ts
|
|
86008
|
-
|
|
86497
|
+
import { AsyncLocalStorage as AsyncLocalStorage3 } from "node:async_hooks";
|
|
86498
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
86499
|
+
init_dbos_owner_fence();
|
|
86500
|
+
init_dbos_process_owner();
|
|
86009
86501
|
function getAtomicExecutorId() {
|
|
86010
|
-
|
|
86502
|
+
const owner = getDbosProcessOwner();
|
|
86503
|
+
owner.executorId ??= `atomic-db-${crypto.randomUUID()}`;
|
|
86504
|
+
return owner.executorId;
|
|
86011
86505
|
}
|
|
86012
|
-
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
86506
|
+
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, fence, previousExecutorIds = []) {
|
|
86013
86507
|
const checkpointId = (workflowId, stepName) => `${workflowId}:checkpoint:${stepName}`;
|
|
86014
86508
|
async function stepRecord(status, prefix) {
|
|
86015
86509
|
if (status.status !== "SUCCESS")
|
|
@@ -86023,10 +86517,18 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
86023
86517
|
const output = status.output === undefined || typeof status.output === "string" ? await dbos.retrieveWorkflow(id).getResult() : status.output;
|
|
86024
86518
|
return { stepName, output, completedAt: status.createdAt };
|
|
86025
86519
|
}
|
|
86026
|
-
|
|
86520
|
+
const raw = {
|
|
86027
86521
|
launch: () => dbos.launch(),
|
|
86028
86522
|
shutdown: () => dbos.shutdown(),
|
|
86029
86523
|
async startWorkflow(workflowId, name, inputs) {
|
|
86524
|
+
if (fence !== undefined) {
|
|
86525
|
+
const existing = await raw.retrieveWorkflow(workflowId);
|
|
86526
|
+
if (existing !== undefined) {
|
|
86527
|
+
if (existing.name !== name || !isDeepStrictEqual2(existing.inputs, inputs))
|
|
86528
|
+
throw new Error("Workflow root identity changed during admission");
|
|
86529
|
+
return;
|
|
86530
|
+
}
|
|
86531
|
+
}
|
|
86030
86532
|
try {
|
|
86031
86533
|
await dbos.startWorkflow(mainWorkflow, { workflowID: workflowId })(name, { ...inputs });
|
|
86032
86534
|
} catch (err) {
|
|
@@ -86045,7 +86547,10 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
86045
86547
|
await dbos.cancelWorkflow(workflowId, { cancelChildren: true });
|
|
86046
86548
|
},
|
|
86047
86549
|
async resumeWorkflow(workflowId) {
|
|
86048
|
-
|
|
86550
|
+
if (fence === undefined)
|
|
86551
|
+
await dbos.resumeWorkflow(workflowId);
|
|
86552
|
+
else if (await raw.retrieveWorkflow(workflowId) === undefined)
|
|
86553
|
+
throw new Error("Workflow root is unavailable for resume");
|
|
86049
86554
|
},
|
|
86050
86555
|
async listAllWorkflows() {
|
|
86051
86556
|
const statuses = await dbos.listWorkflows({
|
|
@@ -86074,6 +86579,12 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
86074
86579
|
return status !== undefined && (status.workflowID ?? status.workflowId) === id ? stepRecord(status, prefix) : undefined;
|
|
86075
86580
|
},
|
|
86076
86581
|
async recordStepOutput(workflowId, stepName, output) {
|
|
86582
|
+
if (fence !== undefined) {
|
|
86583
|
+
const id = checkpointId(workflowId, stepName);
|
|
86584
|
+
const existing = (await dbos.listWorkflows({ workflowIDs: [id], limit: 1 }))[0];
|
|
86585
|
+
if (existing?.status !== undefined && ["PENDING", "ENQUEUED", "CANCELLED", "ERROR", "DELAYED"].includes(existing.status))
|
|
86586
|
+
await dbos.deleteWorkflows([id], false);
|
|
86587
|
+
}
|
|
86077
86588
|
let handle;
|
|
86078
86589
|
try {
|
|
86079
86590
|
handle = await dbos.startWorkflow(checkpointWorkflow, { workflowID: checkpointId(workflowId, stepName) })(workflowId, stepName, output);
|
|
@@ -86097,6 +86608,135 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
86097
86608
|
await dbos.deleteWorkflows([...new Set([workflowId, ...checkpointIds])], true);
|
|
86098
86609
|
}
|
|
86099
86610
|
};
|
|
86611
|
+
if (fence === undefined)
|
|
86612
|
+
return raw;
|
|
86613
|
+
const claims = new AsyncLocalStorage3;
|
|
86614
|
+
const rowAuthority = (workflowId, current) => ({
|
|
86615
|
+
root: workflowId,
|
|
86616
|
+
actor: fence.executorId,
|
|
86617
|
+
...current.kind === "current" ? { owner: current.metadata.ownerExecutorId, generation: current.generation } : {},
|
|
86618
|
+
enroll: current.kind === "unavailable" || current.kind === "current" && isDatabaseExecutor(current.metadata.ownerExecutorId)
|
|
86619
|
+
});
|
|
86620
|
+
const mutate = async (workflowId, callback, step) => fence.write(workflowId, async () => {
|
|
86621
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86622
|
+
const claimed = claims.getStore();
|
|
86623
|
+
if (claimed === undefined && current.kind === "current" && current.metadata.ownerExecutorId !== fence.executorId)
|
|
86624
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
86625
|
+
let output = step === undefined ? undefined : classifyLatestMetadata([{ stepName: step.name, output: step.output }], workflowId);
|
|
86626
|
+
let persistedStep = step;
|
|
86627
|
+
if (step !== undefined && output?.kind === "current" && current.kind === "current" && !step.name.endsWith(":claim") && output.metadata.ownerExecutorId === fence.executorId && output.generation <= current.generation) {
|
|
86628
|
+
const updatedAt = Math.max(current.generation + 1, output.metadata.updatedAt, Date.now());
|
|
86629
|
+
persistedStep = {
|
|
86630
|
+
name: metadataStepName(updatedAt),
|
|
86631
|
+
output: encodeMetadata({ ...output.metadata, updatedAt })
|
|
86632
|
+
};
|
|
86633
|
+
output = classifyLatestMetadata([{ stepName: persistedStep.name, output: persistedStep.output }], workflowId);
|
|
86634
|
+
}
|
|
86635
|
+
const authority = claimed?.authority ?? rowAuthority(workflowId, current);
|
|
86636
|
+
const capability = output?.kind === "current" && step?.name.endsWith(":claim") ? {
|
|
86637
|
+
...authority,
|
|
86638
|
+
claim: output.metadata.transitionClaimId,
|
|
86639
|
+
checkpoint: checkpointId(workflowId, step.name)
|
|
86640
|
+
} : authority;
|
|
86641
|
+
const record = persistedStep;
|
|
86642
|
+
await fence.withRowAuthority(capability, record === undefined ? callback : () => raw.recordStepOutput(workflowId, record.name, record.output));
|
|
86643
|
+
if (claimed !== undefined && output?.kind === "current")
|
|
86644
|
+
claimed.authority = {
|
|
86645
|
+
...capability,
|
|
86646
|
+
owner: output.metadata.ownerExecutorId,
|
|
86647
|
+
generation: output.generation
|
|
86648
|
+
};
|
|
86649
|
+
});
|
|
86650
|
+
return {
|
|
86651
|
+
...raw,
|
|
86652
|
+
executorId: fence.executorId,
|
|
86653
|
+
generationLost: () => fence.invalidated,
|
|
86654
|
+
forkGeneration: () => {
|
|
86655
|
+
const next = getDbosProcessOwner().createExecutorFence?.();
|
|
86656
|
+
if (next === undefined)
|
|
86657
|
+
throw new Error("Workflow ownership fence cannot create a recovery generation");
|
|
86658
|
+
fence.close().catch(() => {});
|
|
86659
|
+
return createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, next, [
|
|
86660
|
+
...previousExecutorIds,
|
|
86661
|
+
fence.executorId
|
|
86662
|
+
]);
|
|
86663
|
+
},
|
|
86664
|
+
adoptPreviousGeneration: async (workflowId) => {
|
|
86665
|
+
if (previousExecutorIds.length === 0)
|
|
86666
|
+
return;
|
|
86667
|
+
return await fence.write(workflowId, async () => {
|
|
86668
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86669
|
+
if (current.kind !== "current" || current.metadata.ownerExecutorId === fence.executorId)
|
|
86670
|
+
return;
|
|
86671
|
+
const previousExecutorId = current.metadata.ownerExecutorId;
|
|
86672
|
+
if (previousExecutorId === undefined || !previousExecutorIds.includes(previousExecutorId))
|
|
86673
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
86674
|
+
const metadata = {
|
|
86675
|
+
...current.metadata,
|
|
86676
|
+
ownerExecutorId: fence.executorId,
|
|
86677
|
+
transitionClaimId: crypto.randomUUID(),
|
|
86678
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
86679
|
+
};
|
|
86680
|
+
const adopted = await fence.recover(previousExecutorId, async () => {
|
|
86681
|
+
await fence.withRowAuthority({
|
|
86682
|
+
...rowAuthority(workflowId, current),
|
|
86683
|
+
claim: metadata.transitionClaimId,
|
|
86684
|
+
checkpoint: checkpointId(workflowId, claimMetadataStepName(current.generation))
|
|
86685
|
+
}, () => raw.recordStepOutput(workflowId, claimMetadataStepName(current.generation), encodeMetadata(metadata)));
|
|
86686
|
+
return metadata;
|
|
86687
|
+
});
|
|
86688
|
+
if (adopted === undefined)
|
|
86689
|
+
throw new Error("Workflow previous execution generation is still active");
|
|
86690
|
+
return adopted;
|
|
86691
|
+
}, true);
|
|
86692
|
+
},
|
|
86693
|
+
enrollLegacyWorkflow: async (workflowId, modelOwner) => fence.write(workflowId, async () => {
|
|
86694
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86695
|
+
if (current.kind !== "current" || isDatabaseExecutor(current.metadata.ownerExecutorId) && current.metadata.legacyRecoveryPending !== true || !["running", "paused", "blocked"].includes(current.metadata.status) || current.metadata.rootWorkflowId !== undefined && current.metadata.rootWorkflowId !== workflowId)
|
|
86696
|
+
return;
|
|
86697
|
+
const metadata = {
|
|
86698
|
+
...current.metadata,
|
|
86699
|
+
modelOwner,
|
|
86700
|
+
ownerExecutorId: fence.executorId,
|
|
86701
|
+
status: "blocked",
|
|
86702
|
+
resumable: true,
|
|
86703
|
+
legacyRecoveryPending: true,
|
|
86704
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
86705
|
+
};
|
|
86706
|
+
const retry = isDatabaseExecutor(current.metadata.ownerExecutorId);
|
|
86707
|
+
const transitionClaimId = crypto.randomUUID();
|
|
86708
|
+
const stepName = retry ? claimMetadataStepName(current.generation) : metadataStepName(Math.max(current.generation + 1, metadata.updatedAt));
|
|
86709
|
+
const persist = () => fence.withRowAuthority({
|
|
86710
|
+
...rowAuthority(workflowId, current),
|
|
86711
|
+
enroll: true,
|
|
86712
|
+
...retry ? { claim: transitionClaimId, checkpoint: checkpointId(workflowId, stepName) } : {}
|
|
86713
|
+
}, () => raw.recordStepOutput(workflowId, stepName, encodeMetadata({ ...metadata, ...retry ? { transitionClaimId } : {} })));
|
|
86714
|
+
if (retry) {
|
|
86715
|
+
const recovered = await fence.recover(current.metadata.ownerExecutorId, async () => {
|
|
86716
|
+
await persist();
|
|
86717
|
+
return true;
|
|
86718
|
+
});
|
|
86719
|
+
if (recovered !== true)
|
|
86720
|
+
return;
|
|
86721
|
+
} else
|
|
86722
|
+
await persist();
|
|
86723
|
+
return metadata;
|
|
86724
|
+
}, true),
|
|
86725
|
+
ownerLiveness: (executorId) => fence.liveness(executorId),
|
|
86726
|
+
withWorkflowClaim: (workflowId, callback) => fence.write(workflowId, async () => {
|
|
86727
|
+
const authoritative = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86728
|
+
const claim = () => claims.run({ authority: { ...rowAuthority(workflowId, authoritative), claim: crypto.randomUUID() } }, callback);
|
|
86729
|
+
if (authoritative.kind === "current" && authoritative.metadata.status === "running" && authoritative.metadata.ownerExecutorId !== fence.executorId) {
|
|
86730
|
+
return await fence.recover(authoritative.metadata.ownerExecutorId, claim) ?? false;
|
|
86731
|
+
}
|
|
86732
|
+
return await claim();
|
|
86733
|
+
}, true),
|
|
86734
|
+
startWorkflow: (id, name, inputs) => mutate(id, () => raw.startWorkflow(id, name, inputs)),
|
|
86735
|
+
cancelWorkflow: (id) => mutate(id, () => raw.cancelWorkflow(id)),
|
|
86736
|
+
resumeWorkflow: (id) => mutate(id, () => raw.resumeWorkflow(id)),
|
|
86737
|
+
recordStepOutput: (id, step, output) => mutate(id, () => raw.recordStepOutput(id, step, output), { name: step, output }),
|
|
86738
|
+
deleteWorkflowData: (id) => mutate(id, () => raw.deleteWorkflowData(id))
|
|
86739
|
+
};
|
|
86100
86740
|
}
|
|
86101
86741
|
function isDbosDuplicateWorkflowError(err) {
|
|
86102
86742
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -86178,6 +86818,8 @@ async function configureDbosDurableBackend(config) {
|
|
|
86178
86818
|
const sdk = await importDbosSdk();
|
|
86179
86819
|
const owner = getDbosProcessOwner();
|
|
86180
86820
|
const existing = owner.wrappers;
|
|
86821
|
+
if (existing !== undefined && owner.executorFence === undefined)
|
|
86822
|
+
throw new Error("Existing workflow executor has no database ownership fence; restart the host before recovering workflows");
|
|
86181
86823
|
let launch = owner.active?.launch ?? (() => sdk.launch());
|
|
86182
86824
|
let checkReady;
|
|
86183
86825
|
if (existing === undefined) {
|
|
@@ -86200,12 +86842,30 @@ async function configureDbosDurableBackend(config) {
|
|
|
86200
86842
|
if (existing === undefined) {
|
|
86201
86843
|
owner.wrappers = { mainWorkflow, checkpointWorkflow };
|
|
86202
86844
|
}
|
|
86845
|
+
let backendFence = owner.executorFence;
|
|
86846
|
+
let backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), {
|
|
86847
|
+
checkReady
|
|
86848
|
+
});
|
|
86203
86849
|
return {
|
|
86204
|
-
backend
|
|
86205
|
-
|
|
86206
|
-
}
|
|
86207
|
-
launch
|
|
86208
|
-
|
|
86850
|
+
get backend() {
|
|
86851
|
+
return backend;
|
|
86852
|
+
},
|
|
86853
|
+
launch: async () => {
|
|
86854
|
+
await launch();
|
|
86855
|
+
if (backendFence !== owner.executorFence && backendFence?.closed) {
|
|
86856
|
+
backendFence = owner.executorFence;
|
|
86857
|
+
backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), { checkReady });
|
|
86858
|
+
}
|
|
86859
|
+
},
|
|
86860
|
+
shutdown: async () => {
|
|
86861
|
+
const results = await Promise.allSettled([
|
|
86862
|
+
sdk.shutdown(),
|
|
86863
|
+
...[...owner.executorFences ?? []].map((fence) => fence.close())
|
|
86864
|
+
]);
|
|
86865
|
+
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason instanceof Error ? result.reason : new Error(String(result.reason)));
|
|
86866
|
+
if (errors.length > 0)
|
|
86867
|
+
throw new AggregateError(errors, "Workflow database shutdown failed");
|
|
86868
|
+
}
|
|
86209
86869
|
};
|
|
86210
86870
|
}
|
|
86211
86871
|
async function importDbosSdk() {
|
|
@@ -86238,13 +86898,18 @@ class DbosDurableBackend {
|
|
|
86238
86898
|
unavailableAdmissions = new Set;
|
|
86239
86899
|
unavailableCheckpoints = new Set;
|
|
86240
86900
|
pendingRecoveryAdmissions = new Set;
|
|
86901
|
+
ownerLiveness = new Map;
|
|
86241
86902
|
admissionMetadataAttempted = new Set;
|
|
86242
86903
|
admissionSettlements = new Map;
|
|
86243
86904
|
admissionRecoveries = new Map;
|
|
86244
86905
|
admissionRecoveryControllers = new Map;
|
|
86906
|
+
fixedGeneration = false;
|
|
86907
|
+
generationIsCurrent;
|
|
86908
|
+
rebindInitialAdmission;
|
|
86909
|
+
admitted = false;
|
|
86245
86910
|
constructor(sdk, options) {
|
|
86246
86911
|
this.sdk = sdk;
|
|
86247
|
-
this.executorId = options?.executorId ?? getAtomicExecutorId();
|
|
86912
|
+
this.executorId = options?.executorId ?? sdk.executorId ?? getAtomicExecutorId();
|
|
86248
86913
|
this.onUnavailable = options?.onUnavailable;
|
|
86249
86914
|
this.checkReady = options?.checkReady;
|
|
86250
86915
|
this.promptReservations = new DbosPromptReservationTracker({
|
|
@@ -86255,6 +86920,55 @@ class DbosDurableBackend {
|
|
|
86255
86920
|
}
|
|
86256
86921
|
});
|
|
86257
86922
|
}
|
|
86923
|
+
async enrollLegacyWorkflow(workflowId, options) {
|
|
86924
|
+
if (options.olderWorkersStopped !== true || this.sdk.enrollLegacyWorkflow === undefined)
|
|
86925
|
+
return false;
|
|
86926
|
+
this.ensureGeneration();
|
|
86927
|
+
return await boundedAdmission((signal) => dbosAdmissionContext.run(signal, async () => {
|
|
86928
|
+
signal.throwIfAborted();
|
|
86929
|
+
const metadata = await this.sdk.enrollLegacyWorkflow(workflowId, options.modelOwner);
|
|
86930
|
+
signal.throwIfAborted();
|
|
86931
|
+
if (metadata === undefined)
|
|
86932
|
+
return false;
|
|
86933
|
+
this.applyMetadata(workflowId, metadata);
|
|
86934
|
+
this.current.add(workflowId);
|
|
86935
|
+
return true;
|
|
86936
|
+
}), options.signal);
|
|
86937
|
+
}
|
|
86938
|
+
executionView() {
|
|
86939
|
+
if (this.fixedGeneration || this.sdk.forkGeneration === undefined)
|
|
86940
|
+
return this;
|
|
86941
|
+
this.ensureGeneration();
|
|
86942
|
+
const view = new DbosDurableBackend(this.sdk, {
|
|
86943
|
+
executorId: this.executorId,
|
|
86944
|
+
checkReady: this.checkReady,
|
|
86945
|
+
onUnavailable: this.onUnavailable
|
|
86946
|
+
});
|
|
86947
|
+
const reservations = view.promptReservations;
|
|
86948
|
+
Object.assign(view, this);
|
|
86949
|
+
Object.assign(view, {
|
|
86950
|
+
promptReservations: reservations,
|
|
86951
|
+
fixedGeneration: true,
|
|
86952
|
+
generationIsCurrent: () => this.sdk === view.sdk,
|
|
86953
|
+
rebindInitialAdmission: () => {
|
|
86954
|
+
this.ensureGeneration();
|
|
86955
|
+
view.sdk = this.sdk;
|
|
86956
|
+
view.executorId = this.executorId;
|
|
86957
|
+
}
|
|
86958
|
+
});
|
|
86959
|
+
return view;
|
|
86960
|
+
}
|
|
86961
|
+
ensureGeneration() {
|
|
86962
|
+
if (this.fixedGeneration) {
|
|
86963
|
+
if (this.generationIsCurrent?.() === false)
|
|
86964
|
+
throw new DbosDependencyError("Workflow database ownership generation changed; stale execution writes are refused");
|
|
86965
|
+
return;
|
|
86966
|
+
}
|
|
86967
|
+
if (this.sdk.generationLost?.() && this.sdk.forkGeneration !== undefined) {
|
|
86968
|
+
this.sdk = this.sdk.forkGeneration();
|
|
86969
|
+
this.executorId = this.sdk.executorId ?? this.executorId;
|
|
86970
|
+
}
|
|
86971
|
+
}
|
|
86258
86972
|
isAdmissionUnavailable(workflowId) {
|
|
86259
86973
|
return this.unavailableAdmissions.has(workflowId);
|
|
86260
86974
|
}
|
|
@@ -86271,6 +86985,8 @@ class DbosDurableBackend {
|
|
|
86271
86985
|
return this.admissionSettlements.has(workflowId);
|
|
86272
86986
|
}
|
|
86273
86987
|
admitWorkflow(workflowId, registration, signal) {
|
|
86988
|
+
if (!this.admitted)
|
|
86989
|
+
this.rebindInitialAdmission?.();
|
|
86274
86990
|
const pending = this.performAdmission(workflowId, registration, signal);
|
|
86275
86991
|
this.admissionSettlements.set(workflowId, pending);
|
|
86276
86992
|
pending.then(() => {
|
|
@@ -86278,6 +86994,7 @@ class DbosDurableBackend {
|
|
|
86278
86994
|
this.admissionSettlements.delete(workflowId);
|
|
86279
86995
|
}, () => {});
|
|
86280
86996
|
return pending.then(async () => {
|
|
86997
|
+
this.admitted = true;
|
|
86281
86998
|
await dbosAdmissionContext.run(signal, () => this.flush(workflowId));
|
|
86282
86999
|
signal.throwIfAborted();
|
|
86283
87000
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
@@ -86298,12 +87015,13 @@ class DbosDurableBackend {
|
|
|
86298
87015
|
await this.flushWrites(workflowId);
|
|
86299
87016
|
signal.throwIfAborted();
|
|
86300
87017
|
}), signal);
|
|
87018
|
+
this.ensureGeneration();
|
|
86301
87019
|
this.admissionUnavailable = false;
|
|
86302
87020
|
this.unavailableAdmissions.delete(workflowId);
|
|
86303
87021
|
this.unavailableCheckpoints.delete(workflowId);
|
|
86304
87022
|
this.pendingRecoveryAdmissions.delete(workflowId);
|
|
86305
87023
|
} catch (error) {
|
|
86306
|
-
if (isDbosDependencyError(error)) {
|
|
87024
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
86307
87025
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
86308
87026
|
this.admissionUnavailable = true;
|
|
86309
87027
|
this.unavailableAdmissions.add(workflowId);
|
|
@@ -86329,6 +87047,7 @@ class DbosDurableBackend {
|
|
|
86329
87047
|
});
|
|
86330
87048
|
}
|
|
86331
87049
|
registerWorkflow(handle) {
|
|
87050
|
+
this.ensureGeneration();
|
|
86332
87051
|
if (this.invalid.has(handle.workflowId)) {
|
|
86333
87052
|
throw new DurableNestedTopologyError(`workflow ${handle.workflowId} contains malformed current DBOS records`);
|
|
86334
87053
|
}
|
|
@@ -86370,6 +87089,7 @@ class DbosDurableBackend {
|
|
|
86370
87089
|
return persisted;
|
|
86371
87090
|
}
|
|
86372
87091
|
recordCheckpoint(checkpoint) {
|
|
87092
|
+
this.ensureGeneration();
|
|
86373
87093
|
if (!this.isWorkflowLoadable(checkpoint.workflowId))
|
|
86374
87094
|
return;
|
|
86375
87095
|
this.mem.recordCheckpoint(checkpoint);
|
|
@@ -86398,7 +87118,7 @@ class DbosDurableBackend {
|
|
|
86398
87118
|
});
|
|
86399
87119
|
signal.throwIfAborted();
|
|
86400
87120
|
}), options?.signal ?? dbosAdmissionContext.getStore(), undefined, "Workflow database checkpoint timed out. Restore PostgreSQL and inspect the run before resuming; external outcomes may be unknown.").catch((error) => {
|
|
86401
|
-
if (isDbosDependencyError(error)) {
|
|
87121
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
86402
87122
|
this.unavailableCheckpoints.add(checkpoint.workflowId);
|
|
86403
87123
|
const errors = this.writeErrors.get(checkpoint.workflowId);
|
|
86404
87124
|
if (errors !== undefined)
|
|
@@ -86446,12 +87166,14 @@ class DbosDurableBackend {
|
|
|
86446
87166
|
return this.mem.listCheckpoints(workflowId);
|
|
86447
87167
|
}
|
|
86448
87168
|
getWorkflow(workflowId) {
|
|
86449
|
-
|
|
87169
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
87170
|
+
return handle === undefined || this.sdk.ownerLiveness === undefined ? handle : { ...handle, ownerLiveness: this.ownerLiveness.get(handle.ownerExecutorId ?? "") ?? "unknown" };
|
|
86450
87171
|
}
|
|
86451
87172
|
getLoadableWorkflow(workflowId) {
|
|
86452
|
-
return this.isWorkflowLoadable(workflowId) ? this.
|
|
87173
|
+
return this.isWorkflowLoadable(workflowId) ? this.getWorkflow(workflowId) : undefined;
|
|
86453
87174
|
}
|
|
86454
87175
|
setWorkflowStatus(workflowId, status, pendingPrompts, resumable, failure) {
|
|
87176
|
+
this.ensureGeneration();
|
|
86455
87177
|
if (status === "cancelled" || status === "completed" || resumable === false) {
|
|
86456
87178
|
this.admissionRecoveryControllers.get(workflowId)?.abort(new Error("Workflow recovery superseded by terminal control."));
|
|
86457
87179
|
}
|
|
@@ -86475,9 +87197,10 @@ class DbosDurableBackend {
|
|
|
86475
87197
|
await this.writeMetadata(workflowId);
|
|
86476
87198
|
});
|
|
86477
87199
|
}
|
|
86478
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
87200
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
87201
|
+
this.ensureGeneration();
|
|
86479
87202
|
let records = [];
|
|
86480
|
-
|
|
87203
|
+
const transition = () => transitionDbosWorkflowStatus({
|
|
86481
87204
|
expectedStatuses: expected,
|
|
86482
87205
|
status,
|
|
86483
87206
|
flush: () => this.flush(workflowId),
|
|
@@ -86491,14 +87214,25 @@ class DbosDurableBackend {
|
|
|
86491
87214
|
const pendingPrompts = this.promptReservations.hydrate(workflowId, entry.pendingPrompts, records, entry.promptReservationEpoch);
|
|
86492
87215
|
this.applyMetadata(workflowId, { ...entry, pendingPrompts });
|
|
86493
87216
|
},
|
|
86494
|
-
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable),
|
|
87217
|
+
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner),
|
|
86495
87218
|
write: async () => {
|
|
87219
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
87220
|
+
if (handle !== undefined)
|
|
87221
|
+
this.mem.registerWorkflow({
|
|
87222
|
+
...handle,
|
|
87223
|
+
legacyRecoveryPending: undefined,
|
|
87224
|
+
...modelOwner !== undefined ? { modelOwner } : {}
|
|
87225
|
+
}, true);
|
|
86496
87226
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
86497
87227
|
await this.flush(workflowId);
|
|
86498
87228
|
}
|
|
86499
87229
|
});
|
|
87230
|
+
if (this.sdk.withWorkflowClaim === undefined)
|
|
87231
|
+
return await transition();
|
|
87232
|
+
await this.flush(workflowId);
|
|
87233
|
+
return await this.sdk.withWorkflowClaim(workflowId, transition);
|
|
86500
87234
|
}
|
|
86501
|
-
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable) {
|
|
87235
|
+
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner) {
|
|
86502
87236
|
const stepName = claimMetadataStepName(generation);
|
|
86503
87237
|
const transitionClaimId = crypto.randomUUID();
|
|
86504
87238
|
const claim = {
|
|
@@ -86506,6 +87240,8 @@ class DbosDurableBackend {
|
|
|
86506
87240
|
status,
|
|
86507
87241
|
...pendingPrompts !== undefined ? { pendingPrompts } : {},
|
|
86508
87242
|
...resumable !== undefined ? { resumable } : {},
|
|
87243
|
+
...modelOwner === undefined ? {} : { modelOwner },
|
|
87244
|
+
legacyRecoveryPending: undefined,
|
|
86509
87245
|
ownerExecutorId: this.executorId,
|
|
86510
87246
|
transitionClaimId,
|
|
86511
87247
|
updatedAt: Math.max(Date.now(), authoritative.updatedAt + 1)
|
|
@@ -86647,7 +87383,7 @@ class DbosDurableBackend {
|
|
|
86647
87383
|
this.mem.recordCheckpoint(classified.checkpoint);
|
|
86648
87384
|
}
|
|
86649
87385
|
const handle = this.getLoadableWorkflow(workflowId);
|
|
86650
|
-
return handle === undefined ? { kind: "malformed" } :
|
|
87386
|
+
return handle === undefined ? { kind: "malformed" } : await this.withOwnerLiveness(handle);
|
|
86651
87387
|
}
|
|
86652
87388
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
86653
87389
|
if (info !== undefined)
|
|
@@ -86670,6 +87406,7 @@ class DbosDurableBackend {
|
|
|
86670
87406
|
}
|
|
86671
87407
|
}
|
|
86672
87408
|
reconcileWorkflowAdmission(workflowId, signal) {
|
|
87409
|
+
this.ensureGeneration();
|
|
86673
87410
|
signal?.throwIfAborted();
|
|
86674
87411
|
if (!this.unavailableAdmissions.has(workflowId) && !this.unavailableCheckpoints.has(workflowId))
|
|
86675
87412
|
return Promise.resolve();
|
|
@@ -86684,11 +87421,13 @@ class DbosDurableBackend {
|
|
|
86684
87421
|
return;
|
|
86685
87422
|
if (metadata.resumable === false || metadata.status === "cancelled" || metadata.status === "completed")
|
|
86686
87423
|
return;
|
|
86687
|
-
if (!await reconcileDbosAdmission(this.sdk, this.promptReservations.metadata(workflowId, metadata), this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
87424
|
+
if (!await reconcileDbosAdmission(this.sdk, { ...this.promptReservations.metadata(workflowId, metadata), ownerExecutorId: this.executorId }, this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
86688
87425
|
await this.suppressWorkflow(workflowId);
|
|
86689
87426
|
return;
|
|
86690
87427
|
}
|
|
86691
87428
|
admissionSignal.throwIfAborted();
|
|
87429
|
+
await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
87430
|
+
admissionSignal.throwIfAborted();
|
|
86692
87431
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
86693
87432
|
if (info === undefined)
|
|
86694
87433
|
return;
|
|
@@ -86725,7 +87464,7 @@ class DbosDurableBackend {
|
|
|
86725
87464
|
return { kind: deletion === "current" ? "deleted" : "malformed" };
|
|
86726
87465
|
}
|
|
86727
87466
|
const metadata = classifyLatestMetadata(records, info.workflowId);
|
|
86728
|
-
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !
|
|
87467
|
+
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !isDeepStrictEqual3(metadata.metadata.inputs, expected.inputs))) {
|
|
86729
87468
|
await this.suppressWorkflow(info.workflowId);
|
|
86730
87469
|
return { kind: "malformed" };
|
|
86731
87470
|
}
|
|
@@ -86759,10 +87498,17 @@ class DbosDurableBackend {
|
|
|
86759
87498
|
});
|
|
86760
87499
|
const handle = this.getLoadableWorkflow(info.workflowId);
|
|
86761
87500
|
if (handle !== undefined)
|
|
86762
|
-
return
|
|
87501
|
+
return await this.withOwnerLiveness(handle);
|
|
86763
87502
|
await this.suppressWorkflow(info.workflowId);
|
|
86764
87503
|
return { kind: "malformed" };
|
|
86765
87504
|
}
|
|
87505
|
+
async withOwnerLiveness(handle) {
|
|
87506
|
+
if (this.sdk.ownerLiveness === undefined)
|
|
87507
|
+
return { kind: "current", handle };
|
|
87508
|
+
const ownerLiveness = await this.sdk.ownerLiveness(handle.ownerExecutorId);
|
|
87509
|
+
this.ownerLiveness.set(handle.ownerExecutorId ?? "", ownerLiveness);
|
|
87510
|
+
return { kind: "current", handle: { ...handle, ownerLiveness } };
|
|
87511
|
+
}
|
|
86766
87512
|
async suppressWorkflow(workflowId) {
|
|
86767
87513
|
this.invalid.add(workflowId);
|
|
86768
87514
|
this.current.delete(workflowId);
|
|
@@ -86776,10 +87522,21 @@ class DbosDurableBackend {
|
|
|
86776
87522
|
return Promise.resolve();
|
|
86777
87523
|
const previous = this.writeQueues.get(workflowId) ?? Promise.resolve();
|
|
86778
87524
|
const next = previous.then(async () => {
|
|
87525
|
+
this.ensureGeneration();
|
|
87526
|
+
const adopted = await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
87527
|
+
const local = this.mem.getWorkflow(workflowId);
|
|
87528
|
+
if (adopted !== undefined && local !== undefined)
|
|
87529
|
+
this.mem.registerWorkflow({
|
|
87530
|
+
...local,
|
|
87531
|
+
ownerExecutorId: adopted.ownerExecutorId,
|
|
87532
|
+
updatedAt: Math.max(local.updatedAt, adopted.updatedAt + 1)
|
|
87533
|
+
}, true);
|
|
86779
87534
|
if (!signal?.aborted)
|
|
86780
87535
|
await fn();
|
|
86781
87536
|
});
|
|
86782
87537
|
const tracked = next.catch((err) => {
|
|
87538
|
+
if (this.generationIsCurrent?.() === false)
|
|
87539
|
+
return;
|
|
86783
87540
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
86784
87541
|
const errors = this.writeErrors.get(workflowId) ?? [];
|
|
86785
87542
|
errors.push(error);
|
|
@@ -86816,7 +87573,7 @@ class DbosDurableBackend {
|
|
|
86816
87573
|
applyMetadata(workflowId, metadata) {
|
|
86817
87574
|
if (metadata.workflowId !== workflowId)
|
|
86818
87575
|
return;
|
|
86819
|
-
this.mem.registerWorkflow(metadata);
|
|
87576
|
+
this.mem.registerWorkflow(metadata, true);
|
|
86820
87577
|
}
|
|
86821
87578
|
}
|
|
86822
87579
|
|
|
@@ -86852,7 +87609,7 @@ import { tmpdir, uptime } from "node:os";
|
|
|
86852
87609
|
import { basename as basename2, dirname as dirname4, isAbsolute as isAbsolute3, join as join5, relative as relative2, sep as sep2 } from "node:path";
|
|
86853
87610
|
|
|
86854
87611
|
// dist/builtin/workflows/src/durable/dbos-embedded-postgres-root.ts
|
|
86855
|
-
import { createHash as
|
|
87612
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
86856
87613
|
import {
|
|
86857
87614
|
chmod,
|
|
86858
87615
|
chown,
|
|
@@ -87335,8 +88092,8 @@ async function snapshotSourceRuntime(root, publisher, progress) {
|
|
|
87335
88092
|
const rootStat = await lstat(root);
|
|
87336
88093
|
assertRuntimeRoot(root, rootStat);
|
|
87337
88094
|
const rootRealPath = await realpath(root);
|
|
87338
|
-
const sourceHash =
|
|
87339
|
-
const sealedHash =
|
|
88095
|
+
const sourceHash = createHash4("sha256");
|
|
88096
|
+
const sealedHash = createHash4("sha256");
|
|
87340
88097
|
await hashSourceEntry(sourceHash, sealedHash, root, ".", rootRealPath, publisher, progress);
|
|
87341
88098
|
return { sourceIdentity: sourceHash.digest("hex"), sealedIdentity: sealedHash.digest("hex") };
|
|
87342
88099
|
}
|
|
@@ -87601,7 +88358,7 @@ async function snapshotSealedManifest(root, progress) {
|
|
|
87601
88358
|
const rootStat = await lstat(root);
|
|
87602
88359
|
assertRuntimeRoot(root, rootStat);
|
|
87603
88360
|
const rootRealPath = await realpath(root);
|
|
87604
|
-
const hash =
|
|
88361
|
+
const hash = createHash4("sha256");
|
|
87605
88362
|
const entries = [];
|
|
87606
88363
|
await hashSealedEntry(hash, root, ".", rootRealPath, progress, entries);
|
|
87607
88364
|
return { version: 1, sealedIdentity: hash.digest("hex"), entries };
|
|
@@ -87835,6 +88592,13 @@ function resolveEmbeddedPostgresTarget(host) {
|
|
|
87835
88592
|
init_dbos_admission();
|
|
87836
88593
|
var HEALTH_INTERVAL_MS = 5000;
|
|
87837
88594
|
var RECOVERY_ATTEMPTS = 3;
|
|
88595
|
+
function isMonitoringConnectionTimeout(error) {
|
|
88596
|
+
if (!(error instanceof Error))
|
|
88597
|
+
return false;
|
|
88598
|
+
if ("code" in error)
|
|
88599
|
+
return error.code === "ETIMEDOUT";
|
|
88600
|
+
return error.message === "timeout expired" || error.message === "Connection terminated due to connection timeout";
|
|
88601
|
+
}
|
|
87838
88602
|
function isMonitoringConnectionFailure(error) {
|
|
87839
88603
|
if (!(error instanceof Error))
|
|
87840
88604
|
return false;
|
|
@@ -87945,7 +88709,7 @@ class PostgresHealth {
|
|
|
87945
88709
|
try {
|
|
87946
88710
|
identity = await this.operations.probe();
|
|
87947
88711
|
} catch (retryError) {
|
|
87948
|
-
if (!isMonitoringConnectionFailure(retryError))
|
|
88712
|
+
if (isMonitoringConnectionTimeout(retryError) || !isMonitoringConnectionFailure(retryError))
|
|
87949
88713
|
throw retryError;
|
|
87950
88714
|
this.retainFailure(retryError);
|
|
87951
88715
|
return;
|
|
@@ -87971,7 +88735,7 @@ class PostgresHealth {
|
|
|
87971
88735
|
if (ready !== undefined)
|
|
87972
88736
|
return ready;
|
|
87973
88737
|
} catch (error) {
|
|
87974
|
-
if (isQueryReadTimeout(error)) {
|
|
88738
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error)) {
|
|
87975
88739
|
this.retainFailure(new Error("Managed PostgreSQL did not answer a health check in time."));
|
|
87976
88740
|
throw this.dependencyFailure("Managed PostgreSQL did not answer a health check in time.");
|
|
87977
88741
|
}
|
|
@@ -88279,7 +89043,7 @@ async function probePostgresIdentity(port) {
|
|
|
88279
89043
|
const result = await client.query(POSTGRES_IDENTITY_SQL);
|
|
88280
89044
|
return result.rows[0];
|
|
88281
89045
|
} catch (error) {
|
|
88282
|
-
if (isQueryReadTimeout(error))
|
|
89046
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
88283
89047
|
throw error;
|
|
88284
89048
|
const code = error instanceof Error && "code" in error ? error.code : undefined;
|
|
88285
89049
|
if (code === "ECONNREFUSED" || code === "ECONNRESET" || code === "EPIPE" || code === "57P01" || code === "57P02" || code === "57P03" || error instanceof Error && /timeout|timed out|connection terminated/i.test(error.message))
|
|
@@ -88452,7 +89216,7 @@ async function ensureCluster(options = {}) {
|
|
|
88452
89216
|
try {
|
|
88453
89217
|
return await existingServerProbe()(probePort);
|
|
88454
89218
|
} catch (error) {
|
|
88455
|
-
if (isQueryReadTimeout(error))
|
|
89219
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
88456
89220
|
return;
|
|
88457
89221
|
throw error;
|
|
88458
89222
|
}
|
|
@@ -90568,6 +91332,9 @@ function createDurableAdmissionSettlement(input, signal) {
|
|
|
90568
91332
|
};
|
|
90569
91333
|
}
|
|
90570
91334
|
|
|
91335
|
+
// dist/builtin/workflows/src/runs/background/quit.ts
|
|
91336
|
+
init_dbos_admission();
|
|
91337
|
+
|
|
90571
91338
|
// dist/builtin/workflows/src/shared/resume-continuation.ts
|
|
90572
91339
|
var RESUME_CONTINUATION_PROMPT = "Continue where you left off. If you believe you are finished with your original task (or a redefined task if the user told you), stop.";
|
|
90573
91340
|
|
|
@@ -91954,7 +92721,8 @@ async function quitRunWithAction(runId, opts, action) {
|
|
|
91954
92721
|
if (await settle() === "refused")
|
|
91955
92722
|
return { ok: false, runId, reason: "already_ended" };
|
|
91956
92723
|
} catch (error) {
|
|
91957
|
-
|
|
92724
|
+
const alreadyDependencyPaused = isDbosDependencyError(error) && current.status === "paused" && current.phase === "blocked_dependency";
|
|
92725
|
+
if (!suspendedByAbort && !alreadyDependencyPaused)
|
|
91958
92726
|
throw error;
|
|
91959
92727
|
const preservedProgress = hasDurableQuitProgress(runId);
|
|
91960
92728
|
publish(preservedProgress);
|
|
@@ -101292,7 +102060,7 @@ function createRegistry(initial = []) {
|
|
|
101292
102060
|
|
|
101293
102061
|
// dist/builtin/workflows/src/extension/discovery-loaders.ts
|
|
101294
102062
|
import { readdir as readdir3, stat as stat2 } from "node:fs/promises";
|
|
101295
|
-
import { extname, isAbsolute as isAbsolute14, join as join30, resolve as
|
|
102063
|
+
import { extname, isAbsolute as isAbsolute14, join as join30, resolve as resolve15 } from "node:path";
|
|
101296
102064
|
|
|
101297
102065
|
// ../../node_modules/jiti/lib/jiti-static.mjs
|
|
101298
102066
|
var import_jiti = __toESM(require_jiti(), 1);
|
|
@@ -101747,12 +102515,12 @@ function raceAbort2(promise, signal) {
|
|
|
101747
102515
|
});
|
|
101748
102516
|
}
|
|
101749
102517
|
// dist/builtin/workflows/src/runs/foreground/executor-hil.ts
|
|
101750
|
-
import { createHash as
|
|
102518
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
101751
102519
|
import { createAskUserQuestionToolDefinition } from "@bastani/atomic";
|
|
101752
102520
|
|
|
101753
102521
|
// dist/builtin/workflows/src/shared/prompt-callsite-context.ts
|
|
101754
|
-
import { AsyncLocalStorage as
|
|
101755
|
-
var promptCallerStacks = new
|
|
102522
|
+
import { AsyncLocalStorage as AsyncLocalStorage4 } from "node:async_hooks";
|
|
102523
|
+
var promptCallerStacks = new AsyncLocalStorage4;
|
|
101756
102524
|
function withPromptCallerStack(stack, delegate) {
|
|
101757
102525
|
if (stack === undefined)
|
|
101758
102526
|
return delegate();
|
|
@@ -102046,7 +102814,7 @@ function makePrompt(descriptor) {
|
|
|
102046
102814
|
};
|
|
102047
102815
|
}
|
|
102048
102816
|
function stableHash(value2) {
|
|
102049
|
-
return
|
|
102817
|
+
return createHash5("sha256").update(JSON.stringify(value2)).digest("hex").slice(0, 32);
|
|
102050
102818
|
}
|
|
102051
102819
|
function promptDescriptorHash(descriptor) {
|
|
102052
102820
|
if (isCustomPromptDescriptor(descriptor)) {
|
|
@@ -106900,6 +107668,16 @@ function workflowStartupFailureMessage(admission, snapshotError, fallback) {
|
|
|
106900
107668
|
// dist/builtin/workflows/src/durable/resume-runtime.ts
|
|
106901
107669
|
init_dbos_admission();
|
|
106902
107670
|
|
|
107671
|
+
// dist/builtin/workflows/src/durable/sdk-recovery-scope.ts
|
|
107672
|
+
init_dbos_owner_fence();
|
|
107673
|
+
import { resolve as resolve13 } from "node:path";
|
|
107674
|
+
function isSdkDurableRunInScope(handle, cwd) {
|
|
107675
|
+
return cwd !== undefined && handle.invocationCwd !== undefined && resolve13(cwd) === resolve13(handle.invocationCwd) && handle.modelOwner !== undefined && (handle.rootWorkflowId === undefined || handle.rootWorkflowId === handle.workflowId);
|
|
107676
|
+
}
|
|
107677
|
+
function isSdkCrashedRunAdoptable(handle, cwd) {
|
|
107678
|
+
return isSdkDurableRunInScope(handle, cwd) && handle.status === "running" && handle.ownerExecutorId !== undefined && handle.ownerExecutorId !== getAtomicExecutorId() && !isLiveRunningWorkflow(handle) && isDatabaseExecutor(handle.ownerExecutorId) && handle.ownerLiveness === "dead";
|
|
107679
|
+
}
|
|
107680
|
+
|
|
106903
107681
|
// dist/builtin/workflows/src/durable/tool-resume-frontier.ts
|
|
106904
107682
|
function resolveToolResumeFrontier(source, backend) {
|
|
106905
107683
|
const fail = (detail) => ({
|
|
@@ -107083,6 +107861,11 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107083
107861
|
message: `Workflow ${resolved.workflowId} is ${handle.status}, not resumable.`
|
|
107084
107862
|
};
|
|
107085
107863
|
}
|
|
107864
|
+
const requestedOwner = deps.baseRunOpts.modelOwner;
|
|
107865
|
+
const adopting = requestedOwner !== undefined && requestedOwner !== handle.modelOwner && (handle.modelOwner !== undefined || handle.origin === "agent");
|
|
107866
|
+
if (adopting && !isSdkCrashedRunAdoptable(handle, deps.baseRunOpts.cwd)) {
|
|
107867
|
+
return { ok: false, reason: "owned_elsewhere", message: "Workflow instance belongs to another caller/session." };
|
|
107868
|
+
}
|
|
107086
107869
|
const def = handle.invocationCwd === undefined ? deps.registry.get(handle.name) : await deps.resolveDefinition?.(handle.name, handle.invocationCwd) ?? deps.registry.get(handle.name);
|
|
107087
107870
|
if (def === undefined) {
|
|
107088
107871
|
return { ok: false, reason: "workflow_not_found", message: `Workflow definition not found: ${handle.name}` };
|
|
@@ -107132,6 +107915,22 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107132
107915
|
failedToolNodeId: handle.failedToolNodeId
|
|
107133
107916
|
};
|
|
107134
107917
|
const sourceSnapshot = toolContinuation?.source ?? deps.baseRunOpts.store?.runs().find((run) => run.id === resolved.workflowId);
|
|
107918
|
+
let adoptedClaimUpdatedAt;
|
|
107919
|
+
const rollbackStartup = async (onRestored) => {
|
|
107920
|
+
if (adopting) {
|
|
107921
|
+
const current = backend.getWorkflow(resolved.workflowId);
|
|
107922
|
+
if (adoptedClaimUpdatedAt === undefined || current?.modelOwner !== requestedOwner)
|
|
107923
|
+
return false;
|
|
107924
|
+
const restored = await backend.transitionWorkflowStatus(resolved.workflowId, ["running"], handle.status, handle.pendingPrompts, handle.resumable, adoptedClaimUpdatedAt);
|
|
107925
|
+
if (restored)
|
|
107926
|
+
onRestored?.();
|
|
107927
|
+
return restored;
|
|
107928
|
+
}
|
|
107929
|
+
backend.setWorkflowStatus(resolved.workflowId, handle.status, handle.pendingPrompts, handle.resumable, sourceFailure);
|
|
107930
|
+
await backend.flush(resolved.workflowId);
|
|
107931
|
+
onRestored?.();
|
|
107932
|
+
return true;
|
|
107933
|
+
};
|
|
107135
107934
|
let claimed;
|
|
107136
107935
|
try {
|
|
107137
107936
|
claimed = await boundedAdmission((signal) => dbosAdmissionContext.run(signal, async () => {
|
|
@@ -107142,10 +107941,14 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107142
107941
|
}
|
|
107143
107942
|
unsettled.add(resolved.workflowId);
|
|
107144
107943
|
try {
|
|
107145
|
-
const accepted = await backend.transitionWorkflowStatus(resolved.workflowId, [handle.status], "running", undefined, undefined, resolved.updatedAt);
|
|
107146
|
-
if (accepted &&
|
|
107147
|
-
backend.
|
|
107148
|
-
|
|
107944
|
+
const accepted = await backend.transitionWorkflowStatus(resolved.workflowId, [handle.status], "running", undefined, undefined, resolved.updatedAt, adopting ? requestedOwner : undefined);
|
|
107945
|
+
if (accepted && adopting) {
|
|
107946
|
+
const current = backend.getWorkflow(resolved.workflowId);
|
|
107947
|
+
if (current?.status === "running" && current.modelOwner === requestedOwner)
|
|
107948
|
+
adoptedClaimUpdatedAt = current.updatedAt;
|
|
107949
|
+
}
|
|
107950
|
+
if (accepted && !adopting && signal.aborted && backend.getWorkflow(resolved.workflowId)?.status === "running") {
|
|
107951
|
+
await rollbackStartup();
|
|
107149
107952
|
}
|
|
107150
107953
|
return accepted;
|
|
107151
107954
|
} finally {
|
|
@@ -107172,7 +107975,7 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107172
107975
|
...deps.baseRunOpts,
|
|
107173
107976
|
...handle.invocationCwd !== undefined ? { cwd: handle.invocationCwd } : {},
|
|
107174
107977
|
...handle.origin !== undefined ? { origin: handle.origin } : {},
|
|
107175
|
-
modelOwner: handle.modelOwner,
|
|
107978
|
+
modelOwner: adopting ? requestedOwner : handle.modelOwner,
|
|
107176
107979
|
runId: resolved.workflowId,
|
|
107177
107980
|
durableBackend: backend,
|
|
107178
107981
|
...toolContinuation === undefined ? {} : { continuation: toolContinuation }
|
|
@@ -107198,10 +108001,10 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107198
108001
|
}
|
|
107199
108002
|
});
|
|
107200
108003
|
} catch (error) {
|
|
107201
|
-
|
|
107202
|
-
|
|
107203
|
-
|
|
107204
|
-
|
|
108004
|
+
await rollbackStartup(() => {
|
|
108005
|
+
if (sourceSnapshot !== undefined)
|
|
108006
|
+
deps.baseRunOpts.store?.recordRunStart(sourceSnapshot);
|
|
108007
|
+
});
|
|
107205
108008
|
return {
|
|
107206
108009
|
ok: false,
|
|
107207
108010
|
reason: "startup_failed",
|
|
@@ -107215,12 +108018,14 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107215
108018
|
const snapshot = deps.baseRunOpts.store?.runs().find((run) => run.id === accepted.runId);
|
|
107216
108019
|
const error = workflowStartupFailureMessage(admission, snapshot?.error, `Workflow ${resolved.workflowId} ended before startup admission`);
|
|
107217
108020
|
if (!backend.isAdmissionUnavailable?.(resolved.workflowId)) {
|
|
107218
|
-
if (sourceSnapshot === undefined)
|
|
108021
|
+
if (!adopting && sourceSnapshot === undefined)
|
|
107219
108022
|
deps.baseRunOpts.store?.removeRun(accepted.runId);
|
|
107220
|
-
|
|
107221
|
-
|
|
107222
|
-
|
|
107223
|
-
|
|
108023
|
+
await rollbackStartup(() => {
|
|
108024
|
+
if (adopting && sourceSnapshot === undefined)
|
|
108025
|
+
deps.baseRunOpts.store?.removeRun(accepted.runId);
|
|
108026
|
+
if (sourceSnapshot !== undefined)
|
|
108027
|
+
deps.baseRunOpts.store?.recordRunStart(sourceSnapshot);
|
|
108028
|
+
});
|
|
107224
108029
|
}
|
|
107225
108030
|
return {
|
|
107226
108031
|
ok: false,
|
|
@@ -112626,9 +113431,9 @@ class StageSessionController {
|
|
|
112626
113431
|
}
|
|
112627
113432
|
|
|
112628
113433
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
112629
|
-
import { createHash as
|
|
113434
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
112630
113435
|
import { mkdir as mkdir7, writeFile as writeFile11 } from "node:fs/promises";
|
|
112631
|
-
import { basename as basename6, dirname as dirname13, isAbsolute as isAbsolute13, join as join29, resolve as
|
|
113436
|
+
import { basename as basename6, dirname as dirname13, isAbsolute as isAbsolute13, join as join29, resolve as resolve14 } from "node:path";
|
|
112632
113437
|
var DEFAULT_MAX_OUTPUT_BYTES = 200 * 1024;
|
|
112633
113438
|
var DEFAULT_MAX_OUTPUT_LINES = 5000;
|
|
112634
113439
|
function normalizeMaxOutput2(maxOutput) {
|
|
@@ -112692,7 +113497,7 @@ function formatByteSize(bytes) {
|
|
|
112692
113497
|
return `${value2.toFixed(1)} ${units[unitIndex]}`;
|
|
112693
113498
|
}
|
|
112694
113499
|
function transcriptPath(runId, outputPath) {
|
|
112695
|
-
const digest =
|
|
113500
|
+
const digest = createHash6("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
112696
113501
|
return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename6(outputPath)}.transcript.md`);
|
|
112697
113502
|
}
|
|
112698
113503
|
function appendContent(lines, content) {
|
|
@@ -112738,7 +113543,7 @@ function renderTranscript(messages, fallbackOutput) {
|
|
|
112738
113543
|
`);
|
|
112739
113544
|
}
|
|
112740
113545
|
function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
|
|
112741
|
-
const absolutePath =
|
|
113546
|
+
const absolutePath = resolve14(outputPath);
|
|
112742
113547
|
const bytes = Buffer.byteLength(fullOutput, "utf8");
|
|
112743
113548
|
const lines = countLines(fullOutput);
|
|
112744
113549
|
const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
|
|
@@ -112753,8 +113558,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
|
|
|
112753
113558
|
return;
|
|
112754
113559
|
if (isAbsolute13(output))
|
|
112755
113560
|
return output;
|
|
112756
|
-
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute13(requestedCwd) ? requestedCwd :
|
|
112757
|
-
return
|
|
113561
|
+
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute13(requestedCwd) ? requestedCwd : resolve14(runtimeCwd, requestedCwd);
|
|
113562
|
+
return resolve14(baseCwd, output);
|
|
112758
113563
|
}
|
|
112759
113564
|
function degenerateWarning(fullOutput) {
|
|
112760
113565
|
return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
|
|
@@ -113794,7 +114599,8 @@ async function run(def, inputs, opts = {}) {
|
|
|
113794
114599
|
ownController.abort(callerSignal?.reason);
|
|
113795
114600
|
};
|
|
113796
114601
|
const exit = createWorkflowExitManager({ runId, exitScope, controller: ownController });
|
|
113797
|
-
const
|
|
114602
|
+
const sourceBackend = opts.durableBackend ?? getDurableBackend();
|
|
114603
|
+
const backendView = sourceBackend.executionView?.() ?? sourceBackend;
|
|
113798
114604
|
const rootBackend = opts.durableRootBackend ?? backendView;
|
|
113799
114605
|
const durableBackend = opts.durableScope !== undefined ? new ScopedDurableBackend(backendView, opts.durableScope) : backendView;
|
|
113800
114606
|
const inheritedElapsedMs = opts.parentRun === undefined ? inheritedRunElapsedMs({ backend: durableBackend, runId, continuationSource: opts.continuation?.source }) : undefined;
|
|
@@ -114554,6 +115360,17 @@ async function run(def, inputs, opts = {}) {
|
|
|
114554
115360
|
const gracefulQuit = observedQuitCancellation() ?? observedTaskTailQuit ?? findWorkflowGracefulQuit(err) ?? findWorkflowGracefulQuit(ownController.signal.reason);
|
|
114555
115361
|
if (gracefulQuit !== undefined)
|
|
114556
115362
|
return suspendForGracefulQuit(gracefulQuit);
|
|
115363
|
+
if (isDbosDependencyError(err) && durableBackend.isCheckpointUnavailable?.(runId) && (durableBackend.getWorkflow(runId)?.completedCheckpoints ?? 0) > 0) {
|
|
115364
|
+
await admittedTools.closeAndDrain();
|
|
115365
|
+
activeStore.recordRunPaused(runId, undefined, { resumable: true });
|
|
115366
|
+
activeStore.recordRunExecutionState(runId, { phase: "blocked_dependency", dependencyError: err.message });
|
|
115367
|
+
return {
|
|
115368
|
+
runId,
|
|
115369
|
+
status: "paused",
|
|
115370
|
+
stages: [...runSnapshot.stages],
|
|
115371
|
+
toolNodes: [...runSnapshot.toolNodes ?? []]
|
|
115372
|
+
};
|
|
115373
|
+
}
|
|
114557
115374
|
await admittedTools.closeAndDrain();
|
|
114558
115375
|
if (ownController.signal.aborted)
|
|
114559
115376
|
await Promise.allSettled([...pendingChildWorkflows]);
|
|
@@ -114892,7 +115709,7 @@ async function loadFromPaths(pathsOrMap, kind, baseCwd, diagnostics) {
|
|
|
114892
115709
|
const all = [];
|
|
114893
115710
|
const entries = Array.isArray(pathsOrMap) ? pathsOrMap.map((p) => ({ rawPath: p })) : Object.entries(pathsOrMap).map(([name, p]) => ({ rawPath: p, configuredName: name }));
|
|
114894
115711
|
for (const { rawPath, configuredName } of entries) {
|
|
114895
|
-
const absPath = isAbsolute14(rawPath) ? rawPath :
|
|
115712
|
+
const absPath = isAbsolute14(rawPath) ? rawPath : resolve15(baseCwd, rawPath);
|
|
114896
115713
|
let pathStats;
|
|
114897
115714
|
try {
|
|
114898
115715
|
pathStats = await stat2(absPath);
|
|
@@ -115665,7 +116482,8 @@ function createDurableResumeRuntime(deps) {
|
|
|
115665
116482
|
baseRunOpts: {
|
|
115666
116483
|
...deps.baseRunOpts(options?.policy),
|
|
115667
116484
|
...options?.budget === undefined ? {} : { budget: options.budget },
|
|
115668
|
-
...options?.actor === undefined ? {} : { resumeActor: options.actor }
|
|
116485
|
+
...options?.actor === undefined ? {} : { resumeActor: options.actor },
|
|
116486
|
+
...options?.modelOwner === undefined ? {} : { modelOwner: options.modelOwner }
|
|
115669
116487
|
},
|
|
115670
116488
|
durableBackend: backend,
|
|
115671
116489
|
resolveDefinition: async (name, cwd) => (await discoverWorkflows({ cwd: cwd ?? deps.runtimeCwd })).registry.get(name),
|
|
@@ -116050,7 +116868,7 @@ function createStatusWriter(store, config, opts = {}) {
|
|
|
116050
116868
|
import { readFileSync as readFileSync7 } from "node:fs";
|
|
116051
116869
|
import { readFile as readFile6 } from "node:fs/promises";
|
|
116052
116870
|
import { homedir as homedir3 } from "node:os";
|
|
116053
|
-
import { join as join33, resolve as
|
|
116871
|
+
import { join as join33, resolve as resolve16 } from "node:path";
|
|
116054
116872
|
import { getAgentDir as getAgentDir3, getAgentDirs as getAgentDirs3, getProjectConfigDirs, SettingsManager } from "@bastani/atomic";
|
|
116055
116873
|
var DATABASE_SETTING_KEYS = new Set(["systemDatabaseUrl", "systemDatabaseUrlFile"]);
|
|
116056
116874
|
function databaseSettings(settings) {
|
|
@@ -116099,7 +116917,7 @@ async function resolveWorkflowDatabaseSettings(global2, project, directories) {
|
|
|
116099
116917
|
if (typeof file !== "string" || file.trim() === "") {
|
|
116100
116918
|
throw new TypeError("workflows.durability.systemDatabaseUrlFile must be a non-empty path");
|
|
116101
116919
|
}
|
|
116102
|
-
const path = file.startsWith("~/") || file.startsWith("~\\") ? join33(homedir3(), file.slice(2)) :
|
|
116920
|
+
const path = file.startsWith("~/") || file.startsWith("~\\") ? join33(homedir3(), file.slice(2)) : resolve16(projectDatabase === undefined ? directories.global : directories.project, file);
|
|
116103
116921
|
try {
|
|
116104
116922
|
value2 = await readFile6(path, "utf8");
|
|
116105
116923
|
} catch {
|
|
@@ -121049,7 +121867,7 @@ function createSessionRunControl(host) {
|
|
|
121049
121867
|
const ctx = executeContext(host.context());
|
|
121050
121868
|
if (args.runId !== undefined && (args.action === "status" || args.action === "stages"))
|
|
121051
121869
|
rejectUnknownRunPrefix(host.store, args.runId);
|
|
121052
|
-
return await host.execute(args, ctx);
|
|
121870
|
+
return await host.execute(args, ctx, undefined, undefined, "sdk");
|
|
121053
121871
|
} catch (error) {
|
|
121054
121872
|
throw error instanceof Error ? translateThrown(error, args.runId) : error;
|
|
121055
121873
|
}
|
|
@@ -121100,7 +121918,8 @@ function createSessionRunControl(host) {
|
|
|
121100
121918
|
action: "resume",
|
|
121101
121919
|
runId,
|
|
121102
121920
|
...options?.stageId === undefined ? {} : { stageId: options.stageId },
|
|
121103
|
-
...options?.message === undefined ? {} : { message: options.message }
|
|
121921
|
+
...options?.message === undefined ? {} : { message: options.message },
|
|
121922
|
+
...options?.legacyRecovery === undefined ? {} : { legacyRecovery: options.legacyRecovery }
|
|
121104
121923
|
}));
|
|
121105
121924
|
}
|
|
121106
121925
|
};
|
|
@@ -121109,6 +121928,32 @@ function createSessionRunControl(host) {
|
|
|
121109
121928
|
// dist/builtin/workflows/src/extension/workflow-tool.ts
|
|
121110
121929
|
import { getSupportedThinkingLevels } from "@bastani/pi-ai/compat";
|
|
121111
121930
|
|
|
121931
|
+
// dist/builtin/workflows/src/extension/workflow-legacy-recovery.ts
|
|
121932
|
+
init_dbos_owner_fence();
|
|
121933
|
+
async function enrollLegacySdkRecovery(args, ctx, access, runtime, owner, signal) {
|
|
121934
|
+
const id = args.runId;
|
|
121935
|
+
signal?.throwIfAborted();
|
|
121936
|
+
if (access !== "sdk" || args.action !== "resume" || args.legacyRecovery?.olderWorkersStopped !== true || args.stageId !== undefined || args.all === true || id === undefined || !isFullRunId(id)) {
|
|
121937
|
+
throw new WorkflowInstanceOwnershipError("Legacy recovery requires an SDK root resume with its full UUID and confirmation that older workers have stopped.");
|
|
121938
|
+
}
|
|
121939
|
+
if (owner.store.runs().some((run) => run.id === id)) {
|
|
121940
|
+
throw new WorkflowInstanceOwnershipError("Legacy enrollment cannot replace a locally attached workflow executor.");
|
|
121941
|
+
}
|
|
121942
|
+
const inspected = await runtime.inspectDurableWorkflow(id);
|
|
121943
|
+
if (inspected.kind !== "found")
|
|
121944
|
+
throw new WorkflowInstanceOwnershipError(inspected.message);
|
|
121945
|
+
const backend = getDurableBackend();
|
|
121946
|
+
const handle = backend.getLoadableWorkflow(id);
|
|
121947
|
+
const interruptedEnrollment = handle?.legacyRecoveryPending === true && handle.ownerLiveness === "dead" && ["blocked", "paused"].includes(handle.status);
|
|
121948
|
+
if (handle === undefined || !isSdkDurableRunInScope(handle, typeof ctx.cwd === "string" ? ctx.cwd : undefined) || handle.ownerExecutorId === undefined || isDatabaseExecutor(handle.ownerExecutorId) && !interruptedEnrollment || !["running", "paused", "blocked"].includes(handle.status) || backend.enrollLegacyWorkflow === undefined) {
|
|
121949
|
+
throw new WorkflowInstanceOwnershipError("Legacy recovery is unavailable for this run or working directory; inspect it without taking ownership.");
|
|
121950
|
+
}
|
|
121951
|
+
signal?.throwIfAborted();
|
|
121952
|
+
if (!await backend.enrollLegacyWorkflow(id, { olderWorkersStopped: true, modelOwner: workflowCaller(ctx), signal })) {
|
|
121953
|
+
throw new WorkflowInstanceOwnershipError("Legacy workflow ownership changed while enrollment was pending; inspect it again before retrying.");
|
|
121954
|
+
}
|
|
121955
|
+
}
|
|
121956
|
+
|
|
121112
121957
|
// dist/builtin/workflows/src/extension/workflow-status-summary.ts
|
|
121113
121958
|
var workflowStatusRenderRuns = new WeakMap;
|
|
121114
121959
|
function setWorkflowStatusRenderRuns(result, runs) {
|
|
@@ -122579,7 +123424,7 @@ function durableInspectionError(action, runId, message) {
|
|
|
122579
123424
|
}
|
|
122580
123425
|
function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflowResourcesLoaded = () => {}, owner = captureWorkflowOwnerResources()) {
|
|
122581
123426
|
const { store, toolControlRegistry } = owner;
|
|
122582
|
-
return async function executeWorkflowTool(args, ctx, signal, onRunAccepted) {
|
|
123427
|
+
return async function executeWorkflowTool(args, ctx, signal, onRunAccepted, access) {
|
|
122583
123428
|
signal?.throwIfAborted();
|
|
122584
123429
|
if (args.action === undefined)
|
|
122585
123430
|
return {
|
|
@@ -122599,7 +123444,13 @@ function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflo
|
|
|
122599
123444
|
stages: []
|
|
122600
123445
|
};
|
|
122601
123446
|
}
|
|
122602
|
-
const
|
|
123447
|
+
const recoveryCwd = typeof ctx.cwd === "string" ? ctx.cwd : undefined;
|
|
123448
|
+
const authorize = (id) => {
|
|
123449
|
+
const handle = getDurableBackend().getLoadableWorkflow(id);
|
|
123450
|
+
if (access === "sdk" && isFullRunId(args.runId?.trim() ?? "") && (action === "status" || action === "stages" || action === "resume" && handle !== undefined && isSdkCrashedRunAdoptable(handle, recoveryCwd)) && !store.runs().some((run) => run.id === id) && handle !== undefined && isSdkDurableRunInScope(handle, recoveryCwd))
|
|
123451
|
+
return;
|
|
123452
|
+
assertWorkflowInstanceOwner(id, ctx, store);
|
|
123453
|
+
};
|
|
122603
123454
|
if (action === "status" && args.runId === undefined) {
|
|
122604
123455
|
for (const run of topLevelWorkflowRuns(store.runs()))
|
|
122605
123456
|
authorize(run.id);
|
|
@@ -122614,8 +123465,16 @@ function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflo
|
|
|
122614
123465
|
}
|
|
122615
123466
|
}
|
|
122616
123467
|
const policy = workflowPolicyFromContext(ctx);
|
|
122617
|
-
const getRuntime = () => typeof runtime === "function" ? runtime(ctx) : runtime;
|
|
122618
123468
|
const awaitRequest = (operation) => raceAbort(operation, signal);
|
|
123469
|
+
const getRuntime = () => {
|
|
123470
|
+
const resolved = typeof runtime === "function" ? runtime(ctx) : runtime;
|
|
123471
|
+
if (access !== "sdk")
|
|
123472
|
+
return resolved;
|
|
123473
|
+
return {
|
|
123474
|
+
...resolved,
|
|
123475
|
+
resumeDurableWorkflow: (id, options) => resolved.resumeDurableWorkflow(id, { ...options, modelOwner: workflowCaller(ctx) })
|
|
123476
|
+
};
|
|
123477
|
+
};
|
|
122619
123478
|
const ensureWorkflowResourcesVisible = async () => {
|
|
122620
123479
|
try {
|
|
122621
123480
|
await awaitRequest(Promise.resolve(ensureWorkflowResourcesLoaded()));
|
|
@@ -122721,6 +123580,9 @@ function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflo
|
|
|
122721
123580
|
case "quit":
|
|
122722
123581
|
return awaitRequest(workflowQuitAction(args, owner, { getRuntime, authorize }));
|
|
122723
123582
|
case "resume":
|
|
123583
|
+
if (args.legacyRecovery !== undefined) {
|
|
123584
|
+
await awaitRequest(enrollLegacySdkRecovery(args, ctx, access, getRuntime(), owner, signal));
|
|
123585
|
+
}
|
|
122724
123586
|
return awaitRequest(workflowResumeAction(args, {
|
|
122725
123587
|
getRuntime,
|
|
122726
123588
|
policy,
|