@bastani/atomic 0.9.26-alpha.5 → 0.9.26-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/builtin/intercom/package.json +2 -2
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +3 -3
- package/dist/builtin/web-access/package.json +2 -2
- package/dist/builtin/workflows/CHANGELOG.md +7 -0
- package/dist/builtin/workflows/builtin/{chunk-613yaa53.js → chunk-9bhs02gm.js} +195 -10
- package/dist/builtin/workflows/builtin/{chunk-6kqrk0pf.js → chunk-gts8wgjr.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-ea6nrbn3.js → chunk-gw7rghda.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +2 -2
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1082 -220
- package/dist/builtin/workflows/src/index.js +967 -186
- package/dist/core/extensions/workflow-run-control.d.ts +3 -0
- package/dist/core/extensions/workflow-run-control.d.ts.map +1 -1
- package/dist/core/extensions/workflow-run-control.js.map +1 -1
- package/dist/core/fast-model-routing.d.ts.map +1 -1
- package/dist/core/fast-model-routing.js +1 -5
- package/dist/core/fast-model-routing.js.map +1 -1
- package/dist/core/model-config.d.ts +36 -0
- package/dist/core/model-config.d.ts.map +1 -1
- package/dist/core/model-config.js +14 -2
- package/dist/core/model-config.js.map +1 -1
- package/dist/core/provider-composer-internal.d.ts +2 -1
- package/dist/core/provider-composer-internal.d.ts.map +1 -1
- package/dist/core/provider-composer-internal.js +13 -0
- package/dist/core/provider-composer-internal.js.map +1 -1
- package/docs/models/reference.md +28 -1
- package/docs/models.md +4 -0
- package/docs/sdk.md +26 -2
- package/docs/workflows/operations.md +15 -1
- package/npm-shrinkwrap.json +67 -67
- package/package.json +8 -8
|
@@ -193,6 +193,602 @@ var init_dbos_admission = __esm(() => {
|
|
|
193
193
|
dbosAdmissionContext = bag[key2];
|
|
194
194
|
});
|
|
195
195
|
|
|
196
|
+
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
197
|
+
function fenceDbosAdmissionPool(pool) {
|
|
198
|
+
const connect = pool.connect.bind(pool);
|
|
199
|
+
async function acquire(signal) {
|
|
200
|
+
if (signal.aborted)
|
|
201
|
+
throw sdkDependencyError();
|
|
202
|
+
const pending = connect().then((client) => {
|
|
203
|
+
if (signal.aborted) {
|
|
204
|
+
client.release(true);
|
|
205
|
+
throw sdkDependencyError();
|
|
206
|
+
}
|
|
207
|
+
return fencedClient(client, signal);
|
|
208
|
+
});
|
|
209
|
+
try {
|
|
210
|
+
return await raceAbort2(pending, signal);
|
|
211
|
+
} catch (error) {
|
|
212
|
+
throw databaseError(error, signal);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function scopedConnect(callback) {
|
|
216
|
+
const signal = dbosAdmissionContext.getStore();
|
|
217
|
+
if (signal === undefined) {
|
|
218
|
+
if (callback === undefined)
|
|
219
|
+
return connect();
|
|
220
|
+
connect(callback);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const pending = acquire(signal);
|
|
224
|
+
if (callback === undefined)
|
|
225
|
+
return pending;
|
|
226
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
227
|
+
}
|
|
228
|
+
pool.connect = scopedConnect;
|
|
229
|
+
return pool;
|
|
230
|
+
}
|
|
231
|
+
function fencedClient(client, signal) {
|
|
232
|
+
let released = false;
|
|
233
|
+
const release = (error) => {
|
|
234
|
+
if (released)
|
|
235
|
+
return;
|
|
236
|
+
released = true;
|
|
237
|
+
signal.removeEventListener("abort", abort);
|
|
238
|
+
client.release(error);
|
|
239
|
+
};
|
|
240
|
+
const abort = () => release(true);
|
|
241
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
242
|
+
const query = new Proxy(client.query, {
|
|
243
|
+
apply(target, _receiver, args) {
|
|
244
|
+
const values = [...args];
|
|
245
|
+
const last = values.at(-1);
|
|
246
|
+
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
247
|
+
const pending = (async () => {
|
|
248
|
+
if (signal.aborted || released)
|
|
249
|
+
throw sdkDependencyError();
|
|
250
|
+
try {
|
|
251
|
+
return await raceAbort2(new Promise((resolve, reject) => {
|
|
252
|
+
Reflect.apply(target, client, [
|
|
253
|
+
...values,
|
|
254
|
+
(error, result) => {
|
|
255
|
+
if (error)
|
|
256
|
+
reject(error);
|
|
257
|
+
else
|
|
258
|
+
resolve(result);
|
|
259
|
+
}
|
|
260
|
+
]);
|
|
261
|
+
}), signal);
|
|
262
|
+
} catch (error) {
|
|
263
|
+
throw databaseError(error, signal);
|
|
264
|
+
}
|
|
265
|
+
})();
|
|
266
|
+
if (callback === undefined)
|
|
267
|
+
return pending;
|
|
268
|
+
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
return new Proxy(client, {
|
|
272
|
+
get(target, property, receiver) {
|
|
273
|
+
if (property === "query")
|
|
274
|
+
return query;
|
|
275
|
+
if (property === "release")
|
|
276
|
+
return release;
|
|
277
|
+
return Reflect.get(target, property, receiver);
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
function sdkDependencyError(message) {
|
|
282
|
+
const reported = message === undefined ? new DbosDependencyError : new DbosDependencyError(message, null);
|
|
283
|
+
return new DbosDependencyError(undefined, null, reported.message);
|
|
284
|
+
}
|
|
285
|
+
function databaseError(error, signal) {
|
|
286
|
+
if (signal?.aborted)
|
|
287
|
+
return sdkDependencyError();
|
|
288
|
+
if (isDbosDependencyError(error))
|
|
289
|
+
return sdkDependencyError(error.admissionDetail ?? error.message);
|
|
290
|
+
const pending = [error];
|
|
291
|
+
const seen = new Set;
|
|
292
|
+
while (pending.length > 0) {
|
|
293
|
+
const value = pending.pop();
|
|
294
|
+
if (typeof value !== "object" || value === null || seen.has(value))
|
|
295
|
+
continue;
|
|
296
|
+
seen.add(value);
|
|
297
|
+
if ("cause" in value)
|
|
298
|
+
pending.push(value.cause);
|
|
299
|
+
if ("error" in value)
|
|
300
|
+
pending.push(value.error);
|
|
301
|
+
if ("errors" in value && Array.isArray(value.errors))
|
|
302
|
+
pending.push(...value.errors);
|
|
303
|
+
const code = "code" in value && typeof value.code === "string" ? value.code : undefined;
|
|
304
|
+
const message = value instanceof Error ? value.message : "";
|
|
305
|
+
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
306
|
+
return sdkDependencyError();
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return error;
|
|
310
|
+
}
|
|
311
|
+
var connectionErrnos, connectionMessages;
|
|
312
|
+
var init_dbos_admission_pool = __esm(() => {
|
|
313
|
+
init_dbos_admission();
|
|
314
|
+
connectionErrnos = new Set([
|
|
315
|
+
"ECONNRESET",
|
|
316
|
+
"ECONNREFUSED",
|
|
317
|
+
"ECONNABORTED",
|
|
318
|
+
"EHOSTUNREACH",
|
|
319
|
+
"ENETUNREACH",
|
|
320
|
+
"ETIMEDOUT",
|
|
321
|
+
"ENOTFOUND",
|
|
322
|
+
"EAI_AGAIN",
|
|
323
|
+
"EPIPE"
|
|
324
|
+
]);
|
|
325
|
+
connectionMessages = new Set([
|
|
326
|
+
"Connection terminated unexpectedly",
|
|
327
|
+
"Connection terminated due to connection timeout",
|
|
328
|
+
"Client has encountered a connection error and is not queryable",
|
|
329
|
+
"timeout exceeded when trying to connect",
|
|
330
|
+
"Client was closed and is not queryable",
|
|
331
|
+
"Postgres connection pool changed while acquiring a connection"
|
|
332
|
+
]);
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
// dist/builtin/workflows/src/durable/dbos-row-guard.ts
|
|
336
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
337
|
+
async function installDbosRowGuard(client) {
|
|
338
|
+
const ready = await client.query("SELECT to_regclass('dbos.workflow_status') IS NOT NULL AS status, to_regclass('dbos.operation_outputs') IS NOT NULL AS operations");
|
|
339
|
+
if (!ready.rows[0]?.status && !ready.rows[0]?.operations)
|
|
340
|
+
return;
|
|
341
|
+
if (!ready.rows[0]?.status || !ready.rows[0]?.operations)
|
|
342
|
+
throw new Error("Atomic workflow database schema is incomplete");
|
|
343
|
+
if (await validateInstalledGuard(client))
|
|
344
|
+
return;
|
|
345
|
+
await client.query("BEGIN");
|
|
346
|
+
try {
|
|
347
|
+
await client.query("SELECT pg_advisory_xact_lock(3419, 9003)");
|
|
348
|
+
if (await validateInstalledGuard(client)) {
|
|
349
|
+
await client.query("COMMIT");
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
await client.query(guardSql);
|
|
353
|
+
await client.query("CREATE TABLE IF NOT EXISTS dbos.atomic_guard_protocol (singleton boolean PRIMARY KEY CHECK(singleton), fingerprint text NOT NULL)");
|
|
354
|
+
await client.query("INSERT INTO dbos.atomic_guard_protocol VALUES (true, $1) ON CONFLICT DO NOTHING", [
|
|
355
|
+
fingerprint
|
|
356
|
+
]);
|
|
357
|
+
await client.query("COMMIT");
|
|
358
|
+
} catch (error) {
|
|
359
|
+
await client.query("ROLLBACK");
|
|
360
|
+
throw error;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
async function validateInstalledGuard(client) {
|
|
364
|
+
const installed = await client.query("SELECT to_regclass('dbos.atomic_guard_protocol') IS NOT NULL AS present");
|
|
365
|
+
if (!installed.rows[0]?.present)
|
|
366
|
+
return false;
|
|
367
|
+
const validated = await client.query("SELECT fingerprint, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid IN ('dbos.workflow_status'::regclass,'dbos.operation_outputs'::regclass) AND tgname = 'atomic_generation_guard' AND tgenabled = 'O' AND tgtype = 31 AND tgfoid = 'dbos.atomic_guard_row()'::regprocedure) AS triggers, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection' AND tgenabled = 'O' AND tgtype = 21 AND tgfoid = 'dbos.atomic_project_metadata()'::regprocedure) AS projections FROM dbos.atomic_guard_protocol WHERE singleton");
|
|
368
|
+
if (validated.rows.length !== 1 || validated.rows[0].fingerprint !== fingerprint || validated.rows[0].triggers !== 2 || validated.rows[0].projections !== 1)
|
|
369
|
+
throw new Error("Atomic database ownership guard schema is incompatible or incomplete");
|
|
370
|
+
return true;
|
|
371
|
+
}
|
|
372
|
+
var guardSql = `
|
|
373
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_owner_generation (
|
|
374
|
+
root text PRIMARY KEY, owner text, generation bigint,
|
|
375
|
+
terminal boolean NOT NULL DEFAULT false, claim text, claimant text, protocol integer NOT NULL DEFAULT 1 CHECK(protocol = 1)
|
|
376
|
+
);
|
|
377
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_guard_rows (workflow_uuid text PRIMARY KEY, root text NOT NULL);
|
|
378
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_metadata(value text) RETURNS jsonb
|
|
379
|
+
LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog AS $$
|
|
380
|
+
DECLARE decoded jsonb;
|
|
381
|
+
BEGIN
|
|
382
|
+
IF value IS NULL THEN RETURN NULL; END IF;
|
|
383
|
+
BEGIN decoded := value::jsonb; EXCEPTION WHEN invalid_text_representation THEN RETURN NULL; END;
|
|
384
|
+
IF decoded->>'__dbos_serializer' = 'superjson' THEN decoded := decoded->'json'; END IF;
|
|
385
|
+
IF decoded->>'__atomicDurableMetadata' IS DISTINCT FROM 'true' OR decoded->>'version' IS DISTINCT FROM '3' THEN RETURN NULL; END IF;
|
|
386
|
+
RETURN decoded->'metadata';
|
|
387
|
+
END $$;
|
|
388
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_guard_row() RETURNS trigger
|
|
389
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
390
|
+
DECLARE
|
|
391
|
+
row_value jsonb; id text; root_id text; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE;
|
|
392
|
+
meta jsonb; prior jsonb; prior_generation bigint; next_generation bigint; absorbing boolean;
|
|
393
|
+
allowed boolean; cap_owner text; cap_generation bigint; claim_token text;
|
|
394
|
+
BEGIN
|
|
395
|
+
row_value := CASE WHEN TG_OP = 'DELETE' THEN to_jsonb(OLD) ELSE to_jsonb(NEW) END;
|
|
396
|
+
id := row_value->>'workflow_uuid';
|
|
397
|
+
root_id := split_part(id, ':checkpoint:', 1);
|
|
398
|
+
IF TG_OP = 'UPDATE' AND to_jsonb(OLD)->>'workflow_uuid' IS DISTINCT FROM id AND (
|
|
399
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid') OR
|
|
400
|
+
(TG_TABLE_NAME = 'workflow_status' AND to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')) OR
|
|
401
|
+
(TG_TABLE_NAME = 'operation_outputs' AND EXISTS(SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid' AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')))
|
|
402
|
+
) THEN
|
|
403
|
+
IF NULLIF(current_setting('atomic.row_authority',true),'') IS NULL OR current_setting('atomic.row_authority',true) = 'null' THEN RETURN NULL; END IF;
|
|
404
|
+
RAISE EXCEPTION 'Atomic workflow identities are immutable';
|
|
405
|
+
END IF;
|
|
406
|
+
IF TG_TABLE_NAME = 'workflow_status' AND NOT (
|
|
407
|
+
COALESCE(row_value->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
408
|
+
COALESCE(to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
409
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id)
|
|
410
|
+
) THEN
|
|
411
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
412
|
+
END IF;
|
|
413
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
414
|
+
IF TG_TABLE_NAME = 'operation_outputs' AND NOT EXISTS (
|
|
415
|
+
SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = id AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')
|
|
416
|
+
) AND NOT EXISTS (SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id) THEN
|
|
417
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
418
|
+
END IF;
|
|
419
|
+
INSERT INTO dbos.atomic_guard_rows VALUES(id, root_id) ON CONFLICT DO NOTHING;
|
|
420
|
+
INSERT INTO dbos.atomic_guard_rows SELECT workflow_uuid, root_id FROM dbos.workflow_status
|
|
421
|
+
WHERE workflow_uuid = root_id AND name = 'atomicWorkflowHandle' ON CONFLICT DO NOTHING;
|
|
422
|
+
INSERT INTO dbos.atomic_owner_generation(root) VALUES(root_id)
|
|
423
|
+
ON CONFLICT(root) DO UPDATE SET root = EXCLUDED.root RETURNING * INTO state;
|
|
424
|
+
IF state.owner IS NULL THEN
|
|
425
|
+
IF TG_TABLE_NAME <> 'workflow_status' OR TG_OP = 'DELETE' OR row_value->>'status' <> 'SUCCESS'
|
|
426
|
+
OR id NOT LIKE root_id || ':checkpoint:__atomic_metadata:%'
|
|
427
|
+
OR authority->>'root' IS DISTINCT FROM root_id OR authority->>'enroll' IS DISTINCT FROM 'true' THEN
|
|
428
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
429
|
+
END IF;
|
|
430
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
431
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id THEN
|
|
432
|
+
RAISE EXCEPTION 'Atomic metadata encoding is unavailable for enrollment';
|
|
433
|
+
END IF;
|
|
434
|
+
SELECT candidate.metadata, candidate.generation INTO prior, prior_generation FROM (
|
|
435
|
+
SELECT dbos.atomic_metadata(output) AS metadata,
|
|
436
|
+
split_part(substring(workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint AS generation
|
|
437
|
+
FROM dbos.workflow_status
|
|
438
|
+
WHERE workflow_uuid LIKE root_id || ':checkpoint:__atomic_metadata:%' AND status = 'SUCCESS'
|
|
439
|
+
) candidate WHERE candidate.metadata IS NOT NULL
|
|
440
|
+
ORDER BY COALESCE(candidate.metadata->>'status' IN ('completed', 'cancelled') OR
|
|
441
|
+
(candidate.metadata->>'status' IN ('failed','blocked') AND candidate.metadata->>'resumable' = 'false'), false) DESC,
|
|
442
|
+
candidate.generation DESC LIMIT 1;
|
|
443
|
+
IF prior_generation IS DISTINCT FROM (authority->>'generation')::bigint
|
|
444
|
+
OR prior->>'ownerExecutorId' IS DISTINCT FROM authority->>'owner' THEN
|
|
445
|
+
RAISE EXCEPTION 'Atomic ownership generation changed before enrollment';
|
|
446
|
+
END IF;
|
|
447
|
+
ELSE
|
|
448
|
+
IF authority->>'root' IS DISTINCT FROM root_id THEN RETURN NULL; END IF;
|
|
449
|
+
cap_owner := authority->>'owner'; cap_generation := (authority->>'generation')::bigint;
|
|
450
|
+
allowed := cap_owner = state.owner AND cap_generation = state.generation AND (
|
|
451
|
+
authority->>'actor' = state.owner OR
|
|
452
|
+
(authority->>'claim' = state.claim AND authority->>'actor' = state.claimant) OR
|
|
453
|
+
(authority->>'claim' IS NOT NULL AND authority->>'checkpoint' = id));
|
|
454
|
+
IF NOT COALESCE(allowed, false) THEN RAISE EXCEPTION 'Atomic ownership generation changed; stale database writes are refused'; END IF;
|
|
455
|
+
END IF;
|
|
456
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP = 'UPDATE' THEN
|
|
457
|
+
IF OLD.status = 'SUCCESS' AND OLD.name = 'atomicWorkflowCheckpoint' THEN
|
|
458
|
+
IF NEW.output IS DISTINCT FROM OLD.output OR NEW.status <> 'SUCCESS' THEN RAISE EXCEPTION 'Completed Atomic checkpoints are immutable'; END IF;
|
|
459
|
+
RETURN NEW;
|
|
460
|
+
END IF;
|
|
461
|
+
END IF;
|
|
462
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP <> 'DELETE' AND row_value->>'status' = 'SUCCESS'
|
|
463
|
+
AND id LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN
|
|
464
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
465
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN RAISE EXCEPTION 'Atomic metadata encoding or owner is unavailable'; END IF;
|
|
466
|
+
next_generation := split_part(substring(id FROM length(root_id) + 13), ':', 2)::bigint;
|
|
467
|
+
absorbing := COALESCE(meta->>'status' IN ('completed', 'cancelled') OR
|
|
468
|
+
(meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
469
|
+
claim_token := meta->>'transitionClaimId';
|
|
470
|
+
IF state.root IS NOT NULL AND (next_generation < state.generation OR (state.terminal AND NOT absorbing)) THEN
|
|
471
|
+
RAISE EXCEPTION 'Atomic terminal or newer generation cannot be overwritten';
|
|
472
|
+
END IF;
|
|
473
|
+
IF state.owner IS NOT NULL AND authority->>'actor' <> state.owner THEN
|
|
474
|
+
IF id NOT LIKE '%:claim' OR next_generation <> state.generation + 1 OR claim_token IS DISTINCT FROM authority->>'claim' OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN
|
|
475
|
+
RAISE EXCEPTION 'Atomic ownership transfer requires an exact generation claim';
|
|
476
|
+
END IF;
|
|
477
|
+
END IF;
|
|
478
|
+
END IF;
|
|
479
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
480
|
+
END $$;
|
|
481
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_project_metadata() RETURNS trigger
|
|
482
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
483
|
+
DECLARE root_id text; meta jsonb; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE; projected_generation bigint; absorbing boolean;
|
|
484
|
+
BEGIN
|
|
485
|
+
IF NEW.name <> 'atomicWorkflowCheckpoint' OR NEW.status <> 'SUCCESS' THEN RETURN NEW; END IF;
|
|
486
|
+
root_id := split_part(NEW.workflow_uuid, ':checkpoint:', 1);
|
|
487
|
+
IF NEW.workflow_uuid NOT LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN RETURN NEW; END IF;
|
|
488
|
+
IF TG_OP = 'UPDATE' THEN IF OLD.status = 'SUCCESS' THEN RETURN NEW; END IF; END IF;
|
|
489
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
490
|
+
SELECT * INTO state FROM dbos.atomic_owner_generation WHERE root = root_id FOR UPDATE;
|
|
491
|
+
IF state.owner IS NULL AND authority->>'enroll' IS DISTINCT FROM 'true' THEN RETURN NEW; END IF;
|
|
492
|
+
meta := dbos.atomic_metadata(NEW.output);
|
|
493
|
+
projected_generation := split_part(substring(NEW.workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint;
|
|
494
|
+
absorbing := COALESCE(meta->>'status' IN ('completed','cancelled') OR (meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
495
|
+
UPDATE dbos.atomic_owner_generation SET owner = meta->>'ownerExecutorId', generation = projected_generation,
|
|
496
|
+
terminal = absorbing, claim = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN meta->>'transitionClaimId' END,
|
|
497
|
+
claimant = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN authority->>'actor' END WHERE root = root_id;
|
|
498
|
+
RETURN NEW;
|
|
499
|
+
END $$;
|
|
500
|
+
DO $$ BEGIN
|
|
501
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
502
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.workflow_status
|
|
503
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
504
|
+
END IF;
|
|
505
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.operation_outputs'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
506
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.operation_outputs
|
|
507
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
508
|
+
END IF;
|
|
509
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection') THEN
|
|
510
|
+
CREATE TRIGGER atomic_generation_projection AFTER INSERT OR UPDATE ON dbos.workflow_status
|
|
511
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_project_metadata();
|
|
512
|
+
END IF;
|
|
513
|
+
END $$;
|
|
514
|
+
`, fingerprint;
|
|
515
|
+
var init_dbos_row_guard = __esm(() => {
|
|
516
|
+
fingerprint = createHash3("sha256").update(guardSql).digest("hex");
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
// dist/builtin/workflows/src/durable/dbos-owner-fence.ts
|
|
520
|
+
import { AsyncLocalStorage as AsyncLocalStorage3 } from "node:async_hooks";
|
|
521
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
522
|
+
function lockKey(identity) {
|
|
523
|
+
const digest = createHash4("sha256").update(`atomic-workflow-fence:${identity}`).digest();
|
|
524
|
+
return [digest.readInt32BE(0), digest.readInt32BE(4)];
|
|
525
|
+
}
|
|
526
|
+
function isDatabaseExecutor(executorId) {
|
|
527
|
+
return /^atomic-db-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(executorId ?? "");
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
class DbosOwnerFence {
|
|
531
|
+
pool;
|
|
532
|
+
executorId;
|
|
533
|
+
endPool;
|
|
534
|
+
lifetime;
|
|
535
|
+
lost = false;
|
|
536
|
+
closing;
|
|
537
|
+
closed = false;
|
|
538
|
+
get invalidated() {
|
|
539
|
+
return this.lost;
|
|
540
|
+
}
|
|
541
|
+
constructor(pool, executorId, endPool) {
|
|
542
|
+
this.pool = pool;
|
|
543
|
+
this.executorId = executorId;
|
|
544
|
+
this.endPool = endPool;
|
|
545
|
+
}
|
|
546
|
+
fail() {
|
|
547
|
+
this.lost = true;
|
|
548
|
+
return new DbosDependencyError("Workflow database unavailable: ownership connection was lost; this execution generation cannot write again");
|
|
549
|
+
}
|
|
550
|
+
owner() {
|
|
551
|
+
if (this.lost)
|
|
552
|
+
return Promise.reject(this.fail());
|
|
553
|
+
this.lifetime ??= (async () => {
|
|
554
|
+
const client = await this.pool().connect();
|
|
555
|
+
client.on("error", () => {
|
|
556
|
+
this.fail();
|
|
557
|
+
});
|
|
558
|
+
client.on("end", () => {
|
|
559
|
+
this.fail();
|
|
560
|
+
});
|
|
561
|
+
try {
|
|
562
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-owner:${this.executorId}`]);
|
|
563
|
+
const held = await client.query("SELECT pg_try_advisory_lock($1, $2) AS held", [
|
|
564
|
+
...lockKey(this.executorId)
|
|
565
|
+
]);
|
|
566
|
+
if (!held.rows[0]?.held || this.lost)
|
|
567
|
+
throw this.fail();
|
|
568
|
+
await client.query("SELECT pg_advisory_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
569
|
+
await client.query("SELECT pg_advisory_unlock($1, $2)", [...lockKey(this.executorId)]);
|
|
570
|
+
return client;
|
|
571
|
+
} catch (error) {
|
|
572
|
+
this.fail();
|
|
573
|
+
client.release(true);
|
|
574
|
+
throw error;
|
|
575
|
+
}
|
|
576
|
+
})().catch((error) => {
|
|
577
|
+
this.fail();
|
|
578
|
+
throw error;
|
|
579
|
+
});
|
|
580
|
+
return this.lifetime;
|
|
581
|
+
}
|
|
582
|
+
async liveness(executorId) {
|
|
583
|
+
if (!isDatabaseExecutor(executorId))
|
|
584
|
+
return "unknown";
|
|
585
|
+
const client = await this.pool().connect();
|
|
586
|
+
try {
|
|
587
|
+
await client.query("BEGIN");
|
|
588
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
589
|
+
...lockKey(executorId)
|
|
590
|
+
]);
|
|
591
|
+
return result.rows[0]?.held ? "dead" : "alive";
|
|
592
|
+
} finally {
|
|
593
|
+
try {
|
|
594
|
+
await client.query("ROLLBACK");
|
|
595
|
+
} finally {
|
|
596
|
+
client.release();
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
async recover(executorId, callback) {
|
|
601
|
+
if (!isDatabaseExecutor(executorId))
|
|
602
|
+
return;
|
|
603
|
+
const client = await this.pool().connect();
|
|
604
|
+
const fail = () => {
|
|
605
|
+
this.fail();
|
|
606
|
+
};
|
|
607
|
+
client.on("error", fail);
|
|
608
|
+
client.on("end", fail);
|
|
609
|
+
try {
|
|
610
|
+
await client.query("BEGIN");
|
|
611
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
612
|
+
...lockKey(executorId)
|
|
613
|
+
]);
|
|
614
|
+
if (!result.rows[0]?.held)
|
|
615
|
+
return;
|
|
616
|
+
return await callback();
|
|
617
|
+
} finally {
|
|
618
|
+
try {
|
|
619
|
+
await client.query("ROLLBACK");
|
|
620
|
+
} finally {
|
|
621
|
+
client.removeListener("error", fail);
|
|
622
|
+
client.removeListener("end", fail);
|
|
623
|
+
client.release();
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
async write(workflowId, callback, claim = false) {
|
|
628
|
+
try {
|
|
629
|
+
return await this.writeLocked(workflowId, callback, claim);
|
|
630
|
+
} catch (error) {
|
|
631
|
+
const mapped = databaseError(error);
|
|
632
|
+
throw this.lost && isDbosDependencyError(mapped) ? this.fail() : mapped;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
async writeLocked(workflowId, callback, claim) {
|
|
636
|
+
const context = writeContext.getStore();
|
|
637
|
+
const active = context?.fence === this ? context.workflows : undefined;
|
|
638
|
+
if (active?.has(workflowId)) {
|
|
639
|
+
if (this.lost)
|
|
640
|
+
throw this.fail();
|
|
641
|
+
return await callback();
|
|
642
|
+
}
|
|
643
|
+
const owner = await this.owner();
|
|
644
|
+
const client = await this.pool().connect();
|
|
645
|
+
const fail = () => {
|
|
646
|
+
this.fail();
|
|
647
|
+
};
|
|
648
|
+
client.on("error", fail);
|
|
649
|
+
client.on("end", fail);
|
|
650
|
+
try {
|
|
651
|
+
await client.query("BEGIN");
|
|
652
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-guard:${workflowId}`]);
|
|
653
|
+
await client.query("SET LOCAL statement_timeout = '10000ms'");
|
|
654
|
+
await client.query("SELECT pg_advisory_xact_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
655
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`workflow:${workflowId}`)]);
|
|
656
|
+
if (claim)
|
|
657
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`sql:${workflowId}`)]);
|
|
658
|
+
if (this.lost)
|
|
659
|
+
throw this.fail();
|
|
660
|
+
try {
|
|
661
|
+
await owner.query("SELECT 1");
|
|
662
|
+
} catch {
|
|
663
|
+
throw this.fail();
|
|
664
|
+
}
|
|
665
|
+
if (this.lost)
|
|
666
|
+
throw this.fail();
|
|
667
|
+
return await writeContext.run({ fence: this, workflows: new Map([...active ?? [], [workflowId, claim]]) }, callback);
|
|
668
|
+
} finally {
|
|
669
|
+
try {
|
|
670
|
+
await client.query("ROLLBACK");
|
|
671
|
+
} finally {
|
|
672
|
+
client.removeListener("error", fail);
|
|
673
|
+
client.removeListener("end", fail);
|
|
674
|
+
client.release();
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
async withRowAuthority(authority, callback) {
|
|
679
|
+
return await rowContext.run(authority, callback);
|
|
680
|
+
}
|
|
681
|
+
protectPool(pool) {
|
|
682
|
+
const connect = pool.connect.bind(pool);
|
|
683
|
+
const acquire = async () => {
|
|
684
|
+
const active = writeContext.getStore();
|
|
685
|
+
const client = await connect();
|
|
686
|
+
try {
|
|
687
|
+
await installDbosRowGuard(client);
|
|
688
|
+
await client.query("SELECT set_config('atomic.row_authority', $1, false)", [
|
|
689
|
+
JSON.stringify(rowContext.getStore() ?? null)
|
|
690
|
+
]);
|
|
691
|
+
} catch (error) {
|
|
692
|
+
client.release(true);
|
|
693
|
+
throw error;
|
|
694
|
+
}
|
|
695
|
+
if (active === undefined)
|
|
696
|
+
return client;
|
|
697
|
+
const fence = active.fence;
|
|
698
|
+
try {
|
|
699
|
+
if (fence.lost)
|
|
700
|
+
throw fence.fail();
|
|
701
|
+
await client.query("SELECT set_config('application_name', $1, false)", [
|
|
702
|
+
`atomic-sql:${[...active.workflows.keys()][0]}`
|
|
703
|
+
]);
|
|
704
|
+
for (const identity of [
|
|
705
|
+
fence.executorId,
|
|
706
|
+
...[...active.workflows].filter(([, claim]) => !claim).map(([id]) => `sql:${id}`)
|
|
707
|
+
]) {
|
|
708
|
+
const held = await client.query("SELECT pg_try_advisory_lock_shared($1, $2) AS held", [...lockKey(identity)]);
|
|
709
|
+
if (!held.rows[0]?.held)
|
|
710
|
+
throw fence.fail();
|
|
711
|
+
}
|
|
712
|
+
try {
|
|
713
|
+
await (await fence.owner()).query("SELECT 1");
|
|
714
|
+
} catch {
|
|
715
|
+
throw fence.fail();
|
|
716
|
+
}
|
|
717
|
+
if (fence.lost)
|
|
718
|
+
throw fence.fail();
|
|
719
|
+
} catch (error) {
|
|
720
|
+
client.release(true);
|
|
721
|
+
throw error;
|
|
722
|
+
}
|
|
723
|
+
let released = false;
|
|
724
|
+
const release = () => {
|
|
725
|
+
if (released)
|
|
726
|
+
return;
|
|
727
|
+
released = true;
|
|
728
|
+
client.release(true);
|
|
729
|
+
};
|
|
730
|
+
const query = new Proxy(client.query, {
|
|
731
|
+
apply: (target, _receiver, args) => {
|
|
732
|
+
if (!fence.lost && !released)
|
|
733
|
+
return Reflect.apply(target, client, args);
|
|
734
|
+
const error = fence.fail();
|
|
735
|
+
const callback = args.at(-1);
|
|
736
|
+
if (typeof callback === "function") {
|
|
737
|
+
queueMicrotask(() => callback(error));
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
return Promise.reject(error);
|
|
741
|
+
}
|
|
742
|
+
});
|
|
743
|
+
return new Proxy(client, {
|
|
744
|
+
get(target, property, receiver) {
|
|
745
|
+
if (property === "release")
|
|
746
|
+
return release;
|
|
747
|
+
if (property === "query")
|
|
748
|
+
return query;
|
|
749
|
+
return Reflect.get(target, property, receiver);
|
|
750
|
+
}
|
|
751
|
+
});
|
|
752
|
+
};
|
|
753
|
+
function scopedConnect(callback) {
|
|
754
|
+
const pending = acquire();
|
|
755
|
+
if (callback === undefined)
|
|
756
|
+
return pending;
|
|
757
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
758
|
+
}
|
|
759
|
+
pool.connect = scopedConnect;
|
|
760
|
+
return pool;
|
|
761
|
+
}
|
|
762
|
+
close() {
|
|
763
|
+
this.closed = true;
|
|
764
|
+
this.lost = true;
|
|
765
|
+
this.closing ??= (async () => {
|
|
766
|
+
if (this.lifetime !== undefined) {
|
|
767
|
+
const client = await this.lifetime.catch(() => {
|
|
768
|
+
return;
|
|
769
|
+
});
|
|
770
|
+
client?.release(true);
|
|
771
|
+
}
|
|
772
|
+
await this.endPool();
|
|
773
|
+
})();
|
|
774
|
+
return this.closing;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
var contextKey, contextBag, writeContext, rowContextKey, rowContextBag, rowContext;
|
|
778
|
+
var init_dbos_owner_fence = __esm(() => {
|
|
779
|
+
init_dbos_admission();
|
|
780
|
+
init_dbos_admission_pool();
|
|
781
|
+
init_dbos_row_guard();
|
|
782
|
+
contextKey = Symbol.for("atomic-workflows/dbos-owner-write-context@1");
|
|
783
|
+
contextBag = globalThis;
|
|
784
|
+
contextBag[contextKey] ??= new AsyncLocalStorage3;
|
|
785
|
+
writeContext = contextBag[contextKey];
|
|
786
|
+
rowContextKey = Symbol.for("atomic-workflows/dbos-row-authority@1");
|
|
787
|
+
rowContextBag = globalThis;
|
|
788
|
+
rowContextBag[rowContextKey] ??= new AsyncLocalStorage3;
|
|
789
|
+
rowContext = rowContextBag[rowContextKey];
|
|
790
|
+
});
|
|
791
|
+
|
|
196
792
|
// ../../node_modules/@dbos-inc/dbos-sdk/dist/src/error.js
|
|
197
793
|
var require_error = __commonJS(function(exports) {
|
|
198
794
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -30349,145 +30945,6 @@ SELECT
|
|
|
30349
30945
|
} });
|
|
30350
30946
|
});
|
|
30351
30947
|
|
|
30352
|
-
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
30353
|
-
function fenceDbosAdmissionPool(pool) {
|
|
30354
|
-
const connect = pool.connect.bind(pool);
|
|
30355
|
-
async function acquire(signal) {
|
|
30356
|
-
if (signal.aborted)
|
|
30357
|
-
throw sdkDependencyError();
|
|
30358
|
-
const pending = connect().then((client) => {
|
|
30359
|
-
if (signal.aborted) {
|
|
30360
|
-
client.release(true);
|
|
30361
|
-
throw sdkDependencyError();
|
|
30362
|
-
}
|
|
30363
|
-
return fencedClient(client, signal);
|
|
30364
|
-
});
|
|
30365
|
-
try {
|
|
30366
|
-
return await raceAbort2(pending, signal);
|
|
30367
|
-
} catch (error) {
|
|
30368
|
-
throw databaseError(error, signal);
|
|
30369
|
-
}
|
|
30370
|
-
}
|
|
30371
|
-
function scopedConnect(callback) {
|
|
30372
|
-
const signal = dbosAdmissionContext.getStore();
|
|
30373
|
-
if (signal === undefined) {
|
|
30374
|
-
if (callback === undefined)
|
|
30375
|
-
return connect();
|
|
30376
|
-
connect(callback);
|
|
30377
|
-
return;
|
|
30378
|
-
}
|
|
30379
|
-
const pending = acquire(signal);
|
|
30380
|
-
if (callback === undefined)
|
|
30381
|
-
return pending;
|
|
30382
|
-
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
30383
|
-
}
|
|
30384
|
-
pool.connect = scopedConnect;
|
|
30385
|
-
return pool;
|
|
30386
|
-
}
|
|
30387
|
-
function fencedClient(client, signal) {
|
|
30388
|
-
let released = false;
|
|
30389
|
-
const release = (error) => {
|
|
30390
|
-
if (released)
|
|
30391
|
-
return;
|
|
30392
|
-
released = true;
|
|
30393
|
-
signal.removeEventListener("abort", abort);
|
|
30394
|
-
client.release(error);
|
|
30395
|
-
};
|
|
30396
|
-
const abort = () => release(true);
|
|
30397
|
-
signal.addEventListener("abort", abort, { once: true });
|
|
30398
|
-
const query = new Proxy(client.query, {
|
|
30399
|
-
apply(target, _receiver, args) {
|
|
30400
|
-
const values = [...args];
|
|
30401
|
-
const last = values.at(-1);
|
|
30402
|
-
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
30403
|
-
const pending = (async () => {
|
|
30404
|
-
if (signal.aborted || released)
|
|
30405
|
-
throw sdkDependencyError();
|
|
30406
|
-
try {
|
|
30407
|
-
return await raceAbort2(new Promise((resolve, reject) => {
|
|
30408
|
-
Reflect.apply(target, client, [
|
|
30409
|
-
...values,
|
|
30410
|
-
(error, result) => {
|
|
30411
|
-
if (error)
|
|
30412
|
-
reject(error);
|
|
30413
|
-
else
|
|
30414
|
-
resolve(result);
|
|
30415
|
-
}
|
|
30416
|
-
]);
|
|
30417
|
-
}), signal);
|
|
30418
|
-
} catch (error) {
|
|
30419
|
-
throw databaseError(error, signal);
|
|
30420
|
-
}
|
|
30421
|
-
})();
|
|
30422
|
-
if (callback === undefined)
|
|
30423
|
-
return pending;
|
|
30424
|
-
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
30425
|
-
}
|
|
30426
|
-
});
|
|
30427
|
-
return new Proxy(client, {
|
|
30428
|
-
get(target, property, receiver) {
|
|
30429
|
-
if (property === "query")
|
|
30430
|
-
return query;
|
|
30431
|
-
if (property === "release")
|
|
30432
|
-
return release;
|
|
30433
|
-
return Reflect.get(target, property, receiver);
|
|
30434
|
-
}
|
|
30435
|
-
});
|
|
30436
|
-
}
|
|
30437
|
-
function sdkDependencyError(message) {
|
|
30438
|
-
const reported = message === undefined ? new DbosDependencyError : new DbosDependencyError(message, null);
|
|
30439
|
-
return new DbosDependencyError(undefined, null, reported.message);
|
|
30440
|
-
}
|
|
30441
|
-
function databaseError(error, signal) {
|
|
30442
|
-
if (signal.aborted)
|
|
30443
|
-
return sdkDependencyError();
|
|
30444
|
-
if (isDbosDependencyError(error))
|
|
30445
|
-
return sdkDependencyError(error.admissionDetail ?? error.message);
|
|
30446
|
-
const pending = [error];
|
|
30447
|
-
const seen = new Set;
|
|
30448
|
-
while (pending.length > 0) {
|
|
30449
|
-
const value = pending.pop();
|
|
30450
|
-
if (typeof value !== "object" || value === null || seen.has(value))
|
|
30451
|
-
continue;
|
|
30452
|
-
seen.add(value);
|
|
30453
|
-
if ("cause" in value)
|
|
30454
|
-
pending.push(value.cause);
|
|
30455
|
-
if ("error" in value)
|
|
30456
|
-
pending.push(value.error);
|
|
30457
|
-
if ("errors" in value && Array.isArray(value.errors))
|
|
30458
|
-
pending.push(...value.errors);
|
|
30459
|
-
const code = "code" in value && typeof value.code === "string" ? value.code : undefined;
|
|
30460
|
-
const message = value instanceof Error ? value.message : "";
|
|
30461
|
-
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
30462
|
-
return sdkDependencyError();
|
|
30463
|
-
}
|
|
30464
|
-
}
|
|
30465
|
-
return error;
|
|
30466
|
-
}
|
|
30467
|
-
var connectionErrnos, connectionMessages;
|
|
30468
|
-
var init_dbos_admission_pool = __esm(() => {
|
|
30469
|
-
init_dbos_admission();
|
|
30470
|
-
connectionErrnos = new Set([
|
|
30471
|
-
"ECONNRESET",
|
|
30472
|
-
"ECONNREFUSED",
|
|
30473
|
-
"ECONNABORTED",
|
|
30474
|
-
"EHOSTUNREACH",
|
|
30475
|
-
"ENETUNREACH",
|
|
30476
|
-
"ETIMEDOUT",
|
|
30477
|
-
"ENOTFOUND",
|
|
30478
|
-
"EAI_AGAIN",
|
|
30479
|
-
"EPIPE"
|
|
30480
|
-
]);
|
|
30481
|
-
connectionMessages = new Set([
|
|
30482
|
-
"Connection terminated unexpectedly",
|
|
30483
|
-
"Connection terminated due to connection timeout",
|
|
30484
|
-
"Client has encountered a connection error and is not queryable",
|
|
30485
|
-
"timeout exceeded when trying to connect",
|
|
30486
|
-
"Client was closed and is not queryable",
|
|
30487
|
-
"Postgres connection pool changed while acquiring a connection"
|
|
30488
|
-
]);
|
|
30489
|
-
});
|
|
30490
|
-
|
|
30491
30948
|
// dist/builtin/workflows/src/durable/dbos-default-postgres-url.ts
|
|
30492
30949
|
function defaultPostgresUrl(database, host = process.env.PGHOST || "localhost", port = process.env.PGPORT || "5432") {
|
|
30493
30950
|
const url = new URL("postgresql://localhost");
|
|
@@ -30777,7 +31234,7 @@ function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock
|
|
|
30777
31234
|
url: health?.endpoint ?? systemDatabaseUrl,
|
|
30778
31235
|
failure: health?.lastFailure?.message
|
|
30779
31236
|
});
|
|
30780
|
-
const createPool = () => {
|
|
31237
|
+
const createPool = (fenced = true) => {
|
|
30781
31238
|
let unsubscribe;
|
|
30782
31239
|
const managed = createRecoverablePostgresPool(systemDatabaseUrl, {
|
|
30783
31240
|
beforeConnect: health === undefined ? undefined : () => health.check(),
|
|
@@ -30787,14 +31244,38 @@ function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock
|
|
|
30787
31244
|
});
|
|
30788
31245
|
unsubscribe = health?.subscribe(managed.invalidate);
|
|
30789
31246
|
health?.start();
|
|
30790
|
-
return fenceDbosAdmissionPool(managed.pool);
|
|
31247
|
+
return fenced ? fenceDbosAdmissionPool(managed.pool) : managed.pool;
|
|
30791
31248
|
};
|
|
30792
31249
|
let pool = createPool();
|
|
31250
|
+
const createFence = (executorId) => {
|
|
31251
|
+
const fencePool = createPool(false);
|
|
31252
|
+
const created = new DbosOwnerFence(() => fencePool, executorId, () => fencePool.end());
|
|
31253
|
+
const owner = getDbosProcessOwner();
|
|
31254
|
+
owner.executorFences ??= new Set;
|
|
31255
|
+
owner.executorFences.add(created);
|
|
31256
|
+
return created;
|
|
31257
|
+
};
|
|
31258
|
+
let fence = createFence(config.executorID);
|
|
31259
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31260
|
+
getDbosProcessOwner().createExecutorFence = () => {
|
|
31261
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31262
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31263
|
+
const next = createFence(executorId);
|
|
31264
|
+
getDbosProcessOwner().executorFence = next;
|
|
31265
|
+
return next;
|
|
31266
|
+
};
|
|
31267
|
+
pool = fence.protectPool(pool);
|
|
30793
31268
|
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30794
31269
|
const launch = async () => {
|
|
31270
|
+
if (fence.closed) {
|
|
31271
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31272
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31273
|
+
fence = createFence(executorId);
|
|
31274
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31275
|
+
}
|
|
30795
31276
|
if (pool.ended) {
|
|
30796
|
-
pool = createPool();
|
|
30797
|
-
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
31277
|
+
pool = fence.protectPool(createPool());
|
|
31278
|
+
sdk.setConfig({ ...config, executorID: fence.executorId, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30798
31279
|
}
|
|
30799
31280
|
const urlToEnsure = health === undefined ? systemDatabaseUrl : await health.check();
|
|
30800
31281
|
const result = await import_datasource2.ensurePGDatabase({ urlToEnsure, logger: () => {} });
|
|
@@ -30814,6 +31295,7 @@ var import_datasource2;
|
|
|
30814
31295
|
var init_dbos_admission_config = __esm(() => {
|
|
30815
31296
|
init_dbos_admission_pool();
|
|
30816
31297
|
init_dbos_launch_lock();
|
|
31298
|
+
init_dbos_owner_fence();
|
|
30817
31299
|
init_dbos_process_owner();
|
|
30818
31300
|
init_dbos_recoverable_pool();
|
|
30819
31301
|
import_datasource2 = __toESM(require_datasource(), 1);
|
|
@@ -35525,7 +36007,7 @@ function isDurableWorkflowResumable(candidate) {
|
|
|
35525
36007
|
}
|
|
35526
36008
|
var FOREIGN_LIVE_WORKFLOW_WINDOW_MS = 120000;
|
|
35527
36009
|
function isLiveRunningWorkflow(candidate, now = Date.now()) {
|
|
35528
|
-
return candidate.status === "running" && now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
36010
|
+
return candidate.status === "running" && (candidate.ownerLiveness === "alive" || now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS);
|
|
35529
36011
|
}
|
|
35530
36012
|
|
|
35531
36013
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
@@ -35575,7 +36057,7 @@ class InMemoryDurableBackend {
|
|
|
35575
36057
|
workflows = new Map;
|
|
35576
36058
|
promptReservations = new Map;
|
|
35577
36059
|
deletedWorkflowIds = new Set;
|
|
35578
|
-
registerWorkflow(handle) {
|
|
36060
|
+
registerWorkflow(handle, authoritativeOwner = false) {
|
|
35579
36061
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
35580
36062
|
const existing = this.workflows.get(handle.workflowId);
|
|
35581
36063
|
const completedCheckpoints = handle.completedCheckpoints ?? existing?.handle.completedCheckpoints ?? 0;
|
|
@@ -35589,7 +36071,7 @@ class InMemoryDurableBackend {
|
|
|
35589
36071
|
status: handle.status,
|
|
35590
36072
|
...handle.invocationCwd !== undefined ? { invocationCwd: handle.invocationCwd } : existing?.handle.invocationCwd !== undefined ? { invocationCwd: existing.handle.invocationCwd } : {},
|
|
35591
36073
|
...handle.origin !== undefined ? { origin: handle.origin } : existing?.handle.origin !== undefined ? { origin: existing.handle.origin } : {},
|
|
35592
|
-
modelOwner: existing?.handle.modelOwner ?? handle.modelOwner,
|
|
36074
|
+
modelOwner: authoritativeOwner ? handle.modelOwner : existing?.handle.modelOwner ?? handle.modelOwner,
|
|
35593
36075
|
...handle.workflowCwd !== undefined ? { workflowCwd: handle.workflowCwd } : existing?.handle.workflowCwd !== undefined ? { workflowCwd: existing.handle.workflowCwd } : {},
|
|
35594
36076
|
...handle.repositoryRoot !== undefined ? { repositoryRoot: handle.repositoryRoot } : existing?.handle.repositoryRoot !== undefined ? { repositoryRoot: existing.handle.repositoryRoot } : {},
|
|
35595
36077
|
...handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: handle.gitWorktreeRoot } : existing?.handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: existing.handle.gitWorktreeRoot } : {},
|
|
@@ -35603,7 +36085,9 @@ class InMemoryDurableBackend {
|
|
|
35603
36085
|
...handle.rootWorkflowId !== undefined ? { rootWorkflowId: handle.rootWorkflowId } : {},
|
|
35604
36086
|
...handle.resumable !== undefined ? { resumable: handle.resumable } : {},
|
|
35605
36087
|
...workflowFailureFields(hasWorkflowFailureMetadata(handle) ? handle : handle.status === "running" ? undefined : existing?.handle),
|
|
35606
|
-
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {}
|
|
36088
|
+
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {},
|
|
36089
|
+
...handle.ownerLiveness !== undefined ? { ownerLiveness: handle.ownerLiveness } : {},
|
|
36090
|
+
...handle.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {}
|
|
35607
36091
|
};
|
|
35608
36092
|
if (existing)
|
|
35609
36093
|
existing.handle = full;
|
|
@@ -35753,11 +36237,14 @@ class InMemoryDurableBackend {
|
|
|
35753
36237
|
if (pendingPrompts !== undefined)
|
|
35754
36238
|
this.promptReservations.delete(workflowId);
|
|
35755
36239
|
}
|
|
35756
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
35757
|
-
const
|
|
36240
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
36241
|
+
const record = this.workflows.get(workflowId);
|
|
36242
|
+
const current = record?.handle;
|
|
35758
36243
|
if (current === undefined || !expected.includes(current.status) || expectedUpdatedAt !== undefined && current.updatedAt !== expectedUpdatedAt)
|
|
35759
36244
|
return false;
|
|
35760
36245
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
36246
|
+
if (modelOwner !== undefined && record !== undefined)
|
|
36247
|
+
record.handle = { ...record.handle, modelOwner };
|
|
35761
36248
|
return true;
|
|
35762
36249
|
}
|
|
35763
36250
|
adjustPendingPrompts(workflowId, delta) {
|
|
@@ -35830,6 +36317,7 @@ class InMemoryDurableBackend {
|
|
|
35830
36317
|
pendingStageMessages: h.pendingStageMessages ?? [],
|
|
35831
36318
|
...h.possibleStages !== undefined ? { possibleStages: h.possibleStages } : {},
|
|
35832
36319
|
...h.ownerExecutorId !== undefined ? { ownerExecutorId: h.ownerExecutorId } : {},
|
|
36320
|
+
...h.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
35833
36321
|
...h.sessionFile !== undefined ? { sessionFile: h.sessionFile } : {},
|
|
35834
36322
|
...h.label !== undefined ? { label: h.label } : {},
|
|
35835
36323
|
...h.rootWorkflowId !== undefined ? { rootWorkflowId: h.rootWorkflowId } : {},
|
|
@@ -37924,7 +38412,7 @@ function getDurableBackendProcessOwner() {
|
|
|
37924
38412
|
}
|
|
37925
38413
|
|
|
37926
38414
|
// dist/builtin/workflows/src/durable/dbos-backend.ts
|
|
37927
|
-
import { isDeepStrictEqual as
|
|
38415
|
+
import { isDeepStrictEqual as isDeepStrictEqual3 } from "node:util";
|
|
37928
38416
|
init_dbos_admission();
|
|
37929
38417
|
|
|
37930
38418
|
// dist/builtin/workflows/src/durable/dbos-admission-recovery.ts
|
|
@@ -38593,6 +39081,7 @@ function encodeMetadata(metadata) {
|
|
|
38593
39081
|
} : {},
|
|
38594
39082
|
...metadata.ownerExecutorId !== undefined ? { ownerExecutorId: metadata.ownerExecutorId } : {},
|
|
38595
39083
|
...metadata.transitionClaimId !== undefined ? { transitionClaimId: metadata.transitionClaimId } : {},
|
|
39084
|
+
...metadata.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
38596
39085
|
...metadata.sessionFile !== undefined ? { sessionFile: metadata.sessionFile } : {},
|
|
38597
39086
|
...metadata.label !== undefined ? { label: metadata.label } : {},
|
|
38598
39087
|
...metadata.rootWorkflowId !== undefined ? { rootWorkflowId: metadata.rootWorkflowId } : {},
|
|
@@ -38655,7 +39144,7 @@ function parseDurableWorkflowMetadata(value, workflowId) {
|
|
|
38655
39144
|
return;
|
|
38656
39145
|
const possibleStages = coercePossibleStages(serialized.possibleStages);
|
|
38657
39146
|
const metadata = value;
|
|
38658
|
-
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
39147
|
+
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.legacyRecoveryPending !== undefined && metadata.legacyRecoveryPending !== true || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
38659
39148
|
return;
|
|
38660
39149
|
const { origin, possibleStages: rawPossibleStages, ...metadataWithoutOrigin } = metadata;
|
|
38661
39150
|
return {
|
|
@@ -39134,11 +39623,16 @@ function classifyDbosDeletionTombstone(records, workflowId) {
|
|
|
39134
39623
|
init_dbos_process_owner();
|
|
39135
39624
|
|
|
39136
39625
|
// dist/builtin/workflows/src/durable/dbos-sdk-handle.ts
|
|
39137
|
-
|
|
39626
|
+
import { AsyncLocalStorage as AsyncLocalStorage4 } from "node:async_hooks";
|
|
39627
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
39628
|
+
init_dbos_owner_fence();
|
|
39629
|
+
init_dbos_process_owner();
|
|
39138
39630
|
function getAtomicExecutorId() {
|
|
39139
|
-
|
|
39631
|
+
const owner = getDbosProcessOwner();
|
|
39632
|
+
owner.executorId ??= `atomic-db-${crypto.randomUUID()}`;
|
|
39633
|
+
return owner.executorId;
|
|
39140
39634
|
}
|
|
39141
|
-
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
39635
|
+
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, fence, previousExecutorIds = []) {
|
|
39142
39636
|
const checkpointId = (workflowId, stepName) => `${workflowId}:checkpoint:${stepName}`;
|
|
39143
39637
|
async function stepRecord(status, prefix) {
|
|
39144
39638
|
if (status.status !== "SUCCESS")
|
|
@@ -39152,10 +39646,18 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39152
39646
|
const output = status.output === undefined || typeof status.output === "string" ? await dbos.retrieveWorkflow(id).getResult() : status.output;
|
|
39153
39647
|
return { stepName, output, completedAt: status.createdAt };
|
|
39154
39648
|
}
|
|
39155
|
-
|
|
39649
|
+
const raw = {
|
|
39156
39650
|
launch: () => dbos.launch(),
|
|
39157
39651
|
shutdown: () => dbos.shutdown(),
|
|
39158
39652
|
async startWorkflow(workflowId, name, inputs) {
|
|
39653
|
+
if (fence !== undefined) {
|
|
39654
|
+
const existing = await raw.retrieveWorkflow(workflowId);
|
|
39655
|
+
if (existing !== undefined) {
|
|
39656
|
+
if (existing.name !== name || !isDeepStrictEqual2(existing.inputs, inputs))
|
|
39657
|
+
throw new Error("Workflow root identity changed during admission");
|
|
39658
|
+
return;
|
|
39659
|
+
}
|
|
39660
|
+
}
|
|
39159
39661
|
try {
|
|
39160
39662
|
await dbos.startWorkflow(mainWorkflow, { workflowID: workflowId })(name, { ...inputs });
|
|
39161
39663
|
} catch (err) {
|
|
@@ -39174,7 +39676,10 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39174
39676
|
await dbos.cancelWorkflow(workflowId, { cancelChildren: true });
|
|
39175
39677
|
},
|
|
39176
39678
|
async resumeWorkflow(workflowId) {
|
|
39177
|
-
|
|
39679
|
+
if (fence === undefined)
|
|
39680
|
+
await dbos.resumeWorkflow(workflowId);
|
|
39681
|
+
else if (await raw.retrieveWorkflow(workflowId) === undefined)
|
|
39682
|
+
throw new Error("Workflow root is unavailable for resume");
|
|
39178
39683
|
},
|
|
39179
39684
|
async listAllWorkflows() {
|
|
39180
39685
|
const statuses = await dbos.listWorkflows({
|
|
@@ -39203,6 +39708,12 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39203
39708
|
return status !== undefined && (status.workflowID ?? status.workflowId) === id ? stepRecord(status, prefix) : undefined;
|
|
39204
39709
|
},
|
|
39205
39710
|
async recordStepOutput(workflowId, stepName, output) {
|
|
39711
|
+
if (fence !== undefined) {
|
|
39712
|
+
const id = checkpointId(workflowId, stepName);
|
|
39713
|
+
const existing = (await dbos.listWorkflows({ workflowIDs: [id], limit: 1 }))[0];
|
|
39714
|
+
if (existing?.status !== undefined && ["PENDING", "ENQUEUED", "CANCELLED", "ERROR", "DELAYED"].includes(existing.status))
|
|
39715
|
+
await dbos.deleteWorkflows([id], false);
|
|
39716
|
+
}
|
|
39206
39717
|
let handle;
|
|
39207
39718
|
try {
|
|
39208
39719
|
handle = await dbos.startWorkflow(checkpointWorkflow, { workflowID: checkpointId(workflowId, stepName) })(workflowId, stepName, output);
|
|
@@ -39226,6 +39737,135 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39226
39737
|
await dbos.deleteWorkflows([...new Set([workflowId, ...checkpointIds])], true);
|
|
39227
39738
|
}
|
|
39228
39739
|
};
|
|
39740
|
+
if (fence === undefined)
|
|
39741
|
+
return raw;
|
|
39742
|
+
const claims = new AsyncLocalStorage4;
|
|
39743
|
+
const rowAuthority = (workflowId, current) => ({
|
|
39744
|
+
root: workflowId,
|
|
39745
|
+
actor: fence.executorId,
|
|
39746
|
+
...current.kind === "current" ? { owner: current.metadata.ownerExecutorId, generation: current.generation } : {},
|
|
39747
|
+
enroll: current.kind === "unavailable" || current.kind === "current" && isDatabaseExecutor(current.metadata.ownerExecutorId)
|
|
39748
|
+
});
|
|
39749
|
+
const mutate = async (workflowId, callback, step) => fence.write(workflowId, async () => {
|
|
39750
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39751
|
+
const claimed = claims.getStore();
|
|
39752
|
+
if (claimed === undefined && current.kind === "current" && current.metadata.ownerExecutorId !== fence.executorId)
|
|
39753
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
39754
|
+
let output = step === undefined ? undefined : classifyLatestMetadata([{ stepName: step.name, output: step.output }], workflowId);
|
|
39755
|
+
let persistedStep = step;
|
|
39756
|
+
if (step !== undefined && output?.kind === "current" && current.kind === "current" && !step.name.endsWith(":claim") && output.metadata.ownerExecutorId === fence.executorId && output.generation <= current.generation) {
|
|
39757
|
+
const updatedAt = Math.max(current.generation + 1, output.metadata.updatedAt, Date.now());
|
|
39758
|
+
persistedStep = {
|
|
39759
|
+
name: metadataStepName(updatedAt),
|
|
39760
|
+
output: encodeMetadata({ ...output.metadata, updatedAt })
|
|
39761
|
+
};
|
|
39762
|
+
output = classifyLatestMetadata([{ stepName: persistedStep.name, output: persistedStep.output }], workflowId);
|
|
39763
|
+
}
|
|
39764
|
+
const authority = claimed?.authority ?? rowAuthority(workflowId, current);
|
|
39765
|
+
const capability = output?.kind === "current" && step?.name.endsWith(":claim") ? {
|
|
39766
|
+
...authority,
|
|
39767
|
+
claim: output.metadata.transitionClaimId,
|
|
39768
|
+
checkpoint: checkpointId(workflowId, step.name)
|
|
39769
|
+
} : authority;
|
|
39770
|
+
const record = persistedStep;
|
|
39771
|
+
await fence.withRowAuthority(capability, record === undefined ? callback : () => raw.recordStepOutput(workflowId, record.name, record.output));
|
|
39772
|
+
if (claimed !== undefined && output?.kind === "current")
|
|
39773
|
+
claimed.authority = {
|
|
39774
|
+
...capability,
|
|
39775
|
+
owner: output.metadata.ownerExecutorId,
|
|
39776
|
+
generation: output.generation
|
|
39777
|
+
};
|
|
39778
|
+
});
|
|
39779
|
+
return {
|
|
39780
|
+
...raw,
|
|
39781
|
+
executorId: fence.executorId,
|
|
39782
|
+
generationLost: () => fence.invalidated,
|
|
39783
|
+
forkGeneration: () => {
|
|
39784
|
+
const next = getDbosProcessOwner().createExecutorFence?.();
|
|
39785
|
+
if (next === undefined)
|
|
39786
|
+
throw new Error("Workflow ownership fence cannot create a recovery generation");
|
|
39787
|
+
fence.close().catch(() => {});
|
|
39788
|
+
return createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, next, [
|
|
39789
|
+
...previousExecutorIds,
|
|
39790
|
+
fence.executorId
|
|
39791
|
+
]);
|
|
39792
|
+
},
|
|
39793
|
+
adoptPreviousGeneration: async (workflowId) => {
|
|
39794
|
+
if (previousExecutorIds.length === 0)
|
|
39795
|
+
return;
|
|
39796
|
+
return await fence.write(workflowId, async () => {
|
|
39797
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39798
|
+
if (current.kind !== "current" || current.metadata.ownerExecutorId === fence.executorId)
|
|
39799
|
+
return;
|
|
39800
|
+
const previousExecutorId = current.metadata.ownerExecutorId;
|
|
39801
|
+
if (previousExecutorId === undefined || !previousExecutorIds.includes(previousExecutorId))
|
|
39802
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
39803
|
+
const metadata = {
|
|
39804
|
+
...current.metadata,
|
|
39805
|
+
ownerExecutorId: fence.executorId,
|
|
39806
|
+
transitionClaimId: crypto.randomUUID(),
|
|
39807
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
39808
|
+
};
|
|
39809
|
+
const adopted = await fence.recover(previousExecutorId, async () => {
|
|
39810
|
+
await fence.withRowAuthority({
|
|
39811
|
+
...rowAuthority(workflowId, current),
|
|
39812
|
+
claim: metadata.transitionClaimId,
|
|
39813
|
+
checkpoint: checkpointId(workflowId, claimMetadataStepName(current.generation))
|
|
39814
|
+
}, () => raw.recordStepOutput(workflowId, claimMetadataStepName(current.generation), encodeMetadata(metadata)));
|
|
39815
|
+
return metadata;
|
|
39816
|
+
});
|
|
39817
|
+
if (adopted === undefined)
|
|
39818
|
+
throw new Error("Workflow previous execution generation is still active");
|
|
39819
|
+
return adopted;
|
|
39820
|
+
}, true);
|
|
39821
|
+
},
|
|
39822
|
+
enrollLegacyWorkflow: async (workflowId, modelOwner) => fence.write(workflowId, async () => {
|
|
39823
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39824
|
+
if (current.kind !== "current" || isDatabaseExecutor(current.metadata.ownerExecutorId) && current.metadata.legacyRecoveryPending !== true || !["running", "paused", "blocked"].includes(current.metadata.status) || current.metadata.rootWorkflowId !== undefined && current.metadata.rootWorkflowId !== workflowId)
|
|
39825
|
+
return;
|
|
39826
|
+
const metadata = {
|
|
39827
|
+
...current.metadata,
|
|
39828
|
+
modelOwner,
|
|
39829
|
+
ownerExecutorId: fence.executorId,
|
|
39830
|
+
status: "blocked",
|
|
39831
|
+
resumable: true,
|
|
39832
|
+
legacyRecoveryPending: true,
|
|
39833
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
39834
|
+
};
|
|
39835
|
+
const retry = isDatabaseExecutor(current.metadata.ownerExecutorId);
|
|
39836
|
+
const transitionClaimId = crypto.randomUUID();
|
|
39837
|
+
const stepName = retry ? claimMetadataStepName(current.generation) : metadataStepName(Math.max(current.generation + 1, metadata.updatedAt));
|
|
39838
|
+
const persist = () => fence.withRowAuthority({
|
|
39839
|
+
...rowAuthority(workflowId, current),
|
|
39840
|
+
enroll: true,
|
|
39841
|
+
...retry ? { claim: transitionClaimId, checkpoint: checkpointId(workflowId, stepName) } : {}
|
|
39842
|
+
}, () => raw.recordStepOutput(workflowId, stepName, encodeMetadata({ ...metadata, ...retry ? { transitionClaimId } : {} })));
|
|
39843
|
+
if (retry) {
|
|
39844
|
+
const recovered = await fence.recover(current.metadata.ownerExecutorId, async () => {
|
|
39845
|
+
await persist();
|
|
39846
|
+
return true;
|
|
39847
|
+
});
|
|
39848
|
+
if (recovered !== true)
|
|
39849
|
+
return;
|
|
39850
|
+
} else
|
|
39851
|
+
await persist();
|
|
39852
|
+
return metadata;
|
|
39853
|
+
}, true),
|
|
39854
|
+
ownerLiveness: (executorId) => fence.liveness(executorId),
|
|
39855
|
+
withWorkflowClaim: (workflowId, callback) => fence.write(workflowId, async () => {
|
|
39856
|
+
const authoritative = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39857
|
+
const claim = () => claims.run({ authority: { ...rowAuthority(workflowId, authoritative), claim: crypto.randomUUID() } }, callback);
|
|
39858
|
+
if (authoritative.kind === "current" && authoritative.metadata.status === "running" && authoritative.metadata.ownerExecutorId !== fence.executorId) {
|
|
39859
|
+
return await fence.recover(authoritative.metadata.ownerExecutorId, claim) ?? false;
|
|
39860
|
+
}
|
|
39861
|
+
return await claim();
|
|
39862
|
+
}, true),
|
|
39863
|
+
startWorkflow: (id, name, inputs) => mutate(id, () => raw.startWorkflow(id, name, inputs)),
|
|
39864
|
+
cancelWorkflow: (id) => mutate(id, () => raw.cancelWorkflow(id)),
|
|
39865
|
+
resumeWorkflow: (id) => mutate(id, () => raw.resumeWorkflow(id)),
|
|
39866
|
+
recordStepOutput: (id, step, output) => mutate(id, () => raw.recordStepOutput(id, step, output), { name: step, output }),
|
|
39867
|
+
deleteWorkflowData: (id) => mutate(id, () => raw.deleteWorkflowData(id))
|
|
39868
|
+
};
|
|
39229
39869
|
}
|
|
39230
39870
|
function isDbosDuplicateWorkflowError(err) {
|
|
39231
39871
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -39307,6 +39947,8 @@ async function configureDbosDurableBackend(config) {
|
|
|
39307
39947
|
const sdk = await importDbosSdk();
|
|
39308
39948
|
const owner = getDbosProcessOwner();
|
|
39309
39949
|
const existing = owner.wrappers;
|
|
39950
|
+
if (existing !== undefined && owner.executorFence === undefined)
|
|
39951
|
+
throw new Error("Existing workflow executor has no database ownership fence; restart the host before recovering workflows");
|
|
39310
39952
|
let launch = owner.active?.launch ?? (() => sdk.launch());
|
|
39311
39953
|
let checkReady;
|
|
39312
39954
|
if (existing === undefined) {
|
|
@@ -39329,12 +39971,30 @@ async function configureDbosDurableBackend(config) {
|
|
|
39329
39971
|
if (existing === undefined) {
|
|
39330
39972
|
owner.wrappers = { mainWorkflow, checkpointWorkflow };
|
|
39331
39973
|
}
|
|
39974
|
+
let backendFence = owner.executorFence;
|
|
39975
|
+
let backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), {
|
|
39976
|
+
checkReady
|
|
39977
|
+
});
|
|
39332
39978
|
return {
|
|
39333
|
-
backend
|
|
39334
|
-
|
|
39335
|
-
}
|
|
39336
|
-
launch
|
|
39337
|
-
|
|
39979
|
+
get backend() {
|
|
39980
|
+
return backend;
|
|
39981
|
+
},
|
|
39982
|
+
launch: async () => {
|
|
39983
|
+
await launch();
|
|
39984
|
+
if (backendFence !== owner.executorFence && backendFence?.closed) {
|
|
39985
|
+
backendFence = owner.executorFence;
|
|
39986
|
+
backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), { checkReady });
|
|
39987
|
+
}
|
|
39988
|
+
},
|
|
39989
|
+
shutdown: async () => {
|
|
39990
|
+
const results = await Promise.allSettled([
|
|
39991
|
+
sdk.shutdown(),
|
|
39992
|
+
...[...owner.executorFences ?? []].map((fence) => fence.close())
|
|
39993
|
+
]);
|
|
39994
|
+
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason instanceof Error ? result.reason : new Error(String(result.reason)));
|
|
39995
|
+
if (errors.length > 0)
|
|
39996
|
+
throw new AggregateError(errors, "Workflow database shutdown failed");
|
|
39997
|
+
}
|
|
39338
39998
|
};
|
|
39339
39999
|
}
|
|
39340
40000
|
async function importDbosSdk() {
|
|
@@ -39367,13 +40027,18 @@ class DbosDurableBackend {
|
|
|
39367
40027
|
unavailableAdmissions = new Set;
|
|
39368
40028
|
unavailableCheckpoints = new Set;
|
|
39369
40029
|
pendingRecoveryAdmissions = new Set;
|
|
40030
|
+
ownerLiveness = new Map;
|
|
39370
40031
|
admissionMetadataAttempted = new Set;
|
|
39371
40032
|
admissionSettlements = new Map;
|
|
39372
40033
|
admissionRecoveries = new Map;
|
|
39373
40034
|
admissionRecoveryControllers = new Map;
|
|
40035
|
+
fixedGeneration = false;
|
|
40036
|
+
generationIsCurrent;
|
|
40037
|
+
rebindInitialAdmission;
|
|
40038
|
+
admitted = false;
|
|
39374
40039
|
constructor(sdk, options) {
|
|
39375
40040
|
this.sdk = sdk;
|
|
39376
|
-
this.executorId = options?.executorId ?? getAtomicExecutorId();
|
|
40041
|
+
this.executorId = options?.executorId ?? sdk.executorId ?? getAtomicExecutorId();
|
|
39377
40042
|
this.onUnavailable = options?.onUnavailable;
|
|
39378
40043
|
this.checkReady = options?.checkReady;
|
|
39379
40044
|
this.promptReservations = new DbosPromptReservationTracker({
|
|
@@ -39384,6 +40049,55 @@ class DbosDurableBackend {
|
|
|
39384
40049
|
}
|
|
39385
40050
|
});
|
|
39386
40051
|
}
|
|
40052
|
+
async enrollLegacyWorkflow(workflowId, options) {
|
|
40053
|
+
if (options.olderWorkersStopped !== true || this.sdk.enrollLegacyWorkflow === undefined)
|
|
40054
|
+
return false;
|
|
40055
|
+
this.ensureGeneration();
|
|
40056
|
+
return await boundedAdmission((signal) => dbosAdmissionContext.run(signal, async () => {
|
|
40057
|
+
signal.throwIfAborted();
|
|
40058
|
+
const metadata = await this.sdk.enrollLegacyWorkflow(workflowId, options.modelOwner);
|
|
40059
|
+
signal.throwIfAborted();
|
|
40060
|
+
if (metadata === undefined)
|
|
40061
|
+
return false;
|
|
40062
|
+
this.applyMetadata(workflowId, metadata);
|
|
40063
|
+
this.current.add(workflowId);
|
|
40064
|
+
return true;
|
|
40065
|
+
}), options.signal);
|
|
40066
|
+
}
|
|
40067
|
+
executionView() {
|
|
40068
|
+
if (this.fixedGeneration || this.sdk.forkGeneration === undefined)
|
|
40069
|
+
return this;
|
|
40070
|
+
this.ensureGeneration();
|
|
40071
|
+
const view = new DbosDurableBackend(this.sdk, {
|
|
40072
|
+
executorId: this.executorId,
|
|
40073
|
+
checkReady: this.checkReady,
|
|
40074
|
+
onUnavailable: this.onUnavailable
|
|
40075
|
+
});
|
|
40076
|
+
const reservations = view.promptReservations;
|
|
40077
|
+
Object.assign(view, this);
|
|
40078
|
+
Object.assign(view, {
|
|
40079
|
+
promptReservations: reservations,
|
|
40080
|
+
fixedGeneration: true,
|
|
40081
|
+
generationIsCurrent: () => this.sdk === view.sdk,
|
|
40082
|
+
rebindInitialAdmission: () => {
|
|
40083
|
+
this.ensureGeneration();
|
|
40084
|
+
view.sdk = this.sdk;
|
|
40085
|
+
view.executorId = this.executorId;
|
|
40086
|
+
}
|
|
40087
|
+
});
|
|
40088
|
+
return view;
|
|
40089
|
+
}
|
|
40090
|
+
ensureGeneration() {
|
|
40091
|
+
if (this.fixedGeneration) {
|
|
40092
|
+
if (this.generationIsCurrent?.() === false)
|
|
40093
|
+
throw new DbosDependencyError("Workflow database ownership generation changed; stale execution writes are refused");
|
|
40094
|
+
return;
|
|
40095
|
+
}
|
|
40096
|
+
if (this.sdk.generationLost?.() && this.sdk.forkGeneration !== undefined) {
|
|
40097
|
+
this.sdk = this.sdk.forkGeneration();
|
|
40098
|
+
this.executorId = this.sdk.executorId ?? this.executorId;
|
|
40099
|
+
}
|
|
40100
|
+
}
|
|
39387
40101
|
isAdmissionUnavailable(workflowId) {
|
|
39388
40102
|
return this.unavailableAdmissions.has(workflowId);
|
|
39389
40103
|
}
|
|
@@ -39400,6 +40114,8 @@ class DbosDurableBackend {
|
|
|
39400
40114
|
return this.admissionSettlements.has(workflowId);
|
|
39401
40115
|
}
|
|
39402
40116
|
admitWorkflow(workflowId, registration, signal) {
|
|
40117
|
+
if (!this.admitted)
|
|
40118
|
+
this.rebindInitialAdmission?.();
|
|
39403
40119
|
const pending = this.performAdmission(workflowId, registration, signal);
|
|
39404
40120
|
this.admissionSettlements.set(workflowId, pending);
|
|
39405
40121
|
pending.then(() => {
|
|
@@ -39407,6 +40123,7 @@ class DbosDurableBackend {
|
|
|
39407
40123
|
this.admissionSettlements.delete(workflowId);
|
|
39408
40124
|
}, () => {});
|
|
39409
40125
|
return pending.then(async () => {
|
|
40126
|
+
this.admitted = true;
|
|
39410
40127
|
await dbosAdmissionContext.run(signal, () => this.flush(workflowId));
|
|
39411
40128
|
signal.throwIfAborted();
|
|
39412
40129
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
@@ -39427,12 +40144,13 @@ class DbosDurableBackend {
|
|
|
39427
40144
|
await this.flushWrites(workflowId);
|
|
39428
40145
|
signal.throwIfAborted();
|
|
39429
40146
|
}), signal);
|
|
40147
|
+
this.ensureGeneration();
|
|
39430
40148
|
this.admissionUnavailable = false;
|
|
39431
40149
|
this.unavailableAdmissions.delete(workflowId);
|
|
39432
40150
|
this.unavailableCheckpoints.delete(workflowId);
|
|
39433
40151
|
this.pendingRecoveryAdmissions.delete(workflowId);
|
|
39434
40152
|
} catch (error) {
|
|
39435
|
-
if (isDbosDependencyError(error)) {
|
|
40153
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
39436
40154
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
39437
40155
|
this.admissionUnavailable = true;
|
|
39438
40156
|
this.unavailableAdmissions.add(workflowId);
|
|
@@ -39458,6 +40176,7 @@ class DbosDurableBackend {
|
|
|
39458
40176
|
});
|
|
39459
40177
|
}
|
|
39460
40178
|
registerWorkflow(handle) {
|
|
40179
|
+
this.ensureGeneration();
|
|
39461
40180
|
if (this.invalid.has(handle.workflowId)) {
|
|
39462
40181
|
throw new DurableNestedTopologyError(`workflow ${handle.workflowId} contains malformed current DBOS records`);
|
|
39463
40182
|
}
|
|
@@ -39499,6 +40218,7 @@ class DbosDurableBackend {
|
|
|
39499
40218
|
return persisted;
|
|
39500
40219
|
}
|
|
39501
40220
|
recordCheckpoint(checkpoint) {
|
|
40221
|
+
this.ensureGeneration();
|
|
39502
40222
|
if (!this.isWorkflowLoadable(checkpoint.workflowId))
|
|
39503
40223
|
return;
|
|
39504
40224
|
this.mem.recordCheckpoint(checkpoint);
|
|
@@ -39527,7 +40247,7 @@ class DbosDurableBackend {
|
|
|
39527
40247
|
});
|
|
39528
40248
|
signal.throwIfAborted();
|
|
39529
40249
|
}), options?.signal ?? dbosAdmissionContext.getStore(), undefined, "Workflow database checkpoint timed out. Restore PostgreSQL and inspect the run before resuming; external outcomes may be unknown.").catch((error) => {
|
|
39530
|
-
if (isDbosDependencyError(error)) {
|
|
40250
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
39531
40251
|
this.unavailableCheckpoints.add(checkpoint.workflowId);
|
|
39532
40252
|
const errors = this.writeErrors.get(checkpoint.workflowId);
|
|
39533
40253
|
if (errors !== undefined)
|
|
@@ -39575,12 +40295,14 @@ class DbosDurableBackend {
|
|
|
39575
40295
|
return this.mem.listCheckpoints(workflowId);
|
|
39576
40296
|
}
|
|
39577
40297
|
getWorkflow(workflowId) {
|
|
39578
|
-
|
|
40298
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
40299
|
+
return handle === undefined || this.sdk.ownerLiveness === undefined ? handle : { ...handle, ownerLiveness: this.ownerLiveness.get(handle.ownerExecutorId ?? "") ?? "unknown" };
|
|
39579
40300
|
}
|
|
39580
40301
|
getLoadableWorkflow(workflowId) {
|
|
39581
|
-
return this.isWorkflowLoadable(workflowId) ? this.
|
|
40302
|
+
return this.isWorkflowLoadable(workflowId) ? this.getWorkflow(workflowId) : undefined;
|
|
39582
40303
|
}
|
|
39583
40304
|
setWorkflowStatus(workflowId, status, pendingPrompts, resumable, failure) {
|
|
40305
|
+
this.ensureGeneration();
|
|
39584
40306
|
if (status === "cancelled" || status === "completed" || resumable === false) {
|
|
39585
40307
|
this.admissionRecoveryControllers.get(workflowId)?.abort(new Error("Workflow recovery superseded by terminal control."));
|
|
39586
40308
|
}
|
|
@@ -39604,9 +40326,10 @@ class DbosDurableBackend {
|
|
|
39604
40326
|
await this.writeMetadata(workflowId);
|
|
39605
40327
|
});
|
|
39606
40328
|
}
|
|
39607
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
40329
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
40330
|
+
this.ensureGeneration();
|
|
39608
40331
|
let records = [];
|
|
39609
|
-
|
|
40332
|
+
const transition = () => transitionDbosWorkflowStatus({
|
|
39610
40333
|
expectedStatuses: expected,
|
|
39611
40334
|
status,
|
|
39612
40335
|
flush: () => this.flush(workflowId),
|
|
@@ -39620,14 +40343,25 @@ class DbosDurableBackend {
|
|
|
39620
40343
|
const pendingPrompts = this.promptReservations.hydrate(workflowId, entry.pendingPrompts, records, entry.promptReservationEpoch);
|
|
39621
40344
|
this.applyMetadata(workflowId, { ...entry, pendingPrompts });
|
|
39622
40345
|
},
|
|
39623
|
-
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable),
|
|
40346
|
+
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner),
|
|
39624
40347
|
write: async () => {
|
|
40348
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
40349
|
+
if (handle !== undefined)
|
|
40350
|
+
this.mem.registerWorkflow({
|
|
40351
|
+
...handle,
|
|
40352
|
+
legacyRecoveryPending: undefined,
|
|
40353
|
+
...modelOwner !== undefined ? { modelOwner } : {}
|
|
40354
|
+
}, true);
|
|
39625
40355
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
39626
40356
|
await this.flush(workflowId);
|
|
39627
40357
|
}
|
|
39628
40358
|
});
|
|
40359
|
+
if (this.sdk.withWorkflowClaim === undefined)
|
|
40360
|
+
return await transition();
|
|
40361
|
+
await this.flush(workflowId);
|
|
40362
|
+
return await this.sdk.withWorkflowClaim(workflowId, transition);
|
|
39629
40363
|
}
|
|
39630
|
-
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable) {
|
|
40364
|
+
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner) {
|
|
39631
40365
|
const stepName = claimMetadataStepName(generation);
|
|
39632
40366
|
const transitionClaimId = crypto.randomUUID();
|
|
39633
40367
|
const claim = {
|
|
@@ -39635,6 +40369,8 @@ class DbosDurableBackend {
|
|
|
39635
40369
|
status,
|
|
39636
40370
|
...pendingPrompts !== undefined ? { pendingPrompts } : {},
|
|
39637
40371
|
...resumable !== undefined ? { resumable } : {},
|
|
40372
|
+
...modelOwner === undefined ? {} : { modelOwner },
|
|
40373
|
+
legacyRecoveryPending: undefined,
|
|
39638
40374
|
ownerExecutorId: this.executorId,
|
|
39639
40375
|
transitionClaimId,
|
|
39640
40376
|
updatedAt: Math.max(Date.now(), authoritative.updatedAt + 1)
|
|
@@ -39776,7 +40512,7 @@ class DbosDurableBackend {
|
|
|
39776
40512
|
this.mem.recordCheckpoint(classified.checkpoint);
|
|
39777
40513
|
}
|
|
39778
40514
|
const handle = this.getLoadableWorkflow(workflowId);
|
|
39779
|
-
return handle === undefined ? { kind: "malformed" } :
|
|
40515
|
+
return handle === undefined ? { kind: "malformed" } : await this.withOwnerLiveness(handle);
|
|
39780
40516
|
}
|
|
39781
40517
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
39782
40518
|
if (info !== undefined)
|
|
@@ -39799,6 +40535,7 @@ class DbosDurableBackend {
|
|
|
39799
40535
|
}
|
|
39800
40536
|
}
|
|
39801
40537
|
reconcileWorkflowAdmission(workflowId, signal) {
|
|
40538
|
+
this.ensureGeneration();
|
|
39802
40539
|
signal?.throwIfAborted();
|
|
39803
40540
|
if (!this.unavailableAdmissions.has(workflowId) && !this.unavailableCheckpoints.has(workflowId))
|
|
39804
40541
|
return Promise.resolve();
|
|
@@ -39813,11 +40550,13 @@ class DbosDurableBackend {
|
|
|
39813
40550
|
return;
|
|
39814
40551
|
if (metadata.resumable === false || metadata.status === "cancelled" || metadata.status === "completed")
|
|
39815
40552
|
return;
|
|
39816
|
-
if (!await reconcileDbosAdmission(this.sdk, this.promptReservations.metadata(workflowId, metadata), this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
40553
|
+
if (!await reconcileDbosAdmission(this.sdk, { ...this.promptReservations.metadata(workflowId, metadata), ownerExecutorId: this.executorId }, this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
39817
40554
|
await this.suppressWorkflow(workflowId);
|
|
39818
40555
|
return;
|
|
39819
40556
|
}
|
|
39820
40557
|
admissionSignal.throwIfAborted();
|
|
40558
|
+
await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
40559
|
+
admissionSignal.throwIfAborted();
|
|
39821
40560
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
39822
40561
|
if (info === undefined)
|
|
39823
40562
|
return;
|
|
@@ -39854,7 +40593,7 @@ class DbosDurableBackend {
|
|
|
39854
40593
|
return { kind: deletion === "current" ? "deleted" : "malformed" };
|
|
39855
40594
|
}
|
|
39856
40595
|
const metadata = classifyLatestMetadata(records, info.workflowId);
|
|
39857
|
-
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !
|
|
40596
|
+
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !isDeepStrictEqual3(metadata.metadata.inputs, expected.inputs))) {
|
|
39858
40597
|
await this.suppressWorkflow(info.workflowId);
|
|
39859
40598
|
return { kind: "malformed" };
|
|
39860
40599
|
}
|
|
@@ -39888,10 +40627,17 @@ class DbosDurableBackend {
|
|
|
39888
40627
|
});
|
|
39889
40628
|
const handle = this.getLoadableWorkflow(info.workflowId);
|
|
39890
40629
|
if (handle !== undefined)
|
|
39891
|
-
return
|
|
40630
|
+
return await this.withOwnerLiveness(handle);
|
|
39892
40631
|
await this.suppressWorkflow(info.workflowId);
|
|
39893
40632
|
return { kind: "malformed" };
|
|
39894
40633
|
}
|
|
40634
|
+
async withOwnerLiveness(handle) {
|
|
40635
|
+
if (this.sdk.ownerLiveness === undefined)
|
|
40636
|
+
return { kind: "current", handle };
|
|
40637
|
+
const ownerLiveness = await this.sdk.ownerLiveness(handle.ownerExecutorId);
|
|
40638
|
+
this.ownerLiveness.set(handle.ownerExecutorId ?? "", ownerLiveness);
|
|
40639
|
+
return { kind: "current", handle: { ...handle, ownerLiveness } };
|
|
40640
|
+
}
|
|
39895
40641
|
async suppressWorkflow(workflowId) {
|
|
39896
40642
|
this.invalid.add(workflowId);
|
|
39897
40643
|
this.current.delete(workflowId);
|
|
@@ -39905,10 +40651,21 @@ class DbosDurableBackend {
|
|
|
39905
40651
|
return Promise.resolve();
|
|
39906
40652
|
const previous = this.writeQueues.get(workflowId) ?? Promise.resolve();
|
|
39907
40653
|
const next = previous.then(async () => {
|
|
40654
|
+
this.ensureGeneration();
|
|
40655
|
+
const adopted = await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
40656
|
+
const local = this.mem.getWorkflow(workflowId);
|
|
40657
|
+
if (adopted !== undefined && local !== undefined)
|
|
40658
|
+
this.mem.registerWorkflow({
|
|
40659
|
+
...local,
|
|
40660
|
+
ownerExecutorId: adopted.ownerExecutorId,
|
|
40661
|
+
updatedAt: Math.max(local.updatedAt, adopted.updatedAt + 1)
|
|
40662
|
+
}, true);
|
|
39908
40663
|
if (!signal?.aborted)
|
|
39909
40664
|
await fn();
|
|
39910
40665
|
});
|
|
39911
40666
|
const tracked = next.catch((err) => {
|
|
40667
|
+
if (this.generationIsCurrent?.() === false)
|
|
40668
|
+
return;
|
|
39912
40669
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
39913
40670
|
const errors = this.writeErrors.get(workflowId) ?? [];
|
|
39914
40671
|
errors.push(error);
|
|
@@ -39945,7 +40702,7 @@ class DbosDurableBackend {
|
|
|
39945
40702
|
applyMetadata(workflowId, metadata) {
|
|
39946
40703
|
if (metadata.workflowId !== workflowId)
|
|
39947
40704
|
return;
|
|
39948
|
-
this.mem.registerWorkflow(metadata);
|
|
40705
|
+
this.mem.registerWorkflow(metadata, true);
|
|
39949
40706
|
}
|
|
39950
40707
|
}
|
|
39951
40708
|
|
|
@@ -39981,7 +40738,7 @@ import { tmpdir, uptime } from "node:os";
|
|
|
39981
40738
|
import { basename as basename3, dirname as dirname3, isAbsolute as isAbsolute3, join as join4, relative as relative2, sep as sep2 } from "node:path";
|
|
39982
40739
|
|
|
39983
40740
|
// dist/builtin/workflows/src/durable/dbos-embedded-postgres-root.ts
|
|
39984
|
-
import { createHash as
|
|
40741
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
39985
40742
|
import {
|
|
39986
40743
|
chmod,
|
|
39987
40744
|
chown,
|
|
@@ -40464,8 +41221,8 @@ async function snapshotSourceRuntime(root, publisher, progress) {
|
|
|
40464
41221
|
const rootStat = await lstat(root);
|
|
40465
41222
|
assertRuntimeRoot(root, rootStat);
|
|
40466
41223
|
const rootRealPath = await realpath(root);
|
|
40467
|
-
const sourceHash =
|
|
40468
|
-
const sealedHash =
|
|
41224
|
+
const sourceHash = createHash5("sha256");
|
|
41225
|
+
const sealedHash = createHash5("sha256");
|
|
40469
41226
|
await hashSourceEntry(sourceHash, sealedHash, root, ".", rootRealPath, publisher, progress);
|
|
40470
41227
|
return { sourceIdentity: sourceHash.digest("hex"), sealedIdentity: sealedHash.digest("hex") };
|
|
40471
41228
|
}
|
|
@@ -40730,7 +41487,7 @@ async function snapshotSealedManifest(root, progress) {
|
|
|
40730
41487
|
const rootStat = await lstat(root);
|
|
40731
41488
|
assertRuntimeRoot(root, rootStat);
|
|
40732
41489
|
const rootRealPath = await realpath(root);
|
|
40733
|
-
const hash =
|
|
41490
|
+
const hash = createHash5("sha256");
|
|
40734
41491
|
const entries = [];
|
|
40735
41492
|
await hashSealedEntry(hash, root, ".", rootRealPath, progress, entries);
|
|
40736
41493
|
return { version: 1, sealedIdentity: hash.digest("hex"), entries };
|
|
@@ -40964,6 +41721,13 @@ function resolveEmbeddedPostgresTarget(host) {
|
|
|
40964
41721
|
init_dbos_admission();
|
|
40965
41722
|
var HEALTH_INTERVAL_MS = 5000;
|
|
40966
41723
|
var RECOVERY_ATTEMPTS = 3;
|
|
41724
|
+
function isMonitoringConnectionTimeout(error) {
|
|
41725
|
+
if (!(error instanceof Error))
|
|
41726
|
+
return false;
|
|
41727
|
+
if ("code" in error)
|
|
41728
|
+
return error.code === "ETIMEDOUT";
|
|
41729
|
+
return error.message === "timeout expired" || error.message === "Connection terminated due to connection timeout";
|
|
41730
|
+
}
|
|
40967
41731
|
function isMonitoringConnectionFailure(error) {
|
|
40968
41732
|
if (!(error instanceof Error))
|
|
40969
41733
|
return false;
|
|
@@ -41074,7 +41838,7 @@ class PostgresHealth {
|
|
|
41074
41838
|
try {
|
|
41075
41839
|
identity = await this.operations.probe();
|
|
41076
41840
|
} catch (retryError) {
|
|
41077
|
-
if (!isMonitoringConnectionFailure(retryError))
|
|
41841
|
+
if (isMonitoringConnectionTimeout(retryError) || !isMonitoringConnectionFailure(retryError))
|
|
41078
41842
|
throw retryError;
|
|
41079
41843
|
this.retainFailure(retryError);
|
|
41080
41844
|
return;
|
|
@@ -41100,7 +41864,7 @@ class PostgresHealth {
|
|
|
41100
41864
|
if (ready !== undefined)
|
|
41101
41865
|
return ready;
|
|
41102
41866
|
} catch (error) {
|
|
41103
|
-
if (isQueryReadTimeout(error)) {
|
|
41867
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error)) {
|
|
41104
41868
|
this.retainFailure(new Error("Managed PostgreSQL did not answer a health check in time."));
|
|
41105
41869
|
throw this.dependencyFailure("Managed PostgreSQL did not answer a health check in time.");
|
|
41106
41870
|
}
|
|
@@ -41408,7 +42172,7 @@ async function probePostgresIdentity(port) {
|
|
|
41408
42172
|
const result = await client.query(POSTGRES_IDENTITY_SQL);
|
|
41409
42173
|
return result.rows[0];
|
|
41410
42174
|
} catch (error) {
|
|
41411
|
-
if (isQueryReadTimeout(error))
|
|
42175
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
41412
42176
|
throw error;
|
|
41413
42177
|
const code = error instanceof Error && "code" in error ? error.code : undefined;
|
|
41414
42178
|
if (code === "ECONNREFUSED" || code === "ECONNRESET" || code === "EPIPE" || code === "57P01" || code === "57P02" || code === "57P03" || error instanceof Error && /timeout|timed out|connection terminated/i.test(error.message))
|
|
@@ -41581,7 +42345,7 @@ async function ensureCluster(options = {}) {
|
|
|
41581
42345
|
try {
|
|
41582
42346
|
return await existingServerProbe()(probePort);
|
|
41583
42347
|
} catch (error) {
|
|
41584
|
-
if (isQueryReadTimeout(error))
|
|
42348
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
41585
42349
|
return;
|
|
41586
42350
|
throw error;
|
|
41587
42351
|
}
|
|
@@ -48639,6 +49403,11 @@ var jobTracker = singleton5.facade;
|
|
|
48639
49403
|
|
|
48640
49404
|
// dist/builtin/workflows/src/durable/resume-runtime.ts
|
|
48641
49405
|
init_dbos_admission();
|
|
49406
|
+
|
|
49407
|
+
// dist/builtin/workflows/src/durable/sdk-recovery-scope.ts
|
|
49408
|
+
init_dbos_owner_fence();
|
|
49409
|
+
|
|
49410
|
+
// dist/builtin/workflows/src/durable/resume-runtime.ts
|
|
48642
49411
|
var pendingResumes = new WeakMap;
|
|
48643
49412
|
var unsettledResumeClaims = new WeakMap;
|
|
48644
49413
|
|
|
@@ -54164,7 +54933,7 @@ class StageSessionController {
|
|
|
54164
54933
|
}
|
|
54165
54934
|
|
|
54166
54935
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
54167
|
-
import { createHash as
|
|
54936
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
54168
54937
|
import { mkdir as mkdir3, writeFile as writeFile2 } from "node:fs/promises";
|
|
54169
54938
|
import { basename as basename6, dirname as dirname10, isAbsolute as isAbsolute11, join as join13, resolve as resolve10 } from "node:path";
|
|
54170
54939
|
var DEFAULT_MAX_OUTPUT_BYTES = 200 * 1024;
|
|
@@ -54230,7 +54999,7 @@ function formatByteSize(bytes) {
|
|
|
54230
54999
|
return `${value.toFixed(1)} ${units[unitIndex]}`;
|
|
54231
55000
|
}
|
|
54232
55001
|
function transcriptPath(runId, outputPath) {
|
|
54233
|
-
const digest =
|
|
55002
|
+
const digest = createHash6("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
54234
55003
|
return join13(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename6(outputPath)}.transcript.md`);
|
|
54235
55004
|
}
|
|
54236
55005
|
function appendContent(lines, content) {
|
|
@@ -55332,7 +56101,8 @@ async function run(def, inputs, opts = {}) {
|
|
|
55332
56101
|
ownController.abort(callerSignal?.reason);
|
|
55333
56102
|
};
|
|
55334
56103
|
const exit = createWorkflowExitManager({ runId, exitScope, controller: ownController });
|
|
55335
|
-
const
|
|
56104
|
+
const sourceBackend = opts.durableBackend ?? getDurableBackend();
|
|
56105
|
+
const backendView = sourceBackend.executionView?.() ?? sourceBackend;
|
|
55336
56106
|
const rootBackend = opts.durableRootBackend ?? backendView;
|
|
55337
56107
|
const durableBackend = opts.durableScope !== undefined ? new ScopedDurableBackend(backendView, opts.durableScope) : backendView;
|
|
55338
56108
|
const inheritedElapsedMs = opts.parentRun === undefined ? inheritedRunElapsedMs({ backend: durableBackend, runId, continuationSource: opts.continuation?.source }) : undefined;
|
|
@@ -56092,6 +56862,17 @@ async function run(def, inputs, opts = {}) {
|
|
|
56092
56862
|
const gracefulQuit = observedQuitCancellation() ?? observedTaskTailQuit ?? findWorkflowGracefulQuit(err) ?? findWorkflowGracefulQuit(ownController.signal.reason);
|
|
56093
56863
|
if (gracefulQuit !== undefined)
|
|
56094
56864
|
return suspendForGracefulQuit(gracefulQuit);
|
|
56865
|
+
if (isDbosDependencyError(err) && durableBackend.isCheckpointUnavailable?.(runId) && (durableBackend.getWorkflow(runId)?.completedCheckpoints ?? 0) > 0) {
|
|
56866
|
+
await admittedTools.closeAndDrain();
|
|
56867
|
+
activeStore.recordRunPaused(runId, undefined, { resumable: true });
|
|
56868
|
+
activeStore.recordRunExecutionState(runId, { phase: "blocked_dependency", dependencyError: err.message });
|
|
56869
|
+
return {
|
|
56870
|
+
runId,
|
|
56871
|
+
status: "paused",
|
|
56872
|
+
stages: [...runSnapshot.stages],
|
|
56873
|
+
toolNodes: [...runSnapshot.toolNodes ?? []]
|
|
56874
|
+
};
|
|
56875
|
+
}
|
|
56095
56876
|
await admittedTools.closeAndDrain();
|
|
56096
56877
|
if (ownController.signal.aborted)
|
|
56097
56878
|
await Promise.allSettled([...pendingChildWorkflows]);
|