@bastani/atomic 0.9.26-alpha.4 → 0.9.26-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/builtin/intercom/package.json +2 -2
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +3 -3
- package/dist/builtin/web-access/package.json +2 -2
- package/dist/builtin/workflows/CHANGELOG.md +17 -0
- package/dist/builtin/workflows/builtin/{chunk-73p2cm5s.js → chunk-9bhs02gm.js} +318 -52
- package/dist/builtin/workflows/builtin/{chunk-n95myjcg.js → chunk-gts8wgjr.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-ycvg8y8k.js → chunk-gw7rghda.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +2 -2
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1383 -266
- package/dist/builtin/workflows/src/index.js +1129 -234
- package/dist/core/agent-session-export.js +1 -1
- package/dist/core/agent-session-export.js.map +1 -1
- package/dist/core/export-html/tool-renderer.d.ts +3 -3
- package/dist/core/export-html/tool-renderer.d.ts.map +1 -1
- package/dist/core/export-html/tool-renderer.js +3 -3
- package/dist/core/export-html/tool-renderer.js.map +1 -1
- package/dist/core/extensions/api-types.d.ts +3 -1
- package/dist/core/extensions/api-types.d.ts.map +1 -1
- package/dist/core/extensions/api-types.js.map +1 -1
- package/dist/core/extensions/index.d.ts +1 -1
- package/dist/core/extensions/index.d.ts.map +1 -1
- package/dist/core/extensions/index.js.map +1 -1
- package/dist/core/extensions/loader-api.d.ts.map +1 -1
- package/dist/core/extensions/loader-api.js +5 -0
- package/dist/core/extensions/loader-api.js.map +1 -1
- package/dist/core/extensions/loader-bindings.d.ts +1 -1
- package/dist/core/extensions/loader-bindings.d.ts.map +1 -1
- package/dist/core/extensions/loader-bindings.js +1 -0
- package/dist/core/extensions/loader-bindings.js.map +1 -1
- package/dist/core/extensions/runner.d.ts +3 -1
- package/dist/core/extensions/runner.d.ts.map +1 -1
- package/dist/core/extensions/runner.js +6 -0
- package/dist/core/extensions/runner.js.map +1 -1
- package/dist/core/extensions/runtime-types.d.ts +2 -1
- package/dist/core/extensions/runtime-types.d.ts.map +1 -1
- package/dist/core/extensions/runtime-types.js.map +1 -1
- package/dist/core/extensions/tool-types.d.ts +3 -0
- package/dist/core/extensions/tool-types.d.ts.map +1 -1
- package/dist/core/extensions/tool-types.js.map +1 -1
- package/dist/core/extensions/ui-types.d.ts +2 -2
- package/dist/core/extensions/ui-types.d.ts.map +1 -1
- package/dist/core/extensions/ui-types.js.map +1 -1
- package/dist/core/extensions/workflow-run-control.d.ts +3 -0
- package/dist/core/extensions/workflow-run-control.d.ts.map +1 -1
- package/dist/core/extensions/workflow-run-control.js.map +1 -1
- package/dist/core/fast-model-routing.d.ts.map +1 -1
- package/dist/core/fast-model-routing.js +1 -5
- package/dist/core/fast-model-routing.js.map +1 -1
- package/dist/core/model-config.d.ts +36 -0
- package/dist/core/model-config.d.ts.map +1 -1
- package/dist/core/model-config.js +14 -2
- package/dist/core/model-config.js.map +1 -1
- package/dist/core/model-resolver-defaults.js +1 -1
- package/dist/core/model-resolver-defaults.js.map +1 -1
- package/dist/core/package-manager-auto-resources.d.ts.map +1 -1
- package/dist/core/package-manager-auto-resources.js +6 -2
- package/dist/core/package-manager-auto-resources.js.map +1 -1
- package/dist/core/package-manager-resolver.d.ts.map +1 -1
- package/dist/core/package-manager-resolver.js +4 -2
- package/dist/core/package-manager-resolver.js.map +1 -1
- package/dist/core/provider-composer-internal.d.ts +2 -1
- package/dist/core/provider-composer-internal.d.ts.map +1 -1
- package/dist/core/provider-composer-internal.js +13 -0
- package/dist/core/provider-composer-internal.js.map +1 -1
- package/dist/core/settings-manager-resource-accessors.js +7 -3
- package/dist/core/settings-manager-resource-accessors.js.map +1 -1
- package/dist/core/settings-merge.d.ts.map +1 -1
- package/dist/core/settings-merge.js +10 -0
- package/dist/core/settings-merge.js.map +1 -1
- package/dist/core/settings-types.d.ts +7 -1
- package/dist/core/settings-types.d.ts.map +1 -1
- package/dist/core/settings-types.js.map +1 -1
- package/dist/extensions/mcp/index.d.ts.map +1 -1
- package/dist/extensions/mcp/index.js +20 -1
- package/dist/extensions/mcp/index.js.map +1 -1
- package/dist/extensions/mcp/tools.d.ts +3 -1
- package/dist/extensions/mcp/tools.d.ts.map +1 -1
- package/dist/extensions/mcp/tools.js +19 -13
- package/dist/extensions/mcp/tools.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modes/interactive/components/chat-message-renderer.d.ts +2 -3
- package/dist/modes/interactive/components/chat-message-renderer.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-message-renderer.js.map +1 -1
- package/dist/modes/interactive/components/config-selector-list.d.ts.map +1 -1
- package/dist/modes/interactive/components/config-selector-list.js +3 -1
- package/dist/modes/interactive/components/config-selector-list.js.map +1 -1
- package/dist/modes/interactive/components/config-selector-project-scope.d.ts.map +1 -1
- package/dist/modes/interactive/components/config-selector-project-scope.js +2 -1
- package/dist/modes/interactive/components/config-selector-project-scope.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +2 -5
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +14 -40
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/interactive-mode-surface.d.ts +3 -3
- package/dist/modes/interactive/interactive-mode-surface.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode-surface.js.map +1 -1
- package/dist/modes/interactive/interactive-session-runtime.js +6 -1
- package/dist/modes/interactive/interactive-session-runtime.js.map +1 -1
- package/dist/modes/interactive/interactive-startup.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-startup.js +5 -0
- package/dist/modes/interactive/interactive-startup.js.map +1 -1
- package/dist/modes/interactive-engine/engine-custom-ui.d.ts.map +1 -1
- package/dist/modes/interactive-engine/engine-custom-ui.js +9 -0
- package/dist/modes/interactive-engine/engine-custom-ui.js.map +1 -1
- package/dist/modes/interactive-engine/engine-render-service.d.ts.map +1 -1
- package/dist/modes/interactive-engine/engine-render-service.js +10 -1
- package/dist/modes/interactive-engine/engine-render-service.js.map +1 -1
- package/dist/utils/image-convert.d.ts +5 -0
- package/dist/utils/image-convert.d.ts.map +1 -1
- package/dist/utils/image-convert.js +36 -6
- package/dist/utils/image-convert.js.map +1 -1
- package/docs/extensions.md +6 -0
- package/docs/models/reference.md +28 -1
- package/docs/models.md +4 -0
- package/docs/providers.md +10 -0
- package/docs/sdk.md +26 -2
- package/docs/settings.md +28 -1
- package/docs/workflows/operations.md +40 -4
- package/npm-shrinkwrap.json +71 -92
- package/package.json +8 -8
|
@@ -76,6 +76,92 @@ function raceAbort(operation, signal) {
|
|
|
76
76
|
});
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
// dist/builtin/workflows/src/durable/dbos-database-diagnostics.ts
|
|
80
|
+
function redactedDatabaseEndpoint(url) {
|
|
81
|
+
try {
|
|
82
|
+
const parsed = new URL(url);
|
|
83
|
+
parsed.username = "";
|
|
84
|
+
parsed.password = "";
|
|
85
|
+
parsed.search = "";
|
|
86
|
+
parsed.hash = "";
|
|
87
|
+
return parsed.toString();
|
|
88
|
+
} catch {
|
|
89
|
+
return "unparseable PostgreSQL endpoint";
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function redactedDatabaseMessage(message, url) {
|
|
93
|
+
let safe = message.replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/gi, (value2) => redactedDatabaseEndpoint(value2));
|
|
94
|
+
let configuredSecrets = [];
|
|
95
|
+
if (url !== undefined) {
|
|
96
|
+
try {
|
|
97
|
+
const parsed = new URL(url);
|
|
98
|
+
configuredSecrets = [parsed.username, parsed.password, ...parsed.searchParams.values()].flatMap((secret) => {
|
|
99
|
+
try {
|
|
100
|
+
return [secret, decodeURIComponent(secret)];
|
|
101
|
+
} catch {
|
|
102
|
+
return [secret];
|
|
103
|
+
}
|
|
104
|
+
}).filter((secret) => secret.length > 0);
|
|
105
|
+
} catch {
|
|
106
|
+
return "PostgreSQL diagnostic could not be safely redacted; check the configured endpoint.";
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
safe = safe.replace(/\b(password|passwd|token|secret|sslpassword|user(?:name)?)\s*(?:([=:])\s*|((?:is|was)\s+)|\s+)(?:"([^"]*)"|'([^']*)'|([^\s,;]+))/gi, (match, key, separator, filler, doubleQuoted, singleQuoted, bare) => {
|
|
110
|
+
const value2 = doubleQuoted ?? singleQuoted ?? bare ?? "";
|
|
111
|
+
const isUserContext = /^user(?:name)?$/i.test(key);
|
|
112
|
+
if (separator === undefined && !isUserContext && !configuredSecrets.includes(value2))
|
|
113
|
+
return match;
|
|
114
|
+
const rendered = doubleQuoted !== undefined ? '"[redacted]"' : singleQuoted !== undefined ? "'[redacted]'" : "[redacted]";
|
|
115
|
+
return `${key}${separator ?? " "}${filler ?? ""}${rendered}`;
|
|
116
|
+
});
|
|
117
|
+
for (const secret of new Set(configuredSecrets)) {
|
|
118
|
+
if (secret.length < 4)
|
|
119
|
+
continue;
|
|
120
|
+
const escaped = secret.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
121
|
+
const expression = new RegExp(`(?<![\\p{L}\\p{N}_\\[])${escaped}(?![\\p{L}\\p{N}_\\]])`, "gu");
|
|
122
|
+
safe = safe.replace(expression, "[redacted]");
|
|
123
|
+
}
|
|
124
|
+
return safe;
|
|
125
|
+
}
|
|
126
|
+
function databaseDependencyMessage(message, diagnostics) {
|
|
127
|
+
const safe = redactedDatabaseMessage(message, diagnostics?.url);
|
|
128
|
+
if (diagnostics === undefined)
|
|
129
|
+
return safe;
|
|
130
|
+
const failure = diagnostics.failure === undefined ? "" : ` Last probe/recovery failure: ${redactedDatabaseMessage(diagnostics.failure, diagnostics.url)}`;
|
|
131
|
+
return `${safe} Provider: ${diagnostics.provider}; endpoint: ${redactedDatabaseEndpoint(diagnostics.url)}.${failure}`;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// dist/builtin/workflows/src/durable/dbos-process-owner.ts
|
|
135
|
+
function emptyOwner2() {
|
|
136
|
+
return {
|
|
137
|
+
version: 1,
|
|
138
|
+
state: "uninitialized",
|
|
139
|
+
configured: undefined,
|
|
140
|
+
active: undefined,
|
|
141
|
+
launchPromise: undefined,
|
|
142
|
+
shutdownPromise: undefined,
|
|
143
|
+
failure: undefined,
|
|
144
|
+
wrappers: undefined,
|
|
145
|
+
leases: new Set
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
function ownerBag2() {
|
|
149
|
+
return globalThis;
|
|
150
|
+
}
|
|
151
|
+
function getDbosProcessOwner() {
|
|
152
|
+
const bag = ownerBag2();
|
|
153
|
+
const existing = bag[DBOS_PROCESS_OWNER_KEY];
|
|
154
|
+
if (existing !== undefined && existing.version === 1)
|
|
155
|
+
return existing;
|
|
156
|
+
const created = emptyOwner2();
|
|
157
|
+
bag[DBOS_PROCESS_OWNER_KEY] = created;
|
|
158
|
+
return created;
|
|
159
|
+
}
|
|
160
|
+
var DBOS_PROCESS_OWNER_KEY;
|
|
161
|
+
var init_dbos_process_owner = __esm(() => {
|
|
162
|
+
DBOS_PROCESS_OWNER_KEY = Symbol.for("atomic-workflows/dbos-process-owner@1");
|
|
163
|
+
});
|
|
164
|
+
|
|
79
165
|
// dist/builtin/workflows/src/durable/dbos-admission.ts
|
|
80
166
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
81
167
|
function isDbosDependencyError(error) {
|
|
@@ -94,6 +180,8 @@ async function boundedAdmission(operation, signal, timeoutMs = DBOS_ADMISSION_TI
|
|
|
94
180
|
return await raceAbort(operation(controller.signal), controller.signal);
|
|
95
181
|
} catch (error) {
|
|
96
182
|
controller.abort(error);
|
|
183
|
+
if (isDbosDependencyError(error) && error.admissionDetail !== undefined)
|
|
184
|
+
throw new DbosDependencyError(error.admissionDetail, null);
|
|
97
185
|
throw error;
|
|
98
186
|
} finally {
|
|
99
187
|
clearTimeout(timer);
|
|
@@ -102,10 +190,13 @@ async function boundedAdmission(operation, signal, timeoutMs = DBOS_ADMISSION_TI
|
|
|
102
190
|
}
|
|
103
191
|
var DBOS_ADMISSION_TIMEOUT_MS = 1e4, DbosDependencyError, key2, bag, dbosAdmissionContext;
|
|
104
192
|
var init_dbos_admission = __esm(() => {
|
|
193
|
+
init_dbos_process_owner();
|
|
105
194
|
DbosDependencyError = class DbosDependencyError extends Error {
|
|
106
195
|
code = "ATOMIC_DBOS_DEPENDENCY";
|
|
107
|
-
|
|
108
|
-
|
|
196
|
+
admissionDetail;
|
|
197
|
+
constructor(message = "Workflow database unavailable during admission. Restore PostgreSQL before retrying.", diagnostics = getDbosProcessOwner().databaseDiagnostics?.(), admissionDetail) {
|
|
198
|
+
super(databaseDependencyMessage(message, diagnostics ?? undefined));
|
|
199
|
+
this.admissionDetail = admissionDetail === undefined ? undefined : redactedDatabaseMessage(admissionDetail);
|
|
109
200
|
this.name = "DbosDependencyError";
|
|
110
201
|
}
|
|
111
202
|
};
|
|
@@ -115,6 +206,602 @@ var init_dbos_admission = __esm(() => {
|
|
|
115
206
|
dbosAdmissionContext = bag[key2];
|
|
116
207
|
});
|
|
117
208
|
|
|
209
|
+
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
210
|
+
function fenceDbosAdmissionPool(pool) {
|
|
211
|
+
const connect = pool.connect.bind(pool);
|
|
212
|
+
async function acquire(signal) {
|
|
213
|
+
if (signal.aborted)
|
|
214
|
+
throw sdkDependencyError();
|
|
215
|
+
const pending = connect().then((client) => {
|
|
216
|
+
if (signal.aborted) {
|
|
217
|
+
client.release(true);
|
|
218
|
+
throw sdkDependencyError();
|
|
219
|
+
}
|
|
220
|
+
return fencedClient(client, signal);
|
|
221
|
+
});
|
|
222
|
+
try {
|
|
223
|
+
return await raceAbort(pending, signal);
|
|
224
|
+
} catch (error) {
|
|
225
|
+
throw databaseError(error, signal);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
function scopedConnect(callback) {
|
|
229
|
+
const signal = dbosAdmissionContext.getStore();
|
|
230
|
+
if (signal === undefined) {
|
|
231
|
+
if (callback === undefined)
|
|
232
|
+
return connect();
|
|
233
|
+
connect(callback);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
const pending = acquire(signal);
|
|
237
|
+
if (callback === undefined)
|
|
238
|
+
return pending;
|
|
239
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
240
|
+
}
|
|
241
|
+
pool.connect = scopedConnect;
|
|
242
|
+
return pool;
|
|
243
|
+
}
|
|
244
|
+
function fencedClient(client, signal) {
|
|
245
|
+
let released = false;
|
|
246
|
+
const release = (error) => {
|
|
247
|
+
if (released)
|
|
248
|
+
return;
|
|
249
|
+
released = true;
|
|
250
|
+
signal.removeEventListener("abort", abort);
|
|
251
|
+
client.release(error);
|
|
252
|
+
};
|
|
253
|
+
const abort = () => release(true);
|
|
254
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
255
|
+
const query = new Proxy(client.query, {
|
|
256
|
+
apply(target, _receiver, args) {
|
|
257
|
+
const values = [...args];
|
|
258
|
+
const last = values.at(-1);
|
|
259
|
+
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
260
|
+
const pending = (async () => {
|
|
261
|
+
if (signal.aborted || released)
|
|
262
|
+
throw sdkDependencyError();
|
|
263
|
+
try {
|
|
264
|
+
return await raceAbort(new Promise((resolve, reject) => {
|
|
265
|
+
Reflect.apply(target, client, [
|
|
266
|
+
...values,
|
|
267
|
+
(error, result) => {
|
|
268
|
+
if (error)
|
|
269
|
+
reject(error);
|
|
270
|
+
else
|
|
271
|
+
resolve(result);
|
|
272
|
+
}
|
|
273
|
+
]);
|
|
274
|
+
}), signal);
|
|
275
|
+
} catch (error) {
|
|
276
|
+
throw databaseError(error, signal);
|
|
277
|
+
}
|
|
278
|
+
})();
|
|
279
|
+
if (callback === undefined)
|
|
280
|
+
return pending;
|
|
281
|
+
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
282
|
+
}
|
|
283
|
+
});
|
|
284
|
+
return new Proxy(client, {
|
|
285
|
+
get(target, property, receiver) {
|
|
286
|
+
if (property === "query")
|
|
287
|
+
return query;
|
|
288
|
+
if (property === "release")
|
|
289
|
+
return release;
|
|
290
|
+
return Reflect.get(target, property, receiver);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
function sdkDependencyError(message) {
|
|
295
|
+
const reported = message === undefined ? new DbosDependencyError : new DbosDependencyError(message, null);
|
|
296
|
+
return new DbosDependencyError(undefined, null, reported.message);
|
|
297
|
+
}
|
|
298
|
+
function databaseError(error, signal) {
|
|
299
|
+
if (signal?.aborted)
|
|
300
|
+
return sdkDependencyError();
|
|
301
|
+
if (isDbosDependencyError(error))
|
|
302
|
+
return sdkDependencyError(error.admissionDetail ?? error.message);
|
|
303
|
+
const pending = [error];
|
|
304
|
+
const seen = new Set;
|
|
305
|
+
while (pending.length > 0) {
|
|
306
|
+
const value2 = pending.pop();
|
|
307
|
+
if (typeof value2 !== "object" || value2 === null || seen.has(value2))
|
|
308
|
+
continue;
|
|
309
|
+
seen.add(value2);
|
|
310
|
+
if ("cause" in value2)
|
|
311
|
+
pending.push(value2.cause);
|
|
312
|
+
if ("error" in value2)
|
|
313
|
+
pending.push(value2.error);
|
|
314
|
+
if ("errors" in value2 && Array.isArray(value2.errors))
|
|
315
|
+
pending.push(...value2.errors);
|
|
316
|
+
const code = "code" in value2 && typeof value2.code === "string" ? value2.code : undefined;
|
|
317
|
+
const message = value2 instanceof Error ? value2.message : "";
|
|
318
|
+
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
319
|
+
return sdkDependencyError();
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
return error;
|
|
323
|
+
}
|
|
324
|
+
var connectionErrnos, connectionMessages;
|
|
325
|
+
var init_dbos_admission_pool = __esm(() => {
|
|
326
|
+
init_dbos_admission();
|
|
327
|
+
connectionErrnos = new Set([
|
|
328
|
+
"ECONNRESET",
|
|
329
|
+
"ECONNREFUSED",
|
|
330
|
+
"ECONNABORTED",
|
|
331
|
+
"EHOSTUNREACH",
|
|
332
|
+
"ENETUNREACH",
|
|
333
|
+
"ETIMEDOUT",
|
|
334
|
+
"ENOTFOUND",
|
|
335
|
+
"EAI_AGAIN",
|
|
336
|
+
"EPIPE"
|
|
337
|
+
]);
|
|
338
|
+
connectionMessages = new Set([
|
|
339
|
+
"Connection terminated unexpectedly",
|
|
340
|
+
"Connection terminated due to connection timeout",
|
|
341
|
+
"Client has encountered a connection error and is not queryable",
|
|
342
|
+
"timeout exceeded when trying to connect",
|
|
343
|
+
"Client was closed and is not queryable",
|
|
344
|
+
"Postgres connection pool changed while acquiring a connection"
|
|
345
|
+
]);
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
// dist/builtin/workflows/src/durable/dbos-row-guard.ts
|
|
349
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
350
|
+
async function installDbosRowGuard(client) {
|
|
351
|
+
const ready = await client.query("SELECT to_regclass('dbos.workflow_status') IS NOT NULL AS status, to_regclass('dbos.operation_outputs') IS NOT NULL AS operations");
|
|
352
|
+
if (!ready.rows[0]?.status && !ready.rows[0]?.operations)
|
|
353
|
+
return;
|
|
354
|
+
if (!ready.rows[0]?.status || !ready.rows[0]?.operations)
|
|
355
|
+
throw new Error("Atomic workflow database schema is incomplete");
|
|
356
|
+
if (await validateInstalledGuard(client))
|
|
357
|
+
return;
|
|
358
|
+
await client.query("BEGIN");
|
|
359
|
+
try {
|
|
360
|
+
await client.query("SELECT pg_advisory_xact_lock(3419, 9003)");
|
|
361
|
+
if (await validateInstalledGuard(client)) {
|
|
362
|
+
await client.query("COMMIT");
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
await client.query(guardSql);
|
|
366
|
+
await client.query("CREATE TABLE IF NOT EXISTS dbos.atomic_guard_protocol (singleton boolean PRIMARY KEY CHECK(singleton), fingerprint text NOT NULL)");
|
|
367
|
+
await client.query("INSERT INTO dbos.atomic_guard_protocol VALUES (true, $1) ON CONFLICT DO NOTHING", [
|
|
368
|
+
fingerprint
|
|
369
|
+
]);
|
|
370
|
+
await client.query("COMMIT");
|
|
371
|
+
} catch (error) {
|
|
372
|
+
await client.query("ROLLBACK");
|
|
373
|
+
throw error;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
async function validateInstalledGuard(client) {
|
|
377
|
+
const installed = await client.query("SELECT to_regclass('dbos.atomic_guard_protocol') IS NOT NULL AS present");
|
|
378
|
+
if (!installed.rows[0]?.present)
|
|
379
|
+
return false;
|
|
380
|
+
const validated = await client.query("SELECT fingerprint, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid IN ('dbos.workflow_status'::regclass,'dbos.operation_outputs'::regclass) AND tgname = 'atomic_generation_guard' AND tgenabled = 'O' AND tgtype = 31 AND tgfoid = 'dbos.atomic_guard_row()'::regprocedure) AS triggers, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection' AND tgenabled = 'O' AND tgtype = 21 AND tgfoid = 'dbos.atomic_project_metadata()'::regprocedure) AS projections FROM dbos.atomic_guard_protocol WHERE singleton");
|
|
381
|
+
if (validated.rows.length !== 1 || validated.rows[0].fingerprint !== fingerprint || validated.rows[0].triggers !== 2 || validated.rows[0].projections !== 1)
|
|
382
|
+
throw new Error("Atomic database ownership guard schema is incompatible or incomplete");
|
|
383
|
+
return true;
|
|
384
|
+
}
|
|
385
|
+
var guardSql = `
|
|
386
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_owner_generation (
|
|
387
|
+
root text PRIMARY KEY, owner text, generation bigint,
|
|
388
|
+
terminal boolean NOT NULL DEFAULT false, claim text, claimant text, protocol integer NOT NULL DEFAULT 1 CHECK(protocol = 1)
|
|
389
|
+
);
|
|
390
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_guard_rows (workflow_uuid text PRIMARY KEY, root text NOT NULL);
|
|
391
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_metadata(value text) RETURNS jsonb
|
|
392
|
+
LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog AS $$
|
|
393
|
+
DECLARE decoded jsonb;
|
|
394
|
+
BEGIN
|
|
395
|
+
IF value IS NULL THEN RETURN NULL; END IF;
|
|
396
|
+
BEGIN decoded := value::jsonb; EXCEPTION WHEN invalid_text_representation THEN RETURN NULL; END;
|
|
397
|
+
IF decoded->>'__dbos_serializer' = 'superjson' THEN decoded := decoded->'json'; END IF;
|
|
398
|
+
IF decoded->>'__atomicDurableMetadata' IS DISTINCT FROM 'true' OR decoded->>'version' IS DISTINCT FROM '3' THEN RETURN NULL; END IF;
|
|
399
|
+
RETURN decoded->'metadata';
|
|
400
|
+
END $$;
|
|
401
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_guard_row() RETURNS trigger
|
|
402
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
403
|
+
DECLARE
|
|
404
|
+
row_value jsonb; id text; root_id text; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE;
|
|
405
|
+
meta jsonb; prior jsonb; prior_generation bigint; next_generation bigint; absorbing boolean;
|
|
406
|
+
allowed boolean; cap_owner text; cap_generation bigint; claim_token text;
|
|
407
|
+
BEGIN
|
|
408
|
+
row_value := CASE WHEN TG_OP = 'DELETE' THEN to_jsonb(OLD) ELSE to_jsonb(NEW) END;
|
|
409
|
+
id := row_value->>'workflow_uuid';
|
|
410
|
+
root_id := split_part(id, ':checkpoint:', 1);
|
|
411
|
+
IF TG_OP = 'UPDATE' AND to_jsonb(OLD)->>'workflow_uuid' IS DISTINCT FROM id AND (
|
|
412
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid') OR
|
|
413
|
+
(TG_TABLE_NAME = 'workflow_status' AND to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')) OR
|
|
414
|
+
(TG_TABLE_NAME = 'operation_outputs' AND EXISTS(SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid' AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')))
|
|
415
|
+
) THEN
|
|
416
|
+
IF NULLIF(current_setting('atomic.row_authority',true),'') IS NULL OR current_setting('atomic.row_authority',true) = 'null' THEN RETURN NULL; END IF;
|
|
417
|
+
RAISE EXCEPTION 'Atomic workflow identities are immutable';
|
|
418
|
+
END IF;
|
|
419
|
+
IF TG_TABLE_NAME = 'workflow_status' AND NOT (
|
|
420
|
+
COALESCE(row_value->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
421
|
+
COALESCE(to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
422
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id)
|
|
423
|
+
) THEN
|
|
424
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
425
|
+
END IF;
|
|
426
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
427
|
+
IF TG_TABLE_NAME = 'operation_outputs' AND NOT EXISTS (
|
|
428
|
+
SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = id AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')
|
|
429
|
+
) AND NOT EXISTS (SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id) THEN
|
|
430
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
431
|
+
END IF;
|
|
432
|
+
INSERT INTO dbos.atomic_guard_rows VALUES(id, root_id) ON CONFLICT DO NOTHING;
|
|
433
|
+
INSERT INTO dbos.atomic_guard_rows SELECT workflow_uuid, root_id FROM dbos.workflow_status
|
|
434
|
+
WHERE workflow_uuid = root_id AND name = 'atomicWorkflowHandle' ON CONFLICT DO NOTHING;
|
|
435
|
+
INSERT INTO dbos.atomic_owner_generation(root) VALUES(root_id)
|
|
436
|
+
ON CONFLICT(root) DO UPDATE SET root = EXCLUDED.root RETURNING * INTO state;
|
|
437
|
+
IF state.owner IS NULL THEN
|
|
438
|
+
IF TG_TABLE_NAME <> 'workflow_status' OR TG_OP = 'DELETE' OR row_value->>'status' <> 'SUCCESS'
|
|
439
|
+
OR id NOT LIKE root_id || ':checkpoint:__atomic_metadata:%'
|
|
440
|
+
OR authority->>'root' IS DISTINCT FROM root_id OR authority->>'enroll' IS DISTINCT FROM 'true' THEN
|
|
441
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
442
|
+
END IF;
|
|
443
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
444
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id THEN
|
|
445
|
+
RAISE EXCEPTION 'Atomic metadata encoding is unavailable for enrollment';
|
|
446
|
+
END IF;
|
|
447
|
+
SELECT candidate.metadata, candidate.generation INTO prior, prior_generation FROM (
|
|
448
|
+
SELECT dbos.atomic_metadata(output) AS metadata,
|
|
449
|
+
split_part(substring(workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint AS generation
|
|
450
|
+
FROM dbos.workflow_status
|
|
451
|
+
WHERE workflow_uuid LIKE root_id || ':checkpoint:__atomic_metadata:%' AND status = 'SUCCESS'
|
|
452
|
+
) candidate WHERE candidate.metadata IS NOT NULL
|
|
453
|
+
ORDER BY COALESCE(candidate.metadata->>'status' IN ('completed', 'cancelled') OR
|
|
454
|
+
(candidate.metadata->>'status' IN ('failed','blocked') AND candidate.metadata->>'resumable' = 'false'), false) DESC,
|
|
455
|
+
candidate.generation DESC LIMIT 1;
|
|
456
|
+
IF prior_generation IS DISTINCT FROM (authority->>'generation')::bigint
|
|
457
|
+
OR prior->>'ownerExecutorId' IS DISTINCT FROM authority->>'owner' THEN
|
|
458
|
+
RAISE EXCEPTION 'Atomic ownership generation changed before enrollment';
|
|
459
|
+
END IF;
|
|
460
|
+
ELSE
|
|
461
|
+
IF authority->>'root' IS DISTINCT FROM root_id THEN RETURN NULL; END IF;
|
|
462
|
+
cap_owner := authority->>'owner'; cap_generation := (authority->>'generation')::bigint;
|
|
463
|
+
allowed := cap_owner = state.owner AND cap_generation = state.generation AND (
|
|
464
|
+
authority->>'actor' = state.owner OR
|
|
465
|
+
(authority->>'claim' = state.claim AND authority->>'actor' = state.claimant) OR
|
|
466
|
+
(authority->>'claim' IS NOT NULL AND authority->>'checkpoint' = id));
|
|
467
|
+
IF NOT COALESCE(allowed, false) THEN RAISE EXCEPTION 'Atomic ownership generation changed; stale database writes are refused'; END IF;
|
|
468
|
+
END IF;
|
|
469
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP = 'UPDATE' THEN
|
|
470
|
+
IF OLD.status = 'SUCCESS' AND OLD.name = 'atomicWorkflowCheckpoint' THEN
|
|
471
|
+
IF NEW.output IS DISTINCT FROM OLD.output OR NEW.status <> 'SUCCESS' THEN RAISE EXCEPTION 'Completed Atomic checkpoints are immutable'; END IF;
|
|
472
|
+
RETURN NEW;
|
|
473
|
+
END IF;
|
|
474
|
+
END IF;
|
|
475
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP <> 'DELETE' AND row_value->>'status' = 'SUCCESS'
|
|
476
|
+
AND id LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN
|
|
477
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
478
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN RAISE EXCEPTION 'Atomic metadata encoding or owner is unavailable'; END IF;
|
|
479
|
+
next_generation := split_part(substring(id FROM length(root_id) + 13), ':', 2)::bigint;
|
|
480
|
+
absorbing := COALESCE(meta->>'status' IN ('completed', 'cancelled') OR
|
|
481
|
+
(meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
482
|
+
claim_token := meta->>'transitionClaimId';
|
|
483
|
+
IF state.root IS NOT NULL AND (next_generation < state.generation OR (state.terminal AND NOT absorbing)) THEN
|
|
484
|
+
RAISE EXCEPTION 'Atomic terminal or newer generation cannot be overwritten';
|
|
485
|
+
END IF;
|
|
486
|
+
IF state.owner IS NOT NULL AND authority->>'actor' <> state.owner THEN
|
|
487
|
+
IF id NOT LIKE '%:claim' OR next_generation <> state.generation + 1 OR claim_token IS DISTINCT FROM authority->>'claim' OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN
|
|
488
|
+
RAISE EXCEPTION 'Atomic ownership transfer requires an exact generation claim';
|
|
489
|
+
END IF;
|
|
490
|
+
END IF;
|
|
491
|
+
END IF;
|
|
492
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
493
|
+
END $$;
|
|
494
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_project_metadata() RETURNS trigger
|
|
495
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
496
|
+
DECLARE root_id text; meta jsonb; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE; projected_generation bigint; absorbing boolean;
|
|
497
|
+
BEGIN
|
|
498
|
+
IF NEW.name <> 'atomicWorkflowCheckpoint' OR NEW.status <> 'SUCCESS' THEN RETURN NEW; END IF;
|
|
499
|
+
root_id := split_part(NEW.workflow_uuid, ':checkpoint:', 1);
|
|
500
|
+
IF NEW.workflow_uuid NOT LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN RETURN NEW; END IF;
|
|
501
|
+
IF TG_OP = 'UPDATE' THEN IF OLD.status = 'SUCCESS' THEN RETURN NEW; END IF; END IF;
|
|
502
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
503
|
+
SELECT * INTO state FROM dbos.atomic_owner_generation WHERE root = root_id FOR UPDATE;
|
|
504
|
+
IF state.owner IS NULL AND authority->>'enroll' IS DISTINCT FROM 'true' THEN RETURN NEW; END IF;
|
|
505
|
+
meta := dbos.atomic_metadata(NEW.output);
|
|
506
|
+
projected_generation := split_part(substring(NEW.workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint;
|
|
507
|
+
absorbing := COALESCE(meta->>'status' IN ('completed','cancelled') OR (meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
508
|
+
UPDATE dbos.atomic_owner_generation SET owner = meta->>'ownerExecutorId', generation = projected_generation,
|
|
509
|
+
terminal = absorbing, claim = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN meta->>'transitionClaimId' END,
|
|
510
|
+
claimant = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN authority->>'actor' END WHERE root = root_id;
|
|
511
|
+
RETURN NEW;
|
|
512
|
+
END $$;
|
|
513
|
+
DO $$ BEGIN
|
|
514
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
515
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.workflow_status
|
|
516
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
517
|
+
END IF;
|
|
518
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.operation_outputs'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
519
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.operation_outputs
|
|
520
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
521
|
+
END IF;
|
|
522
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection') THEN
|
|
523
|
+
CREATE TRIGGER atomic_generation_projection AFTER INSERT OR UPDATE ON dbos.workflow_status
|
|
524
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_project_metadata();
|
|
525
|
+
END IF;
|
|
526
|
+
END $$;
|
|
527
|
+
`, fingerprint;
|
|
528
|
+
var init_dbos_row_guard = __esm(() => {
|
|
529
|
+
fingerprint = createHash2("sha256").update(guardSql).digest("hex");
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
// dist/builtin/workflows/src/durable/dbos-owner-fence.ts
|
|
533
|
+
import { AsyncLocalStorage as AsyncLocalStorage2 } from "node:async_hooks";
|
|
534
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
535
|
+
function lockKey(identity) {
|
|
536
|
+
const digest = createHash3("sha256").update(`atomic-workflow-fence:${identity}`).digest();
|
|
537
|
+
return [digest.readInt32BE(0), digest.readInt32BE(4)];
|
|
538
|
+
}
|
|
539
|
+
function isDatabaseExecutor(executorId) {
|
|
540
|
+
return /^atomic-db-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(executorId ?? "");
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
class DbosOwnerFence {
|
|
544
|
+
pool;
|
|
545
|
+
executorId;
|
|
546
|
+
endPool;
|
|
547
|
+
lifetime;
|
|
548
|
+
lost = false;
|
|
549
|
+
closing;
|
|
550
|
+
closed = false;
|
|
551
|
+
get invalidated() {
|
|
552
|
+
return this.lost;
|
|
553
|
+
}
|
|
554
|
+
constructor(pool, executorId, endPool) {
|
|
555
|
+
this.pool = pool;
|
|
556
|
+
this.executorId = executorId;
|
|
557
|
+
this.endPool = endPool;
|
|
558
|
+
}
|
|
559
|
+
fail() {
|
|
560
|
+
this.lost = true;
|
|
561
|
+
return new DbosDependencyError("Workflow database unavailable: ownership connection was lost; this execution generation cannot write again");
|
|
562
|
+
}
|
|
563
|
+
owner() {
|
|
564
|
+
if (this.lost)
|
|
565
|
+
return Promise.reject(this.fail());
|
|
566
|
+
this.lifetime ??= (async () => {
|
|
567
|
+
const client = await this.pool().connect();
|
|
568
|
+
client.on("error", () => {
|
|
569
|
+
this.fail();
|
|
570
|
+
});
|
|
571
|
+
client.on("end", () => {
|
|
572
|
+
this.fail();
|
|
573
|
+
});
|
|
574
|
+
try {
|
|
575
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-owner:${this.executorId}`]);
|
|
576
|
+
const held = await client.query("SELECT pg_try_advisory_lock($1, $2) AS held", [
|
|
577
|
+
...lockKey(this.executorId)
|
|
578
|
+
]);
|
|
579
|
+
if (!held.rows[0]?.held || this.lost)
|
|
580
|
+
throw this.fail();
|
|
581
|
+
await client.query("SELECT pg_advisory_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
582
|
+
await client.query("SELECT pg_advisory_unlock($1, $2)", [...lockKey(this.executorId)]);
|
|
583
|
+
return client;
|
|
584
|
+
} catch (error) {
|
|
585
|
+
this.fail();
|
|
586
|
+
client.release(true);
|
|
587
|
+
throw error;
|
|
588
|
+
}
|
|
589
|
+
})().catch((error) => {
|
|
590
|
+
this.fail();
|
|
591
|
+
throw error;
|
|
592
|
+
});
|
|
593
|
+
return this.lifetime;
|
|
594
|
+
}
|
|
595
|
+
async liveness(executorId) {
|
|
596
|
+
if (!isDatabaseExecutor(executorId))
|
|
597
|
+
return "unknown";
|
|
598
|
+
const client = await this.pool().connect();
|
|
599
|
+
try {
|
|
600
|
+
await client.query("BEGIN");
|
|
601
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
602
|
+
...lockKey(executorId)
|
|
603
|
+
]);
|
|
604
|
+
return result.rows[0]?.held ? "dead" : "alive";
|
|
605
|
+
} finally {
|
|
606
|
+
try {
|
|
607
|
+
await client.query("ROLLBACK");
|
|
608
|
+
} finally {
|
|
609
|
+
client.release();
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
async recover(executorId, callback) {
|
|
614
|
+
if (!isDatabaseExecutor(executorId))
|
|
615
|
+
return;
|
|
616
|
+
const client = await this.pool().connect();
|
|
617
|
+
const fail = () => {
|
|
618
|
+
this.fail();
|
|
619
|
+
};
|
|
620
|
+
client.on("error", fail);
|
|
621
|
+
client.on("end", fail);
|
|
622
|
+
try {
|
|
623
|
+
await client.query("BEGIN");
|
|
624
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
625
|
+
...lockKey(executorId)
|
|
626
|
+
]);
|
|
627
|
+
if (!result.rows[0]?.held)
|
|
628
|
+
return;
|
|
629
|
+
return await callback();
|
|
630
|
+
} finally {
|
|
631
|
+
try {
|
|
632
|
+
await client.query("ROLLBACK");
|
|
633
|
+
} finally {
|
|
634
|
+
client.removeListener("error", fail);
|
|
635
|
+
client.removeListener("end", fail);
|
|
636
|
+
client.release();
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
async write(workflowId, callback, claim = false) {
|
|
641
|
+
try {
|
|
642
|
+
return await this.writeLocked(workflowId, callback, claim);
|
|
643
|
+
} catch (error) {
|
|
644
|
+
const mapped = databaseError(error);
|
|
645
|
+
throw this.lost && isDbosDependencyError(mapped) ? this.fail() : mapped;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
async writeLocked(workflowId, callback, claim) {
|
|
649
|
+
const context = writeContext.getStore();
|
|
650
|
+
const active = context?.fence === this ? context.workflows : undefined;
|
|
651
|
+
if (active?.has(workflowId)) {
|
|
652
|
+
if (this.lost)
|
|
653
|
+
throw this.fail();
|
|
654
|
+
return await callback();
|
|
655
|
+
}
|
|
656
|
+
const owner = await this.owner();
|
|
657
|
+
const client = await this.pool().connect();
|
|
658
|
+
const fail = () => {
|
|
659
|
+
this.fail();
|
|
660
|
+
};
|
|
661
|
+
client.on("error", fail);
|
|
662
|
+
client.on("end", fail);
|
|
663
|
+
try {
|
|
664
|
+
await client.query("BEGIN");
|
|
665
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-guard:${workflowId}`]);
|
|
666
|
+
await client.query("SET LOCAL statement_timeout = '10000ms'");
|
|
667
|
+
await client.query("SELECT pg_advisory_xact_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
668
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`workflow:${workflowId}`)]);
|
|
669
|
+
if (claim)
|
|
670
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`sql:${workflowId}`)]);
|
|
671
|
+
if (this.lost)
|
|
672
|
+
throw this.fail();
|
|
673
|
+
try {
|
|
674
|
+
await owner.query("SELECT 1");
|
|
675
|
+
} catch {
|
|
676
|
+
throw this.fail();
|
|
677
|
+
}
|
|
678
|
+
if (this.lost)
|
|
679
|
+
throw this.fail();
|
|
680
|
+
return await writeContext.run({ fence: this, workflows: new Map([...active ?? [], [workflowId, claim]]) }, callback);
|
|
681
|
+
} finally {
|
|
682
|
+
try {
|
|
683
|
+
await client.query("ROLLBACK");
|
|
684
|
+
} finally {
|
|
685
|
+
client.removeListener("error", fail);
|
|
686
|
+
client.removeListener("end", fail);
|
|
687
|
+
client.release();
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
async withRowAuthority(authority, callback) {
|
|
692
|
+
return await rowContext.run(authority, callback);
|
|
693
|
+
}
|
|
694
|
+
protectPool(pool) {
|
|
695
|
+
const connect = pool.connect.bind(pool);
|
|
696
|
+
const acquire = async () => {
|
|
697
|
+
const active = writeContext.getStore();
|
|
698
|
+
const client = await connect();
|
|
699
|
+
try {
|
|
700
|
+
await installDbosRowGuard(client);
|
|
701
|
+
await client.query("SELECT set_config('atomic.row_authority', $1, false)", [
|
|
702
|
+
JSON.stringify(rowContext.getStore() ?? null)
|
|
703
|
+
]);
|
|
704
|
+
} catch (error) {
|
|
705
|
+
client.release(true);
|
|
706
|
+
throw error;
|
|
707
|
+
}
|
|
708
|
+
if (active === undefined)
|
|
709
|
+
return client;
|
|
710
|
+
const fence = active.fence;
|
|
711
|
+
try {
|
|
712
|
+
if (fence.lost)
|
|
713
|
+
throw fence.fail();
|
|
714
|
+
await client.query("SELECT set_config('application_name', $1, false)", [
|
|
715
|
+
`atomic-sql:${[...active.workflows.keys()][0]}`
|
|
716
|
+
]);
|
|
717
|
+
for (const identity of [
|
|
718
|
+
fence.executorId,
|
|
719
|
+
...[...active.workflows].filter(([, claim]) => !claim).map(([id]) => `sql:${id}`)
|
|
720
|
+
]) {
|
|
721
|
+
const held = await client.query("SELECT pg_try_advisory_lock_shared($1, $2) AS held", [...lockKey(identity)]);
|
|
722
|
+
if (!held.rows[0]?.held)
|
|
723
|
+
throw fence.fail();
|
|
724
|
+
}
|
|
725
|
+
try {
|
|
726
|
+
await (await fence.owner()).query("SELECT 1");
|
|
727
|
+
} catch {
|
|
728
|
+
throw fence.fail();
|
|
729
|
+
}
|
|
730
|
+
if (fence.lost)
|
|
731
|
+
throw fence.fail();
|
|
732
|
+
} catch (error) {
|
|
733
|
+
client.release(true);
|
|
734
|
+
throw error;
|
|
735
|
+
}
|
|
736
|
+
let released = false;
|
|
737
|
+
const release = () => {
|
|
738
|
+
if (released)
|
|
739
|
+
return;
|
|
740
|
+
released = true;
|
|
741
|
+
client.release(true);
|
|
742
|
+
};
|
|
743
|
+
const query = new Proxy(client.query, {
|
|
744
|
+
apply: (target, _receiver, args) => {
|
|
745
|
+
if (!fence.lost && !released)
|
|
746
|
+
return Reflect.apply(target, client, args);
|
|
747
|
+
const error = fence.fail();
|
|
748
|
+
const callback = args.at(-1);
|
|
749
|
+
if (typeof callback === "function") {
|
|
750
|
+
queueMicrotask(() => callback(error));
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
return Promise.reject(error);
|
|
754
|
+
}
|
|
755
|
+
});
|
|
756
|
+
return new Proxy(client, {
|
|
757
|
+
get(target, property, receiver) {
|
|
758
|
+
if (property === "release")
|
|
759
|
+
return release;
|
|
760
|
+
if (property === "query")
|
|
761
|
+
return query;
|
|
762
|
+
return Reflect.get(target, property, receiver);
|
|
763
|
+
}
|
|
764
|
+
});
|
|
765
|
+
};
|
|
766
|
+
function scopedConnect(callback) {
|
|
767
|
+
const pending = acquire();
|
|
768
|
+
if (callback === undefined)
|
|
769
|
+
return pending;
|
|
770
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
771
|
+
}
|
|
772
|
+
pool.connect = scopedConnect;
|
|
773
|
+
return pool;
|
|
774
|
+
}
|
|
775
|
+
close() {
|
|
776
|
+
this.closed = true;
|
|
777
|
+
this.lost = true;
|
|
778
|
+
this.closing ??= (async () => {
|
|
779
|
+
if (this.lifetime !== undefined) {
|
|
780
|
+
const client = await this.lifetime.catch(() => {
|
|
781
|
+
return;
|
|
782
|
+
});
|
|
783
|
+
client?.release(true);
|
|
784
|
+
}
|
|
785
|
+
await this.endPool();
|
|
786
|
+
})();
|
|
787
|
+
return this.closing;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
var contextKey, contextBag, writeContext, rowContextKey, rowContextBag, rowContext;
|
|
791
|
+
var init_dbos_owner_fence = __esm(() => {
|
|
792
|
+
init_dbos_admission();
|
|
793
|
+
init_dbos_admission_pool();
|
|
794
|
+
init_dbos_row_guard();
|
|
795
|
+
contextKey = Symbol.for("atomic-workflows/dbos-owner-write-context@1");
|
|
796
|
+
contextBag = globalThis;
|
|
797
|
+
contextBag[contextKey] ??= new AsyncLocalStorage2;
|
|
798
|
+
writeContext = contextBag[contextKey];
|
|
799
|
+
rowContextKey = Symbol.for("atomic-workflows/dbos-row-authority@1");
|
|
800
|
+
rowContextBag = globalThis;
|
|
801
|
+
rowContextBag[rowContextKey] ??= new AsyncLocalStorage2;
|
|
802
|
+
rowContext = rowContextBag[rowContextKey];
|
|
803
|
+
});
|
|
804
|
+
|
|
118
805
|
// ../../node_modules/@dbos-inc/dbos-sdk/dist/src/error.js
|
|
119
806
|
var require_error = __commonJS(function(exports) {
|
|
120
807
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -30271,137 +30958,6 @@ SELECT
|
|
|
30271
30958
|
} });
|
|
30272
30959
|
});
|
|
30273
30960
|
|
|
30274
|
-
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
30275
|
-
function fenceDbosAdmissionPool(pool) {
|
|
30276
|
-
const connect = pool.connect.bind(pool);
|
|
30277
|
-
async function acquire(signal) {
|
|
30278
|
-
if (signal.aborted)
|
|
30279
|
-
throw new DbosDependencyError;
|
|
30280
|
-
const pending = connect().then((client) => {
|
|
30281
|
-
if (signal.aborted) {
|
|
30282
|
-
client.release(true);
|
|
30283
|
-
throw new DbosDependencyError;
|
|
30284
|
-
}
|
|
30285
|
-
return fencedClient(client, signal);
|
|
30286
|
-
});
|
|
30287
|
-
try {
|
|
30288
|
-
return await raceAbort(pending, signal);
|
|
30289
|
-
} catch (error) {
|
|
30290
|
-
throw databaseError(error, signal);
|
|
30291
|
-
}
|
|
30292
|
-
}
|
|
30293
|
-
function scopedConnect(callback) {
|
|
30294
|
-
const signal = dbosAdmissionContext.getStore();
|
|
30295
|
-
if (signal === undefined) {
|
|
30296
|
-
if (callback === undefined)
|
|
30297
|
-
return connect();
|
|
30298
|
-
connect(callback);
|
|
30299
|
-
return;
|
|
30300
|
-
}
|
|
30301
|
-
const pending = acquire(signal);
|
|
30302
|
-
if (callback === undefined)
|
|
30303
|
-
return pending;
|
|
30304
|
-
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
30305
|
-
}
|
|
30306
|
-
pool.connect = scopedConnect;
|
|
30307
|
-
return pool;
|
|
30308
|
-
}
|
|
30309
|
-
function fencedClient(client, signal) {
|
|
30310
|
-
let released = false;
|
|
30311
|
-
const release = (error) => {
|
|
30312
|
-
if (released)
|
|
30313
|
-
return;
|
|
30314
|
-
released = true;
|
|
30315
|
-
signal.removeEventListener("abort", abort);
|
|
30316
|
-
client.release(error);
|
|
30317
|
-
};
|
|
30318
|
-
const abort = () => release(true);
|
|
30319
|
-
signal.addEventListener("abort", abort, { once: true });
|
|
30320
|
-
const query = new Proxy(client.query, {
|
|
30321
|
-
apply(target, _receiver, args) {
|
|
30322
|
-
const values = [...args];
|
|
30323
|
-
const last = values.at(-1);
|
|
30324
|
-
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
30325
|
-
const pending = (async () => {
|
|
30326
|
-
if (signal.aborted || released)
|
|
30327
|
-
throw new DbosDependencyError;
|
|
30328
|
-
try {
|
|
30329
|
-
return await raceAbort(new Promise((resolve, reject) => {
|
|
30330
|
-
Reflect.apply(target, client, [
|
|
30331
|
-
...values,
|
|
30332
|
-
(error, result) => {
|
|
30333
|
-
if (error)
|
|
30334
|
-
reject(error);
|
|
30335
|
-
else
|
|
30336
|
-
resolve(result);
|
|
30337
|
-
}
|
|
30338
|
-
]);
|
|
30339
|
-
}), signal);
|
|
30340
|
-
} catch (error) {
|
|
30341
|
-
throw databaseError(error, signal);
|
|
30342
|
-
}
|
|
30343
|
-
})();
|
|
30344
|
-
if (callback === undefined)
|
|
30345
|
-
return pending;
|
|
30346
|
-
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
30347
|
-
}
|
|
30348
|
-
});
|
|
30349
|
-
return new Proxy(client, {
|
|
30350
|
-
get(target, property, receiver) {
|
|
30351
|
-
if (property === "query")
|
|
30352
|
-
return query;
|
|
30353
|
-
if (property === "release")
|
|
30354
|
-
return release;
|
|
30355
|
-
return Reflect.get(target, property, receiver);
|
|
30356
|
-
}
|
|
30357
|
-
});
|
|
30358
|
-
}
|
|
30359
|
-
function databaseError(error, signal) {
|
|
30360
|
-
if (signal.aborted)
|
|
30361
|
-
return new DbosDependencyError;
|
|
30362
|
-
const pending = [error];
|
|
30363
|
-
const seen = new Set;
|
|
30364
|
-
while (pending.length > 0) {
|
|
30365
|
-
const value2 = pending.pop();
|
|
30366
|
-
if (typeof value2 !== "object" || value2 === null || seen.has(value2))
|
|
30367
|
-
continue;
|
|
30368
|
-
seen.add(value2);
|
|
30369
|
-
if ("cause" in value2)
|
|
30370
|
-
pending.push(value2.cause);
|
|
30371
|
-
if ("error" in value2)
|
|
30372
|
-
pending.push(value2.error);
|
|
30373
|
-
if ("errors" in value2 && Array.isArray(value2.errors))
|
|
30374
|
-
pending.push(...value2.errors);
|
|
30375
|
-
const code = "code" in value2 && typeof value2.code === "string" ? value2.code : undefined;
|
|
30376
|
-
const message = value2 instanceof Error ? value2.message : "";
|
|
30377
|
-
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
30378
|
-
return new DbosDependencyError;
|
|
30379
|
-
}
|
|
30380
|
-
}
|
|
30381
|
-
return error;
|
|
30382
|
-
}
|
|
30383
|
-
var connectionErrnos, connectionMessages;
|
|
30384
|
-
var init_dbos_admission_pool = __esm(() => {
|
|
30385
|
-
init_dbos_admission();
|
|
30386
|
-
connectionErrnos = new Set([
|
|
30387
|
-
"ECONNRESET",
|
|
30388
|
-
"ECONNREFUSED",
|
|
30389
|
-
"ECONNABORTED",
|
|
30390
|
-
"EHOSTUNREACH",
|
|
30391
|
-
"ENETUNREACH",
|
|
30392
|
-
"ETIMEDOUT",
|
|
30393
|
-
"ENOTFOUND",
|
|
30394
|
-
"EAI_AGAIN",
|
|
30395
|
-
"EPIPE"
|
|
30396
|
-
]);
|
|
30397
|
-
connectionMessages = new Set([
|
|
30398
|
-
"Connection terminated unexpectedly",
|
|
30399
|
-
"Connection terminated due to connection timeout",
|
|
30400
|
-
"Client has encountered a connection error and is not queryable",
|
|
30401
|
-
"timeout exceeded when trying to connect"
|
|
30402
|
-
]);
|
|
30403
|
-
});
|
|
30404
|
-
|
|
30405
30961
|
// dist/builtin/workflows/src/durable/dbos-default-postgres-url.ts
|
|
30406
30962
|
function defaultPostgresUrl(database, host = process.env.PGHOST || "localhost", port = process.env.PGPORT || "5432") {
|
|
30407
30963
|
const url = new URL("postgresql://localhost");
|
|
@@ -30686,7 +31242,12 @@ function defaultDatabaseUrl(name) {
|
|
|
30686
31242
|
function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock) {
|
|
30687
31243
|
const systemDatabaseUrl = config.systemDatabaseUrl ?? defaultDatabaseUrl(config.name);
|
|
30688
31244
|
const health = resolvedPostgresHealth(systemDatabaseUrl);
|
|
30689
|
-
|
|
31245
|
+
getDbosProcessOwner().databaseDiagnostics = () => ({
|
|
31246
|
+
provider: health === undefined ? config.systemDatabaseUrl === undefined ? "docker" : "configured" : "managed",
|
|
31247
|
+
url: health?.endpoint ?? systemDatabaseUrl,
|
|
31248
|
+
failure: health?.lastFailure?.message
|
|
31249
|
+
});
|
|
31250
|
+
const createPool = (fenced = true) => {
|
|
30690
31251
|
let unsubscribe;
|
|
30691
31252
|
const managed = createRecoverablePostgresPool(systemDatabaseUrl, {
|
|
30692
31253
|
beforeConnect: health === undefined ? undefined : () => health.check(),
|
|
@@ -30696,14 +31257,38 @@ function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock
|
|
|
30696
31257
|
});
|
|
30697
31258
|
unsubscribe = health?.subscribe(managed.invalidate);
|
|
30698
31259
|
health?.start();
|
|
30699
|
-
return fenceDbosAdmissionPool(managed.pool);
|
|
31260
|
+
return fenced ? fenceDbosAdmissionPool(managed.pool) : managed.pool;
|
|
30700
31261
|
};
|
|
30701
31262
|
let pool = createPool();
|
|
31263
|
+
const createFence = (executorId) => {
|
|
31264
|
+
const fencePool = createPool(false);
|
|
31265
|
+
const created = new DbosOwnerFence(() => fencePool, executorId, () => fencePool.end());
|
|
31266
|
+
const owner = getDbosProcessOwner();
|
|
31267
|
+
owner.executorFences ??= new Set;
|
|
31268
|
+
owner.executorFences.add(created);
|
|
31269
|
+
return created;
|
|
31270
|
+
};
|
|
31271
|
+
let fence = createFence(config.executorID);
|
|
31272
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31273
|
+
getDbosProcessOwner().createExecutorFence = () => {
|
|
31274
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31275
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31276
|
+
const next = createFence(executorId);
|
|
31277
|
+
getDbosProcessOwner().executorFence = next;
|
|
31278
|
+
return next;
|
|
31279
|
+
};
|
|
31280
|
+
pool = fence.protectPool(pool);
|
|
30702
31281
|
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30703
31282
|
const launch = async () => {
|
|
31283
|
+
if (fence.closed) {
|
|
31284
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31285
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31286
|
+
fence = createFence(executorId);
|
|
31287
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31288
|
+
}
|
|
30704
31289
|
if (pool.ended) {
|
|
30705
|
-
pool = createPool();
|
|
30706
|
-
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
31290
|
+
pool = fence.protectPool(createPool());
|
|
31291
|
+
sdk.setConfig({ ...config, executorID: fence.executorId, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30707
31292
|
}
|
|
30708
31293
|
const urlToEnsure = health === undefined ? systemDatabaseUrl : await health.check();
|
|
30709
31294
|
const result = await import_datasource2.ensurePGDatabase({ urlToEnsure, logger: () => {} });
|
|
@@ -30723,6 +31308,8 @@ var import_datasource2;
|
|
|
30723
31308
|
var init_dbos_admission_config = __esm(() => {
|
|
30724
31309
|
init_dbos_admission_pool();
|
|
30725
31310
|
init_dbos_launch_lock();
|
|
31311
|
+
init_dbos_owner_fence();
|
|
31312
|
+
init_dbos_process_owner();
|
|
30726
31313
|
init_dbos_recoverable_pool();
|
|
30727
31314
|
import_datasource2 = __toESM(require_datasource(), 1);
|
|
30728
31315
|
});
|
|
@@ -81358,7 +81945,7 @@ function getDurableBackendProcessOwner() {
|
|
|
81358
81945
|
}
|
|
81359
81946
|
|
|
81360
81947
|
// dist/builtin/workflows/src/durable/dbos-backend.ts
|
|
81361
|
-
import { isDeepStrictEqual as
|
|
81948
|
+
import { isDeepStrictEqual as isDeepStrictEqual3 } from "node:util";
|
|
81362
81949
|
|
|
81363
81950
|
// dist/builtin/workflows/src/durable/format-version.ts
|
|
81364
81951
|
var DURABLE_FORMAT_VERSION = 3;
|
|
@@ -82221,10 +82808,10 @@ function isForeignLiveWorkflow(candidate, localExecutorId, now = Date.now()) {
|
|
|
82221
82808
|
return false;
|
|
82222
82809
|
if (candidate.ownerExecutorId === undefined || candidate.ownerExecutorId === localExecutorId)
|
|
82223
82810
|
return false;
|
|
82224
|
-
return now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
82811
|
+
return candidate.ownerLiveness === "alive" || now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
82225
82812
|
}
|
|
82226
82813
|
function isLiveRunningWorkflow(candidate, now = Date.now()) {
|
|
82227
|
-
return candidate.status === "running" && now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
82814
|
+
return candidate.status === "running" && (candidate.ownerLiveness === "alive" || now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS);
|
|
82228
82815
|
}
|
|
82229
82816
|
|
|
82230
82817
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
@@ -82274,7 +82861,7 @@ class InMemoryDurableBackend {
|
|
|
82274
82861
|
workflows = new Map;
|
|
82275
82862
|
promptReservations = new Map;
|
|
82276
82863
|
deletedWorkflowIds = new Set;
|
|
82277
|
-
registerWorkflow(handle) {
|
|
82864
|
+
registerWorkflow(handle, authoritativeOwner = false) {
|
|
82278
82865
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
82279
82866
|
const existing = this.workflows.get(handle.workflowId);
|
|
82280
82867
|
const completedCheckpoints = handle.completedCheckpoints ?? existing?.handle.completedCheckpoints ?? 0;
|
|
@@ -82288,7 +82875,7 @@ class InMemoryDurableBackend {
|
|
|
82288
82875
|
status: handle.status,
|
|
82289
82876
|
...handle.invocationCwd !== undefined ? { invocationCwd: handle.invocationCwd } : existing?.handle.invocationCwd !== undefined ? { invocationCwd: existing.handle.invocationCwd } : {},
|
|
82290
82877
|
...handle.origin !== undefined ? { origin: handle.origin } : existing?.handle.origin !== undefined ? { origin: existing.handle.origin } : {},
|
|
82291
|
-
modelOwner: existing?.handle.modelOwner ?? handle.modelOwner,
|
|
82878
|
+
modelOwner: authoritativeOwner ? handle.modelOwner : existing?.handle.modelOwner ?? handle.modelOwner,
|
|
82292
82879
|
...handle.workflowCwd !== undefined ? { workflowCwd: handle.workflowCwd } : existing?.handle.workflowCwd !== undefined ? { workflowCwd: existing.handle.workflowCwd } : {},
|
|
82293
82880
|
...handle.repositoryRoot !== undefined ? { repositoryRoot: handle.repositoryRoot } : existing?.handle.repositoryRoot !== undefined ? { repositoryRoot: existing.handle.repositoryRoot } : {},
|
|
82294
82881
|
...handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: handle.gitWorktreeRoot } : existing?.handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: existing.handle.gitWorktreeRoot } : {},
|
|
@@ -82302,7 +82889,9 @@ class InMemoryDurableBackend {
|
|
|
82302
82889
|
...handle.rootWorkflowId !== undefined ? { rootWorkflowId: handle.rootWorkflowId } : {},
|
|
82303
82890
|
...handle.resumable !== undefined ? { resumable: handle.resumable } : {},
|
|
82304
82891
|
...workflowFailureFields(hasWorkflowFailureMetadata(handle) ? handle : handle.status === "running" ? undefined : existing?.handle),
|
|
82305
|
-
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {}
|
|
82892
|
+
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {},
|
|
82893
|
+
...handle.ownerLiveness !== undefined ? { ownerLiveness: handle.ownerLiveness } : {},
|
|
82894
|
+
...handle.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {}
|
|
82306
82895
|
};
|
|
82307
82896
|
if (existing)
|
|
82308
82897
|
existing.handle = full;
|
|
@@ -82452,11 +83041,14 @@ class InMemoryDurableBackend {
|
|
|
82452
83041
|
if (pendingPrompts !== undefined)
|
|
82453
83042
|
this.promptReservations.delete(workflowId);
|
|
82454
83043
|
}
|
|
82455
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
82456
|
-
const
|
|
83044
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
83045
|
+
const record = this.workflows.get(workflowId);
|
|
83046
|
+
const current = record?.handle;
|
|
82457
83047
|
if (current === undefined || !expected.includes(current.status) || expectedUpdatedAt !== undefined && current.updatedAt !== expectedUpdatedAt)
|
|
82458
83048
|
return false;
|
|
82459
83049
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
83050
|
+
if (modelOwner !== undefined && record !== undefined)
|
|
83051
|
+
record.handle = { ...record.handle, modelOwner };
|
|
82460
83052
|
return true;
|
|
82461
83053
|
}
|
|
82462
83054
|
adjustPendingPrompts(workflowId, delta) {
|
|
@@ -82529,6 +83121,7 @@ class InMemoryDurableBackend {
|
|
|
82529
83121
|
pendingStageMessages: h.pendingStageMessages ?? [],
|
|
82530
83122
|
...h.possibleStages !== undefined ? { possibleStages: h.possibleStages } : {},
|
|
82531
83123
|
...h.ownerExecutorId !== undefined ? { ownerExecutorId: h.ownerExecutorId } : {},
|
|
83124
|
+
...h.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
82532
83125
|
...h.sessionFile !== undefined ? { sessionFile: h.sessionFile } : {},
|
|
82533
83126
|
...h.label !== undefined ? { label: h.label } : {},
|
|
82534
83127
|
...h.rootWorkflowId !== undefined ? { rootWorkflowId: h.rootWorkflowId } : {},
|
|
@@ -85359,6 +85952,7 @@ function encodeMetadata(metadata) {
|
|
|
85359
85952
|
} : {},
|
|
85360
85953
|
...metadata.ownerExecutorId !== undefined ? { ownerExecutorId: metadata.ownerExecutorId } : {},
|
|
85361
85954
|
...metadata.transitionClaimId !== undefined ? { transitionClaimId: metadata.transitionClaimId } : {},
|
|
85955
|
+
...metadata.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
85362
85956
|
...metadata.sessionFile !== undefined ? { sessionFile: metadata.sessionFile } : {},
|
|
85363
85957
|
...metadata.label !== undefined ? { label: metadata.label } : {},
|
|
85364
85958
|
...metadata.rootWorkflowId !== undefined ? { rootWorkflowId: metadata.rootWorkflowId } : {},
|
|
@@ -85421,7 +86015,7 @@ function parseDurableWorkflowMetadata(value2, workflowId) {
|
|
|
85421
86015
|
return;
|
|
85422
86016
|
const possibleStages = coercePossibleStages(serialized.possibleStages);
|
|
85423
86017
|
const metadata = value2;
|
|
85424
|
-
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
86018
|
+
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.legacyRecoveryPending !== undefined && metadata.legacyRecoveryPending !== true || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
85425
86019
|
return;
|
|
85426
86020
|
const { origin, possibleStages: rawPossibleStages, ...metadataWithoutOrigin } = metadata;
|
|
85427
86021
|
return {
|
|
@@ -85896,40 +86490,20 @@ function classifyDbosDeletionTombstone(records, workflowId) {
|
|
|
85896
86490
|
return isCurrentDurableFormat(raw.version) ? "current" : "unknown";
|
|
85897
86491
|
}
|
|
85898
86492
|
|
|
85899
|
-
// dist/builtin/workflows/src/durable/dbos-
|
|
85900
|
-
|
|
85901
|
-
function emptyOwner2() {
|
|
85902
|
-
return {
|
|
85903
|
-
version: 1,
|
|
85904
|
-
state: "uninitialized",
|
|
85905
|
-
configured: undefined,
|
|
85906
|
-
active: undefined,
|
|
85907
|
-
launchPromise: undefined,
|
|
85908
|
-
shutdownPromise: undefined,
|
|
85909
|
-
failure: undefined,
|
|
85910
|
-
wrappers: undefined,
|
|
85911
|
-
leases: new Set
|
|
85912
|
-
};
|
|
85913
|
-
}
|
|
85914
|
-
function ownerBag2() {
|
|
85915
|
-
return globalThis;
|
|
85916
|
-
}
|
|
85917
|
-
function getDbosProcessOwner() {
|
|
85918
|
-
const bag = ownerBag2();
|
|
85919
|
-
const existing = bag[DBOS_PROCESS_OWNER_KEY];
|
|
85920
|
-
if (existing !== undefined && existing.version === 1)
|
|
85921
|
-
return existing;
|
|
85922
|
-
const created = emptyOwner2();
|
|
85923
|
-
bag[DBOS_PROCESS_OWNER_KEY] = created;
|
|
85924
|
-
return created;
|
|
85925
|
-
}
|
|
86493
|
+
// dist/builtin/workflows/src/durable/dbos-backend.ts
|
|
86494
|
+
init_dbos_process_owner();
|
|
85926
86495
|
|
|
85927
86496
|
// dist/builtin/workflows/src/durable/dbos-sdk-handle.ts
|
|
85928
|
-
|
|
86497
|
+
import { AsyncLocalStorage as AsyncLocalStorage3 } from "node:async_hooks";
|
|
86498
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
86499
|
+
init_dbos_owner_fence();
|
|
86500
|
+
init_dbos_process_owner();
|
|
85929
86501
|
function getAtomicExecutorId() {
|
|
85930
|
-
|
|
86502
|
+
const owner = getDbosProcessOwner();
|
|
86503
|
+
owner.executorId ??= `atomic-db-${crypto.randomUUID()}`;
|
|
86504
|
+
return owner.executorId;
|
|
85931
86505
|
}
|
|
85932
|
-
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
86506
|
+
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, fence, previousExecutorIds = []) {
|
|
85933
86507
|
const checkpointId = (workflowId, stepName) => `${workflowId}:checkpoint:${stepName}`;
|
|
85934
86508
|
async function stepRecord(status, prefix) {
|
|
85935
86509
|
if (status.status !== "SUCCESS")
|
|
@@ -85943,10 +86517,18 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
85943
86517
|
const output = status.output === undefined || typeof status.output === "string" ? await dbos.retrieveWorkflow(id).getResult() : status.output;
|
|
85944
86518
|
return { stepName, output, completedAt: status.createdAt };
|
|
85945
86519
|
}
|
|
85946
|
-
|
|
86520
|
+
const raw = {
|
|
85947
86521
|
launch: () => dbos.launch(),
|
|
85948
86522
|
shutdown: () => dbos.shutdown(),
|
|
85949
86523
|
async startWorkflow(workflowId, name, inputs) {
|
|
86524
|
+
if (fence !== undefined) {
|
|
86525
|
+
const existing = await raw.retrieveWorkflow(workflowId);
|
|
86526
|
+
if (existing !== undefined) {
|
|
86527
|
+
if (existing.name !== name || !isDeepStrictEqual2(existing.inputs, inputs))
|
|
86528
|
+
throw new Error("Workflow root identity changed during admission");
|
|
86529
|
+
return;
|
|
86530
|
+
}
|
|
86531
|
+
}
|
|
85950
86532
|
try {
|
|
85951
86533
|
await dbos.startWorkflow(mainWorkflow, { workflowID: workflowId })(name, { ...inputs });
|
|
85952
86534
|
} catch (err) {
|
|
@@ -85965,7 +86547,10 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
85965
86547
|
await dbos.cancelWorkflow(workflowId, { cancelChildren: true });
|
|
85966
86548
|
},
|
|
85967
86549
|
async resumeWorkflow(workflowId) {
|
|
85968
|
-
|
|
86550
|
+
if (fence === undefined)
|
|
86551
|
+
await dbos.resumeWorkflow(workflowId);
|
|
86552
|
+
else if (await raw.retrieveWorkflow(workflowId) === undefined)
|
|
86553
|
+
throw new Error("Workflow root is unavailable for resume");
|
|
85969
86554
|
},
|
|
85970
86555
|
async listAllWorkflows() {
|
|
85971
86556
|
const statuses = await dbos.listWorkflows({
|
|
@@ -85994,6 +86579,12 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
85994
86579
|
return status !== undefined && (status.workflowID ?? status.workflowId) === id ? stepRecord(status, prefix) : undefined;
|
|
85995
86580
|
},
|
|
85996
86581
|
async recordStepOutput(workflowId, stepName, output) {
|
|
86582
|
+
if (fence !== undefined) {
|
|
86583
|
+
const id = checkpointId(workflowId, stepName);
|
|
86584
|
+
const existing = (await dbos.listWorkflows({ workflowIDs: [id], limit: 1 }))[0];
|
|
86585
|
+
if (existing?.status !== undefined && ["PENDING", "ENQUEUED", "CANCELLED", "ERROR", "DELAYED"].includes(existing.status))
|
|
86586
|
+
await dbos.deleteWorkflows([id], false);
|
|
86587
|
+
}
|
|
85997
86588
|
let handle;
|
|
85998
86589
|
try {
|
|
85999
86590
|
handle = await dbos.startWorkflow(checkpointWorkflow, { workflowID: checkpointId(workflowId, stepName) })(workflowId, stepName, output);
|
|
@@ -86017,6 +86608,135 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
86017
86608
|
await dbos.deleteWorkflows([...new Set([workflowId, ...checkpointIds])], true);
|
|
86018
86609
|
}
|
|
86019
86610
|
};
|
|
86611
|
+
if (fence === undefined)
|
|
86612
|
+
return raw;
|
|
86613
|
+
const claims = new AsyncLocalStorage3;
|
|
86614
|
+
const rowAuthority = (workflowId, current) => ({
|
|
86615
|
+
root: workflowId,
|
|
86616
|
+
actor: fence.executorId,
|
|
86617
|
+
...current.kind === "current" ? { owner: current.metadata.ownerExecutorId, generation: current.generation } : {},
|
|
86618
|
+
enroll: current.kind === "unavailable" || current.kind === "current" && isDatabaseExecutor(current.metadata.ownerExecutorId)
|
|
86619
|
+
});
|
|
86620
|
+
const mutate = async (workflowId, callback, step) => fence.write(workflowId, async () => {
|
|
86621
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86622
|
+
const claimed = claims.getStore();
|
|
86623
|
+
if (claimed === undefined && current.kind === "current" && current.metadata.ownerExecutorId !== fence.executorId)
|
|
86624
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
86625
|
+
let output = step === undefined ? undefined : classifyLatestMetadata([{ stepName: step.name, output: step.output }], workflowId);
|
|
86626
|
+
let persistedStep = step;
|
|
86627
|
+
if (step !== undefined && output?.kind === "current" && current.kind === "current" && !step.name.endsWith(":claim") && output.metadata.ownerExecutorId === fence.executorId && output.generation <= current.generation) {
|
|
86628
|
+
const updatedAt = Math.max(current.generation + 1, output.metadata.updatedAt, Date.now());
|
|
86629
|
+
persistedStep = {
|
|
86630
|
+
name: metadataStepName(updatedAt),
|
|
86631
|
+
output: encodeMetadata({ ...output.metadata, updatedAt })
|
|
86632
|
+
};
|
|
86633
|
+
output = classifyLatestMetadata([{ stepName: persistedStep.name, output: persistedStep.output }], workflowId);
|
|
86634
|
+
}
|
|
86635
|
+
const authority = claimed?.authority ?? rowAuthority(workflowId, current);
|
|
86636
|
+
const capability = output?.kind === "current" && step?.name.endsWith(":claim") ? {
|
|
86637
|
+
...authority,
|
|
86638
|
+
claim: output.metadata.transitionClaimId,
|
|
86639
|
+
checkpoint: checkpointId(workflowId, step.name)
|
|
86640
|
+
} : authority;
|
|
86641
|
+
const record = persistedStep;
|
|
86642
|
+
await fence.withRowAuthority(capability, record === undefined ? callback : () => raw.recordStepOutput(workflowId, record.name, record.output));
|
|
86643
|
+
if (claimed !== undefined && output?.kind === "current")
|
|
86644
|
+
claimed.authority = {
|
|
86645
|
+
...capability,
|
|
86646
|
+
owner: output.metadata.ownerExecutorId,
|
|
86647
|
+
generation: output.generation
|
|
86648
|
+
};
|
|
86649
|
+
});
|
|
86650
|
+
return {
|
|
86651
|
+
...raw,
|
|
86652
|
+
executorId: fence.executorId,
|
|
86653
|
+
generationLost: () => fence.invalidated,
|
|
86654
|
+
forkGeneration: () => {
|
|
86655
|
+
const next = getDbosProcessOwner().createExecutorFence?.();
|
|
86656
|
+
if (next === undefined)
|
|
86657
|
+
throw new Error("Workflow ownership fence cannot create a recovery generation");
|
|
86658
|
+
fence.close().catch(() => {});
|
|
86659
|
+
return createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, next, [
|
|
86660
|
+
...previousExecutorIds,
|
|
86661
|
+
fence.executorId
|
|
86662
|
+
]);
|
|
86663
|
+
},
|
|
86664
|
+
adoptPreviousGeneration: async (workflowId) => {
|
|
86665
|
+
if (previousExecutorIds.length === 0)
|
|
86666
|
+
return;
|
|
86667
|
+
return await fence.write(workflowId, async () => {
|
|
86668
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86669
|
+
if (current.kind !== "current" || current.metadata.ownerExecutorId === fence.executorId)
|
|
86670
|
+
return;
|
|
86671
|
+
const previousExecutorId = current.metadata.ownerExecutorId;
|
|
86672
|
+
if (previousExecutorId === undefined || !previousExecutorIds.includes(previousExecutorId))
|
|
86673
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
86674
|
+
const metadata = {
|
|
86675
|
+
...current.metadata,
|
|
86676
|
+
ownerExecutorId: fence.executorId,
|
|
86677
|
+
transitionClaimId: crypto.randomUUID(),
|
|
86678
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
86679
|
+
};
|
|
86680
|
+
const adopted = await fence.recover(previousExecutorId, async () => {
|
|
86681
|
+
await fence.withRowAuthority({
|
|
86682
|
+
...rowAuthority(workflowId, current),
|
|
86683
|
+
claim: metadata.transitionClaimId,
|
|
86684
|
+
checkpoint: checkpointId(workflowId, claimMetadataStepName(current.generation))
|
|
86685
|
+
}, () => raw.recordStepOutput(workflowId, claimMetadataStepName(current.generation), encodeMetadata(metadata)));
|
|
86686
|
+
return metadata;
|
|
86687
|
+
});
|
|
86688
|
+
if (adopted === undefined)
|
|
86689
|
+
throw new Error("Workflow previous execution generation is still active");
|
|
86690
|
+
return adopted;
|
|
86691
|
+
}, true);
|
|
86692
|
+
},
|
|
86693
|
+
enrollLegacyWorkflow: async (workflowId, modelOwner) => fence.write(workflowId, async () => {
|
|
86694
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86695
|
+
if (current.kind !== "current" || isDatabaseExecutor(current.metadata.ownerExecutorId) && current.metadata.legacyRecoveryPending !== true || !["running", "paused", "blocked"].includes(current.metadata.status) || current.metadata.rootWorkflowId !== undefined && current.metadata.rootWorkflowId !== workflowId)
|
|
86696
|
+
return;
|
|
86697
|
+
const metadata = {
|
|
86698
|
+
...current.metadata,
|
|
86699
|
+
modelOwner,
|
|
86700
|
+
ownerExecutorId: fence.executorId,
|
|
86701
|
+
status: "blocked",
|
|
86702
|
+
resumable: true,
|
|
86703
|
+
legacyRecoveryPending: true,
|
|
86704
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
86705
|
+
};
|
|
86706
|
+
const retry = isDatabaseExecutor(current.metadata.ownerExecutorId);
|
|
86707
|
+
const transitionClaimId = crypto.randomUUID();
|
|
86708
|
+
const stepName = retry ? claimMetadataStepName(current.generation) : metadataStepName(Math.max(current.generation + 1, metadata.updatedAt));
|
|
86709
|
+
const persist = () => fence.withRowAuthority({
|
|
86710
|
+
...rowAuthority(workflowId, current),
|
|
86711
|
+
enroll: true,
|
|
86712
|
+
...retry ? { claim: transitionClaimId, checkpoint: checkpointId(workflowId, stepName) } : {}
|
|
86713
|
+
}, () => raw.recordStepOutput(workflowId, stepName, encodeMetadata({ ...metadata, ...retry ? { transitionClaimId } : {} })));
|
|
86714
|
+
if (retry) {
|
|
86715
|
+
const recovered = await fence.recover(current.metadata.ownerExecutorId, async () => {
|
|
86716
|
+
await persist();
|
|
86717
|
+
return true;
|
|
86718
|
+
});
|
|
86719
|
+
if (recovered !== true)
|
|
86720
|
+
return;
|
|
86721
|
+
} else
|
|
86722
|
+
await persist();
|
|
86723
|
+
return metadata;
|
|
86724
|
+
}, true),
|
|
86725
|
+
ownerLiveness: (executorId) => fence.liveness(executorId),
|
|
86726
|
+
withWorkflowClaim: (workflowId, callback) => fence.write(workflowId, async () => {
|
|
86727
|
+
const authoritative = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
86728
|
+
const claim = () => claims.run({ authority: { ...rowAuthority(workflowId, authoritative), claim: crypto.randomUUID() } }, callback);
|
|
86729
|
+
if (authoritative.kind === "current" && authoritative.metadata.status === "running" && authoritative.metadata.ownerExecutorId !== fence.executorId) {
|
|
86730
|
+
return await fence.recover(authoritative.metadata.ownerExecutorId, claim) ?? false;
|
|
86731
|
+
}
|
|
86732
|
+
return await claim();
|
|
86733
|
+
}, true),
|
|
86734
|
+
startWorkflow: (id, name, inputs) => mutate(id, () => raw.startWorkflow(id, name, inputs)),
|
|
86735
|
+
cancelWorkflow: (id) => mutate(id, () => raw.cancelWorkflow(id)),
|
|
86736
|
+
resumeWorkflow: (id) => mutate(id, () => raw.resumeWorkflow(id)),
|
|
86737
|
+
recordStepOutput: (id, step, output) => mutate(id, () => raw.recordStepOutput(id, step, output), { name: step, output }),
|
|
86738
|
+
deleteWorkflowData: (id) => mutate(id, () => raw.deleteWorkflowData(id))
|
|
86739
|
+
};
|
|
86020
86740
|
}
|
|
86021
86741
|
function isDbosDuplicateWorkflowError(err) {
|
|
86022
86742
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -86043,6 +86763,7 @@ function statusToInfo(status, fallbackId) {
|
|
|
86043
86763
|
}
|
|
86044
86764
|
|
|
86045
86765
|
// dist/builtin/workflows/src/durable/dbos-system-database-url.ts
|
|
86766
|
+
init_dbos_process_owner();
|
|
86046
86767
|
function nonEmptyTrimmed(value2) {
|
|
86047
86768
|
const trimmed = value2?.trim();
|
|
86048
86769
|
return trimmed === undefined || trimmed.length === 0 ? undefined : trimmed;
|
|
@@ -86057,9 +86778,7 @@ class DbosSystemDatabaseConflictError extends Error {
|
|
|
86057
86778
|
function redactDatabaseUrl(url) {
|
|
86058
86779
|
try {
|
|
86059
86780
|
const parsed = new URL(url);
|
|
86060
|
-
|
|
86061
|
-
parsed.password = "***";
|
|
86062
|
-
return parsed.toString();
|
|
86781
|
+
return parsed.host;
|
|
86063
86782
|
} catch {
|
|
86064
86783
|
return "the requested database";
|
|
86065
86784
|
}
|
|
@@ -86099,6 +86818,8 @@ async function configureDbosDurableBackend(config) {
|
|
|
86099
86818
|
const sdk = await importDbosSdk();
|
|
86100
86819
|
const owner = getDbosProcessOwner();
|
|
86101
86820
|
const existing = owner.wrappers;
|
|
86821
|
+
if (existing !== undefined && owner.executorFence === undefined)
|
|
86822
|
+
throw new Error("Existing workflow executor has no database ownership fence; restart the host before recovering workflows");
|
|
86102
86823
|
let launch = owner.active?.launch ?? (() => sdk.launch());
|
|
86103
86824
|
let checkReady;
|
|
86104
86825
|
if (existing === undefined) {
|
|
@@ -86121,12 +86842,30 @@ async function configureDbosDurableBackend(config) {
|
|
|
86121
86842
|
if (existing === undefined) {
|
|
86122
86843
|
owner.wrappers = { mainWorkflow, checkpointWorkflow };
|
|
86123
86844
|
}
|
|
86845
|
+
let backendFence = owner.executorFence;
|
|
86846
|
+
let backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), {
|
|
86847
|
+
checkReady
|
|
86848
|
+
});
|
|
86124
86849
|
return {
|
|
86125
|
-
backend
|
|
86126
|
-
|
|
86127
|
-
}
|
|
86128
|
-
launch
|
|
86129
|
-
|
|
86850
|
+
get backend() {
|
|
86851
|
+
return backend;
|
|
86852
|
+
},
|
|
86853
|
+
launch: async () => {
|
|
86854
|
+
await launch();
|
|
86855
|
+
if (backendFence !== owner.executorFence && backendFence?.closed) {
|
|
86856
|
+
backendFence = owner.executorFence;
|
|
86857
|
+
backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), { checkReady });
|
|
86858
|
+
}
|
|
86859
|
+
},
|
|
86860
|
+
shutdown: async () => {
|
|
86861
|
+
const results = await Promise.allSettled([
|
|
86862
|
+
sdk.shutdown(),
|
|
86863
|
+
...[...owner.executorFences ?? []].map((fence) => fence.close())
|
|
86864
|
+
]);
|
|
86865
|
+
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason instanceof Error ? result.reason : new Error(String(result.reason)));
|
|
86866
|
+
if (errors.length > 0)
|
|
86867
|
+
throw new AggregateError(errors, "Workflow database shutdown failed");
|
|
86868
|
+
}
|
|
86130
86869
|
};
|
|
86131
86870
|
}
|
|
86132
86871
|
async function importDbosSdk() {
|
|
@@ -86159,13 +86898,18 @@ class DbosDurableBackend {
|
|
|
86159
86898
|
unavailableAdmissions = new Set;
|
|
86160
86899
|
unavailableCheckpoints = new Set;
|
|
86161
86900
|
pendingRecoveryAdmissions = new Set;
|
|
86901
|
+
ownerLiveness = new Map;
|
|
86162
86902
|
admissionMetadataAttempted = new Set;
|
|
86163
86903
|
admissionSettlements = new Map;
|
|
86164
86904
|
admissionRecoveries = new Map;
|
|
86165
86905
|
admissionRecoveryControllers = new Map;
|
|
86906
|
+
fixedGeneration = false;
|
|
86907
|
+
generationIsCurrent;
|
|
86908
|
+
rebindInitialAdmission;
|
|
86909
|
+
admitted = false;
|
|
86166
86910
|
constructor(sdk, options) {
|
|
86167
86911
|
this.sdk = sdk;
|
|
86168
|
-
this.executorId = options?.executorId ?? getAtomicExecutorId();
|
|
86912
|
+
this.executorId = options?.executorId ?? sdk.executorId ?? getAtomicExecutorId();
|
|
86169
86913
|
this.onUnavailable = options?.onUnavailable;
|
|
86170
86914
|
this.checkReady = options?.checkReady;
|
|
86171
86915
|
this.promptReservations = new DbosPromptReservationTracker({
|
|
@@ -86176,6 +86920,55 @@ class DbosDurableBackend {
|
|
|
86176
86920
|
}
|
|
86177
86921
|
});
|
|
86178
86922
|
}
|
|
86923
|
+
async enrollLegacyWorkflow(workflowId, options) {
|
|
86924
|
+
if (options.olderWorkersStopped !== true || this.sdk.enrollLegacyWorkflow === undefined)
|
|
86925
|
+
return false;
|
|
86926
|
+
this.ensureGeneration();
|
|
86927
|
+
return await boundedAdmission((signal) => dbosAdmissionContext.run(signal, async () => {
|
|
86928
|
+
signal.throwIfAborted();
|
|
86929
|
+
const metadata = await this.sdk.enrollLegacyWorkflow(workflowId, options.modelOwner);
|
|
86930
|
+
signal.throwIfAborted();
|
|
86931
|
+
if (metadata === undefined)
|
|
86932
|
+
return false;
|
|
86933
|
+
this.applyMetadata(workflowId, metadata);
|
|
86934
|
+
this.current.add(workflowId);
|
|
86935
|
+
return true;
|
|
86936
|
+
}), options.signal);
|
|
86937
|
+
}
|
|
86938
|
+
executionView() {
|
|
86939
|
+
if (this.fixedGeneration || this.sdk.forkGeneration === undefined)
|
|
86940
|
+
return this;
|
|
86941
|
+
this.ensureGeneration();
|
|
86942
|
+
const view = new DbosDurableBackend(this.sdk, {
|
|
86943
|
+
executorId: this.executorId,
|
|
86944
|
+
checkReady: this.checkReady,
|
|
86945
|
+
onUnavailable: this.onUnavailable
|
|
86946
|
+
});
|
|
86947
|
+
const reservations = view.promptReservations;
|
|
86948
|
+
Object.assign(view, this);
|
|
86949
|
+
Object.assign(view, {
|
|
86950
|
+
promptReservations: reservations,
|
|
86951
|
+
fixedGeneration: true,
|
|
86952
|
+
generationIsCurrent: () => this.sdk === view.sdk,
|
|
86953
|
+
rebindInitialAdmission: () => {
|
|
86954
|
+
this.ensureGeneration();
|
|
86955
|
+
view.sdk = this.sdk;
|
|
86956
|
+
view.executorId = this.executorId;
|
|
86957
|
+
}
|
|
86958
|
+
});
|
|
86959
|
+
return view;
|
|
86960
|
+
}
|
|
86961
|
+
ensureGeneration() {
|
|
86962
|
+
if (this.fixedGeneration) {
|
|
86963
|
+
if (this.generationIsCurrent?.() === false)
|
|
86964
|
+
throw new DbosDependencyError("Workflow database ownership generation changed; stale execution writes are refused");
|
|
86965
|
+
return;
|
|
86966
|
+
}
|
|
86967
|
+
if (this.sdk.generationLost?.() && this.sdk.forkGeneration !== undefined) {
|
|
86968
|
+
this.sdk = this.sdk.forkGeneration();
|
|
86969
|
+
this.executorId = this.sdk.executorId ?? this.executorId;
|
|
86970
|
+
}
|
|
86971
|
+
}
|
|
86179
86972
|
isAdmissionUnavailable(workflowId) {
|
|
86180
86973
|
return this.unavailableAdmissions.has(workflowId);
|
|
86181
86974
|
}
|
|
@@ -86192,6 +86985,8 @@ class DbosDurableBackend {
|
|
|
86192
86985
|
return this.admissionSettlements.has(workflowId);
|
|
86193
86986
|
}
|
|
86194
86987
|
admitWorkflow(workflowId, registration, signal) {
|
|
86988
|
+
if (!this.admitted)
|
|
86989
|
+
this.rebindInitialAdmission?.();
|
|
86195
86990
|
const pending = this.performAdmission(workflowId, registration, signal);
|
|
86196
86991
|
this.admissionSettlements.set(workflowId, pending);
|
|
86197
86992
|
pending.then(() => {
|
|
@@ -86199,6 +86994,7 @@ class DbosDurableBackend {
|
|
|
86199
86994
|
this.admissionSettlements.delete(workflowId);
|
|
86200
86995
|
}, () => {});
|
|
86201
86996
|
return pending.then(async () => {
|
|
86997
|
+
this.admitted = true;
|
|
86202
86998
|
await dbosAdmissionContext.run(signal, () => this.flush(workflowId));
|
|
86203
86999
|
signal.throwIfAborted();
|
|
86204
87000
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
@@ -86219,12 +87015,13 @@ class DbosDurableBackend {
|
|
|
86219
87015
|
await this.flushWrites(workflowId);
|
|
86220
87016
|
signal.throwIfAborted();
|
|
86221
87017
|
}), signal);
|
|
87018
|
+
this.ensureGeneration();
|
|
86222
87019
|
this.admissionUnavailable = false;
|
|
86223
87020
|
this.unavailableAdmissions.delete(workflowId);
|
|
86224
87021
|
this.unavailableCheckpoints.delete(workflowId);
|
|
86225
87022
|
this.pendingRecoveryAdmissions.delete(workflowId);
|
|
86226
87023
|
} catch (error) {
|
|
86227
|
-
if (isDbosDependencyError(error)) {
|
|
87024
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
86228
87025
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
86229
87026
|
this.admissionUnavailable = true;
|
|
86230
87027
|
this.unavailableAdmissions.add(workflowId);
|
|
@@ -86250,6 +87047,7 @@ class DbosDurableBackend {
|
|
|
86250
87047
|
});
|
|
86251
87048
|
}
|
|
86252
87049
|
registerWorkflow(handle) {
|
|
87050
|
+
this.ensureGeneration();
|
|
86253
87051
|
if (this.invalid.has(handle.workflowId)) {
|
|
86254
87052
|
throw new DurableNestedTopologyError(`workflow ${handle.workflowId} contains malformed current DBOS records`);
|
|
86255
87053
|
}
|
|
@@ -86291,6 +87089,7 @@ class DbosDurableBackend {
|
|
|
86291
87089
|
return persisted;
|
|
86292
87090
|
}
|
|
86293
87091
|
recordCheckpoint(checkpoint) {
|
|
87092
|
+
this.ensureGeneration();
|
|
86294
87093
|
if (!this.isWorkflowLoadable(checkpoint.workflowId))
|
|
86295
87094
|
return;
|
|
86296
87095
|
this.mem.recordCheckpoint(checkpoint);
|
|
@@ -86319,7 +87118,7 @@ class DbosDurableBackend {
|
|
|
86319
87118
|
});
|
|
86320
87119
|
signal.throwIfAborted();
|
|
86321
87120
|
}), options?.signal ?? dbosAdmissionContext.getStore(), undefined, "Workflow database checkpoint timed out. Restore PostgreSQL and inspect the run before resuming; external outcomes may be unknown.").catch((error) => {
|
|
86322
|
-
if (isDbosDependencyError(error)) {
|
|
87121
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
86323
87122
|
this.unavailableCheckpoints.add(checkpoint.workflowId);
|
|
86324
87123
|
const errors = this.writeErrors.get(checkpoint.workflowId);
|
|
86325
87124
|
if (errors !== undefined)
|
|
@@ -86367,12 +87166,14 @@ class DbosDurableBackend {
|
|
|
86367
87166
|
return this.mem.listCheckpoints(workflowId);
|
|
86368
87167
|
}
|
|
86369
87168
|
getWorkflow(workflowId) {
|
|
86370
|
-
|
|
87169
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
87170
|
+
return handle === undefined || this.sdk.ownerLiveness === undefined ? handle : { ...handle, ownerLiveness: this.ownerLiveness.get(handle.ownerExecutorId ?? "") ?? "unknown" };
|
|
86371
87171
|
}
|
|
86372
87172
|
getLoadableWorkflow(workflowId) {
|
|
86373
|
-
return this.isWorkflowLoadable(workflowId) ? this.
|
|
87173
|
+
return this.isWorkflowLoadable(workflowId) ? this.getWorkflow(workflowId) : undefined;
|
|
86374
87174
|
}
|
|
86375
87175
|
setWorkflowStatus(workflowId, status, pendingPrompts, resumable, failure) {
|
|
87176
|
+
this.ensureGeneration();
|
|
86376
87177
|
if (status === "cancelled" || status === "completed" || resumable === false) {
|
|
86377
87178
|
this.admissionRecoveryControllers.get(workflowId)?.abort(new Error("Workflow recovery superseded by terminal control."));
|
|
86378
87179
|
}
|
|
@@ -86396,9 +87197,10 @@ class DbosDurableBackend {
|
|
|
86396
87197
|
await this.writeMetadata(workflowId);
|
|
86397
87198
|
});
|
|
86398
87199
|
}
|
|
86399
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
87200
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
87201
|
+
this.ensureGeneration();
|
|
86400
87202
|
let records = [];
|
|
86401
|
-
|
|
87203
|
+
const transition = () => transitionDbosWorkflowStatus({
|
|
86402
87204
|
expectedStatuses: expected,
|
|
86403
87205
|
status,
|
|
86404
87206
|
flush: () => this.flush(workflowId),
|
|
@@ -86412,14 +87214,25 @@ class DbosDurableBackend {
|
|
|
86412
87214
|
const pendingPrompts = this.promptReservations.hydrate(workflowId, entry.pendingPrompts, records, entry.promptReservationEpoch);
|
|
86413
87215
|
this.applyMetadata(workflowId, { ...entry, pendingPrompts });
|
|
86414
87216
|
},
|
|
86415
|
-
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable),
|
|
87217
|
+
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner),
|
|
86416
87218
|
write: async () => {
|
|
87219
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
87220
|
+
if (handle !== undefined)
|
|
87221
|
+
this.mem.registerWorkflow({
|
|
87222
|
+
...handle,
|
|
87223
|
+
legacyRecoveryPending: undefined,
|
|
87224
|
+
...modelOwner !== undefined ? { modelOwner } : {}
|
|
87225
|
+
}, true);
|
|
86417
87226
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
86418
87227
|
await this.flush(workflowId);
|
|
86419
87228
|
}
|
|
86420
87229
|
});
|
|
87230
|
+
if (this.sdk.withWorkflowClaim === undefined)
|
|
87231
|
+
return await transition();
|
|
87232
|
+
await this.flush(workflowId);
|
|
87233
|
+
return await this.sdk.withWorkflowClaim(workflowId, transition);
|
|
86421
87234
|
}
|
|
86422
|
-
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable) {
|
|
87235
|
+
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner) {
|
|
86423
87236
|
const stepName = claimMetadataStepName(generation);
|
|
86424
87237
|
const transitionClaimId = crypto.randomUUID();
|
|
86425
87238
|
const claim = {
|
|
@@ -86427,6 +87240,8 @@ class DbosDurableBackend {
|
|
|
86427
87240
|
status,
|
|
86428
87241
|
...pendingPrompts !== undefined ? { pendingPrompts } : {},
|
|
86429
87242
|
...resumable !== undefined ? { resumable } : {},
|
|
87243
|
+
...modelOwner === undefined ? {} : { modelOwner },
|
|
87244
|
+
legacyRecoveryPending: undefined,
|
|
86430
87245
|
ownerExecutorId: this.executorId,
|
|
86431
87246
|
transitionClaimId,
|
|
86432
87247
|
updatedAt: Math.max(Date.now(), authoritative.updatedAt + 1)
|
|
@@ -86568,7 +87383,7 @@ class DbosDurableBackend {
|
|
|
86568
87383
|
this.mem.recordCheckpoint(classified.checkpoint);
|
|
86569
87384
|
}
|
|
86570
87385
|
const handle = this.getLoadableWorkflow(workflowId);
|
|
86571
|
-
return handle === undefined ? { kind: "malformed" } :
|
|
87386
|
+
return handle === undefined ? { kind: "malformed" } : await this.withOwnerLiveness(handle);
|
|
86572
87387
|
}
|
|
86573
87388
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
86574
87389
|
if (info !== undefined)
|
|
@@ -86591,6 +87406,7 @@ class DbosDurableBackend {
|
|
|
86591
87406
|
}
|
|
86592
87407
|
}
|
|
86593
87408
|
reconcileWorkflowAdmission(workflowId, signal) {
|
|
87409
|
+
this.ensureGeneration();
|
|
86594
87410
|
signal?.throwIfAborted();
|
|
86595
87411
|
if (!this.unavailableAdmissions.has(workflowId) && !this.unavailableCheckpoints.has(workflowId))
|
|
86596
87412
|
return Promise.resolve();
|
|
@@ -86605,11 +87421,13 @@ class DbosDurableBackend {
|
|
|
86605
87421
|
return;
|
|
86606
87422
|
if (metadata.resumable === false || metadata.status === "cancelled" || metadata.status === "completed")
|
|
86607
87423
|
return;
|
|
86608
|
-
if (!await reconcileDbosAdmission(this.sdk, this.promptReservations.metadata(workflowId, metadata), this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
87424
|
+
if (!await reconcileDbosAdmission(this.sdk, { ...this.promptReservations.metadata(workflowId, metadata), ownerExecutorId: this.executorId }, this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
86609
87425
|
await this.suppressWorkflow(workflowId);
|
|
86610
87426
|
return;
|
|
86611
87427
|
}
|
|
86612
87428
|
admissionSignal.throwIfAborted();
|
|
87429
|
+
await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
87430
|
+
admissionSignal.throwIfAborted();
|
|
86613
87431
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
86614
87432
|
if (info === undefined)
|
|
86615
87433
|
return;
|
|
@@ -86646,7 +87464,7 @@ class DbosDurableBackend {
|
|
|
86646
87464
|
return { kind: deletion === "current" ? "deleted" : "malformed" };
|
|
86647
87465
|
}
|
|
86648
87466
|
const metadata = classifyLatestMetadata(records, info.workflowId);
|
|
86649
|
-
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !
|
|
87467
|
+
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !isDeepStrictEqual3(metadata.metadata.inputs, expected.inputs))) {
|
|
86650
87468
|
await this.suppressWorkflow(info.workflowId);
|
|
86651
87469
|
return { kind: "malformed" };
|
|
86652
87470
|
}
|
|
@@ -86680,10 +87498,17 @@ class DbosDurableBackend {
|
|
|
86680
87498
|
});
|
|
86681
87499
|
const handle = this.getLoadableWorkflow(info.workflowId);
|
|
86682
87500
|
if (handle !== undefined)
|
|
86683
|
-
return
|
|
87501
|
+
return await this.withOwnerLiveness(handle);
|
|
86684
87502
|
await this.suppressWorkflow(info.workflowId);
|
|
86685
87503
|
return { kind: "malformed" };
|
|
86686
87504
|
}
|
|
87505
|
+
async withOwnerLiveness(handle) {
|
|
87506
|
+
if (this.sdk.ownerLiveness === undefined)
|
|
87507
|
+
return { kind: "current", handle };
|
|
87508
|
+
const ownerLiveness = await this.sdk.ownerLiveness(handle.ownerExecutorId);
|
|
87509
|
+
this.ownerLiveness.set(handle.ownerExecutorId ?? "", ownerLiveness);
|
|
87510
|
+
return { kind: "current", handle: { ...handle, ownerLiveness } };
|
|
87511
|
+
}
|
|
86687
87512
|
async suppressWorkflow(workflowId) {
|
|
86688
87513
|
this.invalid.add(workflowId);
|
|
86689
87514
|
this.current.delete(workflowId);
|
|
@@ -86697,10 +87522,21 @@ class DbosDurableBackend {
|
|
|
86697
87522
|
return Promise.resolve();
|
|
86698
87523
|
const previous = this.writeQueues.get(workflowId) ?? Promise.resolve();
|
|
86699
87524
|
const next = previous.then(async () => {
|
|
87525
|
+
this.ensureGeneration();
|
|
87526
|
+
const adopted = await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
87527
|
+
const local = this.mem.getWorkflow(workflowId);
|
|
87528
|
+
if (adopted !== undefined && local !== undefined)
|
|
87529
|
+
this.mem.registerWorkflow({
|
|
87530
|
+
...local,
|
|
87531
|
+
ownerExecutorId: adopted.ownerExecutorId,
|
|
87532
|
+
updatedAt: Math.max(local.updatedAt, adopted.updatedAt + 1)
|
|
87533
|
+
}, true);
|
|
86700
87534
|
if (!signal?.aborted)
|
|
86701
87535
|
await fn();
|
|
86702
87536
|
});
|
|
86703
87537
|
const tracked = next.catch((err) => {
|
|
87538
|
+
if (this.generationIsCurrent?.() === false)
|
|
87539
|
+
return;
|
|
86704
87540
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
86705
87541
|
const errors = this.writeErrors.get(workflowId) ?? [];
|
|
86706
87542
|
errors.push(error);
|
|
@@ -86737,7 +87573,7 @@ class DbosDurableBackend {
|
|
|
86737
87573
|
applyMetadata(workflowId, metadata) {
|
|
86738
87574
|
if (metadata.workflowId !== workflowId)
|
|
86739
87575
|
return;
|
|
86740
|
-
this.mem.registerWorkflow(metadata);
|
|
87576
|
+
this.mem.registerWorkflow(metadata, true);
|
|
86741
87577
|
}
|
|
86742
87578
|
}
|
|
86743
87579
|
|
|
@@ -86773,7 +87609,7 @@ import { tmpdir, uptime } from "node:os";
|
|
|
86773
87609
|
import { basename as basename2, dirname as dirname4, isAbsolute as isAbsolute3, join as join5, relative as relative2, sep as sep2 } from "node:path";
|
|
86774
87610
|
|
|
86775
87611
|
// dist/builtin/workflows/src/durable/dbos-embedded-postgres-root.ts
|
|
86776
|
-
import { createHash as
|
|
87612
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
86777
87613
|
import {
|
|
86778
87614
|
chmod,
|
|
86779
87615
|
chown,
|
|
@@ -87256,8 +88092,8 @@ async function snapshotSourceRuntime(root, publisher, progress) {
|
|
|
87256
88092
|
const rootStat = await lstat(root);
|
|
87257
88093
|
assertRuntimeRoot(root, rootStat);
|
|
87258
88094
|
const rootRealPath = await realpath(root);
|
|
87259
|
-
const sourceHash =
|
|
87260
|
-
const sealedHash =
|
|
88095
|
+
const sourceHash = createHash4("sha256");
|
|
88096
|
+
const sealedHash = createHash4("sha256");
|
|
87261
88097
|
await hashSourceEntry(sourceHash, sealedHash, root, ".", rootRealPath, publisher, progress);
|
|
87262
88098
|
return { sourceIdentity: sourceHash.digest("hex"), sealedIdentity: sealedHash.digest("hex") };
|
|
87263
88099
|
}
|
|
@@ -87522,7 +88358,7 @@ async function snapshotSealedManifest(root, progress) {
|
|
|
87522
88358
|
const rootStat = await lstat(root);
|
|
87523
88359
|
assertRuntimeRoot(root, rootStat);
|
|
87524
88360
|
const rootRealPath = await realpath(root);
|
|
87525
|
-
const hash =
|
|
88361
|
+
const hash = createHash4("sha256");
|
|
87526
88362
|
const entries = [];
|
|
87527
88363
|
await hashSealedEntry(hash, root, ".", rootRealPath, progress, entries);
|
|
87528
88364
|
return { version: 1, sealedIdentity: hash.digest("hex"), entries };
|
|
@@ -87756,6 +88592,13 @@ function resolveEmbeddedPostgresTarget(host) {
|
|
|
87756
88592
|
init_dbos_admission();
|
|
87757
88593
|
var HEALTH_INTERVAL_MS = 5000;
|
|
87758
88594
|
var RECOVERY_ATTEMPTS = 3;
|
|
88595
|
+
function isMonitoringConnectionTimeout(error) {
|
|
88596
|
+
if (!(error instanceof Error))
|
|
88597
|
+
return false;
|
|
88598
|
+
if ("code" in error)
|
|
88599
|
+
return error.code === "ETIMEDOUT";
|
|
88600
|
+
return error.message === "timeout expired" || error.message === "Connection terminated due to connection timeout";
|
|
88601
|
+
}
|
|
87759
88602
|
function isMonitoringConnectionFailure(error) {
|
|
87760
88603
|
if (!(error instanceof Error))
|
|
87761
88604
|
return false;
|
|
@@ -87776,6 +88619,7 @@ class PostgresHealth {
|
|
|
87776
88619
|
attempts = 0;
|
|
87777
88620
|
nextRecoveryAt = 0;
|
|
87778
88621
|
failure;
|
|
88622
|
+
healthySinceFailure = false;
|
|
87779
88623
|
revision = 0;
|
|
87780
88624
|
listeners = new Set;
|
|
87781
88625
|
constructor(operations) {
|
|
@@ -87784,6 +88628,18 @@ class PostgresHealth {
|
|
|
87784
88628
|
get lastFailure() {
|
|
87785
88629
|
return this.failure;
|
|
87786
88630
|
}
|
|
88631
|
+
get endpoint() {
|
|
88632
|
+
return this.operations.url?.();
|
|
88633
|
+
}
|
|
88634
|
+
retainFailure(error) {
|
|
88635
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
88636
|
+
const safe = redactedDatabaseMessage(message, this.endpoint);
|
|
88637
|
+
this.failure = new Error(safe);
|
|
88638
|
+
this.healthySinceFailure = false;
|
|
88639
|
+
}
|
|
88640
|
+
dependencyFailure(message) {
|
|
88641
|
+
return new DbosDependencyError(message, this.endpoint === undefined ? undefined : { provider: "managed", url: this.endpoint, failure: this.failure?.message });
|
|
88642
|
+
}
|
|
87787
88643
|
subscribe(listener) {
|
|
87788
88644
|
this.listeners.add(listener);
|
|
87789
88645
|
return () => this.listeners.delete(listener);
|
|
@@ -87806,12 +88662,15 @@ class PostgresHealth {
|
|
|
87806
88662
|
await this.operations.validate?.(client);
|
|
87807
88663
|
}
|
|
87808
88664
|
} catch (error) {
|
|
87809
|
-
this.
|
|
88665
|
+
this.retainFailure(error);
|
|
87810
88666
|
if (!isQueryReadTimeout(error)) {
|
|
87811
88667
|
this.invalidate();
|
|
87812
|
-
|
|
88668
|
+
if (this.endpoint === undefined)
|
|
88669
|
+
throw error;
|
|
88670
|
+
const failure = this.dependencyFailure("Managed PostgreSQL connection validation refused. Preserve its data and ownership records.");
|
|
88671
|
+
throw error instanceof DbosDependencyError ? failure : new Error(failure.message);
|
|
87813
88672
|
}
|
|
87814
|
-
throw
|
|
88673
|
+
throw this.dependencyFailure("Managed PostgreSQL did not answer a connection identity check in time.");
|
|
87815
88674
|
}
|
|
87816
88675
|
}
|
|
87817
88676
|
check() {
|
|
@@ -87850,9 +88709,9 @@ class PostgresHealth {
|
|
|
87850
88709
|
try {
|
|
87851
88710
|
identity = await this.operations.probe();
|
|
87852
88711
|
} catch (retryError) {
|
|
87853
|
-
if (!isMonitoringConnectionFailure(retryError))
|
|
88712
|
+
if (isMonitoringConnectionTimeout(retryError) || !isMonitoringConnectionFailure(retryError))
|
|
87854
88713
|
throw retryError;
|
|
87855
|
-
this.
|
|
88714
|
+
this.retainFailure(retryError);
|
|
87856
88715
|
return;
|
|
87857
88716
|
}
|
|
87858
88717
|
}
|
|
@@ -87867,6 +88726,7 @@ class PostgresHealth {
|
|
|
87867
88726
|
this.available = identity;
|
|
87868
88727
|
this.attempts = 0;
|
|
87869
88728
|
this.nextRecoveryAt = 0;
|
|
88729
|
+
this.healthySinceFailure = true;
|
|
87870
88730
|
return identity.url;
|
|
87871
88731
|
}
|
|
87872
88732
|
async refresh() {
|
|
@@ -87875,20 +88735,22 @@ class PostgresHealth {
|
|
|
87875
88735
|
if (ready !== undefined)
|
|
87876
88736
|
return ready;
|
|
87877
88737
|
} catch (error) {
|
|
87878
|
-
if (isQueryReadTimeout(error)) {
|
|
87879
|
-
this.
|
|
87880
|
-
throw this.
|
|
88738
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error)) {
|
|
88739
|
+
this.retainFailure(new Error("Managed PostgreSQL did not answer a health check in time."));
|
|
88740
|
+
throw this.dependencyFailure("Managed PostgreSQL did not answer a health check in time.");
|
|
87881
88741
|
}
|
|
87882
|
-
this.
|
|
88742
|
+
this.retainFailure(error);
|
|
87883
88743
|
if (!(error instanceof Error && ("code" in error) && error.code === "53300"))
|
|
87884
88744
|
this.invalidate();
|
|
87885
|
-
throw error;
|
|
88745
|
+
throw this.endpoint === undefined ? error : this.dependencyFailure("Managed PostgreSQL recovery refused after a health probe failure. Preserve its data and ownership records.");
|
|
88746
|
+
}
|
|
88747
|
+
if (this.attempts === 0 && (this.failure === undefined || this.healthySinceFailure)) {
|
|
88748
|
+
this.retainFailure(new Error("Managed PostgreSQL failed its live health check."));
|
|
88749
|
+
this.healthySinceFailure = false;
|
|
87886
88750
|
}
|
|
87887
|
-
if (this.attempts === 0)
|
|
87888
|
-
this.failure = new DbosDependencyError("Managed PostgreSQL failed its live health check.");
|
|
87889
88751
|
this.invalidate();
|
|
87890
88752
|
if ((this.operations.now ?? Date.now)() < this.nextRecoveryAt)
|
|
87891
|
-
throw
|
|
88753
|
+
throw this.dependencyFailure("Managed Postgres recovery is cooling down after bounded attempts.");
|
|
87892
88754
|
while (!this.stopped && this.attempts < RECOVERY_ATTEMPTS) {
|
|
87893
88755
|
const attempt = this.attempts++;
|
|
87894
88756
|
if (attempt > 0)
|
|
@@ -87901,12 +88763,12 @@ class PostgresHealth {
|
|
|
87901
88763
|
if (ready !== undefined)
|
|
87902
88764
|
return ready;
|
|
87903
88765
|
} catch (error) {
|
|
87904
|
-
this.
|
|
88766
|
+
this.retainFailure(error);
|
|
87905
88767
|
}
|
|
87906
88768
|
}
|
|
87907
88769
|
this.attempts = 0;
|
|
87908
88770
|
this.nextRecoveryAt = (this.operations.now ?? Date.now)() + HEALTH_INTERVAL_MS;
|
|
87909
|
-
throw
|
|
88771
|
+
throw this.dependencyFailure("Managed Postgres is unavailable after bounded recovery. Preserve its data and ownership records.");
|
|
87910
88772
|
}
|
|
87911
88773
|
}
|
|
87912
88774
|
|
|
@@ -88181,7 +89043,7 @@ async function probePostgresIdentity(port) {
|
|
|
88181
89043
|
const result = await client.query(POSTGRES_IDENTITY_SQL);
|
|
88182
89044
|
return result.rows[0];
|
|
88183
89045
|
} catch (error) {
|
|
88184
|
-
if (isQueryReadTimeout(error))
|
|
89046
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
88185
89047
|
throw error;
|
|
88186
89048
|
const code = error instanceof Error && "code" in error ? error.code : undefined;
|
|
88187
89049
|
if (code === "ECONNREFUSED" || code === "ECONNRESET" || code === "EPIPE" || code === "57P01" || code === "57P02" || code === "57P03" || error instanceof Error && /timeout|timed out|connection terminated/i.test(error.message))
|
|
@@ -88354,7 +89216,7 @@ async function ensureCluster(options = {}) {
|
|
|
88354
89216
|
try {
|
|
88355
89217
|
return await existingServerProbe()(probePort);
|
|
88356
89218
|
} catch (error) {
|
|
88357
|
-
if (isQueryReadTimeout(error))
|
|
89219
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
88358
89220
|
return;
|
|
88359
89221
|
throw error;
|
|
88360
89222
|
}
|
|
@@ -88562,6 +89424,8 @@ async function ensureCluster(options = {}) {
|
|
|
88562
89424
|
const server = current.server;
|
|
88563
89425
|
if (!server)
|
|
88564
89426
|
throw new Error("Managed Postgres published identity is missing.");
|
|
89427
|
+
if (managedPostmaster(current) === undefined)
|
|
89428
|
+
return;
|
|
88565
89429
|
if (!managedPostgresRuntimeHealthy(current, server.port)) {
|
|
88566
89430
|
verifyManagedPostmasterProcess(current, server);
|
|
88567
89431
|
return;
|
|
@@ -88578,6 +89442,7 @@ async function ensureCluster(options = {}) {
|
|
|
88578
89442
|
};
|
|
88579
89443
|
health = new PostgresHealth({
|
|
88580
89444
|
probe: () => inspect(),
|
|
89445
|
+
url: embeddedDbosSystemDatabaseUrl,
|
|
88581
89446
|
validate: async (client) => {
|
|
88582
89447
|
const identity = await inspect(async () => {
|
|
88583
89448
|
const query = { text: POSTGRES_IDENTITY_SQL, query_timeout: POSTGRES_HEALTH_QUERY_TIMEOUT_MS };
|
|
@@ -89665,6 +90530,9 @@ async function requireDockerSuccess(action, args) {
|
|
|
89665
90530
|
throw new Error(`Could not ${action}: ${commandFailureDetail(result)}`);
|
|
89666
90531
|
}
|
|
89667
90532
|
|
|
90533
|
+
// dist/builtin/workflows/src/durable/dbos-lifecycle.ts
|
|
90534
|
+
init_dbos_process_owner();
|
|
90535
|
+
|
|
89668
90536
|
// dist/builtin/workflows/src/durable/dbos-registration-diagnostics.ts
|
|
89669
90537
|
var cachedErrorModule;
|
|
89670
90538
|
var errorModuleLoad;
|
|
@@ -90464,6 +91332,9 @@ function createDurableAdmissionSettlement(input, signal) {
|
|
|
90464
91332
|
};
|
|
90465
91333
|
}
|
|
90466
91334
|
|
|
91335
|
+
// dist/builtin/workflows/src/runs/background/quit.ts
|
|
91336
|
+
init_dbos_admission();
|
|
91337
|
+
|
|
90467
91338
|
// dist/builtin/workflows/src/shared/resume-continuation.ts
|
|
90468
91339
|
var RESUME_CONTINUATION_PROMPT = "Continue where you left off. If you believe you are finished with your original task (or a redefined task if the user told you), stop.";
|
|
90469
91340
|
|
|
@@ -91850,7 +92721,8 @@ async function quitRunWithAction(runId, opts, action) {
|
|
|
91850
92721
|
if (await settle() === "refused")
|
|
91851
92722
|
return { ok: false, runId, reason: "already_ended" };
|
|
91852
92723
|
} catch (error) {
|
|
91853
|
-
|
|
92724
|
+
const alreadyDependencyPaused = isDbosDependencyError(error) && current.status === "paused" && current.phase === "blocked_dependency";
|
|
92725
|
+
if (!suspendedByAbort && !alreadyDependencyPaused)
|
|
91854
92726
|
throw error;
|
|
91855
92727
|
const preservedProgress = hasDurableQuitProgress(runId);
|
|
91856
92728
|
publish(preservedProgress);
|
|
@@ -94385,6 +95257,9 @@ function registerWorkflowLifecycleHandlers(pi, deps) {
|
|
|
94385
95257
|
});
|
|
94386
95258
|
}
|
|
94387
95259
|
|
|
95260
|
+
// dist/builtin/workflows/src/extension/extension-runtime-state.ts
|
|
95261
|
+
init_dbos_process_owner();
|
|
95262
|
+
|
|
94388
95263
|
// dist/builtin/workflows/src/durable/workflow-heartbeat-anchor.ts
|
|
94389
95264
|
var WORKFLOW_HEARTBEAT_ANCHOR_CHECKPOINT_NAME = "workflow-heartbeat-anchor";
|
|
94390
95265
|
function asPersistedAnchorOutput(output) {
|
|
@@ -101185,7 +102060,7 @@ function createRegistry(initial = []) {
|
|
|
101185
102060
|
|
|
101186
102061
|
// dist/builtin/workflows/src/extension/discovery-loaders.ts
|
|
101187
102062
|
import { readdir as readdir3, stat as stat2 } from "node:fs/promises";
|
|
101188
|
-
import { extname, isAbsolute as isAbsolute14, join as join30, resolve as
|
|
102063
|
+
import { extname, isAbsolute as isAbsolute14, join as join30, resolve as resolve15 } from "node:path";
|
|
101189
102064
|
|
|
101190
102065
|
// ../../node_modules/jiti/lib/jiti-static.mjs
|
|
101191
102066
|
var import_jiti = __toESM(require_jiti(), 1);
|
|
@@ -101640,12 +102515,12 @@ function raceAbort2(promise, signal) {
|
|
|
101640
102515
|
});
|
|
101641
102516
|
}
|
|
101642
102517
|
// dist/builtin/workflows/src/runs/foreground/executor-hil.ts
|
|
101643
|
-
import { createHash as
|
|
102518
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
101644
102519
|
import { createAskUserQuestionToolDefinition } from "@bastani/atomic";
|
|
101645
102520
|
|
|
101646
102521
|
// dist/builtin/workflows/src/shared/prompt-callsite-context.ts
|
|
101647
|
-
import { AsyncLocalStorage as
|
|
101648
|
-
var promptCallerStacks = new
|
|
102522
|
+
import { AsyncLocalStorage as AsyncLocalStorage4 } from "node:async_hooks";
|
|
102523
|
+
var promptCallerStacks = new AsyncLocalStorage4;
|
|
101649
102524
|
function withPromptCallerStack(stack, delegate) {
|
|
101650
102525
|
if (stack === undefined)
|
|
101651
102526
|
return delegate();
|
|
@@ -101939,7 +102814,7 @@ function makePrompt(descriptor) {
|
|
|
101939
102814
|
};
|
|
101940
102815
|
}
|
|
101941
102816
|
function stableHash(value2) {
|
|
101942
|
-
return
|
|
102817
|
+
return createHash5("sha256").update(JSON.stringify(value2)).digest("hex").slice(0, 32);
|
|
101943
102818
|
}
|
|
101944
102819
|
function promptDescriptorHash(descriptor) {
|
|
101945
102820
|
if (isCustomPromptDescriptor(descriptor)) {
|
|
@@ -102532,6 +103407,7 @@ function hasInvocationData(checkpoints, replayKey) {
|
|
|
102532
103407
|
|
|
102533
103408
|
// dist/builtin/workflows/src/engine/run.ts
|
|
102534
103409
|
init_dbos_admission();
|
|
103410
|
+
init_dbos_process_owner();
|
|
102535
103411
|
|
|
102536
103412
|
// dist/builtin/workflows/src/runs/foreground/executor-direct-helpers.ts
|
|
102537
103413
|
import { isAbsolute as isAbsolute12, join as join28, resolve as resolve12 } from "node:path";
|
|
@@ -106792,6 +107668,16 @@ function workflowStartupFailureMessage(admission, snapshotError, fallback) {
|
|
|
106792
107668
|
// dist/builtin/workflows/src/durable/resume-runtime.ts
|
|
106793
107669
|
init_dbos_admission();
|
|
106794
107670
|
|
|
107671
|
+
// dist/builtin/workflows/src/durable/sdk-recovery-scope.ts
|
|
107672
|
+
init_dbos_owner_fence();
|
|
107673
|
+
import { resolve as resolve13 } from "node:path";
|
|
107674
|
+
function isSdkDurableRunInScope(handle, cwd) {
|
|
107675
|
+
return cwd !== undefined && handle.invocationCwd !== undefined && resolve13(cwd) === resolve13(handle.invocationCwd) && handle.modelOwner !== undefined && (handle.rootWorkflowId === undefined || handle.rootWorkflowId === handle.workflowId);
|
|
107676
|
+
}
|
|
107677
|
+
function isSdkCrashedRunAdoptable(handle, cwd) {
|
|
107678
|
+
return isSdkDurableRunInScope(handle, cwd) && handle.status === "running" && handle.ownerExecutorId !== undefined && handle.ownerExecutorId !== getAtomicExecutorId() && !isLiveRunningWorkflow(handle) && isDatabaseExecutor(handle.ownerExecutorId) && handle.ownerLiveness === "dead";
|
|
107679
|
+
}
|
|
107680
|
+
|
|
106795
107681
|
// dist/builtin/workflows/src/durable/tool-resume-frontier.ts
|
|
106796
107682
|
function resolveToolResumeFrontier(source, backend) {
|
|
106797
107683
|
const fail = (detail) => ({
|
|
@@ -106975,6 +107861,11 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
106975
107861
|
message: `Workflow ${resolved.workflowId} is ${handle.status}, not resumable.`
|
|
106976
107862
|
};
|
|
106977
107863
|
}
|
|
107864
|
+
const requestedOwner = deps.baseRunOpts.modelOwner;
|
|
107865
|
+
const adopting = requestedOwner !== undefined && requestedOwner !== handle.modelOwner && (handle.modelOwner !== undefined || handle.origin === "agent");
|
|
107866
|
+
if (adopting && !isSdkCrashedRunAdoptable(handle, deps.baseRunOpts.cwd)) {
|
|
107867
|
+
return { ok: false, reason: "owned_elsewhere", message: "Workflow instance belongs to another caller/session." };
|
|
107868
|
+
}
|
|
106978
107869
|
const def = handle.invocationCwd === undefined ? deps.registry.get(handle.name) : await deps.resolveDefinition?.(handle.name, handle.invocationCwd) ?? deps.registry.get(handle.name);
|
|
106979
107870
|
if (def === undefined) {
|
|
106980
107871
|
return { ok: false, reason: "workflow_not_found", message: `Workflow definition not found: ${handle.name}` };
|
|
@@ -107024,6 +107915,22 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107024
107915
|
failedToolNodeId: handle.failedToolNodeId
|
|
107025
107916
|
};
|
|
107026
107917
|
const sourceSnapshot = toolContinuation?.source ?? deps.baseRunOpts.store?.runs().find((run) => run.id === resolved.workflowId);
|
|
107918
|
+
let adoptedClaimUpdatedAt;
|
|
107919
|
+
const rollbackStartup = async (onRestored) => {
|
|
107920
|
+
if (adopting) {
|
|
107921
|
+
const current = backend.getWorkflow(resolved.workflowId);
|
|
107922
|
+
if (adoptedClaimUpdatedAt === undefined || current?.modelOwner !== requestedOwner)
|
|
107923
|
+
return false;
|
|
107924
|
+
const restored = await backend.transitionWorkflowStatus(resolved.workflowId, ["running"], handle.status, handle.pendingPrompts, handle.resumable, adoptedClaimUpdatedAt);
|
|
107925
|
+
if (restored)
|
|
107926
|
+
onRestored?.();
|
|
107927
|
+
return restored;
|
|
107928
|
+
}
|
|
107929
|
+
backend.setWorkflowStatus(resolved.workflowId, handle.status, handle.pendingPrompts, handle.resumable, sourceFailure);
|
|
107930
|
+
await backend.flush(resolved.workflowId);
|
|
107931
|
+
onRestored?.();
|
|
107932
|
+
return true;
|
|
107933
|
+
};
|
|
107027
107934
|
let claimed;
|
|
107028
107935
|
try {
|
|
107029
107936
|
claimed = await boundedAdmission((signal) => dbosAdmissionContext.run(signal, async () => {
|
|
@@ -107034,10 +107941,14 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107034
107941
|
}
|
|
107035
107942
|
unsettled.add(resolved.workflowId);
|
|
107036
107943
|
try {
|
|
107037
|
-
const accepted = await backend.transitionWorkflowStatus(resolved.workflowId, [handle.status], "running", undefined, undefined, resolved.updatedAt);
|
|
107038
|
-
if (accepted &&
|
|
107039
|
-
backend.
|
|
107040
|
-
|
|
107944
|
+
const accepted = await backend.transitionWorkflowStatus(resolved.workflowId, [handle.status], "running", undefined, undefined, resolved.updatedAt, adopting ? requestedOwner : undefined);
|
|
107945
|
+
if (accepted && adopting) {
|
|
107946
|
+
const current = backend.getWorkflow(resolved.workflowId);
|
|
107947
|
+
if (current?.status === "running" && current.modelOwner === requestedOwner)
|
|
107948
|
+
adoptedClaimUpdatedAt = current.updatedAt;
|
|
107949
|
+
}
|
|
107950
|
+
if (accepted && !adopting && signal.aborted && backend.getWorkflow(resolved.workflowId)?.status === "running") {
|
|
107951
|
+
await rollbackStartup();
|
|
107041
107952
|
}
|
|
107042
107953
|
return accepted;
|
|
107043
107954
|
} finally {
|
|
@@ -107064,7 +107975,7 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107064
107975
|
...deps.baseRunOpts,
|
|
107065
107976
|
...handle.invocationCwd !== undefined ? { cwd: handle.invocationCwd } : {},
|
|
107066
107977
|
...handle.origin !== undefined ? { origin: handle.origin } : {},
|
|
107067
|
-
modelOwner: handle.modelOwner,
|
|
107978
|
+
modelOwner: adopting ? requestedOwner : handle.modelOwner,
|
|
107068
107979
|
runId: resolved.workflowId,
|
|
107069
107980
|
durableBackend: backend,
|
|
107070
107981
|
...toolContinuation === undefined ? {} : { continuation: toolContinuation }
|
|
@@ -107090,10 +108001,10 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107090
108001
|
}
|
|
107091
108002
|
});
|
|
107092
108003
|
} catch (error) {
|
|
107093
|
-
|
|
107094
|
-
|
|
107095
|
-
|
|
107096
|
-
|
|
108004
|
+
await rollbackStartup(() => {
|
|
108005
|
+
if (sourceSnapshot !== undefined)
|
|
108006
|
+
deps.baseRunOpts.store?.recordRunStart(sourceSnapshot);
|
|
108007
|
+
});
|
|
107097
108008
|
return {
|
|
107098
108009
|
ok: false,
|
|
107099
108010
|
reason: "startup_failed",
|
|
@@ -107107,12 +108018,14 @@ async function resumeDurableWorkflowClaimed(workflowId, deps, catalog) {
|
|
|
107107
108018
|
const snapshot = deps.baseRunOpts.store?.runs().find((run) => run.id === accepted.runId);
|
|
107108
108019
|
const error = workflowStartupFailureMessage(admission, snapshot?.error, `Workflow ${resolved.workflowId} ended before startup admission`);
|
|
107109
108020
|
if (!backend.isAdmissionUnavailable?.(resolved.workflowId)) {
|
|
107110
|
-
if (sourceSnapshot === undefined)
|
|
108021
|
+
if (!adopting && sourceSnapshot === undefined)
|
|
107111
108022
|
deps.baseRunOpts.store?.removeRun(accepted.runId);
|
|
107112
|
-
|
|
107113
|
-
|
|
107114
|
-
|
|
107115
|
-
|
|
108023
|
+
await rollbackStartup(() => {
|
|
108024
|
+
if (adopting && sourceSnapshot === undefined)
|
|
108025
|
+
deps.baseRunOpts.store?.removeRun(accepted.runId);
|
|
108026
|
+
if (sourceSnapshot !== undefined)
|
|
108027
|
+
deps.baseRunOpts.store?.recordRunStart(sourceSnapshot);
|
|
108028
|
+
});
|
|
107116
108029
|
}
|
|
107117
108030
|
return {
|
|
107118
108031
|
ok: false,
|
|
@@ -112518,9 +113431,9 @@ class StageSessionController {
|
|
|
112518
113431
|
}
|
|
112519
113432
|
|
|
112520
113433
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
112521
|
-
import { createHash as
|
|
113434
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
112522
113435
|
import { mkdir as mkdir7, writeFile as writeFile11 } from "node:fs/promises";
|
|
112523
|
-
import { basename as basename6, dirname as dirname13, isAbsolute as isAbsolute13, join as join29, resolve as
|
|
113436
|
+
import { basename as basename6, dirname as dirname13, isAbsolute as isAbsolute13, join as join29, resolve as resolve14 } from "node:path";
|
|
112524
113437
|
var DEFAULT_MAX_OUTPUT_BYTES = 200 * 1024;
|
|
112525
113438
|
var DEFAULT_MAX_OUTPUT_LINES = 5000;
|
|
112526
113439
|
function normalizeMaxOutput2(maxOutput) {
|
|
@@ -112584,7 +113497,7 @@ function formatByteSize(bytes) {
|
|
|
112584
113497
|
return `${value2.toFixed(1)} ${units[unitIndex]}`;
|
|
112585
113498
|
}
|
|
112586
113499
|
function transcriptPath(runId, outputPath) {
|
|
112587
|
-
const digest =
|
|
113500
|
+
const digest = createHash6("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
112588
113501
|
return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename6(outputPath)}.transcript.md`);
|
|
112589
113502
|
}
|
|
112590
113503
|
function appendContent(lines, content) {
|
|
@@ -112630,7 +113543,7 @@ function renderTranscript(messages, fallbackOutput) {
|
|
|
112630
113543
|
`);
|
|
112631
113544
|
}
|
|
112632
113545
|
function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
|
|
112633
|
-
const absolutePath =
|
|
113546
|
+
const absolutePath = resolve14(outputPath);
|
|
112634
113547
|
const bytes = Buffer.byteLength(fullOutput, "utf8");
|
|
112635
113548
|
const lines = countLines(fullOutput);
|
|
112636
113549
|
const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
|
|
@@ -112645,8 +113558,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
|
|
|
112645
113558
|
return;
|
|
112646
113559
|
if (isAbsolute13(output))
|
|
112647
113560
|
return output;
|
|
112648
|
-
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute13(requestedCwd) ? requestedCwd :
|
|
112649
|
-
return
|
|
113561
|
+
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute13(requestedCwd) ? requestedCwd : resolve14(runtimeCwd, requestedCwd);
|
|
113562
|
+
return resolve14(baseCwd, output);
|
|
112650
113563
|
}
|
|
112651
113564
|
function degenerateWarning(fullOutput) {
|
|
112652
113565
|
return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
|
|
@@ -113686,7 +114599,8 @@ async function run(def, inputs, opts = {}) {
|
|
|
113686
114599
|
ownController.abort(callerSignal?.reason);
|
|
113687
114600
|
};
|
|
113688
114601
|
const exit = createWorkflowExitManager({ runId, exitScope, controller: ownController });
|
|
113689
|
-
const
|
|
114602
|
+
const sourceBackend = opts.durableBackend ?? getDurableBackend();
|
|
114603
|
+
const backendView = sourceBackend.executionView?.() ?? sourceBackend;
|
|
113690
114604
|
const rootBackend = opts.durableRootBackend ?? backendView;
|
|
113691
114605
|
const durableBackend = opts.durableScope !== undefined ? new ScopedDurableBackend(backendView, opts.durableScope) : backendView;
|
|
113692
114606
|
const inheritedElapsedMs = opts.parentRun === undefined ? inheritedRunElapsedMs({ backend: durableBackend, runId, continuationSource: opts.continuation?.source }) : undefined;
|
|
@@ -114202,7 +115116,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
114202
115116
|
if (control === latestControl && !ownController.signal.aborted && isDbosDependencyError(error)) {
|
|
114203
115117
|
activeStore.recordRunExecutionState(runId, {
|
|
114204
115118
|
phase: "blocked_dependency",
|
|
114205
|
-
dependencyError: `Workflow database unavailable during ${status === "paused" ? "pause" : "resume"}; persistence not confirmed
|
|
115119
|
+
dependencyError: `Workflow database unavailable during ${status === "paused" ? "pause" : "resume"}; persistence not confirmed. ${error.message}`
|
|
114206
115120
|
});
|
|
114207
115121
|
if (status === "paused")
|
|
114208
115122
|
return;
|
|
@@ -114249,7 +115163,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
114249
115163
|
if (!admission.admitting)
|
|
114250
115164
|
return pausePersistence;
|
|
114251
115165
|
backgroundAdmissionControl(durableBackend, runId, pausePersistence, (error, resumable) => {
|
|
114252
|
-
admissionControlError = unknownErrorMessage(error);
|
|
115166
|
+
admissionControlError = databaseDependencyMessage(unknownErrorMessage(error), getDbosProcessOwner().databaseDiagnostics?.());
|
|
114253
115167
|
runSnapshot.error = admissionControlError;
|
|
114254
115168
|
activeStore.recordRunPaused(runId, undefined, { resumable });
|
|
114255
115169
|
return runSnapshot.error;
|
|
@@ -114266,7 +115180,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
114266
115180
|
} catch (error) {
|
|
114267
115181
|
if (admission.failed) {
|
|
114268
115182
|
scheduler.releaseRun();
|
|
114269
|
-
throw error;
|
|
115183
|
+
throw new Error(admissionControlError ?? databaseDependencyMessage(unknownErrorMessage(error), getDbosProcessOwner().databaseDiagnostics?.()));
|
|
114270
115184
|
}
|
|
114271
115185
|
}
|
|
114272
115186
|
if (generation !== pauseGeneration)
|
|
@@ -114339,12 +115253,21 @@ async function run(def, inputs, opts = {}) {
|
|
|
114339
115253
|
}).catch((error) => {
|
|
114340
115254
|
const isTopologyError = typeof error === "object" && error !== null && "name" in error && error.name === "DurableNestedTopologyError";
|
|
114341
115255
|
if (!isDbosDependencyError(error) && !isTopologyError) {
|
|
114342
|
-
|
|
114343
|
-
|
|
115256
|
+
const admissionDetail = unknownErrorMessage(error);
|
|
115257
|
+
const diagnostics = getDbosProcessOwner().databaseDiagnostics?.();
|
|
115258
|
+
const safeMessage = databaseDependencyMessage(admissionDetail, diagnostics);
|
|
115259
|
+
activeStore.recordRunExecutionState(runId, {
|
|
115260
|
+
phase: "blocked_dependency",
|
|
115261
|
+
dependencyError: safeMessage
|
|
115262
|
+
});
|
|
115263
|
+
const safeError = new Error(safeMessage);
|
|
115264
|
+
durableAdmissionFailure = { error: safeError };
|
|
115265
|
+
classifiedFailures.set(safeError, classifyWorkflowFailure({
|
|
114344
115266
|
code: "ATOMIC_DURABLE_ADMISSION_REJECTED",
|
|
114345
|
-
message:
|
|
114346
|
-
cause:
|
|
115267
|
+
message: safeMessage,
|
|
115268
|
+
cause: safeError
|
|
114347
115269
|
}));
|
|
115270
|
+
throw safeError;
|
|
114348
115271
|
}
|
|
114349
115272
|
throw error;
|
|
114350
115273
|
}), ownController.signal);
|
|
@@ -114437,6 +115360,17 @@ async function run(def, inputs, opts = {}) {
|
|
|
114437
115360
|
const gracefulQuit = observedQuitCancellation() ?? observedTaskTailQuit ?? findWorkflowGracefulQuit(err) ?? findWorkflowGracefulQuit(ownController.signal.reason);
|
|
114438
115361
|
if (gracefulQuit !== undefined)
|
|
114439
115362
|
return suspendForGracefulQuit(gracefulQuit);
|
|
115363
|
+
if (isDbosDependencyError(err) && durableBackend.isCheckpointUnavailable?.(runId) && (durableBackend.getWorkflow(runId)?.completedCheckpoints ?? 0) > 0) {
|
|
115364
|
+
await admittedTools.closeAndDrain();
|
|
115365
|
+
activeStore.recordRunPaused(runId, undefined, { resumable: true });
|
|
115366
|
+
activeStore.recordRunExecutionState(runId, { phase: "blocked_dependency", dependencyError: err.message });
|
|
115367
|
+
return {
|
|
115368
|
+
runId,
|
|
115369
|
+
status: "paused",
|
|
115370
|
+
stages: [...runSnapshot.stages],
|
|
115371
|
+
toolNodes: [...runSnapshot.toolNodes ?? []]
|
|
115372
|
+
};
|
|
115373
|
+
}
|
|
114440
115374
|
await admittedTools.closeAndDrain();
|
|
114441
115375
|
if (ownController.signal.aborted)
|
|
114442
115376
|
await Promise.allSettled([...pendingChildWorkflows]);
|
|
@@ -114775,7 +115709,7 @@ async function loadFromPaths(pathsOrMap, kind, baseCwd, diagnostics) {
|
|
|
114775
115709
|
const all = [];
|
|
114776
115710
|
const entries = Array.isArray(pathsOrMap) ? pathsOrMap.map((p) => ({ rawPath: p })) : Object.entries(pathsOrMap).map(([name, p]) => ({ rawPath: p, configuredName: name }));
|
|
114777
115711
|
for (const { rawPath, configuredName } of entries) {
|
|
114778
|
-
const absPath = isAbsolute14(rawPath) ? rawPath :
|
|
115712
|
+
const absPath = isAbsolute14(rawPath) ? rawPath : resolve15(baseCwd, rawPath);
|
|
114779
115713
|
let pathStats;
|
|
114780
115714
|
try {
|
|
114781
115715
|
pathStats = await stat2(absPath);
|
|
@@ -115548,7 +116482,8 @@ function createDurableResumeRuntime(deps) {
|
|
|
115548
116482
|
baseRunOpts: {
|
|
115549
116483
|
...deps.baseRunOpts(options?.policy),
|
|
115550
116484
|
...options?.budget === undefined ? {} : { budget: options.budget },
|
|
115551
|
-
...options?.actor === undefined ? {} : { resumeActor: options.actor }
|
|
116485
|
+
...options?.actor === undefined ? {} : { resumeActor: options.actor },
|
|
116486
|
+
...options?.modelOwner === undefined ? {} : { modelOwner: options.modelOwner }
|
|
115552
116487
|
},
|
|
115553
116488
|
durableBackend: backend,
|
|
115554
116489
|
resolveDefinition: async (name, cwd) => (await discoverWorkflows({ cwd: cwd ?? deps.runtimeCwd })).registry.get(name),
|
|
@@ -115654,7 +116589,11 @@ function createExtensionRuntime(opts = {}) {
|
|
|
115654
116589
|
const resolvePossibleStageEntry = opts.resolvePossibleStageEntry;
|
|
115655
116590
|
const beforeRestoreCompleted = opts.beforeRestoreCompleted;
|
|
115656
116591
|
const ensureDbosReady = async () => {
|
|
115657
|
-
|
|
116592
|
+
if (opts.prepareDatabase !== undefined)
|
|
116593
|
+
await opts.prepareDatabase();
|
|
116594
|
+
const backend = await initializeDurableBackend(durabilityWarningSink);
|
|
116595
|
+
opts.databaseReady?.();
|
|
116596
|
+
return backend;
|
|
115658
116597
|
};
|
|
115659
116598
|
function runOptions(policy) {
|
|
115660
116599
|
const defaultSessionDir = resolveDefaultStageSessionDir?.();
|
|
@@ -115925,6 +116864,95 @@ function createStatusWriter(store, config, opts = {}) {
|
|
|
115925
116864
|
};
|
|
115926
116865
|
}
|
|
115927
116866
|
|
|
116867
|
+
// dist/builtin/workflows/src/extension/workflow-database-settings.ts
|
|
116868
|
+
import { readFileSync as readFileSync7 } from "node:fs";
|
|
116869
|
+
import { readFile as readFile6 } from "node:fs/promises";
|
|
116870
|
+
import { homedir as homedir3 } from "node:os";
|
|
116871
|
+
import { join as join33, resolve as resolve16 } from "node:path";
|
|
116872
|
+
import { getAgentDir as getAgentDir3, getAgentDirs as getAgentDirs3, getProjectConfigDirs, SettingsManager } from "@bastani/atomic";
|
|
116873
|
+
var DATABASE_SETTING_KEYS = new Set(["systemDatabaseUrl", "systemDatabaseUrlFile"]);
|
|
116874
|
+
function databaseSettings(settings) {
|
|
116875
|
+
const workflows = settings.workflows;
|
|
116876
|
+
if (workflows === undefined || workflows === null || Array.isArray(workflows))
|
|
116877
|
+
return;
|
|
116878
|
+
if (typeof workflows !== "object")
|
|
116879
|
+
throw new TypeError("workflows must be an array of paths or an object");
|
|
116880
|
+
return workflows.durability;
|
|
116881
|
+
}
|
|
116882
|
+
function readSettingsFile(directory) {
|
|
116883
|
+
let parsed;
|
|
116884
|
+
try {
|
|
116885
|
+
parsed = JSON.parse(readFileSync7(join33(directory, "settings.json"), "utf8").replace(/^\uFEFF/, ""));
|
|
116886
|
+
} catch {
|
|
116887
|
+
return;
|
|
116888
|
+
}
|
|
116889
|
+
return parsed !== null && typeof parsed === "object" ? parsed : undefined;
|
|
116890
|
+
}
|
|
116891
|
+
function settingsDirectoryDeclaringDatabase(directories) {
|
|
116892
|
+
let declaring;
|
|
116893
|
+
for (const directory of directories) {
|
|
116894
|
+
const settings = readSettingsFile(directory);
|
|
116895
|
+
if (settings !== undefined && databaseSettings(settings) !== undefined)
|
|
116896
|
+
declaring ??= directory;
|
|
116897
|
+
}
|
|
116898
|
+
return declaring ?? directories[0];
|
|
116899
|
+
}
|
|
116900
|
+
async function resolveWorkflowDatabaseSettings(global2, project, directories) {
|
|
116901
|
+
const projectDatabase = databaseSettings(project);
|
|
116902
|
+
const database = projectDatabase !== undefined ? projectDatabase : databaseSettings(global2);
|
|
116903
|
+
if (database === undefined)
|
|
116904
|
+
return;
|
|
116905
|
+
if (database === null || typeof database !== "object" || Array.isArray(database)) {
|
|
116906
|
+
throw new TypeError("workflows.durability must be an object");
|
|
116907
|
+
}
|
|
116908
|
+
if (Object.keys(database).some((key) => !DATABASE_SETTING_KEYS.has(key))) {
|
|
116909
|
+
throw new TypeError("workflows.durability supports only systemDatabaseUrl or systemDatabaseUrlFile");
|
|
116910
|
+
}
|
|
116911
|
+
const { systemDatabaseUrl: url, systemDatabaseUrlFile: file } = database;
|
|
116912
|
+
if (url !== undefined && file !== undefined) {
|
|
116913
|
+
throw new TypeError("Set only one of workflows.durability.systemDatabaseUrl and systemDatabaseUrlFile");
|
|
116914
|
+
}
|
|
116915
|
+
let value2 = url;
|
|
116916
|
+
if (file !== undefined) {
|
|
116917
|
+
if (typeof file !== "string" || file.trim() === "") {
|
|
116918
|
+
throw new TypeError("workflows.durability.systemDatabaseUrlFile must be a non-empty path");
|
|
116919
|
+
}
|
|
116920
|
+
const path = file.startsWith("~/") || file.startsWith("~\\") ? join33(homedir3(), file.slice(2)) : resolve16(projectDatabase === undefined ? directories.global : directories.project, file);
|
|
116921
|
+
try {
|
|
116922
|
+
value2 = await readFile6(path, "utf8");
|
|
116923
|
+
} catch {
|
|
116924
|
+
throw new Error("Cannot read workflows.durability.systemDatabaseUrlFile; check its path and permissions");
|
|
116925
|
+
}
|
|
116926
|
+
}
|
|
116927
|
+
if (value2 === undefined)
|
|
116928
|
+
return;
|
|
116929
|
+
if (typeof value2 !== "string" || value2.trim() === "") {
|
|
116930
|
+
throw new TypeError("Workflow database selection must contain a non-empty Postgres URL");
|
|
116931
|
+
}
|
|
116932
|
+
try {
|
|
116933
|
+
const parsed = new URL(value2.trim());
|
|
116934
|
+
if (!["postgres:", "postgresql:"].includes(parsed.protocol) || parsed.hostname === "")
|
|
116935
|
+
throw new Error;
|
|
116936
|
+
} catch {
|
|
116937
|
+
throw new TypeError("Workflow database selection must contain a valid Postgres URL");
|
|
116938
|
+
}
|
|
116939
|
+
return value2.trim();
|
|
116940
|
+
}
|
|
116941
|
+
async function prepareWorkflowDatabaseSettings(cwd, projectTrusted) {
|
|
116942
|
+
if (process.env.DBOS_SYSTEM_DATABASE_URL?.trim())
|
|
116943
|
+
return;
|
|
116944
|
+
const manager = SettingsManager.create(cwd, getAgentDir3(), { projectTrusted });
|
|
116945
|
+
if (manager.drainErrors().length > 0)
|
|
116946
|
+
throw new Error("Cannot read workflow database settings; repair settings.json");
|
|
116947
|
+
const projectDirectories = getProjectConfigDirs(cwd);
|
|
116948
|
+
const url = await resolveWorkflowDatabaseSettings(manager.getGlobalSettings(), manager.getProjectSettings(), {
|
|
116949
|
+
global: settingsDirectoryDeclaringDatabase(getAgentDirs3()),
|
|
116950
|
+
project: projectTrusted ? settingsDirectoryDeclaringDatabase(projectDirectories) : projectDirectories[0]
|
|
116951
|
+
});
|
|
116952
|
+
if (url !== undefined)
|
|
116953
|
+
requestDbosSystemDatabaseUrl(url);
|
|
116954
|
+
}
|
|
116955
|
+
|
|
115928
116956
|
// dist/builtin/workflows/src/extension/workflow-heartbeat-notice.ts
|
|
115929
116957
|
var WORKFLOW_HEARTBEAT_TITLE = "WORKFLOW HEARTBEAT";
|
|
115930
116958
|
var WORKFLOW_HEARTBEAT_GLYPH = "♥";
|
|
@@ -117076,6 +118104,45 @@ function createWorkflowExtensionRuntimeState(pi, adapters, options = {}) {
|
|
|
117076
118104
|
const resolveDefaultStageSessionDir = () => hostStageSessionDir.current;
|
|
117077
118105
|
const discoveryRef = { current: null };
|
|
117078
118106
|
const resolvePossibleStageEntry = (normalizedName) => discoveryRef.current?.sources.find((source) => source.id === normalizedName)?.filePath ?? resolveBuiltinDefinitionSource(normalizedName);
|
|
118107
|
+
let databasePreparation;
|
|
118108
|
+
const prepareDatabase = () => {
|
|
118109
|
+
const cwd = resolveCwd();
|
|
118110
|
+
const trusted = pi.getResourceLoaderInheritanceSnapshot?.()?.projectTrusted ?? false;
|
|
118111
|
+
const key = JSON.stringify([cwd, trusted]);
|
|
118112
|
+
if (databasePreparation?.key === key)
|
|
118113
|
+
return databasePreparation.promise;
|
|
118114
|
+
const promise = prepareWorkflowDatabaseSettings(cwd, trusted).catch((error) => {
|
|
118115
|
+
if (databasePreparation?.promise === promise)
|
|
118116
|
+
databasePreparation = undefined;
|
|
118117
|
+
throw error;
|
|
118118
|
+
});
|
|
118119
|
+
databasePreparation = { key, promise };
|
|
118120
|
+
return promise;
|
|
118121
|
+
};
|
|
118122
|
+
const databaseReady = (ctx) => {
|
|
118123
|
+
const owner = getDbosProcessOwner();
|
|
118124
|
+
const describeDatabase = owner.databaseDiagnostics;
|
|
118125
|
+
if (owner.state !== "ready" || describeDatabase === undefined)
|
|
118126
|
+
return;
|
|
118127
|
+
if (owner.announcedDatabaseDiagnostics === describeDatabase)
|
|
118128
|
+
return;
|
|
118129
|
+
const diagnostics = describeDatabase();
|
|
118130
|
+
const dockerFallback = diagnostics.provider === "docker";
|
|
118131
|
+
let host = "unknown host";
|
|
118132
|
+
try {
|
|
118133
|
+
host = new URL(diagnostics.url).host;
|
|
118134
|
+
} catch {}
|
|
118135
|
+
const message = `Workflow database: ${dockerFallback ? "Docker fallback" : diagnostics.provider} (${host})`;
|
|
118136
|
+
if (ctx?.hasUI && ctx.ui?.notify) {
|
|
118137
|
+
try {
|
|
118138
|
+
ctx.ui.notify(message, "info");
|
|
118139
|
+
owner.announcedDatabaseDiagnostics = describeDatabase;
|
|
118140
|
+
} catch {}
|
|
118141
|
+
} else if (dockerFallback) {
|
|
118142
|
+
owner.announcedDatabaseDiagnostics = describeDatabase;
|
|
118143
|
+
console.error(message);
|
|
118144
|
+
}
|
|
118145
|
+
};
|
|
117079
118146
|
const startupDiscovery = discoverStartupWorkflowsSync();
|
|
117080
118147
|
const runtimeRef = {
|
|
117081
118148
|
current: createExtensionRuntime({
|
|
@@ -117089,7 +118156,9 @@ function createWorkflowExtensionRuntimeState(pi, adapters, options = {}) {
|
|
|
117089
118156
|
config: runtimeConfigRef.current,
|
|
117090
118157
|
resolvePossibleStageEntry,
|
|
117091
118158
|
resolveDefaultStageSessionDir,
|
|
117092
|
-
beforeRestoreCompleted
|
|
118159
|
+
beforeRestoreCompleted,
|
|
118160
|
+
prepareDatabase,
|
|
118161
|
+
databaseReady
|
|
117093
118162
|
})
|
|
117094
118163
|
};
|
|
117095
118164
|
const configLoadRef = { current: null };
|
|
@@ -117154,7 +118223,9 @@ function createWorkflowExtensionRuntimeState(pi, adapters, options = {}) {
|
|
|
117154
118223
|
...models === undefined ? {} : { models },
|
|
117155
118224
|
...durabilityWarningSink === undefined ? {} : { durabilityWarningSink },
|
|
117156
118225
|
resolveDefaultStageSessionDir,
|
|
117157
|
-
beforeRestoreCompleted
|
|
118226
|
+
beforeRestoreCompleted,
|
|
118227
|
+
prepareDatabase,
|
|
118228
|
+
databaseReady: () => databaseReady(ctx)
|
|
117158
118229
|
});
|
|
117159
118230
|
}
|
|
117160
118231
|
let lazyDiscoveryPromise = null;
|
|
@@ -117193,7 +118264,9 @@ function createWorkflowExtensionRuntimeState(pi, adapters, options = {}) {
|
|
|
117193
118264
|
config: runtimeConfigRef.current,
|
|
117194
118265
|
resolvePossibleStageEntry,
|
|
117195
118266
|
resolveDefaultStageSessionDir,
|
|
117196
|
-
beforeRestoreCompleted
|
|
118267
|
+
beforeRestoreCompleted,
|
|
118268
|
+
prepareDatabase,
|
|
118269
|
+
databaseReady
|
|
117197
118270
|
});
|
|
117198
118271
|
}
|
|
117199
118272
|
function isWorkflowDiscoveryCurrent(generation) {
|
|
@@ -120794,7 +121867,7 @@ function createSessionRunControl(host) {
|
|
|
120794
121867
|
const ctx = executeContext(host.context());
|
|
120795
121868
|
if (args.runId !== undefined && (args.action === "status" || args.action === "stages"))
|
|
120796
121869
|
rejectUnknownRunPrefix(host.store, args.runId);
|
|
120797
|
-
return await host.execute(args, ctx);
|
|
121870
|
+
return await host.execute(args, ctx, undefined, undefined, "sdk");
|
|
120798
121871
|
} catch (error) {
|
|
120799
121872
|
throw error instanceof Error ? translateThrown(error, args.runId) : error;
|
|
120800
121873
|
}
|
|
@@ -120845,7 +121918,8 @@ function createSessionRunControl(host) {
|
|
|
120845
121918
|
action: "resume",
|
|
120846
121919
|
runId,
|
|
120847
121920
|
...options?.stageId === undefined ? {} : { stageId: options.stageId },
|
|
120848
|
-
...options?.message === undefined ? {} : { message: options.message }
|
|
121921
|
+
...options?.message === undefined ? {} : { message: options.message },
|
|
121922
|
+
...options?.legacyRecovery === undefined ? {} : { legacyRecovery: options.legacyRecovery }
|
|
120849
121923
|
}));
|
|
120850
121924
|
}
|
|
120851
121925
|
};
|
|
@@ -120854,6 +121928,32 @@ function createSessionRunControl(host) {
|
|
|
120854
121928
|
// dist/builtin/workflows/src/extension/workflow-tool.ts
|
|
120855
121929
|
import { getSupportedThinkingLevels } from "@bastani/pi-ai/compat";
|
|
120856
121930
|
|
|
121931
|
+
// dist/builtin/workflows/src/extension/workflow-legacy-recovery.ts
|
|
121932
|
+
init_dbos_owner_fence();
|
|
121933
|
+
async function enrollLegacySdkRecovery(args, ctx, access, runtime, owner, signal) {
|
|
121934
|
+
const id = args.runId;
|
|
121935
|
+
signal?.throwIfAborted();
|
|
121936
|
+
if (access !== "sdk" || args.action !== "resume" || args.legacyRecovery?.olderWorkersStopped !== true || args.stageId !== undefined || args.all === true || id === undefined || !isFullRunId(id)) {
|
|
121937
|
+
throw new WorkflowInstanceOwnershipError("Legacy recovery requires an SDK root resume with its full UUID and confirmation that older workers have stopped.");
|
|
121938
|
+
}
|
|
121939
|
+
if (owner.store.runs().some((run) => run.id === id)) {
|
|
121940
|
+
throw new WorkflowInstanceOwnershipError("Legacy enrollment cannot replace a locally attached workflow executor.");
|
|
121941
|
+
}
|
|
121942
|
+
const inspected = await runtime.inspectDurableWorkflow(id);
|
|
121943
|
+
if (inspected.kind !== "found")
|
|
121944
|
+
throw new WorkflowInstanceOwnershipError(inspected.message);
|
|
121945
|
+
const backend = getDurableBackend();
|
|
121946
|
+
const handle = backend.getLoadableWorkflow(id);
|
|
121947
|
+
const interruptedEnrollment = handle?.legacyRecoveryPending === true && handle.ownerLiveness === "dead" && ["blocked", "paused"].includes(handle.status);
|
|
121948
|
+
if (handle === undefined || !isSdkDurableRunInScope(handle, typeof ctx.cwd === "string" ? ctx.cwd : undefined) || handle.ownerExecutorId === undefined || isDatabaseExecutor(handle.ownerExecutorId) && !interruptedEnrollment || !["running", "paused", "blocked"].includes(handle.status) || backend.enrollLegacyWorkflow === undefined) {
|
|
121949
|
+
throw new WorkflowInstanceOwnershipError("Legacy recovery is unavailable for this run or working directory; inspect it without taking ownership.");
|
|
121950
|
+
}
|
|
121951
|
+
signal?.throwIfAborted();
|
|
121952
|
+
if (!await backend.enrollLegacyWorkflow(id, { olderWorkersStopped: true, modelOwner: workflowCaller(ctx), signal })) {
|
|
121953
|
+
throw new WorkflowInstanceOwnershipError("Legacy workflow ownership changed while enrollment was pending; inspect it again before retrying.");
|
|
121954
|
+
}
|
|
121955
|
+
}
|
|
121956
|
+
|
|
120857
121957
|
// dist/builtin/workflows/src/extension/workflow-status-summary.ts
|
|
120858
121958
|
var workflowStatusRenderRuns = new WeakMap;
|
|
120859
121959
|
function setWorkflowStatusRenderRuns(result, runs) {
|
|
@@ -122324,7 +123424,7 @@ function durableInspectionError(action, runId, message) {
|
|
|
122324
123424
|
}
|
|
122325
123425
|
function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflowResourcesLoaded = () => {}, owner = captureWorkflowOwnerResources()) {
|
|
122326
123426
|
const { store, toolControlRegistry } = owner;
|
|
122327
|
-
return async function executeWorkflowTool(args, ctx, signal, onRunAccepted) {
|
|
123427
|
+
return async function executeWorkflowTool(args, ctx, signal, onRunAccepted, access) {
|
|
122328
123428
|
signal?.throwIfAborted();
|
|
122329
123429
|
if (args.action === undefined)
|
|
122330
123430
|
return {
|
|
@@ -122344,7 +123444,13 @@ function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflo
|
|
|
122344
123444
|
stages: []
|
|
122345
123445
|
};
|
|
122346
123446
|
}
|
|
122347
|
-
const
|
|
123447
|
+
const recoveryCwd = typeof ctx.cwd === "string" ? ctx.cwd : undefined;
|
|
123448
|
+
const authorize = (id) => {
|
|
123449
|
+
const handle = getDurableBackend().getLoadableWorkflow(id);
|
|
123450
|
+
if (access === "sdk" && isFullRunId(args.runId?.trim() ?? "") && (action === "status" || action === "stages" || action === "resume" && handle !== undefined && isSdkCrashedRunAdoptable(handle, recoveryCwd)) && !store.runs().some((run) => run.id === id) && handle !== undefined && isSdkDurableRunInScope(handle, recoveryCwd))
|
|
123451
|
+
return;
|
|
123452
|
+
assertWorkflowInstanceOwner(id, ctx, store);
|
|
123453
|
+
};
|
|
122348
123454
|
if (action === "status" && args.runId === undefined) {
|
|
122349
123455
|
for (const run of topLevelWorkflowRuns(store.runs()))
|
|
122350
123456
|
authorize(run.id);
|
|
@@ -122359,8 +123465,16 @@ function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflo
|
|
|
122359
123465
|
}
|
|
122360
123466
|
}
|
|
122361
123467
|
const policy = workflowPolicyFromContext(ctx);
|
|
122362
|
-
const getRuntime = () => typeof runtime === "function" ? runtime(ctx) : runtime;
|
|
122363
123468
|
const awaitRequest = (operation) => raceAbort(operation, signal);
|
|
123469
|
+
const getRuntime = () => {
|
|
123470
|
+
const resolved = typeof runtime === "function" ? runtime(ctx) : runtime;
|
|
123471
|
+
if (access !== "sdk")
|
|
123472
|
+
return resolved;
|
|
123473
|
+
return {
|
|
123474
|
+
...resolved,
|
|
123475
|
+
resumeDurableWorkflow: (id, options) => resolved.resumeDurableWorkflow(id, { ...options, modelOwner: workflowCaller(ctx) })
|
|
123476
|
+
};
|
|
123477
|
+
};
|
|
122364
123478
|
const ensureWorkflowResourcesVisible = async () => {
|
|
122365
123479
|
try {
|
|
122366
123480
|
await awaitRequest(Promise.resolve(ensureWorkflowResourcesLoaded()));
|
|
@@ -122466,6 +123580,9 @@ function makeExecuteWorkflowTool(runtime, reloadWorkflowResources, ensureWorkflo
|
|
|
122466
123580
|
case "quit":
|
|
122467
123581
|
return awaitRequest(workflowQuitAction(args, owner, { getRuntime, authorize }));
|
|
122468
123582
|
case "resume":
|
|
123583
|
+
if (args.legacyRecovery !== undefined) {
|
|
123584
|
+
await awaitRequest(enrollLegacySdkRecovery(args, ctx, access, getRuntime(), owner, signal));
|
|
123585
|
+
}
|
|
122469
123586
|
return awaitRequest(workflowResumeAction(args, {
|
|
122470
123587
|
getRuntime,
|
|
122471
123588
|
policy,
|