@bastani/atomic 0.9.26-alpha.4 → 0.9.26-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/builtin/intercom/package.json +2 -2
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +3 -3
- package/dist/builtin/web-access/package.json +2 -2
- package/dist/builtin/workflows/CHANGELOG.md +17 -0
- package/dist/builtin/workflows/builtin/{chunk-73p2cm5s.js → chunk-9bhs02gm.js} +318 -52
- package/dist/builtin/workflows/builtin/{chunk-n95myjcg.js → chunk-gts8wgjr.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-ycvg8y8k.js → chunk-gw7rghda.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +2 -2
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1383 -266
- package/dist/builtin/workflows/src/index.js +1129 -234
- package/dist/core/agent-session-export.js +1 -1
- package/dist/core/agent-session-export.js.map +1 -1
- package/dist/core/export-html/tool-renderer.d.ts +3 -3
- package/dist/core/export-html/tool-renderer.d.ts.map +1 -1
- package/dist/core/export-html/tool-renderer.js +3 -3
- package/dist/core/export-html/tool-renderer.js.map +1 -1
- package/dist/core/extensions/api-types.d.ts +3 -1
- package/dist/core/extensions/api-types.d.ts.map +1 -1
- package/dist/core/extensions/api-types.js.map +1 -1
- package/dist/core/extensions/index.d.ts +1 -1
- package/dist/core/extensions/index.d.ts.map +1 -1
- package/dist/core/extensions/index.js.map +1 -1
- package/dist/core/extensions/loader-api.d.ts.map +1 -1
- package/dist/core/extensions/loader-api.js +5 -0
- package/dist/core/extensions/loader-api.js.map +1 -1
- package/dist/core/extensions/loader-bindings.d.ts +1 -1
- package/dist/core/extensions/loader-bindings.d.ts.map +1 -1
- package/dist/core/extensions/loader-bindings.js +1 -0
- package/dist/core/extensions/loader-bindings.js.map +1 -1
- package/dist/core/extensions/runner.d.ts +3 -1
- package/dist/core/extensions/runner.d.ts.map +1 -1
- package/dist/core/extensions/runner.js +6 -0
- package/dist/core/extensions/runner.js.map +1 -1
- package/dist/core/extensions/runtime-types.d.ts +2 -1
- package/dist/core/extensions/runtime-types.d.ts.map +1 -1
- package/dist/core/extensions/runtime-types.js.map +1 -1
- package/dist/core/extensions/tool-types.d.ts +3 -0
- package/dist/core/extensions/tool-types.d.ts.map +1 -1
- package/dist/core/extensions/tool-types.js.map +1 -1
- package/dist/core/extensions/ui-types.d.ts +2 -2
- package/dist/core/extensions/ui-types.d.ts.map +1 -1
- package/dist/core/extensions/ui-types.js.map +1 -1
- package/dist/core/extensions/workflow-run-control.d.ts +3 -0
- package/dist/core/extensions/workflow-run-control.d.ts.map +1 -1
- package/dist/core/extensions/workflow-run-control.js.map +1 -1
- package/dist/core/fast-model-routing.d.ts.map +1 -1
- package/dist/core/fast-model-routing.js +1 -5
- package/dist/core/fast-model-routing.js.map +1 -1
- package/dist/core/model-config.d.ts +36 -0
- package/dist/core/model-config.d.ts.map +1 -1
- package/dist/core/model-config.js +14 -2
- package/dist/core/model-config.js.map +1 -1
- package/dist/core/model-resolver-defaults.js +1 -1
- package/dist/core/model-resolver-defaults.js.map +1 -1
- package/dist/core/package-manager-auto-resources.d.ts.map +1 -1
- package/dist/core/package-manager-auto-resources.js +6 -2
- package/dist/core/package-manager-auto-resources.js.map +1 -1
- package/dist/core/package-manager-resolver.d.ts.map +1 -1
- package/dist/core/package-manager-resolver.js +4 -2
- package/dist/core/package-manager-resolver.js.map +1 -1
- package/dist/core/provider-composer-internal.d.ts +2 -1
- package/dist/core/provider-composer-internal.d.ts.map +1 -1
- package/dist/core/provider-composer-internal.js +13 -0
- package/dist/core/provider-composer-internal.js.map +1 -1
- package/dist/core/settings-manager-resource-accessors.js +7 -3
- package/dist/core/settings-manager-resource-accessors.js.map +1 -1
- package/dist/core/settings-merge.d.ts.map +1 -1
- package/dist/core/settings-merge.js +10 -0
- package/dist/core/settings-merge.js.map +1 -1
- package/dist/core/settings-types.d.ts +7 -1
- package/dist/core/settings-types.d.ts.map +1 -1
- package/dist/core/settings-types.js.map +1 -1
- package/dist/extensions/mcp/index.d.ts.map +1 -1
- package/dist/extensions/mcp/index.js +20 -1
- package/dist/extensions/mcp/index.js.map +1 -1
- package/dist/extensions/mcp/tools.d.ts +3 -1
- package/dist/extensions/mcp/tools.d.ts.map +1 -1
- package/dist/extensions/mcp/tools.js +19 -13
- package/dist/extensions/mcp/tools.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modes/interactive/components/chat-message-renderer.d.ts +2 -3
- package/dist/modes/interactive/components/chat-message-renderer.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-message-renderer.js.map +1 -1
- package/dist/modes/interactive/components/config-selector-list.d.ts.map +1 -1
- package/dist/modes/interactive/components/config-selector-list.js +3 -1
- package/dist/modes/interactive/components/config-selector-list.js.map +1 -1
- package/dist/modes/interactive/components/config-selector-project-scope.d.ts.map +1 -1
- package/dist/modes/interactive/components/config-selector-project-scope.js +2 -1
- package/dist/modes/interactive/components/config-selector-project-scope.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +2 -5
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +14 -40
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/interactive-mode-surface.d.ts +3 -3
- package/dist/modes/interactive/interactive-mode-surface.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode-surface.js.map +1 -1
- package/dist/modes/interactive/interactive-session-runtime.js +6 -1
- package/dist/modes/interactive/interactive-session-runtime.js.map +1 -1
- package/dist/modes/interactive/interactive-startup.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-startup.js +5 -0
- package/dist/modes/interactive/interactive-startup.js.map +1 -1
- package/dist/modes/interactive-engine/engine-custom-ui.d.ts.map +1 -1
- package/dist/modes/interactive-engine/engine-custom-ui.js +9 -0
- package/dist/modes/interactive-engine/engine-custom-ui.js.map +1 -1
- package/dist/modes/interactive-engine/engine-render-service.d.ts.map +1 -1
- package/dist/modes/interactive-engine/engine-render-service.js +10 -1
- package/dist/modes/interactive-engine/engine-render-service.js.map +1 -1
- package/dist/utils/image-convert.d.ts +5 -0
- package/dist/utils/image-convert.d.ts.map +1 -1
- package/dist/utils/image-convert.js +36 -6
- package/dist/utils/image-convert.js.map +1 -1
- package/docs/extensions.md +6 -0
- package/docs/models/reference.md +28 -1
- package/docs/models.md +4 -0
- package/docs/providers.md +10 -0
- package/docs/sdk.md +26 -2
- package/docs/settings.md +28 -1
- package/docs/workflows/operations.md +40 -4
- package/npm-shrinkwrap.json +71 -92
- package/package.json +8 -8
|
@@ -63,6 +63,92 @@ function raceAbort2(operation, signal) {
|
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
// dist/builtin/workflows/src/durable/dbos-database-diagnostics.ts
|
|
67
|
+
function redactedDatabaseEndpoint(url) {
|
|
68
|
+
try {
|
|
69
|
+
const parsed = new URL(url);
|
|
70
|
+
parsed.username = "";
|
|
71
|
+
parsed.password = "";
|
|
72
|
+
parsed.search = "";
|
|
73
|
+
parsed.hash = "";
|
|
74
|
+
return parsed.toString();
|
|
75
|
+
} catch {
|
|
76
|
+
return "unparseable PostgreSQL endpoint";
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function redactedDatabaseMessage(message, url) {
|
|
80
|
+
let safe = message.replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/gi, (value) => redactedDatabaseEndpoint(value));
|
|
81
|
+
let configuredSecrets = [];
|
|
82
|
+
if (url !== undefined) {
|
|
83
|
+
try {
|
|
84
|
+
const parsed = new URL(url);
|
|
85
|
+
configuredSecrets = [parsed.username, parsed.password, ...parsed.searchParams.values()].flatMap((secret) => {
|
|
86
|
+
try {
|
|
87
|
+
return [secret, decodeURIComponent(secret)];
|
|
88
|
+
} catch {
|
|
89
|
+
return [secret];
|
|
90
|
+
}
|
|
91
|
+
}).filter((secret) => secret.length > 0);
|
|
92
|
+
} catch {
|
|
93
|
+
return "PostgreSQL diagnostic could not be safely redacted; check the configured endpoint.";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
safe = safe.replace(/\b(password|passwd|token|secret|sslpassword|user(?:name)?)\s*(?:([=:])\s*|((?:is|was)\s+)|\s+)(?:"([^"]*)"|'([^']*)'|([^\s,;]+))/gi, (match, key, separator, filler, doubleQuoted, singleQuoted, bare) => {
|
|
97
|
+
const value = doubleQuoted ?? singleQuoted ?? bare ?? "";
|
|
98
|
+
const isUserContext = /^user(?:name)?$/i.test(key);
|
|
99
|
+
if (separator === undefined && !isUserContext && !configuredSecrets.includes(value))
|
|
100
|
+
return match;
|
|
101
|
+
const rendered = doubleQuoted !== undefined ? '"[redacted]"' : singleQuoted !== undefined ? "'[redacted]'" : "[redacted]";
|
|
102
|
+
return `${key}${separator ?? " "}${filler ?? ""}${rendered}`;
|
|
103
|
+
});
|
|
104
|
+
for (const secret of new Set(configuredSecrets)) {
|
|
105
|
+
if (secret.length < 4)
|
|
106
|
+
continue;
|
|
107
|
+
const escaped = secret.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
108
|
+
const expression = new RegExp(`(?<![\\p{L}\\p{N}_\\[])${escaped}(?![\\p{L}\\p{N}_\\]])`, "gu");
|
|
109
|
+
safe = safe.replace(expression, "[redacted]");
|
|
110
|
+
}
|
|
111
|
+
return safe;
|
|
112
|
+
}
|
|
113
|
+
function databaseDependencyMessage(message, diagnostics) {
|
|
114
|
+
const safe = redactedDatabaseMessage(message, diagnostics?.url);
|
|
115
|
+
if (diagnostics === undefined)
|
|
116
|
+
return safe;
|
|
117
|
+
const failure = diagnostics.failure === undefined ? "" : ` Last probe/recovery failure: ${redactedDatabaseMessage(diagnostics.failure, diagnostics.url)}`;
|
|
118
|
+
return `${safe} Provider: ${diagnostics.provider}; endpoint: ${redactedDatabaseEndpoint(diagnostics.url)}.${failure}`;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// dist/builtin/workflows/src/durable/dbos-process-owner.ts
|
|
122
|
+
function emptyOwner() {
|
|
123
|
+
return {
|
|
124
|
+
version: 1,
|
|
125
|
+
state: "uninitialized",
|
|
126
|
+
configured: undefined,
|
|
127
|
+
active: undefined,
|
|
128
|
+
launchPromise: undefined,
|
|
129
|
+
shutdownPromise: undefined,
|
|
130
|
+
failure: undefined,
|
|
131
|
+
wrappers: undefined,
|
|
132
|
+
leases: new Set
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
function ownerBag() {
|
|
136
|
+
return globalThis;
|
|
137
|
+
}
|
|
138
|
+
function getDbosProcessOwner() {
|
|
139
|
+
const bag = ownerBag();
|
|
140
|
+
const existing = bag[DBOS_PROCESS_OWNER_KEY];
|
|
141
|
+
if (existing !== undefined && existing.version === 1)
|
|
142
|
+
return existing;
|
|
143
|
+
const created = emptyOwner();
|
|
144
|
+
bag[DBOS_PROCESS_OWNER_KEY] = created;
|
|
145
|
+
return created;
|
|
146
|
+
}
|
|
147
|
+
var DBOS_PROCESS_OWNER_KEY;
|
|
148
|
+
var init_dbos_process_owner = __esm(() => {
|
|
149
|
+
DBOS_PROCESS_OWNER_KEY = Symbol.for("atomic-workflows/dbos-process-owner@1");
|
|
150
|
+
});
|
|
151
|
+
|
|
66
152
|
// dist/builtin/workflows/src/durable/dbos-admission.ts
|
|
67
153
|
import { AsyncLocalStorage as AsyncLocalStorage2 } from "node:async_hooks";
|
|
68
154
|
function isDbosDependencyError(error) {
|
|
@@ -81,6 +167,8 @@ async function boundedAdmission(operation, signal, timeoutMs = DBOS_ADMISSION_TI
|
|
|
81
167
|
return await raceAbort2(operation(controller.signal), controller.signal);
|
|
82
168
|
} catch (error) {
|
|
83
169
|
controller.abort(error);
|
|
170
|
+
if (isDbosDependencyError(error) && error.admissionDetail !== undefined)
|
|
171
|
+
throw new DbosDependencyError(error.admissionDetail, null);
|
|
84
172
|
throw error;
|
|
85
173
|
} finally {
|
|
86
174
|
clearTimeout(timer);
|
|
@@ -89,10 +177,13 @@ async function boundedAdmission(operation, signal, timeoutMs = DBOS_ADMISSION_TI
|
|
|
89
177
|
}
|
|
90
178
|
var DBOS_ADMISSION_TIMEOUT_MS = 1e4, DbosDependencyError, key2, bag, dbosAdmissionContext;
|
|
91
179
|
var init_dbos_admission = __esm(() => {
|
|
180
|
+
init_dbos_process_owner();
|
|
92
181
|
DbosDependencyError = class DbosDependencyError extends Error {
|
|
93
182
|
code = "ATOMIC_DBOS_DEPENDENCY";
|
|
94
|
-
|
|
95
|
-
|
|
183
|
+
admissionDetail;
|
|
184
|
+
constructor(message = "Workflow database unavailable during admission. Restore PostgreSQL before retrying.", diagnostics = getDbosProcessOwner().databaseDiagnostics?.(), admissionDetail) {
|
|
185
|
+
super(databaseDependencyMessage(message, diagnostics ?? undefined));
|
|
186
|
+
this.admissionDetail = admissionDetail === undefined ? undefined : redactedDatabaseMessage(admissionDetail);
|
|
96
187
|
this.name = "DbosDependencyError";
|
|
97
188
|
}
|
|
98
189
|
};
|
|
@@ -102,6 +193,602 @@ var init_dbos_admission = __esm(() => {
|
|
|
102
193
|
dbosAdmissionContext = bag[key2];
|
|
103
194
|
});
|
|
104
195
|
|
|
196
|
+
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
197
|
+
function fenceDbosAdmissionPool(pool) {
|
|
198
|
+
const connect = pool.connect.bind(pool);
|
|
199
|
+
async function acquire(signal) {
|
|
200
|
+
if (signal.aborted)
|
|
201
|
+
throw sdkDependencyError();
|
|
202
|
+
const pending = connect().then((client) => {
|
|
203
|
+
if (signal.aborted) {
|
|
204
|
+
client.release(true);
|
|
205
|
+
throw sdkDependencyError();
|
|
206
|
+
}
|
|
207
|
+
return fencedClient(client, signal);
|
|
208
|
+
});
|
|
209
|
+
try {
|
|
210
|
+
return await raceAbort2(pending, signal);
|
|
211
|
+
} catch (error) {
|
|
212
|
+
throw databaseError(error, signal);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function scopedConnect(callback) {
|
|
216
|
+
const signal = dbosAdmissionContext.getStore();
|
|
217
|
+
if (signal === undefined) {
|
|
218
|
+
if (callback === undefined)
|
|
219
|
+
return connect();
|
|
220
|
+
connect(callback);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const pending = acquire(signal);
|
|
224
|
+
if (callback === undefined)
|
|
225
|
+
return pending;
|
|
226
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
227
|
+
}
|
|
228
|
+
pool.connect = scopedConnect;
|
|
229
|
+
return pool;
|
|
230
|
+
}
|
|
231
|
+
function fencedClient(client, signal) {
|
|
232
|
+
let released = false;
|
|
233
|
+
const release = (error) => {
|
|
234
|
+
if (released)
|
|
235
|
+
return;
|
|
236
|
+
released = true;
|
|
237
|
+
signal.removeEventListener("abort", abort);
|
|
238
|
+
client.release(error);
|
|
239
|
+
};
|
|
240
|
+
const abort = () => release(true);
|
|
241
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
242
|
+
const query = new Proxy(client.query, {
|
|
243
|
+
apply(target, _receiver, args) {
|
|
244
|
+
const values = [...args];
|
|
245
|
+
const last = values.at(-1);
|
|
246
|
+
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
247
|
+
const pending = (async () => {
|
|
248
|
+
if (signal.aborted || released)
|
|
249
|
+
throw sdkDependencyError();
|
|
250
|
+
try {
|
|
251
|
+
return await raceAbort2(new Promise((resolve, reject) => {
|
|
252
|
+
Reflect.apply(target, client, [
|
|
253
|
+
...values,
|
|
254
|
+
(error, result) => {
|
|
255
|
+
if (error)
|
|
256
|
+
reject(error);
|
|
257
|
+
else
|
|
258
|
+
resolve(result);
|
|
259
|
+
}
|
|
260
|
+
]);
|
|
261
|
+
}), signal);
|
|
262
|
+
} catch (error) {
|
|
263
|
+
throw databaseError(error, signal);
|
|
264
|
+
}
|
|
265
|
+
})();
|
|
266
|
+
if (callback === undefined)
|
|
267
|
+
return pending;
|
|
268
|
+
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
return new Proxy(client, {
|
|
272
|
+
get(target, property, receiver) {
|
|
273
|
+
if (property === "query")
|
|
274
|
+
return query;
|
|
275
|
+
if (property === "release")
|
|
276
|
+
return release;
|
|
277
|
+
return Reflect.get(target, property, receiver);
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
function sdkDependencyError(message) {
|
|
282
|
+
const reported = message === undefined ? new DbosDependencyError : new DbosDependencyError(message, null);
|
|
283
|
+
return new DbosDependencyError(undefined, null, reported.message);
|
|
284
|
+
}
|
|
285
|
+
function databaseError(error, signal) {
|
|
286
|
+
if (signal?.aborted)
|
|
287
|
+
return sdkDependencyError();
|
|
288
|
+
if (isDbosDependencyError(error))
|
|
289
|
+
return sdkDependencyError(error.admissionDetail ?? error.message);
|
|
290
|
+
const pending = [error];
|
|
291
|
+
const seen = new Set;
|
|
292
|
+
while (pending.length > 0) {
|
|
293
|
+
const value = pending.pop();
|
|
294
|
+
if (typeof value !== "object" || value === null || seen.has(value))
|
|
295
|
+
continue;
|
|
296
|
+
seen.add(value);
|
|
297
|
+
if ("cause" in value)
|
|
298
|
+
pending.push(value.cause);
|
|
299
|
+
if ("error" in value)
|
|
300
|
+
pending.push(value.error);
|
|
301
|
+
if ("errors" in value && Array.isArray(value.errors))
|
|
302
|
+
pending.push(...value.errors);
|
|
303
|
+
const code = "code" in value && typeof value.code === "string" ? value.code : undefined;
|
|
304
|
+
const message = value instanceof Error ? value.message : "";
|
|
305
|
+
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
306
|
+
return sdkDependencyError();
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return error;
|
|
310
|
+
}
|
|
311
|
+
var connectionErrnos, connectionMessages;
|
|
312
|
+
var init_dbos_admission_pool = __esm(() => {
|
|
313
|
+
init_dbos_admission();
|
|
314
|
+
connectionErrnos = new Set([
|
|
315
|
+
"ECONNRESET",
|
|
316
|
+
"ECONNREFUSED",
|
|
317
|
+
"ECONNABORTED",
|
|
318
|
+
"EHOSTUNREACH",
|
|
319
|
+
"ENETUNREACH",
|
|
320
|
+
"ETIMEDOUT",
|
|
321
|
+
"ENOTFOUND",
|
|
322
|
+
"EAI_AGAIN",
|
|
323
|
+
"EPIPE"
|
|
324
|
+
]);
|
|
325
|
+
connectionMessages = new Set([
|
|
326
|
+
"Connection terminated unexpectedly",
|
|
327
|
+
"Connection terminated due to connection timeout",
|
|
328
|
+
"Client has encountered a connection error and is not queryable",
|
|
329
|
+
"timeout exceeded when trying to connect",
|
|
330
|
+
"Client was closed and is not queryable",
|
|
331
|
+
"Postgres connection pool changed while acquiring a connection"
|
|
332
|
+
]);
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
// dist/builtin/workflows/src/durable/dbos-row-guard.ts
|
|
336
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
337
|
+
async function installDbosRowGuard(client) {
|
|
338
|
+
const ready = await client.query("SELECT to_regclass('dbos.workflow_status') IS NOT NULL AS status, to_regclass('dbos.operation_outputs') IS NOT NULL AS operations");
|
|
339
|
+
if (!ready.rows[0]?.status && !ready.rows[0]?.operations)
|
|
340
|
+
return;
|
|
341
|
+
if (!ready.rows[0]?.status || !ready.rows[0]?.operations)
|
|
342
|
+
throw new Error("Atomic workflow database schema is incomplete");
|
|
343
|
+
if (await validateInstalledGuard(client))
|
|
344
|
+
return;
|
|
345
|
+
await client.query("BEGIN");
|
|
346
|
+
try {
|
|
347
|
+
await client.query("SELECT pg_advisory_xact_lock(3419, 9003)");
|
|
348
|
+
if (await validateInstalledGuard(client)) {
|
|
349
|
+
await client.query("COMMIT");
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
await client.query(guardSql);
|
|
353
|
+
await client.query("CREATE TABLE IF NOT EXISTS dbos.atomic_guard_protocol (singleton boolean PRIMARY KEY CHECK(singleton), fingerprint text NOT NULL)");
|
|
354
|
+
await client.query("INSERT INTO dbos.atomic_guard_protocol VALUES (true, $1) ON CONFLICT DO NOTHING", [
|
|
355
|
+
fingerprint
|
|
356
|
+
]);
|
|
357
|
+
await client.query("COMMIT");
|
|
358
|
+
} catch (error) {
|
|
359
|
+
await client.query("ROLLBACK");
|
|
360
|
+
throw error;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
async function validateInstalledGuard(client) {
|
|
364
|
+
const installed = await client.query("SELECT to_regclass('dbos.atomic_guard_protocol') IS NOT NULL AS present");
|
|
365
|
+
if (!installed.rows[0]?.present)
|
|
366
|
+
return false;
|
|
367
|
+
const validated = await client.query("SELECT fingerprint, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid IN ('dbos.workflow_status'::regclass,'dbos.operation_outputs'::regclass) AND tgname = 'atomic_generation_guard' AND tgenabled = 'O' AND tgtype = 31 AND tgfoid = 'dbos.atomic_guard_row()'::regprocedure) AS triggers, (SELECT count(*)::int FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection' AND tgenabled = 'O' AND tgtype = 21 AND tgfoid = 'dbos.atomic_project_metadata()'::regprocedure) AS projections FROM dbos.atomic_guard_protocol WHERE singleton");
|
|
368
|
+
if (validated.rows.length !== 1 || validated.rows[0].fingerprint !== fingerprint || validated.rows[0].triggers !== 2 || validated.rows[0].projections !== 1)
|
|
369
|
+
throw new Error("Atomic database ownership guard schema is incompatible or incomplete");
|
|
370
|
+
return true;
|
|
371
|
+
}
|
|
372
|
+
var guardSql = `
|
|
373
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_owner_generation (
|
|
374
|
+
root text PRIMARY KEY, owner text, generation bigint,
|
|
375
|
+
terminal boolean NOT NULL DEFAULT false, claim text, claimant text, protocol integer NOT NULL DEFAULT 1 CHECK(protocol = 1)
|
|
376
|
+
);
|
|
377
|
+
CREATE TABLE IF NOT EXISTS dbos.atomic_guard_rows (workflow_uuid text PRIMARY KEY, root text NOT NULL);
|
|
378
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_metadata(value text) RETURNS jsonb
|
|
379
|
+
LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog AS $$
|
|
380
|
+
DECLARE decoded jsonb;
|
|
381
|
+
BEGIN
|
|
382
|
+
IF value IS NULL THEN RETURN NULL; END IF;
|
|
383
|
+
BEGIN decoded := value::jsonb; EXCEPTION WHEN invalid_text_representation THEN RETURN NULL; END;
|
|
384
|
+
IF decoded->>'__dbos_serializer' = 'superjson' THEN decoded := decoded->'json'; END IF;
|
|
385
|
+
IF decoded->>'__atomicDurableMetadata' IS DISTINCT FROM 'true' OR decoded->>'version' IS DISTINCT FROM '3' THEN RETURN NULL; END IF;
|
|
386
|
+
RETURN decoded->'metadata';
|
|
387
|
+
END $$;
|
|
388
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_guard_row() RETURNS trigger
|
|
389
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
390
|
+
DECLARE
|
|
391
|
+
row_value jsonb; id text; root_id text; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE;
|
|
392
|
+
meta jsonb; prior jsonb; prior_generation bigint; next_generation bigint; absorbing boolean;
|
|
393
|
+
allowed boolean; cap_owner text; cap_generation bigint; claim_token text;
|
|
394
|
+
BEGIN
|
|
395
|
+
row_value := CASE WHEN TG_OP = 'DELETE' THEN to_jsonb(OLD) ELSE to_jsonb(NEW) END;
|
|
396
|
+
id := row_value->>'workflow_uuid';
|
|
397
|
+
root_id := split_part(id, ':checkpoint:', 1);
|
|
398
|
+
IF TG_OP = 'UPDATE' AND to_jsonb(OLD)->>'workflow_uuid' IS DISTINCT FROM id AND (
|
|
399
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid') OR
|
|
400
|
+
(TG_TABLE_NAME = 'workflow_status' AND to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')) OR
|
|
401
|
+
(TG_TABLE_NAME = 'operation_outputs' AND EXISTS(SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = to_jsonb(OLD)->>'workflow_uuid' AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')))
|
|
402
|
+
) THEN
|
|
403
|
+
IF NULLIF(current_setting('atomic.row_authority',true),'') IS NULL OR current_setting('atomic.row_authority',true) = 'null' THEN RETURN NULL; END IF;
|
|
404
|
+
RAISE EXCEPTION 'Atomic workflow identities are immutable';
|
|
405
|
+
END IF;
|
|
406
|
+
IF TG_TABLE_NAME = 'workflow_status' AND NOT (
|
|
407
|
+
COALESCE(row_value->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
408
|
+
COALESCE(to_jsonb(OLD)->>'name' IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint'), false) OR
|
|
409
|
+
EXISTS(SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id)
|
|
410
|
+
) THEN
|
|
411
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
412
|
+
END IF;
|
|
413
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
414
|
+
IF TG_TABLE_NAME = 'operation_outputs' AND NOT EXISTS (
|
|
415
|
+
SELECT 1 FROM dbos.workflow_status WHERE workflow_uuid = id AND name IN ('atomicWorkflowHandle','atomicWorkflowCheckpoint')
|
|
416
|
+
) AND NOT EXISTS (SELECT 1 FROM dbos.atomic_guard_rows WHERE workflow_uuid = id) THEN
|
|
417
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
418
|
+
END IF;
|
|
419
|
+
INSERT INTO dbos.atomic_guard_rows VALUES(id, root_id) ON CONFLICT DO NOTHING;
|
|
420
|
+
INSERT INTO dbos.atomic_guard_rows SELECT workflow_uuid, root_id FROM dbos.workflow_status
|
|
421
|
+
WHERE workflow_uuid = root_id AND name = 'atomicWorkflowHandle' ON CONFLICT DO NOTHING;
|
|
422
|
+
INSERT INTO dbos.atomic_owner_generation(root) VALUES(root_id)
|
|
423
|
+
ON CONFLICT(root) DO UPDATE SET root = EXCLUDED.root RETURNING * INTO state;
|
|
424
|
+
IF state.owner IS NULL THEN
|
|
425
|
+
IF TG_TABLE_NAME <> 'workflow_status' OR TG_OP = 'DELETE' OR row_value->>'status' <> 'SUCCESS'
|
|
426
|
+
OR id NOT LIKE root_id || ':checkpoint:__atomic_metadata:%'
|
|
427
|
+
OR authority->>'root' IS DISTINCT FROM root_id OR authority->>'enroll' IS DISTINCT FROM 'true' THEN
|
|
428
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
429
|
+
END IF;
|
|
430
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
431
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id THEN
|
|
432
|
+
RAISE EXCEPTION 'Atomic metadata encoding is unavailable for enrollment';
|
|
433
|
+
END IF;
|
|
434
|
+
SELECT candidate.metadata, candidate.generation INTO prior, prior_generation FROM (
|
|
435
|
+
SELECT dbos.atomic_metadata(output) AS metadata,
|
|
436
|
+
split_part(substring(workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint AS generation
|
|
437
|
+
FROM dbos.workflow_status
|
|
438
|
+
WHERE workflow_uuid LIKE root_id || ':checkpoint:__atomic_metadata:%' AND status = 'SUCCESS'
|
|
439
|
+
) candidate WHERE candidate.metadata IS NOT NULL
|
|
440
|
+
ORDER BY COALESCE(candidate.metadata->>'status' IN ('completed', 'cancelled') OR
|
|
441
|
+
(candidate.metadata->>'status' IN ('failed','blocked') AND candidate.metadata->>'resumable' = 'false'), false) DESC,
|
|
442
|
+
candidate.generation DESC LIMIT 1;
|
|
443
|
+
IF prior_generation IS DISTINCT FROM (authority->>'generation')::bigint
|
|
444
|
+
OR prior->>'ownerExecutorId' IS DISTINCT FROM authority->>'owner' THEN
|
|
445
|
+
RAISE EXCEPTION 'Atomic ownership generation changed before enrollment';
|
|
446
|
+
END IF;
|
|
447
|
+
ELSE
|
|
448
|
+
IF authority->>'root' IS DISTINCT FROM root_id THEN RETURN NULL; END IF;
|
|
449
|
+
cap_owner := authority->>'owner'; cap_generation := (authority->>'generation')::bigint;
|
|
450
|
+
allowed := cap_owner = state.owner AND cap_generation = state.generation AND (
|
|
451
|
+
authority->>'actor' = state.owner OR
|
|
452
|
+
(authority->>'claim' = state.claim AND authority->>'actor' = state.claimant) OR
|
|
453
|
+
(authority->>'claim' IS NOT NULL AND authority->>'checkpoint' = id));
|
|
454
|
+
IF NOT COALESCE(allowed, false) THEN RAISE EXCEPTION 'Atomic ownership generation changed; stale database writes are refused'; END IF;
|
|
455
|
+
END IF;
|
|
456
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP = 'UPDATE' THEN
|
|
457
|
+
IF OLD.status = 'SUCCESS' AND OLD.name = 'atomicWorkflowCheckpoint' THEN
|
|
458
|
+
IF NEW.output IS DISTINCT FROM OLD.output OR NEW.status <> 'SUCCESS' THEN RAISE EXCEPTION 'Completed Atomic checkpoints are immutable'; END IF;
|
|
459
|
+
RETURN NEW;
|
|
460
|
+
END IF;
|
|
461
|
+
END IF;
|
|
462
|
+
IF TG_TABLE_NAME = 'workflow_status' AND TG_OP <> 'DELETE' AND row_value->>'status' = 'SUCCESS'
|
|
463
|
+
AND id LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN
|
|
464
|
+
meta := dbos.atomic_metadata(row_value->>'output');
|
|
465
|
+
IF meta IS NULL OR meta->>'workflowId' IS DISTINCT FROM root_id OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN RAISE EXCEPTION 'Atomic metadata encoding or owner is unavailable'; END IF;
|
|
466
|
+
next_generation := split_part(substring(id FROM length(root_id) + 13), ':', 2)::bigint;
|
|
467
|
+
absorbing := COALESCE(meta->>'status' IN ('completed', 'cancelled') OR
|
|
468
|
+
(meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
469
|
+
claim_token := meta->>'transitionClaimId';
|
|
470
|
+
IF state.root IS NOT NULL AND (next_generation < state.generation OR (state.terminal AND NOT absorbing)) THEN
|
|
471
|
+
RAISE EXCEPTION 'Atomic terminal or newer generation cannot be overwritten';
|
|
472
|
+
END IF;
|
|
473
|
+
IF state.owner IS NOT NULL AND authority->>'actor' <> state.owner THEN
|
|
474
|
+
IF id NOT LIKE '%:claim' OR next_generation <> state.generation + 1 OR claim_token IS DISTINCT FROM authority->>'claim' OR meta->>'ownerExecutorId' IS DISTINCT FROM authority->>'actor' THEN
|
|
475
|
+
RAISE EXCEPTION 'Atomic ownership transfer requires an exact generation claim';
|
|
476
|
+
END IF;
|
|
477
|
+
END IF;
|
|
478
|
+
END IF;
|
|
479
|
+
IF TG_OP = 'DELETE' THEN RETURN OLD; ELSE RETURN NEW; END IF;
|
|
480
|
+
END $$;
|
|
481
|
+
CREATE OR REPLACE FUNCTION dbos.atomic_project_metadata() RETURNS trigger
|
|
482
|
+
LANGUAGE plpgsql SET search_path = pg_catalog AS $$
|
|
483
|
+
DECLARE root_id text; meta jsonb; authority jsonb; state dbos.atomic_owner_generation%ROWTYPE; projected_generation bigint; absorbing boolean;
|
|
484
|
+
BEGIN
|
|
485
|
+
IF NEW.name <> 'atomicWorkflowCheckpoint' OR NEW.status <> 'SUCCESS' THEN RETURN NEW; END IF;
|
|
486
|
+
root_id := split_part(NEW.workflow_uuid, ':checkpoint:', 1);
|
|
487
|
+
IF NEW.workflow_uuid NOT LIKE root_id || ':checkpoint:__atomic_metadata:%' THEN RETURN NEW; END IF;
|
|
488
|
+
IF TG_OP = 'UPDATE' THEN IF OLD.status = 'SUCCESS' THEN RETURN NEW; END IF; END IF;
|
|
489
|
+
authority := NULLIF(current_setting('atomic.row_authority', true), '')::jsonb;
|
|
490
|
+
SELECT * INTO state FROM dbos.atomic_owner_generation WHERE root = root_id FOR UPDATE;
|
|
491
|
+
IF state.owner IS NULL AND authority->>'enroll' IS DISTINCT FROM 'true' THEN RETURN NEW; END IF;
|
|
492
|
+
meta := dbos.atomic_metadata(NEW.output);
|
|
493
|
+
projected_generation := split_part(substring(NEW.workflow_uuid FROM length(root_id) + 13), ':', 2)::bigint;
|
|
494
|
+
absorbing := COALESCE(meta->>'status' IN ('completed','cancelled') OR (meta->>'status' IN ('failed','blocked') AND meta->>'resumable' = 'false'), false);
|
|
495
|
+
UPDATE dbos.atomic_owner_generation SET owner = meta->>'ownerExecutorId', generation = projected_generation,
|
|
496
|
+
terminal = absorbing, claim = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN meta->>'transitionClaimId' END,
|
|
497
|
+
claimant = CASE WHEN NEW.workflow_uuid LIKE '%:claim' THEN authority->>'actor' END WHERE root = root_id;
|
|
498
|
+
RETURN NEW;
|
|
499
|
+
END $$;
|
|
500
|
+
DO $$ BEGIN
|
|
501
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
502
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.workflow_status
|
|
503
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
504
|
+
END IF;
|
|
505
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.operation_outputs'::regclass AND tgname = 'atomic_generation_guard') THEN
|
|
506
|
+
CREATE TRIGGER atomic_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON dbos.operation_outputs
|
|
507
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_guard_row();
|
|
508
|
+
END IF;
|
|
509
|
+
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'dbos.workflow_status'::regclass AND tgname = 'atomic_generation_projection') THEN
|
|
510
|
+
CREATE TRIGGER atomic_generation_projection AFTER INSERT OR UPDATE ON dbos.workflow_status
|
|
511
|
+
FOR EACH ROW EXECUTE FUNCTION dbos.atomic_project_metadata();
|
|
512
|
+
END IF;
|
|
513
|
+
END $$;
|
|
514
|
+
`, fingerprint;
|
|
515
|
+
var init_dbos_row_guard = __esm(() => {
|
|
516
|
+
fingerprint = createHash3("sha256").update(guardSql).digest("hex");
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
// dist/builtin/workflows/src/durable/dbos-owner-fence.ts
|
|
520
|
+
import { AsyncLocalStorage as AsyncLocalStorage3 } from "node:async_hooks";
|
|
521
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
522
|
+
function lockKey(identity) {
|
|
523
|
+
const digest = createHash4("sha256").update(`atomic-workflow-fence:${identity}`).digest();
|
|
524
|
+
return [digest.readInt32BE(0), digest.readInt32BE(4)];
|
|
525
|
+
}
|
|
526
|
+
function isDatabaseExecutor(executorId) {
|
|
527
|
+
return /^atomic-db-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(executorId ?? "");
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
class DbosOwnerFence {
|
|
531
|
+
pool;
|
|
532
|
+
executorId;
|
|
533
|
+
endPool;
|
|
534
|
+
lifetime;
|
|
535
|
+
lost = false;
|
|
536
|
+
closing;
|
|
537
|
+
closed = false;
|
|
538
|
+
get invalidated() {
|
|
539
|
+
return this.lost;
|
|
540
|
+
}
|
|
541
|
+
constructor(pool, executorId, endPool) {
|
|
542
|
+
this.pool = pool;
|
|
543
|
+
this.executorId = executorId;
|
|
544
|
+
this.endPool = endPool;
|
|
545
|
+
}
|
|
546
|
+
fail() {
|
|
547
|
+
this.lost = true;
|
|
548
|
+
return new DbosDependencyError("Workflow database unavailable: ownership connection was lost; this execution generation cannot write again");
|
|
549
|
+
}
|
|
550
|
+
owner() {
|
|
551
|
+
if (this.lost)
|
|
552
|
+
return Promise.reject(this.fail());
|
|
553
|
+
this.lifetime ??= (async () => {
|
|
554
|
+
const client = await this.pool().connect();
|
|
555
|
+
client.on("error", () => {
|
|
556
|
+
this.fail();
|
|
557
|
+
});
|
|
558
|
+
client.on("end", () => {
|
|
559
|
+
this.fail();
|
|
560
|
+
});
|
|
561
|
+
try {
|
|
562
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-owner:${this.executorId}`]);
|
|
563
|
+
const held = await client.query("SELECT pg_try_advisory_lock($1, $2) AS held", [
|
|
564
|
+
...lockKey(this.executorId)
|
|
565
|
+
]);
|
|
566
|
+
if (!held.rows[0]?.held || this.lost)
|
|
567
|
+
throw this.fail();
|
|
568
|
+
await client.query("SELECT pg_advisory_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
569
|
+
await client.query("SELECT pg_advisory_unlock($1, $2)", [...lockKey(this.executorId)]);
|
|
570
|
+
return client;
|
|
571
|
+
} catch (error) {
|
|
572
|
+
this.fail();
|
|
573
|
+
client.release(true);
|
|
574
|
+
throw error;
|
|
575
|
+
}
|
|
576
|
+
})().catch((error) => {
|
|
577
|
+
this.fail();
|
|
578
|
+
throw error;
|
|
579
|
+
});
|
|
580
|
+
return this.lifetime;
|
|
581
|
+
}
|
|
582
|
+
async liveness(executorId) {
|
|
583
|
+
if (!isDatabaseExecutor(executorId))
|
|
584
|
+
return "unknown";
|
|
585
|
+
const client = await this.pool().connect();
|
|
586
|
+
try {
|
|
587
|
+
await client.query("BEGIN");
|
|
588
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
589
|
+
...lockKey(executorId)
|
|
590
|
+
]);
|
|
591
|
+
return result.rows[0]?.held ? "dead" : "alive";
|
|
592
|
+
} finally {
|
|
593
|
+
try {
|
|
594
|
+
await client.query("ROLLBACK");
|
|
595
|
+
} finally {
|
|
596
|
+
client.release();
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
async recover(executorId, callback) {
|
|
601
|
+
if (!isDatabaseExecutor(executorId))
|
|
602
|
+
return;
|
|
603
|
+
const client = await this.pool().connect();
|
|
604
|
+
const fail = () => {
|
|
605
|
+
this.fail();
|
|
606
|
+
};
|
|
607
|
+
client.on("error", fail);
|
|
608
|
+
client.on("end", fail);
|
|
609
|
+
try {
|
|
610
|
+
await client.query("BEGIN");
|
|
611
|
+
const result = await client.query("SELECT pg_try_advisory_xact_lock($1, $2) AS held", [
|
|
612
|
+
...lockKey(executorId)
|
|
613
|
+
]);
|
|
614
|
+
if (!result.rows[0]?.held)
|
|
615
|
+
return;
|
|
616
|
+
return await callback();
|
|
617
|
+
} finally {
|
|
618
|
+
try {
|
|
619
|
+
await client.query("ROLLBACK");
|
|
620
|
+
} finally {
|
|
621
|
+
client.removeListener("error", fail);
|
|
622
|
+
client.removeListener("end", fail);
|
|
623
|
+
client.release();
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
async write(workflowId, callback, claim = false) {
|
|
628
|
+
try {
|
|
629
|
+
return await this.writeLocked(workflowId, callback, claim);
|
|
630
|
+
} catch (error) {
|
|
631
|
+
const mapped = databaseError(error);
|
|
632
|
+
throw this.lost && isDbosDependencyError(mapped) ? this.fail() : mapped;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
async writeLocked(workflowId, callback, claim) {
|
|
636
|
+
const context = writeContext.getStore();
|
|
637
|
+
const active = context?.fence === this ? context.workflows : undefined;
|
|
638
|
+
if (active?.has(workflowId)) {
|
|
639
|
+
if (this.lost)
|
|
640
|
+
throw this.fail();
|
|
641
|
+
return await callback();
|
|
642
|
+
}
|
|
643
|
+
const owner = await this.owner();
|
|
644
|
+
const client = await this.pool().connect();
|
|
645
|
+
const fail = () => {
|
|
646
|
+
this.fail();
|
|
647
|
+
};
|
|
648
|
+
client.on("error", fail);
|
|
649
|
+
client.on("end", fail);
|
|
650
|
+
try {
|
|
651
|
+
await client.query("BEGIN");
|
|
652
|
+
await client.query("SELECT set_config('application_name', $1, false)", [`atomic-guard:${workflowId}`]);
|
|
653
|
+
await client.query("SET LOCAL statement_timeout = '10000ms'");
|
|
654
|
+
await client.query("SELECT pg_advisory_xact_lock_shared($1, $2)", [...lockKey(this.executorId)]);
|
|
655
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`workflow:${workflowId}`)]);
|
|
656
|
+
if (claim)
|
|
657
|
+
await client.query("SELECT pg_advisory_xact_lock($1, $2)", [...lockKey(`sql:${workflowId}`)]);
|
|
658
|
+
if (this.lost)
|
|
659
|
+
throw this.fail();
|
|
660
|
+
try {
|
|
661
|
+
await owner.query("SELECT 1");
|
|
662
|
+
} catch {
|
|
663
|
+
throw this.fail();
|
|
664
|
+
}
|
|
665
|
+
if (this.lost)
|
|
666
|
+
throw this.fail();
|
|
667
|
+
return await writeContext.run({ fence: this, workflows: new Map([...active ?? [], [workflowId, claim]]) }, callback);
|
|
668
|
+
} finally {
|
|
669
|
+
try {
|
|
670
|
+
await client.query("ROLLBACK");
|
|
671
|
+
} finally {
|
|
672
|
+
client.removeListener("error", fail);
|
|
673
|
+
client.removeListener("end", fail);
|
|
674
|
+
client.release();
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
async withRowAuthority(authority, callback) {
|
|
679
|
+
return await rowContext.run(authority, callback);
|
|
680
|
+
}
|
|
681
|
+
protectPool(pool) {
|
|
682
|
+
const connect = pool.connect.bind(pool);
|
|
683
|
+
const acquire = async () => {
|
|
684
|
+
const active = writeContext.getStore();
|
|
685
|
+
const client = await connect();
|
|
686
|
+
try {
|
|
687
|
+
await installDbosRowGuard(client);
|
|
688
|
+
await client.query("SELECT set_config('atomic.row_authority', $1, false)", [
|
|
689
|
+
JSON.stringify(rowContext.getStore() ?? null)
|
|
690
|
+
]);
|
|
691
|
+
} catch (error) {
|
|
692
|
+
client.release(true);
|
|
693
|
+
throw error;
|
|
694
|
+
}
|
|
695
|
+
if (active === undefined)
|
|
696
|
+
return client;
|
|
697
|
+
const fence = active.fence;
|
|
698
|
+
try {
|
|
699
|
+
if (fence.lost)
|
|
700
|
+
throw fence.fail();
|
|
701
|
+
await client.query("SELECT set_config('application_name', $1, false)", [
|
|
702
|
+
`atomic-sql:${[...active.workflows.keys()][0]}`
|
|
703
|
+
]);
|
|
704
|
+
for (const identity of [
|
|
705
|
+
fence.executorId,
|
|
706
|
+
...[...active.workflows].filter(([, claim]) => !claim).map(([id]) => `sql:${id}`)
|
|
707
|
+
]) {
|
|
708
|
+
const held = await client.query("SELECT pg_try_advisory_lock_shared($1, $2) AS held", [...lockKey(identity)]);
|
|
709
|
+
if (!held.rows[0]?.held)
|
|
710
|
+
throw fence.fail();
|
|
711
|
+
}
|
|
712
|
+
try {
|
|
713
|
+
await (await fence.owner()).query("SELECT 1");
|
|
714
|
+
} catch {
|
|
715
|
+
throw fence.fail();
|
|
716
|
+
}
|
|
717
|
+
if (fence.lost)
|
|
718
|
+
throw fence.fail();
|
|
719
|
+
} catch (error) {
|
|
720
|
+
client.release(true);
|
|
721
|
+
throw error;
|
|
722
|
+
}
|
|
723
|
+
let released = false;
|
|
724
|
+
const release = () => {
|
|
725
|
+
if (released)
|
|
726
|
+
return;
|
|
727
|
+
released = true;
|
|
728
|
+
client.release(true);
|
|
729
|
+
};
|
|
730
|
+
const query = new Proxy(client.query, {
|
|
731
|
+
apply: (target, _receiver, args) => {
|
|
732
|
+
if (!fence.lost && !released)
|
|
733
|
+
return Reflect.apply(target, client, args);
|
|
734
|
+
const error = fence.fail();
|
|
735
|
+
const callback = args.at(-1);
|
|
736
|
+
if (typeof callback === "function") {
|
|
737
|
+
queueMicrotask(() => callback(error));
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
return Promise.reject(error);
|
|
741
|
+
}
|
|
742
|
+
});
|
|
743
|
+
return new Proxy(client, {
|
|
744
|
+
get(target, property, receiver) {
|
|
745
|
+
if (property === "release")
|
|
746
|
+
return release;
|
|
747
|
+
if (property === "query")
|
|
748
|
+
return query;
|
|
749
|
+
return Reflect.get(target, property, receiver);
|
|
750
|
+
}
|
|
751
|
+
});
|
|
752
|
+
};
|
|
753
|
+
function scopedConnect(callback) {
|
|
754
|
+
const pending = acquire();
|
|
755
|
+
if (callback === undefined)
|
|
756
|
+
return pending;
|
|
757
|
+
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
758
|
+
}
|
|
759
|
+
pool.connect = scopedConnect;
|
|
760
|
+
return pool;
|
|
761
|
+
}
|
|
762
|
+
close() {
|
|
763
|
+
this.closed = true;
|
|
764
|
+
this.lost = true;
|
|
765
|
+
this.closing ??= (async () => {
|
|
766
|
+
if (this.lifetime !== undefined) {
|
|
767
|
+
const client = await this.lifetime.catch(() => {
|
|
768
|
+
return;
|
|
769
|
+
});
|
|
770
|
+
client?.release(true);
|
|
771
|
+
}
|
|
772
|
+
await this.endPool();
|
|
773
|
+
})();
|
|
774
|
+
return this.closing;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
var contextKey, contextBag, writeContext, rowContextKey, rowContextBag, rowContext;
|
|
778
|
+
var init_dbos_owner_fence = __esm(() => {
|
|
779
|
+
init_dbos_admission();
|
|
780
|
+
init_dbos_admission_pool();
|
|
781
|
+
init_dbos_row_guard();
|
|
782
|
+
contextKey = Symbol.for("atomic-workflows/dbos-owner-write-context@1");
|
|
783
|
+
contextBag = globalThis;
|
|
784
|
+
contextBag[contextKey] ??= new AsyncLocalStorage3;
|
|
785
|
+
writeContext = contextBag[contextKey];
|
|
786
|
+
rowContextKey = Symbol.for("atomic-workflows/dbos-row-authority@1");
|
|
787
|
+
rowContextBag = globalThis;
|
|
788
|
+
rowContextBag[rowContextKey] ??= new AsyncLocalStorage3;
|
|
789
|
+
rowContext = rowContextBag[rowContextKey];
|
|
790
|
+
});
|
|
791
|
+
|
|
105
792
|
// ../../node_modules/@dbos-inc/dbos-sdk/dist/src/error.js
|
|
106
793
|
var require_error = __commonJS(function(exports) {
|
|
107
794
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -30258,137 +30945,6 @@ SELECT
|
|
|
30258
30945
|
} });
|
|
30259
30946
|
});
|
|
30260
30947
|
|
|
30261
|
-
// dist/builtin/workflows/src/durable/dbos-admission-pool.ts
|
|
30262
|
-
function fenceDbosAdmissionPool(pool) {
|
|
30263
|
-
const connect = pool.connect.bind(pool);
|
|
30264
|
-
async function acquire(signal) {
|
|
30265
|
-
if (signal.aborted)
|
|
30266
|
-
throw new DbosDependencyError;
|
|
30267
|
-
const pending = connect().then((client) => {
|
|
30268
|
-
if (signal.aborted) {
|
|
30269
|
-
client.release(true);
|
|
30270
|
-
throw new DbosDependencyError;
|
|
30271
|
-
}
|
|
30272
|
-
return fencedClient(client, signal);
|
|
30273
|
-
});
|
|
30274
|
-
try {
|
|
30275
|
-
return await raceAbort2(pending, signal);
|
|
30276
|
-
} catch (error) {
|
|
30277
|
-
throw databaseError(error, signal);
|
|
30278
|
-
}
|
|
30279
|
-
}
|
|
30280
|
-
function scopedConnect(callback) {
|
|
30281
|
-
const signal = dbosAdmissionContext.getStore();
|
|
30282
|
-
if (signal === undefined) {
|
|
30283
|
-
if (callback === undefined)
|
|
30284
|
-
return connect();
|
|
30285
|
-
connect(callback);
|
|
30286
|
-
return;
|
|
30287
|
-
}
|
|
30288
|
-
const pending = acquire(signal);
|
|
30289
|
-
if (callback === undefined)
|
|
30290
|
-
return pending;
|
|
30291
|
-
pending.then((client) => callback(undefined, client, client.release), (error) => callback(error, undefined, () => {}));
|
|
30292
|
-
}
|
|
30293
|
-
pool.connect = scopedConnect;
|
|
30294
|
-
return pool;
|
|
30295
|
-
}
|
|
30296
|
-
function fencedClient(client, signal) {
|
|
30297
|
-
let released = false;
|
|
30298
|
-
const release = (error) => {
|
|
30299
|
-
if (released)
|
|
30300
|
-
return;
|
|
30301
|
-
released = true;
|
|
30302
|
-
signal.removeEventListener("abort", abort);
|
|
30303
|
-
client.release(error);
|
|
30304
|
-
};
|
|
30305
|
-
const abort = () => release(true);
|
|
30306
|
-
signal.addEventListener("abort", abort, { once: true });
|
|
30307
|
-
const query = new Proxy(client.query, {
|
|
30308
|
-
apply(target, _receiver, args) {
|
|
30309
|
-
const values = [...args];
|
|
30310
|
-
const last = values.at(-1);
|
|
30311
|
-
const callback = typeof last === "function" ? values.pop() : undefined;
|
|
30312
|
-
const pending = (async () => {
|
|
30313
|
-
if (signal.aborted || released)
|
|
30314
|
-
throw new DbosDependencyError;
|
|
30315
|
-
try {
|
|
30316
|
-
return await raceAbort2(new Promise((resolve, reject) => {
|
|
30317
|
-
Reflect.apply(target, client, [
|
|
30318
|
-
...values,
|
|
30319
|
-
(error, result) => {
|
|
30320
|
-
if (error)
|
|
30321
|
-
reject(error);
|
|
30322
|
-
else
|
|
30323
|
-
resolve(result);
|
|
30324
|
-
}
|
|
30325
|
-
]);
|
|
30326
|
-
}), signal);
|
|
30327
|
-
} catch (error) {
|
|
30328
|
-
throw databaseError(error, signal);
|
|
30329
|
-
}
|
|
30330
|
-
})();
|
|
30331
|
-
if (callback === undefined)
|
|
30332
|
-
return pending;
|
|
30333
|
-
pending.then((result) => callback(null, result), (error) => callback(error));
|
|
30334
|
-
}
|
|
30335
|
-
});
|
|
30336
|
-
return new Proxy(client, {
|
|
30337
|
-
get(target, property, receiver) {
|
|
30338
|
-
if (property === "query")
|
|
30339
|
-
return query;
|
|
30340
|
-
if (property === "release")
|
|
30341
|
-
return release;
|
|
30342
|
-
return Reflect.get(target, property, receiver);
|
|
30343
|
-
}
|
|
30344
|
-
});
|
|
30345
|
-
}
|
|
30346
|
-
function databaseError(error, signal) {
|
|
30347
|
-
if (signal.aborted)
|
|
30348
|
-
return new DbosDependencyError;
|
|
30349
|
-
const pending = [error];
|
|
30350
|
-
const seen = new Set;
|
|
30351
|
-
while (pending.length > 0) {
|
|
30352
|
-
const value = pending.pop();
|
|
30353
|
-
if (typeof value !== "object" || value === null || seen.has(value))
|
|
30354
|
-
continue;
|
|
30355
|
-
seen.add(value);
|
|
30356
|
-
if ("cause" in value)
|
|
30357
|
-
pending.push(value.cause);
|
|
30358
|
-
if ("error" in value)
|
|
30359
|
-
pending.push(value.error);
|
|
30360
|
-
if ("errors" in value && Array.isArray(value.errors))
|
|
30361
|
-
pending.push(...value.errors);
|
|
30362
|
-
const code = "code" in value && typeof value.code === "string" ? value.code : undefined;
|
|
30363
|
-
const message = value instanceof Error ? value.message : "";
|
|
30364
|
-
if (code !== undefined && (/^(?:08|53|57)[0-9A-Z]{3}$|^40003$/.test(code) || connectionErrnos.has(code)) || code === undefined && connectionMessages.has(message)) {
|
|
30365
|
-
return new DbosDependencyError;
|
|
30366
|
-
}
|
|
30367
|
-
}
|
|
30368
|
-
return error;
|
|
30369
|
-
}
|
|
30370
|
-
var connectionErrnos, connectionMessages;
|
|
30371
|
-
var init_dbos_admission_pool = __esm(() => {
|
|
30372
|
-
init_dbos_admission();
|
|
30373
|
-
connectionErrnos = new Set([
|
|
30374
|
-
"ECONNRESET",
|
|
30375
|
-
"ECONNREFUSED",
|
|
30376
|
-
"ECONNABORTED",
|
|
30377
|
-
"EHOSTUNREACH",
|
|
30378
|
-
"ENETUNREACH",
|
|
30379
|
-
"ETIMEDOUT",
|
|
30380
|
-
"ENOTFOUND",
|
|
30381
|
-
"EAI_AGAIN",
|
|
30382
|
-
"EPIPE"
|
|
30383
|
-
]);
|
|
30384
|
-
connectionMessages = new Set([
|
|
30385
|
-
"Connection terminated unexpectedly",
|
|
30386
|
-
"Connection terminated due to connection timeout",
|
|
30387
|
-
"Client has encountered a connection error and is not queryable",
|
|
30388
|
-
"timeout exceeded when trying to connect"
|
|
30389
|
-
]);
|
|
30390
|
-
});
|
|
30391
|
-
|
|
30392
30948
|
// dist/builtin/workflows/src/durable/dbos-default-postgres-url.ts
|
|
30393
30949
|
function defaultPostgresUrl(database, host = process.env.PGHOST || "localhost", port = process.env.PGPORT || "5432") {
|
|
30394
30950
|
const url = new URL("postgresql://localhost");
|
|
@@ -30673,7 +31229,12 @@ function defaultDatabaseUrl(name) {
|
|
|
30673
31229
|
function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock) {
|
|
30674
31230
|
const systemDatabaseUrl = config.systemDatabaseUrl ?? defaultDatabaseUrl(config.name);
|
|
30675
31231
|
const health = resolvedPostgresHealth(systemDatabaseUrl);
|
|
30676
|
-
|
|
31232
|
+
getDbosProcessOwner().databaseDiagnostics = () => ({
|
|
31233
|
+
provider: health === undefined ? config.systemDatabaseUrl === undefined ? "docker" : "configured" : "managed",
|
|
31234
|
+
url: health?.endpoint ?? systemDatabaseUrl,
|
|
31235
|
+
failure: health?.lastFailure?.message
|
|
31236
|
+
});
|
|
31237
|
+
const createPool = (fenced = true) => {
|
|
30677
31238
|
let unsubscribe;
|
|
30678
31239
|
const managed = createRecoverablePostgresPool(systemDatabaseUrl, {
|
|
30679
31240
|
beforeConnect: health === undefined ? undefined : () => health.check(),
|
|
@@ -30683,14 +31244,38 @@ function configureAdmissionDatabase(sdk, config, launchLock = withDbosLaunchLock
|
|
|
30683
31244
|
});
|
|
30684
31245
|
unsubscribe = health?.subscribe(managed.invalidate);
|
|
30685
31246
|
health?.start();
|
|
30686
|
-
return fenceDbosAdmissionPool(managed.pool);
|
|
31247
|
+
return fenced ? fenceDbosAdmissionPool(managed.pool) : managed.pool;
|
|
30687
31248
|
};
|
|
30688
31249
|
let pool = createPool();
|
|
31250
|
+
const createFence = (executorId) => {
|
|
31251
|
+
const fencePool = createPool(false);
|
|
31252
|
+
const created = new DbosOwnerFence(() => fencePool, executorId, () => fencePool.end());
|
|
31253
|
+
const owner = getDbosProcessOwner();
|
|
31254
|
+
owner.executorFences ??= new Set;
|
|
31255
|
+
owner.executorFences.add(created);
|
|
31256
|
+
return created;
|
|
31257
|
+
};
|
|
31258
|
+
let fence = createFence(config.executorID);
|
|
31259
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31260
|
+
getDbosProcessOwner().createExecutorFence = () => {
|
|
31261
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31262
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31263
|
+
const next = createFence(executorId);
|
|
31264
|
+
getDbosProcessOwner().executorFence = next;
|
|
31265
|
+
return next;
|
|
31266
|
+
};
|
|
31267
|
+
pool = fence.protectPool(pool);
|
|
30689
31268
|
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30690
31269
|
const launch = async () => {
|
|
31270
|
+
if (fence.closed) {
|
|
31271
|
+
const executorId = `atomic-db-${crypto.randomUUID()}`;
|
|
31272
|
+
getDbosProcessOwner().executorId = executorId;
|
|
31273
|
+
fence = createFence(executorId);
|
|
31274
|
+
getDbosProcessOwner().executorFence = fence;
|
|
31275
|
+
}
|
|
30691
31276
|
if (pool.ended) {
|
|
30692
|
-
pool = createPool();
|
|
30693
|
-
sdk.setConfig({ ...config, systemDatabaseUrl, systemDatabasePool: pool });
|
|
31277
|
+
pool = fence.protectPool(createPool());
|
|
31278
|
+
sdk.setConfig({ ...config, executorID: fence.executorId, systemDatabaseUrl, systemDatabasePool: pool });
|
|
30694
31279
|
}
|
|
30695
31280
|
const urlToEnsure = health === undefined ? systemDatabaseUrl : await health.check();
|
|
30696
31281
|
const result = await import_datasource2.ensurePGDatabase({ urlToEnsure, logger: () => {} });
|
|
@@ -30710,6 +31295,8 @@ var import_datasource2;
|
|
|
30710
31295
|
var init_dbos_admission_config = __esm(() => {
|
|
30711
31296
|
init_dbos_admission_pool();
|
|
30712
31297
|
init_dbos_launch_lock();
|
|
31298
|
+
init_dbos_owner_fence();
|
|
31299
|
+
init_dbos_process_owner();
|
|
30713
31300
|
init_dbos_recoverable_pool();
|
|
30714
31301
|
import_datasource2 = __toESM(require_datasource(), 1);
|
|
30715
31302
|
});
|
|
@@ -35420,7 +36007,7 @@ function isDurableWorkflowResumable(candidate) {
|
|
|
35420
36007
|
}
|
|
35421
36008
|
var FOREIGN_LIVE_WORKFLOW_WINDOW_MS = 120000;
|
|
35422
36009
|
function isLiveRunningWorkflow(candidate, now = Date.now()) {
|
|
35423
|
-
return candidate.status === "running" && now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS;
|
|
36010
|
+
return candidate.status === "running" && (candidate.ownerLiveness === "alive" || now - candidate.updatedAt < FOREIGN_LIVE_WORKFLOW_WINDOW_MS);
|
|
35424
36011
|
}
|
|
35425
36012
|
|
|
35426
36013
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
@@ -35470,7 +36057,7 @@ class InMemoryDurableBackend {
|
|
|
35470
36057
|
workflows = new Map;
|
|
35471
36058
|
promptReservations = new Map;
|
|
35472
36059
|
deletedWorkflowIds = new Set;
|
|
35473
|
-
registerWorkflow(handle) {
|
|
36060
|
+
registerWorkflow(handle, authoritativeOwner = false) {
|
|
35474
36061
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
35475
36062
|
const existing = this.workflows.get(handle.workflowId);
|
|
35476
36063
|
const completedCheckpoints = handle.completedCheckpoints ?? existing?.handle.completedCheckpoints ?? 0;
|
|
@@ -35484,7 +36071,7 @@ class InMemoryDurableBackend {
|
|
|
35484
36071
|
status: handle.status,
|
|
35485
36072
|
...handle.invocationCwd !== undefined ? { invocationCwd: handle.invocationCwd } : existing?.handle.invocationCwd !== undefined ? { invocationCwd: existing.handle.invocationCwd } : {},
|
|
35486
36073
|
...handle.origin !== undefined ? { origin: handle.origin } : existing?.handle.origin !== undefined ? { origin: existing.handle.origin } : {},
|
|
35487
|
-
modelOwner: existing?.handle.modelOwner ?? handle.modelOwner,
|
|
36074
|
+
modelOwner: authoritativeOwner ? handle.modelOwner : existing?.handle.modelOwner ?? handle.modelOwner,
|
|
35488
36075
|
...handle.workflowCwd !== undefined ? { workflowCwd: handle.workflowCwd } : existing?.handle.workflowCwd !== undefined ? { workflowCwd: existing.handle.workflowCwd } : {},
|
|
35489
36076
|
...handle.repositoryRoot !== undefined ? { repositoryRoot: handle.repositoryRoot } : existing?.handle.repositoryRoot !== undefined ? { repositoryRoot: existing.handle.repositoryRoot } : {},
|
|
35490
36077
|
...handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: handle.gitWorktreeRoot } : existing?.handle.gitWorktreeRoot !== undefined ? { gitWorktreeRoot: existing.handle.gitWorktreeRoot } : {},
|
|
@@ -35498,7 +36085,9 @@ class InMemoryDurableBackend {
|
|
|
35498
36085
|
...handle.rootWorkflowId !== undefined ? { rootWorkflowId: handle.rootWorkflowId } : {},
|
|
35499
36086
|
...handle.resumable !== undefined ? { resumable: handle.resumable } : {},
|
|
35500
36087
|
...workflowFailureFields(hasWorkflowFailureMetadata(handle) ? handle : handle.status === "running" ? undefined : existing?.handle),
|
|
35501
|
-
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {}
|
|
36088
|
+
...handle.ownerExecutorId !== undefined ? { ownerExecutorId: handle.ownerExecutorId } : existing?.handle.ownerExecutorId !== undefined ? { ownerExecutorId: existing.handle.ownerExecutorId } : {},
|
|
36089
|
+
...handle.ownerLiveness !== undefined ? { ownerLiveness: handle.ownerLiveness } : {},
|
|
36090
|
+
...handle.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {}
|
|
35502
36091
|
};
|
|
35503
36092
|
if (existing)
|
|
35504
36093
|
existing.handle = full;
|
|
@@ -35648,11 +36237,14 @@ class InMemoryDurableBackend {
|
|
|
35648
36237
|
if (pendingPrompts !== undefined)
|
|
35649
36238
|
this.promptReservations.delete(workflowId);
|
|
35650
36239
|
}
|
|
35651
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
35652
|
-
const
|
|
36240
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
36241
|
+
const record = this.workflows.get(workflowId);
|
|
36242
|
+
const current = record?.handle;
|
|
35653
36243
|
if (current === undefined || !expected.includes(current.status) || expectedUpdatedAt !== undefined && current.updatedAt !== expectedUpdatedAt)
|
|
35654
36244
|
return false;
|
|
35655
36245
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
36246
|
+
if (modelOwner !== undefined && record !== undefined)
|
|
36247
|
+
record.handle = { ...record.handle, modelOwner };
|
|
35656
36248
|
return true;
|
|
35657
36249
|
}
|
|
35658
36250
|
adjustPendingPrompts(workflowId, delta) {
|
|
@@ -35725,6 +36317,7 @@ class InMemoryDurableBackend {
|
|
|
35725
36317
|
pendingStageMessages: h.pendingStageMessages ?? [],
|
|
35726
36318
|
...h.possibleStages !== undefined ? { possibleStages: h.possibleStages } : {},
|
|
35727
36319
|
...h.ownerExecutorId !== undefined ? { ownerExecutorId: h.ownerExecutorId } : {},
|
|
36320
|
+
...h.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
35728
36321
|
...h.sessionFile !== undefined ? { sessionFile: h.sessionFile } : {},
|
|
35729
36322
|
...h.label !== undefined ? { label: h.label } : {},
|
|
35730
36323
|
...h.rootWorkflowId !== undefined ? { rootWorkflowId: h.rootWorkflowId } : {},
|
|
@@ -37791,10 +38384,11 @@ function hasInvocationData(checkpoints, replayKey) {
|
|
|
37791
38384
|
|
|
37792
38385
|
// dist/builtin/workflows/src/engine/run.ts
|
|
37793
38386
|
init_dbos_admission();
|
|
38387
|
+
init_dbos_process_owner();
|
|
37794
38388
|
|
|
37795
38389
|
// dist/builtin/workflows/src/durable/backend-process-owner.ts
|
|
37796
38390
|
var DURABLE_BACKEND_PROCESS_OWNER_KEY = Symbol.for("atomic-workflows/durable-backend-process-owner@1");
|
|
37797
|
-
function
|
|
38391
|
+
function emptyOwner2() {
|
|
37798
38392
|
return {
|
|
37799
38393
|
version: 1,
|
|
37800
38394
|
injectedBackend: undefined,
|
|
@@ -37804,21 +38398,21 @@ function emptyOwner() {
|
|
|
37804
38398
|
warningReported: false
|
|
37805
38399
|
};
|
|
37806
38400
|
}
|
|
37807
|
-
function
|
|
38401
|
+
function ownerBag2() {
|
|
37808
38402
|
return globalThis;
|
|
37809
38403
|
}
|
|
37810
38404
|
function getDurableBackendProcessOwner() {
|
|
37811
|
-
const bag =
|
|
38405
|
+
const bag = ownerBag2();
|
|
37812
38406
|
const existing = bag[DURABLE_BACKEND_PROCESS_OWNER_KEY];
|
|
37813
38407
|
if (existing !== undefined && existing.version === 1)
|
|
37814
38408
|
return existing;
|
|
37815
|
-
const created =
|
|
38409
|
+
const created = emptyOwner2();
|
|
37816
38410
|
bag[DURABLE_BACKEND_PROCESS_OWNER_KEY] = created;
|
|
37817
38411
|
return created;
|
|
37818
38412
|
}
|
|
37819
38413
|
|
|
37820
38414
|
// dist/builtin/workflows/src/durable/dbos-backend.ts
|
|
37821
|
-
import { isDeepStrictEqual as
|
|
38415
|
+
import { isDeepStrictEqual as isDeepStrictEqual3 } from "node:util";
|
|
37822
38416
|
init_dbos_admission();
|
|
37823
38417
|
|
|
37824
38418
|
// dist/builtin/workflows/src/durable/dbos-admission-recovery.ts
|
|
@@ -38487,6 +39081,7 @@ function encodeMetadata(metadata) {
|
|
|
38487
39081
|
} : {},
|
|
38488
39082
|
...metadata.ownerExecutorId !== undefined ? { ownerExecutorId: metadata.ownerExecutorId } : {},
|
|
38489
39083
|
...metadata.transitionClaimId !== undefined ? { transitionClaimId: metadata.transitionClaimId } : {},
|
|
39084
|
+
...metadata.legacyRecoveryPending === true ? { legacyRecoveryPending: true } : {},
|
|
38490
39085
|
...metadata.sessionFile !== undefined ? { sessionFile: metadata.sessionFile } : {},
|
|
38491
39086
|
...metadata.label !== undefined ? { label: metadata.label } : {},
|
|
38492
39087
|
...metadata.rootWorkflowId !== undefined ? { rootWorkflowId: metadata.rootWorkflowId } : {},
|
|
@@ -38549,7 +39144,7 @@ function parseDurableWorkflowMetadata(value, workflowId) {
|
|
|
38549
39144
|
return;
|
|
38550
39145
|
const possibleStages = coercePossibleStages(serialized.possibleStages);
|
|
38551
39146
|
const metadata = value;
|
|
38552
|
-
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
39147
|
+
if (metadata.workflowId !== workflowId || typeof metadata.workflowId !== "string" || typeof metadata.name !== "string" || typeof metadata.inputs !== "object" || metadata.inputs === null || Array.isArray(metadata.inputs) || typeof metadata.status !== "string" || !isDurableWorkflowStatus(metadata.status) || typeof metadata.completedCheckpoints !== "number" || typeof metadata.createdAt !== "number" || typeof metadata.pendingPrompts !== "number" || typeof metadata.promptReservationEpoch !== "string" || typeof metadata.updatedAt !== "number" || metadata.ownerExecutorId !== undefined && typeof metadata.ownerExecutorId !== "string" || metadata.transitionClaimId !== undefined && typeof metadata.transitionClaimId !== "string" || metadata.legacyRecoveryPending !== undefined && metadata.legacyRecoveryPending !== true || metadata.sessionFile !== undefined && typeof metadata.sessionFile !== "string" || metadata.label !== undefined && typeof metadata.label !== "string" || metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string" || metadata.resumable !== undefined && typeof metadata.resumable !== "boolean" || metadata.exited !== undefined && typeof metadata.exited !== "boolean" || metadata.exitReason !== undefined && typeof metadata.exitReason !== "string" || metadata.error !== undefined && typeof metadata.error !== "string" || metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind) || metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode) || metadata.failureRecoverability !== undefined && !isWorkflowFailureRecoverability(metadata.failureRecoverability) || metadata.failureDisposition !== undefined && !isWorkflowFailureDisposition(metadata.failureDisposition) || metadata.failedToolNodeId !== undefined && typeof metadata.failedToolNodeId !== "string" || metadata.invocationCwd !== undefined && typeof metadata.invocationCwd !== "string" || metadata.workflowCwd !== undefined && typeof metadata.workflowCwd !== "string" || metadata.repositoryRoot !== undefined && typeof metadata.repositoryRoot !== "string" || metadata.gitWorktreeRoot !== undefined && typeof metadata.gitWorktreeRoot !== "string")
|
|
38553
39148
|
return;
|
|
38554
39149
|
const { origin, possibleStages: rawPossibleStages, ...metadataWithoutOrigin } = metadata;
|
|
38555
39150
|
return {
|
|
@@ -39024,40 +39619,20 @@ function classifyDbosDeletionTombstone(records, workflowId) {
|
|
|
39024
39619
|
return isCurrentDurableFormat(raw.version) ? "current" : "unknown";
|
|
39025
39620
|
}
|
|
39026
39621
|
|
|
39027
|
-
// dist/builtin/workflows/src/durable/dbos-
|
|
39028
|
-
|
|
39029
|
-
function emptyOwner2() {
|
|
39030
|
-
return {
|
|
39031
|
-
version: 1,
|
|
39032
|
-
state: "uninitialized",
|
|
39033
|
-
configured: undefined,
|
|
39034
|
-
active: undefined,
|
|
39035
|
-
launchPromise: undefined,
|
|
39036
|
-
shutdownPromise: undefined,
|
|
39037
|
-
failure: undefined,
|
|
39038
|
-
wrappers: undefined,
|
|
39039
|
-
leases: new Set
|
|
39040
|
-
};
|
|
39041
|
-
}
|
|
39042
|
-
function ownerBag2() {
|
|
39043
|
-
return globalThis;
|
|
39044
|
-
}
|
|
39045
|
-
function getDbosProcessOwner() {
|
|
39046
|
-
const bag = ownerBag2();
|
|
39047
|
-
const existing = bag[DBOS_PROCESS_OWNER_KEY];
|
|
39048
|
-
if (existing !== undefined && existing.version === 1)
|
|
39049
|
-
return existing;
|
|
39050
|
-
const created = emptyOwner2();
|
|
39051
|
-
bag[DBOS_PROCESS_OWNER_KEY] = created;
|
|
39052
|
-
return created;
|
|
39053
|
-
}
|
|
39622
|
+
// dist/builtin/workflows/src/durable/dbos-backend.ts
|
|
39623
|
+
init_dbos_process_owner();
|
|
39054
39624
|
|
|
39055
39625
|
// dist/builtin/workflows/src/durable/dbos-sdk-handle.ts
|
|
39056
|
-
|
|
39626
|
+
import { AsyncLocalStorage as AsyncLocalStorage4 } from "node:async_hooks";
|
|
39627
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
39628
|
+
init_dbos_owner_fence();
|
|
39629
|
+
init_dbos_process_owner();
|
|
39057
39630
|
function getAtomicExecutorId() {
|
|
39058
|
-
|
|
39631
|
+
const owner = getDbosProcessOwner();
|
|
39632
|
+
owner.executorId ??= `atomic-db-${crypto.randomUUID()}`;
|
|
39633
|
+
return owner.executorId;
|
|
39059
39634
|
}
|
|
39060
|
-
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
39635
|
+
function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, fence, previousExecutorIds = []) {
|
|
39061
39636
|
const checkpointId = (workflowId, stepName) => `${workflowId}:checkpoint:${stepName}`;
|
|
39062
39637
|
async function stepRecord(status, prefix) {
|
|
39063
39638
|
if (status.status !== "SUCCESS")
|
|
@@ -39071,10 +39646,18 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39071
39646
|
const output = status.output === undefined || typeof status.output === "string" ? await dbos.retrieveWorkflow(id).getResult() : status.output;
|
|
39072
39647
|
return { stepName, output, completedAt: status.createdAt };
|
|
39073
39648
|
}
|
|
39074
|
-
|
|
39649
|
+
const raw = {
|
|
39075
39650
|
launch: () => dbos.launch(),
|
|
39076
39651
|
shutdown: () => dbos.shutdown(),
|
|
39077
39652
|
async startWorkflow(workflowId, name, inputs) {
|
|
39653
|
+
if (fence !== undefined) {
|
|
39654
|
+
const existing = await raw.retrieveWorkflow(workflowId);
|
|
39655
|
+
if (existing !== undefined) {
|
|
39656
|
+
if (existing.name !== name || !isDeepStrictEqual2(existing.inputs, inputs))
|
|
39657
|
+
throw new Error("Workflow root identity changed during admission");
|
|
39658
|
+
return;
|
|
39659
|
+
}
|
|
39660
|
+
}
|
|
39078
39661
|
try {
|
|
39079
39662
|
await dbos.startWorkflow(mainWorkflow, { workflowID: workflowId })(name, { ...inputs });
|
|
39080
39663
|
} catch (err) {
|
|
@@ -39093,7 +39676,10 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39093
39676
|
await dbos.cancelWorkflow(workflowId, { cancelChildren: true });
|
|
39094
39677
|
},
|
|
39095
39678
|
async resumeWorkflow(workflowId) {
|
|
39096
|
-
|
|
39679
|
+
if (fence === undefined)
|
|
39680
|
+
await dbos.resumeWorkflow(workflowId);
|
|
39681
|
+
else if (await raw.retrieveWorkflow(workflowId) === undefined)
|
|
39682
|
+
throw new Error("Workflow root is unavailable for resume");
|
|
39097
39683
|
},
|
|
39098
39684
|
async listAllWorkflows() {
|
|
39099
39685
|
const statuses = await dbos.listWorkflows({
|
|
@@ -39122,6 +39708,12 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39122
39708
|
return status !== undefined && (status.workflowID ?? status.workflowId) === id ? stepRecord(status, prefix) : undefined;
|
|
39123
39709
|
},
|
|
39124
39710
|
async recordStepOutput(workflowId, stepName, output) {
|
|
39711
|
+
if (fence !== undefined) {
|
|
39712
|
+
const id = checkpointId(workflowId, stepName);
|
|
39713
|
+
const existing = (await dbos.listWorkflows({ workflowIDs: [id], limit: 1 }))[0];
|
|
39714
|
+
if (existing?.status !== undefined && ["PENDING", "ENQUEUED", "CANCELLED", "ERROR", "DELAYED"].includes(existing.status))
|
|
39715
|
+
await dbos.deleteWorkflows([id], false);
|
|
39716
|
+
}
|
|
39125
39717
|
let handle;
|
|
39126
39718
|
try {
|
|
39127
39719
|
handle = await dbos.startWorkflow(checkpointWorkflow, { workflowID: checkpointId(workflowId, stepName) })(workflowId, stepName, output);
|
|
@@ -39145,6 +39737,135 @@ function createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow) {
|
|
|
39145
39737
|
await dbos.deleteWorkflows([...new Set([workflowId, ...checkpointIds])], true);
|
|
39146
39738
|
}
|
|
39147
39739
|
};
|
|
39740
|
+
if (fence === undefined)
|
|
39741
|
+
return raw;
|
|
39742
|
+
const claims = new AsyncLocalStorage4;
|
|
39743
|
+
const rowAuthority = (workflowId, current) => ({
|
|
39744
|
+
root: workflowId,
|
|
39745
|
+
actor: fence.executorId,
|
|
39746
|
+
...current.kind === "current" ? { owner: current.metadata.ownerExecutorId, generation: current.generation } : {},
|
|
39747
|
+
enroll: current.kind === "unavailable" || current.kind === "current" && isDatabaseExecutor(current.metadata.ownerExecutorId)
|
|
39748
|
+
});
|
|
39749
|
+
const mutate = async (workflowId, callback, step) => fence.write(workflowId, async () => {
|
|
39750
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39751
|
+
const claimed = claims.getStore();
|
|
39752
|
+
if (claimed === undefined && current.kind === "current" && current.metadata.ownerExecutorId !== fence.executorId)
|
|
39753
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
39754
|
+
let output = step === undefined ? undefined : classifyLatestMetadata([{ stepName: step.name, output: step.output }], workflowId);
|
|
39755
|
+
let persistedStep = step;
|
|
39756
|
+
if (step !== undefined && output?.kind === "current" && current.kind === "current" && !step.name.endsWith(":claim") && output.metadata.ownerExecutorId === fence.executorId && output.generation <= current.generation) {
|
|
39757
|
+
const updatedAt = Math.max(current.generation + 1, output.metadata.updatedAt, Date.now());
|
|
39758
|
+
persistedStep = {
|
|
39759
|
+
name: metadataStepName(updatedAt),
|
|
39760
|
+
output: encodeMetadata({ ...output.metadata, updatedAt })
|
|
39761
|
+
};
|
|
39762
|
+
output = classifyLatestMetadata([{ stepName: persistedStep.name, output: persistedStep.output }], workflowId);
|
|
39763
|
+
}
|
|
39764
|
+
const authority = claimed?.authority ?? rowAuthority(workflowId, current);
|
|
39765
|
+
const capability = output?.kind === "current" && step?.name.endsWith(":claim") ? {
|
|
39766
|
+
...authority,
|
|
39767
|
+
claim: output.metadata.transitionClaimId,
|
|
39768
|
+
checkpoint: checkpointId(workflowId, step.name)
|
|
39769
|
+
} : authority;
|
|
39770
|
+
const record = persistedStep;
|
|
39771
|
+
await fence.withRowAuthority(capability, record === undefined ? callback : () => raw.recordStepOutput(workflowId, record.name, record.output));
|
|
39772
|
+
if (claimed !== undefined && output?.kind === "current")
|
|
39773
|
+
claimed.authority = {
|
|
39774
|
+
...capability,
|
|
39775
|
+
owner: output.metadata.ownerExecutorId,
|
|
39776
|
+
generation: output.generation
|
|
39777
|
+
};
|
|
39778
|
+
});
|
|
39779
|
+
return {
|
|
39780
|
+
...raw,
|
|
39781
|
+
executorId: fence.executorId,
|
|
39782
|
+
generationLost: () => fence.invalidated,
|
|
39783
|
+
forkGeneration: () => {
|
|
39784
|
+
const next = getDbosProcessOwner().createExecutorFence?.();
|
|
39785
|
+
if (next === undefined)
|
|
39786
|
+
throw new Error("Workflow ownership fence cannot create a recovery generation");
|
|
39787
|
+
fence.close().catch(() => {});
|
|
39788
|
+
return createRealDbosHandle(dbos, mainWorkflow, checkpointWorkflow, next, [
|
|
39789
|
+
...previousExecutorIds,
|
|
39790
|
+
fence.executorId
|
|
39791
|
+
]);
|
|
39792
|
+
},
|
|
39793
|
+
adoptPreviousGeneration: async (workflowId) => {
|
|
39794
|
+
if (previousExecutorIds.length === 0)
|
|
39795
|
+
return;
|
|
39796
|
+
return await fence.write(workflowId, async () => {
|
|
39797
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39798
|
+
if (current.kind !== "current" || current.metadata.ownerExecutorId === fence.executorId)
|
|
39799
|
+
return;
|
|
39800
|
+
const previousExecutorId = current.metadata.ownerExecutorId;
|
|
39801
|
+
if (previousExecutorId === undefined || !previousExecutorIds.includes(previousExecutorId))
|
|
39802
|
+
throw new Error("Workflow database ownership changed; stale executor writes are refused");
|
|
39803
|
+
const metadata = {
|
|
39804
|
+
...current.metadata,
|
|
39805
|
+
ownerExecutorId: fence.executorId,
|
|
39806
|
+
transitionClaimId: crypto.randomUUID(),
|
|
39807
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
39808
|
+
};
|
|
39809
|
+
const adopted = await fence.recover(previousExecutorId, async () => {
|
|
39810
|
+
await fence.withRowAuthority({
|
|
39811
|
+
...rowAuthority(workflowId, current),
|
|
39812
|
+
claim: metadata.transitionClaimId,
|
|
39813
|
+
checkpoint: checkpointId(workflowId, claimMetadataStepName(current.generation))
|
|
39814
|
+
}, () => raw.recordStepOutput(workflowId, claimMetadataStepName(current.generation), encodeMetadata(metadata)));
|
|
39815
|
+
return metadata;
|
|
39816
|
+
});
|
|
39817
|
+
if (adopted === undefined)
|
|
39818
|
+
throw new Error("Workflow previous execution generation is still active");
|
|
39819
|
+
return adopted;
|
|
39820
|
+
}, true);
|
|
39821
|
+
},
|
|
39822
|
+
enrollLegacyWorkflow: async (workflowId, modelOwner) => fence.write(workflowId, async () => {
|
|
39823
|
+
const current = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39824
|
+
if (current.kind !== "current" || isDatabaseExecutor(current.metadata.ownerExecutorId) && current.metadata.legacyRecoveryPending !== true || !["running", "paused", "blocked"].includes(current.metadata.status) || current.metadata.rootWorkflowId !== undefined && current.metadata.rootWorkflowId !== workflowId)
|
|
39825
|
+
return;
|
|
39826
|
+
const metadata = {
|
|
39827
|
+
...current.metadata,
|
|
39828
|
+
modelOwner,
|
|
39829
|
+
ownerExecutorId: fence.executorId,
|
|
39830
|
+
status: "blocked",
|
|
39831
|
+
resumable: true,
|
|
39832
|
+
legacyRecoveryPending: true,
|
|
39833
|
+
updatedAt: Math.max(Date.now(), current.metadata.updatedAt + 1)
|
|
39834
|
+
};
|
|
39835
|
+
const retry = isDatabaseExecutor(current.metadata.ownerExecutorId);
|
|
39836
|
+
const transitionClaimId = crypto.randomUUID();
|
|
39837
|
+
const stepName = retry ? claimMetadataStepName(current.generation) : metadataStepName(Math.max(current.generation + 1, metadata.updatedAt));
|
|
39838
|
+
const persist = () => fence.withRowAuthority({
|
|
39839
|
+
...rowAuthority(workflowId, current),
|
|
39840
|
+
enroll: true,
|
|
39841
|
+
...retry ? { claim: transitionClaimId, checkpoint: checkpointId(workflowId, stepName) } : {}
|
|
39842
|
+
}, () => raw.recordStepOutput(workflowId, stepName, encodeMetadata({ ...metadata, ...retry ? { transitionClaimId } : {} })));
|
|
39843
|
+
if (retry) {
|
|
39844
|
+
const recovered = await fence.recover(current.metadata.ownerExecutorId, async () => {
|
|
39845
|
+
await persist();
|
|
39846
|
+
return true;
|
|
39847
|
+
});
|
|
39848
|
+
if (recovered !== true)
|
|
39849
|
+
return;
|
|
39850
|
+
} else
|
|
39851
|
+
await persist();
|
|
39852
|
+
return metadata;
|
|
39853
|
+
}, true),
|
|
39854
|
+
ownerLiveness: (executorId) => fence.liveness(executorId),
|
|
39855
|
+
withWorkflowClaim: (workflowId, callback) => fence.write(workflowId, async () => {
|
|
39856
|
+
const authoritative = classifyLatestMetadata(await raw.listStepRecords(workflowId), workflowId);
|
|
39857
|
+
const claim = () => claims.run({ authority: { ...rowAuthority(workflowId, authoritative), claim: crypto.randomUUID() } }, callback);
|
|
39858
|
+
if (authoritative.kind === "current" && authoritative.metadata.status === "running" && authoritative.metadata.ownerExecutorId !== fence.executorId) {
|
|
39859
|
+
return await fence.recover(authoritative.metadata.ownerExecutorId, claim) ?? false;
|
|
39860
|
+
}
|
|
39861
|
+
return await claim();
|
|
39862
|
+
}, true),
|
|
39863
|
+
startWorkflow: (id, name, inputs) => mutate(id, () => raw.startWorkflow(id, name, inputs)),
|
|
39864
|
+
cancelWorkflow: (id) => mutate(id, () => raw.cancelWorkflow(id)),
|
|
39865
|
+
resumeWorkflow: (id) => mutate(id, () => raw.resumeWorkflow(id)),
|
|
39866
|
+
recordStepOutput: (id, step, output) => mutate(id, () => raw.recordStepOutput(id, step, output), { name: step, output }),
|
|
39867
|
+
deleteWorkflowData: (id) => mutate(id, () => raw.deleteWorkflowData(id))
|
|
39868
|
+
};
|
|
39148
39869
|
}
|
|
39149
39870
|
function isDbosDuplicateWorkflowError(err) {
|
|
39150
39871
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -39171,6 +39892,7 @@ function statusToInfo(status, fallbackId) {
|
|
|
39171
39892
|
}
|
|
39172
39893
|
|
|
39173
39894
|
// dist/builtin/workflows/src/durable/dbos-system-database-url.ts
|
|
39895
|
+
init_dbos_process_owner();
|
|
39174
39896
|
function nonEmptyTrimmed(value) {
|
|
39175
39897
|
const trimmed = value?.trim();
|
|
39176
39898
|
return trimmed === undefined || trimmed.length === 0 ? undefined : trimmed;
|
|
@@ -39185,9 +39907,7 @@ class DbosSystemDatabaseConflictError extends Error {
|
|
|
39185
39907
|
function redactDatabaseUrl(url) {
|
|
39186
39908
|
try {
|
|
39187
39909
|
const parsed = new URL(url);
|
|
39188
|
-
|
|
39189
|
-
parsed.password = "***";
|
|
39190
|
-
return parsed.toString();
|
|
39910
|
+
return parsed.host;
|
|
39191
39911
|
} catch {
|
|
39192
39912
|
return "the requested database";
|
|
39193
39913
|
}
|
|
@@ -39227,6 +39947,8 @@ async function configureDbosDurableBackend(config) {
|
|
|
39227
39947
|
const sdk = await importDbosSdk();
|
|
39228
39948
|
const owner = getDbosProcessOwner();
|
|
39229
39949
|
const existing = owner.wrappers;
|
|
39950
|
+
if (existing !== undefined && owner.executorFence === undefined)
|
|
39951
|
+
throw new Error("Existing workflow executor has no database ownership fence; restart the host before recovering workflows");
|
|
39230
39952
|
let launch = owner.active?.launch ?? (() => sdk.launch());
|
|
39231
39953
|
let checkReady;
|
|
39232
39954
|
if (existing === undefined) {
|
|
@@ -39249,12 +39971,30 @@ async function configureDbosDurableBackend(config) {
|
|
|
39249
39971
|
if (existing === undefined) {
|
|
39250
39972
|
owner.wrappers = { mainWorkflow, checkpointWorkflow };
|
|
39251
39973
|
}
|
|
39974
|
+
let backendFence = owner.executorFence;
|
|
39975
|
+
let backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), {
|
|
39976
|
+
checkReady
|
|
39977
|
+
});
|
|
39252
39978
|
return {
|
|
39253
|
-
backend
|
|
39254
|
-
|
|
39255
|
-
}
|
|
39256
|
-
launch
|
|
39257
|
-
|
|
39979
|
+
get backend() {
|
|
39980
|
+
return backend;
|
|
39981
|
+
},
|
|
39982
|
+
launch: async () => {
|
|
39983
|
+
await launch();
|
|
39984
|
+
if (backendFence !== owner.executorFence && backendFence?.closed) {
|
|
39985
|
+
backendFence = owner.executorFence;
|
|
39986
|
+
backend = new DbosDurableBackend(createRealDbosHandle(sdk, mainWorkflow, checkpointWorkflow, backendFence), { checkReady });
|
|
39987
|
+
}
|
|
39988
|
+
},
|
|
39989
|
+
shutdown: async () => {
|
|
39990
|
+
const results = await Promise.allSettled([
|
|
39991
|
+
sdk.shutdown(),
|
|
39992
|
+
...[...owner.executorFences ?? []].map((fence) => fence.close())
|
|
39993
|
+
]);
|
|
39994
|
+
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason instanceof Error ? result.reason : new Error(String(result.reason)));
|
|
39995
|
+
if (errors.length > 0)
|
|
39996
|
+
throw new AggregateError(errors, "Workflow database shutdown failed");
|
|
39997
|
+
}
|
|
39258
39998
|
};
|
|
39259
39999
|
}
|
|
39260
40000
|
async function importDbosSdk() {
|
|
@@ -39287,13 +40027,18 @@ class DbosDurableBackend {
|
|
|
39287
40027
|
unavailableAdmissions = new Set;
|
|
39288
40028
|
unavailableCheckpoints = new Set;
|
|
39289
40029
|
pendingRecoveryAdmissions = new Set;
|
|
40030
|
+
ownerLiveness = new Map;
|
|
39290
40031
|
admissionMetadataAttempted = new Set;
|
|
39291
40032
|
admissionSettlements = new Map;
|
|
39292
40033
|
admissionRecoveries = new Map;
|
|
39293
40034
|
admissionRecoveryControllers = new Map;
|
|
40035
|
+
fixedGeneration = false;
|
|
40036
|
+
generationIsCurrent;
|
|
40037
|
+
rebindInitialAdmission;
|
|
40038
|
+
admitted = false;
|
|
39294
40039
|
constructor(sdk, options) {
|
|
39295
40040
|
this.sdk = sdk;
|
|
39296
|
-
this.executorId = options?.executorId ?? getAtomicExecutorId();
|
|
40041
|
+
this.executorId = options?.executorId ?? sdk.executorId ?? getAtomicExecutorId();
|
|
39297
40042
|
this.onUnavailable = options?.onUnavailable;
|
|
39298
40043
|
this.checkReady = options?.checkReady;
|
|
39299
40044
|
this.promptReservations = new DbosPromptReservationTracker({
|
|
@@ -39304,6 +40049,55 @@ class DbosDurableBackend {
|
|
|
39304
40049
|
}
|
|
39305
40050
|
});
|
|
39306
40051
|
}
|
|
40052
|
+
async enrollLegacyWorkflow(workflowId, options) {
|
|
40053
|
+
if (options.olderWorkersStopped !== true || this.sdk.enrollLegacyWorkflow === undefined)
|
|
40054
|
+
return false;
|
|
40055
|
+
this.ensureGeneration();
|
|
40056
|
+
return await boundedAdmission((signal) => dbosAdmissionContext.run(signal, async () => {
|
|
40057
|
+
signal.throwIfAborted();
|
|
40058
|
+
const metadata = await this.sdk.enrollLegacyWorkflow(workflowId, options.modelOwner);
|
|
40059
|
+
signal.throwIfAborted();
|
|
40060
|
+
if (metadata === undefined)
|
|
40061
|
+
return false;
|
|
40062
|
+
this.applyMetadata(workflowId, metadata);
|
|
40063
|
+
this.current.add(workflowId);
|
|
40064
|
+
return true;
|
|
40065
|
+
}), options.signal);
|
|
40066
|
+
}
|
|
40067
|
+
executionView() {
|
|
40068
|
+
if (this.fixedGeneration || this.sdk.forkGeneration === undefined)
|
|
40069
|
+
return this;
|
|
40070
|
+
this.ensureGeneration();
|
|
40071
|
+
const view = new DbosDurableBackend(this.sdk, {
|
|
40072
|
+
executorId: this.executorId,
|
|
40073
|
+
checkReady: this.checkReady,
|
|
40074
|
+
onUnavailable: this.onUnavailable
|
|
40075
|
+
});
|
|
40076
|
+
const reservations = view.promptReservations;
|
|
40077
|
+
Object.assign(view, this);
|
|
40078
|
+
Object.assign(view, {
|
|
40079
|
+
promptReservations: reservations,
|
|
40080
|
+
fixedGeneration: true,
|
|
40081
|
+
generationIsCurrent: () => this.sdk === view.sdk,
|
|
40082
|
+
rebindInitialAdmission: () => {
|
|
40083
|
+
this.ensureGeneration();
|
|
40084
|
+
view.sdk = this.sdk;
|
|
40085
|
+
view.executorId = this.executorId;
|
|
40086
|
+
}
|
|
40087
|
+
});
|
|
40088
|
+
return view;
|
|
40089
|
+
}
|
|
40090
|
+
ensureGeneration() {
|
|
40091
|
+
if (this.fixedGeneration) {
|
|
40092
|
+
if (this.generationIsCurrent?.() === false)
|
|
40093
|
+
throw new DbosDependencyError("Workflow database ownership generation changed; stale execution writes are refused");
|
|
40094
|
+
return;
|
|
40095
|
+
}
|
|
40096
|
+
if (this.sdk.generationLost?.() && this.sdk.forkGeneration !== undefined) {
|
|
40097
|
+
this.sdk = this.sdk.forkGeneration();
|
|
40098
|
+
this.executorId = this.sdk.executorId ?? this.executorId;
|
|
40099
|
+
}
|
|
40100
|
+
}
|
|
39307
40101
|
isAdmissionUnavailable(workflowId) {
|
|
39308
40102
|
return this.unavailableAdmissions.has(workflowId);
|
|
39309
40103
|
}
|
|
@@ -39320,6 +40114,8 @@ class DbosDurableBackend {
|
|
|
39320
40114
|
return this.admissionSettlements.has(workflowId);
|
|
39321
40115
|
}
|
|
39322
40116
|
admitWorkflow(workflowId, registration, signal) {
|
|
40117
|
+
if (!this.admitted)
|
|
40118
|
+
this.rebindInitialAdmission?.();
|
|
39323
40119
|
const pending = this.performAdmission(workflowId, registration, signal);
|
|
39324
40120
|
this.admissionSettlements.set(workflowId, pending);
|
|
39325
40121
|
pending.then(() => {
|
|
@@ -39327,6 +40123,7 @@ class DbosDurableBackend {
|
|
|
39327
40123
|
this.admissionSettlements.delete(workflowId);
|
|
39328
40124
|
}, () => {});
|
|
39329
40125
|
return pending.then(async () => {
|
|
40126
|
+
this.admitted = true;
|
|
39330
40127
|
await dbosAdmissionContext.run(signal, () => this.flush(workflowId));
|
|
39331
40128
|
signal.throwIfAborted();
|
|
39332
40129
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
@@ -39347,12 +40144,13 @@ class DbosDurableBackend {
|
|
|
39347
40144
|
await this.flushWrites(workflowId);
|
|
39348
40145
|
signal.throwIfAborted();
|
|
39349
40146
|
}), signal);
|
|
40147
|
+
this.ensureGeneration();
|
|
39350
40148
|
this.admissionUnavailable = false;
|
|
39351
40149
|
this.unavailableAdmissions.delete(workflowId);
|
|
39352
40150
|
this.unavailableCheckpoints.delete(workflowId);
|
|
39353
40151
|
this.pendingRecoveryAdmissions.delete(workflowId);
|
|
39354
40152
|
} catch (error) {
|
|
39355
|
-
if (isDbosDependencyError(error)) {
|
|
40153
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
39356
40154
|
this.admissionMetadataAttempted.delete(workflowId);
|
|
39357
40155
|
this.admissionUnavailable = true;
|
|
39358
40156
|
this.unavailableAdmissions.add(workflowId);
|
|
@@ -39378,6 +40176,7 @@ class DbosDurableBackend {
|
|
|
39378
40176
|
});
|
|
39379
40177
|
}
|
|
39380
40178
|
registerWorkflow(handle) {
|
|
40179
|
+
this.ensureGeneration();
|
|
39381
40180
|
if (this.invalid.has(handle.workflowId)) {
|
|
39382
40181
|
throw new DurableNestedTopologyError(`workflow ${handle.workflowId} contains malformed current DBOS records`);
|
|
39383
40182
|
}
|
|
@@ -39419,6 +40218,7 @@ class DbosDurableBackend {
|
|
|
39419
40218
|
return persisted;
|
|
39420
40219
|
}
|
|
39421
40220
|
recordCheckpoint(checkpoint) {
|
|
40221
|
+
this.ensureGeneration();
|
|
39422
40222
|
if (!this.isWorkflowLoadable(checkpoint.workflowId))
|
|
39423
40223
|
return;
|
|
39424
40224
|
this.mem.recordCheckpoint(checkpoint);
|
|
@@ -39447,7 +40247,7 @@ class DbosDurableBackend {
|
|
|
39447
40247
|
});
|
|
39448
40248
|
signal.throwIfAborted();
|
|
39449
40249
|
}), options?.signal ?? dbosAdmissionContext.getStore(), undefined, "Workflow database checkpoint timed out. Restore PostgreSQL and inspect the run before resuming; external outcomes may be unknown.").catch((error) => {
|
|
39450
|
-
if (isDbosDependencyError(error)) {
|
|
40250
|
+
if (isDbosDependencyError(error) && this.generationIsCurrent?.() !== false) {
|
|
39451
40251
|
this.unavailableCheckpoints.add(checkpoint.workflowId);
|
|
39452
40252
|
const errors = this.writeErrors.get(checkpoint.workflowId);
|
|
39453
40253
|
if (errors !== undefined)
|
|
@@ -39495,12 +40295,14 @@ class DbosDurableBackend {
|
|
|
39495
40295
|
return this.mem.listCheckpoints(workflowId);
|
|
39496
40296
|
}
|
|
39497
40297
|
getWorkflow(workflowId) {
|
|
39498
|
-
|
|
40298
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
40299
|
+
return handle === undefined || this.sdk.ownerLiveness === undefined ? handle : { ...handle, ownerLiveness: this.ownerLiveness.get(handle.ownerExecutorId ?? "") ?? "unknown" };
|
|
39499
40300
|
}
|
|
39500
40301
|
getLoadableWorkflow(workflowId) {
|
|
39501
|
-
return this.isWorkflowLoadable(workflowId) ? this.
|
|
40302
|
+
return this.isWorkflowLoadable(workflowId) ? this.getWorkflow(workflowId) : undefined;
|
|
39502
40303
|
}
|
|
39503
40304
|
setWorkflowStatus(workflowId, status, pendingPrompts, resumable, failure) {
|
|
40305
|
+
this.ensureGeneration();
|
|
39504
40306
|
if (status === "cancelled" || status === "completed" || resumable === false) {
|
|
39505
40307
|
this.admissionRecoveryControllers.get(workflowId)?.abort(new Error("Workflow recovery superseded by terminal control."));
|
|
39506
40308
|
}
|
|
@@ -39524,9 +40326,10 @@ class DbosDurableBackend {
|
|
|
39524
40326
|
await this.writeMetadata(workflowId);
|
|
39525
40327
|
});
|
|
39526
40328
|
}
|
|
39527
|
-
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt) {
|
|
40329
|
+
async transitionWorkflowStatus(workflowId, expected, status, pendingPrompts, resumable, expectedUpdatedAt, modelOwner) {
|
|
40330
|
+
this.ensureGeneration();
|
|
39528
40331
|
let records = [];
|
|
39529
|
-
|
|
40332
|
+
const transition = () => transitionDbosWorkflowStatus({
|
|
39530
40333
|
expectedStatuses: expected,
|
|
39531
40334
|
status,
|
|
39532
40335
|
flush: () => this.flush(workflowId),
|
|
@@ -39540,14 +40343,25 @@ class DbosDurableBackend {
|
|
|
39540
40343
|
const pendingPrompts = this.promptReservations.hydrate(workflowId, entry.pendingPrompts, records, entry.promptReservationEpoch);
|
|
39541
40344
|
this.applyMetadata(workflowId, { ...entry, pendingPrompts });
|
|
39542
40345
|
},
|
|
39543
|
-
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable),
|
|
40346
|
+
claim: (authoritative, generation) => this.claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner),
|
|
39544
40347
|
write: async () => {
|
|
40348
|
+
const handle = this.mem.getWorkflow(workflowId);
|
|
40349
|
+
if (handle !== undefined)
|
|
40350
|
+
this.mem.registerWorkflow({
|
|
40351
|
+
...handle,
|
|
40352
|
+
legacyRecoveryPending: undefined,
|
|
40353
|
+
...modelOwner !== undefined ? { modelOwner } : {}
|
|
40354
|
+
}, true);
|
|
39545
40355
|
this.setWorkflowStatus(workflowId, status, pendingPrompts, resumable);
|
|
39546
40356
|
await this.flush(workflowId);
|
|
39547
40357
|
}
|
|
39548
40358
|
});
|
|
40359
|
+
if (this.sdk.withWorkflowClaim === undefined)
|
|
40360
|
+
return await transition();
|
|
40361
|
+
await this.flush(workflowId);
|
|
40362
|
+
return await this.sdk.withWorkflowClaim(workflowId, transition);
|
|
39549
40363
|
}
|
|
39550
|
-
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable) {
|
|
40364
|
+
async claimStatusTransition(workflowId, authoritative, generation, status, pendingPrompts, resumable, modelOwner) {
|
|
39551
40365
|
const stepName = claimMetadataStepName(generation);
|
|
39552
40366
|
const transitionClaimId = crypto.randomUUID();
|
|
39553
40367
|
const claim = {
|
|
@@ -39555,6 +40369,8 @@ class DbosDurableBackend {
|
|
|
39555
40369
|
status,
|
|
39556
40370
|
...pendingPrompts !== undefined ? { pendingPrompts } : {},
|
|
39557
40371
|
...resumable !== undefined ? { resumable } : {},
|
|
40372
|
+
...modelOwner === undefined ? {} : { modelOwner },
|
|
40373
|
+
legacyRecoveryPending: undefined,
|
|
39558
40374
|
ownerExecutorId: this.executorId,
|
|
39559
40375
|
transitionClaimId,
|
|
39560
40376
|
updatedAt: Math.max(Date.now(), authoritative.updatedAt + 1)
|
|
@@ -39696,7 +40512,7 @@ class DbosDurableBackend {
|
|
|
39696
40512
|
this.mem.recordCheckpoint(classified.checkpoint);
|
|
39697
40513
|
}
|
|
39698
40514
|
const handle = this.getLoadableWorkflow(workflowId);
|
|
39699
|
-
return handle === undefined ? { kind: "malformed" } :
|
|
40515
|
+
return handle === undefined ? { kind: "malformed" } : await this.withOwnerLiveness(handle);
|
|
39700
40516
|
}
|
|
39701
40517
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
39702
40518
|
if (info !== undefined)
|
|
@@ -39719,6 +40535,7 @@ class DbosDurableBackend {
|
|
|
39719
40535
|
}
|
|
39720
40536
|
}
|
|
39721
40537
|
reconcileWorkflowAdmission(workflowId, signal) {
|
|
40538
|
+
this.ensureGeneration();
|
|
39722
40539
|
signal?.throwIfAborted();
|
|
39723
40540
|
if (!this.unavailableAdmissions.has(workflowId) && !this.unavailableCheckpoints.has(workflowId))
|
|
39724
40541
|
return Promise.resolve();
|
|
@@ -39733,11 +40550,13 @@ class DbosDurableBackend {
|
|
|
39733
40550
|
return;
|
|
39734
40551
|
if (metadata.resumable === false || metadata.status === "cancelled" || metadata.status === "completed")
|
|
39735
40552
|
return;
|
|
39736
|
-
if (!await reconcileDbosAdmission(this.sdk, this.promptReservations.metadata(workflowId, metadata), this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
40553
|
+
if (!await reconcileDbosAdmission(this.sdk, { ...this.promptReservations.metadata(workflowId, metadata), ownerExecutorId: this.executorId }, this.unavailableAdmissions.has(workflowId), this.checkReady)) {
|
|
39737
40554
|
await this.suppressWorkflow(workflowId);
|
|
39738
40555
|
return;
|
|
39739
40556
|
}
|
|
39740
40557
|
admissionSignal.throwIfAborted();
|
|
40558
|
+
await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
40559
|
+
admissionSignal.throwIfAborted();
|
|
39741
40560
|
const info = await this.sdk.retrieveWorkflow(workflowId);
|
|
39742
40561
|
if (info === undefined)
|
|
39743
40562
|
return;
|
|
@@ -39774,7 +40593,7 @@ class DbosDurableBackend {
|
|
|
39774
40593
|
return { kind: deletion === "current" ? "deleted" : "malformed" };
|
|
39775
40594
|
}
|
|
39776
40595
|
const metadata = classifyLatestMetadata(records, info.workflowId);
|
|
39777
|
-
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !
|
|
40596
|
+
if (metadata.kind !== "current" || expected !== undefined && (metadata.metadata.name !== expected.name || !isDeepStrictEqual3(metadata.metadata.inputs, expected.inputs))) {
|
|
39778
40597
|
await this.suppressWorkflow(info.workflowId);
|
|
39779
40598
|
return { kind: "malformed" };
|
|
39780
40599
|
}
|
|
@@ -39808,10 +40627,17 @@ class DbosDurableBackend {
|
|
|
39808
40627
|
});
|
|
39809
40628
|
const handle = this.getLoadableWorkflow(info.workflowId);
|
|
39810
40629
|
if (handle !== undefined)
|
|
39811
|
-
return
|
|
40630
|
+
return await this.withOwnerLiveness(handle);
|
|
39812
40631
|
await this.suppressWorkflow(info.workflowId);
|
|
39813
40632
|
return { kind: "malformed" };
|
|
39814
40633
|
}
|
|
40634
|
+
async withOwnerLiveness(handle) {
|
|
40635
|
+
if (this.sdk.ownerLiveness === undefined)
|
|
40636
|
+
return { kind: "current", handle };
|
|
40637
|
+
const ownerLiveness = await this.sdk.ownerLiveness(handle.ownerExecutorId);
|
|
40638
|
+
this.ownerLiveness.set(handle.ownerExecutorId ?? "", ownerLiveness);
|
|
40639
|
+
return { kind: "current", handle: { ...handle, ownerLiveness } };
|
|
40640
|
+
}
|
|
39815
40641
|
async suppressWorkflow(workflowId) {
|
|
39816
40642
|
this.invalid.add(workflowId);
|
|
39817
40643
|
this.current.delete(workflowId);
|
|
@@ -39825,10 +40651,21 @@ class DbosDurableBackend {
|
|
|
39825
40651
|
return Promise.resolve();
|
|
39826
40652
|
const previous = this.writeQueues.get(workflowId) ?? Promise.resolve();
|
|
39827
40653
|
const next = previous.then(async () => {
|
|
40654
|
+
this.ensureGeneration();
|
|
40655
|
+
const adopted = await this.sdk.adoptPreviousGeneration?.(workflowId);
|
|
40656
|
+
const local = this.mem.getWorkflow(workflowId);
|
|
40657
|
+
if (adopted !== undefined && local !== undefined)
|
|
40658
|
+
this.mem.registerWorkflow({
|
|
40659
|
+
...local,
|
|
40660
|
+
ownerExecutorId: adopted.ownerExecutorId,
|
|
40661
|
+
updatedAt: Math.max(local.updatedAt, adopted.updatedAt + 1)
|
|
40662
|
+
}, true);
|
|
39828
40663
|
if (!signal?.aborted)
|
|
39829
40664
|
await fn();
|
|
39830
40665
|
});
|
|
39831
40666
|
const tracked = next.catch((err) => {
|
|
40667
|
+
if (this.generationIsCurrent?.() === false)
|
|
40668
|
+
return;
|
|
39832
40669
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
39833
40670
|
const errors = this.writeErrors.get(workflowId) ?? [];
|
|
39834
40671
|
errors.push(error);
|
|
@@ -39865,7 +40702,7 @@ class DbosDurableBackend {
|
|
|
39865
40702
|
applyMetadata(workflowId, metadata) {
|
|
39866
40703
|
if (metadata.workflowId !== workflowId)
|
|
39867
40704
|
return;
|
|
39868
|
-
this.mem.registerWorkflow(metadata);
|
|
40705
|
+
this.mem.registerWorkflow(metadata, true);
|
|
39869
40706
|
}
|
|
39870
40707
|
}
|
|
39871
40708
|
|
|
@@ -39901,7 +40738,7 @@ import { tmpdir, uptime } from "node:os";
|
|
|
39901
40738
|
import { basename as basename3, dirname as dirname3, isAbsolute as isAbsolute3, join as join4, relative as relative2, sep as sep2 } from "node:path";
|
|
39902
40739
|
|
|
39903
40740
|
// dist/builtin/workflows/src/durable/dbos-embedded-postgres-root.ts
|
|
39904
|
-
import { createHash as
|
|
40741
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
39905
40742
|
import {
|
|
39906
40743
|
chmod,
|
|
39907
40744
|
chown,
|
|
@@ -40384,8 +41221,8 @@ async function snapshotSourceRuntime(root, publisher, progress) {
|
|
|
40384
41221
|
const rootStat = await lstat(root);
|
|
40385
41222
|
assertRuntimeRoot(root, rootStat);
|
|
40386
41223
|
const rootRealPath = await realpath(root);
|
|
40387
|
-
const sourceHash =
|
|
40388
|
-
const sealedHash =
|
|
41224
|
+
const sourceHash = createHash5("sha256");
|
|
41225
|
+
const sealedHash = createHash5("sha256");
|
|
40389
41226
|
await hashSourceEntry(sourceHash, sealedHash, root, ".", rootRealPath, publisher, progress);
|
|
40390
41227
|
return { sourceIdentity: sourceHash.digest("hex"), sealedIdentity: sealedHash.digest("hex") };
|
|
40391
41228
|
}
|
|
@@ -40650,7 +41487,7 @@ async function snapshotSealedManifest(root, progress) {
|
|
|
40650
41487
|
const rootStat = await lstat(root);
|
|
40651
41488
|
assertRuntimeRoot(root, rootStat);
|
|
40652
41489
|
const rootRealPath = await realpath(root);
|
|
40653
|
-
const hash =
|
|
41490
|
+
const hash = createHash5("sha256");
|
|
40654
41491
|
const entries = [];
|
|
40655
41492
|
await hashSealedEntry(hash, root, ".", rootRealPath, progress, entries);
|
|
40656
41493
|
return { version: 1, sealedIdentity: hash.digest("hex"), entries };
|
|
@@ -40884,6 +41721,13 @@ function resolveEmbeddedPostgresTarget(host) {
|
|
|
40884
41721
|
init_dbos_admission();
|
|
40885
41722
|
var HEALTH_INTERVAL_MS = 5000;
|
|
40886
41723
|
var RECOVERY_ATTEMPTS = 3;
|
|
41724
|
+
function isMonitoringConnectionTimeout(error) {
|
|
41725
|
+
if (!(error instanceof Error))
|
|
41726
|
+
return false;
|
|
41727
|
+
if ("code" in error)
|
|
41728
|
+
return error.code === "ETIMEDOUT";
|
|
41729
|
+
return error.message === "timeout expired" || error.message === "Connection terminated due to connection timeout";
|
|
41730
|
+
}
|
|
40887
41731
|
function isMonitoringConnectionFailure(error) {
|
|
40888
41732
|
if (!(error instanceof Error))
|
|
40889
41733
|
return false;
|
|
@@ -40904,6 +41748,7 @@ class PostgresHealth {
|
|
|
40904
41748
|
attempts = 0;
|
|
40905
41749
|
nextRecoveryAt = 0;
|
|
40906
41750
|
failure;
|
|
41751
|
+
healthySinceFailure = false;
|
|
40907
41752
|
revision = 0;
|
|
40908
41753
|
listeners = new Set;
|
|
40909
41754
|
constructor(operations) {
|
|
@@ -40912,6 +41757,18 @@ class PostgresHealth {
|
|
|
40912
41757
|
get lastFailure() {
|
|
40913
41758
|
return this.failure;
|
|
40914
41759
|
}
|
|
41760
|
+
get endpoint() {
|
|
41761
|
+
return this.operations.url?.();
|
|
41762
|
+
}
|
|
41763
|
+
retainFailure(error) {
|
|
41764
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
41765
|
+
const safe = redactedDatabaseMessage(message, this.endpoint);
|
|
41766
|
+
this.failure = new Error(safe);
|
|
41767
|
+
this.healthySinceFailure = false;
|
|
41768
|
+
}
|
|
41769
|
+
dependencyFailure(message) {
|
|
41770
|
+
return new DbosDependencyError(message, this.endpoint === undefined ? undefined : { provider: "managed", url: this.endpoint, failure: this.failure?.message });
|
|
41771
|
+
}
|
|
40915
41772
|
subscribe(listener) {
|
|
40916
41773
|
this.listeners.add(listener);
|
|
40917
41774
|
return () => this.listeners.delete(listener);
|
|
@@ -40934,12 +41791,15 @@ class PostgresHealth {
|
|
|
40934
41791
|
await this.operations.validate?.(client);
|
|
40935
41792
|
}
|
|
40936
41793
|
} catch (error) {
|
|
40937
|
-
this.
|
|
41794
|
+
this.retainFailure(error);
|
|
40938
41795
|
if (!isQueryReadTimeout(error)) {
|
|
40939
41796
|
this.invalidate();
|
|
40940
|
-
|
|
41797
|
+
if (this.endpoint === undefined)
|
|
41798
|
+
throw error;
|
|
41799
|
+
const failure = this.dependencyFailure("Managed PostgreSQL connection validation refused. Preserve its data and ownership records.");
|
|
41800
|
+
throw error instanceof DbosDependencyError ? failure : new Error(failure.message);
|
|
40941
41801
|
}
|
|
40942
|
-
throw
|
|
41802
|
+
throw this.dependencyFailure("Managed PostgreSQL did not answer a connection identity check in time.");
|
|
40943
41803
|
}
|
|
40944
41804
|
}
|
|
40945
41805
|
check() {
|
|
@@ -40978,9 +41838,9 @@ class PostgresHealth {
|
|
|
40978
41838
|
try {
|
|
40979
41839
|
identity = await this.operations.probe();
|
|
40980
41840
|
} catch (retryError) {
|
|
40981
|
-
if (!isMonitoringConnectionFailure(retryError))
|
|
41841
|
+
if (isMonitoringConnectionTimeout(retryError) || !isMonitoringConnectionFailure(retryError))
|
|
40982
41842
|
throw retryError;
|
|
40983
|
-
this.
|
|
41843
|
+
this.retainFailure(retryError);
|
|
40984
41844
|
return;
|
|
40985
41845
|
}
|
|
40986
41846
|
}
|
|
@@ -40995,6 +41855,7 @@ class PostgresHealth {
|
|
|
40995
41855
|
this.available = identity;
|
|
40996
41856
|
this.attempts = 0;
|
|
40997
41857
|
this.nextRecoveryAt = 0;
|
|
41858
|
+
this.healthySinceFailure = true;
|
|
40998
41859
|
return identity.url;
|
|
40999
41860
|
}
|
|
41000
41861
|
async refresh() {
|
|
@@ -41003,20 +41864,22 @@ class PostgresHealth {
|
|
|
41003
41864
|
if (ready !== undefined)
|
|
41004
41865
|
return ready;
|
|
41005
41866
|
} catch (error) {
|
|
41006
|
-
if (isQueryReadTimeout(error)) {
|
|
41007
|
-
this.
|
|
41008
|
-
throw this.
|
|
41867
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error)) {
|
|
41868
|
+
this.retainFailure(new Error("Managed PostgreSQL did not answer a health check in time."));
|
|
41869
|
+
throw this.dependencyFailure("Managed PostgreSQL did not answer a health check in time.");
|
|
41009
41870
|
}
|
|
41010
|
-
this.
|
|
41871
|
+
this.retainFailure(error);
|
|
41011
41872
|
if (!(error instanceof Error && ("code" in error) && error.code === "53300"))
|
|
41012
41873
|
this.invalidate();
|
|
41013
|
-
throw error;
|
|
41874
|
+
throw this.endpoint === undefined ? error : this.dependencyFailure("Managed PostgreSQL recovery refused after a health probe failure. Preserve its data and ownership records.");
|
|
41875
|
+
}
|
|
41876
|
+
if (this.attempts === 0 && (this.failure === undefined || this.healthySinceFailure)) {
|
|
41877
|
+
this.retainFailure(new Error("Managed PostgreSQL failed its live health check."));
|
|
41878
|
+
this.healthySinceFailure = false;
|
|
41014
41879
|
}
|
|
41015
|
-
if (this.attempts === 0)
|
|
41016
|
-
this.failure = new DbosDependencyError("Managed PostgreSQL failed its live health check.");
|
|
41017
41880
|
this.invalidate();
|
|
41018
41881
|
if ((this.operations.now ?? Date.now)() < this.nextRecoveryAt)
|
|
41019
|
-
throw
|
|
41882
|
+
throw this.dependencyFailure("Managed Postgres recovery is cooling down after bounded attempts.");
|
|
41020
41883
|
while (!this.stopped && this.attempts < RECOVERY_ATTEMPTS) {
|
|
41021
41884
|
const attempt = this.attempts++;
|
|
41022
41885
|
if (attempt > 0)
|
|
@@ -41029,12 +41892,12 @@ class PostgresHealth {
|
|
|
41029
41892
|
if (ready !== undefined)
|
|
41030
41893
|
return ready;
|
|
41031
41894
|
} catch (error) {
|
|
41032
|
-
this.
|
|
41895
|
+
this.retainFailure(error);
|
|
41033
41896
|
}
|
|
41034
41897
|
}
|
|
41035
41898
|
this.attempts = 0;
|
|
41036
41899
|
this.nextRecoveryAt = (this.operations.now ?? Date.now)() + HEALTH_INTERVAL_MS;
|
|
41037
|
-
throw
|
|
41900
|
+
throw this.dependencyFailure("Managed Postgres is unavailable after bounded recovery. Preserve its data and ownership records.");
|
|
41038
41901
|
}
|
|
41039
41902
|
}
|
|
41040
41903
|
|
|
@@ -41309,7 +42172,7 @@ async function probePostgresIdentity(port) {
|
|
|
41309
42172
|
const result = await client.query(POSTGRES_IDENTITY_SQL);
|
|
41310
42173
|
return result.rows[0];
|
|
41311
42174
|
} catch (error) {
|
|
41312
|
-
if (isQueryReadTimeout(error))
|
|
42175
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
41313
42176
|
throw error;
|
|
41314
42177
|
const code = error instanceof Error && "code" in error ? error.code : undefined;
|
|
41315
42178
|
if (code === "ECONNREFUSED" || code === "ECONNRESET" || code === "EPIPE" || code === "57P01" || code === "57P02" || code === "57P03" || error instanceof Error && /timeout|timed out|connection terminated/i.test(error.message))
|
|
@@ -41482,7 +42345,7 @@ async function ensureCluster(options = {}) {
|
|
|
41482
42345
|
try {
|
|
41483
42346
|
return await existingServerProbe()(probePort);
|
|
41484
42347
|
} catch (error) {
|
|
41485
|
-
if (isQueryReadTimeout(error))
|
|
42348
|
+
if (isQueryReadTimeout(error) || isMonitoringConnectionTimeout(error))
|
|
41486
42349
|
return;
|
|
41487
42350
|
throw error;
|
|
41488
42351
|
}
|
|
@@ -41690,6 +42553,8 @@ async function ensureCluster(options = {}) {
|
|
|
41690
42553
|
const server = current.server;
|
|
41691
42554
|
if (!server)
|
|
41692
42555
|
throw new Error("Managed Postgres published identity is missing.");
|
|
42556
|
+
if (managedPostmaster(current) === undefined)
|
|
42557
|
+
return;
|
|
41693
42558
|
if (!managedPostgresRuntimeHealthy(current, server.port)) {
|
|
41694
42559
|
verifyManagedPostmasterProcess(current, server);
|
|
41695
42560
|
return;
|
|
@@ -41706,6 +42571,7 @@ async function ensureCluster(options = {}) {
|
|
|
41706
42571
|
};
|
|
41707
42572
|
health = new PostgresHealth({
|
|
41708
42573
|
probe: () => inspect(),
|
|
42574
|
+
url: embeddedDbosSystemDatabaseUrl,
|
|
41709
42575
|
validate: async (client) => {
|
|
41710
42576
|
const identity = await inspect(async () => {
|
|
41711
42577
|
const query = { text: POSTGRES_IDENTITY_SQL, query_timeout: POSTGRES_HEALTH_QUERY_TIMEOUT_MS };
|
|
@@ -42793,6 +43659,9 @@ async function requireDockerSuccess(action, args) {
|
|
|
42793
43659
|
throw new Error(`Could not ${action}: ${commandFailureDetail(result)}`);
|
|
42794
43660
|
}
|
|
42795
43661
|
|
|
43662
|
+
// dist/builtin/workflows/src/durable/dbos-lifecycle.ts
|
|
43663
|
+
init_dbos_process_owner();
|
|
43664
|
+
|
|
42796
43665
|
// dist/builtin/workflows/src/durable/dbos-registration-diagnostics.ts
|
|
42797
43666
|
var cachedErrorModule;
|
|
42798
43667
|
var errorModuleLoad;
|
|
@@ -48534,6 +49403,11 @@ var jobTracker = singleton5.facade;
|
|
|
48534
49403
|
|
|
48535
49404
|
// dist/builtin/workflows/src/durable/resume-runtime.ts
|
|
48536
49405
|
init_dbos_admission();
|
|
49406
|
+
|
|
49407
|
+
// dist/builtin/workflows/src/durable/sdk-recovery-scope.ts
|
|
49408
|
+
init_dbos_owner_fence();
|
|
49409
|
+
|
|
49410
|
+
// dist/builtin/workflows/src/durable/resume-runtime.ts
|
|
48537
49411
|
var pendingResumes = new WeakMap;
|
|
48538
49412
|
var unsettledResumeClaims = new WeakMap;
|
|
48539
49413
|
|
|
@@ -54059,7 +54933,7 @@ class StageSessionController {
|
|
|
54059
54933
|
}
|
|
54060
54934
|
|
|
54061
54935
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
54062
|
-
import { createHash as
|
|
54936
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
54063
54937
|
import { mkdir as mkdir3, writeFile as writeFile2 } from "node:fs/promises";
|
|
54064
54938
|
import { basename as basename6, dirname as dirname10, isAbsolute as isAbsolute11, join as join13, resolve as resolve10 } from "node:path";
|
|
54065
54939
|
var DEFAULT_MAX_OUTPUT_BYTES = 200 * 1024;
|
|
@@ -54125,7 +54999,7 @@ function formatByteSize(bytes) {
|
|
|
54125
54999
|
return `${value.toFixed(1)} ${units[unitIndex]}`;
|
|
54126
55000
|
}
|
|
54127
55001
|
function transcriptPath(runId, outputPath) {
|
|
54128
|
-
const digest =
|
|
55002
|
+
const digest = createHash6("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
54129
55003
|
return join13(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename6(outputPath)}.transcript.md`);
|
|
54130
55004
|
}
|
|
54131
55005
|
function appendContent(lines, content) {
|
|
@@ -55227,7 +56101,8 @@ async function run(def, inputs, opts = {}) {
|
|
|
55227
56101
|
ownController.abort(callerSignal?.reason);
|
|
55228
56102
|
};
|
|
55229
56103
|
const exit = createWorkflowExitManager({ runId, exitScope, controller: ownController });
|
|
55230
|
-
const
|
|
56104
|
+
const sourceBackend = opts.durableBackend ?? getDurableBackend();
|
|
56105
|
+
const backendView = sourceBackend.executionView?.() ?? sourceBackend;
|
|
55231
56106
|
const rootBackend = opts.durableRootBackend ?? backendView;
|
|
55232
56107
|
const durableBackend = opts.durableScope !== undefined ? new ScopedDurableBackend(backendView, opts.durableScope) : backendView;
|
|
55233
56108
|
const inheritedElapsedMs = opts.parentRun === undefined ? inheritedRunElapsedMs({ backend: durableBackend, runId, continuationSource: opts.continuation?.source }) : undefined;
|
|
@@ -55743,7 +56618,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
55743
56618
|
if (control === latestControl && !ownController.signal.aborted && isDbosDependencyError(error)) {
|
|
55744
56619
|
activeStore.recordRunExecutionState(runId, {
|
|
55745
56620
|
phase: "blocked_dependency",
|
|
55746
|
-
dependencyError: `Workflow database unavailable during ${status === "paused" ? "pause" : "resume"}; persistence not confirmed
|
|
56621
|
+
dependencyError: `Workflow database unavailable during ${status === "paused" ? "pause" : "resume"}; persistence not confirmed. ${error.message}`
|
|
55747
56622
|
});
|
|
55748
56623
|
if (status === "paused")
|
|
55749
56624
|
return;
|
|
@@ -55790,7 +56665,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
55790
56665
|
if (!admission.admitting)
|
|
55791
56666
|
return pausePersistence;
|
|
55792
56667
|
backgroundAdmissionControl(durableBackend, runId, pausePersistence, (error, resumable) => {
|
|
55793
|
-
admissionControlError = unknownErrorMessage(error);
|
|
56668
|
+
admissionControlError = databaseDependencyMessage(unknownErrorMessage(error), getDbosProcessOwner().databaseDiagnostics?.());
|
|
55794
56669
|
runSnapshot.error = admissionControlError;
|
|
55795
56670
|
activeStore.recordRunPaused(runId, undefined, { resumable });
|
|
55796
56671
|
return runSnapshot.error;
|
|
@@ -55807,7 +56682,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
55807
56682
|
} catch (error) {
|
|
55808
56683
|
if (admission.failed) {
|
|
55809
56684
|
scheduler.releaseRun();
|
|
55810
|
-
throw error;
|
|
56685
|
+
throw new Error(admissionControlError ?? databaseDependencyMessage(unknownErrorMessage(error), getDbosProcessOwner().databaseDiagnostics?.()));
|
|
55811
56686
|
}
|
|
55812
56687
|
}
|
|
55813
56688
|
if (generation !== pauseGeneration)
|
|
@@ -55880,12 +56755,21 @@ async function run(def, inputs, opts = {}) {
|
|
|
55880
56755
|
}).catch((error) => {
|
|
55881
56756
|
const isTopologyError = typeof error === "object" && error !== null && "name" in error && error.name === "DurableNestedTopologyError";
|
|
55882
56757
|
if (!isDbosDependencyError(error) && !isTopologyError) {
|
|
55883
|
-
|
|
55884
|
-
|
|
56758
|
+
const admissionDetail = unknownErrorMessage(error);
|
|
56759
|
+
const diagnostics = getDbosProcessOwner().databaseDiagnostics?.();
|
|
56760
|
+
const safeMessage = databaseDependencyMessage(admissionDetail, diagnostics);
|
|
56761
|
+
activeStore.recordRunExecutionState(runId, {
|
|
56762
|
+
phase: "blocked_dependency",
|
|
56763
|
+
dependencyError: safeMessage
|
|
56764
|
+
});
|
|
56765
|
+
const safeError = new Error(safeMessage);
|
|
56766
|
+
durableAdmissionFailure = { error: safeError };
|
|
56767
|
+
classifiedFailures.set(safeError, classifyWorkflowFailure({
|
|
55885
56768
|
code: "ATOMIC_DURABLE_ADMISSION_REJECTED",
|
|
55886
|
-
message:
|
|
55887
|
-
cause:
|
|
56769
|
+
message: safeMessage,
|
|
56770
|
+
cause: safeError
|
|
55888
56771
|
}));
|
|
56772
|
+
throw safeError;
|
|
55889
56773
|
}
|
|
55890
56774
|
throw error;
|
|
55891
56775
|
}), ownController.signal);
|
|
@@ -55978,6 +56862,17 @@ async function run(def, inputs, opts = {}) {
|
|
|
55978
56862
|
const gracefulQuit = observedQuitCancellation() ?? observedTaskTailQuit ?? findWorkflowGracefulQuit(err) ?? findWorkflowGracefulQuit(ownController.signal.reason);
|
|
55979
56863
|
if (gracefulQuit !== undefined)
|
|
55980
56864
|
return suspendForGracefulQuit(gracefulQuit);
|
|
56865
|
+
if (isDbosDependencyError(err) && durableBackend.isCheckpointUnavailable?.(runId) && (durableBackend.getWorkflow(runId)?.completedCheckpoints ?? 0) > 0) {
|
|
56866
|
+
await admittedTools.closeAndDrain();
|
|
56867
|
+
activeStore.recordRunPaused(runId, undefined, { resumable: true });
|
|
56868
|
+
activeStore.recordRunExecutionState(runId, { phase: "blocked_dependency", dependencyError: err.message });
|
|
56869
|
+
return {
|
|
56870
|
+
runId,
|
|
56871
|
+
status: "paused",
|
|
56872
|
+
stages: [...runSnapshot.stages],
|
|
56873
|
+
toolNodes: [...runSnapshot.toolNodes ?? []]
|
|
56874
|
+
};
|
|
56875
|
+
}
|
|
55981
56876
|
await admittedTools.closeAndDrain();
|
|
55982
56877
|
if (ownController.signal.aborted)
|
|
55983
56878
|
await Promise.allSettled([...pendingChildWorkflows]);
|