@hasna/workflows 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -98
- package/dist/agent-tools.js +152 -959
- package/dist/cli/index.js +4130 -2329
- package/dist/handlers.js +3287 -55
- package/dist/index.js +1534 -370
- package/dist/mcp/index.js +1741 -1660
- package/dist/sdk.js +1620 -0
- package/dist/serve/admin.js +4248 -0
- package/dist/serve/index.js +3416 -1350
- package/dist/service.js +1302 -286
- package/hasna.contract.json +17 -17
- package/package.json +16 -13
- package/types/agent-tools.d.ts +3 -17
- package/types/client-error.d.ts +4 -0
- package/types/client-tools.d.ts +4 -0
- package/types/client.d.ts +60 -0
- package/types/daemon.d.ts +18 -9
- package/types/db/postgres-store.d.ts +54 -0
- package/types/graph.d.ts +7 -0
- package/types/handlers.d.ts +2 -1
- package/types/identity.d.ts +7 -0
- package/types/lanes/types.d.ts +12 -3
- package/types/openapi.d.ts +7 -0
- package/types/paths.d.ts +10 -1
- package/types/sdk/generated-api.d.ts +159 -0
- package/types/sdk.d.ts +6 -0
- package/types/serve/admin.d.ts +2 -0
- package/types/service.d.ts +75 -4
- package/types/session.d.ts +31 -8
- package/types/setup.d.ts +71 -0
- package/types/store.d.ts +35 -0
- package/types/tool-definitions.d.ts +18 -0
- package/types/types.d.ts +8 -0
- package/types/version.d.ts +1 -0
- package/types/wal.d.ts +9 -1
package/dist/service.js
CHANGED
|
@@ -17,22 +17,42 @@ var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
|
17
17
|
var __require = import.meta.require;
|
|
18
18
|
|
|
19
19
|
// src/lanes/types.ts
|
|
20
|
-
import { statSync } from "fs";
|
|
20
|
+
import { statSync as statSync2 } from "fs";
|
|
21
21
|
function binaryOnPath(binary) {
|
|
22
22
|
const pathEntries = (process.env.PATH ?? "").split(":").filter(Boolean);
|
|
23
23
|
for (const entry of pathEntries) {
|
|
24
24
|
try {
|
|
25
|
-
const stat =
|
|
25
|
+
const stat = statSync2(`${entry}/${binary}`);
|
|
26
26
|
if (stat.isFile())
|
|
27
27
|
return true;
|
|
28
28
|
} catch {}
|
|
29
29
|
}
|
|
30
30
|
return false;
|
|
31
31
|
}
|
|
32
|
-
function
|
|
32
|
+
function subscriptionEnvironment(overrides = {}) {
|
|
33
|
+
const paid = /^(OPENAI_API_KEY|ANTHROPIC_API_KEY|ANTHROPIC_AUTH_TOKEN|CODEX_API_KEY|CURSOR_API_KEY)$/;
|
|
34
|
+
const sensitive = /(?:API[_-]?KEY|SECRET|PASSWORD|TOKEN|DATABASE_URL|CREDENTIAL)/i;
|
|
35
|
+
const env = {};
|
|
36
|
+
for (const [name, value] of Object.entries(process.env)) {
|
|
37
|
+
if (value !== undefined && !paid.test(name) && (!sensitive.test(name) || name === "CLAUDE_CODE_OAUTH_TOKEN"))
|
|
38
|
+
env[name] = value;
|
|
39
|
+
}
|
|
40
|
+
for (const [name, value] of Object.entries(overrides)) {
|
|
41
|
+
if (paid.test(name) || sensitive.test(name))
|
|
42
|
+
throw new Error(`lane environment field ${name} requires a supported scoped credential consumer`);
|
|
43
|
+
env[name] = value;
|
|
44
|
+
}
|
|
45
|
+
return env;
|
|
46
|
+
}
|
|
47
|
+
function subscriptionSdkEnvironment(overrides = {}) {
|
|
48
|
+
const safe = subscriptionEnvironment(overrides);
|
|
49
|
+
return { ...Object.fromEntries(Object.keys(process.env).map((name) => [name, undefined])), ...safe };
|
|
50
|
+
}
|
|
51
|
+
function withTimeout(promise, timeoutMs, what, cancel) {
|
|
33
52
|
return new Promise((resolve, reject) => {
|
|
34
53
|
const timer = setTimeout(() => {
|
|
35
|
-
|
|
54
|
+
cancel?.();
|
|
55
|
+
reject(new LaneTimeoutError(what, timeoutMs));
|
|
36
56
|
}, timeoutMs);
|
|
37
57
|
promise.then((value) => {
|
|
38
58
|
clearTimeout(timer);
|
|
@@ -43,15 +63,16 @@ function withTimeout(promise, timeoutMs, what) {
|
|
|
43
63
|
});
|
|
44
64
|
});
|
|
45
65
|
}
|
|
46
|
-
function spawnCli(cmd, opts) {
|
|
66
|
+
async function spawnCli(cmd, opts) {
|
|
47
67
|
const started = Date.now();
|
|
48
68
|
const timeoutMs = opts.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS;
|
|
49
69
|
let proc;
|
|
50
70
|
try {
|
|
51
|
-
proc = Bun.
|
|
71
|
+
proc = Bun.spawn({
|
|
52
72
|
cmd,
|
|
53
73
|
cwd: opts.cwd,
|
|
54
|
-
env:
|
|
74
|
+
env: subscriptionEnvironment(opts.env),
|
|
75
|
+
stdin: "ignore",
|
|
55
76
|
stdout: "pipe",
|
|
56
77
|
stderr: "pipe",
|
|
57
78
|
timeout: timeoutMs
|
|
@@ -63,15 +84,16 @@ function spawnCli(cmd, opts) {
|
|
|
63
84
|
}
|
|
64
85
|
return { ok: false, exitCode: 1, output: "", durationMs: Date.now() - started, error: message };
|
|
65
86
|
}
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
87
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
88
|
+
new Response(proc.stdout).text(),
|
|
89
|
+
new Response(proc.stderr).text(),
|
|
90
|
+
proc.exited
|
|
91
|
+
]);
|
|
69
92
|
const output = stderr ? `${stdout}
|
|
70
93
|
${stderr}`.trim() : stdout.trim();
|
|
71
|
-
|
|
72
|
-
return { ok: exitCode === 0, exitCode, output, durationMs };
|
|
94
|
+
return { ok: exitCode === 0, exitCode, output, durationMs: Date.now() - started };
|
|
73
95
|
}
|
|
74
|
-
var LANE_KINDS2, LaneDependencyMissingError, LaneAdapterShapeError, DEFAULT_LANE_TIMEOUT_MS = 120000;
|
|
96
|
+
var LANE_KINDS2, LaneDependencyMissingError, LaneAdapterShapeError, DEFAULT_LANE_TIMEOUT_MS = 120000, LaneTimeoutError;
|
|
75
97
|
var init_types = __esm(() => {
|
|
76
98
|
LANE_KINDS2 = ["claude", "codex", "cursor", "grok"];
|
|
77
99
|
LaneDependencyMissingError = class LaneDependencyMissingError extends Error {
|
|
@@ -86,6 +108,12 @@ var init_types = __esm(() => {
|
|
|
86
108
|
this.name = "LaneAdapterShapeError";
|
|
87
109
|
}
|
|
88
110
|
};
|
|
111
|
+
LaneTimeoutError = class LaneTimeoutError extends Error {
|
|
112
|
+
constructor(what, timeoutMs) {
|
|
113
|
+
super(`${what} exceeded the ${timeoutMs}ms budget`);
|
|
114
|
+
this.name = "LaneTimeoutError";
|
|
115
|
+
}
|
|
116
|
+
};
|
|
89
117
|
});
|
|
90
118
|
|
|
91
119
|
// src/lanes/claude.ts
|
|
@@ -127,11 +155,12 @@ function createClaudeAdapter(deps = {}) {
|
|
|
127
155
|
throw new LaneAdapterShapeError("@anthropic-ai/claude-agent-sdk exports no query() function");
|
|
128
156
|
}
|
|
129
157
|
const prompt = job.prompt;
|
|
158
|
+
const abortController = new AbortController;
|
|
130
159
|
try {
|
|
131
160
|
const { output, isError } = await withTimeout((async () => {
|
|
132
161
|
const generator = await query({
|
|
133
162
|
prompt,
|
|
134
|
-
options: { model: job.model, maxTurns: 32,
|
|
163
|
+
options: { model: job.model, cwd: job.cwd, maxTurns: 32, abortController, env: subscriptionSdkEnvironment(job.env) }
|
|
135
164
|
});
|
|
136
165
|
let output2 = "";
|
|
137
166
|
let isError2 = false;
|
|
@@ -150,19 +179,19 @@ function createClaudeAdapter(deps = {}) {
|
|
|
150
179
|
isError2 = true;
|
|
151
180
|
}
|
|
152
181
|
return { output: output2, isError: isError2 };
|
|
153
|
-
})(), timeoutMs,
|
|
182
|
+
})(), timeoutMs, "claude lane", () => abortController.abort());
|
|
154
183
|
const durationMs = Date.now() - started;
|
|
155
184
|
return { ok: !isError, exitCode: isError ? 1 : 0, output: output.trim(), durationMs };
|
|
156
185
|
} catch (err) {
|
|
157
186
|
const durationMs = Date.now() - started;
|
|
158
|
-
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err) };
|
|
187
|
+
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err), retryable: !(err instanceof LaneTimeoutError) };
|
|
159
188
|
}
|
|
160
189
|
}
|
|
161
190
|
};
|
|
162
191
|
}
|
|
163
|
-
function runClaudeCli(job, cliPath, started, timeoutMs) {
|
|
192
|
+
async function runClaudeCli(job, cliPath, started, timeoutMs) {
|
|
164
193
|
try {
|
|
165
|
-
const result = spawnCli([cliPath, "-p", job.prompt], {
|
|
194
|
+
const result = await spawnCli([cliPath, "-p", job.prompt], {
|
|
166
195
|
binaryName: cliPath,
|
|
167
196
|
packageHint: "@anthropic-ai/claude-agent-sdk or the claude CLI",
|
|
168
197
|
timeoutMs,
|
|
@@ -219,29 +248,30 @@ function createCodexAdapter(deps = {}) {
|
|
|
219
248
|
if (typeof CodexClass !== "function") {
|
|
220
249
|
throw new LaneAdapterShapeError("@openai/codex-sdk exports no Codex class");
|
|
221
250
|
}
|
|
251
|
+
const abortController = new AbortController;
|
|
222
252
|
try {
|
|
223
253
|
const client = await withTimeout(Promise.resolve(new CodexClass({
|
|
224
254
|
config: { model: job.model },
|
|
225
|
-
env: job.env
|
|
226
|
-
})), timeoutMs,
|
|
227
|
-
const thread = client.startThread?.();
|
|
255
|
+
env: subscriptionEnvironment(job.env)
|
|
256
|
+
})), timeoutMs, "codex lane");
|
|
257
|
+
const thread = client.startThread?.({ model: job.model, workingDirectory: job.cwd });
|
|
228
258
|
if (!thread || typeof thread.run !== "function") {
|
|
229
259
|
throw new LaneAdapterShapeError("@openai/codex-sdk Codex.startThread() returned no run()");
|
|
230
260
|
}
|
|
231
|
-
const turn = await withTimeout(thread.run(job.prompt), timeoutMs,
|
|
261
|
+
const turn = await withTimeout(thread.run(job.prompt, { signal: abortController.signal }), timeoutMs, "codex lane", () => abortController.abort());
|
|
232
262
|
const output = turn?.finalResponse ?? "";
|
|
233
263
|
const durationMs = Date.now() - started;
|
|
234
264
|
return { ok: true, exitCode: 0, output, durationMs };
|
|
235
265
|
} catch (err) {
|
|
236
266
|
const durationMs = Date.now() - started;
|
|
237
|
-
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err) };
|
|
267
|
+
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err), retryable: !(err instanceof LaneTimeoutError) };
|
|
238
268
|
}
|
|
239
269
|
}
|
|
240
270
|
};
|
|
241
271
|
}
|
|
242
|
-
function runCodexCli(job, cliPath, started, timeoutMs) {
|
|
272
|
+
async function runCodexCli(job, cliPath, started, timeoutMs) {
|
|
243
273
|
try {
|
|
244
|
-
const result = spawnCli([cliPath, "exec", "--", job.prompt], {
|
|
274
|
+
const result = await spawnCli([cliPath, "exec", "--", job.prompt], {
|
|
245
275
|
binaryName: cliPath,
|
|
246
276
|
packageHint: "@openai/codex-sdk or the codex CLI",
|
|
247
277
|
timeoutMs,
|
|
@@ -285,6 +315,8 @@ function createCursorAdapter(deps = {}) {
|
|
|
285
315
|
if (!job.prompt) {
|
|
286
316
|
return { ok: false, exitCode: 2, output: "", durationMs: 0, error: "cursor lane requires a prompt" };
|
|
287
317
|
}
|
|
318
|
+
if (!deps.sdkLoader && binaryOnPath(deps.cliPath ?? "cursor-agent"))
|
|
319
|
+
return runCursorCli(job, deps.cliPath ?? "cursor-agent", Date.now(), job.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS);
|
|
288
320
|
const started = Date.now();
|
|
289
321
|
const timeoutMs = job.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS;
|
|
290
322
|
const loader = deps.sdkLoader ?? (async () => await import("@cursor/sdk"));
|
|
@@ -294,38 +326,63 @@ function createCursorAdapter(deps = {}) {
|
|
|
294
326
|
} catch {
|
|
295
327
|
return runCursorCli(job, deps.cliPath ?? "cursor-agent", started, timeoutMs);
|
|
296
328
|
}
|
|
329
|
+
let cancel;
|
|
297
330
|
try {
|
|
298
|
-
|
|
331
|
+
subscriptionEnvironment(job.env);
|
|
332
|
+
if (!deps.sdkLoader && process.env.CURSOR_API_KEY)
|
|
333
|
+
throw new Error("cursor lane refuses ambient API billing; use the native subscription CLI");
|
|
334
|
+
const run2 = await withTimeout(runLocalAgent(sdk, job, (callback) => {
|
|
335
|
+
cancel = callback;
|
|
336
|
+
}), timeoutMs, "cursor lane", () => cancel?.());
|
|
299
337
|
const durationMs = Date.now() - started;
|
|
300
|
-
const status =
|
|
338
|
+
const status = run2.status;
|
|
301
339
|
const ok = status === "finished";
|
|
302
|
-
return { ok, exitCode: ok ? 0 : 1, output:
|
|
340
|
+
return { ok, exitCode: ok ? 0 : 1, output: run2.result, durationMs };
|
|
303
341
|
} catch (err) {
|
|
342
|
+
if (err instanceof LaneAdapterShapeError)
|
|
343
|
+
throw err;
|
|
304
344
|
const durationMs = Date.now() - started;
|
|
305
|
-
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err) };
|
|
345
|
+
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err), retryable: !(err instanceof LaneTimeoutError) };
|
|
306
346
|
}
|
|
307
347
|
}
|
|
308
348
|
};
|
|
309
349
|
}
|
|
310
|
-
async function runLocalAgent(sdk, job) {
|
|
350
|
+
async function runLocalAgent(sdk, job, onCancel) {
|
|
311
351
|
const AgentClass = sdk.Agent;
|
|
312
352
|
const CursorClass = sdk.Cursor;
|
|
313
353
|
if (typeof AgentClass !== "function" && typeof CursorClass !== "function") {
|
|
314
354
|
throw new LaneAdapterShapeError("@cursor/sdk exports no Agent/Cursor class");
|
|
315
355
|
}
|
|
316
|
-
const
|
|
356
|
+
const create = AgentClass.create;
|
|
357
|
+
const agent = typeof create === "function" ? await create.call(AgentClass, { local: { cwd: job.cwd, enableAgentRetries: false } }) : typeof AgentClass === "function" ? new AgentClass : undefined;
|
|
317
358
|
if (!agent || typeof agent.send !== "function") {
|
|
318
359
|
throw new LaneAdapterShapeError("@cursor/sdk Agent has no send()");
|
|
319
360
|
}
|
|
320
|
-
|
|
321
|
-
if (
|
|
361
|
+
let run2 = await agent.send(job.prompt, { model: job.model });
|
|
362
|
+
if (run2 && typeof run2 === "object") {
|
|
363
|
+
const handle = run2;
|
|
364
|
+
if (typeof handle.cancel === "function")
|
|
365
|
+
onCancel(() => {
|
|
366
|
+
handle.cancel().catch(() => {});
|
|
367
|
+
});
|
|
368
|
+
if (typeof handle.wait === "function")
|
|
369
|
+
run2 = await handle.wait();
|
|
370
|
+
}
|
|
371
|
+
if (!run2 || typeof run2 !== "object") {
|
|
322
372
|
throw new LaneAdapterShapeError("@cursor/sdk Agent.send() returned no Run");
|
|
323
373
|
}
|
|
324
|
-
|
|
374
|
+
const candidate = run2;
|
|
375
|
+
if (typeof candidate.status !== "string" || !["finished", "error", "cancelled"].includes(candidate.status)) {
|
|
376
|
+
throw new LaneAdapterShapeError(`@cursor/sdk Agent.send() returned a Run without a terminal status (got ${JSON.stringify(candidate.status)}; expected finished|error|cancelled)`);
|
|
377
|
+
}
|
|
378
|
+
if (typeof candidate.result !== "string") {
|
|
379
|
+
throw new LaneAdapterShapeError("@cursor/sdk Agent.send() returned a Run without a string result");
|
|
380
|
+
}
|
|
381
|
+
return { status: candidate.status, result: candidate.result };
|
|
325
382
|
}
|
|
326
|
-
function runCursorCli(job, cliPath, started, timeoutMs) {
|
|
383
|
+
async function runCursorCli(job, cliPath, started, timeoutMs) {
|
|
327
384
|
try {
|
|
328
|
-
const result = spawnCli([cliPath, "
|
|
385
|
+
const result = await spawnCli([cliPath, "--print", job.prompt, ...job.model ? ["--model", job.model] : []], {
|
|
329
386
|
binaryName: cliPath,
|
|
330
387
|
packageHint: "@cursor/sdk (local mode) or the cursor-agent CLI",
|
|
331
388
|
timeoutMs,
|
|
@@ -368,7 +425,7 @@ function createGrokAdapter(deps = {}) {
|
|
|
368
425
|
const timeoutMs = job.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS;
|
|
369
426
|
const cliPath = deps.cliPath ?? "grok";
|
|
370
427
|
try {
|
|
371
|
-
const result = spawnCli([cliPath, job.prompt], {
|
|
428
|
+
const result = await spawnCli([cliPath, job.prompt], {
|
|
372
429
|
binaryName: cliPath,
|
|
373
430
|
packageHint: "the grok CLI (no xAI Grok SDK exists on npm \u2014 measured 2026-08-25)",
|
|
374
431
|
timeoutMs,
|
|
@@ -439,10 +496,13 @@ var init_lanes = __esm(() => {
|
|
|
439
496
|
["grok", createGrokAdapter()]
|
|
440
497
|
]);
|
|
441
498
|
});
|
|
499
|
+
|
|
500
|
+
// src/service.ts
|
|
501
|
+
import { createHash as createHash3 } from "crypto";
|
|
442
502
|
// package.json
|
|
443
503
|
var package_default = {
|
|
444
504
|
name: "@hasna/workflows",
|
|
445
|
-
version: "0.
|
|
505
|
+
version: "0.2.0",
|
|
446
506
|
description: "Universal graph workflow app \u2014 graph language (start/step/decision/while/end), three-table store, session WAL, daemon with claims/leases, four lane adapters (claude/codex/cursor/grok), and a full CLI on four surfaces",
|
|
447
507
|
type: "module",
|
|
448
508
|
main: "./dist/index.js",
|
|
@@ -458,8 +518,8 @@ var package_default = {
|
|
|
458
518
|
import: "./dist/index.js"
|
|
459
519
|
},
|
|
460
520
|
"./sdk": {
|
|
461
|
-
types: "./types/
|
|
462
|
-
import: "./dist/
|
|
521
|
+
types: "./types/sdk.d.ts",
|
|
522
|
+
import: "./dist/sdk.js"
|
|
463
523
|
},
|
|
464
524
|
"./cli": {
|
|
465
525
|
types: "./types/cli/index.d.ts",
|
|
@@ -474,14 +534,17 @@ var package_default = {
|
|
|
474
534
|
"hasna.contract.json"
|
|
475
535
|
],
|
|
476
536
|
scripts: {
|
|
477
|
-
build: "bun run build:runtime && bun run build:types",
|
|
478
|
-
"
|
|
479
|
-
"
|
|
537
|
+
build: "bun run check:sdk && bun run build:runtime && bun run build:types",
|
|
538
|
+
"generate:sdk": "bun scripts/generate-sdk.ts --write",
|
|
539
|
+
"check:sdk": "bun scripts/generate-sdk.ts",
|
|
540
|
+
"test:artifacts": "bun scripts/artifact-acceptance.ts",
|
|
541
|
+
"build:runtime": "bun build src/cli/index.ts --outdir dist/cli --target bun && bun build src/mcp/index.ts --outdir dist/mcp --target bun && bun build src/serve/index.ts src/serve/admin.ts --outdir dist/serve --target bun --packages external && bun build src/index.ts src/service.ts src/handlers.ts src/agent-tools.ts src/types.ts --root src --outdir dist --target bun --packages external && bun build src/sdk.ts --outdir dist --target node",
|
|
542
|
+
"build:types": "tsc -p tsconfig.build.json --emitDeclarationOnly --outDir types",
|
|
480
543
|
"artifact-scan": "bun scripts/contracts-cli.mjs artifact-scan dist",
|
|
481
544
|
"contract:check": "bun scripts/contracts-cli.mjs repo-conformance .",
|
|
482
545
|
typecheck: "tsc --noEmit",
|
|
483
546
|
test: "bun test",
|
|
484
|
-
"test:pg": 'WORKFLOWS_TEST_DATABASE_URL="${WORKFLOWS_TEST_DATABASE_URL:?point it at a
|
|
547
|
+
"test:pg": 'WORKFLOWS_TEST_DATABASE_URL="${WORKFLOWS_TEST_DATABASE_URL:?point it at a fresh test Postgres database}" WORKFLOWS_REQUIRE_POSTGRES=1 bun test src/db/postgres-store.test.ts src/db/postgres-http.test.ts src/db/postgres-fencing.test.ts',
|
|
485
548
|
prepack: "bun run build && bun run artifact-scan",
|
|
486
549
|
prepublishOnly: "bun run test && bun run build"
|
|
487
550
|
},
|
|
@@ -500,7 +563,8 @@ var package_default = {
|
|
|
500
563
|
},
|
|
501
564
|
repository: {
|
|
502
565
|
type: "git",
|
|
503
|
-
url: "
|
|
566
|
+
url: "https://github.com/hasna/apps.git",
|
|
567
|
+
directory: "apps/workflows"
|
|
504
568
|
},
|
|
505
569
|
homepage: "https://github.com/hasna/apps/tree/main/apps/workflows",
|
|
506
570
|
bugs: {
|
|
@@ -514,21 +578,25 @@ var package_default = {
|
|
|
514
578
|
dependencies: {
|
|
515
579
|
"@anthropic-ai/claude-agent-sdk": "0.3.234",
|
|
516
580
|
"@cursor/sdk": "^1.0.28",
|
|
517
|
-
"@hasna/paths": "0.1.0",
|
|
518
581
|
"@openai/codex-sdk": "0.147.0",
|
|
519
|
-
commander: "^13.1.0"
|
|
582
|
+
commander: "^13.1.0",
|
|
583
|
+
"@hasna/contracts": "1.3.3",
|
|
584
|
+
pg: "^8.16.3"
|
|
520
585
|
},
|
|
521
586
|
devDependencies: {
|
|
522
|
-
"@
|
|
523
|
-
"@types/bun": "latest",
|
|
587
|
+
"@types/bun": "1.3.14",
|
|
524
588
|
"@types/pg": "^8.15.6",
|
|
525
|
-
pg: "^8.16.3",
|
|
526
589
|
typescript: "^5.7.3"
|
|
527
590
|
}
|
|
528
591
|
};
|
|
529
592
|
|
|
593
|
+
// src/version.ts
|
|
594
|
+
function packageVersion() {
|
|
595
|
+
return package_default.version;
|
|
596
|
+
}
|
|
597
|
+
|
|
530
598
|
// src/daemon.ts
|
|
531
|
-
import {
|
|
599
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
532
600
|
|
|
533
601
|
// src/expr.ts
|
|
534
602
|
class ExprSyntaxError extends Error {
|
|
@@ -897,16 +965,49 @@ function findCycles(graph) {
|
|
|
897
965
|
}
|
|
898
966
|
function validateGraph(graph) {
|
|
899
967
|
const issues = [];
|
|
968
|
+
if (graph === null || typeof graph !== "object" || Array.isArray(graph)) {
|
|
969
|
+
return { ok: false, issues: [{ path: "graph", message: "graph must be an object" }] };
|
|
970
|
+
}
|
|
900
971
|
if (typeof graph.name !== "string" || graph.name.trim() === "") {
|
|
901
972
|
issue(issues, "name", "graph requires a non-empty name");
|
|
902
973
|
}
|
|
903
974
|
if (typeof graph.version !== "string" || graph.version.trim() === "") {
|
|
904
975
|
issue(issues, "version", "graph requires a version string");
|
|
905
976
|
}
|
|
977
|
+
if (graph.memoWatch !== undefined) {
|
|
978
|
+
if (!Array.isArray(graph.memoWatch) || graph.memoWatch.length === 0) {
|
|
979
|
+
issue(issues, "memoWatch", "memoWatch must be a non-empty array of file paths or globs");
|
|
980
|
+
} else {
|
|
981
|
+
for (const entry of graph.memoWatch) {
|
|
982
|
+
if (typeof entry !== "string" || entry.trim() === "") {
|
|
983
|
+
issue(issues, "memoWatch", "each memoWatch entry must be a non-empty path or glob string");
|
|
984
|
+
break;
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
}
|
|
906
989
|
if (!Array.isArray(graph.nodes) || graph.nodes.length === 0) {
|
|
907
990
|
issue(issues, "nodes", "graph requires at least one node");
|
|
908
991
|
return { ok: false, issues };
|
|
909
992
|
}
|
|
993
|
+
for (let index = 0;index < graph.nodes.length; index++) {
|
|
994
|
+
const node = graph.nodes[index];
|
|
995
|
+
if (node === null || typeof node !== "object" || Array.isArray(node)) {
|
|
996
|
+
issue(issues, `nodes.${index}`, "node must be an object");
|
|
997
|
+
continue;
|
|
998
|
+
}
|
|
999
|
+
if (!GRAPH_NODE_TYPES.includes(node.type))
|
|
1000
|
+
issue(issues, `nodes.${index}.type`, "type must be start, step, decision, while, or end");
|
|
1001
|
+
for (const key of ["next", "then", "else", "prompt", "command"]) {
|
|
1002
|
+
const value = node[key];
|
|
1003
|
+
if (value !== undefined && (typeof value !== "string" || value.trim() === ""))
|
|
1004
|
+
issue(issues, `nodes.${index}.${key}`, `${key} must be a non-empty string`);
|
|
1005
|
+
}
|
|
1006
|
+
if (node.type === "while" && (!Array.isArray(node.body) || node.body.some((id) => typeof id !== "string")))
|
|
1007
|
+
issue(issues, `nodes.${index}.body`, "body must be an array of node ids");
|
|
1008
|
+
}
|
|
1009
|
+
if (issues.length)
|
|
1010
|
+
return { ok: false, issues };
|
|
910
1011
|
const starts = graph.nodes.filter((n) => n.type === "start");
|
|
911
1012
|
if (starts.length === 0)
|
|
912
1013
|
issue(issues, "nodes", "graph requires exactly one start node (found none)");
|
|
@@ -968,6 +1069,8 @@ function nodeEdges(node) {
|
|
|
968
1069
|
return node.next ? [{ kind: "next", target: node.next }] : [];
|
|
969
1070
|
case "end":
|
|
970
1071
|
return [];
|
|
1072
|
+
default:
|
|
1073
|
+
return [];
|
|
971
1074
|
}
|
|
972
1075
|
}
|
|
973
1076
|
function validateNode(graph, node, issues) {
|
|
@@ -1137,36 +1240,53 @@ function assertNoSecrets(value, what) {
|
|
|
1137
1240
|
|
|
1138
1241
|
// src/session.ts
|
|
1139
1242
|
import { createHash } from "crypto";
|
|
1243
|
+
import { readFileSync, statSync } from "fs";
|
|
1244
|
+
import { isAbsolute, join } from "path";
|
|
1140
1245
|
var DEFAULT_MAX_ATTEMPTS = 3;
|
|
1141
1246
|
function repairTornRuns(store, wal, opts = {}) {
|
|
1247
|
+
const sequence = repairSteps(store, wal, opts);
|
|
1248
|
+
let step = sequence.next();
|
|
1249
|
+
while (!step.done) {
|
|
1250
|
+
if (step.value instanceof Promise) {
|
|
1251
|
+
return (async () => {
|
|
1252
|
+
while (!step.done)
|
|
1253
|
+
step = sequence.next(await step.value);
|
|
1254
|
+
return step.value;
|
|
1255
|
+
})();
|
|
1256
|
+
}
|
|
1257
|
+
step = sequence.next(step.value);
|
|
1258
|
+
}
|
|
1259
|
+
return step.value;
|
|
1260
|
+
}
|
|
1261
|
+
function* repairSteps(store, wal, opts) {
|
|
1142
1262
|
const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
1143
1263
|
const replay = wal.replay();
|
|
1144
1264
|
const liveClaims = replay.liveClaims(opts.now);
|
|
1145
1265
|
const report = { interrupted: 0, requeued: 0, failed: 0 };
|
|
1146
|
-
const running = store.listRuns({ status: "running", limit: 1000 });
|
|
1266
|
+
const running = yield store.listRuns({ status: "running", limit: 1000 });
|
|
1147
1267
|
for (const run of running) {
|
|
1148
1268
|
if (liveClaims.has(run.id))
|
|
1149
1269
|
continue;
|
|
1150
1270
|
report.interrupted++;
|
|
1151
|
-
const nodes = store.listRunNodes(run.id);
|
|
1271
|
+
const nodes = yield store.listRunNodes(run.id);
|
|
1152
1272
|
if (run.attempts < maxAttempts) {
|
|
1153
|
-
store.bumpAttempts(run.id);
|
|
1154
|
-
store.setRunStatus(run.id, "interrupted", {
|
|
1273
|
+
yield store.bumpAttempts(run.id);
|
|
1274
|
+
yield store.setRunStatus(run.id, "interrupted", {
|
|
1155
1275
|
error: `torn run interrupted after claim loss (attempt ${run.attempts + 1}/${maxAttempts})`
|
|
1156
1276
|
});
|
|
1157
1277
|
for (const node of nodes) {
|
|
1158
1278
|
if (node.status === "running") {
|
|
1159
|
-
store.setRunNodeStatus(node.id, "pending", { error: undefined });
|
|
1279
|
+
yield store.setRunNodeStatus(node.id, "pending", { error: undefined });
|
|
1160
1280
|
}
|
|
1161
1281
|
}
|
|
1162
1282
|
report.requeued++;
|
|
1163
1283
|
} else {
|
|
1164
|
-
store.setRunStatus(run.id, "failed", {
|
|
1284
|
+
yield store.setRunStatus(run.id, "failed", {
|
|
1165
1285
|
error: `torn run failed after ${maxAttempts} interrupted attempts`
|
|
1166
1286
|
});
|
|
1167
1287
|
for (const node of nodes) {
|
|
1168
1288
|
if (node.status === "running") {
|
|
1169
|
-
store.setRunNodeStatus(node.id, "failed", { error: "run interrupted beyond retry budget" });
|
|
1289
|
+
yield store.setRunNodeStatus(node.id, "failed", { error: "run interrupted beyond retry budget" });
|
|
1170
1290
|
}
|
|
1171
1291
|
}
|
|
1172
1292
|
report.failed++;
|
|
@@ -1182,30 +1302,429 @@ function memoKey(graphName, nodeId, input) {
|
|
|
1182
1302
|
}
|
|
1183
1303
|
function tryMemoHit(store, graphName, nodeId, input) {
|
|
1184
1304
|
const key = memoKey(graphName, nodeId, input);
|
|
1185
|
-
const
|
|
1186
|
-
if (
|
|
1305
|
+
const value = store.memoGet(key);
|
|
1306
|
+
if (value instanceof Promise) {
|
|
1307
|
+
return value.then(async (row) => {
|
|
1308
|
+
if (row)
|
|
1309
|
+
await store.memoHit(key);
|
|
1310
|
+
return row;
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
if (value)
|
|
1314
|
+
store.memoHit(key);
|
|
1315
|
+
return value;
|
|
1316
|
+
}
|
|
1317
|
+
function memoWatchFingerprint(graph, dataDir) {
|
|
1318
|
+
if (!graph.memoWatch || graph.memoWatch.length === 0)
|
|
1187
1319
|
return;
|
|
1188
|
-
|
|
1189
|
-
|
|
1320
|
+
const entries = [];
|
|
1321
|
+
for (const pattern of graph.memoWatch) {
|
|
1322
|
+
if (hasGlobMagic(pattern)) {
|
|
1323
|
+
const matches = [...new Bun.Glob(pattern).scanSync({ cwd: dataDir, onlyFiles: true })].sort();
|
|
1324
|
+
for (const match of matches) {
|
|
1325
|
+
const path2 = isAbsolute(match) ? match : join(dataDir, match);
|
|
1326
|
+
entries.push(fingerprintFile(pattern, path2));
|
|
1327
|
+
}
|
|
1328
|
+
continue;
|
|
1329
|
+
}
|
|
1330
|
+
const path = isAbsolute(pattern) ? pattern : join(dataDir, pattern);
|
|
1331
|
+
entries.push(fingerprintFile(pattern, path));
|
|
1332
|
+
}
|
|
1333
|
+
return entries;
|
|
1190
1334
|
}
|
|
1191
|
-
function
|
|
1192
|
-
|
|
1193
|
-
store.memoPut(key, graphName, nodeId, inputHash(input), JSON.stringify(output));
|
|
1335
|
+
function hasGlobMagic(pattern) {
|
|
1336
|
+
return /[*?[\]{}]/.test(pattern);
|
|
1194
1337
|
}
|
|
1195
|
-
function
|
|
1196
|
-
|
|
1197
|
-
|
|
1338
|
+
function fingerprintFile(pattern, path) {
|
|
1339
|
+
try {
|
|
1340
|
+
const stat = statSync(path);
|
|
1341
|
+
if (!stat.isFile()) {
|
|
1342
|
+
return { pattern, path, missing: true, mtimeMs: null, size: null, sha256: null };
|
|
1343
|
+
}
|
|
1344
|
+
const content = readFileSync(path);
|
|
1345
|
+
return {
|
|
1346
|
+
pattern,
|
|
1347
|
+
path,
|
|
1348
|
+
missing: false,
|
|
1349
|
+
mtimeMs: stat.mtimeMs,
|
|
1350
|
+
size: content.byteLength,
|
|
1351
|
+
sha256: createHash("sha256").update(content).digest("hex")
|
|
1352
|
+
};
|
|
1353
|
+
} catch {
|
|
1354
|
+
return { pattern, path, missing: true, mtimeMs: null, size: null, sha256: null };
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
// src/db/postgres-store.ts
|
|
1359
|
+
import { randomUUID } from "crypto";
|
|
1360
|
+
import { Pool } from "pg";
|
|
1361
|
+
var run = (r) => ({
|
|
1362
|
+
id: String(r.id),
|
|
1363
|
+
graphName: String(r.graph_name),
|
|
1364
|
+
graphVersion: String(r.graph_version),
|
|
1365
|
+
status: r.status,
|
|
1366
|
+
attempts: Number(r.attempts),
|
|
1367
|
+
contextJson: String(r.context_json),
|
|
1368
|
+
resultJson: r.result_json == null ? null : String(r.result_json),
|
|
1369
|
+
error: r.error == null ? null : String(r.error),
|
|
1370
|
+
startedAt: r.started_at == null ? null : String(r.started_at),
|
|
1371
|
+
finishedAt: r.finished_at == null ? null : String(r.finished_at),
|
|
1372
|
+
createdAt: String(r.created_at),
|
|
1373
|
+
updatedAt: String(r.updated_at)
|
|
1374
|
+
});
|
|
1375
|
+
var node = (r) => ({
|
|
1376
|
+
id: String(r.id),
|
|
1377
|
+
runId: String(r.run_id),
|
|
1378
|
+
nodeId: String(r.node_id),
|
|
1379
|
+
status: r.status,
|
|
1380
|
+
attempts: Number(r.attempts),
|
|
1381
|
+
lane: r.lane == null ? null : String(r.lane),
|
|
1382
|
+
exitCode: r.exit_code == null ? null : Number(r.exit_code),
|
|
1383
|
+
outputJson: r.output_json == null ? null : String(r.output_json),
|
|
1384
|
+
error: r.error == null ? null : String(r.error),
|
|
1385
|
+
startedAt: r.started_at == null ? null : String(r.started_at),
|
|
1386
|
+
finishedAt: r.finished_at == null ? null : String(r.finished_at)
|
|
1387
|
+
});
|
|
1388
|
+
var memo = (r) => ({
|
|
1389
|
+
key: String(r.key),
|
|
1390
|
+
graphName: String(r.graph_name),
|
|
1391
|
+
nodeId: String(r.node_id),
|
|
1392
|
+
inputHash: String(r.input_hash),
|
|
1393
|
+
outputJson: String(r.output_json),
|
|
1394
|
+
hitCount: Number(r.hit_count),
|
|
1395
|
+
createdAt: String(r.created_at)
|
|
1396
|
+
});
|
|
1397
|
+
|
|
1398
|
+
class WorkflowStorageError extends Error {
|
|
1399
|
+
constructor() {
|
|
1400
|
+
super("PostgreSQL workflow storage is unavailable or its schema is invalid");
|
|
1401
|
+
this.name = "WorkflowStorageError";
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
class PostgresWorkflowsStore {
|
|
1406
|
+
pool;
|
|
1407
|
+
prefix;
|
|
1408
|
+
dataDir;
|
|
1409
|
+
execution;
|
|
1410
|
+
constructor(options) {
|
|
1411
|
+
const schema = options.schema ?? "workflows";
|
|
1412
|
+
if (!/^[a-z][a-z0-9_]{0,62}$/.test(schema))
|
|
1413
|
+
throw new Error("PostgreSQL schema must be a lowercase SQL identifier");
|
|
1414
|
+
this.prefix = `"${schema}"`;
|
|
1415
|
+
this.dataDir = options.dataDir;
|
|
1416
|
+
this.pool = new Pool({
|
|
1417
|
+
connectionString: options.databaseUrl,
|
|
1418
|
+
max: 4,
|
|
1419
|
+
connectionTimeoutMillis: 5000,
|
|
1420
|
+
idleTimeoutMillis: 1e4,
|
|
1421
|
+
statement_timeout: 1e4,
|
|
1422
|
+
application_name: "workflows"
|
|
1423
|
+
});
|
|
1424
|
+
}
|
|
1425
|
+
async rows(sql, values = []) {
|
|
1198
1426
|
try {
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
} catch {
|
|
1427
|
+
if (this.execution?.lost)
|
|
1428
|
+
throw new WorkflowStorageError;
|
|
1429
|
+
return (await (this.execution?.client ?? this.pool).query(sql, values)).rows;
|
|
1430
|
+
} catch (error) {
|
|
1431
|
+
if (error?.code === "23505")
|
|
1432
|
+
throw Object.assign(new Error("workflow uniqueness conflict"), { code: "23505" });
|
|
1433
|
+
throw new WorkflowStorageError;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
async close() {
|
|
1437
|
+
await this.releaseExecution();
|
|
1438
|
+
await this.pool.end();
|
|
1439
|
+
}
|
|
1440
|
+
async acquireExecution(runId) {
|
|
1441
|
+
if (this.execution)
|
|
1442
|
+
return this.execution.runId === runId && !this.execution.lost;
|
|
1443
|
+
const client = await this.pool.connect();
|
|
1444
|
+
let lost = false;
|
|
1445
|
+
client.on("error", () => {
|
|
1446
|
+
lost = true;
|
|
1447
|
+
if (this.execution?.client === client)
|
|
1448
|
+
this.execution.lost = true;
|
|
1449
|
+
});
|
|
1450
|
+
try {
|
|
1451
|
+
const acquired = await client.query("SELECT pg_try_advisory_lock(hashtextextended($1,0)) AS owned", [`${this.prefix}:${runId}`]);
|
|
1452
|
+
if (!acquired.rows[0]?.owned) {
|
|
1453
|
+
client.release();
|
|
1454
|
+
return false;
|
|
1455
|
+
}
|
|
1456
|
+
this.execution = { runId, client, lost };
|
|
1457
|
+
return !lost;
|
|
1458
|
+
} catch {
|
|
1459
|
+
client.release(true);
|
|
1460
|
+
throw new WorkflowStorageError;
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
async releaseExecution() {
|
|
1464
|
+
const owned = this.execution;
|
|
1465
|
+
if (!owned)
|
|
1466
|
+
return;
|
|
1467
|
+
this.execution = undefined;
|
|
1468
|
+
try {
|
|
1469
|
+
if (!owned.lost)
|
|
1470
|
+
await owned.client.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [`${this.prefix}:${owned.runId}`]);
|
|
1471
|
+
} finally {
|
|
1472
|
+
owned.client.release(owned.lost);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
async transaction(action) {
|
|
1476
|
+
if (this.execution?.lost)
|
|
1477
|
+
throw new WorkflowStorageError;
|
|
1478
|
+
const client = this.execution?.client ?? await this.pool.connect();
|
|
1479
|
+
const retained = client === this.execution?.client;
|
|
1480
|
+
try {
|
|
1481
|
+
await client.query("BEGIN");
|
|
1482
|
+
const result = await action(client);
|
|
1483
|
+
await client.query("COMMIT");
|
|
1484
|
+
return result;
|
|
1485
|
+
} catch {
|
|
1486
|
+
try {
|
|
1487
|
+
await client.query("ROLLBACK");
|
|
1488
|
+
} catch {}
|
|
1489
|
+
throw new WorkflowStorageError;
|
|
1490
|
+
} finally {
|
|
1491
|
+
if (!retained)
|
|
1492
|
+
client.release();
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
async listTables() {
|
|
1496
|
+
return (await this.rows("SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' ORDER BY table_name", [this.prefix.slice(1, -1)])).map((row) => String(row.table_name));
|
|
1497
|
+
}
|
|
1498
|
+
async checkSchema() {
|
|
1499
|
+
if ((await this.listTables()).join(",") !== "memos,run_nodes,runs")
|
|
1500
|
+
return false;
|
|
1501
|
+
await this.rows(`SELECT id,graph_name,graph_version,status,attempts,context_json,result_json,error,started_at,finished_at,created_at,updated_at FROM ${this.prefix}.runs LIMIT 0`);
|
|
1502
|
+
await this.rows(`SELECT id,sequence,run_id,node_id,status,attempts,lane,exit_code,output_json,error,started_at,finished_at FROM ${this.prefix}.run_nodes LIMIT 0`);
|
|
1503
|
+
await this.rows(`SELECT key,graph_name,node_id,input_hash,output_json,hit_count,created_at FROM ${this.prefix}.memos LIMIT 0`);
|
|
1504
|
+
const schema = this.prefix.slice(1, -1);
|
|
1505
|
+
const indices = await this.rows(`SELECT i.indisunique,i.indisvalid,i.indisready,i.indnkeyatts,i.indnatts,i.indpred IS NULL AS unfiltered,pg_get_expr(i.indexprs,i.indrelid) AS expression FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid JOIN pg_class t ON t.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relname='runs_idempotency' AND t.relname='runs'`, [schema]);
|
|
1506
|
+
const index = indices[0];
|
|
1507
|
+
if (indices.length !== 1 || !index?.indisunique || !index.indisvalid || !index.indisready || !index.unfiltered || Number(index.indnkeyatts) !== 1 || Number(index.indnatts) !== 1 || String(index.expression).replace(/[()\s]/g, "") !== "context_json::jsonb#>>'{__wf,idempotencyKey}'::text[]")
|
|
1508
|
+
return false;
|
|
1509
|
+
const columns = await this.rows("SELECT table_name,column_name,data_type,is_nullable,is_identity,identity_generation FROM information_schema.columns WHERE table_schema=$1", [schema]);
|
|
1510
|
+
const required = {
|
|
1511
|
+
runs: ["id:text", "graph_name:text", "graph_version:text", "status:text", "attempts:integer", "context_json:text", "created_at:text", "updated_at:text"],
|
|
1512
|
+
run_nodes: ["id:text", "sequence:bigint", "run_id:text", "node_id:text", "status:text", "attempts:integer"],
|
|
1513
|
+
memos: ["key:text", "graph_name:text", "node_id:text", "input_hash:text", "output_json:text", "hit_count:integer", "created_at:text"]
|
|
1514
|
+
};
|
|
1515
|
+
for (const [table, fields] of Object.entries(required))
|
|
1516
|
+
for (const field of fields) {
|
|
1517
|
+
const [name, type] = field.split(":");
|
|
1518
|
+
const c = columns.find((row) => row.table_name === table && row.column_name === name);
|
|
1519
|
+
if (!c || c.data_type !== type || c.is_nullable !== "NO")
|
|
1520
|
+
return false;
|
|
1521
|
+
if (name === "sequence" && (c.is_identity !== "YES" || c.identity_generation !== "ALWAYS"))
|
|
1522
|
+
return false;
|
|
1523
|
+
}
|
|
1524
|
+
const constraints = await this.rows(`SELECT c.relname AS table_name,co.contype,pg_get_constraintdef(co.oid) AS definition FROM pg_constraint co JOIN pg_class c ON c.oid=co.conrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1`, [schema]);
|
|
1525
|
+
for (const [table, key] of [["runs", "id"], ["run_nodes", "id"], ["memos", "key"]])
|
|
1526
|
+
if (!constraints.some((row) => row.table_name === table && row.contype === "p" && row.definition === `PRIMARY KEY (${key})`))
|
|
1527
|
+
return false;
|
|
1528
|
+
return constraints.some((row) => row.table_name === "run_nodes" && row.contype === "f" && String(row.definition).replaceAll('"', "") === `FOREIGN KEY (run_id) REFERENCES ${schema}.runs(id)`);
|
|
1529
|
+
}
|
|
1530
|
+
async initialize() {
|
|
1531
|
+
const tables = await this.listTables();
|
|
1532
|
+
if (tables.length) {
|
|
1533
|
+
if (await this.checkSchema())
|
|
1534
|
+
return;
|
|
1535
|
+
throw new WorkflowStorageError;
|
|
1536
|
+
}
|
|
1537
|
+
const client = await this.pool.connect();
|
|
1538
|
+
try {
|
|
1539
|
+
await client.query("BEGIN");
|
|
1540
|
+
await client.query(`CREATE SCHEMA IF NOT EXISTS ${this.prefix};
|
|
1541
|
+
CREATE TABLE IF NOT EXISTS ${this.prefix}.runs (
|
|
1542
|
+
id TEXT PRIMARY KEY, graph_name TEXT NOT NULL, graph_version TEXT NOT NULL, status TEXT NOT NULL,
|
|
1543
|
+
attempts INTEGER NOT NULL DEFAULT 0, context_json TEXT NOT NULL, result_json TEXT, error TEXT,
|
|
1544
|
+
started_at TEXT, finished_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL);
|
|
1545
|
+
CREATE TABLE IF NOT EXISTS ${this.prefix}.run_nodes (
|
|
1546
|
+
id TEXT PRIMARY KEY, sequence BIGINT GENERATED ALWAYS AS IDENTITY, run_id TEXT NOT NULL REFERENCES ${this.prefix}.runs(id), node_id TEXT NOT NULL,
|
|
1547
|
+
status TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, lane TEXT, exit_code INTEGER,
|
|
1548
|
+
output_json TEXT, error TEXT, started_at TEXT, finished_at TEXT);
|
|
1549
|
+
CREATE TABLE IF NOT EXISTS ${this.prefix}.memos (
|
|
1550
|
+
key TEXT PRIMARY KEY, graph_name TEXT NOT NULL, node_id TEXT NOT NULL, input_hash TEXT NOT NULL,
|
|
1551
|
+
output_json TEXT NOT NULL, hit_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL);
|
|
1552
|
+
CREATE UNIQUE INDEX IF NOT EXISTS runs_idempotency ON ${this.prefix}.runs ((context_json::jsonb #>> '{__wf,idempotencyKey}'));
|
|
1553
|
+
CREATE INDEX IF NOT EXISTS runs_status ON ${this.prefix}.runs(status);
|
|
1554
|
+
CREATE INDEX IF NOT EXISTS nodes_run ON ${this.prefix}.run_nodes(run_id);`);
|
|
1555
|
+
await client.query("COMMIT");
|
|
1556
|
+
} catch {
|
|
1557
|
+
await client.query("ROLLBACK");
|
|
1558
|
+
throw new WorkflowStorageError;
|
|
1559
|
+
} finally {
|
|
1560
|
+
client.release();
|
|
1561
|
+
}
|
|
1562
|
+
if (!await this.checkSchema())
|
|
1563
|
+
throw new WorkflowStorageError;
|
|
1564
|
+
}
|
|
1565
|
+
async createRun(input) {
|
|
1566
|
+
const at = new Date().toISOString();
|
|
1567
|
+
const rows = await this.rows(`INSERT INTO ${this.prefix}.runs(id,graph_name,graph_version,status,context_json,created_at,updated_at)
|
|
1568
|
+
VALUES($1,$2,$3,'pending',$4,$5,$5) RETURNING *`, [randomUUID(), input.graphName, input.graphVersion, JSON.stringify(input.context ?? {}), at]);
|
|
1569
|
+
return run(rows[0]);
|
|
1570
|
+
}
|
|
1571
|
+
async getRun(id) {
|
|
1572
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.runs WHERE id=$1`, [id]);
|
|
1573
|
+
return rows[0] ? run(rows[0]) : undefined;
|
|
1574
|
+
}
|
|
1575
|
+
async getRunByIdempotencyKey(key) {
|
|
1576
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.runs WHERE context_json::jsonb #>> '{__wf,idempotencyKey}'=$1`, [key]);
|
|
1577
|
+
return rows[0] ? run(rows[0]) : undefined;
|
|
1578
|
+
}
|
|
1579
|
+
async listRuns(options = {}) {
|
|
1580
|
+
const limit = Math.max(1, Math.min(1000, options.limit ?? 100));
|
|
1581
|
+
return (await this.rows(`SELECT * FROM ${this.prefix}.runs ${options.status ? "WHERE status=$2" : ""} ORDER BY created_at DESC,id DESC LIMIT $1`, options.status ? [limit, options.status] : [limit])).map(run);
|
|
1582
|
+
}
|
|
1583
|
+
async setRunStatus(id, status, patch = {}) {
|
|
1584
|
+
const values = [id, status, new Date().toISOString()];
|
|
1585
|
+
const sets = ["status=$2", "updated_at=$3"];
|
|
1586
|
+
for (const [field, key] of [["result_json", "result"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1587
|
+
if (Object.hasOwn(patch, key)) {
|
|
1588
|
+
values.push(key === "result" ? JSON.stringify(patch.result) : patch[key]);
|
|
1589
|
+
sets.push(`${field}=$${values.length}`);
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
if (status === "running" && !Object.hasOwn(patch, "startedAt"))
|
|
1593
|
+
sets.push("started_at=COALESCE(started_at,$3)");
|
|
1594
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET ${sets.join(",")} WHERE id=$1`, values);
|
|
1595
|
+
}
|
|
1596
|
+
async compareRunStatus(id, expected, status, patch = {}) {
|
|
1597
|
+
const values = [id, expected, status, new Date().toISOString()];
|
|
1598
|
+
const sets = ["status=$3", "updated_at=$4"];
|
|
1599
|
+
for (const [field, key] of [["result_json", "result"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1600
|
+
if (patch && Object.hasOwn(patch, key)) {
|
|
1601
|
+
values.push(key === "result" ? JSON.stringify(patch.result) : patch[key]);
|
|
1602
|
+
sets.push(`${field}=$${values.length}`);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
return (await this.rows(`UPDATE ${this.prefix}.runs SET ${sets.join(",")} WHERE id=$1 AND status=ANY($2::text[]) RETURNING id`, values)).length === 1;
|
|
1606
|
+
}
|
|
1607
|
+
async setRunContext(id, context) {
|
|
1608
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET context_json=$2,updated_at=$3 WHERE id=$1`, [id, JSON.stringify(context), new Date().toISOString()]);
|
|
1609
|
+
}
|
|
1610
|
+
async setRunningContext(id, context) {
|
|
1611
|
+
return (await this.rows(`UPDATE ${this.prefix}.runs SET context_json=$2,updated_at=$3 WHERE id=$1 AND status='running' RETURNING id`, [id, JSON.stringify(context), new Date().toISOString()])).length === 1;
|
|
1612
|
+
}
|
|
1613
|
+
async blockRunResume(id) {
|
|
1614
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET context_json=jsonb_set(context_json::jsonb,'{__wf,resumeBlocked}',to_jsonb($2::text))::text,updated_at=$3 WHERE id=$1`, [id, "execution cancellation has not been confirmed", new Date().toISOString()]);
|
|
1615
|
+
}
|
|
1616
|
+
async dispatchIfRunning(id, action) {
|
|
1617
|
+
const marked = await this.rows(`UPDATE ${this.prefix}.runs SET context_json=jsonb_set(context_json::jsonb,'{__wf,executionMayBeActive}','true'::jsonb)::text WHERE id=$1 AND status='running' RETURNING id`, [id]);
|
|
1618
|
+
if (!marked.length)
|
|
1619
|
+
return { started: false };
|
|
1620
|
+
return this.transaction(async (client) => {
|
|
1621
|
+
const run2 = await client.query(`SELECT status FROM ${this.prefix}.runs WHERE id=$1 FOR UPDATE`, [id]);
|
|
1622
|
+
if (run2.rows[0]?.status !== "running")
|
|
1623
|
+
return { started: false };
|
|
1624
|
+
return { started: true, result: action() };
|
|
1625
|
+
});
|
|
1626
|
+
}
|
|
1627
|
+
async finishDispatch(id, unconfirmed) {
|
|
1628
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET context_json=(CASE WHEN $2 THEN jsonb_set(context_json::jsonb #- '{__wf,executionMayBeActive}','{__wf,resumeBlocked}',to_jsonb($3::text)) ELSE context_json::jsonb #- '{__wf,executionMayBeActive}' END)::text,updated_at=$4 WHERE id=$1`, [id, unconfirmed, "execution cancellation has not been confirmed", new Date().toISOString()]);
|
|
1629
|
+
}
|
|
1630
|
+
async commitStep(input) {
|
|
1631
|
+
return this.transaction(async (client) => {
|
|
1632
|
+
const nodes = await client.query(`SELECT run_id FROM ${this.prefix}.run_nodes WHERE id=$1 FOR UPDATE`, [input.nodeRowId]);
|
|
1633
|
+
if (nodes.rows[0]?.run_id !== input.runId)
|
|
1634
|
+
return false;
|
|
1635
|
+
const run2 = await client.query(`SELECT status FROM ${this.prefix}.runs WHERE id=$1 FOR UPDATE`, [input.runId]);
|
|
1636
|
+
if (run2.rows[0]?.status !== "running")
|
|
1637
|
+
return false;
|
|
1638
|
+
const patch = input.patch ?? {}, values = [input.nodeRowId, input.status];
|
|
1639
|
+
const sets = ["status=$2"];
|
|
1640
|
+
for (const [field, key] of [["exit_code", "exitCode"], ["output_json", "output"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1641
|
+
if (Object.hasOwn(patch, key)) {
|
|
1642
|
+
values.push(key === "output" ? JSON.stringify(patch.output) : patch[key]);
|
|
1643
|
+
sets.push(`${field}=$${values.length}`);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
if (input.status === "running") {
|
|
1647
|
+
values.push(new Date().toISOString());
|
|
1648
|
+
sets.push(`started_at=COALESCE(started_at,$${values.length})`);
|
|
1649
|
+
}
|
|
1650
|
+
if (["completed", "failed", "skipped"].includes(input.status)) {
|
|
1651
|
+
values.push(new Date().toISOString());
|
|
1652
|
+
sets.push(`finished_at=$${values.length}`);
|
|
1653
|
+
}
|
|
1654
|
+
if (input.incrementAttempts)
|
|
1655
|
+
sets.push("attempts=attempts+1");
|
|
1656
|
+
await client.query(`UPDATE ${this.prefix}.run_nodes SET ${sets.join(",")} WHERE id=$1`, values);
|
|
1657
|
+
if (input.context !== undefined)
|
|
1658
|
+
await client.query(`UPDATE ${this.prefix}.runs SET context_json=$2,updated_at=$3 WHERE id=$1`, [input.runId, JSON.stringify(input.context), new Date().toISOString()]);
|
|
1659
|
+
if (input.memo) {
|
|
1660
|
+
const m = input.memo;
|
|
1661
|
+
await client.query(`INSERT INTO ${this.prefix}.memos(key,graph_name,node_id,input_hash,output_json,created_at) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT(key) DO UPDATE SET output_json=EXCLUDED.output_json`, [m.key, m.graphName, m.nodeId, m.inputHash, m.outputJson, new Date().toISOString()]);
|
|
1662
|
+
}
|
|
1663
|
+
return true;
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
async bumpAttempts(id) {
|
|
1667
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET attempts=attempts+1,updated_at=$2 WHERE id=$1`, [id, new Date().toISOString()]);
|
|
1668
|
+
}
|
|
1669
|
+
async createRunNode(input) {
|
|
1670
|
+
const rows = await this.rows(`INSERT INTO ${this.prefix}.run_nodes(id,run_id,node_id,lane,status) VALUES($1,$2,$3,$4,'pending') RETURNING *`, [randomUUID(), input.runId, input.nodeId, input.lane ?? null]);
|
|
1671
|
+
return node(rows[0]);
|
|
1672
|
+
}
|
|
1673
|
+
async getRunNode(id) {
|
|
1674
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.run_nodes WHERE id=$1`, [id]);
|
|
1675
|
+
return rows[0] ? node(rows[0]) : undefined;
|
|
1676
|
+
}
|
|
1677
|
+
async listRunNodes(runId) {
|
|
1678
|
+
return (await this.rows(`SELECT * FROM ${this.prefix}.run_nodes WHERE run_id=$1 ORDER BY sequence`, [runId])).map(node);
|
|
1679
|
+
}
|
|
1680
|
+
async getLatestRunNode(runId, nodeId) {
|
|
1681
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.run_nodes WHERE run_id=$1 AND node_id=$2 ORDER BY sequence DESC LIMIT 1`, [runId, nodeId]);
|
|
1682
|
+
return rows[0] ? node(rows[0]) : undefined;
|
|
1683
|
+
}
|
|
1684
|
+
async setRunNodeStatus(id, status, patch = {}) {
|
|
1685
|
+
const values = [id, status];
|
|
1686
|
+
const sets = ["status=$2"];
|
|
1687
|
+
for (const [field, key] of [["exit_code", "exitCode"], ["output_json", "output"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1688
|
+
if (Object.hasOwn(patch, key)) {
|
|
1689
|
+
values.push(key === "output" ? JSON.stringify(patch.output) : patch[key]);
|
|
1690
|
+
sets.push(`${field}=$${values.length}`);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
if (status === "running" && !Object.hasOwn(patch, "startedAt")) {
|
|
1694
|
+
values.push(new Date().toISOString());
|
|
1695
|
+
sets.push(`started_at=COALESCE(started_at,$${values.length})`);
|
|
1696
|
+
}
|
|
1697
|
+
if (["completed", "failed", "skipped"].includes(status) && !Object.hasOwn(patch, "finishedAt")) {
|
|
1698
|
+
values.push(new Date().toISOString());
|
|
1699
|
+
sets.push(`finished_at=$${values.length}`);
|
|
1700
|
+
}
|
|
1701
|
+
await this.rows(`UPDATE ${this.prefix}.run_nodes SET ${sets.join(",")} WHERE id=$1`, values);
|
|
1702
|
+
}
|
|
1703
|
+
async bumpAttemptsNode(id) {
|
|
1704
|
+
await this.rows(`UPDATE ${this.prefix}.run_nodes SET attempts=attempts+1 WHERE id=$1`, [id]);
|
|
1705
|
+
}
|
|
1706
|
+
async memoPut(key, graphName, nodeId, inputHash2, outputJson) {
|
|
1707
|
+
await this.rows(`INSERT INTO ${this.prefix}.memos(key,graph_name,node_id,input_hash,output_json,created_at) VALUES($1,$2,$3,$4,$5,$6)
|
|
1708
|
+
ON CONFLICT(key) DO UPDATE SET output_json=EXCLUDED.output_json`, [key, graphName, nodeId, inputHash2, outputJson, new Date().toISOString()]);
|
|
1709
|
+
}
|
|
1710
|
+
async memoGet(key) {
|
|
1711
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.memos WHERE key=$1`, [key]);
|
|
1712
|
+
return rows[0] ? memo(rows[0]) : undefined;
|
|
1713
|
+
}
|
|
1714
|
+
async memoHit(key) {
|
|
1715
|
+
await this.rows(`UPDATE ${this.prefix}.memos SET hit_count=hit_count+1 WHERE key=$1`, [key]);
|
|
1716
|
+
}
|
|
1717
|
+
async memoList() {
|
|
1718
|
+
return (await this.rows(`SELECT * FROM ${this.prefix}.memos ORDER BY created_at DESC,key LIMIT 1000`)).map(memo);
|
|
1719
|
+
}
|
|
1720
|
+
async memoClear() {
|
|
1721
|
+
throw new Error("PostgreSQL memo removal requires the reviewed server-operator preservation procedure");
|
|
1203
1722
|
}
|
|
1204
|
-
return;
|
|
1205
1723
|
}
|
|
1206
1724
|
|
|
1207
1725
|
// src/daemon.ts
|
|
1208
1726
|
init_types();
|
|
1727
|
+
init_types();
|
|
1209
1728
|
var DEFAULT_LEASE_TTL_MS = 120000;
|
|
1210
1729
|
var DEFAULT_MAX_ATTEMPTS2 = 3;
|
|
1211
1730
|
var DEFAULT_MAX_DISPATCH = 4;
|
|
@@ -1218,27 +1737,28 @@ function stripRunState(context) {
|
|
|
1218
1737
|
const { __wf: _wf, steps: _steps, ...user } = context;
|
|
1219
1738
|
return user;
|
|
1220
1739
|
}
|
|
1221
|
-
function runCommand(job) {
|
|
1740
|
+
async function runCommand(job) {
|
|
1222
1741
|
if (!job.command) {
|
|
1223
1742
|
return { ok: false, exitCode: 2, output: "", durationMs: 0, error: "no command" };
|
|
1224
1743
|
}
|
|
1225
1744
|
const started = Date.now();
|
|
1226
1745
|
const timeoutMs = job.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
1227
|
-
const proc = Bun.
|
|
1746
|
+
const proc = Bun.spawn({
|
|
1228
1747
|
cmd: ["/bin/sh", "-c", job.command],
|
|
1229
1748
|
cwd: job.cwd,
|
|
1230
|
-
env:
|
|
1749
|
+
env: subscriptionEnvironment(job.env),
|
|
1231
1750
|
stdout: "pipe",
|
|
1232
1751
|
stderr: "pipe",
|
|
1233
1752
|
timeout: timeoutMs
|
|
1234
1753
|
});
|
|
1235
|
-
const
|
|
1236
|
-
|
|
1237
|
-
|
|
1754
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
1755
|
+
new Response(proc.stdout).text(),
|
|
1756
|
+
new Response(proc.stderr).text(),
|
|
1757
|
+
proc.exited
|
|
1758
|
+
]);
|
|
1238
1759
|
const output = stderr ? `${stdout}
|
|
1239
1760
|
${stderr}`.trim() : stdout.trim();
|
|
1240
|
-
|
|
1241
|
-
return { ok: exitCode === 0, exitCode, output, durationMs };
|
|
1761
|
+
return { ok: exitCode === 0, exitCode, output, durationMs: Date.now() - started };
|
|
1242
1762
|
}
|
|
1243
1763
|
|
|
1244
1764
|
class WorkflowsDaemon {
|
|
@@ -1259,7 +1779,7 @@ class WorkflowsDaemon {
|
|
|
1259
1779
|
constructor(store, wal, options = {}) {
|
|
1260
1780
|
this.store = store;
|
|
1261
1781
|
this.wal = wal;
|
|
1262
|
-
this.worker = options.worker ??
|
|
1782
|
+
this.worker = options.worker ?? `worker-${randomUUID2()}`;
|
|
1263
1783
|
this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
|
|
1264
1784
|
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS2;
|
|
1265
1785
|
this.maxDispatchPerCycle = options.maxDispatchPerCycle ?? DEFAULT_MAX_DISPATCH;
|
|
@@ -1291,17 +1811,22 @@ class WorkflowsDaemon {
|
|
|
1291
1811
|
return this.leases;
|
|
1292
1812
|
}
|
|
1293
1813
|
claim(runId) {
|
|
1814
|
+
return this.wal.exclusive(() => this.claimLocked(runId));
|
|
1815
|
+
}
|
|
1816
|
+
claimLocked(runId) {
|
|
1294
1817
|
const now = this.time();
|
|
1295
1818
|
const existing = this.leases.get(runId);
|
|
1296
1819
|
if (existing && existing.worker !== this.worker && existing.expiresAtMs > now) {
|
|
1297
1820
|
return null;
|
|
1298
1821
|
}
|
|
1299
|
-
const
|
|
1822
|
+
const replay = this.wal.replay();
|
|
1823
|
+
const liveClaims = replay.liveClaims(now);
|
|
1300
1824
|
const foreign = liveClaims.get(runId);
|
|
1301
1825
|
if (foreign && foreign.worker !== this.worker && foreign.expiresAtMs > now) {
|
|
1302
1826
|
return null;
|
|
1303
1827
|
}
|
|
1304
|
-
const
|
|
1828
|
+
const lastFencing = replay.entries.reduce((max, entry) => entry.op.op === "claim_acquired" && entry.op.runId === runId ? Math.max(max, entry.op.fencing) : max, 0);
|
|
1829
|
+
const fencing = Math.max(lastFencing, this.fencingCounter.get(runId) ?? 0) + 1;
|
|
1305
1830
|
this.fencingCounter.set(runId, fencing);
|
|
1306
1831
|
const lease = { runId, worker: this.worker, expiresAtMs: now + this.leaseTtlMs, fencing };
|
|
1307
1832
|
this.leases.set(runId, lease);
|
|
@@ -1316,16 +1841,36 @@ class WorkflowsDaemon {
|
|
|
1316
1841
|
return lease;
|
|
1317
1842
|
}
|
|
1318
1843
|
heartbeat(runId, fencing) {
|
|
1844
|
+
return this.wal.exclusive(() => this.heartbeatLocked(runId, fencing));
|
|
1845
|
+
}
|
|
1846
|
+
heartbeatLocked(runId, fencing) {
|
|
1319
1847
|
const lease = this.leases.get(runId);
|
|
1320
1848
|
if (!lease || lease.fencing !== fencing)
|
|
1321
1849
|
return false;
|
|
1850
|
+
const current = this.wal.replay({ repair: false }).liveClaims(this.time()).get(runId);
|
|
1851
|
+
if (!current || current.worker !== lease.worker || current.fencing !== fencing)
|
|
1852
|
+
return false;
|
|
1322
1853
|
lease.expiresAtMs = this.time() + this.leaseTtlMs;
|
|
1854
|
+
this.wal.append({
|
|
1855
|
+
op: "claim_acquired",
|
|
1856
|
+
runId,
|
|
1857
|
+
worker: lease.worker,
|
|
1858
|
+
fencing,
|
|
1859
|
+
expiresAt: new Date(lease.expiresAtMs).toISOString(),
|
|
1860
|
+
at: nowIso()
|
|
1861
|
+
});
|
|
1323
1862
|
return true;
|
|
1324
1863
|
}
|
|
1325
1864
|
release(runId, fencing) {
|
|
1865
|
+
return this.wal.exclusive(() => this.releaseLocked(runId, fencing));
|
|
1866
|
+
}
|
|
1867
|
+
releaseLocked(runId, fencing) {
|
|
1326
1868
|
const lease = this.leases.get(runId);
|
|
1327
1869
|
if (!lease || lease.fencing !== fencing)
|
|
1328
1870
|
return false;
|
|
1871
|
+
const current = this.wal.replay().liveClaims(this.time()).get(runId);
|
|
1872
|
+
if (current && (current.worker !== lease.worker || current.fencing !== fencing))
|
|
1873
|
+
return false;
|
|
1329
1874
|
this.leases.delete(runId);
|
|
1330
1875
|
this.wal.append({ op: "claim_released", runId, fencing, at: nowIso() });
|
|
1331
1876
|
return true;
|
|
@@ -1340,30 +1885,30 @@ class WorkflowsDaemon {
|
|
|
1340
1885
|
}
|
|
1341
1886
|
}
|
|
1342
1887
|
if (this.isolatedRunId === null) {
|
|
1343
|
-
const repair = repairTornRuns(this.store, this.wal, { maxAttempts: this.maxAttempts, now });
|
|
1888
|
+
const repair = await repairTornRuns(this.store, this.wal, { maxAttempts: this.maxAttempts, now });
|
|
1344
1889
|
report.interrupted += repair.interrupted;
|
|
1345
1890
|
report.requeued += repair.requeued;
|
|
1346
1891
|
report.failed += repair.failed;
|
|
1347
1892
|
}
|
|
1348
|
-
const pending = this.store.listRuns({ status: "pending", limit: this.maxDispatchPerCycle }).concat(this.store.listRuns({ status: "interrupted", limit: this.maxDispatchPerCycle })).sort((a, b) => a.createdAt === b.createdAt ? a.id < b.id ? 1 : -1 : a.createdAt < b.createdAt ? 1 : -1).slice(0, this.maxDispatchPerCycle);
|
|
1349
|
-
for (const
|
|
1350
|
-
if (this.isolatedRunId !== null &&
|
|
1893
|
+
const pending = (await this.store.listRuns({ status: "pending", limit: this.maxDispatchPerCycle })).concat(await this.store.listRuns({ status: "interrupted", limit: this.maxDispatchPerCycle })).sort((a, b) => a.createdAt === b.createdAt ? a.id < b.id ? 1 : -1 : a.createdAt < b.createdAt ? 1 : -1).slice(0, this.maxDispatchPerCycle);
|
|
1894
|
+
for (const run2 of pending) {
|
|
1895
|
+
if (this.isolatedRunId !== null && run2.id !== this.isolatedRunId)
|
|
1351
1896
|
continue;
|
|
1352
|
-
const lease = this.claim(
|
|
1897
|
+
const lease = this.claim(run2.id);
|
|
1353
1898
|
if (!lease)
|
|
1354
1899
|
continue;
|
|
1355
|
-
this.store.setRunStatus(
|
|
1356
|
-
this.wal.append({ op: "run_started", runId:
|
|
1900
|
+
await this.store.setRunStatus(run2.id, "running");
|
|
1901
|
+
this.wal.append({ op: "run_started", runId: run2.id, at: nowIso() });
|
|
1357
1902
|
report.dispatched++;
|
|
1358
1903
|
}
|
|
1359
|
-
const running = this.store.listRuns({ status: "running", limit: this.maxStepsPerCycle });
|
|
1904
|
+
const running = await this.store.listRuns({ status: "running", limit: this.maxStepsPerCycle });
|
|
1360
1905
|
let steps = 0;
|
|
1361
|
-
for (const
|
|
1362
|
-
if (this.isolatedRunId !== null &&
|
|
1906
|
+
for (const run2 of running) {
|
|
1907
|
+
if (this.isolatedRunId !== null && run2.id !== this.isolatedRunId)
|
|
1363
1908
|
continue;
|
|
1364
1909
|
if (steps >= this.maxStepsPerCycle)
|
|
1365
1910
|
break;
|
|
1366
|
-
const outcome = await this.advanceRun(
|
|
1911
|
+
const outcome = await this.advanceRun(run2.id);
|
|
1367
1912
|
steps++;
|
|
1368
1913
|
if (outcome.kind === "completed")
|
|
1369
1914
|
report.completed++;
|
|
@@ -1389,66 +1934,78 @@ class WorkflowsDaemon {
|
|
|
1389
1934
|
} else {
|
|
1390
1935
|
this.graphCache.set(graph.name, existing);
|
|
1391
1936
|
}
|
|
1392
|
-
|
|
1937
|
+
const initial = context !== null && typeof context === "object" ? context : {};
|
|
1938
|
+
return this.store.createRun({
|
|
1939
|
+
graphName: graph.name,
|
|
1940
|
+
graphVersion: graph.version,
|
|
1941
|
+
context: { ...initial, __wf: { ...initial.__wf ?? {}, graph } }
|
|
1942
|
+
});
|
|
1393
1943
|
}
|
|
1394
1944
|
async advanceRun(runId) {
|
|
1395
|
-
const
|
|
1396
|
-
if (!
|
|
1945
|
+
const run2 = await this.store.getRun(runId);
|
|
1946
|
+
if (!run2 || run2.status !== "running")
|
|
1397
1947
|
return { kind: "noop" };
|
|
1398
1948
|
const lease = this.leases.get(runId);
|
|
1399
1949
|
if (!lease || lease.expiresAtMs <= this.time())
|
|
1400
1950
|
return { kind: "blocked" };
|
|
1401
|
-
const
|
|
1951
|
+
const storedContext = JSON.parse(run2.contextJson);
|
|
1952
|
+
const pinned = storedContext.__wf?.graph;
|
|
1953
|
+
const graph = pinned ?? this.loadGraph(run2.graphName);
|
|
1402
1954
|
if (!graph) {
|
|
1403
|
-
this.failRun(
|
|
1955
|
+
await this.failRun(run2, `graph ${run2.graphName} not found in daemon graph cache`);
|
|
1404
1956
|
return { kind: "failed", error: "graph not found" };
|
|
1405
1957
|
}
|
|
1406
|
-
const context =
|
|
1958
|
+
const context = storedContext;
|
|
1407
1959
|
const rawWf = context.__wf ?? {};
|
|
1408
1960
|
const wf = {
|
|
1409
1961
|
...rawWf,
|
|
1410
1962
|
cursor: typeof rawWf.cursor === "string" ? rawWf.cursor : undefined,
|
|
1411
|
-
loops: rawWf.loops ?? {},
|
|
1412
|
-
completedLoops: rawWf.completedLoops ?? {}
|
|
1963
|
+
loops: Object.assign(Object.create(null), rawWf.loops ?? {}),
|
|
1964
|
+
completedLoops: Object.assign(Object.create(null), rawWf.completedLoops ?? {})
|
|
1413
1965
|
};
|
|
1414
1966
|
try {
|
|
1967
|
+
if (rawWf.stepTerminal === true) {
|
|
1968
|
+
context.__wf = wf;
|
|
1969
|
+
await this.completeRun(run2, context);
|
|
1970
|
+
return { kind: "completed" };
|
|
1971
|
+
}
|
|
1415
1972
|
const cursor = wf.cursor;
|
|
1416
|
-
const
|
|
1417
|
-
if (!
|
|
1418
|
-
this.failRun(
|
|
1973
|
+
const node2 = cursor === undefined ? graph.nodes.find((n) => n.type === "start") : findNode(graph, cursor);
|
|
1974
|
+
if (!node2) {
|
|
1975
|
+
await this.failRun(run2, `cursor ${cursor ?? "<start>"} resolves to no node`);
|
|
1419
1976
|
return { kind: "failed", error: "no node at cursor" };
|
|
1420
1977
|
}
|
|
1421
|
-
switch (
|
|
1978
|
+
switch (node2.type) {
|
|
1422
1979
|
case "start": {
|
|
1423
|
-
wf.cursor =
|
|
1424
|
-
this.persistContext(
|
|
1425
|
-
return { kind: "advanced", nodeId:
|
|
1980
|
+
wf.cursor = node2.next;
|
|
1981
|
+
await this.persistContext(run2, context, wf);
|
|
1982
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1426
1983
|
}
|
|
1427
1984
|
case "step": {
|
|
1428
|
-
return await this.executeStep(
|
|
1985
|
+
return await this.executeStep(run2, graph, context, wf, node2);
|
|
1429
1986
|
}
|
|
1430
1987
|
case "decision": {
|
|
1431
|
-
const value = evaluateExpr(
|
|
1432
|
-
const next = value ?
|
|
1988
|
+
const value = evaluateExpr(node2.condition, this.conditionScope(context));
|
|
1989
|
+
const next = value ? node2.then : node2.else;
|
|
1433
1990
|
if (!next) {
|
|
1434
|
-
this.failRun(
|
|
1991
|
+
await this.failRun(run2, `decision ${node2.id} evaluated ${value} but no ${value ? "then" : "else"} edge`);
|
|
1435
1992
|
return { kind: "failed", error: "missing decision edge" };
|
|
1436
1993
|
}
|
|
1437
1994
|
wf.cursor = next;
|
|
1438
|
-
this.persistContext(
|
|
1439
|
-
return { kind: "advanced", nodeId:
|
|
1995
|
+
await this.persistContext(run2, context, wf);
|
|
1996
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1440
1997
|
}
|
|
1441
1998
|
case "while": {
|
|
1442
|
-
return this.executeWhile(
|
|
1999
|
+
return await this.executeWhile(run2, graph, context, wf, node2);
|
|
1443
2000
|
}
|
|
1444
2001
|
case "end": {
|
|
1445
|
-
this.completeRun(
|
|
2002
|
+
await this.completeRun(run2, context);
|
|
1446
2003
|
return { kind: "completed" };
|
|
1447
2004
|
}
|
|
1448
2005
|
}
|
|
1449
2006
|
} catch (err) {
|
|
1450
2007
|
if (err instanceof ExprEvalError || err instanceof ExprSyntaxError) {
|
|
1451
|
-
this.failRun(
|
|
2008
|
+
await this.failRun(run2, `condition error at ${wf.cursor ?? "<start>"}: ${err.message}`);
|
|
1452
2009
|
return { kind: "failed", error: err.message };
|
|
1453
2010
|
}
|
|
1454
2011
|
throw err;
|
|
@@ -1460,124 +2017,180 @@ class WorkflowsDaemon {
|
|
|
1460
2017
|
conditionScope(context) {
|
|
1461
2018
|
return { ...context, steps: context.steps ?? {} };
|
|
1462
2019
|
}
|
|
1463
|
-
persistContext(
|
|
2020
|
+
async persistContext(run2, context, wf) {
|
|
1464
2021
|
context.__wf = wf;
|
|
1465
|
-
this.store.
|
|
1466
|
-
}
|
|
1467
|
-
async
|
|
1468
|
-
const
|
|
1469
|
-
const
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
2022
|
+
await this.store.setRunningContext(run2.id, context);
|
|
2023
|
+
}
|
|
2024
|
+
async discardStep(run2, nodeRowId) {
|
|
2025
|
+
const current = await this.store.getRun(run2.id);
|
|
2026
|
+
const node2 = await this.store.getRunNode(nodeRowId);
|
|
2027
|
+
if (current?.status === "cancelled" && node2 && ["pending", "running"].includes(node2.status))
|
|
2028
|
+
await this.store.setRunNodeStatus(nodeRowId, "skipped", { error: "run cancelled", finishedAt: nowIso() });
|
|
2029
|
+
return { kind: "noop" };
|
|
2030
|
+
}
|
|
2031
|
+
async executeStep(run2, graph, context, wf, node2) {
|
|
2032
|
+
const nodeId = node2.id;
|
|
2033
|
+
const latest = await this.store.getLatestRunNode(run2.id, nodeId);
|
|
2034
|
+
const isLoopBody = graph.nodes.some((candidate) => candidate.type === "while" && candidate.body.includes(nodeId));
|
|
2035
|
+
const nodeRow = latest === undefined || isLoopBody && latest.status === "completed" ? await this.store.createRunNode({ runId: run2.id, nodeId, lane: node2.lane ?? "claude" }) : latest;
|
|
2036
|
+
const memoWatch = node2.memo ? memoWatchFingerprint(graph, this.store.dataDir) : undefined;
|
|
2037
|
+
const memoSource = { command: node2.command, prompt: node2.prompt, context: stripRunState(context) };
|
|
2038
|
+
const memoInput = node2.memo ? memoWatch !== undefined ? { ...memoSource, memoWatch } : memoSource : undefined;
|
|
2039
|
+
if (memoInput !== undefined) {
|
|
2040
|
+
const hit = await tryMemoHit(this.store, run2.graphName, nodeId, memoInput);
|
|
1474
2041
|
if (hit) {
|
|
1475
2042
|
const cached = JSON.parse(hit.outputJson);
|
|
1476
|
-
this.store.setRunNodeStatus(nodeRow.id, "completed", { exitCode: Number(cached.exitCode ?? 0), output: { ...cached, memoHit: true } });
|
|
1477
|
-
this.wal.append({ op: "node_finished", runId: run.id, nodeId, status: "completed", at: nowIso() });
|
|
1478
2043
|
this.recordStepResult(context, nodeId, { ...cached, memoHit: true });
|
|
1479
|
-
|
|
2044
|
+
const continuation2 = this.advanceAfterStep(graph, context, wf, node2.next);
|
|
2045
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "completed", patch: { exitCode: Number(cached.exitCode ?? 0), output: { ...cached, memoHit: true } }, context }))
|
|
2046
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2047
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "completed", at: nowIso() });
|
|
2048
|
+
if (continuation2.kind === "completed")
|
|
2049
|
+
await this.completeRun(run2, context);
|
|
2050
|
+
return continuation2;
|
|
1480
2051
|
}
|
|
1481
2052
|
}
|
|
1482
2053
|
if (nodeRow.status !== "running") {
|
|
1483
|
-
this.store.
|
|
1484
|
-
|
|
2054
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "running" }))
|
|
2055
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2056
|
+
this.wal.append({ op: "node_started", runId: run2.id, nodeId, at: nowIso() });
|
|
1485
2057
|
}
|
|
1486
2058
|
const job = {
|
|
1487
|
-
lane:
|
|
1488
|
-
prompt:
|
|
1489
|
-
command:
|
|
1490
|
-
timeoutMs:
|
|
2059
|
+
lane: node2.lane ?? "claude",
|
|
2060
|
+
prompt: node2.prompt,
|
|
2061
|
+
command: node2.command,
|
|
2062
|
+
timeoutMs: node2.timeoutMs,
|
|
1491
2063
|
env: this.env
|
|
1492
2064
|
};
|
|
1493
2065
|
let result;
|
|
2066
|
+
const activeLease = this.leases.get(run2.id);
|
|
2067
|
+
const pulse = setInterval(() => {
|
|
2068
|
+
this.heartbeat(run2.id, activeLease.fencing);
|
|
2069
|
+
}, Math.max(1, Math.floor(this.leaseTtlMs / 3)));
|
|
1494
2070
|
try {
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
2071
|
+
const dispatched = await this.store.dispatchIfRunning(run2.id, async () => {
|
|
2072
|
+
try {
|
|
2073
|
+
return await this.laneRunner(job);
|
|
2074
|
+
} catch (err) {
|
|
2075
|
+
return { ok: false, exitCode: 1, output: "", durationMs: 0, error: String(err instanceof Error ? err.message : err) };
|
|
2076
|
+
}
|
|
2077
|
+
});
|
|
2078
|
+
if (!dispatched.started) {
|
|
2079
|
+
await this.store.finishDispatch(run2.id, false);
|
|
2080
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2081
|
+
}
|
|
2082
|
+
result = await dispatched.result;
|
|
2083
|
+
} finally {
|
|
2084
|
+
clearInterval(pulse);
|
|
2085
|
+
}
|
|
2086
|
+
const unconfirmed = result.retryable === false && /exceeded.*budget/.test(result.error ?? "");
|
|
2087
|
+
await this.store.finishDispatch(run2.id, unconfirmed);
|
|
2088
|
+
if (unconfirmed)
|
|
2089
|
+
Object.assign(wf, { resumeBlocked: "execution cancellation has not been confirmed" });
|
|
2090
|
+
const current = await this.store.getRun(run2.id);
|
|
2091
|
+
const currentClaim = this.wal.replay({ repair: false }).liveClaims(this.time()).get(run2.id);
|
|
2092
|
+
const lease = this.leases.get(run2.id);
|
|
2093
|
+
if (!current || current.status !== "running" || !currentClaim || !lease || currentClaim.worker !== lease.worker || currentClaim.fencing !== lease.fencing) {
|
|
2094
|
+
if (current?.status === "cancelled")
|
|
2095
|
+
await this.store.setRunNodeStatus(nodeRow.id, "skipped", { error: "run cancelled", finishedAt: nowIso() });
|
|
2096
|
+
return { kind: "noop" };
|
|
2097
|
+
}
|
|
2098
|
+
try {
|
|
2099
|
+
assertNoSecrets({ output: result.output, error: result.error }, `node ${nodeId} execution`);
|
|
2100
|
+
} catch {
|
|
2101
|
+
result = { ok: false, exitCode: 1, output: "", durationMs: result.durationMs, error: "execution result refused by secrets write-gate", retryable: false };
|
|
1498
2102
|
}
|
|
1499
2103
|
const output = { ok: result.ok, exitCode: result.exitCode, output: result.output, durationMs: result.durationMs };
|
|
1500
2104
|
if (!result.ok) {
|
|
1501
|
-
const shouldRetry = nodeRow.attempts < (
|
|
1502
|
-
this.store.bumpAttemptsNode(nodeRow.id);
|
|
2105
|
+
const shouldRetry = result.retryable !== false && nodeRow.attempts < (node2.maxRetries ?? 0);
|
|
1503
2106
|
if (shouldRetry) {
|
|
1504
|
-
this.store.
|
|
2107
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "pending", patch: { error: result.error ?? `exit ${result.exitCode}` }, incrementAttempts: true }))
|
|
2108
|
+
return this.discardStep(run2, nodeRow.id);
|
|
1505
2109
|
return { kind: "advanced", nodeId };
|
|
1506
2110
|
}
|
|
1507
2111
|
const error = result.error ?? `step ${nodeId} failed with exit ${result.exitCode}`;
|
|
1508
|
-
this.store.
|
|
1509
|
-
|
|
1510
|
-
this.
|
|
2112
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "failed", patch: { exitCode: result.exitCode, error }, incrementAttempts: true }))
|
|
2113
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2114
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "failed", at: nowIso() });
|
|
2115
|
+
await this.failRun(run2, error);
|
|
1511
2116
|
return { kind: "failed", error };
|
|
1512
2117
|
}
|
|
1513
2118
|
try {
|
|
1514
2119
|
assertNoSecrets(output, `node ${nodeId} output`);
|
|
1515
2120
|
} catch (err) {
|
|
1516
2121
|
const message = err instanceof Error ? err.message : String(err);
|
|
1517
|
-
this.store.
|
|
1518
|
-
|
|
1519
|
-
this.
|
|
2122
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "failed", patch: { error: message } }))
|
|
2123
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2124
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "failed", at: nowIso() });
|
|
2125
|
+
await this.failRun(run2, message);
|
|
1520
2126
|
return { kind: "failed", error: message };
|
|
1521
2127
|
}
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
2128
|
+
let memo2;
|
|
2129
|
+
if (memoInput !== undefined) {
|
|
2130
|
+
const memoInputAfter = node2.memo ? memoWatchFingerprint(graph, this.store.dataDir) : undefined;
|
|
2131
|
+
const stable = memoWatch === undefined && memoInputAfter === undefined || memoWatch !== undefined && memoInputAfter !== undefined && JSON.stringify(memoWatch) === JSON.stringify(memoInputAfter);
|
|
2132
|
+
if (stable) {
|
|
2133
|
+
memo2 = { key: memoKey(run2.graphName, nodeId, memoInput), graphName: run2.graphName, nodeId, inputHash: inputHash(memoInput), outputJson: JSON.stringify(output) };
|
|
2134
|
+
}
|
|
1527
2135
|
}
|
|
1528
2136
|
this.recordStepResult(context, nodeId, output);
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
2137
|
+
const continuation = this.advanceAfterStep(graph, context, wf, node2.next);
|
|
2138
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "completed", patch: { exitCode: result.exitCode, output }, context, memo: memo2 }))
|
|
2139
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2140
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "completed", at: nowIso() });
|
|
2141
|
+
if (memo2)
|
|
2142
|
+
this.wal.append({ op: "memo_set", key: memo2.key, at: nowIso() });
|
|
2143
|
+
if (continuation.kind === "completed")
|
|
2144
|
+
await this.completeRun(run2, context);
|
|
2145
|
+
return continuation;
|
|
2146
|
+
}
|
|
2147
|
+
advanceAfterStep(graph, context, wf, next) {
|
|
2148
|
+
context.__wf = wf;
|
|
1532
2149
|
if (next) {
|
|
1533
2150
|
wf.cursor = next;
|
|
1534
|
-
this.persistContext(run, context, wf);
|
|
1535
2151
|
return { kind: "advanced", nodeId: wf.cursor };
|
|
1536
2152
|
}
|
|
1537
2153
|
const cursor = wf.cursor;
|
|
1538
|
-
const graph = this.loadGraph(run.graphName);
|
|
1539
2154
|
const whileNode = graph.nodes.find((n) => n.type === "while" && n.body.includes(cursor ?? ""));
|
|
1540
2155
|
if (whileNode) {
|
|
1541
2156
|
const index = whileNode.body.indexOf(cursor ?? "");
|
|
1542
2157
|
if (index >= 0 && index < whileNode.body.length - 1) {
|
|
1543
2158
|
wf.cursor = whileNode.body[index + 1];
|
|
1544
|
-
this.persistContext(run, context, wf);
|
|
1545
2159
|
return { kind: "advanced", nodeId: wf.cursor };
|
|
1546
2160
|
}
|
|
1547
2161
|
wf.loops[whileNode.id] = (wf.loops[whileNode.id] ?? 0) + 1;
|
|
1548
2162
|
wf.cursor = whileNode.id;
|
|
1549
|
-
this.persistContext(run, context, wf);
|
|
1550
2163
|
return { kind: "advanced", nodeId: whileNode.id };
|
|
1551
2164
|
}
|
|
1552
|
-
|
|
2165
|
+
wf.stepTerminal = true;
|
|
1553
2166
|
return { kind: "completed" };
|
|
1554
2167
|
}
|
|
1555
|
-
executeWhile(
|
|
1556
|
-
const iteration = wf.loops[
|
|
1557
|
-
if (iteration >=
|
|
1558
|
-
const error = `while node ${
|
|
1559
|
-
this.failRun(
|
|
2168
|
+
async executeWhile(run2, graph, context, wf, node2) {
|
|
2169
|
+
const iteration = wf.loops[node2.id] ?? 0;
|
|
2170
|
+
if (iteration >= node2.maxIterations) {
|
|
2171
|
+
const error = `while node ${node2.id} exceeded maxIterations (${node2.maxIterations})`;
|
|
2172
|
+
await this.failRun(run2, error);
|
|
1560
2173
|
return { kind: "failed", error };
|
|
1561
2174
|
}
|
|
1562
2175
|
const scope = { ...this.conditionScope(context), i: iteration };
|
|
1563
|
-
const keepGoing = evaluateExpr(
|
|
2176
|
+
const keepGoing = evaluateExpr(node2.condition, scope);
|
|
1564
2177
|
if (!keepGoing) {
|
|
1565
|
-
wf.completedLoops[
|
|
1566
|
-
delete wf.loops[
|
|
1567
|
-
wf.cursor =
|
|
1568
|
-
this.persistContext(
|
|
1569
|
-
return { kind: "advanced", nodeId:
|
|
2178
|
+
wf.completedLoops[node2.id] = iteration;
|
|
2179
|
+
delete wf.loops[node2.id];
|
|
2180
|
+
wf.cursor = node2.next;
|
|
2181
|
+
await this.persistContext(run2, context, wf);
|
|
2182
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1570
2183
|
}
|
|
1571
|
-
wf.cursor =
|
|
1572
|
-
this.persistContext(
|
|
1573
|
-
return { kind: "advanced", nodeId:
|
|
2184
|
+
wf.cursor = node2.body[0];
|
|
2185
|
+
await this.persistContext(run2, context, wf);
|
|
2186
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1574
2187
|
}
|
|
1575
2188
|
recordStepResult(context, nodeId, output) {
|
|
1576
|
-
const steps = context.steps ?? {};
|
|
2189
|
+
const steps = Object.assign(Object.create(null), context.steps ?? {});
|
|
1577
2190
|
steps[nodeId] = output;
|
|
1578
2191
|
context.steps = steps;
|
|
1579
2192
|
}
|
|
1580
|
-
completeRun(
|
|
2193
|
+
async completeRun(run2, context) {
|
|
1581
2194
|
const result = {
|
|
1582
2195
|
steps: context.steps ?? {},
|
|
1583
2196
|
iterations: context.__wf?.completedLoops ?? {}
|
|
@@ -1586,21 +2199,23 @@ class WorkflowsDaemon {
|
|
|
1586
2199
|
assertNoSecrets(result, "run result");
|
|
1587
2200
|
} catch (err) {
|
|
1588
2201
|
const message = err instanceof Error ? err.message : String(err);
|
|
1589
|
-
this.failRun(
|
|
2202
|
+
await this.failRun(run2, message);
|
|
1590
2203
|
return;
|
|
1591
2204
|
}
|
|
1592
|
-
this.store.
|
|
1593
|
-
|
|
1594
|
-
|
|
2205
|
+
if (!await this.store.compareRunStatus(run2.id, ["running"], "completed", { result, finishedAt: nowIso(), error: null }))
|
|
2206
|
+
return;
|
|
2207
|
+
this.wal.append({ op: "run_finished", runId: run2.id, status: "completed", at: nowIso() });
|
|
2208
|
+
const lease = this.leases.get(run2.id);
|
|
1595
2209
|
if (lease)
|
|
1596
|
-
this.release(
|
|
2210
|
+
this.release(run2.id, lease.fencing);
|
|
1597
2211
|
}
|
|
1598
|
-
failRun(
|
|
1599
|
-
this.store.
|
|
1600
|
-
|
|
1601
|
-
|
|
2212
|
+
async failRun(run2, error) {
|
|
2213
|
+
if (!await this.store.compareRunStatus(run2.id, ["running"], "failed", { error, finishedAt: nowIso() }))
|
|
2214
|
+
return;
|
|
2215
|
+
this.wal.append({ op: "run_finished", runId: run2.id, status: "failed", at: nowIso() });
|
|
2216
|
+
const lease = this.leases.get(run2.id);
|
|
1602
2217
|
if (lease)
|
|
1603
|
-
this.release(
|
|
2218
|
+
this.release(run2.id, lease.fencing);
|
|
1604
2219
|
}
|
|
1605
2220
|
}
|
|
1606
2221
|
var defaultLaneRunner = async (job) => {
|
|
@@ -1611,48 +2226,125 @@ var defaultLaneRunner = async (job) => {
|
|
|
1611
2226
|
};
|
|
1612
2227
|
async function runGraphToCompletion(store, wal, graph, context, options = {}) {
|
|
1613
2228
|
const daemon = new WorkflowsDaemon(store, wal, {
|
|
1614
|
-
worker:
|
|
2229
|
+
worker: `foreground-${randomUUID2()}`,
|
|
1615
2230
|
laneRunner: options.laneRunner,
|
|
1616
2231
|
time: options.time,
|
|
1617
2232
|
env: options.env,
|
|
1618
2233
|
isolatedRunId: options.isolatedRunId
|
|
1619
2234
|
});
|
|
1620
|
-
const
|
|
1621
|
-
|
|
2235
|
+
const run2 = options.existingRunId ? await store.getRun(options.existingRunId) : await daemon.startRun(graph, context);
|
|
2236
|
+
if (!run2)
|
|
2237
|
+
throw new Error("run not found");
|
|
2238
|
+
options.onRunCreated?.(run2);
|
|
2239
|
+
daemon.isolateRun(run2.id);
|
|
1622
2240
|
const maxCycles = options.maxCycles ?? 500;
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
2241
|
+
const hosted = store instanceof PostgresWorkflowsStore ? store : undefined;
|
|
2242
|
+
if (hosted && !await hosted.acquireExecution(run2.id))
|
|
2243
|
+
return await store.getRun(run2.id);
|
|
2244
|
+
const lease = daemon.claim(run2.id);
|
|
2245
|
+
if (!lease) {
|
|
2246
|
+
await hosted?.releaseExecution();
|
|
2247
|
+
return await store.getRun(run2.id);
|
|
2248
|
+
}
|
|
2249
|
+
try {
|
|
2250
|
+
if (!await store.compareRunStatus(run2.id, ["pending"], "running", { startedAt: nowIso() }))
|
|
2251
|
+
return await store.getRun(run2.id);
|
|
2252
|
+
wal.append({ op: "run_started", runId: run2.id, at: nowIso() });
|
|
2253
|
+
for (let cycle = 0;cycle < maxCycles; cycle++) {
|
|
2254
|
+
daemon.heartbeat(run2.id, lease.fencing);
|
|
2255
|
+
const outcome = await daemon.advanceRun(run2.id);
|
|
2256
|
+
const current = await store.getRun(run2.id);
|
|
2257
|
+
if (current.status !== "running" || outcome.kind === "blocked" || outcome.kind === "noop")
|
|
2258
|
+
return current;
|
|
2259
|
+
}
|
|
2260
|
+
await store.compareRunStatus(run2.id, ["running"], "interrupted", { error: "run reached maxCycles; resume to continue" });
|
|
2261
|
+
return await store.getRun(run2.id);
|
|
2262
|
+
} finally {
|
|
2263
|
+
daemon.release(run2.id, lease.fencing);
|
|
2264
|
+
await hosted?.releaseExecution();
|
|
1631
2265
|
}
|
|
1632
|
-
return store.getRun(run.id);
|
|
1633
2266
|
}
|
|
1634
2267
|
|
|
1635
2268
|
// src/paths.ts
|
|
1636
2269
|
import { existsSync } from "fs";
|
|
1637
2270
|
import { homedir } from "os";
|
|
1638
|
-
import { join } from "path";
|
|
1639
|
-
import {
|
|
2271
|
+
import { join as join2 } from "path";
|
|
2272
|
+
import { homedir as pathsResolverHomedir } from "os";
|
|
2273
|
+
import { join as pathsResolverJoin } from "path";
|
|
2274
|
+
var PATHS_RESOLVER_KIND_ENV = {
|
|
2275
|
+
config: "HASNA_CONFIG_HOME",
|
|
2276
|
+
data: "HASNA_DATA_HOME",
|
|
2277
|
+
state: "HASNA_STATE_HOME",
|
|
2278
|
+
cache: "HASNA_CACHE_HOME"
|
|
2279
|
+
};
|
|
2280
|
+
var PATHS_RESOLVER_APP_SLUG_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
2281
|
+
function pathsResolverAssertApp(app) {
|
|
2282
|
+
if (typeof app !== "string" || app.length === 0) {
|
|
2283
|
+
throw new TypeError("paths: app must be a non-empty string");
|
|
2284
|
+
}
|
|
2285
|
+
if (!PATHS_RESOLVER_APP_SLUG_RE.test(app)) {
|
|
2286
|
+
throw new TypeError(`paths: invalid app slug "${app}" \u2014 expected lowercase kebab-case ([a-z0-9]+(-[a-z0-9]+)*)`);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
function pathsResolverAssertKind(kind) {
|
|
2290
|
+
if (!Object.keys(PATHS_RESOLVER_KIND_ENV).includes(kind)) {
|
|
2291
|
+
throw new TypeError(`paths: invalid path kind "${kind}" \u2014 expected one of ${Object.keys(PATHS_RESOLVER_KIND_ENV).join(", ")}`);
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
function pathsResolverBaseDir(kind, options) {
|
|
2295
|
+
pathsResolverAssertKind(kind);
|
|
2296
|
+
const env = options.env ?? process.env;
|
|
2297
|
+
const override = env[PATHS_RESOLVER_KIND_ENV[kind]];
|
|
2298
|
+
if (typeof override === "string" && override.length > 0)
|
|
2299
|
+
return override;
|
|
2300
|
+
const home = options.home ?? pathsResolverHomedir();
|
|
2301
|
+
const platform = options.platform ?? process.platform;
|
|
2302
|
+
if (platform === "darwin") {
|
|
2303
|
+
switch (kind) {
|
|
2304
|
+
case "config":
|
|
2305
|
+
case "data":
|
|
2306
|
+
return pathsResolverJoin(home, "Library", "Application Support", "Hasna");
|
|
2307
|
+
case "cache":
|
|
2308
|
+
return pathsResolverJoin(home, "Library", "Caches", "Hasna");
|
|
2309
|
+
case "state":
|
|
2310
|
+
return pathsResolverJoin(home, "Library", "Logs", "Hasna");
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
switch (kind) {
|
|
2314
|
+
case "config":
|
|
2315
|
+
return pathsResolverJoin(home, ".config", "hasna");
|
|
2316
|
+
case "data":
|
|
2317
|
+
return pathsResolverJoin(home, ".local", "share", "hasna");
|
|
2318
|
+
case "state":
|
|
2319
|
+
return pathsResolverJoin(home, ".local", "state", "hasna");
|
|
2320
|
+
case "cache":
|
|
2321
|
+
return pathsResolverJoin(home, ".cache", "hasna");
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
function pathsResolverResolve(kind, options) {
|
|
2325
|
+
pathsResolverAssertApp(options.app);
|
|
2326
|
+
const appSegment = options.internal === true ? pathsResolverJoin("internal", options.app) : options.app;
|
|
2327
|
+
return pathsResolverJoin(pathsResolverBaseDir(kind, options), appSegment);
|
|
2328
|
+
}
|
|
2329
|
+
function dataDir(options) {
|
|
2330
|
+
return pathsResolverResolve("data", options);
|
|
2331
|
+
}
|
|
1640
2332
|
var APP = "workflows";
|
|
1641
2333
|
var STORE_DB_FILE = "workflows.db";
|
|
1642
2334
|
function getHomeDir(env = process.env) {
|
|
1643
2335
|
return env["HOME"] || env["USERPROFILE"] || homedir();
|
|
1644
2336
|
}
|
|
1645
2337
|
function getLegacyDataDir(env = process.env) {
|
|
1646
|
-
return
|
|
2338
|
+
return join2(getHomeDir(env), ".hasna", APP);
|
|
1647
2339
|
}
|
|
1648
|
-
function getResolverDataDir(env = process.env) {
|
|
1649
|
-
return dataDir({ app: APP, home: getHomeDir(env), env });
|
|
2340
|
+
function getResolverDataDir(env = process.env, platform = process.platform) {
|
|
2341
|
+
return dataDir({ app: APP, home: getHomeDir(env), env, platform });
|
|
1650
2342
|
}
|
|
1651
2343
|
function adoptResolverDataDir(resolved, env = process.env) {
|
|
1652
2344
|
const dataOverride = env.HASNA_DATA_HOME;
|
|
1653
2345
|
if (typeof dataOverride === "string" && dataOverride.trim().length > 0)
|
|
1654
2346
|
return true;
|
|
1655
|
-
return existsSync(
|
|
2347
|
+
return existsSync(join2(resolved, STORE_DB_FILE));
|
|
1656
2348
|
}
|
|
1657
2349
|
function getEffectiveDataDir(env = process.env) {
|
|
1658
2350
|
const resolved = getResolverDataDir(env);
|
|
@@ -1662,17 +2354,42 @@ function getEffectiveDataDir(env = process.env) {
|
|
|
1662
2354
|
// src/store.ts
|
|
1663
2355
|
import { Database } from "bun:sqlite";
|
|
1664
2356
|
import { mkdirSync } from "fs";
|
|
1665
|
-
import { join as
|
|
1666
|
-
import { randomUUID } from "crypto";
|
|
2357
|
+
import { join as join3 } from "path";
|
|
2358
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
2359
|
+
var RUN_STATUSES = ["pending", "running", "completed", "failed", "cancelled", "interrupted"];
|
|
1667
2360
|
function nowIso2() {
|
|
1668
2361
|
return new Date().toISOString();
|
|
1669
2362
|
}
|
|
2363
|
+
var BUSY_TIMEOUT_MS = 3000;
|
|
2364
|
+
var BUSY_RETRIES = 2;
|
|
2365
|
+
var BUSY_RETRY_SLEEP_MS = 50;
|
|
2366
|
+
function isBusyError(err) {
|
|
2367
|
+
if (!(err instanceof Error))
|
|
2368
|
+
return false;
|
|
2369
|
+
const code = String(err.code ?? "");
|
|
2370
|
+
return code.includes("BUSY") || /\bdatabase is locked\b/i.test(err.message);
|
|
2371
|
+
}
|
|
2372
|
+
function retryBusy(fn) {
|
|
2373
|
+
let last;
|
|
2374
|
+
for (let attempt = 0;attempt <= BUSY_RETRIES; attempt++) {
|
|
2375
|
+
try {
|
|
2376
|
+
return fn();
|
|
2377
|
+
} catch (err) {
|
|
2378
|
+
if (!isBusyError(err))
|
|
2379
|
+
throw err;
|
|
2380
|
+
last = err;
|
|
2381
|
+
Bun.sleepSync(BUSY_RETRY_SLEEP_MS);
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
throw last;
|
|
2385
|
+
}
|
|
1670
2386
|
function openStore(dataDir2) {
|
|
1671
2387
|
mkdirSync(dataDir2, { recursive: true });
|
|
1672
|
-
const db = new Database(
|
|
1673
|
-
db.exec(
|
|
1674
|
-
db.exec("PRAGMA
|
|
1675
|
-
db.exec(
|
|
2388
|
+
const db = new Database(join3(dataDir2, "workflows.db"));
|
|
2389
|
+
retryBusy(() => db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`));
|
|
2390
|
+
retryBusy(() => db.exec("PRAGMA journal_mode = WAL"));
|
|
2391
|
+
retryBusy(() => db.exec("PRAGMA foreign_keys = ON"));
|
|
2392
|
+
retryBusy(() => db.exec(`
|
|
1676
2393
|
CREATE TABLE IF NOT EXISTS runs (
|
|
1677
2394
|
id TEXT PRIMARY KEY,
|
|
1678
2395
|
graph_name TEXT NOT NULL,
|
|
@@ -1709,10 +2426,12 @@ function openStore(dataDir2) {
|
|
|
1709
2426
|
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
1710
2427
|
created_at TEXT NOT NULL
|
|
1711
2428
|
);
|
|
2429
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_runs_idempotency ON runs(json_extract(context_json, '$.__wf.idempotencyKey'))
|
|
2430
|
+
WHERE json_type(context_json, '$.__wf.idempotencyKey') = 'text';
|
|
1712
2431
|
CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status);
|
|
1713
2432
|
CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(created_at);
|
|
1714
2433
|
CREATE INDEX IF NOT EXISTS idx_run_nodes_run ON run_nodes(run_id);
|
|
1715
|
-
`);
|
|
2434
|
+
`));
|
|
1716
2435
|
const rowToRun = (r) => ({
|
|
1717
2436
|
id: String(r.id),
|
|
1718
2437
|
graphName: String(r.graph_name),
|
|
@@ -1750,31 +2469,37 @@ function openStore(dataDir2) {
|
|
|
1750
2469
|
createdAt: String(r.created_at)
|
|
1751
2470
|
});
|
|
1752
2471
|
return {
|
|
2472
|
+
dataDir: dataDir2,
|
|
1753
2473
|
close() {
|
|
1754
2474
|
db.close();
|
|
1755
2475
|
},
|
|
1756
2476
|
listTables() {
|
|
1757
|
-
const rows = db.query("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
|
|
2477
|
+
const rows = retryBusy(() => db.query("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all());
|
|
1758
2478
|
return rows.map((r) => r.name);
|
|
1759
2479
|
},
|
|
1760
2480
|
createRun(input) {
|
|
1761
|
-
const id =
|
|
2481
|
+
const id = randomUUID3();
|
|
1762
2482
|
const now = nowIso2();
|
|
1763
|
-
db.query(`INSERT INTO runs (id, graph_name, graph_version, status, attempts, context_json, created_at, updated_at)
|
|
1764
|
-
|
|
2483
|
+
retryBusy(() => db.query(`INSERT INTO runs (id, graph_name, graph_version, status, attempts, context_json, created_at, updated_at)
|
|
2484
|
+
VALUES (?, ?, ?, 'pending', 0, ?, ?, ?)`).run(id, input.graphName, input.graphVersion, JSON.stringify(input.context ?? {}), now, now));
|
|
1765
2485
|
return this.getRun(id);
|
|
1766
2486
|
},
|
|
1767
2487
|
getRun(id) {
|
|
1768
|
-
const row = db.query(`SELECT * FROM runs WHERE id = ?`).get(id);
|
|
2488
|
+
const row = retryBusy(() => db.query(`SELECT * FROM runs WHERE id = ?`).get(id));
|
|
2489
|
+
return row ? rowToRun(row) : undefined;
|
|
2490
|
+
},
|
|
2491
|
+
getRunByIdempotencyKey(key) {
|
|
2492
|
+
const row = retryBusy(() => db.query(`SELECT * FROM runs WHERE json_valid(context_json) AND json_extract(context_json, '$.__wf.idempotencyKey') = ? ORDER BY created_at DESC LIMIT 1`).get(key));
|
|
1769
2493
|
return row ? rowToRun(row) : undefined;
|
|
1770
2494
|
},
|
|
1771
2495
|
listRuns(opts = {}) {
|
|
1772
2496
|
const limit = Math.min(opts.limit ?? 100, 1000);
|
|
1773
2497
|
if (opts.status) {
|
|
1774
|
-
const
|
|
2498
|
+
const status = opts.status;
|
|
2499
|
+
const rows2 = retryBusy(() => db.query(`SELECT * FROM runs WHERE status = ? ORDER BY created_at DESC, id DESC LIMIT ?`).all(status, limit));
|
|
1775
2500
|
return rows2.map((r) => rowToRun(r));
|
|
1776
2501
|
}
|
|
1777
|
-
const rows = db.query(`SELECT * FROM runs ORDER BY created_at DESC, id DESC LIMIT ?`).all(limit);
|
|
2502
|
+
const rows = retryBusy(() => db.query(`SELECT * FROM runs ORDER BY created_at DESC, id DESC LIMIT ?`).all(limit));
|
|
1778
2503
|
return rows.map((r) => rowToRun(r));
|
|
1779
2504
|
},
|
|
1780
2505
|
setRunStatus(id, status, patch = {}) {
|
|
@@ -1785,28 +2510,99 @@ function openStore(dataDir2) {
|
|
|
1785
2510
|
const startedAt = patch.startedAt !== undefined ? patch.startedAt : status === "running" && existing?.startedAt === null ? now : existing?.startedAt ?? null;
|
|
1786
2511
|
const terminal = status === "completed" || status === "failed" || status === "cancelled" || status === "interrupted";
|
|
1787
2512
|
const finishedAt = patch.finishedAt !== undefined ? patch.finishedAt : terminal && existing?.finishedAt === null ? now : existing?.finishedAt ?? null;
|
|
1788
|
-
db.query(`UPDATE runs SET status = ?, result_json = ?, error = ?, started_at = ?, finished_at = ?, updated_at = ? WHERE id = ?`).run(status, resultJson, error, startedAt, finishedAt, now, id);
|
|
2513
|
+
retryBusy(() => db.query(`UPDATE runs SET status = ?, result_json = ?, error = ?, started_at = ?, finished_at = ?, updated_at = ? WHERE id = ?`).run(status, resultJson, error, startedAt, finishedAt, now, id));
|
|
2514
|
+
},
|
|
2515
|
+
compareRunStatus(id, expected, status, patch = {}) {
|
|
2516
|
+
return retryBusy(() => db.transaction(() => {
|
|
2517
|
+
const current = this.getRun(id);
|
|
2518
|
+
if (!current || !expected.includes(current.status))
|
|
2519
|
+
return false;
|
|
2520
|
+
this.setRunStatus(id, status, patch);
|
|
2521
|
+
return true;
|
|
2522
|
+
}).immediate());
|
|
1789
2523
|
},
|
|
1790
2524
|
bumpAttempts(id) {
|
|
1791
|
-
db.query(`UPDATE runs SET attempts = attempts + 1, updated_at = ? WHERE id = ?`).run(nowIso2(), id);
|
|
2525
|
+
retryBusy(() => db.query(`UPDATE runs SET attempts = attempts + 1, updated_at = ? WHERE id = ?`).run(nowIso2(), id));
|
|
1792
2526
|
},
|
|
1793
2527
|
setRunContext(id, context) {
|
|
1794
|
-
db.query(`UPDATE runs SET context_json = ?, updated_at = ? WHERE id = ?`).run(JSON.stringify(context), nowIso2(), id);
|
|
2528
|
+
retryBusy(() => db.query(`UPDATE runs SET context_json = ?, updated_at = ? WHERE id = ?`).run(JSON.stringify(context), nowIso2(), id));
|
|
2529
|
+
},
|
|
2530
|
+
setRunningContext(id, context) {
|
|
2531
|
+
return retryBusy(() => db.transaction(() => {
|
|
2532
|
+
if (this.getRun(id)?.status !== "running")
|
|
2533
|
+
return false;
|
|
2534
|
+
this.setRunContext(id, context);
|
|
2535
|
+
return true;
|
|
2536
|
+
}).immediate());
|
|
2537
|
+
},
|
|
2538
|
+
blockRunResume(id) {
|
|
2539
|
+
retryBusy(() => db.transaction(() => {
|
|
2540
|
+
const run2 = this.getRun(id);
|
|
2541
|
+
if (!run2)
|
|
2542
|
+
return;
|
|
2543
|
+
const context = JSON.parse(run2.contextJson);
|
|
2544
|
+
context.__wf = { ...context.__wf, resumeBlocked: "execution cancellation has not been confirmed" };
|
|
2545
|
+
this.setRunContext(id, context);
|
|
2546
|
+
}).immediate());
|
|
2547
|
+
},
|
|
2548
|
+
dispatchIfRunning(id, action) {
|
|
2549
|
+
return retryBusy(() => db.transaction(() => {
|
|
2550
|
+
const run2 = this.getRun(id);
|
|
2551
|
+
if (run2?.status !== "running")
|
|
2552
|
+
return { started: false };
|
|
2553
|
+
const context = JSON.parse(run2.contextJson);
|
|
2554
|
+
context.__wf = { ...context.__wf, executionMayBeActive: true };
|
|
2555
|
+
this.setRunContext(id, context);
|
|
2556
|
+
return { started: true, result: action() };
|
|
2557
|
+
}).immediate());
|
|
2558
|
+
},
|
|
2559
|
+
finishDispatch(id, unconfirmed) {
|
|
2560
|
+
retryBusy(() => db.transaction(() => {
|
|
2561
|
+
const run2 = this.getRun(id);
|
|
2562
|
+
if (!run2)
|
|
2563
|
+
return;
|
|
2564
|
+
const context = JSON.parse(run2.contextJson);
|
|
2565
|
+
delete context.__wf?.executionMayBeActive;
|
|
2566
|
+
if (unconfirmed)
|
|
2567
|
+
context.__wf = { ...context.__wf, resumeBlocked: "execution cancellation has not been confirmed" };
|
|
2568
|
+
this.setRunContext(id, context);
|
|
2569
|
+
}).immediate());
|
|
2570
|
+
},
|
|
2571
|
+
commitStep(input) {
|
|
2572
|
+
return retryBusy(() => db.transaction(() => {
|
|
2573
|
+
if (this.getRun(input.runId)?.status !== "running")
|
|
2574
|
+
return false;
|
|
2575
|
+
const node2 = this.getRunNode(input.nodeRowId);
|
|
2576
|
+
if (!node2 || node2.runId !== input.runId)
|
|
2577
|
+
return false;
|
|
2578
|
+
this.setRunNodeStatus(input.nodeRowId, input.status, input.patch);
|
|
2579
|
+
if (input.incrementAttempts)
|
|
2580
|
+
this.bumpAttemptsNode(input.nodeRowId);
|
|
2581
|
+
if (input.context !== undefined)
|
|
2582
|
+
this.setRunContext(input.runId, input.context);
|
|
2583
|
+
if (input.memo)
|
|
2584
|
+
this.memoPut(input.memo.key, input.memo.graphName, input.memo.nodeId, input.memo.inputHash, input.memo.outputJson);
|
|
2585
|
+
return true;
|
|
2586
|
+
}).immediate());
|
|
1795
2587
|
},
|
|
1796
2588
|
createRunNode(input) {
|
|
1797
|
-
const id =
|
|
1798
|
-
db.query(`INSERT INTO run_nodes (id, run_id, node_id, status, attempts, lane)
|
|
1799
|
-
|
|
2589
|
+
const id = randomUUID3();
|
|
2590
|
+
retryBusy(() => db.query(`INSERT INTO run_nodes (id, run_id, node_id, status, attempts, lane)
|
|
2591
|
+
VALUES (?, ?, ?, 'pending', 0, ?)`).run(id, input.runId, input.nodeId, input.lane ?? null));
|
|
1800
2592
|
return this.getRunNode(id);
|
|
1801
2593
|
},
|
|
1802
2594
|
getRunNode(id) {
|
|
1803
|
-
const row = db.query(`SELECT * FROM run_nodes WHERE id = ?`).get(id);
|
|
2595
|
+
const row = retryBusy(() => db.query(`SELECT * FROM run_nodes WHERE id = ?`).get(id));
|
|
1804
2596
|
return row ? rowToNode(row) : undefined;
|
|
1805
2597
|
},
|
|
1806
2598
|
listRunNodes(runId) {
|
|
1807
|
-
const rows = db.query(`SELECT * FROM run_nodes WHERE run_id = ? ORDER BY rowid ASC`).all(runId);
|
|
2599
|
+
const rows = retryBusy(() => db.query(`SELECT * FROM run_nodes WHERE run_id = ? ORDER BY rowid ASC`).all(runId));
|
|
1808
2600
|
return rows.map((r) => rowToNode(r));
|
|
1809
2601
|
},
|
|
2602
|
+
getLatestRunNode(runId, nodeId) {
|
|
2603
|
+
const row = retryBusy(() => db.query(`SELECT * FROM run_nodes WHERE run_id = ? AND node_id = ? ORDER BY rowid DESC LIMIT 1`).get(runId, nodeId));
|
|
2604
|
+
return row ? rowToNode(row) : undefined;
|
|
2605
|
+
},
|
|
1810
2606
|
setRunNodeStatus(id, status, patch = {}) {
|
|
1811
2607
|
const now = nowIso2();
|
|
1812
2608
|
const existing = this.getRunNode(id);
|
|
@@ -1818,57 +2614,60 @@ function openStore(dataDir2) {
|
|
|
1818
2614
|
const startedAt = patch.startedAt !== undefined ? patch.startedAt : status === "running" && existing.startedAt === null ? now : existing.startedAt;
|
|
1819
2615
|
const terminal = status === "completed" || status === "failed" || status === "skipped";
|
|
1820
2616
|
const finishedAt = patch.finishedAt !== undefined ? patch.finishedAt : terminal && existing.finishedAt === null ? now : existing.finishedAt;
|
|
1821
|
-
db.query(`UPDATE run_nodes SET status = ?, exit_code = ?, output_json = ?, error = ?, started_at = ?, finished_at = ? WHERE id = ?`).run(status, exitCode, outputJson, error, startedAt, finishedAt, id);
|
|
2617
|
+
retryBusy(() => db.query(`UPDATE run_nodes SET status = ?, exit_code = ?, output_json = ?, error = ?, started_at = ?, finished_at = ? WHERE id = ?`).run(status, exitCode, outputJson, error, startedAt, finishedAt, id));
|
|
1822
2618
|
},
|
|
1823
2619
|
bumpAttemptsNode(id) {
|
|
1824
|
-
db.query(`UPDATE run_nodes SET attempts = attempts + 1 WHERE id = ?`).run(id);
|
|
2620
|
+
retryBusy(() => db.query(`UPDATE run_nodes SET attempts = attempts + 1 WHERE id = ?`).run(id));
|
|
1825
2621
|
},
|
|
1826
2622
|
memoPut(key, graphName, nodeId, inputHash2, outputJson) {
|
|
1827
|
-
db.query(`INSERT INTO memos (key, graph_name, node_id, input_hash, output_json, hit_count, created_at)
|
|
1828
|
-
|
|
1829
|
-
|
|
2623
|
+
retryBusy(() => db.query(`INSERT INTO memos (key, graph_name, node_id, input_hash, output_json, hit_count, created_at)
|
|
2624
|
+
VALUES (?, ?, ?, ?, ?, 0, ?)
|
|
2625
|
+
ON CONFLICT(key) DO UPDATE SET output_json = excluded.output_json`).run(key, graphName, nodeId, inputHash2, outputJson, nowIso2()));
|
|
1830
2626
|
},
|
|
1831
2627
|
memoGet(key) {
|
|
1832
|
-
const row = db.query(`SELECT * FROM memos WHERE key = ?`).get(key);
|
|
2628
|
+
const row = retryBusy(() => db.query(`SELECT * FROM memos WHERE key = ?`).get(key));
|
|
1833
2629
|
return row ? rowToMemo(row) : undefined;
|
|
1834
2630
|
},
|
|
1835
2631
|
memoHit(key) {
|
|
1836
|
-
db.query(`UPDATE memos SET hit_count = hit_count + 1 WHERE key = ?`).run(key);
|
|
2632
|
+
retryBusy(() => db.query(`UPDATE memos SET hit_count = hit_count + 1 WHERE key = ?`).run(key));
|
|
1837
2633
|
},
|
|
1838
2634
|
memoList() {
|
|
1839
|
-
const rows = db.query(`SELECT * FROM memos ORDER BY created_at DESC LIMIT 1000`).all();
|
|
2635
|
+
const rows = retryBusy(() => db.query(`SELECT * FROM memos ORDER BY created_at DESC LIMIT 1000`).all());
|
|
1840
2636
|
return rows.map((r) => rowToMemo(r));
|
|
1841
2637
|
},
|
|
1842
2638
|
memoClear() {
|
|
1843
|
-
db.exec(`DELETE FROM memos`);
|
|
2639
|
+
retryBusy(() => db.exec(`DELETE FROM memos`));
|
|
1844
2640
|
}
|
|
1845
2641
|
};
|
|
1846
2642
|
}
|
|
1847
2643
|
|
|
1848
2644
|
// src/wal.ts
|
|
1849
|
-
import {
|
|
1850
|
-
import {
|
|
1851
|
-
import {
|
|
2645
|
+
import { Database as Database2 } from "bun:sqlite";
|
|
2646
|
+
import { createHash as createHash2 } from "crypto";
|
|
2647
|
+
import { appendFileSync, copyFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, renameSync, truncateSync, writeFileSync, constants } from "fs";
|
|
2648
|
+
import { basename, dirname, join as join4 } from "path";
|
|
1852
2649
|
var WAL_FILE_NAME = "session.wal";
|
|
1853
2650
|
var SESSIONS_DIR_NAME = "sessions";
|
|
1854
2651
|
var MAX_WAL_BYTES = 256 * 1024 * 1024;
|
|
1855
2652
|
function checksumOf(op) {
|
|
1856
2653
|
if (op === undefined || op === null)
|
|
1857
2654
|
return "";
|
|
1858
|
-
return
|
|
2655
|
+
return createHash2("sha256").update(JSON.stringify(op)).digest("hex");
|
|
1859
2656
|
}
|
|
1860
2657
|
|
|
1861
2658
|
class SessionWAL {
|
|
1862
2659
|
filePath;
|
|
2660
|
+
journalLocked = false;
|
|
1863
2661
|
constructor(filePath) {
|
|
1864
2662
|
this.filePath = filePath;
|
|
1865
2663
|
}
|
|
1866
2664
|
static open(dataDir2) {
|
|
1867
|
-
mkdirSync2(
|
|
1868
|
-
const canonical =
|
|
1869
|
-
const legacy =
|
|
2665
|
+
mkdirSync2(join4(dataDir2, SESSIONS_DIR_NAME), { recursive: true, mode: 448 });
|
|
2666
|
+
const canonical = join4(dataDir2, SESSIONS_DIR_NAME, WAL_FILE_NAME);
|
|
2667
|
+
const legacy = join4(dataDir2, WAL_FILE_NAME);
|
|
1870
2668
|
if (existsSync2(legacy) && !existsSync2(canonical)) {
|
|
1871
2669
|
try {
|
|
2670
|
+
preserveOriginal(legacy);
|
|
1872
2671
|
renameSync(legacy, canonical);
|
|
1873
2672
|
} catch {
|
|
1874
2673
|
return new SessionWAL(legacy);
|
|
@@ -1880,25 +2679,59 @@ class SessionWAL {
|
|
|
1880
2679
|
return new SessionWAL(filePath);
|
|
1881
2680
|
}
|
|
1882
2681
|
append(op) {
|
|
2682
|
+
return this.withJournalLock(() => this.appendLocked(op));
|
|
2683
|
+
}
|
|
2684
|
+
withJournalLock(operation) {
|
|
2685
|
+
if (this.journalLocked)
|
|
2686
|
+
return operation();
|
|
2687
|
+
const parent = dirname(this.filePath);
|
|
2688
|
+
const dataDir2 = basename(parent) === SESSIONS_DIR_NAME ? dirname(parent) : parent;
|
|
2689
|
+
mkdirSync2(dataDir2, { recursive: true, mode: 448 });
|
|
2690
|
+
const mutex = new Database2(join4(dataDir2, "workflows.db"));
|
|
2691
|
+
try {
|
|
2692
|
+
mutex.exec("PRAGMA busy_timeout = 5000");
|
|
2693
|
+
mutex.exec("BEGIN IMMEDIATE");
|
|
2694
|
+
this.journalLocked = true;
|
|
2695
|
+
try {
|
|
2696
|
+
const result = operation();
|
|
2697
|
+
mutex.exec("COMMIT");
|
|
2698
|
+
return result;
|
|
2699
|
+
} catch (error) {
|
|
2700
|
+
mutex.exec("ROLLBACK");
|
|
2701
|
+
throw error;
|
|
2702
|
+
} finally {
|
|
2703
|
+
this.journalLocked = false;
|
|
2704
|
+
}
|
|
2705
|
+
} finally {
|
|
2706
|
+
mutex.close();
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
exclusive(operation) {
|
|
2710
|
+
return this.withJournalLock(operation);
|
|
2711
|
+
}
|
|
2712
|
+
appendLocked(op) {
|
|
1883
2713
|
const at = op.at ?? new Date().toISOString();
|
|
1884
2714
|
const full = { ...op, at };
|
|
1885
2715
|
const nextSeq = this.nextSeq();
|
|
1886
2716
|
const entry = { seq: nextSeq, at, checksum: checksumOf(full), op: full };
|
|
1887
2717
|
appendFileSync(this.filePath, JSON.stringify(entry) + `
|
|
1888
|
-
`, "utf8");
|
|
2718
|
+
`, { encoding: "utf8", mode: 384 });
|
|
1889
2719
|
return entry;
|
|
1890
2720
|
}
|
|
1891
2721
|
nextSeq() {
|
|
1892
2722
|
const replay = this.replay();
|
|
1893
2723
|
return replay.entries.length + 1;
|
|
1894
2724
|
}
|
|
1895
|
-
replay() {
|
|
2725
|
+
replay(options = {}) {
|
|
2726
|
+
return this.withJournalLock(() => this.replayLocked(options));
|
|
2727
|
+
}
|
|
2728
|
+
replayLocked(options) {
|
|
1896
2729
|
const entries = [];
|
|
1897
2730
|
let torn = false;
|
|
1898
2731
|
let repaired = false;
|
|
1899
2732
|
let content;
|
|
1900
2733
|
try {
|
|
1901
|
-
content =
|
|
2734
|
+
content = readFileSync2(this.filePath, "utf8");
|
|
1902
2735
|
} catch {
|
|
1903
2736
|
return { entries, torn, repaired, liveClaims: () => new Map };
|
|
1904
2737
|
}
|
|
@@ -1936,6 +2769,9 @@ class SessionWAL {
|
|
|
1936
2769
|
if (badOffset !== null) {
|
|
1937
2770
|
torn = true;
|
|
1938
2771
|
try {
|
|
2772
|
+
if (options.repair === false)
|
|
2773
|
+
return { entries, torn, repaired, liveClaims: (nowMs) => reconstructLiveClaims(entries, nowMs) };
|
|
2774
|
+
preserveOriginal(this.filePath);
|
|
1939
2775
|
truncateSync(this.filePath, badOffset);
|
|
1940
2776
|
repaired = true;
|
|
1941
2777
|
} catch {}
|
|
@@ -1975,13 +2811,22 @@ function reconstructLiveClaims(entries, nowMs = Date.now()) {
|
|
|
1975
2811
|
}
|
|
1976
2812
|
return claims;
|
|
1977
2813
|
}
|
|
2814
|
+
function preserveOriginal(filePath) {
|
|
2815
|
+
const original = readFileSync2(filePath);
|
|
2816
|
+
const digest = createHash2("sha256").update(original).digest("hex");
|
|
2817
|
+
const backup = `${filePath}.preserved-${digest}`;
|
|
2818
|
+
if (!existsSync2(backup))
|
|
2819
|
+
copyFileSync(filePath, backup, constants.COPYFILE_EXCL);
|
|
2820
|
+
if (!readFileSync2(backup).equals(original))
|
|
2821
|
+
throw new Error("session WAL preservation readback failed; source retained");
|
|
2822
|
+
return backup;
|
|
2823
|
+
}
|
|
1978
2824
|
|
|
1979
2825
|
// src/service.ts
|
|
2826
|
+
import { hostname } from "os";
|
|
2827
|
+
import { resolveDatabaseUrl } from "@hasna/contracts";
|
|
1980
2828
|
var DEFAULT_PORT = 8790;
|
|
1981
2829
|
var DEFAULT_HOST = "127.0.0.1";
|
|
1982
|
-
function packageVersion() {
|
|
1983
|
-
return package_default.version;
|
|
1984
|
-
}
|
|
1985
2830
|
function envString(names) {
|
|
1986
2831
|
for (const name of names) {
|
|
1987
2832
|
const value = process.env[name];
|
|
@@ -1998,69 +2843,218 @@ function envInt(names, fallback) {
|
|
|
1998
2843
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
1999
2844
|
}
|
|
2000
2845
|
function resolveWorkflowsConfig(overrides = {}) {
|
|
2846
|
+
const originAuth = overrides.originAuth ?? process.env.HASNA_WORKFLOWS_ORIGIN_AUTH;
|
|
2847
|
+
if (originAuth !== undefined && !/^[A-Za-z0-9_-]{32,256}$/.test(originAuth))
|
|
2848
|
+
throw new Error("HASNA_WORKFLOWS_ORIGIN_AUTH must be a valid gateway origin identity");
|
|
2001
2849
|
return {
|
|
2002
2850
|
port: overrides.port ?? envInt(["HASNA_WORKFLOWS_PORT", "WORKFLOWS_PORT"], DEFAULT_PORT),
|
|
2003
2851
|
host: overrides.host ?? envString(["HASNA_WORKFLOWS_HOST", "WORKFLOWS_HOST"]) ?? DEFAULT_HOST,
|
|
2004
2852
|
dataDir: overrides.dataDir ?? envString(["HASNA_WORKFLOWS_DATA_DIR", "WORKFLOWS_DATA_DIR"]) ?? getEffectiveDataDir(),
|
|
2005
|
-
apiUrl: overrides.apiUrl ?? envString(["WORKFLOWS_API_URL"]),
|
|
2006
|
-
apiKey: overrides.apiKey ?? envString(["WORKFLOWS_API_KEY"])
|
|
2853
|
+
apiUrl: overrides.apiUrl ?? envString(["HASNA_WORKFLOWS_API_URL", "WORKFLOWS_API_URL"]),
|
|
2854
|
+
apiKey: overrides.apiKey ?? envString(["HASNA_WORKFLOWS_API_KEY", "WORKFLOWS_API_KEY"]),
|
|
2855
|
+
originAuth,
|
|
2856
|
+
storageBackend: overrides.storageBackend ?? "postgresql",
|
|
2857
|
+
databaseUrl: overrides.databaseUrl ?? envString(["HASNA_WORKFLOWS_DATABASE_URL", "WORKFLOWS_DATABASE_URL"]),
|
|
2858
|
+
databaseSchema: overrides.databaseSchema ?? envString(["HASNA_WORKFLOWS_DATABASE_SCHEMA", "WORKFLOWS_DATABASE_SCHEMA"])
|
|
2007
2859
|
};
|
|
2008
2860
|
}
|
|
2009
2861
|
|
|
2010
2862
|
class WorkflowsService {
|
|
2863
|
+
execution;
|
|
2011
2864
|
name = "workflows";
|
|
2012
2865
|
version;
|
|
2013
2866
|
config;
|
|
2014
2867
|
startedAt = Date.now();
|
|
2015
|
-
|
|
2868
|
+
databaseEnv;
|
|
2869
|
+
constructor(config = {}, execution = {}) {
|
|
2870
|
+
this.execution = execution;
|
|
2016
2871
|
this.config = resolveWorkflowsConfig(config);
|
|
2872
|
+
this.databaseEnv = config.databaseUrl !== undefined ? { HASNA_WORKFLOWS_DATABASE_URL: config.databaseUrl } : {
|
|
2873
|
+
HASNA_WORKFLOWS_DATABASE_URL: process.env.HASNA_WORKFLOWS_DATABASE_URL,
|
|
2874
|
+
WORKFLOWS_DATABASE_URL: process.env.WORKFLOWS_DATABASE_URL
|
|
2875
|
+
};
|
|
2017
2876
|
this.version = packageVersion();
|
|
2018
2877
|
}
|
|
2019
2878
|
health() {
|
|
2020
2879
|
return {
|
|
2021
2880
|
ok: true,
|
|
2881
|
+
status: "ok",
|
|
2882
|
+
backend: this.config.storageBackend,
|
|
2022
2883
|
service: this.name,
|
|
2023
2884
|
version: this.version,
|
|
2024
2885
|
pid: process.pid,
|
|
2025
2886
|
uptimeMs: Date.now() - this.startedAt
|
|
2026
2887
|
};
|
|
2027
2888
|
}
|
|
2028
|
-
ready() {
|
|
2889
|
+
async ready() {
|
|
2890
|
+
const checks = { version: "ok", auth: this.config.apiKey ? "ok" : "missing", storage: "not-checked" };
|
|
2891
|
+
if (this.config.apiKey) {
|
|
2892
|
+
try {
|
|
2893
|
+
checks.storage = await this.withStore(async (store) => (store instanceof PostgresWorkflowsStore ? await store.checkSchema() : (await store.listTables()).sort().join(",") === "memos,run_nodes,runs") ? "ok" : "invalid-schema");
|
|
2894
|
+
} catch (error) {
|
|
2895
|
+
checks.storage = error instanceof WorkflowRequestError && error.code === "invalid_schema" ? "invalid-schema" : "unavailable";
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
return { ok: checks.auth === "ok" && checks.storage === "ok", service: this.name, version: this.version, checks };
|
|
2899
|
+
}
|
|
2900
|
+
async openEngineStore(verifySchema = true) {
|
|
2901
|
+
if (this.config.storageBackend === "sqlite")
|
|
2902
|
+
return openStore(this.config.dataDir);
|
|
2903
|
+
if (this.config.storageBackend !== "postgresql")
|
|
2904
|
+
throw new WorkflowRequestError("invalid_storage_backend", "storage backend must be postgresql or explicit embedded sqlite", 503);
|
|
2905
|
+
let databaseUrl;
|
|
2906
|
+
try {
|
|
2907
|
+
databaseUrl = resolveDatabaseUrl("workflows", this.databaseEnv);
|
|
2908
|
+
} catch (error) {
|
|
2909
|
+
throw new WorkflowRequestError("invalid_database_configuration", error instanceof Error ? error.message : "HASNA_WORKFLOWS_DATABASE_URL must configure PostgreSQL", 503);
|
|
2910
|
+
}
|
|
2911
|
+
let store;
|
|
2912
|
+
try {
|
|
2913
|
+
store = new PostgresWorkflowsStore({ databaseUrl, dataDir: this.config.dataDir, schema: this.config.databaseSchema });
|
|
2914
|
+
} catch {
|
|
2915
|
+
throw new WorkflowRequestError("invalid_database_schema", "HASNA_WORKFLOWS_DATABASE_SCHEMA must be a lowercase SQL identifier", 503);
|
|
2916
|
+
}
|
|
2917
|
+
try {
|
|
2918
|
+
if (verifySchema && !await store.checkSchema())
|
|
2919
|
+
throw new WorkflowRequestError("invalid_schema", "PostgreSQL workflow schema requires explicit initialization or a reviewed migration", 503);
|
|
2920
|
+
return store;
|
|
2921
|
+
} catch (error) {
|
|
2922
|
+
await store.close();
|
|
2923
|
+
throw error;
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
async withStore(operation) {
|
|
2927
|
+
const store = await this.openEngineStore();
|
|
2928
|
+
try {
|
|
2929
|
+
return await operation(store);
|
|
2930
|
+
} finally {
|
|
2931
|
+
await store.close();
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
async initializeStorage() {
|
|
2935
|
+
const store = await this.openEngineStore(false);
|
|
2936
|
+
try {
|
|
2937
|
+
if (!(store instanceof PostgresWorkflowsStore))
|
|
2938
|
+
throw new WorkflowRequestError("invalid_storage_backend", "init-db requires PostgreSQL", 503);
|
|
2939
|
+
await store.initialize();
|
|
2940
|
+
} finally {
|
|
2941
|
+
await store.close();
|
|
2942
|
+
}
|
|
2943
|
+
}
|
|
2944
|
+
listRuns(options = {}) {
|
|
2945
|
+
return this.withStore((store) => store.listRuns(options));
|
|
2946
|
+
}
|
|
2947
|
+
getRun(id) {
|
|
2948
|
+
return this.withStore((store) => store.getRun(id));
|
|
2949
|
+
}
|
|
2950
|
+
listNodes(id) {
|
|
2951
|
+
return this.withStore((store) => store.listRunNodes(id));
|
|
2952
|
+
}
|
|
2953
|
+
events(id) {
|
|
2954
|
+
return SessionWAL.open(this.config.dataDir).replay({ repair: false }).entries.filter((entry) => ("runId" in entry.op) && entry.op.runId === id);
|
|
2955
|
+
}
|
|
2956
|
+
cancelRun(id) {
|
|
2957
|
+
return this.withStore(async (store) => {
|
|
2958
|
+
const run2 = await store.getRun(id);
|
|
2959
|
+
if (!run2)
|
|
2960
|
+
return;
|
|
2961
|
+
if (["completed", "failed", "cancelled"].includes(run2.status))
|
|
2962
|
+
throw new WorkflowRequestError("invalid_run_status", "cancellation requires a pending, running, or interrupted run", 409);
|
|
2963
|
+
if (!await store.compareRunStatus(id, ["pending", "running", "interrupted"], "cancelled", { finishedAt: new Date().toISOString() }))
|
|
2964
|
+
throw new WorkflowRequestError("run_status_conflict", "run changed before cancellation", 409);
|
|
2965
|
+
return store.getRun(id);
|
|
2966
|
+
});
|
|
2967
|
+
}
|
|
2968
|
+
async resumeRun(id) {
|
|
2969
|
+
const store = await this.openEngineStore();
|
|
2970
|
+
const wal = SessionWAL.open(this.config.dataDir);
|
|
2971
|
+
try {
|
|
2972
|
+
const run2 = await store.getRun(id);
|
|
2973
|
+
if (!run2)
|
|
2974
|
+
throw new WorkflowRequestError("run_not_found", "run id does not exist", 404);
|
|
2975
|
+
if (!["failed", "cancelled", "interrupted"].includes(run2.status))
|
|
2976
|
+
throw new WorkflowRequestError("invalid_run_status", "resume requires a failed, cancelled, or interrupted run", 409);
|
|
2977
|
+
const context = JSON.parse(run2.contextJson);
|
|
2978
|
+
if (context.__wf?.resumeBlocked || context.__wf?.executionMayBeActive || wal.replay({ repair: false }).liveClaims().has(id))
|
|
2979
|
+
throw new WorkflowRequestError("execution_active", "previous execution must finish before resume", 409);
|
|
2980
|
+
if (store instanceof PostgresWorkflowsStore && !await store.acquireExecution(id))
|
|
2981
|
+
throw new WorkflowRequestError("execution_active", "another executor still owns this run", 409);
|
|
2982
|
+
const graph = context.__wf?.graph;
|
|
2983
|
+
if (!graph)
|
|
2984
|
+
throw new WorkflowRequestError("graph_unavailable", "legacy run needs its original graph registered by the server operator", 409);
|
|
2985
|
+
if (!await store.compareRunStatus(id, ["failed", "cancelled", "interrupted"], "pending", { error: null }))
|
|
2986
|
+
throw new WorkflowRequestError("run_status_conflict", "run changed before resume", 409);
|
|
2987
|
+
for (const node2 of await store.listRunNodes(id)) {
|
|
2988
|
+
if (node2.status !== "completed")
|
|
2989
|
+
await store.setRunNodeStatus(node2.id, "pending", { error: null });
|
|
2990
|
+
}
|
|
2991
|
+
const final = await runGraphToCompletion(store, wal, graph, context, { ...this.execution, existingRunId: id });
|
|
2992
|
+
return runSummary(final, false);
|
|
2993
|
+
} finally {
|
|
2994
|
+
await store.close();
|
|
2995
|
+
}
|
|
2996
|
+
}
|
|
2997
|
+
sessions(action) {
|
|
2998
|
+
const replay = SessionWAL.open(this.config.dataDir).replay({ repair: false });
|
|
2999
|
+
if (action === "list")
|
|
3000
|
+
return replay.entries.slice(-100).map(({ seq, at, op }) => ({ seq, at, operation: op.op }));
|
|
2029
3001
|
return {
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
3002
|
+
entries: replay.entries.length,
|
|
3003
|
+
torn: replay.torn,
|
|
3004
|
+
repaired: false,
|
|
3005
|
+
liveClaims: [...replay.liveClaims().entries()].map(([runId, claim]) => ({ runId, worker: claim.worker, fencing: claim.fencing, expiresAt: new Date(claim.expiresAtMs).toISOString() }))
|
|
2034
3006
|
};
|
|
2035
3007
|
}
|
|
3008
|
+
machines(action) {
|
|
3009
|
+
const local = { name: hostname(), role: "server", version: this.version };
|
|
3010
|
+
if (action === "list")
|
|
3011
|
+
return [local];
|
|
3012
|
+
return { ...local, health: this.health(), sessions: this.sessions("pull") };
|
|
3013
|
+
}
|
|
3014
|
+
daemonStatus() {
|
|
3015
|
+
const replay = SessionWAL.open(this.config.dataDir).replay({ repair: false });
|
|
3016
|
+
return { activeClaims: replay.liveClaims().size, mode: "bounded-trigger", continuousDaemon: false };
|
|
3017
|
+
}
|
|
3018
|
+
listMemos() {
|
|
3019
|
+
return this.withStore(async (store) => (await store.memoList()).slice(0, 100));
|
|
3020
|
+
}
|
|
2036
3021
|
async triggerRun(graph, context = {}, opts = {}) {
|
|
2037
|
-
const store =
|
|
3022
|
+
const store = await this.openEngineStore();
|
|
2038
3023
|
const wal = SessionWAL.open(this.config.dataDir);
|
|
2039
3024
|
try {
|
|
2040
|
-
if (
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
3025
|
+
if (context === null || typeof context !== "object" || Array.isArray(context))
|
|
3026
|
+
throw new WorkflowRequestError("invalid_context", "context must be an object");
|
|
3027
|
+
if (Object.hasOwn(context, "__wf") || Object.hasOwn(context, "steps"))
|
|
3028
|
+
throw new WorkflowRequestError("reserved_context", "context.__wf and context.steps are reserved for the workflow engine");
|
|
3029
|
+
const maxCycles = opts.maxCycles ?? 500;
|
|
3030
|
+
if (!Number.isSafeInteger(maxCycles) || maxCycles < 1 || maxCycles > 2000)
|
|
3031
|
+
throw new WorkflowRequestError("invalid_max_cycles", "maxCycles must be an integer from 1 to 2000");
|
|
3032
|
+
if (opts.idempotencyKey !== undefined && !/^[A-Za-z0-9_.:-]{1,128}$/.test(opts.idempotencyKey))
|
|
3033
|
+
throw new WorkflowRequestError("invalid_idempotency_key", "idempotencyKey must contain 1 to 128 ASCII letters, digits, or _.:-");
|
|
3034
|
+
const requestHash = createHash3("sha256").update(canonicalJson({ graph, context, maxCycles })).digest("hex");
|
|
3035
|
+
const reuse = async () => {
|
|
3036
|
+
const existing2 = opts.idempotencyKey ? await store.getRunByIdempotencyKey(opts.idempotencyKey) : undefined;
|
|
3037
|
+
if (!existing2)
|
|
3038
|
+
return;
|
|
3039
|
+
if (JSON.parse(existing2.contextJson).__wf?.requestHash !== requestHash)
|
|
3040
|
+
throw new WorkflowRequestError("idempotency_conflict", "idempotencyKey is already bound to a different request", 409);
|
|
3041
|
+
return runSummary(existing2, true);
|
|
3042
|
+
};
|
|
3043
|
+
const existing = await reuse();
|
|
3044
|
+
if (existing)
|
|
3045
|
+
return existing;
|
|
3046
|
+
const contextWithKey = { ...context, __wf: { requestHash, ...opts.idempotencyKey ? { idempotencyKey: opts.idempotencyKey } : {} } };
|
|
3047
|
+
let final;
|
|
3048
|
+
try {
|
|
3049
|
+
final = await runGraphToCompletion(store, wal, graph, contextWithKey, { ...this.execution, maxCycles });
|
|
3050
|
+
} catch (err) {
|
|
3051
|
+
if (err instanceof Error && (/UNIQUE constraint failed/.test(err.message) || err.code === "23505")) {
|
|
3052
|
+
const winner = await reuse();
|
|
3053
|
+
if (winner)
|
|
3054
|
+
return winner;
|
|
2050
3055
|
}
|
|
3056
|
+
throw err;
|
|
2051
3057
|
}
|
|
2052
|
-
const contextWithKey = opts.idempotencyKey ? {
|
|
2053
|
-
...context,
|
|
2054
|
-
__wf: {
|
|
2055
|
-
cursor: undefined,
|
|
2056
|
-
loops: {},
|
|
2057
|
-
completedLoops: {},
|
|
2058
|
-
...context?.__wf ?? {},
|
|
2059
|
-
idempotencyKey: opts.idempotencyKey
|
|
2060
|
-
}
|
|
2061
|
-
} : context;
|
|
2062
|
-
const maxCycles = Math.min(Math.max(opts.maxCycles ?? 500, 1), 2000);
|
|
2063
|
-
const final = await runGraphToCompletion(store, wal, graph, contextWithKey, { maxCycles });
|
|
2064
3058
|
return {
|
|
2065
3059
|
runId: final.id,
|
|
2066
3060
|
status: final.status,
|
|
@@ -2069,16 +3063,38 @@ class WorkflowsService {
|
|
|
2069
3063
|
reused: false
|
|
2070
3064
|
};
|
|
2071
3065
|
} finally {
|
|
2072
|
-
store.close();
|
|
3066
|
+
await store.close();
|
|
2073
3067
|
}
|
|
2074
3068
|
}
|
|
2075
3069
|
}
|
|
2076
|
-
function createWorkflowsService(config = {}) {
|
|
2077
|
-
return new WorkflowsService(config);
|
|
3070
|
+
function createWorkflowsService(config = {}, execution = {}) {
|
|
3071
|
+
return new WorkflowsService(config, execution);
|
|
3072
|
+
}
|
|
3073
|
+
|
|
3074
|
+
class WorkflowRequestError extends Error {
|
|
3075
|
+
code;
|
|
3076
|
+
status;
|
|
3077
|
+
constructor(code, message, status = 400) {
|
|
3078
|
+
super(message);
|
|
3079
|
+
this.code = code;
|
|
3080
|
+
this.status = status;
|
|
3081
|
+
this.name = "WorkflowRequestError";
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
function canonicalJson(value) {
|
|
3085
|
+
if (Array.isArray(value))
|
|
3086
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
3087
|
+
if (value && typeof value === "object")
|
|
3088
|
+
return `{${Object.keys(value).sort().filter((k) => value[k] !== undefined).map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(",")}}`;
|
|
3089
|
+
return JSON.stringify(value) ?? "null";
|
|
3090
|
+
}
|
|
3091
|
+
function runSummary(run2, reused) {
|
|
3092
|
+
return { runId: run2.id, status: run2.status, error: run2.error ?? null, result: run2.resultJson ? JSON.parse(run2.resultJson) : null, reused };
|
|
2078
3093
|
}
|
|
2079
3094
|
export {
|
|
2080
3095
|
resolveWorkflowsConfig,
|
|
2081
3096
|
packageVersion,
|
|
2082
3097
|
createWorkflowsService,
|
|
2083
|
-
WorkflowsService
|
|
3098
|
+
WorkflowsService,
|
|
3099
|
+
WorkflowRequestError
|
|
2084
3100
|
};
|