@hasna/workflows 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -98
- package/dist/agent-tools.js +130 -1035
- package/dist/cli/index.js +4108 -2473
- package/dist/handlers.js +3287 -55
- package/dist/index.js +1403 -337
- package/dist/mcp/index.js +1725 -1742
- package/dist/sdk.js +1620 -0
- package/dist/serve/admin.js +4248 -0
- package/dist/serve/index.js +3349 -1381
- package/dist/service.js +1168 -250
- package/hasna.contract.json +17 -17
- package/package.json +15 -12
- package/types/agent-tools.d.ts +3 -17
- package/types/client-error.d.ts +4 -0
- package/types/client-tools.d.ts +4 -0
- package/types/client.d.ts +60 -0
- package/types/daemon.d.ts +15 -9
- package/types/db/postgres-store.d.ts +54 -0
- package/types/handlers.d.ts +2 -1
- package/types/identity.d.ts +7 -0
- package/types/lanes/types.d.ts +12 -3
- package/types/openapi.d.ts +7 -0
- package/types/paths.d.ts +10 -1
- package/types/sdk/generated-api.d.ts +159 -0
- package/types/sdk.d.ts +6 -0
- package/types/serve/admin.d.ts +2 -0
- package/types/service.d.ts +75 -4
- package/types/session.d.ts +7 -8
- package/types/setup.d.ts +71 -0
- package/types/store.d.ts +30 -0
- package/types/tool-definitions.d.ts +18 -0
- package/types/types.d.ts +8 -0
- package/types/version.d.ts +1 -0
- package/types/wal.d.ts +9 -1
package/dist/service.js
CHANGED
|
@@ -29,10 +29,30 @@ function binaryOnPath(binary) {
|
|
|
29
29
|
}
|
|
30
30
|
return false;
|
|
31
31
|
}
|
|
32
|
-
function
|
|
32
|
+
function subscriptionEnvironment(overrides = {}) {
|
|
33
|
+
const paid = /^(OPENAI_API_KEY|ANTHROPIC_API_KEY|ANTHROPIC_AUTH_TOKEN|CODEX_API_KEY|CURSOR_API_KEY)$/;
|
|
34
|
+
const sensitive = /(?:API[_-]?KEY|SECRET|PASSWORD|TOKEN|DATABASE_URL|CREDENTIAL)/i;
|
|
35
|
+
const env = {};
|
|
36
|
+
for (const [name, value] of Object.entries(process.env)) {
|
|
37
|
+
if (value !== undefined && !paid.test(name) && (!sensitive.test(name) || name === "CLAUDE_CODE_OAUTH_TOKEN"))
|
|
38
|
+
env[name] = value;
|
|
39
|
+
}
|
|
40
|
+
for (const [name, value] of Object.entries(overrides)) {
|
|
41
|
+
if (paid.test(name) || sensitive.test(name))
|
|
42
|
+
throw new Error(`lane environment field ${name} requires a supported scoped credential consumer`);
|
|
43
|
+
env[name] = value;
|
|
44
|
+
}
|
|
45
|
+
return env;
|
|
46
|
+
}
|
|
47
|
+
function subscriptionSdkEnvironment(overrides = {}) {
|
|
48
|
+
const safe = subscriptionEnvironment(overrides);
|
|
49
|
+
return { ...Object.fromEntries(Object.keys(process.env).map((name) => [name, undefined])), ...safe };
|
|
50
|
+
}
|
|
51
|
+
function withTimeout(promise, timeoutMs, what, cancel) {
|
|
33
52
|
return new Promise((resolve, reject) => {
|
|
34
53
|
const timer = setTimeout(() => {
|
|
35
|
-
|
|
54
|
+
cancel?.();
|
|
55
|
+
reject(new LaneTimeoutError(what, timeoutMs));
|
|
36
56
|
}, timeoutMs);
|
|
37
57
|
promise.then((value) => {
|
|
38
58
|
clearTimeout(timer);
|
|
@@ -43,15 +63,16 @@ function withTimeout(promise, timeoutMs, what) {
|
|
|
43
63
|
});
|
|
44
64
|
});
|
|
45
65
|
}
|
|
46
|
-
function spawnCli(cmd, opts) {
|
|
66
|
+
async function spawnCli(cmd, opts) {
|
|
47
67
|
const started = Date.now();
|
|
48
68
|
const timeoutMs = opts.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS;
|
|
49
69
|
let proc;
|
|
50
70
|
try {
|
|
51
|
-
proc = Bun.
|
|
71
|
+
proc = Bun.spawn({
|
|
52
72
|
cmd,
|
|
53
73
|
cwd: opts.cwd,
|
|
54
|
-
env:
|
|
74
|
+
env: subscriptionEnvironment(opts.env),
|
|
75
|
+
stdin: "ignore",
|
|
55
76
|
stdout: "pipe",
|
|
56
77
|
stderr: "pipe",
|
|
57
78
|
timeout: timeoutMs
|
|
@@ -63,15 +84,16 @@ function spawnCli(cmd, opts) {
|
|
|
63
84
|
}
|
|
64
85
|
return { ok: false, exitCode: 1, output: "", durationMs: Date.now() - started, error: message };
|
|
65
86
|
}
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
87
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
88
|
+
new Response(proc.stdout).text(),
|
|
89
|
+
new Response(proc.stderr).text(),
|
|
90
|
+
proc.exited
|
|
91
|
+
]);
|
|
69
92
|
const output = stderr ? `${stdout}
|
|
70
93
|
${stderr}`.trim() : stdout.trim();
|
|
71
|
-
|
|
72
|
-
return { ok: exitCode === 0, exitCode, output, durationMs };
|
|
94
|
+
return { ok: exitCode === 0, exitCode, output, durationMs: Date.now() - started };
|
|
73
95
|
}
|
|
74
|
-
var LANE_KINDS2, LaneDependencyMissingError, LaneAdapterShapeError, DEFAULT_LANE_TIMEOUT_MS = 120000;
|
|
96
|
+
var LANE_KINDS2, LaneDependencyMissingError, LaneAdapterShapeError, DEFAULT_LANE_TIMEOUT_MS = 120000, LaneTimeoutError;
|
|
75
97
|
var init_types = __esm(() => {
|
|
76
98
|
LANE_KINDS2 = ["claude", "codex", "cursor", "grok"];
|
|
77
99
|
LaneDependencyMissingError = class LaneDependencyMissingError extends Error {
|
|
@@ -86,6 +108,12 @@ var init_types = __esm(() => {
|
|
|
86
108
|
this.name = "LaneAdapterShapeError";
|
|
87
109
|
}
|
|
88
110
|
};
|
|
111
|
+
LaneTimeoutError = class LaneTimeoutError extends Error {
|
|
112
|
+
constructor(what, timeoutMs) {
|
|
113
|
+
super(`${what} exceeded the ${timeoutMs}ms budget`);
|
|
114
|
+
this.name = "LaneTimeoutError";
|
|
115
|
+
}
|
|
116
|
+
};
|
|
89
117
|
});
|
|
90
118
|
|
|
91
119
|
// src/lanes/claude.ts
|
|
@@ -127,11 +155,12 @@ function createClaudeAdapter(deps = {}) {
|
|
|
127
155
|
throw new LaneAdapterShapeError("@anthropic-ai/claude-agent-sdk exports no query() function");
|
|
128
156
|
}
|
|
129
157
|
const prompt = job.prompt;
|
|
158
|
+
const abortController = new AbortController;
|
|
130
159
|
try {
|
|
131
160
|
const { output, isError } = await withTimeout((async () => {
|
|
132
161
|
const generator = await query({
|
|
133
162
|
prompt,
|
|
134
|
-
options: { model: job.model, maxTurns: 32,
|
|
163
|
+
options: { model: job.model, cwd: job.cwd, maxTurns: 32, abortController, env: subscriptionSdkEnvironment(job.env) }
|
|
135
164
|
});
|
|
136
165
|
let output2 = "";
|
|
137
166
|
let isError2 = false;
|
|
@@ -150,19 +179,19 @@ function createClaudeAdapter(deps = {}) {
|
|
|
150
179
|
isError2 = true;
|
|
151
180
|
}
|
|
152
181
|
return { output: output2, isError: isError2 };
|
|
153
|
-
})(), timeoutMs,
|
|
182
|
+
})(), timeoutMs, "claude lane", () => abortController.abort());
|
|
154
183
|
const durationMs = Date.now() - started;
|
|
155
184
|
return { ok: !isError, exitCode: isError ? 1 : 0, output: output.trim(), durationMs };
|
|
156
185
|
} catch (err) {
|
|
157
186
|
const durationMs = Date.now() - started;
|
|
158
|
-
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err) };
|
|
187
|
+
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err), retryable: !(err instanceof LaneTimeoutError) };
|
|
159
188
|
}
|
|
160
189
|
}
|
|
161
190
|
};
|
|
162
191
|
}
|
|
163
|
-
function runClaudeCli(job, cliPath, started, timeoutMs) {
|
|
192
|
+
async function runClaudeCli(job, cliPath, started, timeoutMs) {
|
|
164
193
|
try {
|
|
165
|
-
const result = spawnCli([cliPath, "-p", job.prompt], {
|
|
194
|
+
const result = await spawnCli([cliPath, "-p", job.prompt], {
|
|
166
195
|
binaryName: cliPath,
|
|
167
196
|
packageHint: "@anthropic-ai/claude-agent-sdk or the claude CLI",
|
|
168
197
|
timeoutMs,
|
|
@@ -219,29 +248,30 @@ function createCodexAdapter(deps = {}) {
|
|
|
219
248
|
if (typeof CodexClass !== "function") {
|
|
220
249
|
throw new LaneAdapterShapeError("@openai/codex-sdk exports no Codex class");
|
|
221
250
|
}
|
|
251
|
+
const abortController = new AbortController;
|
|
222
252
|
try {
|
|
223
253
|
const client = await withTimeout(Promise.resolve(new CodexClass({
|
|
224
254
|
config: { model: job.model },
|
|
225
|
-
env: job.env
|
|
226
|
-
})), timeoutMs,
|
|
227
|
-
const thread = client.startThread?.();
|
|
255
|
+
env: subscriptionEnvironment(job.env)
|
|
256
|
+
})), timeoutMs, "codex lane");
|
|
257
|
+
const thread = client.startThread?.({ model: job.model, workingDirectory: job.cwd });
|
|
228
258
|
if (!thread || typeof thread.run !== "function") {
|
|
229
259
|
throw new LaneAdapterShapeError("@openai/codex-sdk Codex.startThread() returned no run()");
|
|
230
260
|
}
|
|
231
|
-
const turn = await withTimeout(thread.run(job.prompt), timeoutMs,
|
|
261
|
+
const turn = await withTimeout(thread.run(job.prompt, { signal: abortController.signal }), timeoutMs, "codex lane", () => abortController.abort());
|
|
232
262
|
const output = turn?.finalResponse ?? "";
|
|
233
263
|
const durationMs = Date.now() - started;
|
|
234
264
|
return { ok: true, exitCode: 0, output, durationMs };
|
|
235
265
|
} catch (err) {
|
|
236
266
|
const durationMs = Date.now() - started;
|
|
237
|
-
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err) };
|
|
267
|
+
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err), retryable: !(err instanceof LaneTimeoutError) };
|
|
238
268
|
}
|
|
239
269
|
}
|
|
240
270
|
};
|
|
241
271
|
}
|
|
242
|
-
function runCodexCli(job, cliPath, started, timeoutMs) {
|
|
272
|
+
async function runCodexCli(job, cliPath, started, timeoutMs) {
|
|
243
273
|
try {
|
|
244
|
-
const result = spawnCli([cliPath, "exec", "--", job.prompt], {
|
|
274
|
+
const result = await spawnCli([cliPath, "exec", "--", job.prompt], {
|
|
245
275
|
binaryName: cliPath,
|
|
246
276
|
packageHint: "@openai/codex-sdk or the codex CLI",
|
|
247
277
|
timeoutMs,
|
|
@@ -285,6 +315,8 @@ function createCursorAdapter(deps = {}) {
|
|
|
285
315
|
if (!job.prompt) {
|
|
286
316
|
return { ok: false, exitCode: 2, output: "", durationMs: 0, error: "cursor lane requires a prompt" };
|
|
287
317
|
}
|
|
318
|
+
if (!deps.sdkLoader && binaryOnPath(deps.cliPath ?? "cursor-agent"))
|
|
319
|
+
return runCursorCli(job, deps.cliPath ?? "cursor-agent", Date.now(), job.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS);
|
|
288
320
|
const started = Date.now();
|
|
289
321
|
const timeoutMs = job.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS;
|
|
290
322
|
const loader = deps.sdkLoader ?? (async () => await import("@cursor/sdk"));
|
|
@@ -294,36 +326,52 @@ function createCursorAdapter(deps = {}) {
|
|
|
294
326
|
} catch {
|
|
295
327
|
return runCursorCli(job, deps.cliPath ?? "cursor-agent", started, timeoutMs);
|
|
296
328
|
}
|
|
329
|
+
let cancel;
|
|
297
330
|
try {
|
|
298
|
-
|
|
331
|
+
subscriptionEnvironment(job.env);
|
|
332
|
+
if (!deps.sdkLoader && process.env.CURSOR_API_KEY)
|
|
333
|
+
throw new Error("cursor lane refuses ambient API billing; use the native subscription CLI");
|
|
334
|
+
const run2 = await withTimeout(runLocalAgent(sdk, job, (callback) => {
|
|
335
|
+
cancel = callback;
|
|
336
|
+
}), timeoutMs, "cursor lane", () => cancel?.());
|
|
299
337
|
const durationMs = Date.now() - started;
|
|
300
|
-
const status =
|
|
338
|
+
const status = run2.status;
|
|
301
339
|
const ok = status === "finished";
|
|
302
|
-
return { ok, exitCode: ok ? 0 : 1, output:
|
|
340
|
+
return { ok, exitCode: ok ? 0 : 1, output: run2.result, durationMs };
|
|
303
341
|
} catch (err) {
|
|
304
342
|
if (err instanceof LaneAdapterShapeError)
|
|
305
343
|
throw err;
|
|
306
344
|
const durationMs = Date.now() - started;
|
|
307
|
-
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err) };
|
|
345
|
+
return { ok: false, exitCode: 1, output: "", durationMs, error: String(err instanceof Error ? err.message : err), retryable: !(err instanceof LaneTimeoutError) };
|
|
308
346
|
}
|
|
309
347
|
}
|
|
310
348
|
};
|
|
311
349
|
}
|
|
312
|
-
async function runLocalAgent(sdk, job) {
|
|
350
|
+
async function runLocalAgent(sdk, job, onCancel) {
|
|
313
351
|
const AgentClass = sdk.Agent;
|
|
314
352
|
const CursorClass = sdk.Cursor;
|
|
315
353
|
if (typeof AgentClass !== "function" && typeof CursorClass !== "function") {
|
|
316
354
|
throw new LaneAdapterShapeError("@cursor/sdk exports no Agent/Cursor class");
|
|
317
355
|
}
|
|
318
|
-
const
|
|
356
|
+
const create = AgentClass.create;
|
|
357
|
+
const agent = typeof create === "function" ? await create.call(AgentClass, { local: { cwd: job.cwd, enableAgentRetries: false } }) : typeof AgentClass === "function" ? new AgentClass : undefined;
|
|
319
358
|
if (!agent || typeof agent.send !== "function") {
|
|
320
359
|
throw new LaneAdapterShapeError("@cursor/sdk Agent has no send()");
|
|
321
360
|
}
|
|
322
|
-
|
|
323
|
-
if (
|
|
361
|
+
let run2 = await agent.send(job.prompt, { model: job.model });
|
|
362
|
+
if (run2 && typeof run2 === "object") {
|
|
363
|
+
const handle = run2;
|
|
364
|
+
if (typeof handle.cancel === "function")
|
|
365
|
+
onCancel(() => {
|
|
366
|
+
handle.cancel().catch(() => {});
|
|
367
|
+
});
|
|
368
|
+
if (typeof handle.wait === "function")
|
|
369
|
+
run2 = await handle.wait();
|
|
370
|
+
}
|
|
371
|
+
if (!run2 || typeof run2 !== "object") {
|
|
324
372
|
throw new LaneAdapterShapeError("@cursor/sdk Agent.send() returned no Run");
|
|
325
373
|
}
|
|
326
|
-
const candidate =
|
|
374
|
+
const candidate = run2;
|
|
327
375
|
if (typeof candidate.status !== "string" || !["finished", "error", "cancelled"].includes(candidate.status)) {
|
|
328
376
|
throw new LaneAdapterShapeError(`@cursor/sdk Agent.send() returned a Run without a terminal status (got ${JSON.stringify(candidate.status)}; expected finished|error|cancelled)`);
|
|
329
377
|
}
|
|
@@ -332,9 +380,9 @@ async function runLocalAgent(sdk, job) {
|
|
|
332
380
|
}
|
|
333
381
|
return { status: candidate.status, result: candidate.result };
|
|
334
382
|
}
|
|
335
|
-
function runCursorCli(job, cliPath, started, timeoutMs) {
|
|
383
|
+
async function runCursorCli(job, cliPath, started, timeoutMs) {
|
|
336
384
|
try {
|
|
337
|
-
const result = spawnCli([cliPath, "
|
|
385
|
+
const result = await spawnCli([cliPath, "--print", job.prompt, ...job.model ? ["--model", job.model] : []], {
|
|
338
386
|
binaryName: cliPath,
|
|
339
387
|
packageHint: "@cursor/sdk (local mode) or the cursor-agent CLI",
|
|
340
388
|
timeoutMs,
|
|
@@ -377,7 +425,7 @@ function createGrokAdapter(deps = {}) {
|
|
|
377
425
|
const timeoutMs = job.timeoutMs ?? DEFAULT_LANE_TIMEOUT_MS;
|
|
378
426
|
const cliPath = deps.cliPath ?? "grok";
|
|
379
427
|
try {
|
|
380
|
-
const result = spawnCli([cliPath, job.prompt], {
|
|
428
|
+
const result = await spawnCli([cliPath, job.prompt], {
|
|
381
429
|
binaryName: cliPath,
|
|
382
430
|
packageHint: "the grok CLI (no xAI Grok SDK exists on npm \u2014 measured 2026-08-25)",
|
|
383
431
|
timeoutMs,
|
|
@@ -448,10 +496,13 @@ var init_lanes = __esm(() => {
|
|
|
448
496
|
["grok", createGrokAdapter()]
|
|
449
497
|
]);
|
|
450
498
|
});
|
|
499
|
+
|
|
500
|
+
// src/service.ts
|
|
501
|
+
import { createHash as createHash3 } from "crypto";
|
|
451
502
|
// package.json
|
|
452
503
|
var package_default = {
|
|
453
504
|
name: "@hasna/workflows",
|
|
454
|
-
version: "0.
|
|
505
|
+
version: "0.2.0",
|
|
455
506
|
description: "Universal graph workflow app \u2014 graph language (start/step/decision/while/end), three-table store, session WAL, daemon with claims/leases, four lane adapters (claude/codex/cursor/grok), and a full CLI on four surfaces",
|
|
456
507
|
type: "module",
|
|
457
508
|
main: "./dist/index.js",
|
|
@@ -467,8 +518,8 @@ var package_default = {
|
|
|
467
518
|
import: "./dist/index.js"
|
|
468
519
|
},
|
|
469
520
|
"./sdk": {
|
|
470
|
-
types: "./types/
|
|
471
|
-
import: "./dist/
|
|
521
|
+
types: "./types/sdk.d.ts",
|
|
522
|
+
import: "./dist/sdk.js"
|
|
472
523
|
},
|
|
473
524
|
"./cli": {
|
|
474
525
|
types: "./types/cli/index.d.ts",
|
|
@@ -483,14 +534,17 @@ var package_default = {
|
|
|
483
534
|
"hasna.contract.json"
|
|
484
535
|
],
|
|
485
536
|
scripts: {
|
|
486
|
-
build: "bun run build:runtime && bun run build:types",
|
|
487
|
-
"
|
|
488
|
-
"
|
|
537
|
+
build: "bun run check:sdk && bun run build:runtime && bun run build:types",
|
|
538
|
+
"generate:sdk": "bun scripts/generate-sdk.ts --write",
|
|
539
|
+
"check:sdk": "bun scripts/generate-sdk.ts",
|
|
540
|
+
"test:artifacts": "bun scripts/artifact-acceptance.ts",
|
|
541
|
+
"build:runtime": "bun build src/cli/index.ts --outdir dist/cli --target bun && bun build src/mcp/index.ts --outdir dist/mcp --target bun && bun build src/serve/index.ts src/serve/admin.ts --outdir dist/serve --target bun --packages external && bun build src/index.ts src/service.ts src/handlers.ts src/agent-tools.ts src/types.ts --root src --outdir dist --target bun --packages external && bun build src/sdk.ts --outdir dist --target node",
|
|
542
|
+
"build:types": "tsc -p tsconfig.build.json --emitDeclarationOnly --outDir types",
|
|
489
543
|
"artifact-scan": "bun scripts/contracts-cli.mjs artifact-scan dist",
|
|
490
544
|
"contract:check": "bun scripts/contracts-cli.mjs repo-conformance .",
|
|
491
545
|
typecheck: "tsc --noEmit",
|
|
492
546
|
test: "bun test",
|
|
493
|
-
"test:pg": 'WORKFLOWS_TEST_DATABASE_URL="${WORKFLOWS_TEST_DATABASE_URL:?point it at a
|
|
547
|
+
"test:pg": 'WORKFLOWS_TEST_DATABASE_URL="${WORKFLOWS_TEST_DATABASE_URL:?point it at a fresh test Postgres database}" WORKFLOWS_REQUIRE_POSTGRES=1 bun test src/db/postgres-store.test.ts src/db/postgres-http.test.ts src/db/postgres-fencing.test.ts',
|
|
494
548
|
prepack: "bun run build && bun run artifact-scan",
|
|
495
549
|
prepublishOnly: "bun run test && bun run build"
|
|
496
550
|
},
|
|
@@ -509,7 +563,8 @@ var package_default = {
|
|
|
509
563
|
},
|
|
510
564
|
repository: {
|
|
511
565
|
type: "git",
|
|
512
|
-
url: "
|
|
566
|
+
url: "https://github.com/hasna/apps.git",
|
|
567
|
+
directory: "apps/workflows"
|
|
513
568
|
},
|
|
514
569
|
homepage: "https://github.com/hasna/apps/tree/main/apps/workflows",
|
|
515
570
|
bugs: {
|
|
@@ -523,19 +578,26 @@ var package_default = {
|
|
|
523
578
|
dependencies: {
|
|
524
579
|
"@anthropic-ai/claude-agent-sdk": "0.3.234",
|
|
525
580
|
"@cursor/sdk": "^1.0.28",
|
|
526
|
-
"@hasna/paths": "0.1.0",
|
|
527
581
|
"@openai/codex-sdk": "0.147.0",
|
|
528
|
-
commander: "^13.1.0"
|
|
582
|
+
commander: "^13.1.0",
|
|
583
|
+
"@hasna/contracts": "1.3.3",
|
|
584
|
+
pg: "^8.16.3"
|
|
529
585
|
},
|
|
530
586
|
devDependencies: {
|
|
531
|
-
"@hasna/contracts": "0.14.2",
|
|
532
587
|
"@types/bun": "1.3.14",
|
|
533
588
|
"@types/pg": "^8.15.6",
|
|
534
|
-
pg: "^8.16.3",
|
|
535
589
|
typescript: "^5.7.3"
|
|
536
590
|
}
|
|
537
591
|
};
|
|
538
592
|
|
|
593
|
+
// src/version.ts
|
|
594
|
+
function packageVersion() {
|
|
595
|
+
return package_default.version;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// src/daemon.ts
|
|
599
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
600
|
+
|
|
539
601
|
// src/expr.ts
|
|
540
602
|
class ExprSyntaxError extends Error {
|
|
541
603
|
constructor(message) {
|
|
@@ -903,6 +965,9 @@ function findCycles(graph) {
|
|
|
903
965
|
}
|
|
904
966
|
function validateGraph(graph) {
|
|
905
967
|
const issues = [];
|
|
968
|
+
if (graph === null || typeof graph !== "object" || Array.isArray(graph)) {
|
|
969
|
+
return { ok: false, issues: [{ path: "graph", message: "graph must be an object" }] };
|
|
970
|
+
}
|
|
906
971
|
if (typeof graph.name !== "string" || graph.name.trim() === "") {
|
|
907
972
|
issue(issues, "name", "graph requires a non-empty name");
|
|
908
973
|
}
|
|
@@ -925,6 +990,24 @@ function validateGraph(graph) {
|
|
|
925
990
|
issue(issues, "nodes", "graph requires at least one node");
|
|
926
991
|
return { ok: false, issues };
|
|
927
992
|
}
|
|
993
|
+
for (let index = 0;index < graph.nodes.length; index++) {
|
|
994
|
+
const node = graph.nodes[index];
|
|
995
|
+
if (node === null || typeof node !== "object" || Array.isArray(node)) {
|
|
996
|
+
issue(issues, `nodes.${index}`, "node must be an object");
|
|
997
|
+
continue;
|
|
998
|
+
}
|
|
999
|
+
if (!GRAPH_NODE_TYPES.includes(node.type))
|
|
1000
|
+
issue(issues, `nodes.${index}.type`, "type must be start, step, decision, while, or end");
|
|
1001
|
+
for (const key of ["next", "then", "else", "prompt", "command"]) {
|
|
1002
|
+
const value = node[key];
|
|
1003
|
+
if (value !== undefined && (typeof value !== "string" || value.trim() === ""))
|
|
1004
|
+
issue(issues, `nodes.${index}.${key}`, `${key} must be a non-empty string`);
|
|
1005
|
+
}
|
|
1006
|
+
if (node.type === "while" && (!Array.isArray(node.body) || node.body.some((id) => typeof id !== "string")))
|
|
1007
|
+
issue(issues, `nodes.${index}.body`, "body must be an array of node ids");
|
|
1008
|
+
}
|
|
1009
|
+
if (issues.length)
|
|
1010
|
+
return { ok: false, issues };
|
|
928
1011
|
const starts = graph.nodes.filter((n) => n.type === "start");
|
|
929
1012
|
if (starts.length === 0)
|
|
930
1013
|
issue(issues, "nodes", "graph requires exactly one start node (found none)");
|
|
@@ -986,6 +1069,8 @@ function nodeEdges(node) {
|
|
|
986
1069
|
return node.next ? [{ kind: "next", target: node.next }] : [];
|
|
987
1070
|
case "end":
|
|
988
1071
|
return [];
|
|
1072
|
+
default:
|
|
1073
|
+
return [];
|
|
989
1074
|
}
|
|
990
1075
|
}
|
|
991
1076
|
function validateNode(graph, node, issues) {
|
|
@@ -1159,34 +1244,49 @@ import { readFileSync, statSync } from "fs";
|
|
|
1159
1244
|
import { isAbsolute, join } from "path";
|
|
1160
1245
|
var DEFAULT_MAX_ATTEMPTS = 3;
|
|
1161
1246
|
function repairTornRuns(store, wal, opts = {}) {
|
|
1247
|
+
const sequence = repairSteps(store, wal, opts);
|
|
1248
|
+
let step = sequence.next();
|
|
1249
|
+
while (!step.done) {
|
|
1250
|
+
if (step.value instanceof Promise) {
|
|
1251
|
+
return (async () => {
|
|
1252
|
+
while (!step.done)
|
|
1253
|
+
step = sequence.next(await step.value);
|
|
1254
|
+
return step.value;
|
|
1255
|
+
})();
|
|
1256
|
+
}
|
|
1257
|
+
step = sequence.next(step.value);
|
|
1258
|
+
}
|
|
1259
|
+
return step.value;
|
|
1260
|
+
}
|
|
1261
|
+
function* repairSteps(store, wal, opts) {
|
|
1162
1262
|
const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
1163
1263
|
const replay = wal.replay();
|
|
1164
1264
|
const liveClaims = replay.liveClaims(opts.now);
|
|
1165
1265
|
const report = { interrupted: 0, requeued: 0, failed: 0 };
|
|
1166
|
-
const running = store.listRuns({ status: "running", limit: 1000 });
|
|
1266
|
+
const running = yield store.listRuns({ status: "running", limit: 1000 });
|
|
1167
1267
|
for (const run of running) {
|
|
1168
1268
|
if (liveClaims.has(run.id))
|
|
1169
1269
|
continue;
|
|
1170
1270
|
report.interrupted++;
|
|
1171
|
-
const nodes = store.listRunNodes(run.id);
|
|
1271
|
+
const nodes = yield store.listRunNodes(run.id);
|
|
1172
1272
|
if (run.attempts < maxAttempts) {
|
|
1173
|
-
store.bumpAttempts(run.id);
|
|
1174
|
-
store.setRunStatus(run.id, "interrupted", {
|
|
1273
|
+
yield store.bumpAttempts(run.id);
|
|
1274
|
+
yield store.setRunStatus(run.id, "interrupted", {
|
|
1175
1275
|
error: `torn run interrupted after claim loss (attempt ${run.attempts + 1}/${maxAttempts})`
|
|
1176
1276
|
});
|
|
1177
1277
|
for (const node of nodes) {
|
|
1178
1278
|
if (node.status === "running") {
|
|
1179
|
-
store.setRunNodeStatus(node.id, "pending", { error: undefined });
|
|
1279
|
+
yield store.setRunNodeStatus(node.id, "pending", { error: undefined });
|
|
1180
1280
|
}
|
|
1181
1281
|
}
|
|
1182
1282
|
report.requeued++;
|
|
1183
1283
|
} else {
|
|
1184
|
-
store.setRunStatus(run.id, "failed", {
|
|
1284
|
+
yield store.setRunStatus(run.id, "failed", {
|
|
1185
1285
|
error: `torn run failed after ${maxAttempts} interrupted attempts`
|
|
1186
1286
|
});
|
|
1187
1287
|
for (const node of nodes) {
|
|
1188
1288
|
if (node.status === "running") {
|
|
1189
|
-
store.setRunNodeStatus(node.id, "failed", { error: "run interrupted beyond retry budget" });
|
|
1289
|
+
yield store.setRunNodeStatus(node.id, "failed", { error: "run interrupted beyond retry budget" });
|
|
1190
1290
|
}
|
|
1191
1291
|
}
|
|
1192
1292
|
report.failed++;
|
|
@@ -1202,15 +1302,17 @@ function memoKey(graphName, nodeId, input) {
|
|
|
1202
1302
|
}
|
|
1203
1303
|
function tryMemoHit(store, graphName, nodeId, input) {
|
|
1204
1304
|
const key = memoKey(graphName, nodeId, input);
|
|
1205
|
-
const
|
|
1206
|
-
if (
|
|
1207
|
-
return
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1305
|
+
const value = store.memoGet(key);
|
|
1306
|
+
if (value instanceof Promise) {
|
|
1307
|
+
return value.then(async (row) => {
|
|
1308
|
+
if (row)
|
|
1309
|
+
await store.memoHit(key);
|
|
1310
|
+
return row;
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
if (value)
|
|
1314
|
+
store.memoHit(key);
|
|
1315
|
+
return value;
|
|
1214
1316
|
}
|
|
1215
1317
|
function memoWatchFingerprint(graph, dataDir) {
|
|
1216
1318
|
if (!graph.memoWatch || graph.memoWatch.length === 0)
|
|
@@ -1252,20 +1354,377 @@ function fingerprintFile(pattern, path) {
|
|
|
1252
1354
|
return { pattern, path, missing: true, mtimeMs: null, size: null, sha256: null };
|
|
1253
1355
|
}
|
|
1254
1356
|
}
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1357
|
+
|
|
1358
|
+
// src/db/postgres-store.ts
|
|
1359
|
+
import { randomUUID } from "crypto";
|
|
1360
|
+
import { Pool } from "pg";
|
|
1361
|
+
var run = (r) => ({
|
|
1362
|
+
id: String(r.id),
|
|
1363
|
+
graphName: String(r.graph_name),
|
|
1364
|
+
graphVersion: String(r.graph_version),
|
|
1365
|
+
status: r.status,
|
|
1366
|
+
attempts: Number(r.attempts),
|
|
1367
|
+
contextJson: String(r.context_json),
|
|
1368
|
+
resultJson: r.result_json == null ? null : String(r.result_json),
|
|
1369
|
+
error: r.error == null ? null : String(r.error),
|
|
1370
|
+
startedAt: r.started_at == null ? null : String(r.started_at),
|
|
1371
|
+
finishedAt: r.finished_at == null ? null : String(r.finished_at),
|
|
1372
|
+
createdAt: String(r.created_at),
|
|
1373
|
+
updatedAt: String(r.updated_at)
|
|
1374
|
+
});
|
|
1375
|
+
var node = (r) => ({
|
|
1376
|
+
id: String(r.id),
|
|
1377
|
+
runId: String(r.run_id),
|
|
1378
|
+
nodeId: String(r.node_id),
|
|
1379
|
+
status: r.status,
|
|
1380
|
+
attempts: Number(r.attempts),
|
|
1381
|
+
lane: r.lane == null ? null : String(r.lane),
|
|
1382
|
+
exitCode: r.exit_code == null ? null : Number(r.exit_code),
|
|
1383
|
+
outputJson: r.output_json == null ? null : String(r.output_json),
|
|
1384
|
+
error: r.error == null ? null : String(r.error),
|
|
1385
|
+
startedAt: r.started_at == null ? null : String(r.started_at),
|
|
1386
|
+
finishedAt: r.finished_at == null ? null : String(r.finished_at)
|
|
1387
|
+
});
|
|
1388
|
+
var memo = (r) => ({
|
|
1389
|
+
key: String(r.key),
|
|
1390
|
+
graphName: String(r.graph_name),
|
|
1391
|
+
nodeId: String(r.node_id),
|
|
1392
|
+
inputHash: String(r.input_hash),
|
|
1393
|
+
outputJson: String(r.output_json),
|
|
1394
|
+
hitCount: Number(r.hit_count),
|
|
1395
|
+
createdAt: String(r.created_at)
|
|
1396
|
+
});
|
|
1397
|
+
|
|
1398
|
+
class WorkflowStorageError extends Error {
|
|
1399
|
+
constructor() {
|
|
1400
|
+
super("PostgreSQL workflow storage is unavailable or its schema is invalid");
|
|
1401
|
+
this.name = "WorkflowStorageError";
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
class PostgresWorkflowsStore {
|
|
1406
|
+
pool;
|
|
1407
|
+
prefix;
|
|
1408
|
+
dataDir;
|
|
1409
|
+
execution;
|
|
1410
|
+
constructor(options) {
|
|
1411
|
+
const schema = options.schema ?? "workflows";
|
|
1412
|
+
if (!/^[a-z][a-z0-9_]{0,62}$/.test(schema))
|
|
1413
|
+
throw new Error("PostgreSQL schema must be a lowercase SQL identifier");
|
|
1414
|
+
this.prefix = `"${schema}"`;
|
|
1415
|
+
this.dataDir = options.dataDir;
|
|
1416
|
+
this.pool = new Pool({
|
|
1417
|
+
connectionString: options.databaseUrl,
|
|
1418
|
+
max: 4,
|
|
1419
|
+
connectionTimeoutMillis: 5000,
|
|
1420
|
+
idleTimeoutMillis: 1e4,
|
|
1421
|
+
statement_timeout: 1e4,
|
|
1422
|
+
application_name: "workflows"
|
|
1423
|
+
});
|
|
1424
|
+
}
|
|
1425
|
+
async rows(sql, values = []) {
|
|
1258
1426
|
try {
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
} catch {
|
|
1427
|
+
if (this.execution?.lost)
|
|
1428
|
+
throw new WorkflowStorageError;
|
|
1429
|
+
return (await (this.execution?.client ?? this.pool).query(sql, values)).rows;
|
|
1430
|
+
} catch (error) {
|
|
1431
|
+
if (error?.code === "23505")
|
|
1432
|
+
throw Object.assign(new Error("workflow uniqueness conflict"), { code: "23505" });
|
|
1433
|
+
throw new WorkflowStorageError;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
async close() {
|
|
1437
|
+
await this.releaseExecution();
|
|
1438
|
+
await this.pool.end();
|
|
1439
|
+
}
|
|
1440
|
+
async acquireExecution(runId) {
|
|
1441
|
+
if (this.execution)
|
|
1442
|
+
return this.execution.runId === runId && !this.execution.lost;
|
|
1443
|
+
const client = await this.pool.connect();
|
|
1444
|
+
let lost = false;
|
|
1445
|
+
client.on("error", () => {
|
|
1446
|
+
lost = true;
|
|
1447
|
+
if (this.execution?.client === client)
|
|
1448
|
+
this.execution.lost = true;
|
|
1449
|
+
});
|
|
1450
|
+
try {
|
|
1451
|
+
const acquired = await client.query("SELECT pg_try_advisory_lock(hashtextextended($1,0)) AS owned", [`${this.prefix}:${runId}`]);
|
|
1452
|
+
if (!acquired.rows[0]?.owned) {
|
|
1453
|
+
client.release();
|
|
1454
|
+
return false;
|
|
1455
|
+
}
|
|
1456
|
+
this.execution = { runId, client, lost };
|
|
1457
|
+
return !lost;
|
|
1458
|
+
} catch {
|
|
1459
|
+
client.release(true);
|
|
1460
|
+
throw new WorkflowStorageError;
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
async releaseExecution() {
|
|
1464
|
+
const owned = this.execution;
|
|
1465
|
+
if (!owned)
|
|
1466
|
+
return;
|
|
1467
|
+
this.execution = undefined;
|
|
1468
|
+
try {
|
|
1469
|
+
if (!owned.lost)
|
|
1470
|
+
await owned.client.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [`${this.prefix}:${owned.runId}`]);
|
|
1471
|
+
} finally {
|
|
1472
|
+
owned.client.release(owned.lost);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
async transaction(action) {
|
|
1476
|
+
if (this.execution?.lost)
|
|
1477
|
+
throw new WorkflowStorageError;
|
|
1478
|
+
const client = this.execution?.client ?? await this.pool.connect();
|
|
1479
|
+
const retained = client === this.execution?.client;
|
|
1480
|
+
try {
|
|
1481
|
+
await client.query("BEGIN");
|
|
1482
|
+
const result = await action(client);
|
|
1483
|
+
await client.query("COMMIT");
|
|
1484
|
+
return result;
|
|
1485
|
+
} catch {
|
|
1486
|
+
try {
|
|
1487
|
+
await client.query("ROLLBACK");
|
|
1488
|
+
} catch {}
|
|
1489
|
+
throw new WorkflowStorageError;
|
|
1490
|
+
} finally {
|
|
1491
|
+
if (!retained)
|
|
1492
|
+
client.release();
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
async listTables() {
|
|
1496
|
+
return (await this.rows("SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' ORDER BY table_name", [this.prefix.slice(1, -1)])).map((row) => String(row.table_name));
|
|
1497
|
+
}
|
|
1498
|
+
async checkSchema() {
|
|
1499
|
+
if ((await this.listTables()).join(",") !== "memos,run_nodes,runs")
|
|
1500
|
+
return false;
|
|
1501
|
+
await this.rows(`SELECT id,graph_name,graph_version,status,attempts,context_json,result_json,error,started_at,finished_at,created_at,updated_at FROM ${this.prefix}.runs LIMIT 0`);
|
|
1502
|
+
await this.rows(`SELECT id,sequence,run_id,node_id,status,attempts,lane,exit_code,output_json,error,started_at,finished_at FROM ${this.prefix}.run_nodes LIMIT 0`);
|
|
1503
|
+
await this.rows(`SELECT key,graph_name,node_id,input_hash,output_json,hit_count,created_at FROM ${this.prefix}.memos LIMIT 0`);
|
|
1504
|
+
const schema = this.prefix.slice(1, -1);
|
|
1505
|
+
const indices = await this.rows(`SELECT i.indisunique,i.indisvalid,i.indisready,i.indnkeyatts,i.indnatts,i.indpred IS NULL AS unfiltered,pg_get_expr(i.indexprs,i.indrelid) AS expression FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid JOIN pg_class t ON t.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relname='runs_idempotency' AND t.relname='runs'`, [schema]);
|
|
1506
|
+
const index = indices[0];
|
|
1507
|
+
if (indices.length !== 1 || !index?.indisunique || !index.indisvalid || !index.indisready || !index.unfiltered || Number(index.indnkeyatts) !== 1 || Number(index.indnatts) !== 1 || String(index.expression).replace(/[()\s]/g, "") !== "context_json::jsonb#>>'{__wf,idempotencyKey}'::text[]")
|
|
1508
|
+
return false;
|
|
1509
|
+
const columns = await this.rows("SELECT table_name,column_name,data_type,is_nullable,is_identity,identity_generation FROM information_schema.columns WHERE table_schema=$1", [schema]);
|
|
1510
|
+
const required = {
|
|
1511
|
+
runs: ["id:text", "graph_name:text", "graph_version:text", "status:text", "attempts:integer", "context_json:text", "created_at:text", "updated_at:text"],
|
|
1512
|
+
run_nodes: ["id:text", "sequence:bigint", "run_id:text", "node_id:text", "status:text", "attempts:integer"],
|
|
1513
|
+
memos: ["key:text", "graph_name:text", "node_id:text", "input_hash:text", "output_json:text", "hit_count:integer", "created_at:text"]
|
|
1514
|
+
};
|
|
1515
|
+
for (const [table, fields] of Object.entries(required))
|
|
1516
|
+
for (const field of fields) {
|
|
1517
|
+
const [name, type] = field.split(":");
|
|
1518
|
+
const c = columns.find((row) => row.table_name === table && row.column_name === name);
|
|
1519
|
+
if (!c || c.data_type !== type || c.is_nullable !== "NO")
|
|
1520
|
+
return false;
|
|
1521
|
+
if (name === "sequence" && (c.is_identity !== "YES" || c.identity_generation !== "ALWAYS"))
|
|
1522
|
+
return false;
|
|
1523
|
+
}
|
|
1524
|
+
const constraints = await this.rows(`SELECT c.relname AS table_name,co.contype,pg_get_constraintdef(co.oid) AS definition FROM pg_constraint co JOIN pg_class c ON c.oid=co.conrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1`, [schema]);
|
|
1525
|
+
for (const [table, key] of [["runs", "id"], ["run_nodes", "id"], ["memos", "key"]])
|
|
1526
|
+
if (!constraints.some((row) => row.table_name === table && row.contype === "p" && row.definition === `PRIMARY KEY (${key})`))
|
|
1527
|
+
return false;
|
|
1528
|
+
return constraints.some((row) => row.table_name === "run_nodes" && row.contype === "f" && String(row.definition).replaceAll('"', "") === `FOREIGN KEY (run_id) REFERENCES ${schema}.runs(id)`);
|
|
1529
|
+
}
|
|
1530
|
+
async initialize() {
|
|
1531
|
+
const tables = await this.listTables();
|
|
1532
|
+
if (tables.length) {
|
|
1533
|
+
if (await this.checkSchema())
|
|
1534
|
+
return;
|
|
1535
|
+
throw new WorkflowStorageError;
|
|
1536
|
+
}
|
|
1537
|
+
const client = await this.pool.connect();
|
|
1538
|
+
try {
|
|
1539
|
+
await client.query("BEGIN");
|
|
1540
|
+
await client.query(`CREATE SCHEMA IF NOT EXISTS ${this.prefix};
|
|
1541
|
+
CREATE TABLE IF NOT EXISTS ${this.prefix}.runs (
|
|
1542
|
+
id TEXT PRIMARY KEY, graph_name TEXT NOT NULL, graph_version TEXT NOT NULL, status TEXT NOT NULL,
|
|
1543
|
+
attempts INTEGER NOT NULL DEFAULT 0, context_json TEXT NOT NULL, result_json TEXT, error TEXT,
|
|
1544
|
+
started_at TEXT, finished_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL);
|
|
1545
|
+
CREATE TABLE IF NOT EXISTS ${this.prefix}.run_nodes (
|
|
1546
|
+
id TEXT PRIMARY KEY, sequence BIGINT GENERATED ALWAYS AS IDENTITY, run_id TEXT NOT NULL REFERENCES ${this.prefix}.runs(id), node_id TEXT NOT NULL,
|
|
1547
|
+
status TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, lane TEXT, exit_code INTEGER,
|
|
1548
|
+
output_json TEXT, error TEXT, started_at TEXT, finished_at TEXT);
|
|
1549
|
+
CREATE TABLE IF NOT EXISTS ${this.prefix}.memos (
|
|
1550
|
+
key TEXT PRIMARY KEY, graph_name TEXT NOT NULL, node_id TEXT NOT NULL, input_hash TEXT NOT NULL,
|
|
1551
|
+
output_json TEXT NOT NULL, hit_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL);
|
|
1552
|
+
CREATE UNIQUE INDEX IF NOT EXISTS runs_idempotency ON ${this.prefix}.runs ((context_json::jsonb #>> '{__wf,idempotencyKey}'));
|
|
1553
|
+
CREATE INDEX IF NOT EXISTS runs_status ON ${this.prefix}.runs(status);
|
|
1554
|
+
CREATE INDEX IF NOT EXISTS nodes_run ON ${this.prefix}.run_nodes(run_id);`);
|
|
1555
|
+
await client.query("COMMIT");
|
|
1556
|
+
} catch {
|
|
1557
|
+
await client.query("ROLLBACK");
|
|
1558
|
+
throw new WorkflowStorageError;
|
|
1559
|
+
} finally {
|
|
1560
|
+
client.release();
|
|
1561
|
+
}
|
|
1562
|
+
if (!await this.checkSchema())
|
|
1563
|
+
throw new WorkflowStorageError;
|
|
1564
|
+
}
|
|
1565
|
+
async createRun(input) {
|
|
1566
|
+
const at = new Date().toISOString();
|
|
1567
|
+
const rows = await this.rows(`INSERT INTO ${this.prefix}.runs(id,graph_name,graph_version,status,context_json,created_at,updated_at)
|
|
1568
|
+
VALUES($1,$2,$3,'pending',$4,$5,$5) RETURNING *`, [randomUUID(), input.graphName, input.graphVersion, JSON.stringify(input.context ?? {}), at]);
|
|
1569
|
+
return run(rows[0]);
|
|
1570
|
+
}
|
|
1571
|
+
async getRun(id) {
|
|
1572
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.runs WHERE id=$1`, [id]);
|
|
1573
|
+
return rows[0] ? run(rows[0]) : undefined;
|
|
1574
|
+
}
|
|
1575
|
+
async getRunByIdempotencyKey(key) {
|
|
1576
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.runs WHERE context_json::jsonb #>> '{__wf,idempotencyKey}'=$1`, [key]);
|
|
1577
|
+
return rows[0] ? run(rows[0]) : undefined;
|
|
1578
|
+
}
|
|
1579
|
+
async listRuns(options = {}) {
|
|
1580
|
+
const limit = Math.max(1, Math.min(1000, options.limit ?? 100));
|
|
1581
|
+
return (await this.rows(`SELECT * FROM ${this.prefix}.runs ${options.status ? "WHERE status=$2" : ""} ORDER BY created_at DESC,id DESC LIMIT $1`, options.status ? [limit, options.status] : [limit])).map(run);
|
|
1582
|
+
}
|
|
1583
|
+
async setRunStatus(id, status, patch = {}) {
|
|
1584
|
+
const values = [id, status, new Date().toISOString()];
|
|
1585
|
+
const sets = ["status=$2", "updated_at=$3"];
|
|
1586
|
+
for (const [field, key] of [["result_json", "result"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1587
|
+
if (Object.hasOwn(patch, key)) {
|
|
1588
|
+
values.push(key === "result" ? JSON.stringify(patch.result) : patch[key]);
|
|
1589
|
+
sets.push(`${field}=$${values.length}`);
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
if (status === "running" && !Object.hasOwn(patch, "startedAt"))
|
|
1593
|
+
sets.push("started_at=COALESCE(started_at,$3)");
|
|
1594
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET ${sets.join(",")} WHERE id=$1`, values);
|
|
1595
|
+
}
|
|
1596
|
+
async compareRunStatus(id, expected, status, patch = {}) {
|
|
1597
|
+
const values = [id, expected, status, new Date().toISOString()];
|
|
1598
|
+
const sets = ["status=$3", "updated_at=$4"];
|
|
1599
|
+
for (const [field, key] of [["result_json", "result"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1600
|
+
if (patch && Object.hasOwn(patch, key)) {
|
|
1601
|
+
values.push(key === "result" ? JSON.stringify(patch.result) : patch[key]);
|
|
1602
|
+
sets.push(`${field}=$${values.length}`);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
return (await this.rows(`UPDATE ${this.prefix}.runs SET ${sets.join(",")} WHERE id=$1 AND status=ANY($2::text[]) RETURNING id`, values)).length === 1;
|
|
1606
|
+
}
|
|
1607
|
+
async setRunContext(id, context) {
|
|
1608
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET context_json=$2,updated_at=$3 WHERE id=$1`, [id, JSON.stringify(context), new Date().toISOString()]);
|
|
1609
|
+
}
|
|
1610
|
+
async setRunningContext(id, context) {
|
|
1611
|
+
return (await this.rows(`UPDATE ${this.prefix}.runs SET context_json=$2,updated_at=$3 WHERE id=$1 AND status='running' RETURNING id`, [id, JSON.stringify(context), new Date().toISOString()])).length === 1;
|
|
1612
|
+
}
|
|
1613
|
+
async blockRunResume(id) {
|
|
1614
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET context_json=jsonb_set(context_json::jsonb,'{__wf,resumeBlocked}',to_jsonb($2::text))::text,updated_at=$3 WHERE id=$1`, [id, "execution cancellation has not been confirmed", new Date().toISOString()]);
|
|
1615
|
+
}
|
|
1616
|
+
async dispatchIfRunning(id, action) {
|
|
1617
|
+
const marked = await this.rows(`UPDATE ${this.prefix}.runs SET context_json=jsonb_set(context_json::jsonb,'{__wf,executionMayBeActive}','true'::jsonb)::text WHERE id=$1 AND status='running' RETURNING id`, [id]);
|
|
1618
|
+
if (!marked.length)
|
|
1619
|
+
return { started: false };
|
|
1620
|
+
return this.transaction(async (client) => {
|
|
1621
|
+
const run2 = await client.query(`SELECT status FROM ${this.prefix}.runs WHERE id=$1 FOR UPDATE`, [id]);
|
|
1622
|
+
if (run2.rows[0]?.status !== "running")
|
|
1623
|
+
return { started: false };
|
|
1624
|
+
return { started: true, result: action() };
|
|
1625
|
+
});
|
|
1626
|
+
}
|
|
1627
|
+
async finishDispatch(id, unconfirmed) {
|
|
1628
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET context_json=(CASE WHEN $2 THEN jsonb_set(context_json::jsonb #- '{__wf,executionMayBeActive}','{__wf,resumeBlocked}',to_jsonb($3::text)) ELSE context_json::jsonb #- '{__wf,executionMayBeActive}' END)::text,updated_at=$4 WHERE id=$1`, [id, unconfirmed, "execution cancellation has not been confirmed", new Date().toISOString()]);
|
|
1629
|
+
}
|
|
1630
|
+
async commitStep(input) {
|
|
1631
|
+
return this.transaction(async (client) => {
|
|
1632
|
+
const nodes = await client.query(`SELECT run_id FROM ${this.prefix}.run_nodes WHERE id=$1 FOR UPDATE`, [input.nodeRowId]);
|
|
1633
|
+
if (nodes.rows[0]?.run_id !== input.runId)
|
|
1634
|
+
return false;
|
|
1635
|
+
const run2 = await client.query(`SELECT status FROM ${this.prefix}.runs WHERE id=$1 FOR UPDATE`, [input.runId]);
|
|
1636
|
+
if (run2.rows[0]?.status !== "running")
|
|
1637
|
+
return false;
|
|
1638
|
+
const patch = input.patch ?? {}, values = [input.nodeRowId, input.status];
|
|
1639
|
+
const sets = ["status=$2"];
|
|
1640
|
+
for (const [field, key] of [["exit_code", "exitCode"], ["output_json", "output"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1641
|
+
if (Object.hasOwn(patch, key)) {
|
|
1642
|
+
values.push(key === "output" ? JSON.stringify(patch.output) : patch[key]);
|
|
1643
|
+
sets.push(`${field}=$${values.length}`);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
if (input.status === "running") {
|
|
1647
|
+
values.push(new Date().toISOString());
|
|
1648
|
+
sets.push(`started_at=COALESCE(started_at,$${values.length})`);
|
|
1649
|
+
}
|
|
1650
|
+
if (["completed", "failed", "skipped"].includes(input.status)) {
|
|
1651
|
+
values.push(new Date().toISOString());
|
|
1652
|
+
sets.push(`finished_at=$${values.length}`);
|
|
1653
|
+
}
|
|
1654
|
+
if (input.incrementAttempts)
|
|
1655
|
+
sets.push("attempts=attempts+1");
|
|
1656
|
+
await client.query(`UPDATE ${this.prefix}.run_nodes SET ${sets.join(",")} WHERE id=$1`, values);
|
|
1657
|
+
if (input.context !== undefined)
|
|
1658
|
+
await client.query(`UPDATE ${this.prefix}.runs SET context_json=$2,updated_at=$3 WHERE id=$1`, [input.runId, JSON.stringify(input.context), new Date().toISOString()]);
|
|
1659
|
+
if (input.memo) {
|
|
1660
|
+
const m = input.memo;
|
|
1661
|
+
await client.query(`INSERT INTO ${this.prefix}.memos(key,graph_name,node_id,input_hash,output_json,created_at) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT(key) DO UPDATE SET output_json=EXCLUDED.output_json`, [m.key, m.graphName, m.nodeId, m.inputHash, m.outputJson, new Date().toISOString()]);
|
|
1662
|
+
}
|
|
1663
|
+
return true;
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
async bumpAttempts(id) {
|
|
1667
|
+
await this.rows(`UPDATE ${this.prefix}.runs SET attempts=attempts+1,updated_at=$2 WHERE id=$1`, [id, new Date().toISOString()]);
|
|
1668
|
+
}
|
|
1669
|
+
async createRunNode(input) {
|
|
1670
|
+
const rows = await this.rows(`INSERT INTO ${this.prefix}.run_nodes(id,run_id,node_id,lane,status) VALUES($1,$2,$3,$4,'pending') RETURNING *`, [randomUUID(), input.runId, input.nodeId, input.lane ?? null]);
|
|
1671
|
+
return node(rows[0]);
|
|
1672
|
+
}
|
|
1673
|
+
async getRunNode(id) {
|
|
1674
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.run_nodes WHERE id=$1`, [id]);
|
|
1675
|
+
return rows[0] ? node(rows[0]) : undefined;
|
|
1676
|
+
}
|
|
1677
|
+
async listRunNodes(runId) {
|
|
1678
|
+
return (await this.rows(`SELECT * FROM ${this.prefix}.run_nodes WHERE run_id=$1 ORDER BY sequence`, [runId])).map(node);
|
|
1679
|
+
}
|
|
1680
|
+
async getLatestRunNode(runId, nodeId) {
|
|
1681
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.run_nodes WHERE run_id=$1 AND node_id=$2 ORDER BY sequence DESC LIMIT 1`, [runId, nodeId]);
|
|
1682
|
+
return rows[0] ? node(rows[0]) : undefined;
|
|
1683
|
+
}
|
|
1684
|
+
async setRunNodeStatus(id, status, patch = {}) {
|
|
1685
|
+
const values = [id, status];
|
|
1686
|
+
const sets = ["status=$2"];
|
|
1687
|
+
for (const [field, key] of [["exit_code", "exitCode"], ["output_json", "output"], ["error", "error"], ["started_at", "startedAt"], ["finished_at", "finishedAt"]]) {
|
|
1688
|
+
if (Object.hasOwn(patch, key)) {
|
|
1689
|
+
values.push(key === "output" ? JSON.stringify(patch.output) : patch[key]);
|
|
1690
|
+
sets.push(`${field}=$${values.length}`);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
if (status === "running" && !Object.hasOwn(patch, "startedAt")) {
|
|
1694
|
+
values.push(new Date().toISOString());
|
|
1695
|
+
sets.push(`started_at=COALESCE(started_at,$${values.length})`);
|
|
1696
|
+
}
|
|
1697
|
+
if (["completed", "failed", "skipped"].includes(status) && !Object.hasOwn(patch, "finishedAt")) {
|
|
1698
|
+
values.push(new Date().toISOString());
|
|
1699
|
+
sets.push(`finished_at=$${values.length}`);
|
|
1700
|
+
}
|
|
1701
|
+
await this.rows(`UPDATE ${this.prefix}.run_nodes SET ${sets.join(",")} WHERE id=$1`, values);
|
|
1702
|
+
}
|
|
1703
|
+
async bumpAttemptsNode(id) {
|
|
1704
|
+
await this.rows(`UPDATE ${this.prefix}.run_nodes SET attempts=attempts+1 WHERE id=$1`, [id]);
|
|
1705
|
+
}
|
|
1706
|
+
async memoPut(key, graphName, nodeId, inputHash2, outputJson) {
|
|
1707
|
+
await this.rows(`INSERT INTO ${this.prefix}.memos(key,graph_name,node_id,input_hash,output_json,created_at) VALUES($1,$2,$3,$4,$5,$6)
|
|
1708
|
+
ON CONFLICT(key) DO UPDATE SET output_json=EXCLUDED.output_json`, [key, graphName, nodeId, inputHash2, outputJson, new Date().toISOString()]);
|
|
1709
|
+
}
|
|
1710
|
+
async memoGet(key) {
|
|
1711
|
+
const rows = await this.rows(`SELECT * FROM ${this.prefix}.memos WHERE key=$1`, [key]);
|
|
1712
|
+
return rows[0] ? memo(rows[0]) : undefined;
|
|
1713
|
+
}
|
|
1714
|
+
async memoHit(key) {
|
|
1715
|
+
await this.rows(`UPDATE ${this.prefix}.memos SET hit_count=hit_count+1 WHERE key=$1`, [key]);
|
|
1716
|
+
}
|
|
1717
|
+
async memoList() {
|
|
1718
|
+
return (await this.rows(`SELECT * FROM ${this.prefix}.memos ORDER BY created_at DESC,key LIMIT 1000`)).map(memo);
|
|
1719
|
+
}
|
|
1720
|
+
async memoClear() {
|
|
1721
|
+
throw new Error("PostgreSQL memo removal requires the reviewed server-operator preservation procedure");
|
|
1263
1722
|
}
|
|
1264
|
-
return;
|
|
1265
1723
|
}
|
|
1266
1724
|
|
|
1267
1725
|
// src/daemon.ts
|
|
1268
1726
|
init_types();
|
|
1727
|
+
init_types();
|
|
1269
1728
|
var DEFAULT_LEASE_TTL_MS = 120000;
|
|
1270
1729
|
var DEFAULT_MAX_ATTEMPTS2 = 3;
|
|
1271
1730
|
var DEFAULT_MAX_DISPATCH = 4;
|
|
@@ -1278,27 +1737,28 @@ function stripRunState(context) {
|
|
|
1278
1737
|
const { __wf: _wf, steps: _steps, ...user } = context;
|
|
1279
1738
|
return user;
|
|
1280
1739
|
}
|
|
1281
|
-
function runCommand(job) {
|
|
1740
|
+
async function runCommand(job) {
|
|
1282
1741
|
if (!job.command) {
|
|
1283
1742
|
return { ok: false, exitCode: 2, output: "", durationMs: 0, error: "no command" };
|
|
1284
1743
|
}
|
|
1285
1744
|
const started = Date.now();
|
|
1286
1745
|
const timeoutMs = job.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
1287
|
-
const proc = Bun.
|
|
1746
|
+
const proc = Bun.spawn({
|
|
1288
1747
|
cmd: ["/bin/sh", "-c", job.command],
|
|
1289
1748
|
cwd: job.cwd,
|
|
1290
|
-
env:
|
|
1749
|
+
env: subscriptionEnvironment(job.env),
|
|
1291
1750
|
stdout: "pipe",
|
|
1292
1751
|
stderr: "pipe",
|
|
1293
1752
|
timeout: timeoutMs
|
|
1294
1753
|
});
|
|
1295
|
-
const
|
|
1296
|
-
|
|
1297
|
-
|
|
1754
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
1755
|
+
new Response(proc.stdout).text(),
|
|
1756
|
+
new Response(proc.stderr).text(),
|
|
1757
|
+
proc.exited
|
|
1758
|
+
]);
|
|
1298
1759
|
const output = stderr ? `${stdout}
|
|
1299
1760
|
${stderr}`.trim() : stdout.trim();
|
|
1300
|
-
|
|
1301
|
-
return { ok: exitCode === 0, exitCode, output, durationMs };
|
|
1761
|
+
return { ok: exitCode === 0, exitCode, output, durationMs: Date.now() - started };
|
|
1302
1762
|
}
|
|
1303
1763
|
|
|
1304
1764
|
class WorkflowsDaemon {
|
|
@@ -1319,7 +1779,7 @@ class WorkflowsDaemon {
|
|
|
1319
1779
|
constructor(store, wal, options = {}) {
|
|
1320
1780
|
this.store = store;
|
|
1321
1781
|
this.wal = wal;
|
|
1322
|
-
this.worker = options.worker ??
|
|
1782
|
+
this.worker = options.worker ?? `worker-${randomUUID2()}`;
|
|
1323
1783
|
this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
|
|
1324
1784
|
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS2;
|
|
1325
1785
|
this.maxDispatchPerCycle = options.maxDispatchPerCycle ?? DEFAULT_MAX_DISPATCH;
|
|
@@ -1351,17 +1811,22 @@ class WorkflowsDaemon {
|
|
|
1351
1811
|
return this.leases;
|
|
1352
1812
|
}
|
|
1353
1813
|
claim(runId) {
|
|
1814
|
+
return this.wal.exclusive(() => this.claimLocked(runId));
|
|
1815
|
+
}
|
|
1816
|
+
claimLocked(runId) {
|
|
1354
1817
|
const now = this.time();
|
|
1355
1818
|
const existing = this.leases.get(runId);
|
|
1356
1819
|
if (existing && existing.worker !== this.worker && existing.expiresAtMs > now) {
|
|
1357
1820
|
return null;
|
|
1358
1821
|
}
|
|
1359
|
-
const
|
|
1822
|
+
const replay = this.wal.replay();
|
|
1823
|
+
const liveClaims = replay.liveClaims(now);
|
|
1360
1824
|
const foreign = liveClaims.get(runId);
|
|
1361
1825
|
if (foreign && foreign.worker !== this.worker && foreign.expiresAtMs > now) {
|
|
1362
1826
|
return null;
|
|
1363
1827
|
}
|
|
1364
|
-
const
|
|
1828
|
+
const lastFencing = replay.entries.reduce((max, entry) => entry.op.op === "claim_acquired" && entry.op.runId === runId ? Math.max(max, entry.op.fencing) : max, 0);
|
|
1829
|
+
const fencing = Math.max(lastFencing, this.fencingCounter.get(runId) ?? 0) + 1;
|
|
1365
1830
|
this.fencingCounter.set(runId, fencing);
|
|
1366
1831
|
const lease = { runId, worker: this.worker, expiresAtMs: now + this.leaseTtlMs, fencing };
|
|
1367
1832
|
this.leases.set(runId, lease);
|
|
@@ -1376,16 +1841,36 @@ class WorkflowsDaemon {
|
|
|
1376
1841
|
return lease;
|
|
1377
1842
|
}
|
|
1378
1843
|
heartbeat(runId, fencing) {
|
|
1844
|
+
return this.wal.exclusive(() => this.heartbeatLocked(runId, fencing));
|
|
1845
|
+
}
|
|
1846
|
+
heartbeatLocked(runId, fencing) {
|
|
1379
1847
|
const lease = this.leases.get(runId);
|
|
1380
1848
|
if (!lease || lease.fencing !== fencing)
|
|
1381
1849
|
return false;
|
|
1850
|
+
const current = this.wal.replay({ repair: false }).liveClaims(this.time()).get(runId);
|
|
1851
|
+
if (!current || current.worker !== lease.worker || current.fencing !== fencing)
|
|
1852
|
+
return false;
|
|
1382
1853
|
lease.expiresAtMs = this.time() + this.leaseTtlMs;
|
|
1854
|
+
this.wal.append({
|
|
1855
|
+
op: "claim_acquired",
|
|
1856
|
+
runId,
|
|
1857
|
+
worker: lease.worker,
|
|
1858
|
+
fencing,
|
|
1859
|
+
expiresAt: new Date(lease.expiresAtMs).toISOString(),
|
|
1860
|
+
at: nowIso()
|
|
1861
|
+
});
|
|
1383
1862
|
return true;
|
|
1384
1863
|
}
|
|
1385
1864
|
release(runId, fencing) {
|
|
1865
|
+
return this.wal.exclusive(() => this.releaseLocked(runId, fencing));
|
|
1866
|
+
}
|
|
1867
|
+
releaseLocked(runId, fencing) {
|
|
1386
1868
|
const lease = this.leases.get(runId);
|
|
1387
1869
|
if (!lease || lease.fencing !== fencing)
|
|
1388
1870
|
return false;
|
|
1871
|
+
const current = this.wal.replay().liveClaims(this.time()).get(runId);
|
|
1872
|
+
if (current && (current.worker !== lease.worker || current.fencing !== fencing))
|
|
1873
|
+
return false;
|
|
1389
1874
|
this.leases.delete(runId);
|
|
1390
1875
|
this.wal.append({ op: "claim_released", runId, fencing, at: nowIso() });
|
|
1391
1876
|
return true;
|
|
@@ -1400,30 +1885,30 @@ class WorkflowsDaemon {
|
|
|
1400
1885
|
}
|
|
1401
1886
|
}
|
|
1402
1887
|
if (this.isolatedRunId === null) {
|
|
1403
|
-
const repair = repairTornRuns(this.store, this.wal, { maxAttempts: this.maxAttempts, now });
|
|
1888
|
+
const repair = await repairTornRuns(this.store, this.wal, { maxAttempts: this.maxAttempts, now });
|
|
1404
1889
|
report.interrupted += repair.interrupted;
|
|
1405
1890
|
report.requeued += repair.requeued;
|
|
1406
1891
|
report.failed += repair.failed;
|
|
1407
1892
|
}
|
|
1408
|
-
const pending = this.store.listRuns({ status: "pending", limit: this.maxDispatchPerCycle }).concat(this.store.listRuns({ status: "interrupted", limit: this.maxDispatchPerCycle })).sort((a, b) => a.createdAt === b.createdAt ? a.id < b.id ? 1 : -1 : a.createdAt < b.createdAt ? 1 : -1).slice(0, this.maxDispatchPerCycle);
|
|
1409
|
-
for (const
|
|
1410
|
-
if (this.isolatedRunId !== null &&
|
|
1893
|
+
const pending = (await this.store.listRuns({ status: "pending", limit: this.maxDispatchPerCycle })).concat(await this.store.listRuns({ status: "interrupted", limit: this.maxDispatchPerCycle })).sort((a, b) => a.createdAt === b.createdAt ? a.id < b.id ? 1 : -1 : a.createdAt < b.createdAt ? 1 : -1).slice(0, this.maxDispatchPerCycle);
|
|
1894
|
+
for (const run2 of pending) {
|
|
1895
|
+
if (this.isolatedRunId !== null && run2.id !== this.isolatedRunId)
|
|
1411
1896
|
continue;
|
|
1412
|
-
const lease = this.claim(
|
|
1897
|
+
const lease = this.claim(run2.id);
|
|
1413
1898
|
if (!lease)
|
|
1414
1899
|
continue;
|
|
1415
|
-
this.store.setRunStatus(
|
|
1416
|
-
this.wal.append({ op: "run_started", runId:
|
|
1900
|
+
await this.store.setRunStatus(run2.id, "running");
|
|
1901
|
+
this.wal.append({ op: "run_started", runId: run2.id, at: nowIso() });
|
|
1417
1902
|
report.dispatched++;
|
|
1418
1903
|
}
|
|
1419
|
-
const running = this.store.listRuns({ status: "running", limit: this.maxStepsPerCycle });
|
|
1904
|
+
const running = await this.store.listRuns({ status: "running", limit: this.maxStepsPerCycle });
|
|
1420
1905
|
let steps = 0;
|
|
1421
|
-
for (const
|
|
1422
|
-
if (this.isolatedRunId !== null &&
|
|
1906
|
+
for (const run2 of running) {
|
|
1907
|
+
if (this.isolatedRunId !== null && run2.id !== this.isolatedRunId)
|
|
1423
1908
|
continue;
|
|
1424
1909
|
if (steps >= this.maxStepsPerCycle)
|
|
1425
1910
|
break;
|
|
1426
|
-
const outcome = await this.advanceRun(
|
|
1911
|
+
const outcome = await this.advanceRun(run2.id);
|
|
1427
1912
|
steps++;
|
|
1428
1913
|
if (outcome.kind === "completed")
|
|
1429
1914
|
report.completed++;
|
|
@@ -1449,66 +1934,78 @@ class WorkflowsDaemon {
|
|
|
1449
1934
|
} else {
|
|
1450
1935
|
this.graphCache.set(graph.name, existing);
|
|
1451
1936
|
}
|
|
1452
|
-
|
|
1937
|
+
const initial = context !== null && typeof context === "object" ? context : {};
|
|
1938
|
+
return this.store.createRun({
|
|
1939
|
+
graphName: graph.name,
|
|
1940
|
+
graphVersion: graph.version,
|
|
1941
|
+
context: { ...initial, __wf: { ...initial.__wf ?? {}, graph } }
|
|
1942
|
+
});
|
|
1453
1943
|
}
|
|
1454
1944
|
async advanceRun(runId) {
|
|
1455
|
-
const
|
|
1456
|
-
if (!
|
|
1945
|
+
const run2 = await this.store.getRun(runId);
|
|
1946
|
+
if (!run2 || run2.status !== "running")
|
|
1457
1947
|
return { kind: "noop" };
|
|
1458
1948
|
const lease = this.leases.get(runId);
|
|
1459
1949
|
if (!lease || lease.expiresAtMs <= this.time())
|
|
1460
1950
|
return { kind: "blocked" };
|
|
1461
|
-
const
|
|
1951
|
+
const storedContext = JSON.parse(run2.contextJson);
|
|
1952
|
+
const pinned = storedContext.__wf?.graph;
|
|
1953
|
+
const graph = pinned ?? this.loadGraph(run2.graphName);
|
|
1462
1954
|
if (!graph) {
|
|
1463
|
-
this.failRun(
|
|
1955
|
+
await this.failRun(run2, `graph ${run2.graphName} not found in daemon graph cache`);
|
|
1464
1956
|
return { kind: "failed", error: "graph not found" };
|
|
1465
1957
|
}
|
|
1466
|
-
const context =
|
|
1958
|
+
const context = storedContext;
|
|
1467
1959
|
const rawWf = context.__wf ?? {};
|
|
1468
1960
|
const wf = {
|
|
1469
1961
|
...rawWf,
|
|
1470
1962
|
cursor: typeof rawWf.cursor === "string" ? rawWf.cursor : undefined,
|
|
1471
|
-
loops: rawWf.loops ?? {},
|
|
1472
|
-
completedLoops: rawWf.completedLoops ?? {}
|
|
1963
|
+
loops: Object.assign(Object.create(null), rawWf.loops ?? {}),
|
|
1964
|
+
completedLoops: Object.assign(Object.create(null), rawWf.completedLoops ?? {})
|
|
1473
1965
|
};
|
|
1474
1966
|
try {
|
|
1967
|
+
if (rawWf.stepTerminal === true) {
|
|
1968
|
+
context.__wf = wf;
|
|
1969
|
+
await this.completeRun(run2, context);
|
|
1970
|
+
return { kind: "completed" };
|
|
1971
|
+
}
|
|
1475
1972
|
const cursor = wf.cursor;
|
|
1476
|
-
const
|
|
1477
|
-
if (!
|
|
1478
|
-
this.failRun(
|
|
1973
|
+
const node2 = cursor === undefined ? graph.nodes.find((n) => n.type === "start") : findNode(graph, cursor);
|
|
1974
|
+
if (!node2) {
|
|
1975
|
+
await this.failRun(run2, `cursor ${cursor ?? "<start>"} resolves to no node`);
|
|
1479
1976
|
return { kind: "failed", error: "no node at cursor" };
|
|
1480
1977
|
}
|
|
1481
|
-
switch (
|
|
1978
|
+
switch (node2.type) {
|
|
1482
1979
|
case "start": {
|
|
1483
|
-
wf.cursor =
|
|
1484
|
-
this.persistContext(
|
|
1485
|
-
return { kind: "advanced", nodeId:
|
|
1980
|
+
wf.cursor = node2.next;
|
|
1981
|
+
await this.persistContext(run2, context, wf);
|
|
1982
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1486
1983
|
}
|
|
1487
1984
|
case "step": {
|
|
1488
|
-
return await this.executeStep(
|
|
1985
|
+
return await this.executeStep(run2, graph, context, wf, node2);
|
|
1489
1986
|
}
|
|
1490
1987
|
case "decision": {
|
|
1491
|
-
const value = evaluateExpr(
|
|
1492
|
-
const next = value ?
|
|
1988
|
+
const value = evaluateExpr(node2.condition, this.conditionScope(context));
|
|
1989
|
+
const next = value ? node2.then : node2.else;
|
|
1493
1990
|
if (!next) {
|
|
1494
|
-
this.failRun(
|
|
1991
|
+
await this.failRun(run2, `decision ${node2.id} evaluated ${value} but no ${value ? "then" : "else"} edge`);
|
|
1495
1992
|
return { kind: "failed", error: "missing decision edge" };
|
|
1496
1993
|
}
|
|
1497
1994
|
wf.cursor = next;
|
|
1498
|
-
this.persistContext(
|
|
1499
|
-
return { kind: "advanced", nodeId:
|
|
1995
|
+
await this.persistContext(run2, context, wf);
|
|
1996
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1500
1997
|
}
|
|
1501
1998
|
case "while": {
|
|
1502
|
-
return this.executeWhile(
|
|
1999
|
+
return await this.executeWhile(run2, graph, context, wf, node2);
|
|
1503
2000
|
}
|
|
1504
2001
|
case "end": {
|
|
1505
|
-
this.completeRun(
|
|
2002
|
+
await this.completeRun(run2, context);
|
|
1506
2003
|
return { kind: "completed" };
|
|
1507
2004
|
}
|
|
1508
2005
|
}
|
|
1509
2006
|
} catch (err) {
|
|
1510
2007
|
if (err instanceof ExprEvalError || err instanceof ExprSyntaxError) {
|
|
1511
|
-
this.failRun(
|
|
2008
|
+
await this.failRun(run2, `condition error at ${wf.cursor ?? "<start>"}: ${err.message}`);
|
|
1512
2009
|
return { kind: "failed", error: err.message };
|
|
1513
2010
|
}
|
|
1514
2011
|
throw err;
|
|
@@ -1520,131 +2017,180 @@ class WorkflowsDaemon {
|
|
|
1520
2017
|
conditionScope(context) {
|
|
1521
2018
|
return { ...context, steps: context.steps ?? {} };
|
|
1522
2019
|
}
|
|
1523
|
-
persistContext(
|
|
2020
|
+
async persistContext(run2, context, wf) {
|
|
1524
2021
|
context.__wf = wf;
|
|
1525
|
-
this.store.
|
|
1526
|
-
}
|
|
1527
|
-
async
|
|
1528
|
-
const
|
|
1529
|
-
const
|
|
2022
|
+
await this.store.setRunningContext(run2.id, context);
|
|
2023
|
+
}
|
|
2024
|
+
async discardStep(run2, nodeRowId) {
|
|
2025
|
+
const current = await this.store.getRun(run2.id);
|
|
2026
|
+
const node2 = await this.store.getRunNode(nodeRowId);
|
|
2027
|
+
if (current?.status === "cancelled" && node2 && ["pending", "running"].includes(node2.status))
|
|
2028
|
+
await this.store.setRunNodeStatus(nodeRowId, "skipped", { error: "run cancelled", finishedAt: nowIso() });
|
|
2029
|
+
return { kind: "noop" };
|
|
2030
|
+
}
|
|
2031
|
+
async executeStep(run2, graph, context, wf, node2) {
|
|
2032
|
+
const nodeId = node2.id;
|
|
2033
|
+
const latest = await this.store.getLatestRunNode(run2.id, nodeId);
|
|
1530
2034
|
const isLoopBody = graph.nodes.some((candidate) => candidate.type === "while" && candidate.body.includes(nodeId));
|
|
1531
|
-
const nodeRow = latest === undefined || isLoopBody && latest.status === "completed" ? this.store.createRunNode({ runId:
|
|
1532
|
-
const memoWatch =
|
|
1533
|
-
const memoSource = { command:
|
|
1534
|
-
const memoInput =
|
|
2035
|
+
const nodeRow = latest === undefined || isLoopBody && latest.status === "completed" ? await this.store.createRunNode({ runId: run2.id, nodeId, lane: node2.lane ?? "claude" }) : latest;
|
|
2036
|
+
const memoWatch = node2.memo ? memoWatchFingerprint(graph, this.store.dataDir) : undefined;
|
|
2037
|
+
const memoSource = { command: node2.command, prompt: node2.prompt, context: stripRunState(context) };
|
|
2038
|
+
const memoInput = node2.memo ? memoWatch !== undefined ? { ...memoSource, memoWatch } : memoSource : undefined;
|
|
1535
2039
|
if (memoInput !== undefined) {
|
|
1536
|
-
const hit = tryMemoHit(this.store,
|
|
2040
|
+
const hit = await tryMemoHit(this.store, run2.graphName, nodeId, memoInput);
|
|
1537
2041
|
if (hit) {
|
|
1538
2042
|
const cached = JSON.parse(hit.outputJson);
|
|
1539
|
-
this.store.setRunNodeStatus(nodeRow.id, "completed", { exitCode: Number(cached.exitCode ?? 0), output: { ...cached, memoHit: true } });
|
|
1540
|
-
this.wal.append({ op: "node_finished", runId: run.id, nodeId, status: "completed", at: nowIso() });
|
|
1541
2043
|
this.recordStepResult(context, nodeId, { ...cached, memoHit: true });
|
|
1542
|
-
|
|
2044
|
+
const continuation2 = this.advanceAfterStep(graph, context, wf, node2.next);
|
|
2045
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "completed", patch: { exitCode: Number(cached.exitCode ?? 0), output: { ...cached, memoHit: true } }, context }))
|
|
2046
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2047
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "completed", at: nowIso() });
|
|
2048
|
+
if (continuation2.kind === "completed")
|
|
2049
|
+
await this.completeRun(run2, context);
|
|
2050
|
+
return continuation2;
|
|
1543
2051
|
}
|
|
1544
2052
|
}
|
|
1545
2053
|
if (nodeRow.status !== "running") {
|
|
1546
|
-
this.store.
|
|
1547
|
-
|
|
2054
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "running" }))
|
|
2055
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2056
|
+
this.wal.append({ op: "node_started", runId: run2.id, nodeId, at: nowIso() });
|
|
1548
2057
|
}
|
|
1549
2058
|
const job = {
|
|
1550
|
-
lane:
|
|
1551
|
-
prompt:
|
|
1552
|
-
command:
|
|
1553
|
-
timeoutMs:
|
|
2059
|
+
lane: node2.lane ?? "claude",
|
|
2060
|
+
prompt: node2.prompt,
|
|
2061
|
+
command: node2.command,
|
|
2062
|
+
timeoutMs: node2.timeoutMs,
|
|
1554
2063
|
env: this.env
|
|
1555
2064
|
};
|
|
1556
2065
|
let result;
|
|
2066
|
+
const activeLease = this.leases.get(run2.id);
|
|
2067
|
+
const pulse = setInterval(() => {
|
|
2068
|
+
this.heartbeat(run2.id, activeLease.fencing);
|
|
2069
|
+
}, Math.max(1, Math.floor(this.leaseTtlMs / 3)));
|
|
1557
2070
|
try {
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
2071
|
+
const dispatched = await this.store.dispatchIfRunning(run2.id, async () => {
|
|
2072
|
+
try {
|
|
2073
|
+
return await this.laneRunner(job);
|
|
2074
|
+
} catch (err) {
|
|
2075
|
+
return { ok: false, exitCode: 1, output: "", durationMs: 0, error: String(err instanceof Error ? err.message : err) };
|
|
2076
|
+
}
|
|
2077
|
+
});
|
|
2078
|
+
if (!dispatched.started) {
|
|
2079
|
+
await this.store.finishDispatch(run2.id, false);
|
|
2080
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2081
|
+
}
|
|
2082
|
+
result = await dispatched.result;
|
|
2083
|
+
} finally {
|
|
2084
|
+
clearInterval(pulse);
|
|
2085
|
+
}
|
|
2086
|
+
const unconfirmed = result.retryable === false && /exceeded.*budget/.test(result.error ?? "");
|
|
2087
|
+
await this.store.finishDispatch(run2.id, unconfirmed);
|
|
2088
|
+
if (unconfirmed)
|
|
2089
|
+
Object.assign(wf, { resumeBlocked: "execution cancellation has not been confirmed" });
|
|
2090
|
+
const current = await this.store.getRun(run2.id);
|
|
2091
|
+
const currentClaim = this.wal.replay({ repair: false }).liveClaims(this.time()).get(run2.id);
|
|
2092
|
+
const lease = this.leases.get(run2.id);
|
|
2093
|
+
if (!current || current.status !== "running" || !currentClaim || !lease || currentClaim.worker !== lease.worker || currentClaim.fencing !== lease.fencing) {
|
|
2094
|
+
if (current?.status === "cancelled")
|
|
2095
|
+
await this.store.setRunNodeStatus(nodeRow.id, "skipped", { error: "run cancelled", finishedAt: nowIso() });
|
|
2096
|
+
return { kind: "noop" };
|
|
2097
|
+
}
|
|
2098
|
+
try {
|
|
2099
|
+
assertNoSecrets({ output: result.output, error: result.error }, `node ${nodeId} execution`);
|
|
2100
|
+
} catch {
|
|
2101
|
+
result = { ok: false, exitCode: 1, output: "", durationMs: result.durationMs, error: "execution result refused by secrets write-gate", retryable: false };
|
|
1561
2102
|
}
|
|
1562
2103
|
const output = { ok: result.ok, exitCode: result.exitCode, output: result.output, durationMs: result.durationMs };
|
|
1563
2104
|
if (!result.ok) {
|
|
1564
|
-
const shouldRetry = nodeRow.attempts < (
|
|
1565
|
-
this.store.bumpAttemptsNode(nodeRow.id);
|
|
2105
|
+
const shouldRetry = result.retryable !== false && nodeRow.attempts < (node2.maxRetries ?? 0);
|
|
1566
2106
|
if (shouldRetry) {
|
|
1567
|
-
this.store.
|
|
2107
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "pending", patch: { error: result.error ?? `exit ${result.exitCode}` }, incrementAttempts: true }))
|
|
2108
|
+
return this.discardStep(run2, nodeRow.id);
|
|
1568
2109
|
return { kind: "advanced", nodeId };
|
|
1569
2110
|
}
|
|
1570
2111
|
const error = result.error ?? `step ${nodeId} failed with exit ${result.exitCode}`;
|
|
1571
|
-
this.store.
|
|
1572
|
-
|
|
1573
|
-
this.
|
|
2112
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "failed", patch: { exitCode: result.exitCode, error }, incrementAttempts: true }))
|
|
2113
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2114
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "failed", at: nowIso() });
|
|
2115
|
+
await this.failRun(run2, error);
|
|
1574
2116
|
return { kind: "failed", error };
|
|
1575
2117
|
}
|
|
1576
2118
|
try {
|
|
1577
2119
|
assertNoSecrets(output, `node ${nodeId} output`);
|
|
1578
2120
|
} catch (err) {
|
|
1579
2121
|
const message = err instanceof Error ? err.message : String(err);
|
|
1580
|
-
this.store.
|
|
1581
|
-
|
|
1582
|
-
this.
|
|
2122
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "failed", patch: { error: message } }))
|
|
2123
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2124
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "failed", at: nowIso() });
|
|
2125
|
+
await this.failRun(run2, message);
|
|
1583
2126
|
return { kind: "failed", error: message };
|
|
1584
2127
|
}
|
|
1585
|
-
|
|
1586
|
-
this.wal.append({ op: "node_finished", runId: run.id, nodeId, status: "completed", at: nowIso() });
|
|
2128
|
+
let memo2;
|
|
1587
2129
|
if (memoInput !== undefined) {
|
|
1588
|
-
const memoInputAfter =
|
|
2130
|
+
const memoInputAfter = node2.memo ? memoWatchFingerprint(graph, this.store.dataDir) : undefined;
|
|
1589
2131
|
const stable = memoWatch === undefined && memoInputAfter === undefined || memoWatch !== undefined && memoInputAfter !== undefined && JSON.stringify(memoWatch) === JSON.stringify(memoInputAfter);
|
|
1590
2132
|
if (stable) {
|
|
1591
|
-
|
|
1592
|
-
this.wal.append({ op: "memo_set", key: memoKey(run.graphName, nodeId, memoInput), at: nowIso() });
|
|
2133
|
+
memo2 = { key: memoKey(run2.graphName, nodeId, memoInput), graphName: run2.graphName, nodeId, inputHash: inputHash(memoInput), outputJson: JSON.stringify(output) };
|
|
1593
2134
|
}
|
|
1594
2135
|
}
|
|
1595
2136
|
this.recordStepResult(context, nodeId, output);
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
2137
|
+
const continuation = this.advanceAfterStep(graph, context, wf, node2.next);
|
|
2138
|
+
if (!await this.store.commitStep({ runId: run2.id, nodeRowId: nodeRow.id, status: "completed", patch: { exitCode: result.exitCode, output }, context, memo: memo2 }))
|
|
2139
|
+
return this.discardStep(run2, nodeRow.id);
|
|
2140
|
+
this.wal.append({ op: "node_finished", runId: run2.id, nodeId, status: "completed", at: nowIso() });
|
|
2141
|
+
if (memo2)
|
|
2142
|
+
this.wal.append({ op: "memo_set", key: memo2.key, at: nowIso() });
|
|
2143
|
+
if (continuation.kind === "completed")
|
|
2144
|
+
await this.completeRun(run2, context);
|
|
2145
|
+
return continuation;
|
|
2146
|
+
}
|
|
2147
|
+
advanceAfterStep(graph, context, wf, next) {
|
|
2148
|
+
context.__wf = wf;
|
|
1599
2149
|
if (next) {
|
|
1600
2150
|
wf.cursor = next;
|
|
1601
|
-
this.persistContext(run, context, wf);
|
|
1602
2151
|
return { kind: "advanced", nodeId: wf.cursor };
|
|
1603
2152
|
}
|
|
1604
2153
|
const cursor = wf.cursor;
|
|
1605
|
-
const graph = this.loadGraph(run.graphName);
|
|
1606
2154
|
const whileNode = graph.nodes.find((n) => n.type === "while" && n.body.includes(cursor ?? ""));
|
|
1607
2155
|
if (whileNode) {
|
|
1608
2156
|
const index = whileNode.body.indexOf(cursor ?? "");
|
|
1609
2157
|
if (index >= 0 && index < whileNode.body.length - 1) {
|
|
1610
2158
|
wf.cursor = whileNode.body[index + 1];
|
|
1611
|
-
this.persistContext(run, context, wf);
|
|
1612
2159
|
return { kind: "advanced", nodeId: wf.cursor };
|
|
1613
2160
|
}
|
|
1614
2161
|
wf.loops[whileNode.id] = (wf.loops[whileNode.id] ?? 0) + 1;
|
|
1615
2162
|
wf.cursor = whileNode.id;
|
|
1616
|
-
this.persistContext(run, context, wf);
|
|
1617
2163
|
return { kind: "advanced", nodeId: whileNode.id };
|
|
1618
2164
|
}
|
|
1619
|
-
|
|
2165
|
+
wf.stepTerminal = true;
|
|
1620
2166
|
return { kind: "completed" };
|
|
1621
2167
|
}
|
|
1622
|
-
executeWhile(
|
|
1623
|
-
const iteration = wf.loops[
|
|
1624
|
-
if (iteration >=
|
|
1625
|
-
const error = `while node ${
|
|
1626
|
-
this.failRun(
|
|
2168
|
+
async executeWhile(run2, graph, context, wf, node2) {
|
|
2169
|
+
const iteration = wf.loops[node2.id] ?? 0;
|
|
2170
|
+
if (iteration >= node2.maxIterations) {
|
|
2171
|
+
const error = `while node ${node2.id} exceeded maxIterations (${node2.maxIterations})`;
|
|
2172
|
+
await this.failRun(run2, error);
|
|
1627
2173
|
return { kind: "failed", error };
|
|
1628
2174
|
}
|
|
1629
2175
|
const scope = { ...this.conditionScope(context), i: iteration };
|
|
1630
|
-
const keepGoing = evaluateExpr(
|
|
2176
|
+
const keepGoing = evaluateExpr(node2.condition, scope);
|
|
1631
2177
|
if (!keepGoing) {
|
|
1632
|
-
wf.completedLoops[
|
|
1633
|
-
delete wf.loops[
|
|
1634
|
-
wf.cursor =
|
|
1635
|
-
this.persistContext(
|
|
1636
|
-
return { kind: "advanced", nodeId:
|
|
2178
|
+
wf.completedLoops[node2.id] = iteration;
|
|
2179
|
+
delete wf.loops[node2.id];
|
|
2180
|
+
wf.cursor = node2.next;
|
|
2181
|
+
await this.persistContext(run2, context, wf);
|
|
2182
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1637
2183
|
}
|
|
1638
|
-
wf.cursor =
|
|
1639
|
-
this.persistContext(
|
|
1640
|
-
return { kind: "advanced", nodeId:
|
|
2184
|
+
wf.cursor = node2.body[0];
|
|
2185
|
+
await this.persistContext(run2, context, wf);
|
|
2186
|
+
return { kind: "advanced", nodeId: node2.id };
|
|
1641
2187
|
}
|
|
1642
2188
|
recordStepResult(context, nodeId, output) {
|
|
1643
|
-
const steps = context.steps ?? {};
|
|
2189
|
+
const steps = Object.assign(Object.create(null), context.steps ?? {});
|
|
1644
2190
|
steps[nodeId] = output;
|
|
1645
2191
|
context.steps = steps;
|
|
1646
2192
|
}
|
|
1647
|
-
completeRun(
|
|
2193
|
+
async completeRun(run2, context) {
|
|
1648
2194
|
const result = {
|
|
1649
2195
|
steps: context.steps ?? {},
|
|
1650
2196
|
iterations: context.__wf?.completedLoops ?? {}
|
|
@@ -1653,21 +2199,23 @@ class WorkflowsDaemon {
|
|
|
1653
2199
|
assertNoSecrets(result, "run result");
|
|
1654
2200
|
} catch (err) {
|
|
1655
2201
|
const message = err instanceof Error ? err.message : String(err);
|
|
1656
|
-
this.failRun(
|
|
2202
|
+
await this.failRun(run2, message);
|
|
1657
2203
|
return;
|
|
1658
2204
|
}
|
|
1659
|
-
this.store.
|
|
1660
|
-
|
|
1661
|
-
|
|
2205
|
+
if (!await this.store.compareRunStatus(run2.id, ["running"], "completed", { result, finishedAt: nowIso(), error: null }))
|
|
2206
|
+
return;
|
|
2207
|
+
this.wal.append({ op: "run_finished", runId: run2.id, status: "completed", at: nowIso() });
|
|
2208
|
+
const lease = this.leases.get(run2.id);
|
|
1662
2209
|
if (lease)
|
|
1663
|
-
this.release(
|
|
2210
|
+
this.release(run2.id, lease.fencing);
|
|
1664
2211
|
}
|
|
1665
|
-
failRun(
|
|
1666
|
-
this.store.
|
|
1667
|
-
|
|
1668
|
-
|
|
2212
|
+
async failRun(run2, error) {
|
|
2213
|
+
if (!await this.store.compareRunStatus(run2.id, ["running"], "failed", { error, finishedAt: nowIso() }))
|
|
2214
|
+
return;
|
|
2215
|
+
this.wal.append({ op: "run_finished", runId: run2.id, status: "failed", at: nowIso() });
|
|
2216
|
+
const lease = this.leases.get(run2.id);
|
|
1669
2217
|
if (lease)
|
|
1670
|
-
this.release(
|
|
2218
|
+
this.release(run2.id, lease.fencing);
|
|
1671
2219
|
}
|
|
1672
2220
|
}
|
|
1673
2221
|
var defaultLaneRunner = async (job) => {
|
|
@@ -1678,33 +2226,109 @@ var defaultLaneRunner = async (job) => {
|
|
|
1678
2226
|
};
|
|
1679
2227
|
async function runGraphToCompletion(store, wal, graph, context, options = {}) {
|
|
1680
2228
|
const daemon = new WorkflowsDaemon(store, wal, {
|
|
1681
|
-
worker:
|
|
2229
|
+
worker: `foreground-${randomUUID2()}`,
|
|
1682
2230
|
laneRunner: options.laneRunner,
|
|
1683
2231
|
time: options.time,
|
|
1684
2232
|
env: options.env,
|
|
1685
2233
|
isolatedRunId: options.isolatedRunId
|
|
1686
2234
|
});
|
|
1687
|
-
const
|
|
1688
|
-
|
|
1689
|
-
|
|
2235
|
+
const run2 = options.existingRunId ? await store.getRun(options.existingRunId) : await daemon.startRun(graph, context);
|
|
2236
|
+
if (!run2)
|
|
2237
|
+
throw new Error("run not found");
|
|
2238
|
+
options.onRunCreated?.(run2);
|
|
2239
|
+
daemon.isolateRun(run2.id);
|
|
1690
2240
|
const maxCycles = options.maxCycles ?? 500;
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
2241
|
+
const hosted = store instanceof PostgresWorkflowsStore ? store : undefined;
|
|
2242
|
+
if (hosted && !await hosted.acquireExecution(run2.id))
|
|
2243
|
+
return await store.getRun(run2.id);
|
|
2244
|
+
const lease = daemon.claim(run2.id);
|
|
2245
|
+
if (!lease) {
|
|
2246
|
+
await hosted?.releaseExecution();
|
|
2247
|
+
return await store.getRun(run2.id);
|
|
2248
|
+
}
|
|
2249
|
+
try {
|
|
2250
|
+
if (!await store.compareRunStatus(run2.id, ["pending"], "running", { startedAt: nowIso() }))
|
|
2251
|
+
return await store.getRun(run2.id);
|
|
2252
|
+
wal.append({ op: "run_started", runId: run2.id, at: nowIso() });
|
|
2253
|
+
for (let cycle = 0;cycle < maxCycles; cycle++) {
|
|
2254
|
+
daemon.heartbeat(run2.id, lease.fencing);
|
|
2255
|
+
const outcome = await daemon.advanceRun(run2.id);
|
|
2256
|
+
const current = await store.getRun(run2.id);
|
|
2257
|
+
if (current.status !== "running" || outcome.kind === "blocked" || outcome.kind === "noop")
|
|
2258
|
+
return current;
|
|
2259
|
+
}
|
|
2260
|
+
await store.compareRunStatus(run2.id, ["running"], "interrupted", { error: "run reached maxCycles; resume to continue" });
|
|
2261
|
+
return await store.getRun(run2.id);
|
|
2262
|
+
} finally {
|
|
2263
|
+
daemon.release(run2.id, lease.fencing);
|
|
2264
|
+
await hosted?.releaseExecution();
|
|
1699
2265
|
}
|
|
1700
|
-
return store.getRun(run.id);
|
|
1701
2266
|
}
|
|
1702
2267
|
|
|
1703
2268
|
// src/paths.ts
|
|
1704
2269
|
import { existsSync } from "fs";
|
|
1705
2270
|
import { homedir } from "os";
|
|
1706
2271
|
import { join as join2 } from "path";
|
|
1707
|
-
import {
|
|
2272
|
+
import { homedir as pathsResolverHomedir } from "os";
|
|
2273
|
+
import { join as pathsResolverJoin } from "path";
|
|
2274
|
+
var PATHS_RESOLVER_KIND_ENV = {
|
|
2275
|
+
config: "HASNA_CONFIG_HOME",
|
|
2276
|
+
data: "HASNA_DATA_HOME",
|
|
2277
|
+
state: "HASNA_STATE_HOME",
|
|
2278
|
+
cache: "HASNA_CACHE_HOME"
|
|
2279
|
+
};
|
|
2280
|
+
var PATHS_RESOLVER_APP_SLUG_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
2281
|
+
function pathsResolverAssertApp(app) {
|
|
2282
|
+
if (typeof app !== "string" || app.length === 0) {
|
|
2283
|
+
throw new TypeError("paths: app must be a non-empty string");
|
|
2284
|
+
}
|
|
2285
|
+
if (!PATHS_RESOLVER_APP_SLUG_RE.test(app)) {
|
|
2286
|
+
throw new TypeError(`paths: invalid app slug "${app}" \u2014 expected lowercase kebab-case ([a-z0-9]+(-[a-z0-9]+)*)`);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
function pathsResolverAssertKind(kind) {
|
|
2290
|
+
if (!Object.keys(PATHS_RESOLVER_KIND_ENV).includes(kind)) {
|
|
2291
|
+
throw new TypeError(`paths: invalid path kind "${kind}" \u2014 expected one of ${Object.keys(PATHS_RESOLVER_KIND_ENV).join(", ")}`);
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
function pathsResolverBaseDir(kind, options) {
|
|
2295
|
+
pathsResolverAssertKind(kind);
|
|
2296
|
+
const env = options.env ?? process.env;
|
|
2297
|
+
const override = env[PATHS_RESOLVER_KIND_ENV[kind]];
|
|
2298
|
+
if (typeof override === "string" && override.length > 0)
|
|
2299
|
+
return override;
|
|
2300
|
+
const home = options.home ?? pathsResolverHomedir();
|
|
2301
|
+
const platform = options.platform ?? process.platform;
|
|
2302
|
+
if (platform === "darwin") {
|
|
2303
|
+
switch (kind) {
|
|
2304
|
+
case "config":
|
|
2305
|
+
case "data":
|
|
2306
|
+
return pathsResolverJoin(home, "Library", "Application Support", "Hasna");
|
|
2307
|
+
case "cache":
|
|
2308
|
+
return pathsResolverJoin(home, "Library", "Caches", "Hasna");
|
|
2309
|
+
case "state":
|
|
2310
|
+
return pathsResolverJoin(home, "Library", "Logs", "Hasna");
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
switch (kind) {
|
|
2314
|
+
case "config":
|
|
2315
|
+
return pathsResolverJoin(home, ".config", "hasna");
|
|
2316
|
+
case "data":
|
|
2317
|
+
return pathsResolverJoin(home, ".local", "share", "hasna");
|
|
2318
|
+
case "state":
|
|
2319
|
+
return pathsResolverJoin(home, ".local", "state", "hasna");
|
|
2320
|
+
case "cache":
|
|
2321
|
+
return pathsResolverJoin(home, ".cache", "hasna");
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
function pathsResolverResolve(kind, options) {
|
|
2325
|
+
pathsResolverAssertApp(options.app);
|
|
2326
|
+
const appSegment = options.internal === true ? pathsResolverJoin("internal", options.app) : options.app;
|
|
2327
|
+
return pathsResolverJoin(pathsResolverBaseDir(kind, options), appSegment);
|
|
2328
|
+
}
|
|
2329
|
+
function dataDir(options) {
|
|
2330
|
+
return pathsResolverResolve("data", options);
|
|
2331
|
+
}
|
|
1708
2332
|
var APP = "workflows";
|
|
1709
2333
|
var STORE_DB_FILE = "workflows.db";
|
|
1710
2334
|
function getHomeDir(env = process.env) {
|
|
@@ -1713,8 +2337,8 @@ function getHomeDir(env = process.env) {
|
|
|
1713
2337
|
function getLegacyDataDir(env = process.env) {
|
|
1714
2338
|
return join2(getHomeDir(env), ".hasna", APP);
|
|
1715
2339
|
}
|
|
1716
|
-
function getResolverDataDir(env = process.env) {
|
|
1717
|
-
return dataDir({ app: APP, home: getHomeDir(env), env });
|
|
2340
|
+
function getResolverDataDir(env = process.env, platform = process.platform) {
|
|
2341
|
+
return dataDir({ app: APP, home: getHomeDir(env), env, platform });
|
|
1718
2342
|
}
|
|
1719
2343
|
function adoptResolverDataDir(resolved, env = process.env) {
|
|
1720
2344
|
const dataOverride = env.HASNA_DATA_HOME;
|
|
@@ -1731,7 +2355,8 @@ function getEffectiveDataDir(env = process.env) {
|
|
|
1731
2355
|
import { Database } from "bun:sqlite";
|
|
1732
2356
|
import { mkdirSync } from "fs";
|
|
1733
2357
|
import { join as join3 } from "path";
|
|
1734
|
-
import { randomUUID } from "crypto";
|
|
2358
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
2359
|
+
var RUN_STATUSES = ["pending", "running", "completed", "failed", "cancelled", "interrupted"];
|
|
1735
2360
|
function nowIso2() {
|
|
1736
2361
|
return new Date().toISOString();
|
|
1737
2362
|
}
|
|
@@ -1801,6 +2426,8 @@ function openStore(dataDir2) {
|
|
|
1801
2426
|
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
1802
2427
|
created_at TEXT NOT NULL
|
|
1803
2428
|
);
|
|
2429
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_runs_idempotency ON runs(json_extract(context_json, '$.__wf.idempotencyKey'))
|
|
2430
|
+
WHERE json_type(context_json, '$.__wf.idempotencyKey') = 'text';
|
|
1804
2431
|
CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status);
|
|
1805
2432
|
CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(created_at);
|
|
1806
2433
|
CREATE INDEX IF NOT EXISTS idx_run_nodes_run ON run_nodes(run_id);
|
|
@@ -1851,7 +2478,7 @@ function openStore(dataDir2) {
|
|
|
1851
2478
|
return rows.map((r) => r.name);
|
|
1852
2479
|
},
|
|
1853
2480
|
createRun(input) {
|
|
1854
|
-
const id =
|
|
2481
|
+
const id = randomUUID3();
|
|
1855
2482
|
const now = nowIso2();
|
|
1856
2483
|
retryBusy(() => db.query(`INSERT INTO runs (id, graph_name, graph_version, status, attempts, context_json, created_at, updated_at)
|
|
1857
2484
|
VALUES (?, ?, ?, 'pending', 0, ?, ?, ?)`).run(id, input.graphName, input.graphVersion, JSON.stringify(input.context ?? {}), now, now));
|
|
@@ -1861,6 +2488,10 @@ function openStore(dataDir2) {
|
|
|
1861
2488
|
const row = retryBusy(() => db.query(`SELECT * FROM runs WHERE id = ?`).get(id));
|
|
1862
2489
|
return row ? rowToRun(row) : undefined;
|
|
1863
2490
|
},
|
|
2491
|
+
getRunByIdempotencyKey(key) {
|
|
2492
|
+
const row = retryBusy(() => db.query(`SELECT * FROM runs WHERE json_valid(context_json) AND json_extract(context_json, '$.__wf.idempotencyKey') = ? ORDER BY created_at DESC LIMIT 1`).get(key));
|
|
2493
|
+
return row ? rowToRun(row) : undefined;
|
|
2494
|
+
},
|
|
1864
2495
|
listRuns(opts = {}) {
|
|
1865
2496
|
const limit = Math.min(opts.limit ?? 100, 1000);
|
|
1866
2497
|
if (opts.status) {
|
|
@@ -1881,14 +2512,81 @@ function openStore(dataDir2) {
|
|
|
1881
2512
|
const finishedAt = patch.finishedAt !== undefined ? patch.finishedAt : terminal && existing?.finishedAt === null ? now : existing?.finishedAt ?? null;
|
|
1882
2513
|
retryBusy(() => db.query(`UPDATE runs SET status = ?, result_json = ?, error = ?, started_at = ?, finished_at = ?, updated_at = ? WHERE id = ?`).run(status, resultJson, error, startedAt, finishedAt, now, id));
|
|
1883
2514
|
},
|
|
2515
|
+
compareRunStatus(id, expected, status, patch = {}) {
|
|
2516
|
+
return retryBusy(() => db.transaction(() => {
|
|
2517
|
+
const current = this.getRun(id);
|
|
2518
|
+
if (!current || !expected.includes(current.status))
|
|
2519
|
+
return false;
|
|
2520
|
+
this.setRunStatus(id, status, patch);
|
|
2521
|
+
return true;
|
|
2522
|
+
}).immediate());
|
|
2523
|
+
},
|
|
1884
2524
|
bumpAttempts(id) {
|
|
1885
2525
|
retryBusy(() => db.query(`UPDATE runs SET attempts = attempts + 1, updated_at = ? WHERE id = ?`).run(nowIso2(), id));
|
|
1886
2526
|
},
|
|
1887
2527
|
setRunContext(id, context) {
|
|
1888
2528
|
retryBusy(() => db.query(`UPDATE runs SET context_json = ?, updated_at = ? WHERE id = ?`).run(JSON.stringify(context), nowIso2(), id));
|
|
1889
2529
|
},
|
|
2530
|
+
setRunningContext(id, context) {
|
|
2531
|
+
return retryBusy(() => db.transaction(() => {
|
|
2532
|
+
if (this.getRun(id)?.status !== "running")
|
|
2533
|
+
return false;
|
|
2534
|
+
this.setRunContext(id, context);
|
|
2535
|
+
return true;
|
|
2536
|
+
}).immediate());
|
|
2537
|
+
},
|
|
2538
|
+
blockRunResume(id) {
|
|
2539
|
+
retryBusy(() => db.transaction(() => {
|
|
2540
|
+
const run2 = this.getRun(id);
|
|
2541
|
+
if (!run2)
|
|
2542
|
+
return;
|
|
2543
|
+
const context = JSON.parse(run2.contextJson);
|
|
2544
|
+
context.__wf = { ...context.__wf, resumeBlocked: "execution cancellation has not been confirmed" };
|
|
2545
|
+
this.setRunContext(id, context);
|
|
2546
|
+
}).immediate());
|
|
2547
|
+
},
|
|
2548
|
+
dispatchIfRunning(id, action) {
|
|
2549
|
+
return retryBusy(() => db.transaction(() => {
|
|
2550
|
+
const run2 = this.getRun(id);
|
|
2551
|
+
if (run2?.status !== "running")
|
|
2552
|
+
return { started: false };
|
|
2553
|
+
const context = JSON.parse(run2.contextJson);
|
|
2554
|
+
context.__wf = { ...context.__wf, executionMayBeActive: true };
|
|
2555
|
+
this.setRunContext(id, context);
|
|
2556
|
+
return { started: true, result: action() };
|
|
2557
|
+
}).immediate());
|
|
2558
|
+
},
|
|
2559
|
+
finishDispatch(id, unconfirmed) {
|
|
2560
|
+
retryBusy(() => db.transaction(() => {
|
|
2561
|
+
const run2 = this.getRun(id);
|
|
2562
|
+
if (!run2)
|
|
2563
|
+
return;
|
|
2564
|
+
const context = JSON.parse(run2.contextJson);
|
|
2565
|
+
delete context.__wf?.executionMayBeActive;
|
|
2566
|
+
if (unconfirmed)
|
|
2567
|
+
context.__wf = { ...context.__wf, resumeBlocked: "execution cancellation has not been confirmed" };
|
|
2568
|
+
this.setRunContext(id, context);
|
|
2569
|
+
}).immediate());
|
|
2570
|
+
},
|
|
2571
|
+
commitStep(input) {
|
|
2572
|
+
return retryBusy(() => db.transaction(() => {
|
|
2573
|
+
if (this.getRun(input.runId)?.status !== "running")
|
|
2574
|
+
return false;
|
|
2575
|
+
const node2 = this.getRunNode(input.nodeRowId);
|
|
2576
|
+
if (!node2 || node2.runId !== input.runId)
|
|
2577
|
+
return false;
|
|
2578
|
+
this.setRunNodeStatus(input.nodeRowId, input.status, input.patch);
|
|
2579
|
+
if (input.incrementAttempts)
|
|
2580
|
+
this.bumpAttemptsNode(input.nodeRowId);
|
|
2581
|
+
if (input.context !== undefined)
|
|
2582
|
+
this.setRunContext(input.runId, input.context);
|
|
2583
|
+
if (input.memo)
|
|
2584
|
+
this.memoPut(input.memo.key, input.memo.graphName, input.memo.nodeId, input.memo.inputHash, input.memo.outputJson);
|
|
2585
|
+
return true;
|
|
2586
|
+
}).immediate());
|
|
2587
|
+
},
|
|
1890
2588
|
createRunNode(input) {
|
|
1891
|
-
const id =
|
|
2589
|
+
const id = randomUUID3();
|
|
1892
2590
|
retryBusy(() => db.query(`INSERT INTO run_nodes (id, run_id, node_id, status, attempts, lane)
|
|
1893
2591
|
VALUES (?, ?, ?, 'pending', 0, ?)`).run(id, input.runId, input.nodeId, input.lane ?? null));
|
|
1894
2592
|
return this.getRunNode(id);
|
|
@@ -1944,9 +2642,10 @@ function openStore(dataDir2) {
|
|
|
1944
2642
|
}
|
|
1945
2643
|
|
|
1946
2644
|
// src/wal.ts
|
|
2645
|
+
import { Database as Database2 } from "bun:sqlite";
|
|
1947
2646
|
import { createHash as createHash2 } from "crypto";
|
|
1948
|
-
import { appendFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, renameSync, truncateSync, writeFileSync } from "fs";
|
|
1949
|
-
import { join as join4 } from "path";
|
|
2647
|
+
import { appendFileSync, copyFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, renameSync, truncateSync, writeFileSync, constants } from "fs";
|
|
2648
|
+
import { basename, dirname, join as join4 } from "path";
|
|
1950
2649
|
var WAL_FILE_NAME = "session.wal";
|
|
1951
2650
|
var SESSIONS_DIR_NAME = "sessions";
|
|
1952
2651
|
var MAX_WAL_BYTES = 256 * 1024 * 1024;
|
|
@@ -1958,15 +2657,17 @@ function checksumOf(op) {
|
|
|
1958
2657
|
|
|
1959
2658
|
class SessionWAL {
|
|
1960
2659
|
filePath;
|
|
2660
|
+
journalLocked = false;
|
|
1961
2661
|
constructor(filePath) {
|
|
1962
2662
|
this.filePath = filePath;
|
|
1963
2663
|
}
|
|
1964
2664
|
static open(dataDir2) {
|
|
1965
|
-
mkdirSync2(join4(dataDir2, SESSIONS_DIR_NAME), { recursive: true });
|
|
2665
|
+
mkdirSync2(join4(dataDir2, SESSIONS_DIR_NAME), { recursive: true, mode: 448 });
|
|
1966
2666
|
const canonical = join4(dataDir2, SESSIONS_DIR_NAME, WAL_FILE_NAME);
|
|
1967
2667
|
const legacy = join4(dataDir2, WAL_FILE_NAME);
|
|
1968
2668
|
if (existsSync2(legacy) && !existsSync2(canonical)) {
|
|
1969
2669
|
try {
|
|
2670
|
+
preserveOriginal(legacy);
|
|
1970
2671
|
renameSync(legacy, canonical);
|
|
1971
2672
|
} catch {
|
|
1972
2673
|
return new SessionWAL(legacy);
|
|
@@ -1978,19 +2679,53 @@ class SessionWAL {
|
|
|
1978
2679
|
return new SessionWAL(filePath);
|
|
1979
2680
|
}
|
|
1980
2681
|
append(op) {
|
|
2682
|
+
return this.withJournalLock(() => this.appendLocked(op));
|
|
2683
|
+
}
|
|
2684
|
+
withJournalLock(operation) {
|
|
2685
|
+
if (this.journalLocked)
|
|
2686
|
+
return operation();
|
|
2687
|
+
const parent = dirname(this.filePath);
|
|
2688
|
+
const dataDir2 = basename(parent) === SESSIONS_DIR_NAME ? dirname(parent) : parent;
|
|
2689
|
+
mkdirSync2(dataDir2, { recursive: true, mode: 448 });
|
|
2690
|
+
const mutex = new Database2(join4(dataDir2, "workflows.db"));
|
|
2691
|
+
try {
|
|
2692
|
+
mutex.exec("PRAGMA busy_timeout = 5000");
|
|
2693
|
+
mutex.exec("BEGIN IMMEDIATE");
|
|
2694
|
+
this.journalLocked = true;
|
|
2695
|
+
try {
|
|
2696
|
+
const result = operation();
|
|
2697
|
+
mutex.exec("COMMIT");
|
|
2698
|
+
return result;
|
|
2699
|
+
} catch (error) {
|
|
2700
|
+
mutex.exec("ROLLBACK");
|
|
2701
|
+
throw error;
|
|
2702
|
+
} finally {
|
|
2703
|
+
this.journalLocked = false;
|
|
2704
|
+
}
|
|
2705
|
+
} finally {
|
|
2706
|
+
mutex.close();
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
exclusive(operation) {
|
|
2710
|
+
return this.withJournalLock(operation);
|
|
2711
|
+
}
|
|
2712
|
+
appendLocked(op) {
|
|
1981
2713
|
const at = op.at ?? new Date().toISOString();
|
|
1982
2714
|
const full = { ...op, at };
|
|
1983
2715
|
const nextSeq = this.nextSeq();
|
|
1984
2716
|
const entry = { seq: nextSeq, at, checksum: checksumOf(full), op: full };
|
|
1985
2717
|
appendFileSync(this.filePath, JSON.stringify(entry) + `
|
|
1986
|
-
`, "utf8");
|
|
2718
|
+
`, { encoding: "utf8", mode: 384 });
|
|
1987
2719
|
return entry;
|
|
1988
2720
|
}
|
|
1989
2721
|
nextSeq() {
|
|
1990
2722
|
const replay = this.replay();
|
|
1991
2723
|
return replay.entries.length + 1;
|
|
1992
2724
|
}
|
|
1993
|
-
replay() {
|
|
2725
|
+
replay(options = {}) {
|
|
2726
|
+
return this.withJournalLock(() => this.replayLocked(options));
|
|
2727
|
+
}
|
|
2728
|
+
replayLocked(options) {
|
|
1994
2729
|
const entries = [];
|
|
1995
2730
|
let torn = false;
|
|
1996
2731
|
let repaired = false;
|
|
@@ -2034,6 +2769,9 @@ class SessionWAL {
|
|
|
2034
2769
|
if (badOffset !== null) {
|
|
2035
2770
|
torn = true;
|
|
2036
2771
|
try {
|
|
2772
|
+
if (options.repair === false)
|
|
2773
|
+
return { entries, torn, repaired, liveClaims: (nowMs) => reconstructLiveClaims(entries, nowMs) };
|
|
2774
|
+
preserveOriginal(this.filePath);
|
|
2037
2775
|
truncateSync(this.filePath, badOffset);
|
|
2038
2776
|
repaired = true;
|
|
2039
2777
|
} catch {}
|
|
@@ -2073,13 +2811,22 @@ function reconstructLiveClaims(entries, nowMs = Date.now()) {
|
|
|
2073
2811
|
}
|
|
2074
2812
|
return claims;
|
|
2075
2813
|
}
|
|
2814
|
+
function preserveOriginal(filePath) {
|
|
2815
|
+
const original = readFileSync2(filePath);
|
|
2816
|
+
const digest = createHash2("sha256").update(original).digest("hex");
|
|
2817
|
+
const backup = `${filePath}.preserved-${digest}`;
|
|
2818
|
+
if (!existsSync2(backup))
|
|
2819
|
+
copyFileSync(filePath, backup, constants.COPYFILE_EXCL);
|
|
2820
|
+
if (!readFileSync2(backup).equals(original))
|
|
2821
|
+
throw new Error("session WAL preservation readback failed; source retained");
|
|
2822
|
+
return backup;
|
|
2823
|
+
}
|
|
2076
2824
|
|
|
2077
2825
|
// src/service.ts
|
|
2826
|
+
import { hostname } from "os";
|
|
2827
|
+
import { resolveDatabaseUrl } from "@hasna/contracts";
|
|
2078
2828
|
var DEFAULT_PORT = 8790;
|
|
2079
2829
|
var DEFAULT_HOST = "127.0.0.1";
|
|
2080
|
-
function packageVersion() {
|
|
2081
|
-
return package_default.version;
|
|
2082
|
-
}
|
|
2083
2830
|
function envString(names) {
|
|
2084
2831
|
for (const name of names) {
|
|
2085
2832
|
const value = process.env[name];
|
|
@@ -2096,69 +2843,218 @@ function envInt(names, fallback) {
|
|
|
2096
2843
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
2097
2844
|
}
|
|
2098
2845
|
function resolveWorkflowsConfig(overrides = {}) {
|
|
2846
|
+
const originAuth = overrides.originAuth ?? process.env.HASNA_WORKFLOWS_ORIGIN_AUTH;
|
|
2847
|
+
if (originAuth !== undefined && !/^[A-Za-z0-9_-]{32,256}$/.test(originAuth))
|
|
2848
|
+
throw new Error("HASNA_WORKFLOWS_ORIGIN_AUTH must be a valid gateway origin identity");
|
|
2099
2849
|
return {
|
|
2100
2850
|
port: overrides.port ?? envInt(["HASNA_WORKFLOWS_PORT", "WORKFLOWS_PORT"], DEFAULT_PORT),
|
|
2101
2851
|
host: overrides.host ?? envString(["HASNA_WORKFLOWS_HOST", "WORKFLOWS_HOST"]) ?? DEFAULT_HOST,
|
|
2102
2852
|
dataDir: overrides.dataDir ?? envString(["HASNA_WORKFLOWS_DATA_DIR", "WORKFLOWS_DATA_DIR"]) ?? getEffectiveDataDir(),
|
|
2103
|
-
apiUrl: overrides.apiUrl ?? envString(["WORKFLOWS_API_URL"]),
|
|
2104
|
-
apiKey: overrides.apiKey ?? envString(["WORKFLOWS_API_KEY"])
|
|
2853
|
+
apiUrl: overrides.apiUrl ?? envString(["HASNA_WORKFLOWS_API_URL", "WORKFLOWS_API_URL"]),
|
|
2854
|
+
apiKey: overrides.apiKey ?? envString(["HASNA_WORKFLOWS_API_KEY", "WORKFLOWS_API_KEY"]),
|
|
2855
|
+
originAuth,
|
|
2856
|
+
storageBackend: overrides.storageBackend ?? "postgresql",
|
|
2857
|
+
databaseUrl: overrides.databaseUrl ?? envString(["HASNA_WORKFLOWS_DATABASE_URL", "WORKFLOWS_DATABASE_URL"]),
|
|
2858
|
+
databaseSchema: overrides.databaseSchema ?? envString(["HASNA_WORKFLOWS_DATABASE_SCHEMA", "WORKFLOWS_DATABASE_SCHEMA"])
|
|
2105
2859
|
};
|
|
2106
2860
|
}
|
|
2107
2861
|
|
|
2108
2862
|
class WorkflowsService {
|
|
2863
|
+
execution;
|
|
2109
2864
|
name = "workflows";
|
|
2110
2865
|
version;
|
|
2111
2866
|
config;
|
|
2112
2867
|
startedAt = Date.now();
|
|
2113
|
-
|
|
2868
|
+
databaseEnv;
|
|
2869
|
+
constructor(config = {}, execution = {}) {
|
|
2870
|
+
this.execution = execution;
|
|
2114
2871
|
this.config = resolveWorkflowsConfig(config);
|
|
2872
|
+
this.databaseEnv = config.databaseUrl !== undefined ? { HASNA_WORKFLOWS_DATABASE_URL: config.databaseUrl } : {
|
|
2873
|
+
HASNA_WORKFLOWS_DATABASE_URL: process.env.HASNA_WORKFLOWS_DATABASE_URL,
|
|
2874
|
+
WORKFLOWS_DATABASE_URL: process.env.WORKFLOWS_DATABASE_URL
|
|
2875
|
+
};
|
|
2115
2876
|
this.version = packageVersion();
|
|
2116
2877
|
}
|
|
2117
2878
|
health() {
|
|
2118
2879
|
return {
|
|
2119
2880
|
ok: true,
|
|
2881
|
+
status: "ok",
|
|
2882
|
+
backend: this.config.storageBackend,
|
|
2120
2883
|
service: this.name,
|
|
2121
2884
|
version: this.version,
|
|
2122
2885
|
pid: process.pid,
|
|
2123
2886
|
uptimeMs: Date.now() - this.startedAt
|
|
2124
2887
|
};
|
|
2125
2888
|
}
|
|
2126
|
-
ready() {
|
|
2889
|
+
async ready() {
|
|
2890
|
+
const checks = { version: "ok", auth: this.config.apiKey ? "ok" : "missing", storage: "not-checked" };
|
|
2891
|
+
if (this.config.apiKey) {
|
|
2892
|
+
try {
|
|
2893
|
+
checks.storage = await this.withStore(async (store) => (store instanceof PostgresWorkflowsStore ? await store.checkSchema() : (await store.listTables()).sort().join(",") === "memos,run_nodes,runs") ? "ok" : "invalid-schema");
|
|
2894
|
+
} catch (error) {
|
|
2895
|
+
checks.storage = error instanceof WorkflowRequestError && error.code === "invalid_schema" ? "invalid-schema" : "unavailable";
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
return { ok: checks.auth === "ok" && checks.storage === "ok", service: this.name, version: this.version, checks };
|
|
2899
|
+
}
|
|
2900
|
+
async openEngineStore(verifySchema = true) {
|
|
2901
|
+
if (this.config.storageBackend === "sqlite")
|
|
2902
|
+
return openStore(this.config.dataDir);
|
|
2903
|
+
if (this.config.storageBackend !== "postgresql")
|
|
2904
|
+
throw new WorkflowRequestError("invalid_storage_backend", "storage backend must be postgresql or explicit embedded sqlite", 503);
|
|
2905
|
+
let databaseUrl;
|
|
2906
|
+
try {
|
|
2907
|
+
databaseUrl = resolveDatabaseUrl("workflows", this.databaseEnv);
|
|
2908
|
+
} catch (error) {
|
|
2909
|
+
throw new WorkflowRequestError("invalid_database_configuration", error instanceof Error ? error.message : "HASNA_WORKFLOWS_DATABASE_URL must configure PostgreSQL", 503);
|
|
2910
|
+
}
|
|
2911
|
+
let store;
|
|
2912
|
+
try {
|
|
2913
|
+
store = new PostgresWorkflowsStore({ databaseUrl, dataDir: this.config.dataDir, schema: this.config.databaseSchema });
|
|
2914
|
+
} catch {
|
|
2915
|
+
throw new WorkflowRequestError("invalid_database_schema", "HASNA_WORKFLOWS_DATABASE_SCHEMA must be a lowercase SQL identifier", 503);
|
|
2916
|
+
}
|
|
2917
|
+
try {
|
|
2918
|
+
if (verifySchema && !await store.checkSchema())
|
|
2919
|
+
throw new WorkflowRequestError("invalid_schema", "PostgreSQL workflow schema requires explicit initialization or a reviewed migration", 503);
|
|
2920
|
+
return store;
|
|
2921
|
+
} catch (error) {
|
|
2922
|
+
await store.close();
|
|
2923
|
+
throw error;
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
async withStore(operation) {
|
|
2927
|
+
const store = await this.openEngineStore();
|
|
2928
|
+
try {
|
|
2929
|
+
return await operation(store);
|
|
2930
|
+
} finally {
|
|
2931
|
+
await store.close();
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
async initializeStorage() {
|
|
2935
|
+
const store = await this.openEngineStore(false);
|
|
2936
|
+
try {
|
|
2937
|
+
if (!(store instanceof PostgresWorkflowsStore))
|
|
2938
|
+
throw new WorkflowRequestError("invalid_storage_backend", "init-db requires PostgreSQL", 503);
|
|
2939
|
+
await store.initialize();
|
|
2940
|
+
} finally {
|
|
2941
|
+
await store.close();
|
|
2942
|
+
}
|
|
2943
|
+
}
|
|
2944
|
+
listRuns(options = {}) {
|
|
2945
|
+
return this.withStore((store) => store.listRuns(options));
|
|
2946
|
+
}
|
|
2947
|
+
getRun(id) {
|
|
2948
|
+
return this.withStore((store) => store.getRun(id));
|
|
2949
|
+
}
|
|
2950
|
+
listNodes(id) {
|
|
2951
|
+
return this.withStore((store) => store.listRunNodes(id));
|
|
2952
|
+
}
|
|
2953
|
+
events(id) {
|
|
2954
|
+
return SessionWAL.open(this.config.dataDir).replay({ repair: false }).entries.filter((entry) => ("runId" in entry.op) && entry.op.runId === id);
|
|
2955
|
+
}
|
|
2956
|
+
cancelRun(id) {
|
|
2957
|
+
return this.withStore(async (store) => {
|
|
2958
|
+
const run2 = await store.getRun(id);
|
|
2959
|
+
if (!run2)
|
|
2960
|
+
return;
|
|
2961
|
+
if (["completed", "failed", "cancelled"].includes(run2.status))
|
|
2962
|
+
throw new WorkflowRequestError("invalid_run_status", "cancellation requires a pending, running, or interrupted run", 409);
|
|
2963
|
+
if (!await store.compareRunStatus(id, ["pending", "running", "interrupted"], "cancelled", { finishedAt: new Date().toISOString() }))
|
|
2964
|
+
throw new WorkflowRequestError("run_status_conflict", "run changed before cancellation", 409);
|
|
2965
|
+
return store.getRun(id);
|
|
2966
|
+
});
|
|
2967
|
+
}
|
|
2968
|
+
async resumeRun(id) {
|
|
2969
|
+
const store = await this.openEngineStore();
|
|
2970
|
+
const wal = SessionWAL.open(this.config.dataDir);
|
|
2971
|
+
try {
|
|
2972
|
+
const run2 = await store.getRun(id);
|
|
2973
|
+
if (!run2)
|
|
2974
|
+
throw new WorkflowRequestError("run_not_found", "run id does not exist", 404);
|
|
2975
|
+
if (!["failed", "cancelled", "interrupted"].includes(run2.status))
|
|
2976
|
+
throw new WorkflowRequestError("invalid_run_status", "resume requires a failed, cancelled, or interrupted run", 409);
|
|
2977
|
+
const context = JSON.parse(run2.contextJson);
|
|
2978
|
+
if (context.__wf?.resumeBlocked || context.__wf?.executionMayBeActive || wal.replay({ repair: false }).liveClaims().has(id))
|
|
2979
|
+
throw new WorkflowRequestError("execution_active", "previous execution must finish before resume", 409);
|
|
2980
|
+
if (store instanceof PostgresWorkflowsStore && !await store.acquireExecution(id))
|
|
2981
|
+
throw new WorkflowRequestError("execution_active", "another executor still owns this run", 409);
|
|
2982
|
+
const graph = context.__wf?.graph;
|
|
2983
|
+
if (!graph)
|
|
2984
|
+
throw new WorkflowRequestError("graph_unavailable", "legacy run needs its original graph registered by the server operator", 409);
|
|
2985
|
+
if (!await store.compareRunStatus(id, ["failed", "cancelled", "interrupted"], "pending", { error: null }))
|
|
2986
|
+
throw new WorkflowRequestError("run_status_conflict", "run changed before resume", 409);
|
|
2987
|
+
for (const node2 of await store.listRunNodes(id)) {
|
|
2988
|
+
if (node2.status !== "completed")
|
|
2989
|
+
await store.setRunNodeStatus(node2.id, "pending", { error: null });
|
|
2990
|
+
}
|
|
2991
|
+
const final = await runGraphToCompletion(store, wal, graph, context, { ...this.execution, existingRunId: id });
|
|
2992
|
+
return runSummary(final, false);
|
|
2993
|
+
} finally {
|
|
2994
|
+
await store.close();
|
|
2995
|
+
}
|
|
2996
|
+
}
|
|
2997
|
+
sessions(action) {
|
|
2998
|
+
const replay = SessionWAL.open(this.config.dataDir).replay({ repair: false });
|
|
2999
|
+
if (action === "list")
|
|
3000
|
+
return replay.entries.slice(-100).map(({ seq, at, op }) => ({ seq, at, operation: op.op }));
|
|
2127
3001
|
return {
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
3002
|
+
entries: replay.entries.length,
|
|
3003
|
+
torn: replay.torn,
|
|
3004
|
+
repaired: false,
|
|
3005
|
+
liveClaims: [...replay.liveClaims().entries()].map(([runId, claim]) => ({ runId, worker: claim.worker, fencing: claim.fencing, expiresAt: new Date(claim.expiresAtMs).toISOString() }))
|
|
2132
3006
|
};
|
|
2133
3007
|
}
|
|
3008
|
+
machines(action) {
|
|
3009
|
+
const local = { name: hostname(), role: "server", version: this.version };
|
|
3010
|
+
if (action === "list")
|
|
3011
|
+
return [local];
|
|
3012
|
+
return { ...local, health: this.health(), sessions: this.sessions("pull") };
|
|
3013
|
+
}
|
|
3014
|
+
daemonStatus() {
|
|
3015
|
+
const replay = SessionWAL.open(this.config.dataDir).replay({ repair: false });
|
|
3016
|
+
return { activeClaims: replay.liveClaims().size, mode: "bounded-trigger", continuousDaemon: false };
|
|
3017
|
+
}
|
|
3018
|
+
listMemos() {
|
|
3019
|
+
return this.withStore(async (store) => (await store.memoList()).slice(0, 100));
|
|
3020
|
+
}
|
|
2134
3021
|
async triggerRun(graph, context = {}, opts = {}) {
|
|
2135
|
-
const store =
|
|
3022
|
+
const store = await this.openEngineStore();
|
|
2136
3023
|
const wal = SessionWAL.open(this.config.dataDir);
|
|
2137
3024
|
try {
|
|
2138
|
-
if (
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
3025
|
+
if (context === null || typeof context !== "object" || Array.isArray(context))
|
|
3026
|
+
throw new WorkflowRequestError("invalid_context", "context must be an object");
|
|
3027
|
+
if (Object.hasOwn(context, "__wf") || Object.hasOwn(context, "steps"))
|
|
3028
|
+
throw new WorkflowRequestError("reserved_context", "context.__wf and context.steps are reserved for the workflow engine");
|
|
3029
|
+
const maxCycles = opts.maxCycles ?? 500;
|
|
3030
|
+
if (!Number.isSafeInteger(maxCycles) || maxCycles < 1 || maxCycles > 2000)
|
|
3031
|
+
throw new WorkflowRequestError("invalid_max_cycles", "maxCycles must be an integer from 1 to 2000");
|
|
3032
|
+
if (opts.idempotencyKey !== undefined && !/^[A-Za-z0-9_.:-]{1,128}$/.test(opts.idempotencyKey))
|
|
3033
|
+
throw new WorkflowRequestError("invalid_idempotency_key", "idempotencyKey must contain 1 to 128 ASCII letters, digits, or _.:-");
|
|
3034
|
+
const requestHash = createHash3("sha256").update(canonicalJson({ graph, context, maxCycles })).digest("hex");
|
|
3035
|
+
const reuse = async () => {
|
|
3036
|
+
const existing2 = opts.idempotencyKey ? await store.getRunByIdempotencyKey(opts.idempotencyKey) : undefined;
|
|
3037
|
+
if (!existing2)
|
|
3038
|
+
return;
|
|
3039
|
+
if (JSON.parse(existing2.contextJson).__wf?.requestHash !== requestHash)
|
|
3040
|
+
throw new WorkflowRequestError("idempotency_conflict", "idempotencyKey is already bound to a different request", 409);
|
|
3041
|
+
return runSummary(existing2, true);
|
|
3042
|
+
};
|
|
3043
|
+
const existing = await reuse();
|
|
3044
|
+
if (existing)
|
|
3045
|
+
return existing;
|
|
3046
|
+
const contextWithKey = { ...context, __wf: { requestHash, ...opts.idempotencyKey ? { idempotencyKey: opts.idempotencyKey } : {} } };
|
|
3047
|
+
let final;
|
|
3048
|
+
try {
|
|
3049
|
+
final = await runGraphToCompletion(store, wal, graph, contextWithKey, { ...this.execution, maxCycles });
|
|
3050
|
+
} catch (err) {
|
|
3051
|
+
if (err instanceof Error && (/UNIQUE constraint failed/.test(err.message) || err.code === "23505")) {
|
|
3052
|
+
const winner = await reuse();
|
|
3053
|
+
if (winner)
|
|
3054
|
+
return winner;
|
|
2148
3055
|
}
|
|
3056
|
+
throw err;
|
|
2149
3057
|
}
|
|
2150
|
-
const contextWithKey = opts.idempotencyKey ? {
|
|
2151
|
-
...context,
|
|
2152
|
-
__wf: {
|
|
2153
|
-
cursor: undefined,
|
|
2154
|
-
loops: {},
|
|
2155
|
-
completedLoops: {},
|
|
2156
|
-
...context?.__wf ?? {},
|
|
2157
|
-
idempotencyKey: opts.idempotencyKey
|
|
2158
|
-
}
|
|
2159
|
-
} : context;
|
|
2160
|
-
const maxCycles = Math.min(Math.max(opts.maxCycles ?? 500, 1), 2000);
|
|
2161
|
-
const final = await runGraphToCompletion(store, wal, graph, contextWithKey, { maxCycles });
|
|
2162
3058
|
return {
|
|
2163
3059
|
runId: final.id,
|
|
2164
3060
|
status: final.status,
|
|
@@ -2167,16 +3063,38 @@ class WorkflowsService {
|
|
|
2167
3063
|
reused: false
|
|
2168
3064
|
};
|
|
2169
3065
|
} finally {
|
|
2170
|
-
store.close();
|
|
3066
|
+
await store.close();
|
|
2171
3067
|
}
|
|
2172
3068
|
}
|
|
2173
3069
|
}
|
|
2174
|
-
function createWorkflowsService(config = {}) {
|
|
2175
|
-
return new WorkflowsService(config);
|
|
3070
|
+
function createWorkflowsService(config = {}, execution = {}) {
|
|
3071
|
+
return new WorkflowsService(config, execution);
|
|
3072
|
+
}
|
|
3073
|
+
|
|
3074
|
+
class WorkflowRequestError extends Error {
|
|
3075
|
+
code;
|
|
3076
|
+
status;
|
|
3077
|
+
constructor(code, message, status = 400) {
|
|
3078
|
+
super(message);
|
|
3079
|
+
this.code = code;
|
|
3080
|
+
this.status = status;
|
|
3081
|
+
this.name = "WorkflowRequestError";
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
function canonicalJson(value) {
|
|
3085
|
+
if (Array.isArray(value))
|
|
3086
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
3087
|
+
if (value && typeof value === "object")
|
|
3088
|
+
return `{${Object.keys(value).sort().filter((k) => value[k] !== undefined).map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(",")}}`;
|
|
3089
|
+
return JSON.stringify(value) ?? "null";
|
|
3090
|
+
}
|
|
3091
|
+
function runSummary(run2, reused) {
|
|
3092
|
+
return { runId: run2.id, status: run2.status, error: run2.error ?? null, result: run2.resultJson ? JSON.parse(run2.resultJson) : null, reused };
|
|
2176
3093
|
}
|
|
2177
3094
|
export {
|
|
2178
3095
|
resolveWorkflowsConfig,
|
|
2179
3096
|
packageVersion,
|
|
2180
3097
|
createWorkflowsService,
|
|
2181
|
-
WorkflowsService
|
|
3098
|
+
WorkflowsService,
|
|
3099
|
+
WorkflowRequestError
|
|
2182
3100
|
};
|