@hasna/loops 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -3
- package/dist/cli/index.js +1479 -75
- package/dist/daemon/daemon.d.ts +1 -0
- package/dist/daemon/index.js +1156 -52
- package/dist/daemon/install.d.ts +8 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +1335 -57
- package/dist/lib/accounts.d.ts +4 -0
- package/dist/lib/doctor.d.ts +13 -0
- package/dist/lib/executor.d.ts +20 -1
- package/dist/lib/format.d.ts +5 -1
- package/dist/lib/scheduler.d.ts +12 -0
- package/dist/lib/store.d.ts +47 -2
- package/dist/lib/store.js +627 -6
- package/dist/lib/workflow-runner.d.ts +14 -0
- package/dist/lib/workflow-spec.d.ts +5 -0
- package/dist/sdk/index.js +1126 -57
- package/dist/types.d.ts +88 -2
- package/docs/USAGE.md +89 -4
- package/package.json +2 -2
package/dist/daemon/index.js
CHANGED
|
@@ -236,6 +236,103 @@ function parseDuration(input) {
|
|
|
236
236
|
return Math.round(value * multiplier);
|
|
237
237
|
}
|
|
238
238
|
|
|
239
|
+
// src/lib/workflow-spec.ts
|
|
240
|
+
function assertObject(value, label) {
|
|
241
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
242
|
+
throw new Error(`${label} must be an object`);
|
|
243
|
+
}
|
|
244
|
+
function assertString(value, label) {
|
|
245
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
246
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
247
|
+
}
|
|
248
|
+
function validateTarget(value, label) {
|
|
249
|
+
assertObject(value, label);
|
|
250
|
+
if (value.type === "command") {
|
|
251
|
+
assertString(value.command, `${label}.command`);
|
|
252
|
+
if (value.shell !== true && /\s/.test(value.command.trim())) {
|
|
253
|
+
throw new Error(`${label}.command must be an executable without spaces when shell is false; put flags in args or set shell true`);
|
|
254
|
+
}
|
|
255
|
+
return value;
|
|
256
|
+
}
|
|
257
|
+
if (value.type === "agent") {
|
|
258
|
+
assertString(value.provider, `${label}.provider`);
|
|
259
|
+
assertString(value.prompt, `${label}.prompt`);
|
|
260
|
+
const providers = ["claude", "cursor", "codewith", "aicopilot", "opencode", "codex"];
|
|
261
|
+
if (!providers.includes(value.provider))
|
|
262
|
+
throw new Error(`${label}.provider must be one of ${providers.join(", ")}`);
|
|
263
|
+
return value;
|
|
264
|
+
}
|
|
265
|
+
throw new Error(`${label}.type must be command or agent`);
|
|
266
|
+
}
|
|
267
|
+
function normalizeCreateWorkflowInput(input) {
|
|
268
|
+
assertString(input.name, "workflow.name");
|
|
269
|
+
if (!Array.isArray(input.steps) || input.steps.length === 0)
|
|
270
|
+
throw new Error("workflow.steps must contain at least one step");
|
|
271
|
+
const seen = new Set;
|
|
272
|
+
const steps = input.steps.map((step, index) => {
|
|
273
|
+
assertObject(step, `workflow.steps[${index}]`);
|
|
274
|
+
assertString(step.id, `workflow.steps[${index}].id`);
|
|
275
|
+
if (seen.has(step.id))
|
|
276
|
+
throw new Error(`duplicate workflow step id: ${step.id}`);
|
|
277
|
+
seen.add(step.id);
|
|
278
|
+
return {
|
|
279
|
+
...step,
|
|
280
|
+
id: step.id,
|
|
281
|
+
target: validateTarget(step.target, `workflow.steps[${index}].target`),
|
|
282
|
+
dependsOn: step.dependsOn ?? [],
|
|
283
|
+
continueOnFailure: step.continueOnFailure ?? false
|
|
284
|
+
};
|
|
285
|
+
});
|
|
286
|
+
for (const step of steps) {
|
|
287
|
+
for (const dependency of step.dependsOn ?? []) {
|
|
288
|
+
if (!seen.has(dependency))
|
|
289
|
+
throw new Error(`step ${step.id} depends on missing step ${dependency}`);
|
|
290
|
+
if (dependency === step.id)
|
|
291
|
+
throw new Error(`step ${step.id} cannot depend on itself`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
workflowExecutionOrder({ steps });
|
|
295
|
+
return { ...input, name: input.name.trim(), version: input.version ?? 1, steps };
|
|
296
|
+
}
|
|
297
|
+
function workflowExecutionOrder(workflow) {
|
|
298
|
+
const byId = new Map(workflow.steps.map((step) => [step.id, step]));
|
|
299
|
+
const visiting = new Set;
|
|
300
|
+
const visited = new Set;
|
|
301
|
+
const order = [];
|
|
302
|
+
function visit(step) {
|
|
303
|
+
if (visited.has(step.id))
|
|
304
|
+
return;
|
|
305
|
+
if (visiting.has(step.id))
|
|
306
|
+
throw new Error(`workflow dependency cycle includes step ${step.id}`);
|
|
307
|
+
visiting.add(step.id);
|
|
308
|
+
for (const dependencyId of step.dependsOn ?? []) {
|
|
309
|
+
const dependency = byId.get(dependencyId);
|
|
310
|
+
if (!dependency)
|
|
311
|
+
throw new Error(`step ${step.id} depends on missing step ${dependencyId}`);
|
|
312
|
+
visit(dependency);
|
|
313
|
+
}
|
|
314
|
+
visiting.delete(step.id);
|
|
315
|
+
visited.add(step.id);
|
|
316
|
+
order.push(step);
|
|
317
|
+
}
|
|
318
|
+
for (const step of workflow.steps)
|
|
319
|
+
visit(step);
|
|
320
|
+
return order;
|
|
321
|
+
}
|
|
322
|
+
function workflowBodyFromJson(value, fallbackName) {
|
|
323
|
+
assertObject(value, "workflow file");
|
|
324
|
+
const rawName = fallbackName ?? value.name;
|
|
325
|
+
assertString(rawName, "workflow.name");
|
|
326
|
+
if (!Array.isArray(value.steps))
|
|
327
|
+
throw new Error("workflow.steps must be an array");
|
|
328
|
+
return normalizeCreateWorkflowInput({
|
|
329
|
+
name: rawName,
|
|
330
|
+
description: typeof value.description === "string" ? value.description : undefined,
|
|
331
|
+
version: typeof value.version === "number" ? value.version : undefined,
|
|
332
|
+
steps: value.steps
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
|
|
239
336
|
// src/lib/store.ts
|
|
240
337
|
function rowToLoop(row) {
|
|
241
338
|
return {
|
|
@@ -280,6 +377,76 @@ function rowToRun(row) {
|
|
|
280
377
|
updatedAt: row.updated_at
|
|
281
378
|
};
|
|
282
379
|
}
|
|
380
|
+
function rowToWorkflow(row) {
|
|
381
|
+
return {
|
|
382
|
+
id: row.id,
|
|
383
|
+
name: row.name,
|
|
384
|
+
description: row.description ?? undefined,
|
|
385
|
+
version: row.version,
|
|
386
|
+
status: row.status,
|
|
387
|
+
steps: JSON.parse(row.steps_json),
|
|
388
|
+
createdAt: row.created_at,
|
|
389
|
+
updatedAt: row.updated_at
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
function rowToWorkflowRun(row) {
|
|
393
|
+
return {
|
|
394
|
+
id: row.id,
|
|
395
|
+
workflowId: row.workflow_id,
|
|
396
|
+
workflowName: row.workflow_name,
|
|
397
|
+
loopId: row.loop_id ?? undefined,
|
|
398
|
+
loopRunId: row.loop_run_id ?? undefined,
|
|
399
|
+
scheduledFor: row.scheduled_for ?? undefined,
|
|
400
|
+
idempotencyKey: row.idempotency_key ?? undefined,
|
|
401
|
+
status: row.status,
|
|
402
|
+
startedAt: row.started_at ?? undefined,
|
|
403
|
+
finishedAt: row.finished_at ?? undefined,
|
|
404
|
+
durationMs: row.duration_ms ?? undefined,
|
|
405
|
+
error: row.error ?? undefined,
|
|
406
|
+
createdAt: row.created_at,
|
|
407
|
+
updatedAt: row.updated_at
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
function rowToWorkflowStepRun(row) {
|
|
411
|
+
return {
|
|
412
|
+
id: row.id,
|
|
413
|
+
workflowRunId: row.workflow_run_id,
|
|
414
|
+
stepId: row.step_id,
|
|
415
|
+
sequence: row.sequence,
|
|
416
|
+
status: row.status,
|
|
417
|
+
startedAt: row.started_at ?? undefined,
|
|
418
|
+
finishedAt: row.finished_at ?? undefined,
|
|
419
|
+
exitCode: row.exit_code ?? undefined,
|
|
420
|
+
pid: row.pid ?? undefined,
|
|
421
|
+
durationMs: row.duration_ms ?? undefined,
|
|
422
|
+
stdout: row.stdout ?? undefined,
|
|
423
|
+
stderr: row.stderr ?? undefined,
|
|
424
|
+
error: row.error ?? undefined,
|
|
425
|
+
accountProfile: row.account_profile ?? undefined,
|
|
426
|
+
accountTool: row.account_tool ?? undefined,
|
|
427
|
+
createdAt: row.created_at,
|
|
428
|
+
updatedAt: row.updated_at
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
function rowToWorkflowEvent(row) {
|
|
432
|
+
return {
|
|
433
|
+
id: row.id,
|
|
434
|
+
workflowRunId: row.workflow_run_id,
|
|
435
|
+
sequence: row.sequence,
|
|
436
|
+
eventType: row.event_type,
|
|
437
|
+
stepId: row.step_id ?? undefined,
|
|
438
|
+
payload: row.payload_json ? JSON.parse(row.payload_json) : undefined,
|
|
439
|
+
createdAt: row.created_at
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
function isProcessAlive(pid) {
|
|
443
|
+
try {
|
|
444
|
+
process.kill(pid, 0);
|
|
445
|
+
return true;
|
|
446
|
+
} catch {
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
283
450
|
function rowToLease(row) {
|
|
284
451
|
return {
|
|
285
452
|
id: row.id,
|
|
@@ -305,6 +472,11 @@ class Store {
|
|
|
305
472
|
}
|
|
306
473
|
migrate() {
|
|
307
474
|
this.db.exec(`
|
|
475
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
476
|
+
id TEXT PRIMARY KEY,
|
|
477
|
+
applied_at TEXT NOT NULL
|
|
478
|
+
);
|
|
479
|
+
|
|
308
480
|
CREATE TABLE IF NOT EXISTS loops (
|
|
309
481
|
id TEXT PRIMARY KEY,
|
|
310
482
|
name TEXT NOT NULL,
|
|
@@ -361,7 +533,82 @@ class Store {
|
|
|
361
533
|
created_at TEXT NOT NULL,
|
|
362
534
|
updated_at TEXT NOT NULL
|
|
363
535
|
);
|
|
536
|
+
|
|
537
|
+
CREATE TABLE IF NOT EXISTS workflow_specs (
|
|
538
|
+
id TEXT PRIMARY KEY,
|
|
539
|
+
name TEXT NOT NULL,
|
|
540
|
+
description TEXT,
|
|
541
|
+
version INTEGER NOT NULL,
|
|
542
|
+
status TEXT NOT NULL,
|
|
543
|
+
steps_json TEXT NOT NULL,
|
|
544
|
+
created_at TEXT NOT NULL,
|
|
545
|
+
updated_at TEXT NOT NULL
|
|
546
|
+
);
|
|
547
|
+
CREATE INDEX IF NOT EXISTS idx_workflows_status_name ON workflow_specs(status, name);
|
|
548
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_workflows_name_active ON workflow_specs(name) WHERE status = 'active';
|
|
549
|
+
|
|
550
|
+
CREATE TABLE IF NOT EXISTS workflow_runs (
|
|
551
|
+
id TEXT PRIMARY KEY,
|
|
552
|
+
workflow_id TEXT NOT NULL REFERENCES workflow_specs(id) ON DELETE CASCADE,
|
|
553
|
+
workflow_name TEXT NOT NULL,
|
|
554
|
+
loop_id TEXT REFERENCES loops(id) ON DELETE SET NULL,
|
|
555
|
+
loop_run_id TEXT REFERENCES loop_runs(id) ON DELETE SET NULL,
|
|
556
|
+
scheduled_for TEXT,
|
|
557
|
+
idempotency_key TEXT,
|
|
558
|
+
status TEXT NOT NULL,
|
|
559
|
+
started_at TEXT,
|
|
560
|
+
finished_at TEXT,
|
|
561
|
+
duration_ms INTEGER,
|
|
562
|
+
error TEXT,
|
|
563
|
+
created_at TEXT NOT NULL,
|
|
564
|
+
updated_at TEXT NOT NULL
|
|
565
|
+
);
|
|
566
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_workflow_runs_idempotency
|
|
567
|
+
ON workflow_runs(workflow_id, idempotency_key)
|
|
568
|
+
WHERE idempotency_key IS NOT NULL;
|
|
569
|
+
CREATE INDEX IF NOT EXISTS idx_workflow_runs_workflow_created ON workflow_runs(workflow_id, created_at DESC);
|
|
570
|
+
CREATE INDEX IF NOT EXISTS idx_workflow_runs_loop_run ON workflow_runs(loop_run_id);
|
|
571
|
+
CREATE INDEX IF NOT EXISTS idx_workflow_runs_status ON workflow_runs(status);
|
|
572
|
+
|
|
573
|
+
CREATE TABLE IF NOT EXISTS workflow_step_runs (
|
|
574
|
+
id TEXT PRIMARY KEY,
|
|
575
|
+
workflow_run_id TEXT NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE,
|
|
576
|
+
step_id TEXT NOT NULL,
|
|
577
|
+
sequence INTEGER NOT NULL,
|
|
578
|
+
status TEXT NOT NULL,
|
|
579
|
+
started_at TEXT,
|
|
580
|
+
finished_at TEXT,
|
|
581
|
+
exit_code INTEGER,
|
|
582
|
+
pid INTEGER,
|
|
583
|
+
duration_ms INTEGER,
|
|
584
|
+
stdout TEXT,
|
|
585
|
+
stderr TEXT,
|
|
586
|
+
error TEXT,
|
|
587
|
+
account_profile TEXT,
|
|
588
|
+
account_tool TEXT,
|
|
589
|
+
created_at TEXT NOT NULL,
|
|
590
|
+
updated_at TEXT NOT NULL,
|
|
591
|
+
UNIQUE(workflow_run_id, step_id)
|
|
592
|
+
);
|
|
593
|
+
CREATE INDEX IF NOT EXISTS idx_workflow_step_runs_run_sequence ON workflow_step_runs(workflow_run_id, sequence);
|
|
594
|
+
CREATE INDEX IF NOT EXISTS idx_workflow_step_runs_status ON workflow_step_runs(status);
|
|
595
|
+
|
|
596
|
+
CREATE TABLE IF NOT EXISTS workflow_events (
|
|
597
|
+
id TEXT PRIMARY KEY,
|
|
598
|
+
workflow_run_id TEXT NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE,
|
|
599
|
+
sequence INTEGER NOT NULL,
|
|
600
|
+
event_type TEXT NOT NULL,
|
|
601
|
+
step_id TEXT,
|
|
602
|
+
payload_json TEXT,
|
|
603
|
+
created_at TEXT NOT NULL,
|
|
604
|
+
UNIQUE(workflow_run_id, sequence)
|
|
605
|
+
);
|
|
606
|
+
CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
|
|
364
607
|
`);
|
|
608
|
+
try {
|
|
609
|
+
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
610
|
+
} catch {}
|
|
611
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
365
612
|
}
|
|
366
613
|
createLoop(input, from = new Date) {
|
|
367
614
|
const now = nowIso();
|
|
@@ -453,10 +700,343 @@ class Store {
|
|
|
453
700
|
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
454
701
|
return res.changes > 0;
|
|
455
702
|
}
|
|
703
|
+
createWorkflow(input) {
|
|
704
|
+
const normalized = normalizeCreateWorkflowInput(input);
|
|
705
|
+
const now = nowIso();
|
|
706
|
+
const workflow = {
|
|
707
|
+
id: genId(),
|
|
708
|
+
name: normalized.name,
|
|
709
|
+
description: normalized.description,
|
|
710
|
+
version: normalized.version ?? 1,
|
|
711
|
+
status: "active",
|
|
712
|
+
steps: normalized.steps,
|
|
713
|
+
createdAt: now,
|
|
714
|
+
updatedAt: now
|
|
715
|
+
};
|
|
716
|
+
this.db.query(`INSERT INTO workflow_specs (id, name, description, version, status, steps_json, created_at, updated_at)
|
|
717
|
+
VALUES ($id, $name, $description, $version, $status, $steps, $created, $updated)`).run({
|
|
718
|
+
$id: workflow.id,
|
|
719
|
+
$name: workflow.name,
|
|
720
|
+
$description: workflow.description ?? null,
|
|
721
|
+
$version: workflow.version,
|
|
722
|
+
$status: workflow.status,
|
|
723
|
+
$steps: JSON.stringify(workflow.steps),
|
|
724
|
+
$created: workflow.createdAt,
|
|
725
|
+
$updated: workflow.updatedAt
|
|
726
|
+
});
|
|
727
|
+
return workflow;
|
|
728
|
+
}
|
|
729
|
+
getWorkflow(id) {
|
|
730
|
+
const row = this.db.query("SELECT * FROM workflow_specs WHERE id = ?").get(id);
|
|
731
|
+
return row ? rowToWorkflow(row) : undefined;
|
|
732
|
+
}
|
|
733
|
+
findWorkflowByName(name) {
|
|
734
|
+
const row = this.db.query("SELECT * FROM workflow_specs WHERE name = ? AND status = 'active' ORDER BY updated_at DESC LIMIT 1").get(name);
|
|
735
|
+
return row ? rowToWorkflow(row) : undefined;
|
|
736
|
+
}
|
|
737
|
+
requireWorkflow(idOrName) {
|
|
738
|
+
return this.getWorkflow(idOrName) ?? this.findWorkflowByName(idOrName) ?? (() => {
|
|
739
|
+
throw new Error(`workflow not found: ${idOrName}`);
|
|
740
|
+
})();
|
|
741
|
+
}
|
|
742
|
+
listWorkflows(opts = {}) {
|
|
743
|
+
const limit = opts.limit ?? 200;
|
|
744
|
+
const rows = opts.status ? this.db.query("SELECT * FROM workflow_specs WHERE status = ? ORDER BY updated_at DESC LIMIT ?").all(opts.status, limit) : this.db.query("SELECT * FROM workflow_specs ORDER BY status ASC, updated_at DESC LIMIT ?").all(limit);
|
|
745
|
+
return rows.map(rowToWorkflow);
|
|
746
|
+
}
|
|
747
|
+
archiveWorkflow(idOrName) {
|
|
748
|
+
const workflow = this.requireWorkflow(idOrName);
|
|
749
|
+
const updated = nowIso();
|
|
750
|
+
this.db.query("UPDATE workflow_specs SET status='archived', updated_at=? WHERE id=?").run(updated, workflow.id);
|
|
751
|
+
const archived = this.getWorkflow(workflow.id);
|
|
752
|
+
if (!archived)
|
|
753
|
+
throw new Error(`workflow not found after archive: ${workflow.id}`);
|
|
754
|
+
return archived;
|
|
755
|
+
}
|
|
756
|
+
createWorkflowRun(input) {
|
|
757
|
+
const now = nowIso();
|
|
758
|
+
if (input.idempotencyKey) {
|
|
759
|
+
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
760
|
+
if (existing)
|
|
761
|
+
return rowToWorkflowRun(existing);
|
|
762
|
+
}
|
|
763
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
764
|
+
try {
|
|
765
|
+
if (input.idempotencyKey) {
|
|
766
|
+
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
767
|
+
if (existing) {
|
|
768
|
+
this.db.exec("COMMIT");
|
|
769
|
+
return rowToWorkflowRun(existing);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
const runId = genId();
|
|
773
|
+
this.db.query(`INSERT INTO workflow_runs (id, workflow_id, workflow_name, loop_id, loop_run_id, scheduled_for, idempotency_key,
|
|
774
|
+
status, started_at, finished_at, duration_ms, error, created_at, updated_at)
|
|
775
|
+
VALUES ($id, $workflowId, $workflowName, $loopId, $loopRunId, $scheduledFor, $idempotencyKey,
|
|
776
|
+
'running', $started, NULL, NULL, NULL, $created, $updated)`).run({
|
|
777
|
+
$id: runId,
|
|
778
|
+
$workflowId: input.workflow.id,
|
|
779
|
+
$workflowName: input.workflow.name,
|
|
780
|
+
$loopId: input.loop?.id ?? null,
|
|
781
|
+
$loopRunId: input.loopRun?.id ?? null,
|
|
782
|
+
$scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor ?? null,
|
|
783
|
+
$idempotencyKey: input.idempotencyKey ?? null,
|
|
784
|
+
$started: now,
|
|
785
|
+
$created: now,
|
|
786
|
+
$updated: now
|
|
787
|
+
});
|
|
788
|
+
input.workflow.steps.forEach((step, sequence) => {
|
|
789
|
+
const account = step.account ?? step.target.account;
|
|
790
|
+
this.db.query(`INSERT INTO workflow_step_runs (id, workflow_run_id, step_id, sequence, status, started_at, finished_at,
|
|
791
|
+
exit_code, pid, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
|
|
792
|
+
VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
|
|
793
|
+
$accountProfile, $accountTool, $created, $updated)`).run({
|
|
794
|
+
$id: genId(),
|
|
795
|
+
$workflowRunId: runId,
|
|
796
|
+
$stepId: step.id,
|
|
797
|
+
$sequence: sequence,
|
|
798
|
+
$accountProfile: account?.profile ?? null,
|
|
799
|
+
$accountTool: account?.tool ?? null,
|
|
800
|
+
$created: now,
|
|
801
|
+
$updated: now
|
|
802
|
+
});
|
|
803
|
+
});
|
|
804
|
+
this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
|
|
805
|
+
VALUES ($id, $workflowRunId, 1, 'created', NULL, $payload, $created)`).run({
|
|
806
|
+
$id: genId(),
|
|
807
|
+
$workflowRunId: runId,
|
|
808
|
+
$payload: JSON.stringify({
|
|
809
|
+
workflowId: input.workflow.id,
|
|
810
|
+
workflowName: input.workflow.name,
|
|
811
|
+
stepCount: input.workflow.steps.length,
|
|
812
|
+
loopId: input.loop?.id,
|
|
813
|
+
loopRunId: input.loopRun?.id
|
|
814
|
+
}),
|
|
815
|
+
$created: now
|
|
816
|
+
});
|
|
817
|
+
this.db.exec("COMMIT");
|
|
818
|
+
const run = this.getWorkflowRun(runId);
|
|
819
|
+
if (!run)
|
|
820
|
+
throw new Error(`workflow run not found after create: ${runId}`);
|
|
821
|
+
return run;
|
|
822
|
+
} catch (error) {
|
|
823
|
+
try {
|
|
824
|
+
this.db.exec("ROLLBACK");
|
|
825
|
+
} catch {}
|
|
826
|
+
throw error;
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
getWorkflowRun(id) {
|
|
830
|
+
const row = this.db.query("SELECT * FROM workflow_runs WHERE id = ?").get(id);
|
|
831
|
+
return row ? rowToWorkflowRun(row) : undefined;
|
|
832
|
+
}
|
|
833
|
+
requireWorkflowRun(id) {
|
|
834
|
+
const run = this.getWorkflowRun(id);
|
|
835
|
+
if (!run)
|
|
836
|
+
throw new Error(`workflow run not found: ${id}`);
|
|
837
|
+
return run;
|
|
838
|
+
}
|
|
839
|
+
listWorkflowRuns(opts = {}) {
|
|
840
|
+
const limit = opts.limit ?? 100;
|
|
841
|
+
let rows;
|
|
842
|
+
if (opts.workflowId) {
|
|
843
|
+
rows = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.workflowId, limit);
|
|
844
|
+
} else if (opts.loopRunId) {
|
|
845
|
+
rows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopRunId, limit);
|
|
846
|
+
} else {
|
|
847
|
+
rows = this.db.query("SELECT * FROM workflow_runs ORDER BY created_at DESC LIMIT ?").all(limit);
|
|
848
|
+
}
|
|
849
|
+
return rows.map(rowToWorkflowRun);
|
|
850
|
+
}
|
|
851
|
+
listWorkflowStepRuns(workflowRunId) {
|
|
852
|
+
const rows = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? ORDER BY sequence ASC").all(workflowRunId);
|
|
853
|
+
return rows.map(rowToWorkflowStepRun);
|
|
854
|
+
}
|
|
855
|
+
getWorkflowStepRun(workflowRunId, stepId) {
|
|
856
|
+
const row = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? AND step_id = ?").get(workflowRunId, stepId);
|
|
857
|
+
return row ? rowToWorkflowStepRun(row) : undefined;
|
|
858
|
+
}
|
|
859
|
+
isWorkflowRunTerminal(workflowRunId) {
|
|
860
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
861
|
+
return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
|
|
862
|
+
}
|
|
863
|
+
startWorkflowStepRun(workflowRunId, stepId) {
|
|
864
|
+
const now = nowIso();
|
|
865
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
866
|
+
SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
|
|
867
|
+
pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
868
|
+
WHERE workflow_run_id=$workflowRunId
|
|
869
|
+
AND step_id=$stepId
|
|
870
|
+
AND status IN ('pending', 'failed', 'timed_out')
|
|
871
|
+
AND EXISTS (
|
|
872
|
+
SELECT 1 FROM workflow_runs
|
|
873
|
+
WHERE id=$workflowRunId AND status='running'
|
|
874
|
+
)`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
|
|
875
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
876
|
+
if (!run)
|
|
877
|
+
throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
|
|
878
|
+
if (res.changes !== 1) {
|
|
879
|
+
throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
|
|
880
|
+
}
|
|
881
|
+
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
882
|
+
return run;
|
|
883
|
+
}
|
|
884
|
+
markWorkflowStepPid(workflowRunId, stepId, pid) {
|
|
885
|
+
const now = nowIso();
|
|
886
|
+
this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
|
|
887
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
|
|
888
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
889
|
+
if (!run)
|
|
890
|
+
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
891
|
+
return run;
|
|
892
|
+
}
|
|
893
|
+
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
894
|
+
const now = nowIso();
|
|
895
|
+
const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
|
|
896
|
+
const live = before.filter((step) => step.pid !== undefined && isProcessAlive(step.pid));
|
|
897
|
+
if (live.length > 0) {
|
|
898
|
+
throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
|
|
899
|
+
}
|
|
900
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
901
|
+
SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
|
|
902
|
+
stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
|
|
903
|
+
WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
|
|
904
|
+
if (before.length > 0) {
|
|
905
|
+
this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
|
|
906
|
+
reason,
|
|
907
|
+
recoveredSteps: before.map((step) => step.stepId)
|
|
908
|
+
});
|
|
909
|
+
}
|
|
910
|
+
return {
|
|
911
|
+
run: this.requireWorkflowRun(workflowRunId),
|
|
912
|
+
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
913
|
+
};
|
|
914
|
+
}
|
|
915
|
+
finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
|
|
916
|
+
const finishedAt = patch.finishedAt ?? nowIso();
|
|
917
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
918
|
+
pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
919
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
|
|
920
|
+
$workflowRunId: workflowRunId,
|
|
921
|
+
$stepId: stepId,
|
|
922
|
+
$status: patch.status,
|
|
923
|
+
$finished: finishedAt,
|
|
924
|
+
$exitCode: patch.exitCode ?? null,
|
|
925
|
+
$durationMs: patch.durationMs ?? null,
|
|
926
|
+
$stdout: patch.stdout ?? null,
|
|
927
|
+
$stderr: patch.stderr ?? null,
|
|
928
|
+
$error: patch.error ?? null,
|
|
929
|
+
$updated: finishedAt
|
|
930
|
+
});
|
|
931
|
+
if (res.changes === 1) {
|
|
932
|
+
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
933
|
+
exitCode: patch.exitCode,
|
|
934
|
+
error: patch.error
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
938
|
+
if (!run)
|
|
939
|
+
throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
|
|
940
|
+
return run;
|
|
941
|
+
}
|
|
942
|
+
skipWorkflowStepRun(workflowRunId, stepId, reason) {
|
|
943
|
+
const now = nowIso();
|
|
944
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
|
|
945
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
|
|
946
|
+
if (res.changes === 1)
|
|
947
|
+
this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
|
|
948
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
949
|
+
if (!run)
|
|
950
|
+
throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
|
|
951
|
+
return run;
|
|
952
|
+
}
|
|
953
|
+
finalizeWorkflowRun(workflowRunId, status, patch = {}) {
|
|
954
|
+
const finishedAt = patch.finishedAt ?? nowIso();
|
|
955
|
+
const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
|
|
956
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
|
|
957
|
+
$id: workflowRunId,
|
|
958
|
+
$status: status,
|
|
959
|
+
$finished: finishedAt,
|
|
960
|
+
$durationMs: patch.durationMs ?? null,
|
|
961
|
+
$error: patch.error ?? null,
|
|
962
|
+
$updated: finishedAt
|
|
963
|
+
});
|
|
964
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
965
|
+
if (!run)
|
|
966
|
+
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
967
|
+
if (res.changes === 1)
|
|
968
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
969
|
+
return run;
|
|
970
|
+
}
|
|
971
|
+
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
972
|
+
const now = nowIso();
|
|
973
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
974
|
+
try {
|
|
975
|
+
const run = this.requireWorkflowRun(workflowRunId);
|
|
976
|
+
if (!["succeeded", "failed", "timed_out", "cancelled"].includes(run.status)) {
|
|
977
|
+
this.db.query(`UPDATE workflow_runs
|
|
978
|
+
SET status='cancelled', finished_at=$finished, error=$reason, updated_at=$updated
|
|
979
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRunId, $finished: now, $reason: reason, $updated: now });
|
|
980
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
981
|
+
SET status='cancelled', finished_at=$finished, pid=NULL, error=$reason, updated_at=$updated
|
|
982
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $finished: now, $reason: reason, $updated: now });
|
|
983
|
+
this.appendWorkflowEvent(workflowRunId, "cancelled", undefined, { reason });
|
|
984
|
+
}
|
|
985
|
+
this.db.exec("COMMIT");
|
|
986
|
+
return this.requireWorkflowRun(workflowRunId);
|
|
987
|
+
} catch (error) {
|
|
988
|
+
try {
|
|
989
|
+
this.db.exec("ROLLBACK");
|
|
990
|
+
} catch {}
|
|
991
|
+
throw error;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
|
|
995
|
+
const now = nowIso();
|
|
996
|
+
const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
|
|
997
|
+
const sequence = (current?.sequence ?? 0) + 1;
|
|
998
|
+
const id = genId();
|
|
999
|
+
this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
|
|
1000
|
+
VALUES ($id, $workflowRunId, $sequence, $eventType, $stepId, $payload, $created)`).run({
|
|
1001
|
+
$id: id,
|
|
1002
|
+
$workflowRunId: workflowRunId,
|
|
1003
|
+
$sequence: sequence,
|
|
1004
|
+
$eventType: eventType,
|
|
1005
|
+
$stepId: stepId ?? null,
|
|
1006
|
+
$payload: payload ? JSON.stringify(payload) : null,
|
|
1007
|
+
$created: now
|
|
1008
|
+
});
|
|
1009
|
+
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
1010
|
+
if (!event)
|
|
1011
|
+
throw new Error(`workflow event not found after append: ${id}`);
|
|
1012
|
+
return rowToWorkflowEvent(event);
|
|
1013
|
+
}
|
|
1014
|
+
listWorkflowEvents(workflowRunId, limit = 200) {
|
|
1015
|
+
const rows = this.db.query("SELECT * FROM workflow_events WHERE workflow_run_id = ? ORDER BY sequence ASC LIMIT ?").all(workflowRunId, limit);
|
|
1016
|
+
return rows.map(rowToWorkflowEvent);
|
|
1017
|
+
}
|
|
456
1018
|
hasRunningRun(loopId) {
|
|
457
1019
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
458
1020
|
return (row?.count ?? 0) > 0;
|
|
459
1021
|
}
|
|
1022
|
+
markRunPid(id, pid, claimedBy) {
|
|
1023
|
+
const now = nowIso();
|
|
1024
|
+
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1025
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
|
|
1026
|
+
if (res.changes !== 1)
|
|
1027
|
+
return;
|
|
1028
|
+
return this.getRun(id);
|
|
1029
|
+
}
|
|
1030
|
+
hasLiveWorkflowStepProcesses(loopRunId) {
|
|
1031
|
+
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
|
|
1032
|
+
FROM workflow_runs wr
|
|
1033
|
+
JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
|
|
1034
|
+
WHERE wr.loop_run_id = ?
|
|
1035
|
+
AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1036
|
+
AND wsr.status = 'running'
|
|
1037
|
+
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
1038
|
+
return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
|
|
1039
|
+
}
|
|
460
1040
|
createSkippedRun(loop, scheduledFor, reason) {
|
|
461
1041
|
const now = nowIso();
|
|
462
1042
|
const run = {
|
|
@@ -504,6 +1084,14 @@ class Store {
|
|
|
504
1084
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
505
1085
|
if (existing) {
|
|
506
1086
|
if (existing.status === "running") {
|
|
1087
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid && isProcessAlive(existing.pid)) {
|
|
1088
|
+
this.db.exec("COMMIT");
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && this.hasLiveWorkflowStepProcesses(existing.id)) {
|
|
1092
|
+
this.db.exec("COMMIT");
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
507
1095
|
const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
|
|
508
1096
|
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
509
1097
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
@@ -573,10 +1161,9 @@ class Store {
|
|
|
573
1161
|
throw error;
|
|
574
1162
|
}
|
|
575
1163
|
}
|
|
576
|
-
finalizeRun(id, patch) {
|
|
1164
|
+
finalizeRun(id, patch, opts = {}) {
|
|
577
1165
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
578
|
-
|
|
579
|
-
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run({
|
|
1166
|
+
const params = {
|
|
580
1167
|
$id: id,
|
|
581
1168
|
$status: patch.status,
|
|
582
1169
|
$finished: finishedAt,
|
|
@@ -586,13 +1173,29 @@ class Store {
|
|
|
586
1173
|
$stdout: patch.stdout ?? null,
|
|
587
1174
|
$stderr: patch.stderr ?? null,
|
|
588
1175
|
$error: patch.error ?? null,
|
|
589
|
-
$updated: finishedAt
|
|
590
|
-
|
|
1176
|
+
$updated: finishedAt,
|
|
1177
|
+
$claimedBy: opts.claimedBy ?? null,
|
|
1178
|
+
$now: (opts.now ?? new Date).toISOString()
|
|
1179
|
+
};
|
|
1180
|
+
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1181
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
1182
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1183
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
591
1184
|
const run = this.getRun(id);
|
|
592
1185
|
if (!run)
|
|
593
1186
|
throw new Error(`run not found after finalize: ${id}`);
|
|
1187
|
+
if (opts.claimedBy && res.changes !== 1)
|
|
1188
|
+
return run;
|
|
594
1189
|
return run;
|
|
595
1190
|
}
|
|
1191
|
+
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
|
|
1192
|
+
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
1193
|
+
const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
|
|
1194
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
|
|
1195
|
+
if (res.changes !== 1)
|
|
1196
|
+
return;
|
|
1197
|
+
return this.getRun(id);
|
|
1198
|
+
}
|
|
596
1199
|
listRuns(opts = {}) {
|
|
597
1200
|
const limit = opts.limit ?? 100;
|
|
598
1201
|
let rows;
|
|
@@ -611,8 +1214,26 @@ class Store {
|
|
|
611
1214
|
const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
|
|
612
1215
|
const recovered = [];
|
|
613
1216
|
for (const row of rows) {
|
|
1217
|
+
if (row.pid && isProcessAlive(row.pid))
|
|
1218
|
+
continue;
|
|
1219
|
+
if (this.hasLiveWorkflowStepProcesses(row.id))
|
|
1220
|
+
continue;
|
|
1221
|
+
const finished = now.toISOString();
|
|
614
1222
|
this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
|
|
615
|
-
error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished:
|
|
1223
|
+
error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
|
|
1224
|
+
const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
|
|
1225
|
+
for (const workflowRow of workflowRows) {
|
|
1226
|
+
this.db.query(`UPDATE workflow_runs
|
|
1227
|
+
SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1228
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
|
|
1229
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
1230
|
+
SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1231
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
|
|
1232
|
+
this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
|
|
1233
|
+
error: "parent loop run lease expired before completion",
|
|
1234
|
+
loopRunId: row.id
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
616
1237
|
const run = this.getRun(row.id);
|
|
617
1238
|
if (run)
|
|
618
1239
|
recovered.push(run);
|
|
@@ -692,10 +1313,126 @@ import { hostname as hostname2 } from "os";
|
|
|
692
1313
|
import { spawn as spawn2 } from "child_process";
|
|
693
1314
|
|
|
694
1315
|
// src/lib/executor.ts
|
|
695
|
-
import { spawn } from "child_process";
|
|
1316
|
+
import { spawn, spawnSync as spawnSync2 } from "child_process";
|
|
696
1317
|
import { once } from "events";
|
|
1318
|
+
import { existsSync as existsSync2 } from "fs";
|
|
1319
|
+
|
|
1320
|
+
// src/lib/accounts.ts
|
|
1321
|
+
import { spawnSync } from "child_process";
|
|
1322
|
+
import { existsSync } from "fs";
|
|
1323
|
+
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
1324
|
+
function accountToolForProvider(provider) {
|
|
1325
|
+
switch (provider) {
|
|
1326
|
+
case "claude":
|
|
1327
|
+
return "claude";
|
|
1328
|
+
case "cursor":
|
|
1329
|
+
return "cursor";
|
|
1330
|
+
case "codewith":
|
|
1331
|
+
return "codewith";
|
|
1332
|
+
case "aicopilot":
|
|
1333
|
+
return "aicopilot";
|
|
1334
|
+
case "opencode":
|
|
1335
|
+
return "opencode";
|
|
1336
|
+
case "codex":
|
|
1337
|
+
return "codex";
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
function parseExportValue(raw) {
|
|
1341
|
+
try {
|
|
1342
|
+
return JSON.parse(raw);
|
|
1343
|
+
} catch {
|
|
1344
|
+
return raw.replace(/^['"]|['"]$/g, "");
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
function parseAccountExportLines(output) {
|
|
1348
|
+
const env = {};
|
|
1349
|
+
for (const line of output.split(/\r?\n/)) {
|
|
1350
|
+
const match = EXPORT_RE.exec(line.trim());
|
|
1351
|
+
if (!match)
|
|
1352
|
+
continue;
|
|
1353
|
+
env[match[1]] = parseExportValue(match[2]);
|
|
1354
|
+
}
|
|
1355
|
+
return env;
|
|
1356
|
+
}
|
|
1357
|
+
function primaryAccountDir(output) {
|
|
1358
|
+
for (const line of output.split(/\r?\n/)) {
|
|
1359
|
+
const match = EXPORT_RE.exec(line.trim());
|
|
1360
|
+
if (!match)
|
|
1361
|
+
continue;
|
|
1362
|
+
return parseExportValue(match[2]);
|
|
1363
|
+
}
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
function accountDirEnvVar(tool) {
|
|
1367
|
+
switch (tool) {
|
|
1368
|
+
case "claude":
|
|
1369
|
+
return "CLAUDE_CONFIG_DIR";
|
|
1370
|
+
case "codex":
|
|
1371
|
+
case "codex-app":
|
|
1372
|
+
return "CODEX_HOME";
|
|
1373
|
+
case "cursor":
|
|
1374
|
+
return "CURSOR_CONFIG_DIR";
|
|
1375
|
+
case "opencode":
|
|
1376
|
+
return "OPENCODE_CONFIG_DIR";
|
|
1377
|
+
case "codewith":
|
|
1378
|
+
return "CODEWITH_HOME";
|
|
1379
|
+
case "aicopilot":
|
|
1380
|
+
return "AICOPILOT_CONFIG_DIR";
|
|
1381
|
+
default:
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
function resolveAccountEnv(account, toolHint, env) {
|
|
1386
|
+
if (!account)
|
|
1387
|
+
return {};
|
|
1388
|
+
const tool = account.tool ?? toolHint;
|
|
1389
|
+
if (!tool)
|
|
1390
|
+
throw new Error("account.tool is required when no provider tool can be inferred");
|
|
1391
|
+
const result = spawnSync("accounts", ["env", account.profile, "--tool", tool], {
|
|
1392
|
+
encoding: "utf8",
|
|
1393
|
+
env,
|
|
1394
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1395
|
+
});
|
|
1396
|
+
if (result.error) {
|
|
1397
|
+
throw new Error(`failed to run accounts env for ${account.profile}/${tool}: ${result.error.message}`);
|
|
1398
|
+
}
|
|
1399
|
+
if ((result.status ?? 0) !== 0) {
|
|
1400
|
+
const stderr = result.stderr.trim();
|
|
1401
|
+
throw new Error(`accounts env failed for ${account.profile}/${tool}${stderr ? `: ${stderr}` : ""}`);
|
|
1402
|
+
}
|
|
1403
|
+
const accountEnv = parseAccountExportLines(result.stdout);
|
|
1404
|
+
const profileDir = (accountDirEnvVar(tool) ? accountEnv[accountDirEnvVar(tool)] : undefined) ?? primaryAccountDir(result.stdout);
|
|
1405
|
+
if (!profileDir)
|
|
1406
|
+
throw new Error(`accounts env returned no profile directory for ${account.profile}/${tool}`);
|
|
1407
|
+
if (!existsSync(profileDir))
|
|
1408
|
+
throw new Error(`account profile directory does not exist for ${account.profile}/${tool}: ${profileDir}`);
|
|
1409
|
+
return {
|
|
1410
|
+
...accountEnv,
|
|
1411
|
+
LOOPS_ACCOUNT_PROFILE: account.profile,
|
|
1412
|
+
LOOPS_ACCOUNT_TOOL: tool
|
|
1413
|
+
};
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
// src/lib/executor.ts
|
|
697
1417
|
var DEFAULT_TIMEOUT_MS = 30 * 60000;
|
|
698
1418
|
var DEFAULT_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
1419
|
+
var AUTH_ENV_KEYS = [
|
|
1420
|
+
"CLAUDE_CONFIG_DIR",
|
|
1421
|
+
"CODEWITH_HOME",
|
|
1422
|
+
"CODEX_HOME",
|
|
1423
|
+
"CURSOR_CONFIG_DIR",
|
|
1424
|
+
"OPENCODE_CONFIG_DIR",
|
|
1425
|
+
"AICOPILOT_CONFIG_DIR",
|
|
1426
|
+
"ANTHROPIC_API_KEY",
|
|
1427
|
+
"OPENAI_API_KEY",
|
|
1428
|
+
"OPENROUTER_API_KEY",
|
|
1429
|
+
"GITHUB_TOKEN",
|
|
1430
|
+
"GH_TOKEN",
|
|
1431
|
+
"XDG_CONFIG_HOME",
|
|
1432
|
+
"XDG_DATA_HOME",
|
|
1433
|
+
"XDG_STATE_HOME",
|
|
1434
|
+
"XDG_CACHE_HOME"
|
|
1435
|
+
];
|
|
699
1436
|
function appendBounded(current, chunk, maxBytes) {
|
|
700
1437
|
const next = current + chunk.toString("utf8");
|
|
701
1438
|
if (Buffer.byteLength(next, "utf8") <= maxBytes)
|
|
@@ -734,6 +1471,8 @@ function providerCommand(provider) {
|
|
|
734
1471
|
return "aicopilot";
|
|
735
1472
|
case "opencode":
|
|
736
1473
|
return "opencode";
|
|
1474
|
+
case "codex":
|
|
1475
|
+
return "codex";
|
|
737
1476
|
}
|
|
738
1477
|
}
|
|
739
1478
|
function agentArgs(target) {
|
|
@@ -770,6 +1509,16 @@ function agentArgs(target) {
|
|
|
770
1509
|
args.push("--agent", target.agent);
|
|
771
1510
|
args.push(...target.extraArgs ?? [], target.prompt);
|
|
772
1511
|
return args;
|
|
1512
|
+
case "codex":
|
|
1513
|
+
args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
|
|
1514
|
+
if (isolation === "safe")
|
|
1515
|
+
args.push("--ignore-rules");
|
|
1516
|
+
if (target.cwd)
|
|
1517
|
+
args.push("--cd", target.cwd);
|
|
1518
|
+
if (target.model)
|
|
1519
|
+
args.push("--model", target.model);
|
|
1520
|
+
args.push(...target.extraArgs ?? [], target.prompt);
|
|
1521
|
+
return args;
|
|
773
1522
|
case "aicopilot":
|
|
774
1523
|
args.push("run", "--format", "json");
|
|
775
1524
|
if (isolation === "safe")
|
|
@@ -796,8 +1545,7 @@ function agentArgs(target) {
|
|
|
796
1545
|
return args;
|
|
797
1546
|
}
|
|
798
1547
|
}
|
|
799
|
-
function commandSpec(
|
|
800
|
-
const target = loop.target;
|
|
1548
|
+
function commandSpec(target) {
|
|
801
1549
|
if (target.type === "command") {
|
|
802
1550
|
const commandTarget = target;
|
|
803
1551
|
return {
|
|
@@ -806,7 +1554,9 @@ function commandSpec(loop) {
|
|
|
806
1554
|
cwd: commandTarget.cwd,
|
|
807
1555
|
shell: commandTarget.shell,
|
|
808
1556
|
env: commandTarget.env,
|
|
809
|
-
timeoutMs: commandTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
|
1557
|
+
timeoutMs: commandTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
1558
|
+
account: commandTarget.account,
|
|
1559
|
+
accountTool: commandTarget.account?.tool
|
|
810
1560
|
};
|
|
811
1561
|
}
|
|
812
1562
|
const agentTarget = target;
|
|
@@ -814,11 +1564,61 @@ function commandSpec(loop) {
|
|
|
814
1564
|
command: providerCommand(agentTarget.provider),
|
|
815
1565
|
args: agentArgs(agentTarget),
|
|
816
1566
|
cwd: agentTarget.cwd,
|
|
817
|
-
timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
|
1567
|
+
timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
1568
|
+
account: agentTarget.account,
|
|
1569
|
+
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
|
|
818
1570
|
};
|
|
819
1571
|
}
|
|
820
|
-
|
|
821
|
-
const
|
|
1572
|
+
function executionEnv(spec, metadata, opts) {
|
|
1573
|
+
const env = { ...opts.env ?? process.env };
|
|
1574
|
+
if (spec.account) {
|
|
1575
|
+
const accountEnv = resolveAccountEnv(spec.account, spec.accountTool, env);
|
|
1576
|
+
for (const key of AUTH_ENV_KEYS)
|
|
1577
|
+
delete env[key];
|
|
1578
|
+
Object.assign(env, accountEnv);
|
|
1579
|
+
}
|
|
1580
|
+
Object.assign(env, spec.env ?? {});
|
|
1581
|
+
if (metadata.loopId)
|
|
1582
|
+
env.LOOPS_LOOP_ID = metadata.loopId;
|
|
1583
|
+
if (metadata.loopName)
|
|
1584
|
+
env.LOOPS_LOOP_NAME = metadata.loopName;
|
|
1585
|
+
if (metadata.runId)
|
|
1586
|
+
env.LOOPS_RUN_ID = metadata.runId;
|
|
1587
|
+
if (metadata.scheduledFor)
|
|
1588
|
+
env.LOOPS_SCHEDULED_FOR = metadata.scheduledFor;
|
|
1589
|
+
if (metadata.workflowId)
|
|
1590
|
+
env.LOOPS_WORKFLOW_ID = metadata.workflowId;
|
|
1591
|
+
if (metadata.workflowName)
|
|
1592
|
+
env.LOOPS_WORKFLOW_NAME = metadata.workflowName;
|
|
1593
|
+
if (metadata.workflowRunId)
|
|
1594
|
+
env.LOOPS_WORKFLOW_RUN_ID = metadata.workflowRunId;
|
|
1595
|
+
if (metadata.workflowStepId)
|
|
1596
|
+
env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
|
|
1597
|
+
return env;
|
|
1598
|
+
}
|
|
1599
|
+
function commandExists(command, env) {
|
|
1600
|
+
if (command.includes("/") && existsSync2(command))
|
|
1601
|
+
return true;
|
|
1602
|
+
const result = spawnSync2("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], {
|
|
1603
|
+
env,
|
|
1604
|
+
stdio: "ignore"
|
|
1605
|
+
});
|
|
1606
|
+
return (result.status ?? 1) === 0;
|
|
1607
|
+
}
|
|
1608
|
+
function preflightTarget(target, metadata = {}, opts = {}) {
|
|
1609
|
+
const spec = commandSpec(target);
|
|
1610
|
+
const env = executionEnv(spec, metadata, opts);
|
|
1611
|
+
if (!spec.shell && !commandExists(spec.command, env)) {
|
|
1612
|
+
throw new Error(`Executable not found in PATH: ${spec.command}`);
|
|
1613
|
+
}
|
|
1614
|
+
return {
|
|
1615
|
+
command: spec.command,
|
|
1616
|
+
accountProfile: spec.account?.profile,
|
|
1617
|
+
accountTool: spec.accountTool
|
|
1618
|
+
};
|
|
1619
|
+
}
|
|
1620
|
+
async function executeTarget(target, metadata = {}, opts = {}) {
|
|
1621
|
+
const spec = commandSpec(target);
|
|
822
1622
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
823
1623
|
const startedAt = nowIso();
|
|
824
1624
|
let stdout = "";
|
|
@@ -826,14 +1626,18 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
826
1626
|
let timedOut = false;
|
|
827
1627
|
let exitCode;
|
|
828
1628
|
let error;
|
|
829
|
-
const env =
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
1629
|
+
const env = executionEnv(spec, metadata, opts);
|
|
1630
|
+
if (!spec.shell && !commandExists(spec.command, env)) {
|
|
1631
|
+
return {
|
|
1632
|
+
status: "failed",
|
|
1633
|
+
stdout: "",
|
|
1634
|
+
stderr: "",
|
|
1635
|
+
error: `Executable not found in PATH: ${spec.command}`,
|
|
1636
|
+
startedAt,
|
|
1637
|
+
finishedAt: nowIso(),
|
|
1638
|
+
durationMs: 0
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
837
1641
|
const child = spawn(spec.command, spec.args, {
|
|
838
1642
|
cwd: spec.cwd,
|
|
839
1643
|
env,
|
|
@@ -841,6 +1645,16 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
841
1645
|
detached: true,
|
|
842
1646
|
stdio: ["ignore", "pipe", "pipe"]
|
|
843
1647
|
});
|
|
1648
|
+
if (child.pid)
|
|
1649
|
+
opts.onSpawn?.(child.pid);
|
|
1650
|
+
const abortHandler = () => {
|
|
1651
|
+
error = "cancelled";
|
|
1652
|
+
if (child.pid)
|
|
1653
|
+
killProcessGroup(child.pid);
|
|
1654
|
+
};
|
|
1655
|
+
if (opts.signal?.aborted)
|
|
1656
|
+
abortHandler();
|
|
1657
|
+
opts.signal?.addEventListener("abort", abortHandler, { once: true });
|
|
844
1658
|
child.stdout.on("data", (chunk) => {
|
|
845
1659
|
stdout = appendBounded(stdout, chunk, maxOutputBytes);
|
|
846
1660
|
});
|
|
@@ -863,6 +1677,7 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
863
1677
|
error = err instanceof Error ? err.message : String(err);
|
|
864
1678
|
} finally {
|
|
865
1679
|
clearTimeout(timer);
|
|
1680
|
+
opts.signal?.removeEventListener("abort", abortHandler);
|
|
866
1681
|
}
|
|
867
1682
|
const finishedAt = nowIso();
|
|
868
1683
|
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
@@ -903,12 +1718,243 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
903
1718
|
durationMs
|
|
904
1719
|
};
|
|
905
1720
|
}
|
|
1721
|
+
async function executeLoop(loop, run, opts = {}) {
|
|
1722
|
+
if (loop.target.type === "workflow") {
|
|
1723
|
+
throw new Error("workflow loop targets must be executed with executeLoopTarget");
|
|
1724
|
+
}
|
|
1725
|
+
return executeTarget(loop.target, {
|
|
1726
|
+
loopId: loop.id,
|
|
1727
|
+
loopName: loop.name,
|
|
1728
|
+
runId: run.id,
|
|
1729
|
+
scheduledFor: run.scheduledFor
|
|
1730
|
+
}, opts);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
// src/lib/workflow-runner.ts
|
|
1734
|
+
function targetWithStepAccount(step) {
|
|
1735
|
+
const account = step.account ?? step.target.account;
|
|
1736
|
+
const timeoutMs = step.timeoutMs ?? step.target.timeoutMs;
|
|
1737
|
+
if (!account && timeoutMs === step.target.timeoutMs)
|
|
1738
|
+
return step.target;
|
|
1739
|
+
return { ...step.target, account, timeoutMs };
|
|
1740
|
+
}
|
|
1741
|
+
function workflowResult(workflowRun, status, startedAt, finishedAt, stdout, error) {
|
|
1742
|
+
const executorStatus = status === "succeeded" ? "succeeded" : status === "timed_out" ? "timed_out" : "failed";
|
|
1743
|
+
return {
|
|
1744
|
+
status: executorStatus,
|
|
1745
|
+
exitCode: executorStatus === "succeeded" ? 0 : 1,
|
|
1746
|
+
stdout,
|
|
1747
|
+
stderr: "",
|
|
1748
|
+
error,
|
|
1749
|
+
startedAt,
|
|
1750
|
+
finishedAt,
|
|
1751
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
async function executeWorkflow(store, workflow, opts = {}) {
|
|
1755
|
+
const run = store.createWorkflowRun({
|
|
1756
|
+
workflow,
|
|
1757
|
+
loop: opts.loop,
|
|
1758
|
+
loopRun: opts.loopRun,
|
|
1759
|
+
scheduledFor: opts.scheduledFor,
|
|
1760
|
+
idempotencyKey: opts.idempotencyKey
|
|
1761
|
+
});
|
|
1762
|
+
const startedAt = run.startedAt ?? nowIso();
|
|
1763
|
+
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
1764
|
+
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
1765
|
+
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), JSON.stringify({ workflowRun: run, steps: steps2 }, null, 2), run.error);
|
|
1766
|
+
}
|
|
1767
|
+
const ordered = workflowExecutionOrder(workflow);
|
|
1768
|
+
const byId = new Map(workflow.steps.map((step) => [step.id, step]));
|
|
1769
|
+
let blockingError;
|
|
1770
|
+
let terminalStatus = "succeeded";
|
|
1771
|
+
for (const step of ordered) {
|
|
1772
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
1773
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
1774
|
+
blockingError = "workflow run was cancelled";
|
|
1775
|
+
break;
|
|
1776
|
+
}
|
|
1777
|
+
const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
1778
|
+
if (pendingTimeout) {
|
|
1779
|
+
terminalStatus = "timed_out";
|
|
1780
|
+
blockingError = pendingTimeout;
|
|
1781
|
+
break;
|
|
1782
|
+
}
|
|
1783
|
+
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
1784
|
+
if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
|
|
1785
|
+
continue;
|
|
1786
|
+
const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
|
|
1787
|
+
const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
|
|
1788
|
+
const dependencyStep = byId.get(dependencyId);
|
|
1789
|
+
if (dependencyRun?.status === "succeeded")
|
|
1790
|
+
return false;
|
|
1791
|
+
return !dependencyStep?.continueOnFailure;
|
|
1792
|
+
});
|
|
1793
|
+
if (blockedBy) {
|
|
1794
|
+
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
|
|
1795
|
+
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
1796
|
+
terminalStatus = "failed";
|
|
1797
|
+
continue;
|
|
1798
|
+
}
|
|
1799
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id);
|
|
1800
|
+
if (startedStep.status !== "running") {
|
|
1801
|
+
terminalStatus = "failed";
|
|
1802
|
+
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
1803
|
+
break;
|
|
1804
|
+
}
|
|
1805
|
+
const metadata = {
|
|
1806
|
+
loopId: opts.loop?.id,
|
|
1807
|
+
loopName: opts.loop?.name,
|
|
1808
|
+
runId: opts.loopRun?.id,
|
|
1809
|
+
scheduledFor: opts.loopRun?.scheduledFor ?? opts.scheduledFor,
|
|
1810
|
+
workflowId: workflow.id,
|
|
1811
|
+
workflowName: workflow.name,
|
|
1812
|
+
workflowRunId: run.id,
|
|
1813
|
+
workflowStepId: step.id
|
|
1814
|
+
};
|
|
1815
|
+
let result;
|
|
1816
|
+
const controller = new AbortController;
|
|
1817
|
+
const externalAbort = () => controller.abort();
|
|
1818
|
+
if (opts.signal?.aborted)
|
|
1819
|
+
controller.abort();
|
|
1820
|
+
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
1821
|
+
const cancelTimer = setInterval(() => {
|
|
1822
|
+
if (store.getWorkflowRun(run.id)?.status === "cancelled")
|
|
1823
|
+
controller.abort();
|
|
1824
|
+
}, opts.cancelPollMs ?? 500);
|
|
1825
|
+
cancelTimer.unref();
|
|
1826
|
+
try {
|
|
1827
|
+
result = await executeTarget(targetWithStepAccount(step), metadata, {
|
|
1828
|
+
...opts,
|
|
1829
|
+
signal: controller.signal,
|
|
1830
|
+
onSpawn: (pid) => {
|
|
1831
|
+
store.markWorkflowStepPid(run.id, step.id, pid);
|
|
1832
|
+
opts.onSpawn?.(pid);
|
|
1833
|
+
}
|
|
1834
|
+
});
|
|
1835
|
+
} catch (error) {
|
|
1836
|
+
const finishedAt2 = nowIso();
|
|
1837
|
+
result = {
|
|
1838
|
+
status: "failed",
|
|
1839
|
+
stdout: "",
|
|
1840
|
+
stderr: "",
|
|
1841
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1842
|
+
startedAt: startedStep.startedAt ?? finishedAt2,
|
|
1843
|
+
finishedAt: finishedAt2,
|
|
1844
|
+
durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
|
|
1845
|
+
};
|
|
1846
|
+
} finally {
|
|
1847
|
+
clearInterval(cancelTimer);
|
|
1848
|
+
opts.signal?.removeEventListener("abort", externalAbort);
|
|
1849
|
+
}
|
|
1850
|
+
const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
1851
|
+
if (timeoutMessage && result.status === "failed") {
|
|
1852
|
+
result = { ...result, status: "timed_out", error: timeoutMessage };
|
|
1853
|
+
}
|
|
1854
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
1855
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
1856
|
+
blockingError = "workflow run was cancelled";
|
|
1857
|
+
break;
|
|
1858
|
+
}
|
|
1859
|
+
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
1860
|
+
status: result.status,
|
|
1861
|
+
finishedAt: result.finishedAt,
|
|
1862
|
+
durationMs: result.durationMs,
|
|
1863
|
+
stdout: result.stdout,
|
|
1864
|
+
stderr: result.stderr,
|
|
1865
|
+
exitCode: result.exitCode,
|
|
1866
|
+
error: result.error
|
|
1867
|
+
});
|
|
1868
|
+
if (result.status !== "succeeded" && !step.continueOnFailure) {
|
|
1869
|
+
terminalStatus = result.status;
|
|
1870
|
+
blockingError = `step ${step.id} ${result.status}${result.error ? `: ${result.error}` : ""}`;
|
|
1871
|
+
break;
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
if (terminalStatus !== "succeeded") {
|
|
1875
|
+
for (const step of ordered) {
|
|
1876
|
+
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
1877
|
+
if (existing?.status === "pending" || existing?.status === "running") {
|
|
1878
|
+
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
const finishedAt = nowIso();
|
|
1883
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
1884
|
+
const terminalRun = store.requireWorkflowRun(run.id);
|
|
1885
|
+
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
1886
|
+
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
1887
|
+
}
|
|
1888
|
+
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
1889
|
+
finishedAt,
|
|
1890
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
1891
|
+
error: blockingError
|
|
1892
|
+
});
|
|
1893
|
+
const steps = store.listWorkflowStepRuns(run.id);
|
|
1894
|
+
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
1895
|
+
}
|
|
1896
|
+
function preflightWorkflow(workflow, opts = {}) {
|
|
1897
|
+
return workflowExecutionOrder(workflow).map((step) => preflightTarget(targetWithStepAccount(step), {
|
|
1898
|
+
workflowId: workflow.id,
|
|
1899
|
+
workflowName: workflow.name,
|
|
1900
|
+
workflowStepId: step.id
|
|
1901
|
+
}, opts));
|
|
1902
|
+
}
|
|
1903
|
+
async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
1904
|
+
if (loop.target.type !== "workflow")
|
|
1905
|
+
return executeLoop(loop, run, opts);
|
|
1906
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
1907
|
+
const controller = loop.target.timeoutMs ? new AbortController : undefined;
|
|
1908
|
+
let workflowTimedOut = false;
|
|
1909
|
+
const externalAbort = () => controller?.abort();
|
|
1910
|
+
if (controller && opts.signal?.aborted)
|
|
1911
|
+
controller.abort();
|
|
1912
|
+
if (controller)
|
|
1913
|
+
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
1914
|
+
const timer = controller ? setTimeout(() => {
|
|
1915
|
+
workflowTimedOut = true;
|
|
1916
|
+
controller.abort();
|
|
1917
|
+
}, loop.target.timeoutMs) : undefined;
|
|
1918
|
+
timer?.unref();
|
|
1919
|
+
try {
|
|
1920
|
+
return await executeWorkflow(store, workflow, {
|
|
1921
|
+
...opts,
|
|
1922
|
+
signal: controller?.signal ?? opts.signal,
|
|
1923
|
+
signalTimeoutMessage: () => workflowTimedOut && loop.target.type === "workflow" ? `workflow timed out after ${loop.target.timeoutMs}ms` : undefined,
|
|
1924
|
+
loop,
|
|
1925
|
+
loopRun: run,
|
|
1926
|
+
scheduledFor: run.scheduledFor,
|
|
1927
|
+
idempotencyKey: `${loop.id}:${run.scheduledFor}:attempt:${run.attempt}`
|
|
1928
|
+
});
|
|
1929
|
+
} finally {
|
|
1930
|
+
if (timer)
|
|
1931
|
+
clearTimeout(timer);
|
|
1932
|
+
if (controller)
|
|
1933
|
+
opts.signal?.removeEventListener("abort", externalAbort);
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
906
1936
|
|
|
907
1937
|
// src/lib/scheduler.ts
|
|
1938
|
+
function manualRunScheduledFor(loop, now = new Date) {
|
|
1939
|
+
if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
1940
|
+
return loop.retryScheduledFor ?? loop.nextRunAt;
|
|
1941
|
+
}
|
|
1942
|
+
return now.toISOString();
|
|
1943
|
+
}
|
|
1944
|
+
function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
|
|
1945
|
+
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
1946
|
+
return false;
|
|
1947
|
+
return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
|
|
1948
|
+
}
|
|
908
1949
|
function nextAfterRetry(loop, now) {
|
|
909
1950
|
return new Date(now.getTime() + loop.retryDelayMs).toISOString();
|
|
910
1951
|
}
|
|
911
1952
|
function advanceLoop(store, loop, run, finishedAt, succeeded) {
|
|
1953
|
+
if (run.status === "running")
|
|
1954
|
+
return;
|
|
1955
|
+
const current = store.getLoop(loop.id);
|
|
1956
|
+
if (!current || current.status !== "active")
|
|
1957
|
+
return;
|
|
912
1958
|
const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
|
|
913
1959
|
if (shouldRetry) {
|
|
914
1960
|
store.updateLoop(loop.id, {
|
|
@@ -925,21 +1971,18 @@ function advanceLoop(store, loop, run, finishedAt, succeeded) {
|
|
|
925
1971
|
retryScheduledFor: undefined
|
|
926
1972
|
});
|
|
927
1973
|
}
|
|
928
|
-
async function
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
}
|
|
936
|
-
const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
|
|
937
|
-
if (!claim)
|
|
938
|
-
return;
|
|
939
|
-
deps.onRun?.(claim.run);
|
|
1974
|
+
async function executeClaimedRun(deps) {
|
|
1975
|
+
let heartbeat;
|
|
1976
|
+
const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
|
|
1977
|
+
heartbeat = setInterval(() => {
|
|
1978
|
+
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
|
|
1979
|
+
}, heartbeatEveryMs);
|
|
1980
|
+
heartbeat.unref();
|
|
940
1981
|
try {
|
|
941
|
-
const result = await (deps.execute ??
|
|
942
|
-
|
|
1982
|
+
const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
|
|
1983
|
+
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
|
|
1984
|
+
})))(deps.loop, deps.run);
|
|
1985
|
+
return deps.store.finalizeRun(deps.run.id, {
|
|
943
1986
|
status: result.status,
|
|
944
1987
|
finishedAt: result.finishedAt,
|
|
945
1988
|
durationMs: result.durationMs,
|
|
@@ -948,25 +1991,53 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
948
1991
|
exitCode: result.exitCode,
|
|
949
1992
|
error: result.error,
|
|
950
1993
|
pid: result.pid
|
|
1994
|
+
}, {
|
|
1995
|
+
claimedBy: deps.runnerId,
|
|
1996
|
+
now: deps.now?.() ?? new Date(result.finishedAt)
|
|
951
1997
|
});
|
|
952
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(result.finishedAt), result.status === "succeeded");
|
|
953
|
-
deps.onRun?.(finalRun);
|
|
954
|
-
return finalRun;
|
|
955
1998
|
} catch (err) {
|
|
956
|
-
deps.onError?.(
|
|
1999
|
+
deps.onError?.(deps.loop, err);
|
|
957
2000
|
const finishedAt = new Date;
|
|
958
|
-
|
|
2001
|
+
return deps.store.finalizeRun(deps.run.id, {
|
|
959
2002
|
status: "failed",
|
|
960
2003
|
finishedAt: finishedAt.toISOString(),
|
|
961
|
-
durationMs: finishedAt.getTime() - new Date(
|
|
2004
|
+
durationMs: finishedAt.getTime() - new Date(deps.run.startedAt ?? deps.run.createdAt).getTime(),
|
|
962
2005
|
stdout: "",
|
|
963
2006
|
stderr: "",
|
|
964
2007
|
error: err instanceof Error ? err.message : String(err)
|
|
2008
|
+
}, {
|
|
2009
|
+
claimedBy: deps.runnerId,
|
|
2010
|
+
now: deps.now?.() ?? finishedAt
|
|
965
2011
|
});
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
2012
|
+
} finally {
|
|
2013
|
+
if (heartbeat)
|
|
2014
|
+
clearInterval(heartbeat);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
async function runSlot(deps, loop, scheduledFor) {
|
|
2018
|
+
const now = deps.now?.() ?? new Date;
|
|
2019
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
2020
|
+
const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
|
|
2021
|
+
advanceLoop(deps.store, loop, skipped, now, true);
|
|
2022
|
+
deps.onRun?.(skipped);
|
|
2023
|
+
return skipped;
|
|
969
2024
|
}
|
|
2025
|
+
const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
|
|
2026
|
+
if (!claim)
|
|
2027
|
+
return;
|
|
2028
|
+
deps.onRun?.(claim.run);
|
|
2029
|
+
const finalRun = await executeClaimedRun({
|
|
2030
|
+
store: deps.store,
|
|
2031
|
+
runnerId: deps.runnerId,
|
|
2032
|
+
loop: claim.loop,
|
|
2033
|
+
run: claim.run,
|
|
2034
|
+
now: deps.now,
|
|
2035
|
+
execute: deps.execute,
|
|
2036
|
+
onError: deps.onError
|
|
2037
|
+
});
|
|
2038
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
|
|
2039
|
+
deps.onRun?.(finalRun);
|
|
2040
|
+
return finalRun;
|
|
970
2041
|
}
|
|
971
2042
|
async function tick(deps) {
|
|
972
2043
|
const now = deps.now?.() ?? new Date;
|
|
@@ -992,13 +2063,15 @@ async function tick(deps) {
|
|
|
992
2063
|
skipped.push(run);
|
|
993
2064
|
else
|
|
994
2065
|
completed.push(run);
|
|
2066
|
+
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
2067
|
+
break;
|
|
995
2068
|
}
|
|
996
2069
|
}
|
|
997
2070
|
return { claimed, completed, skipped, recovered, expired };
|
|
998
2071
|
}
|
|
999
2072
|
|
|
1000
2073
|
// src/daemon/control.ts
|
|
1001
|
-
import { existsSync, mkdirSync as mkdirSync3, readFileSync, rmSync, writeFileSync } from "fs";
|
|
2074
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync, rmSync, writeFileSync } from "fs";
|
|
1002
2075
|
import { hostname } from "os";
|
|
1003
2076
|
import { dirname as dirname2 } from "path";
|
|
1004
2077
|
|
|
@@ -1026,7 +2099,7 @@ async function runLoop(opts) {
|
|
|
1026
2099
|
|
|
1027
2100
|
// src/daemon/control.ts
|
|
1028
2101
|
function readPid(path = pidFilePath()) {
|
|
1029
|
-
if (!
|
|
2102
|
+
if (!existsSync3(path))
|
|
1030
2103
|
return;
|
|
1031
2104
|
try {
|
|
1032
2105
|
const pid = Number(readFileSync(path, "utf8").trim());
|
|
@@ -1141,6 +2214,7 @@ async function runDaemon(opts = {}) {
|
|
|
1141
2214
|
const ownStore = !opts.store;
|
|
1142
2215
|
const store = opts.store ?? new Store;
|
|
1143
2216
|
const leaseId = genId();
|
|
2217
|
+
const runnerId = `${hostname2()}:${process.pid}:${leaseId}`;
|
|
1144
2218
|
const intervalMs = opts.intervalMs ?? intervalFromEnv() ?? 1000;
|
|
1145
2219
|
const leaseTtlMs = opts.leaseTtlMs ?? Math.max(60000, intervalMs * 10);
|
|
1146
2220
|
const log = opts.log ?? ((message) => console.error(`[loops-daemon] ${message}`));
|
|
@@ -1156,18 +2230,28 @@ async function runDaemon(opts = {}) {
|
|
|
1156
2230
|
log(`started pid=${process.pid} interval=${intervalMs}ms lease=${leaseId}`);
|
|
1157
2231
|
let stopFlag = false;
|
|
1158
2232
|
let leaseLost = false;
|
|
2233
|
+
const runAbort = new AbortController;
|
|
2234
|
+
const requestStop = (message) => {
|
|
2235
|
+
stopFlag = true;
|
|
2236
|
+
if (!runAbort.signal.aborted)
|
|
2237
|
+
runAbort.abort();
|
|
2238
|
+
if (message)
|
|
2239
|
+
log(message);
|
|
2240
|
+
};
|
|
1159
2241
|
const ensureLease = () => {
|
|
1160
2242
|
const current = store.heartbeatDaemonLease(leaseId, leaseTtlMs);
|
|
1161
2243
|
if (!current || current.id !== leaseId) {
|
|
1162
2244
|
leaseLost = true;
|
|
1163
|
-
|
|
2245
|
+
requestStop("daemon lease lost");
|
|
1164
2246
|
throw new Error("daemon lease lost");
|
|
1165
2247
|
}
|
|
1166
2248
|
};
|
|
1167
2249
|
const onSignal = () => {
|
|
1168
|
-
|
|
1169
|
-
log("stop signal received");
|
|
2250
|
+
requestStop("stop signal received");
|
|
1170
2251
|
};
|
|
2252
|
+
if (opts.signal?.aborted)
|
|
2253
|
+
onSignal();
|
|
2254
|
+
opts.signal?.addEventListener("abort", onSignal, { once: true });
|
|
1171
2255
|
process.on("SIGINT", onSignal);
|
|
1172
2256
|
process.on("SIGTERM", onSignal);
|
|
1173
2257
|
try {
|
|
@@ -1180,7 +2264,7 @@ async function runDaemon(opts = {}) {
|
|
|
1180
2264
|
ensureLease();
|
|
1181
2265
|
const result = await tick({
|
|
1182
2266
|
store,
|
|
1183
|
-
runnerId
|
|
2267
|
+
runnerId,
|
|
1184
2268
|
execute: async (loop, run) => {
|
|
1185
2269
|
const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs / 3));
|
|
1186
2270
|
const timer = setInterval(() => {
|
|
@@ -1192,7 +2276,10 @@ async function runDaemon(opts = {}) {
|
|
|
1192
2276
|
}, heartbeatMs);
|
|
1193
2277
|
timer.unref();
|
|
1194
2278
|
try {
|
|
1195
|
-
const result2 = await
|
|
2279
|
+
const result2 = await executeLoopTarget(store, loop, run, {
|
|
2280
|
+
signal: runAbort.signal,
|
|
2281
|
+
onSpawn: (pid) => store.markRunPid(run.id, pid, runnerId)
|
|
2282
|
+
});
|
|
1196
2283
|
if (leaseLost)
|
|
1197
2284
|
throw new Error("daemon lease lost during run");
|
|
1198
2285
|
return result2;
|
|
@@ -1209,6 +2296,7 @@ async function runDaemon(opts = {}) {
|
|
|
1209
2296
|
}
|
|
1210
2297
|
});
|
|
1211
2298
|
} finally {
|
|
2299
|
+
opts.signal?.removeEventListener("abort", onSignal);
|
|
1212
2300
|
process.off("SIGINT", onSignal);
|
|
1213
2301
|
process.off("SIGTERM", onSignal);
|
|
1214
2302
|
store.releaseDaemonLease(leaseId);
|
|
@@ -1244,6 +2332,7 @@ async function startDaemon(opts) {
|
|
|
1244
2332
|
|
|
1245
2333
|
// src/daemon/install.ts
|
|
1246
2334
|
import { chmodSync, mkdirSync as mkdirSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
2335
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
1247
2336
|
import { dirname as dirname3 } from "path";
|
|
1248
2337
|
function installStartup(cliEntry, execPath = process.execPath, args = ["daemon", "run"]) {
|
|
1249
2338
|
const command = [execPath, cliEntry, ...args].join(" ");
|
|
@@ -1305,10 +2394,25 @@ ${args.map((arg) => ` <string>${arg}</string>`).join(`
|
|
|
1305
2394
|
}
|
|
1306
2395
|
throw new Error(`startup install is not implemented for ${process.platform}`);
|
|
1307
2396
|
}
|
|
2397
|
+
function enableStartup(result) {
|
|
2398
|
+
const commands = result.platform === "linux" ? ["systemctl --user daemon-reload", "systemctl --user enable --now loops-daemon.service"] : result.platform === "darwin" ? [`launchctl load -w ${result.path}`] : [];
|
|
2399
|
+
return commands.map((command) => {
|
|
2400
|
+
const run = spawnSync3("sh", ["-c", command], {
|
|
2401
|
+
encoding: "utf8",
|
|
2402
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
2403
|
+
});
|
|
2404
|
+
return {
|
|
2405
|
+
command,
|
|
2406
|
+
status: run.status,
|
|
2407
|
+
stdout: run.stdout.trim(),
|
|
2408
|
+
stderr: run.stderr.trim()
|
|
2409
|
+
};
|
|
2410
|
+
});
|
|
2411
|
+
}
|
|
1308
2412
|
|
|
1309
2413
|
// src/daemon/index.ts
|
|
1310
2414
|
var program = new Command;
|
|
1311
|
-
program.name("loops-daemon").description("OpenLoops daemon helper").version("0.
|
|
2415
|
+
program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.0");
|
|
1312
2416
|
program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
|
|
1313
2417
|
program.command("start").action(async () => {
|
|
1314
2418
|
const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops-daemon", args: ["run"] });
|