@hasna/loops 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -4
- package/dist/cli/index.js +686 -88
- package/dist/daemon/daemon.d.ts +1 -0
- package/dist/daemon/index.js +474 -71
- package/dist/daemon/install.d.ts +8 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +644 -79
- package/dist/lib/doctor.d.ts +13 -0
- package/dist/lib/env.d.ts +4 -0
- package/dist/lib/executor.d.ts +8 -0
- package/dist/lib/format.d.ts +3 -0
- package/dist/lib/scheduler.d.ts +12 -0
- package/dist/lib/store.d.ts +10 -0
- package/dist/lib/store.js +157 -20
- package/dist/lib/workflow-runner.d.ts +4 -1
- package/dist/sdk/index.js +440 -79
- package/dist/types.d.ts +3 -2
- package/docs/USAGE.md +25 -4
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -249,6 +249,9 @@ function validateTarget(value, label) {
|
|
|
249
249
|
assertObject(value, label);
|
|
250
250
|
if (value.type === "command") {
|
|
251
251
|
assertString(value.command, `${label}.command`);
|
|
252
|
+
if (value.shell !== true && /\s/.test(value.command.trim())) {
|
|
253
|
+
throw new Error(`${label}.command must be an executable without spaces when shell is false; put flags in args or set shell true`);
|
|
254
|
+
}
|
|
252
255
|
return value;
|
|
253
256
|
}
|
|
254
257
|
if (value.type === "agent") {
|
|
@@ -414,6 +417,7 @@ function rowToWorkflowStepRun(row) {
|
|
|
414
417
|
startedAt: row.started_at ?? undefined,
|
|
415
418
|
finishedAt: row.finished_at ?? undefined,
|
|
416
419
|
exitCode: row.exit_code ?? undefined,
|
|
420
|
+
pid: row.pid ?? undefined,
|
|
417
421
|
durationMs: row.duration_ms ?? undefined,
|
|
418
422
|
stdout: row.stdout ?? undefined,
|
|
419
423
|
stderr: row.stderr ?? undefined,
|
|
@@ -435,6 +439,14 @@ function rowToWorkflowEvent(row) {
|
|
|
435
439
|
createdAt: row.created_at
|
|
436
440
|
};
|
|
437
441
|
}
|
|
442
|
+
function isProcessAlive(pid) {
|
|
443
|
+
try {
|
|
444
|
+
process.kill(pid, 0);
|
|
445
|
+
return true;
|
|
446
|
+
} catch {
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
438
450
|
function rowToLease(row) {
|
|
439
451
|
return {
|
|
440
452
|
id: row.id,
|
|
@@ -567,6 +579,7 @@ class Store {
|
|
|
567
579
|
started_at TEXT,
|
|
568
580
|
finished_at TEXT,
|
|
569
581
|
exit_code INTEGER,
|
|
582
|
+
pid INTEGER,
|
|
570
583
|
duration_ms INTEGER,
|
|
571
584
|
stdout TEXT,
|
|
572
585
|
stderr TEXT,
|
|
@@ -592,6 +605,9 @@ class Store {
|
|
|
592
605
|
);
|
|
593
606
|
CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
|
|
594
607
|
`);
|
|
608
|
+
try {
|
|
609
|
+
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
610
|
+
} catch {}
|
|
595
611
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
596
612
|
}
|
|
597
613
|
createLoop(input, from = new Date) {
|
|
@@ -772,8 +788,8 @@ class Store {
|
|
|
772
788
|
input.workflow.steps.forEach((step, sequence) => {
|
|
773
789
|
const account = step.account ?? step.target.account;
|
|
774
790
|
this.db.query(`INSERT INTO workflow_step_runs (id, workflow_run_id, step_id, sequence, status, started_at, finished_at,
|
|
775
|
-
exit_code, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
|
|
776
|
-
VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL,
|
|
791
|
+
exit_code, pid, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
|
|
792
|
+
VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
|
|
777
793
|
$accountProfile, $accountTool, $created, $updated)`).run({
|
|
778
794
|
$id: genId(),
|
|
779
795
|
$workflowRunId: runId,
|
|
@@ -814,6 +830,12 @@ class Store {
|
|
|
814
830
|
const row = this.db.query("SELECT * FROM workflow_runs WHERE id = ?").get(id);
|
|
815
831
|
return row ? rowToWorkflowRun(row) : undefined;
|
|
816
832
|
}
|
|
833
|
+
requireWorkflowRun(id) {
|
|
834
|
+
const run = this.getWorkflowRun(id);
|
|
835
|
+
if (!run)
|
|
836
|
+
throw new Error(`workflow run not found: ${id}`);
|
|
837
|
+
return run;
|
|
838
|
+
}
|
|
817
839
|
listWorkflowRuns(opts = {}) {
|
|
818
840
|
const limit = opts.limit ?? 100;
|
|
819
841
|
let rows;
|
|
@@ -834,23 +856,67 @@ class Store {
|
|
|
834
856
|
const row = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? AND step_id = ?").get(workflowRunId, stepId);
|
|
835
857
|
return row ? rowToWorkflowStepRun(row) : undefined;
|
|
836
858
|
}
|
|
859
|
+
isWorkflowRunTerminal(workflowRunId) {
|
|
860
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
861
|
+
return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
|
|
862
|
+
}
|
|
837
863
|
startWorkflowStepRun(workflowRunId, stepId) {
|
|
838
864
|
const now = nowIso();
|
|
839
|
-
this.db.query(`UPDATE workflow_step_runs
|
|
865
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
840
866
|
SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
|
|
841
|
-
stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
842
|
-
WHERE workflow_run_id=$workflowRunId
|
|
843
|
-
|
|
867
|
+
pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
868
|
+
WHERE workflow_run_id=$workflowRunId
|
|
869
|
+
AND step_id=$stepId
|
|
870
|
+
AND status IN ('pending', 'failed', 'timed_out')
|
|
871
|
+
AND EXISTS (
|
|
872
|
+
SELECT 1 FROM workflow_runs
|
|
873
|
+
WHERE id=$workflowRunId AND status='running'
|
|
874
|
+
)`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
|
|
844
875
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
845
876
|
if (!run)
|
|
846
877
|
throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
|
|
878
|
+
if (res.changes !== 1) {
|
|
879
|
+
throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
|
|
880
|
+
}
|
|
881
|
+
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
847
882
|
return run;
|
|
848
883
|
}
|
|
884
|
+
markWorkflowStepPid(workflowRunId, stepId, pid) {
|
|
885
|
+
const now = nowIso();
|
|
886
|
+
this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
|
|
887
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
|
|
888
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
889
|
+
if (!run)
|
|
890
|
+
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
891
|
+
return run;
|
|
892
|
+
}
|
|
893
|
+
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
894
|
+
const now = nowIso();
|
|
895
|
+
const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
|
|
896
|
+
const live = before.filter((step) => step.pid !== undefined && isProcessAlive(step.pid));
|
|
897
|
+
if (live.length > 0) {
|
|
898
|
+
throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
|
|
899
|
+
}
|
|
900
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
901
|
+
SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
|
|
902
|
+
stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
|
|
903
|
+
WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
|
|
904
|
+
if (before.length > 0) {
|
|
905
|
+
this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
|
|
906
|
+
reason,
|
|
907
|
+
recoveredSteps: before.map((step) => step.stepId)
|
|
908
|
+
});
|
|
909
|
+
}
|
|
910
|
+
return {
|
|
911
|
+
run: this.requireWorkflowRun(workflowRunId),
|
|
912
|
+
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
913
|
+
};
|
|
914
|
+
}
|
|
849
915
|
finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
|
|
850
916
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
851
|
-
this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
852
|
-
stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
853
|
-
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId`).run({
|
|
917
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
918
|
+
pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
919
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
|
|
854
920
|
$workflowRunId: workflowRunId,
|
|
855
921
|
$stepId: stepId,
|
|
856
922
|
$status: patch.status,
|
|
@@ -862,10 +928,12 @@ class Store {
|
|
|
862
928
|
$error: patch.error ?? null,
|
|
863
929
|
$updated: finishedAt
|
|
864
930
|
});
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
931
|
+
if (res.changes === 1) {
|
|
932
|
+
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
933
|
+
exitCode: patch.exitCode,
|
|
934
|
+
error: patch.error
|
|
935
|
+
});
|
|
936
|
+
}
|
|
869
937
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
870
938
|
if (!run)
|
|
871
939
|
throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
|
|
@@ -873,9 +941,10 @@ class Store {
|
|
|
873
941
|
}
|
|
874
942
|
skipWorkflowStepRun(workflowRunId, stepId, reason) {
|
|
875
943
|
const now = nowIso();
|
|
876
|
-
this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, error=$error, updated_at=$updated
|
|
877
|
-
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
|
|
878
|
-
|
|
944
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
|
|
945
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
|
|
946
|
+
if (res.changes === 1)
|
|
947
|
+
this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
|
|
879
948
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
880
949
|
if (!run)
|
|
881
950
|
throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
|
|
@@ -883,8 +952,8 @@ class Store {
|
|
|
883
952
|
}
|
|
884
953
|
finalizeWorkflowRun(workflowRunId, status, patch = {}) {
|
|
885
954
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
886
|
-
this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
|
|
887
|
-
WHERE id=$id`).run({
|
|
955
|
+
const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
|
|
956
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
|
|
888
957
|
$id: workflowRunId,
|
|
889
958
|
$status: status,
|
|
890
959
|
$finished: finishedAt,
|
|
@@ -892,12 +961,36 @@ class Store {
|
|
|
892
961
|
$error: patch.error ?? null,
|
|
893
962
|
$updated: finishedAt
|
|
894
963
|
});
|
|
895
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
896
964
|
const run = this.getWorkflowRun(workflowRunId);
|
|
897
965
|
if (!run)
|
|
898
966
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
967
|
+
if (res.changes === 1)
|
|
968
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
899
969
|
return run;
|
|
900
970
|
}
|
|
971
|
+
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
972
|
+
const now = nowIso();
|
|
973
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
974
|
+
try {
|
|
975
|
+
const run = this.requireWorkflowRun(workflowRunId);
|
|
976
|
+
if (!["succeeded", "failed", "timed_out", "cancelled"].includes(run.status)) {
|
|
977
|
+
this.db.query(`UPDATE workflow_runs
|
|
978
|
+
SET status='cancelled', finished_at=$finished, error=$reason, updated_at=$updated
|
|
979
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRunId, $finished: now, $reason: reason, $updated: now });
|
|
980
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
981
|
+
SET status='cancelled', finished_at=$finished, pid=NULL, error=$reason, updated_at=$updated
|
|
982
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $finished: now, $reason: reason, $updated: now });
|
|
983
|
+
this.appendWorkflowEvent(workflowRunId, "cancelled", undefined, { reason });
|
|
984
|
+
}
|
|
985
|
+
this.db.exec("COMMIT");
|
|
986
|
+
return this.requireWorkflowRun(workflowRunId);
|
|
987
|
+
} catch (error) {
|
|
988
|
+
try {
|
|
989
|
+
this.db.exec("ROLLBACK");
|
|
990
|
+
} catch {}
|
|
991
|
+
throw error;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
901
994
|
appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
|
|
902
995
|
const now = nowIso();
|
|
903
996
|
const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
|
|
@@ -926,6 +1019,24 @@ class Store {
|
|
|
926
1019
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
927
1020
|
return (row?.count ?? 0) > 0;
|
|
928
1021
|
}
|
|
1022
|
+
markRunPid(id, pid, claimedBy) {
|
|
1023
|
+
const now = nowIso();
|
|
1024
|
+
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1025
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
|
|
1026
|
+
if (res.changes !== 1)
|
|
1027
|
+
return;
|
|
1028
|
+
return this.getRun(id);
|
|
1029
|
+
}
|
|
1030
|
+
hasLiveWorkflowStepProcesses(loopRunId) {
|
|
1031
|
+
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
|
|
1032
|
+
FROM workflow_runs wr
|
|
1033
|
+
JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
|
|
1034
|
+
WHERE wr.loop_run_id = ?
|
|
1035
|
+
AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1036
|
+
AND wsr.status = 'running'
|
|
1037
|
+
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
1038
|
+
return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
|
|
1039
|
+
}
|
|
929
1040
|
createSkippedRun(loop, scheduledFor, reason) {
|
|
930
1041
|
const now = nowIso();
|
|
931
1042
|
const run = {
|
|
@@ -973,6 +1084,14 @@ class Store {
|
|
|
973
1084
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
974
1085
|
if (existing) {
|
|
975
1086
|
if (existing.status === "running") {
|
|
1087
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid && isProcessAlive(existing.pid)) {
|
|
1088
|
+
this.db.exec("COMMIT");
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && this.hasLiveWorkflowStepProcesses(existing.id)) {
|
|
1092
|
+
this.db.exec("COMMIT");
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
976
1095
|
const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
|
|
977
1096
|
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
978
1097
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
@@ -1095,8 +1214,26 @@ class Store {
|
|
|
1095
1214
|
const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
|
|
1096
1215
|
const recovered = [];
|
|
1097
1216
|
for (const row of rows) {
|
|
1217
|
+
if (row.pid && isProcessAlive(row.pid))
|
|
1218
|
+
continue;
|
|
1219
|
+
if (this.hasLiveWorkflowStepProcesses(row.id))
|
|
1220
|
+
continue;
|
|
1221
|
+
const finished = now.toISOString();
|
|
1098
1222
|
this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
|
|
1099
|
-
error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished:
|
|
1223
|
+
error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
|
|
1224
|
+
const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
|
|
1225
|
+
for (const workflowRow of workflowRows) {
|
|
1226
|
+
this.db.query(`UPDATE workflow_runs
|
|
1227
|
+
SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1228
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
|
|
1229
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
1230
|
+
SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1231
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
|
|
1232
|
+
this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
|
|
1233
|
+
error: "parent loop run lease expired before completion",
|
|
1234
|
+
loopRunId: row.id
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
1100
1237
|
const run = this.getRun(row.id);
|
|
1101
1238
|
if (run)
|
|
1102
1239
|
recovered.push(run);
|
|
@@ -1172,6 +1309,7 @@ import { existsSync as existsSync3, readFileSync as readFileSync2 } from "fs";
|
|
|
1172
1309
|
import { Command } from "commander";
|
|
1173
1310
|
|
|
1174
1311
|
// src/lib/format.ts
|
|
1312
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1175
1313
|
function redact(value, visible = 80) {
|
|
1176
1314
|
if (!value)
|
|
1177
1315
|
return value;
|
|
@@ -1179,6 +1317,29 @@ function redact(value, visible = 80) {
|
|
|
1179
1317
|
return value;
|
|
1180
1318
|
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1181
1319
|
}
|
|
1320
|
+
function truncateTextOutput(value) {
|
|
1321
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1322
|
+
return value;
|
|
1323
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1324
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1325
|
+
}
|
|
1326
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1327
|
+
const indent = opts.indent ?? "";
|
|
1328
|
+
const nested = `${indent} `;
|
|
1329
|
+
const blocks = [];
|
|
1330
|
+
for (const [label, output] of [
|
|
1331
|
+
["stdout", value.stdout],
|
|
1332
|
+
["stderr", value.stderr]
|
|
1333
|
+
]) {
|
|
1334
|
+
if (!output)
|
|
1335
|
+
continue;
|
|
1336
|
+
blocks.push(`${indent}${label}:`);
|
|
1337
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1338
|
+
blocks.push(`${nested}${line}`);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
return blocks;
|
|
1342
|
+
}
|
|
1182
1343
|
function publicLoop(loop) {
|
|
1183
1344
|
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1184
1345
|
return {
|
|
@@ -1266,6 +1427,25 @@ function primaryAccountDir(output) {
|
|
|
1266
1427
|
}
|
|
1267
1428
|
return;
|
|
1268
1429
|
}
|
|
1430
|
+
function accountDirEnvVar(tool) {
|
|
1431
|
+
switch (tool) {
|
|
1432
|
+
case "claude":
|
|
1433
|
+
return "CLAUDE_CONFIG_DIR";
|
|
1434
|
+
case "codex":
|
|
1435
|
+
case "codex-app":
|
|
1436
|
+
return "CODEX_HOME";
|
|
1437
|
+
case "cursor":
|
|
1438
|
+
return "CURSOR_CONFIG_DIR";
|
|
1439
|
+
case "opencode":
|
|
1440
|
+
return "OPENCODE_CONFIG_DIR";
|
|
1441
|
+
case "codewith":
|
|
1442
|
+
return "CODEWITH_HOME";
|
|
1443
|
+
case "aicopilot":
|
|
1444
|
+
return "AICOPILOT_CONFIG_DIR";
|
|
1445
|
+
default:
|
|
1446
|
+
return;
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1269
1449
|
function resolveAccountEnv(account, toolHint, env) {
|
|
1270
1450
|
if (!account)
|
|
1271
1451
|
return {};
|
|
@@ -1284,18 +1464,78 @@ function resolveAccountEnv(account, toolHint, env) {
|
|
|
1284
1464
|
const stderr = result.stderr.trim();
|
|
1285
1465
|
throw new Error(`accounts env failed for ${account.profile}/${tool}${stderr ? `: ${stderr}` : ""}`);
|
|
1286
1466
|
}
|
|
1287
|
-
const
|
|
1467
|
+
const accountEnv = parseAccountExportLines(result.stdout);
|
|
1468
|
+
const profileDir = (accountDirEnvVar(tool) ? accountEnv[accountDirEnvVar(tool)] : undefined) ?? primaryAccountDir(result.stdout);
|
|
1288
1469
|
if (!profileDir)
|
|
1289
1470
|
throw new Error(`accounts env returned no profile directory for ${account.profile}/${tool}`);
|
|
1290
1471
|
if (!existsSync(profileDir))
|
|
1291
1472
|
throw new Error(`account profile directory does not exist for ${account.profile}/${tool}: ${profileDir}`);
|
|
1292
1473
|
return {
|
|
1293
|
-
...
|
|
1474
|
+
...accountEnv,
|
|
1294
1475
|
LOOPS_ACCOUNT_PROFILE: account.profile,
|
|
1295
1476
|
LOOPS_ACCOUNT_TOOL: tool
|
|
1296
1477
|
};
|
|
1297
1478
|
}
|
|
1298
1479
|
|
|
1480
|
+
// src/lib/env.ts
|
|
1481
|
+
import { accessSync, constants } from "fs";
|
|
1482
|
+
import { homedir as homedir2 } from "os";
|
|
1483
|
+
import { delimiter, join as join2 } from "path";
|
|
1484
|
+
function compactPathParts(parts) {
|
|
1485
|
+
const seen = new Set;
|
|
1486
|
+
const result = [];
|
|
1487
|
+
for (const part of parts) {
|
|
1488
|
+
const value = part?.trim();
|
|
1489
|
+
if (!value || seen.has(value))
|
|
1490
|
+
continue;
|
|
1491
|
+
seen.add(value);
|
|
1492
|
+
result.push(value);
|
|
1493
|
+
}
|
|
1494
|
+
return result;
|
|
1495
|
+
}
|
|
1496
|
+
function commonExecutableDirs(env = process.env) {
|
|
1497
|
+
const home = env.HOME || homedir2();
|
|
1498
|
+
return compactPathParts([
|
|
1499
|
+
join2(home, ".local", "bin"),
|
|
1500
|
+
join2(home, ".bun", "bin"),
|
|
1501
|
+
join2(home, ".cargo", "bin"),
|
|
1502
|
+
join2(home, ".npm-global", "bin"),
|
|
1503
|
+
join2(home, "bin"),
|
|
1504
|
+
env.BUN_INSTALL ? join2(env.BUN_INSTALL, "bin") : undefined,
|
|
1505
|
+
env.PNPM_HOME,
|
|
1506
|
+
env.NPM_CONFIG_PREFIX ? join2(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
1507
|
+
"/opt/homebrew/bin",
|
|
1508
|
+
"/usr/local/bin",
|
|
1509
|
+
"/usr/bin",
|
|
1510
|
+
"/bin",
|
|
1511
|
+
"/usr/sbin",
|
|
1512
|
+
"/sbin"
|
|
1513
|
+
]);
|
|
1514
|
+
}
|
|
1515
|
+
function normalizeExecutionPath(env = process.env) {
|
|
1516
|
+
return compactPathParts([...(env.PATH ?? "").split(delimiter), ...commonExecutableDirs(env)]).join(delimiter);
|
|
1517
|
+
}
|
|
1518
|
+
function isExecutable(path) {
|
|
1519
|
+
try {
|
|
1520
|
+
accessSync(path, constants.X_OK);
|
|
1521
|
+
return true;
|
|
1522
|
+
} catch {
|
|
1523
|
+
return false;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
function executableExists(command, env = process.env) {
|
|
1527
|
+
if (command.includes("/"))
|
|
1528
|
+
return isExecutable(command);
|
|
1529
|
+
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
1530
|
+
if (dir && isExecutable(join2(dir, command)))
|
|
1531
|
+
return true;
|
|
1532
|
+
}
|
|
1533
|
+
return false;
|
|
1534
|
+
}
|
|
1535
|
+
function commandNotFoundMessage(command, env = process.env) {
|
|
1536
|
+
return `Executable not found in PATH: ${command}. Effective PATH=${env.PATH || "(empty)"}`;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1299
1539
|
// src/lib/executor.ts
|
|
1300
1540
|
var DEFAULT_TIMEOUT_MS = 30 * 60000;
|
|
1301
1541
|
var DEFAULT_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
@@ -1381,7 +1621,7 @@ function agentArgs(target) {
|
|
|
1381
1621
|
args.push(...target.extraArgs ?? [], target.prompt);
|
|
1382
1622
|
return args;
|
|
1383
1623
|
case "codewith":
|
|
1384
|
-
args.push("
|
|
1624
|
+
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
|
|
1385
1625
|
if (isolation === "safe")
|
|
1386
1626
|
args.push("--ignore-rules");
|
|
1387
1627
|
if (target.cwd)
|
|
@@ -1461,6 +1701,7 @@ function executionEnv(spec, metadata, opts) {
|
|
|
1461
1701
|
Object.assign(env, accountEnv);
|
|
1462
1702
|
}
|
|
1463
1703
|
Object.assign(env, spec.env ?? {});
|
|
1704
|
+
env.PATH = normalizeExecutionPath(env);
|
|
1464
1705
|
if (metadata.loopId)
|
|
1465
1706
|
env.LOOPS_LOOP_ID = metadata.loopId;
|
|
1466
1707
|
if (metadata.loopName)
|
|
@@ -1479,6 +1720,18 @@ function executionEnv(spec, metadata, opts) {
|
|
|
1479
1720
|
env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
|
|
1480
1721
|
return env;
|
|
1481
1722
|
}
|
|
1723
|
+
function preflightTarget(target, metadata = {}, opts = {}) {
|
|
1724
|
+
const spec = commandSpec(target);
|
|
1725
|
+
const env = executionEnv(spec, metadata, opts);
|
|
1726
|
+
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
1727
|
+
throw new Error(commandNotFoundMessage(spec.command, env));
|
|
1728
|
+
}
|
|
1729
|
+
return {
|
|
1730
|
+
command: spec.command,
|
|
1731
|
+
accountProfile: spec.account?.profile,
|
|
1732
|
+
accountTool: spec.accountTool
|
|
1733
|
+
};
|
|
1734
|
+
}
|
|
1482
1735
|
async function executeTarget(target, metadata = {}, opts = {}) {
|
|
1483
1736
|
const spec = commandSpec(target);
|
|
1484
1737
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
@@ -1489,6 +1742,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1489
1742
|
let exitCode;
|
|
1490
1743
|
let error;
|
|
1491
1744
|
const env = executionEnv(spec, metadata, opts);
|
|
1745
|
+
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
1746
|
+
return {
|
|
1747
|
+
status: "failed",
|
|
1748
|
+
stdout: "",
|
|
1749
|
+
stderr: "",
|
|
1750
|
+
error: commandNotFoundMessage(spec.command, env),
|
|
1751
|
+
startedAt,
|
|
1752
|
+
finishedAt: nowIso(),
|
|
1753
|
+
durationMs: 0
|
|
1754
|
+
};
|
|
1755
|
+
}
|
|
1492
1756
|
const child = spawn(spec.command, spec.args, {
|
|
1493
1757
|
cwd: spec.cwd,
|
|
1494
1758
|
env,
|
|
@@ -1496,6 +1760,16 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1496
1760
|
detached: true,
|
|
1497
1761
|
stdio: ["ignore", "pipe", "pipe"]
|
|
1498
1762
|
});
|
|
1763
|
+
if (child.pid)
|
|
1764
|
+
opts.onSpawn?.(child.pid);
|
|
1765
|
+
const abortHandler = () => {
|
|
1766
|
+
error = "cancelled";
|
|
1767
|
+
if (child.pid)
|
|
1768
|
+
killProcessGroup(child.pid);
|
|
1769
|
+
};
|
|
1770
|
+
if (opts.signal?.aborted)
|
|
1771
|
+
abortHandler();
|
|
1772
|
+
opts.signal?.addEventListener("abort", abortHandler, { once: true });
|
|
1499
1773
|
child.stdout.on("data", (chunk) => {
|
|
1500
1774
|
stdout = appendBounded(stdout, chunk, maxOutputBytes);
|
|
1501
1775
|
});
|
|
@@ -1518,6 +1792,7 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1518
1792
|
error = err instanceof Error ? err.message : String(err);
|
|
1519
1793
|
} finally {
|
|
1520
1794
|
clearTimeout(timer);
|
|
1795
|
+
opts.signal?.removeEventListener("abort", abortHandler);
|
|
1521
1796
|
}
|
|
1522
1797
|
const finishedAt = nowIso();
|
|
1523
1798
|
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
@@ -1573,14 +1848,16 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
1573
1848
|
// src/lib/workflow-runner.ts
|
|
1574
1849
|
function targetWithStepAccount(step) {
|
|
1575
1850
|
const account = step.account ?? step.target.account;
|
|
1576
|
-
|
|
1851
|
+
const timeoutMs = step.timeoutMs ?? step.target.timeoutMs;
|
|
1852
|
+
if (!account && timeoutMs === step.target.timeoutMs)
|
|
1577
1853
|
return step.target;
|
|
1578
|
-
return { ...step.target, account };
|
|
1854
|
+
return { ...step.target, account, timeoutMs };
|
|
1579
1855
|
}
|
|
1580
1856
|
function workflowResult(workflowRun, status, startedAt, finishedAt, stdout, error) {
|
|
1857
|
+
const executorStatus = status === "succeeded" ? "succeeded" : status === "timed_out" ? "timed_out" : "failed";
|
|
1581
1858
|
return {
|
|
1582
|
-
status,
|
|
1583
|
-
exitCode:
|
|
1859
|
+
status: executorStatus,
|
|
1860
|
+
exitCode: executorStatus === "succeeded" ? 0 : 1,
|
|
1584
1861
|
stdout,
|
|
1585
1862
|
stderr: "",
|
|
1586
1863
|
error,
|
|
@@ -1598,7 +1875,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1598
1875
|
idempotencyKey: opts.idempotencyKey
|
|
1599
1876
|
});
|
|
1600
1877
|
const startedAt = run.startedAt ?? nowIso();
|
|
1601
|
-
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out") {
|
|
1878
|
+
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
1602
1879
|
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
1603
1880
|
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), JSON.stringify({ workflowRun: run, steps: steps2 }, null, 2), run.error);
|
|
1604
1881
|
}
|
|
@@ -1607,8 +1884,19 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1607
1884
|
let blockingError;
|
|
1608
1885
|
let terminalStatus = "succeeded";
|
|
1609
1886
|
for (const step of ordered) {
|
|
1887
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
1888
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
1889
|
+
blockingError = "workflow run was cancelled";
|
|
1890
|
+
break;
|
|
1891
|
+
}
|
|
1892
|
+
const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
1893
|
+
if (pendingTimeout) {
|
|
1894
|
+
terminalStatus = "timed_out";
|
|
1895
|
+
blockingError = pendingTimeout;
|
|
1896
|
+
break;
|
|
1897
|
+
}
|
|
1610
1898
|
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
1611
|
-
if (existing?.status === "succeeded" || existing?.status === "skipped")
|
|
1899
|
+
if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
|
|
1612
1900
|
continue;
|
|
1613
1901
|
const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
|
|
1614
1902
|
const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
|
|
@@ -1623,8 +1911,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1623
1911
|
terminalStatus = "failed";
|
|
1624
1912
|
continue;
|
|
1625
1913
|
}
|
|
1626
|
-
store.startWorkflowStepRun(run.id, step.id);
|
|
1627
|
-
|
|
1914
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id);
|
|
1915
|
+
if (startedStep.status !== "running") {
|
|
1916
|
+
terminalStatus = "failed";
|
|
1917
|
+
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
1918
|
+
break;
|
|
1919
|
+
}
|
|
1920
|
+
const metadata = {
|
|
1628
1921
|
loopId: opts.loop?.id,
|
|
1629
1922
|
loopName: opts.loop?.name,
|
|
1630
1923
|
runId: opts.loopRun?.id,
|
|
@@ -1633,7 +1926,51 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1633
1926
|
workflowName: workflow.name,
|
|
1634
1927
|
workflowRunId: run.id,
|
|
1635
1928
|
workflowStepId: step.id
|
|
1636
|
-
}
|
|
1929
|
+
};
|
|
1930
|
+
let result;
|
|
1931
|
+
const controller = new AbortController;
|
|
1932
|
+
const externalAbort = () => controller.abort();
|
|
1933
|
+
if (opts.signal?.aborted)
|
|
1934
|
+
controller.abort();
|
|
1935
|
+
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
1936
|
+
const cancelTimer = setInterval(() => {
|
|
1937
|
+
if (store.getWorkflowRun(run.id)?.status === "cancelled")
|
|
1938
|
+
controller.abort();
|
|
1939
|
+
}, opts.cancelPollMs ?? 500);
|
|
1940
|
+
cancelTimer.unref();
|
|
1941
|
+
try {
|
|
1942
|
+
result = await executeTarget(targetWithStepAccount(step), metadata, {
|
|
1943
|
+
...opts,
|
|
1944
|
+
signal: controller.signal,
|
|
1945
|
+
onSpawn: (pid) => {
|
|
1946
|
+
store.markWorkflowStepPid(run.id, step.id, pid);
|
|
1947
|
+
opts.onSpawn?.(pid);
|
|
1948
|
+
}
|
|
1949
|
+
});
|
|
1950
|
+
} catch (error) {
|
|
1951
|
+
const finishedAt2 = nowIso();
|
|
1952
|
+
result = {
|
|
1953
|
+
status: "failed",
|
|
1954
|
+
stdout: "",
|
|
1955
|
+
stderr: "",
|
|
1956
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1957
|
+
startedAt: startedStep.startedAt ?? finishedAt2,
|
|
1958
|
+
finishedAt: finishedAt2,
|
|
1959
|
+
durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
|
|
1960
|
+
};
|
|
1961
|
+
} finally {
|
|
1962
|
+
clearInterval(cancelTimer);
|
|
1963
|
+
opts.signal?.removeEventListener("abort", externalAbort);
|
|
1964
|
+
}
|
|
1965
|
+
const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
1966
|
+
if (timeoutMessage && result.status === "failed") {
|
|
1967
|
+
result = { ...result, status: "timed_out", error: timeoutMessage };
|
|
1968
|
+
}
|
|
1969
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
1970
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
1971
|
+
blockingError = "workflow run was cancelled";
|
|
1972
|
+
break;
|
|
1973
|
+
}
|
|
1637
1974
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
1638
1975
|
status: result.status,
|
|
1639
1976
|
finishedAt: result.finishedAt,
|
|
@@ -1658,6 +1995,11 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1658
1995
|
}
|
|
1659
1996
|
}
|
|
1660
1997
|
const finishedAt = nowIso();
|
|
1998
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
1999
|
+
const terminalRun = store.requireWorkflowRun(run.id);
|
|
2000
|
+
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
2001
|
+
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
2002
|
+
}
|
|
1661
2003
|
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
1662
2004
|
finishedAt,
|
|
1663
2005
|
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
@@ -1666,20 +2008,59 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1666
2008
|
const steps = store.listWorkflowStepRuns(run.id);
|
|
1667
2009
|
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
1668
2010
|
}
|
|
2011
|
+
function preflightWorkflow(workflow, opts = {}) {
|
|
2012
|
+
return workflowExecutionOrder(workflow).map((step) => preflightTarget(targetWithStepAccount(step), {
|
|
2013
|
+
workflowId: workflow.id,
|
|
2014
|
+
workflowName: workflow.name,
|
|
2015
|
+
workflowStepId: step.id
|
|
2016
|
+
}, opts));
|
|
2017
|
+
}
|
|
1669
2018
|
async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
1670
2019
|
if (loop.target.type !== "workflow")
|
|
1671
2020
|
return executeLoop(loop, run, opts);
|
|
1672
2021
|
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
2022
|
+
const controller = loop.target.timeoutMs ? new AbortController : undefined;
|
|
2023
|
+
let workflowTimedOut = false;
|
|
2024
|
+
const externalAbort = () => controller?.abort();
|
|
2025
|
+
if (controller && opts.signal?.aborted)
|
|
2026
|
+
controller.abort();
|
|
2027
|
+
if (controller)
|
|
2028
|
+
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
2029
|
+
const timer = controller ? setTimeout(() => {
|
|
2030
|
+
workflowTimedOut = true;
|
|
2031
|
+
controller.abort();
|
|
2032
|
+
}, loop.target.timeoutMs) : undefined;
|
|
2033
|
+
timer?.unref();
|
|
2034
|
+
try {
|
|
2035
|
+
return await executeWorkflow(store, workflow, {
|
|
2036
|
+
...opts,
|
|
2037
|
+
signal: controller?.signal ?? opts.signal,
|
|
2038
|
+
signalTimeoutMessage: () => workflowTimedOut && loop.target.type === "workflow" ? `workflow timed out after ${loop.target.timeoutMs}ms` : undefined,
|
|
2039
|
+
loop,
|
|
2040
|
+
loopRun: run,
|
|
2041
|
+
scheduledFor: run.scheduledFor,
|
|
2042
|
+
idempotencyKey: `${loop.id}:${run.scheduledFor}:attempt:${run.attempt}`
|
|
2043
|
+
});
|
|
2044
|
+
} finally {
|
|
2045
|
+
if (timer)
|
|
2046
|
+
clearTimeout(timer);
|
|
2047
|
+
if (controller)
|
|
2048
|
+
opts.signal?.removeEventListener("abort", externalAbort);
|
|
2049
|
+
}
|
|
1680
2050
|
}
|
|
1681
2051
|
|
|
1682
2052
|
// src/lib/scheduler.ts
|
|
2053
|
+
function manualRunScheduledFor(loop, now = new Date) {
|
|
2054
|
+
if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
2055
|
+
return loop.retryScheduledFor ?? loop.nextRunAt;
|
|
2056
|
+
}
|
|
2057
|
+
return now.toISOString();
|
|
2058
|
+
}
|
|
2059
|
+
function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
|
|
2060
|
+
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
2061
|
+
return false;
|
|
2062
|
+
return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
|
|
2063
|
+
}
|
|
1683
2064
|
function nextAfterRetry(loop, now) {
|
|
1684
2065
|
return new Date(now.getTime() + loop.retryDelayMs).toISOString();
|
|
1685
2066
|
}
|
|
@@ -1705,27 +2086,18 @@ function advanceLoop(store, loop, run, finishedAt, succeeded) {
|
|
|
1705
2086
|
retryScheduledFor: undefined
|
|
1706
2087
|
});
|
|
1707
2088
|
}
|
|
1708
|
-
async function
|
|
1709
|
-
const now = deps.now?.() ?? new Date;
|
|
1710
|
-
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
1711
|
-
const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
|
|
1712
|
-
advanceLoop(deps.store, loop, skipped, now, true);
|
|
1713
|
-
deps.onRun?.(skipped);
|
|
1714
|
-
return skipped;
|
|
1715
|
-
}
|
|
1716
|
-
const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
|
|
1717
|
-
if (!claim)
|
|
1718
|
-
return;
|
|
1719
|
-
deps.onRun?.(claim.run);
|
|
2089
|
+
async function executeClaimedRun(deps) {
|
|
1720
2090
|
let heartbeat;
|
|
1721
|
-
const heartbeatEveryMs = Math.max(
|
|
2091
|
+
const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
|
|
1722
2092
|
heartbeat = setInterval(() => {
|
|
1723
|
-
deps.store.heartbeatRunLease(
|
|
2093
|
+
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
|
|
1724
2094
|
}, heartbeatEveryMs);
|
|
1725
2095
|
heartbeat.unref();
|
|
1726
2096
|
try {
|
|
1727
|
-
const result = await (deps.execute ?? ((
|
|
1728
|
-
|
|
2097
|
+
const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
|
|
2098
|
+
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
|
|
2099
|
+
})))(deps.loop, deps.run);
|
|
2100
|
+
return deps.store.finalizeRun(deps.run.id, {
|
|
1729
2101
|
status: result.status,
|
|
1730
2102
|
finishedAt: result.finishedAt,
|
|
1731
2103
|
durationMs: result.durationMs,
|
|
@@ -1738,16 +2110,13 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
1738
2110
|
claimedBy: deps.runnerId,
|
|
1739
2111
|
now: deps.now?.() ?? new Date(result.finishedAt)
|
|
1740
2112
|
});
|
|
1741
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(result.finishedAt), finalRun.status === "succeeded");
|
|
1742
|
-
deps.onRun?.(finalRun);
|
|
1743
|
-
return finalRun;
|
|
1744
2113
|
} catch (err) {
|
|
1745
|
-
deps.onError?.(
|
|
2114
|
+
deps.onError?.(deps.loop, err);
|
|
1746
2115
|
const finishedAt = new Date;
|
|
1747
|
-
|
|
2116
|
+
return deps.store.finalizeRun(deps.run.id, {
|
|
1748
2117
|
status: "failed",
|
|
1749
2118
|
finishedAt: finishedAt.toISOString(),
|
|
1750
|
-
durationMs: finishedAt.getTime() - new Date(
|
|
2119
|
+
durationMs: finishedAt.getTime() - new Date(deps.run.startedAt ?? deps.run.createdAt).getTime(),
|
|
1751
2120
|
stdout: "",
|
|
1752
2121
|
stderr: "",
|
|
1753
2122
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -1755,14 +2124,36 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
1755
2124
|
claimedBy: deps.runnerId,
|
|
1756
2125
|
now: deps.now?.() ?? finishedAt
|
|
1757
2126
|
});
|
|
1758
|
-
advanceLoop(deps.store, claim.loop, finalRun, finishedAt, false);
|
|
1759
|
-
deps.onRun?.(finalRun);
|
|
1760
|
-
return finalRun;
|
|
1761
2127
|
} finally {
|
|
1762
2128
|
if (heartbeat)
|
|
1763
2129
|
clearInterval(heartbeat);
|
|
1764
2130
|
}
|
|
1765
2131
|
}
|
|
2132
|
+
async function runSlot(deps, loop, scheduledFor) {
|
|
2133
|
+
const now = deps.now?.() ?? new Date;
|
|
2134
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
2135
|
+
const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
|
|
2136
|
+
advanceLoop(deps.store, loop, skipped, now, true);
|
|
2137
|
+
deps.onRun?.(skipped);
|
|
2138
|
+
return skipped;
|
|
2139
|
+
}
|
|
2140
|
+
const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
|
|
2141
|
+
if (!claim)
|
|
2142
|
+
return;
|
|
2143
|
+
deps.onRun?.(claim.run);
|
|
2144
|
+
const finalRun = await executeClaimedRun({
|
|
2145
|
+
store: deps.store,
|
|
2146
|
+
runnerId: deps.runnerId,
|
|
2147
|
+
loop: claim.loop,
|
|
2148
|
+
run: claim.run,
|
|
2149
|
+
now: deps.now,
|
|
2150
|
+
execute: deps.execute,
|
|
2151
|
+
onError: deps.onError
|
|
2152
|
+
});
|
|
2153
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
|
|
2154
|
+
deps.onRun?.(finalRun);
|
|
2155
|
+
return finalRun;
|
|
2156
|
+
}
|
|
1766
2157
|
async function tick(deps) {
|
|
1767
2158
|
const now = deps.now?.() ?? new Date;
|
|
1768
2159
|
const recovered = deps.store.recoverExpiredRunLeases(now);
|
|
@@ -1941,6 +2332,7 @@ async function runDaemon(opts = {}) {
|
|
|
1941
2332
|
const ownStore = !opts.store;
|
|
1942
2333
|
const store = opts.store ?? new Store;
|
|
1943
2334
|
const leaseId = genId();
|
|
2335
|
+
const runnerId = `${hostname2()}:${process.pid}:${leaseId}`;
|
|
1944
2336
|
const intervalMs = opts.intervalMs ?? intervalFromEnv() ?? 1000;
|
|
1945
2337
|
const leaseTtlMs = opts.leaseTtlMs ?? Math.max(60000, intervalMs * 10);
|
|
1946
2338
|
const log = opts.log ?? ((message) => console.error(`[loops-daemon] ${message}`));
|
|
@@ -1956,18 +2348,28 @@ async function runDaemon(opts = {}) {
|
|
|
1956
2348
|
log(`started pid=${process.pid} interval=${intervalMs}ms lease=${leaseId}`);
|
|
1957
2349
|
let stopFlag = false;
|
|
1958
2350
|
let leaseLost = false;
|
|
2351
|
+
const runAbort = new AbortController;
|
|
2352
|
+
const requestStop = (message) => {
|
|
2353
|
+
stopFlag = true;
|
|
2354
|
+
if (!runAbort.signal.aborted)
|
|
2355
|
+
runAbort.abort();
|
|
2356
|
+
if (message)
|
|
2357
|
+
log(message);
|
|
2358
|
+
};
|
|
1959
2359
|
const ensureLease = () => {
|
|
1960
2360
|
const current = store.heartbeatDaemonLease(leaseId, leaseTtlMs);
|
|
1961
2361
|
if (!current || current.id !== leaseId) {
|
|
1962
2362
|
leaseLost = true;
|
|
1963
|
-
|
|
2363
|
+
requestStop("daemon lease lost");
|
|
1964
2364
|
throw new Error("daemon lease lost");
|
|
1965
2365
|
}
|
|
1966
2366
|
};
|
|
1967
2367
|
const onSignal = () => {
|
|
1968
|
-
|
|
1969
|
-
log("stop signal received");
|
|
2368
|
+
requestStop("stop signal received");
|
|
1970
2369
|
};
|
|
2370
|
+
if (opts.signal?.aborted)
|
|
2371
|
+
onSignal();
|
|
2372
|
+
opts.signal?.addEventListener("abort", onSignal, { once: true });
|
|
1971
2373
|
process.on("SIGINT", onSignal);
|
|
1972
2374
|
process.on("SIGTERM", onSignal);
|
|
1973
2375
|
try {
|
|
@@ -1980,7 +2382,7 @@ async function runDaemon(opts = {}) {
|
|
|
1980
2382
|
ensureLease();
|
|
1981
2383
|
const result = await tick({
|
|
1982
2384
|
store,
|
|
1983
|
-
runnerId
|
|
2385
|
+
runnerId,
|
|
1984
2386
|
execute: async (loop, run) => {
|
|
1985
2387
|
const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs / 3));
|
|
1986
2388
|
const timer = setInterval(() => {
|
|
@@ -1992,7 +2394,10 @@ async function runDaemon(opts = {}) {
|
|
|
1992
2394
|
}, heartbeatMs);
|
|
1993
2395
|
timer.unref();
|
|
1994
2396
|
try {
|
|
1995
|
-
const result2 = await
|
|
2397
|
+
const result2 = await executeLoopTarget(store, loop, run, {
|
|
2398
|
+
signal: runAbort.signal,
|
|
2399
|
+
onSpawn: (pid) => store.markRunPid(run.id, pid, runnerId)
|
|
2400
|
+
});
|
|
1996
2401
|
if (leaseLost)
|
|
1997
2402
|
throw new Error("daemon lease lost during run");
|
|
1998
2403
|
return result2;
|
|
@@ -2009,6 +2414,7 @@ async function runDaemon(opts = {}) {
|
|
|
2009
2414
|
}
|
|
2010
2415
|
});
|
|
2011
2416
|
} finally {
|
|
2417
|
+
opts.signal?.removeEventListener("abort", onSignal);
|
|
2012
2418
|
process.off("SIGINT", onSignal);
|
|
2013
2419
|
process.off("SIGTERM", onSignal);
|
|
2014
2420
|
store.releaseDaemonLease(leaseId);
|
|
@@ -2044,9 +2450,11 @@ async function startDaemon(opts) {
|
|
|
2044
2450
|
|
|
2045
2451
|
// src/daemon/install.ts
|
|
2046
2452
|
import { chmodSync, mkdirSync as mkdirSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
2453
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
2047
2454
|
import { dirname as dirname3 } from "path";
|
|
2048
2455
|
function installStartup(cliEntry, execPath = process.execPath, args = ["daemon", "run"]) {
|
|
2049
2456
|
const command = [execPath, cliEntry, ...args].join(" ");
|
|
2457
|
+
const pathEnv = normalizeExecutionPath(process.env);
|
|
2050
2458
|
if (process.platform === "linux") {
|
|
2051
2459
|
const path = systemdServicePath();
|
|
2052
2460
|
mkdirSync4(dirname3(path), { recursive: true, mode: 448 });
|
|
@@ -2059,7 +2467,7 @@ Type=simple
|
|
|
2059
2467
|
ExecStart=${command}
|
|
2060
2468
|
Restart=always
|
|
2061
2469
|
RestartSec=5
|
|
2062
|
-
Environment=PATH=${
|
|
2470
|
+
Environment=PATH=${pathEnv}
|
|
2063
2471
|
|
|
2064
2472
|
[Install]
|
|
2065
2473
|
WantedBy=default.target
|
|
@@ -2091,6 +2499,10 @@ ${args.map((arg) => ` <string>${arg}</string>`).join(`
|
|
|
2091
2499
|
</array>
|
|
2092
2500
|
<key>RunAtLoad</key><true/>
|
|
2093
2501
|
<key>KeepAlive</key><true/>
|
|
2502
|
+
<key>EnvironmentVariables</key>
|
|
2503
|
+
<dict>
|
|
2504
|
+
<key>PATH</key><string>${pathEnv}</string>
|
|
2505
|
+
</dict>
|
|
2094
2506
|
<key>StandardOutPath</key><string>${daemonLogPath()}</string>
|
|
2095
2507
|
<key>StandardErrorPath</key><string>${daemonLogPath()}</string>
|
|
2096
2508
|
</dict>
|
|
@@ -2105,10 +2517,100 @@ ${args.map((arg) => ` <string>${arg}</string>`).join(`
|
|
|
2105
2517
|
}
|
|
2106
2518
|
throw new Error(`startup install is not implemented for ${process.platform}`);
|
|
2107
2519
|
}
|
|
2520
|
+
function enableStartup(result) {
|
|
2521
|
+
const commands = result.platform === "linux" ? ["systemctl --user daemon-reload", "systemctl --user enable --now loops-daemon.service"] : result.platform === "darwin" ? [`launchctl load -w ${result.path}`] : [];
|
|
2522
|
+
return commands.map((command) => {
|
|
2523
|
+
const run = spawnSync2("sh", ["-c", command], {
|
|
2524
|
+
encoding: "utf8",
|
|
2525
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
2526
|
+
});
|
|
2527
|
+
return {
|
|
2528
|
+
command,
|
|
2529
|
+
status: run.status,
|
|
2530
|
+
stdout: run.stdout.trim(),
|
|
2531
|
+
stderr: run.stderr.trim()
|
|
2532
|
+
};
|
|
2533
|
+
});
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
// src/lib/doctor.ts
|
|
2537
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
2538
|
+
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
2539
|
+
var PROVIDER_COMMANDS = [
|
|
2540
|
+
"claude",
|
|
2541
|
+
"cursor-agent",
|
|
2542
|
+
"codewith",
|
|
2543
|
+
"aicopilot",
|
|
2544
|
+
"opencode",
|
|
2545
|
+
"codex"
|
|
2546
|
+
];
|
|
2547
|
+
function hasCommand(command) {
|
|
2548
|
+
const result = spawnSync3("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
2549
|
+
return (result.status ?? 1) === 0;
|
|
2550
|
+
}
|
|
2551
|
+
function commandVersion(command) {
|
|
2552
|
+
const result = spawnSync3(command, ["--version"], {
|
|
2553
|
+
encoding: "utf8",
|
|
2554
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
2555
|
+
});
|
|
2556
|
+
if ((result.status ?? 1) !== 0)
|
|
2557
|
+
return;
|
|
2558
|
+
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
2559
|
+
}
|
|
2560
|
+
function runDoctor(store) {
|
|
2561
|
+
const checks = [];
|
|
2562
|
+
try {
|
|
2563
|
+
const dir = ensureDataDir();
|
|
2564
|
+
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
2565
|
+
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
2566
|
+
} catch (error) {
|
|
2567
|
+
checks.push({
|
|
2568
|
+
id: "data-dir",
|
|
2569
|
+
status: "fail",
|
|
2570
|
+
message: "data directory is not writable",
|
|
2571
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
2572
|
+
});
|
|
2573
|
+
}
|
|
2574
|
+
const bunVersion = commandVersion("bun");
|
|
2575
|
+
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
2576
|
+
const accountsVersion = commandVersion("accounts");
|
|
2577
|
+
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
2578
|
+
for (const command of PROVIDER_COMMANDS) {
|
|
2579
|
+
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
2580
|
+
}
|
|
2581
|
+
const status = daemonStatus(store);
|
|
2582
|
+
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
2583
|
+
const failedRuns = store.countRuns("failed");
|
|
2584
|
+
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
2585
|
+
for (const loop of store.listLoops({ status: "active" })) {
|
|
2586
|
+
try {
|
|
2587
|
+
if (loop.target.type === "workflow") {
|
|
2588
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
2589
|
+
for (const step of workflowExecutionOrder(workflow)) {
|
|
2590
|
+
preflightTarget({ ...step.target, account: step.account ?? step.target.account, timeoutMs: step.timeoutMs ?? step.target.timeoutMs }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id });
|
|
2591
|
+
}
|
|
2592
|
+
} else {
|
|
2593
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name });
|
|
2594
|
+
}
|
|
2595
|
+
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
2596
|
+
} catch (error) {
|
|
2597
|
+
checks.push({
|
|
2598
|
+
id: `loop:${loop.id}:preflight`,
|
|
2599
|
+
status: "warn",
|
|
2600
|
+
message: `active loop target preflight failed: ${loop.name}`,
|
|
2601
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
2602
|
+
});
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
return {
|
|
2606
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
2607
|
+
checks
|
|
2608
|
+
};
|
|
2609
|
+
}
|
|
2108
2610
|
|
|
2109
2611
|
// src/cli/index.ts
|
|
2110
2612
|
var program = new Command;
|
|
2111
|
-
program.name("loops").description("Persistent local loops for commands and headless coding agents").version("0.
|
|
2613
|
+
program.name("loops").description("Persistent local loops for commands and headless coding agents").version("0.3.1");
|
|
2112
2614
|
program.option("-j, --json", "print JSON");
|
|
2113
2615
|
function isJson() {
|
|
2114
2616
|
return Boolean(program.opts().json);
|
|
@@ -2119,6 +2621,10 @@ function print(value, human) {
|
|
|
2119
2621
|
else
|
|
2120
2622
|
console.log(human);
|
|
2121
2623
|
}
|
|
2624
|
+
function printTextOutput(value) {
|
|
2625
|
+
for (const line of textOutputBlocks(value, { indent: " " }))
|
|
2626
|
+
console.log(line);
|
|
2627
|
+
}
|
|
2122
2628
|
function parseSchedule(opts) {
|
|
2123
2629
|
const count = [opts.at, opts.every, opts.cron, opts.dynamic ? "dynamic" : undefined].filter(Boolean).length;
|
|
2124
2630
|
if (count !== 1)
|
|
@@ -2258,6 +2764,22 @@ addScheduleOptions(create.command("workflow <name>").description("schedule a sto
|
|
|
2258
2764
|
}
|
|
2259
2765
|
});
|
|
2260
2766
|
var workflows = program.command("workflows").alias("workflow").description("manage workflow specs and runs");
|
|
2767
|
+
workflows.command("validate <file>").description("validate a workflow JSON file without storing or running it").option("--name <name>", "override workflow name from the file").option("--preflight", "also check account env and target executables").action((file, opts) => {
|
|
2768
|
+
const body = workflowBodyFromJson(JSON.parse(readFileSync2(file, "utf8")), opts.name);
|
|
2769
|
+
const now = new Date().toISOString();
|
|
2770
|
+
const workflow = {
|
|
2771
|
+
id: "validation",
|
|
2772
|
+
name: body.name,
|
|
2773
|
+
description: body.description,
|
|
2774
|
+
version: body.version ?? 1,
|
|
2775
|
+
status: "active",
|
|
2776
|
+
steps: body.steps,
|
|
2777
|
+
createdAt: now,
|
|
2778
|
+
updatedAt: now
|
|
2779
|
+
};
|
|
2780
|
+
const preflight = opts.preflight ? preflightWorkflow(workflow) : undefined;
|
|
2781
|
+
print({ valid: true, workflow: publicWorkflow(workflow), preflight }, `valid workflow ${workflow.name} steps=${workflow.steps.length}`);
|
|
2782
|
+
});
|
|
2261
2783
|
workflows.command("create <file>").description("validate and store a workflow JSON file").option("--name <name>", "override workflow name from the file").action((file, opts) => {
|
|
2262
2784
|
const store = new Store;
|
|
2263
2785
|
try {
|
|
@@ -2291,6 +2813,30 @@ workflows.command("show <idOrName>").action((idOrName) => {
|
|
|
2291
2813
|
store.close();
|
|
2292
2814
|
}
|
|
2293
2815
|
});
|
|
2816
|
+
workflows.command("inspect <runId>").description("show a workflow run with steps and events").action((runId) => {
|
|
2817
|
+
const store = new Store;
|
|
2818
|
+
try {
|
|
2819
|
+
const run = store.requireWorkflowRun(runId);
|
|
2820
|
+
const steps = store.listWorkflowStepRuns(run.id);
|
|
2821
|
+
const events = store.listWorkflowEvents(run.id);
|
|
2822
|
+
const value = {
|
|
2823
|
+
workflowRun: publicWorkflowRun(run),
|
|
2824
|
+
steps: steps.map((step) => publicWorkflowStepRun(step, isJson())),
|
|
2825
|
+
events: events.map(publicWorkflowEvent)
|
|
2826
|
+
};
|
|
2827
|
+
if (isJson())
|
|
2828
|
+
print(value);
|
|
2829
|
+
else {
|
|
2830
|
+
console.log(`${run.id} ${run.status} ${run.workflowName}`);
|
|
2831
|
+
for (const step of steps) {
|
|
2832
|
+
console.log(` ${String(step.sequence).padStart(2, "0")} ${step.status.padEnd(10)} ${step.stepId} ${step.error ?? ""}`);
|
|
2833
|
+
}
|
|
2834
|
+
console.log(` events=${events.length}`);
|
|
2835
|
+
}
|
|
2836
|
+
} finally {
|
|
2837
|
+
store.close();
|
|
2838
|
+
}
|
|
2839
|
+
});
|
|
2294
2840
|
workflows.command("run <idOrName>").option("--show-output", "show step stdout/stderr").action(async (idOrName, opts) => {
|
|
2295
2841
|
const store = new Store;
|
|
2296
2842
|
try {
|
|
@@ -2303,7 +2849,16 @@ workflows.command("run <idOrName>").option("--show-output", "show step stdout/st
|
|
|
2303
2849
|
workflowRun: run ? publicWorkflowRun(run) : undefined,
|
|
2304
2850
|
steps: steps.map((step) => publicWorkflowStepRun(step, opts.showOutput))
|
|
2305
2851
|
};
|
|
2306
|
-
|
|
2852
|
+
if (isJson())
|
|
2853
|
+
print(value);
|
|
2854
|
+
else {
|
|
2855
|
+
console.log(`${run?.id ?? workflow.id} ${result.status}`);
|
|
2856
|
+
for (const step of steps) {
|
|
2857
|
+
console.log(` ${String(step.sequence).padStart(2, "0")} ${step.status.padEnd(10)} ${step.stepId} ${step.error ?? ""}`);
|
|
2858
|
+
if (opts.showOutput)
|
|
2859
|
+
printTextOutput(step);
|
|
2860
|
+
}
|
|
2861
|
+
}
|
|
2307
2862
|
} finally {
|
|
2308
2863
|
store.close();
|
|
2309
2864
|
}
|
|
@@ -2339,6 +2894,27 @@ workflows.command("events <runId>").option("--limit <n>", "limit", "200").action
|
|
|
2339
2894
|
store.close();
|
|
2340
2895
|
}
|
|
2341
2896
|
});
|
|
2897
|
+
workflows.command("cancel <runId>").description("mark a workflow run cancelled and cancel pending/running steps").option("--reason <reason>", "cancellation reason", "cancelled by user").action((runId, opts) => {
|
|
2898
|
+
const store = new Store;
|
|
2899
|
+
try {
|
|
2900
|
+
const run = store.cancelWorkflowRun(runId, opts.reason);
|
|
2901
|
+
print(publicWorkflowRun(run), `${run.id} ${run.status}`);
|
|
2902
|
+
} finally {
|
|
2903
|
+
store.close();
|
|
2904
|
+
}
|
|
2905
|
+
});
|
|
2906
|
+
workflows.command("recover <runId>").description("reset interrupted running workflow steps to pending").option("--reason <reason>", "recovery reason", "manual recovery").action((runId, opts) => {
|
|
2907
|
+
const store = new Store;
|
|
2908
|
+
try {
|
|
2909
|
+
const result = store.recoverWorkflowRun(runId, opts.reason);
|
|
2910
|
+
print({
|
|
2911
|
+
workflowRun: publicWorkflowRun(result.run),
|
|
2912
|
+
recoveredSteps: result.recoveredSteps.map((step) => publicWorkflowStepRun(step))
|
|
2913
|
+
}, `${result.run.id} recovered=${result.recoveredSteps.length}`);
|
|
2914
|
+
} finally {
|
|
2915
|
+
store.close();
|
|
2916
|
+
}
|
|
2917
|
+
});
|
|
2342
2918
|
workflows.command("archive <idOrName>").action((idOrName) => {
|
|
2343
2919
|
const store = new Store;
|
|
2344
2920
|
try {
|
|
@@ -2381,6 +2957,8 @@ program.command("runs [idOrName]").option("--limit <n>", "limit", "50").option("
|
|
|
2381
2957
|
else {
|
|
2382
2958
|
for (const run of runs) {
|
|
2383
2959
|
console.log(`${run.id} ${run.status.padEnd(10)} attempt=${run.attempt} slot=${run.scheduledFor} ${run.loopName}`);
|
|
2960
|
+
if (opts.showOutput)
|
|
2961
|
+
printTextOutput(run);
|
|
2384
2962
|
}
|
|
2385
2963
|
}
|
|
2386
2964
|
} finally {
|
|
@@ -2413,23 +2991,20 @@ program.command("run-now <idOrName>").option("--show-output", "show stdout/stder
|
|
|
2413
2991
|
const store = new Store;
|
|
2414
2992
|
try {
|
|
2415
2993
|
const loop = store.requireLoop(idOrName);
|
|
2416
|
-
const
|
|
2994
|
+
const runnerId = `manual:${process.pid}`;
|
|
2995
|
+
const now = new Date;
|
|
2996
|
+
const scheduledFor = manualRunScheduledFor(loop, now);
|
|
2997
|
+
const shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
|
|
2998
|
+
const claim = store.claimRun(loop, scheduledFor, runnerId, now);
|
|
2417
2999
|
if (!claim)
|
|
2418
3000
|
throw new Error("could not claim manual run");
|
|
2419
|
-
const
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
durationMs: result.durationMs,
|
|
2424
|
-
stdout: result.stdout,
|
|
2425
|
-
stderr: result.stderr,
|
|
2426
|
-
exitCode: result.exitCode,
|
|
2427
|
-
error: result.error,
|
|
2428
|
-
pid: result.pid
|
|
2429
|
-
}, {
|
|
2430
|
-
claimedBy: claim.run.claimedBy
|
|
2431
|
-
});
|
|
3001
|
+
const run = await executeClaimedRun({ store, runnerId, loop: claim.loop, run: claim.run });
|
|
3002
|
+
if (shouldAdvance) {
|
|
3003
|
+
advanceLoop(store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
|
|
3004
|
+
}
|
|
2432
3005
|
print(publicRun(run, opts.showOutput), `${run.id} ${run.status}`);
|
|
3006
|
+
if (!isJson() && opts.showOutput)
|
|
3007
|
+
printTextOutput(run);
|
|
2433
3008
|
} finally {
|
|
2434
3009
|
store.close();
|
|
2435
3010
|
}
|
|
@@ -2443,6 +3018,24 @@ program.command("tick").description("run one scheduler tick").action(async () =>
|
|
|
2443
3018
|
store.close();
|
|
2444
3019
|
}
|
|
2445
3020
|
});
|
|
3021
|
+
program.command("doctor").description("check local OpenLoops runtime dependencies and state").action(() => {
|
|
3022
|
+
const store = new Store;
|
|
3023
|
+
try {
|
|
3024
|
+
const report = runDoctor(store);
|
|
3025
|
+
if (isJson())
|
|
3026
|
+
print(report);
|
|
3027
|
+
else {
|
|
3028
|
+
for (const check of report.checks) {
|
|
3029
|
+
const marker = check.status === "ok" ? "ok" : check.status === "warn" ? "warn" : "fail";
|
|
3030
|
+
console.log(`${marker.padEnd(4)} ${check.id.padEnd(22)} ${check.message}${check.detail ? ` (${check.detail})` : ""}`);
|
|
3031
|
+
}
|
|
3032
|
+
if (!report.ok)
|
|
3033
|
+
process.exitCode = 1;
|
|
3034
|
+
}
|
|
3035
|
+
} finally {
|
|
3036
|
+
store.close();
|
|
3037
|
+
}
|
|
3038
|
+
});
|
|
2446
3039
|
var daemon = program.command("daemon").description("manage the local daemon");
|
|
2447
3040
|
daemon.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
|
|
2448
3041
|
daemon.command("start").action(async () => {
|
|
@@ -2461,11 +3054,16 @@ daemon.command("status").action(() => {
|
|
|
2461
3054
|
store.close();
|
|
2462
3055
|
}
|
|
2463
3056
|
});
|
|
2464
|
-
daemon.command("install").description("write a systemd user service or launchd plist").action(() => {
|
|
3057
|
+
daemon.command("install").description("write a systemd user service or launchd plist").option("--enable", "also enable/start the user service when supported").action((opts) => {
|
|
2465
3058
|
const result = installStartup(process.argv[1] ?? "loops");
|
|
3059
|
+
if (opts.enable)
|
|
3060
|
+
result.enableResults = enableStartup(result);
|
|
3061
|
+
const enableText = result.enableResults ? `
|
|
3062
|
+
${result.enableResults.map((item) => `${item.command} -> ${item.status === 0 ? "ok" : `exit ${item.status}`}`).join(`
|
|
3063
|
+
`)}` : "";
|
|
2466
3064
|
print(result, `wrote ${result.path}
|
|
2467
3065
|
${result.instructions.join(`
|
|
2468
|
-
`)}`);
|
|
3066
|
+
`)}${enableText}`);
|
|
2469
3067
|
});
|
|
2470
3068
|
daemon.command("logs").option("-n, --lines <n>", "lines", "80").action((opts) => {
|
|
2471
3069
|
const path = daemonLogPath();
|