@hasna/loops 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/dist/cli/index.js +510 -159
- package/dist/daemon/index.js +447 -146
- package/dist/index.js +446 -146
- package/dist/lib/executor.d.ts +2 -2
- package/dist/lib/format.d.ts +2 -1
- package/dist/lib/scheduler.d.ts +10 -1
- package/dist/lib/store.d.ts +21 -12
- package/dist/lib/store.js +304 -107
- package/dist/sdk/index.js +446 -146
- package/dist/types.d.ts +5 -0
- package/docs/USAGE.md +21 -1
- package/package.json +1 -1
package/dist/daemon/index.js
CHANGED
|
@@ -260,6 +260,11 @@ function validateTarget(value, label) {
|
|
|
260
260
|
const providers = ["claude", "cursor", "codewith", "aicopilot", "opencode", "codex"];
|
|
261
261
|
if (!providers.includes(value.provider))
|
|
262
262
|
throw new Error(`${label}.provider must be one of ${providers.join(", ")}`);
|
|
263
|
+
if (value.authProfile !== undefined) {
|
|
264
|
+
assertString(value.authProfile, `${label}.authProfile`);
|
|
265
|
+
if (value.provider !== "codewith")
|
|
266
|
+
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
267
|
+
}
|
|
263
268
|
return value;
|
|
264
269
|
}
|
|
265
270
|
throw new Error(`${label}.type must be command or agent`);
|
|
@@ -610,6 +615,13 @@ class Store {
|
|
|
610
615
|
} catch {}
|
|
611
616
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
612
617
|
}
|
|
618
|
+
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
619
|
+
if (!opts.daemonLeaseId)
|
|
620
|
+
return;
|
|
621
|
+
const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
|
|
622
|
+
if (!row)
|
|
623
|
+
throw new Error("daemon lease lost");
|
|
624
|
+
}
|
|
613
625
|
createLoop(input, from = new Date) {
|
|
614
626
|
const now = nowIso();
|
|
615
627
|
const loop = {
|
|
@@ -679,21 +691,31 @@ class Store {
|
|
|
679
691
|
ORDER BY next_run_at ASC`).all(now.toISOString());
|
|
680
692
|
return rows.map(rowToLoop);
|
|
681
693
|
}
|
|
682
|
-
updateLoop(id, patch) {
|
|
694
|
+
updateLoop(id, patch, opts = {}) {
|
|
683
695
|
const current = this.getLoop(id);
|
|
684
696
|
if (!current)
|
|
685
697
|
throw new Error(`loop not found: ${id}`);
|
|
686
|
-
const
|
|
698
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
699
|
+
const merged = { ...current, ...patch, updatedAt: updated };
|
|
687
700
|
this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
688
|
-
expires_at=$expiresAt, updated_at=$updated
|
|
701
|
+
expires_at=$expiresAt, updated_at=$updated
|
|
702
|
+
WHERE id=$id
|
|
703
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
704
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
705
|
+
))`).run({
|
|
689
706
|
$id: id,
|
|
690
707
|
$status: merged.status,
|
|
691
708
|
$nextRun: merged.nextRunAt ?? null,
|
|
692
709
|
$retrySlot: merged.retryScheduledFor ?? null,
|
|
693
710
|
$expiresAt: merged.expiresAt ?? null,
|
|
694
|
-
$updated: merged.updatedAt
|
|
711
|
+
$updated: merged.updatedAt,
|
|
712
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
713
|
+
$now: updated
|
|
695
714
|
});
|
|
696
|
-
|
|
715
|
+
const after = this.getLoop(id);
|
|
716
|
+
if (!after)
|
|
717
|
+
throw new Error(`loop not found after update: ${id}`);
|
|
718
|
+
return after;
|
|
697
719
|
}
|
|
698
720
|
deleteLoop(idOrName) {
|
|
699
721
|
const loop = this.requireLoop(idOrName);
|
|
@@ -757,11 +779,14 @@ class Store {
|
|
|
757
779
|
const now = nowIso();
|
|
758
780
|
if (input.idempotencyKey) {
|
|
759
781
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
760
|
-
if (existing)
|
|
782
|
+
if (existing) {
|
|
783
|
+
this.assertDaemonLeaseFence(input);
|
|
761
784
|
return rowToWorkflowRun(existing);
|
|
785
|
+
}
|
|
762
786
|
}
|
|
763
787
|
this.db.exec("BEGIN IMMEDIATE");
|
|
764
788
|
try {
|
|
789
|
+
this.assertDaemonLeaseFence(input, now);
|
|
765
790
|
if (input.idempotencyKey) {
|
|
766
791
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
767
792
|
if (existing) {
|
|
@@ -860,31 +885,60 @@ class Store {
|
|
|
860
885
|
const run = this.getWorkflowRun(workflowRunId);
|
|
861
886
|
return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
|
|
862
887
|
}
|
|
863
|
-
startWorkflowStepRun(workflowRunId, stepId) {
|
|
864
|
-
const now =
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
888
|
+
startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
|
|
889
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
890
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
891
|
+
try {
|
|
892
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
893
|
+
SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
|
|
894
|
+
pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
895
|
+
WHERE workflow_run_id=$workflowRunId
|
|
896
|
+
AND step_id=$stepId
|
|
897
|
+
AND status IN ('pending', 'failed', 'timed_out')
|
|
898
|
+
AND EXISTS (
|
|
899
|
+
SELECT 1 FROM workflow_runs
|
|
900
|
+
WHERE id=$workflowRunId AND status='running'
|
|
901
|
+
)
|
|
902
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
903
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
904
|
+
))`).run({
|
|
905
|
+
$workflowRunId: workflowRunId,
|
|
906
|
+
$stepId: stepId,
|
|
907
|
+
$started: now,
|
|
908
|
+
$updated: now,
|
|
909
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
910
|
+
$now: now
|
|
911
|
+
});
|
|
912
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
913
|
+
if (!run)
|
|
914
|
+
throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
|
|
915
|
+
if (res.changes !== 1) {
|
|
916
|
+
throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
|
|
917
|
+
}
|
|
918
|
+
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
919
|
+
this.db.exec("COMMIT");
|
|
920
|
+
return run;
|
|
921
|
+
} catch (error) {
|
|
922
|
+
try {
|
|
923
|
+
this.db.exec("ROLLBACK");
|
|
924
|
+
} catch {}
|
|
925
|
+
throw error;
|
|
880
926
|
}
|
|
881
|
-
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
882
|
-
return run;
|
|
883
927
|
}
|
|
884
|
-
markWorkflowStepPid(workflowRunId, stepId, pid) {
|
|
885
|
-
const now =
|
|
928
|
+
markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
|
|
929
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
886
930
|
this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
|
|
887
|
-
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
931
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
932
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
933
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
934
|
+
))`).run({
|
|
935
|
+
$workflowRunId: workflowRunId,
|
|
936
|
+
$stepId: stepId,
|
|
937
|
+
$pid: pid,
|
|
938
|
+
$updated: now,
|
|
939
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
940
|
+
$now: now
|
|
941
|
+
});
|
|
888
942
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
889
943
|
if (!run)
|
|
890
944
|
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
@@ -912,60 +966,110 @@ class Store {
|
|
|
912
966
|
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
913
967
|
};
|
|
914
968
|
}
|
|
915
|
-
finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
|
|
969
|
+
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
916
970
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
971
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
972
|
+
try {
|
|
973
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
974
|
+
pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
975
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
976
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
977
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
978
|
+
))`).run({
|
|
979
|
+
$workflowRunId: workflowRunId,
|
|
980
|
+
$stepId: stepId,
|
|
981
|
+
$status: patch.status,
|
|
982
|
+
$finished: finishedAt,
|
|
983
|
+
$exitCode: patch.exitCode ?? null,
|
|
984
|
+
$durationMs: patch.durationMs ?? null,
|
|
985
|
+
$stdout: patch.stdout ?? null,
|
|
986
|
+
$stderr: patch.stderr ?? null,
|
|
987
|
+
$error: patch.error ?? null,
|
|
988
|
+
$updated: finishedAt,
|
|
989
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
990
|
+
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
935
991
|
});
|
|
992
|
+
if (res.changes === 1) {
|
|
993
|
+
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
994
|
+
exitCode: patch.exitCode,
|
|
995
|
+
error: patch.error
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
this.db.exec("COMMIT");
|
|
999
|
+
} catch (error) {
|
|
1000
|
+
try {
|
|
1001
|
+
this.db.exec("ROLLBACK");
|
|
1002
|
+
} catch {}
|
|
1003
|
+
throw error;
|
|
936
1004
|
}
|
|
937
1005
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
938
1006
|
if (!run)
|
|
939
1007
|
throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
|
|
940
1008
|
return run;
|
|
941
1009
|
}
|
|
942
|
-
skipWorkflowStepRun(workflowRunId, stepId, reason) {
|
|
943
|
-
const now =
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
1010
|
+
skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
|
|
1011
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1012
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1013
|
+
try {
|
|
1014
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
|
|
1015
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
|
|
1016
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1017
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1018
|
+
))`).run({
|
|
1019
|
+
$workflowRunId: workflowRunId,
|
|
1020
|
+
$stepId: stepId,
|
|
1021
|
+
$finished: now,
|
|
1022
|
+
$error: reason,
|
|
1023
|
+
$updated: now,
|
|
1024
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1025
|
+
$now: now
|
|
1026
|
+
});
|
|
1027
|
+
if (res.changes === 1)
|
|
1028
|
+
this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
|
|
1029
|
+
this.db.exec("COMMIT");
|
|
1030
|
+
} catch (error) {
|
|
1031
|
+
try {
|
|
1032
|
+
this.db.exec("ROLLBACK");
|
|
1033
|
+
} catch {}
|
|
1034
|
+
throw error;
|
|
1035
|
+
}
|
|
948
1036
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
949
1037
|
if (!run)
|
|
950
1038
|
throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
|
|
951
1039
|
return run;
|
|
952
1040
|
}
|
|
953
|
-
finalizeWorkflowRun(workflowRunId, status, patch = {}) {
|
|
1041
|
+
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
954
1042
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1043
|
+
let changed = false;
|
|
1044
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1045
|
+
try {
|
|
1046
|
+
const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
|
|
1047
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1048
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1049
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1050
|
+
))`).run({
|
|
1051
|
+
$id: workflowRunId,
|
|
1052
|
+
$status: status,
|
|
1053
|
+
$finished: finishedAt,
|
|
1054
|
+
$durationMs: patch.durationMs ?? null,
|
|
1055
|
+
$error: patch.error ?? null,
|
|
1056
|
+
$updated: finishedAt,
|
|
1057
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1058
|
+
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
1059
|
+
});
|
|
1060
|
+
changed = res.changes === 1;
|
|
1061
|
+
if (changed)
|
|
1062
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
1063
|
+
this.db.exec("COMMIT");
|
|
1064
|
+
} catch (error) {
|
|
1065
|
+
try {
|
|
1066
|
+
this.db.exec("ROLLBACK");
|
|
1067
|
+
} catch {}
|
|
1068
|
+
throw error;
|
|
1069
|
+
}
|
|
964
1070
|
const run = this.getWorkflowRun(workflowRunId);
|
|
965
1071
|
if (!run)
|
|
966
1072
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
967
|
-
if (res.changes === 1)
|
|
968
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
969
1073
|
return run;
|
|
970
1074
|
}
|
|
971
1075
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -1019,10 +1123,24 @@ class Store {
|
|
|
1019
1123
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1020
1124
|
return (row?.count ?? 0) > 0;
|
|
1021
1125
|
}
|
|
1022
|
-
markRunPid(id, pid, claimedBy) {
|
|
1023
|
-
const now =
|
|
1126
|
+
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1127
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1024
1128
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1025
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1129
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1130
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1131
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1132
|
+
))`).run({
|
|
1133
|
+
$id: id,
|
|
1134
|
+
$pid: pid,
|
|
1135
|
+
$updated: now,
|
|
1136
|
+
$claimedBy: claimedBy,
|
|
1137
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1138
|
+
$now: now
|
|
1139
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1140
|
+
WHERE id=$id AND status='running'
|
|
1141
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1142
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1143
|
+
))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
|
|
1026
1144
|
if (res.changes !== 1)
|
|
1027
1145
|
return;
|
|
1028
1146
|
return this.getRun(id);
|
|
@@ -1037,7 +1155,7 @@ class Store {
|
|
|
1037
1155
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
1038
1156
|
return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
|
|
1039
1157
|
}
|
|
1040
|
-
createSkippedRun(loop, scheduledFor, reason) {
|
|
1158
|
+
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
1041
1159
|
const now = nowIso();
|
|
1042
1160
|
const run = {
|
|
1043
1161
|
id: genId(),
|
|
@@ -1051,21 +1169,31 @@ class Store {
|
|
|
1051
1169
|
createdAt: now,
|
|
1052
1170
|
updatedAt: now
|
|
1053
1171
|
};
|
|
1054
|
-
this.db.
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1172
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1173
|
+
try {
|
|
1174
|
+
this.assertDaemonLeaseFence(opts, now);
|
|
1175
|
+
this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
|
|
1176
|
+
claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
|
|
1177
|
+
VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
|
|
1178
|
+
NULL, NULL, $error, $created, $updated)`).run({
|
|
1179
|
+
$id: run.id,
|
|
1180
|
+
$loopId: run.loopId,
|
|
1181
|
+
$loopName: run.loopName,
|
|
1182
|
+
$scheduledFor: run.scheduledFor,
|
|
1183
|
+
$attempt: run.attempt,
|
|
1184
|
+
$status: run.status,
|
|
1185
|
+
$finished: run.finishedAt ?? null,
|
|
1186
|
+
$error: run.error ?? null,
|
|
1187
|
+
$created: run.createdAt,
|
|
1188
|
+
$updated: run.updatedAt
|
|
1189
|
+
});
|
|
1190
|
+
this.db.exec("COMMIT");
|
|
1191
|
+
} catch (error) {
|
|
1192
|
+
try {
|
|
1193
|
+
this.db.exec("ROLLBACK");
|
|
1194
|
+
} catch {}
|
|
1195
|
+
throw error;
|
|
1196
|
+
}
|
|
1069
1197
|
return this.getRunBySlot(loop.id, scheduledFor) ?? run;
|
|
1070
1198
|
}
|
|
1071
1199
|
getRun(id) {
|
|
@@ -1076,11 +1204,20 @@ class Store {
|
|
|
1076
1204
|
const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
|
|
1077
1205
|
return row ? rowToRun(row) : undefined;
|
|
1078
1206
|
}
|
|
1079
|
-
|
|
1207
|
+
nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
|
|
1208
|
+
const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
|
|
1209
|
+
WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
|
|
1210
|
+
ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
|
|
1211
|
+
WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
|
|
1212
|
+
ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
|
|
1213
|
+
return row ? rowToRun(row) : undefined;
|
|
1214
|
+
}
|
|
1215
|
+
claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
|
|
1080
1216
|
const startedAt = now.toISOString();
|
|
1081
1217
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
1082
1218
|
this.db.exec("BEGIN IMMEDIATE");
|
|
1083
1219
|
try {
|
|
1220
|
+
this.assertDaemonLeaseFence(opts, startedAt);
|
|
1084
1221
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
1085
1222
|
if (existing) {
|
|
1086
1223
|
if (existing.status === "running") {
|
|
@@ -1175,11 +1312,15 @@ class Store {
|
|
|
1175
1312
|
$error: patch.error ?? null,
|
|
1176
1313
|
$updated: finishedAt,
|
|
1177
1314
|
$claimedBy: opts.claimedBy ?? null,
|
|
1178
|
-
$now: (opts.now ?? new Date).toISOString()
|
|
1315
|
+
$now: (opts.now ?? new Date).toISOString(),
|
|
1316
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1179
1317
|
};
|
|
1180
1318
|
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1181
1319
|
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
1182
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1320
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1321
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1322
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1323
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1183
1324
|
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
1184
1325
|
const run = this.getRun(id);
|
|
1185
1326
|
if (!run)
|
|
@@ -1188,10 +1329,20 @@ class Store {
|
|
|
1188
1329
|
return run;
|
|
1189
1330
|
return run;
|
|
1190
1331
|
}
|
|
1191
|
-
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
|
|
1332
|
+
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
1192
1333
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
1193
1334
|
const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
|
|
1194
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1335
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1336
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1337
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1338
|
+
))`).run({
|
|
1339
|
+
$id: id,
|
|
1340
|
+
$claimedBy: claimedBy,
|
|
1341
|
+
$expires: expiresAt,
|
|
1342
|
+
$updated: now.toISOString(),
|
|
1343
|
+
$now: now.toISOString(),
|
|
1344
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1345
|
+
});
|
|
1195
1346
|
if (res.changes !== 1)
|
|
1196
1347
|
return;
|
|
1197
1348
|
return this.getRun(id);
|
|
@@ -1210,7 +1361,7 @@ class Store {
|
|
|
1210
1361
|
}
|
|
1211
1362
|
return rows.map(rowToRun);
|
|
1212
1363
|
}
|
|
1213
|
-
recoverExpiredRunLeases(now = new Date) {
|
|
1364
|
+
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
1214
1365
|
const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
|
|
1215
1366
|
const recovered = [];
|
|
1216
1367
|
for (const row of rows) {
|
|
@@ -1219,20 +1370,63 @@ class Store {
|
|
|
1219
1370
|
if (this.hasLiveWorkflowStepProcesses(row.id))
|
|
1220
1371
|
continue;
|
|
1221
1372
|
const finished = now.toISOString();
|
|
1222
|
-
this.db.
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1373
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1374
|
+
try {
|
|
1375
|
+
const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
|
|
1376
|
+
error='run lease expired before completion', updated_at=$updated
|
|
1377
|
+
WHERE id=$id AND status='running' AND lease_expires_at <= $now
|
|
1378
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1379
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1380
|
+
))`).run({
|
|
1381
|
+
$id: row.id,
|
|
1382
|
+
$finished: finished,
|
|
1383
|
+
$updated: finished,
|
|
1384
|
+
$now: finished,
|
|
1385
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1235
1386
|
});
|
|
1387
|
+
if (res.changes !== 1) {
|
|
1388
|
+
this.db.exec("COMMIT");
|
|
1389
|
+
continue;
|
|
1390
|
+
}
|
|
1391
|
+
const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
|
|
1392
|
+
for (const workflowRow of workflowRows) {
|
|
1393
|
+
const workflowRes = this.db.query(`UPDATE workflow_runs
|
|
1394
|
+
SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1395
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1396
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1397
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1398
|
+
))`).run({
|
|
1399
|
+
$id: workflowRow.id,
|
|
1400
|
+
$finished: finished,
|
|
1401
|
+
$updated: finished,
|
|
1402
|
+
$now: finished,
|
|
1403
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1404
|
+
});
|
|
1405
|
+
if (workflowRes.changes !== 1)
|
|
1406
|
+
continue;
|
|
1407
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
1408
|
+
SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1409
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
|
|
1410
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1411
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1412
|
+
))`).run({
|
|
1413
|
+
$workflowRunId: workflowRow.id,
|
|
1414
|
+
$finished: finished,
|
|
1415
|
+
$updated: finished,
|
|
1416
|
+
$now: finished,
|
|
1417
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1418
|
+
});
|
|
1419
|
+
this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
|
|
1420
|
+
error: "parent loop run lease expired before completion",
|
|
1421
|
+
loopRunId: row.id
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
this.db.exec("COMMIT");
|
|
1425
|
+
} catch (error) {
|
|
1426
|
+
try {
|
|
1427
|
+
this.db.exec("ROLLBACK");
|
|
1428
|
+
} catch {}
|
|
1429
|
+
throw error;
|
|
1236
1430
|
}
|
|
1237
1431
|
const run = this.getRun(row.id);
|
|
1238
1432
|
if (run)
|
|
@@ -1240,11 +1434,14 @@ class Store {
|
|
|
1240
1434
|
}
|
|
1241
1435
|
return recovered;
|
|
1242
1436
|
}
|
|
1243
|
-
expireLoops(now = new Date) {
|
|
1437
|
+
expireLoops(now = new Date, opts = {}) {
|
|
1244
1438
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
1245
1439
|
const expired = [];
|
|
1246
|
-
for (const row of rows)
|
|
1247
|
-
|
|
1440
|
+
for (const row of rows) {
|
|
1441
|
+
const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
|
|
1442
|
+
if (updated.status === "expired")
|
|
1443
|
+
expired.push(updated);
|
|
1444
|
+
}
|
|
1248
1445
|
return expired;
|
|
1249
1446
|
}
|
|
1250
1447
|
countLoops(status) {
|
|
@@ -1287,7 +1484,7 @@ class Store {
|
|
|
1287
1484
|
}
|
|
1288
1485
|
heartbeatDaemonLease(id, ttlMs, now = new Date) {
|
|
1289
1486
|
const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
|
|
1290
|
-
const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
|
|
1487
|
+
const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
|
|
1291
1488
|
if (res.changes !== 1)
|
|
1292
1489
|
return;
|
|
1293
1490
|
return this.getDaemonLease();
|
|
@@ -1545,7 +1742,7 @@ function agentArgs(target) {
|
|
|
1545
1742
|
args.push("--model", target.model);
|
|
1546
1743
|
if (target.agent)
|
|
1547
1744
|
args.push("--agent", target.agent);
|
|
1548
|
-
args.push(...target.extraArgs ?? []
|
|
1745
|
+
args.push(...target.extraArgs ?? []);
|
|
1549
1746
|
return args;
|
|
1550
1747
|
case "cursor":
|
|
1551
1748
|
args.push("-p");
|
|
@@ -1553,10 +1750,10 @@ function agentArgs(target) {
|
|
|
1553
1750
|
args.push("--model", target.model);
|
|
1554
1751
|
if (target.agent)
|
|
1555
1752
|
args.push("--agent", target.agent);
|
|
1556
|
-
args.push(...target.extraArgs ?? []
|
|
1753
|
+
args.push(...target.extraArgs ?? []);
|
|
1557
1754
|
return args;
|
|
1558
1755
|
case "codewith":
|
|
1559
|
-
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
|
|
1756
|
+
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
|
|
1560
1757
|
if (isolation === "safe")
|
|
1561
1758
|
args.push("--ignore-rules");
|
|
1562
1759
|
if (target.cwd)
|
|
@@ -1565,7 +1762,7 @@ function agentArgs(target) {
|
|
|
1565
1762
|
args.push("--model", target.model);
|
|
1566
1763
|
if (target.agent)
|
|
1567
1764
|
args.push("--agent", target.agent);
|
|
1568
|
-
args.push(...target.extraArgs ?? []
|
|
1765
|
+
args.push(...target.extraArgs ?? []);
|
|
1569
1766
|
return args;
|
|
1570
1767
|
case "codex":
|
|
1571
1768
|
args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
|
|
@@ -1575,7 +1772,7 @@ function agentArgs(target) {
|
|
|
1575
1772
|
args.push("--cd", target.cwd);
|
|
1576
1773
|
if (target.model)
|
|
1577
1774
|
args.push("--model", target.model);
|
|
1578
|
-
args.push(...target.extraArgs ?? []
|
|
1775
|
+
args.push(...target.extraArgs ?? []);
|
|
1579
1776
|
return args;
|
|
1580
1777
|
case "aicopilot":
|
|
1581
1778
|
args.push("run", "--format", "json");
|
|
@@ -1587,7 +1784,7 @@ function agentArgs(target) {
|
|
|
1587
1784
|
args.push("--model", target.model);
|
|
1588
1785
|
if (target.agent)
|
|
1589
1786
|
args.push("--agent", target.agent);
|
|
1590
|
-
args.push(...target.extraArgs ?? []
|
|
1787
|
+
args.push(...target.extraArgs ?? []);
|
|
1591
1788
|
return args;
|
|
1592
1789
|
case "opencode":
|
|
1593
1790
|
args.push("run", "--format", "json");
|
|
@@ -1599,7 +1796,7 @@ function agentArgs(target) {
|
|
|
1599
1796
|
args.push("--model", target.model);
|
|
1600
1797
|
if (target.agent)
|
|
1601
1798
|
args.push("--agent", target.agent);
|
|
1602
|
-
args.push(...target.extraArgs ?? []
|
|
1799
|
+
args.push(...target.extraArgs ?? []);
|
|
1603
1800
|
return args;
|
|
1604
1801
|
}
|
|
1605
1802
|
}
|
|
@@ -1624,7 +1821,8 @@ function commandSpec(target) {
|
|
|
1624
1821
|
cwd: agentTarget.cwd,
|
|
1625
1822
|
timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
1626
1823
|
account: agentTarget.account,
|
|
1627
|
-
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
|
|
1824
|
+
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
|
|
1825
|
+
stdin: agentTarget.prompt
|
|
1628
1826
|
};
|
|
1629
1827
|
}
|
|
1630
1828
|
function executionEnv(spec, metadata, opts) {
|
|
@@ -1693,10 +1891,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1693
1891
|
env,
|
|
1694
1892
|
shell: spec.shell ?? false,
|
|
1695
1893
|
detached: true,
|
|
1696
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
1894
|
+
stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
|
|
1697
1895
|
});
|
|
1698
1896
|
if (child.pid)
|
|
1699
1897
|
opts.onSpawn?.(child.pid);
|
|
1898
|
+
if (spec.stdin !== undefined && child.stdin) {
|
|
1899
|
+
child.stdin.on("error", (err) => {
|
|
1900
|
+
if (err.code !== "EPIPE")
|
|
1901
|
+
error = err.message;
|
|
1902
|
+
});
|
|
1903
|
+
child.stdin.end(spec.stdin);
|
|
1904
|
+
}
|
|
1700
1905
|
const abortHandler = () => {
|
|
1701
1906
|
error = "cancelled";
|
|
1702
1907
|
if (child.pid)
|
|
@@ -1705,10 +1910,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1705
1910
|
if (opts.signal?.aborted)
|
|
1706
1911
|
abortHandler();
|
|
1707
1912
|
opts.signal?.addEventListener("abort", abortHandler, { once: true });
|
|
1708
|
-
child.stdout
|
|
1913
|
+
child.stdout?.on("data", (chunk) => {
|
|
1709
1914
|
stdout = appendBounded(stdout, chunk, maxOutputBytes);
|
|
1710
1915
|
});
|
|
1711
|
-
child.stderr
|
|
1916
|
+
child.stderr?.on("data", (chunk) => {
|
|
1712
1917
|
stderr = appendBounded(stderr, chunk, maxOutputBytes);
|
|
1713
1918
|
});
|
|
1714
1919
|
const timer = setTimeout(() => {
|
|
@@ -1807,7 +2012,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1807
2012
|
loop: opts.loop,
|
|
1808
2013
|
loopRun: opts.loopRun,
|
|
1809
2014
|
scheduledFor: opts.scheduledFor,
|
|
1810
|
-
idempotencyKey: opts.idempotencyKey
|
|
2015
|
+
idempotencyKey: opts.idempotencyKey,
|
|
2016
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1811
2017
|
});
|
|
1812
2018
|
const startedAt = run.startedAt ?? nowIso();
|
|
1813
2019
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
@@ -1841,12 +2047,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1841
2047
|
return !dependencyStep?.continueOnFailure;
|
|
1842
2048
|
});
|
|
1843
2049
|
if (blockedBy) {
|
|
1844
|
-
|
|
2050
|
+
opts.beforePersist?.();
|
|
2051
|
+
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
1845
2052
|
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
1846
2053
|
terminalStatus = "failed";
|
|
1847
2054
|
continue;
|
|
1848
2055
|
}
|
|
1849
|
-
|
|
2056
|
+
opts.beforePersist?.();
|
|
2057
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
|
|
1850
2058
|
if (startedStep.status !== "running") {
|
|
1851
2059
|
terminalStatus = "failed";
|
|
1852
2060
|
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
@@ -1878,7 +2086,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1878
2086
|
...opts,
|
|
1879
2087
|
signal: controller.signal,
|
|
1880
2088
|
onSpawn: (pid) => {
|
|
1881
|
-
|
|
2089
|
+
opts.beforePersist?.();
|
|
2090
|
+
store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
|
|
1882
2091
|
opts.onSpawn?.(pid);
|
|
1883
2092
|
}
|
|
1884
2093
|
});
|
|
@@ -1906,6 +2115,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1906
2115
|
blockingError = "workflow run was cancelled";
|
|
1907
2116
|
break;
|
|
1908
2117
|
}
|
|
2118
|
+
opts.beforePersist?.();
|
|
1909
2119
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
1910
2120
|
status: result.status,
|
|
1911
2121
|
finishedAt: result.finishedAt,
|
|
@@ -1914,6 +2124,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1914
2124
|
stderr: result.stderr,
|
|
1915
2125
|
exitCode: result.exitCode,
|
|
1916
2126
|
error: result.error
|
|
2127
|
+
}, {
|
|
2128
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1917
2129
|
});
|
|
1918
2130
|
if (result.status !== "succeeded" && !step.continueOnFailure) {
|
|
1919
2131
|
terminalStatus = result.status;
|
|
@@ -1925,7 +2137,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1925
2137
|
for (const step of ordered) {
|
|
1926
2138
|
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
1927
2139
|
if (existing?.status === "pending" || existing?.status === "running") {
|
|
1928
|
-
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run"
|
|
2140
|
+
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
|
|
2141
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
2142
|
+
});
|
|
1929
2143
|
}
|
|
1930
2144
|
}
|
|
1931
2145
|
}
|
|
@@ -1935,10 +2149,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1935
2149
|
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
1936
2150
|
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
1937
2151
|
}
|
|
2152
|
+
opts.beforePersist?.();
|
|
1938
2153
|
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
1939
2154
|
finishedAt,
|
|
1940
2155
|
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
1941
2156
|
error: blockingError
|
|
2157
|
+
}, {
|
|
2158
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1942
2159
|
});
|
|
1943
2160
|
const steps = store.listWorkflowStepRuns(run.id);
|
|
1944
2161
|
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
@@ -1986,52 +2203,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
1986
2203
|
|
|
1987
2204
|
// src/lib/scheduler.ts
|
|
1988
2205
|
function manualRunScheduledFor(loop, now = new Date) {
|
|
1989
|
-
if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
2206
|
+
if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
1990
2207
|
return loop.retryScheduledFor ?? loop.nextRunAt;
|
|
1991
2208
|
}
|
|
1992
2209
|
return now.toISOString();
|
|
1993
2210
|
}
|
|
1994
2211
|
function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
|
|
2212
|
+
if (loop.status !== "active")
|
|
2213
|
+
return false;
|
|
1995
2214
|
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
1996
2215
|
return false;
|
|
1997
2216
|
return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
|
|
1998
2217
|
}
|
|
2218
|
+
function manualRunSource(loop, scheduledFor, now = new Date) {
|
|
2219
|
+
if (loop.status !== "active")
|
|
2220
|
+
return "ad_hoc";
|
|
2221
|
+
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
2222
|
+
return "ad_hoc";
|
|
2223
|
+
if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
|
|
2224
|
+
return "retry_slot";
|
|
2225
|
+
return "due_slot";
|
|
2226
|
+
}
|
|
1999
2227
|
function nextAfterRetry(loop, now) {
|
|
2000
2228
|
return new Date(now.getTime() + loop.retryDelayMs).toISOString();
|
|
2001
2229
|
}
|
|
2002
|
-
function
|
|
2230
|
+
function isDaemonLeaseLost(error) {
|
|
2231
|
+
return error instanceof Error && error.message === "daemon lease lost";
|
|
2232
|
+
}
|
|
2233
|
+
function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
2003
2234
|
if (run.status === "running")
|
|
2004
2235
|
return;
|
|
2005
2236
|
const current = store.getLoop(loop.id);
|
|
2006
2237
|
if (!current || current.status !== "active")
|
|
2007
2238
|
return;
|
|
2008
|
-
|
|
2239
|
+
if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
|
|
2240
|
+
return;
|
|
2241
|
+
const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
|
|
2009
2242
|
if (shouldRetry) {
|
|
2010
|
-
store.updateLoop(
|
|
2243
|
+
store.updateLoop(current.id, {
|
|
2011
2244
|
status: "active",
|
|
2012
|
-
nextRunAt: nextAfterRetry(
|
|
2245
|
+
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
2013
2246
|
retryScheduledFor: run.scheduledFor
|
|
2014
|
-
});
|
|
2247
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2015
2248
|
return;
|
|
2016
2249
|
}
|
|
2017
|
-
const
|
|
2018
|
-
|
|
2250
|
+
const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
|
|
2251
|
+
if (deferredRetry) {
|
|
2252
|
+
store.updateLoop(current.id, {
|
|
2253
|
+
status: "active",
|
|
2254
|
+
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
2255
|
+
retryScheduledFor: deferredRetry.scheduledFor
|
|
2256
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2257
|
+
return;
|
|
2258
|
+
}
|
|
2259
|
+
const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
|
|
2260
|
+
store.updateLoop(current.id, {
|
|
2019
2261
|
status: nextRunAt ? "active" : "stopped",
|
|
2020
2262
|
nextRunAt,
|
|
2021
2263
|
retryScheduledFor: undefined
|
|
2022
|
-
});
|
|
2264
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2023
2265
|
}
|
|
2024
2266
|
async function executeClaimedRun(deps) {
|
|
2025
2267
|
let heartbeat;
|
|
2026
2268
|
const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
|
|
2027
2269
|
heartbeat = setInterval(() => {
|
|
2028
|
-
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs
|
|
2270
|
+
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
|
|
2271
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2272
|
+
});
|
|
2029
2273
|
}, heartbeatEveryMs);
|
|
2030
2274
|
heartbeat.unref();
|
|
2031
2275
|
try {
|
|
2032
2276
|
const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
|
|
2033
|
-
|
|
2277
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2278
|
+
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
2034
2279
|
})))(deps.loop, deps.run);
|
|
2280
|
+
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
2035
2281
|
return deps.store.finalizeRun(deps.run.id, {
|
|
2036
2282
|
status: result.status,
|
|
2037
2283
|
finishedAt: result.finishedAt,
|
|
@@ -2043,10 +2289,16 @@ async function executeClaimedRun(deps) {
|
|
|
2043
2289
|
pid: result.pid
|
|
2044
2290
|
}, {
|
|
2045
2291
|
claimedBy: deps.runnerId,
|
|
2292
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2046
2293
|
now: deps.now?.() ?? new Date(result.finishedAt)
|
|
2047
2294
|
});
|
|
2048
2295
|
} catch (err) {
|
|
2049
2296
|
deps.onError?.(deps.loop, err);
|
|
2297
|
+
try {
|
|
2298
|
+
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
2299
|
+
} catch {
|
|
2300
|
+
return deps.store.getRun(deps.run.id) ?? deps.run;
|
|
2301
|
+
}
|
|
2050
2302
|
const finishedAt = new Date;
|
|
2051
2303
|
return deps.store.finalizeRun(deps.run.id, {
|
|
2052
2304
|
status: "failed",
|
|
@@ -2057,6 +2309,7 @@ async function executeClaimedRun(deps) {
|
|
|
2057
2309
|
error: err instanceof Error ? err.message : String(err)
|
|
2058
2310
|
}, {
|
|
2059
2311
|
claimedBy: deps.runnerId,
|
|
2312
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2060
2313
|
now: deps.now?.() ?? finishedAt
|
|
2061
2314
|
});
|
|
2062
2315
|
} finally {
|
|
@@ -2066,15 +2319,33 @@ async function executeClaimedRun(deps) {
|
|
|
2066
2319
|
}
|
|
2067
2320
|
async function runSlot(deps, loop, scheduledFor) {
|
|
2068
2321
|
const now = deps.now?.() ?? new Date;
|
|
2322
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
2069
2323
|
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
2070
|
-
|
|
2071
|
-
|
|
2324
|
+
let skipped;
|
|
2325
|
+
try {
|
|
2326
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
2327
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2328
|
+
});
|
|
2329
|
+
} catch (error) {
|
|
2330
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
2331
|
+
return;
|
|
2332
|
+
throw error;
|
|
2333
|
+
}
|
|
2334
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
2072
2335
|
deps.onRun?.(skipped);
|
|
2073
2336
|
return skipped;
|
|
2074
2337
|
}
|
|
2075
|
-
|
|
2338
|
+
let claim;
|
|
2339
|
+
try {
|
|
2340
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2341
|
+
} catch (error) {
|
|
2342
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
2343
|
+
return;
|
|
2344
|
+
throw error;
|
|
2345
|
+
}
|
|
2076
2346
|
if (!claim)
|
|
2077
2347
|
return;
|
|
2348
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
2078
2349
|
deps.onRun?.(claim.run);
|
|
2079
2350
|
const finalRun = await executeClaimedRun({
|
|
2080
2351
|
store: deps.store,
|
|
@@ -2083,21 +2354,42 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
2083
2354
|
run: claim.run,
|
|
2084
2355
|
now: deps.now,
|
|
2085
2356
|
execute: deps.execute,
|
|
2357
|
+
beforeFinalize: deps.beforeFinalize,
|
|
2358
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2086
2359
|
onError: deps.onError
|
|
2087
2360
|
});
|
|
2088
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
|
|
2361
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
|
|
2089
2362
|
deps.onRun?.(finalRun);
|
|
2090
2363
|
return finalRun;
|
|
2091
2364
|
}
|
|
2092
2365
|
async function tick(deps) {
|
|
2093
2366
|
const now = deps.now?.() ?? new Date;
|
|
2094
|
-
const recovered = deps.store.recoverExpiredRunLeases(now);
|
|
2367
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2368
|
+
const recoveredByLoop = new Map;
|
|
2095
2369
|
for (const run of recovered) {
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2370
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
2371
|
+
}
|
|
2372
|
+
for (const runs of recoveredByLoop.values()) {
|
|
2373
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
2374
|
+
if (!loop)
|
|
2375
|
+
continue;
|
|
2376
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
2377
|
+
if (retryable) {
|
|
2378
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
2379
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2380
|
+
});
|
|
2381
|
+
continue;
|
|
2382
|
+
}
|
|
2383
|
+
for (const run of runs) {
|
|
2384
|
+
const current = deps.store.getLoop(run.loopId);
|
|
2385
|
+
if (current) {
|
|
2386
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
2387
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2388
|
+
});
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2099
2391
|
}
|
|
2100
|
-
const expired = deps.store.expireLoops(now);
|
|
2392
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2101
2393
|
const claimed = [];
|
|
2102
2394
|
const completed = [];
|
|
2103
2395
|
const skipped = [];
|
|
@@ -2315,8 +2607,10 @@ async function runDaemon(opts = {}) {
|
|
|
2315
2607
|
const result = await tick({
|
|
2316
2608
|
store,
|
|
2317
2609
|
runnerId,
|
|
2610
|
+
daemonLeaseId: leaseId,
|
|
2611
|
+
beforeRun: () => ensureLease(),
|
|
2318
2612
|
execute: async (loop, run) => {
|
|
2319
|
-
const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs /
|
|
2613
|
+
const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
|
|
2320
2614
|
const timer = setInterval(() => {
|
|
2321
2615
|
try {
|
|
2322
2616
|
ensureLease();
|
|
@@ -2328,8 +2622,14 @@ async function runDaemon(opts = {}) {
|
|
|
2328
2622
|
try {
|
|
2329
2623
|
const result2 = await executeLoopTarget(store, loop, run, {
|
|
2330
2624
|
signal: runAbort.signal,
|
|
2331
|
-
|
|
2625
|
+
beforePersist: () => ensureLease(),
|
|
2626
|
+
daemonLeaseId: leaseId,
|
|
2627
|
+
onSpawn: (pid) => {
|
|
2628
|
+
ensureLease();
|
|
2629
|
+
store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
|
|
2630
|
+
}
|
|
2332
2631
|
});
|
|
2632
|
+
ensureLease();
|
|
2333
2633
|
if (leaseLost)
|
|
2334
2634
|
throw new Error("daemon lease lost during run");
|
|
2335
2635
|
return result2;
|
|
@@ -2337,6 +2637,7 @@ async function runDaemon(opts = {}) {
|
|
|
2337
2637
|
clearInterval(timer);
|
|
2338
2638
|
}
|
|
2339
2639
|
},
|
|
2640
|
+
beforeFinalize: () => ensureLease(),
|
|
2340
2641
|
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
2341
2642
|
});
|
|
2342
2643
|
const changed = result.completed.length + result.skipped.length + result.recovered.length + result.expired.length;
|
|
@@ -2467,7 +2768,7 @@ function enableStartup(result) {
|
|
|
2467
2768
|
|
|
2468
2769
|
// src/daemon/index.ts
|
|
2469
2770
|
var program = new Command;
|
|
2470
|
-
program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.
|
|
2771
|
+
program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.3");
|
|
2471
2772
|
program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
|
|
2472
2773
|
program.command("start").action(async () => {
|
|
2473
2774
|
const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops-daemon", args: ["run"] });
|