@hasna/loops 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/cli/index.js +493 -157
- package/dist/daemon/index.js +441 -145
- package/dist/index.js +440 -145
- package/dist/lib/executor.d.ts +2 -2
- package/dist/lib/format.d.ts +2 -1
- package/dist/lib/scheduler.d.ts +10 -1
- package/dist/lib/store.d.ts +21 -12
- package/dist/lib/store.js +299 -107
- package/dist/sdk/index.js +440 -145
- package/dist/types.d.ts +4 -0
- package/docs/USAGE.md +8 -0
- package/package.json +1 -1
package/dist/daemon/index.js
CHANGED
|
@@ -615,6 +615,13 @@ class Store {
|
|
|
615
615
|
} catch {}
|
|
616
616
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
617
617
|
}
|
|
618
|
+
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
619
|
+
if (!opts.daemonLeaseId)
|
|
620
|
+
return;
|
|
621
|
+
const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
|
|
622
|
+
if (!row)
|
|
623
|
+
throw new Error("daemon lease lost");
|
|
624
|
+
}
|
|
618
625
|
createLoop(input, from = new Date) {
|
|
619
626
|
const now = nowIso();
|
|
620
627
|
const loop = {
|
|
@@ -684,21 +691,31 @@ class Store {
|
|
|
684
691
|
ORDER BY next_run_at ASC`).all(now.toISOString());
|
|
685
692
|
return rows.map(rowToLoop);
|
|
686
693
|
}
|
|
687
|
-
updateLoop(id, patch) {
|
|
694
|
+
updateLoop(id, patch, opts = {}) {
|
|
688
695
|
const current = this.getLoop(id);
|
|
689
696
|
if (!current)
|
|
690
697
|
throw new Error(`loop not found: ${id}`);
|
|
691
|
-
const
|
|
698
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
699
|
+
const merged = { ...current, ...patch, updatedAt: updated };
|
|
692
700
|
this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
693
|
-
expires_at=$expiresAt, updated_at=$updated
|
|
701
|
+
expires_at=$expiresAt, updated_at=$updated
|
|
702
|
+
WHERE id=$id
|
|
703
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
704
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
705
|
+
))`).run({
|
|
694
706
|
$id: id,
|
|
695
707
|
$status: merged.status,
|
|
696
708
|
$nextRun: merged.nextRunAt ?? null,
|
|
697
709
|
$retrySlot: merged.retryScheduledFor ?? null,
|
|
698
710
|
$expiresAt: merged.expiresAt ?? null,
|
|
699
|
-
$updated: merged.updatedAt
|
|
711
|
+
$updated: merged.updatedAt,
|
|
712
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
713
|
+
$now: updated
|
|
700
714
|
});
|
|
701
|
-
|
|
715
|
+
const after = this.getLoop(id);
|
|
716
|
+
if (!after)
|
|
717
|
+
throw new Error(`loop not found after update: ${id}`);
|
|
718
|
+
return after;
|
|
702
719
|
}
|
|
703
720
|
deleteLoop(idOrName) {
|
|
704
721
|
const loop = this.requireLoop(idOrName);
|
|
@@ -762,11 +779,14 @@ class Store {
|
|
|
762
779
|
const now = nowIso();
|
|
763
780
|
if (input.idempotencyKey) {
|
|
764
781
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
765
|
-
if (existing)
|
|
782
|
+
if (existing) {
|
|
783
|
+
this.assertDaemonLeaseFence(input);
|
|
766
784
|
return rowToWorkflowRun(existing);
|
|
785
|
+
}
|
|
767
786
|
}
|
|
768
787
|
this.db.exec("BEGIN IMMEDIATE");
|
|
769
788
|
try {
|
|
789
|
+
this.assertDaemonLeaseFence(input, now);
|
|
770
790
|
if (input.idempotencyKey) {
|
|
771
791
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
772
792
|
if (existing) {
|
|
@@ -865,31 +885,60 @@ class Store {
|
|
|
865
885
|
const run = this.getWorkflowRun(workflowRunId);
|
|
866
886
|
return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
|
|
867
887
|
}
|
|
868
|
-
startWorkflowStepRun(workflowRunId, stepId) {
|
|
869
|
-
const now =
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
888
|
+
startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
|
|
889
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
890
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
891
|
+
try {
|
|
892
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
893
|
+
SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
|
|
894
|
+
pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
895
|
+
WHERE workflow_run_id=$workflowRunId
|
|
896
|
+
AND step_id=$stepId
|
|
897
|
+
AND status IN ('pending', 'failed', 'timed_out')
|
|
898
|
+
AND EXISTS (
|
|
899
|
+
SELECT 1 FROM workflow_runs
|
|
900
|
+
WHERE id=$workflowRunId AND status='running'
|
|
901
|
+
)
|
|
902
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
903
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
904
|
+
))`).run({
|
|
905
|
+
$workflowRunId: workflowRunId,
|
|
906
|
+
$stepId: stepId,
|
|
907
|
+
$started: now,
|
|
908
|
+
$updated: now,
|
|
909
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
910
|
+
$now: now
|
|
911
|
+
});
|
|
912
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
913
|
+
if (!run)
|
|
914
|
+
throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
|
|
915
|
+
if (res.changes !== 1) {
|
|
916
|
+
throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
|
|
917
|
+
}
|
|
918
|
+
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
919
|
+
this.db.exec("COMMIT");
|
|
920
|
+
return run;
|
|
921
|
+
} catch (error) {
|
|
922
|
+
try {
|
|
923
|
+
this.db.exec("ROLLBACK");
|
|
924
|
+
} catch {}
|
|
925
|
+
throw error;
|
|
885
926
|
}
|
|
886
|
-
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
887
|
-
return run;
|
|
888
927
|
}
|
|
889
|
-
markWorkflowStepPid(workflowRunId, stepId, pid) {
|
|
890
|
-
const now =
|
|
928
|
+
markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
|
|
929
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
891
930
|
this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
|
|
892
|
-
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
931
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
932
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
933
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
934
|
+
))`).run({
|
|
935
|
+
$workflowRunId: workflowRunId,
|
|
936
|
+
$stepId: stepId,
|
|
937
|
+
$pid: pid,
|
|
938
|
+
$updated: now,
|
|
939
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
940
|
+
$now: now
|
|
941
|
+
});
|
|
893
942
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
894
943
|
if (!run)
|
|
895
944
|
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
@@ -917,60 +966,110 @@ class Store {
|
|
|
917
966
|
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
918
967
|
};
|
|
919
968
|
}
|
|
920
|
-
finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
|
|
969
|
+
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
921
970
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
971
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
972
|
+
try {
|
|
973
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
974
|
+
pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
975
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
976
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
977
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
978
|
+
))`).run({
|
|
979
|
+
$workflowRunId: workflowRunId,
|
|
980
|
+
$stepId: stepId,
|
|
981
|
+
$status: patch.status,
|
|
982
|
+
$finished: finishedAt,
|
|
983
|
+
$exitCode: patch.exitCode ?? null,
|
|
984
|
+
$durationMs: patch.durationMs ?? null,
|
|
985
|
+
$stdout: patch.stdout ?? null,
|
|
986
|
+
$stderr: patch.stderr ?? null,
|
|
987
|
+
$error: patch.error ?? null,
|
|
988
|
+
$updated: finishedAt,
|
|
989
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
990
|
+
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
940
991
|
});
|
|
992
|
+
if (res.changes === 1) {
|
|
993
|
+
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
994
|
+
exitCode: patch.exitCode,
|
|
995
|
+
error: patch.error
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
this.db.exec("COMMIT");
|
|
999
|
+
} catch (error) {
|
|
1000
|
+
try {
|
|
1001
|
+
this.db.exec("ROLLBACK");
|
|
1002
|
+
} catch {}
|
|
1003
|
+
throw error;
|
|
941
1004
|
}
|
|
942
1005
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
943
1006
|
if (!run)
|
|
944
1007
|
throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
|
|
945
1008
|
return run;
|
|
946
1009
|
}
|
|
947
|
-
skipWorkflowStepRun(workflowRunId, stepId, reason) {
|
|
948
|
-
const now =
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
1010
|
+
skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
|
|
1011
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1012
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1013
|
+
try {
|
|
1014
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
|
|
1015
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
|
|
1016
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1017
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1018
|
+
))`).run({
|
|
1019
|
+
$workflowRunId: workflowRunId,
|
|
1020
|
+
$stepId: stepId,
|
|
1021
|
+
$finished: now,
|
|
1022
|
+
$error: reason,
|
|
1023
|
+
$updated: now,
|
|
1024
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1025
|
+
$now: now
|
|
1026
|
+
});
|
|
1027
|
+
if (res.changes === 1)
|
|
1028
|
+
this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
|
|
1029
|
+
this.db.exec("COMMIT");
|
|
1030
|
+
} catch (error) {
|
|
1031
|
+
try {
|
|
1032
|
+
this.db.exec("ROLLBACK");
|
|
1033
|
+
} catch {}
|
|
1034
|
+
throw error;
|
|
1035
|
+
}
|
|
953
1036
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
954
1037
|
if (!run)
|
|
955
1038
|
throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
|
|
956
1039
|
return run;
|
|
957
1040
|
}
|
|
958
|
-
finalizeWorkflowRun(workflowRunId, status, patch = {}) {
|
|
1041
|
+
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
959
1042
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
1043
|
+
let changed = false;
|
|
1044
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1045
|
+
try {
|
|
1046
|
+
const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
|
|
1047
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1048
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1049
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1050
|
+
))`).run({
|
|
1051
|
+
$id: workflowRunId,
|
|
1052
|
+
$status: status,
|
|
1053
|
+
$finished: finishedAt,
|
|
1054
|
+
$durationMs: patch.durationMs ?? null,
|
|
1055
|
+
$error: patch.error ?? null,
|
|
1056
|
+
$updated: finishedAt,
|
|
1057
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1058
|
+
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
1059
|
+
});
|
|
1060
|
+
changed = res.changes === 1;
|
|
1061
|
+
if (changed)
|
|
1062
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
1063
|
+
this.db.exec("COMMIT");
|
|
1064
|
+
} catch (error) {
|
|
1065
|
+
try {
|
|
1066
|
+
this.db.exec("ROLLBACK");
|
|
1067
|
+
} catch {}
|
|
1068
|
+
throw error;
|
|
1069
|
+
}
|
|
969
1070
|
const run = this.getWorkflowRun(workflowRunId);
|
|
970
1071
|
if (!run)
|
|
971
1072
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
972
|
-
if (res.changes === 1)
|
|
973
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
974
1073
|
return run;
|
|
975
1074
|
}
|
|
976
1075
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -1024,10 +1123,24 @@ class Store {
|
|
|
1024
1123
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1025
1124
|
return (row?.count ?? 0) > 0;
|
|
1026
1125
|
}
|
|
1027
|
-
markRunPid(id, pid, claimedBy) {
|
|
1028
|
-
const now =
|
|
1126
|
+
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1127
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1029
1128
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1030
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1129
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1130
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1131
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1132
|
+
))`).run({
|
|
1133
|
+
$id: id,
|
|
1134
|
+
$pid: pid,
|
|
1135
|
+
$updated: now,
|
|
1136
|
+
$claimedBy: claimedBy,
|
|
1137
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1138
|
+
$now: now
|
|
1139
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1140
|
+
WHERE id=$id AND status='running'
|
|
1141
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1142
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1143
|
+
))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
|
|
1031
1144
|
if (res.changes !== 1)
|
|
1032
1145
|
return;
|
|
1033
1146
|
return this.getRun(id);
|
|
@@ -1042,7 +1155,7 @@ class Store {
|
|
|
1042
1155
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
1043
1156
|
return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
|
|
1044
1157
|
}
|
|
1045
|
-
createSkippedRun(loop, scheduledFor, reason) {
|
|
1158
|
+
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
1046
1159
|
const now = nowIso();
|
|
1047
1160
|
const run = {
|
|
1048
1161
|
id: genId(),
|
|
@@ -1056,21 +1169,31 @@ class Store {
|
|
|
1056
1169
|
createdAt: now,
|
|
1057
1170
|
updatedAt: now
|
|
1058
1171
|
};
|
|
1059
|
-
this.db.
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1172
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1173
|
+
try {
|
|
1174
|
+
this.assertDaemonLeaseFence(opts, now);
|
|
1175
|
+
this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
|
|
1176
|
+
claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
|
|
1177
|
+
VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
|
|
1178
|
+
NULL, NULL, $error, $created, $updated)`).run({
|
|
1179
|
+
$id: run.id,
|
|
1180
|
+
$loopId: run.loopId,
|
|
1181
|
+
$loopName: run.loopName,
|
|
1182
|
+
$scheduledFor: run.scheduledFor,
|
|
1183
|
+
$attempt: run.attempt,
|
|
1184
|
+
$status: run.status,
|
|
1185
|
+
$finished: run.finishedAt ?? null,
|
|
1186
|
+
$error: run.error ?? null,
|
|
1187
|
+
$created: run.createdAt,
|
|
1188
|
+
$updated: run.updatedAt
|
|
1189
|
+
});
|
|
1190
|
+
this.db.exec("COMMIT");
|
|
1191
|
+
} catch (error) {
|
|
1192
|
+
try {
|
|
1193
|
+
this.db.exec("ROLLBACK");
|
|
1194
|
+
} catch {}
|
|
1195
|
+
throw error;
|
|
1196
|
+
}
|
|
1074
1197
|
return this.getRunBySlot(loop.id, scheduledFor) ?? run;
|
|
1075
1198
|
}
|
|
1076
1199
|
getRun(id) {
|
|
@@ -1081,11 +1204,20 @@ class Store {
|
|
|
1081
1204
|
const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
|
|
1082
1205
|
return row ? rowToRun(row) : undefined;
|
|
1083
1206
|
}
|
|
1084
|
-
|
|
1207
|
+
nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
|
|
1208
|
+
const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
|
|
1209
|
+
WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
|
|
1210
|
+
ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
|
|
1211
|
+
WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
|
|
1212
|
+
ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
|
|
1213
|
+
return row ? rowToRun(row) : undefined;
|
|
1214
|
+
}
|
|
1215
|
+
claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
|
|
1085
1216
|
const startedAt = now.toISOString();
|
|
1086
1217
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
1087
1218
|
this.db.exec("BEGIN IMMEDIATE");
|
|
1088
1219
|
try {
|
|
1220
|
+
this.assertDaemonLeaseFence(opts, startedAt);
|
|
1089
1221
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
1090
1222
|
if (existing) {
|
|
1091
1223
|
if (existing.status === "running") {
|
|
@@ -1180,11 +1312,15 @@ class Store {
|
|
|
1180
1312
|
$error: patch.error ?? null,
|
|
1181
1313
|
$updated: finishedAt,
|
|
1182
1314
|
$claimedBy: opts.claimedBy ?? null,
|
|
1183
|
-
$now: (opts.now ?? new Date).toISOString()
|
|
1315
|
+
$now: (opts.now ?? new Date).toISOString(),
|
|
1316
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1184
1317
|
};
|
|
1185
1318
|
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1186
1319
|
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
1187
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1320
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1321
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1322
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1323
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1188
1324
|
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
1189
1325
|
const run = this.getRun(id);
|
|
1190
1326
|
if (!run)
|
|
@@ -1193,10 +1329,20 @@ class Store {
|
|
|
1193
1329
|
return run;
|
|
1194
1330
|
return run;
|
|
1195
1331
|
}
|
|
1196
|
-
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
|
|
1332
|
+
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
1197
1333
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
1198
1334
|
const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
|
|
1199
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1335
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1336
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1337
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1338
|
+
))`).run({
|
|
1339
|
+
$id: id,
|
|
1340
|
+
$claimedBy: claimedBy,
|
|
1341
|
+
$expires: expiresAt,
|
|
1342
|
+
$updated: now.toISOString(),
|
|
1343
|
+
$now: now.toISOString(),
|
|
1344
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1345
|
+
});
|
|
1200
1346
|
if (res.changes !== 1)
|
|
1201
1347
|
return;
|
|
1202
1348
|
return this.getRun(id);
|
|
@@ -1215,7 +1361,7 @@ class Store {
|
|
|
1215
1361
|
}
|
|
1216
1362
|
return rows.map(rowToRun);
|
|
1217
1363
|
}
|
|
1218
|
-
recoverExpiredRunLeases(now = new Date) {
|
|
1364
|
+
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
1219
1365
|
const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
|
|
1220
1366
|
const recovered = [];
|
|
1221
1367
|
for (const row of rows) {
|
|
@@ -1224,20 +1370,63 @@ class Store {
|
|
|
1224
1370
|
if (this.hasLiveWorkflowStepProcesses(row.id))
|
|
1225
1371
|
continue;
|
|
1226
1372
|
const finished = now.toISOString();
|
|
1227
|
-
this.db.
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1373
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1374
|
+
try {
|
|
1375
|
+
const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
|
|
1376
|
+
error='run lease expired before completion', updated_at=$updated
|
|
1377
|
+
WHERE id=$id AND status='running' AND lease_expires_at <= $now
|
|
1378
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1379
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1380
|
+
))`).run({
|
|
1381
|
+
$id: row.id,
|
|
1382
|
+
$finished: finished,
|
|
1383
|
+
$updated: finished,
|
|
1384
|
+
$now: finished,
|
|
1385
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1240
1386
|
});
|
|
1387
|
+
if (res.changes !== 1) {
|
|
1388
|
+
this.db.exec("COMMIT");
|
|
1389
|
+
continue;
|
|
1390
|
+
}
|
|
1391
|
+
const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
|
|
1392
|
+
for (const workflowRow of workflowRows) {
|
|
1393
|
+
const workflowRes = this.db.query(`UPDATE workflow_runs
|
|
1394
|
+
SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1395
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1396
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1397
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1398
|
+
))`).run({
|
|
1399
|
+
$id: workflowRow.id,
|
|
1400
|
+
$finished: finished,
|
|
1401
|
+
$updated: finished,
|
|
1402
|
+
$now: finished,
|
|
1403
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1404
|
+
});
|
|
1405
|
+
if (workflowRes.changes !== 1)
|
|
1406
|
+
continue;
|
|
1407
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
1408
|
+
SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1409
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
|
|
1410
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1411
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1412
|
+
))`).run({
|
|
1413
|
+
$workflowRunId: workflowRow.id,
|
|
1414
|
+
$finished: finished,
|
|
1415
|
+
$updated: finished,
|
|
1416
|
+
$now: finished,
|
|
1417
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1418
|
+
});
|
|
1419
|
+
this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
|
|
1420
|
+
error: "parent loop run lease expired before completion",
|
|
1421
|
+
loopRunId: row.id
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
this.db.exec("COMMIT");
|
|
1425
|
+
} catch (error) {
|
|
1426
|
+
try {
|
|
1427
|
+
this.db.exec("ROLLBACK");
|
|
1428
|
+
} catch {}
|
|
1429
|
+
throw error;
|
|
1241
1430
|
}
|
|
1242
1431
|
const run = this.getRun(row.id);
|
|
1243
1432
|
if (run)
|
|
@@ -1245,11 +1434,14 @@ class Store {
|
|
|
1245
1434
|
}
|
|
1246
1435
|
return recovered;
|
|
1247
1436
|
}
|
|
1248
|
-
expireLoops(now = new Date) {
|
|
1437
|
+
expireLoops(now = new Date, opts = {}) {
|
|
1249
1438
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
1250
1439
|
const expired = [];
|
|
1251
|
-
for (const row of rows)
|
|
1252
|
-
|
|
1440
|
+
for (const row of rows) {
|
|
1441
|
+
const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
|
|
1442
|
+
if (updated.status === "expired")
|
|
1443
|
+
expired.push(updated);
|
|
1444
|
+
}
|
|
1253
1445
|
return expired;
|
|
1254
1446
|
}
|
|
1255
1447
|
countLoops(status) {
|
|
@@ -1292,7 +1484,7 @@ class Store {
|
|
|
1292
1484
|
}
|
|
1293
1485
|
heartbeatDaemonLease(id, ttlMs, now = new Date) {
|
|
1294
1486
|
const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
|
|
1295
|
-
const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
|
|
1487
|
+
const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
|
|
1296
1488
|
if (res.changes !== 1)
|
|
1297
1489
|
return;
|
|
1298
1490
|
return this.getDaemonLease();
|
|
@@ -1550,7 +1742,7 @@ function agentArgs(target) {
|
|
|
1550
1742
|
args.push("--model", target.model);
|
|
1551
1743
|
if (target.agent)
|
|
1552
1744
|
args.push("--agent", target.agent);
|
|
1553
|
-
args.push(...target.extraArgs ?? []
|
|
1745
|
+
args.push(...target.extraArgs ?? []);
|
|
1554
1746
|
return args;
|
|
1555
1747
|
case "cursor":
|
|
1556
1748
|
args.push("-p");
|
|
@@ -1558,7 +1750,7 @@ function agentArgs(target) {
|
|
|
1558
1750
|
args.push("--model", target.model);
|
|
1559
1751
|
if (target.agent)
|
|
1560
1752
|
args.push("--agent", target.agent);
|
|
1561
|
-
args.push(...target.extraArgs ?? []
|
|
1753
|
+
args.push(...target.extraArgs ?? []);
|
|
1562
1754
|
return args;
|
|
1563
1755
|
case "codewith":
|
|
1564
1756
|
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
|
|
@@ -1570,7 +1762,7 @@ function agentArgs(target) {
|
|
|
1570
1762
|
args.push("--model", target.model);
|
|
1571
1763
|
if (target.agent)
|
|
1572
1764
|
args.push("--agent", target.agent);
|
|
1573
|
-
args.push(...target.extraArgs ?? []
|
|
1765
|
+
args.push(...target.extraArgs ?? []);
|
|
1574
1766
|
return args;
|
|
1575
1767
|
case "codex":
|
|
1576
1768
|
args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
|
|
@@ -1580,7 +1772,7 @@ function agentArgs(target) {
|
|
|
1580
1772
|
args.push("--cd", target.cwd);
|
|
1581
1773
|
if (target.model)
|
|
1582
1774
|
args.push("--model", target.model);
|
|
1583
|
-
args.push(...target.extraArgs ?? []
|
|
1775
|
+
args.push(...target.extraArgs ?? []);
|
|
1584
1776
|
return args;
|
|
1585
1777
|
case "aicopilot":
|
|
1586
1778
|
args.push("run", "--format", "json");
|
|
@@ -1592,7 +1784,7 @@ function agentArgs(target) {
|
|
|
1592
1784
|
args.push("--model", target.model);
|
|
1593
1785
|
if (target.agent)
|
|
1594
1786
|
args.push("--agent", target.agent);
|
|
1595
|
-
args.push(...target.extraArgs ?? []
|
|
1787
|
+
args.push(...target.extraArgs ?? []);
|
|
1596
1788
|
return args;
|
|
1597
1789
|
case "opencode":
|
|
1598
1790
|
args.push("run", "--format", "json");
|
|
@@ -1604,7 +1796,7 @@ function agentArgs(target) {
|
|
|
1604
1796
|
args.push("--model", target.model);
|
|
1605
1797
|
if (target.agent)
|
|
1606
1798
|
args.push("--agent", target.agent);
|
|
1607
|
-
args.push(...target.extraArgs ?? []
|
|
1799
|
+
args.push(...target.extraArgs ?? []);
|
|
1608
1800
|
return args;
|
|
1609
1801
|
}
|
|
1610
1802
|
}
|
|
@@ -1629,7 +1821,8 @@ function commandSpec(target) {
|
|
|
1629
1821
|
cwd: agentTarget.cwd,
|
|
1630
1822
|
timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
1631
1823
|
account: agentTarget.account,
|
|
1632
|
-
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
|
|
1824
|
+
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
|
|
1825
|
+
stdin: agentTarget.prompt
|
|
1633
1826
|
};
|
|
1634
1827
|
}
|
|
1635
1828
|
function executionEnv(spec, metadata, opts) {
|
|
@@ -1698,10 +1891,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1698
1891
|
env,
|
|
1699
1892
|
shell: spec.shell ?? false,
|
|
1700
1893
|
detached: true,
|
|
1701
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
1894
|
+
stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
|
|
1702
1895
|
});
|
|
1703
1896
|
if (child.pid)
|
|
1704
1897
|
opts.onSpawn?.(child.pid);
|
|
1898
|
+
if (spec.stdin !== undefined && child.stdin) {
|
|
1899
|
+
child.stdin.on("error", (err) => {
|
|
1900
|
+
if (err.code !== "EPIPE")
|
|
1901
|
+
error = err.message;
|
|
1902
|
+
});
|
|
1903
|
+
child.stdin.end(spec.stdin);
|
|
1904
|
+
}
|
|
1705
1905
|
const abortHandler = () => {
|
|
1706
1906
|
error = "cancelled";
|
|
1707
1907
|
if (child.pid)
|
|
@@ -1710,10 +1910,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1710
1910
|
if (opts.signal?.aborted)
|
|
1711
1911
|
abortHandler();
|
|
1712
1912
|
opts.signal?.addEventListener("abort", abortHandler, { once: true });
|
|
1713
|
-
child.stdout
|
|
1913
|
+
child.stdout?.on("data", (chunk) => {
|
|
1714
1914
|
stdout = appendBounded(stdout, chunk, maxOutputBytes);
|
|
1715
1915
|
});
|
|
1716
|
-
child.stderr
|
|
1916
|
+
child.stderr?.on("data", (chunk) => {
|
|
1717
1917
|
stderr = appendBounded(stderr, chunk, maxOutputBytes);
|
|
1718
1918
|
});
|
|
1719
1919
|
const timer = setTimeout(() => {
|
|
@@ -1812,7 +2012,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1812
2012
|
loop: opts.loop,
|
|
1813
2013
|
loopRun: opts.loopRun,
|
|
1814
2014
|
scheduledFor: opts.scheduledFor,
|
|
1815
|
-
idempotencyKey: opts.idempotencyKey
|
|
2015
|
+
idempotencyKey: opts.idempotencyKey,
|
|
2016
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1816
2017
|
});
|
|
1817
2018
|
const startedAt = run.startedAt ?? nowIso();
|
|
1818
2019
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
@@ -1846,12 +2047,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1846
2047
|
return !dependencyStep?.continueOnFailure;
|
|
1847
2048
|
});
|
|
1848
2049
|
if (blockedBy) {
|
|
1849
|
-
|
|
2050
|
+
opts.beforePersist?.();
|
|
2051
|
+
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
1850
2052
|
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
1851
2053
|
terminalStatus = "failed";
|
|
1852
2054
|
continue;
|
|
1853
2055
|
}
|
|
1854
|
-
|
|
2056
|
+
opts.beforePersist?.();
|
|
2057
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
|
|
1855
2058
|
if (startedStep.status !== "running") {
|
|
1856
2059
|
terminalStatus = "failed";
|
|
1857
2060
|
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
@@ -1883,7 +2086,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1883
2086
|
...opts,
|
|
1884
2087
|
signal: controller.signal,
|
|
1885
2088
|
onSpawn: (pid) => {
|
|
1886
|
-
|
|
2089
|
+
opts.beforePersist?.();
|
|
2090
|
+
store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
|
|
1887
2091
|
opts.onSpawn?.(pid);
|
|
1888
2092
|
}
|
|
1889
2093
|
});
|
|
@@ -1911,6 +2115,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1911
2115
|
blockingError = "workflow run was cancelled";
|
|
1912
2116
|
break;
|
|
1913
2117
|
}
|
|
2118
|
+
opts.beforePersist?.();
|
|
1914
2119
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
1915
2120
|
status: result.status,
|
|
1916
2121
|
finishedAt: result.finishedAt,
|
|
@@ -1919,6 +2124,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1919
2124
|
stderr: result.stderr,
|
|
1920
2125
|
exitCode: result.exitCode,
|
|
1921
2126
|
error: result.error
|
|
2127
|
+
}, {
|
|
2128
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1922
2129
|
});
|
|
1923
2130
|
if (result.status !== "succeeded" && !step.continueOnFailure) {
|
|
1924
2131
|
terminalStatus = result.status;
|
|
@@ -1930,7 +2137,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1930
2137
|
for (const step of ordered) {
|
|
1931
2138
|
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
1932
2139
|
if (existing?.status === "pending" || existing?.status === "running") {
|
|
1933
|
-
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run"
|
|
2140
|
+
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
|
|
2141
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
2142
|
+
});
|
|
1934
2143
|
}
|
|
1935
2144
|
}
|
|
1936
2145
|
}
|
|
@@ -1940,10 +2149,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1940
2149
|
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
1941
2150
|
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
1942
2151
|
}
|
|
2152
|
+
opts.beforePersist?.();
|
|
1943
2153
|
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
1944
2154
|
finishedAt,
|
|
1945
2155
|
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
1946
2156
|
error: blockingError
|
|
2157
|
+
}, {
|
|
2158
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1947
2159
|
});
|
|
1948
2160
|
const steps = store.listWorkflowStepRuns(run.id);
|
|
1949
2161
|
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
@@ -1991,52 +2203,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
1991
2203
|
|
|
1992
2204
|
// src/lib/scheduler.ts
|
|
1993
2205
|
function manualRunScheduledFor(loop, now = new Date) {
|
|
1994
|
-
if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
2206
|
+
if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
1995
2207
|
return loop.retryScheduledFor ?? loop.nextRunAt;
|
|
1996
2208
|
}
|
|
1997
2209
|
return now.toISOString();
|
|
1998
2210
|
}
|
|
1999
2211
|
function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
|
|
2212
|
+
if (loop.status !== "active")
|
|
2213
|
+
return false;
|
|
2000
2214
|
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
2001
2215
|
return false;
|
|
2002
2216
|
return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
|
|
2003
2217
|
}
|
|
2218
|
+
function manualRunSource(loop, scheduledFor, now = new Date) {
|
|
2219
|
+
if (loop.status !== "active")
|
|
2220
|
+
return "ad_hoc";
|
|
2221
|
+
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
2222
|
+
return "ad_hoc";
|
|
2223
|
+
if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
|
|
2224
|
+
return "retry_slot";
|
|
2225
|
+
return "due_slot";
|
|
2226
|
+
}
|
|
2004
2227
|
function nextAfterRetry(loop, now) {
|
|
2005
2228
|
return new Date(now.getTime() + loop.retryDelayMs).toISOString();
|
|
2006
2229
|
}
|
|
2007
|
-
function
|
|
2230
|
+
function isDaemonLeaseLost(error) {
|
|
2231
|
+
return error instanceof Error && error.message === "daemon lease lost";
|
|
2232
|
+
}
|
|
2233
|
+
function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
2008
2234
|
if (run.status === "running")
|
|
2009
2235
|
return;
|
|
2010
2236
|
const current = store.getLoop(loop.id);
|
|
2011
2237
|
if (!current || current.status !== "active")
|
|
2012
2238
|
return;
|
|
2013
|
-
|
|
2239
|
+
if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
|
|
2240
|
+
return;
|
|
2241
|
+
const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
|
|
2014
2242
|
if (shouldRetry) {
|
|
2015
|
-
store.updateLoop(
|
|
2243
|
+
store.updateLoop(current.id, {
|
|
2016
2244
|
status: "active",
|
|
2017
|
-
nextRunAt: nextAfterRetry(
|
|
2245
|
+
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
2018
2246
|
retryScheduledFor: run.scheduledFor
|
|
2019
|
-
});
|
|
2247
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2020
2248
|
return;
|
|
2021
2249
|
}
|
|
2022
|
-
const
|
|
2023
|
-
|
|
2250
|
+
const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
|
|
2251
|
+
if (deferredRetry) {
|
|
2252
|
+
store.updateLoop(current.id, {
|
|
2253
|
+
status: "active",
|
|
2254
|
+
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
2255
|
+
retryScheduledFor: deferredRetry.scheduledFor
|
|
2256
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2257
|
+
return;
|
|
2258
|
+
}
|
|
2259
|
+
const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
|
|
2260
|
+
store.updateLoop(current.id, {
|
|
2024
2261
|
status: nextRunAt ? "active" : "stopped",
|
|
2025
2262
|
nextRunAt,
|
|
2026
2263
|
retryScheduledFor: undefined
|
|
2027
|
-
});
|
|
2264
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2028
2265
|
}
|
|
2029
2266
|
async function executeClaimedRun(deps) {
|
|
2030
2267
|
let heartbeat;
|
|
2031
2268
|
const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
|
|
2032
2269
|
heartbeat = setInterval(() => {
|
|
2033
|
-
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs
|
|
2270
|
+
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
|
|
2271
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2272
|
+
});
|
|
2034
2273
|
}, heartbeatEveryMs);
|
|
2035
2274
|
heartbeat.unref();
|
|
2036
2275
|
try {
|
|
2037
2276
|
const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
|
|
2038
|
-
|
|
2277
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2278
|
+
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
2039
2279
|
})))(deps.loop, deps.run);
|
|
2280
|
+
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
2040
2281
|
return deps.store.finalizeRun(deps.run.id, {
|
|
2041
2282
|
status: result.status,
|
|
2042
2283
|
finishedAt: result.finishedAt,
|
|
@@ -2048,10 +2289,16 @@ async function executeClaimedRun(deps) {
|
|
|
2048
2289
|
pid: result.pid
|
|
2049
2290
|
}, {
|
|
2050
2291
|
claimedBy: deps.runnerId,
|
|
2292
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2051
2293
|
now: deps.now?.() ?? new Date(result.finishedAt)
|
|
2052
2294
|
});
|
|
2053
2295
|
} catch (err) {
|
|
2054
2296
|
deps.onError?.(deps.loop, err);
|
|
2297
|
+
try {
|
|
2298
|
+
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
2299
|
+
} catch {
|
|
2300
|
+
return deps.store.getRun(deps.run.id) ?? deps.run;
|
|
2301
|
+
}
|
|
2055
2302
|
const finishedAt = new Date;
|
|
2056
2303
|
return deps.store.finalizeRun(deps.run.id, {
|
|
2057
2304
|
status: "failed",
|
|
@@ -2062,6 +2309,7 @@ async function executeClaimedRun(deps) {
|
|
|
2062
2309
|
error: err instanceof Error ? err.message : String(err)
|
|
2063
2310
|
}, {
|
|
2064
2311
|
claimedBy: deps.runnerId,
|
|
2312
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2065
2313
|
now: deps.now?.() ?? finishedAt
|
|
2066
2314
|
});
|
|
2067
2315
|
} finally {
|
|
@@ -2071,15 +2319,33 @@ async function executeClaimedRun(deps) {
|
|
|
2071
2319
|
}
|
|
2072
2320
|
async function runSlot(deps, loop, scheduledFor) {
|
|
2073
2321
|
const now = deps.now?.() ?? new Date;
|
|
2322
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
2074
2323
|
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
2075
|
-
|
|
2076
|
-
|
|
2324
|
+
let skipped;
|
|
2325
|
+
try {
|
|
2326
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
2327
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2328
|
+
});
|
|
2329
|
+
} catch (error) {
|
|
2330
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
2331
|
+
return;
|
|
2332
|
+
throw error;
|
|
2333
|
+
}
|
|
2334
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
2077
2335
|
deps.onRun?.(skipped);
|
|
2078
2336
|
return skipped;
|
|
2079
2337
|
}
|
|
2080
|
-
|
|
2338
|
+
let claim;
|
|
2339
|
+
try {
|
|
2340
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2341
|
+
} catch (error) {
|
|
2342
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
2343
|
+
return;
|
|
2344
|
+
throw error;
|
|
2345
|
+
}
|
|
2081
2346
|
if (!claim)
|
|
2082
2347
|
return;
|
|
2348
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
2083
2349
|
deps.onRun?.(claim.run);
|
|
2084
2350
|
const finalRun = await executeClaimedRun({
|
|
2085
2351
|
store: deps.store,
|
|
@@ -2088,21 +2354,42 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
2088
2354
|
run: claim.run,
|
|
2089
2355
|
now: deps.now,
|
|
2090
2356
|
execute: deps.execute,
|
|
2357
|
+
beforeFinalize: deps.beforeFinalize,
|
|
2358
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2091
2359
|
onError: deps.onError
|
|
2092
2360
|
});
|
|
2093
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
|
|
2361
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
|
|
2094
2362
|
deps.onRun?.(finalRun);
|
|
2095
2363
|
return finalRun;
|
|
2096
2364
|
}
|
|
2097
2365
|
async function tick(deps) {
|
|
2098
2366
|
const now = deps.now?.() ?? new Date;
|
|
2099
|
-
const recovered = deps.store.recoverExpiredRunLeases(now);
|
|
2367
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2368
|
+
const recoveredByLoop = new Map;
|
|
2100
2369
|
for (const run of recovered) {
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2370
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
2371
|
+
}
|
|
2372
|
+
for (const runs of recoveredByLoop.values()) {
|
|
2373
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
2374
|
+
if (!loop)
|
|
2375
|
+
continue;
|
|
2376
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
2377
|
+
if (retryable) {
|
|
2378
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
2379
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2380
|
+
});
|
|
2381
|
+
continue;
|
|
2382
|
+
}
|
|
2383
|
+
for (const run of runs) {
|
|
2384
|
+
const current = deps.store.getLoop(run.loopId);
|
|
2385
|
+
if (current) {
|
|
2386
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
2387
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2388
|
+
});
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2104
2391
|
}
|
|
2105
|
-
const expired = deps.store.expireLoops(now);
|
|
2392
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2106
2393
|
const claimed = [];
|
|
2107
2394
|
const completed = [];
|
|
2108
2395
|
const skipped = [];
|
|
@@ -2320,8 +2607,10 @@ async function runDaemon(opts = {}) {
|
|
|
2320
2607
|
const result = await tick({
|
|
2321
2608
|
store,
|
|
2322
2609
|
runnerId,
|
|
2610
|
+
daemonLeaseId: leaseId,
|
|
2611
|
+
beforeRun: () => ensureLease(),
|
|
2323
2612
|
execute: async (loop, run) => {
|
|
2324
|
-
const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs /
|
|
2613
|
+
const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
|
|
2325
2614
|
const timer = setInterval(() => {
|
|
2326
2615
|
try {
|
|
2327
2616
|
ensureLease();
|
|
@@ -2333,8 +2622,14 @@ async function runDaemon(opts = {}) {
|
|
|
2333
2622
|
try {
|
|
2334
2623
|
const result2 = await executeLoopTarget(store, loop, run, {
|
|
2335
2624
|
signal: runAbort.signal,
|
|
2336
|
-
|
|
2625
|
+
beforePersist: () => ensureLease(),
|
|
2626
|
+
daemonLeaseId: leaseId,
|
|
2627
|
+
onSpawn: (pid) => {
|
|
2628
|
+
ensureLease();
|
|
2629
|
+
store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
|
|
2630
|
+
}
|
|
2337
2631
|
});
|
|
2632
|
+
ensureLease();
|
|
2338
2633
|
if (leaseLost)
|
|
2339
2634
|
throw new Error("daemon lease lost during run");
|
|
2340
2635
|
return result2;
|
|
@@ -2342,6 +2637,7 @@ async function runDaemon(opts = {}) {
|
|
|
2342
2637
|
clearInterval(timer);
|
|
2343
2638
|
}
|
|
2344
2639
|
},
|
|
2640
|
+
beforeFinalize: () => ensureLease(),
|
|
2345
2641
|
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
2346
2642
|
});
|
|
2347
2643
|
const changed = result.completed.length + result.skipped.length + result.recovered.length + result.expired.length;
|
|
@@ -2472,7 +2768,7 @@ function enableStartup(result) {
|
|
|
2472
2768
|
|
|
2473
2769
|
// src/daemon/index.ts
|
|
2474
2770
|
var program = new Command;
|
|
2475
|
-
program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.
|
|
2771
|
+
program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.3");
|
|
2476
2772
|
program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
|
|
2477
2773
|
program.command("start").action(async () => {
|
|
2478
2774
|
const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops-daemon", args: ["run"] });
|