@hasna/loops 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/dist/cli/index.js +510 -159
- package/dist/daemon/index.js +447 -146
- package/dist/index.js +446 -146
- package/dist/lib/executor.d.ts +2 -2
- package/dist/lib/format.d.ts +2 -1
- package/dist/lib/scheduler.d.ts +10 -1
- package/dist/lib/store.d.ts +21 -12
- package/dist/lib/store.js +304 -107
- package/dist/sdk/index.js +446 -146
- package/dist/types.d.ts +5 -0
- package/docs/USAGE.md +21 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -258,6 +258,11 @@ function validateTarget(value, label) {
|
|
|
258
258
|
const providers = ["claude", "cursor", "codewith", "aicopilot", "opencode", "codex"];
|
|
259
259
|
if (!providers.includes(value.provider))
|
|
260
260
|
throw new Error(`${label}.provider must be one of ${providers.join(", ")}`);
|
|
261
|
+
if (value.authProfile !== undefined) {
|
|
262
|
+
assertString(value.authProfile, `${label}.authProfile`);
|
|
263
|
+
if (value.provider !== "codewith")
|
|
264
|
+
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
265
|
+
}
|
|
261
266
|
return value;
|
|
262
267
|
}
|
|
263
268
|
throw new Error(`${label}.type must be command or agent`);
|
|
@@ -608,6 +613,13 @@ class Store {
|
|
|
608
613
|
} catch {}
|
|
609
614
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
610
615
|
}
|
|
616
|
+
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
617
|
+
if (!opts.daemonLeaseId)
|
|
618
|
+
return;
|
|
619
|
+
const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
|
|
620
|
+
if (!row)
|
|
621
|
+
throw new Error("daemon lease lost");
|
|
622
|
+
}
|
|
611
623
|
createLoop(input, from = new Date) {
|
|
612
624
|
const now = nowIso();
|
|
613
625
|
const loop = {
|
|
@@ -677,21 +689,31 @@ class Store {
|
|
|
677
689
|
ORDER BY next_run_at ASC`).all(now.toISOString());
|
|
678
690
|
return rows.map(rowToLoop);
|
|
679
691
|
}
|
|
680
|
-
updateLoop(id, patch) {
|
|
692
|
+
updateLoop(id, patch, opts = {}) {
|
|
681
693
|
const current = this.getLoop(id);
|
|
682
694
|
if (!current)
|
|
683
695
|
throw new Error(`loop not found: ${id}`);
|
|
684
|
-
const
|
|
696
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
697
|
+
const merged = { ...current, ...patch, updatedAt: updated };
|
|
685
698
|
this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
686
|
-
expires_at=$expiresAt, updated_at=$updated
|
|
699
|
+
expires_at=$expiresAt, updated_at=$updated
|
|
700
|
+
WHERE id=$id
|
|
701
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
702
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
703
|
+
))`).run({
|
|
687
704
|
$id: id,
|
|
688
705
|
$status: merged.status,
|
|
689
706
|
$nextRun: merged.nextRunAt ?? null,
|
|
690
707
|
$retrySlot: merged.retryScheduledFor ?? null,
|
|
691
708
|
$expiresAt: merged.expiresAt ?? null,
|
|
692
|
-
$updated: merged.updatedAt
|
|
709
|
+
$updated: merged.updatedAt,
|
|
710
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
711
|
+
$now: updated
|
|
693
712
|
});
|
|
694
|
-
|
|
713
|
+
const after = this.getLoop(id);
|
|
714
|
+
if (!after)
|
|
715
|
+
throw new Error(`loop not found after update: ${id}`);
|
|
716
|
+
return after;
|
|
695
717
|
}
|
|
696
718
|
deleteLoop(idOrName) {
|
|
697
719
|
const loop = this.requireLoop(idOrName);
|
|
@@ -755,11 +777,14 @@ class Store {
|
|
|
755
777
|
const now = nowIso();
|
|
756
778
|
if (input.idempotencyKey) {
|
|
757
779
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
758
|
-
if (existing)
|
|
780
|
+
if (existing) {
|
|
781
|
+
this.assertDaemonLeaseFence(input);
|
|
759
782
|
return rowToWorkflowRun(existing);
|
|
783
|
+
}
|
|
760
784
|
}
|
|
761
785
|
this.db.exec("BEGIN IMMEDIATE");
|
|
762
786
|
try {
|
|
787
|
+
this.assertDaemonLeaseFence(input, now);
|
|
763
788
|
if (input.idempotencyKey) {
|
|
764
789
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
765
790
|
if (existing) {
|
|
@@ -858,31 +883,60 @@ class Store {
|
|
|
858
883
|
const run = this.getWorkflowRun(workflowRunId);
|
|
859
884
|
return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
|
|
860
885
|
}
|
|
861
|
-
startWorkflowStepRun(workflowRunId, stepId) {
|
|
862
|
-
const now =
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
886
|
+
startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
|
|
887
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
888
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
889
|
+
try {
|
|
890
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
891
|
+
SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
|
|
892
|
+
pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
893
|
+
WHERE workflow_run_id=$workflowRunId
|
|
894
|
+
AND step_id=$stepId
|
|
895
|
+
AND status IN ('pending', 'failed', 'timed_out')
|
|
896
|
+
AND EXISTS (
|
|
897
|
+
SELECT 1 FROM workflow_runs
|
|
898
|
+
WHERE id=$workflowRunId AND status='running'
|
|
899
|
+
)
|
|
900
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
901
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
902
|
+
))`).run({
|
|
903
|
+
$workflowRunId: workflowRunId,
|
|
904
|
+
$stepId: stepId,
|
|
905
|
+
$started: now,
|
|
906
|
+
$updated: now,
|
|
907
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
908
|
+
$now: now
|
|
909
|
+
});
|
|
910
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
911
|
+
if (!run)
|
|
912
|
+
throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
|
|
913
|
+
if (res.changes !== 1) {
|
|
914
|
+
throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
|
|
915
|
+
}
|
|
916
|
+
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
917
|
+
this.db.exec("COMMIT");
|
|
918
|
+
return run;
|
|
919
|
+
} catch (error) {
|
|
920
|
+
try {
|
|
921
|
+
this.db.exec("ROLLBACK");
|
|
922
|
+
} catch {}
|
|
923
|
+
throw error;
|
|
878
924
|
}
|
|
879
|
-
this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
|
|
880
|
-
return run;
|
|
881
925
|
}
|
|
882
|
-
markWorkflowStepPid(workflowRunId, stepId, pid) {
|
|
883
|
-
const now =
|
|
926
|
+
markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
|
|
927
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
884
928
|
this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
|
|
885
|
-
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
929
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
930
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
931
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
932
|
+
))`).run({
|
|
933
|
+
$workflowRunId: workflowRunId,
|
|
934
|
+
$stepId: stepId,
|
|
935
|
+
$pid: pid,
|
|
936
|
+
$updated: now,
|
|
937
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
938
|
+
$now: now
|
|
939
|
+
});
|
|
886
940
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
887
941
|
if (!run)
|
|
888
942
|
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
@@ -910,60 +964,110 @@ class Store {
|
|
|
910
964
|
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
911
965
|
};
|
|
912
966
|
}
|
|
913
|
-
finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
|
|
967
|
+
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
914
968
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
969
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
970
|
+
try {
|
|
971
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
972
|
+
pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
973
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
974
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
975
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
976
|
+
))`).run({
|
|
977
|
+
$workflowRunId: workflowRunId,
|
|
978
|
+
$stepId: stepId,
|
|
979
|
+
$status: patch.status,
|
|
980
|
+
$finished: finishedAt,
|
|
981
|
+
$exitCode: patch.exitCode ?? null,
|
|
982
|
+
$durationMs: patch.durationMs ?? null,
|
|
983
|
+
$stdout: patch.stdout ?? null,
|
|
984
|
+
$stderr: patch.stderr ?? null,
|
|
985
|
+
$error: patch.error ?? null,
|
|
986
|
+
$updated: finishedAt,
|
|
987
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
988
|
+
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
933
989
|
});
|
|
990
|
+
if (res.changes === 1) {
|
|
991
|
+
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
992
|
+
exitCode: patch.exitCode,
|
|
993
|
+
error: patch.error
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
this.db.exec("COMMIT");
|
|
997
|
+
} catch (error) {
|
|
998
|
+
try {
|
|
999
|
+
this.db.exec("ROLLBACK");
|
|
1000
|
+
} catch {}
|
|
1001
|
+
throw error;
|
|
934
1002
|
}
|
|
935
1003
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
936
1004
|
if (!run)
|
|
937
1005
|
throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
|
|
938
1006
|
return run;
|
|
939
1007
|
}
|
|
940
|
-
skipWorkflowStepRun(workflowRunId, stepId, reason) {
|
|
941
|
-
const now =
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1008
|
+
skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
|
|
1009
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1010
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1011
|
+
try {
|
|
1012
|
+
const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
|
|
1013
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
|
|
1014
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1015
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1016
|
+
))`).run({
|
|
1017
|
+
$workflowRunId: workflowRunId,
|
|
1018
|
+
$stepId: stepId,
|
|
1019
|
+
$finished: now,
|
|
1020
|
+
$error: reason,
|
|
1021
|
+
$updated: now,
|
|
1022
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1023
|
+
$now: now
|
|
1024
|
+
});
|
|
1025
|
+
if (res.changes === 1)
|
|
1026
|
+
this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
|
|
1027
|
+
this.db.exec("COMMIT");
|
|
1028
|
+
} catch (error) {
|
|
1029
|
+
try {
|
|
1030
|
+
this.db.exec("ROLLBACK");
|
|
1031
|
+
} catch {}
|
|
1032
|
+
throw error;
|
|
1033
|
+
}
|
|
946
1034
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
947
1035
|
if (!run)
|
|
948
1036
|
throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
|
|
949
1037
|
return run;
|
|
950
1038
|
}
|
|
951
|
-
finalizeWorkflowRun(workflowRunId, status, patch = {}) {
|
|
1039
|
+
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
952
1040
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1041
|
+
let changed = false;
|
|
1042
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1043
|
+
try {
|
|
1044
|
+
const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
|
|
1045
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1046
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1047
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1048
|
+
))`).run({
|
|
1049
|
+
$id: workflowRunId,
|
|
1050
|
+
$status: status,
|
|
1051
|
+
$finished: finishedAt,
|
|
1052
|
+
$durationMs: patch.durationMs ?? null,
|
|
1053
|
+
$error: patch.error ?? null,
|
|
1054
|
+
$updated: finishedAt,
|
|
1055
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1056
|
+
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
1057
|
+
});
|
|
1058
|
+
changed = res.changes === 1;
|
|
1059
|
+
if (changed)
|
|
1060
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
1061
|
+
this.db.exec("COMMIT");
|
|
1062
|
+
} catch (error) {
|
|
1063
|
+
try {
|
|
1064
|
+
this.db.exec("ROLLBACK");
|
|
1065
|
+
} catch {}
|
|
1066
|
+
throw error;
|
|
1067
|
+
}
|
|
962
1068
|
const run = this.getWorkflowRun(workflowRunId);
|
|
963
1069
|
if (!run)
|
|
964
1070
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
965
|
-
if (res.changes === 1)
|
|
966
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
|
|
967
1071
|
return run;
|
|
968
1072
|
}
|
|
969
1073
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -1017,10 +1121,24 @@ class Store {
|
|
|
1017
1121
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1018
1122
|
return (row?.count ?? 0) > 0;
|
|
1019
1123
|
}
|
|
1020
|
-
markRunPid(id, pid, claimedBy) {
|
|
1021
|
-
const now =
|
|
1124
|
+
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1125
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1022
1126
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1023
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1127
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1128
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1129
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1130
|
+
))`).run({
|
|
1131
|
+
$id: id,
|
|
1132
|
+
$pid: pid,
|
|
1133
|
+
$updated: now,
|
|
1134
|
+
$claimedBy: claimedBy,
|
|
1135
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
1136
|
+
$now: now
|
|
1137
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
1138
|
+
WHERE id=$id AND status='running'
|
|
1139
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1140
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1141
|
+
))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
|
|
1024
1142
|
if (res.changes !== 1)
|
|
1025
1143
|
return;
|
|
1026
1144
|
return this.getRun(id);
|
|
@@ -1035,7 +1153,7 @@ class Store {
|
|
|
1035
1153
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
1036
1154
|
return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
|
|
1037
1155
|
}
|
|
1038
|
-
createSkippedRun(loop, scheduledFor, reason) {
|
|
1156
|
+
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
1039
1157
|
const now = nowIso();
|
|
1040
1158
|
const run = {
|
|
1041
1159
|
id: genId(),
|
|
@@ -1049,21 +1167,31 @@ class Store {
|
|
|
1049
1167
|
createdAt: now,
|
|
1050
1168
|
updatedAt: now
|
|
1051
1169
|
};
|
|
1052
|
-
this.db.
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1170
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1171
|
+
try {
|
|
1172
|
+
this.assertDaemonLeaseFence(opts, now);
|
|
1173
|
+
this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
|
|
1174
|
+
claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
|
|
1175
|
+
VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
|
|
1176
|
+
NULL, NULL, $error, $created, $updated)`).run({
|
|
1177
|
+
$id: run.id,
|
|
1178
|
+
$loopId: run.loopId,
|
|
1179
|
+
$loopName: run.loopName,
|
|
1180
|
+
$scheduledFor: run.scheduledFor,
|
|
1181
|
+
$attempt: run.attempt,
|
|
1182
|
+
$status: run.status,
|
|
1183
|
+
$finished: run.finishedAt ?? null,
|
|
1184
|
+
$error: run.error ?? null,
|
|
1185
|
+
$created: run.createdAt,
|
|
1186
|
+
$updated: run.updatedAt
|
|
1187
|
+
});
|
|
1188
|
+
this.db.exec("COMMIT");
|
|
1189
|
+
} catch (error) {
|
|
1190
|
+
try {
|
|
1191
|
+
this.db.exec("ROLLBACK");
|
|
1192
|
+
} catch {}
|
|
1193
|
+
throw error;
|
|
1194
|
+
}
|
|
1067
1195
|
return this.getRunBySlot(loop.id, scheduledFor) ?? run;
|
|
1068
1196
|
}
|
|
1069
1197
|
getRun(id) {
|
|
@@ -1074,11 +1202,20 @@ class Store {
|
|
|
1074
1202
|
const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
|
|
1075
1203
|
return row ? rowToRun(row) : undefined;
|
|
1076
1204
|
}
|
|
1077
|
-
|
|
1205
|
+
nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
|
|
1206
|
+
const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
|
|
1207
|
+
WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
|
|
1208
|
+
ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
|
|
1209
|
+
WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
|
|
1210
|
+
ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
|
|
1211
|
+
return row ? rowToRun(row) : undefined;
|
|
1212
|
+
}
|
|
1213
|
+
claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
|
|
1078
1214
|
const startedAt = now.toISOString();
|
|
1079
1215
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
1080
1216
|
this.db.exec("BEGIN IMMEDIATE");
|
|
1081
1217
|
try {
|
|
1218
|
+
this.assertDaemonLeaseFence(opts, startedAt);
|
|
1082
1219
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
1083
1220
|
if (existing) {
|
|
1084
1221
|
if (existing.status === "running") {
|
|
@@ -1173,11 +1310,15 @@ class Store {
|
|
|
1173
1310
|
$error: patch.error ?? null,
|
|
1174
1311
|
$updated: finishedAt,
|
|
1175
1312
|
$claimedBy: opts.claimedBy ?? null,
|
|
1176
|
-
$now: (opts.now ?? new Date).toISOString()
|
|
1313
|
+
$now: (opts.now ?? new Date).toISOString(),
|
|
1314
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1177
1315
|
};
|
|
1178
1316
|
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1179
1317
|
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
1180
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1318
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1319
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1320
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1321
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
1181
1322
|
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
1182
1323
|
const run = this.getRun(id);
|
|
1183
1324
|
if (!run)
|
|
@@ -1186,10 +1327,20 @@ class Store {
|
|
|
1186
1327
|
return run;
|
|
1187
1328
|
return run;
|
|
1188
1329
|
}
|
|
1189
|
-
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
|
|
1330
|
+
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
1190
1331
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
1191
1332
|
const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
|
|
1192
|
-
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
1333
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
1334
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1335
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1336
|
+
))`).run({
|
|
1337
|
+
$id: id,
|
|
1338
|
+
$claimedBy: claimedBy,
|
|
1339
|
+
$expires: expiresAt,
|
|
1340
|
+
$updated: now.toISOString(),
|
|
1341
|
+
$now: now.toISOString(),
|
|
1342
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1343
|
+
});
|
|
1193
1344
|
if (res.changes !== 1)
|
|
1194
1345
|
return;
|
|
1195
1346
|
return this.getRun(id);
|
|
@@ -1208,7 +1359,7 @@ class Store {
|
|
|
1208
1359
|
}
|
|
1209
1360
|
return rows.map(rowToRun);
|
|
1210
1361
|
}
|
|
1211
|
-
recoverExpiredRunLeases(now = new Date) {
|
|
1362
|
+
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
1212
1363
|
const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
|
|
1213
1364
|
const recovered = [];
|
|
1214
1365
|
for (const row of rows) {
|
|
@@ -1217,20 +1368,63 @@ class Store {
|
|
|
1217
1368
|
if (this.hasLiveWorkflowStepProcesses(row.id))
|
|
1218
1369
|
continue;
|
|
1219
1370
|
const finished = now.toISOString();
|
|
1220
|
-
this.db.
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1371
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1372
|
+
try {
|
|
1373
|
+
const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
|
|
1374
|
+
error='run lease expired before completion', updated_at=$updated
|
|
1375
|
+
WHERE id=$id AND status='running' AND lease_expires_at <= $now
|
|
1376
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1377
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1378
|
+
))`).run({
|
|
1379
|
+
$id: row.id,
|
|
1380
|
+
$finished: finished,
|
|
1381
|
+
$updated: finished,
|
|
1382
|
+
$now: finished,
|
|
1383
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1233
1384
|
});
|
|
1385
|
+
if (res.changes !== 1) {
|
|
1386
|
+
this.db.exec("COMMIT");
|
|
1387
|
+
continue;
|
|
1388
|
+
}
|
|
1389
|
+
const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
|
|
1390
|
+
for (const workflowRow of workflowRows) {
|
|
1391
|
+
const workflowRes = this.db.query(`UPDATE workflow_runs
|
|
1392
|
+
SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1393
|
+
WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
1394
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1395
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1396
|
+
))`).run({
|
|
1397
|
+
$id: workflowRow.id,
|
|
1398
|
+
$finished: finished,
|
|
1399
|
+
$updated: finished,
|
|
1400
|
+
$now: finished,
|
|
1401
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1402
|
+
});
|
|
1403
|
+
if (workflowRes.changes !== 1)
|
|
1404
|
+
continue;
|
|
1405
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
1406
|
+
SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
|
|
1407
|
+
WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
|
|
1408
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
1409
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
1410
|
+
))`).run({
|
|
1411
|
+
$workflowRunId: workflowRow.id,
|
|
1412
|
+
$finished: finished,
|
|
1413
|
+
$updated: finished,
|
|
1414
|
+
$now: finished,
|
|
1415
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
1416
|
+
});
|
|
1417
|
+
this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
|
|
1418
|
+
error: "parent loop run lease expired before completion",
|
|
1419
|
+
loopRunId: row.id
|
|
1420
|
+
});
|
|
1421
|
+
}
|
|
1422
|
+
this.db.exec("COMMIT");
|
|
1423
|
+
} catch (error) {
|
|
1424
|
+
try {
|
|
1425
|
+
this.db.exec("ROLLBACK");
|
|
1426
|
+
} catch {}
|
|
1427
|
+
throw error;
|
|
1234
1428
|
}
|
|
1235
1429
|
const run = this.getRun(row.id);
|
|
1236
1430
|
if (run)
|
|
@@ -1238,11 +1432,14 @@ class Store {
|
|
|
1238
1432
|
}
|
|
1239
1433
|
return recovered;
|
|
1240
1434
|
}
|
|
1241
|
-
expireLoops(now = new Date) {
|
|
1435
|
+
expireLoops(now = new Date, opts = {}) {
|
|
1242
1436
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
1243
1437
|
const expired = [];
|
|
1244
|
-
for (const row of rows)
|
|
1245
|
-
|
|
1438
|
+
for (const row of rows) {
|
|
1439
|
+
const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
|
|
1440
|
+
if (updated.status === "expired")
|
|
1441
|
+
expired.push(updated);
|
|
1442
|
+
}
|
|
1246
1443
|
return expired;
|
|
1247
1444
|
}
|
|
1248
1445
|
countLoops(status) {
|
|
@@ -1285,7 +1482,7 @@ class Store {
|
|
|
1285
1482
|
}
|
|
1286
1483
|
heartbeatDaemonLease(id, ttlMs, now = new Date) {
|
|
1287
1484
|
const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
|
|
1288
|
-
const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
|
|
1485
|
+
const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
|
|
1289
1486
|
if (res.changes !== 1)
|
|
1290
1487
|
return;
|
|
1291
1488
|
return this.getDaemonLease();
|
|
@@ -1535,7 +1732,7 @@ function agentArgs(target) {
|
|
|
1535
1732
|
args.push("--model", target.model);
|
|
1536
1733
|
if (target.agent)
|
|
1537
1734
|
args.push("--agent", target.agent);
|
|
1538
|
-
args.push(...target.extraArgs ?? []
|
|
1735
|
+
args.push(...target.extraArgs ?? []);
|
|
1539
1736
|
return args;
|
|
1540
1737
|
case "cursor":
|
|
1541
1738
|
args.push("-p");
|
|
@@ -1543,10 +1740,10 @@ function agentArgs(target) {
|
|
|
1543
1740
|
args.push("--model", target.model);
|
|
1544
1741
|
if (target.agent)
|
|
1545
1742
|
args.push("--agent", target.agent);
|
|
1546
|
-
args.push(...target.extraArgs ?? []
|
|
1743
|
+
args.push(...target.extraArgs ?? []);
|
|
1547
1744
|
return args;
|
|
1548
1745
|
case "codewith":
|
|
1549
|
-
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
|
|
1746
|
+
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
|
|
1550
1747
|
if (isolation === "safe")
|
|
1551
1748
|
args.push("--ignore-rules");
|
|
1552
1749
|
if (target.cwd)
|
|
@@ -1555,7 +1752,7 @@ function agentArgs(target) {
|
|
|
1555
1752
|
args.push("--model", target.model);
|
|
1556
1753
|
if (target.agent)
|
|
1557
1754
|
args.push("--agent", target.agent);
|
|
1558
|
-
args.push(...target.extraArgs ?? []
|
|
1755
|
+
args.push(...target.extraArgs ?? []);
|
|
1559
1756
|
return args;
|
|
1560
1757
|
case "codex":
|
|
1561
1758
|
args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
|
|
@@ -1565,7 +1762,7 @@ function agentArgs(target) {
|
|
|
1565
1762
|
args.push("--cd", target.cwd);
|
|
1566
1763
|
if (target.model)
|
|
1567
1764
|
args.push("--model", target.model);
|
|
1568
|
-
args.push(...target.extraArgs ?? []
|
|
1765
|
+
args.push(...target.extraArgs ?? []);
|
|
1569
1766
|
return args;
|
|
1570
1767
|
case "aicopilot":
|
|
1571
1768
|
args.push("run", "--format", "json");
|
|
@@ -1577,7 +1774,7 @@ function agentArgs(target) {
|
|
|
1577
1774
|
args.push("--model", target.model);
|
|
1578
1775
|
if (target.agent)
|
|
1579
1776
|
args.push("--agent", target.agent);
|
|
1580
|
-
args.push(...target.extraArgs ?? []
|
|
1777
|
+
args.push(...target.extraArgs ?? []);
|
|
1581
1778
|
return args;
|
|
1582
1779
|
case "opencode":
|
|
1583
1780
|
args.push("run", "--format", "json");
|
|
@@ -1589,7 +1786,7 @@ function agentArgs(target) {
|
|
|
1589
1786
|
args.push("--model", target.model);
|
|
1590
1787
|
if (target.agent)
|
|
1591
1788
|
args.push("--agent", target.agent);
|
|
1592
|
-
args.push(...target.extraArgs ?? []
|
|
1789
|
+
args.push(...target.extraArgs ?? []);
|
|
1593
1790
|
return args;
|
|
1594
1791
|
}
|
|
1595
1792
|
}
|
|
@@ -1614,7 +1811,8 @@ function commandSpec(target) {
|
|
|
1614
1811
|
cwd: agentTarget.cwd,
|
|
1615
1812
|
timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
1616
1813
|
account: agentTarget.account,
|
|
1617
|
-
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
|
|
1814
|
+
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
|
|
1815
|
+
stdin: agentTarget.prompt
|
|
1618
1816
|
};
|
|
1619
1817
|
}
|
|
1620
1818
|
function executionEnv(spec, metadata, opts) {
|
|
@@ -1683,10 +1881,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1683
1881
|
env,
|
|
1684
1882
|
shell: spec.shell ?? false,
|
|
1685
1883
|
detached: true,
|
|
1686
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
1884
|
+
stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
|
|
1687
1885
|
});
|
|
1688
1886
|
if (child.pid)
|
|
1689
1887
|
opts.onSpawn?.(child.pid);
|
|
1888
|
+
if (spec.stdin !== undefined && child.stdin) {
|
|
1889
|
+
child.stdin.on("error", (err) => {
|
|
1890
|
+
if (err.code !== "EPIPE")
|
|
1891
|
+
error = err.message;
|
|
1892
|
+
});
|
|
1893
|
+
child.stdin.end(spec.stdin);
|
|
1894
|
+
}
|
|
1690
1895
|
const abortHandler = () => {
|
|
1691
1896
|
error = "cancelled";
|
|
1692
1897
|
if (child.pid)
|
|
@@ -1695,10 +1900,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
1695
1900
|
if (opts.signal?.aborted)
|
|
1696
1901
|
abortHandler();
|
|
1697
1902
|
opts.signal?.addEventListener("abort", abortHandler, { once: true });
|
|
1698
|
-
child.stdout
|
|
1903
|
+
child.stdout?.on("data", (chunk) => {
|
|
1699
1904
|
stdout = appendBounded(stdout, chunk, maxOutputBytes);
|
|
1700
1905
|
});
|
|
1701
|
-
child.stderr
|
|
1906
|
+
child.stderr?.on("data", (chunk) => {
|
|
1702
1907
|
stderr = appendBounded(stderr, chunk, maxOutputBytes);
|
|
1703
1908
|
});
|
|
1704
1909
|
const timer = setTimeout(() => {
|
|
@@ -1797,7 +2002,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1797
2002
|
loop: opts.loop,
|
|
1798
2003
|
loopRun: opts.loopRun,
|
|
1799
2004
|
scheduledFor: opts.scheduledFor,
|
|
1800
|
-
idempotencyKey: opts.idempotencyKey
|
|
2005
|
+
idempotencyKey: opts.idempotencyKey,
|
|
2006
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1801
2007
|
});
|
|
1802
2008
|
const startedAt = run.startedAt ?? nowIso();
|
|
1803
2009
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
@@ -1831,12 +2037,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1831
2037
|
return !dependencyStep?.continueOnFailure;
|
|
1832
2038
|
});
|
|
1833
2039
|
if (blockedBy) {
|
|
1834
|
-
|
|
2040
|
+
opts.beforePersist?.();
|
|
2041
|
+
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
1835
2042
|
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
1836
2043
|
terminalStatus = "failed";
|
|
1837
2044
|
continue;
|
|
1838
2045
|
}
|
|
1839
|
-
|
|
2046
|
+
opts.beforePersist?.();
|
|
2047
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
|
|
1840
2048
|
if (startedStep.status !== "running") {
|
|
1841
2049
|
terminalStatus = "failed";
|
|
1842
2050
|
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
@@ -1868,7 +2076,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1868
2076
|
...opts,
|
|
1869
2077
|
signal: controller.signal,
|
|
1870
2078
|
onSpawn: (pid) => {
|
|
1871
|
-
|
|
2079
|
+
opts.beforePersist?.();
|
|
2080
|
+
store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
|
|
1872
2081
|
opts.onSpawn?.(pid);
|
|
1873
2082
|
}
|
|
1874
2083
|
});
|
|
@@ -1896,6 +2105,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1896
2105
|
blockingError = "workflow run was cancelled";
|
|
1897
2106
|
break;
|
|
1898
2107
|
}
|
|
2108
|
+
opts.beforePersist?.();
|
|
1899
2109
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
1900
2110
|
status: result.status,
|
|
1901
2111
|
finishedAt: result.finishedAt,
|
|
@@ -1904,6 +2114,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1904
2114
|
stderr: result.stderr,
|
|
1905
2115
|
exitCode: result.exitCode,
|
|
1906
2116
|
error: result.error
|
|
2117
|
+
}, {
|
|
2118
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1907
2119
|
});
|
|
1908
2120
|
if (result.status !== "succeeded" && !step.continueOnFailure) {
|
|
1909
2121
|
terminalStatus = result.status;
|
|
@@ -1915,7 +2127,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1915
2127
|
for (const step of ordered) {
|
|
1916
2128
|
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
1917
2129
|
if (existing?.status === "pending" || existing?.status === "running") {
|
|
1918
|
-
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run"
|
|
2130
|
+
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
|
|
2131
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
2132
|
+
});
|
|
1919
2133
|
}
|
|
1920
2134
|
}
|
|
1921
2135
|
}
|
|
@@ -1925,10 +2139,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
1925
2139
|
const steps2 = store.listWorkflowStepRuns(run.id);
|
|
1926
2140
|
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
1927
2141
|
}
|
|
2142
|
+
opts.beforePersist?.();
|
|
1928
2143
|
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
1929
2144
|
finishedAt,
|
|
1930
2145
|
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
1931
2146
|
error: blockingError
|
|
2147
|
+
}, {
|
|
2148
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
1932
2149
|
});
|
|
1933
2150
|
const steps = store.listWorkflowStepRuns(run.id);
|
|
1934
2151
|
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
@@ -1976,52 +2193,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
1976
2193
|
|
|
1977
2194
|
// src/lib/scheduler.ts
|
|
1978
2195
|
function manualRunScheduledFor(loop, now = new Date) {
|
|
1979
|
-
if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
2196
|
+
if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
|
|
1980
2197
|
return loop.retryScheduledFor ?? loop.nextRunAt;
|
|
1981
2198
|
}
|
|
1982
2199
|
return now.toISOString();
|
|
1983
2200
|
}
|
|
1984
2201
|
function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
|
|
2202
|
+
if (loop.status !== "active")
|
|
2203
|
+
return false;
|
|
1985
2204
|
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
1986
2205
|
return false;
|
|
1987
2206
|
return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
|
|
1988
2207
|
}
|
|
2208
|
+
function manualRunSource(loop, scheduledFor, now = new Date) {
|
|
2209
|
+
if (loop.status !== "active")
|
|
2210
|
+
return "ad_hoc";
|
|
2211
|
+
if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
|
|
2212
|
+
return "ad_hoc";
|
|
2213
|
+
if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
|
|
2214
|
+
return "retry_slot";
|
|
2215
|
+
return "due_slot";
|
|
2216
|
+
}
|
|
1989
2217
|
function nextAfterRetry(loop, now) {
|
|
1990
2218
|
return new Date(now.getTime() + loop.retryDelayMs).toISOString();
|
|
1991
2219
|
}
|
|
1992
|
-
function
|
|
2220
|
+
function isDaemonLeaseLost(error) {
|
|
2221
|
+
return error instanceof Error && error.message === "daemon lease lost";
|
|
2222
|
+
}
|
|
2223
|
+
function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
1993
2224
|
if (run.status === "running")
|
|
1994
2225
|
return;
|
|
1995
2226
|
const current = store.getLoop(loop.id);
|
|
1996
2227
|
if (!current || current.status !== "active")
|
|
1997
2228
|
return;
|
|
1998
|
-
|
|
2229
|
+
if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
|
|
2230
|
+
return;
|
|
2231
|
+
const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
|
|
1999
2232
|
if (shouldRetry) {
|
|
2000
|
-
store.updateLoop(
|
|
2233
|
+
store.updateLoop(current.id, {
|
|
2001
2234
|
status: "active",
|
|
2002
|
-
nextRunAt: nextAfterRetry(
|
|
2235
|
+
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
2003
2236
|
retryScheduledFor: run.scheduledFor
|
|
2004
|
-
});
|
|
2237
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2238
|
+
return;
|
|
2239
|
+
}
|
|
2240
|
+
const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
|
|
2241
|
+
if (deferredRetry) {
|
|
2242
|
+
store.updateLoop(current.id, {
|
|
2243
|
+
status: "active",
|
|
2244
|
+
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
2245
|
+
retryScheduledFor: deferredRetry.scheduledFor
|
|
2246
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2005
2247
|
return;
|
|
2006
2248
|
}
|
|
2007
|
-
const nextRunAt = computeNextAfter(
|
|
2008
|
-
store.updateLoop(
|
|
2249
|
+
const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
|
|
2250
|
+
store.updateLoop(current.id, {
|
|
2009
2251
|
status: nextRunAt ? "active" : "stopped",
|
|
2010
2252
|
nextRunAt,
|
|
2011
2253
|
retryScheduledFor: undefined
|
|
2012
|
-
});
|
|
2254
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
2013
2255
|
}
|
|
2014
2256
|
async function executeClaimedRun(deps) {
|
|
2015
2257
|
let heartbeat;
|
|
2016
2258
|
const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
|
|
2017
2259
|
heartbeat = setInterval(() => {
|
|
2018
|
-
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs
|
|
2260
|
+
deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
|
|
2261
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2262
|
+
});
|
|
2019
2263
|
}, heartbeatEveryMs);
|
|
2020
2264
|
heartbeat.unref();
|
|
2021
2265
|
try {
|
|
2022
2266
|
const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
|
|
2023
|
-
|
|
2267
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2268
|
+
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
2024
2269
|
})))(deps.loop, deps.run);
|
|
2270
|
+
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
2025
2271
|
return deps.store.finalizeRun(deps.run.id, {
|
|
2026
2272
|
status: result.status,
|
|
2027
2273
|
finishedAt: result.finishedAt,
|
|
@@ -2033,10 +2279,16 @@ async function executeClaimedRun(deps) {
|
|
|
2033
2279
|
pid: result.pid
|
|
2034
2280
|
}, {
|
|
2035
2281
|
claimedBy: deps.runnerId,
|
|
2282
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2036
2283
|
now: deps.now?.() ?? new Date(result.finishedAt)
|
|
2037
2284
|
});
|
|
2038
2285
|
} catch (err) {
|
|
2039
2286
|
deps.onError?.(deps.loop, err);
|
|
2287
|
+
try {
|
|
2288
|
+
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
2289
|
+
} catch {
|
|
2290
|
+
return deps.store.getRun(deps.run.id) ?? deps.run;
|
|
2291
|
+
}
|
|
2040
2292
|
const finishedAt = new Date;
|
|
2041
2293
|
return deps.store.finalizeRun(deps.run.id, {
|
|
2042
2294
|
status: "failed",
|
|
@@ -2047,6 +2299,7 @@ async function executeClaimedRun(deps) {
|
|
|
2047
2299
|
error: err instanceof Error ? err.message : String(err)
|
|
2048
2300
|
}, {
|
|
2049
2301
|
claimedBy: deps.runnerId,
|
|
2302
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2050
2303
|
now: deps.now?.() ?? finishedAt
|
|
2051
2304
|
});
|
|
2052
2305
|
} finally {
|
|
@@ -2056,15 +2309,33 @@ async function executeClaimedRun(deps) {
|
|
|
2056
2309
|
}
|
|
2057
2310
|
async function runSlot(deps, loop, scheduledFor) {
|
|
2058
2311
|
const now = deps.now?.() ?? new Date;
|
|
2312
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
2059
2313
|
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
2060
|
-
|
|
2061
|
-
|
|
2314
|
+
let skipped;
|
|
2315
|
+
try {
|
|
2316
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
2317
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2318
|
+
});
|
|
2319
|
+
} catch (error) {
|
|
2320
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
2321
|
+
return;
|
|
2322
|
+
throw error;
|
|
2323
|
+
}
|
|
2324
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
2062
2325
|
deps.onRun?.(skipped);
|
|
2063
2326
|
return skipped;
|
|
2064
2327
|
}
|
|
2065
|
-
|
|
2328
|
+
let claim;
|
|
2329
|
+
try {
|
|
2330
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2331
|
+
} catch (error) {
|
|
2332
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
2333
|
+
return;
|
|
2334
|
+
throw error;
|
|
2335
|
+
}
|
|
2066
2336
|
if (!claim)
|
|
2067
2337
|
return;
|
|
2338
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
2068
2339
|
deps.onRun?.(claim.run);
|
|
2069
2340
|
const finalRun = await executeClaimedRun({
|
|
2070
2341
|
store: deps.store,
|
|
@@ -2073,21 +2344,42 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
2073
2344
|
run: claim.run,
|
|
2074
2345
|
now: deps.now,
|
|
2075
2346
|
execute: deps.execute,
|
|
2347
|
+
beforeFinalize: deps.beforeFinalize,
|
|
2348
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
2076
2349
|
onError: deps.onError
|
|
2077
2350
|
});
|
|
2078
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
|
|
2351
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
|
|
2079
2352
|
deps.onRun?.(finalRun);
|
|
2080
2353
|
return finalRun;
|
|
2081
2354
|
}
|
|
2082
2355
|
async function tick(deps) {
|
|
2083
2356
|
const now = deps.now?.() ?? new Date;
|
|
2084
|
-
const recovered = deps.store.recoverExpiredRunLeases(now);
|
|
2357
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2358
|
+
const recoveredByLoop = new Map;
|
|
2085
2359
|
for (const run of recovered) {
|
|
2086
|
-
|
|
2087
|
-
if (loop)
|
|
2088
|
-
advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
|
|
2360
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
2089
2361
|
}
|
|
2090
|
-
const
|
|
2362
|
+
for (const runs of recoveredByLoop.values()) {
|
|
2363
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
2364
|
+
if (!loop)
|
|
2365
|
+
continue;
|
|
2366
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
2367
|
+
if (retryable) {
|
|
2368
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
2369
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2370
|
+
});
|
|
2371
|
+
continue;
|
|
2372
|
+
}
|
|
2373
|
+
for (const run of runs) {
|
|
2374
|
+
const current = deps.store.getLoop(run.loopId);
|
|
2375
|
+
if (current) {
|
|
2376
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
2377
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
2378
|
+
});
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2382
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
2091
2383
|
const claimed = [];
|
|
2092
2384
|
const completed = [];
|
|
2093
2385
|
const skipped = [];
|
|
@@ -2153,9 +2445,17 @@ class LoopsClient {
|
|
|
2153
2445
|
async runNow(idOrName) {
|
|
2154
2446
|
const loop = this.get(idOrName);
|
|
2155
2447
|
const now = new Date;
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2448
|
+
let scheduledFor = manualRunScheduledFor(loop, now);
|
|
2449
|
+
let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
|
|
2450
|
+
let claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
|
|
2451
|
+
if (!claim && shouldAdvance) {
|
|
2452
|
+
const existing = this.store.getRunBySlot(loop.id, scheduledFor);
|
|
2453
|
+
if (existing && existing.status !== "running") {
|
|
2454
|
+
scheduledFor = now.toISOString();
|
|
2455
|
+
shouldAdvance = false;
|
|
2456
|
+
claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2159
2459
|
if (!claim)
|
|
2160
2460
|
throw new Error(`could not claim manual run for ${idOrName}`);
|
|
2161
2461
|
const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });
|