@hasna/loops 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -615,6 +615,13 @@ class Store {
615
615
  } catch {}
616
616
  this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
617
617
  }
618
+ assertDaemonLeaseFence(opts = {}, now = nowIso()) {
619
+ if (!opts.daemonLeaseId)
620
+ return;
621
+ const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
622
+ if (!row)
623
+ throw new Error("daemon lease lost");
624
+ }
618
625
  createLoop(input, from = new Date) {
619
626
  const now = nowIso();
620
627
  const loop = {
@@ -684,21 +691,31 @@ class Store {
684
691
  ORDER BY next_run_at ASC`).all(now.toISOString());
685
692
  return rows.map(rowToLoop);
686
693
  }
687
- updateLoop(id, patch) {
694
+ updateLoop(id, patch, opts = {}) {
688
695
  const current = this.getLoop(id);
689
696
  if (!current)
690
697
  throw new Error(`loop not found: ${id}`);
691
- const merged = { ...current, ...patch, updatedAt: nowIso() };
698
+ const updated = (opts.now ?? new Date).toISOString();
699
+ const merged = { ...current, ...patch, updatedAt: updated };
692
700
  this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
693
- expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
701
+ expires_at=$expiresAt, updated_at=$updated
702
+ WHERE id=$id
703
+ AND ($daemonLeaseId IS NULL OR EXISTS (
704
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
705
+ ))`).run({
694
706
  $id: id,
695
707
  $status: merged.status,
696
708
  $nextRun: merged.nextRunAt ?? null,
697
709
  $retrySlot: merged.retryScheduledFor ?? null,
698
710
  $expiresAt: merged.expiresAt ?? null,
699
- $updated: merged.updatedAt
711
+ $updated: merged.updatedAt,
712
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
713
+ $now: updated
700
714
  });
701
- return merged;
715
+ const after = this.getLoop(id);
716
+ if (!after)
717
+ throw new Error(`loop not found after update: ${id}`);
718
+ return after;
702
719
  }
703
720
  deleteLoop(idOrName) {
704
721
  const loop = this.requireLoop(idOrName);
@@ -762,11 +779,14 @@ class Store {
762
779
  const now = nowIso();
763
780
  if (input.idempotencyKey) {
764
781
  const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
765
- if (existing)
782
+ if (existing) {
783
+ this.assertDaemonLeaseFence(input);
766
784
  return rowToWorkflowRun(existing);
785
+ }
767
786
  }
768
787
  this.db.exec("BEGIN IMMEDIATE");
769
788
  try {
789
+ this.assertDaemonLeaseFence(input, now);
770
790
  if (input.idempotencyKey) {
771
791
  const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
772
792
  if (existing) {
@@ -865,31 +885,60 @@ class Store {
865
885
  const run = this.getWorkflowRun(workflowRunId);
866
886
  return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
867
887
  }
868
- startWorkflowStepRun(workflowRunId, stepId) {
869
- const now = nowIso();
870
- const res = this.db.query(`UPDATE workflow_step_runs
871
- SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
872
- pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
873
- WHERE workflow_run_id=$workflowRunId
874
- AND step_id=$stepId
875
- AND status IN ('pending', 'failed', 'timed_out')
876
- AND EXISTS (
877
- SELECT 1 FROM workflow_runs
878
- WHERE id=$workflowRunId AND status='running'
879
- )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
880
- const run = this.getWorkflowStepRun(workflowRunId, stepId);
881
- if (!run)
882
- throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
883
- if (res.changes !== 1) {
884
- throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
888
+ startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
889
+ const now = (opts.now ?? new Date).toISOString();
890
+ this.db.exec("BEGIN IMMEDIATE");
891
+ try {
892
+ const res = this.db.query(`UPDATE workflow_step_runs
893
+ SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
894
+ pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
895
+ WHERE workflow_run_id=$workflowRunId
896
+ AND step_id=$stepId
897
+ AND status IN ('pending', 'failed', 'timed_out')
898
+ AND EXISTS (
899
+ SELECT 1 FROM workflow_runs
900
+ WHERE id=$workflowRunId AND status='running'
901
+ )
902
+ AND ($daemonLeaseId IS NULL OR EXISTS (
903
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
904
+ ))`).run({
905
+ $workflowRunId: workflowRunId,
906
+ $stepId: stepId,
907
+ $started: now,
908
+ $updated: now,
909
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
910
+ $now: now
911
+ });
912
+ const run = this.getWorkflowStepRun(workflowRunId, stepId);
913
+ if (!run)
914
+ throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
915
+ if (res.changes !== 1) {
916
+ throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
917
+ }
918
+ this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
919
+ this.db.exec("COMMIT");
920
+ return run;
921
+ } catch (error) {
922
+ try {
923
+ this.db.exec("ROLLBACK");
924
+ } catch {}
925
+ throw error;
885
926
  }
886
- this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
887
- return run;
888
927
  }
889
- markWorkflowStepPid(workflowRunId, stepId, pid) {
890
- const now = nowIso();
928
+ markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
929
+ const now = (opts.now ?? new Date).toISOString();
891
930
  this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
892
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
931
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
932
+ AND ($daemonLeaseId IS NULL OR EXISTS (
933
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
934
+ ))`).run({
935
+ $workflowRunId: workflowRunId,
936
+ $stepId: stepId,
937
+ $pid: pid,
938
+ $updated: now,
939
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
940
+ $now: now
941
+ });
893
942
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
894
943
  if (!run)
895
944
  throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -917,60 +966,110 @@ class Store {
917
966
  recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
918
967
  };
919
968
  }
920
- finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
969
+ finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
921
970
  const finishedAt = patch.finishedAt ?? nowIso();
922
- const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
923
- pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
924
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
925
- $workflowRunId: workflowRunId,
926
- $stepId: stepId,
927
- $status: patch.status,
928
- $finished: finishedAt,
929
- $exitCode: patch.exitCode ?? null,
930
- $durationMs: patch.durationMs ?? null,
931
- $stdout: patch.stdout ?? null,
932
- $stderr: patch.stderr ?? null,
933
- $error: patch.error ?? null,
934
- $updated: finishedAt
935
- });
936
- if (res.changes === 1) {
937
- this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
938
- exitCode: patch.exitCode,
939
- error: patch.error
971
+ this.db.exec("BEGIN IMMEDIATE");
972
+ try {
973
+ const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
974
+ pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
975
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
976
+ AND ($daemonLeaseId IS NULL OR EXISTS (
977
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
978
+ ))`).run({
979
+ $workflowRunId: workflowRunId,
980
+ $stepId: stepId,
981
+ $status: patch.status,
982
+ $finished: finishedAt,
983
+ $exitCode: patch.exitCode ?? null,
984
+ $durationMs: patch.durationMs ?? null,
985
+ $stdout: patch.stdout ?? null,
986
+ $stderr: patch.stderr ?? null,
987
+ $error: patch.error ?? null,
988
+ $updated: finishedAt,
989
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
990
+ $now: (opts.now ?? new Date(finishedAt)).toISOString()
940
991
  });
992
+ if (res.changes === 1) {
993
+ this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
994
+ exitCode: patch.exitCode,
995
+ error: patch.error
996
+ });
997
+ }
998
+ this.db.exec("COMMIT");
999
+ } catch (error) {
1000
+ try {
1001
+ this.db.exec("ROLLBACK");
1002
+ } catch {}
1003
+ throw error;
941
1004
  }
942
1005
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
943
1006
  if (!run)
944
1007
  throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
945
1008
  return run;
946
1009
  }
947
- skipWorkflowStepRun(workflowRunId, stepId, reason) {
948
- const now = nowIso();
949
- const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
950
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
951
- if (res.changes === 1)
952
- this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
1010
+ skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
1011
+ const now = (opts.now ?? new Date).toISOString();
1012
+ this.db.exec("BEGIN IMMEDIATE");
1013
+ try {
1014
+ const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
1015
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
1016
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1017
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1018
+ ))`).run({
1019
+ $workflowRunId: workflowRunId,
1020
+ $stepId: stepId,
1021
+ $finished: now,
1022
+ $error: reason,
1023
+ $updated: now,
1024
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1025
+ $now: now
1026
+ });
1027
+ if (res.changes === 1)
1028
+ this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
1029
+ this.db.exec("COMMIT");
1030
+ } catch (error) {
1031
+ try {
1032
+ this.db.exec("ROLLBACK");
1033
+ } catch {}
1034
+ throw error;
1035
+ }
953
1036
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
954
1037
  if (!run)
955
1038
  throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
956
1039
  return run;
957
1040
  }
958
- finalizeWorkflowRun(workflowRunId, status, patch = {}) {
1041
+ finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
959
1042
  const finishedAt = patch.finishedAt ?? nowIso();
960
- const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
961
- WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
962
- $id: workflowRunId,
963
- $status: status,
964
- $finished: finishedAt,
965
- $durationMs: patch.durationMs ?? null,
966
- $error: patch.error ?? null,
967
- $updated: finishedAt
968
- });
1043
+ let changed = false;
1044
+ this.db.exec("BEGIN IMMEDIATE");
1045
+ try {
1046
+ const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
1047
+ WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
1048
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1049
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1050
+ ))`).run({
1051
+ $id: workflowRunId,
1052
+ $status: status,
1053
+ $finished: finishedAt,
1054
+ $durationMs: patch.durationMs ?? null,
1055
+ $error: patch.error ?? null,
1056
+ $updated: finishedAt,
1057
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1058
+ $now: (opts.now ?? new Date(finishedAt)).toISOString()
1059
+ });
1060
+ changed = res.changes === 1;
1061
+ if (changed)
1062
+ this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
1063
+ this.db.exec("COMMIT");
1064
+ } catch (error) {
1065
+ try {
1066
+ this.db.exec("ROLLBACK");
1067
+ } catch {}
1068
+ throw error;
1069
+ }
969
1070
  const run = this.getWorkflowRun(workflowRunId);
970
1071
  if (!run)
971
1072
  throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
972
- if (res.changes === 1)
973
- this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
974
1073
  return run;
975
1074
  }
976
1075
  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1024,10 +1123,24 @@ class Store {
1024
1123
  const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
1025
1124
  return (row?.count ?? 0) > 0;
1026
1125
  }
1027
- markRunPid(id, pid, claimedBy) {
1028
- const now = nowIso();
1126
+ markRunPid(id, pid, claimedBy, opts = {}) {
1127
+ const now = (opts.now ?? new Date).toISOString();
1029
1128
  const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
1030
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
1129
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy
1130
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1131
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1132
+ ))`).run({
1133
+ $id: id,
1134
+ $pid: pid,
1135
+ $updated: now,
1136
+ $claimedBy: claimedBy,
1137
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1138
+ $now: now
1139
+ }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
1140
+ WHERE id=$id AND status='running'
1141
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1142
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1143
+ ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
1031
1144
  if (res.changes !== 1)
1032
1145
  return;
1033
1146
  return this.getRun(id);
@@ -1042,7 +1155,7 @@ class Store {
1042
1155
  AND wsr.pid IS NOT NULL`).all(loopRunId);
1043
1156
  return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
1044
1157
  }
1045
- createSkippedRun(loop, scheduledFor, reason) {
1158
+ createSkippedRun(loop, scheduledFor, reason, opts = {}) {
1046
1159
  const now = nowIso();
1047
1160
  const run = {
1048
1161
  id: genId(),
@@ -1056,21 +1169,31 @@ class Store {
1056
1169
  createdAt: now,
1057
1170
  updatedAt: now
1058
1171
  };
1059
- this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
1060
- claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
1061
- VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
1062
- NULL, NULL, $error, $created, $updated)`).run({
1063
- $id: run.id,
1064
- $loopId: run.loopId,
1065
- $loopName: run.loopName,
1066
- $scheduledFor: run.scheduledFor,
1067
- $attempt: run.attempt,
1068
- $status: run.status,
1069
- $finished: run.finishedAt ?? null,
1070
- $error: run.error ?? null,
1071
- $created: run.createdAt,
1072
- $updated: run.updatedAt
1073
- });
1172
+ this.db.exec("BEGIN IMMEDIATE");
1173
+ try {
1174
+ this.assertDaemonLeaseFence(opts, now);
1175
+ this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
1176
+ claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
1177
+ VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
1178
+ NULL, NULL, $error, $created, $updated)`).run({
1179
+ $id: run.id,
1180
+ $loopId: run.loopId,
1181
+ $loopName: run.loopName,
1182
+ $scheduledFor: run.scheduledFor,
1183
+ $attempt: run.attempt,
1184
+ $status: run.status,
1185
+ $finished: run.finishedAt ?? null,
1186
+ $error: run.error ?? null,
1187
+ $created: run.createdAt,
1188
+ $updated: run.updatedAt
1189
+ });
1190
+ this.db.exec("COMMIT");
1191
+ } catch (error) {
1192
+ try {
1193
+ this.db.exec("ROLLBACK");
1194
+ } catch {}
1195
+ throw error;
1196
+ }
1074
1197
  return this.getRunBySlot(loop.id, scheduledFor) ?? run;
1075
1198
  }
1076
1199
  getRun(id) {
@@ -1081,11 +1204,20 @@ class Store {
1081
1204
  const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
1082
1205
  return row ? rowToRun(row) : undefined;
1083
1206
  }
1084
- claimRun(loop, scheduledFor, runnerId, now = new Date) {
1207
+ nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
1208
+ const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
1209
+ WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
1210
+ ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
1211
+ WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
1212
+ ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
1213
+ return row ? rowToRun(row) : undefined;
1214
+ }
1215
+ claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
1085
1216
  const startedAt = now.toISOString();
1086
1217
  const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
1087
1218
  this.db.exec("BEGIN IMMEDIATE");
1088
1219
  try {
1220
+ this.assertDaemonLeaseFence(opts, startedAt);
1089
1221
  const existing = this.getRunBySlot(loop.id, scheduledFor);
1090
1222
  if (existing) {
1091
1223
  if (existing.status === "running") {
@@ -1180,11 +1312,15 @@ class Store {
1180
1312
  $error: patch.error ?? null,
1181
1313
  $updated: finishedAt,
1182
1314
  $claimedBy: opts.claimedBy ?? null,
1183
- $now: (opts.now ?? new Date).toISOString()
1315
+ $now: (opts.now ?? new Date).toISOString(),
1316
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1184
1317
  };
1185
1318
  const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1186
1319
  duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
1187
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1320
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
1321
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1322
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1323
+ ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1188
1324
  duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
1189
1325
  const run = this.getRun(id);
1190
1326
  if (!run)
@@ -1193,10 +1329,20 @@ class Store {
1193
1329
  return run;
1194
1330
  return run;
1195
1331
  }
1196
- heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
1332
+ heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
1197
1333
  const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
1198
1334
  const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
1199
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
1335
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
1336
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1337
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1338
+ ))`).run({
1339
+ $id: id,
1340
+ $claimedBy: claimedBy,
1341
+ $expires: expiresAt,
1342
+ $updated: now.toISOString(),
1343
+ $now: now.toISOString(),
1344
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1345
+ });
1200
1346
  if (res.changes !== 1)
1201
1347
  return;
1202
1348
  return this.getRun(id);
@@ -1215,7 +1361,7 @@ class Store {
1215
1361
  }
1216
1362
  return rows.map(rowToRun);
1217
1363
  }
1218
- recoverExpiredRunLeases(now = new Date) {
1364
+ recoverExpiredRunLeases(now = new Date, opts = {}) {
1219
1365
  const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
1220
1366
  const recovered = [];
1221
1367
  for (const row of rows) {
@@ -1224,20 +1370,63 @@ class Store {
1224
1370
  if (this.hasLiveWorkflowStepProcesses(row.id))
1225
1371
  continue;
1226
1372
  const finished = now.toISOString();
1227
- this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
1228
- error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
1229
- const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
1230
- for (const workflowRow of workflowRows) {
1231
- this.db.query(`UPDATE workflow_runs
1232
- SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
1233
- WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
1234
- this.db.query(`UPDATE workflow_step_runs
1235
- SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
1236
- WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
1237
- this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
1238
- error: "parent loop run lease expired before completion",
1239
- loopRunId: row.id
1373
+ this.db.exec("BEGIN IMMEDIATE");
1374
+ try {
1375
+ const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
1376
+ error='run lease expired before completion', updated_at=$updated
1377
+ WHERE id=$id AND status='running' AND lease_expires_at <= $now
1378
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1379
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1380
+ ))`).run({
1381
+ $id: row.id,
1382
+ $finished: finished,
1383
+ $updated: finished,
1384
+ $now: finished,
1385
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1240
1386
  });
1387
+ if (res.changes !== 1) {
1388
+ this.db.exec("COMMIT");
1389
+ continue;
1390
+ }
1391
+ const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
1392
+ for (const workflowRow of workflowRows) {
1393
+ const workflowRes = this.db.query(`UPDATE workflow_runs
1394
+ SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
1395
+ WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
1396
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1397
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1398
+ ))`).run({
1399
+ $id: workflowRow.id,
1400
+ $finished: finished,
1401
+ $updated: finished,
1402
+ $now: finished,
1403
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1404
+ });
1405
+ if (workflowRes.changes !== 1)
1406
+ continue;
1407
+ this.db.query(`UPDATE workflow_step_runs
1408
+ SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
1409
+ WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
1410
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1411
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1412
+ ))`).run({
1413
+ $workflowRunId: workflowRow.id,
1414
+ $finished: finished,
1415
+ $updated: finished,
1416
+ $now: finished,
1417
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1418
+ });
1419
+ this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
1420
+ error: "parent loop run lease expired before completion",
1421
+ loopRunId: row.id
1422
+ });
1423
+ }
1424
+ this.db.exec("COMMIT");
1425
+ } catch (error) {
1426
+ try {
1427
+ this.db.exec("ROLLBACK");
1428
+ } catch {}
1429
+ throw error;
1241
1430
  }
1242
1431
  const run = this.getRun(row.id);
1243
1432
  if (run)
@@ -1245,11 +1434,14 @@ class Store {
1245
1434
  }
1246
1435
  return recovered;
1247
1436
  }
1248
- expireLoops(now = new Date) {
1437
+ expireLoops(now = new Date, opts = {}) {
1249
1438
  const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
1250
1439
  const expired = [];
1251
- for (const row of rows)
1252
- expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
1440
+ for (const row of rows) {
1441
+ const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
1442
+ if (updated.status === "expired")
1443
+ expired.push(updated);
1444
+ }
1253
1445
  return expired;
1254
1446
  }
1255
1447
  countLoops(status) {
@@ -1292,7 +1484,7 @@ class Store {
1292
1484
  }
1293
1485
  heartbeatDaemonLease(id, ttlMs, now = new Date) {
1294
1486
  const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
1295
- const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
1487
+ const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
1296
1488
  if (res.changes !== 1)
1297
1489
  return;
1298
1490
  return this.getDaemonLease();
@@ -1550,7 +1742,7 @@ function agentArgs(target) {
1550
1742
  args.push("--model", target.model);
1551
1743
  if (target.agent)
1552
1744
  args.push("--agent", target.agent);
1553
- args.push(...target.extraArgs ?? [], target.prompt);
1745
+ args.push(...target.extraArgs ?? []);
1554
1746
  return args;
1555
1747
  case "cursor":
1556
1748
  args.push("-p");
@@ -1558,7 +1750,7 @@ function agentArgs(target) {
1558
1750
  args.push("--model", target.model);
1559
1751
  if (target.agent)
1560
1752
  args.push("--agent", target.agent);
1561
- args.push(...target.extraArgs ?? [], target.prompt);
1753
+ args.push(...target.extraArgs ?? []);
1562
1754
  return args;
1563
1755
  case "codewith":
1564
1756
  args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
@@ -1570,7 +1762,7 @@ function agentArgs(target) {
1570
1762
  args.push("--model", target.model);
1571
1763
  if (target.agent)
1572
1764
  args.push("--agent", target.agent);
1573
- args.push(...target.extraArgs ?? [], target.prompt);
1765
+ args.push(...target.extraArgs ?? []);
1574
1766
  return args;
1575
1767
  case "codex":
1576
1768
  args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
@@ -1580,7 +1772,7 @@ function agentArgs(target) {
1580
1772
  args.push("--cd", target.cwd);
1581
1773
  if (target.model)
1582
1774
  args.push("--model", target.model);
1583
- args.push(...target.extraArgs ?? [], target.prompt);
1775
+ args.push(...target.extraArgs ?? []);
1584
1776
  return args;
1585
1777
  case "aicopilot":
1586
1778
  args.push("run", "--format", "json");
@@ -1592,7 +1784,7 @@ function agentArgs(target) {
1592
1784
  args.push("--model", target.model);
1593
1785
  if (target.agent)
1594
1786
  args.push("--agent", target.agent);
1595
- args.push(...target.extraArgs ?? [], target.prompt);
1787
+ args.push(...target.extraArgs ?? []);
1596
1788
  return args;
1597
1789
  case "opencode":
1598
1790
  args.push("run", "--format", "json");
@@ -1604,7 +1796,7 @@ function agentArgs(target) {
1604
1796
  args.push("--model", target.model);
1605
1797
  if (target.agent)
1606
1798
  args.push("--agent", target.agent);
1607
- args.push(...target.extraArgs ?? [], target.prompt);
1799
+ args.push(...target.extraArgs ?? []);
1608
1800
  return args;
1609
1801
  }
1610
1802
  }
@@ -1629,7 +1821,8 @@ function commandSpec(target) {
1629
1821
  cwd: agentTarget.cwd,
1630
1822
  timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
1631
1823
  account: agentTarget.account,
1632
- accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
1824
+ accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
1825
+ stdin: agentTarget.prompt
1633
1826
  };
1634
1827
  }
1635
1828
  function executionEnv(spec, metadata, opts) {
@@ -1698,10 +1891,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
1698
1891
  env,
1699
1892
  shell: spec.shell ?? false,
1700
1893
  detached: true,
1701
- stdio: ["ignore", "pipe", "pipe"]
1894
+ stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
1702
1895
  });
1703
1896
  if (child.pid)
1704
1897
  opts.onSpawn?.(child.pid);
1898
+ if (spec.stdin !== undefined && child.stdin) {
1899
+ child.stdin.on("error", (err) => {
1900
+ if (err.code !== "EPIPE")
1901
+ error = err.message;
1902
+ });
1903
+ child.stdin.end(spec.stdin);
1904
+ }
1705
1905
  const abortHandler = () => {
1706
1906
  error = "cancelled";
1707
1907
  if (child.pid)
@@ -1710,10 +1910,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
1710
1910
  if (opts.signal?.aborted)
1711
1911
  abortHandler();
1712
1912
  opts.signal?.addEventListener("abort", abortHandler, { once: true });
1713
- child.stdout.on("data", (chunk) => {
1913
+ child.stdout?.on("data", (chunk) => {
1714
1914
  stdout = appendBounded(stdout, chunk, maxOutputBytes);
1715
1915
  });
1716
- child.stderr.on("data", (chunk) => {
1916
+ child.stderr?.on("data", (chunk) => {
1717
1917
  stderr = appendBounded(stderr, chunk, maxOutputBytes);
1718
1918
  });
1719
1919
  const timer = setTimeout(() => {
@@ -1812,7 +2012,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1812
2012
  loop: opts.loop,
1813
2013
  loopRun: opts.loopRun,
1814
2014
  scheduledFor: opts.scheduledFor,
1815
- idempotencyKey: opts.idempotencyKey
2015
+ idempotencyKey: opts.idempotencyKey,
2016
+ daemonLeaseId: opts.daemonLeaseId
1816
2017
  });
1817
2018
  const startedAt = run.startedAt ?? nowIso();
1818
2019
  if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
@@ -1846,12 +2047,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
1846
2047
  return !dependencyStep?.continueOnFailure;
1847
2048
  });
1848
2049
  if (blockedBy) {
1849
- store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
2050
+ opts.beforePersist?.();
2051
+ store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
1850
2052
  blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
1851
2053
  terminalStatus = "failed";
1852
2054
  continue;
1853
2055
  }
1854
- const startedStep = store.startWorkflowStepRun(run.id, step.id);
2056
+ opts.beforePersist?.();
2057
+ const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
1855
2058
  if (startedStep.status !== "running") {
1856
2059
  terminalStatus = "failed";
1857
2060
  blockingError = `step ${step.id} could not start because workflow is no longer running`;
@@ -1883,7 +2086,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1883
2086
  ...opts,
1884
2087
  signal: controller.signal,
1885
2088
  onSpawn: (pid) => {
1886
- store.markWorkflowStepPid(run.id, step.id, pid);
2089
+ opts.beforePersist?.();
2090
+ store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
1887
2091
  opts.onSpawn?.(pid);
1888
2092
  }
1889
2093
  });
@@ -1911,6 +2115,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
1911
2115
  blockingError = "workflow run was cancelled";
1912
2116
  break;
1913
2117
  }
2118
+ opts.beforePersist?.();
1914
2119
  store.finalizeWorkflowStepRun(run.id, step.id, {
1915
2120
  status: result.status,
1916
2121
  finishedAt: result.finishedAt,
@@ -1919,6 +2124,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1919
2124
  stderr: result.stderr,
1920
2125
  exitCode: result.exitCode,
1921
2126
  error: result.error
2127
+ }, {
2128
+ daemonLeaseId: opts.daemonLeaseId
1922
2129
  });
1923
2130
  if (result.status !== "succeeded" && !step.continueOnFailure) {
1924
2131
  terminalStatus = result.status;
@@ -1930,7 +2137,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
1930
2137
  for (const step of ordered) {
1931
2138
  const existing = store.getWorkflowStepRun(run.id, step.id);
1932
2139
  if (existing?.status === "pending" || existing?.status === "running") {
1933
- store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
2140
+ store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
2141
+ daemonLeaseId: opts.daemonLeaseId
2142
+ });
1934
2143
  }
1935
2144
  }
1936
2145
  }
@@ -1940,10 +2149,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
1940
2149
  const steps2 = store.listWorkflowStepRuns(run.id);
1941
2150
  return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
1942
2151
  }
2152
+ opts.beforePersist?.();
1943
2153
  const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
1944
2154
  finishedAt,
1945
2155
  durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
1946
2156
  error: blockingError
2157
+ }, {
2158
+ daemonLeaseId: opts.daemonLeaseId
1947
2159
  });
1948
2160
  const steps = store.listWorkflowStepRuns(run.id);
1949
2161
  return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
@@ -1991,52 +2203,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
1991
2203
 
1992
2204
  // src/lib/scheduler.ts
1993
2205
  function manualRunScheduledFor(loop, now = new Date) {
1994
- if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
2206
+ if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
1995
2207
  return loop.retryScheduledFor ?? loop.nextRunAt;
1996
2208
  }
1997
2209
  return now.toISOString();
1998
2210
  }
1999
2211
  function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
2212
+ if (loop.status !== "active")
2213
+ return false;
2000
2214
  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
2001
2215
  return false;
2002
2216
  return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
2003
2217
  }
2218
+ function manualRunSource(loop, scheduledFor, now = new Date) {
2219
+ if (loop.status !== "active")
2220
+ return "ad_hoc";
2221
+ if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
2222
+ return "ad_hoc";
2223
+ if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
2224
+ return "retry_slot";
2225
+ return "due_slot";
2226
+ }
2004
2227
  function nextAfterRetry(loop, now) {
2005
2228
  return new Date(now.getTime() + loop.retryDelayMs).toISOString();
2006
2229
  }
2007
- function advanceLoop(store, loop, run, finishedAt, succeeded) {
2230
+ function isDaemonLeaseLost(error) {
2231
+ return error instanceof Error && error.message === "daemon lease lost";
2232
+ }
2233
+ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
2008
2234
  if (run.status === "running")
2009
2235
  return;
2010
2236
  const current = store.getLoop(loop.id);
2011
2237
  if (!current || current.status !== "active")
2012
2238
  return;
2013
- const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
2239
+ if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
2240
+ return;
2241
+ const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
2014
2242
  if (shouldRetry) {
2015
- store.updateLoop(loop.id, {
2243
+ store.updateLoop(current.id, {
2016
2244
  status: "active",
2017
- nextRunAt: nextAfterRetry(loop, finishedAt),
2245
+ nextRunAt: nextAfterRetry(current, finishedAt),
2018
2246
  retryScheduledFor: run.scheduledFor
2019
- });
2247
+ }, { daemonLeaseId: opts.daemonLeaseId });
2020
2248
  return;
2021
2249
  }
2022
- const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
2023
- store.updateLoop(loop.id, {
2250
+ const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
2251
+ if (deferredRetry) {
2252
+ store.updateLoop(current.id, {
2253
+ status: "active",
2254
+ nextRunAt: nextAfterRetry(current, finishedAt),
2255
+ retryScheduledFor: deferredRetry.scheduledFor
2256
+ }, { daemonLeaseId: opts.daemonLeaseId });
2257
+ return;
2258
+ }
2259
+ const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
2260
+ store.updateLoop(current.id, {
2024
2261
  status: nextRunAt ? "active" : "stopped",
2025
2262
  nextRunAt,
2026
2263
  retryScheduledFor: undefined
2027
- });
2264
+ }, { daemonLeaseId: opts.daemonLeaseId });
2028
2265
  }
2029
2266
  async function executeClaimedRun(deps) {
2030
2267
  let heartbeat;
2031
2268
  const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
2032
2269
  heartbeat = setInterval(() => {
2033
- deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
2270
+ deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
2271
+ daemonLeaseId: deps.daemonLeaseId
2272
+ });
2034
2273
  }, heartbeatEveryMs);
2035
2274
  heartbeat.unref();
2036
2275
  try {
2037
2276
  const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
2038
- onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
2277
+ daemonLeaseId: deps.daemonLeaseId,
2278
+ onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
2039
2279
  })))(deps.loop, deps.run);
2280
+ deps.beforeFinalize?.(deps.loop, deps.run);
2040
2281
  return deps.store.finalizeRun(deps.run.id, {
2041
2282
  status: result.status,
2042
2283
  finishedAt: result.finishedAt,
@@ -2048,10 +2289,16 @@ async function executeClaimedRun(deps) {
2048
2289
  pid: result.pid
2049
2290
  }, {
2050
2291
  claimedBy: deps.runnerId,
2292
+ daemonLeaseId: deps.daemonLeaseId,
2051
2293
  now: deps.now?.() ?? new Date(result.finishedAt)
2052
2294
  });
2053
2295
  } catch (err) {
2054
2296
  deps.onError?.(deps.loop, err);
2297
+ try {
2298
+ deps.beforeFinalize?.(deps.loop, deps.run);
2299
+ } catch {
2300
+ return deps.store.getRun(deps.run.id) ?? deps.run;
2301
+ }
2055
2302
  const finishedAt = new Date;
2056
2303
  return deps.store.finalizeRun(deps.run.id, {
2057
2304
  status: "failed",
@@ -2062,6 +2309,7 @@ async function executeClaimedRun(deps) {
2062
2309
  error: err instanceof Error ? err.message : String(err)
2063
2310
  }, {
2064
2311
  claimedBy: deps.runnerId,
2312
+ daemonLeaseId: deps.daemonLeaseId,
2065
2313
  now: deps.now?.() ?? finishedAt
2066
2314
  });
2067
2315
  } finally {
@@ -2071,15 +2319,33 @@ async function executeClaimedRun(deps) {
2071
2319
  }
2072
2320
  async function runSlot(deps, loop, scheduledFor) {
2073
2321
  const now = deps.now?.() ?? new Date;
2322
+ deps.beforeRun?.(loop, scheduledFor);
2074
2323
  if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
2075
- const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
2076
- advanceLoop(deps.store, loop, skipped, now, true);
2324
+ let skipped;
2325
+ try {
2326
+ skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
2327
+ daemonLeaseId: deps.daemonLeaseId
2328
+ });
2329
+ } catch (error) {
2330
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
2331
+ return;
2332
+ throw error;
2333
+ }
2334
+ advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
2077
2335
  deps.onRun?.(skipped);
2078
2336
  return skipped;
2079
2337
  }
2080
- const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
2338
+ let claim;
2339
+ try {
2340
+ claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
2341
+ } catch (error) {
2342
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
2343
+ return;
2344
+ throw error;
2345
+ }
2081
2346
  if (!claim)
2082
2347
  return;
2348
+ deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
2083
2349
  deps.onRun?.(claim.run);
2084
2350
  const finalRun = await executeClaimedRun({
2085
2351
  store: deps.store,
@@ -2088,21 +2354,42 @@ async function runSlot(deps, loop, scheduledFor) {
2088
2354
  run: claim.run,
2089
2355
  now: deps.now,
2090
2356
  execute: deps.execute,
2357
+ beforeFinalize: deps.beforeFinalize,
2358
+ daemonLeaseId: deps.daemonLeaseId,
2091
2359
  onError: deps.onError
2092
2360
  });
2093
- advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
2361
+ advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
2094
2362
  deps.onRun?.(finalRun);
2095
2363
  return finalRun;
2096
2364
  }
2097
2365
  async function tick(deps) {
2098
2366
  const now = deps.now?.() ?? new Date;
2099
- const recovered = deps.store.recoverExpiredRunLeases(now);
2367
+ const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
2368
+ const recoveredByLoop = new Map;
2100
2369
  for (const run of recovered) {
2101
- const loop = deps.store.getLoop(run.loopId);
2102
- if (loop)
2103
- advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
2370
+ recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
2371
+ }
2372
+ for (const runs of recoveredByLoop.values()) {
2373
+ const loop = deps.store.getLoop(runs[0].loopId);
2374
+ if (!loop)
2375
+ continue;
2376
+ const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
2377
+ if (retryable) {
2378
+ advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
2379
+ daemonLeaseId: deps.daemonLeaseId
2380
+ });
2381
+ continue;
2382
+ }
2383
+ for (const run of runs) {
2384
+ const current = deps.store.getLoop(run.loopId);
2385
+ if (current) {
2386
+ advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
2387
+ daemonLeaseId: deps.daemonLeaseId
2388
+ });
2389
+ }
2390
+ }
2104
2391
  }
2105
- const expired = deps.store.expireLoops(now);
2392
+ const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
2106
2393
  const claimed = [];
2107
2394
  const completed = [];
2108
2395
  const skipped = [];
@@ -2320,8 +2607,10 @@ async function runDaemon(opts = {}) {
2320
2607
  const result = await tick({
2321
2608
  store,
2322
2609
  runnerId,
2610
+ daemonLeaseId: leaseId,
2611
+ beforeRun: () => ensureLease(),
2323
2612
  execute: async (loop, run) => {
2324
- const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs / 3));
2613
+ const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
2325
2614
  const timer = setInterval(() => {
2326
2615
  try {
2327
2616
  ensureLease();
@@ -2333,8 +2622,14 @@ async function runDaemon(opts = {}) {
2333
2622
  try {
2334
2623
  const result2 = await executeLoopTarget(store, loop, run, {
2335
2624
  signal: runAbort.signal,
2336
- onSpawn: (pid) => store.markRunPid(run.id, pid, runnerId)
2625
+ beforePersist: () => ensureLease(),
2626
+ daemonLeaseId: leaseId,
2627
+ onSpawn: (pid) => {
2628
+ ensureLease();
2629
+ store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
2630
+ }
2337
2631
  });
2632
+ ensureLease();
2338
2633
  if (leaseLost)
2339
2634
  throw new Error("daemon lease lost during run");
2340
2635
  return result2;
@@ -2342,6 +2637,7 @@ async function runDaemon(opts = {}) {
2342
2637
  clearInterval(timer);
2343
2638
  }
2344
2639
  },
2640
+ beforeFinalize: () => ensureLease(),
2345
2641
  onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
2346
2642
  });
2347
2643
  const changed = result.completed.length + result.skipped.length + result.recovered.length + result.expired.length;
@@ -2472,7 +2768,7 @@ function enableStartup(result) {
2472
2768
 
2473
2769
  // src/daemon/index.ts
2474
2770
  var program = new Command;
2475
- program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.2");
2771
+ program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.3");
2476
2772
  program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
2477
2773
  program.command("start").action(async () => {
2478
2774
  const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops-daemon", args: ["run"] });