@hasna/loops 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -260,6 +260,11 @@ function validateTarget(value, label) {
260
260
  const providers = ["claude", "cursor", "codewith", "aicopilot", "opencode", "codex"];
261
261
  if (!providers.includes(value.provider))
262
262
  throw new Error(`${label}.provider must be one of ${providers.join(", ")}`);
263
+ if (value.authProfile !== undefined) {
264
+ assertString(value.authProfile, `${label}.authProfile`);
265
+ if (value.provider !== "codewith")
266
+ throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
267
+ }
263
268
  return value;
264
269
  }
265
270
  throw new Error(`${label}.type must be command or agent`);
@@ -610,6 +615,13 @@ class Store {
610
615
  } catch {}
611
616
  this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
612
617
  }
618
+ assertDaemonLeaseFence(opts = {}, now = nowIso()) {
619
+ if (!opts.daemonLeaseId)
620
+ return;
621
+ const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
622
+ if (!row)
623
+ throw new Error("daemon lease lost");
624
+ }
613
625
  createLoop(input, from = new Date) {
614
626
  const now = nowIso();
615
627
  const loop = {
@@ -679,21 +691,31 @@ class Store {
679
691
  ORDER BY next_run_at ASC`).all(now.toISOString());
680
692
  return rows.map(rowToLoop);
681
693
  }
682
- updateLoop(id, patch) {
694
+ updateLoop(id, patch, opts = {}) {
683
695
  const current = this.getLoop(id);
684
696
  if (!current)
685
697
  throw new Error(`loop not found: ${id}`);
686
- const merged = { ...current, ...patch, updatedAt: nowIso() };
698
+ const updated = (opts.now ?? new Date).toISOString();
699
+ const merged = { ...current, ...patch, updatedAt: updated };
687
700
  this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
688
- expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
701
+ expires_at=$expiresAt, updated_at=$updated
702
+ WHERE id=$id
703
+ AND ($daemonLeaseId IS NULL OR EXISTS (
704
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
705
+ ))`).run({
689
706
  $id: id,
690
707
  $status: merged.status,
691
708
  $nextRun: merged.nextRunAt ?? null,
692
709
  $retrySlot: merged.retryScheduledFor ?? null,
693
710
  $expiresAt: merged.expiresAt ?? null,
694
- $updated: merged.updatedAt
711
+ $updated: merged.updatedAt,
712
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
713
+ $now: updated
695
714
  });
696
- return merged;
715
+ const after = this.getLoop(id);
716
+ if (!after)
717
+ throw new Error(`loop not found after update: ${id}`);
718
+ return after;
697
719
  }
698
720
  deleteLoop(idOrName) {
699
721
  const loop = this.requireLoop(idOrName);
@@ -757,11 +779,14 @@ class Store {
757
779
  const now = nowIso();
758
780
  if (input.idempotencyKey) {
759
781
  const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
760
- if (existing)
782
+ if (existing) {
783
+ this.assertDaemonLeaseFence(input);
761
784
  return rowToWorkflowRun(existing);
785
+ }
762
786
  }
763
787
  this.db.exec("BEGIN IMMEDIATE");
764
788
  try {
789
+ this.assertDaemonLeaseFence(input, now);
765
790
  if (input.idempotencyKey) {
766
791
  const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
767
792
  if (existing) {
@@ -860,31 +885,60 @@ class Store {
860
885
  const run = this.getWorkflowRun(workflowRunId);
861
886
  return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
862
887
  }
863
- startWorkflowStepRun(workflowRunId, stepId) {
864
- const now = nowIso();
865
- const res = this.db.query(`UPDATE workflow_step_runs
866
- SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
867
- pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
868
- WHERE workflow_run_id=$workflowRunId
869
- AND step_id=$stepId
870
- AND status IN ('pending', 'failed', 'timed_out')
871
- AND EXISTS (
872
- SELECT 1 FROM workflow_runs
873
- WHERE id=$workflowRunId AND status='running'
874
- )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
875
- const run = this.getWorkflowStepRun(workflowRunId, stepId);
876
- if (!run)
877
- throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
878
- if (res.changes !== 1) {
879
- throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
888
+ startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
889
+ const now = (opts.now ?? new Date).toISOString();
890
+ this.db.exec("BEGIN IMMEDIATE");
891
+ try {
892
+ const res = this.db.query(`UPDATE workflow_step_runs
893
+ SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
894
+ pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
895
+ WHERE workflow_run_id=$workflowRunId
896
+ AND step_id=$stepId
897
+ AND status IN ('pending', 'failed', 'timed_out')
898
+ AND EXISTS (
899
+ SELECT 1 FROM workflow_runs
900
+ WHERE id=$workflowRunId AND status='running'
901
+ )
902
+ AND ($daemonLeaseId IS NULL OR EXISTS (
903
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
904
+ ))`).run({
905
+ $workflowRunId: workflowRunId,
906
+ $stepId: stepId,
907
+ $started: now,
908
+ $updated: now,
909
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
910
+ $now: now
911
+ });
912
+ const run = this.getWorkflowStepRun(workflowRunId, stepId);
913
+ if (!run)
914
+ throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
915
+ if (res.changes !== 1) {
916
+ throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
917
+ }
918
+ this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
919
+ this.db.exec("COMMIT");
920
+ return run;
921
+ } catch (error) {
922
+ try {
923
+ this.db.exec("ROLLBACK");
924
+ } catch {}
925
+ throw error;
880
926
  }
881
- this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
882
- return run;
883
927
  }
884
- markWorkflowStepPid(workflowRunId, stepId, pid) {
885
- const now = nowIso();
928
+ markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
929
+ const now = (opts.now ?? new Date).toISOString();
886
930
  this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
887
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
931
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
932
+ AND ($daemonLeaseId IS NULL OR EXISTS (
933
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
934
+ ))`).run({
935
+ $workflowRunId: workflowRunId,
936
+ $stepId: stepId,
937
+ $pid: pid,
938
+ $updated: now,
939
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
940
+ $now: now
941
+ });
888
942
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
889
943
  if (!run)
890
944
  throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -912,60 +966,110 @@ class Store {
912
966
  recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
913
967
  };
914
968
  }
915
- finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
969
+ finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
916
970
  const finishedAt = patch.finishedAt ?? nowIso();
917
- const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
918
- pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
919
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
920
- $workflowRunId: workflowRunId,
921
- $stepId: stepId,
922
- $status: patch.status,
923
- $finished: finishedAt,
924
- $exitCode: patch.exitCode ?? null,
925
- $durationMs: patch.durationMs ?? null,
926
- $stdout: patch.stdout ?? null,
927
- $stderr: patch.stderr ?? null,
928
- $error: patch.error ?? null,
929
- $updated: finishedAt
930
- });
931
- if (res.changes === 1) {
932
- this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
933
- exitCode: patch.exitCode,
934
- error: patch.error
971
+ this.db.exec("BEGIN IMMEDIATE");
972
+ try {
973
+ const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
974
+ pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
975
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
976
+ AND ($daemonLeaseId IS NULL OR EXISTS (
977
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
978
+ ))`).run({
979
+ $workflowRunId: workflowRunId,
980
+ $stepId: stepId,
981
+ $status: patch.status,
982
+ $finished: finishedAt,
983
+ $exitCode: patch.exitCode ?? null,
984
+ $durationMs: patch.durationMs ?? null,
985
+ $stdout: patch.stdout ?? null,
986
+ $stderr: patch.stderr ?? null,
987
+ $error: patch.error ?? null,
988
+ $updated: finishedAt,
989
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
990
+ $now: (opts.now ?? new Date(finishedAt)).toISOString()
935
991
  });
992
+ if (res.changes === 1) {
993
+ this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
994
+ exitCode: patch.exitCode,
995
+ error: patch.error
996
+ });
997
+ }
998
+ this.db.exec("COMMIT");
999
+ } catch (error) {
1000
+ try {
1001
+ this.db.exec("ROLLBACK");
1002
+ } catch {}
1003
+ throw error;
936
1004
  }
937
1005
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
938
1006
  if (!run)
939
1007
  throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
940
1008
  return run;
941
1009
  }
942
- skipWorkflowStepRun(workflowRunId, stepId, reason) {
943
- const now = nowIso();
944
- const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
945
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
946
- if (res.changes === 1)
947
- this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
1010
+ skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
1011
+ const now = (opts.now ?? new Date).toISOString();
1012
+ this.db.exec("BEGIN IMMEDIATE");
1013
+ try {
1014
+ const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
1015
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
1016
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1017
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1018
+ ))`).run({
1019
+ $workflowRunId: workflowRunId,
1020
+ $stepId: stepId,
1021
+ $finished: now,
1022
+ $error: reason,
1023
+ $updated: now,
1024
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1025
+ $now: now
1026
+ });
1027
+ if (res.changes === 1)
1028
+ this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
1029
+ this.db.exec("COMMIT");
1030
+ } catch (error) {
1031
+ try {
1032
+ this.db.exec("ROLLBACK");
1033
+ } catch {}
1034
+ throw error;
1035
+ }
948
1036
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
949
1037
  if (!run)
950
1038
  throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
951
1039
  return run;
952
1040
  }
953
- finalizeWorkflowRun(workflowRunId, status, patch = {}) {
1041
+ finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
954
1042
  const finishedAt = patch.finishedAt ?? nowIso();
955
- const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
956
- WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
957
- $id: workflowRunId,
958
- $status: status,
959
- $finished: finishedAt,
960
- $durationMs: patch.durationMs ?? null,
961
- $error: patch.error ?? null,
962
- $updated: finishedAt
963
- });
1043
+ let changed = false;
1044
+ this.db.exec("BEGIN IMMEDIATE");
1045
+ try {
1046
+ const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
1047
+ WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
1048
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1049
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1050
+ ))`).run({
1051
+ $id: workflowRunId,
1052
+ $status: status,
1053
+ $finished: finishedAt,
1054
+ $durationMs: patch.durationMs ?? null,
1055
+ $error: patch.error ?? null,
1056
+ $updated: finishedAt,
1057
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1058
+ $now: (opts.now ?? new Date(finishedAt)).toISOString()
1059
+ });
1060
+ changed = res.changes === 1;
1061
+ if (changed)
1062
+ this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
1063
+ this.db.exec("COMMIT");
1064
+ } catch (error) {
1065
+ try {
1066
+ this.db.exec("ROLLBACK");
1067
+ } catch {}
1068
+ throw error;
1069
+ }
964
1070
  const run = this.getWorkflowRun(workflowRunId);
965
1071
  if (!run)
966
1072
  throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
967
- if (res.changes === 1)
968
- this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
969
1073
  return run;
970
1074
  }
971
1075
  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1019,10 +1123,24 @@ class Store {
1019
1123
  const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
1020
1124
  return (row?.count ?? 0) > 0;
1021
1125
  }
1022
- markRunPid(id, pid, claimedBy) {
1023
- const now = nowIso();
1126
+ markRunPid(id, pid, claimedBy, opts = {}) {
1127
+ const now = (opts.now ?? new Date).toISOString();
1024
1128
  const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
1025
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
1129
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy
1130
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1131
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1132
+ ))`).run({
1133
+ $id: id,
1134
+ $pid: pid,
1135
+ $updated: now,
1136
+ $claimedBy: claimedBy,
1137
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1138
+ $now: now
1139
+ }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
1140
+ WHERE id=$id AND status='running'
1141
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1142
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1143
+ ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
1026
1144
  if (res.changes !== 1)
1027
1145
  return;
1028
1146
  return this.getRun(id);
@@ -1037,7 +1155,7 @@ class Store {
1037
1155
  AND wsr.pid IS NOT NULL`).all(loopRunId);
1038
1156
  return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
1039
1157
  }
1040
- createSkippedRun(loop, scheduledFor, reason) {
1158
+ createSkippedRun(loop, scheduledFor, reason, opts = {}) {
1041
1159
  const now = nowIso();
1042
1160
  const run = {
1043
1161
  id: genId(),
@@ -1051,21 +1169,31 @@ class Store {
1051
1169
  createdAt: now,
1052
1170
  updatedAt: now
1053
1171
  };
1054
- this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
1055
- claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
1056
- VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
1057
- NULL, NULL, $error, $created, $updated)`).run({
1058
- $id: run.id,
1059
- $loopId: run.loopId,
1060
- $loopName: run.loopName,
1061
- $scheduledFor: run.scheduledFor,
1062
- $attempt: run.attempt,
1063
- $status: run.status,
1064
- $finished: run.finishedAt ?? null,
1065
- $error: run.error ?? null,
1066
- $created: run.createdAt,
1067
- $updated: run.updatedAt
1068
- });
1172
+ this.db.exec("BEGIN IMMEDIATE");
1173
+ try {
1174
+ this.assertDaemonLeaseFence(opts, now);
1175
+ this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
1176
+ claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
1177
+ VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
1178
+ NULL, NULL, $error, $created, $updated)`).run({
1179
+ $id: run.id,
1180
+ $loopId: run.loopId,
1181
+ $loopName: run.loopName,
1182
+ $scheduledFor: run.scheduledFor,
1183
+ $attempt: run.attempt,
1184
+ $status: run.status,
1185
+ $finished: run.finishedAt ?? null,
1186
+ $error: run.error ?? null,
1187
+ $created: run.createdAt,
1188
+ $updated: run.updatedAt
1189
+ });
1190
+ this.db.exec("COMMIT");
1191
+ } catch (error) {
1192
+ try {
1193
+ this.db.exec("ROLLBACK");
1194
+ } catch {}
1195
+ throw error;
1196
+ }
1069
1197
  return this.getRunBySlot(loop.id, scheduledFor) ?? run;
1070
1198
  }
1071
1199
  getRun(id) {
@@ -1076,11 +1204,20 @@ class Store {
1076
1204
  const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
1077
1205
  return row ? rowToRun(row) : undefined;
1078
1206
  }
1079
- claimRun(loop, scheduledFor, runnerId, now = new Date) {
1207
+ nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
1208
+ const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
1209
+ WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
1210
+ ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
1211
+ WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
1212
+ ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
1213
+ return row ? rowToRun(row) : undefined;
1214
+ }
1215
+ claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
1080
1216
  const startedAt = now.toISOString();
1081
1217
  const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
1082
1218
  this.db.exec("BEGIN IMMEDIATE");
1083
1219
  try {
1220
+ this.assertDaemonLeaseFence(opts, startedAt);
1084
1221
  const existing = this.getRunBySlot(loop.id, scheduledFor);
1085
1222
  if (existing) {
1086
1223
  if (existing.status === "running") {
@@ -1175,11 +1312,15 @@ class Store {
1175
1312
  $error: patch.error ?? null,
1176
1313
  $updated: finishedAt,
1177
1314
  $claimedBy: opts.claimedBy ?? null,
1178
- $now: (opts.now ?? new Date).toISOString()
1315
+ $now: (opts.now ?? new Date).toISOString(),
1316
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1179
1317
  };
1180
1318
  const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1181
1319
  duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
1182
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1320
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
1321
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1322
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1323
+ ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1183
1324
  duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
1184
1325
  const run = this.getRun(id);
1185
1326
  if (!run)
@@ -1188,10 +1329,20 @@ class Store {
1188
1329
  return run;
1189
1330
  return run;
1190
1331
  }
1191
- heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
1332
+ heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
1192
1333
  const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
1193
1334
  const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
1194
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
1335
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
1336
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1337
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1338
+ ))`).run({
1339
+ $id: id,
1340
+ $claimedBy: claimedBy,
1341
+ $expires: expiresAt,
1342
+ $updated: now.toISOString(),
1343
+ $now: now.toISOString(),
1344
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1345
+ });
1195
1346
  if (res.changes !== 1)
1196
1347
  return;
1197
1348
  return this.getRun(id);
@@ -1210,7 +1361,7 @@ class Store {
1210
1361
  }
1211
1362
  return rows.map(rowToRun);
1212
1363
  }
1213
- recoverExpiredRunLeases(now = new Date) {
1364
+ recoverExpiredRunLeases(now = new Date, opts = {}) {
1214
1365
  const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
1215
1366
  const recovered = [];
1216
1367
  for (const row of rows) {
@@ -1219,20 +1370,63 @@ class Store {
1219
1370
  if (this.hasLiveWorkflowStepProcesses(row.id))
1220
1371
  continue;
1221
1372
  const finished = now.toISOString();
1222
- this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
1223
- error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
1224
- const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
1225
- for (const workflowRow of workflowRows) {
1226
- this.db.query(`UPDATE workflow_runs
1227
- SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
1228
- WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
1229
- this.db.query(`UPDATE workflow_step_runs
1230
- SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
1231
- WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
1232
- this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
1233
- error: "parent loop run lease expired before completion",
1234
- loopRunId: row.id
1373
+ this.db.exec("BEGIN IMMEDIATE");
1374
+ try {
1375
+ const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
1376
+ error='run lease expired before completion', updated_at=$updated
1377
+ WHERE id=$id AND status='running' AND lease_expires_at <= $now
1378
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1379
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1380
+ ))`).run({
1381
+ $id: row.id,
1382
+ $finished: finished,
1383
+ $updated: finished,
1384
+ $now: finished,
1385
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1235
1386
  });
1387
+ if (res.changes !== 1) {
1388
+ this.db.exec("COMMIT");
1389
+ continue;
1390
+ }
1391
+ const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
1392
+ for (const workflowRow of workflowRows) {
1393
+ const workflowRes = this.db.query(`UPDATE workflow_runs
1394
+ SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
1395
+ WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
1396
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1397
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1398
+ ))`).run({
1399
+ $id: workflowRow.id,
1400
+ $finished: finished,
1401
+ $updated: finished,
1402
+ $now: finished,
1403
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1404
+ });
1405
+ if (workflowRes.changes !== 1)
1406
+ continue;
1407
+ this.db.query(`UPDATE workflow_step_runs
1408
+ SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
1409
+ WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
1410
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1411
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1412
+ ))`).run({
1413
+ $workflowRunId: workflowRow.id,
1414
+ $finished: finished,
1415
+ $updated: finished,
1416
+ $now: finished,
1417
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1418
+ });
1419
+ this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
1420
+ error: "parent loop run lease expired before completion",
1421
+ loopRunId: row.id
1422
+ });
1423
+ }
1424
+ this.db.exec("COMMIT");
1425
+ } catch (error) {
1426
+ try {
1427
+ this.db.exec("ROLLBACK");
1428
+ } catch {}
1429
+ throw error;
1236
1430
  }
1237
1431
  const run = this.getRun(row.id);
1238
1432
  if (run)
@@ -1240,11 +1434,14 @@ class Store {
1240
1434
  }
1241
1435
  return recovered;
1242
1436
  }
1243
- expireLoops(now = new Date) {
1437
+ expireLoops(now = new Date, opts = {}) {
1244
1438
  const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
1245
1439
  const expired = [];
1246
- for (const row of rows)
1247
- expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
1440
+ for (const row of rows) {
1441
+ const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
1442
+ if (updated.status === "expired")
1443
+ expired.push(updated);
1444
+ }
1248
1445
  return expired;
1249
1446
  }
1250
1447
  countLoops(status) {
@@ -1287,7 +1484,7 @@ class Store {
1287
1484
  }
1288
1485
  heartbeatDaemonLease(id, ttlMs, now = new Date) {
1289
1486
  const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
1290
- const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
1487
+ const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
1291
1488
  if (res.changes !== 1)
1292
1489
  return;
1293
1490
  return this.getDaemonLease();
@@ -1545,7 +1742,7 @@ function agentArgs(target) {
1545
1742
  args.push("--model", target.model);
1546
1743
  if (target.agent)
1547
1744
  args.push("--agent", target.agent);
1548
- args.push(...target.extraArgs ?? [], target.prompt);
1745
+ args.push(...target.extraArgs ?? []);
1549
1746
  return args;
1550
1747
  case "cursor":
1551
1748
  args.push("-p");
@@ -1553,10 +1750,10 @@ function agentArgs(target) {
1553
1750
  args.push("--model", target.model);
1554
1751
  if (target.agent)
1555
1752
  args.push("--agent", target.agent);
1556
- args.push(...target.extraArgs ?? [], target.prompt);
1753
+ args.push(...target.extraArgs ?? []);
1557
1754
  return args;
1558
1755
  case "codewith":
1559
- args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
1756
+ args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
1560
1757
  if (isolation === "safe")
1561
1758
  args.push("--ignore-rules");
1562
1759
  if (target.cwd)
@@ -1565,7 +1762,7 @@ function agentArgs(target) {
1565
1762
  args.push("--model", target.model);
1566
1763
  if (target.agent)
1567
1764
  args.push("--agent", target.agent);
1568
- args.push(...target.extraArgs ?? [], target.prompt);
1765
+ args.push(...target.extraArgs ?? []);
1569
1766
  return args;
1570
1767
  case "codex":
1571
1768
  args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
@@ -1575,7 +1772,7 @@ function agentArgs(target) {
1575
1772
  args.push("--cd", target.cwd);
1576
1773
  if (target.model)
1577
1774
  args.push("--model", target.model);
1578
- args.push(...target.extraArgs ?? [], target.prompt);
1775
+ args.push(...target.extraArgs ?? []);
1579
1776
  return args;
1580
1777
  case "aicopilot":
1581
1778
  args.push("run", "--format", "json");
@@ -1587,7 +1784,7 @@ function agentArgs(target) {
1587
1784
  args.push("--model", target.model);
1588
1785
  if (target.agent)
1589
1786
  args.push("--agent", target.agent);
1590
- args.push(...target.extraArgs ?? [], target.prompt);
1787
+ args.push(...target.extraArgs ?? []);
1591
1788
  return args;
1592
1789
  case "opencode":
1593
1790
  args.push("run", "--format", "json");
@@ -1599,7 +1796,7 @@ function agentArgs(target) {
1599
1796
  args.push("--model", target.model);
1600
1797
  if (target.agent)
1601
1798
  args.push("--agent", target.agent);
1602
- args.push(...target.extraArgs ?? [], target.prompt);
1799
+ args.push(...target.extraArgs ?? []);
1603
1800
  return args;
1604
1801
  }
1605
1802
  }
@@ -1624,7 +1821,8 @@ function commandSpec(target) {
1624
1821
  cwd: agentTarget.cwd,
1625
1822
  timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
1626
1823
  account: agentTarget.account,
1627
- accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
1824
+ accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
1825
+ stdin: agentTarget.prompt
1628
1826
  };
1629
1827
  }
1630
1828
  function executionEnv(spec, metadata, opts) {
@@ -1693,10 +1891,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
1693
1891
  env,
1694
1892
  shell: spec.shell ?? false,
1695
1893
  detached: true,
1696
- stdio: ["ignore", "pipe", "pipe"]
1894
+ stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
1697
1895
  });
1698
1896
  if (child.pid)
1699
1897
  opts.onSpawn?.(child.pid);
1898
+ if (spec.stdin !== undefined && child.stdin) {
1899
+ child.stdin.on("error", (err) => {
1900
+ if (err.code !== "EPIPE")
1901
+ error = err.message;
1902
+ });
1903
+ child.stdin.end(spec.stdin);
1904
+ }
1700
1905
  const abortHandler = () => {
1701
1906
  error = "cancelled";
1702
1907
  if (child.pid)
@@ -1705,10 +1910,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
1705
1910
  if (opts.signal?.aborted)
1706
1911
  abortHandler();
1707
1912
  opts.signal?.addEventListener("abort", abortHandler, { once: true });
1708
- child.stdout.on("data", (chunk) => {
1913
+ child.stdout?.on("data", (chunk) => {
1709
1914
  stdout = appendBounded(stdout, chunk, maxOutputBytes);
1710
1915
  });
1711
- child.stderr.on("data", (chunk) => {
1916
+ child.stderr?.on("data", (chunk) => {
1712
1917
  stderr = appendBounded(stderr, chunk, maxOutputBytes);
1713
1918
  });
1714
1919
  const timer = setTimeout(() => {
@@ -1807,7 +2012,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1807
2012
  loop: opts.loop,
1808
2013
  loopRun: opts.loopRun,
1809
2014
  scheduledFor: opts.scheduledFor,
1810
- idempotencyKey: opts.idempotencyKey
2015
+ idempotencyKey: opts.idempotencyKey,
2016
+ daemonLeaseId: opts.daemonLeaseId
1811
2017
  });
1812
2018
  const startedAt = run.startedAt ?? nowIso();
1813
2019
  if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
@@ -1841,12 +2047,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
1841
2047
  return !dependencyStep?.continueOnFailure;
1842
2048
  });
1843
2049
  if (blockedBy) {
1844
- store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
2050
+ opts.beforePersist?.();
2051
+ store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
1845
2052
  blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
1846
2053
  terminalStatus = "failed";
1847
2054
  continue;
1848
2055
  }
1849
- const startedStep = store.startWorkflowStepRun(run.id, step.id);
2056
+ opts.beforePersist?.();
2057
+ const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
1850
2058
  if (startedStep.status !== "running") {
1851
2059
  terminalStatus = "failed";
1852
2060
  blockingError = `step ${step.id} could not start because workflow is no longer running`;
@@ -1878,7 +2086,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1878
2086
  ...opts,
1879
2087
  signal: controller.signal,
1880
2088
  onSpawn: (pid) => {
1881
- store.markWorkflowStepPid(run.id, step.id, pid);
2089
+ opts.beforePersist?.();
2090
+ store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
1882
2091
  opts.onSpawn?.(pid);
1883
2092
  }
1884
2093
  });
@@ -1906,6 +2115,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
1906
2115
  blockingError = "workflow run was cancelled";
1907
2116
  break;
1908
2117
  }
2118
+ opts.beforePersist?.();
1909
2119
  store.finalizeWorkflowStepRun(run.id, step.id, {
1910
2120
  status: result.status,
1911
2121
  finishedAt: result.finishedAt,
@@ -1914,6 +2124,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1914
2124
  stderr: result.stderr,
1915
2125
  exitCode: result.exitCode,
1916
2126
  error: result.error
2127
+ }, {
2128
+ daemonLeaseId: opts.daemonLeaseId
1917
2129
  });
1918
2130
  if (result.status !== "succeeded" && !step.continueOnFailure) {
1919
2131
  terminalStatus = result.status;
@@ -1925,7 +2137,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
1925
2137
  for (const step of ordered) {
1926
2138
  const existing = store.getWorkflowStepRun(run.id, step.id);
1927
2139
  if (existing?.status === "pending" || existing?.status === "running") {
1928
- store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
2140
+ store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
2141
+ daemonLeaseId: opts.daemonLeaseId
2142
+ });
1929
2143
  }
1930
2144
  }
1931
2145
  }
@@ -1935,10 +2149,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
1935
2149
  const steps2 = store.listWorkflowStepRuns(run.id);
1936
2150
  return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
1937
2151
  }
2152
+ opts.beforePersist?.();
1938
2153
  const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
1939
2154
  finishedAt,
1940
2155
  durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
1941
2156
  error: blockingError
2157
+ }, {
2158
+ daemonLeaseId: opts.daemonLeaseId
1942
2159
  });
1943
2160
  const steps = store.listWorkflowStepRuns(run.id);
1944
2161
  return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
@@ -1986,52 +2203,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
1986
2203
 
1987
2204
  // src/lib/scheduler.ts
1988
2205
  function manualRunScheduledFor(loop, now = new Date) {
1989
- if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
2206
+ if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
1990
2207
  return loop.retryScheduledFor ?? loop.nextRunAt;
1991
2208
  }
1992
2209
  return now.toISOString();
1993
2210
  }
1994
2211
  function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
2212
+ if (loop.status !== "active")
2213
+ return false;
1995
2214
  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
1996
2215
  return false;
1997
2216
  return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
1998
2217
  }
2218
+ function manualRunSource(loop, scheduledFor, now = new Date) {
2219
+ if (loop.status !== "active")
2220
+ return "ad_hoc";
2221
+ if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
2222
+ return "ad_hoc";
2223
+ if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
2224
+ return "retry_slot";
2225
+ return "due_slot";
2226
+ }
1999
2227
  function nextAfterRetry(loop, now) {
2000
2228
  return new Date(now.getTime() + loop.retryDelayMs).toISOString();
2001
2229
  }
2002
- function advanceLoop(store, loop, run, finishedAt, succeeded) {
2230
+ function isDaemonLeaseLost(error) {
2231
+ return error instanceof Error && error.message === "daemon lease lost";
2232
+ }
2233
+ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
2003
2234
  if (run.status === "running")
2004
2235
  return;
2005
2236
  const current = store.getLoop(loop.id);
2006
2237
  if (!current || current.status !== "active")
2007
2238
  return;
2008
- const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
2239
+ if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
2240
+ return;
2241
+ const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
2009
2242
  if (shouldRetry) {
2010
- store.updateLoop(loop.id, {
2243
+ store.updateLoop(current.id, {
2011
2244
  status: "active",
2012
- nextRunAt: nextAfterRetry(loop, finishedAt),
2245
+ nextRunAt: nextAfterRetry(current, finishedAt),
2013
2246
  retryScheduledFor: run.scheduledFor
2014
- });
2247
+ }, { daemonLeaseId: opts.daemonLeaseId });
2015
2248
  return;
2016
2249
  }
2017
- const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
2018
- store.updateLoop(loop.id, {
2250
+ const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
2251
+ if (deferredRetry) {
2252
+ store.updateLoop(current.id, {
2253
+ status: "active",
2254
+ nextRunAt: nextAfterRetry(current, finishedAt),
2255
+ retryScheduledFor: deferredRetry.scheduledFor
2256
+ }, { daemonLeaseId: opts.daemonLeaseId });
2257
+ return;
2258
+ }
2259
+ const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
2260
+ store.updateLoop(current.id, {
2019
2261
  status: nextRunAt ? "active" : "stopped",
2020
2262
  nextRunAt,
2021
2263
  retryScheduledFor: undefined
2022
- });
2264
+ }, { daemonLeaseId: opts.daemonLeaseId });
2023
2265
  }
2024
2266
  async function executeClaimedRun(deps) {
2025
2267
  let heartbeat;
2026
2268
  const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
2027
2269
  heartbeat = setInterval(() => {
2028
- deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
2270
+ deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
2271
+ daemonLeaseId: deps.daemonLeaseId
2272
+ });
2029
2273
  }, heartbeatEveryMs);
2030
2274
  heartbeat.unref();
2031
2275
  try {
2032
2276
  const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
2033
- onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
2277
+ daemonLeaseId: deps.daemonLeaseId,
2278
+ onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
2034
2279
  })))(deps.loop, deps.run);
2280
+ deps.beforeFinalize?.(deps.loop, deps.run);
2035
2281
  return deps.store.finalizeRun(deps.run.id, {
2036
2282
  status: result.status,
2037
2283
  finishedAt: result.finishedAt,
@@ -2043,10 +2289,16 @@ async function executeClaimedRun(deps) {
2043
2289
  pid: result.pid
2044
2290
  }, {
2045
2291
  claimedBy: deps.runnerId,
2292
+ daemonLeaseId: deps.daemonLeaseId,
2046
2293
  now: deps.now?.() ?? new Date(result.finishedAt)
2047
2294
  });
2048
2295
  } catch (err) {
2049
2296
  deps.onError?.(deps.loop, err);
2297
+ try {
2298
+ deps.beforeFinalize?.(deps.loop, deps.run);
2299
+ } catch {
2300
+ return deps.store.getRun(deps.run.id) ?? deps.run;
2301
+ }
2050
2302
  const finishedAt = new Date;
2051
2303
  return deps.store.finalizeRun(deps.run.id, {
2052
2304
  status: "failed",
@@ -2057,6 +2309,7 @@ async function executeClaimedRun(deps) {
2057
2309
  error: err instanceof Error ? err.message : String(err)
2058
2310
  }, {
2059
2311
  claimedBy: deps.runnerId,
2312
+ daemonLeaseId: deps.daemonLeaseId,
2060
2313
  now: deps.now?.() ?? finishedAt
2061
2314
  });
2062
2315
  } finally {
@@ -2066,15 +2319,33 @@ async function executeClaimedRun(deps) {
2066
2319
  }
2067
2320
  async function runSlot(deps, loop, scheduledFor) {
2068
2321
  const now = deps.now?.() ?? new Date;
2322
+ deps.beforeRun?.(loop, scheduledFor);
2069
2323
  if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
2070
- const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
2071
- advanceLoop(deps.store, loop, skipped, now, true);
2324
+ let skipped;
2325
+ try {
2326
+ skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
2327
+ daemonLeaseId: deps.daemonLeaseId
2328
+ });
2329
+ } catch (error) {
2330
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
2331
+ return;
2332
+ throw error;
2333
+ }
2334
+ advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
2072
2335
  deps.onRun?.(skipped);
2073
2336
  return skipped;
2074
2337
  }
2075
- const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
2338
+ let claim;
2339
+ try {
2340
+ claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
2341
+ } catch (error) {
2342
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
2343
+ return;
2344
+ throw error;
2345
+ }
2076
2346
  if (!claim)
2077
2347
  return;
2348
+ deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
2078
2349
  deps.onRun?.(claim.run);
2079
2350
  const finalRun = await executeClaimedRun({
2080
2351
  store: deps.store,
@@ -2083,21 +2354,42 @@ async function runSlot(deps, loop, scheduledFor) {
2083
2354
  run: claim.run,
2084
2355
  now: deps.now,
2085
2356
  execute: deps.execute,
2357
+ beforeFinalize: deps.beforeFinalize,
2358
+ daemonLeaseId: deps.daemonLeaseId,
2086
2359
  onError: deps.onError
2087
2360
  });
2088
- advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
2361
+ advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
2089
2362
  deps.onRun?.(finalRun);
2090
2363
  return finalRun;
2091
2364
  }
2092
2365
  async function tick(deps) {
2093
2366
  const now = deps.now?.() ?? new Date;
2094
- const recovered = deps.store.recoverExpiredRunLeases(now);
2367
+ const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
2368
+ const recoveredByLoop = new Map;
2095
2369
  for (const run of recovered) {
2096
- const loop = deps.store.getLoop(run.loopId);
2097
- if (loop)
2098
- advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
2370
+ recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
2371
+ }
2372
+ for (const runs of recoveredByLoop.values()) {
2373
+ const loop = deps.store.getLoop(runs[0].loopId);
2374
+ if (!loop)
2375
+ continue;
2376
+ const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
2377
+ if (retryable) {
2378
+ advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
2379
+ daemonLeaseId: deps.daemonLeaseId
2380
+ });
2381
+ continue;
2382
+ }
2383
+ for (const run of runs) {
2384
+ const current = deps.store.getLoop(run.loopId);
2385
+ if (current) {
2386
+ advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
2387
+ daemonLeaseId: deps.daemonLeaseId
2388
+ });
2389
+ }
2390
+ }
2099
2391
  }
2100
- const expired = deps.store.expireLoops(now);
2392
+ const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
2101
2393
  const claimed = [];
2102
2394
  const completed = [];
2103
2395
  const skipped = [];
@@ -2315,8 +2607,10 @@ async function runDaemon(opts = {}) {
2315
2607
  const result = await tick({
2316
2608
  store,
2317
2609
  runnerId,
2610
+ daemonLeaseId: leaseId,
2611
+ beforeRun: () => ensureLease(),
2318
2612
  execute: async (loop, run) => {
2319
- const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs / 3));
2613
+ const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
2320
2614
  const timer = setInterval(() => {
2321
2615
  try {
2322
2616
  ensureLease();
@@ -2328,8 +2622,14 @@ async function runDaemon(opts = {}) {
2328
2622
  try {
2329
2623
  const result2 = await executeLoopTarget(store, loop, run, {
2330
2624
  signal: runAbort.signal,
2331
- onSpawn: (pid) => store.markRunPid(run.id, pid, runnerId)
2625
+ beforePersist: () => ensureLease(),
2626
+ daemonLeaseId: leaseId,
2627
+ onSpawn: (pid) => {
2628
+ ensureLease();
2629
+ store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
2630
+ }
2332
2631
  });
2632
+ ensureLease();
2333
2633
  if (leaseLost)
2334
2634
  throw new Error("daemon lease lost during run");
2335
2635
  return result2;
@@ -2337,6 +2637,7 @@ async function runDaemon(opts = {}) {
2337
2637
  clearInterval(timer);
2338
2638
  }
2339
2639
  },
2640
+ beforeFinalize: () => ensureLease(),
2340
2641
  onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
2341
2642
  });
2342
2643
  const changed = result.completed.length + result.skipped.length + result.recovered.length + result.expired.length;
@@ -2467,7 +2768,7 @@ function enableStartup(result) {
2467
2768
 
2468
2769
  // src/daemon/index.ts
2469
2770
  var program = new Command;
2470
- program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.1");
2771
+ program.name("loops-daemon").description("OpenLoops daemon helper").version("0.3.3");
2471
2772
  program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
2472
2773
  program.command("start").action(async () => {
2473
2774
  const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops-daemon", args: ["run"] });