@hasna/loops 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/sdk/index.js CHANGED
@@ -258,6 +258,11 @@ function validateTarget(value, label) {
258
258
  const providers = ["claude", "cursor", "codewith", "aicopilot", "opencode", "codex"];
259
259
  if (!providers.includes(value.provider))
260
260
  throw new Error(`${label}.provider must be one of ${providers.join(", ")}`);
261
+ if (value.authProfile !== undefined) {
262
+ assertString(value.authProfile, `${label}.authProfile`);
263
+ if (value.provider !== "codewith")
264
+ throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
265
+ }
261
266
  return value;
262
267
  }
263
268
  throw new Error(`${label}.type must be command or agent`);
@@ -608,6 +613,13 @@ class Store {
608
613
  } catch {}
609
614
  this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
610
615
  }
616
+ assertDaemonLeaseFence(opts = {}, now = nowIso()) {
617
+ if (!opts.daemonLeaseId)
618
+ return;
619
+ const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
620
+ if (!row)
621
+ throw new Error("daemon lease lost");
622
+ }
611
623
  createLoop(input, from = new Date) {
612
624
  const now = nowIso();
613
625
  const loop = {
@@ -677,21 +689,31 @@ class Store {
677
689
  ORDER BY next_run_at ASC`).all(now.toISOString());
678
690
  return rows.map(rowToLoop);
679
691
  }
680
- updateLoop(id, patch) {
692
+ updateLoop(id, patch, opts = {}) {
681
693
  const current = this.getLoop(id);
682
694
  if (!current)
683
695
  throw new Error(`loop not found: ${id}`);
684
- const merged = { ...current, ...patch, updatedAt: nowIso() };
696
+ const updated = (opts.now ?? new Date).toISOString();
697
+ const merged = { ...current, ...patch, updatedAt: updated };
685
698
  this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
686
- expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
699
+ expires_at=$expiresAt, updated_at=$updated
700
+ WHERE id=$id
701
+ AND ($daemonLeaseId IS NULL OR EXISTS (
702
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
703
+ ))`).run({
687
704
  $id: id,
688
705
  $status: merged.status,
689
706
  $nextRun: merged.nextRunAt ?? null,
690
707
  $retrySlot: merged.retryScheduledFor ?? null,
691
708
  $expiresAt: merged.expiresAt ?? null,
692
- $updated: merged.updatedAt
709
+ $updated: merged.updatedAt,
710
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
711
+ $now: updated
693
712
  });
694
- return merged;
713
+ const after = this.getLoop(id);
714
+ if (!after)
715
+ throw new Error(`loop not found after update: ${id}`);
716
+ return after;
695
717
  }
696
718
  deleteLoop(idOrName) {
697
719
  const loop = this.requireLoop(idOrName);
@@ -755,11 +777,14 @@ class Store {
755
777
  const now = nowIso();
756
778
  if (input.idempotencyKey) {
757
779
  const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
758
- if (existing)
780
+ if (existing) {
781
+ this.assertDaemonLeaseFence(input);
759
782
  return rowToWorkflowRun(existing);
783
+ }
760
784
  }
761
785
  this.db.exec("BEGIN IMMEDIATE");
762
786
  try {
787
+ this.assertDaemonLeaseFence(input, now);
763
788
  if (input.idempotencyKey) {
764
789
  const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
765
790
  if (existing) {
@@ -858,31 +883,60 @@ class Store {
858
883
  const run = this.getWorkflowRun(workflowRunId);
859
884
  return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
860
885
  }
861
- startWorkflowStepRun(workflowRunId, stepId) {
862
- const now = nowIso();
863
- const res = this.db.query(`UPDATE workflow_step_runs
864
- SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
865
- pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
866
- WHERE workflow_run_id=$workflowRunId
867
- AND step_id=$stepId
868
- AND status IN ('pending', 'failed', 'timed_out')
869
- AND EXISTS (
870
- SELECT 1 FROM workflow_runs
871
- WHERE id=$workflowRunId AND status='running'
872
- )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
873
- const run = this.getWorkflowStepRun(workflowRunId, stepId);
874
- if (!run)
875
- throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
876
- if (res.changes !== 1) {
877
- throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
886
+ startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
887
+ const now = (opts.now ?? new Date).toISOString();
888
+ this.db.exec("BEGIN IMMEDIATE");
889
+ try {
890
+ const res = this.db.query(`UPDATE workflow_step_runs
891
+ SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
892
+ pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
893
+ WHERE workflow_run_id=$workflowRunId
894
+ AND step_id=$stepId
895
+ AND status IN ('pending', 'failed', 'timed_out')
896
+ AND EXISTS (
897
+ SELECT 1 FROM workflow_runs
898
+ WHERE id=$workflowRunId AND status='running'
899
+ )
900
+ AND ($daemonLeaseId IS NULL OR EXISTS (
901
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
902
+ ))`).run({
903
+ $workflowRunId: workflowRunId,
904
+ $stepId: stepId,
905
+ $started: now,
906
+ $updated: now,
907
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
908
+ $now: now
909
+ });
910
+ const run = this.getWorkflowStepRun(workflowRunId, stepId);
911
+ if (!run)
912
+ throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
913
+ if (res.changes !== 1) {
914
+ throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
915
+ }
916
+ this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
917
+ this.db.exec("COMMIT");
918
+ return run;
919
+ } catch (error) {
920
+ try {
921
+ this.db.exec("ROLLBACK");
922
+ } catch {}
923
+ throw error;
878
924
  }
879
- this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
880
- return run;
881
925
  }
882
- markWorkflowStepPid(workflowRunId, stepId, pid) {
883
- const now = nowIso();
926
+ markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
927
+ const now = (opts.now ?? new Date).toISOString();
884
928
  this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
885
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
929
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
930
+ AND ($daemonLeaseId IS NULL OR EXISTS (
931
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
932
+ ))`).run({
933
+ $workflowRunId: workflowRunId,
934
+ $stepId: stepId,
935
+ $pid: pid,
936
+ $updated: now,
937
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
938
+ $now: now
939
+ });
886
940
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
887
941
  if (!run)
888
942
  throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -910,60 +964,110 @@ class Store {
910
964
  recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
911
965
  };
912
966
  }
913
- finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
967
+ finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
914
968
  const finishedAt = patch.finishedAt ?? nowIso();
915
- const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
916
- pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
917
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
918
- $workflowRunId: workflowRunId,
919
- $stepId: stepId,
920
- $status: patch.status,
921
- $finished: finishedAt,
922
- $exitCode: patch.exitCode ?? null,
923
- $durationMs: patch.durationMs ?? null,
924
- $stdout: patch.stdout ?? null,
925
- $stderr: patch.stderr ?? null,
926
- $error: patch.error ?? null,
927
- $updated: finishedAt
928
- });
929
- if (res.changes === 1) {
930
- this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
931
- exitCode: patch.exitCode,
932
- error: patch.error
969
+ this.db.exec("BEGIN IMMEDIATE");
970
+ try {
971
+ const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
972
+ pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
973
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
974
+ AND ($daemonLeaseId IS NULL OR EXISTS (
975
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
976
+ ))`).run({
977
+ $workflowRunId: workflowRunId,
978
+ $stepId: stepId,
979
+ $status: patch.status,
980
+ $finished: finishedAt,
981
+ $exitCode: patch.exitCode ?? null,
982
+ $durationMs: patch.durationMs ?? null,
983
+ $stdout: patch.stdout ?? null,
984
+ $stderr: patch.stderr ?? null,
985
+ $error: patch.error ?? null,
986
+ $updated: finishedAt,
987
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
988
+ $now: (opts.now ?? new Date(finishedAt)).toISOString()
933
989
  });
990
+ if (res.changes === 1) {
991
+ this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
992
+ exitCode: patch.exitCode,
993
+ error: patch.error
994
+ });
995
+ }
996
+ this.db.exec("COMMIT");
997
+ } catch (error) {
998
+ try {
999
+ this.db.exec("ROLLBACK");
1000
+ } catch {}
1001
+ throw error;
934
1002
  }
935
1003
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
936
1004
  if (!run)
937
1005
  throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
938
1006
  return run;
939
1007
  }
940
- skipWorkflowStepRun(workflowRunId, stepId, reason) {
941
- const now = nowIso();
942
- const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
943
- WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
944
- if (res.changes === 1)
945
- this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
1008
+ skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
1009
+ const now = (opts.now ?? new Date).toISOString();
1010
+ this.db.exec("BEGIN IMMEDIATE");
1011
+ try {
1012
+ const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
1013
+ WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
1014
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1015
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1016
+ ))`).run({
1017
+ $workflowRunId: workflowRunId,
1018
+ $stepId: stepId,
1019
+ $finished: now,
1020
+ $error: reason,
1021
+ $updated: now,
1022
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1023
+ $now: now
1024
+ });
1025
+ if (res.changes === 1)
1026
+ this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
1027
+ this.db.exec("COMMIT");
1028
+ } catch (error) {
1029
+ try {
1030
+ this.db.exec("ROLLBACK");
1031
+ } catch {}
1032
+ throw error;
1033
+ }
946
1034
  const run = this.getWorkflowStepRun(workflowRunId, stepId);
947
1035
  if (!run)
948
1036
  throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
949
1037
  return run;
950
1038
  }
951
- finalizeWorkflowRun(workflowRunId, status, patch = {}) {
1039
+ finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
952
1040
  const finishedAt = patch.finishedAt ?? nowIso();
953
- const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
954
- WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
955
- $id: workflowRunId,
956
- $status: status,
957
- $finished: finishedAt,
958
- $durationMs: patch.durationMs ?? null,
959
- $error: patch.error ?? null,
960
- $updated: finishedAt
961
- });
1041
+ let changed = false;
1042
+ this.db.exec("BEGIN IMMEDIATE");
1043
+ try {
1044
+ const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
1045
+ WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
1046
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1047
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1048
+ ))`).run({
1049
+ $id: workflowRunId,
1050
+ $status: status,
1051
+ $finished: finishedAt,
1052
+ $durationMs: patch.durationMs ?? null,
1053
+ $error: patch.error ?? null,
1054
+ $updated: finishedAt,
1055
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1056
+ $now: (opts.now ?? new Date(finishedAt)).toISOString()
1057
+ });
1058
+ changed = res.changes === 1;
1059
+ if (changed)
1060
+ this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
1061
+ this.db.exec("COMMIT");
1062
+ } catch (error) {
1063
+ try {
1064
+ this.db.exec("ROLLBACK");
1065
+ } catch {}
1066
+ throw error;
1067
+ }
962
1068
  const run = this.getWorkflowRun(workflowRunId);
963
1069
  if (!run)
964
1070
  throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
965
- if (res.changes === 1)
966
- this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
967
1071
  return run;
968
1072
  }
969
1073
  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1017,10 +1121,24 @@ class Store {
1017
1121
  const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
1018
1122
  return (row?.count ?? 0) > 0;
1019
1123
  }
1020
- markRunPid(id, pid, claimedBy) {
1021
- const now = nowIso();
1124
+ markRunPid(id, pid, claimedBy, opts = {}) {
1125
+ const now = (opts.now ?? new Date).toISOString();
1022
1126
  const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
1023
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
1127
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy
1128
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1129
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1130
+ ))`).run({
1131
+ $id: id,
1132
+ $pid: pid,
1133
+ $updated: now,
1134
+ $claimedBy: claimedBy,
1135
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
1136
+ $now: now
1137
+ }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
1138
+ WHERE id=$id AND status='running'
1139
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1140
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1141
+ ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
1024
1142
  if (res.changes !== 1)
1025
1143
  return;
1026
1144
  return this.getRun(id);
@@ -1035,7 +1153,7 @@ class Store {
1035
1153
  AND wsr.pid IS NOT NULL`).all(loopRunId);
1036
1154
  return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
1037
1155
  }
1038
- createSkippedRun(loop, scheduledFor, reason) {
1156
+ createSkippedRun(loop, scheduledFor, reason, opts = {}) {
1039
1157
  const now = nowIso();
1040
1158
  const run = {
1041
1159
  id: genId(),
@@ -1049,21 +1167,31 @@ class Store {
1049
1167
  createdAt: now,
1050
1168
  updatedAt: now
1051
1169
  };
1052
- this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
1053
- claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
1054
- VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
1055
- NULL, NULL, $error, $created, $updated)`).run({
1056
- $id: run.id,
1057
- $loopId: run.loopId,
1058
- $loopName: run.loopName,
1059
- $scheduledFor: run.scheduledFor,
1060
- $attempt: run.attempt,
1061
- $status: run.status,
1062
- $finished: run.finishedAt ?? null,
1063
- $error: run.error ?? null,
1064
- $created: run.createdAt,
1065
- $updated: run.updatedAt
1066
- });
1170
+ this.db.exec("BEGIN IMMEDIATE");
1171
+ try {
1172
+ this.assertDaemonLeaseFence(opts, now);
1173
+ this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
1174
+ claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
1175
+ VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
1176
+ NULL, NULL, $error, $created, $updated)`).run({
1177
+ $id: run.id,
1178
+ $loopId: run.loopId,
1179
+ $loopName: run.loopName,
1180
+ $scheduledFor: run.scheduledFor,
1181
+ $attempt: run.attempt,
1182
+ $status: run.status,
1183
+ $finished: run.finishedAt ?? null,
1184
+ $error: run.error ?? null,
1185
+ $created: run.createdAt,
1186
+ $updated: run.updatedAt
1187
+ });
1188
+ this.db.exec("COMMIT");
1189
+ } catch (error) {
1190
+ try {
1191
+ this.db.exec("ROLLBACK");
1192
+ } catch {}
1193
+ throw error;
1194
+ }
1067
1195
  return this.getRunBySlot(loop.id, scheduledFor) ?? run;
1068
1196
  }
1069
1197
  getRun(id) {
@@ -1074,11 +1202,20 @@ class Store {
1074
1202
  const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
1075
1203
  return row ? rowToRun(row) : undefined;
1076
1204
  }
1077
- claimRun(loop, scheduledFor, runnerId, now = new Date) {
1205
+ nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
1206
+ const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
1207
+ WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
1208
+ ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
1209
+ WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
1210
+ ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
1211
+ return row ? rowToRun(row) : undefined;
1212
+ }
1213
+ claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
1078
1214
  const startedAt = now.toISOString();
1079
1215
  const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
1080
1216
  this.db.exec("BEGIN IMMEDIATE");
1081
1217
  try {
1218
+ this.assertDaemonLeaseFence(opts, startedAt);
1082
1219
  const existing = this.getRunBySlot(loop.id, scheduledFor);
1083
1220
  if (existing) {
1084
1221
  if (existing.status === "running") {
@@ -1173,11 +1310,15 @@ class Store {
1173
1310
  $error: patch.error ?? null,
1174
1311
  $updated: finishedAt,
1175
1312
  $claimedBy: opts.claimedBy ?? null,
1176
- $now: (opts.now ?? new Date).toISOString()
1313
+ $now: (opts.now ?? new Date).toISOString(),
1314
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1177
1315
  };
1178
1316
  const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1179
1317
  duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
1180
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1318
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
1319
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1320
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1321
+ ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
1181
1322
  duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
1182
1323
  const run = this.getRun(id);
1183
1324
  if (!run)
@@ -1186,10 +1327,20 @@ class Store {
1186
1327
  return run;
1187
1328
  return run;
1188
1329
  }
1189
- heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
1330
+ heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
1190
1331
  const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
1191
1332
  const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
1192
- WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
1333
+ WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
1334
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1335
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1336
+ ))`).run({
1337
+ $id: id,
1338
+ $claimedBy: claimedBy,
1339
+ $expires: expiresAt,
1340
+ $updated: now.toISOString(),
1341
+ $now: now.toISOString(),
1342
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1343
+ });
1193
1344
  if (res.changes !== 1)
1194
1345
  return;
1195
1346
  return this.getRun(id);
@@ -1208,7 +1359,7 @@ class Store {
1208
1359
  }
1209
1360
  return rows.map(rowToRun);
1210
1361
  }
1211
- recoverExpiredRunLeases(now = new Date) {
1362
+ recoverExpiredRunLeases(now = new Date, opts = {}) {
1212
1363
  const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
1213
1364
  const recovered = [];
1214
1365
  for (const row of rows) {
@@ -1217,20 +1368,63 @@ class Store {
1217
1368
  if (this.hasLiveWorkflowStepProcesses(row.id))
1218
1369
  continue;
1219
1370
  const finished = now.toISOString();
1220
- this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
1221
- error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
1222
- const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
1223
- for (const workflowRow of workflowRows) {
1224
- this.db.query(`UPDATE workflow_runs
1225
- SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
1226
- WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
1227
- this.db.query(`UPDATE workflow_step_runs
1228
- SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
1229
- WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
1230
- this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
1231
- error: "parent loop run lease expired before completion",
1232
- loopRunId: row.id
1371
+ this.db.exec("BEGIN IMMEDIATE");
1372
+ try {
1373
+ const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
1374
+ error='run lease expired before completion', updated_at=$updated
1375
+ WHERE id=$id AND status='running' AND lease_expires_at <= $now
1376
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1377
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1378
+ ))`).run({
1379
+ $id: row.id,
1380
+ $finished: finished,
1381
+ $updated: finished,
1382
+ $now: finished,
1383
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1233
1384
  });
1385
+ if (res.changes !== 1) {
1386
+ this.db.exec("COMMIT");
1387
+ continue;
1388
+ }
1389
+ const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
1390
+ for (const workflowRow of workflowRows) {
1391
+ const workflowRes = this.db.query(`UPDATE workflow_runs
1392
+ SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
1393
+ WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
1394
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1395
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1396
+ ))`).run({
1397
+ $id: workflowRow.id,
1398
+ $finished: finished,
1399
+ $updated: finished,
1400
+ $now: finished,
1401
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1402
+ });
1403
+ if (workflowRes.changes !== 1)
1404
+ continue;
1405
+ this.db.query(`UPDATE workflow_step_runs
1406
+ SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
1407
+ WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
1408
+ AND ($daemonLeaseId IS NULL OR EXISTS (
1409
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
1410
+ ))`).run({
1411
+ $workflowRunId: workflowRow.id,
1412
+ $finished: finished,
1413
+ $updated: finished,
1414
+ $now: finished,
1415
+ $daemonLeaseId: opts.daemonLeaseId ?? null
1416
+ });
1417
+ this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
1418
+ error: "parent loop run lease expired before completion",
1419
+ loopRunId: row.id
1420
+ });
1421
+ }
1422
+ this.db.exec("COMMIT");
1423
+ } catch (error) {
1424
+ try {
1425
+ this.db.exec("ROLLBACK");
1426
+ } catch {}
1427
+ throw error;
1234
1428
  }
1235
1429
  const run = this.getRun(row.id);
1236
1430
  if (run)
@@ -1238,11 +1432,14 @@ class Store {
1238
1432
  }
1239
1433
  return recovered;
1240
1434
  }
1241
- expireLoops(now = new Date) {
1435
+ expireLoops(now = new Date, opts = {}) {
1242
1436
  const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
1243
1437
  const expired = [];
1244
- for (const row of rows)
1245
- expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
1438
+ for (const row of rows) {
1439
+ const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
1440
+ if (updated.status === "expired")
1441
+ expired.push(updated);
1442
+ }
1246
1443
  return expired;
1247
1444
  }
1248
1445
  countLoops(status) {
@@ -1285,7 +1482,7 @@ class Store {
1285
1482
  }
1286
1483
  heartbeatDaemonLease(id, ttlMs, now = new Date) {
1287
1484
  const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
1288
- const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
1485
+ const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
1289
1486
  if (res.changes !== 1)
1290
1487
  return;
1291
1488
  return this.getDaemonLease();
@@ -1535,7 +1732,7 @@ function agentArgs(target) {
1535
1732
  args.push("--model", target.model);
1536
1733
  if (target.agent)
1537
1734
  args.push("--agent", target.agent);
1538
- args.push(...target.extraArgs ?? [], target.prompt);
1735
+ args.push(...target.extraArgs ?? []);
1539
1736
  return args;
1540
1737
  case "cursor":
1541
1738
  args.push("-p");
@@ -1543,10 +1740,10 @@ function agentArgs(target) {
1543
1740
  args.push("--model", target.model);
1544
1741
  if (target.agent)
1545
1742
  args.push("--agent", target.agent);
1546
- args.push(...target.extraArgs ?? [], target.prompt);
1743
+ args.push(...target.extraArgs ?? []);
1547
1744
  return args;
1548
1745
  case "codewith":
1549
- args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
1746
+ args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
1550
1747
  if (isolation === "safe")
1551
1748
  args.push("--ignore-rules");
1552
1749
  if (target.cwd)
@@ -1555,7 +1752,7 @@ function agentArgs(target) {
1555
1752
  args.push("--model", target.model);
1556
1753
  if (target.agent)
1557
1754
  args.push("--agent", target.agent);
1558
- args.push(...target.extraArgs ?? [], target.prompt);
1755
+ args.push(...target.extraArgs ?? []);
1559
1756
  return args;
1560
1757
  case "codex":
1561
1758
  args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
@@ -1565,7 +1762,7 @@ function agentArgs(target) {
1565
1762
  args.push("--cd", target.cwd);
1566
1763
  if (target.model)
1567
1764
  args.push("--model", target.model);
1568
- args.push(...target.extraArgs ?? [], target.prompt);
1765
+ args.push(...target.extraArgs ?? []);
1569
1766
  return args;
1570
1767
  case "aicopilot":
1571
1768
  args.push("run", "--format", "json");
@@ -1577,7 +1774,7 @@ function agentArgs(target) {
1577
1774
  args.push("--model", target.model);
1578
1775
  if (target.agent)
1579
1776
  args.push("--agent", target.agent);
1580
- args.push(...target.extraArgs ?? [], target.prompt);
1777
+ args.push(...target.extraArgs ?? []);
1581
1778
  return args;
1582
1779
  case "opencode":
1583
1780
  args.push("run", "--format", "json");
@@ -1589,7 +1786,7 @@ function agentArgs(target) {
1589
1786
  args.push("--model", target.model);
1590
1787
  if (target.agent)
1591
1788
  args.push("--agent", target.agent);
1592
- args.push(...target.extraArgs ?? [], target.prompt);
1789
+ args.push(...target.extraArgs ?? []);
1593
1790
  return args;
1594
1791
  }
1595
1792
  }
@@ -1614,7 +1811,8 @@ function commandSpec(target) {
1614
1811
  cwd: agentTarget.cwd,
1615
1812
  timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
1616
1813
  account: agentTarget.account,
1617
- accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
1814
+ accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
1815
+ stdin: agentTarget.prompt
1618
1816
  };
1619
1817
  }
1620
1818
  function executionEnv(spec, metadata, opts) {
@@ -1683,10 +1881,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
1683
1881
  env,
1684
1882
  shell: spec.shell ?? false,
1685
1883
  detached: true,
1686
- stdio: ["ignore", "pipe", "pipe"]
1884
+ stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
1687
1885
  });
1688
1886
  if (child.pid)
1689
1887
  opts.onSpawn?.(child.pid);
1888
+ if (spec.stdin !== undefined && child.stdin) {
1889
+ child.stdin.on("error", (err) => {
1890
+ if (err.code !== "EPIPE")
1891
+ error = err.message;
1892
+ });
1893
+ child.stdin.end(spec.stdin);
1894
+ }
1690
1895
  const abortHandler = () => {
1691
1896
  error = "cancelled";
1692
1897
  if (child.pid)
@@ -1695,10 +1900,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
1695
1900
  if (opts.signal?.aborted)
1696
1901
  abortHandler();
1697
1902
  opts.signal?.addEventListener("abort", abortHandler, { once: true });
1698
- child.stdout.on("data", (chunk) => {
1903
+ child.stdout?.on("data", (chunk) => {
1699
1904
  stdout = appendBounded(stdout, chunk, maxOutputBytes);
1700
1905
  });
1701
- child.stderr.on("data", (chunk) => {
1906
+ child.stderr?.on("data", (chunk) => {
1702
1907
  stderr = appendBounded(stderr, chunk, maxOutputBytes);
1703
1908
  });
1704
1909
  const timer = setTimeout(() => {
@@ -1797,7 +2002,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1797
2002
  loop: opts.loop,
1798
2003
  loopRun: opts.loopRun,
1799
2004
  scheduledFor: opts.scheduledFor,
1800
- idempotencyKey: opts.idempotencyKey
2005
+ idempotencyKey: opts.idempotencyKey,
2006
+ daemonLeaseId: opts.daemonLeaseId
1801
2007
  });
1802
2008
  const startedAt = run.startedAt ?? nowIso();
1803
2009
  if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
@@ -1831,12 +2037,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
1831
2037
  return !dependencyStep?.continueOnFailure;
1832
2038
  });
1833
2039
  if (blockedBy) {
1834
- store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
2040
+ opts.beforePersist?.();
2041
+ store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
1835
2042
  blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
1836
2043
  terminalStatus = "failed";
1837
2044
  continue;
1838
2045
  }
1839
- const startedStep = store.startWorkflowStepRun(run.id, step.id);
2046
+ opts.beforePersist?.();
2047
+ const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
1840
2048
  if (startedStep.status !== "running") {
1841
2049
  terminalStatus = "failed";
1842
2050
  blockingError = `step ${step.id} could not start because workflow is no longer running`;
@@ -1868,7 +2076,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1868
2076
  ...opts,
1869
2077
  signal: controller.signal,
1870
2078
  onSpawn: (pid) => {
1871
- store.markWorkflowStepPid(run.id, step.id, pid);
2079
+ opts.beforePersist?.();
2080
+ store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
1872
2081
  opts.onSpawn?.(pid);
1873
2082
  }
1874
2083
  });
@@ -1896,6 +2105,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
1896
2105
  blockingError = "workflow run was cancelled";
1897
2106
  break;
1898
2107
  }
2108
+ opts.beforePersist?.();
1899
2109
  store.finalizeWorkflowStepRun(run.id, step.id, {
1900
2110
  status: result.status,
1901
2111
  finishedAt: result.finishedAt,
@@ -1904,6 +2114,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
1904
2114
  stderr: result.stderr,
1905
2115
  exitCode: result.exitCode,
1906
2116
  error: result.error
2117
+ }, {
2118
+ daemonLeaseId: opts.daemonLeaseId
1907
2119
  });
1908
2120
  if (result.status !== "succeeded" && !step.continueOnFailure) {
1909
2121
  terminalStatus = result.status;
@@ -1915,7 +2127,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
1915
2127
  for (const step of ordered) {
1916
2128
  const existing = store.getWorkflowStepRun(run.id, step.id);
1917
2129
  if (existing?.status === "pending" || existing?.status === "running") {
1918
- store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
2130
+ store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
2131
+ daemonLeaseId: opts.daemonLeaseId
2132
+ });
1919
2133
  }
1920
2134
  }
1921
2135
  }
@@ -1925,10 +2139,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
1925
2139
  const steps2 = store.listWorkflowStepRuns(run.id);
1926
2140
  return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
1927
2141
  }
2142
+ opts.beforePersist?.();
1928
2143
  const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
1929
2144
  finishedAt,
1930
2145
  durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
1931
2146
  error: blockingError
2147
+ }, {
2148
+ daemonLeaseId: opts.daemonLeaseId
1932
2149
  });
1933
2150
  const steps = store.listWorkflowStepRuns(run.id);
1934
2151
  return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
@@ -1976,52 +2193,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
1976
2193
 
1977
2194
  // src/lib/scheduler.ts
1978
2195
  function manualRunScheduledFor(loop, now = new Date) {
1979
- if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
2196
+ if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
1980
2197
  return loop.retryScheduledFor ?? loop.nextRunAt;
1981
2198
  }
1982
2199
  return now.toISOString();
1983
2200
  }
1984
2201
  function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
2202
+ if (loop.status !== "active")
2203
+ return false;
1985
2204
  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
1986
2205
  return false;
1987
2206
  return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
1988
2207
  }
2208
+ function manualRunSource(loop, scheduledFor, now = new Date) {
2209
+ if (loop.status !== "active")
2210
+ return "ad_hoc";
2211
+ if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
2212
+ return "ad_hoc";
2213
+ if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
2214
+ return "retry_slot";
2215
+ return "due_slot";
2216
+ }
1989
2217
  function nextAfterRetry(loop, now) {
1990
2218
  return new Date(now.getTime() + loop.retryDelayMs).toISOString();
1991
2219
  }
1992
- function advanceLoop(store, loop, run, finishedAt, succeeded) {
2220
+ function isDaemonLeaseLost(error) {
2221
+ return error instanceof Error && error.message === "daemon lease lost";
2222
+ }
2223
+ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
1993
2224
  if (run.status === "running")
1994
2225
  return;
1995
2226
  const current = store.getLoop(loop.id);
1996
2227
  if (!current || current.status !== "active")
1997
2228
  return;
1998
- const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
2229
+ if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
2230
+ return;
2231
+ const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
1999
2232
  if (shouldRetry) {
2000
- store.updateLoop(loop.id, {
2233
+ store.updateLoop(current.id, {
2001
2234
  status: "active",
2002
- nextRunAt: nextAfterRetry(loop, finishedAt),
2235
+ nextRunAt: nextAfterRetry(current, finishedAt),
2003
2236
  retryScheduledFor: run.scheduledFor
2004
- });
2237
+ }, { daemonLeaseId: opts.daemonLeaseId });
2005
2238
  return;
2006
2239
  }
2007
- const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
2008
- store.updateLoop(loop.id, {
2240
+ const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
2241
+ if (deferredRetry) {
2242
+ store.updateLoop(current.id, {
2243
+ status: "active",
2244
+ nextRunAt: nextAfterRetry(current, finishedAt),
2245
+ retryScheduledFor: deferredRetry.scheduledFor
2246
+ }, { daemonLeaseId: opts.daemonLeaseId });
2247
+ return;
2248
+ }
2249
+ const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
2250
+ store.updateLoop(current.id, {
2009
2251
  status: nextRunAt ? "active" : "stopped",
2010
2252
  nextRunAt,
2011
2253
  retryScheduledFor: undefined
2012
- });
2254
+ }, { daemonLeaseId: opts.daemonLeaseId });
2013
2255
  }
2014
2256
  async function executeClaimedRun(deps) {
2015
2257
  let heartbeat;
2016
2258
  const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
2017
2259
  heartbeat = setInterval(() => {
2018
- deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
2260
+ deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
2261
+ daemonLeaseId: deps.daemonLeaseId
2262
+ });
2019
2263
  }, heartbeatEveryMs);
2020
2264
  heartbeat.unref();
2021
2265
  try {
2022
2266
  const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
2023
- onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
2267
+ daemonLeaseId: deps.daemonLeaseId,
2268
+ onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
2024
2269
  })))(deps.loop, deps.run);
2270
+ deps.beforeFinalize?.(deps.loop, deps.run);
2025
2271
  return deps.store.finalizeRun(deps.run.id, {
2026
2272
  status: result.status,
2027
2273
  finishedAt: result.finishedAt,
@@ -2033,10 +2279,16 @@ async function executeClaimedRun(deps) {
2033
2279
  pid: result.pid
2034
2280
  }, {
2035
2281
  claimedBy: deps.runnerId,
2282
+ daemonLeaseId: deps.daemonLeaseId,
2036
2283
  now: deps.now?.() ?? new Date(result.finishedAt)
2037
2284
  });
2038
2285
  } catch (err) {
2039
2286
  deps.onError?.(deps.loop, err);
2287
+ try {
2288
+ deps.beforeFinalize?.(deps.loop, deps.run);
2289
+ } catch {
2290
+ return deps.store.getRun(deps.run.id) ?? deps.run;
2291
+ }
2040
2292
  const finishedAt = new Date;
2041
2293
  return deps.store.finalizeRun(deps.run.id, {
2042
2294
  status: "failed",
@@ -2047,6 +2299,7 @@ async function executeClaimedRun(deps) {
2047
2299
  error: err instanceof Error ? err.message : String(err)
2048
2300
  }, {
2049
2301
  claimedBy: deps.runnerId,
2302
+ daemonLeaseId: deps.daemonLeaseId,
2050
2303
  now: deps.now?.() ?? finishedAt
2051
2304
  });
2052
2305
  } finally {
@@ -2056,15 +2309,33 @@ async function executeClaimedRun(deps) {
2056
2309
  }
2057
2310
  async function runSlot(deps, loop, scheduledFor) {
2058
2311
  const now = deps.now?.() ?? new Date;
2312
+ deps.beforeRun?.(loop, scheduledFor);
2059
2313
  if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
2060
- const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
2061
- advanceLoop(deps.store, loop, skipped, now, true);
2314
+ let skipped;
2315
+ try {
2316
+ skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
2317
+ daemonLeaseId: deps.daemonLeaseId
2318
+ });
2319
+ } catch (error) {
2320
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
2321
+ return;
2322
+ throw error;
2323
+ }
2324
+ advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
2062
2325
  deps.onRun?.(skipped);
2063
2326
  return skipped;
2064
2327
  }
2065
- const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
2328
+ let claim;
2329
+ try {
2330
+ claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
2331
+ } catch (error) {
2332
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
2333
+ return;
2334
+ throw error;
2335
+ }
2066
2336
  if (!claim)
2067
2337
  return;
2338
+ deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
2068
2339
  deps.onRun?.(claim.run);
2069
2340
  const finalRun = await executeClaimedRun({
2070
2341
  store: deps.store,
@@ -2073,21 +2344,42 @@ async function runSlot(deps, loop, scheduledFor) {
2073
2344
  run: claim.run,
2074
2345
  now: deps.now,
2075
2346
  execute: deps.execute,
2347
+ beforeFinalize: deps.beforeFinalize,
2348
+ daemonLeaseId: deps.daemonLeaseId,
2076
2349
  onError: deps.onError
2077
2350
  });
2078
- advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
2351
+ advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
2079
2352
  deps.onRun?.(finalRun);
2080
2353
  return finalRun;
2081
2354
  }
2082
2355
  async function tick(deps) {
2083
2356
  const now = deps.now?.() ?? new Date;
2084
- const recovered = deps.store.recoverExpiredRunLeases(now);
2357
+ const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
2358
+ const recoveredByLoop = new Map;
2085
2359
  for (const run of recovered) {
2086
- const loop = deps.store.getLoop(run.loopId);
2087
- if (loop)
2088
- advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
2360
+ recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
2089
2361
  }
2090
- const expired = deps.store.expireLoops(now);
2362
+ for (const runs of recoveredByLoop.values()) {
2363
+ const loop = deps.store.getLoop(runs[0].loopId);
2364
+ if (!loop)
2365
+ continue;
2366
+ const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
2367
+ if (retryable) {
2368
+ advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
2369
+ daemonLeaseId: deps.daemonLeaseId
2370
+ });
2371
+ continue;
2372
+ }
2373
+ for (const run of runs) {
2374
+ const current = deps.store.getLoop(run.loopId);
2375
+ if (current) {
2376
+ advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
2377
+ daemonLeaseId: deps.daemonLeaseId
2378
+ });
2379
+ }
2380
+ }
2381
+ }
2382
+ const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
2091
2383
  const claimed = [];
2092
2384
  const completed = [];
2093
2385
  const skipped = [];
@@ -2153,9 +2445,17 @@ class LoopsClient {
2153
2445
  async runNow(idOrName) {
2154
2446
  const loop = this.get(idOrName);
2155
2447
  const now = new Date;
2156
- const scheduledFor = manualRunScheduledFor(loop, now);
2157
- const shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
2158
- const claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
2448
+ let scheduledFor = manualRunScheduledFor(loop, now);
2449
+ let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
2450
+ let claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
2451
+ if (!claim && shouldAdvance) {
2452
+ const existing = this.store.getRunBySlot(loop.id, scheduledFor);
2453
+ if (existing && existing.status !== "running") {
2454
+ scheduledFor = now.toISOString();
2455
+ shouldAdvance = false;
2456
+ claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
2457
+ }
2458
+ }
2159
2459
  if (!claim)
2160
2460
  throw new Error(`could not claim manual run for ${idOrName}`);
2161
2461
  const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });