@alphacoder-v0/pi-loops 0.22.7 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +681 -0
  2. package/CONTRACT.md +7 -7
  3. package/README.md +91 -63
  4. package/README.zh-CN.md +59 -39
  5. package/docs/cli.md +272 -83
  6. package/docs/configuration.md +86 -24
  7. package/docs/design.md +395 -56
  8. package/docs/downstream.md +2 -2
  9. package/docs/goal.md +12 -3
  10. package/docs/hooks.md +6 -7
  11. package/docs/loops.md +288 -59
  12. package/docs/mcp.md +22 -14
  13. package/docs/recipes.md +6 -0
  14. package/docs/screenshot.png +0 -0
  15. package/docs/triggers.md +120 -26
  16. package/docs/troubleshooting.md +37 -26
  17. package/docs/web-ui-parity.md +691 -65
  18. package/package.json +4 -3
  19. package/recipes/autoresearch/recipe.toml +1 -0
  20. package/recipes/changelog-draft/recipe.toml +1 -0
  21. package/recipes/ci-sweeper/recipe.toml +1 -0
  22. package/recipes/daily-digest/recipe.toml +1 -0
  23. package/recipes/deps-sweeper/recipe.toml +1 -0
  24. package/recipes/ecosystem/recipe.toml +1 -0
  25. package/recipes/issue-loop/recipe.toml +1 -0
  26. package/recipes/pr-watch/recipe.toml +1 -0
  27. package/skills/pi-loops/SKILL.md +6 -4
  28. package/src/activity.ts +199 -0
  29. package/src/archive.ts +21 -9
  30. package/src/args.ts +6 -2
  31. package/src/cli-entry.mjs +12 -3
  32. package/src/cli.ts +746 -125
  33. package/src/config.ts +53 -5
  34. package/src/conversation-rows.ts +469 -0
  35. package/src/danger.ts +227 -6
  36. package/src/engine.ts +1240 -0
  37. package/src/goal.ts +71 -0
  38. package/src/held-read.ts +27 -0
  39. package/src/hooks.ts +49 -18
  40. package/src/host-control-channel.ts +455 -40
  41. package/src/host-control.ts +602 -111
  42. package/src/host-entry.mjs +5 -3
  43. package/src/host-runtime.ts +455 -41
  44. package/src/host.ts +659 -123
  45. package/src/inbox-card.ts +278 -0
  46. package/src/inbox.ts +139 -62
  47. package/src/job-edit.ts +7 -1
  48. package/src/job-health.ts +14 -2
  49. package/src/job-lines.ts +201 -13
  50. package/src/job-owner.ts +74 -13
  51. package/src/job-tune.ts +110 -0
  52. package/src/lease.ts +537 -0
  53. package/src/lock.ts +7 -2
  54. package/src/mcp-pool.ts +66 -14
  55. package/src/mcp.ts +1 -1
  56. package/src/open-file.ts +54 -0
  57. package/src/page-children.ts +1507 -0
  58. package/src/page-devices.ts +192 -0
  59. package/src/page-server.ts +3150 -0
  60. package/src/path-precheck.ts +215 -0
  61. package/src/paths.ts +24 -0
  62. package/src/pi-floor.ts +10 -0
  63. package/src/pi-loops.ts +1301 -450
  64. package/src/pi-resolver.d.mts +1 -0
  65. package/src/pi-resolver.mjs +110 -10
  66. package/src/presence.ts +12 -63
  67. package/src/protocol.ts +10 -1
  68. package/src/recipe.ts +43 -1
  69. package/src/requests.ts +354 -0
  70. package/src/rule-lines.ts +68 -0
  71. package/src/run-card.ts +105 -0
  72. package/src/runner.ts +18 -1
  73. package/src/schedule.ts +1 -2
  74. package/src/scheduler.ts +598 -245
  75. package/src/sdk-runner.ts +321 -27
  76. package/src/seen.ts +278 -0
  77. package/src/session-head.ts +21 -0
  78. package/src/share.ts +12 -2
  79. package/src/snapshot.ts +3 -2
  80. package/src/store.ts +81 -8
  81. package/src/tools.ts +196 -65
  82. package/src/trigger-runtime.ts +164 -127
  83. package/src/triggers.ts +39 -7
  84. package/src/trust.ts +46 -2
  85. package/src/ui-prefs.ts +16 -5
  86. package/src/watch-plan.ts +174 -0
  87. package/src/web.d.mts +3 -0
  88. package/src/web.mjs +2260 -2340
  89. package/src/window-executor.ts +459 -0
  90. package/src/window-guard.ts +158 -0
  91. package/docs/agents/issue-tracker.md +0 -34
package/src/scheduler.ts CHANGED
@@ -1,36 +1,34 @@
1
1
  /**
2
- * The heartbeat. One pi process on the machine owns the timer at a time
3
- * (leadership lives in scheduler.json with a heartbeat); every other process
4
- * with this extension stands by and takes over when the owner exits or dies.
5
- * Loop jobs are therefore host-agnostic: close the pi that created a loop and
6
- * any other open pi keeps it ticking; restart pi and missed ticks are caught up
7
- * (once, collapsed) unless the job opted out.
8
- *
9
- * Inject-mode jobs are session-bound: they only fire from the process whose
10
- * current session is the one that created them.
2
+ * The clock. The same scheduler runs in two roles, and only what it owns differs: the always-on
3
+ * host runs every loop, and a pi window runs the plain jobs of the conversation whose lease it
4
+ * holds (src/lease.ts). Nobody is elected. A missed tick is caught up once, collapsed, unless the
5
+ * job opted out.
11
6
  */
12
7
  import * as fs from "node:fs";
13
8
  import * as os from "node:os";
14
9
  import { PresenceRegistry, type PresenceEntry, type PresenceKind, type PresenceSelf } from "./presence.ts";
15
10
  import * as path from "node:path";
16
11
  import { Inbox } from "./inbox.ts";
17
- import { pidAlive, withFileLock, writeFileAtomic } from "./lock.ts";
12
+ import { pidAlive, writeFileAtomic } from "./lock.ts";
18
13
  import { composeCheckerPrompt, composeLoopPrompt, findingKind, parseCheckerOutput, parseRunOutput, stripProtocolTags } from "./protocol.ts";
19
14
  import { type RunnerResult, type SubagentRunner, failedRun } from "./runner.ts";
20
15
  import { previewRedacted, redact } from "./redact.ts";
21
16
  import { type SubagentSlot, SubagentSlots } from "./slots.ts";
22
17
  import { backoffWaitMs, computeDue, computeNext, formatLocal, formatLocalZoned, formatSchedule, isValidSchedule, stamp } from "./schedule.ts";
23
- import { type CheckerRecord, JobStore, type LoopJob, type RunRecord, newId } from "./store.ts";
24
- import { ownerSession, runsIn } from "./job-owner.ts";
18
+ import { type CheckerRecord, JobStore, type LoopJob, type RunRecord, type RunRequester, newId } from "./store.ts";
19
+ import { openWhere, ownerSession, runsHere, whereRuns, type OwnerView } from "./job-owner.ts";
20
+ import { shortSessionId } from "./session-head.ts";
21
+ import { DEAD_AFTER_ATTEMPTS, backoffFor, plainCollapseKey, plainJobRow, plainRunId, sessionFileHas } from "./engine.ts";
22
+ import { RequestStore, type SessionRequest } from "./requests.ts";
23
+ import { PI_LOOPS_VERSION } from "./version.ts";
25
24
 
26
25
  export const DEFAULT_TICK_MS = 30_000;
27
- export const LEADER_STALE_MS = 90_000;
26
+ /** Interrupted runs in a row (a process that died mid-run) after which a job is parked. */
27
+ export const INTERRUPTIONS_BEFORE_PARKING = 3;
28
28
  /** Consecutive failures before a job is retried on a widening gap rather than every due tick. */
29
29
  export const FAILURE_BACKOFF_AFTER = 3;
30
30
  export const FAILURE_BACKOFF_BASE_MS = 5 * 60_000;
31
31
  export const FAILURE_BACKOFF_MAX_MS = 6 * 60 * 60_000;
32
- /** A run claimed by another machine (shared $HOME) is only assumed dead after this long. */
33
- export const FOREIGN_RUN_STALE_MS = 24 * 60 * 60 * 1000;
34
32
  export const DEFAULT_RUN_TIMEOUT_MS = 15 * 60_000;
35
33
  export const MAX_CONCURRENT_RUNS = 3;
36
34
  /**
@@ -47,6 +45,8 @@ export interface SessionSnapshot {
47
45
  thinking?: string;
48
46
  /** The interactive pi trusted this project (`pi --approve` / trust prompt); children in the same cwd inherit it. */
49
47
  trusted?: boolean;
48
+ /** The built-in tools active in this chat (`--tools`, `--exclude-tools`); a loop made here keeps them. */
49
+ builtinTools?: string[];
50
50
  }
51
51
 
52
52
  export interface SchedulerSettings {
@@ -57,7 +57,7 @@ export interface SchedulerSettings {
57
57
  maxConcurrentRuns: number;
58
58
  /** Stop dispatching once today's automation has cost this much. 0 or absent = no cap. */
59
59
  dailyBudgetUsd?: number;
60
- /** Global switch for firing ticks missed while no pi was open (`[cron] catch_up`); per-job `catchUp` still applies. */
60
+ /** Global switch for firing ticks missed while this scheduler was not running (`[cron] catch_up`); per-job `catchUp` still applies. */
61
61
  catchUp: boolean;
62
62
  }
63
63
 
@@ -69,21 +69,28 @@ export interface RunOutcome {
69
69
  }
70
70
 
71
71
  export interface SchedulerHooks {
72
- /** Inject-mode job is due: deliver `prompt` to the current session. `runId` is the id in its `[Trigger …]` prefix. */
73
- onInject?: (job: LoopJob, prompt: string, runId: string) => void | Promise<void>;
72
+ /**
73
+ * The host: run `prompt` as a turn in the closed conversation it now holds. Resolves when the turn
74
+ * is over with the run's result; `preempted` when a window took the conversation back before the
75
+ * turn completed (a turn that completed is done, whoever holds the lease now). `landed` says
76
+ * whether the prompt's `[Trigger <id>]` entry is in the conversation; absent, a completed turn is
77
+ * taken to have landed it.
78
+ */
79
+ onHeldTurn?: (job: LoopJob, prompt: string, runId: string, held: HeldSession, signal: AbortSignal) => Promise<{ preempted: boolean; result: RunnerResult; landed?: boolean }>;
74
80
  onRunStart?: (job: LoopJob, runId: string) => void;
75
- /** A run is being fired for a tick that was missed while no pi was open. */
81
+ /** A run is being fired for a tick that was missed while this scheduler was not running. */
76
82
  onCatchUp?: (job: LoopJob, dueAt: number) => void;
77
83
  onRunFinished?: (outcome: RunOutcome) => void;
78
84
  /** A one-shot removed itself: the control plane records it the way a person's removal is recorded. */
79
85
  onJobRetired?: (job: LoopJob, reason: string) => void;
86
+ /** A plain job's row this process wrote into its conversation's queue (the clock's, or `/cron run`'s): the window records it as its own, and says so when it waits. */
87
+ onRowWritten?: (row: SessionRequest, job: LoopJob) => void;
80
88
  onInboxChanged?: () => void;
81
- /** Every tick, after leadership is settled. Other subsystems (dynamic triggers) piggyback on it. */
82
- onTick?: (now: number, leader: boolean) => void | Promise<void>;
83
- /** Leadership gained or lost. */
84
- onLeadership?: (leader: boolean) => void | Promise<void>;
89
+ /** Every tick, before dispatch. Other subsystems (leases, dynamic triggers) piggyback on it. */
90
+ onTick?: (now: number) => void | Promise<void>;
85
91
  /** A whole tick failed: nothing ran. A failure that needs a person, where `log` is a diagnostic at either level. */
86
- onSchedulerError?: (message: string) => void;
92
+ /** `job` is the one the message is about, when there is one (its cwd decides which windows hear it). */
93
+ onSchedulerError?: (message: string, job?: Pick<LoopJob, "cwd">) => void;
87
94
  /** Today's spend passed the configured cap; nothing more will be dispatched today. */
88
95
  onBudgetExceeded?: (spent: number, cap: number) => void;
89
96
  /**
@@ -92,7 +99,46 @@ export interface SchedulerHooks {
92
99
  * a regex over these strings. Absent `level` means `"warning"`: a line that does not say it is
93
100
  * routine is not.
94
101
  */
95
- log?: (message: string, level?: "info" | "warning") => void;
102
+ /** `job` is the one the line is about, so a window of its project can say it (src/host-control-channel.ts Notice). */
103
+ log?: (message: string, level?: "info" | "warning", job?: Pick<LoopJob, "cwd">) => void;
104
+ }
105
+
106
+ /**
107
+ * The lease side of plain-job ownership (src/lease.ts): a plain job fires only in the instance that
108
+ * holds its session's lease, and its claim re-checks that the lease has not moved since.
109
+ */
110
+ export interface PlainJobLeases {
111
+ /** The epoch of this instance's lease on `sessionId`, when it holds it. */
112
+ heldEpoch(sessionId: string): number | undefined;
113
+ /** The pid of the live holder of `sessionId`'s lease, when there is one. */
114
+ holderOf(sessionId: string): number | undefined;
115
+ /** The kind of the live holder, when there is one. */
116
+ holderKind?(sessionId: string): "terminal" | "page" | "host" | undefined;
117
+ /** Windows: whether this one holds its conversation's lease but stands aside (OwnerView.standsAside). */
118
+ standsAside?(): boolean;
119
+ /** Windows: whether the host fires this closed conversation's plain jobs (it runs, and knows the file). */
120
+ hostRuns?(sessionId: string): boolean;
121
+ /** Why no host runs anything right now, when that is so (see OwnerView.hostPaused). */
122
+ hostPaused?(): string | undefined;
123
+ /** The host: take a closed conversation for one turn (src/lease.ts acquireForHost), or undefined. */
124
+ acquireForWork?(sessionId: string, runId: string, jobId: string): HeldSession | undefined;
125
+ /** The host: give the conversation back after the turn. A window that took it meanwhile keeps it. */
126
+ releaseWork?(sessionId: string, epoch: number): void;
127
+ }
128
+
129
+ /**
130
+ * The `AbortSignal.reason` of a run a person stopped: `/triggers abort`, `pi-loops
131
+ * abort`, a stop row. A held turn stopped so is done, not handed back: no successor row, no owed slot.
132
+ * A quit, an upgrade, a preemption or the budget aborts with no reason and hands the turn back.
133
+ */
134
+ export const STOPPED_BY_PERSON = "stopped by a person";
135
+
136
+ /** A closed conversation the host holds for one turn. */
137
+ export interface HeldSession {
138
+ sessionId: string;
139
+ epoch: number;
140
+ sessionFile: string;
141
+ cwd?: string;
96
142
  }
97
143
 
98
144
  export interface SchedulerOptions {
@@ -107,16 +153,27 @@ export interface SchedulerOptions {
107
153
  hop?: number;
108
154
  /** Live settings (re-read on every use so a config reload takes effect). */
109
155
  getSettings?: () => SchedulerSettings;
156
+ /** A window: whether it should write next-runs.json this tick (no host is running to do it). */
157
+ writesNextRuns?: () => boolean;
110
158
  /** Does pi still have this session? Plain jobs of deleted sessions are disabled (and `gc()` removes them). */
111
159
  sessionExists?: (sessionId: string) => boolean;
112
- /** "host" = the headless keeper of the clock; any interactive pi preempts it. Default "interactive". */
160
+ /** "host" = the always-on process: it also writes next-runs.json and sweeps dead sessions. Default "interactive". */
113
161
  kind?: PresenceKind;
162
+ /**
163
+ * Which jobs this scheduler dispatches. The host passes "loops only", a window "the plain jobs of
164
+ * the conversation it holds". Omitted (tests), it owns every loop and the plain jobs `view` says run here.
165
+ */
166
+ owns?: (job: LoopJob, view: OwnerView) => boolean;
114
167
  /**
115
168
  * The process-wide sub-agent admission counter, shared with the trigger runtime and the /goal
116
169
  * evaluator (src/slots.ts). Omitted, the scheduler bounds only itself against
117
170
  * `maxConcurrentRuns` — which is the bug this exists to close, so production always passes one.
118
171
  */
119
172
  slots?: SubagentSlots;
173
+ /** Omitted, the session of `getSession()` counts as held by this instance (tests, the host, which has none). */
174
+ leases?: PlainJobLeases;
175
+ /** The extension instance this scheduler runs for: the same id its session lease names. */
176
+ instance?: string;
120
177
  }
121
178
 
122
179
  export const DEAD_SESSION_MARKER = "no longer exists (/cron gc removes it)";
@@ -130,23 +187,12 @@ const DEAD_SESSION_SCAN_MS = 10 * 60_000;
130
187
  */
131
188
  const CWD_GRACE_MS = 30 * 60_000;
132
189
 
133
- interface LeaderRecord {
134
- pid: number;
135
- host: string;
136
- /** Distinguishes scheduler instances inside one process (reload, tests). */
137
- instance: string;
138
- sessionId?: string;
139
- kind?: PresenceKind;
140
- startedAt: string;
141
- heartbeatAt: string;
142
- }
143
-
144
190
  export class LoopScheduler {
145
191
  readonly store: JobStore;
146
192
  readonly inbox: Inbox;
193
+ /** The conversations' request queues: a plain job's prompt is a row here before its slot is spent. */
194
+ readonly requests: RequestStore;
147
195
  private readonly dir: string;
148
- private readonly leaderFile: string;
149
- private readonly leaderLock: string;
150
196
  private readonly getSession: () => SessionSnapshot;
151
197
  private readonly hooks: SchedulerHooks;
152
198
  private readonly tickMs: number;
@@ -154,6 +200,7 @@ export class LoopScheduler {
154
200
  private readonly now: () => number;
155
201
  private readonly hop: number;
156
202
  private readonly getSettings: () => SchedulerSettings;
203
+ private readonly writesNextRuns?: () => boolean;
157
204
  /** Every sub-agent this process starts takes a slot from here, loop runs and trigger checks alike. */
158
205
  readonly slots: SubagentSlots;
159
206
  private readonly sessionExists?: (sessionId: string) => boolean;
@@ -164,14 +211,15 @@ export class LoopScheduler {
164
211
  private readonly nextRunsFile: string;
165
212
  private timer: NodeJS.Timeout | undefined;
166
213
  private ticking = false;
167
- private leader = false;
168
214
  private startedAt = 0;
169
- private readonly inflight = new Map<string, { ctrl: AbortController; label: string; jobId: string; startedAt: string; promptPreview: string; sessionFile?: string }>();
215
+ private readonly inflight = new Map<string, { ctrl: AbortController; label: string; jobId: string; startedAt: string; promptPreview: string; sessionFile?: string; stoppedFrom?: "page" | "cli" }>();
170
216
  /** Promises of runs in flight, so stop() can wait for their records to land. */
171
217
  private readonly runs = new Set<Promise<void>>();
172
218
  private stopped = false;
173
219
  private tickPromise?: Promise<void>;
174
- private readonly instance = newId("sched");
220
+ readonly instance: string;
221
+ private readonly leases?: PlainJobLeases;
222
+ private readonly owns: (job: LoopJob, view: OwnerView) => boolean;
175
223
  /** This process in the machine-wide presence registry (which pi is open where). */
176
224
  readonly presence: PresenceRegistry;
177
225
  readonly self: PresenceSelf;
@@ -180,16 +228,11 @@ export class LoopScheduler {
180
228
  this.dir = opts.dir;
181
229
  this.store = new JobStore(opts.dir);
182
230
  this.inbox = new Inbox(opts.dir);
183
- // One leader at a time, for the machine: every process here elects through this one file. Two
184
- // machines sharing a $HOME contend for it and both end up running every job, which is not
185
- // supported (docs/design.md §19).
186
- this.leaderFile = path.join(opts.dir, "scheduler.json");
187
- // One plain file beside the leader record. A cron expression is matched against local time, so
188
- // if two machines sharing a `$HOME` both ran, they would write different answers over one
189
- // another and the panel would show whichever wrote last — which is why that arrangement is
190
- // unsupported rather than handled (docs/design.md §19).
231
+ this.requests = new RequestStore(opts.dir);
232
+ // Written by the host alone. A cron expression is matched against local time, so if two
233
+ // machines sharing a `$HOME` both ran, they would write different answers over one another —
234
+ // which is why that arrangement is unsupported rather than handled (docs/design.md §19).
191
235
  this.nextRunsFile = path.join(opts.dir, "next-runs.json");
192
- this.leaderLock = path.join(opts.dir, "scheduler.lock");
193
236
  this.getSession = opts.getSession;
194
237
  this.hooks = opts.hooks ?? {};
195
238
  this.tickMs = opts.tickMs ?? DEFAULT_TICK_MS;
@@ -197,23 +240,31 @@ export class LoopScheduler {
197
240
  this.now = opts.now ?? Date.now;
198
241
  this.hop = opts.hop ?? 0;
199
242
  this.getSettings = opts.getSettings ?? (() => ({ maxConcurrentRuns: MAX_CONCURRENT_RUNS, catchUp: true }));
243
+ this.writesNextRuns = opts.writesNextRuns;
200
244
  this.slots = opts.slots ?? new SubagentSlots(() => this.getSettings().maxConcurrentRuns);
245
+ this.instance = opts.instance ?? newId("sched");
246
+ this.leases = opts.leases;
247
+ this.owns = opts.owns ?? ((job, view) => runsHere(job, view));
201
248
  this.sessionExists = opts.sessionExists;
202
249
  this.kind = opts.kind ?? "interactive";
203
250
  const session = opts.getSession();
204
- this.self = { pid: process.pid, host: os.hostname(), instance: this.instance, sessionId: session.sessionId, cwd: session.cwd, kind: this.kind };
251
+ this.self = { pid: process.pid, host: os.hostname(), instance: this.instance, sessionId: session.sessionId, cwd: session.cwd, kind: this.kind, leases: true };
205
252
  this.presence = new PresenceRegistry(opts.dir, this.self);
206
253
  }
207
254
 
255
+ /** What this instance knows about who holds which session: the lists and dispatch ask it the same way. */
256
+ ownerView(session: SessionSnapshot = this.getSession()): OwnerView {
257
+ const sessionId = session.sessionId;
258
+ const leases = this.leases;
259
+ if (!leases) return { sessionId, holdsOwn: !!sessionId };
260
+ return { sessionId, holdsOwn: !!sessionId && leases.heldEpoch(sessionId) !== undefined, holderOf: (id) => leases.holderOf(id), standsAside: leases.standsAside?.(), holderKind: leases.holderKind ? (id) => leases.holderKind!(id) : undefined, hostRuns: leases.hostRuns ? (id) => leases.hostRuns!(id) : undefined, hostPaused: leases.hostPaused?.() };
261
+ }
262
+
208
263
  /** Live pi processes on every host (stale entries pruned). */
209
264
  presenceList(now = this.now()): PresenceEntry[] {
210
265
  return this.presence.list(now);
211
266
  }
212
267
 
213
- get isLeader(): boolean {
214
- return this.leader;
215
- }
216
-
217
268
  get runningCount(): number {
218
269
  return this.inflight.size;
219
270
  }
@@ -228,14 +279,26 @@ export class LoopScheduler {
228
279
  return [...this.inflight.entries()].map(([runId, r]) => ({ runId, label: r.label, jobId: r.jobId, startedAt: r.startedAt, promptPreview: r.promptPreview, sessionFile: r.sessionFile }));
229
280
  }
230
281
 
231
- /** Abort one in-flight run (kills the sub-agent). */
232
- abortRun(runId: string): boolean {
282
+ /** Abort one in-flight run (kills the sub-agent), as a person asked (STOPPED_BY_PERSON), from `from`. */
283
+ abortRun(runId: string, from: "page" | "cli" = "cli"): boolean {
233
284
  const r = this.inflight.get(runId);
234
285
  if (!r) return false;
235
- r.ctrl.abort();
286
+ r.stoppedFrom ??= from;
287
+ r.ctrl.abort(STOPPED_BY_PERSON);
236
288
  return true;
237
289
  }
238
290
 
291
+ /** Where a person's stop of `runId` came from, while the run is still in flight. */
292
+ stopFrom(runId: string): "page" | "cli" | undefined {
293
+ return this.inflight.get(runId)?.stoppedFrom;
294
+ }
295
+
296
+ /** Where the stop on its way to `runId` came from, for a stop that arrives through a request row. */
297
+ noteStopFrom(runId: string, from: "page" | "cli"): void {
298
+ const r = this.inflight.get(runId);
299
+ if (r) r.stoppedFrom = from;
300
+ }
301
+
239
302
  /**
240
303
  * Remove a job. A run of it that is in flight is aborted first and given up to
241
304
  * REMOVE_DRAIN_MS to write its record, so the removal does not race the run's end.
@@ -256,7 +319,7 @@ export class LoopScheduler {
256
319
  return { job, aborted: runIds.length };
257
320
  }
258
321
 
259
- /** Idempotent. Starts the tick timer and takes leadership if free. */
322
+ /** Idempotent. Starts the tick timer. */
260
323
  start(): void {
261
324
  if (this.timer) return;
262
325
  this.stopped = false;
@@ -270,93 +333,23 @@ export class LoopScheduler {
270
333
  safeTick();
271
334
  }
272
335
 
273
- /** Idempotent. Stops the timer, releases leadership, aborts in-flight runs. */
336
+ /** Idempotent. Stops the timer and aborts in-flight runs. */
274
337
  async stop(): Promise<void> {
275
338
  this.stopped = true;
276
339
  if (this.timer) {
277
340
  clearInterval(this.timer);
278
341
  this.timer = undefined;
279
342
  }
280
- await this.tickPromise; // a tick past its leadership claim must not launch after we leave
343
+ await this.tickPromise; // a tick already dispatching must not launch after we leave
281
344
  for (const { ctrl } of this.inflight.values()) ctrl.abort();
282
345
  // What is lost past this is the record, not the run: the sub-agent is already aborted, and
283
346
  // waiting longer only delays the quit that asked for it.
284
347
  const DRAIN_TIMEOUT_MS = 5_000;
285
348
  await this.drain(DRAIN_TIMEOUT_MS);
286
- await this.releaseLeadership();
287
- this.presence.remove();
288
349
  }
289
350
 
290
- readLeader(): LeaderRecord | undefined {
291
- try {
292
- return JSON.parse(fs.readFileSync(this.leaderFile, "utf8")) as LeaderRecord;
293
- } catch {
294
- return undefined;
295
- }
296
- }
297
-
298
- private isMine(rec: LeaderRecord | undefined): boolean {
299
- return !!rec && rec.pid === process.pid && rec.host === os.hostname() && rec.instance === this.instance;
300
- }
301
-
302
- private leaderIsStale(rec: LeaderRecord | undefined, now: number): boolean {
303
- if (!rec) return true;
304
- if (this.isMine(rec)) return false;
305
- // The headless host only keeps the clock while nobody is around: an interactive pi takes it
306
- // back the moment it opens, and the host exits when it sees that (see host.ts).
307
- if (this.kind === "interactive" && rec.kind === "host") return true;
308
- const age = now - Date.parse(rec.heartbeatAt);
309
- if (Number.isNaN(age) || age > LEADER_STALE_MS) return true;
310
- // One machine: a pid that is not alive here is not alive. (A record from a renamed or rebuilt
311
- // box used to be untouchable until its heartbeat aged out; now it is taken over like any other.)
312
- if (!pidAlive(rec.pid)) return true;
313
- return false;
314
- }
315
-
316
- private async claimOrRenewLeadership(now: number): Promise<boolean> {
317
- return withFileLock(this.leaderLock, () => {
318
- const rec = this.readLeader();
319
- const mine = this.isMine(rec);
320
- if (!mine && !this.leaderIsStale(rec, now)) {
321
- this.leader = false;
322
- return false;
323
- }
324
- const next: LeaderRecord = {
325
- pid: process.pid,
326
- host: os.hostname(),
327
- instance: this.instance,
328
- sessionId: this.getSession().sessionId,
329
- kind: this.kind,
330
- startedAt: mine && rec ? rec.startedAt : stamp(now),
331
- heartbeatAt: stamp(now),
332
- };
333
- writeFileAtomic(this.leaderFile, `${JSON.stringify(next, null, 2)}\n`);
334
- // Routine: which process holds the timer is bookkeeping, not something gone wrong.
335
- if (!this.leader) this.log(`took over the loop scheduler (pid ${process.pid})`, "info");
336
- this.leader = true;
337
- return true;
338
- });
339
- }
340
-
341
- private async releaseLeadership(): Promise<void> {
342
- if (!this.leader) return;
343
- this.leader = false;
344
- try {
345
- await this.hooks.onLeadership?.(false);
346
- } catch (err: any) {
347
- this.log(`leadership hook failed: ${err?.message ?? err}`);
348
- }
349
- try {
350
- await withFileLock(this.leaderLock, () => {
351
- if (this.isMine(this.readLeader())) fs.rmSync(this.leaderFile, { force: true });
352
- });
353
- } catch {
354
- /* best effort */
355
- }
356
- }
357
-
358
- private log(message: string, level?: "info" | "warning"): void {
359
- this.hooks.log?.(message, level);
351
+ private log(message: string, level?: "info" | "warning", job?: Pick<LoopJob, "cwd">): void {
352
+ this.hooks.log?.(message, level, job);
360
353
  }
361
354
 
362
355
  /**
@@ -395,30 +388,12 @@ export class LoopScheduler {
395
388
  this.tickPromise = new Promise<void>((r) => (finish = r));
396
389
  try {
397
390
  const now = this.now();
398
- let leader = false;
399
- const wasLeader = this.leader;
400
- try {
401
- leader = await this.claimOrRenewLeadership(now);
402
- } catch (err: any) {
403
- this.log(`leadership check failed: ${err?.message ?? err}`);
404
- }
405
- // A UI hook that throws must not escape the tick: `start()` runs it as `void this.tick()`
406
- // and pi installs no unhandledRejection handler, so an unguarded throw here kills pi.
407
- if (leader !== wasLeader) {
408
- try {
409
- await this.hooks.onLeadership?.(leader);
410
- } catch (err: any) {
411
- this.log(`leadership hook failed: ${err?.message ?? err}`);
412
- }
413
- }
391
+ const host = this.kind === "host";
414
392
  const session = this.getSession();
415
393
  try {
416
- this.presence.heartbeat(now, { cwd: session.cwd, sessionId: session.sessionId });
417
- } catch (err: any) {
418
- this.log(`presence heartbeat failed: ${err?.message ?? err}`);
419
- }
420
- try {
421
- await this.hooks.onTick?.(now, leader);
394
+ // A hook that throws must not escape the tick: `start()` runs it as `void this.tick()` and
395
+ // pi installs no unhandledRejection handler, so an unguarded throw here kills pi.
396
+ await this.hooks.onTick?.(now);
422
397
  } catch (err: any) {
423
398
  this.log(`tick hook failed: ${err?.message ?? err}`);
424
399
  }
@@ -426,21 +401,40 @@ export class LoopScheduler {
426
401
  try {
427
402
  // `mutate` writes only when the list really changed, so a tick with nothing to clear
428
403
  // costs one read and no write (store.ts).
404
+ let retired: LoopJob[] = [];
429
405
  jobs = await this.store.mutate((all) => {
430
- this.clearStaleRunning(all);
406
+ retired = this.clearStaleRunning(all);
431
407
  return { jobs: all, result: all };
432
408
  });
409
+ for (const job of retired) {
410
+ try {
411
+ this.hooks.onJobRetired?.(job, "one-shot finished before its process ended");
412
+ } catch (err: any) {
413
+ this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
414
+ }
415
+ }
433
416
  } catch (err: any) {
434
417
  this.log(`cannot read jobs: ${err?.message ?? err}`);
435
418
  return;
436
419
  }
420
+ // A job removed in another process (a window, the CLI) takes its run here with it, as
421
+ // `removeJob` does in this one: the run used to go on to its deadline, spending as it went.
422
+ const present = new Set(jobs.map((j) => j.id));
423
+ for (const [runId, r] of this.inflight) {
424
+ if (present.has(r.jobId) || r.ctrl.signal.aborted) continue;
425
+ r.ctrl.abort();
426
+ this.log(`cron ${r.label}: removed while it ran; stopped run ${runId}`);
427
+ }
437
428
  // What the store cannot say and every reader wants: when each job runs next. It is a pure
438
429
  // function of the job, but computing it needs the cron evaluator — which the browser
439
- // front end does not have and should not grow a second copy of. The leader writes the
430
+ // front end does not have and should not grow a second copy of. The host writes the
440
431
  // answers where that front end already reads pi-loops' own files, and only when one of
441
432
  // them changes, which for a nightly job is once a day.
442
- if (leader) this.writeNextRuns(jobs, now);
443
- if (leader && this.sessionExists && now - this.lastDeadSessionScan >= DEAD_SESSION_SCAN_MS) {
433
+ // The host writes it; with no host running a window does, or the browser front end of a person
434
+ // with only plain jobs (who never starts a host) showed no next run at all. Every writer
435
+ // computes the same function of the same store, so two windows writing it agree.
436
+ if (host || this.writesNextRuns?.()) this.writeNextRuns(jobs, now);
437
+ if (host && this.sessionExists && now - this.lastDeadSessionScan >= DEAD_SESSION_SCAN_MS) {
444
438
  this.lastDeadSessionScan = now;
445
439
  try {
446
440
  await this.disableDeadSessionJobs(jobs);
@@ -448,10 +442,23 @@ export class LoopScheduler {
448
442
  this.log(`dead-session check failed: ${err?.message ?? err}`);
449
443
  }
450
444
  }
445
+ const view = this.ownerView(session);
451
446
  for (const job of jobs) {
447
+ if (!this.owns(job, view)) continue;
448
+ // A fired one-shot stays until its row is done (its prompt is in the conversation, or the row
449
+ // died): removed at fire, a quit before the hand-over deleted it unfired.
450
+ if (this.plainOneShotDone(job)) {
451
+ await this.store.remove(job.id);
452
+ try {
453
+ this.hooks.onJobRetired?.(job, "one-shot fired");
454
+ } catch (err: any) {
455
+ this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
456
+ }
457
+ continue;
458
+ }
459
+ // A disabled job's queued rows wait: its prompt runs when the job is enabled again, which
460
+ // is what a person who disabled it expects. A removed job's rows the engine removes.
452
461
  if (!job.enabled) continue;
453
- const owned = job.stateful ? leader : runsIn(job, session.sessionId);
454
- if (!owned) continue;
455
462
  // One unusable job must never stop the clock for the others: a schedule that cannot be
456
463
  // evaluated disables that job and says why, instead of throwing out of the tick (pi
457
464
  // installs no unhandledRejection handler, so that would take the whole session down).
@@ -466,11 +473,12 @@ export class LoopScheduler {
466
473
  },
467
474
  now,
468
475
  );
469
- if (due === undefined) continue;
476
+ // Nothing due, but a row a window left behind when it quit is the host's to run (launchHeld).
477
+ if (due === undefined && !(host && !job.stateful && this.heldTurnRow(job))) continue;
470
478
  await this.dispatch(job, due, now, session);
471
479
  } catch (err: any) {
472
480
  const message = `disabled: ${err?.message ?? err} (${formatSchedule(job.schedule)})`;
473
- this.log(`loop ${job.id}: ${message}`);
481
+ this.log(`loop ${job.id}: ${message}`, undefined, job);
474
482
  await this.store.update(job.id, (j) => {
475
483
  j.enabled = false;
476
484
  j.lastError = message;
@@ -496,9 +504,9 @@ export class LoopScheduler {
496
504
  for (const job of dead) {
497
505
  await this.store.update(job.id, (j) => {
498
506
  j.enabled = false;
499
- j.lastError = `disabled: session ${job.sessionId!.slice(0, 8)} ${DEAD_SESSION_MARKER}`;
507
+ j.lastError = `disabled: session ${shortSessionId(job.sessionId!)} ${DEAD_SESSION_MARKER}`;
500
508
  });
501
- this.log(`cron ${job.name ?? job.id}: disabled, its session ${job.sessionId!.slice(0, 8)} no longer exists`);
509
+ this.log(`cron ${job.name ?? job.id}: disabled, its session ${shortSessionId(job.sessionId!)} no longer exists`, undefined, job);
502
510
  }
503
511
  }
504
512
 
@@ -539,39 +547,86 @@ export class LoopScheduler {
539
547
  return this.store.removeWhere((j) => inScope(j) && !j.enabled && !!j.lastError?.endsWith(DEAD_SESSION_MARKER), { purge: true });
540
548
  }
541
549
 
542
- /** Clear running markers left behind by processes that are gone. Mutates `jobs` in place. */
543
- private clearStaleRunning(jobs: LoopJob[]): void {
550
+ /**
551
+ * Clear running markers left behind by processes that are gone. Mutates `jobs` in place and
552
+ * returns the one-shots it retired, for the caller to report once the store is written.
553
+ *
554
+ * A dead marker is looked up in the run log first. A run whose record is there finished, and only
555
+ * its last write to the store was lost (the process died between `appendRun` and the update after
556
+ * it): its slot is spent, so the bookkeeping is finished from the record instead of rolling back
557
+ * and running it a second time. Only a run with no record is owed again.
558
+ */
559
+ private clearStaleRunning(jobs: LoopJob[]): LoopJob[] {
544
560
  // The wall clock, not `this.now()`: `os.uptime()` is measured against it, so the boot instant
545
- // is only meaningful in the same terms — and so is the `recycled` comparison against it below,
546
- // which is why a test cannot move that one. The foreign-host age check does go through
547
- // `this.now()`: it compares a stamp against our own notion of now, with no machine fact in it.
561
+ // is only meaningful in the same terms — which is why a test cannot move this comparison.
548
562
  const booted = Date.now() - os.uptime() * 1000;
563
+ let records: Map<string, RunRecord> | undefined;
564
+ const recordOf = (runId: string): RunRecord | undefined => {
565
+ records ??= new Map(this.store.allRuns().map((r) => [r.runId, r]));
566
+ return records.get(runId);
567
+ };
568
+ const retired: LoopJob[] = [];
569
+ const dead = (job: LoopJob): boolean => {
570
+ const { pid, runId, startedAt } = job.running!;
571
+ // A pid is only evidence while it can still be the same process: after a reboot the
572
+ // numbers are handed out again, and a recycled pid would park the job forever.
573
+ const recycled = Number.isFinite(booted) && Date.parse(startedAt) < booted - 60_000;
574
+ return !(pid === process.pid ? this.inflight.has(runId) : !recycled && pidAlive(pid));
575
+ };
576
+ // How many runs each dead process took with it. One that died with a single run in flight
577
+ // points at that run; one that died with several says nothing about any of them, and counting
578
+ // each would park every loop that happened to run beside the one that took it down.
579
+ const diedWith = new Map<number, number>();
580
+ for (const job of jobs) if (job.running && dead(job) && !recordOf(job.running.runId)) diedWith.set(job.running.pid, (diedWith.get(job.running.pid) ?? 0) + 1);
549
581
  for (const job of jobs) {
550
582
  if (!job.running) continue;
551
583
  const { pid, runId, startedAt } = job.running;
552
- // Another machine's pid table says nothing about this run (shared $HOME); leave it to the
553
- // host that owns it, and fall back to a generous age check so it cannot stick forever.
554
- if (job.running.host && job.running.host !== this.self.host) {
555
- if (Date.parse(startedAt) > this.now() - FOREIGN_RUN_STALE_MS) continue;
556
- } else {
557
- const isMine = pid === process.pid;
558
- // A pid is only evidence while it can still be the same process: after a reboot the
559
- // numbers are handed out again, and a recycled pid would park the job forever.
560
- const recycled = Number.isFinite(booted) && Date.parse(startedAt) < booted - 60_000;
561
- const alive = isMine ? this.inflight.has(runId) : !recycled && pidAlive(pid);
562
- if (alive) continue;
584
+ if (!dead(job)) continue;
585
+ job.running = undefined;
586
+ const record = recordOf(runId);
587
+ if (record) {
588
+ job.lastCompletedAt = record.finishedAt;
589
+ job.lastError = record.ok ? undefined : record.error;
590
+ job.runCount++;
591
+ job.consecutiveFailures = record.ok ? undefined : (job.consecutiveFailures ?? 0) + 1;
592
+ job.interruptions = undefined;
593
+ if (job.schedule.kind === "once" && (record.ok || job.runCount >= 2)) retired.push(job);
594
+ continue;
563
595
  }
564
596
  // The run that died is owed again: roll the bookkeeping back so the next tick re-fires it
565
- // (collapsed like any other missed tick) instead of silently skipping to the next slot.
566
- job.running = undefined;
567
- job.lastDueAt = undefined;
568
- job.lastFiredAt = undefined;
597
+ // (collapsed like any other missed tick) instead of silently skipping to the next slot. It
598
+ // was tried, so a process that starts after its due time must not call it missed. A run of a
599
+ // row that is still queued (a held turn of what a window left, launchHeld) rolls nothing
600
+ // back: the row is what runs again, and the stamps the window moved stand.
601
+ if (!(job.sessionId && this.requests.list(job.sessionId).some((row) => row.id === runId))) {
602
+ job.owedSlot = job.lastDueAt ?? startedAt;
603
+ job.lastDueAt = undefined;
604
+ job.lastFiredAt = undefined;
605
+ }
606
+ if ((diedWith.get(pid) ?? 0) > 1) {
607
+ job.lastError = "cleared stale running state (the process ended mid-run, with other runs beside this one); the run will be retried";
608
+ continue;
609
+ }
610
+ job.interruptions = (job.interruptions ?? 0) + 1;
611
+ if (job.interruptions >= INTERRUPTIONS_BEFORE_PARKING) {
612
+ // Parked rather than retried on every start: each time, it was the only loop run in flight
613
+ // when the process stopped (a rule check may have been running beside it). Still not proof (a reboot mid-run counts too), so the reason says what
614
+ // is known, and enabling it again tries once more.
615
+ job.enabled = false;
616
+ job.owedSlot = undefined;
617
+ job.lastError = `parked: its last ${job.interruptions} runs were interrupted before they finished (the process running them stopped each time, with no other loop running; see host.log) — /cron enable ${job.id} to try again`;
618
+ this.log(`cron ${job.name ?? job.id}: parked after ${job.interruptions} interrupted runs`, undefined, job);
619
+ continue;
620
+ }
569
621
  job.lastError = "cleared stale running state (previous pi process ended mid-run); the run will be retried";
570
622
  }
623
+ for (const job of retired) jobs.splice(jobs.indexOf(job), 1);
624
+ return retired;
571
625
  }
572
626
 
573
- private async dispatch(job: LoopJob, due: number, now: number, session: SessionSnapshot): Promise<void> {
574
- const dueIso = stamp(due);
627
+ /** `due` is the slot the schedule owes; absent, the host runs a queued row of the job and spends no slot. */
628
+ private async dispatch(job: LoopJob, due: number | undefined, now: number, session: SessionSnapshot): Promise<void> {
629
+ const dueIso = due === undefined ? undefined : stamp(due);
575
630
  if (job.stateful && !fs.existsSync(job.cwd)) {
576
631
  /**
577
632
  * The checkout is not there. That is usually a deleted worktree or a moved project — but
@@ -592,13 +647,13 @@ export class LoopScheduler {
592
647
  j.cwdMissingSince = stamp(since);
593
648
  if (gone) {
594
649
  j.enabled = false;
595
- j.lastDueAt = dueIso;
650
+ if (dueIso) j.lastDueAt = dueIso;
596
651
  j.lastError = `disabled: cwd ${job.cwd} has been missing since ${formatLocal(since)} (re-enable after /cron add --cwd or restoring it)`;
597
652
  } else {
598
653
  j.lastError = `waiting: cwd ${job.cwd} is not there yet (since ${formatLocal(since)}; disabled if it stays missing)`;
599
654
  }
600
655
  });
601
- if (gone) this.log(`cron ${job.name ?? job.id}: disabled, cwd ${job.cwd} missing since ${formatLocal(since)}`);
656
+ if (gone) this.log(`cron ${job.name ?? job.id}: disabled, cwd ${job.cwd} missing since ${formatLocal(since)}`, undefined, job);
602
657
  return;
603
658
  }
604
659
  // Back, and the run that was owed is about to happen: the marker must not outlive the outage.
@@ -609,21 +664,27 @@ export class LoopScheduler {
609
664
  });
610
665
  }
611
666
  if (job.running) {
667
+ // A plain job's marker is the host's turn in its conversation: that turn either completes the
668
+ // slot or hands it back owed, so there is nothing to skip — wait for it.
669
+ if (!job.stateful) return;
612
670
  await this.store.update(job.id, (j) => {
613
671
  j.skippedOverlap++;
614
- j.lastDueAt = dueIso;
672
+ if (dueIso) j.lastDueAt = dueIso;
615
673
  j.lastError = "skipped: previous run still active";
616
674
  });
617
675
  return;
618
676
  }
619
- const missedWhileDown = due < this.startedAt - this.tickMs;
620
- if (missedWhileDown && !(job.catchUp && this.getSettings().catchUp)) {
677
+ // A slot handed back unrun was tried, not missed: it runs now whatever the catch-up setting says.
678
+ const owed = due !== undefined && !!job.owedSlot && Date.parse(job.owedSlot) <= due;
679
+ const missedDue = due !== undefined && !owed && due < this.startedAt - this.tickMs ? due : undefined;
680
+ if (missedDue !== undefined && !(job.catchUp && this.getSettings().catchUp)) {
621
681
  await this.store.update(job.id, (j) => {
622
- j.lastDueAt = dueIso;
623
- j.lastError = `missed ${formatLocal(due)} while no pi was running (catch-up disabled)`;
682
+ if (dueIso) j.lastDueAt = dueIso;
683
+ j.lastError = `missed ${formatLocal(missedDue)} while ${this.kind === "host" ? "pi-loops was not running" : "no pi was running"} (catch-up disabled)`;
624
684
  });
625
685
  return;
626
686
  }
687
+ const note = missedDue !== undefined ? `\n(catching up a run due ${formatLocal(missedDue)})` : "";
627
688
  const budget = this.budgetState(now);
628
689
  if (budget.over) {
629
690
  // Plain jobs are gated too: injecting one makes the parent agent take a turn, which is
@@ -638,8 +699,8 @@ export class LoopScheduler {
638
699
  }
639
700
  return;
640
701
  }
641
- if (!job.stateful) {
642
- await this.injectPlainJob(job, dueIso, now, missedWhileDown ? `\n(catching up a run due ${formatLocal(due)})` : "");
702
+ if (!job.stateful && this.kind !== "host") {
703
+ await this.injectPlainJob(job, dueIso, now, note);
643
704
  return;
644
705
  }
645
706
  // A failing job is retried with a widening gap instead of at every due tick: a loop whose
@@ -666,51 +727,310 @@ export class LoopScheduler {
666
727
  slot.release();
667
728
  return;
668
729
  }
669
- if (missedWhileDown) this.hooks.onCatchUp?.(job, due);
670
- // Never await a run inside a tick: heartbeats, leadership, presence and trigger checks keep
730
+ if (missedDue !== undefined) this.hooks.onCatchUp?.(job, missedDue);
731
+ // Never await a run inside a tick: leases, presence and trigger checks keep
671
732
  // going while sub-agents work, and several loops really do run at once.
672
- this.track(this.launch(job, dueIso, now, session, missedWhileDown), slot);
733
+ if (!job.stateful) {
734
+ this.track(this.launchHeld(job, dueIso, now, note), slot);
735
+ return;
736
+ }
737
+ this.track(this.launch(job, dueIso, now, session, missedDue !== undefined), slot);
738
+ }
739
+
740
+ /** Who wrote a row: this process, as `created_by` names it. */
741
+ private createdBy(): SessionRequest["created_by"] {
742
+ return { pid: process.pid, instance: this.instance, version: PI_LOOPS_VERSION };
743
+ }
744
+
745
+ /** The job's queued `turn` row, when one is: the prompt waiting to run in its conversation. */
746
+ private queuedTurnRow(job: LoopJob): SessionRequest | undefined {
747
+ if (!job.sessionId) return undefined;
748
+ const key = plainCollapseKey(job.id);
749
+ return this.requests.list(job.sessionId).find((row) => row.kind === "input" && row.wake === "turn" && row.collapse_key === key);
750
+ }
751
+
752
+ /**
753
+ * The job's queued row the background may run as a held turn: not an attended one (`/cron run`,
754
+ * typed in the conversation), which waits for a window and is no reason for the clock's slots to.
755
+ */
756
+ private heldTurnRow(job: LoopJob): SessionRequest | undefined {
757
+ const row = this.queuedTurnRow(job);
758
+ return row?.mode === "attended" ? undefined : row;
759
+ }
760
+
761
+ /**
762
+ * A fired plain one-shot whose row is done: its prompt is in the conversation, or the row died.
763
+ * A failed held turn is not done; it keeps the retry `launchHeld` gives a one-shot.
764
+ */
765
+ private plainOneShotDone(job: LoopJob): boolean {
766
+ if (job.stateful || job.schedule.kind !== "once" || job.running || job.runCount < 1 || job.consecutiveFailures) return false;
767
+ return this.queuedTurnRow(job) === undefined;
673
768
  }
674
769
 
675
770
  /**
676
- * A plain job firing: the bookkeeping, the injection, and the retirement of a one-shot. Both the
677
- * timer (`dispatch`) and `/cron run` come through here — they were two copies, and the second one
678
- * had drifted: no `once` retirement, no `lastDueAt` / `lastCompletedAt`, `lastError` never cleared.
771
+ * A plain job firing in a window: its prompt is a row in the conversation's requests first, then
772
+ * the slot is spent. Both the timer (`dispatch`) and `/cron run` come through here. The engine
773
+ * hands the row to pi once pi is idle (src/engine.ts, src/window-executor.ts); a one-shot is
774
+ * retired when the row is done (plainOneShotDone), not here.
775
+ *
776
+ * The row's id is derived from the slot (plainRunId), so a crash between the row and the stamps
777
+ * leaves a queued row and an unspent slot: the next tick decides the same slot, its append is a
778
+ * no-op, and the swap commits. While an earlier row of the job is still queued the slot is spent
779
+ * without a new row and counted in `skippedOverlap`: collapse, never replay.
679
780
  *
680
781
  * `dueIso` is the slot being spent, absent when the run was asked for rather than scheduled;
681
- * `note` is appended to the injected text (the catch-up line). Returns whether the job was
682
- * actually injected, so `/cron run` does not report a run that never happened.
782
+ * `note` is appended to the prompt (the catch-up line). Returns what became of the slot, so
783
+ * `/cron run` does not report a run that never happened.
683
784
  */
684
- private async injectPlainJob(job: LoopJob, dueIso: string | undefined, now: number, note = ""): Promise<boolean> {
685
- // A plain job's whole point is to land in a chat, and the headless host has none. The
686
- // ownership test in `dispatch` already excludes these (the host's snapshot has no sessionId), so
687
- // this is belt and braces: the bookkeeping below commits before `onInject` runs, and a
688
- // host that ever gained a session id would otherwise record an undelivered job as a
689
- // completed run and delete a one-shot unfired.
690
- if (this.kind === "host") return false;
785
+ private async injectPlainJob(job: LoopJob, dueIso: string | undefined, now: number, note = "", source: "cron" | "cron-run" = "cron"): Promise<"fired" | "collapsed" | "lost"> {
786
+ // A plain job's whole point is to land in a chat, and the headless host has none: a closed
787
+ // conversation's plain job is a held turn (launchHeld).
788
+ if (this.kind === "host") return "lost";
789
+ const sessionId = job.sessionId;
790
+ if (!sessionId) return "lost";
791
+ // The epoch the lease had when this instance decided the job was its own. Read again inside
792
+ // the claim: a lease that moved since (this window closed its session, another took it) means
793
+ // the slot is no longer ours to fire.
794
+ const epoch = this.leases?.heldEpoch(sessionId);
795
+ if (this.leases && epoch === undefined) return "lost";
796
+ const runId = plainRunId(job.id, dueIso ?? stamp(now));
797
+ const queued = this.queuedTurnRow(job);
798
+ // A row pi did not take (`attempts` counts its refusals) is not what a new slot collapses onto:
799
+ // that slot is lost, as 0.22.8 lost a slot pi refused, and the new one asks in its place.
800
+ // A `/cron run` row is the person's: a slot collapses onto it, refused or not, so it keeps running as theirs.
801
+ const refused = queued !== undefined && queued.id !== runId && queued.attempts > 0 && queued.source.startsWith("cron:");
802
+ const collapsed = !!queued && !refused && queued.id !== runId;
803
+ // `/cron run` while the prompt is already queued asks for nothing new: no slot, no stamp.
804
+ if (collapsed && source === "cron-run") return "collapsed";
805
+ // The row before the stamps: a crash between the two costs nothing, where the reverse order
806
+ // spent the slot with nothing to show for it.
807
+ const row = plainJobRow(job, runId, `${source}:${job.id}`, note, this.createdBy(), now);
808
+ let wrote = false;
809
+ if (!collapsed) {
810
+ if (refused) this.requests.remove(sessionId, [queued.id]);
811
+ wrote = this.requests.appendIfAbsent(row);
812
+ // Said before anything awaits: a drain that hands the row over meanwhile must already know
813
+ // this window wrote it, or a person's `/cron run` would start as a scheduled turn.
814
+ if (wrote) this.hooks.onRowWritten?.(row, job);
815
+ }
816
+ let claimed = false;
691
817
  await this.store.update(job.id, (j) => {
818
+ // The slot was decided from a snapshot, as in `launch`: if another instance fired or skipped
819
+ // this job since, its stamps moved, and firing again would put the message in the chat twice.
820
+ if (j.lastDueAt !== job.lastDueAt || j.lastFiredAt !== job.lastFiredAt) return;
821
+ if (this.leases && (!j.sessionId || this.leases.heldEpoch(j.sessionId) !== epoch)) return;
822
+ claimed = true;
692
823
  if (dueIso) j.lastDueAt = dueIso;
693
824
  j.lastFiredAt = stamp(now);
825
+ // The count is the record of a collapse; an error here outlived the prompt it was about.
826
+ if (collapsed) {
827
+ j.skippedOverlap++;
828
+ return;
829
+ }
694
830
  j.lastCompletedAt = j.lastFiredAt;
831
+ j.owedSlot = undefined;
695
832
  // Only an enabled job's error is stale news. A disabled one's is why it is disabled — and
696
833
  // for a job the dead-session sweep parked it is the marker `gc()` matches on.
697
834
  if (j.enabled) j.lastError = undefined;
698
835
  j.runCount++;
699
836
  });
700
- // Inject-and-run: the job's action lands in the parent chat as a user message with
701
- // the engine-enforced `[Trigger <run id>] ` prefix. The id is handed over as well: it is minted
702
- // here, and the caller used to read it back out of the string this just formatted.
703
- const runId = newId("run");
704
- await this.hooks.onInject?.(job, `[Trigger ${runId}] ${job.prompt}${note}`, runId);
705
- if (job.schedule.kind === "once") {
706
- await this.store.remove(job.id);
837
+ if (!claimed) {
838
+ // Another process fired this slot first (a pi-loops 0.22 window on the same conversation during
839
+ // an upgrade): the row this call wrote would run the slot a second time. Only that row goes.
840
+ if (wrote) this.requests.remove(sessionId, [runId]);
841
+ return "lost";
842
+ }
843
+ // Its next run moved: written now, so the browser panel does not show a blank one until the
844
+ // next tick of whichever process writes it. Every writer computes the same answer.
845
+ try {
846
+ this.writeNextRuns(this.store.load(), now);
847
+ } catch {
848
+ /* a derived file is a courtesy (see writeNextRuns); the tick writes it too */
849
+ }
850
+ return collapsed ? "collapsed" : "fired";
851
+ }
852
+
853
+ /**
854
+ * The host firing a plain job of a closed conversation: take the conversation, claim the slot
855
+ * against the stamps and the lease's epoch, run the turn in it, give the conversation back. A
856
+ * window that opens the conversation meanwhile takes it at once; the turn stops at its next event
857
+ * and the slot is handed back, so the window fires it instead — a slot runs to completion once.
858
+ *
859
+ * A row a window wrote and never handed over (it quit first) runs before any slot of the host's
860
+ * own, with the row's id as the run id, and leaves the queue once its prompt is in the file. A slot
861
+ * due meanwhile is spent onto that row (collapse, as the window does), never run after it. A turn
862
+ * handed back unrun whose prompt did land is not owed: a successor row asks again, once, in
863
+ * whichever process holds the conversation next. An attended row (`/cron run`) is a window's to
864
+ * run and is left where it is; the clock's slots run as if it were not there. A row in backoff
865
+ * waits, and so does the slot behind it.
866
+ */
867
+ private async launchHeld(job: LoopJob, dueIso: string | undefined, now: number, note: string): Promise<void> {
868
+ const leases = this.leases;
869
+ const sessionId = job.sessionId;
870
+ if (!sessionId || !leases?.acquireForWork || !this.hooks.onHeldTurn) return;
871
+ const row = this.heldTurnRow(job);
872
+ if (!row && dueIso === undefined) return;
873
+ if (row && (row.attempts >= DEAD_AFTER_ATTEMPTS || (row.not_before !== undefined && Date.parse(row.not_before) > now))) return;
874
+ const runId = row?.id ?? newId("run");
875
+ const spendsSlot = dueIso !== undefined;
876
+ // The slot due is not the row's own: spent onto it, and counted as the overlap it is.
877
+ const collapsedSlot = spendsSlot && !!row && row.id !== plainRunId(job.id, dueIso);
878
+ const held = leases.acquireForWork(sessionId, runId, job.id);
879
+ if (!held) return; // the slot stays owed: the conversation is busy, open or recently written
880
+ try {
881
+ // A process before this one ran the row and died before removing it: the prompt is in the
882
+ // conversation, so the row is done, and running it again would answer it twice.
883
+ if (row && sessionFileHas(held.sessionFile, row.id)) {
884
+ this.requests.remove(sessionId, [row.id]);
885
+ await this.store.update(job.id, (j) => {
886
+ if (j.running?.runId === row.id) j.running = undefined;
887
+ });
888
+ return;
889
+ }
890
+ const startedAt = stamp(now);
891
+ let claimed = false;
892
+ await this.store.update(job.id, (j) => {
893
+ if (j.running) return;
894
+ if (j.lastDueAt !== job.lastDueAt || j.lastFiredAt !== job.lastFiredAt) return;
895
+ if (leases.heldEpoch(sessionId) !== held.epoch) return;
896
+ claimed = true;
897
+ // Written before the turn starts, as a loop's marker is: a host that dies mid-turn leaves
898
+ // it behind, and the next host finds an unfinished run instead of a slot spent in silence.
899
+ j.running = { runId, pid: process.pid, host: this.self.host, startedAt };
900
+ if (!spendsSlot) return;
901
+ j.lastDueAt = dueIso;
902
+ j.lastFiredAt = startedAt;
903
+ j.owedSlot = undefined;
904
+ if (collapsedSlot) j.skippedOverlap++;
905
+ });
906
+ if (!claimed) return;
907
+ const ctrl = new AbortController();
908
+ this.inflight.set(runId, { ctrl, label: job.name ?? job.id, jobId: job.id, startedAt, promptPreview: previewRedacted(job.prompt, 120) });
909
+ let outcome: { preempted: boolean; result: RunnerResult; landed?: boolean };
707
910
  try {
708
- this.hooks.onJobRetired?.(job, "one-shot fired");
911
+ outcome = await this.hooks.onHeldTurn(job, row?.payload?.text ?? `[Trigger ${runId}] ${job.prompt}${note}`, runId, held, ctrl.signal);
912
+ } catch (err: any) {
913
+ outcome = { preempted: false, result: failedRun(err?.message ?? String(err)) };
914
+ }
915
+ const { result } = outcome;
916
+ const landed = outcome.landed ?? result.ok;
917
+ // Handed back unrun: a window took the conversation (and runs it now), or this host stopped
918
+ // the turn itself — a quit, an upgrade, a preemption, the daily budget (a person's stop is done, above). The slot is owed
919
+ // only when the prompt never landed; landed, the successor row below is the owed slot.
920
+ const budgetStopped = !result.ok && (result.errorMessage ?? "").startsWith("stopped by today's");
921
+ // A person stopped this turn: it is done, whatever landed. Asking the prompt again would run
922
+ // what they just stopped.
923
+ const stopped = !result.ok && ctrl.signal.aborted && ctrl.signal.reason === STOPPED_BY_PERSON;
924
+ const handedBack = !result.ok && !stopped && (outcome.preempted || ctrl.signal.aborted || budgetStopped);
925
+ const record: RunRecord = {
926
+ runId,
927
+ jobId: job.id,
928
+ jobName: job.name,
929
+ stateful: false,
930
+ cwd: held.cwd ?? job.cwd,
931
+ pid: process.pid,
932
+ startedAt,
933
+ finishedAt: stamp(this.now()),
934
+ ok: result.ok,
935
+ exitCode: result.exitCode,
936
+ error: result.ok ? undefined : stopped ? STOPPED_BY_PERSON : redact(result.errorMessage ?? "unknown error"),
937
+ findings: 0,
938
+ droppedFindings: 0,
939
+ stateUpdated: false,
940
+ model: result.model ?? job.model,
941
+ usage: result.usage,
942
+ warning: result.warning,
943
+ retries: result.retries,
944
+ compactions: result.compactions,
945
+ summary: previewRedacted(result.text, 400) || undefined,
946
+ sessionId,
947
+ sessionFile: held.sessionFile,
948
+ forSession: job.createdBy?.sessionId ?? sessionId,
949
+ ...(stopped || handedBack ? { aborted: true as const } : {}),
950
+ ...(stopped ? { stoppedBy: "person" as const, stoppedFrom: this.inflight.get(runId)?.stoppedFrom ?? "cli" } : {}),
951
+ };
952
+ // Every turn is recorded, a stopped one too: it was billed, and the daily budget, `/cron runs`
953
+ // and the status page all read the run log.
954
+ try {
955
+ this.store.appendRun(record);
956
+ } catch (err: any) {
957
+ this.log(`run log append failed: ${err?.message ?? err}`);
958
+ }
959
+ let updated: LoopJob | undefined;
960
+ try {
961
+ updated = await this.store.update(job.id, (j) => {
962
+ if (j.running?.runId === runId) j.running = undefined;
963
+ if (handedBack) {
964
+ // Given back with a compare: only while the stamps are still the ones this claim wrote.
965
+ if (!spendsSlot || j.lastFiredAt !== startedAt) return;
966
+ // Landed, the slot is spent (the successor row asks for its answer); unlanded, it is owed.
967
+ if (landed) {
968
+ if (!collapsedSlot) j.runCount++;
969
+ return;
970
+ }
971
+ j.lastDueAt = job.lastDueAt;
972
+ j.lastFiredAt = job.lastFiredAt;
973
+ j.owedSlot = dueIso;
974
+ return;
975
+ }
976
+ j.lastCompletedAt = record.finishedAt;
977
+ // A window's row counted its slot when the row was written; a slot collapsed onto it is an overlap.
978
+ // A person's stop is not a run and not a failure, as for a loop's run: the count and the
979
+ // last error stay what they were.
980
+ if (spendsSlot && !collapsedSlot && !stopped) j.runCount++;
981
+ // Only an enabled job's error is stale news: a disabled one's says why it is disabled.
982
+ if (!stopped && (j.enabled || !result.ok)) j.lastError = record.error;
983
+ // A stop is not a failure of the job's: the streak stays what it was.
984
+ if (!stopped) j.consecutiveFailures = result.ok ? undefined : (j.consecutiveFailures ?? 0) + 1;
985
+ j.interruptions = undefined;
986
+ // A one-shot that failed gets one retry, as a one-shot loop does: its stamps go back so
987
+ // it is due again, and the second failure retires it (below). A stopped one does not.
988
+ if (job.schedule.kind === "once" && !result.ok && !stopped && (j.consecutiveFailures ?? 0) < 2) {
989
+ j.lastDueAt = undefined;
990
+ j.lastFiredAt = undefined;
991
+ j.lastError = `${record.error ?? "run failed"} (one-shot: retrying once)`;
992
+ }
993
+ });
994
+ } catch (err: any) {
995
+ this.log(`cron ${job.id}: could not write the store after its turn (${err?.message ?? err})`, undefined, job);
996
+ }
997
+ this.inflight.delete(runId);
998
+ try {
999
+ if (landed || stopped) {
1000
+ // The prompt is in the conversation, or a person stopped the turn: the row is done.
1001
+ // Handed back with the prompt there, the answer was cut off, so a successor row asks
1002
+ // again, once.
1003
+ if (row) this.requests.remove(sessionId, [row.id]);
1004
+ if (handedBack) this.requests.append(plainJobRow(job, newId("run"), `cron:${job.id}`, "", this.createdBy(), this.now()));
1005
+ } else if (row && !handedBack) {
1006
+ // The row's turn failed before its prompt was written: an attempt, as a failed hand-over
1007
+ // in a window is, with the engine's backoff; at the third the engine's drain buries it.
1008
+ const attempts = row.attempts + 1;
1009
+ this.requests.update(sessionId, row.id, { attempts, last_error: record.error, not_before: stamp(this.now() + backoffFor(attempts)) });
1010
+ }
709
1011
  } catch (err: any) {
710
- this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
1012
+ this.log(`cron ${job.id}: could not update its requests after its turn (${err?.message ?? err})`, undefined, job);
711
1013
  }
1014
+ if (handedBack) {
1015
+ this.log(`cron ${job.name ?? job.id}: ${outcome.preempted ? `a window opened session ${shortSessionId(sessionId)} during its turn; that window runs it` : `turn stopped (${record.error}); ${landed ? "a fresh prompt is queued for it" : "the slot is owed"}`}`, "info");
1016
+ return;
1017
+ }
1018
+ if (stopped) this.log(`cron ${job.name ?? job.id}: turn stopped; it is done`, "info");
1019
+ if (updated && !result.ok && !stopped && (updated.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER) {
1020
+ this.hooks.onSchedulerError?.(`${updated.name ?? updated.id} has failed ${updated.consecutiveFailures} times in a row; backing off (last: ${record.error ?? "unknown"})`, updated);
1021
+ }
1022
+ // A one-shot is retired once it ran, was stopped, or failed twice (the retry above was its second chance).
1023
+ if (updated && job.schedule.kind === "once" && (result.ok || stopped || (updated.consecutiveFailures ?? 0) >= 2)) {
1024
+ await this.store.remove(job.id);
1025
+ try {
1026
+ this.hooks.onJobRetired?.(job, result.ok ? "one-shot fired" : stopped ? "one-shot stopped" : `one-shot failed twice (${record.error})`);
1027
+ } catch (err: any) {
1028
+ this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
1029
+ }
1030
+ }
1031
+ } finally {
1032
+ leases.releaseWork?.(sessionId, held.epoch);
712
1033
  }
713
- return true;
714
1034
  }
715
1035
 
716
1036
  private track(run: Promise<void>, slot: SubagentSlot): void {
@@ -741,9 +1061,10 @@ export class LoopScheduler {
741
1061
  /**
742
1062
  * Fire a job now, ignoring its schedule. Returns `true`, or the reason it did not run — phrased
743
1063
  * to follow the job's name, which is the shape `/cron run` has always reported a refusal in
744
- * ("nightly is already running").
1064
+ * ("nightly is already running"). `requestedBy` is the conversation a person asked from; a loop's
1065
+ * record carries it, so the run's card goes there (src/run-card.ts).
745
1066
  */
746
- async runNow(jobId: string): Promise<true | string> {
1067
+ async runNow(jobId: string, requestedBy?: RunRequester): Promise<true | string> {
747
1068
  const job = this.store.load().find((j) => j.id === jobId);
748
1069
  if (!job) return "is no longer in the store";
749
1070
  if (job.running) return "is already running";
@@ -757,18 +1078,24 @@ export class LoopScheduler {
757
1078
  if (!job.enabled) return "is disabled (/cron enable it first)";
758
1079
  // The same reasoning for a job that is enabled but belongs to another session: its prompt is
759
1080
  // written for that conversation, and running it here would land it in this one.
760
- if (!runsIn(job, this.getSession().sessionId)) return `belongs to session ${ownerSession(job)}, which is not open here — resume it, or /cron add the job again in this chat`;
1081
+ const where = whereRuns(job, this.ownerView());
1082
+ if (where.kind === "aside") return `belongs to this conversation, but this window shows an older copy of it — /resume this session to run it`;
1083
+ if (where.kind === "terminal") return `belongs to session ${ownerSession(job)}, which is ${openWhere(where.page ? "page" : "terminal")} — run it there`;
1084
+ if (where.kind === "host") return `belongs to session ${ownerSession(job)}, which is closed: it runs in the background when due — resume that session to run it now`;
1085
+ if (where.kind !== "here") return `belongs to session ${ownerSession(job)}, which is not open here — resume it, or /cron add the job again in this chat`;
761
1086
  // No `dueIso`: this run was asked for, so whatever slot the schedule owes is still owed.
762
- if (!(await this.injectPlainJob(job, undefined, now))) return "has no chat to run in (the background host has no session)";
1087
+ const fired = await this.injectPlainJob(job, undefined, now, "", "cron-run");
1088
+ if (fired === "collapsed") return "already has its prompt queued in this conversation";
1089
+ if (fired === "lost") return this.kind === "host" ? "has no chat to run in (pi-loops in the background has no session)" : "was fired by another window at the same moment";
763
1090
  return true;
764
1091
  }
765
1092
  // `/cron run` is a direct instruction, so a busy machine does not get to refuse it — but it
766
1093
  // still occupies a slot, so the shared count stays honest even when that overruns the limit.
767
- this.track(this.launch(job, undefined, now, this.getSession(), false), this.slots.occupy());
1094
+ this.track(this.launch(job, undefined, now, this.getSession(), false, requestedBy), this.slots.occupy());
768
1095
  return true;
769
1096
  }
770
1097
 
771
- private async launch(job: LoopJob, dueIso: string | undefined, now: number, session: SessionSnapshot, catchingUp: boolean): Promise<void> {
1098
+ private async launch(job: LoopJob, dueIso: string | undefined, now: number, session: SessionSnapshot, catchingUp: boolean, requestedBy?: RunRequester): Promise<void> {
772
1099
  const runId = newId("run");
773
1100
  const startedAt = stamp(now);
774
1101
  // Kept so an aborted run can hand its slot back rather than skipping to the next one.
@@ -776,10 +1103,14 @@ export class LoopScheduler {
776
1103
  const priorFiredAt = job.lastFiredAt;
777
1104
  const claimed = await this.store.update(job.id, (j) => {
778
1105
  if (j.running) return;
1106
+ // The slot was decided from a snapshot. If another process ran or skipped this job since,
1107
+ // its stamps moved: that slot is spent, and running it again would run it twice.
1108
+ if (j.lastDueAt !== priorDueAt || j.lastFiredAt !== priorFiredAt) return;
779
1109
  j.running = { runId, pid: process.pid, host: this.self.host, startedAt };
780
1110
  if (dueIso) j.lastDueAt = dueIso;
781
1111
  j.lastFiredAt = startedAt;
782
1112
  j.lastError = undefined;
1113
+ j.owedSlot = undefined;
783
1114
  });
784
1115
  if (!claimed || claimed.running?.runId !== runId) return;
785
1116
 
@@ -791,7 +1122,7 @@ export class LoopScheduler {
791
1122
  this.hooks.log?.(`onRunStart hook failed: ${err?.message ?? err}`);
792
1123
  }
793
1124
  try {
794
- await this.runOnce(job, claimed, runId, startedAt, ctrl, session, now, catchingUp, { dueAt: priorDueAt, firedAt: priorFiredAt });
1125
+ await this.runOnce(job, claimed, runId, startedAt, ctrl, session, now, catchingUp, { dueAt: priorDueAt, firedAt: priorFiredAt }, requestedBy);
795
1126
  } finally {
796
1127
  // Whatever failed (a state file that cannot be read, a hook that threw), the run id is
797
1128
  // released — otherwise `clearStaleRunning` would call the job alive forever.
@@ -806,7 +1137,7 @@ export class LoopScheduler {
806
1137
  }
807
1138
 
808
1139
  /** The body of one run: everything between claiming the job and writing its record. */
809
- private async runOnce(job: LoopJob, claimed: LoopJob, runId: string, startedAt: string, ctrl: AbortController, session: SessionSnapshot, now: number, catchingUp: boolean, prior: { dueAt?: string; firedAt?: string }): Promise<void> {
1140
+ private async runOnce(job: LoopJob, claimed: LoopJob, runId: string, startedAt: string, ctrl: AbortController, session: SessionSnapshot, now: number, catchingUp: boolean, prior: { dueAt?: string; firedAt?: string }, requestedBy?: RunRequester): Promise<void> {
810
1141
  const previousState = this.store.readState(job.id);
811
1142
  // `job` is the pre-claim snapshot, so its `lastFiredAt` is when the previous run started —
812
1143
  // a reason given before that was already in front of that run.
@@ -831,6 +1162,7 @@ export class LoopScheduler {
831
1162
  model,
832
1163
  thinking,
833
1164
  tools: job.tools,
1165
+ builtinTools: job.builtinTools,
834
1166
  timeoutMs: job.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS,
835
1167
  signal: ctrl.signal,
836
1168
  sessionDir: this.store.sessionDirFor(job.id),
@@ -862,9 +1194,9 @@ export class LoopScheduler {
862
1194
  try {
863
1195
  stillThere = this.store.load().some((j) => j.id === job.id);
864
1196
  } catch (err: any) {
865
- this.log(`loop ${job.id}: could not read the store after the run (${err?.message ?? err}); keeping its state and findings`);
1197
+ this.log(`loop ${job.id}: could not read the store after the run (${err?.message ?? err}); keeping its state and findings`, undefined, job);
866
1198
  }
867
- if (!stillThere) this.log(`loop ${job.id}: removed during the run; its state and findings are discarded`);
1199
+ if (!stillThere) this.log(`loop ${job.id}: removed during the run; its state and findings are discarded`, undefined, job);
868
1200
 
869
1201
  // Tag extraction never fails a run: malformed/missing tags leave the state untouched.
870
1202
  const parsed = parseRunOutput(result.text);
@@ -874,7 +1206,7 @@ export class LoopScheduler {
874
1206
  this.store.writeState(job.id, parsed.state);
875
1207
  stateUpdated = true;
876
1208
  } catch (err: any) {
877
- this.log(`loop ${job.id}: state write failed: ${err?.message ?? err}`);
1209
+ this.log(`loop ${job.id}: state write failed: ${err?.message ?? err}`, undefined, job);
878
1210
  }
879
1211
  }
880
1212
  // Maker/checker: with verify=true a second, adversarial sub-agent reviews the
@@ -893,15 +1225,23 @@ export class LoopScheduler {
893
1225
  const source = `cron:${job.name ?? job.id.slice(0, "cron-".length + 8)}`;
894
1226
  for (const f of reviewed) {
895
1227
  try {
896
- await this.inbox.append({ source, text: f.text, run_id: runId, job_id: job.id, cwd: job.cwd, kind: findingKind(f.text), verified: f.verified, verified_reason: f.reason });
1228
+ await this.inbox.append({ source, text: f.text, run_id: runId, job_id: job.id, cwd: job.cwd, kind: findingKind(f.text), verified: f.verified, verified_reason: f.reason, session_id: job.createdBy?.sessionId, transcript: result.sessionFile });
897
1229
  findings.push(f.text);
898
1230
  } catch (err: any) {
899
- this.log(`loop ${job.id}: inbox append failed: ${err?.message ?? err}`);
1231
+ this.log(`loop ${job.id}: inbox append failed: ${err?.message ?? err}`, undefined, job);
900
1232
  }
901
1233
  }
902
1234
  }
903
1235
 
904
1236
  const finishedAt = stamp(this.now());
1237
+ // An abort is not a run: quitting, a session swap, a preemption or the budget interrupted it,
1238
+ // so the slot it claimed is given back and the next tick re-fires it (a tick is never lost this
1239
+ // way because its runs die with the session that owned them).
1240
+ const aborted = !result.ok && (result.stopReason === "aborted" || ctrl.signal.aborted);
1241
+ // A person's stop is the exception: they stopped this slot, so it is spent and the next run is
1242
+ // on schedule. Giving it back would run what they just stopped again on the next tick. A run
1243
+ // that finished before the stop reached it is `ok`, and its record says so.
1244
+ const byPerson = aborted && ctrl.signal.aborted && ctrl.signal.reason === STOPPED_BY_PERSON;
905
1245
  const record: RunRecord = {
906
1246
  runId,
907
1247
  jobId: job.id,
@@ -913,7 +1253,7 @@ export class LoopScheduler {
913
1253
  finishedAt,
914
1254
  ok: result.ok,
915
1255
  exitCode: result.exitCode,
916
- error: result.ok ? undefined : redact(result.errorMessage ?? "unknown error"),
1256
+ error: result.ok ? undefined : byPerson ? STOPPED_BY_PERSON : redact(result.errorMessage ?? "unknown error"),
917
1257
  findings: findings.length,
918
1258
  droppedFindings: parsed.droppedFindings,
919
1259
  stateUpdated,
@@ -926,17 +1266,18 @@ export class LoopScheduler {
926
1266
  sessionId: result.sessionId,
927
1267
  sessionFile: result.sessionFile,
928
1268
  checker,
1269
+ forSession: job.createdBy?.sessionId,
1270
+ ordinal: claimed.runCount + 1,
1271
+ ...(requestedBy ? { requestedBy } : {}),
1272
+ ...(aborted ? { aborted: true as const } : {}),
1273
+ ...(byPerson ? { stoppedBy: "person" as const, stoppedFrom: this.inflight.get(runId)?.stoppedFrom ?? "cli" } : {}),
929
1274
  };
930
- this.store.pruneSessions(job.id);
1275
+ this.store.pruneSessions(job.id, undefined, this.inbox.pinnedTranscripts(job.id));
931
1276
  try {
932
1277
  this.store.appendRun(record);
933
1278
  } catch (err: any) {
934
1279
  this.log(`run log append failed: ${err?.message ?? err}`);
935
1280
  }
936
- // An abort is not a run: quitting, a session swap or `/cron abort` interrupted it, so the
937
- // slot it claimed is given back and the next tick re-fires it (a tick is never lost this
938
- // way because its runs die with the session that owned them).
939
- const aborted = !result.ok && (result.stopReason === "aborted" || ctrl.signal.aborted);
940
1281
  // The same unreadable store that was tolerated above must not cost the run its `onRunFinished`
941
1282
  // either: the record is already written, and the job's own bookkeeping is the one part that
942
1283
  // can be skipped when the file cannot be read.
@@ -945,6 +1286,16 @@ export class LoopScheduler {
945
1286
  updated = await this.store.update(job.id, (j) => {
946
1287
  if (j.running?.runId === runId) j.running = undefined;
947
1288
  j.lastCompletedAt = finishedAt;
1289
+ if (byPerson) {
1290
+ // The slot is spent; `runCount` stays (the `ordinal` comment on RunRecord), and a stop is
1291
+ // not a failure, so the streak and the last error stay what they were. A one-shot has no
1292
+ // next slot: it is turned off, not deleted, so its notes are kept.
1293
+ if (job.schedule.kind === "once") {
1294
+ j.enabled = false;
1295
+ j.lastError = `${STOPPED_BY_PERSON}; /cron run ${job.name ?? job.id} runs it now`;
1296
+ }
1297
+ return;
1298
+ }
948
1299
  j.lastError = result.ok ? undefined : record.error;
949
1300
  if (aborted) {
950
1301
  j.lastDueAt = prior.dueAt;
@@ -954,13 +1305,14 @@ export class LoopScheduler {
954
1305
  // A job that fails every time costs money every time. Count the streak so `dispatch`
955
1306
  // can back off, and clear it the moment one run works.
956
1307
  j.consecutiveFailures = result.ok ? undefined : (j.consecutiveFailures ?? 0) + 1;
1308
+ j.interruptions = undefined;
957
1309
  }
958
1310
  });
959
1311
  } catch (err: any) {
960
- this.log(`loop ${job.id}: could not write the store after the run (${err?.message ?? err})`);
1312
+ this.log(`loop ${job.id}: could not write the store after the run (${err?.message ?? err})`, undefined, job);
961
1313
  }
962
- if (updated && !result.ok && (updated.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER) {
963
- this.hooks.onSchedulerError?.(`${updated.name ?? updated.id} has failed ${updated.consecutiveFailures} times in a row; backing off (last: ${record.error ?? "unknown"})`);
1314
+ if (updated && !result.ok && !byPerson && (updated.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER) {
1315
+ this.hooks.onSchedulerError?.(`${updated.name ?? updated.id} has failed ${updated.consecutiveFailures} times in a row; backing off (last: ${record.error ?? "unknown"})`, updated);
964
1316
  }
965
1317
  // Only now: while the run id is in `inflight`, `clearStaleRunning` treats the job as alive,
966
1318
  // which is exactly right until its `running` marker is gone from the store.
@@ -1013,6 +1365,7 @@ export class LoopScheduler {
1013
1365
  model,
1014
1366
  thinking,
1015
1367
  tools: job.tools,
1368
+ builtinTools: job.builtinTools,
1016
1369
  timeoutMs: job.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS,
1017
1370
  signal,
1018
1371
  sessionDir: this.store.sessionDirFor(job.id),
@@ -1049,7 +1402,7 @@ export class LoopScheduler {
1049
1402
  model: result.model ?? model,
1050
1403
  sessionFile: result.sessionFile,
1051
1404
  };
1052
- if (!result.ok) this.log(`loop ${job.name ?? job.id}: checker failed (${record.error}); findings enter the inbox unverified`);
1405
+ if (!result.ok) this.log(`loop ${job.name ?? job.id}: checker failed (${record.error}); findings enter the inbox unverified`, undefined, job);
1053
1406
  return { record, reviewed };
1054
1407
  }
1055
1408
  }