@alphacoder-v0/pi-loops 0.22.7 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +681 -0
- package/CONTRACT.md +7 -7
- package/README.md +91 -63
- package/README.zh-CN.md +59 -39
- package/docs/cli.md +272 -83
- package/docs/configuration.md +86 -24
- package/docs/design.md +395 -56
- package/docs/downstream.md +2 -2
- package/docs/goal.md +12 -3
- package/docs/hooks.md +6 -7
- package/docs/loops.md +288 -59
- package/docs/mcp.md +22 -14
- package/docs/recipes.md +6 -0
- package/docs/screenshot.png +0 -0
- package/docs/triggers.md +120 -26
- package/docs/troubleshooting.md +37 -26
- package/docs/web-ui-parity.md +691 -65
- package/package.json +4 -3
- package/recipes/autoresearch/recipe.toml +1 -0
- package/recipes/changelog-draft/recipe.toml +1 -0
- package/recipes/ci-sweeper/recipe.toml +1 -0
- package/recipes/daily-digest/recipe.toml +1 -0
- package/recipes/deps-sweeper/recipe.toml +1 -0
- package/recipes/ecosystem/recipe.toml +1 -0
- package/recipes/issue-loop/recipe.toml +1 -0
- package/recipes/pr-watch/recipe.toml +1 -0
- package/skills/pi-loops/SKILL.md +6 -4
- package/src/activity.ts +199 -0
- package/src/archive.ts +21 -9
- package/src/args.ts +6 -2
- package/src/cli-entry.mjs +12 -3
- package/src/cli.ts +746 -125
- package/src/config.ts +53 -5
- package/src/conversation-rows.ts +469 -0
- package/src/danger.ts +227 -6
- package/src/engine.ts +1240 -0
- package/src/goal.ts +71 -0
- package/src/held-read.ts +27 -0
- package/src/hooks.ts +49 -18
- package/src/host-control-channel.ts +455 -40
- package/src/host-control.ts +602 -111
- package/src/host-entry.mjs +5 -3
- package/src/host-runtime.ts +455 -41
- package/src/host.ts +659 -123
- package/src/inbox-card.ts +278 -0
- package/src/inbox.ts +139 -62
- package/src/job-edit.ts +7 -1
- package/src/job-health.ts +14 -2
- package/src/job-lines.ts +201 -13
- package/src/job-owner.ts +74 -13
- package/src/job-tune.ts +110 -0
- package/src/lease.ts +537 -0
- package/src/lock.ts +7 -2
- package/src/mcp-pool.ts +66 -14
- package/src/mcp.ts +1 -1
- package/src/open-file.ts +54 -0
- package/src/page-children.ts +1507 -0
- package/src/page-devices.ts +192 -0
- package/src/page-server.ts +3150 -0
- package/src/path-precheck.ts +215 -0
- package/src/paths.ts +24 -0
- package/src/pi-floor.ts +10 -0
- package/src/pi-loops.ts +1301 -450
- package/src/pi-resolver.d.mts +1 -0
- package/src/pi-resolver.mjs +110 -10
- package/src/presence.ts +12 -63
- package/src/protocol.ts +10 -1
- package/src/recipe.ts +43 -1
- package/src/requests.ts +354 -0
- package/src/rule-lines.ts +68 -0
- package/src/run-card.ts +105 -0
- package/src/runner.ts +18 -1
- package/src/schedule.ts +1 -2
- package/src/scheduler.ts +598 -245
- package/src/sdk-runner.ts +321 -27
- package/src/seen.ts +278 -0
- package/src/session-head.ts +21 -0
- package/src/share.ts +12 -2
- package/src/snapshot.ts +3 -2
- package/src/store.ts +81 -8
- package/src/tools.ts +196 -65
- package/src/trigger-runtime.ts +164 -127
- package/src/triggers.ts +39 -7
- package/src/trust.ts +46 -2
- package/src/ui-prefs.ts +16 -5
- package/src/watch-plan.ts +174 -0
- package/src/web.d.mts +3 -0
- package/src/web.mjs +2260 -2340
- package/src/window-executor.ts +459 -0
- package/src/window-guard.ts +158 -0
- package/docs/agents/issue-tracker.md +0 -34
package/src/scheduler.ts
CHANGED
|
@@ -1,36 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* any other open pi keeps it ticking; restart pi and missed ticks are caught up
|
|
7
|
-
* (once, collapsed) unless the job opted out.
|
|
8
|
-
*
|
|
9
|
-
* Inject-mode jobs are session-bound: they only fire from the process whose
|
|
10
|
-
* current session is the one that created them.
|
|
2
|
+
* The clock. The same scheduler runs in two roles, and only what it owns differs: the always-on
|
|
3
|
+
* host runs every loop, and a pi window runs the plain jobs of the conversation whose lease it
|
|
4
|
+
* holds (src/lease.ts). Nobody is elected. A missed tick is caught up once, collapsed, unless the
|
|
5
|
+
* job opted out.
|
|
11
6
|
*/
|
|
12
7
|
import * as fs from "node:fs";
|
|
13
8
|
import * as os from "node:os";
|
|
14
9
|
import { PresenceRegistry, type PresenceEntry, type PresenceKind, type PresenceSelf } from "./presence.ts";
|
|
15
10
|
import * as path from "node:path";
|
|
16
11
|
import { Inbox } from "./inbox.ts";
|
|
17
|
-
import { pidAlive,
|
|
12
|
+
import { pidAlive, writeFileAtomic } from "./lock.ts";
|
|
18
13
|
import { composeCheckerPrompt, composeLoopPrompt, findingKind, parseCheckerOutput, parseRunOutput, stripProtocolTags } from "./protocol.ts";
|
|
19
14
|
import { type RunnerResult, type SubagentRunner, failedRun } from "./runner.ts";
|
|
20
15
|
import { previewRedacted, redact } from "./redact.ts";
|
|
21
16
|
import { type SubagentSlot, SubagentSlots } from "./slots.ts";
|
|
22
17
|
import { backoffWaitMs, computeDue, computeNext, formatLocal, formatLocalZoned, formatSchedule, isValidSchedule, stamp } from "./schedule.ts";
|
|
23
|
-
import { type CheckerRecord, JobStore, type LoopJob, type RunRecord, newId } from "./store.ts";
|
|
24
|
-
import { ownerSession,
|
|
18
|
+
import { type CheckerRecord, JobStore, type LoopJob, type RunRecord, type RunRequester, newId } from "./store.ts";
|
|
19
|
+
import { openWhere, ownerSession, runsHere, whereRuns, type OwnerView } from "./job-owner.ts";
|
|
20
|
+
import { shortSessionId } from "./session-head.ts";
|
|
21
|
+
import { DEAD_AFTER_ATTEMPTS, backoffFor, plainCollapseKey, plainJobRow, plainRunId, sessionFileHas } from "./engine.ts";
|
|
22
|
+
import { RequestStore, type SessionRequest } from "./requests.ts";
|
|
23
|
+
import { PI_LOOPS_VERSION } from "./version.ts";
|
|
25
24
|
|
|
26
25
|
export const DEFAULT_TICK_MS = 30_000;
|
|
27
|
-
|
|
26
|
+
/** Interrupted runs in a row (a process that died mid-run) after which a job is parked. */
|
|
27
|
+
export const INTERRUPTIONS_BEFORE_PARKING = 3;
|
|
28
28
|
/** Consecutive failures before a job is retried on a widening gap rather than every due tick. */
|
|
29
29
|
export const FAILURE_BACKOFF_AFTER = 3;
|
|
30
30
|
export const FAILURE_BACKOFF_BASE_MS = 5 * 60_000;
|
|
31
31
|
export const FAILURE_BACKOFF_MAX_MS = 6 * 60 * 60_000;
|
|
32
|
-
/** A run claimed by another machine (shared $HOME) is only assumed dead after this long. */
|
|
33
|
-
export const FOREIGN_RUN_STALE_MS = 24 * 60 * 60 * 1000;
|
|
34
32
|
export const DEFAULT_RUN_TIMEOUT_MS = 15 * 60_000;
|
|
35
33
|
export const MAX_CONCURRENT_RUNS = 3;
|
|
36
34
|
/**
|
|
@@ -47,6 +45,8 @@ export interface SessionSnapshot {
|
|
|
47
45
|
thinking?: string;
|
|
48
46
|
/** The interactive pi trusted this project (`pi --approve` / trust prompt); children in the same cwd inherit it. */
|
|
49
47
|
trusted?: boolean;
|
|
48
|
+
/** The built-in tools active in this chat (`--tools`, `--exclude-tools`); a loop made here keeps them. */
|
|
49
|
+
builtinTools?: string[];
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
export interface SchedulerSettings {
|
|
@@ -57,7 +57,7 @@ export interface SchedulerSettings {
|
|
|
57
57
|
maxConcurrentRuns: number;
|
|
58
58
|
/** Stop dispatching once today's automation has cost this much. 0 or absent = no cap. */
|
|
59
59
|
dailyBudgetUsd?: number;
|
|
60
|
-
/** Global switch for firing ticks missed while
|
|
60
|
+
/** Global switch for firing ticks missed while this scheduler was not running (`[cron] catch_up`); per-job `catchUp` still applies. */
|
|
61
61
|
catchUp: boolean;
|
|
62
62
|
}
|
|
63
63
|
|
|
@@ -69,21 +69,28 @@ export interface RunOutcome {
|
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
export interface SchedulerHooks {
|
|
72
|
-
/**
|
|
73
|
-
|
|
72
|
+
/**
|
|
73
|
+
* The host: run `prompt` as a turn in the closed conversation it now holds. Resolves when the turn
|
|
74
|
+
* is over with the run's result; `preempted` when a window took the conversation back before the
|
|
75
|
+
* turn completed (a turn that completed is done, whoever holds the lease now). `landed` says
|
|
76
|
+
* whether the prompt's `[Trigger <id>]` entry is in the conversation; absent, a completed turn is
|
|
77
|
+
* taken to have landed it.
|
|
78
|
+
*/
|
|
79
|
+
onHeldTurn?: (job: LoopJob, prompt: string, runId: string, held: HeldSession, signal: AbortSignal) => Promise<{ preempted: boolean; result: RunnerResult; landed?: boolean }>;
|
|
74
80
|
onRunStart?: (job: LoopJob, runId: string) => void;
|
|
75
|
-
/** A run is being fired for a tick that was missed while
|
|
81
|
+
/** A run is being fired for a tick that was missed while this scheduler was not running. */
|
|
76
82
|
onCatchUp?: (job: LoopJob, dueAt: number) => void;
|
|
77
83
|
onRunFinished?: (outcome: RunOutcome) => void;
|
|
78
84
|
/** A one-shot removed itself: the control plane records it the way a person's removal is recorded. */
|
|
79
85
|
onJobRetired?: (job: LoopJob, reason: string) => void;
|
|
86
|
+
/** A plain job's row this process wrote into its conversation's queue (the clock's, or `/cron run`'s): the window records it as its own, and says so when it waits. */
|
|
87
|
+
onRowWritten?: (row: SessionRequest, job: LoopJob) => void;
|
|
80
88
|
onInboxChanged?: () => void;
|
|
81
|
-
/** Every tick,
|
|
82
|
-
onTick?: (now: number
|
|
83
|
-
/** Leadership gained or lost. */
|
|
84
|
-
onLeadership?: (leader: boolean) => void | Promise<void>;
|
|
89
|
+
/** Every tick, before dispatch. Other subsystems (leases, dynamic triggers) piggyback on it. */
|
|
90
|
+
onTick?: (now: number) => void | Promise<void>;
|
|
85
91
|
/** A whole tick failed: nothing ran. A failure that needs a person, where `log` is a diagnostic at either level. */
|
|
86
|
-
|
|
92
|
+
/** `job` is the one the message is about, when there is one (its cwd decides which windows hear it). */
|
|
93
|
+
onSchedulerError?: (message: string, job?: Pick<LoopJob, "cwd">) => void;
|
|
87
94
|
/** Today's spend passed the configured cap; nothing more will be dispatched today. */
|
|
88
95
|
onBudgetExceeded?: (spent: number, cap: number) => void;
|
|
89
96
|
/**
|
|
@@ -92,7 +99,46 @@ export interface SchedulerHooks {
|
|
|
92
99
|
* a regex over these strings. Absent `level` means `"warning"`: a line that does not say it is
|
|
93
100
|
* routine is not.
|
|
94
101
|
*/
|
|
95
|
-
|
|
102
|
+
/** `job` is the one the line is about, so a window of its project can say it (src/host-control-channel.ts Notice). */
|
|
103
|
+
log?: (message: string, level?: "info" | "warning", job?: Pick<LoopJob, "cwd">) => void;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The lease side of plain-job ownership (src/lease.ts): a plain job fires only in the instance that
|
|
108
|
+
* holds its session's lease, and its claim re-checks that the lease has not moved since.
|
|
109
|
+
*/
|
|
110
|
+
export interface PlainJobLeases {
|
|
111
|
+
/** The epoch of this instance's lease on `sessionId`, when it holds it. */
|
|
112
|
+
heldEpoch(sessionId: string): number | undefined;
|
|
113
|
+
/** The pid of the live holder of `sessionId`'s lease, when there is one. */
|
|
114
|
+
holderOf(sessionId: string): number | undefined;
|
|
115
|
+
/** The kind of the live holder, when there is one. */
|
|
116
|
+
holderKind?(sessionId: string): "terminal" | "page" | "host" | undefined;
|
|
117
|
+
/** Windows: whether this one holds its conversation's lease but stands aside (OwnerView.standsAside). */
|
|
118
|
+
standsAside?(): boolean;
|
|
119
|
+
/** Windows: whether the host fires this closed conversation's plain jobs (it runs, and knows the file). */
|
|
120
|
+
hostRuns?(sessionId: string): boolean;
|
|
121
|
+
/** Why no host runs anything right now, when that is so (see OwnerView.hostPaused). */
|
|
122
|
+
hostPaused?(): string | undefined;
|
|
123
|
+
/** The host: take a closed conversation for one turn (src/lease.ts acquireForHost), or undefined. */
|
|
124
|
+
acquireForWork?(sessionId: string, runId: string, jobId: string): HeldSession | undefined;
|
|
125
|
+
/** The host: give the conversation back after the turn. A window that took it meanwhile keeps it. */
|
|
126
|
+
releaseWork?(sessionId: string, epoch: number): void;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The `AbortSignal.reason` of a run a person stopped: `/triggers abort`, `pi-loops
|
|
131
|
+
* abort`, a stop row. A held turn stopped so is done, not handed back: no successor row, no owed slot.
|
|
132
|
+
* A quit, an upgrade, a preemption or the budget aborts with no reason and hands the turn back.
|
|
133
|
+
*/
|
|
134
|
+
export const STOPPED_BY_PERSON = "stopped by a person";
|
|
135
|
+
|
|
136
|
+
/** A closed conversation the host holds for one turn. */
|
|
137
|
+
export interface HeldSession {
|
|
138
|
+
sessionId: string;
|
|
139
|
+
epoch: number;
|
|
140
|
+
sessionFile: string;
|
|
141
|
+
cwd?: string;
|
|
96
142
|
}
|
|
97
143
|
|
|
98
144
|
export interface SchedulerOptions {
|
|
@@ -107,16 +153,27 @@ export interface SchedulerOptions {
|
|
|
107
153
|
hop?: number;
|
|
108
154
|
/** Live settings (re-read on every use so a config reload takes effect). */
|
|
109
155
|
getSettings?: () => SchedulerSettings;
|
|
156
|
+
/** A window: whether it should write next-runs.json this tick (no host is running to do it). */
|
|
157
|
+
writesNextRuns?: () => boolean;
|
|
110
158
|
/** Does pi still have this session? Plain jobs of deleted sessions are disabled (and `gc()` removes them). */
|
|
111
159
|
sessionExists?: (sessionId: string) => boolean;
|
|
112
|
-
/** "host" = the
|
|
160
|
+
/** "host" = the always-on process: it also writes next-runs.json and sweeps dead sessions. Default "interactive". */
|
|
113
161
|
kind?: PresenceKind;
|
|
162
|
+
/**
|
|
163
|
+
* Which jobs this scheduler dispatches. The host passes "loops only", a window "the plain jobs of
|
|
164
|
+
* the conversation it holds". Omitted (tests), it owns every loop and the plain jobs `view` says run here.
|
|
165
|
+
*/
|
|
166
|
+
owns?: (job: LoopJob, view: OwnerView) => boolean;
|
|
114
167
|
/**
|
|
115
168
|
* The process-wide sub-agent admission counter, shared with the trigger runtime and the /goal
|
|
116
169
|
* evaluator (src/slots.ts). Omitted, the scheduler bounds only itself against
|
|
117
170
|
* `maxConcurrentRuns` — which is the bug this exists to close, so production always passes one.
|
|
118
171
|
*/
|
|
119
172
|
slots?: SubagentSlots;
|
|
173
|
+
/** Omitted, the session of `getSession()` counts as held by this instance (tests, the host, which has none). */
|
|
174
|
+
leases?: PlainJobLeases;
|
|
175
|
+
/** The extension instance this scheduler runs for: the same id its session lease names. */
|
|
176
|
+
instance?: string;
|
|
120
177
|
}
|
|
121
178
|
|
|
122
179
|
export const DEAD_SESSION_MARKER = "no longer exists (/cron gc removes it)";
|
|
@@ -130,23 +187,12 @@ const DEAD_SESSION_SCAN_MS = 10 * 60_000;
|
|
|
130
187
|
*/
|
|
131
188
|
const CWD_GRACE_MS = 30 * 60_000;
|
|
132
189
|
|
|
133
|
-
interface LeaderRecord {
|
|
134
|
-
pid: number;
|
|
135
|
-
host: string;
|
|
136
|
-
/** Distinguishes scheduler instances inside one process (reload, tests). */
|
|
137
|
-
instance: string;
|
|
138
|
-
sessionId?: string;
|
|
139
|
-
kind?: PresenceKind;
|
|
140
|
-
startedAt: string;
|
|
141
|
-
heartbeatAt: string;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
190
|
export class LoopScheduler {
|
|
145
191
|
readonly store: JobStore;
|
|
146
192
|
readonly inbox: Inbox;
|
|
193
|
+
/** The conversations' request queues: a plain job's prompt is a row here before its slot is spent. */
|
|
194
|
+
readonly requests: RequestStore;
|
|
147
195
|
private readonly dir: string;
|
|
148
|
-
private readonly leaderFile: string;
|
|
149
|
-
private readonly leaderLock: string;
|
|
150
196
|
private readonly getSession: () => SessionSnapshot;
|
|
151
197
|
private readonly hooks: SchedulerHooks;
|
|
152
198
|
private readonly tickMs: number;
|
|
@@ -154,6 +200,7 @@ export class LoopScheduler {
|
|
|
154
200
|
private readonly now: () => number;
|
|
155
201
|
private readonly hop: number;
|
|
156
202
|
private readonly getSettings: () => SchedulerSettings;
|
|
203
|
+
private readonly writesNextRuns?: () => boolean;
|
|
157
204
|
/** Every sub-agent this process starts takes a slot from here, loop runs and trigger checks alike. */
|
|
158
205
|
readonly slots: SubagentSlots;
|
|
159
206
|
private readonly sessionExists?: (sessionId: string) => boolean;
|
|
@@ -164,14 +211,15 @@ export class LoopScheduler {
|
|
|
164
211
|
private readonly nextRunsFile: string;
|
|
165
212
|
private timer: NodeJS.Timeout | undefined;
|
|
166
213
|
private ticking = false;
|
|
167
|
-
private leader = false;
|
|
168
214
|
private startedAt = 0;
|
|
169
|
-
private readonly inflight = new Map<string, { ctrl: AbortController; label: string; jobId: string; startedAt: string; promptPreview: string; sessionFile?: string }>();
|
|
215
|
+
private readonly inflight = new Map<string, { ctrl: AbortController; label: string; jobId: string; startedAt: string; promptPreview: string; sessionFile?: string; stoppedFrom?: "page" | "cli" }>();
|
|
170
216
|
/** Promises of runs in flight, so stop() can wait for their records to land. */
|
|
171
217
|
private readonly runs = new Set<Promise<void>>();
|
|
172
218
|
private stopped = false;
|
|
173
219
|
private tickPromise?: Promise<void>;
|
|
174
|
-
|
|
220
|
+
readonly instance: string;
|
|
221
|
+
private readonly leases?: PlainJobLeases;
|
|
222
|
+
private readonly owns: (job: LoopJob, view: OwnerView) => boolean;
|
|
175
223
|
/** This process in the machine-wide presence registry (which pi is open where). */
|
|
176
224
|
readonly presence: PresenceRegistry;
|
|
177
225
|
readonly self: PresenceSelf;
|
|
@@ -180,16 +228,11 @@ export class LoopScheduler {
|
|
|
180
228
|
this.dir = opts.dir;
|
|
181
229
|
this.store = new JobStore(opts.dir);
|
|
182
230
|
this.inbox = new Inbox(opts.dir);
|
|
183
|
-
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
|
|
187
|
-
// One plain file beside the leader record. A cron expression is matched against local time, so
|
|
188
|
-
// if two machines sharing a `$HOME` both ran, they would write different answers over one
|
|
189
|
-
// another and the panel would show whichever wrote last — which is why that arrangement is
|
|
190
|
-
// unsupported rather than handled (docs/design.md §19).
|
|
231
|
+
this.requests = new RequestStore(opts.dir);
|
|
232
|
+
// Written by the host alone. A cron expression is matched against local time, so if two
|
|
233
|
+
// machines sharing a `$HOME` both ran, they would write different answers over one another —
|
|
234
|
+
// which is why that arrangement is unsupported rather than handled (docs/design.md §19).
|
|
191
235
|
this.nextRunsFile = path.join(opts.dir, "next-runs.json");
|
|
192
|
-
this.leaderLock = path.join(opts.dir, "scheduler.lock");
|
|
193
236
|
this.getSession = opts.getSession;
|
|
194
237
|
this.hooks = opts.hooks ?? {};
|
|
195
238
|
this.tickMs = opts.tickMs ?? DEFAULT_TICK_MS;
|
|
@@ -197,23 +240,31 @@ export class LoopScheduler {
|
|
|
197
240
|
this.now = opts.now ?? Date.now;
|
|
198
241
|
this.hop = opts.hop ?? 0;
|
|
199
242
|
this.getSettings = opts.getSettings ?? (() => ({ maxConcurrentRuns: MAX_CONCURRENT_RUNS, catchUp: true }));
|
|
243
|
+
this.writesNextRuns = opts.writesNextRuns;
|
|
200
244
|
this.slots = opts.slots ?? new SubagentSlots(() => this.getSettings().maxConcurrentRuns);
|
|
245
|
+
this.instance = opts.instance ?? newId("sched");
|
|
246
|
+
this.leases = opts.leases;
|
|
247
|
+
this.owns = opts.owns ?? ((job, view) => runsHere(job, view));
|
|
201
248
|
this.sessionExists = opts.sessionExists;
|
|
202
249
|
this.kind = opts.kind ?? "interactive";
|
|
203
250
|
const session = opts.getSession();
|
|
204
|
-
this.self = { pid: process.pid, host: os.hostname(), instance: this.instance, sessionId: session.sessionId, cwd: session.cwd, kind: this.kind };
|
|
251
|
+
this.self = { pid: process.pid, host: os.hostname(), instance: this.instance, sessionId: session.sessionId, cwd: session.cwd, kind: this.kind, leases: true };
|
|
205
252
|
this.presence = new PresenceRegistry(opts.dir, this.self);
|
|
206
253
|
}
|
|
207
254
|
|
|
255
|
+
/** What this instance knows about who holds which session: the lists and dispatch ask it the same way. */
|
|
256
|
+
ownerView(session: SessionSnapshot = this.getSession()): OwnerView {
|
|
257
|
+
const sessionId = session.sessionId;
|
|
258
|
+
const leases = this.leases;
|
|
259
|
+
if (!leases) return { sessionId, holdsOwn: !!sessionId };
|
|
260
|
+
return { sessionId, holdsOwn: !!sessionId && leases.heldEpoch(sessionId) !== undefined, holderOf: (id) => leases.holderOf(id), standsAside: leases.standsAside?.(), holderKind: leases.holderKind ? (id) => leases.holderKind!(id) : undefined, hostRuns: leases.hostRuns ? (id) => leases.hostRuns!(id) : undefined, hostPaused: leases.hostPaused?.() };
|
|
261
|
+
}
|
|
262
|
+
|
|
208
263
|
/** Live pi processes on every host (stale entries pruned). */
|
|
209
264
|
presenceList(now = this.now()): PresenceEntry[] {
|
|
210
265
|
return this.presence.list(now);
|
|
211
266
|
}
|
|
212
267
|
|
|
213
|
-
get isLeader(): boolean {
|
|
214
|
-
return this.leader;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
268
|
get runningCount(): number {
|
|
218
269
|
return this.inflight.size;
|
|
219
270
|
}
|
|
@@ -228,14 +279,26 @@ export class LoopScheduler {
|
|
|
228
279
|
return [...this.inflight.entries()].map(([runId, r]) => ({ runId, label: r.label, jobId: r.jobId, startedAt: r.startedAt, promptPreview: r.promptPreview, sessionFile: r.sessionFile }));
|
|
229
280
|
}
|
|
230
281
|
|
|
231
|
-
/** Abort one in-flight run (kills the sub-agent)
|
|
232
|
-
abortRun(runId: string): boolean {
|
|
282
|
+
/** Abort one in-flight run (kills the sub-agent), as a person asked (STOPPED_BY_PERSON), from `from`. */
|
|
283
|
+
abortRun(runId: string, from: "page" | "cli" = "cli"): boolean {
|
|
233
284
|
const r = this.inflight.get(runId);
|
|
234
285
|
if (!r) return false;
|
|
235
|
-
r.
|
|
286
|
+
r.stoppedFrom ??= from;
|
|
287
|
+
r.ctrl.abort(STOPPED_BY_PERSON);
|
|
236
288
|
return true;
|
|
237
289
|
}
|
|
238
290
|
|
|
291
|
+
/** Where a person's stop of `runId` came from, while the run is still in flight. */
|
|
292
|
+
stopFrom(runId: string): "page" | "cli" | undefined {
|
|
293
|
+
return this.inflight.get(runId)?.stoppedFrom;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Where the stop on its way to `runId` came from, for a stop that arrives through a request row. */
|
|
297
|
+
noteStopFrom(runId: string, from: "page" | "cli"): void {
|
|
298
|
+
const r = this.inflight.get(runId);
|
|
299
|
+
if (r) r.stoppedFrom = from;
|
|
300
|
+
}
|
|
301
|
+
|
|
239
302
|
/**
|
|
240
303
|
* Remove a job. A run of it that is in flight is aborted first and given up to
|
|
241
304
|
* REMOVE_DRAIN_MS to write its record, so the removal does not race the run's end.
|
|
@@ -256,7 +319,7 @@ export class LoopScheduler {
|
|
|
256
319
|
return { job, aborted: runIds.length };
|
|
257
320
|
}
|
|
258
321
|
|
|
259
|
-
/** Idempotent. Starts the tick timer
|
|
322
|
+
/** Idempotent. Starts the tick timer. */
|
|
260
323
|
start(): void {
|
|
261
324
|
if (this.timer) return;
|
|
262
325
|
this.stopped = false;
|
|
@@ -270,93 +333,23 @@ export class LoopScheduler {
|
|
|
270
333
|
safeTick();
|
|
271
334
|
}
|
|
272
335
|
|
|
273
|
-
/** Idempotent. Stops the timer
|
|
336
|
+
/** Idempotent. Stops the timer and aborts in-flight runs. */
|
|
274
337
|
async stop(): Promise<void> {
|
|
275
338
|
this.stopped = true;
|
|
276
339
|
if (this.timer) {
|
|
277
340
|
clearInterval(this.timer);
|
|
278
341
|
this.timer = undefined;
|
|
279
342
|
}
|
|
280
|
-
await this.tickPromise; // a tick
|
|
343
|
+
await this.tickPromise; // a tick already dispatching must not launch after we leave
|
|
281
344
|
for (const { ctrl } of this.inflight.values()) ctrl.abort();
|
|
282
345
|
// What is lost past this is the record, not the run: the sub-agent is already aborted, and
|
|
283
346
|
// waiting longer only delays the quit that asked for it.
|
|
284
347
|
const DRAIN_TIMEOUT_MS = 5_000;
|
|
285
348
|
await this.drain(DRAIN_TIMEOUT_MS);
|
|
286
|
-
await this.releaseLeadership();
|
|
287
|
-
this.presence.remove();
|
|
288
349
|
}
|
|
289
350
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
return JSON.parse(fs.readFileSync(this.leaderFile, "utf8")) as LeaderRecord;
|
|
293
|
-
} catch {
|
|
294
|
-
return undefined;
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
private isMine(rec: LeaderRecord | undefined): boolean {
|
|
299
|
-
return !!rec && rec.pid === process.pid && rec.host === os.hostname() && rec.instance === this.instance;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
private leaderIsStale(rec: LeaderRecord | undefined, now: number): boolean {
|
|
303
|
-
if (!rec) return true;
|
|
304
|
-
if (this.isMine(rec)) return false;
|
|
305
|
-
// The headless host only keeps the clock while nobody is around: an interactive pi takes it
|
|
306
|
-
// back the moment it opens, and the host exits when it sees that (see host.ts).
|
|
307
|
-
if (this.kind === "interactive" && rec.kind === "host") return true;
|
|
308
|
-
const age = now - Date.parse(rec.heartbeatAt);
|
|
309
|
-
if (Number.isNaN(age) || age > LEADER_STALE_MS) return true;
|
|
310
|
-
// One machine: a pid that is not alive here is not alive. (A record from a renamed or rebuilt
|
|
311
|
-
// box used to be untouchable until its heartbeat aged out; now it is taken over like any other.)
|
|
312
|
-
if (!pidAlive(rec.pid)) return true;
|
|
313
|
-
return false;
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
private async claimOrRenewLeadership(now: number): Promise<boolean> {
|
|
317
|
-
return withFileLock(this.leaderLock, () => {
|
|
318
|
-
const rec = this.readLeader();
|
|
319
|
-
const mine = this.isMine(rec);
|
|
320
|
-
if (!mine && !this.leaderIsStale(rec, now)) {
|
|
321
|
-
this.leader = false;
|
|
322
|
-
return false;
|
|
323
|
-
}
|
|
324
|
-
const next: LeaderRecord = {
|
|
325
|
-
pid: process.pid,
|
|
326
|
-
host: os.hostname(),
|
|
327
|
-
instance: this.instance,
|
|
328
|
-
sessionId: this.getSession().sessionId,
|
|
329
|
-
kind: this.kind,
|
|
330
|
-
startedAt: mine && rec ? rec.startedAt : stamp(now),
|
|
331
|
-
heartbeatAt: stamp(now),
|
|
332
|
-
};
|
|
333
|
-
writeFileAtomic(this.leaderFile, `${JSON.stringify(next, null, 2)}\n`);
|
|
334
|
-
// Routine: which process holds the timer is bookkeeping, not something gone wrong.
|
|
335
|
-
if (!this.leader) this.log(`took over the loop scheduler (pid ${process.pid})`, "info");
|
|
336
|
-
this.leader = true;
|
|
337
|
-
return true;
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
private async releaseLeadership(): Promise<void> {
|
|
342
|
-
if (!this.leader) return;
|
|
343
|
-
this.leader = false;
|
|
344
|
-
try {
|
|
345
|
-
await this.hooks.onLeadership?.(false);
|
|
346
|
-
} catch (err: any) {
|
|
347
|
-
this.log(`leadership hook failed: ${err?.message ?? err}`);
|
|
348
|
-
}
|
|
349
|
-
try {
|
|
350
|
-
await withFileLock(this.leaderLock, () => {
|
|
351
|
-
if (this.isMine(this.readLeader())) fs.rmSync(this.leaderFile, { force: true });
|
|
352
|
-
});
|
|
353
|
-
} catch {
|
|
354
|
-
/* best effort */
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
private log(message: string, level?: "info" | "warning"): void {
|
|
359
|
-
this.hooks.log?.(message, level);
|
|
351
|
+
private log(message: string, level?: "info" | "warning", job?: Pick<LoopJob, "cwd">): void {
|
|
352
|
+
this.hooks.log?.(message, level, job);
|
|
360
353
|
}
|
|
361
354
|
|
|
362
355
|
/**
|
|
@@ -395,30 +388,12 @@ export class LoopScheduler {
|
|
|
395
388
|
this.tickPromise = new Promise<void>((r) => (finish = r));
|
|
396
389
|
try {
|
|
397
390
|
const now = this.now();
|
|
398
|
-
|
|
399
|
-
const wasLeader = this.leader;
|
|
400
|
-
try {
|
|
401
|
-
leader = await this.claimOrRenewLeadership(now);
|
|
402
|
-
} catch (err: any) {
|
|
403
|
-
this.log(`leadership check failed: ${err?.message ?? err}`);
|
|
404
|
-
}
|
|
405
|
-
// A UI hook that throws must not escape the tick: `start()` runs it as `void this.tick()`
|
|
406
|
-
// and pi installs no unhandledRejection handler, so an unguarded throw here kills pi.
|
|
407
|
-
if (leader !== wasLeader) {
|
|
408
|
-
try {
|
|
409
|
-
await this.hooks.onLeadership?.(leader);
|
|
410
|
-
} catch (err: any) {
|
|
411
|
-
this.log(`leadership hook failed: ${err?.message ?? err}`);
|
|
412
|
-
}
|
|
413
|
-
}
|
|
391
|
+
const host = this.kind === "host";
|
|
414
392
|
const session = this.getSession();
|
|
415
393
|
try {
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
this.
|
|
419
|
-
}
|
|
420
|
-
try {
|
|
421
|
-
await this.hooks.onTick?.(now, leader);
|
|
394
|
+
// A hook that throws must not escape the tick: `start()` runs it as `void this.tick()` and
|
|
395
|
+
// pi installs no unhandledRejection handler, so an unguarded throw here kills pi.
|
|
396
|
+
await this.hooks.onTick?.(now);
|
|
422
397
|
} catch (err: any) {
|
|
423
398
|
this.log(`tick hook failed: ${err?.message ?? err}`);
|
|
424
399
|
}
|
|
@@ -426,21 +401,40 @@ export class LoopScheduler {
|
|
|
426
401
|
try {
|
|
427
402
|
// `mutate` writes only when the list really changed, so a tick with nothing to clear
|
|
428
403
|
// costs one read and no write (store.ts).
|
|
404
|
+
let retired: LoopJob[] = [];
|
|
429
405
|
jobs = await this.store.mutate((all) => {
|
|
430
|
-
this.clearStaleRunning(all);
|
|
406
|
+
retired = this.clearStaleRunning(all);
|
|
431
407
|
return { jobs: all, result: all };
|
|
432
408
|
});
|
|
409
|
+
for (const job of retired) {
|
|
410
|
+
try {
|
|
411
|
+
this.hooks.onJobRetired?.(job, "one-shot finished before its process ended");
|
|
412
|
+
} catch (err: any) {
|
|
413
|
+
this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
433
416
|
} catch (err: any) {
|
|
434
417
|
this.log(`cannot read jobs: ${err?.message ?? err}`);
|
|
435
418
|
return;
|
|
436
419
|
}
|
|
420
|
+
// A job removed in another process (a window, the CLI) takes its run here with it, as
|
|
421
|
+
// `removeJob` does in this one: the run used to go on to its deadline, spending as it went.
|
|
422
|
+
const present = new Set(jobs.map((j) => j.id));
|
|
423
|
+
for (const [runId, r] of this.inflight) {
|
|
424
|
+
if (present.has(r.jobId) || r.ctrl.signal.aborted) continue;
|
|
425
|
+
r.ctrl.abort();
|
|
426
|
+
this.log(`cron ${r.label}: removed while it ran; stopped run ${runId}`);
|
|
427
|
+
}
|
|
437
428
|
// What the store cannot say and every reader wants: when each job runs next. It is a pure
|
|
438
429
|
// function of the job, but computing it needs the cron evaluator — which the browser
|
|
439
|
-
// front end does not have and should not grow a second copy of. The
|
|
430
|
+
// front end does not have and should not grow a second copy of. The host writes the
|
|
440
431
|
// answers where that front end already reads pi-loops' own files, and only when one of
|
|
441
432
|
// them changes, which for a nightly job is once a day.
|
|
442
|
-
|
|
443
|
-
|
|
433
|
+
// The host writes it; with no host running a window does, or the browser front end of a person
|
|
434
|
+
// with only plain jobs (who never starts a host) showed no next run at all. Every writer
|
|
435
|
+
// computes the same function of the same store, so two windows writing it agree.
|
|
436
|
+
if (host || this.writesNextRuns?.()) this.writeNextRuns(jobs, now);
|
|
437
|
+
if (host && this.sessionExists && now - this.lastDeadSessionScan >= DEAD_SESSION_SCAN_MS) {
|
|
444
438
|
this.lastDeadSessionScan = now;
|
|
445
439
|
try {
|
|
446
440
|
await this.disableDeadSessionJobs(jobs);
|
|
@@ -448,10 +442,23 @@ export class LoopScheduler {
|
|
|
448
442
|
this.log(`dead-session check failed: ${err?.message ?? err}`);
|
|
449
443
|
}
|
|
450
444
|
}
|
|
445
|
+
const view = this.ownerView(session);
|
|
451
446
|
for (const job of jobs) {
|
|
447
|
+
if (!this.owns(job, view)) continue;
|
|
448
|
+
// A fired one-shot stays until its row is done (its prompt is in the conversation, or the row
|
|
449
|
+
// died): removed at fire, a quit before the hand-over deleted it unfired.
|
|
450
|
+
if (this.plainOneShotDone(job)) {
|
|
451
|
+
await this.store.remove(job.id);
|
|
452
|
+
try {
|
|
453
|
+
this.hooks.onJobRetired?.(job, "one-shot fired");
|
|
454
|
+
} catch (err: any) {
|
|
455
|
+
this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
|
|
456
|
+
}
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
// A disabled job's queued rows wait: its prompt runs when the job is enabled again, which
|
|
460
|
+
// is what a person who disabled it expects. A removed job's rows the engine removes.
|
|
452
461
|
if (!job.enabled) continue;
|
|
453
|
-
const owned = job.stateful ? leader : runsIn(job, session.sessionId);
|
|
454
|
-
if (!owned) continue;
|
|
455
462
|
// One unusable job must never stop the clock for the others: a schedule that cannot be
|
|
456
463
|
// evaluated disables that job and says why, instead of throwing out of the tick (pi
|
|
457
464
|
// installs no unhandledRejection handler, so that would take the whole session down).
|
|
@@ -466,11 +473,12 @@ export class LoopScheduler {
|
|
|
466
473
|
},
|
|
467
474
|
now,
|
|
468
475
|
);
|
|
469
|
-
|
|
476
|
+
// Nothing due, but a row a window left behind when it quit is the host's to run (launchHeld).
|
|
477
|
+
if (due === undefined && !(host && !job.stateful && this.heldTurnRow(job))) continue;
|
|
470
478
|
await this.dispatch(job, due, now, session);
|
|
471
479
|
} catch (err: any) {
|
|
472
480
|
const message = `disabled: ${err?.message ?? err} (${formatSchedule(job.schedule)})`;
|
|
473
|
-
this.log(`loop ${job.id}: ${message}
|
|
481
|
+
this.log(`loop ${job.id}: ${message}`, undefined, job);
|
|
474
482
|
await this.store.update(job.id, (j) => {
|
|
475
483
|
j.enabled = false;
|
|
476
484
|
j.lastError = message;
|
|
@@ -496,9 +504,9 @@ export class LoopScheduler {
|
|
|
496
504
|
for (const job of dead) {
|
|
497
505
|
await this.store.update(job.id, (j) => {
|
|
498
506
|
j.enabled = false;
|
|
499
|
-
j.lastError = `disabled: session ${job.sessionId
|
|
507
|
+
j.lastError = `disabled: session ${shortSessionId(job.sessionId!)} ${DEAD_SESSION_MARKER}`;
|
|
500
508
|
});
|
|
501
|
-
this.log(`cron ${job.name ?? job.id}: disabled, its session ${job.sessionId
|
|
509
|
+
this.log(`cron ${job.name ?? job.id}: disabled, its session ${shortSessionId(job.sessionId!)} no longer exists`, undefined, job);
|
|
502
510
|
}
|
|
503
511
|
}
|
|
504
512
|
|
|
@@ -539,39 +547,86 @@ export class LoopScheduler {
|
|
|
539
547
|
return this.store.removeWhere((j) => inScope(j) && !j.enabled && !!j.lastError?.endsWith(DEAD_SESSION_MARKER), { purge: true });
|
|
540
548
|
}
|
|
541
549
|
|
|
542
|
-
/**
|
|
543
|
-
|
|
550
|
+
/**
|
|
551
|
+
* Clear running markers left behind by processes that are gone. Mutates `jobs` in place and
|
|
552
|
+
* returns the one-shots it retired, for the caller to report once the store is written.
|
|
553
|
+
*
|
|
554
|
+
* A dead marker is looked up in the run log first. A run whose record is there finished, and only
|
|
555
|
+
* its last write to the store was lost (the process died between `appendRun` and the update after
|
|
556
|
+
* it): its slot is spent, so the bookkeeping is finished from the record instead of rolling back
|
|
557
|
+
* and running it a second time. Only a run with no record is owed again.
|
|
558
|
+
*/
|
|
559
|
+
private clearStaleRunning(jobs: LoopJob[]): LoopJob[] {
|
|
544
560
|
// The wall clock, not `this.now()`: `os.uptime()` is measured against it, so the boot instant
|
|
545
|
-
// is only meaningful in the same terms —
|
|
546
|
-
// which is why a test cannot move that one. The foreign-host age check does go through
|
|
547
|
-
// `this.now()`: it compares a stamp against our own notion of now, with no machine fact in it.
|
|
561
|
+
// is only meaningful in the same terms — which is why a test cannot move this comparison.
|
|
548
562
|
const booted = Date.now() - os.uptime() * 1000;
|
|
563
|
+
let records: Map<string, RunRecord> | undefined;
|
|
564
|
+
const recordOf = (runId: string): RunRecord | undefined => {
|
|
565
|
+
records ??= new Map(this.store.allRuns().map((r) => [r.runId, r]));
|
|
566
|
+
return records.get(runId);
|
|
567
|
+
};
|
|
568
|
+
const retired: LoopJob[] = [];
|
|
569
|
+
const dead = (job: LoopJob): boolean => {
|
|
570
|
+
const { pid, runId, startedAt } = job.running!;
|
|
571
|
+
// A pid is only evidence while it can still be the same process: after a reboot the
|
|
572
|
+
// numbers are handed out again, and a recycled pid would park the job forever.
|
|
573
|
+
const recycled = Number.isFinite(booted) && Date.parse(startedAt) < booted - 60_000;
|
|
574
|
+
return !(pid === process.pid ? this.inflight.has(runId) : !recycled && pidAlive(pid));
|
|
575
|
+
};
|
|
576
|
+
// How many runs each dead process took with it. One that died with a single run in flight
|
|
577
|
+
// points at that run; one that died with several says nothing about any of them, and counting
|
|
578
|
+
// each would park every loop that happened to run beside the one that took it down.
|
|
579
|
+
const diedWith = new Map<number, number>();
|
|
580
|
+
for (const job of jobs) if (job.running && dead(job) && !recordOf(job.running.runId)) diedWith.set(job.running.pid, (diedWith.get(job.running.pid) ?? 0) + 1);
|
|
549
581
|
for (const job of jobs) {
|
|
550
582
|
if (!job.running) continue;
|
|
551
583
|
const { pid, runId, startedAt } = job.running;
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
584
|
+
if (!dead(job)) continue;
|
|
585
|
+
job.running = undefined;
|
|
586
|
+
const record = recordOf(runId);
|
|
587
|
+
if (record) {
|
|
588
|
+
job.lastCompletedAt = record.finishedAt;
|
|
589
|
+
job.lastError = record.ok ? undefined : record.error;
|
|
590
|
+
job.runCount++;
|
|
591
|
+
job.consecutiveFailures = record.ok ? undefined : (job.consecutiveFailures ?? 0) + 1;
|
|
592
|
+
job.interruptions = undefined;
|
|
593
|
+
if (job.schedule.kind === "once" && (record.ok || job.runCount >= 2)) retired.push(job);
|
|
594
|
+
continue;
|
|
563
595
|
}
|
|
564
596
|
// The run that died is owed again: roll the bookkeeping back so the next tick re-fires it
|
|
565
|
-
// (collapsed like any other missed tick) instead of silently skipping to the next slot.
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
597
|
+
// (collapsed like any other missed tick) instead of silently skipping to the next slot. It
|
|
598
|
+
// was tried, so a process that starts after its due time must not call it missed. A run of a
|
|
599
|
+
// row that is still queued (a held turn of what a window left, launchHeld) rolls nothing
|
|
600
|
+
// back: the row is what runs again, and the stamps the window moved stand.
|
|
601
|
+
if (!(job.sessionId && this.requests.list(job.sessionId).some((row) => row.id === runId))) {
|
|
602
|
+
job.owedSlot = job.lastDueAt ?? startedAt;
|
|
603
|
+
job.lastDueAt = undefined;
|
|
604
|
+
job.lastFiredAt = undefined;
|
|
605
|
+
}
|
|
606
|
+
if ((diedWith.get(pid) ?? 0) > 1) {
|
|
607
|
+
job.lastError = "cleared stale running state (the process ended mid-run, with other runs beside this one); the run will be retried";
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
job.interruptions = (job.interruptions ?? 0) + 1;
|
|
611
|
+
if (job.interruptions >= INTERRUPTIONS_BEFORE_PARKING) {
|
|
612
|
+
// Parked rather than retried on every start: each time, it was the only loop run in flight
|
|
613
|
+
// when the process stopped (a rule check may have been running beside it). Still not proof (a reboot mid-run counts too), so the reason says what
|
|
614
|
+
// is known, and enabling it again tries once more.
|
|
615
|
+
job.enabled = false;
|
|
616
|
+
job.owedSlot = undefined;
|
|
617
|
+
job.lastError = `parked: its last ${job.interruptions} runs were interrupted before they finished (the process running them stopped each time, with no other loop running; see host.log) — /cron enable ${job.id} to try again`;
|
|
618
|
+
this.log(`cron ${job.name ?? job.id}: parked after ${job.interruptions} interrupted runs`, undefined, job);
|
|
619
|
+
continue;
|
|
620
|
+
}
|
|
569
621
|
job.lastError = "cleared stale running state (previous pi process ended mid-run); the run will be retried";
|
|
570
622
|
}
|
|
623
|
+
for (const job of retired) jobs.splice(jobs.indexOf(job), 1);
|
|
624
|
+
return retired;
|
|
571
625
|
}
|
|
572
626
|
|
|
573
|
-
|
|
574
|
-
|
|
627
|
+
/** `due` is the slot the schedule owes; absent, the host runs a queued row of the job and spends no slot. */
|
|
628
|
+
private async dispatch(job: LoopJob, due: number | undefined, now: number, session: SessionSnapshot): Promise<void> {
|
|
629
|
+
const dueIso = due === undefined ? undefined : stamp(due);
|
|
575
630
|
if (job.stateful && !fs.existsSync(job.cwd)) {
|
|
576
631
|
/**
|
|
577
632
|
* The checkout is not there. That is usually a deleted worktree or a moved project — but
|
|
@@ -592,13 +647,13 @@ export class LoopScheduler {
|
|
|
592
647
|
j.cwdMissingSince = stamp(since);
|
|
593
648
|
if (gone) {
|
|
594
649
|
j.enabled = false;
|
|
595
|
-
j.lastDueAt = dueIso;
|
|
650
|
+
if (dueIso) j.lastDueAt = dueIso;
|
|
596
651
|
j.lastError = `disabled: cwd ${job.cwd} has been missing since ${formatLocal(since)} (re-enable after /cron add --cwd or restoring it)`;
|
|
597
652
|
} else {
|
|
598
653
|
j.lastError = `waiting: cwd ${job.cwd} is not there yet (since ${formatLocal(since)}; disabled if it stays missing)`;
|
|
599
654
|
}
|
|
600
655
|
});
|
|
601
|
-
if (gone) this.log(`cron ${job.name ?? job.id}: disabled, cwd ${job.cwd} missing since ${formatLocal(since)}
|
|
656
|
+
if (gone) this.log(`cron ${job.name ?? job.id}: disabled, cwd ${job.cwd} missing since ${formatLocal(since)}`, undefined, job);
|
|
602
657
|
return;
|
|
603
658
|
}
|
|
604
659
|
// Back, and the run that was owed is about to happen: the marker must not outlive the outage.
|
|
@@ -609,21 +664,27 @@ export class LoopScheduler {
|
|
|
609
664
|
});
|
|
610
665
|
}
|
|
611
666
|
if (job.running) {
|
|
667
|
+
// A plain job's marker is the host's turn in its conversation: that turn either completes the
|
|
668
|
+
// slot or hands it back owed, so there is nothing to skip — wait for it.
|
|
669
|
+
if (!job.stateful) return;
|
|
612
670
|
await this.store.update(job.id, (j) => {
|
|
613
671
|
j.skippedOverlap++;
|
|
614
|
-
j.lastDueAt = dueIso;
|
|
672
|
+
if (dueIso) j.lastDueAt = dueIso;
|
|
615
673
|
j.lastError = "skipped: previous run still active";
|
|
616
674
|
});
|
|
617
675
|
return;
|
|
618
676
|
}
|
|
619
|
-
|
|
620
|
-
|
|
677
|
+
// A slot handed back unrun was tried, not missed: it runs now whatever the catch-up setting says.
|
|
678
|
+
const owed = due !== undefined && !!job.owedSlot && Date.parse(job.owedSlot) <= due;
|
|
679
|
+
const missedDue = due !== undefined && !owed && due < this.startedAt - this.tickMs ? due : undefined;
|
|
680
|
+
if (missedDue !== undefined && !(job.catchUp && this.getSettings().catchUp)) {
|
|
621
681
|
await this.store.update(job.id, (j) => {
|
|
622
|
-
j.lastDueAt = dueIso;
|
|
623
|
-
j.lastError = `missed ${formatLocal(
|
|
682
|
+
if (dueIso) j.lastDueAt = dueIso;
|
|
683
|
+
j.lastError = `missed ${formatLocal(missedDue)} while ${this.kind === "host" ? "pi-loops was not running" : "no pi was running"} (catch-up disabled)`;
|
|
624
684
|
});
|
|
625
685
|
return;
|
|
626
686
|
}
|
|
687
|
+
const note = missedDue !== undefined ? `\n(catching up a run due ${formatLocal(missedDue)})` : "";
|
|
627
688
|
const budget = this.budgetState(now);
|
|
628
689
|
if (budget.over) {
|
|
629
690
|
// Plain jobs are gated too: injecting one makes the parent agent take a turn, which is
|
|
@@ -638,8 +699,8 @@ export class LoopScheduler {
|
|
|
638
699
|
}
|
|
639
700
|
return;
|
|
640
701
|
}
|
|
641
|
-
if (!job.stateful) {
|
|
642
|
-
await this.injectPlainJob(job, dueIso, now,
|
|
702
|
+
if (!job.stateful && this.kind !== "host") {
|
|
703
|
+
await this.injectPlainJob(job, dueIso, now, note);
|
|
643
704
|
return;
|
|
644
705
|
}
|
|
645
706
|
// A failing job is retried with a widening gap instead of at every due tick: a loop whose
|
|
@@ -666,51 +727,310 @@ export class LoopScheduler {
|
|
|
666
727
|
slot.release();
|
|
667
728
|
return;
|
|
668
729
|
}
|
|
669
|
-
if (
|
|
670
|
-
// Never await a run inside a tick:
|
|
730
|
+
if (missedDue !== undefined) this.hooks.onCatchUp?.(job, missedDue);
|
|
731
|
+
// Never await a run inside a tick: leases, presence and trigger checks keep
|
|
671
732
|
// going while sub-agents work, and several loops really do run at once.
|
|
672
|
-
|
|
733
|
+
if (!job.stateful) {
|
|
734
|
+
this.track(this.launchHeld(job, dueIso, now, note), slot);
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
this.track(this.launch(job, dueIso, now, session, missedDue !== undefined), slot);
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
/** Who wrote a row: this process, as `created_by` names it. */
|
|
741
|
+
private createdBy(): SessionRequest["created_by"] {
|
|
742
|
+
return { pid: process.pid, instance: this.instance, version: PI_LOOPS_VERSION };
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/** The job's queued `turn` row, when one is: the prompt waiting to run in its conversation. */
|
|
746
|
+
private queuedTurnRow(job: LoopJob): SessionRequest | undefined {
|
|
747
|
+
if (!job.sessionId) return undefined;
|
|
748
|
+
const key = plainCollapseKey(job.id);
|
|
749
|
+
return this.requests.list(job.sessionId).find((row) => row.kind === "input" && row.wake === "turn" && row.collapse_key === key);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* The job's queued row the background may run as a held turn: not an attended one (`/cron run`,
|
|
754
|
+
* typed in the conversation), which waits for a window and is no reason for the clock's slots to.
|
|
755
|
+
*/
|
|
756
|
+
private heldTurnRow(job: LoopJob): SessionRequest | undefined {
|
|
757
|
+
const row = this.queuedTurnRow(job);
|
|
758
|
+
return row?.mode === "attended" ? undefined : row;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* A fired plain one-shot whose row is done: its prompt is in the conversation, or the row died.
|
|
763
|
+
* A failed held turn is not done; it keeps the retry `launchHeld` gives a one-shot.
|
|
764
|
+
*/
|
|
765
|
+
private plainOneShotDone(job: LoopJob): boolean {
|
|
766
|
+
if (job.stateful || job.schedule.kind !== "once" || job.running || job.runCount < 1 || job.consecutiveFailures) return false;
|
|
767
|
+
return this.queuedTurnRow(job) === undefined;
|
|
673
768
|
}
|
|
674
769
|
|
|
675
770
|
/**
|
|
676
|
-
* A plain job firing:
|
|
677
|
-
* timer (`dispatch`) and `/cron run` come through here
|
|
678
|
-
*
|
|
771
|
+
* A plain job firing in a window: its prompt is a row in the conversation's requests first, then
|
|
772
|
+
* the slot is spent. Both the timer (`dispatch`) and `/cron run` come through here. The engine
|
|
773
|
+
* hands the row to pi once pi is idle (src/engine.ts, src/window-executor.ts); a one-shot is
|
|
774
|
+
* retired when the row is done (plainOneShotDone), not here.
|
|
775
|
+
*
|
|
776
|
+
* The row's id is derived from the slot (plainRunId), so a crash between the row and the stamps
|
|
777
|
+
* leaves a queued row and an unspent slot: the next tick decides the same slot, its append is a
|
|
778
|
+
* no-op, and the swap commits. While an earlier row of the job is still queued the slot is spent
|
|
779
|
+
* without a new row and counted in `skippedOverlap`: collapse, never replay.
|
|
679
780
|
*
|
|
680
781
|
* `dueIso` is the slot being spent, absent when the run was asked for rather than scheduled;
|
|
681
|
-
* `note` is appended to the
|
|
682
|
-
*
|
|
782
|
+
* `note` is appended to the prompt (the catch-up line). Returns what became of the slot, so
|
|
783
|
+
* `/cron run` does not report a run that never happened.
|
|
683
784
|
*/
|
|
684
|
-
private async injectPlainJob(job: LoopJob, dueIso: string | undefined, now: number, note = ""): Promise<
|
|
685
|
-
// A plain job's whole point is to land in a chat, and the headless host has none
|
|
686
|
-
//
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
785
|
+
private async injectPlainJob(job: LoopJob, dueIso: string | undefined, now: number, note = "", source: "cron" | "cron-run" = "cron"): Promise<"fired" | "collapsed" | "lost"> {
|
|
786
|
+
// A plain job's whole point is to land in a chat, and the headless host has none: a closed
|
|
787
|
+
// conversation's plain job is a held turn (launchHeld).
|
|
788
|
+
if (this.kind === "host") return "lost";
|
|
789
|
+
const sessionId = job.sessionId;
|
|
790
|
+
if (!sessionId) return "lost";
|
|
791
|
+
// The epoch the lease had when this instance decided the job was its own. Read again inside
|
|
792
|
+
// the claim: a lease that moved since (this window closed its session, another took it) means
|
|
793
|
+
// the slot is no longer ours to fire.
|
|
794
|
+
const epoch = this.leases?.heldEpoch(sessionId);
|
|
795
|
+
if (this.leases && epoch === undefined) return "lost";
|
|
796
|
+
const runId = plainRunId(job.id, dueIso ?? stamp(now));
|
|
797
|
+
const queued = this.queuedTurnRow(job);
|
|
798
|
+
// A row pi did not take (`attempts` counts its refusals) is not what a new slot collapses onto:
|
|
799
|
+
// that slot is lost, as 0.22.8 lost a slot pi refused, and the new one asks in its place.
|
|
800
|
+
// A `/cron run` row is the person's: a slot collapses onto it, refused or not, so it keeps running as theirs.
|
|
801
|
+
const refused = queued !== undefined && queued.id !== runId && queued.attempts > 0 && queued.source.startsWith("cron:");
|
|
802
|
+
const collapsed = !!queued && !refused && queued.id !== runId;
|
|
803
|
+
// `/cron run` while the prompt is already queued asks for nothing new: no slot, no stamp.
|
|
804
|
+
if (collapsed && source === "cron-run") return "collapsed";
|
|
805
|
+
// The row before the stamps: a crash between the two costs nothing, where the reverse order
|
|
806
|
+
// spent the slot with nothing to show for it.
|
|
807
|
+
const row = plainJobRow(job, runId, `${source}:${job.id}`, note, this.createdBy(), now);
|
|
808
|
+
let wrote = false;
|
|
809
|
+
if (!collapsed) {
|
|
810
|
+
if (refused) this.requests.remove(sessionId, [queued.id]);
|
|
811
|
+
wrote = this.requests.appendIfAbsent(row);
|
|
812
|
+
// Said before anything awaits: a drain that hands the row over meanwhile must already know
|
|
813
|
+
// this window wrote it, or a person's `/cron run` would start as a scheduled turn.
|
|
814
|
+
if (wrote) this.hooks.onRowWritten?.(row, job);
|
|
815
|
+
}
|
|
816
|
+
let claimed = false;
|
|
691
817
|
await this.store.update(job.id, (j) => {
|
|
818
|
+
// The slot was decided from a snapshot, as in `launch`: if another instance fired or skipped
|
|
819
|
+
// this job since, its stamps moved, and firing again would put the message in the chat twice.
|
|
820
|
+
if (j.lastDueAt !== job.lastDueAt || j.lastFiredAt !== job.lastFiredAt) return;
|
|
821
|
+
if (this.leases && (!j.sessionId || this.leases.heldEpoch(j.sessionId) !== epoch)) return;
|
|
822
|
+
claimed = true;
|
|
692
823
|
if (dueIso) j.lastDueAt = dueIso;
|
|
693
824
|
j.lastFiredAt = stamp(now);
|
|
825
|
+
// The count is the record of a collapse; an error here outlived the prompt it was about.
|
|
826
|
+
if (collapsed) {
|
|
827
|
+
j.skippedOverlap++;
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
694
830
|
j.lastCompletedAt = j.lastFiredAt;
|
|
831
|
+
j.owedSlot = undefined;
|
|
695
832
|
// Only an enabled job's error is stale news. A disabled one's is why it is disabled — and
|
|
696
833
|
// for a job the dead-session sweep parked it is the marker `gc()` matches on.
|
|
697
834
|
if (j.enabled) j.lastError = undefined;
|
|
698
835
|
j.runCount++;
|
|
699
836
|
});
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
837
|
+
if (!claimed) {
|
|
838
|
+
// Another process fired this slot first (a pi-loops 0.22 window on the same conversation during
|
|
839
|
+
// an upgrade): the row this call wrote would run the slot a second time. Only that row goes.
|
|
840
|
+
if (wrote) this.requests.remove(sessionId, [runId]);
|
|
841
|
+
return "lost";
|
|
842
|
+
}
|
|
843
|
+
// Its next run moved: written now, so the browser panel does not show a blank one until the
|
|
844
|
+
// next tick of whichever process writes it. Every writer computes the same answer.
|
|
845
|
+
try {
|
|
846
|
+
this.writeNextRuns(this.store.load(), now);
|
|
847
|
+
} catch {
|
|
848
|
+
/* a derived file is a courtesy (see writeNextRuns); the tick writes it too */
|
|
849
|
+
}
|
|
850
|
+
return collapsed ? "collapsed" : "fired";
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* The host firing a plain job of a closed conversation: take the conversation, claim the slot
|
|
855
|
+
* against the stamps and the lease's epoch, run the turn in it, give the conversation back. A
|
|
856
|
+
* window that opens the conversation meanwhile takes it at once; the turn stops at its next event
|
|
857
|
+
* and the slot is handed back, so the window fires it instead — a slot runs to completion once.
|
|
858
|
+
*
|
|
859
|
+
* A row a window wrote and never handed over (it quit first) runs before any slot of the host's
|
|
860
|
+
* own, with the row's id as the run id, and leaves the queue once its prompt is in the file. A slot
|
|
861
|
+
* due meanwhile is spent onto that row (collapse, as the window does), never run after it. A turn
|
|
862
|
+
* handed back unrun whose prompt did land is not owed: a successor row asks again, once, in
|
|
863
|
+
* whichever process holds the conversation next. An attended row (`/cron run`) is a window's to
|
|
864
|
+
* run and is left where it is; the clock's slots run as if it were not there. A row in backoff
|
|
865
|
+
* waits, and so does the slot behind it.
|
|
866
|
+
*/
|
|
867
|
+
private async launchHeld(job: LoopJob, dueIso: string | undefined, now: number, note: string): Promise<void> {
|
|
868
|
+
const leases = this.leases;
|
|
869
|
+
const sessionId = job.sessionId;
|
|
870
|
+
if (!sessionId || !leases?.acquireForWork || !this.hooks.onHeldTurn) return;
|
|
871
|
+
const row = this.heldTurnRow(job);
|
|
872
|
+
if (!row && dueIso === undefined) return;
|
|
873
|
+
if (row && (row.attempts >= DEAD_AFTER_ATTEMPTS || (row.not_before !== undefined && Date.parse(row.not_before) > now))) return;
|
|
874
|
+
const runId = row?.id ?? newId("run");
|
|
875
|
+
const spendsSlot = dueIso !== undefined;
|
|
876
|
+
// The slot due is not the row's own: spent onto it, and counted as the overlap it is.
|
|
877
|
+
const collapsedSlot = spendsSlot && !!row && row.id !== plainRunId(job.id, dueIso);
|
|
878
|
+
const held = leases.acquireForWork(sessionId, runId, job.id);
|
|
879
|
+
if (!held) return; // the slot stays owed: the conversation is busy, open or recently written
|
|
880
|
+
try {
|
|
881
|
+
// A process before this one ran the row and died before removing it: the prompt is in the
|
|
882
|
+
// conversation, so the row is done, and running it again would answer it twice.
|
|
883
|
+
if (row && sessionFileHas(held.sessionFile, row.id)) {
|
|
884
|
+
this.requests.remove(sessionId, [row.id]);
|
|
885
|
+
await this.store.update(job.id, (j) => {
|
|
886
|
+
if (j.running?.runId === row.id) j.running = undefined;
|
|
887
|
+
});
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
const startedAt = stamp(now);
|
|
891
|
+
let claimed = false;
|
|
892
|
+
await this.store.update(job.id, (j) => {
|
|
893
|
+
if (j.running) return;
|
|
894
|
+
if (j.lastDueAt !== job.lastDueAt || j.lastFiredAt !== job.lastFiredAt) return;
|
|
895
|
+
if (leases.heldEpoch(sessionId) !== held.epoch) return;
|
|
896
|
+
claimed = true;
|
|
897
|
+
// Written before the turn starts, as a loop's marker is: a host that dies mid-turn leaves
|
|
898
|
+
// it behind, and the next host finds an unfinished run instead of a slot spent in silence.
|
|
899
|
+
j.running = { runId, pid: process.pid, host: this.self.host, startedAt };
|
|
900
|
+
if (!spendsSlot) return;
|
|
901
|
+
j.lastDueAt = dueIso;
|
|
902
|
+
j.lastFiredAt = startedAt;
|
|
903
|
+
j.owedSlot = undefined;
|
|
904
|
+
if (collapsedSlot) j.skippedOverlap++;
|
|
905
|
+
});
|
|
906
|
+
if (!claimed) return;
|
|
907
|
+
const ctrl = new AbortController();
|
|
908
|
+
this.inflight.set(runId, { ctrl, label: job.name ?? job.id, jobId: job.id, startedAt, promptPreview: previewRedacted(job.prompt, 120) });
|
|
909
|
+
let outcome: { preempted: boolean; result: RunnerResult; landed?: boolean };
|
|
707
910
|
try {
|
|
708
|
-
this.hooks.
|
|
911
|
+
outcome = await this.hooks.onHeldTurn(job, row?.payload?.text ?? `[Trigger ${runId}] ${job.prompt}${note}`, runId, held, ctrl.signal);
|
|
912
|
+
} catch (err: any) {
|
|
913
|
+
outcome = { preempted: false, result: failedRun(err?.message ?? String(err)) };
|
|
914
|
+
}
|
|
915
|
+
const { result } = outcome;
|
|
916
|
+
const landed = outcome.landed ?? result.ok;
|
|
917
|
+
// Handed back unrun: a window took the conversation (and runs it now), or this host stopped
|
|
918
|
+
// the turn itself — a quit, an upgrade, a preemption, the daily budget (a person's stop is done, above). The slot is owed
|
|
919
|
+
// only when the prompt never landed; landed, the successor row below is the owed slot.
|
|
920
|
+
const budgetStopped = !result.ok && (result.errorMessage ?? "").startsWith("stopped by today's");
|
|
921
|
+
// A person stopped this turn: it is done, whatever landed. Asking the prompt again would run
|
|
922
|
+
// what they just stopped.
|
|
923
|
+
const stopped = !result.ok && ctrl.signal.aborted && ctrl.signal.reason === STOPPED_BY_PERSON;
|
|
924
|
+
const handedBack = !result.ok && !stopped && (outcome.preempted || ctrl.signal.aborted || budgetStopped);
|
|
925
|
+
const record: RunRecord = {
|
|
926
|
+
runId,
|
|
927
|
+
jobId: job.id,
|
|
928
|
+
jobName: job.name,
|
|
929
|
+
stateful: false,
|
|
930
|
+
cwd: held.cwd ?? job.cwd,
|
|
931
|
+
pid: process.pid,
|
|
932
|
+
startedAt,
|
|
933
|
+
finishedAt: stamp(this.now()),
|
|
934
|
+
ok: result.ok,
|
|
935
|
+
exitCode: result.exitCode,
|
|
936
|
+
error: result.ok ? undefined : stopped ? STOPPED_BY_PERSON : redact(result.errorMessage ?? "unknown error"),
|
|
937
|
+
findings: 0,
|
|
938
|
+
droppedFindings: 0,
|
|
939
|
+
stateUpdated: false,
|
|
940
|
+
model: result.model ?? job.model,
|
|
941
|
+
usage: result.usage,
|
|
942
|
+
warning: result.warning,
|
|
943
|
+
retries: result.retries,
|
|
944
|
+
compactions: result.compactions,
|
|
945
|
+
summary: previewRedacted(result.text, 400) || undefined,
|
|
946
|
+
sessionId,
|
|
947
|
+
sessionFile: held.sessionFile,
|
|
948
|
+
forSession: job.createdBy?.sessionId ?? sessionId,
|
|
949
|
+
...(stopped || handedBack ? { aborted: true as const } : {}),
|
|
950
|
+
...(stopped ? { stoppedBy: "person" as const, stoppedFrom: this.inflight.get(runId)?.stoppedFrom ?? "cli" } : {}),
|
|
951
|
+
};
|
|
952
|
+
// Every turn is recorded, a stopped one too: it was billed, and the daily budget, `/cron runs`
|
|
953
|
+
// and the status page all read the run log.
|
|
954
|
+
try {
|
|
955
|
+
this.store.appendRun(record);
|
|
956
|
+
} catch (err: any) {
|
|
957
|
+
this.log(`run log append failed: ${err?.message ?? err}`);
|
|
958
|
+
}
|
|
959
|
+
let updated: LoopJob | undefined;
|
|
960
|
+
try {
|
|
961
|
+
updated = await this.store.update(job.id, (j) => {
|
|
962
|
+
if (j.running?.runId === runId) j.running = undefined;
|
|
963
|
+
if (handedBack) {
|
|
964
|
+
// Given back with a compare: only while the stamps are still the ones this claim wrote.
|
|
965
|
+
if (!spendsSlot || j.lastFiredAt !== startedAt) return;
|
|
966
|
+
// Landed, the slot is spent (the successor row asks for its answer); unlanded, it is owed.
|
|
967
|
+
if (landed) {
|
|
968
|
+
if (!collapsedSlot) j.runCount++;
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
j.lastDueAt = job.lastDueAt;
|
|
972
|
+
j.lastFiredAt = job.lastFiredAt;
|
|
973
|
+
j.owedSlot = dueIso;
|
|
974
|
+
return;
|
|
975
|
+
}
|
|
976
|
+
j.lastCompletedAt = record.finishedAt;
|
|
977
|
+
// A window's row counted its slot when the row was written; a slot collapsed onto it is an overlap.
|
|
978
|
+
// A person's stop is not a run and not a failure, as for a loop's run: the count and the
|
|
979
|
+
// last error stay what they were.
|
|
980
|
+
if (spendsSlot && !collapsedSlot && !stopped) j.runCount++;
|
|
981
|
+
// Only an enabled job's error is stale news: a disabled one's says why it is disabled.
|
|
982
|
+
if (!stopped && (j.enabled || !result.ok)) j.lastError = record.error;
|
|
983
|
+
// A stop is not a failure of the job's: the streak stays what it was.
|
|
984
|
+
if (!stopped) j.consecutiveFailures = result.ok ? undefined : (j.consecutiveFailures ?? 0) + 1;
|
|
985
|
+
j.interruptions = undefined;
|
|
986
|
+
// A one-shot that failed gets one retry, as a one-shot loop does: its stamps go back so
|
|
987
|
+
// it is due again, and the second failure retires it (below). A stopped one does not.
|
|
988
|
+
if (job.schedule.kind === "once" && !result.ok && !stopped && (j.consecutiveFailures ?? 0) < 2) {
|
|
989
|
+
j.lastDueAt = undefined;
|
|
990
|
+
j.lastFiredAt = undefined;
|
|
991
|
+
j.lastError = `${record.error ?? "run failed"} (one-shot: retrying once)`;
|
|
992
|
+
}
|
|
993
|
+
});
|
|
994
|
+
} catch (err: any) {
|
|
995
|
+
this.log(`cron ${job.id}: could not write the store after its turn (${err?.message ?? err})`, undefined, job);
|
|
996
|
+
}
|
|
997
|
+
this.inflight.delete(runId);
|
|
998
|
+
try {
|
|
999
|
+
if (landed || stopped) {
|
|
1000
|
+
// The prompt is in the conversation, or a person stopped the turn: the row is done.
|
|
1001
|
+
// Handed back with the prompt there, the answer was cut off, so a successor row asks
|
|
1002
|
+
// again, once.
|
|
1003
|
+
if (row) this.requests.remove(sessionId, [row.id]);
|
|
1004
|
+
if (handedBack) this.requests.append(plainJobRow(job, newId("run"), `cron:${job.id}`, "", this.createdBy(), this.now()));
|
|
1005
|
+
} else if (row && !handedBack) {
|
|
1006
|
+
// The row's turn failed before its prompt was written: an attempt, as a failed hand-over
|
|
1007
|
+
// in a window is, with the engine's backoff; at the third the engine's drain buries it.
|
|
1008
|
+
const attempts = row.attempts + 1;
|
|
1009
|
+
this.requests.update(sessionId, row.id, { attempts, last_error: record.error, not_before: stamp(this.now() + backoffFor(attempts)) });
|
|
1010
|
+
}
|
|
709
1011
|
} catch (err: any) {
|
|
710
|
-
this.log(`
|
|
1012
|
+
this.log(`cron ${job.id}: could not update its requests after its turn (${err?.message ?? err})`, undefined, job);
|
|
711
1013
|
}
|
|
1014
|
+
if (handedBack) {
|
|
1015
|
+
this.log(`cron ${job.name ?? job.id}: ${outcome.preempted ? `a window opened session ${shortSessionId(sessionId)} during its turn; that window runs it` : `turn stopped (${record.error}); ${landed ? "a fresh prompt is queued for it" : "the slot is owed"}`}`, "info");
|
|
1016
|
+
return;
|
|
1017
|
+
}
|
|
1018
|
+
if (stopped) this.log(`cron ${job.name ?? job.id}: turn stopped; it is done`, "info");
|
|
1019
|
+
if (updated && !result.ok && !stopped && (updated.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER) {
|
|
1020
|
+
this.hooks.onSchedulerError?.(`${updated.name ?? updated.id} has failed ${updated.consecutiveFailures} times in a row; backing off (last: ${record.error ?? "unknown"})`, updated);
|
|
1021
|
+
}
|
|
1022
|
+
// A one-shot is retired once it ran, was stopped, or failed twice (the retry above was its second chance).
|
|
1023
|
+
if (updated && job.schedule.kind === "once" && (result.ok || stopped || (updated.consecutiveFailures ?? 0) >= 2)) {
|
|
1024
|
+
await this.store.remove(job.id);
|
|
1025
|
+
try {
|
|
1026
|
+
this.hooks.onJobRetired?.(job, result.ok ? "one-shot fired" : stopped ? "one-shot stopped" : `one-shot failed twice (${record.error})`);
|
|
1027
|
+
} catch (err: any) {
|
|
1028
|
+
this.log(`onJobRetired hook failed: ${err?.message ?? err}`);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
} finally {
|
|
1032
|
+
leases.releaseWork?.(sessionId, held.epoch);
|
|
712
1033
|
}
|
|
713
|
-
return true;
|
|
714
1034
|
}
|
|
715
1035
|
|
|
716
1036
|
private track(run: Promise<void>, slot: SubagentSlot): void {
|
|
@@ -741,9 +1061,10 @@ export class LoopScheduler {
|
|
|
741
1061
|
/**
|
|
742
1062
|
* Fire a job now, ignoring its schedule. Returns `true`, or the reason it did not run — phrased
|
|
743
1063
|
* to follow the job's name, which is the shape `/cron run` has always reported a refusal in
|
|
744
|
-
* ("nightly is already running").
|
|
1064
|
+
* ("nightly is already running"). `requestedBy` is the conversation a person asked from; a loop's
|
|
1065
|
+
* record carries it, so the run's card goes there (src/run-card.ts).
|
|
745
1066
|
*/
|
|
746
|
-
async runNow(jobId: string): Promise<true | string> {
|
|
1067
|
+
async runNow(jobId: string, requestedBy?: RunRequester): Promise<true | string> {
|
|
747
1068
|
const job = this.store.load().find((j) => j.id === jobId);
|
|
748
1069
|
if (!job) return "is no longer in the store";
|
|
749
1070
|
if (job.running) return "is already running";
|
|
@@ -757,18 +1078,24 @@ export class LoopScheduler {
|
|
|
757
1078
|
if (!job.enabled) return "is disabled (/cron enable it first)";
|
|
758
1079
|
// The same reasoning for a job that is enabled but belongs to another session: its prompt is
|
|
759
1080
|
// written for that conversation, and running it here would land it in this one.
|
|
760
|
-
|
|
1081
|
+
const where = whereRuns(job, this.ownerView());
|
|
1082
|
+
if (where.kind === "aside") return `belongs to this conversation, but this window shows an older copy of it — /resume this session to run it`;
|
|
1083
|
+
if (where.kind === "terminal") return `belongs to session ${ownerSession(job)}, which is ${openWhere(where.page ? "page" : "terminal")} — run it there`;
|
|
1084
|
+
if (where.kind === "host") return `belongs to session ${ownerSession(job)}, which is closed: it runs in the background when due — resume that session to run it now`;
|
|
1085
|
+
if (where.kind !== "here") return `belongs to session ${ownerSession(job)}, which is not open here — resume it, or /cron add the job again in this chat`;
|
|
761
1086
|
// No `dueIso`: this run was asked for, so whatever slot the schedule owes is still owed.
|
|
762
|
-
|
|
1087
|
+
const fired = await this.injectPlainJob(job, undefined, now, "", "cron-run");
|
|
1088
|
+
if (fired === "collapsed") return "already has its prompt queued in this conversation";
|
|
1089
|
+
if (fired === "lost") return this.kind === "host" ? "has no chat to run in (pi-loops in the background has no session)" : "was fired by another window at the same moment";
|
|
763
1090
|
return true;
|
|
764
1091
|
}
|
|
765
1092
|
// `/cron run` is a direct instruction, so a busy machine does not get to refuse it — but it
|
|
766
1093
|
// still occupies a slot, so the shared count stays honest even when that overruns the limit.
|
|
767
|
-
this.track(this.launch(job, undefined, now, this.getSession(), false), this.slots.occupy());
|
|
1094
|
+
this.track(this.launch(job, undefined, now, this.getSession(), false, requestedBy), this.slots.occupy());
|
|
768
1095
|
return true;
|
|
769
1096
|
}
|
|
770
1097
|
|
|
771
|
-
private async launch(job: LoopJob, dueIso: string | undefined, now: number, session: SessionSnapshot, catchingUp: boolean): Promise<void> {
|
|
1098
|
+
private async launch(job: LoopJob, dueIso: string | undefined, now: number, session: SessionSnapshot, catchingUp: boolean, requestedBy?: RunRequester): Promise<void> {
|
|
772
1099
|
const runId = newId("run");
|
|
773
1100
|
const startedAt = stamp(now);
|
|
774
1101
|
// Kept so an aborted run can hand its slot back rather than skipping to the next one.
|
|
@@ -776,10 +1103,14 @@ export class LoopScheduler {
|
|
|
776
1103
|
const priorFiredAt = job.lastFiredAt;
|
|
777
1104
|
const claimed = await this.store.update(job.id, (j) => {
|
|
778
1105
|
if (j.running) return;
|
|
1106
|
+
// The slot was decided from a snapshot. If another process ran or skipped this job since,
|
|
1107
|
+
// its stamps moved: that slot is spent, and running it again would run it twice.
|
|
1108
|
+
if (j.lastDueAt !== priorDueAt || j.lastFiredAt !== priorFiredAt) return;
|
|
779
1109
|
j.running = { runId, pid: process.pid, host: this.self.host, startedAt };
|
|
780
1110
|
if (dueIso) j.lastDueAt = dueIso;
|
|
781
1111
|
j.lastFiredAt = startedAt;
|
|
782
1112
|
j.lastError = undefined;
|
|
1113
|
+
j.owedSlot = undefined;
|
|
783
1114
|
});
|
|
784
1115
|
if (!claimed || claimed.running?.runId !== runId) return;
|
|
785
1116
|
|
|
@@ -791,7 +1122,7 @@ export class LoopScheduler {
|
|
|
791
1122
|
this.hooks.log?.(`onRunStart hook failed: ${err?.message ?? err}`);
|
|
792
1123
|
}
|
|
793
1124
|
try {
|
|
794
|
-
await this.runOnce(job, claimed, runId, startedAt, ctrl, session, now, catchingUp, { dueAt: priorDueAt, firedAt: priorFiredAt });
|
|
1125
|
+
await this.runOnce(job, claimed, runId, startedAt, ctrl, session, now, catchingUp, { dueAt: priorDueAt, firedAt: priorFiredAt }, requestedBy);
|
|
795
1126
|
} finally {
|
|
796
1127
|
// Whatever failed (a state file that cannot be read, a hook that threw), the run id is
|
|
797
1128
|
// released — otherwise `clearStaleRunning` would call the job alive forever.
|
|
@@ -806,7 +1137,7 @@ export class LoopScheduler {
|
|
|
806
1137
|
}
|
|
807
1138
|
|
|
808
1139
|
/** The body of one run: everything between claiming the job and writing its record. */
|
|
809
|
-
private async runOnce(job: LoopJob, claimed: LoopJob, runId: string, startedAt: string, ctrl: AbortController, session: SessionSnapshot, now: number, catchingUp: boolean, prior: { dueAt?: string; firedAt?: string }): Promise<void> {
|
|
1140
|
+
private async runOnce(job: LoopJob, claimed: LoopJob, runId: string, startedAt: string, ctrl: AbortController, session: SessionSnapshot, now: number, catchingUp: boolean, prior: { dueAt?: string; firedAt?: string }, requestedBy?: RunRequester): Promise<void> {
|
|
810
1141
|
const previousState = this.store.readState(job.id);
|
|
811
1142
|
// `job` is the pre-claim snapshot, so its `lastFiredAt` is when the previous run started —
|
|
812
1143
|
// a reason given before that was already in front of that run.
|
|
@@ -831,6 +1162,7 @@ export class LoopScheduler {
|
|
|
831
1162
|
model,
|
|
832
1163
|
thinking,
|
|
833
1164
|
tools: job.tools,
|
|
1165
|
+
builtinTools: job.builtinTools,
|
|
834
1166
|
timeoutMs: job.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS,
|
|
835
1167
|
signal: ctrl.signal,
|
|
836
1168
|
sessionDir: this.store.sessionDirFor(job.id),
|
|
@@ -862,9 +1194,9 @@ export class LoopScheduler {
|
|
|
862
1194
|
try {
|
|
863
1195
|
stillThere = this.store.load().some((j) => j.id === job.id);
|
|
864
1196
|
} catch (err: any) {
|
|
865
|
-
this.log(`loop ${job.id}: could not read the store after the run (${err?.message ?? err}); keeping its state and findings
|
|
1197
|
+
this.log(`loop ${job.id}: could not read the store after the run (${err?.message ?? err}); keeping its state and findings`, undefined, job);
|
|
866
1198
|
}
|
|
867
|
-
if (!stillThere) this.log(`loop ${job.id}: removed during the run; its state and findings are discarded
|
|
1199
|
+
if (!stillThere) this.log(`loop ${job.id}: removed during the run; its state and findings are discarded`, undefined, job);
|
|
868
1200
|
|
|
869
1201
|
// Tag extraction never fails a run: malformed/missing tags leave the state untouched.
|
|
870
1202
|
const parsed = parseRunOutput(result.text);
|
|
@@ -874,7 +1206,7 @@ export class LoopScheduler {
|
|
|
874
1206
|
this.store.writeState(job.id, parsed.state);
|
|
875
1207
|
stateUpdated = true;
|
|
876
1208
|
} catch (err: any) {
|
|
877
|
-
this.log(`loop ${job.id}: state write failed: ${err?.message ?? err}
|
|
1209
|
+
this.log(`loop ${job.id}: state write failed: ${err?.message ?? err}`, undefined, job);
|
|
878
1210
|
}
|
|
879
1211
|
}
|
|
880
1212
|
// Maker/checker: with verify=true a second, adversarial sub-agent reviews the
|
|
@@ -893,15 +1225,23 @@ export class LoopScheduler {
|
|
|
893
1225
|
const source = `cron:${job.name ?? job.id.slice(0, "cron-".length + 8)}`;
|
|
894
1226
|
for (const f of reviewed) {
|
|
895
1227
|
try {
|
|
896
|
-
await this.inbox.append({ source, text: f.text, run_id: runId, job_id: job.id, cwd: job.cwd, kind: findingKind(f.text), verified: f.verified, verified_reason: f.reason });
|
|
1228
|
+
await this.inbox.append({ source, text: f.text, run_id: runId, job_id: job.id, cwd: job.cwd, kind: findingKind(f.text), verified: f.verified, verified_reason: f.reason, session_id: job.createdBy?.sessionId, transcript: result.sessionFile });
|
|
897
1229
|
findings.push(f.text);
|
|
898
1230
|
} catch (err: any) {
|
|
899
|
-
this.log(`loop ${job.id}: inbox append failed: ${err?.message ?? err}
|
|
1231
|
+
this.log(`loop ${job.id}: inbox append failed: ${err?.message ?? err}`, undefined, job);
|
|
900
1232
|
}
|
|
901
1233
|
}
|
|
902
1234
|
}
|
|
903
1235
|
|
|
904
1236
|
const finishedAt = stamp(this.now());
|
|
1237
|
+
// An abort is not a run: quitting, a session swap, a preemption or the budget interrupted it,
|
|
1238
|
+
// so the slot it claimed is given back and the next tick re-fires it (a tick is never lost this
|
|
1239
|
+
// way because its runs die with the session that owned them).
|
|
1240
|
+
const aborted = !result.ok && (result.stopReason === "aborted" || ctrl.signal.aborted);
|
|
1241
|
+
// A person's stop is the exception: they stopped this slot, so it is spent and the next run is
|
|
1242
|
+
// on schedule. Giving it back would run what they just stopped again on the next tick. A run
|
|
1243
|
+
// that finished before the stop reached it is `ok`, and its record says so.
|
|
1244
|
+
const byPerson = aborted && ctrl.signal.aborted && ctrl.signal.reason === STOPPED_BY_PERSON;
|
|
905
1245
|
const record: RunRecord = {
|
|
906
1246
|
runId,
|
|
907
1247
|
jobId: job.id,
|
|
@@ -913,7 +1253,7 @@ export class LoopScheduler {
|
|
|
913
1253
|
finishedAt,
|
|
914
1254
|
ok: result.ok,
|
|
915
1255
|
exitCode: result.exitCode,
|
|
916
|
-
error: result.ok ? undefined : redact(result.errorMessage ?? "unknown error"),
|
|
1256
|
+
error: result.ok ? undefined : byPerson ? STOPPED_BY_PERSON : redact(result.errorMessage ?? "unknown error"),
|
|
917
1257
|
findings: findings.length,
|
|
918
1258
|
droppedFindings: parsed.droppedFindings,
|
|
919
1259
|
stateUpdated,
|
|
@@ -926,17 +1266,18 @@ export class LoopScheduler {
|
|
|
926
1266
|
sessionId: result.sessionId,
|
|
927
1267
|
sessionFile: result.sessionFile,
|
|
928
1268
|
checker,
|
|
1269
|
+
forSession: job.createdBy?.sessionId,
|
|
1270
|
+
ordinal: claimed.runCount + 1,
|
|
1271
|
+
...(requestedBy ? { requestedBy } : {}),
|
|
1272
|
+
...(aborted ? { aborted: true as const } : {}),
|
|
1273
|
+
...(byPerson ? { stoppedBy: "person" as const, stoppedFrom: this.inflight.get(runId)?.stoppedFrom ?? "cli" } : {}),
|
|
929
1274
|
};
|
|
930
|
-
this.store.pruneSessions(job.id);
|
|
1275
|
+
this.store.pruneSessions(job.id, undefined, this.inbox.pinnedTranscripts(job.id));
|
|
931
1276
|
try {
|
|
932
1277
|
this.store.appendRun(record);
|
|
933
1278
|
} catch (err: any) {
|
|
934
1279
|
this.log(`run log append failed: ${err?.message ?? err}`);
|
|
935
1280
|
}
|
|
936
|
-
// An abort is not a run: quitting, a session swap or `/cron abort` interrupted it, so the
|
|
937
|
-
// slot it claimed is given back and the next tick re-fires it (a tick is never lost this
|
|
938
|
-
// way because its runs die with the session that owned them).
|
|
939
|
-
const aborted = !result.ok && (result.stopReason === "aborted" || ctrl.signal.aborted);
|
|
940
1281
|
// The same unreadable store that was tolerated above must not cost the run its `onRunFinished`
|
|
941
1282
|
// either: the record is already written, and the job's own bookkeeping is the one part that
|
|
942
1283
|
// can be skipped when the file cannot be read.
|
|
@@ -945,6 +1286,16 @@ export class LoopScheduler {
|
|
|
945
1286
|
updated = await this.store.update(job.id, (j) => {
|
|
946
1287
|
if (j.running?.runId === runId) j.running = undefined;
|
|
947
1288
|
j.lastCompletedAt = finishedAt;
|
|
1289
|
+
if (byPerson) {
|
|
1290
|
+
// The slot is spent; `runCount` stays (the `ordinal` comment on RunRecord), and a stop is
|
|
1291
|
+
// not a failure, so the streak and the last error stay what they were. A one-shot has no
|
|
1292
|
+
// next slot: it is turned off, not deleted, so its notes are kept.
|
|
1293
|
+
if (job.schedule.kind === "once") {
|
|
1294
|
+
j.enabled = false;
|
|
1295
|
+
j.lastError = `${STOPPED_BY_PERSON}; /cron run ${job.name ?? job.id} runs it now`;
|
|
1296
|
+
}
|
|
1297
|
+
return;
|
|
1298
|
+
}
|
|
948
1299
|
j.lastError = result.ok ? undefined : record.error;
|
|
949
1300
|
if (aborted) {
|
|
950
1301
|
j.lastDueAt = prior.dueAt;
|
|
@@ -954,13 +1305,14 @@ export class LoopScheduler {
|
|
|
954
1305
|
// A job that fails every time costs money every time. Count the streak so `dispatch`
|
|
955
1306
|
// can back off, and clear it the moment one run works.
|
|
956
1307
|
j.consecutiveFailures = result.ok ? undefined : (j.consecutiveFailures ?? 0) + 1;
|
|
1308
|
+
j.interruptions = undefined;
|
|
957
1309
|
}
|
|
958
1310
|
});
|
|
959
1311
|
} catch (err: any) {
|
|
960
|
-
this.log(`loop ${job.id}: could not write the store after the run (${err?.message ?? err})
|
|
1312
|
+
this.log(`loop ${job.id}: could not write the store after the run (${err?.message ?? err})`, undefined, job);
|
|
961
1313
|
}
|
|
962
|
-
if (updated && !result.ok && (updated.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER) {
|
|
963
|
-
this.hooks.onSchedulerError?.(`${updated.name ?? updated.id} has failed ${updated.consecutiveFailures} times in a row; backing off (last: ${record.error ?? "unknown"})
|
|
1314
|
+
if (updated && !result.ok && !byPerson && (updated.consecutiveFailures ?? 0) >= FAILURE_BACKOFF_AFTER) {
|
|
1315
|
+
this.hooks.onSchedulerError?.(`${updated.name ?? updated.id} has failed ${updated.consecutiveFailures} times in a row; backing off (last: ${record.error ?? "unknown"})`, updated);
|
|
964
1316
|
}
|
|
965
1317
|
// Only now: while the run id is in `inflight`, `clearStaleRunning` treats the job as alive,
|
|
966
1318
|
// which is exactly right until its `running` marker is gone from the store.
|
|
@@ -1013,6 +1365,7 @@ export class LoopScheduler {
|
|
|
1013
1365
|
model,
|
|
1014
1366
|
thinking,
|
|
1015
1367
|
tools: job.tools,
|
|
1368
|
+
builtinTools: job.builtinTools,
|
|
1016
1369
|
timeoutMs: job.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS,
|
|
1017
1370
|
signal,
|
|
1018
1371
|
sessionDir: this.store.sessionDirFor(job.id),
|
|
@@ -1049,7 +1402,7 @@ export class LoopScheduler {
|
|
|
1049
1402
|
model: result.model ?? model,
|
|
1050
1403
|
sessionFile: result.sessionFile,
|
|
1051
1404
|
};
|
|
1052
|
-
if (!result.ok) this.log(`loop ${job.name ?? job.id}: checker failed (${record.error}); findings enter the inbox unverified
|
|
1405
|
+
if (!result.ok) this.log(`loop ${job.name ?? job.id}: checker failed (${record.error}); findings enter the inbox unverified`, undefined, job);
|
|
1053
1406
|
return { record, reviewed };
|
|
1054
1407
|
}
|
|
1055
1408
|
}
|