agent-accelerator 0.5.6 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -17
- package/package.json +1 -1
- package/src/agent/agent.ts +92 -57
- package/src/agent/architecture.md +25 -0
- package/src/agent/context.ts +7 -3
- package/src/agent/delegation.ts +30 -96
- package/src/agent/loop.ts +54 -56
- package/src/agent/subagent.ts +4 -4
- package/src/agent/worker.ts +1 -3
- package/src/architecture.md +28 -0
- package/src/index.ts +13 -80
- package/src/models/architecture.md +14 -0
- package/src/models/catalog-cache.ts +10 -7
- package/src/models/catalog.ts +20 -13
- package/src/providers/architecture.md +24 -0
- package/src/providers/google.ts +24 -4
- package/src/providers/openai-compat.ts +29 -7
- package/src/providers/openai.ts +22 -6
- package/src/providers/openrouter.ts +14 -2
- package/src/providers/registry.ts +1 -0
- package/src/providers/shared.ts +4 -4
- package/src/providers.ts +9 -53
- package/src/scheduler/architecture.md +22 -0
- package/src/scheduler/scheduler.ts +131 -56
- package/src/scheduler/store.ts +274 -54
- package/src/scheduler/time.ts +2 -0
- package/src/session/architecture.md +19 -0
- package/src/session/store.ts +37 -22
- package/src/session/usage.ts +1 -0
- package/src/streaming/architecture.md +13 -0
- package/src/streaming/event-stream.ts +16 -11
- package/src/telemetry-file.ts +74 -24
- package/src/telemetry.ts +3 -2
- package/src/tools/architecture.md +19 -0
- package/src/tools/executor.ts +14 -1
- package/src/tools/schema.ts +1 -0
- package/src/tools/subagent-result.ts +1 -0
- package/src/tools/tool.ts +8 -8
- package/src/types/agent.ts +22 -20
- package/src/types/architecture.md +20 -0
- package/src/types/response.ts +8 -8
- package/src/types/scheduler.ts +2 -2
- package/src/types/tool.ts +5 -5
- package/src/update-models.ts +2 -2
- package/src/utils/architecture.md +25 -0
- package/src/utils/base64.ts +2 -0
- package/src/utils/cache.ts +6 -4
- package/src/utils/durable.ts +78 -3
- package/src/utils/env.ts +1 -1
- package/src/utils/errors.ts +4 -0
- package/src/utils/headers.ts +7 -4
- package/src/utils/home.ts +1 -1
- package/src/utils/media.ts +1 -0
- package/src/utils/raw-capture.ts +3 -0
- package/src/utils/retry.ts +2 -1
- package/src/utils/safe-io.ts +4 -0
- package/src/utils/session.ts +10 -166
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* One clock per home folder, ticking once a second against the machine's own
|
|
6
6
|
* time. Each tick only asks "what is overdue?", so drift, sleep or a clock
|
|
7
|
-
* change cannot skip a run. The files under `<home>/
|
|
7
|
+
* change cannot skip a run. The files under `<home>/scheduled/` are the
|
|
8
8
|
* source of truth (see `store.ts`); nothing waits in memory, and the timer is
|
|
9
9
|
* unref'd and runs only while something is pending.
|
|
10
10
|
*
|
|
@@ -12,12 +12,13 @@
|
|
|
12
12
|
* | |
|
|
13
13
|
* +--> failed (recorded, moves on) +--> waiting (repeat) | done
|
|
14
14
|
*/
|
|
15
|
+
import * as os from "node:os";
|
|
15
16
|
import type { Agent } from "../agent/agent.ts";
|
|
16
17
|
import { createChildSessionId } from "../agent/delegation.ts";
|
|
17
18
|
import { buildDynamicWorkerConfig, grantWorkerTools, scheduledRunTimeoutMs } from "../agent/worker.ts";
|
|
18
19
|
import { telemetryBus } from "../telemetry.ts";
|
|
19
20
|
import { readUsage, usageLine, type UsageTotals } from "../session/usage.ts";
|
|
20
|
-
import {
|
|
21
|
+
import { processAlive } from "../utils/durable.ts";
|
|
21
22
|
import { ConfigError, ValidationError } from "../types/errors.ts";
|
|
22
23
|
import type { ToolDefinition } from "../types/tool.ts";
|
|
23
24
|
import type { AgentResponse } from "../types/response.ts";
|
|
@@ -31,18 +32,24 @@ import type {
|
|
|
31
32
|
} from "../types/scheduler.ts";
|
|
32
33
|
import {
|
|
33
34
|
TASK_ID_PATTERN,
|
|
35
|
+
clearHeartbeat,
|
|
36
|
+
createTask,
|
|
37
|
+
deleteTask,
|
|
34
38
|
findRunFile,
|
|
35
|
-
migrateLegacyFolder,
|
|
36
39
|
readDeliveries,
|
|
40
|
+
readHeartbeat,
|
|
37
41
|
readRun,
|
|
42
|
+
readTask,
|
|
38
43
|
readTasks,
|
|
39
44
|
removeTaskFolder,
|
|
40
45
|
resolveHome,
|
|
41
46
|
runPath,
|
|
47
|
+
touchHeartbeat,
|
|
42
48
|
updateDeliveries,
|
|
43
|
-
|
|
49
|
+
updateTask,
|
|
44
50
|
validateDelivery,
|
|
45
51
|
validateRecord,
|
|
52
|
+
writeHeartbeat,
|
|
46
53
|
writeRunFile,
|
|
47
54
|
type PendingDelivery,
|
|
48
55
|
type RunHeader,
|
|
@@ -61,6 +68,11 @@ import {
|
|
|
61
68
|
/** Default lead time (10 minutes) when the developer API omits `startCookingInAdvance`. */
|
|
62
69
|
const DEFAULT_COOK_LEAD = "00:00:10";
|
|
63
70
|
const SLUG_MAX = 48;
|
|
71
|
+
/** A heartbeat older than this means its owner is gone (it is refreshed every 5 s). */
|
|
72
|
+
const HEARTBEAT_STALE_MS = 60_000;
|
|
73
|
+
const HEARTBEAT_EVERY_MS = 5_000;
|
|
74
|
+
/** How often a live process looks for runs whose owner process died. */
|
|
75
|
+
const DEAD_SWEEP_EVERY_MS = 15_000;
|
|
64
76
|
const ID_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789";
|
|
65
77
|
|
|
66
78
|
/** Optional `{ home }` accepted by every public function except `createScheduledAction` (the agent knows its home). */
|
|
@@ -81,6 +93,7 @@ interface RunningEntry {
|
|
|
81
93
|
missed: boolean;
|
|
82
94
|
}
|
|
83
95
|
|
|
96
|
+
// The message of any thrown value; in: catch blocks here, out: event and error text.
|
|
84
97
|
const errText = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
|
85
98
|
|
|
86
99
|
// A run's exact spending as run-header fields, the numbers `usageReport()` keeps for workers (tokens split, cache hit rate, cost and where the price came from); in: the one-shot worker's totals over all attempts, out: finish's RunHeader.
|
|
@@ -106,10 +119,12 @@ function usageFields(totals: UsageTotals | undefined): Partial<RunHeader> {
|
|
|
106
119
|
// Whole seconds since the epoch: stored times have no milliseconds, so lateness is compared at this precision; in: a clock reading or stored time, out: finish and deliver.
|
|
107
120
|
const secondOf = (d: Date): number => Math.floor(d.getTime() / 1000);
|
|
108
121
|
|
|
122
|
+
// Emits one `scheduled_action.<code>` event on the telemetry bus; in: the scheduler and public functions, out: telemetryBus.
|
|
109
123
|
function emit(code: string, message: string, data: Record<string, unknown>, level: "info" | "warn" = "info"): void {
|
|
110
124
|
telemetryBus.emit({ code: `scheduled_action.${code}`, message: `[Agent Accelerator] ${message}`, level, data });
|
|
111
125
|
}
|
|
112
126
|
|
|
127
|
+
// The ValidationError for an unknown action id; in: lookups by id, out: a thrown error.
|
|
113
128
|
function notFound(id: string): ValidationError {
|
|
114
129
|
return new ValidationError(`No scheduled action with id "${id}".`);
|
|
115
130
|
}
|
|
@@ -118,6 +133,7 @@ function notFound(id: string): ValidationError {
|
|
|
118
133
|
// Validation (collects EVERY problem so the model can fix a call in one go)
|
|
119
134
|
// ---------------------------------------------------------------------------
|
|
120
135
|
|
|
136
|
+
// Runs one validation step and records its message instead of throwing, so one call can report every problem; in: validateInput/checkSchedule, out: the problems list.
|
|
121
137
|
function collect(problems: string[], fn: () => void): void {
|
|
122
138
|
try {
|
|
123
139
|
fn();
|
|
@@ -126,10 +142,12 @@ function collect(problems: string[], fn: () => void): void {
|
|
|
126
142
|
}
|
|
127
143
|
}
|
|
128
144
|
|
|
145
|
+
// Throws unless the value is a non-empty string; in: field validation, out: collect.
|
|
129
146
|
function requireText(field: string, value: unknown): void {
|
|
130
147
|
if (typeof value !== "string" || !value.trim()) throw new ValidationError(`${field}: required, a non-empty string.`);
|
|
131
148
|
}
|
|
132
149
|
|
|
150
|
+
// Throws unless every requested worker tool name exists in the developer's pool; in: create/update validation, out: collect.
|
|
133
151
|
function checkTools(value: unknown, pool: PoolTools): void {
|
|
134
152
|
if (!Array.isArray(value) || value.some((n) => typeof n !== "string")) {
|
|
135
153
|
throw new ValidationError("tools: must be an array of worker tool names, e.g. [\"recent_news\"].");
|
|
@@ -174,6 +192,7 @@ function checkSchedule(
|
|
|
174
192
|
}
|
|
175
193
|
}
|
|
176
194
|
|
|
195
|
+
// Validates role, tools and timeoutMs of a create or update; in: validateInput/updateScheduledAction, out: the problems list.
|
|
177
196
|
function checkOptionalFields(input: Pick<ScheduledActionPatch, "role" | "tools" | "timeoutMs">, pool: PoolTools | undefined, problems: string[]): void {
|
|
178
197
|
if (input.role !== undefined && typeof input.role !== "string") problems.push("role: must be a string.");
|
|
179
198
|
if (input.tools !== undefined && pool) collect(problems, () => checkTools(input.tools, pool));
|
|
@@ -208,9 +227,11 @@ export function validateInput(
|
|
|
208
227
|
return problems;
|
|
209
228
|
}
|
|
210
229
|
|
|
230
|
+
// Joins validation problems into one numbered ValidationError message; in: create/update failures, out: the error text.
|
|
211
231
|
const joinProblems = (what: string, problems: string[]): string =>
|
|
212
232
|
`${what} (${problems.length} problem${problems.length === 1 ? "" : "s"}): ${problems.map((p, i) => `(${i + 1}) ${p}`).join(" ")}`;
|
|
213
233
|
|
|
234
|
+
// Six random lowercase/digit characters appended to an action id; in: createScheduledAction, out: the unique id suffix.
|
|
214
235
|
function randomSuffix(): string {
|
|
215
236
|
const bytes = new Uint8Array(6);
|
|
216
237
|
globalThis.crypto.getRandomValues(bytes);
|
|
@@ -231,6 +252,10 @@ export class Scheduler {
|
|
|
231
252
|
tickMs = 1000;
|
|
232
253
|
private timer: ReturnType<typeof setInterval> | undefined;
|
|
233
254
|
private swept = false;
|
|
255
|
+
/** Names this scheduler instance in the heartbeat files of the runs it owns. */
|
|
256
|
+
private readonly token = Math.random().toString(36).slice(2, 10);
|
|
257
|
+
private lastDeadSweep = 0;
|
|
258
|
+
private lastBeat = 0;
|
|
234
259
|
private readonly warned = new Set<string>();
|
|
235
260
|
|
|
236
261
|
constructor(readonly home: string) {}
|
|
@@ -244,6 +269,7 @@ export class Scheduler {
|
|
|
244
269
|
(this.timer as { unref?: () => void }).unref?.();
|
|
245
270
|
}
|
|
246
271
|
|
|
272
|
+
// Stops the clock's interval; in: tick when nothing is pending and resetSchedulers, out: no more ticks.
|
|
247
273
|
stop(): void {
|
|
248
274
|
if (this.timer) clearInterval(this.timer);
|
|
249
275
|
this.timer = undefined;
|
|
@@ -258,6 +284,8 @@ export class Scheduler {
|
|
|
258
284
|
this.swept = true;
|
|
259
285
|
this.startupSweep(now);
|
|
260
286
|
}
|
|
287
|
+
this.keepHeartbeats(now);
|
|
288
|
+
this.sweepDeadRuns(now);
|
|
261
289
|
this.deliverDue(now);
|
|
262
290
|
this.startDue(now);
|
|
263
291
|
if (!this.hasPending()) this.stop();
|
|
@@ -266,7 +294,8 @@ export class Scheduler {
|
|
|
266
294
|
}
|
|
267
295
|
}
|
|
268
296
|
|
|
269
|
-
|
|
297
|
+
// Reports an invalid record or delivery once per file/id/problem and leaves the file untouched; in: snapshot/startCook, out: a scheduled_action.invalid warning.
|
|
298
|
+
private warnInvalid(id: string, problem: string, file = "task.json"): void {
|
|
270
299
|
const key = `${file}:${id}:${problem}`;
|
|
271
300
|
if (this.warned.has(key)) return;
|
|
272
301
|
this.warned.add(key);
|
|
@@ -286,12 +315,13 @@ export class Scheduler {
|
|
|
286
315
|
});
|
|
287
316
|
const pending = readDeliveries(this.home).filter((p) => {
|
|
288
317
|
const problem = validateDelivery(p);
|
|
289
|
-
if (problem) this.warnInvalid(String((p as { id?: unknown }).id), problem, "
|
|
318
|
+
if (problem) this.warnInvalid(String((p as { id?: unknown }).id), problem, "delivery.json");
|
|
290
319
|
return !problem;
|
|
291
320
|
});
|
|
292
321
|
return { live, pending };
|
|
293
322
|
}
|
|
294
323
|
|
|
324
|
+
// True while anything keeps the clock useful (a run in flight, a saved delivery, a waiting or cooking action); in: ensureRunning and tick, out: start/stop of the interval.
|
|
295
325
|
private hasPending(): boolean {
|
|
296
326
|
const { live, pending } = this.snapshot();
|
|
297
327
|
return this.running.size > 0 || pending.length > 0 || live.some((t) => t.state === "waiting" || t.state === "cooking");
|
|
@@ -306,23 +336,54 @@ export class Scheduler {
|
|
|
306
336
|
*/
|
|
307
337
|
private startupSweep(now: Date): void {
|
|
308
338
|
for (const t of this.snapshot().live) {
|
|
309
|
-
if (t.
|
|
310
|
-
|
|
311
|
-
this.markMissed(t.id, "process_down", t.schedule.onMissed);
|
|
339
|
+
if (!this.ownsHere(t.agent)) continue; // another process's action (or nobody's: startDue handles that)
|
|
340
|
+
if (t.state === "waiting" && t.nextDeliverAt && parseLocalIso(t.nextDeliverAt).getTime() <= now.getTime()) {
|
|
341
|
+
this.markMissed(t.id, "process_down", "waiting", t.schedule.onMissed);
|
|
312
342
|
}
|
|
313
343
|
}
|
|
314
344
|
}
|
|
315
345
|
|
|
346
|
+
// True when a live Agent with this name is registered in THIS process: only then may this process start, deliver or recover its actions; in: the sweeps, startDue and deliverDue, out: a boolean.
|
|
347
|
+
private ownsHere(agentName: string): boolean {
|
|
348
|
+
return !!this.agents.get(agentName)?.deref();
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// True while the process that started a run is still running it: its heartbeat is fresh and (same host) its pid exists; this process counts only for runs in this.running; in: the dead-run sweep and delete, out: the heartbeat or undefined.
|
|
352
|
+
private liveOwner(id: string): { pid: number } | undefined {
|
|
353
|
+
if (this.running.has(id)) return { pid: process.pid };
|
|
354
|
+
const hb = readHeartbeat(this.home, id);
|
|
355
|
+
if (!hb || hb.ageMs > HEARTBEAT_STALE_MS) return undefined;
|
|
356
|
+
if (hb.pid === process.pid) return undefined; // an earlier scheduler of this very process is gone
|
|
357
|
+
if (hb.host === os.hostname() && !processAlive(hb.pid)) return undefined;
|
|
358
|
+
return { pid: hb.pid };
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Refreshes the heartbeat of every run in flight (at most every 5 s); in: tick, out: the .heartbeat files.
|
|
362
|
+
private keepHeartbeats(now: Date): void {
|
|
363
|
+
if (this.running.size === 0 || Math.abs(now.getTime() - this.lastBeat) < HEARTBEAT_EVERY_MS) return;
|
|
364
|
+
this.lastBeat = now.getTime();
|
|
365
|
+
for (const id of this.running.keys()) touchHeartbeat(this.home, id);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
// Recovers the runs whose owner is gone (a killed process), but only for actions this process owns, at start and then every 15 s; in: tick, out: recoverCooking calls.
|
|
369
|
+
private sweepDeadRuns(now: Date): void {
|
|
370
|
+
if (this.lastDeadSweep !== 0 && Math.abs(now.getTime() - this.lastDeadSweep) < DEAD_SWEEP_EVERY_MS) return;
|
|
371
|
+
this.lastDeadSweep = now.getTime();
|
|
372
|
+
for (const t of this.snapshot().live) {
|
|
373
|
+
if (t.state === "cooking" && this.ownsHere(t.agent) && !this.liveOwner(t.id)) this.recoverCooking(t);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
316
377
|
/** The process died mid-cook: rebuild the delivery if the run file was already written, else it is missed. */
|
|
317
378
|
private recoverCooking(t: ScheduledActionRecord): void {
|
|
318
379
|
const current = t.current!;
|
|
319
380
|
const file = findRunFile(this.home, t.id, parseLocalIso(current.deliverAt), current.cookStartedAt);
|
|
320
381
|
const run = file ? readRun(this.home, file) : undefined;
|
|
321
382
|
if (!file || !run) {
|
|
322
|
-
this.markMissed(t.id, "interrupted", t.schedule.onMissed);
|
|
383
|
+
this.markMissed(t.id, "interrupted", "cooking", t.schedule.onMissed);
|
|
323
384
|
return;
|
|
324
385
|
}
|
|
325
|
-
updateDeliveries(this.home, (pending) => {
|
|
386
|
+
updateDeliveries(this.home, t.id, (pending) => {
|
|
326
387
|
if (pending.some((p) => p.file === file)) return;
|
|
327
388
|
pending.push({
|
|
328
389
|
id: t.id,
|
|
@@ -333,24 +394,24 @@ export class Scheduler {
|
|
|
333
394
|
...(run.header["late"] === "true" ? { late: true } : {}),
|
|
334
395
|
});
|
|
335
396
|
});
|
|
336
|
-
|
|
337
|
-
const rec = tasks.find((x) => x.id === t.id);
|
|
397
|
+
updateTask(this.home, t.id, (rec) => {
|
|
338
398
|
if (!rec || rec.state !== "cooking") return;
|
|
339
399
|
delete rec.current;
|
|
340
400
|
rec.state = rec.nextDeliverAt ? "waiting" : "done";
|
|
341
401
|
});
|
|
342
402
|
}
|
|
343
403
|
|
|
344
|
-
// Marks a run whose moment passed as missed, then applies the action's onMissed policy: `skip` moves on to the next time, `run` runs it now, unset waits for run_now / skip; in
|
|
345
|
-
private markMissed(id: string, reason: string, policy?: "skip" | "run"): void {
|
|
404
|
+
// Marks a run whose moment passed as missed, then applies the action's onMissed policy: `skip` moves on to the next time, `run` runs it now, unset waits for run_now / skip; only when the action is still in the state the caller saw (`from`), so two processes never both mark it; in: the sweeps, out: task.json + the missed (and skipped / started) events.
|
|
405
|
+
private markMissed(id: string, reason: string, from: "waiting" | "cooking", policy?: "skip" | "run"): void {
|
|
346
406
|
let name = id;
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
if (!t) return;
|
|
407
|
+
const changed = updateTask(this.home, id, (t) => {
|
|
408
|
+
if (!t || t.state !== from) return false; // another process already dealt with it
|
|
350
409
|
name = t.name;
|
|
351
410
|
t.state = "missed";
|
|
352
411
|
delete t.current;
|
|
412
|
+
return true;
|
|
353
413
|
});
|
|
414
|
+
if (!changed) return;
|
|
354
415
|
const next = policy === "skip" ? "Skipping it (onMissed: skip)." : policy === "run" ? "Running it now (onMissed: run)." : "Use run_now or skip.";
|
|
355
416
|
emit("missed", `"${name}" (${id}) missed its run (${reason}). ${next}`, { id, name, reason, ...(policy ? { onMissed: policy } : {}) }, "warn");
|
|
356
417
|
if (policy === "skip") this.skip(id);
|
|
@@ -359,23 +420,28 @@ export class Scheduler {
|
|
|
359
420
|
|
|
360
421
|
// ----- delivery ----------------------------------------------------------
|
|
361
422
|
|
|
423
|
+
// Delivers every saved run whose delivery time has come; in: tick, out: deliver calls.
|
|
362
424
|
private deliverDue(now: Date): void {
|
|
363
425
|
for (const entry of this.snapshot().pending) {
|
|
364
|
-
if (parseLocalIso(entry.deliverAt).getTime()
|
|
426
|
+
if (parseLocalIso(entry.deliverAt).getTime() > now.getTime()) continue;
|
|
427
|
+
const owner = readTask(this.home, entry.id)?.agent; // the delivered event must fire in the process that owns the action
|
|
428
|
+
if (owner === undefined || this.ownsHere(owner)) this.deliver(entry, now);
|
|
365
429
|
}
|
|
366
430
|
}
|
|
367
431
|
|
|
368
|
-
// Removes the entry, records `lastRun`, THEN emits (whether or not anyone listens); late = it replaced a missed occurrence, was delivered over two ticks after its time, or finished after it; in: deliverDue, out:
|
|
432
|
+
// Removes the entry (only the process that removed it goes on), records `lastRun`, THEN emits (whether or not anyone listens); late = it replaced a missed occurrence, was delivered over two ticks after its time, or finished after it; in: deliverDue, out: task.json + the delivered event.
|
|
369
433
|
private deliver(entry: PendingDelivery, now: Date): void {
|
|
370
|
-
updateDeliveries(this.home, (pending) => {
|
|
434
|
+
const removed = updateDeliveries(this.home, entry.id, (pending) => {
|
|
371
435
|
const i = pending.findIndex((p) => p.id === entry.id && p.file === entry.file);
|
|
372
|
-
if (i
|
|
436
|
+
if (i < 0) return false;
|
|
437
|
+
pending.splice(i, 1);
|
|
438
|
+
return true;
|
|
373
439
|
});
|
|
440
|
+
if (!removed) return; // another process took this delivery: exactly one process delivers it
|
|
374
441
|
const deliverAt = parseLocalIso(entry.deliverAt).getTime();
|
|
375
442
|
const late = entry.late === true || now.getTime() - deliverAt > 2 * this.tickMs || parseLocalIso(entry.readyAt).getTime() > deliverAt;
|
|
376
443
|
let name = entry.id;
|
|
377
|
-
|
|
378
|
-
const t = tasks.find((x) => x.id === entry.id);
|
|
444
|
+
updateTask(this.home, entry.id, (t) => {
|
|
379
445
|
if (!t) return;
|
|
380
446
|
name = t.name;
|
|
381
447
|
t.lastRun = { status: entry.status, file: entry.file, deliverAt: entry.deliverAt, deliveredAt: toLocalIso(now), late };
|
|
@@ -394,8 +460,16 @@ export class Scheduler {
|
|
|
394
460
|
|
|
395
461
|
// ----- starting runs -----------------------------------------------------
|
|
396
462
|
|
|
463
|
+
// Starts waiting actions whose cook time has come and skips the overlap of those still cooking; in: tick, out: startCook/skipOverlap calls.
|
|
397
464
|
private startDue(now: Date): void {
|
|
398
465
|
for (const t of this.snapshot().live) {
|
|
466
|
+
if (!this.ownsHere(t.agent)) {
|
|
467
|
+
// Nobody here can run it. Another process may own it, so only once its delivery time has passed with no run started is it missed.
|
|
468
|
+
if (t.state === "waiting" && t.nextDeliverAt && parseLocalIso(t.nextDeliverAt).getTime() <= now.getTime()) {
|
|
469
|
+
this.markMissed(t.id, "no_agent", "waiting");
|
|
470
|
+
}
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
399
473
|
if (!t.nextCookAt || parseLocalIso(t.nextCookAt).getTime() > now.getTime()) continue;
|
|
400
474
|
if (t.state === "waiting") this.startCook(t.id, false);
|
|
401
475
|
else if (t.state === "cooking") this.skipOverlap(t.id, now);
|
|
@@ -419,8 +493,7 @@ export class Scheduler {
|
|
|
419
493
|
private startCook(id: string, manual: boolean): ScheduledActionRecord | undefined {
|
|
420
494
|
const now = this.now();
|
|
421
495
|
const ctx: { agent?: Agent; noAgent?: boolean; replacesMissed?: boolean } = {};
|
|
422
|
-
const started =
|
|
423
|
-
const t = tasks.find((x) => x.id === id);
|
|
496
|
+
const started = updateTask(this.home, id, (t) => {
|
|
424
497
|
if (!t) return undefined;
|
|
425
498
|
const problem = validateRecord(t);
|
|
426
499
|
if (problem) {
|
|
@@ -428,6 +501,8 @@ export class Scheduler {
|
|
|
428
501
|
return undefined;
|
|
429
502
|
}
|
|
430
503
|
if (t.state === "cooking") return undefined;
|
|
504
|
+
// Another process may have started (and even finished) this run since the caller looked: a scheduled start needs the action to be waiting and due right now.
|
|
505
|
+
if (!manual && (t.state !== "waiting" || !t.nextCookAt || parseLocalIso(t.nextCookAt).getTime() > now.getTime())) return undefined;
|
|
431
506
|
ctx.replacesMissed = manual && t.state === "missed";
|
|
432
507
|
ctx.agent = this.agents.get(t.agent)?.deref();
|
|
433
508
|
if (!ctx.agent) {
|
|
@@ -451,11 +526,12 @@ export class Scheduler {
|
|
|
451
526
|
t.current = { cookStartedAt: toLocalIso(now), deliverAt: toLocalIso(deliverAt) };
|
|
452
527
|
t.nextCookAt = next.cook;
|
|
453
528
|
t.nextDeliverAt = next.deliver;
|
|
529
|
+
writeHeartbeat(this.home, id, this.token); // before the record shows `cooking`, so no process sees a cooking action without an owner
|
|
454
530
|
return t;
|
|
455
531
|
});
|
|
456
532
|
if (ctx.noAgent) {
|
|
457
533
|
const name = this.nameOf(id);
|
|
458
|
-
emit("missed", `"${name}" (${id}) cannot run: no live Agent named in
|
|
534
|
+
emit("missed", `"${name}" (${id}) cannot run: no live Agent named in its task.json is registered.`, { id, name, reason: "no_agent" }, "warn");
|
|
459
535
|
}
|
|
460
536
|
if (!started || !ctx.agent) return undefined;
|
|
461
537
|
emit("started", `"${started.name}" (${id}) started cooking.`, {
|
|
@@ -468,15 +544,15 @@ export class Scheduler {
|
|
|
468
544
|
return started;
|
|
469
545
|
}
|
|
470
546
|
|
|
547
|
+
// The display name of an action from task.json, else its id; in: startCook's no-agent message, out: event text.
|
|
471
548
|
private nameOf(id: string): string {
|
|
472
|
-
return
|
|
549
|
+
return readTask(this.home, id)?.name ?? id;
|
|
473
550
|
}
|
|
474
551
|
|
|
475
552
|
/** The next run is due while this one still cooks: never stack. Skip that occurrence. */
|
|
476
553
|
private skipOverlap(id: string, now: Date): void {
|
|
477
554
|
const ctx: { skipped?: { name: string; deliverAt: string } } = {};
|
|
478
|
-
|
|
479
|
-
const t = tasks.find((x) => x.id === id);
|
|
555
|
+
updateTask(this.home, id, (t) => {
|
|
480
556
|
if (!t || t.state !== "cooking" || !t.nextCookAt || !t.nextDeliverAt) return;
|
|
481
557
|
ctx.skipped = { name: t.name, deliverAt: t.nextDeliverAt };
|
|
482
558
|
if (!t.schedule.repeatEvery) {
|
|
@@ -505,12 +581,14 @@ export class Scheduler {
|
|
|
505
581
|
})
|
|
506
582
|
.finally(() => {
|
|
507
583
|
this.running.delete(task.id);
|
|
584
|
+
clearHeartbeat(this.home, task.id);
|
|
508
585
|
});
|
|
509
586
|
this.ensureRunning();
|
|
510
587
|
}
|
|
511
588
|
|
|
512
589
|
// ----- the run -----------------------------------------------------------
|
|
513
590
|
|
|
591
|
+
// Runs one action to its outcome and writes the result unless the action was deleted meanwhile; in: launch, out: finish.
|
|
514
592
|
private async runWorker(task: ScheduledActionRecord, agent: Agent, entry: RunningEntry): Promise<void> {
|
|
515
593
|
const deliverAt = parseLocalIso(task.current!.deliverAt);
|
|
516
594
|
const outcome = await this.execute(task, agent, entry, deliverAt);
|
|
@@ -556,6 +634,7 @@ export class Scheduler {
|
|
|
556
634
|
}
|
|
557
635
|
}
|
|
558
636
|
|
|
637
|
+
// One worker run with a per-attempt timeout and the delete signal, raced so a hung provider cannot hold it; in: execute, out: the worker's AgentResponse.
|
|
559
638
|
private async attempt(worker: Agent, task: ScheduledActionRecord, agent: Agent, entry: RunningEntry): Promise<AgentResponse> {
|
|
560
639
|
const ms = scheduledRunTimeoutMs(agent.dynamicSubagents?.workerTimeoutMs ?? Infinity, task.timeoutMs);
|
|
561
640
|
// One controller per attempt (the timeout is per attempt), fed by the entry's delete controller.
|
|
@@ -576,7 +655,7 @@ export class Scheduler {
|
|
|
576
655
|
}
|
|
577
656
|
}
|
|
578
657
|
|
|
579
|
-
// Writes a finished run: run file, then the `
|
|
658
|
+
// Writes a finished run: run file, then the `delivery.json` entry, then `task.json` (a crash in between is repaired on start); a manual run is due when it finishes and is late only if it replaced a missed occurrence, a scheduled one is late only past its second; in: runWorker, out: the files and the ready/failed event.
|
|
580
659
|
private finish(task: ScheduledActionRecord, outcome: Outcome, startedAt: string, deliverAt: Date, run: Pick<RunningEntry, "manual" | "missed">): void {
|
|
581
660
|
const now = this.now();
|
|
582
661
|
const ok = "res" in outcome;
|
|
@@ -597,11 +676,10 @@ export class Scheduler {
|
|
|
597
676
|
};
|
|
598
677
|
const body = ok ? outcome.res.text.trim() || "[The worker returned no text.]" : `The run failed: ${outcome.error}`;
|
|
599
678
|
const file = writeRunFile(this.home, task.id, deliverAt, header, body);
|
|
600
|
-
updateDeliveries(this.home, (pending) => {
|
|
679
|
+
updateDeliveries(this.home, task.id, (pending) => {
|
|
601
680
|
pending.push({ id: task.id, file, status: header.status, readyAt: finishedAt, deliverAt: header.deliverAt, ...(run.missed ? { late: true } : {}) });
|
|
602
681
|
});
|
|
603
|
-
|
|
604
|
-
const t = tasks.find((x) => x.id === task.id);
|
|
682
|
+
updateTask(this.home, task.id, (t) => {
|
|
605
683
|
if (!t || t.state !== "cooking") return;
|
|
606
684
|
delete t.current;
|
|
607
685
|
t.state = t.nextDeliverAt ? "waiting" : "done";
|
|
@@ -613,8 +691,9 @@ export class Scheduler {
|
|
|
613
691
|
|
|
614
692
|
// ----- commands ----------------------------------------------------------
|
|
615
693
|
|
|
694
|
+
// Starts an action immediately outside its schedule (also resolves a missed one); in: runScheduledAction, out: the cooking record.
|
|
616
695
|
runNow(id: string): ScheduledActionRecord {
|
|
617
|
-
const rec =
|
|
696
|
+
const rec = readTask(this.home, id);
|
|
618
697
|
if (!rec) throw notFound(id);
|
|
619
698
|
if (rec.state === "cooking" || this.running.has(id)) {
|
|
620
699
|
throw new ValidationError(`Scheduled action "${id}" is already cooking; wait for it to finish.`);
|
|
@@ -629,10 +708,10 @@ export class Scheduler {
|
|
|
629
708
|
return started;
|
|
630
709
|
}
|
|
631
710
|
|
|
711
|
+
// Ignores a missed occurrence and schedules the next one (a one-shot becomes done); in: skipScheduledAction and onMissed: skip, out: the updated record.
|
|
632
712
|
skip(id: string): ScheduledActionRecord {
|
|
633
713
|
const now = this.now();
|
|
634
|
-
const rec =
|
|
635
|
-
const t = tasks.find((x) => x.id === id);
|
|
714
|
+
const rec = updateTask(this.home, id, (t) => {
|
|
636
715
|
if (!t) throw notFound(id);
|
|
637
716
|
if (t.state !== "missed") throw new ValidationError(`Scheduled action "${id}" is ${t.state}, not missed: nothing to skip.`);
|
|
638
717
|
const next = this.afterMissed(t, now);
|
|
@@ -647,10 +726,19 @@ export class Scheduler {
|
|
|
647
726
|
return rec;
|
|
648
727
|
}
|
|
649
728
|
|
|
729
|
+
// Deletes an action (aborting a cooking run only with force) and optionally purges its run files; in: deleteScheduledAction, out: a DeleteResult.
|
|
650
730
|
async delete(id: string, opts: { force?: boolean; purge?: boolean }): Promise<DeleteResult> {
|
|
651
|
-
const rec =
|
|
731
|
+
const rec = readTask(this.home, id);
|
|
652
732
|
if (!rec) return { ok: false, reason: "not_found" };
|
|
653
733
|
const run = this.running.get(id);
|
|
734
|
+
const other = !run && rec.state === "cooking" ? this.liveOwner(id) : undefined;
|
|
735
|
+
if (other) {
|
|
736
|
+
return {
|
|
737
|
+
ok: false,
|
|
738
|
+
reason: "running",
|
|
739
|
+
message: `"${id}" is cooking in another process (pid ${other.pid}), so nothing was changed. Only that process can abort it: wait for it to finish, or delete it from there.`,
|
|
740
|
+
};
|
|
741
|
+
}
|
|
654
742
|
if (run && !opts.force) {
|
|
655
743
|
return {
|
|
656
744
|
ok: false,
|
|
@@ -663,13 +751,7 @@ export class Scheduler {
|
|
|
663
751
|
run.controller.abort(new Error("deleted"));
|
|
664
752
|
await run.done;
|
|
665
753
|
}
|
|
666
|
-
|
|
667
|
-
const i = tasks.findIndex((t) => t.id === id);
|
|
668
|
-
if (i >= 0) tasks.splice(i, 1);
|
|
669
|
-
});
|
|
670
|
-
updateDeliveries(this.home, (pending) => {
|
|
671
|
-
for (let i = pending.length - 1; i >= 0; i--) if (pending[i]!.id === id) pending.splice(i, 1);
|
|
672
|
-
});
|
|
754
|
+
if (!deleteTask(this.home, id)) return { ok: false, reason: "not_found" }; // another process deleted it first; only the one that removed it reports it
|
|
673
755
|
if (opts.purge) removeTaskFolder(this.home, id);
|
|
674
756
|
emit("deleted", `"${rec.name}" (${id}) was deleted.`, { id, name: rec.name, forced: !!run, purged: !!opts.purge });
|
|
675
757
|
return { ok: true };
|
|
@@ -686,8 +768,6 @@ const schedulers = new Map<string, Scheduler>();
|
|
|
686
768
|
export function getScheduler(home: string): Scheduler {
|
|
687
769
|
let s = schedulers.get(home);
|
|
688
770
|
if (!s) {
|
|
689
|
-
lockFolder(home); // every scheduler function goes through here: one process per home
|
|
690
|
-
migrateLegacyFolder(home);
|
|
691
771
|
schedulers.set(home, (s = new Scheduler(home)));
|
|
692
772
|
}
|
|
693
773
|
return s;
|
|
@@ -699,6 +779,7 @@ export function resetSchedulers(): void {
|
|
|
699
779
|
schedulers.clear();
|
|
700
780
|
}
|
|
701
781
|
|
|
782
|
+
// The scheduler of a resolved home (option, setHome, env or default); in: the public functions, out: a Scheduler.
|
|
702
783
|
const schedulerFor = (opts?: HomeOptions): Scheduler => getScheduler(resolveHome(opts?.home));
|
|
703
784
|
|
|
704
785
|
/** Makes `agent` the live runner for actions created under its name, and starts the clock if work is pending. */
|
|
@@ -728,11 +809,7 @@ export function createScheduledAction(agent: Agent, input: ScheduledActionInput)
|
|
|
728
809
|
const sa = input.scheduled_action;
|
|
729
810
|
const lead = sa.startCookingInAdvance ?? DEFAULT_COOK_LEAD;
|
|
730
811
|
const deliverAt = resolveDelivery(now, sa.timeToDeliver, sa.meridiem);
|
|
731
|
-
const record =
|
|
732
|
-
const taken = new Set(tasks.map((t) => t.id));
|
|
733
|
-
let id: string;
|
|
734
|
-
do id = `${sa.id}-${randomSuffix()}`;
|
|
735
|
-
while (taken.has(id));
|
|
812
|
+
const record = createTask(s.home, () => `${sa.id}-${randomSuffix()}`, (id) => {
|
|
736
813
|
const rec: ScheduledActionRecord = {
|
|
737
814
|
id,
|
|
738
815
|
name: sa.name.trim(),
|
|
@@ -755,7 +832,6 @@ export function createScheduledAction(agent: Agent, input: ScheduledActionInput)
|
|
|
755
832
|
nextDeliverAt: toLocalIso(deliverAt),
|
|
756
833
|
createdAt: toLocalIso(now),
|
|
757
834
|
};
|
|
758
|
-
tasks.push(rec);
|
|
759
835
|
return rec;
|
|
760
836
|
});
|
|
761
837
|
s.agents.set(agent.name, new WeakRef(agent));
|
|
@@ -786,11 +862,10 @@ export function updateScheduledAction(id: string, patch: ScheduledActionPatch, o
|
|
|
786
862
|
const now = s.now();
|
|
787
863
|
const sa = patch.scheduled_action ?? {};
|
|
788
864
|
const { name, ...scheduleChange } = sa;
|
|
789
|
-
const rec =
|
|
790
|
-
const t = tasks.find((x) => x.id === id);
|
|
865
|
+
const rec = updateTask(s.home, id, (t) => {
|
|
791
866
|
if (!t) throw notFound(id);
|
|
792
867
|
const stored = validateRecord(t);
|
|
793
|
-
if (stored) throw new ValidationError(`Scheduled action "${id}" is invalid in
|
|
868
|
+
if (stored) throw new ValidationError(`Scheduled action "${id}" is invalid in its task.json: ${stored}`);
|
|
794
869
|
const scheduleChanged = Object.values(scheduleChange).some((v) => v !== undefined);
|
|
795
870
|
if (scheduleChanged && t.state === "cooking") {
|
|
796
871
|
throw new ValidationError(`Scheduled action "${id}" is cooking right now; schedule changes are refused while it runs. Try again after it finishes.`);
|