agent-accelerator 0.5.7 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/package.json +1 -1
- package/src/agent/agent.ts +76 -39
- package/src/agent/context.ts +3 -3
- package/src/agent/delegation.ts +12 -12
- package/src/agent/loop.ts +41 -41
- package/src/agent/subagent.ts +4 -4
- package/src/architecture.md +1 -1
- package/src/index.ts +7 -7
- package/src/models/catalog-cache.ts +7 -7
- package/src/models/catalog.ts +13 -13
- package/src/providers/google.ts +4 -4
- package/src/providers/openai-compat.ts +7 -7
- package/src/providers/openai.ts +5 -5
- package/src/providers/openrouter.ts +2 -2
- package/src/providers/shared.ts +4 -4
- package/src/scheduler/architecture.md +5 -2
- package/src/scheduler/scheduler.ts +111 -57
- package/src/scheduler/store.ts +273 -59
- package/src/session/architecture.md +1 -1
- package/src/session/store.ts +22 -22
- package/src/streaming/event-stream.ts +11 -11
- package/src/telemetry-file.ts +74 -24
- package/src/telemetry.ts +2 -2
- package/src/tools/executor.ts +1 -1
- package/src/tools/tool.ts +8 -8
- package/src/types/agent.ts +22 -20
- package/src/types/response.ts +8 -8
- package/src/types/scheduler.ts +2 -2
- package/src/types/tool.ts +5 -5
- package/src/update-models.ts +2 -2
- package/src/utils/architecture.md +1 -1
- package/src/utils/cache.ts +4 -4
- package/src/utils/durable.ts +78 -3
- package/src/utils/env.ts +1 -1
- package/src/utils/headers.ts +4 -4
- package/src/utils/home.ts +1 -1
- package/src/utils/retry.ts +1 -1
- package/src/utils/session.ts +5 -5
|
@@ -406,7 +406,7 @@ function resultToOutput(result: unknown): string {
|
|
|
406
406
|
* resent — except echoed Gemini thought signatures, which strict
|
|
407
407
|
* OpenAI-compatible endpoints require on tool calls they generated.
|
|
408
408
|
*
|
|
409
|
-
*
|
|
409
|
+
* Signatures are namespaced — only echo when the stored provider is
|
|
410
410
|
* `google` (or untagged legacy). Signatures minted by other providers never
|
|
411
411
|
* leak onto this transport.
|
|
412
412
|
*/
|
|
@@ -468,7 +468,7 @@ async function fullHistoryMessages(context: ProviderContext): Promise<ChatMessag
|
|
|
468
468
|
// Echo provider-issued thought signatures verbatim so endpoints
|
|
469
469
|
// that minted them (Gemini behind a compat proxy) keep working.
|
|
470
470
|
// Only present when a previous turn captured one — other endpoints
|
|
471
|
-
// never see this field.
|
|
471
|
+
// never see this field. Echo only google/untagged signatures.
|
|
472
472
|
const storedProvider = (part as { thoughtSignatureProvider?: unknown }).thoughtSignatureProvider;
|
|
473
473
|
if (part.thoughtSignature && shouldEchoSignature(storedProvider, "google")) {
|
|
474
474
|
call.extra_content = { google: { thought_signature: part.thoughtSignature } };
|
|
@@ -540,7 +540,7 @@ function extractSignature(rawTc: {
|
|
|
540
540
|
return typeof sig === "string" && sig ? sig : undefined;
|
|
541
541
|
}
|
|
542
542
|
|
|
543
|
-
/** Tags a captured compat signature with its issuing provider (`google
|
|
543
|
+
/** Tags a captured compat signature with its issuing provider (`google`). */
|
|
544
544
|
function withCompatSignature(sig: string | undefined): { thoughtSignature?: string; thoughtSignatureProvider?: string } {
|
|
545
545
|
if (!sig) return {};
|
|
546
546
|
return { thoughtSignature: sig, thoughtSignatureProvider: "google" };
|
|
@@ -575,7 +575,7 @@ function parseResponse(
|
|
|
575
575
|
|
|
576
576
|
const choice = response.choices?.[0];
|
|
577
577
|
const message = choice?.message;
|
|
578
|
-
//
|
|
578
|
+
// Refusal content surfaces as text with a `refusal` finish reason.
|
|
579
579
|
const refusal = typeof message?.refusal === "string" && message.refusal ? message.refusal : "";
|
|
580
580
|
const text = typeof message?.content === "string" && message.content ? message.content : refusal;
|
|
581
581
|
// Thinking tolerance: plain `reasoning` plus per-block texts (deepseek-style
|
|
@@ -789,7 +789,7 @@ export class OpenAICompatibleChatProvider implements Provider {
|
|
|
789
789
|
// why: invalid output shapes fail fast at Agent construction/loop validation.
|
|
790
790
|
}
|
|
791
791
|
}
|
|
792
|
-
//
|
|
792
|
+
// Request usage on streams where supported so terminal accounting
|
|
793
793
|
// arrives on the final chunk. Strict servers that reject unknown fields
|
|
794
794
|
// opt out via `COMPAT_NO_STREAM_OPTIONS=1`; an explicit per-call
|
|
795
795
|
// `(options as any).streamOptions === false` also skips it.
|
|
@@ -1068,7 +1068,7 @@ export class OpenAICompatibleChatProvider implements Provider {
|
|
|
1068
1068
|
const choice = (Array.isArray(msg["choices"]) ? (msg["choices"] as Record<string, unknown>[])[0] : undefined) ?? {};
|
|
1069
1069
|
const delta = (choice["delta"] as Record<string, unknown>) ?? {};
|
|
1070
1070
|
// Text delta (content-free accounting frames carry "" — ignored).
|
|
1071
|
-
//
|
|
1071
|
+
// Refusal deltas surface as text with a `refusal` finish.
|
|
1072
1072
|
const deltaRefusal = delta["refusal"];
|
|
1073
1073
|
if (typeof deltaRefusal === "string" && deltaRefusal) {
|
|
1074
1074
|
text += deltaRefusal;
|
|
@@ -1187,7 +1187,7 @@ export class OpenAICompatibleChatProvider implements Provider {
|
|
|
1187
1187
|
sawFinishReason = true;
|
|
1188
1188
|
}
|
|
1189
1189
|
|
|
1190
|
-
//
|
|
1190
|
+
// Terminal-state guard — fail `incomplete_stream` (retryable)
|
|
1191
1191
|
// instead of resolving empty success. Skipped for non-SSE payloads
|
|
1192
1192
|
// (JSON mocks) where no SSE frames were ever seen.
|
|
1193
1193
|
const cleanThinking = thinking.replace(/\n{3,}/g, "\n\n").trim();
|
package/src/providers/openai.ts
CHANGED
|
@@ -472,7 +472,7 @@ function parseResponse(
|
|
|
472
472
|
}
|
|
473
473
|
}
|
|
474
474
|
|
|
475
|
-
//
|
|
475
|
+
// `incomplete` -> length (content-filter truncations surface as
|
|
476
476
|
// refusal when refusal text is present).
|
|
477
477
|
const incompleteReason = (response as { incomplete_details?: { reason?: string } }).incomplete_details?.reason;
|
|
478
478
|
const finishReason =
|
|
@@ -828,7 +828,7 @@ export class OpenAIResponsesProvider implements Provider {
|
|
|
828
828
|
body,
|
|
829
829
|
};
|
|
830
830
|
|
|
831
|
-
//
|
|
831
|
+
// Optional per-call timeout races the linked abort signal.
|
|
832
832
|
const streamTimeoutMs = timeoutFor(options as { timeoutMs?: number } | undefined);
|
|
833
833
|
// Transient failures (429/5xx/network) are retried here, before the
|
|
834
834
|
// first byte; once the stream is live it is never retried.
|
|
@@ -895,7 +895,7 @@ export class OpenAIResponsesProvider implements Provider {
|
|
|
895
895
|
type === "response.refusal.delta" ||
|
|
896
896
|
type === "response.output_refusal.delta"
|
|
897
897
|
) {
|
|
898
|
-
//
|
|
898
|
+
// Refusal deltas carry the filter text; surface as text and
|
|
899
899
|
// remember for a `refusal` finish reason.
|
|
900
900
|
if (typeof msg["delta"] === "string" && msg["delta"]) {
|
|
901
901
|
text += msg["delta"] as string;
|
|
@@ -955,7 +955,7 @@ export class OpenAIResponsesProvider implements Provider {
|
|
|
955
955
|
}
|
|
956
956
|
// Non-streaming maps `incomplete` → `length` (max tokens). Streaming
|
|
957
957
|
// must do the same — otherwise truncated turns misreport `stop`.
|
|
958
|
-
//
|
|
958
|
+
// Content-filter incompletes surface as `refusal`.
|
|
959
959
|
const incompleteReason = (response as { incomplete_details?: { reason?: string } }).incomplete_details?.reason;
|
|
960
960
|
if (response.status === "incomplete") {
|
|
961
961
|
finishReason = incompleteReason === "content_filter" ? "refusal" : "length";
|
|
@@ -1039,7 +1039,7 @@ export class OpenAIResponsesProvider implements Provider {
|
|
|
1039
1039
|
sawFinishReason = true;
|
|
1040
1040
|
}
|
|
1041
1041
|
|
|
1042
|
-
//
|
|
1042
|
+
// Terminal-state guard. Loops ending with no completed body, no
|
|
1043
1043
|
// finish_reason, and no usage while carrying no content must fail as
|
|
1044
1044
|
// `incomplete_stream` (retryable) instead of resolving empty success.
|
|
1045
1045
|
// Skipped for non-SSE payloads (e.g. JSON mocks in delegation tests)
|
|
@@ -741,7 +741,7 @@ export class OpenRouterChatCompletionsProvider implements Provider {
|
|
|
741
741
|
const choice = (Array.isArray(msg["choices"]) ? (msg["choices"] as Record<string, unknown>[])[0] : undefined) ?? {};
|
|
742
742
|
const delta = (choice["delta"] as Record<string, unknown>) ?? {};
|
|
743
743
|
// Text delta (content-free accounting frames carry "" — ignored).
|
|
744
|
-
//
|
|
744
|
+
// Refusal deltas surface as text with a `refusal` finish.
|
|
745
745
|
const deltaRefusal = delta["refusal"];
|
|
746
746
|
if (typeof deltaRefusal === "string" && deltaRefusal) {
|
|
747
747
|
text += deltaRefusal;
|
|
@@ -854,7 +854,7 @@ export class OpenRouterChatCompletionsProvider implements Provider {
|
|
|
854
854
|
sawFinishReason = true;
|
|
855
855
|
}
|
|
856
856
|
|
|
857
|
-
//
|
|
857
|
+
// Terminal-state guard — fail `incomplete_stream` (retryable)
|
|
858
858
|
// instead of resolving empty success. Skipped for non-SSE payloads
|
|
859
859
|
// (JSON mocks) where no SSE frames were ever seen.
|
|
860
860
|
const cleanThinking = thinking.replace(/\n{3,}/g, "\n\n").trim();
|
package/src/providers/shared.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Shared provider helpers
|
|
2
|
+
* Shared provider helpers.
|
|
3
3
|
*
|
|
4
4
|
* Single home for the `redactedHeaders` / `readErrorPayload` /
|
|
5
5
|
* `parseArguments` / `normalizeThinkingParts` helpers previously duplicated
|
|
@@ -78,7 +78,7 @@ export function readErrorPayload(
|
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
/** Retryable incomplete-stream error for truncated SSE
|
|
81
|
+
/** Retryable incomplete-stream error for truncated SSE. */
|
|
82
82
|
export function incompleteStreamError(provider: string, modelId: string): Error {
|
|
83
83
|
const err = toConciseProviderError(
|
|
84
84
|
Object.assign(new Error("Stream ended without terminal event"), {
|
|
@@ -118,7 +118,7 @@ export function isNotFoundChainError(error: unknown): boolean {
|
|
|
118
118
|
return code === "not_found";
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
/** Per-request timeout ms from provider options
|
|
121
|
+
/** Per-request timeout ms from provider options. */
|
|
122
122
|
export function timeoutFor(options: { timeoutMs?: number } | undefined): number | undefined {
|
|
123
123
|
const t = (options as { timeoutMs?: unknown } | undefined)?.timeoutMs;
|
|
124
124
|
return typeof t === "number" && Number.isFinite(t) && t > 0 ? Math.floor(t) : undefined;
|
|
@@ -149,7 +149,7 @@ export function combinedSignal(signal?: AbortSignal, timeoutMs?: number): { sign
|
|
|
149
149
|
return { signal: ctrl.signal, cleanup };
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
-
/**
|
|
152
|
+
/** Crypto-backed tool-call id, no Math.random primary. */
|
|
153
153
|
export function newToolCallId(prefix = "call"): string {
|
|
154
154
|
try {
|
|
155
155
|
const c = (globalThis as any)?.crypto;
|
|
@@ -5,15 +5,18 @@ Scheduled actions: a worker that runs later (once or repeating) and saves its an
|
|
|
5
5
|
| File | What it is |
|
|
6
6
|
| :-- | :-- |
|
|
7
7
|
| `scheduler.ts` | One `Scheduler` per home folder: a 1 s unref'd clock that runs only while something is pending, startup sweep, cook/deliver state machine, and the public functions (`createScheduledAction`, `list/update/run/skip/deleteScheduledAction`). |
|
|
8
|
-
| `store.ts` | Disk layer under `<home>/
|
|
8
|
+
| `store.ts` | Disk layer under `<home>/scheduled/`: `<id>/task.json`, `<id>/delivery.json`, run files, one short mutex per action; validation of records. |
|
|
9
9
|
| `time.ts` | Pure `DD:HH:MM` and local-clock maths. No timezone, DST or `Intl`; `now` is always passed in. |
|
|
10
10
|
| `tool.ts` + `scheduled-actions.md` | The model-facing `scheduled_actions` tool and `get_datetime`; the `.md` is the tool description. |
|
|
11
11
|
|
|
12
12
|
## Invariants
|
|
13
13
|
- The files are the truth: every write re-reads first and goes through the durable writer. The file format is a stable public contract: see `docs/scheduled-action-files.md` and add a changelog row for any change.
|
|
14
|
-
- Write order when a run ends: run file, then
|
|
14
|
+
- Write order when a run ends: run file, then `<id>/delivery.json`, then `<id>/task.json`. The startup sweep repairs a crash in between.
|
|
15
15
|
- `startCook` advances the next times BEFORE the run starts; a run still cooking when the next is due is skipped, never stacked.
|
|
16
16
|
- Runs are fresh stateless workers built by `buildDynamicWorkerConfig`; their spending guardrail is `workerBudget` only.
|
|
17
17
|
- The Agent registry holds `WeakRef`s by name: keep every Agent that owns actions referenced, or its runs become `missed`.
|
|
18
|
+
- Several processes may share a home (the file store is safe for it; the home lock is lifted in a later change of this release). Ownership: a process only starts, delivers or recovers actions whose Agent it registered. An action nobody here owns is left alone until its delivery time passed with no run started, then marked `missed` (`no_agent`).
|
|
19
|
+
- Exactly once: a scheduled start re-checks under the action's mutex that it is still `waiting` and due; `markMissed` only changes the state the caller saw; a delivery is emitted only by the process that removed its entry from `delivery.json`.
|
|
20
|
+
- A run in flight keeps `<id>/.heartbeat` (pid, host, token; mtime refreshed every 5 s, stale after 60 s), written before the record shows `cooking`. A cooking action whose owner is dead (pid gone, stale, or none) is recovered by a live process that owns its Agent, at start and every 15 s. `delete` of a run another live process is cooking is refused, even with `force`.
|
|
18
21
|
- Times come from `Date`, local clock only. Tests inject `scheduler.now` / `tickMs` and call `tick()`.
|
|
19
22
|
- Events: `scheduled_action.{created,updated,started,ready,failed,delivered,missed,skipped,deleted,invalid,store_corrupt,error}`.
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* One clock per home folder, ticking once a second against the machine's own
|
|
6
6
|
* time. Each tick only asks "what is overdue?", so drift, sleep or a clock
|
|
7
|
-
* change cannot skip a run. The files under `<home>/
|
|
7
|
+
* change cannot skip a run. The files under `<home>/scheduled/` are the
|
|
8
8
|
* source of truth (see `store.ts`); nothing waits in memory, and the timer is
|
|
9
9
|
* unref'd and runs only while something is pending.
|
|
10
10
|
*
|
|
@@ -12,12 +12,13 @@
|
|
|
12
12
|
* | |
|
|
13
13
|
* +--> failed (recorded, moves on) +--> waiting (repeat) | done
|
|
14
14
|
*/
|
|
15
|
+
import * as os from "node:os";
|
|
15
16
|
import type { Agent } from "../agent/agent.ts";
|
|
16
17
|
import { createChildSessionId } from "../agent/delegation.ts";
|
|
17
18
|
import { buildDynamicWorkerConfig, grantWorkerTools, scheduledRunTimeoutMs } from "../agent/worker.ts";
|
|
18
19
|
import { telemetryBus } from "../telemetry.ts";
|
|
19
20
|
import { readUsage, usageLine, type UsageTotals } from "../session/usage.ts";
|
|
20
|
-
import {
|
|
21
|
+
import { processAlive } from "../utils/durable.ts";
|
|
21
22
|
import { ConfigError, ValidationError } from "../types/errors.ts";
|
|
22
23
|
import type { ToolDefinition } from "../types/tool.ts";
|
|
23
24
|
import type { AgentResponse } from "../types/response.ts";
|
|
@@ -31,18 +32,24 @@ import type {
|
|
|
31
32
|
} from "../types/scheduler.ts";
|
|
32
33
|
import {
|
|
33
34
|
TASK_ID_PATTERN,
|
|
35
|
+
clearHeartbeat,
|
|
36
|
+
createTask,
|
|
37
|
+
deleteTask,
|
|
34
38
|
findRunFile,
|
|
35
|
-
migrateLegacyFolder,
|
|
36
39
|
readDeliveries,
|
|
40
|
+
readHeartbeat,
|
|
37
41
|
readRun,
|
|
42
|
+
readTask,
|
|
38
43
|
readTasks,
|
|
39
44
|
removeTaskFolder,
|
|
40
45
|
resolveHome,
|
|
41
46
|
runPath,
|
|
47
|
+
touchHeartbeat,
|
|
42
48
|
updateDeliveries,
|
|
43
|
-
|
|
49
|
+
updateTask,
|
|
44
50
|
validateDelivery,
|
|
45
51
|
validateRecord,
|
|
52
|
+
writeHeartbeat,
|
|
46
53
|
writeRunFile,
|
|
47
54
|
type PendingDelivery,
|
|
48
55
|
type RunHeader,
|
|
@@ -61,6 +68,11 @@ import {
|
|
|
61
68
|
/** Default lead time (10 minutes) when the developer API omits `startCookingInAdvance`. */
|
|
62
69
|
const DEFAULT_COOK_LEAD = "00:00:10";
|
|
63
70
|
const SLUG_MAX = 48;
|
|
71
|
+
/** A heartbeat older than this means its owner is gone (it is refreshed every 5 s). */
|
|
72
|
+
const HEARTBEAT_STALE_MS = 60_000;
|
|
73
|
+
const HEARTBEAT_EVERY_MS = 5_000;
|
|
74
|
+
/** How often a live process looks for runs whose owner process died. */
|
|
75
|
+
const DEAD_SWEEP_EVERY_MS = 15_000;
|
|
64
76
|
const ID_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789";
|
|
65
77
|
|
|
66
78
|
/** Optional `{ home }` accepted by every public function except `createScheduledAction` (the agent knows its home). */
|
|
@@ -240,6 +252,10 @@ export class Scheduler {
|
|
|
240
252
|
tickMs = 1000;
|
|
241
253
|
private timer: ReturnType<typeof setInterval> | undefined;
|
|
242
254
|
private swept = false;
|
|
255
|
+
/** Names this scheduler instance in the heartbeat files of the runs it owns. */
|
|
256
|
+
private readonly token = Math.random().toString(36).slice(2, 10);
|
|
257
|
+
private lastDeadSweep = 0;
|
|
258
|
+
private lastBeat = 0;
|
|
243
259
|
private readonly warned = new Set<string>();
|
|
244
260
|
|
|
245
261
|
constructor(readonly home: string) {}
|
|
@@ -268,6 +284,8 @@ export class Scheduler {
|
|
|
268
284
|
this.swept = true;
|
|
269
285
|
this.startupSweep(now);
|
|
270
286
|
}
|
|
287
|
+
this.keepHeartbeats(now);
|
|
288
|
+
this.sweepDeadRuns(now);
|
|
271
289
|
this.deliverDue(now);
|
|
272
290
|
this.startDue(now);
|
|
273
291
|
if (!this.hasPending()) this.stop();
|
|
@@ -277,7 +295,7 @@ export class Scheduler {
|
|
|
277
295
|
}
|
|
278
296
|
|
|
279
297
|
// Reports an invalid record or delivery once per file/id/problem and leaves the file untouched; in: snapshot/startCook, out: a scheduled_action.invalid warning.
|
|
280
|
-
private warnInvalid(id: string, problem: string, file = "
|
|
298
|
+
private warnInvalid(id: string, problem: string, file = "task.json"): void {
|
|
281
299
|
const key = `${file}:${id}:${problem}`;
|
|
282
300
|
if (this.warned.has(key)) return;
|
|
283
301
|
this.warned.add(key);
|
|
@@ -297,7 +315,7 @@ export class Scheduler {
|
|
|
297
315
|
});
|
|
298
316
|
const pending = readDeliveries(this.home).filter((p) => {
|
|
299
317
|
const problem = validateDelivery(p);
|
|
300
|
-
if (problem) this.warnInvalid(String((p as { id?: unknown }).id), problem, "
|
|
318
|
+
if (problem) this.warnInvalid(String((p as { id?: unknown }).id), problem, "delivery.json");
|
|
301
319
|
return !problem;
|
|
302
320
|
});
|
|
303
321
|
return { live, pending };
|
|
@@ -318,23 +336,54 @@ export class Scheduler {
|
|
|
318
336
|
*/
|
|
319
337
|
private startupSweep(now: Date): void {
|
|
320
338
|
for (const t of this.snapshot().live) {
|
|
321
|
-
if (t.
|
|
322
|
-
|
|
323
|
-
this.markMissed(t.id, "process_down", t.schedule.onMissed);
|
|
339
|
+
if (!this.ownsHere(t.agent)) continue; // another process's action (or nobody's: startDue handles that)
|
|
340
|
+
if (t.state === "waiting" && t.nextDeliverAt && parseLocalIso(t.nextDeliverAt).getTime() <= now.getTime()) {
|
|
341
|
+
this.markMissed(t.id, "process_down", "waiting", t.schedule.onMissed);
|
|
324
342
|
}
|
|
325
343
|
}
|
|
326
344
|
}
|
|
327
345
|
|
|
346
|
+
// True when a live Agent with this name is registered in THIS process: only then may this process start, deliver or recover its actions; in: the sweeps, startDue and deliverDue, out: a boolean.
|
|
347
|
+
private ownsHere(agentName: string): boolean {
|
|
348
|
+
return !!this.agents.get(agentName)?.deref();
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// True while the process that started a run is still running it: its heartbeat is fresh and (same host) its pid exists; this process counts only for runs in this.running; in: the dead-run sweep and delete, out: the heartbeat or undefined.
|
|
352
|
+
private liveOwner(id: string): { pid: number } | undefined {
|
|
353
|
+
if (this.running.has(id)) return { pid: process.pid };
|
|
354
|
+
const hb = readHeartbeat(this.home, id);
|
|
355
|
+
if (!hb || hb.ageMs > HEARTBEAT_STALE_MS) return undefined;
|
|
356
|
+
if (hb.pid === process.pid) return undefined; // an earlier scheduler of this very process is gone
|
|
357
|
+
if (hb.host === os.hostname() && !processAlive(hb.pid)) return undefined;
|
|
358
|
+
return { pid: hb.pid };
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Refreshes the heartbeat of every run in flight (at most every 5 s); in: tick, out: the .heartbeat files.
|
|
362
|
+
private keepHeartbeats(now: Date): void {
|
|
363
|
+
if (this.running.size === 0 || Math.abs(now.getTime() - this.lastBeat) < HEARTBEAT_EVERY_MS) return;
|
|
364
|
+
this.lastBeat = now.getTime();
|
|
365
|
+
for (const id of this.running.keys()) touchHeartbeat(this.home, id);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
// Recovers the runs whose owner is gone (a killed process), but only for actions this process owns, at start and then every 15 s; in: tick, out: recoverCooking calls.
|
|
369
|
+
private sweepDeadRuns(now: Date): void {
|
|
370
|
+
if (this.lastDeadSweep !== 0 && Math.abs(now.getTime() - this.lastDeadSweep) < DEAD_SWEEP_EVERY_MS) return;
|
|
371
|
+
this.lastDeadSweep = now.getTime();
|
|
372
|
+
for (const t of this.snapshot().live) {
|
|
373
|
+
if (t.state === "cooking" && this.ownsHere(t.agent) && !this.liveOwner(t.id)) this.recoverCooking(t);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
328
377
|
/** The process died mid-cook: rebuild the delivery if the run file was already written, else it is missed. */
|
|
329
378
|
private recoverCooking(t: ScheduledActionRecord): void {
|
|
330
379
|
const current = t.current!;
|
|
331
380
|
const file = findRunFile(this.home, t.id, parseLocalIso(current.deliverAt), current.cookStartedAt);
|
|
332
381
|
const run = file ? readRun(this.home, file) : undefined;
|
|
333
382
|
if (!file || !run) {
|
|
334
|
-
this.markMissed(t.id, "interrupted", t.schedule.onMissed);
|
|
383
|
+
this.markMissed(t.id, "interrupted", "cooking", t.schedule.onMissed);
|
|
335
384
|
return;
|
|
336
385
|
}
|
|
337
|
-
updateDeliveries(this.home, (pending) => {
|
|
386
|
+
updateDeliveries(this.home, t.id, (pending) => {
|
|
338
387
|
if (pending.some((p) => p.file === file)) return;
|
|
339
388
|
pending.push({
|
|
340
389
|
id: t.id,
|
|
@@ -345,24 +394,24 @@ export class Scheduler {
|
|
|
345
394
|
...(run.header["late"] === "true" ? { late: true } : {}),
|
|
346
395
|
});
|
|
347
396
|
});
|
|
348
|
-
|
|
349
|
-
const rec = tasks.find((x) => x.id === t.id);
|
|
397
|
+
updateTask(this.home, t.id, (rec) => {
|
|
350
398
|
if (!rec || rec.state !== "cooking") return;
|
|
351
399
|
delete rec.current;
|
|
352
400
|
rec.state = rec.nextDeliverAt ? "waiting" : "done";
|
|
353
401
|
});
|
|
354
402
|
}
|
|
355
403
|
|
|
356
|
-
// Marks a run whose moment passed as missed, then applies the action's onMissed policy: `skip` moves on to the next time, `run` runs it now, unset waits for run_now / skip; in
|
|
357
|
-
private markMissed(id: string, reason: string, policy?: "skip" | "run"): void {
|
|
404
|
+
// Marks a run whose moment passed as missed, then applies the action's onMissed policy: `skip` moves on to the next time, `run` runs it now, unset waits for run_now / skip; only when the action is still in the state the caller saw (`from`), so two processes never both mark it; in: the sweeps, out: task.json + the missed (and skipped / started) events.
|
|
405
|
+
private markMissed(id: string, reason: string, from: "waiting" | "cooking", policy?: "skip" | "run"): void {
|
|
358
406
|
let name = id;
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
if (!t) return;
|
|
407
|
+
const changed = updateTask(this.home, id, (t) => {
|
|
408
|
+
if (!t || t.state !== from) return false; // another process already dealt with it
|
|
362
409
|
name = t.name;
|
|
363
410
|
t.state = "missed";
|
|
364
411
|
delete t.current;
|
|
412
|
+
return true;
|
|
365
413
|
});
|
|
414
|
+
if (!changed) return;
|
|
366
415
|
const next = policy === "skip" ? "Skipping it (onMissed: skip)." : policy === "run" ? "Running it now (onMissed: run)." : "Use run_now or skip.";
|
|
367
416
|
emit("missed", `"${name}" (${id}) missed its run (${reason}). ${next}`, { id, name, reason, ...(policy ? { onMissed: policy } : {}) }, "warn");
|
|
368
417
|
if (policy === "skip") this.skip(id);
|
|
@@ -374,21 +423,25 @@ export class Scheduler {
|
|
|
374
423
|
// Delivers every saved run whose delivery time has come; in: tick, out: deliver calls.
|
|
375
424
|
private deliverDue(now: Date): void {
|
|
376
425
|
for (const entry of this.snapshot().pending) {
|
|
377
|
-
if (parseLocalIso(entry.deliverAt).getTime()
|
|
426
|
+
if (parseLocalIso(entry.deliverAt).getTime() > now.getTime()) continue;
|
|
427
|
+
const owner = readTask(this.home, entry.id)?.agent; // the delivered event must fire in the process that owns the action
|
|
428
|
+
if (owner === undefined || this.ownsHere(owner)) this.deliver(entry, now);
|
|
378
429
|
}
|
|
379
430
|
}
|
|
380
431
|
|
|
381
|
-
// Removes the entry, records `lastRun`, THEN emits (whether or not anyone listens); late = it replaced a missed occurrence, was delivered over two ticks after its time, or finished after it; in: deliverDue, out:
|
|
432
|
+
// Removes the entry (only the process that removed it goes on), records `lastRun`, THEN emits (whether or not anyone listens); late = it replaced a missed occurrence, was delivered over two ticks after its time, or finished after it; in: deliverDue, out: task.json + the delivered event.
|
|
382
433
|
private deliver(entry: PendingDelivery, now: Date): void {
|
|
383
|
-
updateDeliveries(this.home, (pending) => {
|
|
434
|
+
const removed = updateDeliveries(this.home, entry.id, (pending) => {
|
|
384
435
|
const i = pending.findIndex((p) => p.id === entry.id && p.file === entry.file);
|
|
385
|
-
if (i
|
|
436
|
+
if (i < 0) return false;
|
|
437
|
+
pending.splice(i, 1);
|
|
438
|
+
return true;
|
|
386
439
|
});
|
|
440
|
+
if (!removed) return; // another process took this delivery: exactly one process delivers it
|
|
387
441
|
const deliverAt = parseLocalIso(entry.deliverAt).getTime();
|
|
388
442
|
const late = entry.late === true || now.getTime() - deliverAt > 2 * this.tickMs || parseLocalIso(entry.readyAt).getTime() > deliverAt;
|
|
389
443
|
let name = entry.id;
|
|
390
|
-
|
|
391
|
-
const t = tasks.find((x) => x.id === entry.id);
|
|
444
|
+
updateTask(this.home, entry.id, (t) => {
|
|
392
445
|
if (!t) return;
|
|
393
446
|
name = t.name;
|
|
394
447
|
t.lastRun = { status: entry.status, file: entry.file, deliverAt: entry.deliverAt, deliveredAt: toLocalIso(now), late };
|
|
@@ -410,6 +463,13 @@ export class Scheduler {
|
|
|
410
463
|
// Starts waiting actions whose cook time has come and skips the overlap of those still cooking; in: tick, out: startCook/skipOverlap calls.
|
|
411
464
|
private startDue(now: Date): void {
|
|
412
465
|
for (const t of this.snapshot().live) {
|
|
466
|
+
if (!this.ownsHere(t.agent)) {
|
|
467
|
+
// Nobody here can run it. Another process may own it, so only once its delivery time has passed with no run started is it missed.
|
|
468
|
+
if (t.state === "waiting" && t.nextDeliverAt && parseLocalIso(t.nextDeliverAt).getTime() <= now.getTime()) {
|
|
469
|
+
this.markMissed(t.id, "no_agent", "waiting");
|
|
470
|
+
}
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
413
473
|
if (!t.nextCookAt || parseLocalIso(t.nextCookAt).getTime() > now.getTime()) continue;
|
|
414
474
|
if (t.state === "waiting") this.startCook(t.id, false);
|
|
415
475
|
else if (t.state === "cooking") this.skipOverlap(t.id, now);
|
|
@@ -433,8 +493,7 @@ export class Scheduler {
|
|
|
433
493
|
private startCook(id: string, manual: boolean): ScheduledActionRecord | undefined {
|
|
434
494
|
const now = this.now();
|
|
435
495
|
const ctx: { agent?: Agent; noAgent?: boolean; replacesMissed?: boolean } = {};
|
|
436
|
-
const started =
|
|
437
|
-
const t = tasks.find((x) => x.id === id);
|
|
496
|
+
const started = updateTask(this.home, id, (t) => {
|
|
438
497
|
if (!t) return undefined;
|
|
439
498
|
const problem = validateRecord(t);
|
|
440
499
|
if (problem) {
|
|
@@ -442,6 +501,8 @@ export class Scheduler {
|
|
|
442
501
|
return undefined;
|
|
443
502
|
}
|
|
444
503
|
if (t.state === "cooking") return undefined;
|
|
504
|
+
// Another process may have started (and even finished) this run since the caller looked: a scheduled start needs the action to be waiting and due right now.
|
|
505
|
+
if (!manual && (t.state !== "waiting" || !t.nextCookAt || parseLocalIso(t.nextCookAt).getTime() > now.getTime())) return undefined;
|
|
445
506
|
ctx.replacesMissed = manual && t.state === "missed";
|
|
446
507
|
ctx.agent = this.agents.get(t.agent)?.deref();
|
|
447
508
|
if (!ctx.agent) {
|
|
@@ -465,11 +526,12 @@ export class Scheduler {
|
|
|
465
526
|
t.current = { cookStartedAt: toLocalIso(now), deliverAt: toLocalIso(deliverAt) };
|
|
466
527
|
t.nextCookAt = next.cook;
|
|
467
528
|
t.nextDeliverAt = next.deliver;
|
|
529
|
+
writeHeartbeat(this.home, id, this.token); // before the record shows `cooking`, so no process sees a cooking action without an owner
|
|
468
530
|
return t;
|
|
469
531
|
});
|
|
470
532
|
if (ctx.noAgent) {
|
|
471
533
|
const name = this.nameOf(id);
|
|
472
|
-
emit("missed", `"${name}" (${id}) cannot run: no live Agent named in
|
|
534
|
+
emit("missed", `"${name}" (${id}) cannot run: no live Agent named in its task.json is registered.`, { id, name, reason: "no_agent" }, "warn");
|
|
473
535
|
}
|
|
474
536
|
if (!started || !ctx.agent) return undefined;
|
|
475
537
|
emit("started", `"${started.name}" (${id}) started cooking.`, {
|
|
@@ -482,16 +544,15 @@ export class Scheduler {
|
|
|
482
544
|
return started;
|
|
483
545
|
}
|
|
484
546
|
|
|
485
|
-
// The display name of an action from
|
|
547
|
+
// The display name of an action from task.json, else its id; in: startCook's no-agent message, out: event text.
|
|
486
548
|
private nameOf(id: string): string {
|
|
487
|
-
return
|
|
549
|
+
return readTask(this.home, id)?.name ?? id;
|
|
488
550
|
}
|
|
489
551
|
|
|
490
552
|
/** The next run is due while this one still cooks: never stack. Skip that occurrence. */
|
|
491
553
|
private skipOverlap(id: string, now: Date): void {
|
|
492
554
|
const ctx: { skipped?: { name: string; deliverAt: string } } = {};
|
|
493
|
-
|
|
494
|
-
const t = tasks.find((x) => x.id === id);
|
|
555
|
+
updateTask(this.home, id, (t) => {
|
|
495
556
|
if (!t || t.state !== "cooking" || !t.nextCookAt || !t.nextDeliverAt) return;
|
|
496
557
|
ctx.skipped = { name: t.name, deliverAt: t.nextDeliverAt };
|
|
497
558
|
if (!t.schedule.repeatEvery) {
|
|
@@ -520,6 +581,7 @@ export class Scheduler {
|
|
|
520
581
|
})
|
|
521
582
|
.finally(() => {
|
|
522
583
|
this.running.delete(task.id);
|
|
584
|
+
clearHeartbeat(this.home, task.id);
|
|
523
585
|
});
|
|
524
586
|
this.ensureRunning();
|
|
525
587
|
}
|
|
@@ -593,7 +655,7 @@ export class Scheduler {
|
|
|
593
655
|
}
|
|
594
656
|
}
|
|
595
657
|
|
|
596
|
-
// Writes a finished run: run file, then the `
|
|
658
|
+
// Writes a finished run: run file, then the `delivery.json` entry, then `task.json` (a crash in between is repaired on start); a manual run is due when it finishes and is late only if it replaced a missed occurrence, a scheduled one is late only past its second; in: runWorker, out: the files and the ready/failed event.
|
|
597
659
|
private finish(task: ScheduledActionRecord, outcome: Outcome, startedAt: string, deliverAt: Date, run: Pick<RunningEntry, "manual" | "missed">): void {
|
|
598
660
|
const now = this.now();
|
|
599
661
|
const ok = "res" in outcome;
|
|
@@ -614,11 +676,10 @@ export class Scheduler {
|
|
|
614
676
|
};
|
|
615
677
|
const body = ok ? outcome.res.text.trim() || "[The worker returned no text.]" : `The run failed: ${outcome.error}`;
|
|
616
678
|
const file = writeRunFile(this.home, task.id, deliverAt, header, body);
|
|
617
|
-
updateDeliveries(this.home, (pending) => {
|
|
679
|
+
updateDeliveries(this.home, task.id, (pending) => {
|
|
618
680
|
pending.push({ id: task.id, file, status: header.status, readyAt: finishedAt, deliverAt: header.deliverAt, ...(run.missed ? { late: true } : {}) });
|
|
619
681
|
});
|
|
620
|
-
|
|
621
|
-
const t = tasks.find((x) => x.id === task.id);
|
|
682
|
+
updateTask(this.home, task.id, (t) => {
|
|
622
683
|
if (!t || t.state !== "cooking") return;
|
|
623
684
|
delete t.current;
|
|
624
685
|
t.state = t.nextDeliverAt ? "waiting" : "done";
|
|
@@ -632,7 +693,7 @@ export class Scheduler {
|
|
|
632
693
|
|
|
633
694
|
// Starts an action immediately outside its schedule (also resolves a missed one); in: runScheduledAction, out: the cooking record.
|
|
634
695
|
runNow(id: string): ScheduledActionRecord {
|
|
635
|
-
const rec =
|
|
696
|
+
const rec = readTask(this.home, id);
|
|
636
697
|
if (!rec) throw notFound(id);
|
|
637
698
|
if (rec.state === "cooking" || this.running.has(id)) {
|
|
638
699
|
throw new ValidationError(`Scheduled action "${id}" is already cooking; wait for it to finish.`);
|
|
@@ -650,8 +711,7 @@ export class Scheduler {
|
|
|
650
711
|
// Ignores a missed occurrence and schedules the next one (a one-shot becomes done); in: skipScheduledAction and onMissed: skip, out: the updated record.
|
|
651
712
|
skip(id: string): ScheduledActionRecord {
|
|
652
713
|
const now = this.now();
|
|
653
|
-
const rec =
|
|
654
|
-
const t = tasks.find((x) => x.id === id);
|
|
714
|
+
const rec = updateTask(this.home, id, (t) => {
|
|
655
715
|
if (!t) throw notFound(id);
|
|
656
716
|
if (t.state !== "missed") throw new ValidationError(`Scheduled action "${id}" is ${t.state}, not missed: nothing to skip.`);
|
|
657
717
|
const next = this.afterMissed(t, now);
|
|
@@ -668,9 +728,17 @@ export class Scheduler {
|
|
|
668
728
|
|
|
669
729
|
// Deletes an action (aborting a cooking run only with force) and optionally purges its run files; in: deleteScheduledAction, out: a DeleteResult.
|
|
670
730
|
async delete(id: string, opts: { force?: boolean; purge?: boolean }): Promise<DeleteResult> {
|
|
671
|
-
const rec =
|
|
731
|
+
const rec = readTask(this.home, id);
|
|
672
732
|
if (!rec) return { ok: false, reason: "not_found" };
|
|
673
733
|
const run = this.running.get(id);
|
|
734
|
+
const other = !run && rec.state === "cooking" ? this.liveOwner(id) : undefined;
|
|
735
|
+
if (other) {
|
|
736
|
+
return {
|
|
737
|
+
ok: false,
|
|
738
|
+
reason: "running",
|
|
739
|
+
message: `"${id}" is cooking in another process (pid ${other.pid}), so nothing was changed. Only that process can abort it: wait for it to finish, or delete it from there.`,
|
|
740
|
+
};
|
|
741
|
+
}
|
|
674
742
|
if (run && !opts.force) {
|
|
675
743
|
return {
|
|
676
744
|
ok: false,
|
|
@@ -683,13 +751,7 @@ export class Scheduler {
|
|
|
683
751
|
run.controller.abort(new Error("deleted"));
|
|
684
752
|
await run.done;
|
|
685
753
|
}
|
|
686
|
-
|
|
687
|
-
const i = tasks.findIndex((t) => t.id === id);
|
|
688
|
-
if (i >= 0) tasks.splice(i, 1);
|
|
689
|
-
});
|
|
690
|
-
updateDeliveries(this.home, (pending) => {
|
|
691
|
-
for (let i = pending.length - 1; i >= 0; i--) if (pending[i]!.id === id) pending.splice(i, 1);
|
|
692
|
-
});
|
|
754
|
+
if (!deleteTask(this.home, id)) return { ok: false, reason: "not_found" }; // another process deleted it first; only the one that removed it reports it
|
|
693
755
|
if (opts.purge) removeTaskFolder(this.home, id);
|
|
694
756
|
emit("deleted", `"${rec.name}" (${id}) was deleted.`, { id, name: rec.name, forced: !!run, purged: !!opts.purge });
|
|
695
757
|
return { ok: true };
|
|
@@ -706,8 +768,6 @@ const schedulers = new Map<string, Scheduler>();
|
|
|
706
768
|
export function getScheduler(home: string): Scheduler {
|
|
707
769
|
let s = schedulers.get(home);
|
|
708
770
|
if (!s) {
|
|
709
|
-
lockFolder(home); // every scheduler function goes through here: one process per home
|
|
710
|
-
migrateLegacyFolder(home);
|
|
711
771
|
schedulers.set(home, (s = new Scheduler(home)));
|
|
712
772
|
}
|
|
713
773
|
return s;
|
|
@@ -749,11 +809,7 @@ export function createScheduledAction(agent: Agent, input: ScheduledActionInput)
|
|
|
749
809
|
const sa = input.scheduled_action;
|
|
750
810
|
const lead = sa.startCookingInAdvance ?? DEFAULT_COOK_LEAD;
|
|
751
811
|
const deliverAt = resolveDelivery(now, sa.timeToDeliver, sa.meridiem);
|
|
752
|
-
const record =
|
|
753
|
-
const taken = new Set(tasks.map((t) => t.id));
|
|
754
|
-
let id: string;
|
|
755
|
-
do id = `${sa.id}-${randomSuffix()}`;
|
|
756
|
-
while (taken.has(id));
|
|
812
|
+
const record = createTask(s.home, () => `${sa.id}-${randomSuffix()}`, (id) => {
|
|
757
813
|
const rec: ScheduledActionRecord = {
|
|
758
814
|
id,
|
|
759
815
|
name: sa.name.trim(),
|
|
@@ -776,7 +832,6 @@ export function createScheduledAction(agent: Agent, input: ScheduledActionInput)
|
|
|
776
832
|
nextDeliverAt: toLocalIso(deliverAt),
|
|
777
833
|
createdAt: toLocalIso(now),
|
|
778
834
|
};
|
|
779
|
-
tasks.push(rec);
|
|
780
835
|
return rec;
|
|
781
836
|
});
|
|
782
837
|
s.agents.set(agent.name, new WeakRef(agent));
|
|
@@ -807,11 +862,10 @@ export function updateScheduledAction(id: string, patch: ScheduledActionPatch, o
|
|
|
807
862
|
const now = s.now();
|
|
808
863
|
const sa = patch.scheduled_action ?? {};
|
|
809
864
|
const { name, ...scheduleChange } = sa;
|
|
810
|
-
const rec =
|
|
811
|
-
const t = tasks.find((x) => x.id === id);
|
|
865
|
+
const rec = updateTask(s.home, id, (t) => {
|
|
812
866
|
if (!t) throw notFound(id);
|
|
813
867
|
const stored = validateRecord(t);
|
|
814
|
-
if (stored) throw new ValidationError(`Scheduled action "${id}" is invalid in
|
|
868
|
+
if (stored) throw new ValidationError(`Scheduled action "${id}" is invalid in its task.json: ${stored}`);
|
|
815
869
|
const scheduleChanged = Object.values(scheduleChange).some((v) => v !== undefined);
|
|
816
870
|
if (scheduleChanged && t.state === "cooking") {
|
|
817
871
|
throw new ValidationError(`Scheduled action "${id}" is cooking right now; schedule changes are refused while it runs. Try again after it finishes.`);
|