agent-accelerator 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -5
- package/package.json +1 -1
- package/src/agent/agent.ts +283 -15
- package/src/agent/context.ts +2 -0
- package/src/agent/loop.ts +127 -8
- package/src/index.ts +2 -1
- package/src/models/catalog-cache.ts +3 -10
- package/src/scheduler/scheduler.ts +2 -0
- package/src/scheduler/store.ts +34 -21
- package/src/session/journal.ts +201 -0
- package/src/session/store.ts +181 -82
- package/src/session/tool-log.ts +62 -0
- package/src/telemetry-file.ts +4 -1
- package/src/tools/executor.ts +72 -15
- package/src/tools/tool.ts +0 -6
- package/src/types/agent.ts +26 -0
- package/src/types/message.ts +4 -0
- package/src/types/response.ts +2 -0
- package/src/types/tool.ts +18 -6
- package/src/utils/durable.ts +218 -0
- package/src/utils/headers.ts +1 -1
- package/src/utils/retry.ts +15 -1
package/src/agent/loop.ts
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
shouldBlockRepeat,
|
|
19
19
|
toolCallFingerprint,
|
|
20
20
|
traceToolEvent,
|
|
21
|
+
type ExecuteToolsOptions,
|
|
21
22
|
} from "../tools/executor.ts";
|
|
22
23
|
import { getModelFromCatalog, ensureModelCatalogFresh, validateModelThinking } from "../models/catalog.ts";
|
|
23
24
|
import { enforceFileAccess } from "../utils/safe-io.ts";
|
|
@@ -70,6 +71,15 @@ export interface AgentLoopConfig {
|
|
|
70
71
|
enforceBudget?: (usage?: TokenUsage) => void;
|
|
71
72
|
/** Called after every turn for durable persistence checkpoints (never fails a turn). */
|
|
72
73
|
onProgress?: () => void;
|
|
74
|
+
/** Called the moment each tool call finishes, before the rest of its batch (the crash-proof record of finished work). */
|
|
75
|
+
onToolResult?: (result: ToolResultRecord) => void;
|
|
76
|
+
/** Opens the log file of one tool call (see `ToolLogs`); unset when the Agent does not persist to a folder. */
|
|
77
|
+
openToolLog?: ExecuteToolsOptions["openToolLog"];
|
|
78
|
+
/**
|
|
79
|
+
* Continue a run that a crash cut off instead of starting from a new prompt: tool calls the last assistant turn made
|
|
80
|
+
* without a stored result are finished first (a result in `stored` is reused, anything else runs now), then the loop goes on.
|
|
81
|
+
*/
|
|
82
|
+
resume?: { stored: Map<string, ToolResultRecord> };
|
|
73
83
|
/**
|
|
74
84
|
* Shared steer inbox for mid-session redirection. Entries are drained
|
|
75
85
|
* between turns (after the current provider/tool work finishes) and
|
|
@@ -438,7 +448,7 @@ export function computeCostFromPricing(usage: TokenUsage, spec?: ModelSpec): Tok
|
|
|
438
448
|
// This function never mutates `source` (Q-35): the priced cost is cloned
|
|
439
449
|
// into a local before folding into the target.
|
|
440
450
|
// ---------------------------------------------------------------------------
|
|
441
|
-
function accumulateUsage(target: TokenUsage, source: TokenUsage, spec?: ModelSpec): void {
|
|
451
|
+
export function accumulateUsage(target: TokenUsage, source: TokenUsage, spec?: ModelSpec): void {
|
|
442
452
|
target.inputTokens += source.inputTokens || 0;
|
|
443
453
|
target.outputTokens += source.outputTokens || 0;
|
|
444
454
|
target.totalTokens += source.totalTokens || 0;
|
|
@@ -506,6 +516,8 @@ function sanitizeToolResult(r: ToolResultRecord): { sanitized: ToolResultRecord;
|
|
|
506
516
|
async function executeToolCallsWithRepeatGuard(options: {
|
|
507
517
|
tools: Record<string, ToolDefinition>;
|
|
508
518
|
toolCalls: ToolCallRecord[];
|
|
519
|
+
onResult?: (result: ToolResultRecord) => void;
|
|
520
|
+
openToolLog?: ExecuteToolsOptions["openToolLog"];
|
|
509
521
|
previousFingerprints: Set<string>;
|
|
510
522
|
repeatCounts: Map<string, number>;
|
|
511
523
|
agentName?: string;
|
|
@@ -546,6 +558,8 @@ async function executeToolCallsWithRepeatGuard(options: {
|
|
|
546
558
|
signal: options.signal,
|
|
547
559
|
sessionId: options.sessionId,
|
|
548
560
|
onSubagentEvent: options.onSubagentEvent,
|
|
561
|
+
onResult: options.onResult,
|
|
562
|
+
openToolLog: options.openToolLog,
|
|
549
563
|
})
|
|
550
564
|
: [];
|
|
551
565
|
const byId = new Map<string, ToolResultRecord[]>();
|
|
@@ -572,6 +586,73 @@ async function executeToolCallsWithRepeatGuard(options: {
|
|
|
572
586
|
};
|
|
573
587
|
}
|
|
574
588
|
|
|
589
|
+
/** Tool calls of the LAST assistant message that have no result in the conversation yet (a crash or interrupt cut them off). */
|
|
590
|
+
export function unfinishedToolCalls(messages: AgentContext["messages"]): ToolCallRecord[] {
|
|
591
|
+
let lastAssistant = -1;
|
|
592
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
593
|
+
if (messages[i]!.role === "assistant") {
|
|
594
|
+
lastAssistant = i;
|
|
595
|
+
break;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
const calls = lastAssistant < 0 ? [] : messages[lastAssistant]!.content;
|
|
599
|
+
if (!Array.isArray(calls)) return [];
|
|
600
|
+
const answered = new Set<string>();
|
|
601
|
+
for (const m of messages.slice(lastAssistant + 1)) {
|
|
602
|
+
if (m.role !== "tool" || !Array.isArray(m.content)) continue;
|
|
603
|
+
for (const p of m.content) if (p.type === "tool_result") answered.add(p.id);
|
|
604
|
+
}
|
|
605
|
+
const open: ToolCallRecord[] = [];
|
|
606
|
+
for (const p of calls) {
|
|
607
|
+
if (p.type === "tool_call" && !answered.has(p.id)) {
|
|
608
|
+
open.push({ id: p.id, name: p.name, arguments: p.arguments, ...(p.rawArguments ? { rawArguments: p.rawArguments } : {}), ...(p.callId ? { callId: p.callId } : {}) });
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
return open;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** True when the conversation ends with tool results of a batch in which a tool called `ctx.stop` (the run was meant to end there). */
|
|
615
|
+
export function endsWithStop(messages: AgentContext["messages"]): boolean {
|
|
616
|
+
for (let i = messages.length - 1; i >= 0 && messages[i]!.role === "tool"; i--) {
|
|
617
|
+
const content = messages[i]!.content;
|
|
618
|
+
if (Array.isArray(content) && content.some((p) => p.type === "tool_result" && p.stopped === true)) return true;
|
|
619
|
+
}
|
|
620
|
+
return false;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// Finishes the tool calls a crash left open (stored result reused, anything else executed now) and records the results in the conversation; in: a resumed run's config, out: tool results in context + what the response should report.
|
|
624
|
+
async function finishInterruptedTools(
|
|
625
|
+
config: AgentLoopConfig
|
|
626
|
+
): Promise<{ calls: ToolCallRecord[]; results: ToolResultRecord[]; subagents: SubAgentExecutionMetadata[] }> {
|
|
627
|
+
const open = unfinishedToolCalls(config.context.messages);
|
|
628
|
+
const none = { calls: [], results: [], subagents: [] };
|
|
629
|
+
if (open.length === 0 || !config.resume) return none;
|
|
630
|
+
const stored = config.resume.stored;
|
|
631
|
+
const toRun = open.filter((c) => !stored.has(c.id));
|
|
632
|
+
const executed = toRun.length
|
|
633
|
+
? await executeToolCalls({
|
|
634
|
+
tools: config.tools,
|
|
635
|
+
toolCalls: toRun,
|
|
636
|
+
agentName: config.agentName,
|
|
637
|
+
parallel: true,
|
|
638
|
+
signal: config.runOptions?.signal,
|
|
639
|
+
sessionId: config.runOptions?.sessionId || config.options?.sessionId,
|
|
640
|
+
onResult: config.onToolResult,
|
|
641
|
+
openToolLog: config.openToolLog,
|
|
642
|
+
})
|
|
643
|
+
: [];
|
|
644
|
+
const ran = new Map(executed.map((r) => [r.id, r]));
|
|
645
|
+
const subagents: SubAgentExecutionMetadata[] = [];
|
|
646
|
+
const results = open.map((call) => {
|
|
647
|
+
const { sanitized, metas } = sanitizeToolResult(ran.get(call.id) ?? stored.get(call.id)!);
|
|
648
|
+
subagents.push(...metas);
|
|
649
|
+
return sanitized;
|
|
650
|
+
});
|
|
651
|
+
config.context.addToolResults(results);
|
|
652
|
+
emitProgress(config);
|
|
653
|
+
return { calls: open, results, subagents };
|
|
654
|
+
}
|
|
655
|
+
|
|
575
656
|
/**
|
|
576
657
|
* Runs a single non-streaming agent turn or multi-turn loop
|
|
577
658
|
*/
|
|
@@ -614,13 +695,28 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
614
695
|
let finalResult: any = null;
|
|
615
696
|
let turns = 0;
|
|
616
697
|
let toolsExecuted = false;
|
|
698
|
+
let stoppedByTool = false;
|
|
617
699
|
let previousToolCallFingerprints = new Set<string>();
|
|
618
700
|
const repeatCounts = new Map<string, number>();
|
|
619
701
|
|
|
620
702
|
try {
|
|
621
703
|
const standardTools = toStandardToolDeclarations(tools);
|
|
622
704
|
|
|
623
|
-
|
|
705
|
+
if (config.resume) {
|
|
706
|
+
const finished = await finishInterruptedTools(config);
|
|
707
|
+
stoppedByTool = finished.results.some((r) => r.stopped) || endsWithStop(context.messages);
|
|
708
|
+
if (finished.calls.length > 0) {
|
|
709
|
+
toolsExecuted = true;
|
|
710
|
+
allToolCalls.push(...finished.calls);
|
|
711
|
+
allToolResults.push(...finished.results);
|
|
712
|
+
for (const meta of finished.subagents) {
|
|
713
|
+
allSubagents.push(meta);
|
|
714
|
+
accumulateUsage(accumulatedUsage, meta.usage, getModelFromCatalog(meta.provider, meta.model));
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
while (!stoppedByTool && turns < maxTurns) {
|
|
624
720
|
// Q-21: fail fast on abort (throws AbortError with partial).
|
|
625
721
|
throwIfAborted(runOptions?.signal, { usage: accumulatedUsage, turns, text: finalResult?.text });
|
|
626
722
|
checkDeadline(startTime, limits.deadlineMs);
|
|
@@ -758,6 +854,7 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
758
854
|
// Execute tool calls — always parallel (model-driven, bloatfree DX7)
|
|
759
855
|
// Q-21: check abort before tools (throws AbortError with partial).
|
|
760
856
|
throwIfAborted(runOptions?.signal, { usage: accumulatedUsage, turns, text: genResult.text });
|
|
857
|
+
emitProgress(config); // the calls are on record BEFORE they run, so a crash can finish them instead of forgetting them
|
|
761
858
|
allToolCalls.push(...genResult.toolCalls);
|
|
762
859
|
// Q-22: per-run tool-call budget.
|
|
763
860
|
if (allToolCalls.length > limits.maxToolCallsPerRun) {
|
|
@@ -774,6 +871,8 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
774
871
|
agentName,
|
|
775
872
|
signal: runOptions?.signal,
|
|
776
873
|
sessionId: runOptions?.sessionId || options?.sessionId,
|
|
874
|
+
onResult: config.onToolResult,
|
|
875
|
+
openToolLog: config.openToolLog,
|
|
777
876
|
});
|
|
778
877
|
const results = guarded.results;
|
|
779
878
|
previousToolCallFingerprints = guarded.fingerprints;
|
|
@@ -811,6 +910,12 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
811
910
|
toolCalls: genResult.toolCalls,
|
|
812
911
|
toolResults: sanitizedResults,
|
|
813
912
|
});
|
|
913
|
+
// A tool ended the run (`ctx.stop`): the whole batch has finished, so stop here. Waiting steers become follow-ups.
|
|
914
|
+
if (sanitizedResults.some((r) => r.stopped)) {
|
|
915
|
+
stoppedByTool = true;
|
|
916
|
+
emitProgress(config);
|
|
917
|
+
break;
|
|
918
|
+
}
|
|
814
919
|
// Steer drain AFTER tool results so call/result pairs stay adjacent.
|
|
815
920
|
// The next loop iteration sends the injected user message to the model
|
|
816
921
|
// within the same run (same usage/turn budget).
|
|
@@ -829,7 +934,7 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
829
934
|
// results above were never sent back, so the run is truncated — surface
|
|
830
935
|
// it via finishReason + warning instead of silently returning.
|
|
831
936
|
const truncatedByMaxTurns =
|
|
832
|
-
turns >= maxTurns && !!finalResult?.toolCalls && finalResult.toolCalls.length > 0;
|
|
937
|
+
!stoppedByTool && turns >= maxTurns && !!finalResult?.toolCalls && finalResult.toolCalls.length > 0;
|
|
833
938
|
if (truncatedByMaxTurns) {
|
|
834
939
|
warnMaxTurnsHit(
|
|
835
940
|
agentName,
|
|
@@ -841,7 +946,7 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
841
946
|
// Structured outputs: validate final text only (tool turns skip this).
|
|
842
947
|
// Strict-fail throws ValidationError with partial attached via catch below.
|
|
843
948
|
let parsedOutput: unknown | undefined;
|
|
844
|
-
if (loopOutput.raw !== undefined && !truncatedByMaxTurns) {
|
|
949
|
+
if (loopOutput.raw !== undefined && !truncatedByMaxTurns && !stoppedByTool) {
|
|
845
950
|
parsedOutput = parseFinalStructuredOutput(finalResult?.text || "", loopOutput.raw, loopOutput.spec);
|
|
846
951
|
}
|
|
847
952
|
|
|
@@ -858,7 +963,9 @@ export async function runAgentLoop(config: AgentLoopConfig): Promise<AgentRespon
|
|
|
858
963
|
model: modelId,
|
|
859
964
|
provider: provider.id,
|
|
860
965
|
// Q-04: canonical finish reason + raw vendor passthrough.
|
|
861
|
-
...
|
|
966
|
+
...(stoppedByTool
|
|
967
|
+
? { finishReason: "stopped" as const, ...(finalResult?.finishReason ? { rawFinishReason: finalResult.finishReason } : {}) }
|
|
968
|
+
: canonicalFinishReason(finalResult?.finishReason, truncatedByMaxTurns)),
|
|
862
969
|
durationMs: Date.now() - startTime,
|
|
863
970
|
raw: applyCaptureRaw(finalResult?.raw, config.captureRaw) as ProviderRawData,
|
|
864
971
|
turns,
|
|
@@ -1016,6 +1123,7 @@ export function streamAgentLoop(config: AgentLoopConfig): AssistantMessageEventS
|
|
|
1016
1123
|
turns = 0;
|
|
1017
1124
|
let previousToolCallFingerprints = new Set<string>();
|
|
1018
1125
|
const repeatCounts = new Map<string, number>();
|
|
1126
|
+
let stoppedByTool = false;
|
|
1019
1127
|
|
|
1020
1128
|
while (turns < maxTurns) {
|
|
1021
1129
|
throwIfCancelled({ usage: accumulatedUsage, turns, text: lastResponse?.text });
|
|
@@ -1198,6 +1306,7 @@ export function streamAgentLoop(config: AgentLoopConfig): AssistantMessageEventS
|
|
|
1198
1306
|
}
|
|
1199
1307
|
|
|
1200
1308
|
throwIfCancelled({ usage: accumulatedUsage, turns, text: turnResponse.text });
|
|
1309
|
+
emitProgress(config); // the calls are on record BEFORE they run, so a crash can finish them instead of forgetting them
|
|
1201
1310
|
allToolCalls.push(...turnResponse.toolCalls);
|
|
1202
1311
|
if (allToolCalls.length > streamLimits.maxToolCallsPerRun) {
|
|
1203
1312
|
throw new Error(
|
|
@@ -1219,6 +1328,8 @@ export function streamAgentLoop(config: AgentLoopConfig): AssistantMessageEventS
|
|
|
1219
1328
|
agentName,
|
|
1220
1329
|
signal: linked.signal,
|
|
1221
1330
|
sessionId: runOptions?.sessionId || options?.sessionId,
|
|
1331
|
+
onResult: config.onToolResult,
|
|
1332
|
+
openToolLog: config.openToolLog,
|
|
1222
1333
|
onSubagentEvent: (ev) => {
|
|
1223
1334
|
try {
|
|
1224
1335
|
outerStream.push({
|
|
@@ -1284,6 +1395,12 @@ export function streamAgentLoop(config: AgentLoopConfig): AssistantMessageEventS
|
|
|
1284
1395
|
toolCalls: turnResponse.toolCalls,
|
|
1285
1396
|
toolResults: sanitizedResults,
|
|
1286
1397
|
});
|
|
1398
|
+
// A tool ended the run (`ctx.stop`): the whole batch has finished, so stop here. Waiting steers become follow-ups.
|
|
1399
|
+
if (sanitizedResults.some((r) => r.stopped)) {
|
|
1400
|
+
stoppedByTool = true;
|
|
1401
|
+
emitProgress(config);
|
|
1402
|
+
break;
|
|
1403
|
+
}
|
|
1287
1404
|
const toolSteered = drainSteerInbox(context, config.steerInbox);
|
|
1288
1405
|
if (toolSteered.length > 0) {
|
|
1289
1406
|
for (const preview of toolSteered) {
|
|
@@ -1301,7 +1418,7 @@ export function streamAgentLoop(config: AgentLoopConfig): AssistantMessageEventS
|
|
|
1301
1418
|
}
|
|
1302
1419
|
|
|
1303
1420
|
const truncatedByMaxTurns =
|
|
1304
|
-
turns >= maxTurns && !!lastResponse?.toolCalls && lastResponse.toolCalls.length > 0;
|
|
1421
|
+
!stoppedByTool && turns >= maxTurns && !!lastResponse?.toolCalls && lastResponse.toolCalls.length > 0;
|
|
1305
1422
|
if (truncatedByMaxTurns) {
|
|
1306
1423
|
warnMaxTurnsHit(
|
|
1307
1424
|
agentName,
|
|
@@ -1309,10 +1426,12 @@ export function streamAgentLoop(config: AgentLoopConfig): AssistantMessageEventS
|
|
|
1309
1426
|
lastResponse!.toolCalls.map((c) => c.name)
|
|
1310
1427
|
);
|
|
1311
1428
|
}
|
|
1312
|
-
const truncatedFinish =
|
|
1429
|
+
const truncatedFinish = stoppedByTool
|
|
1430
|
+
? { finishReason: "stopped" as const, ...(lastResponse?.finishReason ? { rawFinishReason: lastResponse.finishReason } : {}) }
|
|
1431
|
+
: canonicalFinishReason(lastResponse?.finishReason, truncatedByMaxTurns);
|
|
1313
1432
|
|
|
1314
1433
|
let streamParsed: unknown | undefined;
|
|
1315
|
-
if (streamOutputRaw !== undefined && !truncatedByMaxTurns) {
|
|
1434
|
+
if (streamOutputRaw !== undefined && !truncatedByMaxTurns && !stoppedByTool) {
|
|
1316
1435
|
try {
|
|
1317
1436
|
streamParsed = parseFinalStructuredOutput(lastResponse?.text || "", streamOutputRaw, streamOutputSpec);
|
|
1318
1437
|
} catch (err) {
|
package/src/index.ts
CHANGED
|
@@ -317,7 +317,7 @@ export {
|
|
|
317
317
|
formatSessionBanner,
|
|
318
318
|
emptyTotals,
|
|
319
319
|
} from "./session/store.ts";
|
|
320
|
-
export type { SessionLimits, SessionPayloadStatus, PersistedAgentSession, SessionTotals, SessionAgentLike, LoadedSessionDir, PersistedSubAgentTrace } from "./session/store.ts";
|
|
320
|
+
export type { SessionLimits, SessionPayloadStatus, PersistedAgentSession, SessionTotals, SessionAgentLike, LoadedSessionDir, PersistedSubAgentTrace, SessionRunMark, SessionRunEnd, SessionPending, SessionPendingPrompt, SessionToolRecord } from "./session/store.ts";
|
|
321
321
|
|
|
322
322
|
// Utilities
|
|
323
323
|
/**
|
|
@@ -414,6 +414,7 @@ export type {
|
|
|
414
414
|
Budget,
|
|
415
415
|
BudgetState,
|
|
416
416
|
SubAgentRunIds,
|
|
417
|
+
InterruptedRun,
|
|
417
418
|
} from "./types/agent.ts";
|
|
418
419
|
|
|
419
420
|
// Minimalist core: harness modules (RLM/RSI, OTel, testing, CLI doctor, hooks,
|
|
@@ -4,6 +4,7 @@ import { createHash } from "node:crypto";
|
|
|
4
4
|
import { getEnv } from "../utils/env.ts";
|
|
5
5
|
import { telemetryBus } from "../telemetry.ts";
|
|
6
6
|
import { onHomeChange, resolveHome } from "../utils/home.ts";
|
|
7
|
+
import { writeFileDurable } from "../utils/durable.ts";
|
|
7
8
|
|
|
8
9
|
/** Default TTL: 12 Hours (in milliseconds). */
|
|
9
10
|
export const DEFAULT_CATALOG_TTL_MS = 12 * 60 * 60 * 1000;
|
|
@@ -102,17 +103,9 @@ function readJsonFileSync(filePath: string): any {
|
|
|
102
103
|
}
|
|
103
104
|
}
|
|
104
105
|
|
|
106
|
+
// Writes the catalog cache durably but without a `.bak` (it is re-downloadable); in: path + payload from refreshModelCatalog, out: the cache file.
|
|
105
107
|
function writeJsonFileSync(filePath: string, data: any): void {
|
|
106
|
-
|
|
107
|
-
if (!fs.existsSync(dir)) {
|
|
108
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
109
|
-
}
|
|
110
|
-
// Atomic tmp+rename with 0600 so a crash never leaves half-written JSON (Q-16a).
|
|
111
|
-
const rand = Math.random().toString(36).slice(2, 10);
|
|
112
|
-
const tmp = `${filePath}.tmp-${process.pid}-${rand}`;
|
|
113
|
-
fs.writeFileSync(tmp, JSON.stringify(data), { encoding: "utf-8", mode: 0o600 });
|
|
114
|
-
try { fs.chmodSync(tmp, 0o600); } catch {}
|
|
115
|
-
fs.renameSync(tmp, filePath);
|
|
108
|
+
writeFileDurable(filePath, JSON.stringify(data), { backup: false });
|
|
116
109
|
}
|
|
117
110
|
|
|
118
111
|
/** Verifies optional sha256 pin from MODELS_DEV_SHA256 (Q-16a). Returns false on mismatch. */
|
|
@@ -16,6 +16,7 @@ import type { Agent } from "../agent/agent.ts";
|
|
|
16
16
|
import { createChildSessionId } from "../agent/delegation.ts";
|
|
17
17
|
import { buildDynamicWorkerConfig, grantWorkerTools, normalizeWorkerTimeoutValue } from "../agent/worker.ts";
|
|
18
18
|
import { telemetryBus } from "../telemetry.ts";
|
|
19
|
+
import { lockFolder } from "../utils/durable.ts";
|
|
19
20
|
import { ConfigError, ValidationError } from "../types/errors.ts";
|
|
20
21
|
import type { ToolDefinition } from "../types/tool.ts";
|
|
21
22
|
import type { AgentResponse } from "../types/response.ts";
|
|
@@ -649,6 +650,7 @@ const schedulers = new Map<string, Scheduler>();
|
|
|
649
650
|
export function getScheduler(home: string): Scheduler {
|
|
650
651
|
let s = schedulers.get(home);
|
|
651
652
|
if (!s) {
|
|
653
|
+
lockFolder(home); // every scheduler function goes through here: one process per home
|
|
652
654
|
migrateLegacyFolder(home);
|
|
653
655
|
schedulers.set(home, (s = new Scheduler(home)));
|
|
654
656
|
}
|
package/src/scheduler/store.ts
CHANGED
|
@@ -6,13 +6,13 @@
|
|
|
6
6
|
* <home>/scheduled_actions/<id>/<date>.md one file per run
|
|
7
7
|
*
|
|
8
8
|
* The files are the source of truth: every write re-reads first (never writes
|
|
9
|
-
* from a stale copy) and goes through `
|
|
9
|
+
* from a stale copy) and goes through `writeFileDurable` (temp file, fsync, `.bak`, rename).
|
|
10
10
|
* Sync `fs`, like the session store.
|
|
11
11
|
*/
|
|
12
12
|
import * as fs from "node:fs";
|
|
13
13
|
import * as path from "node:path";
|
|
14
14
|
import { z } from "zod";
|
|
15
|
-
import {
|
|
15
|
+
import { readWithBackup, writeFileDurable } from "../utils/durable.ts";
|
|
16
16
|
import { telemetryBus } from "../telemetry.ts";
|
|
17
17
|
import { resolveHome } from "../utils/home.ts";
|
|
18
18
|
export { resolveHome };
|
|
@@ -73,36 +73,49 @@ export function migrateLegacyFolder(home: string): void {
|
|
|
73
73
|
|
|
74
74
|
type Raw = Record<string, unknown>;
|
|
75
75
|
|
|
76
|
-
|
|
77
|
-
function
|
|
78
|
-
let text: string;
|
|
76
|
+
// Parses a `{ version, <key>: [...] }` file into its object entries; undefined when the text is not that shape; in: raw file text, out: readList.
|
|
77
|
+
function parseList(text: string, key: "tasks" | "pending"): Raw[] | undefined {
|
|
79
78
|
try {
|
|
80
|
-
|
|
79
|
+
const list = (JSON.parse(text) as Raw | null)?.[key];
|
|
80
|
+
if (!Array.isArray(list)) return undefined;
|
|
81
|
+
return list.filter((x): x is Raw => !!x && typeof x === "object" && !Array.isArray(x));
|
|
81
82
|
} catch {
|
|
82
|
-
return
|
|
83
|
+
return undefined;
|
|
83
84
|
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Reads a `{ version, <key>: [...] }` file. Missing = empty. A damaged file is renamed `*.corrupt-<ts>`; if its `.bak`
|
|
89
|
+
* (the last good version) is valid it is restored, else the list starts empty.
|
|
90
|
+
*/
|
|
91
|
+
function readList(file: string, key: "tasks" | "pending"): Raw[] {
|
|
92
|
+
const read = readWithBackup(file, (t) => parseList(t, key) !== undefined);
|
|
93
|
+
if (!read) return [];
|
|
94
|
+
const list = parseList(read.text, key);
|
|
95
|
+
if (list && !read.fromBackup) return list;
|
|
96
|
+
const moved = `${file}.corrupt-${Date.now()}`;
|
|
84
97
|
try {
|
|
85
|
-
|
|
86
|
-
if (!Array.isArray(list)) throw new Error(`"${key}" is not an array`);
|
|
87
|
-
return list.filter((x): x is Raw => !!x && typeof x === "object" && !Array.isArray(x));
|
|
98
|
+
fs.renameSync(file, moved);
|
|
88
99
|
} catch {
|
|
89
|
-
|
|
100
|
+
// why: quarantine is best-effort; the file is ignored either way.
|
|
101
|
+
}
|
|
102
|
+
if (list) {
|
|
90
103
|
try {
|
|
91
|
-
|
|
104
|
+
writeFileDurable(file, read.text, { backup: false }); // keep the good .bak where it is
|
|
92
105
|
} catch {
|
|
93
|
-
// why:
|
|
106
|
+
// why: the backup content is still returned for this read.
|
|
94
107
|
}
|
|
95
|
-
telemetryBus.emitWarning(
|
|
96
|
-
"scheduled_action.store_corrupt",
|
|
97
|
-
`[Agent Accelerator] ${path.basename(file)} is not valid; moved to ${path.basename(moved)} and starting empty.`,
|
|
98
|
-
{ file: moved }
|
|
99
|
-
);
|
|
100
|
-
return [];
|
|
101
108
|
}
|
|
109
|
+
telemetryBus.emitWarning(
|
|
110
|
+
"scheduled_action.store_corrupt",
|
|
111
|
+
`[Agent Accelerator] ${path.basename(file)} is not valid; moved to ${path.basename(moved)} and ${list ? "restored from its backup" : "starting empty"}.`,
|
|
112
|
+
{ file: moved, restored: !!list }
|
|
113
|
+
);
|
|
114
|
+
return list ?? [];
|
|
102
115
|
}
|
|
103
116
|
|
|
104
117
|
function writeList(file: string, key: "tasks" | "pending", list: Raw[]): void {
|
|
105
|
-
|
|
118
|
+
writeFileDurable(file, JSON.stringify({ version: 1, [key]: list }, null, 2) + "\n");
|
|
106
119
|
}
|
|
107
120
|
|
|
108
121
|
/** Every record as stored. Records may be invalid: check with {@link validateRecord} before acting on one. */
|
|
@@ -212,7 +225,7 @@ export function writeRunFile(home: string, id: string, deliverAt: Date, header:
|
|
|
212
225
|
const base = runFileName(deliverAt);
|
|
213
226
|
let name = `${base}.md`;
|
|
214
227
|
for (let n = 2; fs.existsSync(path.join(dir, name)); n++) name = `${base}-${n}.md`;
|
|
215
|
-
|
|
228
|
+
writeFileDurable(path.join(dir, name), renderRun(header, body), { backup: false }); // written once, never rewritten
|
|
216
229
|
return `${id}/${name}`;
|
|
217
230
|
}
|
|
218
231
|
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only step log for a persisted session (dir mode). `session.json` stays the snapshot; every step after it
|
|
3
|
+
* adds ONE short line to `journal.ndjson` instead of rewriting the whole file, so a long chat costs the same per step
|
|
4
|
+
* as a short one. When history changes any other way (rollback, repair, reset, import) or the journal grows past a
|
|
5
|
+
* limit, a fresh snapshot replaces both. The snapshot carries an `epoch` id and the journal header repeats it, so a
|
|
6
|
+
* journal left over from before a snapshot is ignored on load (see `replayJournal` in `store.ts`).
|
|
7
|
+
*/
|
|
8
|
+
import * as fs from "node:fs";
|
|
9
|
+
import * as path from "node:path";
|
|
10
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
11
|
+
import type { Message, ContentPart } from "../types/message.ts";
|
|
12
|
+
import { writeFileDurable } from "../utils/durable.ts";
|
|
13
|
+
import {
|
|
14
|
+
JOURNAL_FILE,
|
|
15
|
+
buildSessionData,
|
|
16
|
+
embeddedMedia,
|
|
17
|
+
quarantineIfCorrupt,
|
|
18
|
+
serializeSession,
|
|
19
|
+
stripRedundantRawArguments,
|
|
20
|
+
type SessionAgentLike,
|
|
21
|
+
type SessionPending,
|
|
22
|
+
type SessionPendingPrompt,
|
|
23
|
+
type SessionToolRecord,
|
|
24
|
+
} from "./store.ts";
|
|
25
|
+
|
|
26
|
+
/** A journal past this many lines is folded into a fresh snapshot. */
|
|
27
|
+
const MAX_JOURNAL_LINES = 500;
|
|
28
|
+
/** A journal past this many bytes is folded into a fresh snapshot. */
|
|
29
|
+
const MAX_JOURNAL_BYTES = 4 * 1024 * 1024;
|
|
30
|
+
|
|
31
|
+
const EMPTY_PENDING = JSON.stringify({ steer: [], queue: [] });
|
|
32
|
+
|
|
33
|
+
// One `tool` journal line; the result is saved as given (tool results are JSON-safe by the time the executor returns them); in: a finished tool call, out: the journal.
|
|
34
|
+
function toolLine(t: SessionToolRecord): string {
|
|
35
|
+
return JSON.stringify({ t: "tool", call: t.id, name: t.name, result: t.result, ...(t.isError ? { isError: true } : {}), ...(t.stopped ? { stopped: true } : {}) });
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Persists one session folder (`sessions/<id>/`) step by step. */
|
|
39
|
+
export class SessionJournal {
|
|
40
|
+
/** Epoch of the snapshot the journal belongs to; unset until the first snapshot (and after a failed append). */
|
|
41
|
+
private epoch: string | undefined;
|
|
42
|
+
/** How many messages are on disk, and the last one (identity check: history only grew since the last sync). */
|
|
43
|
+
private count = 0;
|
|
44
|
+
private last: Message | undefined;
|
|
45
|
+
private lines = 0;
|
|
46
|
+
private bytes = 0;
|
|
47
|
+
/** JSON of the last written non-message state (config, system prompt, worker traces). */
|
|
48
|
+
private meta = "";
|
|
49
|
+
/** How many run marks are on disk, and the last one. */
|
|
50
|
+
private runCount = 0;
|
|
51
|
+
private lastRun: unknown;
|
|
52
|
+
/** JSON of the last journaled waiting prompts. */
|
|
53
|
+
private pending = EMPTY_PENDING;
|
|
54
|
+
/** Run ids whose ending is on disk. */
|
|
55
|
+
private readonly endsOnDisk = new Set<string>();
|
|
56
|
+
/** How many of the current run's finished tool results are on disk, and the last one. */
|
|
57
|
+
private toolCount = 0;
|
|
58
|
+
private lastTool: unknown;
|
|
59
|
+
/** Disk form of each message already prepared (media replaced by `media/` files, redundant raw arguments dropped). */
|
|
60
|
+
private readonly onDisk = new WeakMap<Message, Message>();
|
|
61
|
+
/** Same for waiting prompts made of content parts. */
|
|
62
|
+
private readonly promptOnDisk = new WeakMap<ContentPart[], ContentPart[]>();
|
|
63
|
+
|
|
64
|
+
constructor(readonly dir: string) {}
|
|
65
|
+
|
|
66
|
+
// Brings the folder up to date with the agent: appends new messages and changed state, or writes a fresh snapshot when history changed another way; in: Agent.doPersist after every step, out: files in this.dir.
|
|
67
|
+
sync(agent: SessionAgentLike): void {
|
|
68
|
+
const messages = agent.context.messages;
|
|
69
|
+
const runs = agent.runMarks ?? [];
|
|
70
|
+
const grew =
|
|
71
|
+
this.epoch !== undefined &&
|
|
72
|
+
messages.length >= this.count &&
|
|
73
|
+
(this.count === 0 || messages[this.count - 1] === this.last) &&
|
|
74
|
+
runs.length >= this.runCount &&
|
|
75
|
+
(this.runCount === 0 || runs[this.runCount - 1] === this.lastRun);
|
|
76
|
+
if (!grew) {
|
|
77
|
+
this.snapshot(agent);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const records: string[] = [];
|
|
81
|
+
for (let i = this.count; i < messages.length; i++) records.push(JSON.stringify({ t: "msg", i, m: this.toDisk(messages[i]!) }));
|
|
82
|
+
for (let r = this.runCount; r < runs.length; r++) records.push(JSON.stringify({ t: "run", id: runs[r]!.id, i: runs[r]!.i, at: runs[r]!.at }));
|
|
83
|
+
for (const mark of runs) {
|
|
84
|
+
if (mark.end && !this.endsOnDisk.has(mark.id)) records.push(JSON.stringify({ t: "end", id: mark.id, status: mark.end }));
|
|
85
|
+
}
|
|
86
|
+
const tools = agent.toolRecords ?? [];
|
|
87
|
+
const firstNewTool = this.toolCount > 0 && tools[this.toolCount - 1] === this.lastTool ? this.toolCount : 0; // a new run starts the list over
|
|
88
|
+
for (let k = firstNewTool; k < tools.length; k++) records.push(toolLine(tools[k]!));
|
|
89
|
+
const meta = this.metaOf(agent);
|
|
90
|
+
if (meta !== this.meta) records.push(`{"t":"meta","m":${meta}}`);
|
|
91
|
+
const pending = this.pendingOf(agent);
|
|
92
|
+
if (pending !== this.pending) records.push(`{"t":"pending","p":${pending}}`);
|
|
93
|
+
if (records.length === 0) return;
|
|
94
|
+
const text = records.join("\n") + "\n";
|
|
95
|
+
try {
|
|
96
|
+
const fd = fs.openSync(path.join(this.dir, JOURNAL_FILE), "a", 0o600);
|
|
97
|
+
try {
|
|
98
|
+
fs.writeFileSync(fd, text);
|
|
99
|
+
fs.fsyncSync(fd);
|
|
100
|
+
} finally {
|
|
101
|
+
fs.closeSync(fd);
|
|
102
|
+
}
|
|
103
|
+
} catch (err) {
|
|
104
|
+
this.epoch = undefined; // a half-written line may be on disk: the next sync starts over with a snapshot
|
|
105
|
+
throw err;
|
|
106
|
+
}
|
|
107
|
+
this.count = messages.length;
|
|
108
|
+
this.last = messages[messages.length - 1];
|
|
109
|
+
this.meta = meta;
|
|
110
|
+
this.runCount = runs.length;
|
|
111
|
+
this.lastRun = runs[runs.length - 1];
|
|
112
|
+
this.pending = pending;
|
|
113
|
+
this.toolCount = tools.length;
|
|
114
|
+
this.lastTool = tools[tools.length - 1];
|
|
115
|
+
for (const mark of runs) if (mark.end) this.endsOnDisk.add(mark.id);
|
|
116
|
+
this.lines += records.length;
|
|
117
|
+
this.bytes += text.length;
|
|
118
|
+
if (this.lines >= MAX_JOURNAL_LINES || this.bytes >= MAX_JOURNAL_BYTES) this.snapshot(agent);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Writes `session.json` (new epoch) and then restarts the journal under that epoch; a crash in between leaves a journal with the old epoch, which load ignores; in: sync, out: both files.
|
|
122
|
+
private snapshot(agent: SessionAgentLike): void {
|
|
123
|
+
const messages = agent.context.messages;
|
|
124
|
+
const epoch = randomUUID();
|
|
125
|
+
const target = path.join(this.dir, "session.json");
|
|
126
|
+
if (this.epoch === undefined) quarantineIfCorrupt(target);
|
|
127
|
+
const data = buildSessionData({ ...agent, context: { ...agent.context, messages: messages.map((m) => this.toDisk(m)) } }, null);
|
|
128
|
+
writeFileDurable(target, serializeSession({ ...data, epoch }) + "\n");
|
|
129
|
+
const pending = this.pendingOf(agent);
|
|
130
|
+
const tools = agent.toolRecords ?? [];
|
|
131
|
+
// Waiting prompts and the current run's finished tool results live only in the journal, so a new journal starts with them.
|
|
132
|
+
const header =
|
|
133
|
+
JSON.stringify({ t: "base", e: epoch }) + "\n" + (pending === EMPTY_PENDING ? "" : `{"t":"pending","p":${pending}}\n`) + tools.map((t) => toolLine(t) + "\n").join("");
|
|
134
|
+
writeFileDurable(path.join(this.dir, JOURNAL_FILE), header, { backup: false });
|
|
135
|
+
this.epoch = epoch;
|
|
136
|
+
this.pending = pending;
|
|
137
|
+
this.runs(agent);
|
|
138
|
+
this.count = messages.length;
|
|
139
|
+
this.last = messages[messages.length - 1];
|
|
140
|
+
this.lines = (pending === EMPTY_PENDING ? 0 : 1) + tools.length;
|
|
141
|
+
this.toolCount = tools.length;
|
|
142
|
+
this.lastTool = tools[tools.length - 1];
|
|
143
|
+
this.endsOnDisk.clear();
|
|
144
|
+
for (const mark of agent.runMarks ?? []) if (mark.end) this.endsOnDisk.add(mark.id); // the snapshot holds them
|
|
145
|
+
this.bytes = 0;
|
|
146
|
+
this.meta = this.metaOf(agent);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Remembers which run marks the snapshot already holds; in: snapshot, out: the next sync's "which marks are new".
|
|
150
|
+
private runs(agent: SessionAgentLike): void {
|
|
151
|
+
const runs = agent.runMarks ?? [];
|
|
152
|
+
this.runCount = runs.length;
|
|
153
|
+
this.lastRun = runs[runs.length - 1];
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// JSON of the waiting prompts with media moved to files; in: the agent's steer inbox + follow-up queue, out: change detection + the `pending` record.
|
|
157
|
+
private pendingOf(agent: SessionAgentLike): string {
|
|
158
|
+
const waiting = agent.pendingPrompts?.();
|
|
159
|
+
if (!waiting || waiting.steer.length + waiting.queue.length === 0) return EMPTY_PENDING;
|
|
160
|
+
const parts = (prompt: ContentPart[]): ContentPart[] => {
|
|
161
|
+
let disk = this.promptOnDisk.get(prompt);
|
|
162
|
+
if (!disk) this.promptOnDisk.set(prompt, (disk = this.toDisk({ role: "user", content: prompt }).content as ContentPart[]));
|
|
163
|
+
return disk;
|
|
164
|
+
};
|
|
165
|
+
const slim = (list: SessionPendingPrompt[]): SessionPendingPrompt[] =>
|
|
166
|
+
list.map((w) => ({ ...w, prompt: typeof w.prompt === "string" ? w.prompt : parts(w.prompt) }));
|
|
167
|
+
return JSON.stringify({ steer: slim(waiting.steer), queue: slim(waiting.queue) } satisfies SessionPending);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// JSON of everything in the snapshot that is not a message, identity or totals; in: the live agent, out: change detection + the `meta` record.
|
|
171
|
+
private metaOf(agent: SessionAgentLike): string {
|
|
172
|
+
const { version: _v, sessionId: _s, messages: _m, totals: _t, runs: _r, ...rest } = buildSessionData(
|
|
173
|
+
{ ...agent, context: { ...agent.context, messages: [] } },
|
|
174
|
+
null
|
|
175
|
+
);
|
|
176
|
+
return JSON.stringify(rest);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// The form of a message that goes to disk: embedded media written once as `media/<kind>-h<hash>.<ext>` (content-addressed, never deleted or renumbered), redundant raw arguments dropped; in: a live message, out: journal lines and snapshots.
|
|
180
|
+
private toDisk(message: Message): Message {
|
|
181
|
+
const known = this.onDisk.get(message);
|
|
182
|
+
if (known) return known;
|
|
183
|
+
const slim = stripRedundantRawArguments([message])[0]!;
|
|
184
|
+
let out = slim;
|
|
185
|
+
if (Array.isArray(slim.content)) {
|
|
186
|
+
let changed = false;
|
|
187
|
+
const content = (slim.content as ContentPart[]).map((part) => {
|
|
188
|
+
const found = embeddedMedia(part);
|
|
189
|
+
if (!found) return part;
|
|
190
|
+
const file = `${found.key}-h${createHash("sha256").update(found.bytes).digest("hex").slice(0, 16)}.${found.ext}`;
|
|
191
|
+
const dest = path.join(this.dir, "media", file);
|
|
192
|
+
if (!fs.existsSync(dest)) writeFileDurable(dest, found.bytes, { backup: false });
|
|
193
|
+
changed = true;
|
|
194
|
+
return { ...(part as unknown as Record<string, unknown>), [found.key]: `media/${file}`, ...(found.mimeType ? { mimeType: found.mimeType } : {}) } as unknown as ContentPart;
|
|
195
|
+
});
|
|
196
|
+
if (changed) out = { ...slim, content };
|
|
197
|
+
}
|
|
198
|
+
this.onDisk.set(message, out);
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
}
|