@sema-agent/core 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/teacher.js +51 -23
- package/dist/core/runner/assemble-result.d.ts +1 -0
- package/dist/core/runner/assemble-result.js +12 -8
- package/dist/core/runner/prepare-task.js +12 -6
- package/dist/core/runner/runtask.js +18 -5
- package/dist/core/task-registry-agent.d.ts +3 -1
- package/dist/core/task-registry-agent.js +18 -2
- package/dist/core/task-registry-monitor.js +71 -6
- package/dist/core/task-registry-shared.d.ts +12 -1
- package/dist/core/task-registry-shared.js +5 -2
- package/dist/core/task-registry.d.ts +3 -0
- package/dist/core/task-registry.js +4 -3
- package/dist/core/tools.d.ts +2 -0
- package/dist/core/tools.js +9 -0
- package/dist/core/workflow-journal-store.d.ts +16 -0
- package/dist/core/workflow-journal-store.js +28 -0
- package/dist/orchestration/workflow.d.ts +15 -0
- package/dist/orchestration/workflow.js +63 -19
- package/dist/tools/monitor.d.ts +2 -0
- package/dist/tools/monitor.js +2 -0
- package/package.json +1 -1
package/dist/agents/teacher.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { mapNestedSuspend, isDurablePause } from "./suspend-guard.js";
|
|
3
|
+
import { isDefineToolProduct, stampDefineToolBrand } from "../core/tools.js";
|
|
3
4
|
import { delimitUntrusted, inlineUntrusted } from "../core/untrusted-text.js";
|
|
4
5
|
export const TEACHER_PROMPT = `You are an expert advisor to a less-capable "student" agent that got stuck.
|
|
5
6
|
You receive the task and the student's recent failed attempts. Your job is to help the student RECOVER,
|
|
@@ -119,29 +120,56 @@ async function runTeacherCore(runner, studentSpec, teacher) {
|
|
|
119
120
|
errorStreak = sig === lastErrSig ? errorStreak + 1 : 1;
|
|
120
121
|
lastErrSig = sig;
|
|
121
122
|
};
|
|
122
|
-
const wrap = (t) =>
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
?
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
123
|
+
const wrap = (t) => {
|
|
124
|
+
if (isDefineToolProduct(t)) {
|
|
125
|
+
const product = t;
|
|
126
|
+
return stampDefineToolBrand({
|
|
127
|
+
...t,
|
|
128
|
+
execute: async (toolCallId, rawParams, signal, onUpdate) => {
|
|
129
|
+
try {
|
|
130
|
+
const out = await product.execute(toolCallId, rawParams, signal, onUpdate);
|
|
131
|
+
const blocks = out && typeof out === "object" && "content" in out ? out.content : out;
|
|
132
|
+
const text = Array.isArray(blocks)
|
|
133
|
+
? blocks.map((b) => (b && typeof b === "object" && "text" in b ? String(b.text) : "")).join("\n")
|
|
134
|
+
: String(blocks ?? "");
|
|
135
|
+
logTool(t.name, rawParams, text);
|
|
136
|
+
errorStreak = 0;
|
|
137
|
+
lastErrSig = "";
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
catch (e) {
|
|
141
|
+
const errText = String(e);
|
|
142
|
+
logTool(t.name, rawParams, `ERROR: ${errText}`);
|
|
143
|
+
onToolFail(t.name, errText);
|
|
144
|
+
throw e;
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
...t,
|
|
151
|
+
execute: async (args, ctx) => {
|
|
152
|
+
try {
|
|
153
|
+
const out = await t.execute(args, ctx);
|
|
154
|
+
const text = typeof out === "string"
|
|
155
|
+
? out
|
|
156
|
+
: out && typeof out === "object" && "content" in out
|
|
157
|
+
? String(out.content)
|
|
158
|
+
: (JSON.stringify(out) ?? "");
|
|
159
|
+
logTool(t.name, args, text);
|
|
160
|
+
errorStreak = 0;
|
|
161
|
+
lastErrSig = "";
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
catch (e) {
|
|
165
|
+
const errText = String(e);
|
|
166
|
+
logTool(t.name, args, `ERROR: ${errText}`);
|
|
167
|
+
onToolFail(t.name, errText);
|
|
168
|
+
throw e;
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
};
|
|
145
173
|
const tools = studentSpec.tools?.map(wrap);
|
|
146
174
|
const helperBase = () => teacher.helperModel
|
|
147
175
|
? { model: teacher.helperModel }
|
|
@@ -36,14 +36,16 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
|
|
|
36
36
|
const taskId = spec.taskId ?? sessionId;
|
|
37
37
|
const text = final ? assistantText(final) : "";
|
|
38
38
|
stats.totalInputTokens = stats.promptTokens;
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
39
|
+
if (!flags.unpricedSpend) {
|
|
40
|
+
const compactionMicroUsd = stats.compactionMicroUsd ?? 0;
|
|
41
|
+
const nestedSubagentMicroUsd = stats.nested?.costMicroUsd ?? 0;
|
|
42
|
+
stats.costBreakdown = {
|
|
43
|
+
llmRootMicroUsd: Math.max(0, stats.costMicroUsd - compactionMicroUsd),
|
|
44
|
+
nestedSubagentMicroUsd,
|
|
45
|
+
memoryConsolidationMicroUsd: 0,
|
|
46
|
+
compactionMicroUsd,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
47
49
|
let status;
|
|
48
50
|
const result = text;
|
|
49
51
|
let errorMessage;
|
|
@@ -139,5 +141,7 @@ export function assembleResult(spec, sessionId, final, stats, flags) {
|
|
|
139
141
|
}
|
|
140
142
|
const { compactionMicroUsd: _internalCompaction, ...publicStats } = stats;
|
|
141
143
|
void _internalCompaction;
|
|
144
|
+
if (flags.unpricedSpend)
|
|
145
|
+
delete publicStats.costMicroUsd;
|
|
142
146
|
return { taskId, sessionId, status, ...(flags.model !== undefined ? { model: flags.model } : {}), result: result.trim(), salvagedOutput, blockedReason, errorMessage, errorCode, checkpointToken, checkpointGate, stats: publicStats };
|
|
143
147
|
}
|
|
@@ -28,7 +28,7 @@ import { cloneObserverInput, formatHookFeedback, runToolGate } from "../hooks.js
|
|
|
28
28
|
import { reconcileInterruptedSession } from "../session-reconcile.js";
|
|
29
29
|
import { CacheBreakDetector, toolsToFingerprintInputs } from "../cache-break-detector.js";
|
|
30
30
|
import { reservedCollisions, reservedFor } from "../../brain/request-params.js";
|
|
31
|
-
import { defineTool } from "../tools.js";
|
|
31
|
+
import { defineTool, isDefineToolProduct } from "../tools.js";
|
|
32
32
|
import { canonicalToolName } from "../tool-name-aliases.js";
|
|
33
33
|
import { pathToUri } from "../lsp-protocol.js";
|
|
34
34
|
import { DEFAULT_TOOL_RESULT_THRESHOLD_CHARS, buildToolResultRef, firstPartyOffloadPolicy, InMemoryToolResultStore, RunnerSharedToolResultStore, ScopedToolResultStore, isVolatileOffloadStore, OFFLOAD_TOOL_NAME, createReadToolResultTool, withToolResultOffload, } from "../tool-result-store.js";
|
|
@@ -796,10 +796,15 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
796
796
|
requestStopAfterTurn,
|
|
797
797
|
...(spec.enablePlanMode === true ? { enterPlanMode } : {}),
|
|
798
798
|
});
|
|
799
|
-
const tools = (spec.tools ?? []).map((t) =>
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
799
|
+
const tools = (spec.tools ?? []).map((t) => {
|
|
800
|
+
if (isDefineToolProduct(t)) {
|
|
801
|
+
return maybeOffload(t, t);
|
|
802
|
+
}
|
|
803
|
+
return maybeOffload(defineTool({
|
|
804
|
+
...t,
|
|
805
|
+
execute: (args, ctx) => t.execute(args, enrichSpecToolCtx(ctx)),
|
|
806
|
+
}), t);
|
|
807
|
+
});
|
|
803
808
|
const blockedRef = {};
|
|
804
809
|
if (spec.enableBlockedReport !== false) {
|
|
805
810
|
tools.push(createReportBlockedTool(blockedRef));
|
|
@@ -1217,7 +1222,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
1217
1222
|
if (backgroundTaskToolsActive || workflowToolsActive) {
|
|
1218
1223
|
toolEffects.set("TaskOutput", "read");
|
|
1219
1224
|
toolEffects.set("TaskStop", "write");
|
|
1220
|
-
tools.push(firstPartyOffload(createTaskOutputTool({ registry: defaultTaskRegistry, owner: hostTaskId, scope: taskScope, sessionId, workflowStore: deps.workflowRunStore, agentStore: deps.backgroundAgentStore, notificationWired: internals?.onTaskNotification !== undefined, oneShot: spec.oneShot, ...(callCapRef ? { deadlineMs: () => toolCutDeadlineMs(callCapRef, Date.now()) } : {}) })), firstPartyOffload(createTaskStopTool({ registry: defaultTaskRegistry, owner: hostTaskId, scope: taskScope, sessionId, workflowStore: deps.workflowRunStore, agentStore: deps.backgroundAgentStore })));
|
|
1225
|
+
tools.push(firstPartyOffload(createTaskOutputTool({ registry: defaultTaskRegistry, owner: hostTaskId, scope: taskScope, sessionId, workflowStore: deps.workflowRunStore, agentStore: deps.backgroundAgentStore, notificationWired: internals?.onTaskNotification !== undefined, oneShot: spec.oneShot, toolResultStore: offloadStore, ...(callCapRef ? { deadlineMs: () => toolCutDeadlineMs(callCapRef, Date.now()) } : {}) })), firstPartyOffload(createTaskStopTool({ registry: defaultTaskRegistry, owner: hostTaskId, scope: taskScope, sessionId, workflowStore: deps.workflowRunStore, agentStore: deps.backgroundAgentStore })));
|
|
1221
1226
|
if (runnerSelf && !(spec.tools ?? []).some((t) => t.name === SEND_MESSAGE_TOOL_NAME)) {
|
|
1222
1227
|
const delegationForRevive = (spec.tools ?? []).find((t) => t.agentListing !== undefined);
|
|
1223
1228
|
const reviveSpawn = delegationForRevive !== undefined && deps.backgroundAgentStore !== undefined && deps.mailboxStore !== undefined
|
|
@@ -1277,6 +1282,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
1277
1282
|
...(sessionId !== undefined ? { sessionId } : {}),
|
|
1278
1283
|
...(internals?.onTaskNotification !== undefined ? { onTaskNotification: internals.onTaskNotification } : {}),
|
|
1279
1284
|
...(handsCwdRef !== undefined ? { cwdRef: handsCwdRef } : {}),
|
|
1285
|
+
...(offloadStore !== undefined ? { toolResultStore: offloadStore } : {}),
|
|
1280
1286
|
})));
|
|
1281
1287
|
}
|
|
1282
1288
|
if (handsCwdRef !== undefined) {
|
|
@@ -45,7 +45,7 @@ import { structuredFrom, toolOutputFrom } from "./tool-output-projection.js";
|
|
|
45
45
|
export { DEFAULT_MAX_TURNS };
|
|
46
46
|
function createRunState() {
|
|
47
47
|
return {
|
|
48
|
-
telemetry: { cacheFamily: "input-excludes-cached", pricing: { inputPer1M: 0, outputPer1M: 0 }, tracer: undefined, taskId: "", taskStart: 0, taskStartMonotonic: 0, cacheBreakReported: false },
|
|
48
|
+
telemetry: { cacheFamily: "input-excludes-cached", pricing: { inputPer1M: 0, outputPer1M: 0 }, pricingConfigured: true, unpricedSpend: false, tracer: undefined, taskId: "", taskStart: 0, taskStartMonotonic: 0, cacheBreakReported: false },
|
|
49
49
|
degrade: { degradeToModel: undefined, degraded: undefined, outputErrorStreak: 0, outputInvalid: false, recordDegraded: () => { } },
|
|
50
50
|
limits: { turnsExceeded: false, budgetHit: undefined, outputRetryCap: 0, effectiveMaxTurns: undefined },
|
|
51
51
|
budget: { remainingMicroUsd: undefined, maxCostMicroUsd: undefined, overBudget: () => false, streamCancel: false, callOutputChars: 0, lastStreamBudgetCheck: 0, projectedOverBudget: () => false },
|
|
@@ -788,6 +788,8 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
788
788
|
const u = m.usage;
|
|
789
789
|
stats.tokens += u.totalTokens || 0;
|
|
790
790
|
const { totalInputTokens: turnInput, costMicroUsd: turnCostMicroUsd } = usageCostMicroUsd(rs.telemetry.cacheFamily, u, rs.telemetry.pricing);
|
|
791
|
+
if (!rs.telemetry.pricingConfigured)
|
|
792
|
+
rs.telemetry.unpricedSpend = true;
|
|
791
793
|
stats.promptTokens += turnInput;
|
|
792
794
|
stats.cachedTokens += u.cacheRead || 0;
|
|
793
795
|
stats.cacheWriteTokens += u.cacheWrite || 0;
|
|
@@ -826,7 +828,7 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
826
828
|
latencyMs: rs.turn.callStartAt !== undefined ? callEndAt - rs.turn.callStartAt : 0,
|
|
827
829
|
firstTokenMs: rs.turn.callStartAt !== undefined && rs.turn.firstTokenAt !== undefined ? rs.turn.firstTokenAt - rs.turn.callStartAt : undefined,
|
|
828
830
|
...(callIssuedAt !== undefined ? { callStartedAt: callIssuedAt } : {}),
|
|
829
|
-
costMicroUsd: turnCostMicroUsd,
|
|
831
|
+
...(rs.telemetry.pricingConfigured ? { costMicroUsd: turnCostMicroUsd } : {}),
|
|
830
832
|
...(capLast !== undefined ? { callCap: capLast.cap, capThinkingSkipped: capLast.thinkingSkipped } : {}),
|
|
831
833
|
...(m.role === "assistant" && typeof m.stopReason === "string"
|
|
832
834
|
? { stopReason: m.stopReason }
|
|
@@ -1628,6 +1630,7 @@ export class Runner {
|
|
|
1628
1630
|
const rs = createRunState();
|
|
1629
1631
|
rs.telemetry.cacheFamily = cacheFamilyOf(prepared.model);
|
|
1630
1632
|
rs.telemetry.pricing = this.deps.pricing?.[prepared.model.id] ?? modelCostToPricing(prepared.model.cost);
|
|
1633
|
+
rs.telemetry.pricingConfigured = this.deps.pricing?.[prepared.model.id] !== undefined || prepared.model.cost !== undefined;
|
|
1631
1634
|
if (spec.degrade) {
|
|
1632
1635
|
try {
|
|
1633
1636
|
rs.degrade.degradeToModel = resolveModel(spec.degrade.to, this.deps.models);
|
|
@@ -1669,11 +1672,17 @@ export class Runner {
|
|
|
1669
1672
|
}
|
|
1670
1673
|
if (m) {
|
|
1671
1674
|
rs.telemetry.pricing = this.deps.pricing?.[m.id] ?? modelCostToPricing(m.cost);
|
|
1675
|
+
rs.telemetry.pricingConfigured = this.deps.pricing?.[m.id] !== undefined || m.cost !== undefined;
|
|
1672
1676
|
rs.telemetry.cacheFamily = cacheFamilyOf(m);
|
|
1673
1677
|
}
|
|
1674
1678
|
else {
|
|
1675
|
-
if (this.deps.pricing?.[info.to])
|
|
1679
|
+
if (this.deps.pricing?.[info.to]) {
|
|
1676
1680
|
rs.telemetry.pricing = this.deps.pricing[info.to];
|
|
1681
|
+
rs.telemetry.pricingConfigured = true;
|
|
1682
|
+
}
|
|
1683
|
+
else {
|
|
1684
|
+
rs.telemetry.pricingConfigured = false;
|
|
1685
|
+
}
|
|
1677
1686
|
rs.telemetry.cacheFamily = "input-excludes-cached";
|
|
1678
1687
|
}
|
|
1679
1688
|
rs.degrade.degraded = info;
|
|
@@ -2076,6 +2085,9 @@ export class Runner {
|
|
|
2076
2085
|
return;
|
|
2077
2086
|
const fam = cacheFamilyOf(m);
|
|
2078
2087
|
const price = this.deps.pricing?.[m.id] ?? modelCostToPricing(m.cost);
|
|
2088
|
+
const priced = this.deps.pricing?.[m.id] !== undefined || m.cost !== undefined;
|
|
2089
|
+
if (!priced)
|
|
2090
|
+
rs.telemetry.unpricedSpend = true;
|
|
2079
2091
|
const { totalInputTokens, costMicroUsd } = usageCostMicroUsd(fam, u, price);
|
|
2080
2092
|
stats.tokens += u.totalTokens || 0;
|
|
2081
2093
|
stats.promptTokens += totalInputTokens;
|
|
@@ -2088,7 +2100,7 @@ export class Runner {
|
|
|
2088
2100
|
kind: "brain.call", version: 1, taskId: rs.telemetry.taskId, model: m.id, provider: m.provider,
|
|
2089
2101
|
promptTokens: totalInputTokens, completionTokens: u.output || 0,
|
|
2090
2102
|
cacheRead: u.cacheRead || 0, cacheWrite: u.cacheWrite || 0,
|
|
2091
|
-
latencyMs: 0, costMicroUsd, ...(typeof msg?.stopReason === "string" ? { stopReason: msg.stopReason } : {}), ts: Date.now(),
|
|
2103
|
+
latencyMs: 0, ...(priced ? { costMicroUsd } : {}), ...(typeof msg?.stopReason === "string" ? { stopReason: msg.stopReason } : {}), ts: Date.now(),
|
|
2092
2104
|
}));
|
|
2093
2105
|
};
|
|
2094
2106
|
const compactionBrain = {
|
|
@@ -2717,6 +2729,7 @@ export class Runner {
|
|
|
2717
2729
|
const result = assembleResult(spec, prepared.sessionId, final, stats, {
|
|
2718
2730
|
threw,
|
|
2719
2731
|
model: prepared.model.id,
|
|
2732
|
+
unpricedSpend: rs.telemetry.unpricedSpend,
|
|
2720
2733
|
abortedForTimeout: timeout.fired,
|
|
2721
2734
|
abortedForTurns: rs.limits.turnsExceeded,
|
|
2722
2735
|
abortedLive,
|
|
@@ -2835,7 +2848,7 @@ export class Runner {
|
|
|
2835
2848
|
errorCode: result.errorCode,
|
|
2836
2849
|
turns: stats.turns,
|
|
2837
2850
|
tokens: stats.tokens,
|
|
2838
|
-
costMicroUsd: stats.costMicroUsd,
|
|
2851
|
+
...(rs.telemetry.unpricedSpend ? {} : { costMicroUsd: stats.costMicroUsd }),
|
|
2839
2852
|
durationMs: Date.now() - rs.telemetry.taskStart,
|
|
2840
2853
|
...(prepared.outputRef.set ? { hasStructuredOutput: true } : {}),
|
|
2841
2854
|
...(stats.mechanisms !== undefined
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type BackgroundAgentRecord, type BackgroundAgentStore } from "./background-agent-store.js";
|
|
2
2
|
import { type StopSource, type TaskAccess, type UnifiedTaskResult, type BackgroundAgentTaskHandle, type DurableAgentCore, type ParkedClaimTicket, type RegisterBackgroundAgentInput } from "./task-registry-shared.js";
|
|
3
|
+
import { type ToolResultStore } from "./tool-result-store.js";
|
|
3
4
|
export declare function ensureDurableHeartbeatLane(core: DurableAgentCore): void;
|
|
4
5
|
export declare function durableAgentWriteLane(handle: BackgroundAgentTaskHandle, patch: Partial<BackgroundAgentRecord>, clear?: (keyof BackgroundAgentRecord)[]): void;
|
|
5
6
|
export declare function durableAgentArmedLane(core: DurableAgentCore, id: string): boolean;
|
|
@@ -119,5 +120,6 @@ export declare function notFoundRunningAgentsTail(footer: {
|
|
|
119
120
|
background: string[];
|
|
120
121
|
}): string;
|
|
121
122
|
export declare function serveDurableAgentRowLane(row: BackgroundAgentRecord): UnifiedTaskResult;
|
|
122
|
-
export declare function
|
|
123
|
+
export declare function spillClippedAgentResult(handle: BackgroundAgentTaskHandle, full: string, clipped: string, store: ToolResultStore | undefined, sessionId: string | undefined): Promise<string>;
|
|
124
|
+
export declare function pollBackgroundAgentLane(handle: BackgroundAgentTaskHandle, deadline?: number, signal?: AbortSignal, oneShot?: boolean, store?: ToolResultStore, sessionId?: string): Promise<UnifiedTaskResult>;
|
|
123
125
|
export declare function stopBackgroundAgentLane(core: DurableAgentCore, handle: BackgroundAgentTaskHandle): Promise<UnifiedTaskResult>;
|
|
@@ -4,6 +4,7 @@ import { canAccessAgentRecord, BackgroundAgentStoreError, } from "./background-a
|
|
|
4
4
|
import { shutdownDebug } from "./shutdown-debug.js";
|
|
5
5
|
import { delimitUntrusted } from "./untrusted-text.js";
|
|
6
6
|
import { mintCompletionId, commitCompletionIdIfEmpty, clipTaskOutput, assertOwnership, sleepPollStep, alreadyTerminalStopNote, canAccess, normalizeAgentName, closestName, DURABLE_AGENT_HEARTBEAT_MS, DURABLE_AGENT_HANDLE_RE, BG_AGENT_REAP_STOP_ERROR, } from "./task-registry-shared.js";
|
|
7
|
+
import { buildToolResultRef, OFFLOAD_TOOL_NAME } from "./tool-result-store.js";
|
|
7
8
|
export function ensureDurableHeartbeatLane(core) {
|
|
8
9
|
if (core.durableHeartbeatTimer !== undefined)
|
|
9
10
|
return;
|
|
@@ -834,6 +835,7 @@ export function reviveBackgroundAgentLane(core, id, access, abort) {
|
|
|
834
835
|
handle.abort = abort;
|
|
835
836
|
handle.result = undefined;
|
|
836
837
|
handle.resultFull = undefined;
|
|
838
|
+
handle.spillRef = undefined;
|
|
837
839
|
handle.error = undefined;
|
|
838
840
|
handle.resultIsPartial = undefined;
|
|
839
841
|
handle.stopSource = undefined;
|
|
@@ -1022,7 +1024,19 @@ ${clipTaskOutput(row.finalOutputFull ?? row.finalOutput)}` : "(no result text)"}
|
|
|
1022
1024
|
},
|
|
1023
1025
|
};
|
|
1024
1026
|
}
|
|
1025
|
-
export async function
|
|
1027
|
+
export async function spillClippedAgentResult(handle, full, clipped, store, sessionId) {
|
|
1028
|
+
if (clipped === full)
|
|
1029
|
+
return clipped;
|
|
1030
|
+
if (store === undefined)
|
|
1031
|
+
return clipped;
|
|
1032
|
+
if (handle.spillRef === undefined) {
|
|
1033
|
+
const ref = buildToolResultRef(sessionId ?? "no-session", `${handle.id}_c${handle.reviveCycle ?? 0}`);
|
|
1034
|
+
await store.put(ref, full);
|
|
1035
|
+
handle.spillRef = ref;
|
|
1036
|
+
}
|
|
1037
|
+
return `${clipped}\n\n[full output persisted — call ${OFFLOAD_TOOL_NAME} with ref "${handle.spillRef}" to read it back.]`;
|
|
1038
|
+
}
|
|
1039
|
+
export async function pollBackgroundAgentLane(handle, deadline, signal, oneShot, store, sessionId) {
|
|
1026
1040
|
while (handle.status === "running" && deadline !== undefined && Date.now() < deadline && !signal?.aborted) {
|
|
1027
1041
|
await sleepPollStep(deadline, signal);
|
|
1028
1042
|
}
|
|
@@ -1041,6 +1055,8 @@ The agent is durably suspended, waiting for an approval decision. It resumes whe
|
|
|
1041
1055
|
},
|
|
1042
1056
|
};
|
|
1043
1057
|
}
|
|
1058
|
+
const fullResult = handle.result ? (handle.resultFull ?? handle.result) : undefined;
|
|
1059
|
+
const resultText = fullResult !== undefined ? await spillClippedAgentResult(handle, fullResult, clipTaskOutput(fullResult, handle.outputFile), store, sessionId) : undefined;
|
|
1044
1060
|
const body = running
|
|
1045
1061
|
? oneShot === true
|
|
1046
1062
|
? `status: running
|
|
@@ -1050,7 +1066,7 @@ The agent is still working — you will be notified when it completes.`
|
|
|
1050
1066
|
: `status: ${handle.status}
|
|
1051
1067
|
${handle.error ? `error: ${handle.error}
|
|
1052
1068
|
` : ""}${handle.result ? `--- result${handle.resultIsPartial ? " (partial — produced before the task was stopped)" : ""} ---
|
|
1053
|
-
${
|
|
1069
|
+
${resultText}` : "(no result text)"}`;
|
|
1054
1070
|
return {
|
|
1055
1071
|
content: delimitUntrusted(`TaskOutput ${handle.id}`, body),
|
|
1056
1072
|
details: {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { delimitUntrusted } from "./untrusted-text.js";
|
|
2
|
-
import { assertOwnership, defaultMonitorTimers, MONITOR_MAX_TIMEOUT_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_BATCH_WINDOW_MS, MONITOR_MAX_BATCHES_PER_MINUTE, MONITOR_STORM_BURST, MONITOR_STORM_KILL_AFTER_MS, MONITOR_LINE_BUF_CAP, TASK_OUTPUT_MAX_CHARS, mintCompletionId, clipMonitorEvent, clipMonitorLine, terminalTaskSummary, accountDroppedBytes, rollSpoolText, statusFromBackground, droppedGapNote, firstDropNote, alreadyTerminalStopNote, clipTaskOutput, sleepPollStep, } from "./task-registry-shared.js";
|
|
2
|
+
import { assertOwnership, defaultMonitorTimers, MONITOR_MAX_TIMEOUT_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_BATCH_WINDOW_MS, MONITOR_MAX_BATCHES_PER_MINUTE, MONITOR_STORM_BURST, MONITOR_STORM_KILL_AFTER_MS, MONITOR_LINE_BUF_CAP, MONITOR_SPILL_CAP_BYTES, TASK_OUTPUT_MAX_CHARS, mintCompletionId, clipMonitorEvent, clipMonitorLine, terminalTaskSummary, accountDroppedBytes, rollSpoolText, statusFromBackground, droppedGapNote, firstDropNote, alreadyTerminalStopNote, clipTaskOutput, sleepPollStep, } from "./task-registry-shared.js";
|
|
3
|
+
import { buildToolResultRef, OFFLOAD_TOOL_NAME } from "./tool-result-store.js";
|
|
3
4
|
export function registerMonitorLane(core, input) {
|
|
4
5
|
assertOwnership(input, "registerMonitor");
|
|
5
6
|
const id = core.mintTaskId("monitor");
|
|
@@ -37,17 +38,66 @@ export function registerMonitorLane(core, input) {
|
|
|
37
38
|
...(timeoutMs !== undefined ? { timeoutMs, deadlineAt: timers.now() + timeoutMs } : {}),
|
|
38
39
|
timers,
|
|
39
40
|
...(input.onEvent !== undefined ? { onEvent: input.onEvent } : {}),
|
|
41
|
+
...(input.toolResultStore !== undefined ? { toolResultStore: input.toolResultStore } : {}),
|
|
42
|
+
...(input.sessionId !== undefined ? { spillSessionId: input.sessionId } : {}),
|
|
40
43
|
};
|
|
41
44
|
core.handles.set(id, handle);
|
|
42
45
|
startMonitorWatcherLane(core, id);
|
|
43
46
|
return id;
|
|
44
47
|
}
|
|
48
|
+
function spillRolledMonitorChunk(handle, stream, dropped) {
|
|
49
|
+
const store = handle.toolResultStore;
|
|
50
|
+
if (store === undefined)
|
|
51
|
+
return;
|
|
52
|
+
const used = handle.spillBytesUsed ?? 0;
|
|
53
|
+
if (used >= MONITOR_SPILL_CAP_BYTES) {
|
|
54
|
+
handle.spillCapped = true;
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const n = stream === "out" ? (handle.spillSegCount ?? 0) : (handle.spillErrSegCount ?? 0);
|
|
58
|
+
const ref = buildToolResultRef(handle.spillSessionId ?? "no-session", `${handle.id}_${stream}_seg${n}`);
|
|
59
|
+
handle.spillBytesUsed = used + dropped.length;
|
|
60
|
+
if (stream === "out")
|
|
61
|
+
handle.spillSegCount = n + 1;
|
|
62
|
+
else
|
|
63
|
+
handle.spillErrSegCount = n + 1;
|
|
64
|
+
try {
|
|
65
|
+
void Promise.resolve(store.put(ref, dropped)).catch(() => {
|
|
66
|
+
handle.spillFailed = true;
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
handle.spillFailed = true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function monitorSpillNote(handle) {
|
|
74
|
+
const outN = handle.spillSegCount ?? 0;
|
|
75
|
+
const errN = handle.spillErrSegCount ?? 0;
|
|
76
|
+
if (outN === 0 && errN === 0)
|
|
77
|
+
return "";
|
|
78
|
+
const sid = handle.spillSessionId ?? "no-session";
|
|
79
|
+
const segLabel = (stream, n) => {
|
|
80
|
+
const first = buildToolResultRef(sid, `${handle.id}_${stream}_seg0`);
|
|
81
|
+
return n <= 1 ? `ref "${first}"` : `refs "${first}" .. "${buildToolResultRef(sid, `${handle.id}_${stream}_seg${n - 1}`)}"`;
|
|
82
|
+
};
|
|
83
|
+
const clauses = [];
|
|
84
|
+
if (outN > 0)
|
|
85
|
+
clauses.push(`stdout ${segLabel("out", outN)}`);
|
|
86
|
+
if (errN > 0)
|
|
87
|
+
clauses.push(`stderr ${segLabel("err", errN)}`);
|
|
88
|
+
const coverage = handle.spillFailed === true
|
|
89
|
+
? " — a write failed partway through; the ref chain may be INCOMPLETE, read what is there"
|
|
90
|
+
: handle.spillCapped === true
|
|
91
|
+
? ` — spill cap (${MONITOR_SPILL_CAP_BYTES} bytes) reached; earlier segments retained, later rolls were not spilled`
|
|
92
|
+
: "";
|
|
93
|
+
return `; spilled to ${clauses.join(", ")} (read back via ${OFFLOAD_TOOL_NAME})${coverage}`;
|
|
94
|
+
}
|
|
45
95
|
export function absorbMonitorPollLane(handle, v) {
|
|
46
96
|
const spool = handle.spool;
|
|
47
97
|
accountDroppedBytes(spool, v);
|
|
48
98
|
const ROLL_CAP = 2 * TASK_OUTPUT_MAX_CHARS;
|
|
49
|
-
spool.stdout = rollSpoolText(spool, spool.stdout + v.stdout, ROLL_CAP);
|
|
50
|
-
spool.stderr = rollSpoolText(spool, spool.stderr + v.stderr, ROLL_CAP);
|
|
99
|
+
spool.stdout = rollSpoolText(spool, spool.stdout + v.stdout, ROLL_CAP, (dropped) => spillRolledMonitorChunk(handle, "out", dropped));
|
|
100
|
+
spool.stderr = rollSpoolText(spool, spool.stderr + v.stderr, ROLL_CAP, (dropped) => spillRolledMonitorChunk(handle, "err", dropped));
|
|
51
101
|
handle.lineBuf += v.stdout;
|
|
52
102
|
if (handle.lineBuf.length > MONITOR_LINE_BUF_CAP)
|
|
53
103
|
handle.lineBuf = handle.lineBuf.slice(-MONITOR_LINE_BUF_CAP);
|
|
@@ -264,7 +314,7 @@ export async function pollMonitorLane(handle, filter, deadline, signal) {
|
|
|
264
314
|
const retrieval = running && deadline !== undefined && Date.now() >= deadline ? "timeout" : "success";
|
|
265
315
|
const timedOutClause = spool.timedOut === true ? ` (timed out after ${spool.timeoutSec !== undefined ? `${spool.timeoutSec}s` : "its background time budget"})` : "";
|
|
266
316
|
const displayStatus = (spool.exitCode !== undefined ? `exited(code ${spool.exitCode})` : running ? "running" : handle.status) + timedOutClause;
|
|
267
|
-
const rolledNote = spool.rolledChars > 0 ? `[!] ${spool.rolledChars} earlier char(s) dropped from the spool (memory bound)\n` : "";
|
|
317
|
+
const rolledNote = spool.rolledChars > 0 ? `[!] ${spool.rolledChars} earlier char(s) dropped from the spool (memory bound)${monitorSpillNote(handle)}\n` : "";
|
|
268
318
|
const droppedNote = (spool.droppedBytes ?? 0) > 0
|
|
269
319
|
? `[!] ${spool.droppedBytes} earlier byte(s) permanently dropped before this window (8MB buffer evicted)\n`
|
|
270
320
|
: spool.dropUnknown === true
|
|
@@ -299,6 +349,21 @@ function consumePendingSuppressionNote(handle) {
|
|
|
299
349
|
handle.suppressedBatches = 0;
|
|
300
350
|
return ` [${suppressed} event batch(es) suppressed before the watch ended — output rate was too high; the suppressed output is still readable via TaskOutput.]`;
|
|
301
351
|
}
|
|
352
|
+
function explicitStopTerminalNote(handle, alreadyGone) {
|
|
353
|
+
const label = (handle.description ?? "monitor").slice(0, 200);
|
|
354
|
+
const detail = alreadyGone
|
|
355
|
+
? "killed before completion (stopped via TaskStop; the process was already gone); watch ended."
|
|
356
|
+
: "killed before completion (stopped via TaskStop); watch ended.";
|
|
357
|
+
emitMonitorEventLane(handle, {
|
|
358
|
+
task_id: handle.id,
|
|
359
|
+
task_type: "monitor",
|
|
360
|
+
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
|
|
361
|
+
status: "killed",
|
|
362
|
+
stoppedBy: handle.stoppedBy,
|
|
363
|
+
summary: `${terminalTaskSummary("monitor", label, "killed")} — ${detail}${droppedGapNote(handle.spool)}`,
|
|
364
|
+
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
|
|
365
|
+
});
|
|
366
|
+
}
|
|
302
367
|
export async function stopMonitorLane(core, handle) {
|
|
303
368
|
const wasRunning = handle.status === "running";
|
|
304
369
|
if (handle.watcher !== undefined) {
|
|
@@ -358,7 +423,7 @@ export async function stopMonitorLane(core, handle) {
|
|
|
358
423
|
handle.status = "killed";
|
|
359
424
|
handle.updatedAt = Date.now();
|
|
360
425
|
mintCompletionId(handle);
|
|
361
|
-
core.
|
|
426
|
+
core.notifyTerminalOnce(handle, () => explicitStopTerminalNote(handle, true));
|
|
362
427
|
return {
|
|
363
428
|
content: `Terminated ${handle.id} (the process was already gone).${drainNote}${consumePendingSuppressionNote(handle)}`,
|
|
364
429
|
details: {
|
|
@@ -386,7 +451,7 @@ export async function stopMonitorLane(core, handle) {
|
|
|
386
451
|
handle.status = "killed";
|
|
387
452
|
handle.updatedAt = Date.now();
|
|
388
453
|
mintCompletionId(handle);
|
|
389
|
-
core.
|
|
454
|
+
core.notifyTerminalOnce(handle, () => explicitStopTerminalNote(handle, false));
|
|
390
455
|
}
|
|
391
456
|
return {
|
|
392
457
|
content: `Terminated ${handle.id}.${drainNote}${consumePendingSuppressionNote(handle)}`,
|
|
@@ -4,6 +4,7 @@ import { type WorkflowRunStore } from "./workflow-run-store.js";
|
|
|
4
4
|
import type { BackgroundAgentRecord, BackgroundAgentStore } from "./background-agent-store.js";
|
|
5
5
|
import type { WorkflowHandle, WorkflowRun } from "../orchestration/workflow.js";
|
|
6
6
|
import type { TaskNotificationPayload } from "./task-notification.js";
|
|
7
|
+
import type { ToolResultStore } from "./tool-result-store.js";
|
|
7
8
|
export type SemaTaskType = "background_bash" | "workflow" | "background_agent" | "monitor";
|
|
8
9
|
export type StopSource = "user" | "parent" | "system" | (string & {});
|
|
9
10
|
export type SemaTaskStatus = "pending" | "running" | "parked" | "completed" | "failed" | "killed" | "cancelled";
|
|
@@ -132,6 +133,7 @@ export interface BackgroundAgentTaskHandle extends SemaTaskHandle {
|
|
|
132
133
|
sessionScoped?: true;
|
|
133
134
|
result?: string;
|
|
134
135
|
resultFull?: string;
|
|
136
|
+
spillRef?: string;
|
|
135
137
|
error?: string;
|
|
136
138
|
resultIsPartial?: boolean;
|
|
137
139
|
stopSource?: StopSource;
|
|
@@ -182,6 +184,13 @@ export interface MonitorTaskHandle extends SemaTaskHandle {
|
|
|
182
184
|
stopSource?: StopSource;
|
|
183
185
|
stoppedBy?: StopSource;
|
|
184
186
|
terminalNotified?: true;
|
|
187
|
+
toolResultStore?: ToolResultStore;
|
|
188
|
+
spillSessionId?: string;
|
|
189
|
+
spillSegCount?: number;
|
|
190
|
+
spillErrSegCount?: number;
|
|
191
|
+
spillBytesUsed?: number;
|
|
192
|
+
spillCapped?: true;
|
|
193
|
+
spillFailed?: true;
|
|
185
194
|
}
|
|
186
195
|
export type RegisteredTaskHandle = BackgroundBashTaskHandle | WorkflowTaskHandle | BackgroundAgentTaskHandle | MonitorTaskHandle;
|
|
187
196
|
export interface RegisterMonitorInput extends TaskAccess {
|
|
@@ -198,6 +207,7 @@ export interface RegisterMonitorInput extends TaskAccess {
|
|
|
198
207
|
maxBatchesPerMinute?: number;
|
|
199
208
|
stormBurst?: number;
|
|
200
209
|
now?: number;
|
|
210
|
+
toolResultStore?: ToolResultStore;
|
|
201
211
|
}
|
|
202
212
|
export interface RegisterWorkflowInput extends TaskAccess {
|
|
203
213
|
runId: string;
|
|
@@ -226,7 +236,8 @@ export declare function clipTaskOutput(s: string, fullOutputPath?: string): stri
|
|
|
226
236
|
export declare function statusFromBackground(status: string, exitCode?: number): SemaTaskStatus;
|
|
227
237
|
export declare function rollSpoolText(spool: {
|
|
228
238
|
rolledChars: number;
|
|
229
|
-
}, s: string, cap: number): string;
|
|
239
|
+
}, s: string, cap: number, onDrop?: (dropped: string) => void): string;
|
|
240
|
+
export declare const MONITOR_SPILL_CAP_BYTES: number;
|
|
230
241
|
export declare function accountDroppedBytes(spool: {
|
|
231
242
|
droppedBytes?: number;
|
|
232
243
|
dropUnknown?: true;
|
|
@@ -85,13 +85,16 @@ export function statusFromBackground(status, exitCode) {
|
|
|
85
85
|
return exitCode === 0 ? "completed" : "failed";
|
|
86
86
|
return "failed";
|
|
87
87
|
}
|
|
88
|
-
export function rollSpoolText(spool, s, cap) {
|
|
88
|
+
export function rollSpoolText(spool, s, cap, onDrop) {
|
|
89
89
|
if (s.length <= cap)
|
|
90
90
|
return s;
|
|
91
91
|
const half = Math.floor(cap / 2);
|
|
92
|
-
|
|
92
|
+
const dropped = s.slice(half, s.length - half);
|
|
93
|
+
spool.rolledChars += dropped.length;
|
|
94
|
+
onDrop?.(dropped);
|
|
93
95
|
return s.slice(0, half) + s.slice(s.length - half);
|
|
94
96
|
}
|
|
97
|
+
export const MONITOR_SPILL_CAP_BYTES = 64 * 1024 * 1024;
|
|
95
98
|
export function accountDroppedBytes(spool, poll) {
|
|
96
99
|
const d = poll.bytesDroppedBeforeCursor ?? 0;
|
|
97
100
|
if (d > 0)
|
|
@@ -4,6 +4,7 @@ import { type BackgroundShellCapability, type BackgroundShellId } from "./backgr
|
|
|
4
4
|
import type { WorkflowRunStore } from "./workflow-run-store.js";
|
|
5
5
|
import { type BackgroundAgentStore } from "./background-agent-store.js";
|
|
6
6
|
import type { TaskNotificationPayload } from "./task-notification.js";
|
|
7
|
+
import type { ToolResultStore } from "./tool-result-store.js";
|
|
7
8
|
import { type SemaTaskType, type StopSource, type SemaTaskStatus, type TaskRetrievalStatus, type UnifiedTaskOutput, type UnifiedTaskResult, type SemaTaskHandle, type TaskAccess, type BackgroundAgentTaskHandle, type MonitorTimers, type RegisterMonitorInput, type RegisterWorkflowInput, canAccessWorkflowRun, clipTaskOutput, MONITOR_BATCH_WINDOW_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_MAX_TIMEOUT_MS, MONITOR_MAX_BATCHES_PER_MINUTE } from "./task-registry-shared.js";
|
|
8
9
|
export { normalizeAgentName, DURABLE_AGENT_HEARTBEAT_MS, DURABLE_AGENT_HANDLE_RE, BG_AGENT_REAP_STOP_ERROR } from "./task-registry-shared.js";
|
|
9
10
|
export type { ParkedClaimTicket, RegisterBackgroundAgentInput } from "./task-registry-shared.js";
|
|
@@ -33,6 +34,7 @@ export interface TaskPollOptions {
|
|
|
33
34
|
timeoutMs?: number;
|
|
34
35
|
signal?: AbortSignal;
|
|
35
36
|
oneShot?: boolean;
|
|
37
|
+
toolResultStore?: ToolResultStore;
|
|
36
38
|
}
|
|
37
39
|
export interface TaskStopOptions {
|
|
38
40
|
workflowStore?: WorkflowRunStore;
|
|
@@ -45,6 +47,7 @@ export interface TaskToolOptions extends TaskAccess {
|
|
|
45
47
|
deadlineMs?: () => number | undefined;
|
|
46
48
|
notificationWired?: boolean;
|
|
47
49
|
oneShot?: boolean;
|
|
50
|
+
toolResultStore?: ToolResultStore;
|
|
48
51
|
}
|
|
49
52
|
export interface AccessibleTaskRow {
|
|
50
53
|
task_id: string;
|
|
@@ -151,8 +151,8 @@ export class TaskRegistry {
|
|
|
151
151
|
runningBackgroundAgentLabels(access) {
|
|
152
152
|
return runningBackgroundAgentLabelsLane(this.core, access);
|
|
153
153
|
}
|
|
154
|
-
async pollBackgroundAgent(handle, deadline, signal, oneShot) {
|
|
155
|
-
return pollBackgroundAgentLane(handle, deadline, signal, oneShot);
|
|
154
|
+
async pollBackgroundAgent(handle, deadline, signal, oneShot, toolResultStore, sessionId) {
|
|
155
|
+
return pollBackgroundAgentLane(handle, deadline, signal, oneShot, toolResultStore, sessionId);
|
|
156
156
|
}
|
|
157
157
|
async stopBackgroundAgent(handle) {
|
|
158
158
|
return stopBackgroundAgentLane(this.core, handle);
|
|
@@ -748,7 +748,7 @@ export class TaskRegistry {
|
|
|
748
748
|
if (handle.type === "background_bash")
|
|
749
749
|
return this.pollBackgroundBash(handle, taskId, access, opts.filter, deadline, opts.signal);
|
|
750
750
|
if (handle.type === "background_agent")
|
|
751
|
-
return this.pollBackgroundAgent(handle, deadline, opts.signal, opts.oneShot);
|
|
751
|
+
return this.pollBackgroundAgent(handle, deadline, opts.signal, opts.oneShot, opts.toolResultStore, access.sessionId);
|
|
752
752
|
if (handle.type === "monitor")
|
|
753
753
|
return this.pollMonitor(handle, opts.filter, deadline, opts.signal);
|
|
754
754
|
return this.pollWorkflow(handle, access, deadline, opts.signal);
|
|
@@ -1214,6 +1214,7 @@ export function createTaskOutputTool(opts) {
|
|
|
1214
1214
|
timeoutMs: effectiveTimeoutMs,
|
|
1215
1215
|
signal: ctx.signal,
|
|
1216
1216
|
oneShot: opts.oneShot,
|
|
1217
|
+
toolResultStore: opts.toolResultStore,
|
|
1217
1218
|
});
|
|
1218
1219
|
const { type: taskType, ...rest } = r.details;
|
|
1219
1220
|
return {
|
package/dist/core/tools.d.ts
CHANGED
|
@@ -6,4 +6,6 @@ export declare function errorResult(text: string, details?: unknown): {
|
|
|
6
6
|
isError: true;
|
|
7
7
|
details?: unknown;
|
|
8
8
|
};
|
|
9
|
+
export declare function isDefineToolProduct(x: unknown): x is AgentTool;
|
|
10
|
+
export declare function stampDefineToolBrand<T extends object>(tool: T): T;
|
|
9
11
|
export declare function defineTool<TParams extends TSchema = TSchema>(spec: ToolSpec<TParams>): AgentTool<TParams>;
|
package/dist/core/tools.js
CHANGED
|
@@ -25,6 +25,14 @@ function isEmptyToolContent(content) {
|
|
|
25
25
|
export function errorResult(text, details) {
|
|
26
26
|
return details === undefined ? { content: text, isError: true } : { content: text, isError: true, details };
|
|
27
27
|
}
|
|
28
|
+
const DEFINE_TOOL_BRAND = Symbol("sema.core.defineTool.product");
|
|
29
|
+
export function isDefineToolProduct(x) {
|
|
30
|
+
return typeof x === "object" && x !== null && x[DEFINE_TOOL_BRAND] === true;
|
|
31
|
+
}
|
|
32
|
+
export function stampDefineToolBrand(tool) {
|
|
33
|
+
Object.defineProperty(tool, DEFINE_TOOL_BRAND, { value: true, enumerable: false });
|
|
34
|
+
return tool;
|
|
35
|
+
}
|
|
28
36
|
export function defineTool(spec) {
|
|
29
37
|
const executionMode = spec.executionMode ?? (spec.effect === "read" ? "parallel" : "sequential");
|
|
30
38
|
const tool = {
|
|
@@ -72,5 +80,6 @@ export function defineTool(spec) {
|
|
|
72
80
|
...(spec.aliases && spec.aliases.length > 0 ? { aliases: spec.aliases } : {}),
|
|
73
81
|
});
|
|
74
82
|
}
|
|
83
|
+
stampDefineToolBrand(tool);
|
|
75
84
|
return tool;
|
|
76
85
|
}
|
|
@@ -26,8 +26,24 @@ export declare function oversizeJournalResult(serialized: string): boolean;
|
|
|
26
26
|
export declare function callKeyOrdinal(callKey: string): number;
|
|
27
27
|
export declare const JOURNAL_OVERSIZE_ERROR_CODE = "workflow.journal_oversize";
|
|
28
28
|
export declare function journalOversizeTombstone(result: TaskResult, bytes: number): TaskResult;
|
|
29
|
+
export declare const IN_MEMORY_RESUME_CLAIM_TTL_MS: number;
|
|
29
30
|
export declare class InMemoryWorkflowJournalStore implements WorkflowJournalStore {
|
|
30
31
|
private readonly runs;
|
|
32
|
+
private readonly resumeClaims;
|
|
33
|
+
private claimsForScope;
|
|
34
|
+
resumeClaim(input: {
|
|
35
|
+
sourceRunId: string;
|
|
36
|
+
newRunId: string;
|
|
37
|
+
scope: string;
|
|
38
|
+
}): Promise<{
|
|
39
|
+
granted: boolean;
|
|
40
|
+
holder?: string;
|
|
41
|
+
}>;
|
|
42
|
+
releaseResumeClaim(input: {
|
|
43
|
+
sourceRunId: string;
|
|
44
|
+
newRunId: string;
|
|
45
|
+
scope: string;
|
|
46
|
+
}): Promise<void>;
|
|
31
47
|
load(runId: string, scope: string): Promise<WorkflowJournalEntry[]>;
|
|
32
48
|
append(runId: string, scope: string, entry: WorkflowJournalEntry): Promise<void>;
|
|
33
49
|
}
|
|
@@ -29,8 +29,36 @@ function snapshot(entry) {
|
|
|
29
29
|
return JSON.parse(JSON.stringify(entry));
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
|
+
export const IN_MEMORY_RESUME_CLAIM_TTL_MS = 60 * 60 * 1000;
|
|
32
33
|
export class InMemoryWorkflowJournalStore {
|
|
33
34
|
runs = new Map();
|
|
35
|
+
resumeClaims = new Map();
|
|
36
|
+
claimsForScope(scope) {
|
|
37
|
+
let m = this.resumeClaims.get(scope);
|
|
38
|
+
if (m === undefined) {
|
|
39
|
+
m = new Map();
|
|
40
|
+
this.resumeClaims.set(scope, m);
|
|
41
|
+
}
|
|
42
|
+
return m;
|
|
43
|
+
}
|
|
44
|
+
async resumeClaim(input) {
|
|
45
|
+
const table = this.claimsForScope(input.scope);
|
|
46
|
+
const now = Date.now();
|
|
47
|
+
const existing = table.get(input.sourceRunId);
|
|
48
|
+
if (existing !== undefined && existing.expiresAt > now && existing.holder !== input.newRunId) {
|
|
49
|
+
return { granted: false, holder: existing.holder };
|
|
50
|
+
}
|
|
51
|
+
table.set(input.sourceRunId, { holder: input.newRunId, expiresAt: now + IN_MEMORY_RESUME_CLAIM_TTL_MS });
|
|
52
|
+
return { granted: true };
|
|
53
|
+
}
|
|
54
|
+
async releaseResumeClaim(input) {
|
|
55
|
+
const table = this.resumeClaims.get(input.scope);
|
|
56
|
+
if (table === undefined)
|
|
57
|
+
return;
|
|
58
|
+
const existing = table.get(input.sourceRunId);
|
|
59
|
+
if (existing !== undefined && existing.holder === input.newRunId)
|
|
60
|
+
table.delete(input.sourceRunId);
|
|
61
|
+
}
|
|
34
62
|
async load(runId, scope) {
|
|
35
63
|
const rec = this.runs.get(runId);
|
|
36
64
|
if (!rec || rec.scope !== scope)
|
|
@@ -129,5 +129,20 @@ export interface WorkflowTimers {
|
|
|
129
129
|
setTimeout(fn: () => void, ms: number): unknown;
|
|
130
130
|
clearTimeout(handle: unknown): void;
|
|
131
131
|
}
|
|
132
|
+
export declare const workflowResumeClaimFallback: {
|
|
133
|
+
acquire(store: WorkflowJournalStore, input: {
|
|
134
|
+
sourceRunId: string;
|
|
135
|
+
newRunId: string;
|
|
136
|
+
scope: string;
|
|
137
|
+
}): {
|
|
138
|
+
granted: boolean;
|
|
139
|
+
holder?: string;
|
|
140
|
+
};
|
|
141
|
+
release(store: WorkflowJournalStore, input: {
|
|
142
|
+
sourceRunId: string;
|
|
143
|
+
newRunId: string;
|
|
144
|
+
scope: string;
|
|
145
|
+
}): void;
|
|
146
|
+
};
|
|
132
147
|
export declare function startWorkflow<T>(runner: Runner, fn: (ctx: WorkflowRunContext) => Promise<T>, opts?: RunWorkflowOptions, internals?: WorkflowInternals): WorkflowHandle<T>;
|
|
133
148
|
export declare function runWorkflow<T>(runner: Runner, fn: (ctx: WorkflowRunContext) => Promise<T>, opts?: RunWorkflowOptions, internals?: WorkflowInternals): Promise<RunWorkflowResult<T>>;
|
|
@@ -20,6 +20,7 @@ const MAX_TRANSCRIPT_CHARS = 4000;
|
|
|
20
20
|
const WORKFLOW_RESULT_MAX = 4000;
|
|
21
21
|
const WORKFLOW_RESULT_FULL_MAX = 200_000;
|
|
22
22
|
const MAX_ACTIVITY = 30;
|
|
23
|
+
const RUNNING_AGENT_PERSIST_EVERY_BEATS = 4;
|
|
23
24
|
function workflowModelLabel(spec) {
|
|
24
25
|
const model = spec.model;
|
|
25
26
|
if (model === undefined)
|
|
@@ -206,6 +207,33 @@ function createSemaphore(max) {
|
|
|
206
207
|
function clampRunIdText(raw) {
|
|
207
208
|
return raw.replace(/[^A-Za-z0-9_.:-]/g, "").slice(0, 64);
|
|
208
209
|
}
|
|
210
|
+
const fallbackResumeClaimTables = new WeakMap();
|
|
211
|
+
function fallbackResumeClaimKey(scope, sourceRunId) {
|
|
212
|
+
return JSON.stringify([scope, sourceRunId]);
|
|
213
|
+
}
|
|
214
|
+
export const workflowResumeClaimFallback = {
|
|
215
|
+
acquire(store, input) {
|
|
216
|
+
let table = fallbackResumeClaimTables.get(store);
|
|
217
|
+
if (table === undefined) {
|
|
218
|
+
table = new Map();
|
|
219
|
+
fallbackResumeClaimTables.set(store, table);
|
|
220
|
+
}
|
|
221
|
+
const key = fallbackResumeClaimKey(input.scope, input.sourceRunId);
|
|
222
|
+
const holder = table.get(key);
|
|
223
|
+
if (holder !== undefined && holder !== input.newRunId)
|
|
224
|
+
return { granted: false, holder };
|
|
225
|
+
table.set(key, input.newRunId);
|
|
226
|
+
return { granted: true };
|
|
227
|
+
},
|
|
228
|
+
release(store, input) {
|
|
229
|
+
const table = fallbackResumeClaimTables.get(store);
|
|
230
|
+
if (table === undefined)
|
|
231
|
+
return;
|
|
232
|
+
const key = fallbackResumeClaimKey(input.scope, input.sourceRunId);
|
|
233
|
+
if (table.get(key) === input.newRunId)
|
|
234
|
+
table.delete(key);
|
|
235
|
+
},
|
|
236
|
+
};
|
|
209
237
|
export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
210
238
|
const alsDepth = currentWorkflowDepth();
|
|
211
239
|
const internalDepth = internals?.workflowDepth;
|
|
@@ -566,8 +594,9 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
566
594
|
throw err;
|
|
567
595
|
}
|
|
568
596
|
};
|
|
569
|
-
const createActivityCapture = (callKey, label, groupId) => {
|
|
597
|
+
const createActivityCapture = (callKey, label, groupId, rec) => {
|
|
570
598
|
const tail = [];
|
|
599
|
+
let beatCount = 0;
|
|
571
600
|
const onActivity = (a) => {
|
|
572
601
|
if (finalized)
|
|
573
602
|
return;
|
|
@@ -587,6 +616,13 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
587
616
|
tail.push(a);
|
|
588
617
|
if (tail.length > MAX_ACTIVITY)
|
|
589
618
|
tail.shift();
|
|
619
|
+
if (a.phase === "start")
|
|
620
|
+
rec.toolCalls = (rec.toolCalls ?? 0) + 1;
|
|
621
|
+
rec.activity = tail;
|
|
622
|
+
beatCount += 1;
|
|
623
|
+
if (beatCount === 1 || beatCount % RUNNING_AGENT_PERSIST_EVERY_BEATS === 0) {
|
|
624
|
+
void persist("update");
|
|
625
|
+
}
|
|
590
626
|
};
|
|
591
627
|
return { tail, onActivity };
|
|
592
628
|
};
|
|
@@ -734,8 +770,8 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
734
770
|
if (budgetTotal !== null && spent() >= budgetTotal) {
|
|
735
771
|
throw new WorkflowBudgetExceededError(spent(), budgetTotal);
|
|
736
772
|
}
|
|
737
|
-
const { tail: activityTail, onActivity } = createActivityCapture(callKey, label, groupId);
|
|
738
773
|
const rec = { label, callKey, ...(groupId !== undefined ? { groupId } : {}), phase, prompt, ...(model !== undefined ? { model } : {}), status: "running", queuedAt: now() };
|
|
774
|
+
const { tail: activityTail, onActivity } = createActivityCapture(callKey, label, groupId, rec);
|
|
739
775
|
run.agents.push(rec);
|
|
740
776
|
if (phaseInstance)
|
|
741
777
|
agentPhaseOf.set(rec, phaseInstance);
|
|
@@ -1047,8 +1083,8 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1047
1083
|
const model = workflowModelLabel(specForIdentity);
|
|
1048
1084
|
noteDivergence(run.agents.length, "ctx.agentStream results are never replayed");
|
|
1049
1085
|
diverged = true;
|
|
1050
|
-
const { tail: activityTail, onActivity } = createActivityCapture(callKey, label, groupId);
|
|
1051
1086
|
const rec = { label, callKey, ...(groupId !== undefined ? { groupId } : {}), phase, prompt, ...(model !== undefined ? { model } : {}), status: "running", queuedAt: now() };
|
|
1087
|
+
const { tail: activityTail, onActivity } = createActivityCapture(callKey, label, groupId, rec);
|
|
1052
1088
|
run.agents.push(rec);
|
|
1053
1089
|
if (phaseInstance)
|
|
1054
1090
|
agentPhaseOf.set(rec, phaseInstance);
|
|
@@ -1388,18 +1424,19 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1388
1424
|
try {
|
|
1389
1425
|
const runBody = async () => {
|
|
1390
1426
|
if (opts.resumeFromRunId !== undefined && journalStore) {
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1427
|
+
const usingFallback = journalStore.resumeClaim === undefined;
|
|
1428
|
+
const verdict = usingFallback
|
|
1429
|
+
? Promise.resolve(workflowResumeClaimFallback.acquire(journalStore, { sourceRunId: opts.resumeFromRunId, newRunId: runId, scope }))
|
|
1430
|
+
: journalStore.resumeClaim({ sourceRunId: opts.resumeFromRunId, newRunId: runId, scope });
|
|
1431
|
+
resumeClaim = { sourceRunId: opts.resumeFromRunId, verdict, fallback: usingFallback };
|
|
1432
|
+
const decision = await verdict;
|
|
1433
|
+
if (!decision.granted) {
|
|
1434
|
+
const holder = decision.holder !== undefined ? clampRunIdText(decision.holder) : "";
|
|
1435
|
+
throw new Error(`startWorkflow: resume from "${clampRunIdText(opts.resumeFromRunId)}" was REFUSED — ` +
|
|
1436
|
+
`another run already holds the resume claim on it${holder ? ` (holder: ${holder})` : " (holder unknown to the store)"}. ` +
|
|
1437
|
+
"Two concurrent resumes of one source run fork its execution: both replay the same prefix and then " +
|
|
1438
|
+
"re-run the whole suffix live, duplicating every side effect. Wait for the holder to reach a terminal " +
|
|
1439
|
+
"state (or stop it) and resume again.");
|
|
1403
1440
|
}
|
|
1404
1441
|
const entries = await journalStore.load(opts.resumeFromRunId, scope);
|
|
1405
1442
|
for (const e of entries)
|
|
@@ -1501,13 +1538,20 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1501
1538
|
finally {
|
|
1502
1539
|
finalized = true;
|
|
1503
1540
|
await journalTail.catch(() => undefined);
|
|
1504
|
-
if (resumeClaim !== undefined
|
|
1541
|
+
if (resumeClaim !== undefined) {
|
|
1505
1542
|
const granted = await finalizeWithin(timers, resumeClaim.verdict.then((v) => v.granted, () => false), false);
|
|
1506
1543
|
if (granted) {
|
|
1507
|
-
|
|
1508
|
-
|
|
1544
|
+
if (resumeClaim.fallback) {
|
|
1545
|
+
if (journalStore) {
|
|
1546
|
+
workflowResumeClaimFallback.release(journalStore, { sourceRunId: resumeClaim.sourceRunId, newRunId: runId, scope });
|
|
1547
|
+
}
|
|
1509
1548
|
}
|
|
1510
|
-
|
|
1549
|
+
else if (journalStore?.releaseResumeClaim) {
|
|
1550
|
+
try {
|
|
1551
|
+
await finalizeWithin(timers, journalStore.releaseResumeClaim({ sourceRunId: resumeClaim.sourceRunId, newRunId: runId, scope }), undefined);
|
|
1552
|
+
}
|
|
1553
|
+
catch {
|
|
1554
|
+
}
|
|
1511
1555
|
}
|
|
1512
1556
|
}
|
|
1513
1557
|
}
|
package/dist/tools/monitor.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AgentTool, ExecutionEnv } from "../internal/harness-types.js";
|
|
2
2
|
import { type MonitorTimers, type TaskRegistry } from "../core/task-registry.js";
|
|
3
3
|
import type { TaskNotificationPayload } from "../core/task-notification.js";
|
|
4
|
+
import type { ToolResultStore } from "../core/tool-result-store.js";
|
|
4
5
|
export interface MonitorToolOptions {
|
|
5
6
|
registry: TaskRegistry;
|
|
6
7
|
owner?: string;
|
|
@@ -15,5 +16,6 @@ export interface MonitorToolOptions {
|
|
|
15
16
|
timers?: MonitorTimers;
|
|
16
17
|
batchWindowMs?: number;
|
|
17
18
|
maxBatchesPerMinute?: number;
|
|
19
|
+
toolResultStore?: ToolResultStore;
|
|
18
20
|
}
|
|
19
21
|
export declare function createMonitorTool(env: ExecutionEnv, opts: MonitorToolOptions): AgentTool;
|
package/dist/tools/monitor.js
CHANGED
|
@@ -82,6 +82,8 @@ export function createMonitorTool(env, opts) {
|
|
|
82
82
|
...(opts.timers !== undefined ? { timers: opts.timers } : {}),
|
|
83
83
|
...(opts.batchWindowMs !== undefined ? { batchWindowMs: opts.batchWindowMs } : {}),
|
|
84
84
|
...(opts.maxBatchesPerMinute !== undefined ? { maxBatchesPerMinute: opts.maxBatchesPerMinute } : {}),
|
|
85
|
+
...(opts.sessionId !== undefined ? { sessionId: opts.sessionId } : {}),
|
|
86
|
+
...(opts.toolResultStore !== undefined ? { toolResultStore: opts.toolResultStore } : {}),
|
|
85
87
|
});
|
|
86
88
|
}
|
|
87
89
|
catch (e) {
|