@inflexa-ai/harness 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/agents/conversation-agent.d.ts +7 -0
- package/dist/agents/conversation-agent.js +5 -3
- package/dist/app/synthesize-run.d.ts +3 -0
- package/dist/app/synthesize-run.js +2 -1
- package/dist/billing/noop-usage-recorder.d.ts +7 -0
- package/dist/billing/noop-usage-recorder.js +10 -0
- package/dist/billing/usage-recorder.d.ts +63 -0
- package/dist/billing/usage-recorder.js +15 -0
- package/dist/contracts/chat-events.d.ts +44 -3
- package/dist/contracts/chat-parts.d.ts +38 -2
- package/dist/contracts/chat-parts.js +1 -1
- package/dist/contracts/data-profile.d.ts +26 -0
- package/dist/contracts/data-profile.js +26 -0
- package/dist/contracts/index.d.ts +3 -1
- package/dist/contracts/index.js +1 -0
- package/dist/contracts/message.d.ts +21 -1
- package/dist/contracts/part-registry.js +1 -0
- package/dist/contracts/schemas/chat-events.d.ts +48 -2
- package/dist/contracts/schemas/chat-events.js +8 -1
- package/dist/contracts/schemas/chat-parts.d.ts +46 -3
- package/dist/contracts/schemas/chat-parts.js +13 -0
- package/dist/contracts/schemas/index.d.ts +2 -1
- package/dist/contracts/schemas/index.js +2 -1
- package/dist/contracts/schemas/usage.d.ts +11 -0
- package/dist/contracts/schemas/usage.js +18 -0
- package/dist/contracts/usage.d.ts +51 -0
- package/dist/contracts/usage.js +41 -0
- package/dist/execution/artifact-metadata.d.ts +3 -0
- package/dist/execution/artifact-metadata.js +1 -0
- package/dist/execution/dbos-workflow-purger.d.ts +30 -0
- package/dist/execution/dbos-workflow-purger.js +193 -0
- package/dist/execution/post-step-pipeline.d.ts +3 -0
- package/dist/execution/post-step-pipeline.js +2 -0
- package/dist/execution/reconcile-manifest.js +28 -15
- package/dist/execution/report-runner.d.ts +16 -0
- package/dist/execution/report-runner.js +4 -0
- package/dist/execution/run-synthesis.d.ts +3 -0
- package/dist/execution/run-synthesis.js +2 -0
- package/dist/execution/step-summary.d.ts +3 -0
- package/dist/execution/step-summary.js +1 -0
- package/dist/execution/workflow-purger.d.ts +38 -0
- package/dist/execution/workflow-purger.js +24 -0
- package/dist/index.d.ts +24 -2
- package/dist/index.js +27 -0
- package/dist/lib/metrics.js +2 -1
- package/dist/loop/metrics.d.ts +18 -0
- package/dist/loop/metrics.js +18 -1
- package/dist/loop/run-agent.d.ts +35 -0
- package/dist/loop/run-agent.js +149 -23
- package/dist/loop/run-to-terminal.d.ts +5 -2
- package/dist/loop/run-to-terminal.js +36 -2
- package/dist/loop/tool-detail.d.ts +54 -0
- package/dist/loop/tool-detail.js +105 -0
- package/dist/loop/tool-outcome.d.ts +21 -0
- package/dist/loop/tool-outcome.js +25 -0
- package/dist/loop/types.d.ts +15 -1
- package/dist/memory/content-to-cortex.d.ts +28 -4
- package/dist/memory/content-to-cortex.js +70 -14
- package/dist/memory/thread-history.d.ts +21 -1
- package/dist/memory/thread-history.js +29 -7
- package/dist/memory/thread-store.d.ts +65 -8
- package/dist/memory/thread-store.js +81 -17
- package/dist/provenance/collector.d.ts +5 -3
- package/dist/provenance/collector.js +5 -3
- package/dist/providers/ai-sdk.js +91 -16
- package/dist/providers/streaming-chat.js +5 -0
- package/dist/providers/types.d.ts +23 -0
- package/dist/providers/types.js +1 -0
- package/dist/runtime/assemble.d.ts +25 -8
- package/dist/runtime/assemble.js +7 -4
- package/dist/sandbox/sandbox-step-translate.d.ts +22 -1
- package/dist/sandbox/sandbox-step-translate.js +23 -36
- package/dist/schemas/run-synthesis.d.ts +2 -2
- package/dist/state/index.d.ts +2 -0
- package/dist/state/index.js +3 -0
- package/dist/state/init.js +28 -0
- package/dist/state/purge-analysis.d.ts +74 -0
- package/dist/state/purge-analysis.js +205 -0
- package/dist/state/vector-store.d.ts +6 -0
- package/dist/state/vector-store.js +22 -2
- package/dist/tasks/data-profile-activity.d.ts +14 -2
- package/dist/tasks/data-profile-activity.js +1 -1
- package/dist/tasks/data-profile.d.ts +3 -0
- package/dist/tasks/data-profile.js +9 -3
- package/dist/tools/bio/pubmed.js +15 -0
- package/dist/tools/bio/search-gene.js +12 -0
- package/dist/tools/define-tool.d.ts +41 -3
- package/dist/tools/define-tool.js +4 -1
- package/dist/tools/detail-resolver.d.ts +35 -0
- package/dist/tools/detail-resolver.js +39 -0
- package/dist/tools/execute-analysis.js +3 -0
- package/dist/tools/iterate-report.d.ts +3 -0
- package/dist/tools/iterate-report.js +3 -0
- package/dist/tools/memory/update-working-memory.js +8 -0
- package/dist/tools/research/generate-analogy-report.d.ts +3 -0
- package/dist/tools/research/generate-analogy-report.js +6 -0
- package/dist/tools/research/generate-plan.d.ts +3 -0
- package/dist/tools/research/generate-plan.js +6 -0
- package/dist/tools/research/inspect-run.js +4 -0
- package/dist/tools/research/literature-reviewer.d.ts +3 -0
- package/dist/tools/research/literature-reviewer.js +6 -0
- package/dist/tools/workspace/edit-file.js +2 -0
- package/dist/tools/workspace/execute-command.js +25 -0
- package/dist/tools/workspace/grep.js +3 -0
- package/dist/tools/workspace/list-files.js +2 -0
- package/dist/tools/workspace/read-file.js +7 -0
- package/dist/tools/workspace/workspace-search.js +1 -0
- package/dist/tools/workspace/write-file.js +3 -0
- package/dist/workflows/execute-analysis.d.ts +7 -0
- package/dist/workflows/execute-analysis.js +31 -2
- package/dist/workflows/execute-target-assessment.d.ts +3 -0
- package/dist/workflows/sandbox-step.d.ts +23 -0
- package/dist/workflows/sandbox-step.js +33 -2
- package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.d.ts +3 -0
- package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.js +1 -0
- package/package.json +1 -1
package/dist/loop/run-agent.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { jsonSchema, tool as aiTool } from "ai";
|
|
3
|
+
import { createNoopUsageRecorder } from "../billing/noop-usage-recorder.js";
|
|
2
4
|
import { stripNulCharacters } from "../input-sanitization.js";
|
|
3
5
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
4
6
|
import { hintForZodIssue, repairToolInput } from "../lib/zod-issues.js";
|
|
@@ -8,7 +10,9 @@ import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/p
|
|
|
8
10
|
import { resultStep } from "./run-step.js";
|
|
9
11
|
import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
|
|
10
12
|
import { isToolError } from "../tools/define-tool.js";
|
|
11
|
-
import { addChatUsage, recordAgentRun } from "./metrics.js";
|
|
13
|
+
import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
|
|
14
|
+
import { computeDetail } from "./tool-detail.js";
|
|
15
|
+
import { toolOutcomeForOutputType } from "./tool-outcome.js";
|
|
12
16
|
const TRUNCATED_PROSE_STEER = "Your previous reply was cut off at the output-token limit; continue concisely, or finish via your terminal tool.";
|
|
13
17
|
const TRUNCATED_TOOL_USE_ERROR = "Your previous tool call was cut off at the output-token limit and was not executed. Retry with a smaller payload, writing in incremental pieces.";
|
|
14
18
|
export const DEFAULT_STEP_NAME_FORMATTER = {
|
|
@@ -50,6 +54,11 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
50
54
|
// answer that cannot come.
|
|
51
55
|
const unavailableAsk = new UnavailableAsk();
|
|
52
56
|
const ask = opts.ask ?? ((request) => unavailableAsk.ask(request));
|
|
57
|
+
// A run handed no accumulator is the turn's root: it creates the one every
|
|
58
|
+
// descendant loop folds into, and it is the only loop entitled to report a
|
|
59
|
+
// turn total. Descendants receive the root's object and mutate it in place.
|
|
60
|
+
const turnUsage = opts.turnUsage ?? {};
|
|
61
|
+
const isTurnRoot = opts.turnUsage === undefined;
|
|
53
62
|
const toolCtx = (tu) => ({
|
|
54
63
|
invocationId: tu.toolCallId,
|
|
55
64
|
session,
|
|
@@ -57,6 +66,7 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
57
66
|
emit,
|
|
58
67
|
runStep: (name, fn) => runStep(`${formatStepName.tool(tu.toolName, tu.toolCallId)}:${name}`, fn),
|
|
59
68
|
ask,
|
|
69
|
+
turnUsage,
|
|
60
70
|
});
|
|
61
71
|
let iterations = 0;
|
|
62
72
|
let truncationRecoveries = 0;
|
|
@@ -77,6 +87,44 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
77
87
|
const logFinish = (level, reason, cappedOut) => {
|
|
78
88
|
log[level]("run finished", { iterations, reason, cappedOut, truncationRecoveries, toolCalls: toolCallCount, toolErrors: toolErrorCount, usage });
|
|
79
89
|
};
|
|
90
|
+
const usageRecorder = opts.usageRecorder ?? createNoopUsageRecorder();
|
|
91
|
+
/**
|
|
92
|
+
* Fold one completed call into both rollups and hand it to the recorder.
|
|
93
|
+
* Called at the fold point, with the reply in hand — before any branch that
|
|
94
|
+
* can end the run — so a call that completed is accounted for even when the
|
|
95
|
+
* run later aborts or dies.
|
|
96
|
+
*
|
|
97
|
+
* A call that reported nothing produces no record. Model ids are identity,
|
|
98
|
+
* not usage: a reply carrying only `requestedModelId`/`servedModelId` still
|
|
99
|
+
* reported nothing to account for, so it is folded (to no effect) and left
|
|
100
|
+
* out of the ledger rather than entered as an all-absent record.
|
|
101
|
+
*
|
|
102
|
+
* Delivery is bare — no `await`, no `try` — because the `UsageRecorder`
|
|
103
|
+
* contract forbids `record` to throw or block.
|
|
104
|
+
*/
|
|
105
|
+
const accountForCall = (reply, stepName) => {
|
|
106
|
+
addChatUsage(usage, reply.usage);
|
|
107
|
+
addChatUsage(turnUsage, reply.usage);
|
|
108
|
+
const reported = reply.usage;
|
|
109
|
+
if (reported === undefined || !hasReportedUsage(reported))
|
|
110
|
+
return;
|
|
111
|
+
usageRecorder.record({
|
|
112
|
+
recordKey: recordKeyFor(session, opts.invocationId, stepName),
|
|
113
|
+
agentId: source.agentId,
|
|
114
|
+
callPath: source.callPath,
|
|
115
|
+
scope: session.scope,
|
|
116
|
+
...(session.runFrame?.runId === undefined ? {} : { runId: session.runFrame.runId }),
|
|
117
|
+
...(session.runFrame?.stepId === undefined ? {} : { stepId: session.runFrame.stepId }),
|
|
118
|
+
...(reply.requestedModelId === undefined ? {} : { requestedModelId: reply.requestedModelId }),
|
|
119
|
+
...(reply.servedModelId === undefined ? {} : { servedModelId: reply.servedModelId }),
|
|
120
|
+
usage: reported,
|
|
121
|
+
});
|
|
122
|
+
};
|
|
123
|
+
/** The rollups this loop stamps on its finish — each absent when nothing reported. */
|
|
124
|
+
const finishUsage = () => ({
|
|
125
|
+
...(hasReportedUsage(usage) ? { usage: { ...usage } } : {}),
|
|
126
|
+
...(isTurnRoot && hasReportedUsage(turnUsage) ? { turnUsage: { ...turnUsage } } : {}),
|
|
127
|
+
});
|
|
80
128
|
// The user said no. A subsequent model call would only let the agent argue
|
|
81
129
|
// with the decision, or spend a call acknowledging it; the denial tool result
|
|
82
130
|
// is itself what the surface renders, so the turn ends the moment a denial
|
|
@@ -86,8 +134,19 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
86
134
|
await emit({ type: "iteration", source, index: i, final: true });
|
|
87
135
|
recordAgentRun({ agentId: agent.id, iterations, cappedOut: false, usage });
|
|
88
136
|
logFinish("warn", "denied", false);
|
|
89
|
-
return { messages, finish: { reason: "denied", cappedOut: false, truncationRecoveries } };
|
|
137
|
+
return { messages, finish: { reason: "denied", cappedOut: false, truncationRecoveries, ...finishUsage() } };
|
|
90
138
|
};
|
|
139
|
+
/**
|
|
140
|
+
* The call details for one dispatch round, positionally aligned with `calls`.
|
|
141
|
+
*
|
|
142
|
+
* Computed once per round and carried onto both the `tool-started` and the
|
|
143
|
+
* `tool-finished` event, so the pair a host renders as one chip cannot show
|
|
144
|
+
* two different descriptions of the same call.
|
|
145
|
+
*/
|
|
146
|
+
const roundDetails = (calls) => calls.map((tu) => {
|
|
147
|
+
const tool = toolsById.get(tu.toolName);
|
|
148
|
+
return tool === undefined ? undefined : computeDetail(tool, tu.input, log);
|
|
149
|
+
});
|
|
91
150
|
/**
|
|
92
151
|
* Fold one dispatch round into the run's counters and the event sink, returning the
|
|
93
152
|
* tools whose results the model will read as errors.
|
|
@@ -97,17 +156,20 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
97
156
|
* it is not a thrown failure, so nothing reports it and the loop simply feeds it
|
|
98
157
|
* back — and a run that ends badly is usually a run whose tool calls were failing.
|
|
99
158
|
*/
|
|
100
|
-
const settleRound = async (calls, results) => {
|
|
159
|
+
const settleRound = async (calls, results, details) => {
|
|
101
160
|
const errored = [];
|
|
102
161
|
for (let idx = 0; idx < calls.length; idx++) {
|
|
103
162
|
const tu = calls[idx];
|
|
104
|
-
const
|
|
163
|
+
const outcome = outcomeOf(results[idx]);
|
|
105
164
|
toolCallCount++;
|
|
106
|
-
|
|
165
|
+
// A denial counts here with the errors: the model reads it as a result
|
|
166
|
+
// it did not get to act on, which is what these counters describe. The
|
|
167
|
+
// event keeps the two apart for the user-facing surface.
|
|
168
|
+
if (outcome !== "ok") {
|
|
107
169
|
toolErrorCount++;
|
|
108
170
|
errored.push(tu.toolName);
|
|
109
171
|
}
|
|
110
|
-
await emit({ type: "tool-finished", source, toolUseId: tu.toolCallId, name: tu.toolName,
|
|
172
|
+
await emit({ type: "tool-finished", source, toolUseId: tu.toolCallId, name: tu.toolName, outcome, ...detailField(details[idx]) });
|
|
111
173
|
}
|
|
112
174
|
return errored;
|
|
113
175
|
};
|
|
@@ -125,8 +187,9 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
125
187
|
...(opts.toolChoice !== undefined ? { toolChoice: opts.toolChoice } : {}),
|
|
126
188
|
providerOptions,
|
|
127
189
|
};
|
|
128
|
-
const
|
|
129
|
-
|
|
190
|
+
const llmStepName = formatStepName.llm(i);
|
|
191
|
+
const reply = await resultStep(runStep)(llmStepName, () => provider.chat(request, session, signal));
|
|
192
|
+
accountForCall(reply, llmStepName);
|
|
130
193
|
if (reply.finishReason === "aborted") {
|
|
131
194
|
// An interrupted turn keeps whatever the model produced before the cut, but
|
|
132
195
|
// never an empty shell: a partial with no content adds no message, so a
|
|
@@ -159,11 +222,12 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
159
222
|
}
|
|
160
223
|
const trailing = toolCalls[toolCalls.length - 1];
|
|
161
224
|
const earlier = toolCalls.slice(0, -1);
|
|
162
|
-
|
|
163
|
-
|
|
225
|
+
const earlierDetails = roundDetails(earlier);
|
|
226
|
+
for (const [idx, tu] of earlier.entries()) {
|
|
227
|
+
await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(earlierDetails[idx]) });
|
|
164
228
|
}
|
|
165
229
|
const results = await dispatchTools(earlier, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool);
|
|
166
|
-
const errored = await settleRound(earlier, results);
|
|
230
|
+
const errored = await settleRound(earlier, results, earlierDetails);
|
|
167
231
|
results.push(errorResult(trailing, TRUNCATED_TOOL_USE_ERROR));
|
|
168
232
|
// The trailing call was never dispatched, but it reaches the model as an error
|
|
169
233
|
// result like any other — so it counts, or `toolErrors` reports a cleaner run
|
|
@@ -184,15 +248,16 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
184
248
|
await emit({ type: "iteration", source, index: i, final: true });
|
|
185
249
|
recordAgentRun({ agentId: agent.id, iterations, cappedOut: false, usage });
|
|
186
250
|
logFinish("info", reply.finishReason, false);
|
|
187
|
-
return { messages, finish: { reason: reply.finishReason, cappedOut: false, truncationRecoveries } };
|
|
251
|
+
return { messages, finish: { reason: reply.finishReason, cappedOut: false, truncationRecoveries, ...finishUsage() } };
|
|
188
252
|
}
|
|
189
253
|
await emit({ type: "iteration", source, index: i, final: false });
|
|
190
254
|
log.debug("iteration", { iteration: i, tools: toolCalls.map((t) => t.toolName) });
|
|
191
|
-
|
|
192
|
-
|
|
255
|
+
const details = roundDetails(toolCalls);
|
|
256
|
+
for (const [idx, tu] of toolCalls.entries()) {
|
|
257
|
+
await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(details[idx]) });
|
|
193
258
|
}
|
|
194
259
|
const results = await dispatchTools(toolCalls, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool);
|
|
195
|
-
const errored = await settleRound(toolCalls, results);
|
|
260
|
+
const errored = await settleRound(toolCalls, results, details);
|
|
196
261
|
if (errored.length > 0)
|
|
197
262
|
log.debug("tool results returned errors", { iteration: i, tools: errored });
|
|
198
263
|
messages.push({ role: "tool", content: results });
|
|
@@ -207,8 +272,9 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
207
272
|
// rewrites the whole prefix from scratch. It still carries the cache options
|
|
208
273
|
// because it is the one call whose write is pure waste, and the
|
|
209
274
|
// cache_write_tokens counter is what makes that waste visible.
|
|
210
|
-
const
|
|
211
|
-
|
|
275
|
+
const wrapUpStepName = formatStepName.llm(agent.maxIterations);
|
|
276
|
+
const wrapUp = await resultStep(runStep)(wrapUpStepName, () => provider.chat({ system: agent.systemPrompt, messages, tools: {}, toolChoice: "none", providerOptions }, session, signal));
|
|
277
|
+
accountForCall(wrapUp, wrapUpStepName);
|
|
212
278
|
if (wrapUp.finishReason === "aborted") {
|
|
213
279
|
// An abort during the tool-less wrap-up is still the user cutting the turn — the
|
|
214
280
|
// same event the in-loop path handles — so it gets the identical treatment: keep a
|
|
@@ -222,13 +288,65 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
222
288
|
await emit({ type: "iteration", source, index: agent.maxIterations, final: true });
|
|
223
289
|
recordAgentRun({ agentId: agent.id, iterations, cappedOut: true, usage });
|
|
224
290
|
logFinish("warn", "aborted", true);
|
|
225
|
-
return { messages, finish: { reason: "aborted", cappedOut: true, truncationRecoveries } };
|
|
291
|
+
return { messages, finish: { reason: "aborted", cappedOut: true, truncationRecoveries, ...finishUsage() } };
|
|
226
292
|
}
|
|
227
293
|
messages.push(wrapUp.message);
|
|
228
294
|
await emit({ type: "iteration", source, index: agent.maxIterations, final: true });
|
|
229
295
|
recordAgentRun({ agentId: agent.id, iterations, cappedOut: true, usage });
|
|
230
296
|
logFinish("warn", "max_iterations", true);
|
|
231
|
-
return { messages, finish: { reason: "max_iterations", cappedOut: true, truncationRecoveries } };
|
|
297
|
+
return { messages, finish: { reason: "max_iterations", cappedOut: true, truncationRecoveries, ...finishUsage() } };
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Delimiter joining the call path into one key segment.
|
|
301
|
+
*
|
|
302
|
+
* Deliberately not the `:` that separates the key's segments, so a call path
|
|
303
|
+
* can never be read as one: agent ids are kebab-case identifiers, run ids are
|
|
304
|
+
* UUIDs or the profiler's `data-profile` literal, step ids are path-safe plan
|
|
305
|
+
* ids (`T1S1`, `synthesis`, `profile`), tool-call ids are provider-issued
|
|
306
|
+
* opaque tokens, and the loop's step names are `llm-{n}` / `tool-{name}-{id}`
|
|
307
|
+
* with an optional `salvage:` prefix — none of them can contain `>`.
|
|
308
|
+
*/
|
|
309
|
+
const CALL_PATH_DELIMITER = ">";
|
|
310
|
+
/**
|
|
311
|
+
* Idempotency key for one call's usage record:
|
|
312
|
+
* `{runId}[:{stepId}]:{callPath}[:{invocationId}]:{stepName}`.
|
|
313
|
+
*
|
|
314
|
+
* Every component is a fact the same call carries again on every replay, so a
|
|
315
|
+
* re-executed body re-fires `record` with the byte-identical key and an
|
|
316
|
+
* upserting sink counts the call once. Each earns its place by a collision it
|
|
317
|
+
* alone resolves:
|
|
318
|
+
*
|
|
319
|
+
* - `stepId` — `executeAnalysis` runs one child workflow per step under a
|
|
320
|
+
* single shared `runId`, and each step's loop names its first call `llm-0`.
|
|
321
|
+
* - `callPath` — a step name is unique only within ONE loop invocation (every
|
|
322
|
+
* `runAgent` restarts its names at `llm-0`), and several loops routinely run
|
|
323
|
+
* under one frame: a step's post-step describer and its summary writer share
|
|
324
|
+
* `{runId, stepId}`, and the run-synthesis loop shares a bare `{runId}` with
|
|
325
|
+
* the sub-agents its tools run. The provenance path is what separates those
|
|
326
|
+
* agent chains.
|
|
327
|
+
* - `invocationId` — the same sub-agent dispatched twice in one round has an
|
|
328
|
+
* identical call path, so only the dispatching tool-call id tells the two
|
|
329
|
+
* child loops apart. It is replay-stable by the harness-tools contract: a
|
|
330
|
+
* redelivered call carries the id it was issued under.
|
|
331
|
+
* - `stepName` — the loop's own deterministic name, reused rather than
|
|
332
|
+
* re-derived. Minting a second *naming* scheme here would create two things
|
|
333
|
+
* that must agree about what "the same call" is, and they would drift.
|
|
334
|
+
*
|
|
335
|
+
* Outside a `RunFrame` there is no replay to be safe against (the HTTP chat
|
|
336
|
+
* path runs a call exactly once), so a fresh id is enough and is what keeps two
|
|
337
|
+
* calls in one turn distinct.
|
|
338
|
+
*/
|
|
339
|
+
function recordKeyFor(session, invocationId, stepName) {
|
|
340
|
+
const frame = session.runFrame;
|
|
341
|
+
if (frame === undefined)
|
|
342
|
+
return randomUUID();
|
|
343
|
+
return [
|
|
344
|
+
frame.runId,
|
|
345
|
+
...(frame.stepId === undefined ? [] : [frame.stepId]),
|
|
346
|
+
session.provenance.callPath.join(CALL_PATH_DELIMITER),
|
|
347
|
+
...(invocationId === undefined ? [] : [invocationId]),
|
|
348
|
+
stepName,
|
|
349
|
+
].join(":");
|
|
232
350
|
}
|
|
233
351
|
function toolCallParts(message) {
|
|
234
352
|
if (typeof message.content === "string")
|
|
@@ -357,8 +475,8 @@ function isAskRejected(err) {
|
|
|
357
475
|
}
|
|
358
476
|
/**
|
|
359
477
|
* Map a rejected approval to a model-visible `execution-denied` tool result. The
|
|
360
|
-
* prose is the model's only account of the denial; `
|
|
361
|
-
*
|
|
478
|
+
* prose is the model's only account of the denial; `outcomeOf` reports it as
|
|
479
|
+
* `denied`, distinct from a fault.
|
|
362
480
|
*/
|
|
363
481
|
function deniedResult(toolCall, feedback) {
|
|
364
482
|
const reason = feedback === undefined || feedback.length === 0
|
|
@@ -371,8 +489,16 @@ function deniedResult(toolCall, feedback) {
|
|
|
371
489
|
output: { type: "execution-denied", reason },
|
|
372
490
|
};
|
|
373
491
|
}
|
|
374
|
-
function
|
|
375
|
-
return result.output.type
|
|
492
|
+
function outcomeOf(result) {
|
|
493
|
+
return toolOutcomeForOutputType(result.output.type);
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* The optional `detail` field, present only when there is one. Spread rather
|
|
497
|
+
* than assigned so an undescribed call emits an event with no `detail` key at
|
|
498
|
+
* all, which is what the contract promises: absent, never empty.
|
|
499
|
+
*/
|
|
500
|
+
function detailField(detail) {
|
|
501
|
+
return detail === undefined ? {} : { detail };
|
|
376
502
|
}
|
|
377
503
|
function hasDenial(results) {
|
|
378
504
|
return results.some((r) => r.output.type === "execution-denied");
|
|
@@ -33,7 +33,9 @@ export declare const DEFAULT_SALVAGE_ITERATIONS = 3;
|
|
|
33
33
|
* first run that ended `max_iterations` spent its whole budget failing to submit,
|
|
34
34
|
* while one that ended `stop` gave up on prose after a single turn. Those call for
|
|
35
35
|
* opposite responses, and the returned `RunAgentResult` carries only the second
|
|
36
|
-
* run's finish, which is the same for either.
|
|
36
|
+
* run's finish, which is the same for either. Its token rollups are the sole
|
|
37
|
+
* exception — those are summed across both attempts, since they measure the run
|
|
38
|
+
* rather than diagnose it.
|
|
37
39
|
*/
|
|
38
40
|
export interface SalvageRecord {
|
|
39
41
|
/** How the FIRST run ended — the reason a salvage was needed at all. */
|
|
@@ -59,6 +61,7 @@ export interface TerminalSalvage {
|
|
|
59
61
|
/**
|
|
60
62
|
* Drive `agent` to its terminal tool, salvaging once if it doesn't get there.
|
|
61
63
|
* Returns the salvage run's result when a salvage occurred (its message array
|
|
62
|
-
* already includes the first run's
|
|
64
|
+
* already includes the first run's, and its token rollups are summed across both
|
|
65
|
+
* attempts), otherwise the first run's result.
|
|
63
66
|
*/
|
|
64
67
|
export declare function runToTerminal(agent: AgentDefinition, initial: readonly LoopMessage[], session: AgentSession, opts: RunAgentOptions, salvage: TerminalSalvage): Promise<RunToTerminalResult>;
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
* run's — replaying them would silently return the first run's cached results.
|
|
22
22
|
*/
|
|
23
23
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
24
|
+
import { addChatUsage } from "./metrics.js";
|
|
24
25
|
import { DEFAULT_STEP_NAME_FORMATTER, runAgent } from "./run-agent.js";
|
|
25
26
|
/** Default salvage budget: one submit plus a validation-fix retry or two. */
|
|
26
27
|
export const DEFAULT_SALVAGE_ITERATIONS = 3;
|
|
@@ -33,7 +34,8 @@ function salvageStepNames(base) {
|
|
|
33
34
|
/**
|
|
34
35
|
* Drive `agent` to its terminal tool, salvaging once if it doesn't get there.
|
|
35
36
|
* Returns the salvage run's result when a salvage occurred (its message array
|
|
36
|
-
* already includes the first run's
|
|
37
|
+
* already includes the first run's, and its token rollups are summed across both
|
|
38
|
+
* attempts), otherwise the first run's result.
|
|
37
39
|
*/
|
|
38
40
|
export async function runToTerminal(agent, initial, session, opts, salvage) {
|
|
39
41
|
const first = await runAgent(agent, initial, session, opts);
|
|
@@ -63,5 +65,37 @@ export async function runToTerminal(agent, initial, session, opts, salvage) {
|
|
|
63
65
|
formatStepName: salvageStepNames(opts.formatStepName ?? DEFAULT_STEP_NAME_FORMATTER),
|
|
64
66
|
};
|
|
65
67
|
const salvaged = await runAgent(salvageAgent, [...first.messages, { role: "user", content: salvage.nudge }], session, salvageOpts);
|
|
66
|
-
|
|
68
|
+
// The continuation is the same logical run as the first attempt — its
|
|
69
|
+
// message array already carries the first run's — so its rollups must too,
|
|
70
|
+
// or the caller reads the salvage turn's tokens as the whole cost.
|
|
71
|
+
//
|
|
72
|
+
// `salvage` deliberately keeps each attempt's OWN finish, unsummed: it is the
|
|
73
|
+
// per-attempt diagnosis, not a second accounting of the same tokens. The two
|
|
74
|
+
// views are therefore NOT additive — `finish` is the run's total and already
|
|
75
|
+
// covers everything `salvage.firstFinish` and `salvage.finish` report.
|
|
76
|
+
return {
|
|
77
|
+
...salvaged,
|
|
78
|
+
finish: {
|
|
79
|
+
...salvaged.finish,
|
|
80
|
+
...sumUsage("usage", first.finish, salvaged.finish),
|
|
81
|
+
...sumUsage("turnUsage", first.finish, salvaged.finish),
|
|
82
|
+
},
|
|
83
|
+
salvage: { firstFinish: first.finish, finish: salvaged.finish },
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Sum one rollup field across the two runs, keeping it absent when neither
|
|
88
|
+
* reported. Both runs are roots or neither is — `runToTerminal` forwards its
|
|
89
|
+
* caller's options unchanged — so summing `turnUsage` never mixes a turn total
|
|
90
|
+
* with a bare own-rollup.
|
|
91
|
+
*/
|
|
92
|
+
function sumUsage(field, first, second) {
|
|
93
|
+
const a = first[field];
|
|
94
|
+
const b = second[field];
|
|
95
|
+
if (a === undefined && b === undefined)
|
|
96
|
+
return {};
|
|
97
|
+
const total = {};
|
|
98
|
+
addChatUsage(total, a);
|
|
99
|
+
addChatUsage(total, b);
|
|
100
|
+
return { [field]: total };
|
|
67
101
|
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The call detail: one line naming what a tool call is doing.
|
|
3
|
+
*
|
|
4
|
+
* A tool states it through its own `describeCall` hook, colocated with the Zod
|
|
5
|
+
* `inputSchema` the compiler checks it against. Everything else about the
|
|
6
|
+
* detail happens here, once, for every tool: validate, guard, normalize.
|
|
7
|
+
*
|
|
8
|
+
* Two rules make this the only place that matters:
|
|
9
|
+
*
|
|
10
|
+
* 1. **The detail is a diagnostic and must never fail a call.** A hook that
|
|
11
|
+
* throws, returns a non-string, or returns nothing usable yields no detail
|
|
12
|
+
* and the tool dispatches unchanged.
|
|
13
|
+
* 2. **Normalization is not delegated to tool authors.** Thirty authors would
|
|
14
|
+
* each get the line breaks, the control characters, the secret redaction, and
|
|
15
|
+
* the length cap slightly wrong. Here it is one auditable path.
|
|
16
|
+
*/
|
|
17
|
+
import type { ToolCallDetail } from "../contracts/chat-events.js";
|
|
18
|
+
import type { Logger } from "../lib/logger.js";
|
|
19
|
+
import type { Tool } from "../tools/define-tool.js";
|
|
20
|
+
export type { ToolCallDetail };
|
|
21
|
+
/**
|
|
22
|
+
* The length cap, in characters. A full workspace path with a line range fits
|
|
23
|
+
* comfortably; a `write_file` content field does not, which is the point — a
|
|
24
|
+
* hook author who returns a payload cannot push it to a host through this
|
|
25
|
+
* channel.
|
|
26
|
+
*/
|
|
27
|
+
export declare const DETAIL_MAX_LENGTH = 120;
|
|
28
|
+
/**
|
|
29
|
+
* Normalize whatever a hook returned into an emittable detail, or `undefined`.
|
|
30
|
+
*
|
|
31
|
+
* In order: reject anything that is not a usable string, strip control
|
|
32
|
+
* characters, collapse to one line, redact secrets, then cap the length.
|
|
33
|
+
*
|
|
34
|
+
* The control-character strip is the harness's existing `normalizeUnicode`,
|
|
35
|
+
* which is also what the chat route applies to user prose — a detail is
|
|
36
|
+
* rendered by a host, so a terminal escape sequence in it is an injection.
|
|
37
|
+
* Redaction runs before the cap, so a secret cannot survive by being cut in
|
|
38
|
+
* half, and after the strip, so an embedded control byte cannot break a pattern.
|
|
39
|
+
*/
|
|
40
|
+
export declare function normalizeDetail(raw: unknown): ToolCallDetail | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* Compute a tool call's detail, best-effort.
|
|
43
|
+
*
|
|
44
|
+
* `rawInput` is unvalidated model output: the loop emits `tool-started` before
|
|
45
|
+
* `dispatchTools` parses anything, so the value here has satisfied no schema
|
|
46
|
+
* yet. The hook declares its parameter as `z.infer<Schema>`, so it is called
|
|
47
|
+
* ONLY on a successful `safeParse` — without that, the declared type would be a
|
|
48
|
+
* lie and author-written code would run over arbitrary shapes.
|
|
49
|
+
*
|
|
50
|
+
* A call the loop repairs later (`repairToolInput`) therefore emits no detail.
|
|
51
|
+
* That is accepted: repair is a rare recovery path, and computing the detail a
|
|
52
|
+
* second time from the repaired value costs more than the case is worth.
|
|
53
|
+
*/
|
|
54
|
+
export declare function computeDetail(tool: Tool, rawInput: unknown, log: Logger): ToolCallDetail | undefined;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The call detail: one line naming what a tool call is doing.
|
|
3
|
+
*
|
|
4
|
+
* A tool states it through its own `describeCall` hook, colocated with the Zod
|
|
5
|
+
* `inputSchema` the compiler checks it against. Everything else about the
|
|
6
|
+
* detail happens here, once, for every tool: validate, guard, normalize.
|
|
7
|
+
*
|
|
8
|
+
* Two rules make this the only place that matters:
|
|
9
|
+
*
|
|
10
|
+
* 1. **The detail is a diagnostic and must never fail a call.** A hook that
|
|
11
|
+
* throws, returns a non-string, or returns nothing usable yields no detail
|
|
12
|
+
* and the tool dispatches unchanged.
|
|
13
|
+
* 2. **Normalization is not delegated to tool authors.** Thirty authors would
|
|
14
|
+
* each get the line breaks, the control characters, the secret redaction, and
|
|
15
|
+
* the length cap slightly wrong. Here it is one auditable path.
|
|
16
|
+
*/
|
|
17
|
+
import { normalizeUnicode, redactSecrets } from "../input-sanitization.js";
|
|
18
|
+
/**
|
|
19
|
+
* The length cap, in characters. A full workspace path with a line range fits
|
|
20
|
+
* comfortably; a `write_file` content field does not, which is the point — a
|
|
21
|
+
* hook author who returns a payload cannot push it to a host through this
|
|
22
|
+
* channel.
|
|
23
|
+
*/
|
|
24
|
+
export const DETAIL_MAX_LENGTH = 120;
|
|
25
|
+
/**
|
|
26
|
+
* Cap `text` at `max` CODE POINTS, not UTF-16 units.
|
|
27
|
+
*
|
|
28
|
+
* `String.prototype.slice` cuts at a fixed unit index, which can land between the
|
|
29
|
+
* two halves of a surrogate pair and emit a lone surrogate — a terminal paints
|
|
30
|
+
* that as a replacement character, so the cap would corrupt the very tail it is
|
|
31
|
+
* supposed to merely shorten. Iterating the string yields whole code points.
|
|
32
|
+
*
|
|
33
|
+
* Code points, not display columns, is the right unit here: this cap is a safety
|
|
34
|
+
* valve against a runaway hook, so its job is to bound the payload. A column
|
|
35
|
+
* measurement would claim knowledge of the renderer's font metrics that the
|
|
36
|
+
* harness does not have — the host measures columns, because only the host knows
|
|
37
|
+
* how wide its own line is.
|
|
38
|
+
*/
|
|
39
|
+
function capCodePoints(text, max) {
|
|
40
|
+
const points = Array.from(text);
|
|
41
|
+
return points.length <= max ? text : points.slice(0, max).join("").trimEnd();
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Normalize whatever a hook returned into an emittable detail, or `undefined`.
|
|
45
|
+
*
|
|
46
|
+
* In order: reject anything that is not a usable string, strip control
|
|
47
|
+
* characters, collapse to one line, redact secrets, then cap the length.
|
|
48
|
+
*
|
|
49
|
+
* The control-character strip is the harness's existing `normalizeUnicode`,
|
|
50
|
+
* which is also what the chat route applies to user prose — a detail is
|
|
51
|
+
* rendered by a host, so a terminal escape sequence in it is an injection.
|
|
52
|
+
* Redaction runs before the cap, so a secret cannot survive by being cut in
|
|
53
|
+
* half, and after the strip, so an embedded control byte cannot break a pattern.
|
|
54
|
+
*/
|
|
55
|
+
export function normalizeDetail(raw) {
|
|
56
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
57
|
+
return undefined;
|
|
58
|
+
// `normalizeUnicode` leaves tab, newline and carriage return in place (they
|
|
59
|
+
// are legitimate whitespace in prose); the collapse below is what turns them
|
|
60
|
+
// into the single line this contract promises.
|
|
61
|
+
const oneLine = normalizeUnicode(raw).replace(/\s+/g, " ").trim();
|
|
62
|
+
if (oneLine.length === 0)
|
|
63
|
+
return undefined;
|
|
64
|
+
const capped = capCodePoints(redactSecrets(oneLine), DETAIL_MAX_LENGTH);
|
|
65
|
+
return capped.length === 0 ? undefined : capped;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Compute a tool call's detail, best-effort.
|
|
69
|
+
*
|
|
70
|
+
* `rawInput` is unvalidated model output: the loop emits `tool-started` before
|
|
71
|
+
* `dispatchTools` parses anything, so the value here has satisfied no schema
|
|
72
|
+
* yet. The hook declares its parameter as `z.infer<Schema>`, so it is called
|
|
73
|
+
* ONLY on a successful `safeParse` — without that, the declared type would be a
|
|
74
|
+
* lie and author-written code would run over arbitrary shapes.
|
|
75
|
+
*
|
|
76
|
+
* A call the loop repairs later (`repairToolInput`) therefore emits no detail.
|
|
77
|
+
* That is accepted: repair is a rare recovery path, and computing the detail a
|
|
78
|
+
* second time from the repaired value costs more than the case is worth.
|
|
79
|
+
*/
|
|
80
|
+
export function computeDetail(tool, rawInput, log) {
|
|
81
|
+
const describeCall = tool.describeCall;
|
|
82
|
+
if (describeCall === undefined)
|
|
83
|
+
return undefined;
|
|
84
|
+
try {
|
|
85
|
+
// The parse is INSIDE the guard, not before it. `safeParse` returns an
|
|
86
|
+
// error for a rejected value but THROWS for a schema it cannot run
|
|
87
|
+
// synchronously: an async refinement raises "Encountered Promise during
|
|
88
|
+
// synchronous parse", and a refinement whose predicate throws raises
|
|
89
|
+
// whatever it threw. Refinements survive `defineTool` because
|
|
90
|
+
// `z.toJSONSchema` ignores them, so such a schema reaches here intact —
|
|
91
|
+
// and a tool list is open, since an embedder contributes its own through
|
|
92
|
+
// the host-tools seam. Outside the guard that throw would escape the loop
|
|
93
|
+
// and kill the turn. A description must never be able to fail a call.
|
|
94
|
+
const parsed = tool.inputSchema.safeParse(rawInput);
|
|
95
|
+
if (!parsed.success)
|
|
96
|
+
return undefined;
|
|
97
|
+
return normalizeDetail(describeCall(parsed.data));
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
// `debug`, not `warn`: a broken description is a cosmetic defect in one
|
|
101
|
+
// tool, and the turn it happened in succeeded.
|
|
102
|
+
log.debug("describeCall failed", { tool: tool.id, ...log.errorFields(err) });
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One classification of how a tool call ended, shared by every surface that
|
|
3
|
+
* reports it.
|
|
4
|
+
*
|
|
5
|
+
* The live loop reads it off the `ToolResultPart` it just built; the reload
|
|
6
|
+
* converter reads it off the same part after a round trip through storage. They
|
|
7
|
+
* resolve through this function so a reloaded call cannot report an outcome the
|
|
8
|
+
* live call never had — the defect this replaces was exactly that, with reload
|
|
9
|
+
* reporting success for every call because it never looked at the result at all.
|
|
10
|
+
*/
|
|
11
|
+
import type { ToolOutcome } from "../contracts/chat-events.js";
|
|
12
|
+
export type { ToolOutcome };
|
|
13
|
+
/**
|
|
14
|
+
* Classify a tool result's output type.
|
|
15
|
+
*
|
|
16
|
+
* `execution-denied` is the user's own decision and stays distinct from a fault.
|
|
17
|
+
* `error-text` / `error-json` are the model-visible error results the loop
|
|
18
|
+
* builds from a thrown failure, an `err(ToolError)`, or rejected input.
|
|
19
|
+
* Everything else — including an expected "not found" data variant — is `ok`.
|
|
20
|
+
*/
|
|
21
|
+
export declare function toolOutcomeForOutputType(outputType: string): ToolOutcome;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One classification of how a tool call ended, shared by every surface that
|
|
3
|
+
* reports it.
|
|
4
|
+
*
|
|
5
|
+
* The live loop reads it off the `ToolResultPart` it just built; the reload
|
|
6
|
+
* converter reads it off the same part after a round trip through storage. They
|
|
7
|
+
* resolve through this function so a reloaded call cannot report an outcome the
|
|
8
|
+
* live call never had — the defect this replaces was exactly that, with reload
|
|
9
|
+
* reporting success for every call because it never looked at the result at all.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Classify a tool result's output type.
|
|
13
|
+
*
|
|
14
|
+
* `execution-denied` is the user's own decision and stays distinct from a fault.
|
|
15
|
+
* `error-text` / `error-json` are the model-visible error results the loop
|
|
16
|
+
* builds from a thrown failure, an `err(ToolError)`, or rejected input.
|
|
17
|
+
* Everything else — including an expected "not found" data variant — is `ok`.
|
|
18
|
+
*/
|
|
19
|
+
export function toolOutcomeForOutputType(outputType) {
|
|
20
|
+
if (outputType === "execution-denied")
|
|
21
|
+
return "denied";
|
|
22
|
+
if (outputType === "error-text" || outputType === "error-json")
|
|
23
|
+
return "error";
|
|
24
|
+
return "ok";
|
|
25
|
+
}
|
package/dist/loop/types.d.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* The loop body itself knows nothing about DBOS, HTTP, or memory.
|
|
9
9
|
*/
|
|
10
10
|
import type { ModelMessage } from "ai";
|
|
11
|
+
import type { ToolCallDetail, ToolOutcome } from "../contracts/chat-events.js";
|
|
11
12
|
import type { ChatStreamEvent } from "../providers/types.js";
|
|
12
13
|
import type { Tool } from "../tools/define-tool.js";
|
|
13
14
|
/**
|
|
@@ -61,12 +62,25 @@ export type EmitEvent = {
|
|
|
61
62
|
readonly toolUseId: string;
|
|
62
63
|
readonly name: string;
|
|
63
64
|
readonly input: unknown;
|
|
65
|
+
/**
|
|
66
|
+
* One line naming what this call is doing, from the tool's own
|
|
67
|
+
* `describeCall` hook. Absent — never empty — when the tool declares no
|
|
68
|
+
* hook or the detail could not be produced (see `tool-detail.ts`).
|
|
69
|
+
*/
|
|
70
|
+
readonly detail?: ToolCallDetail;
|
|
64
71
|
} | {
|
|
65
72
|
readonly type: "tool-finished";
|
|
66
73
|
readonly source: EventSource;
|
|
67
74
|
readonly toolUseId: string;
|
|
68
75
|
readonly name: string;
|
|
69
|
-
|
|
76
|
+
/**
|
|
77
|
+
* How the call ended. Three states rather than an error boolean: a
|
|
78
|
+
* denial is the user's decision, not a fault, and the loop already
|
|
79
|
+
* treats the two differently in control flow.
|
|
80
|
+
*/
|
|
81
|
+
readonly outcome: ToolOutcome;
|
|
82
|
+
/** The same detail the matching `tool-started` carried. */
|
|
83
|
+
readonly detail?: ToolCallDetail;
|
|
70
84
|
};
|
|
71
85
|
/**
|
|
72
86
|
* A UI presentation event a tool streams to chat — an AI-SDK `data-*`
|