@inflexa-ai/harness 0.14.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/README.md +2 -1
  2. package/dist/agents/conversation-agent.d.ts +7 -0
  3. package/dist/agents/conversation-agent.js +5 -3
  4. package/dist/app/synthesize-run.d.ts +3 -0
  5. package/dist/app/synthesize-run.js +2 -1
  6. package/dist/billing/noop-usage-recorder.d.ts +7 -0
  7. package/dist/billing/noop-usage-recorder.js +10 -0
  8. package/dist/billing/usage-recorder.d.ts +63 -0
  9. package/dist/billing/usage-recorder.js +15 -0
  10. package/dist/contracts/chat-events.d.ts +44 -3
  11. package/dist/contracts/chat-parts.d.ts +38 -2
  12. package/dist/contracts/chat-parts.js +1 -1
  13. package/dist/contracts/data-profile.d.ts +26 -0
  14. package/dist/contracts/data-profile.js +26 -0
  15. package/dist/contracts/index.d.ts +3 -1
  16. package/dist/contracts/index.js +1 -0
  17. package/dist/contracts/message.d.ts +21 -1
  18. package/dist/contracts/part-registry.js +1 -0
  19. package/dist/contracts/schemas/chat-events.d.ts +48 -2
  20. package/dist/contracts/schemas/chat-events.js +8 -1
  21. package/dist/contracts/schemas/chat-parts.d.ts +46 -3
  22. package/dist/contracts/schemas/chat-parts.js +13 -0
  23. package/dist/contracts/schemas/index.d.ts +2 -1
  24. package/dist/contracts/schemas/index.js +2 -1
  25. package/dist/contracts/schemas/usage.d.ts +11 -0
  26. package/dist/contracts/schemas/usage.js +18 -0
  27. package/dist/contracts/usage.d.ts +51 -0
  28. package/dist/contracts/usage.js +41 -0
  29. package/dist/execution/artifact-metadata.d.ts +3 -0
  30. package/dist/execution/artifact-metadata.js +1 -0
  31. package/dist/execution/dbos-workflow-purger.d.ts +30 -0
  32. package/dist/execution/dbos-workflow-purger.js +193 -0
  33. package/dist/execution/post-step-pipeline.d.ts +3 -0
  34. package/dist/execution/post-step-pipeline.js +2 -0
  35. package/dist/execution/reconcile-manifest.js +28 -15
  36. package/dist/execution/report-runner.d.ts +16 -0
  37. package/dist/execution/report-runner.js +4 -0
  38. package/dist/execution/run-synthesis.d.ts +3 -0
  39. package/dist/execution/run-synthesis.js +2 -0
  40. package/dist/execution/step-summary.d.ts +3 -0
  41. package/dist/execution/step-summary.js +1 -0
  42. package/dist/execution/workflow-purger.d.ts +38 -0
  43. package/dist/execution/workflow-purger.js +24 -0
  44. package/dist/index.d.ts +24 -2
  45. package/dist/index.js +27 -0
  46. package/dist/lib/metrics.js +2 -1
  47. package/dist/loop/metrics.d.ts +18 -0
  48. package/dist/loop/metrics.js +18 -1
  49. package/dist/loop/run-agent.d.ts +35 -0
  50. package/dist/loop/run-agent.js +149 -23
  51. package/dist/loop/run-to-terminal.d.ts +5 -2
  52. package/dist/loop/run-to-terminal.js +36 -2
  53. package/dist/loop/tool-detail.d.ts +54 -0
  54. package/dist/loop/tool-detail.js +105 -0
  55. package/dist/loop/tool-outcome.d.ts +21 -0
  56. package/dist/loop/tool-outcome.js +25 -0
  57. package/dist/loop/types.d.ts +15 -1
  58. package/dist/memory/content-to-cortex.d.ts +28 -4
  59. package/dist/memory/content-to-cortex.js +70 -14
  60. package/dist/memory/thread-history.d.ts +21 -1
  61. package/dist/memory/thread-history.js +29 -7
  62. package/dist/memory/thread-store.d.ts +65 -8
  63. package/dist/memory/thread-store.js +81 -17
  64. package/dist/provenance/collector.d.ts +5 -3
  65. package/dist/provenance/collector.js +5 -3
  66. package/dist/providers/ai-sdk.js +91 -16
  67. package/dist/providers/streaming-chat.js +5 -0
  68. package/dist/providers/types.d.ts +23 -0
  69. package/dist/providers/types.js +1 -0
  70. package/dist/runtime/assemble.d.ts +25 -8
  71. package/dist/runtime/assemble.js +7 -4
  72. package/dist/sandbox/sandbox-step-translate.d.ts +22 -1
  73. package/dist/sandbox/sandbox-step-translate.js +23 -36
  74. package/dist/schemas/run-synthesis.d.ts +2 -2
  75. package/dist/state/index.d.ts +2 -0
  76. package/dist/state/index.js +3 -0
  77. package/dist/state/init.js +28 -0
  78. package/dist/state/purge-analysis.d.ts +74 -0
  79. package/dist/state/purge-analysis.js +205 -0
  80. package/dist/state/vector-store.d.ts +6 -0
  81. package/dist/state/vector-store.js +22 -2
  82. package/dist/tasks/data-profile-activity.d.ts +14 -2
  83. package/dist/tasks/data-profile-activity.js +1 -1
  84. package/dist/tasks/data-profile.d.ts +3 -0
  85. package/dist/tasks/data-profile.js +9 -3
  86. package/dist/tools/bio/pubmed.js +15 -0
  87. package/dist/tools/bio/search-gene.js +12 -0
  88. package/dist/tools/define-tool.d.ts +41 -3
  89. package/dist/tools/define-tool.js +4 -1
  90. package/dist/tools/detail-resolver.d.ts +35 -0
  91. package/dist/tools/detail-resolver.js +39 -0
  92. package/dist/tools/execute-analysis.js +3 -0
  93. package/dist/tools/iterate-report.d.ts +3 -0
  94. package/dist/tools/iterate-report.js +3 -0
  95. package/dist/tools/memory/update-working-memory.js +8 -0
  96. package/dist/tools/research/generate-analogy-report.d.ts +3 -0
  97. package/dist/tools/research/generate-analogy-report.js +6 -0
  98. package/dist/tools/research/generate-plan.d.ts +3 -0
  99. package/dist/tools/research/generate-plan.js +6 -0
  100. package/dist/tools/research/inspect-run.js +4 -0
  101. package/dist/tools/research/literature-reviewer.d.ts +3 -0
  102. package/dist/tools/research/literature-reviewer.js +6 -0
  103. package/dist/tools/workspace/edit-file.js +2 -0
  104. package/dist/tools/workspace/execute-command.js +25 -0
  105. package/dist/tools/workspace/grep.js +3 -0
  106. package/dist/tools/workspace/list-files.js +2 -0
  107. package/dist/tools/workspace/read-file.js +7 -0
  108. package/dist/tools/workspace/workspace-search.js +1 -0
  109. package/dist/tools/workspace/write-file.js +3 -0
  110. package/dist/workflows/execute-analysis.d.ts +7 -0
  111. package/dist/workflows/execute-analysis.js +31 -2
  112. package/dist/workflows/execute-target-assessment.d.ts +3 -0
  113. package/dist/workflows/sandbox-step.d.ts +23 -0
  114. package/dist/workflows/sandbox-step.js +33 -2
  115. package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.d.ts +3 -0
  116. package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.js +1 -0
  117. package/package.json +1 -1
@@ -1,4 +1,6 @@
1
+ import { randomUUID } from "node:crypto";
1
2
  import { jsonSchema, tool as aiTool } from "ai";
3
+ import { createNoopUsageRecorder } from "../billing/noop-usage-recorder.js";
2
4
  import { stripNulCharacters } from "../input-sanitization.js";
3
5
  import { createNoopLogger } from "../lib/console-logger.js";
4
6
  import { hintForZodIssue, repairToolInput } from "../lib/zod-issues.js";
@@ -8,7 +10,9 @@ import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/p
8
10
  import { resultStep } from "./run-step.js";
9
11
  import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
10
12
  import { isToolError } from "../tools/define-tool.js";
11
- import { addChatUsage, recordAgentRun } from "./metrics.js";
13
+ import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
14
+ import { computeDetail } from "./tool-detail.js";
15
+ import { toolOutcomeForOutputType } from "./tool-outcome.js";
12
16
  const TRUNCATED_PROSE_STEER = "Your previous reply was cut off at the output-token limit; continue concisely, or finish via your terminal tool.";
13
17
  const TRUNCATED_TOOL_USE_ERROR = "Your previous tool call was cut off at the output-token limit and was not executed. Retry with a smaller payload, writing in incremental pieces.";
14
18
  export const DEFAULT_STEP_NAME_FORMATTER = {
@@ -50,6 +54,11 @@ export async function runAgent(agent, initial, session, opts) {
50
54
  // answer that cannot come.
51
55
  const unavailableAsk = new UnavailableAsk();
52
56
  const ask = opts.ask ?? ((request) => unavailableAsk.ask(request));
57
+ // A run handed no accumulator is the turn's root: it creates the one every
58
+ // descendant loop folds into, and it is the only loop entitled to report a
59
+ // turn total. Descendants receive the root's object and mutate it in place.
60
+ const turnUsage = opts.turnUsage ?? {};
61
+ const isTurnRoot = opts.turnUsage === undefined;
53
62
  const toolCtx = (tu) => ({
54
63
  invocationId: tu.toolCallId,
55
64
  session,
@@ -57,6 +66,7 @@ export async function runAgent(agent, initial, session, opts) {
57
66
  emit,
58
67
  runStep: (name, fn) => runStep(`${formatStepName.tool(tu.toolName, tu.toolCallId)}:${name}`, fn),
59
68
  ask,
69
+ turnUsage,
60
70
  });
61
71
  let iterations = 0;
62
72
  let truncationRecoveries = 0;
@@ -77,6 +87,44 @@ export async function runAgent(agent, initial, session, opts) {
77
87
  const logFinish = (level, reason, cappedOut) => {
78
88
  log[level]("run finished", { iterations, reason, cappedOut, truncationRecoveries, toolCalls: toolCallCount, toolErrors: toolErrorCount, usage });
79
89
  };
90
+ const usageRecorder = opts.usageRecorder ?? createNoopUsageRecorder();
91
+ /**
92
+ * Fold one completed call into both rollups and hand it to the recorder.
93
+ * Called at the fold point, with the reply in hand — before any branch that
94
+ * can end the run — so a call that completed is accounted for even when the
95
+ * run later aborts or dies.
96
+ *
97
+ * A call that reported nothing produces no record. Model ids are identity,
98
+ * not usage: a reply carrying only `requestedModelId`/`servedModelId` still
99
+ * reported nothing to account for, so it is folded (to no effect) and left
100
+ * out of the ledger rather than entered as an all-absent record.
101
+ *
102
+ * Delivery is bare — no `await`, no `try` — because the `UsageRecorder`
103
+ * contract forbids `record` to throw or block.
104
+ */
105
+ const accountForCall = (reply, stepName) => {
106
+ addChatUsage(usage, reply.usage);
107
+ addChatUsage(turnUsage, reply.usage);
108
+ const reported = reply.usage;
109
+ if (reported === undefined || !hasReportedUsage(reported))
110
+ return;
111
+ usageRecorder.record({
112
+ recordKey: recordKeyFor(session, opts.invocationId, stepName),
113
+ agentId: source.agentId,
114
+ callPath: source.callPath,
115
+ scope: session.scope,
116
+ ...(session.runFrame?.runId === undefined ? {} : { runId: session.runFrame.runId }),
117
+ ...(session.runFrame?.stepId === undefined ? {} : { stepId: session.runFrame.stepId }),
118
+ ...(reply.requestedModelId === undefined ? {} : { requestedModelId: reply.requestedModelId }),
119
+ ...(reply.servedModelId === undefined ? {} : { servedModelId: reply.servedModelId }),
120
+ usage: reported,
121
+ });
122
+ };
123
+ /** The rollups this loop stamps on its finish — each absent when nothing reported. */
124
+ const finishUsage = () => ({
125
+ ...(hasReportedUsage(usage) ? { usage: { ...usage } } : {}),
126
+ ...(isTurnRoot && hasReportedUsage(turnUsage) ? { turnUsage: { ...turnUsage } } : {}),
127
+ });
80
128
  // The user said no. A subsequent model call would only let the agent argue
81
129
  // with the decision, or spend a call acknowledging it; the denial tool result
82
130
  // is itself what the surface renders, so the turn ends the moment a denial
@@ -86,8 +134,19 @@ export async function runAgent(agent, initial, session, opts) {
86
134
  await emit({ type: "iteration", source, index: i, final: true });
87
135
  recordAgentRun({ agentId: agent.id, iterations, cappedOut: false, usage });
88
136
  logFinish("warn", "denied", false);
89
- return { messages, finish: { reason: "denied", cappedOut: false, truncationRecoveries } };
137
+ return { messages, finish: { reason: "denied", cappedOut: false, truncationRecoveries, ...finishUsage() } };
90
138
  };
139
+ /**
140
+ * The call details for one dispatch round, positionally aligned with `calls`.
141
+ *
142
+ * Computed once per round and carried onto both the `tool-started` and the
143
+ * `tool-finished` event, so the pair a host renders as one chip cannot show
144
+ * two different descriptions of the same call.
145
+ */
146
+ const roundDetails = (calls) => calls.map((tu) => {
147
+ const tool = toolsById.get(tu.toolName);
148
+ return tool === undefined ? undefined : computeDetail(tool, tu.input, log);
149
+ });
91
150
  /**
92
151
  * Fold one dispatch round into the run's counters and the event sink, returning the
93
152
  * tools whose results the model will read as errors.
@@ -97,17 +156,20 @@ export async function runAgent(agent, initial, session, opts) {
97
156
  * it is not a thrown failure, so nothing reports it and the loop simply feeds it
98
157
  * back — and a run that ends badly is usually a run whose tool calls were failing.
99
158
  */
100
- const settleRound = async (calls, results) => {
159
+ const settleRound = async (calls, results, details) => {
101
160
  const errored = [];
102
161
  for (let idx = 0; idx < calls.length; idx++) {
103
162
  const tu = calls[idx];
104
- const isError = isErrorOutput(results[idx]);
163
+ const outcome = outcomeOf(results[idx]);
105
164
  toolCallCount++;
106
- if (isError) {
165
+ // A denial counts here with the errors: the model reads it as a result
166
+ // it did not get to act on, which is what these counters describe. The
167
+ // event keeps the two apart for the user-facing surface.
168
+ if (outcome !== "ok") {
107
169
  toolErrorCount++;
108
170
  errored.push(tu.toolName);
109
171
  }
110
- await emit({ type: "tool-finished", source, toolUseId: tu.toolCallId, name: tu.toolName, isError });
172
+ await emit({ type: "tool-finished", source, toolUseId: tu.toolCallId, name: tu.toolName, outcome, ...detailField(details[idx]) });
111
173
  }
112
174
  return errored;
113
175
  };
@@ -125,8 +187,9 @@ export async function runAgent(agent, initial, session, opts) {
125
187
  ...(opts.toolChoice !== undefined ? { toolChoice: opts.toolChoice } : {}),
126
188
  providerOptions,
127
189
  };
128
- const reply = await resultStep(runStep)(formatStepName.llm(i), () => provider.chat(request, session, signal));
129
- addChatUsage(usage, reply.usage);
190
+ const llmStepName = formatStepName.llm(i);
191
+ const reply = await resultStep(runStep)(llmStepName, () => provider.chat(request, session, signal));
192
+ accountForCall(reply, llmStepName);
130
193
  if (reply.finishReason === "aborted") {
131
194
  // An interrupted turn keeps whatever the model produced before the cut, but
132
195
  // never an empty shell: a partial with no content adds no message, so a
@@ -159,11 +222,12 @@ export async function runAgent(agent, initial, session, opts) {
159
222
  }
160
223
  const trailing = toolCalls[toolCalls.length - 1];
161
224
  const earlier = toolCalls.slice(0, -1);
162
- for (const tu of earlier) {
163
- await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input });
225
+ const earlierDetails = roundDetails(earlier);
226
+ for (const [idx, tu] of earlier.entries()) {
227
+ await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(earlierDetails[idx]) });
164
228
  }
165
229
  const results = await dispatchTools(earlier, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool);
166
- const errored = await settleRound(earlier, results);
230
+ const errored = await settleRound(earlier, results, earlierDetails);
167
231
  results.push(errorResult(trailing, TRUNCATED_TOOL_USE_ERROR));
168
232
  // The trailing call was never dispatched, but it reaches the model as an error
169
233
  // result like any other — so it counts, or `toolErrors` reports a cleaner run
@@ -184,15 +248,16 @@ export async function runAgent(agent, initial, session, opts) {
184
248
  await emit({ type: "iteration", source, index: i, final: true });
185
249
  recordAgentRun({ agentId: agent.id, iterations, cappedOut: false, usage });
186
250
  logFinish("info", reply.finishReason, false);
187
- return { messages, finish: { reason: reply.finishReason, cappedOut: false, truncationRecoveries } };
251
+ return { messages, finish: { reason: reply.finishReason, cappedOut: false, truncationRecoveries, ...finishUsage() } };
188
252
  }
189
253
  await emit({ type: "iteration", source, index: i, final: false });
190
254
  log.debug("iteration", { iteration: i, tools: toolCalls.map((t) => t.toolName) });
191
- for (const tu of toolCalls) {
192
- await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input });
255
+ const details = roundDetails(toolCalls);
256
+ for (const [idx, tu] of toolCalls.entries()) {
257
+ await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(details[idx]) });
193
258
  }
194
259
  const results = await dispatchTools(toolCalls, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool);
195
- const errored = await settleRound(toolCalls, results);
260
+ const errored = await settleRound(toolCalls, results, details);
196
261
  if (errored.length > 0)
197
262
  log.debug("tool results returned errors", { iteration: i, tools: errored });
198
263
  messages.push({ role: "tool", content: results });
@@ -207,8 +272,9 @@ export async function runAgent(agent, initial, session, opts) {
207
272
  // rewrites the whole prefix from scratch. It still carries the cache options
208
273
  // because it is the one call whose write is pure waste, and the
209
274
  // cache_write_tokens counter is what makes that waste visible.
210
- const wrapUp = await resultStep(runStep)(formatStepName.llm(agent.maxIterations), () => provider.chat({ system: agent.systemPrompt, messages, tools: {}, toolChoice: "none", providerOptions }, session, signal));
211
- addChatUsage(usage, wrapUp.usage);
275
+ const wrapUpStepName = formatStepName.llm(agent.maxIterations);
276
+ const wrapUp = await resultStep(runStep)(wrapUpStepName, () => provider.chat({ system: agent.systemPrompt, messages, tools: {}, toolChoice: "none", providerOptions }, session, signal));
277
+ accountForCall(wrapUp, wrapUpStepName);
212
278
  if (wrapUp.finishReason === "aborted") {
213
279
  // An abort during the tool-less wrap-up is still the user cutting the turn — the
214
280
  // same event the in-loop path handles — so it gets the identical treatment: keep a
@@ -222,13 +288,65 @@ export async function runAgent(agent, initial, session, opts) {
222
288
  await emit({ type: "iteration", source, index: agent.maxIterations, final: true });
223
289
  recordAgentRun({ agentId: agent.id, iterations, cappedOut: true, usage });
224
290
  logFinish("warn", "aborted", true);
225
- return { messages, finish: { reason: "aborted", cappedOut: true, truncationRecoveries } };
291
+ return { messages, finish: { reason: "aborted", cappedOut: true, truncationRecoveries, ...finishUsage() } };
226
292
  }
227
293
  messages.push(wrapUp.message);
228
294
  await emit({ type: "iteration", source, index: agent.maxIterations, final: true });
229
295
  recordAgentRun({ agentId: agent.id, iterations, cappedOut: true, usage });
230
296
  logFinish("warn", "max_iterations", true);
231
- return { messages, finish: { reason: "max_iterations", cappedOut: true, truncationRecoveries } };
297
+ return { messages, finish: { reason: "max_iterations", cappedOut: true, truncationRecoveries, ...finishUsage() } };
298
+ }
299
+ /**
300
+ * Delimiter joining the call path into one key segment.
301
+ *
302
+ * Deliberately not the `:` that separates the key's segments, so a call path
303
+ * can never be read as one: agent ids are kebab-case identifiers, run ids are
304
+ * UUIDs or the profiler's `data-profile` literal, step ids are path-safe plan
305
+ * ids (`T1S1`, `synthesis`, `profile`), tool-call ids are provider-issued
306
+ * opaque tokens, and the loop's step names are `llm-{n}` / `tool-{name}-{id}`
307
+ * with an optional `salvage:` prefix — none of them can contain `>`.
308
+ */
309
+ const CALL_PATH_DELIMITER = ">";
310
+ /**
311
+ * Idempotency key for one call's usage record:
312
+ * `{runId}[:{stepId}]:{callPath}[:{invocationId}]:{stepName}`.
313
+ *
314
+ * Every component is a fact the same call carries again on every replay, so a
315
+ * re-executed body re-fires `record` with the byte-identical key and an
316
+ * upserting sink counts the call once. Each earns its place by a collision it
317
+ * alone resolves:
318
+ *
319
+ * - `stepId` — `executeAnalysis` runs one child workflow per step under a
320
+ * single shared `runId`, and each step's loop names its first call `llm-0`.
321
+ * - `callPath` — a step name is unique only within ONE loop invocation (every
322
+ * `runAgent` restarts its names at `llm-0`), and several loops routinely run
323
+ * under one frame: a step's post-step describer and its summary writer share
324
+ * `{runId, stepId}`, and the run-synthesis loop shares a bare `{runId}` with
325
+ * the sub-agents its tools run. The provenance path is what separates those
326
+ * agent chains.
327
+ * - `invocationId` — the same sub-agent dispatched twice in one round has an
328
+ * identical call path, so only the dispatching tool-call id tells the two
329
+ * child loops apart. It is replay-stable by the harness-tools contract: a
330
+ * redelivered call carries the id it was issued under.
331
+ * - `stepName` — the loop's own deterministic name, reused rather than
332
+ * re-derived. Minting a second *naming* scheme here would create two things
333
+ * that must agree about what "the same call" is, and they would drift.
334
+ *
335
+ * Outside a `RunFrame` there is no replay to be safe against (the HTTP chat
336
+ * path runs a call exactly once), so a fresh id is enough and is what keeps two
337
+ * calls in one turn distinct.
338
+ */
339
+ function recordKeyFor(session, invocationId, stepName) {
340
+ const frame = session.runFrame;
341
+ if (frame === undefined)
342
+ return randomUUID();
343
+ return [
344
+ frame.runId,
345
+ ...(frame.stepId === undefined ? [] : [frame.stepId]),
346
+ session.provenance.callPath.join(CALL_PATH_DELIMITER),
347
+ ...(invocationId === undefined ? [] : [invocationId]),
348
+ stepName,
349
+ ].join(":");
232
350
  }
233
351
  function toolCallParts(message) {
234
352
  if (typeof message.content === "string")
@@ -357,8 +475,8 @@ function isAskRejected(err) {
357
475
  }
358
476
  /**
359
477
  * Map a rejected approval to a model-visible `execution-denied` tool result. The
360
- * prose is the model's only account of the denial; `isErrorOutput` already
361
- * treats `execution-denied` as an error result.
478
+ * prose is the model's only account of the denial; `outcomeOf` reports it as
479
+ * `denied`, distinct from a fault.
362
480
  */
363
481
  function deniedResult(toolCall, feedback) {
364
482
  const reason = feedback === undefined || feedback.length === 0
@@ -371,8 +489,16 @@ function deniedResult(toolCall, feedback) {
371
489
  output: { type: "execution-denied", reason },
372
490
  };
373
491
  }
374
- function isErrorOutput(result) {
375
- return result.output.type === "error-text" || result.output.type === "error-json" || result.output.type === "execution-denied";
492
+ function outcomeOf(result) {
493
+ return toolOutcomeForOutputType(result.output.type);
494
+ }
495
+ /**
496
+ * The optional `detail` field, present only when there is one. Spread rather
497
+ * than assigned so an undescribed call emits an event with no `detail` key at
498
+ * all, which is what the contract promises: absent, never empty.
499
+ */
500
+ function detailField(detail) {
501
+ return detail === undefined ? {} : { detail };
376
502
  }
377
503
  function hasDenial(results) {
378
504
  return results.some((r) => r.output.type === "execution-denied");
@@ -33,7 +33,9 @@ export declare const DEFAULT_SALVAGE_ITERATIONS = 3;
33
33
  * first run that ended `max_iterations` spent its whole budget failing to submit,
34
34
  * while one that ended `stop` gave up on prose after a single turn. Those call for
35
35
  * opposite responses, and the returned `RunAgentResult` carries only the second
36
- * run's finish, which is the same for either.
36
+ * run's finish, which is the same for either. Its token rollups are the sole
37
+ * exception — those are summed across both attempts, since they measure the run
38
+ * rather than diagnose it.
37
39
  */
38
40
  export interface SalvageRecord {
39
41
  /** How the FIRST run ended — the reason a salvage was needed at all. */
@@ -59,6 +61,7 @@ export interface TerminalSalvage {
59
61
  /**
60
62
  * Drive `agent` to its terminal tool, salvaging once if it doesn't get there.
61
63
  * Returns the salvage run's result when a salvage occurred (its message array
62
- * already includes the first run's), otherwise the first run's result.
64
+ * already includes the first run's, and its token rollups are summed across both
65
+ * attempts), otherwise the first run's result.
63
66
  */
64
67
  export declare function runToTerminal(agent: AgentDefinition, initial: readonly LoopMessage[], session: AgentSession, opts: RunAgentOptions, salvage: TerminalSalvage): Promise<RunToTerminalResult>;
@@ -21,6 +21,7 @@
21
21
  * run's — replaying them would silently return the first run's cached results.
22
22
  */
23
23
  import { createNoopLogger } from "../lib/console-logger.js";
24
+ import { addChatUsage } from "./metrics.js";
24
25
  import { DEFAULT_STEP_NAME_FORMATTER, runAgent } from "./run-agent.js";
25
26
  /** Default salvage budget: one submit plus a validation-fix retry or two. */
26
27
  export const DEFAULT_SALVAGE_ITERATIONS = 3;
@@ -33,7 +34,8 @@ function salvageStepNames(base) {
33
34
  /**
34
35
  * Drive `agent` to its terminal tool, salvaging once if it doesn't get there.
35
36
  * Returns the salvage run's result when a salvage occurred (its message array
36
- * already includes the first run's), otherwise the first run's result.
37
+ * already includes the first run's, and its token rollups are summed across both
38
+ * attempts), otherwise the first run's result.
37
39
  */
38
40
  export async function runToTerminal(agent, initial, session, opts, salvage) {
39
41
  const first = await runAgent(agent, initial, session, opts);
@@ -63,5 +65,37 @@ export async function runToTerminal(agent, initial, session, opts, salvage) {
63
65
  formatStepName: salvageStepNames(opts.formatStepName ?? DEFAULT_STEP_NAME_FORMATTER),
64
66
  };
65
67
  const salvaged = await runAgent(salvageAgent, [...first.messages, { role: "user", content: salvage.nudge }], session, salvageOpts);
66
- return { ...salvaged, salvage: { firstFinish: first.finish, finish: salvaged.finish } };
68
+ // The continuation is the same logical run as the first attempt — its
69
+ // message array already carries the first run's — so its rollups must too,
70
+ // or the caller reads the salvage turn's tokens as the whole cost.
71
+ //
72
+ // `salvage` deliberately keeps each attempt's OWN finish, unsummed: it is the
73
+ // per-attempt diagnosis, not a second accounting of the same tokens. The two
74
+ // views are therefore NOT additive — `finish` is the run's total and already
75
+ // covers everything `salvage.firstFinish` and `salvage.finish` report.
76
+ return {
77
+ ...salvaged,
78
+ finish: {
79
+ ...salvaged.finish,
80
+ ...sumUsage("usage", first.finish, salvaged.finish),
81
+ ...sumUsage("turnUsage", first.finish, salvaged.finish),
82
+ },
83
+ salvage: { firstFinish: first.finish, finish: salvaged.finish },
84
+ };
85
+ }
86
+ /**
87
+ * Sum one rollup field across the two runs, keeping it absent when neither
88
+ * reported. Both runs are roots or neither is — `runToTerminal` forwards its
89
+ * caller's options unchanged — so summing `turnUsage` never mixes a turn total
90
+ * with a bare own-rollup.
91
+ */
92
+ function sumUsage(field, first, second) {
93
+ const a = first[field];
94
+ const b = second[field];
95
+ if (a === undefined && b === undefined)
96
+ return {};
97
+ const total = {};
98
+ addChatUsage(total, a);
99
+ addChatUsage(total, b);
100
+ return { [field]: total };
67
101
  }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * The call detail: one line naming what a tool call is doing.
3
+ *
4
+ * A tool states it through its own `describeCall` hook, colocated with the Zod
5
+ * `inputSchema` the compiler checks it against. Everything else about the
6
+ * detail happens here, once, for every tool: validate, guard, normalize.
7
+ *
8
+ * Two rules make this the only place that matters:
9
+ *
10
+ * 1. **The detail is a diagnostic and must never fail a call.** A hook that
11
+ * throws, returns a non-string, or returns nothing usable yields no detail
12
+ * and the tool dispatches unchanged.
13
+ * 2. **Normalization is not delegated to tool authors.** Thirty authors would
14
+ * each get the line breaks, the control characters, the secret redaction, and
15
+ * the length cap slightly wrong. Here it is one auditable path.
16
+ */
17
+ import type { ToolCallDetail } from "../contracts/chat-events.js";
18
+ import type { Logger } from "../lib/logger.js";
19
+ import type { Tool } from "../tools/define-tool.js";
20
+ export type { ToolCallDetail };
21
+ /**
22
+ * The length cap, in characters. A full workspace path with a line range fits
23
+ * comfortably; a `write_file` content field does not, which is the point — a
24
+ * hook author who returns a payload cannot push it to a host through this
25
+ * channel.
26
+ */
27
+ export declare const DETAIL_MAX_LENGTH = 120;
28
+ /**
29
+ * Normalize whatever a hook returned into an emittable detail, or `undefined`.
30
+ *
31
+ * In order: reject anything that is not a usable string, strip control
32
+ * characters, collapse to one line, redact secrets, then cap the length.
33
+ *
34
+ * The control-character strip is the harness's existing `normalizeUnicode`,
35
+ * which is also what the chat route applies to user prose — a detail is
36
+ * rendered by a host, so a terminal escape sequence in it is an injection.
37
+ * Redaction runs before the cap, so a secret cannot survive by being cut in
38
+ * half, and after the strip, so an embedded control byte cannot break a pattern.
39
+ */
40
+ export declare function normalizeDetail(raw: unknown): ToolCallDetail | undefined;
41
+ /**
42
+ * Compute a tool call's detail, best-effort.
43
+ *
44
+ * `rawInput` is unvalidated model output: the loop emits `tool-started` before
45
+ * `dispatchTools` parses anything, so the value here has satisfied no schema
46
+ * yet. The hook declares its parameter as `z.infer<Schema>`, so it is called
47
+ * ONLY on a successful `safeParse` — without that, the declared type would be a
48
+ * lie and author-written code would run over arbitrary shapes.
49
+ *
50
+ * A call the loop repairs later (`repairToolInput`) therefore emits no detail.
51
+ * That is accepted: repair is a rare recovery path, and computing the detail a
52
+ * second time from the repaired value costs more than the case is worth.
53
+ */
54
+ export declare function computeDetail(tool: Tool, rawInput: unknown, log: Logger): ToolCallDetail | undefined;
@@ -0,0 +1,105 @@
1
+ /**
2
+ * The call detail: one line naming what a tool call is doing.
3
+ *
4
+ * A tool states it through its own `describeCall` hook, colocated with the Zod
5
+ * `inputSchema` the compiler checks it against. Everything else about the
6
+ * detail happens here, once, for every tool: validate, guard, normalize.
7
+ *
8
+ * Two rules make this the only place that matters:
9
+ *
10
+ * 1. **The detail is a diagnostic and must never fail a call.** A hook that
11
+ * throws, returns a non-string, or returns nothing usable yields no detail
12
+ * and the tool dispatches unchanged.
13
+ * 2. **Normalization is not delegated to tool authors.** Thirty authors would
14
+ * each get the line breaks, the control characters, the secret redaction, and
15
+ * the length cap slightly wrong. Here it is one auditable path.
16
+ */
17
+ import { normalizeUnicode, redactSecrets } from "../input-sanitization.js";
18
+ /**
19
+ * The length cap, in characters. A full workspace path with a line range fits
20
+ * comfortably; a `write_file` content field does not, which is the point — a
21
+ * hook author who returns a payload cannot push it to a host through this
22
+ * channel.
23
+ */
24
+ export const DETAIL_MAX_LENGTH = 120;
25
+ /**
26
+ * Cap `text` at `max` CODE POINTS, not UTF-16 units.
27
+ *
28
+ * `String.prototype.slice` cuts at a fixed unit index, which can land between the
29
+ * two halves of a surrogate pair and emit a lone surrogate — a terminal paints
30
+ * that as a replacement character, so the cap would corrupt the very tail it is
31
+ * supposed to merely shorten. Iterating the string yields whole code points.
32
+ *
33
+ * Code points, not display columns, is the right unit here: this cap is a safety
34
+ * valve against a runaway hook, so its job is to bound the payload. A column
35
+ * measurement would claim knowledge of the renderer's font metrics that the
36
+ * harness does not have — the host measures columns, because only the host knows
37
+ * how wide its own line is.
38
+ */
39
+ function capCodePoints(text, max) {
40
+ const points = Array.from(text);
41
+ return points.length <= max ? text : points.slice(0, max).join("").trimEnd();
42
+ }
43
+ /**
44
+ * Normalize whatever a hook returned into an emittable detail, or `undefined`.
45
+ *
46
+ * In order: reject anything that is not a usable string, strip control
47
+ * characters, collapse to one line, redact secrets, then cap the length.
48
+ *
49
+ * The control-character strip is the harness's existing `normalizeUnicode`,
50
+ * which is also what the chat route applies to user prose — a detail is
51
+ * rendered by a host, so a terminal escape sequence in it is an injection.
52
+ * Redaction runs before the cap, so a secret cannot survive by being cut in
53
+ * half, and after the strip, so an embedded control byte cannot break a pattern.
54
+ */
55
+ export function normalizeDetail(raw) {
56
+ if (typeof raw !== "string" || raw.length === 0)
57
+ return undefined;
58
+ // `normalizeUnicode` leaves tab, newline and carriage return in place (they
59
+ // are legitimate whitespace in prose); the collapse below is what turns them
60
+ // into the single line this contract promises.
61
+ const oneLine = normalizeUnicode(raw).replace(/\s+/g, " ").trim();
62
+ if (oneLine.length === 0)
63
+ return undefined;
64
+ const capped = capCodePoints(redactSecrets(oneLine), DETAIL_MAX_LENGTH);
65
+ return capped.length === 0 ? undefined : capped;
66
+ }
67
+ /**
68
+ * Compute a tool call's detail, best-effort.
69
+ *
70
+ * `rawInput` is unvalidated model output: the loop emits `tool-started` before
71
+ * `dispatchTools` parses anything, so the value here has satisfied no schema
72
+ * yet. The hook declares its parameter as `z.infer<Schema>`, so it is called
73
+ * ONLY on a successful `safeParse` — without that, the declared type would be a
74
+ * lie and author-written code would run over arbitrary shapes.
75
+ *
76
+ * A call the loop repairs later (`repairToolInput`) therefore emits no detail.
77
+ * That is accepted: repair is a rare recovery path, and computing the detail a
78
+ * second time from the repaired value costs more than the case is worth.
79
+ */
80
+ export function computeDetail(tool, rawInput, log) {
81
+ const describeCall = tool.describeCall;
82
+ if (describeCall === undefined)
83
+ return undefined;
84
+ try {
85
+ // The parse is INSIDE the guard, not before it. `safeParse` returns an
86
+ // error for a rejected value but THROWS for a schema it cannot run
87
+ // synchronously: an async refinement raises "Encountered Promise during
88
+ // synchronous parse", and a refinement whose predicate throws raises
89
+ // whatever it threw. Refinements survive `defineTool` because
90
+ // `z.toJSONSchema` ignores them, so such a schema reaches here intact —
91
+ // and a tool list is open, since an embedder contributes its own through
92
+ // the host-tools seam. Outside the guard that throw would escape the loop
93
+ // and kill the turn. A description must never be able to fail a call.
94
+ const parsed = tool.inputSchema.safeParse(rawInput);
95
+ if (!parsed.success)
96
+ return undefined;
97
+ return normalizeDetail(describeCall(parsed.data));
98
+ }
99
+ catch (err) {
100
+ // `debug`, not `warn`: a broken description is a cosmetic defect in one
101
+ // tool, and the turn it happened in succeeded.
102
+ log.debug("describeCall failed", { tool: tool.id, ...log.errorFields(err) });
103
+ return undefined;
104
+ }
105
+ }
@@ -0,0 +1,21 @@
1
+ /**
2
+ * One classification of how a tool call ended, shared by every surface that
3
+ * reports it.
4
+ *
5
+ * The live loop reads it off the `ToolResultPart` it just built; the reload
6
+ * converter reads it off the same part after a round trip through storage. They
7
+ * resolve through this function so a reloaded call cannot report an outcome the
8
+ * live call never had — the defect this replaces was exactly that, with reload
9
+ * reporting success for every call because it never looked at the result at all.
10
+ */
11
+ import type { ToolOutcome } from "../contracts/chat-events.js";
12
+ export type { ToolOutcome };
13
+ /**
14
+ * Classify a tool result's output type.
15
+ *
16
+ * `execution-denied` is the user's own decision and stays distinct from a fault.
17
+ * `error-text` / `error-json` are the model-visible error results the loop
18
+ * builds from a thrown failure, an `err(ToolError)`, or rejected input.
19
+ * Everything else — including an expected "not found" data variant — is `ok`.
20
+ */
21
+ export declare function toolOutcomeForOutputType(outputType: string): ToolOutcome;
@@ -0,0 +1,25 @@
1
+ /**
2
+ * One classification of how a tool call ended, shared by every surface that
3
+ * reports it.
4
+ *
5
+ * The live loop reads it off the `ToolResultPart` it just built; the reload
6
+ * converter reads it off the same part after a round trip through storage. They
7
+ * resolve through this function so a reloaded call cannot report an outcome the
8
+ * live call never had — the defect this replaces was exactly that, with reload
9
+ * reporting success for every call because it never looked at the result at all.
10
+ */
11
+ /**
12
+ * Classify a tool result's output type.
13
+ *
14
+ * `execution-denied` is the user's own decision and stays distinct from a fault.
15
+ * `error-text` / `error-json` are the model-visible error results the loop
16
+ * builds from a thrown failure, an `err(ToolError)`, or rejected input.
17
+ * Everything else — including an expected "not found" data variant — is `ok`.
18
+ */
19
+ export function toolOutcomeForOutputType(outputType) {
20
+ if (outputType === "execution-denied")
21
+ return "denied";
22
+ if (outputType === "error-text" || outputType === "error-json")
23
+ return "error";
24
+ return "ok";
25
+ }
@@ -8,6 +8,7 @@
8
8
  * The loop body itself knows nothing about DBOS, HTTP, or memory.
9
9
  */
10
10
  import type { ModelMessage } from "ai";
11
+ import type { ToolCallDetail, ToolOutcome } from "../contracts/chat-events.js";
11
12
  import type { ChatStreamEvent } from "../providers/types.js";
12
13
  import type { Tool } from "../tools/define-tool.js";
13
14
  /**
@@ -61,12 +62,25 @@ export type EmitEvent = {
61
62
  readonly toolUseId: string;
62
63
  readonly name: string;
63
64
  readonly input: unknown;
65
+ /**
66
+ * One line naming what this call is doing, from the tool's own
67
+ * `describeCall` hook. Absent — never empty — when the tool declares no
68
+ * hook or the detail could not be produced (see `tool-detail.ts`).
69
+ */
70
+ readonly detail?: ToolCallDetail;
64
71
  } | {
65
72
  readonly type: "tool-finished";
66
73
  readonly source: EventSource;
67
74
  readonly toolUseId: string;
68
75
  readonly name: string;
69
- readonly isError: boolean;
76
+ /**
77
+ * How the call ended. Three states rather than an error boolean: a
78
+ * denial is the user's decision, not a fault, and the loop already
79
+ * treats the two differently in control flow.
80
+ */
81
+ readonly outcome: ToolOutcome;
82
+ /** The same detail the matching `tool-started` carried. */
83
+ readonly detail?: ToolCallDetail;
70
84
  };
71
85
  /**
72
86
  * A UI presentation event a tool streams to chat — an AI-SDK `data-*`