@inflexa-ai/harness 0.39.2 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/agents/conversation-agent.d.ts +3 -0
- package/dist/agents/conversation-agent.js +13 -2
- package/dist/agents/report-session-agent.d.ts +3 -0
- package/dist/agents/report-session-agent.js +2 -0
- package/dist/agents/sandbox/cheminformatics-agent.js +0 -3
- package/dist/agents/sandbox/shared.d.ts +11 -2
- package/dist/agents/sandbox/shared.js +4 -0
- package/dist/app/chat-turn.d.ts +104 -12
- package/dist/app/chat-turn.js +180 -21
- package/dist/app/message-assembly.d.ts +16 -52
- package/dist/app/message-assembly.js +41 -68
- package/dist/app/run-activity.d.ts +4 -6
- package/dist/app/run-activity.js +7 -27
- package/dist/app/spawn-report-session.d.ts +6 -11
- package/dist/app/spawn-report-session.js +5 -6
- package/dist/app/synthesize-run.d.ts +3 -0
- package/dist/app/synthesize-run.js +1 -0
- package/dist/billing/usage-recorder.d.ts +3 -4
- package/dist/billing/usage-recorder.js +3 -4
- package/dist/contracts/chat-parts.d.ts +11 -1
- package/dist/contracts/chat-parts.js +1 -0
- package/dist/contracts/index.d.ts +1 -1
- package/dist/contracts/part-registry.js +1 -0
- package/dist/contracts/report-blocks.d.ts +8 -8
- package/dist/contracts/schemas/chat-parts.d.ts +32 -9
- package/dist/contracts/schemas/chat-parts.js +10 -0
- package/dist/execution/artifact-metadata.d.ts +26 -33
- package/dist/execution/artifact-metadata.js +44 -106
- package/dist/execution/post-step-pipeline.d.ts +23 -16
- package/dist/execution/post-step-pipeline.js +19 -21
- package/dist/execution/run-synthesis.d.ts +3 -0
- package/dist/execution/run-synthesis.js +11 -1
- package/dist/execution/step-summary.d.ts +21 -30
- package/dist/execution/step-summary.js +30 -57
- package/dist/index.d.ts +9 -8
- package/dist/index.js +15 -34
- package/dist/lib/zod-issues.js +2 -4
- package/dist/loop/compaction.d.ts +20 -0
- package/dist/loop/compaction.js +2 -0
- package/dist/loop/continue-agent.d.ts +40 -0
- package/dist/loop/continue-agent.js +38 -0
- package/dist/loop/metrics.d.ts +30 -22
- package/dist/loop/metrics.js +51 -22
- package/dist/loop/run-agent.d.ts +111 -13
- package/dist/loop/run-agent.js +467 -220
- package/dist/loop/run-to-terminal.d.ts +16 -17
- package/dist/loop/run-to-terminal.js +24 -35
- package/dist/loop/tool-mask.d.ts +23 -0
- package/dist/loop/tool-mask.js +46 -0
- package/dist/loop/tool-output.d.ts +48 -0
- package/dist/loop/tool-output.js +79 -0
- package/dist/loop/types.d.ts +6 -3
- package/dist/memory/ai-sdk-message-storage.d.ts +49 -2
- package/dist/memory/ai-sdk-message-storage.js +78 -2
- package/dist/memory/conversation-display-recorder.d.ts +6 -0
- package/dist/memory/conversation-display-recorder.js +64 -22
- package/dist/memory/conversation-display-replay.d.ts +5 -14
- package/dist/memory/conversation-display-replay.js +66 -32
- package/dist/memory/conversation-display-storage.d.ts +2 -1
- package/dist/memory/conversation-display-storage.js +2 -1
- package/dist/memory/conversation-view.d.ts +53 -0
- package/dist/memory/conversation-view.js +130 -0
- package/dist/memory/count-tokens.d.ts +5 -7
- package/dist/memory/count-tokens.js +7 -9
- package/dist/memory/thread-history.d.ts +74 -102
- package/dist/memory/thread-history.js +181 -246
- package/dist/memory/thread-store.d.ts +3 -3
- package/dist/memory/thread-store.js +7 -2
- package/dist/memory/tool-call-integrity.d.ts +15 -21
- package/dist/memory/tool-call-integrity.js +31 -38
- package/dist/memory/working-memory.d.ts +2 -5
- package/dist/memory/working-memory.js +2 -5
- package/dist/prompts/SOUL.d.ts +2 -2
- package/dist/prompts/SOUL.js +2 -7
- package/dist/prompts/analogical-reasoner.d.ts +1 -1
- package/dist/prompts/analogical-reasoner.js +16 -32
- package/dist/prompts/briefing.js +1 -1
- package/dist/prompts/compaction.d.ts +4 -0
- package/dist/prompts/compaction.js +24 -0
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +26 -35
- package/dist/prompts/execute-analysis/step-summary.js +7 -5
- package/dist/prompts/literature-reviewer.d.ts +1 -1
- package/dist/prompts/literature-reviewer.js +29 -34
- package/dist/prompts/planner.js +15 -29
- package/dist/prompts/report-session.d.ts +1 -1
- package/dist/prompts/report-session.js +2 -33
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.js +1 -2
- package/dist/prompts/sandbox/cheminformatics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/cheminformatics-agent.js +10 -29
- package/dist/prompts/sandbox/chromatin-agent.d.ts +1 -1
- package/dist/prompts/sandbox/chromatin-agent.js +1 -2
- package/dist/prompts/sandbox/data-profiler.js +6 -3
- package/dist/prompts/sandbox/dna-methylation-agent.d.ts +1 -1
- package/dist/prompts/sandbox/dna-methylation-agent.js +2 -7
- package/dist/prompts/sandbox/drug-repurposing-agent.d.ts +1 -1
- package/dist/prompts/sandbox/drug-repurposing-agent.js +20 -24
- package/dist/prompts/sandbox/enrichment-agent.d.ts +1 -1
- package/dist/prompts/sandbox/enrichment-agent.js +3 -2
- package/dist/prompts/sandbox/genomic-variant-agent.d.ts +1 -1
- package/dist/prompts/sandbox/genomic-variant-agent.js +0 -2
- package/dist/prompts/sandbox/immune-profiling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/immune-profiling-agent.js +7 -14
- package/dist/prompts/sandbox/metabolomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/metabolomics-agent.js +0 -2
- package/dist/prompts/sandbox/microbiome-agent.d.ts +1 -1
- package/dist/prompts/sandbox/microbiome-agent.js +14 -21
- package/dist/prompts/sandbox/multi-omics-integration-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multi-omics-integration-agent.js +9 -19
- package/dist/prompts/sandbox/multimodal-sc-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multimodal-sc-agent.js +4 -5
- package/dist/prompts/sandbox/network-agent.d.ts +1 -1
- package/dist/prompts/sandbox/network-agent.js +7 -8
- package/dist/prompts/sandbox/pkpd-clinical-response-agent.d.ts +1 -1
- package/dist/prompts/sandbox/pkpd-clinical-response-agent.js +19 -19
- package/dist/prompts/sandbox/proteomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/proteomics-agent.js +3 -4
- package/dist/prompts/sandbox/scientific-executor.d.ts +1 -1
- package/dist/prompts/sandbox/scientific-executor.js +7 -10
- package/dist/prompts/sandbox/single-cell-agent.d.ts +1 -1
- package/dist/prompts/sandbox/single-cell-agent.js +2 -4
- package/dist/prompts/sandbox/spatial-omics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/spatial-omics-agent.js +3 -5
- package/dist/prompts/sandbox/statistical-modeling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/statistical-modeling-agent.js +1 -1
- package/dist/prompts/sandbox/translational-safety-agent.d.ts +1 -1
- package/dist/prompts/sandbox/translational-safety-agent.js +10 -19
- package/dist/prompts/sandbox-standards.d.ts +7 -7
- package/dist/prompts/sandbox-standards.js +74 -76
- package/dist/prompts/synthesis-agent.d.ts +1 -1
- package/dist/prompts/synthesis-agent.js +16 -42
- package/dist/providers/ai-sdk.d.ts +39 -6
- package/dist/providers/ai-sdk.js +117 -18
- package/dist/providers/errors.d.ts +2 -0
- package/dist/providers/errors.js +5 -0
- package/dist/providers/prompt-cache.d.ts +43 -41
- package/dist/providers/prompt-cache.js +47 -41
- package/dist/providers/reasoning.d.ts +10 -21
- package/dist/providers/reasoning.js +10 -21
- package/dist/providers/types.d.ts +21 -5
- package/dist/report-model/authoring-grammar.d.ts +2 -2
- package/dist/runtime/assemble.d.ts +9 -7
- package/dist/runtime/assemble.js +7 -3
- package/dist/sandbox/create-sandbox.d.ts +2 -2
- package/dist/sandbox/submit-exec.d.ts +1 -1
- package/dist/schemas/artifact-manifest.d.ts +4 -4
- package/dist/state/init.js +39 -4
- package/dist/state/purge-analysis.js +4 -0
- package/dist/state/tool-outputs.d.ts +4 -0
- package/dist/state/tool-outputs.js +30 -0
- package/dist/tasks/data-profile.d.ts +3 -0
- package/dist/tasks/data-profile.js +2 -0
- package/dist/tools/ad-hoc-router.d.ts +8 -0
- package/dist/tools/ad-hoc-router.js +21 -2
- package/dist/tools/display/show-user.d.ts +1 -1
- package/dist/tools/execute-analysis.d.ts +3 -0
- package/dist/tools/execute-analysis.js +9 -1
- package/dist/tools/read-tool-output.d.ts +7 -0
- package/dist/tools/read-tool-output.js +130 -0
- package/dist/tools/research/generate-analogy-report.d.ts +10 -26
- package/dist/tools/research/generate-analogy-report.js +76 -163
- package/dist/tools/research/generate-plan.d.ts +3 -0
- package/dist/tools/research/generate-plan.js +4 -1
- package/dist/tools/research/literature-reviewer.d.ts +3 -0
- package/dist/tools/research/literature-reviewer.js +3 -0
- package/dist/tools/sandbox/report-blocker.d.ts +18 -5
- package/dist/tools/sandbox/report-blocker.js +54 -6
- package/dist/tools/sandbox/submit-file-metadata.d.ts +30 -0
- package/dist/tools/sandbox/submit-file-metadata.js +67 -0
- package/dist/tools/start-report-session.d.ts +1 -3
- package/dist/tools/start-report-session.js +0 -2
- package/dist/tools/workspace/execute-command.js +7 -4
- package/dist/tools/workspace/result-bounds.d.ts +4 -4
- package/dist/tools/workspace/result-bounds.js +6 -5
- package/dist/tools/workspace/workspace-search.d.ts +1 -1
- package/dist/workflows/execute-analysis.d.ts +3 -0
- package/dist/workflows/execute-analysis.js +1 -0
- package/dist/workflows/sandbox-step.d.ts +30 -9
- package/dist/workflows/sandbox-step.js +37 -8
- package/package.json +9 -9
- package/dist/providers/anthropic.d.ts +0 -25
- package/dist/providers/anthropic.js +0 -28
|
@@ -37,6 +37,7 @@ import { decodeObjectString } from "../lib/zod-issues.js";
|
|
|
37
37
|
import { runToTerminal } from "../loop/run-to-terminal.js";
|
|
38
38
|
import { passthroughStep } from "../loop/run-step.js";
|
|
39
39
|
import { defineTool } from "../tools/define-tool.js";
|
|
40
|
+
import { createReadToolOutputTool } from "../tools/read-tool-output.js";
|
|
40
41
|
import { createReportBlockerToolFor } from "../tools/sandbox/report-blocker.js";
|
|
41
42
|
import { createLiteratureReviewerTool } from "../tools/research/literature-reviewer.js";
|
|
42
43
|
// ── Agent identity / budgets ────────────────────────────────────────
|
|
@@ -52,6 +53,11 @@ export const SYNTHESIS_AGENT_ID = "run-synthesizer";
|
|
|
52
53
|
* + draft + 1–2 submit/fix cycles + headroom.
|
|
53
54
|
*/
|
|
54
55
|
const SYNTHESIZER_MAX_ITERATIONS = 25;
|
|
56
|
+
/**
|
|
57
|
+
* Cap on `literature_reviewer` calls per run. Not named in the prompt: a
|
|
58
|
+
* model treats a stated call-count limit as a target to hit, not a maximum.
|
|
59
|
+
*/
|
|
60
|
+
const LITERATURE_REVIEWER_BUDGET = 3;
|
|
55
61
|
// ── Validation ──────────────────────────────────────────────────────
|
|
56
62
|
function zodIssuesToValidationIssues(error, input, rootPath = "synthesis") {
|
|
57
63
|
return error.issues.map((i) => ({
|
|
@@ -358,11 +364,13 @@ export async function generateRunSynthesis(input) {
|
|
|
358
364
|
bioKeys: input.bioKeys,
|
|
359
365
|
citationResolver: input.citationResolver,
|
|
360
366
|
usageRecorder: input.usageRecorder,
|
|
367
|
+
...(input.toolOutputStore ? { toolOutputStore: input.toolOutputStore } : {}),
|
|
361
368
|
});
|
|
362
369
|
const validateTool = buildValidateTool(innerCtx);
|
|
363
370
|
const submitTool = buildSubmitTool(holder, innerCtx);
|
|
364
371
|
const blockerTool = buildBlockerTool(holder);
|
|
365
|
-
const
|
|
372
|
+
const readTools = input.toolOutputStore ? [createReadToolOutputTool(input.toolOutputStore)] : [];
|
|
373
|
+
const tools = [validateTool, submitTool, blockerTool, reviewer, ...readTools];
|
|
366
374
|
const agent = {
|
|
367
375
|
id: SYNTHESIS_AGENT_ID,
|
|
368
376
|
systemPrompt: composeSystemPrompt(synthesisAgentPrompt),
|
|
@@ -380,6 +388,8 @@ export async function generateRunSynthesis(input) {
|
|
|
380
388
|
runStep: passthroughStep,
|
|
381
389
|
resolved: () => holder.outcome !== null,
|
|
382
390
|
usageRecorder: input.usageRecorder,
|
|
391
|
+
toolOutputStore: input.toolOutputStore,
|
|
392
|
+
toolBudget: { [reviewer.id]: LITERATURE_REVIEWER_BUDGET },
|
|
383
393
|
};
|
|
384
394
|
await runToTerminal(agent, [{ role: "user", content: prompt }], input.session, loopDeps, {
|
|
385
395
|
tools: [submitTool, blockerTool],
|
|
@@ -1,61 +1,52 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Per-step interpretive
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* scoped `read_file` so it grounds every quantitative claim in the persisted
|
|
6
|
-
* output files rather than confabulating from `execute_command` stdout.
|
|
2
|
+
* Per-step interpretive summary. Continues the step agent's conversation
|
|
3
|
+
* (`continueAgent`) so the request reuses the cached prefix and keeps
|
|
4
|
+
* signed thinking blocks valid.
|
|
7
5
|
*
|
|
8
6
|
* Contract:
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
* The summarizer reads any output file it needs, then emits the markdown
|
|
7
|
+
* - Sends the step agent's system prompt and tools, masked to `read_file`
|
|
8
|
+
* and `grep` in the step's output tree plus `read_tool_output`, and emits the markdown
|
|
12
9
|
* summary as its final assistant text.
|
|
13
10
|
* - Returns `{ stepId, agentId, markdown }` on non-empty final text;
|
|
14
|
-
* `undefined` on empty
|
|
15
|
-
*
|
|
16
|
-
* whether to write `output/summary.md`.
|
|
11
|
+
* `undefined` on empty text, on a throw, or when the request cap is hit
|
|
12
|
+
* before a final answer (a continuation runs no wrap-up). Non-fatal.
|
|
17
13
|
* - `Session` is taken explicitly (see the harness-durable-runtime spec).
|
|
18
14
|
*
|
|
19
|
-
* Honest empty: with no output artifacts the
|
|
15
|
+
* Honest empty: with no output artifacts the request directs the model to
|
|
20
16
|
* state plainly that the step produced no output files — no synthesized
|
|
21
17
|
* results. Blocked steps skip this producer entirely (handled upstream).
|
|
22
18
|
*
|
|
23
19
|
* Transcript-from-memory: per the harness-thread-store spec, workflow loops have no `messages`
|
|
24
20
|
* table; reconstruction from `operation_outputs` is read-side only. At
|
|
25
|
-
* step time the workflow body already holds the array — pass it in.
|
|
21
|
+
* step time the workflow body already holds the array — pass it in. The task
|
|
22
|
+
* always closes its unanswered calls, so the transcript needs no repair.
|
|
26
23
|
*/
|
|
27
24
|
import type { AgentSession } from "../auth/types.js";
|
|
28
|
-
import type
|
|
25
|
+
import { type ToolOutputStore } from "../loop/tool-output.js";
|
|
26
|
+
import type { AgentDefinition, LoopMessage } from "../loop/types.js";
|
|
29
27
|
import type { AgentChat } from "../providers/types.js";
|
|
30
|
-
import type { WorkspaceFilesystem } from "../workspace/filesystem.js";
|
|
31
28
|
import { type StepSummary } from "../schemas/step-summary.js";
|
|
32
29
|
import type { Logger } from "../lib/logger.js";
|
|
33
30
|
import type { UsageRecorder } from "../billing/usage-recorder.js";
|
|
34
31
|
export interface GenerateStepSummaryOptions {
|
|
35
32
|
/** Operational logging seam; omitted falls back to no-op. */
|
|
36
33
|
readonly logger?: Logger;
|
|
37
|
-
/** LLM usage-accounting seam for the
|
|
34
|
+
/** LLM usage-accounting seam for the continuation; omitted falls back to the no-op recorder. */
|
|
38
35
|
readonly usageRecorder?: UsageRecorder;
|
|
36
|
+
/** The store of the task, thus the loop of the continuation keeps its texts where the read tool of the agent reads them. */
|
|
37
|
+
readonly toolOutputStore?: ToolOutputStore;
|
|
38
|
+
/** The provider of the task, thus each request extends the prefix that the task cached. */
|
|
39
39
|
readonly provider: AgentChat;
|
|
40
40
|
readonly session: AgentSession;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
/** The step agent: the continuation sends its system prompt and its declared tools. */
|
|
42
|
+
readonly agent: AgentDefinition;
|
|
43
|
+
/** Task transcript, then the file-metadata exchange messages if one ran. */
|
|
44
|
+
readonly conversation: readonly LoopMessage[];
|
|
44
45
|
readonly artifactPaths: readonly string[];
|
|
45
|
-
/** Workspace read seam — backs the scoped `read_file` tool. */
|
|
46
|
-
readonly workspaceFs: WorkspaceFilesystem;
|
|
47
|
-
/** Absolute host path to the step's writable output tree — `read_file`'s working dir. */
|
|
48
|
-
readonly workingDir: string;
|
|
49
46
|
readonly stepId: string;
|
|
50
47
|
readonly agentId: string;
|
|
51
48
|
readonly runId: string;
|
|
52
|
-
readonly maxIterations?: number;
|
|
53
49
|
readonly signal?: AbortSignal;
|
|
54
50
|
}
|
|
55
|
-
/**
|
|
56
|
-
* Run the post-step summary loop on the harness provider against the supplied
|
|
57
|
-
* transcript, grounding claims in persisted files via `read_file`. Returns
|
|
58
|
-
* `undefined` on any non-fatal failure mode; the workflow body proceeds
|
|
59
|
-
* without `summary.md` in that case.
|
|
60
|
-
*/
|
|
51
|
+
/** Returns `undefined` on non-fatal failure instead of throwing. */
|
|
61
52
|
export declare function generateStepSummary(opts: GenerateStepSummaryOptions): Promise<StepSummary | undefined>;
|
|
@@ -1,35 +1,31 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Per-step interpretive
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* scoped `read_file` so it grounds every quantitative claim in the persisted
|
|
6
|
-
* output files rather than confabulating from `execute_command` stdout.
|
|
2
|
+
* Per-step interpretive summary. Continues the step agent's conversation
|
|
3
|
+
* (`continueAgent`) so the request reuses the cached prefix and keeps
|
|
4
|
+
* signed thinking blocks valid.
|
|
7
5
|
*
|
|
8
6
|
* Contract:
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
* The summarizer reads any output file it needs, then emits the markdown
|
|
7
|
+
* - Sends the step agent's system prompt and tools, masked to `read_file`
|
|
8
|
+
* and `grep` in the step's output tree plus `read_tool_output`, and emits the markdown
|
|
12
9
|
* summary as its final assistant text.
|
|
13
10
|
* - Returns `{ stepId, agentId, markdown }` on non-empty final text;
|
|
14
|
-
* `undefined` on empty
|
|
15
|
-
*
|
|
16
|
-
* whether to write `output/summary.md`.
|
|
11
|
+
* `undefined` on empty text, on a throw, or when the request cap is hit
|
|
12
|
+
* before a final answer (a continuation runs no wrap-up). Non-fatal.
|
|
17
13
|
* - `Session` is taken explicitly (see the harness-durable-runtime spec).
|
|
18
14
|
*
|
|
19
|
-
* Honest empty: with no output artifacts the
|
|
15
|
+
* Honest empty: with no output artifacts the request directs the model to
|
|
20
16
|
* state plainly that the step produced no output files — no synthesized
|
|
21
17
|
* results. Blocked steps skip this producer entirely (handled upstream).
|
|
22
18
|
*
|
|
23
19
|
* Transcript-from-memory: per the harness-thread-store spec, workflow loops have no `messages`
|
|
24
20
|
* table; reconstruction from `operation_outputs` is read-side only. At
|
|
25
|
-
* step time the workflow body already holds the array — pass it in.
|
|
21
|
+
* step time the workflow body already holds the array — pass it in. The task
|
|
22
|
+
* always closes its unanswered calls, so the transcript needs no repair.
|
|
26
23
|
*/
|
|
27
|
-
import {
|
|
28
|
-
import { finalText
|
|
24
|
+
import { continueAgent } from "../loop/continue-agent.js";
|
|
25
|
+
import { finalText } from "../loop/run-agent.js";
|
|
29
26
|
import { passthroughStep } from "../loop/run-step.js";
|
|
27
|
+
import { READ_TOOL_OUTPUT_TOOL_ID } from "../loop/tool-output.js";
|
|
30
28
|
import { stepSummaryPrompt } from "../prompts/execute-analysis/step-summary.js";
|
|
31
|
-
import { composeSystemPrompt } from "../agents/system-prompt.js";
|
|
32
|
-
import { createReadFileTool } from "../tools/workspace/read-file.js";
|
|
33
29
|
import { StepSummarySchema } from "../schemas/step-summary.js";
|
|
34
30
|
import { incrementSummaryNullCount } from "./step-summary-metrics.js";
|
|
35
31
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
@@ -40,59 +36,36 @@ Your prior conversation — tool calls, code output, intermediate results — is
|
|
|
40
36
|
Ground every quantitative claim in a PERSISTED output file: open it with the read_file tool and report what the file actually contains. A number that appears only in command stdout is NOT a citable result — read the artifact that holds it. If there are no output files, say so plainly; do not invent numbers and do not synthesize results no artifact backs.
|
|
41
37
|
|
|
42
38
|
When finished, write the markdown summary as your final message. It is stored as the step summary exactly as you write it.`;
|
|
43
|
-
/** Sub-agent identity for the summary loop — provenance only. */
|
|
44
39
|
const SUMMARY_AGENT_ID = "step-summary-writer";
|
|
45
|
-
/**
|
|
46
|
-
const
|
|
47
|
-
/**
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
*/
|
|
52
|
-
function sanitizeTranscript(messages) {
|
|
53
|
-
const out = [...messages];
|
|
54
|
-
while (out.length > 0) {
|
|
55
|
-
const last = out[out.length - 1];
|
|
56
|
-
const blocks = Array.isArray(last.content) ? last.content : [];
|
|
57
|
-
const hasOpenToolUse = last.role === "assistant" && blocks.some((b) => b.type === "tool-call");
|
|
58
|
-
if (hasOpenToolUse) {
|
|
59
|
-
out.pop();
|
|
60
|
-
continue;
|
|
61
|
-
}
|
|
62
|
-
break;
|
|
63
|
-
}
|
|
64
|
-
return out;
|
|
40
|
+
/** A handful of read_file reads plus the final write-up. */
|
|
41
|
+
const SUMMARY_MAX_REQUESTS = 12;
|
|
42
|
+
/** Must already be declared tools of the step agent — the mask only narrows. */
|
|
43
|
+
const SUMMARY_TOOLS = ["read_file", "grep", READ_TOOL_OUTPUT_TOOL_ID];
|
|
44
|
+
function summaryRequest(artifactPaths) {
|
|
45
|
+
return `${SYSTEM_PROMPT}\n\n${stepSummaryPrompt(artifactPaths.join("\n"))}`;
|
|
65
46
|
}
|
|
66
|
-
/**
|
|
67
|
-
* Run the post-step summary loop on the harness provider against the supplied
|
|
68
|
-
* transcript, grounding claims in persisted files via `read_file`. Returns
|
|
69
|
-
* `undefined` on any non-fatal failure mode; the workflow body proceeds
|
|
70
|
-
* without `summary.md` in that case.
|
|
71
|
-
*/
|
|
47
|
+
/** Returns `undefined` on non-fatal failure instead of throwing. */
|
|
72
48
|
export async function generateStepSummary(opts) {
|
|
73
49
|
const logger = (opts.logger ?? createNoopLogger()).named("step-summary").with({ runId: opts.runId, stepId: opts.stepId, agentId: opts.agentId });
|
|
74
|
-
const transcript = sanitizeTranscript(opts.messages);
|
|
75
|
-
const userPrompt = stepSummaryPrompt(opts.artifactPaths.join("\n"));
|
|
76
|
-
const writer = {
|
|
77
|
-
id: SUMMARY_AGENT_ID,
|
|
78
|
-
systemPrompt: composeSystemPrompt(SYSTEM_PROMPT),
|
|
79
|
-
model: opts.modelId,
|
|
80
|
-
tools: [createReadFileTool(opts.workspaceFs, opts.workingDir)],
|
|
81
|
-
maxIterations: opts.maxIterations ?? DEFAULT_MAX_ITERATIONS,
|
|
82
|
-
};
|
|
83
|
-
const signal = opts.signal ?? new AbortController().signal;
|
|
84
50
|
let result;
|
|
85
51
|
try {
|
|
86
|
-
result = await
|
|
52
|
+
result = await continueAgent(opts.agent, opts.conversation, {
|
|
53
|
+
text: summaryRequest(opts.artifactPaths),
|
|
54
|
+
mask: { allow: SUMMARY_TOOLS },
|
|
55
|
+
maxRequests: SUMMARY_MAX_REQUESTS,
|
|
56
|
+
stepNamespace: "step-summary",
|
|
57
|
+
accountingAgentId: SUMMARY_AGENT_ID,
|
|
58
|
+
}, opts.session, {
|
|
87
59
|
provider: opts.provider,
|
|
88
|
-
signal,
|
|
60
|
+
signal: opts.signal ?? new AbortController().signal,
|
|
89
61
|
emit: () => { },
|
|
90
62
|
runStep: passthroughStep,
|
|
91
63
|
usageRecorder: opts.usageRecorder,
|
|
64
|
+
toolOutputStore: opts.toolOutputStore,
|
|
92
65
|
});
|
|
93
66
|
}
|
|
94
67
|
catch (err) {
|
|
95
|
-
logger.warn("summary
|
|
68
|
+
logger.warn("summary continuation failed", logger.errorFields(err));
|
|
96
69
|
incrementSummaryNullCount(opts.agentId, "throw");
|
|
97
70
|
return undefined;
|
|
98
71
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -59,20 +59,21 @@ export type { AskGateway, AskGatewayDeps, AskContext, AnswerOutcome, PendingAsk
|
|
|
59
59
|
export { UnavailableAsk, AskRejectedError } from "./tools/approval/contract.js";
|
|
60
60
|
export type { Ask, AskRequest, AskReply, AskApproval } from "./tools/approval/contract.js";
|
|
61
61
|
export { runAgent, finalText } from "./loop/run-agent.js";
|
|
62
|
-
export type { RunAgentOptions, RunAgentResult, AgentFinish } from "./loop/run-agent.js";
|
|
62
|
+
export type { RunAgentOptions, RunAgentResult, AgentFinish, AgentRound } from "./loop/run-agent.js";
|
|
63
|
+
export type { CompactionPolicy } from "./loop/compaction.js";
|
|
64
|
+
export type { ToolMask } from "./loop/tool-mask.js";
|
|
63
65
|
export type { AgentDefinition, RunStep, EmitFn, EmitEvent, EventSource } from "./loop/types.js";
|
|
66
|
+
export type { KeptToolOutput, ToolOutputStore } from "./loop/tool-output.js";
|
|
64
67
|
export { passthroughStep } from "./loop/run-step.js";
|
|
65
68
|
export { forStep, forSubAgent, scopeResource, scopeWorkloadId } from "./auth/types.js";
|
|
66
69
|
export type { AgentSession, RequestSession, RunSession, Identity, Scope, Credential, Provenance, RunFrame, AuthContext, ResourceCoordinates, HookSessionView, ResolvableSession, } from "./auth/types.js";
|
|
67
70
|
export { makeLocalAuth } from "./auth/local-auth-context.js";
|
|
68
|
-
export { createAnthropicProvider } from "./providers/anthropic.js";
|
|
69
|
-
export type { AnthropicProviderDeps } from "./providers/anthropic.js";
|
|
70
71
|
export { createConfiguredAiSdkProvider, DEFAULT_MAX_OUTPUT_TOKENS } from "./providers/ai-sdk.js";
|
|
71
72
|
export type { AiSdkProviderConfig, ConfiguredAiSdkProviderDeps, ProviderHostPolicy } from "./providers/ai-sdk.js";
|
|
72
73
|
export { createEmbeddingProvider } from "./providers/embedding.js";
|
|
73
74
|
export type { EmbeddingProviderDeps } from "./providers/embedding.js";
|
|
74
75
|
export type { AgentChat, ChatProvider, EmbeddingProvider, ChatRequest, ChatResponse, ChatStreamEvent, ChatUsage, ModelMessage, PromptCachePolicy, ReasoningPolicy, } from "./providers/types.js";
|
|
75
|
-
export { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions, withPromptCacheBreakpoint } from "./providers/prompt-cache.js";
|
|
76
|
+
export { CONVERSATION_PROMPT_CACHE, DEFAULT_PROMPT_CACHE, promptCacheProviderOptions, withPromptCacheBreakpoint } from "./providers/prompt-cache.js";
|
|
76
77
|
export { DEFAULT_REASONING } from "./providers/reasoning.js";
|
|
77
78
|
export { toProviderError, isProviderError } from "./providers/errors.js";
|
|
78
79
|
export type { ProviderError } from "./providers/errors.js";
|
|
@@ -81,12 +82,12 @@ export type { SuspendOn } from "./providers/errors.js";
|
|
|
81
82
|
export { suspensionOfFailure } from "./workflows/suspension.js";
|
|
82
83
|
export type { Suspension } from "./workflows/suspension.js";
|
|
83
84
|
export { createStreamingChat } from "./providers/streaming-chat.js";
|
|
84
|
-
export { prepareChatTurn } from "./app/chat-turn.js";
|
|
85
|
-
export type { PrepareChatTurnDeps, PrepareChatTurnParams, PrepareChatTurnResult } from "./app/chat-turn.js";
|
|
85
|
+
export { DEFAULT_CONVERSATION_BUDGET, openChatTurn, prepareChatTurn, runChatTurn } from "./app/chat-turn.js";
|
|
86
|
+
export type { ChatTurnOutcome, ChatTurnRan, ChatTurnRefusal, ChatTurnResult, ChatTurnSession, OpenChatTurn, OpenChatTurnParams, PrepareChatTurnDeps, PrepareChatTurnParams, PrepareChatTurnResult, RunChatTurnDeps, RunChatTurnParams, RunOpenChatTurnParams, } from "./app/chat-turn.js";
|
|
86
87
|
export { createThreadStore } from "./memory/thread-store.js";
|
|
87
88
|
export type { ThreadStore, Thread, ThreadType, ThreadInputError, CreateThreadInput, ListThreadsInput, ThreadPage } from "./memory/thread-store.js";
|
|
88
89
|
export { createThreadHistory, conversationRecordTurn } from "./memory/thread-history.js";
|
|
89
|
-
export type { ThreadHistory, StoredMessage, RetractOutcome, ConversationTurn } from "./memory/thread-history.js";
|
|
90
|
+
export type { ThreadHistory, StoredMessage, RetractOutcome, ConversationTurn, TurnWrite, TurnWriteResult, TurnClose, TurnStatus, StoredTurnRecord, } from "./memory/thread-history.js";
|
|
90
91
|
export { syntheticUserMessage, syntheticRecordMessage, isSyntheticUserMessage, isSyntheticRecordMessage } from "./memory/ai-sdk-message-storage.js";
|
|
91
92
|
export { createConversationDisplayRecorder } from "./memory/conversation-display-recorder.js";
|
|
92
93
|
export type { ConversationDisplayRecorder, ConversationDisplayRecorderOptions } from "./memory/conversation-display-recorder.js";
|
|
@@ -95,7 +96,7 @@ export type { ConversationDisplayMetadata, ConversationUIData, ConversationUIMes
|
|
|
95
96
|
export { storedMessagesToCortex } from "./memory/conversation-display-replay.js";
|
|
96
97
|
export type { CortexChatEvent, TextDeltaEvent, ToolStartedEvent, ToolFinishedEvent, ToolOutcome, ToolCallOutcome, ToolCallDetail, FinishEvent, ChatErrorEvent, } from "./contracts/chat-events.js";
|
|
97
98
|
export type { TokenUsageRollup } from "./contracts/usage.js";
|
|
98
|
-
export type { CortexChatPart, PresentationPart, PresentationContent, PlanPart, RunCardPart, FileReferencePart, FileReferenceEntry, RunStartedPart, DagStatePart, DagStepState, StepStatus, StepActivityPart, StepPhase, StepFileTreePart, FileTreeEntry, StepOutputPart, StepOutputFile, StepSummaryPart, StepUsagePart, StepBlockedPart, RunSynthesisPart, SynthesizedFinding, BiologicalTheme, SynthesisProgressPart, SynthesisPhase, RunCompletedPart, RunCompletedFinding, RunFailedPart, } from "./contracts/chat-parts.js";
|
|
99
|
+
export type { CortexChatPart, PresentationPart, PresentationContent, PlanPart, RunCardPart, FileReferencePart, FileReferenceEntry, RunStartedPart, DagStatePart, DagStepState, StepStatus, StepActivityPart, StepPhase, StepFileTreePart, FileTreeEntry, StepOutputPart, StepOutputFile, StepSummaryPart, StepUsagePart, StepBlockedPart, RunSynthesisPart, SynthesizedFinding, BiologicalTheme, SynthesisProgressPart, SynthesisPhase, RunCompletedPart, RunCompletedFinding, RunFailedPart, CompactionPart, } from "./contracts/chat-parts.js";
|
|
99
100
|
export { PART_REGISTRY, isTransient, isReconciling, isSidebarPart } from "./contracts/part-registry.js";
|
|
100
101
|
export type { CortexChatPartType, PartDescriptor, PartEmitter, PartConsumer } from "./contracts/part-registry.js";
|
|
101
102
|
export { parseStructureUrl, alphafoldEntryUrl, alphafoldPredictionUrl } from "./contracts/structure-source.js";
|
package/dist/index.js
CHANGED
|
@@ -94,36 +94,18 @@ export { passthroughStep } from "./loop/run-step.js";
|
|
|
94
94
|
export { forStep, forSubAgent, scopeResource, scopeWorkloadId } from "./auth/types.js";
|
|
95
95
|
export { makeLocalAuth } from "./auth/local-auth-context.js";
|
|
96
96
|
// Providers.
|
|
97
|
-
|
|
98
|
-
//
|
|
99
|
-
// builds a `ChatProvider` from an `AiSdkProviderConfig` — the connection+model
|
|
100
|
-
// union discriminated over the `anthropic` and `openai-compatible` kinds — with
|
|
101
|
-
// `ConfiguredAiSdkProviderDeps` as its argument shape. `createAnthropicProvider`
|
|
102
|
-
// above is a convenience over this union's `anthropic` arm.
|
|
97
|
+
// `AiSdkProviderConfig` discriminates by `kind`: `anthropic`, `openai`, or
|
|
98
|
+
// `openai-compatible`.
|
|
103
99
|
export { createConfiguredAiSdkProvider, DEFAULT_MAX_OUTPUT_TOKENS } from "./providers/ai-sdk.js";
|
|
104
100
|
export { createEmbeddingProvider } from "./providers/embedding.js";
|
|
105
|
-
//
|
|
106
|
-
//
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
// can count it, in the namespace of each vendor that needs an explicit
|
|
114
|
-
// breakpoint (Anthropic and Bedrock; the OpenAI family and Gemini cache without
|
|
115
|
-
// one). Never attach `promptCacheProviderOptions` to a request itself — that
|
|
116
|
-
// emits a top-level field an intermediary cannot see, and a proxy that trims to
|
|
117
|
-
// Anthropic's cap of four breakpoints then sends five.
|
|
118
|
-
export { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions, withPromptCacheBreakpoint } from "./providers/prompt-cache.js";
|
|
119
|
-
// Reasoning depth. `ReasoningPolicy` is the vendor-neutral name for how deep a
|
|
120
|
-
// model reasons; a composition root sets it per run through
|
|
121
|
-
// `RunAgentOptions.reasoning`. It defaults to `DEFAULT_REASONING` — `xhigh` —
|
|
122
|
-
// because an agent loop drives tools over many iterations, and a shallow turn
|
|
123
|
-
// there wastes more calls than the deeper turn costs in tokens. The provider
|
|
124
|
-
// package resolves the name for the model that it is bound to, thus a model
|
|
125
|
-
// that accepts no `xhigh` gets the nearest name that it accepts. A host on a
|
|
126
|
-
// model with no reasoning support passes `"provider-default"`.
|
|
101
|
+
// Two cache markers (loop + `withPromptCacheBreakpoint`) count against a vendor's
|
|
102
|
+
// breakpoint cap (four for Anthropic). Never attach `promptCacheProviderOptions`
|
|
103
|
+
// to the request itself, or a proxy miscounts. The root chat loop gets
|
|
104
|
+
// `CONVERSATION_PROMPT_CACHE` (1h), because a person can reply minutes later.
|
|
105
|
+
export { CONVERSATION_PROMPT_CACHE, DEFAULT_PROMPT_CACHE, promptCacheProviderOptions, withPromptCacheBreakpoint } from "./providers/prompt-cache.js";
|
|
106
|
+
// Effort resolves in order: `ChatRequest.reasoning`, the provider config's
|
|
107
|
+
// `reasoning`, then `DEFAULT_REASONING`, downgraded to what the model accepts.
|
|
108
|
+
// A model with no reasoning support takes `"provider-default"`.
|
|
127
109
|
export { DEFAULT_REASONING } from "./providers/reasoning.js";
|
|
128
110
|
// Provider error channel. `ProviderError` is the value `chat`/`embed` fail
|
|
129
111
|
// with; `toProviderError` is its sole constructor. Exposed so an embedder
|
|
@@ -144,12 +126,11 @@ export { suspensionOfFailure } from "./workflows/suspension.js";
|
|
|
144
126
|
// `chatStream` and forwards each text delta to `onText`, so a same-process host
|
|
145
127
|
// gets live token streaming out of the otherwise non-streaming loop.
|
|
146
128
|
export { createStreamingChat } from "./providers/streaming-chat.js";
|
|
147
|
-
// Conversation turn + thread memory
|
|
148
|
-
//
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
|
|
152
|
-
export { prepareChatTurn } from "./app/chat-turn.js";
|
|
129
|
+
// Conversation turn + thread memory. A host runs a chat turn with `runChatTurn`
|
|
130
|
+
// and gives only its transport values: the harness stores the opening, each round,
|
|
131
|
+
// and the outcome. `appendTurn` stays for a record of a host. `createThreadStore`
|
|
132
|
+
// owns thread metadata (create/list/title).
|
|
133
|
+
export { DEFAULT_CONVERSATION_BUDGET, openChatTurn, prepareChatTurn, runChatTurn } from "./app/chat-turn.js";
|
|
153
134
|
export { createThreadStore } from "./memory/thread-store.js";
|
|
154
135
|
export { createThreadHistory, conversationRecordTurn } from "./memory/thread-history.js";
|
|
155
136
|
// The synthetic-message primitives. A host appends a record of out-of-band work (an analysis run's
|
package/dist/lib/zod-issues.js
CHANGED
|
@@ -201,10 +201,8 @@ function parseAs(raw, expected) {
|
|
|
201
201
|
* collects what came off, for the hint to quote back; it is empty when the
|
|
202
202
|
* string carried no recognized wrapper.
|
|
203
203
|
*
|
|
204
|
-
* Deliberately narrow
|
|
205
|
-
*
|
|
206
|
-
* a fragment with a *leading* `<parameter name=…>` opener is not a whole value
|
|
207
|
-
* with decoration on it, and pretending otherwise would invent content.
|
|
204
|
+
* Deliberately narrow: only these two shapes. A fragment with a *leading*
|
|
205
|
+
* `<parameter name=…>` opener is not salvageable without inventing content.
|
|
208
206
|
*/
|
|
209
207
|
function stripWrappers(raw) {
|
|
210
208
|
const removed = [];
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AgentChat } from "../providers/types.js";
|
|
2
|
+
import type { ToolMask } from "./tool-mask.js";
|
|
3
|
+
import type { LoopMessage } from "./types.js";
|
|
4
|
+
/** The cap of requests of one compaction exchange: up to 3 rounds of memory edits, and then the summary. */
|
|
5
|
+
export declare const COMPACTION_MAX_REQUESTS = 4;
|
|
6
|
+
/** How the loop compacts its conversation when the view passes the budget. */
|
|
7
|
+
export interface CompactionPolicy {
|
|
8
|
+
/** The budget of the view, in estimated tokens. */
|
|
9
|
+
readonly budget: number;
|
|
10
|
+
/** The provider of the conversation, with no text stream to the surface: the summary is no reply. */
|
|
11
|
+
readonly provider: AgentChat;
|
|
12
|
+
/** The text of the compaction request. */
|
|
13
|
+
readonly request: string;
|
|
14
|
+
/** The tools that can run in the exchange. */
|
|
15
|
+
readonly mask: ToolMask;
|
|
16
|
+
/** Keep the first turn, the seed of a report thread, in front of each view. */
|
|
17
|
+
readonly keepFirstTurn: boolean;
|
|
18
|
+
/** The context records that come after a new marker, for the view that starts at the marker. */
|
|
19
|
+
readonly recordsAfter: (view: readonly LoopMessage[]) => Promise<readonly LoopMessage[]>;
|
|
20
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A signed thinking block is bound to the exact prefix of its request, and the
|
|
3
|
+
* prompt cache keys on that same prefix.
|
|
4
|
+
* A continuation must extend the conversation, never replay it under a new
|
|
5
|
+
* system prompt or tool set.
|
|
6
|
+
*/
|
|
7
|
+
import type { AgentSession } from "../auth/types.js";
|
|
8
|
+
import { type AgentFinish, type RequestRefusal, type RunAgentOptions, type StepNameFormatter } from "./run-agent.js";
|
|
9
|
+
import type { ToolMask } from "./tool-mask.js";
|
|
10
|
+
import type { AgentDefinition, LoopMessage } from "./types.js";
|
|
11
|
+
export interface ContinuationRequest {
|
|
12
|
+
/** Appended as a synthetic user message. */
|
|
13
|
+
readonly text: string;
|
|
14
|
+
/** The requests still declare every tool of the agent, even when masked. */
|
|
15
|
+
readonly mask: ToolMask;
|
|
16
|
+
/** The continuation runs no wrap-up at its cap. */
|
|
17
|
+
readonly maxRequests: number;
|
|
18
|
+
/** Prefixes each step name (`<namespace>:<name>`) so its cache key cannot match the conversation's own. */
|
|
19
|
+
readonly stepNamespace: string;
|
|
20
|
+
/**
|
|
21
|
+
* When set, metrics and token counts are attributed to this id instead of
|
|
22
|
+
* `agent.id`, through `forSubAgent`. Omitted, both stay the agent's own.
|
|
23
|
+
*/
|
|
24
|
+
readonly accountingAgentId?: string;
|
|
25
|
+
/** A `provider` error with the status `400` or `413` ends the continuation with the finish reason `error`, and no throw. */
|
|
26
|
+
readonly endOnRequestRefusal?: boolean;
|
|
27
|
+
}
|
|
28
|
+
export interface ContinuationResult {
|
|
29
|
+
/** The new messages only, starting with the synthetic request. */
|
|
30
|
+
readonly messages: LoopMessage[];
|
|
31
|
+
readonly finish: AgentFinish;
|
|
32
|
+
/** The refusal that ended a continuation with `endOnRequestRefusal`. */
|
|
33
|
+
readonly refusal?: RequestRefusal;
|
|
34
|
+
}
|
|
35
|
+
export declare function namespacedStepNames(base: StepNameFormatter, namespace: string): StepNameFormatter;
|
|
36
|
+
/**
|
|
37
|
+
* `agent` and `opts` must match the conversation's own, since each request
|
|
38
|
+
* extends its cached prefix. `conversation` itself is never changed.
|
|
39
|
+
*/
|
|
40
|
+
export declare function continueAgent(agent: AgentDefinition, conversation: readonly LoopMessage[], request: ContinuationRequest, session: AgentSession, opts: RunAgentOptions): Promise<ContinuationResult>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A signed thinking block is bound to the exact prefix of its request, and the
|
|
3
|
+
* prompt cache keys on that same prefix.
|
|
4
|
+
* A continuation must extend the conversation, never replay it under a new
|
|
5
|
+
* system prompt or tool set.
|
|
6
|
+
*/
|
|
7
|
+
import { forSubAgent } from "../auth/types.js";
|
|
8
|
+
import { traceAgentRun } from "./genai-spans.js";
|
|
9
|
+
import { DEFAULT_STEP_NAME_FORMATTER, isTurnRoot, openLoop, } from "./run-agent.js";
|
|
10
|
+
export function namespacedStepNames(base, namespace) {
|
|
11
|
+
return {
|
|
12
|
+
llm: (i) => `${namespace}:${base.llm(i)}`,
|
|
13
|
+
tool: (name, id) => `${namespace}:${base.tool(name, id)}`,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* `agent` and `opts` must match the conversation's own, since each request
|
|
18
|
+
* extends its cached prefix. `conversation` itself is never changed.
|
|
19
|
+
*/
|
|
20
|
+
export function continueAgent(agent, conversation, request, session, opts) {
|
|
21
|
+
const accountingAgentId = request.accountingAgentId;
|
|
22
|
+
const runSession = accountingAgentId === undefined ? session : forSubAgent(session, accountingAgentId);
|
|
23
|
+
const metricAgentId = accountingAgentId ?? agent.id;
|
|
24
|
+
return traceAgentRun(metricAgentId, runSession, isTurnRoot(opts), async () => {
|
|
25
|
+
const loop = openLoop(agent, conversation, runSession, opts, metricAgentId);
|
|
26
|
+
const outcome = await loop.runSegment({
|
|
27
|
+
mask: request.mask,
|
|
28
|
+
maxRequests: request.maxRequests,
|
|
29
|
+
stepNames: namespacedStepNames(opts.formatStepName ?? DEFAULT_STEP_NAME_FORMATTER, request.stepNamespace),
|
|
30
|
+
firstIndex: 0,
|
|
31
|
+
requestText: request.text,
|
|
32
|
+
...(request.endOnRequestRefusal === true ? { endsOnRequestRefusal: true } : {}),
|
|
33
|
+
});
|
|
34
|
+
const result = outcome === "capped" ? await loop.endCapped() : outcome;
|
|
35
|
+
const refusal = outcome === "capped" ? undefined : outcome.refusal;
|
|
36
|
+
return { messages: result.messages.slice(conversation.length), finish: result.finish, ...(refusal === undefined ? {} : { refusal }) };
|
|
37
|
+
});
|
|
38
|
+
}
|
package/dist/loop/metrics.d.ts
CHANGED
|
@@ -1,25 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Agent-loop OTel metrics, exported through the global MeterProvider.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Each LLM call records its token counters when it completes. A durable step
|
|
5
|
+
* that a recovery replays returns its stored reply and skips the body, so a
|
|
6
|
+
* replay never double-counts a call.
|
|
7
|
+
*
|
|
8
|
+
* Cache-write tokens are billed at a premium and only pay off once something
|
|
9
|
+
* reads them. Each counter carries `agent_id`, plus `model` and `provider`
|
|
10
|
+
* when the response reports them. Session provenance is not a label: agents
|
|
11
|
+
* that share one root provenance also share one series. Providers do not
|
|
12
|
+
* agree on whether `outputTokens` includes the reasoning tokens, so the
|
|
13
|
+
* reasoning counter keeps a `provider` label.
|
|
14
|
+
*
|
|
15
|
+
* Each `runAgent` completion also records, with the `agent_id` label only:
|
|
5
16
|
* - cortex.harness.agent.iterations — histogram of LLM iterations per run.
|
|
6
17
|
* A creeping distribution is a leading indicator of prompt drift.
|
|
7
|
-
* - cortex.harness.agent.cap_hits
|
|
8
|
-
*
|
|
9
|
-
* - cortex.harness.agent.input_tokens / .output_tokens — counters, summed
|
|
10
|
-
* over every LLM call the run made (the wrap-up included).
|
|
11
|
-
* - cortex.harness.agent.cache_read_tokens — counter, prefix tokens served
|
|
12
|
-
* from the prompt cache.
|
|
13
|
-
* - cortex.harness.agent.cache_write_tokens — counter, prefix tokens written
|
|
14
|
-
* into it (billed at a premium; only pays off once something reads it).
|
|
18
|
+
* - cortex.harness.agent.cap_hits — counter, incremented once per run that
|
|
19
|
+
* exhausted `maxIterations` and took the forced wrap-up path.
|
|
15
20
|
*
|
|
16
21
|
* The two cache counters are what make prompt caching *observable*: the hit
|
|
17
22
|
* rate for an agent type is `cache_read_tokens / input_tokens` (the harness's
|
|
18
23
|
* `inputTokens` is the total billed prefix, cache reads included). A flat-zero
|
|
19
24
|
* `cache_read_tokens` against a non-zero `cache_write_tokens` means every write
|
|
20
25
|
* is being thrown away — either a cache defeater is shifting the prefix (see
|
|
21
|
-
* `providers/prompt-cache.ts`) or the endpoint ignores cache directives
|
|
22
|
-
* outright, as the Claude Max OAuth path does.
|
|
26
|
+
* `providers/prompt-cache.ts`) or the endpoint ignores cache directives.
|
|
23
27
|
*
|
|
24
28
|
* Token counters are recorded only from what a provider actually reports; a
|
|
25
29
|
* provider that reports no usage contributes nothing rather than zero.
|
|
@@ -29,16 +33,11 @@
|
|
|
29
33
|
* exporter from `harness/lib/otel.ts`; tests: an in-memory reader).
|
|
30
34
|
*/
|
|
31
35
|
import { type TokenUsageRollup } from "../contracts/usage.js";
|
|
32
|
-
import type { ChatUsage } from "../providers/types.js";
|
|
36
|
+
import type { ChatResponse, ChatUsage } from "../providers/types.js";
|
|
33
37
|
/**
|
|
34
|
-
* Token usage summed over
|
|
35
|
-
* `undefined` until
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
* The fold is wider than the instruments: `reasoningTokens` is carried for the
|
|
39
|
-
* usage records and the finish rollups, and deliberately has no OTel counter —
|
|
40
|
-
* whether a provider counts reasoning inside `outputTokens` varies, so a
|
|
41
|
-
* cross-agent counter would sum figures that do not mean the same thing.
|
|
38
|
+
* Token usage summed over LLM calls, either one `runAgent` run or a whole
|
|
39
|
+
* turn. A field stays `undefined` until a call reports it, so an unreported
|
|
40
|
+
* field never masquerades as a measured zero.
|
|
42
41
|
*/
|
|
43
42
|
export interface AgentRunUsage {
|
|
44
43
|
inputTokens?: number;
|
|
@@ -60,15 +59,24 @@ export declare function addChatUsage(total: AgentRunUsage, usage: ChatUsage | un
|
|
|
60
59
|
* at either end is how the two would come to disagree.
|
|
61
60
|
*/
|
|
62
61
|
export declare function hasReportedUsage(usage: AgentRunUsage | ChatUsage | TokenUsageRollup | undefined): boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Maps a step's response to grow its token counters. Use inside the step
|
|
64
|
+
* body, `provider.chat(...).map(countChatTokens(agentId))`. A call that gives
|
|
65
|
+
* `err` counts nothing.
|
|
66
|
+
*/
|
|
67
|
+
export declare function countChatTokens(agentId: string): (response: ChatResponse) => ChatResponse;
|
|
68
|
+
export declare function recordChatCall(call: {
|
|
69
|
+
readonly agentId: string;
|
|
70
|
+
readonly response: ChatResponse;
|
|
71
|
+
}): void;
|
|
63
72
|
/** Record one completed `runAgent` invocation. */
|
|
64
73
|
export declare function recordAgentRun(run: {
|
|
65
74
|
readonly agentId: string;
|
|
66
75
|
readonly iterations: number;
|
|
67
76
|
readonly cappedOut: boolean;
|
|
68
|
-
readonly usage?: AgentRunUsage;
|
|
69
77
|
}): void;
|
|
70
78
|
/**
|
|
71
|
-
* Drop the memoized instruments so the next
|
|
79
|
+
* Drop the memoized instruments so the next record rebinds to a
|
|
72
80
|
* freshly-registered `MeterProvider`. Test-only — production registers its
|
|
73
81
|
* provider once at startup.
|
|
74
82
|
*/
|