@inflexa-ai/harness 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -0
- package/dist/agents/conversation-agent.d.ts +15 -11
- package/dist/agents/conversation-agent.js +31 -34
- package/dist/agents/report-builder.js +2 -1
- package/dist/agents/sandbox/bulk-transcriptomics-agent.js +1 -1
- package/dist/agents/sandbox/cheminformatics-agent.js +1 -19
- package/dist/agents/sandbox/chromatin-agent.js +1 -10
- package/dist/agents/sandbox/dna-methylation-agent.js +1 -10
- package/dist/agents/sandbox/drug-repurposing-agent.js +4 -14
- package/dist/agents/sandbox/enrichment-agent.js +1 -10
- package/dist/agents/sandbox/ephemeral-executor.js +1 -10
- package/dist/agents/sandbox/genomic-variant-agent.js +1 -3
- package/dist/agents/sandbox/immune-profiling-agent.js +1 -13
- package/dist/agents/sandbox/metabolomics-agent.js +1 -1
- package/dist/agents/sandbox/microbiome-agent.js +1 -1
- package/dist/agents/sandbox/multi-omics-integration-agent.js +1 -12
- package/dist/agents/sandbox/multimodal-sc-agent.js +1 -1
- package/dist/agents/sandbox/network-agent.js +1 -10
- package/dist/agents/sandbox/pkpd-clinical-response-agent.js +2 -8
- package/dist/agents/sandbox/proteomics-agent.js +1 -1
- package/dist/agents/sandbox/scientific-executor.js +1 -10
- package/dist/agents/sandbox/shared.d.ts +20 -6
- package/dist/agents/sandbox/shared.js +50 -39
- package/dist/agents/sandbox/single-cell-agent.js +1 -1
- package/dist/agents/sandbox/spatial-omics-agent.js +1 -1
- package/dist/agents/sandbox/statistical-modeling-agent.js +1 -1
- package/dist/agents/sandbox/translational-safety-agent.js +3 -9
- package/dist/agents/sandbox/types.d.ts +1 -1
- package/dist/agents/system-prompt.d.ts +37 -11
- package/dist/agents/system-prompt.js +35 -12
- package/dist/app/chat-turn.d.ts +3 -0
- package/dist/app/chat-turn.js +3 -1
- package/dist/app/data-profile-orientation.d.ts +34 -0
- package/dist/app/data-profile-orientation.js +117 -0
- package/dist/app/synthesize-run.d.ts +3 -0
- package/dist/app/synthesize-run.js +3 -1
- package/dist/contracts/schemas/chat-parts.d.ts +9 -9
- package/dist/execution/artifact-metadata.d.ts +3 -0
- package/dist/execution/artifact-metadata.js +10 -3
- package/dist/execution/artifact-registration.d.ts +2 -1
- package/dist/execution/artifact-registration.js +8 -2
- package/dist/execution/ephemeral-runner.d.ts +16 -0
- package/dist/execution/ephemeral-runner.js +31 -5
- package/dist/execution/post-step-pipeline.d.ts +3 -0
- package/dist/execution/post-step-pipeline.js +24 -4
- package/dist/execution/reconcile-manifest.d.ts +7 -0
- package/dist/execution/reconcile-manifest.js +19 -5
- package/dist/execution/report-runner.d.ts +3 -0
- package/dist/execution/report-runner.js +4 -4
- package/dist/execution/run-synthesis.js +20 -19
- package/dist/execution/step-summary.d.ts +3 -0
- package/dist/execution/step-summary.js +6 -3
- package/dist/index.d.ts +9 -2
- package/dist/index.js +24 -5
- package/dist/lib/chrome.d.ts +4 -1
- package/dist/lib/chrome.js +10 -7
- package/dist/lib/console-logger.d.ts +19 -0
- package/dist/lib/console-logger.js +62 -0
- package/dist/lib/logger.d.ts +88 -0
- package/dist/lib/logger.js +37 -0
- package/dist/lib/otel.d.ts +2 -1
- package/dist/lib/otel.js +8 -3
- package/dist/loop/__fixtures__/scripted-provider.d.ts +2 -2
- package/dist/loop/__fixtures__/scripted-provider.js +2 -1
- package/dist/loop/metrics.d.ts +33 -1
- package/dist/loop/metrics.js +56 -2
- package/dist/loop/run-agent.d.ts +13 -1
- package/dist/loop/run-agent.js +19 -4
- package/dist/memory/card-builders.d.ts +8 -1
- package/dist/memory/card-builders.js +20 -3
- package/dist/memory/reconstruct-cards.js +4 -3
- package/dist/memory/thread-history.d.ts +27 -2
- package/dist/memory/thread-history.js +45 -5
- package/dist/memory/working-memory.d.ts +93 -10
- package/dist/memory/working-memory.js +220 -84
- package/dist/prompts/SOUL.d.ts +22 -9
- package/dist/prompts/SOUL.js +56 -37
- package/dist/prompts/analogical-reasoner.d.ts +1 -1
- package/dist/prompts/analogical-reasoner.js +2 -2
- package/dist/prompts/briefing.d.ts +100 -0
- package/dist/prompts/briefing.js +166 -0
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +94 -440
- package/dist/prompts/literature-reviewer.d.ts +1 -1
- package/dist/prompts/literature-reviewer.js +10 -10
- package/dist/prompts/planner.js +28 -59
- package/dist/prompts/report-builder.d.ts +1 -1
- package/dist/prompts/report-builder.js +71 -24
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.js +2 -7
- package/dist/prompts/sandbox/cheminformatics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/cheminformatics-agent.js +11 -15
- package/dist/prompts/sandbox/chromatin-agent.d.ts +1 -1
- package/dist/prompts/sandbox/chromatin-agent.js +3 -7
- package/dist/prompts/sandbox/data-profiler.d.ts +1 -1
- package/dist/prompts/sandbox/data-profiler.js +13 -8
- package/dist/prompts/sandbox/dna-methylation-agent.d.ts +1 -1
- package/dist/prompts/sandbox/dna-methylation-agent.js +2 -7
- package/dist/prompts/sandbox/drug-repurposing-agent.d.ts +1 -1
- package/dist/prompts/sandbox/drug-repurposing-agent.js +15 -16
- package/dist/prompts/sandbox/enrichment-agent.d.ts +1 -1
- package/dist/prompts/sandbox/enrichment-agent.js +2 -9
- package/dist/prompts/sandbox/genomic-variant-agent.d.ts +1 -1
- package/dist/prompts/sandbox/genomic-variant-agent.js +2 -6
- package/dist/prompts/sandbox/immune-profiling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/immune-profiling-agent.js +3 -8
- package/dist/prompts/sandbox/metabolomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/metabolomics-agent.js +1 -7
- package/dist/prompts/sandbox/microbiome-agent.d.ts +1 -1
- package/dist/prompts/sandbox/microbiome-agent.js +2 -7
- package/dist/prompts/sandbox/multi-omics-integration-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multi-omics-integration-agent.js +2 -8
- package/dist/prompts/sandbox/multimodal-sc-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multimodal-sc-agent.js +2 -7
- package/dist/prompts/sandbox/network-agent.d.ts +1 -1
- package/dist/prompts/sandbox/network-agent.js +2 -7
- package/dist/prompts/sandbox/proteomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/proteomics-agent.js +2 -9
- package/dist/prompts/sandbox/single-cell-agent.d.ts +1 -1
- package/dist/prompts/sandbox/single-cell-agent.js +2 -9
- package/dist/prompts/sandbox/spatial-omics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/spatial-omics-agent.js +3 -6
- package/dist/prompts/sandbox/statistical-modeling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/statistical-modeling-agent.js +2 -7
- package/dist/prompts/sandbox/translational-safety-agent.d.ts +1 -1
- package/dist/prompts/sandbox/translational-safety-agent.js +13 -14
- package/dist/prompts/sandbox-standards.d.ts +22 -14
- package/dist/prompts/sandbox-standards.js +91 -71
- package/dist/prompts/synthesis-agent.d.ts +1 -1
- package/dist/prompts/synthesis-agent.js +33 -131
- package/dist/provenance/collector.js +1 -1
- package/dist/providers/ai-sdk.js +31 -4
- package/dist/providers/prompt-cache.d.ts +74 -0
- package/dist/providers/prompt-cache.js +77 -0
- package/dist/providers/types.d.ts +33 -0
- package/dist/reference-data/catalog.d.ts +131 -0
- package/dist/reference-data/catalog.js +348 -0
- package/dist/reference-data/receipt.d.ts +41 -0
- package/dist/reference-data/receipt.js +40 -0
- package/dist/runtime/boot.d.ts +2 -2
- package/dist/runtime/boot.js +1 -1
- package/dist/runtime/connection-budget.d.ts +2 -2
- package/dist/runtime/connection-budget.js +5 -3
- package/dist/runtime/dbos.d.ts +7 -7
- package/dist/runtime/dbos.js +14 -11
- package/dist/runtime/shutdown.d.ts +2 -2
- package/dist/runtime/shutdown.js +4 -3
- package/dist/sandbox/create-sandbox.d.ts +2 -2
- package/dist/sandbox/docker-client.d.ts +2 -2
- package/dist/sandbox/docker-client.js +4 -1
- package/dist/sandbox/k8s-client.d.ts +3 -0
- package/dist/sandbox/k8s-client.js +5 -4
- package/dist/sandbox/notification-sweep.d.ts +3 -3
- package/dist/sandbox/notification-sweep.js +4 -2
- package/dist/sandbox/reaper.d.ts +3 -3
- package/dist/sandbox/reaper.js +7 -4
- package/dist/sandbox/watchdog.d.ts +3 -3
- package/dist/sandbox/watchdog.js +8 -2
- package/dist/schemas/data-profile-schemas.js +5 -3
- package/dist/schemas/plan-schemas.d.ts +21 -21
- package/dist/schemas/plan-schemas.js +9 -3
- package/dist/schemas/run-synthesis.d.ts +8 -4
- package/dist/schemas/run-synthesis.js +92 -56
- package/dist/schemas/validate-plan.js +1 -1
- package/dist/schemas/workflow-state.d.ts +7 -1
- package/dist/schemas/workflow-state.js +33 -19
- package/dist/state/artifacts.d.ts +13 -0
- package/dist/state/artifacts.js +27 -0
- package/dist/state/data-profile.d.ts +64 -5
- package/dist/state/index.d.ts +3 -3
- package/dist/state/index.js +1 -1
- package/dist/state/init.d.ts +2 -1
- package/dist/state/init.js +5 -6
- package/dist/tasks/data-profile.d.ts +25 -2
- package/dist/tasks/data-profile.js +95 -39
- package/dist/tools/bio/chembl.d.ts +38 -0
- package/dist/tools/bio/chembl.js +138 -0
- package/dist/tools/bio/comptox.d.ts +50 -0
- package/dist/tools/bio/comptox.js +149 -0
- package/dist/tools/bio/get-impc-ko-profile.js +9 -6
- package/dist/tools/bio/index.d.ts +5 -17
- package/dist/tools/bio/index.js +5 -17
- package/dist/tools/bio/keys.d.ts +14 -9
- package/dist/tools/bio/keys.js +16 -16
- package/dist/tools/bio/opentargets.d.ts +38 -0
- package/dist/tools/bio/opentargets.js +89 -0
- package/dist/tools/bio/pubchem.d.ts +29 -0
- package/dist/tools/bio/pubchem.js +92 -0
- package/dist/tools/bio/pubmed.d.ts +51 -0
- package/dist/tools/bio/pubmed.js +100 -0
- package/dist/tools/bio/search-bgee-expression.js +12 -6
- package/dist/tools/bio/search-clinical-trials.js +16 -5
- package/dist/tools/bio/search-dgidb.d.ts +11 -1
- package/dist/tools/bio/search-dgidb.js +92 -78
- package/dist/tools/bio/search-faers.js +8 -4
- package/dist/tools/bio/search-geo-datasets.js +20 -6
- package/dist/tools/bio/search-gwas-catalog.d.ts +1 -1
- package/dist/tools/bio/search-pharmgkb.js +10 -4
- package/dist/tools/display/index.d.ts +1 -0
- package/dist/tools/display/index.js +1 -0
- package/dist/tools/display/normalize-echart-spec.d.ts +61 -0
- package/dist/tools/display/normalize-echart-spec.js +211 -0
- package/dist/tools/display/show-user.d.ts +6 -1
- package/dist/tools/display/show-user.js +25 -12
- package/dist/tools/execute-plan.js +5 -2
- package/dist/tools/iterate-report.d.ts +171 -118
- package/dist/tools/iterate-report.js +336 -96
- package/dist/tools/lib/chembl-client.d.ts +12 -0
- package/dist/tools/lib/chembl-client.js +127 -0
- package/dist/tools/lib/ctx-ops.d.ts +224 -0
- package/dist/tools/lib/ctx-ops.js +450 -0
- package/dist/tools/lib/iuphar-client.js +10 -1
- package/dist/tools/lib/openfda-client.js +11 -3
- package/dist/tools/lib/pathway-client.js +10 -4
- package/dist/tools/lib/pubchem-ops.d.ts +74 -0
- package/dist/tools/lib/pubchem-ops.js +224 -0
- package/dist/tools/memory/update-working-memory.d.ts +45 -7
- package/dist/tools/memory/update-working-memory.js +111 -41
- package/dist/tools/report/index.d.ts +3 -2
- package/dist/tools/report/index.js +3 -2
- package/dist/tools/research/generate-analogy-report.js +25 -12
- package/dist/tools/research/generate-plan.d.ts +12 -5
- package/dist/tools/research/generate-plan.js +168 -37
- package/dist/tools/research/index.d.ts +1 -0
- package/dist/tools/research/index.js +1 -0
- package/dist/tools/research/inspect-data-profile.d.ts +85 -0
- package/dist/tools/research/inspect-data-profile.js +161 -0
- package/dist/tools/research/literature-reviewer.d.ts +3 -0
- package/dist/tools/research/literature-reviewer.js +5 -6
- package/dist/tools/sandbox/list-available-packages.d.ts +79 -5
- package/dist/tools/sandbox/list-available-packages.js +152 -10
- package/dist/tools/sandbox/list-available-refs.d.ts +141 -5
- package/dist/tools/sandbox/list-available-refs.js +647 -134
- package/dist/tools/sandbox/report-blocker.d.ts +39 -9
- package/dist/tools/sandbox/report-blocker.js +43 -20
- package/dist/tools/workspace/execute-command.d.ts +3 -0
- package/dist/tools/workspace/execute-command.js +3 -1
- package/dist/tools/workspace/read-file.js +4 -5
- package/dist/tools/workspace/run-exec.d.ts +4 -4
- package/dist/tools/workspace/run-exec.js +4 -4
- package/dist/tools/workspace/show-file.js +10 -6
- package/dist/tools/workspace/show-plan.js +12 -6
- package/dist/tools/workspace/workspace-search.d.ts +1 -1
- package/dist/tools/workspace/workspace-search.js +11 -5
- package/dist/workflows/execute-analysis.d.ts +46 -4
- package/dist/workflows/execute-analysis.js +118 -13
- package/dist/workflows/execute-target-assessment.d.ts +3 -0
- package/dist/workflows/execute-target-assessment.js +19 -15
- package/dist/workflows/sandbox-step.d.ts +9 -0
- package/dist/workflows/sandbox-step.js +39 -17
- package/dist/workflows/target-assessment/lib/citation-validator.js +2 -2
- package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.js +2 -1
- package/dist/workflows/target-assessment/lib/llm-step.d.ts +3 -0
- package/dist/workflows/target-assessment/lib/llm-step.js +3 -1
- package/dist/workflows/target-assessment/lib/structured-llm.d.ts +3 -0
- package/dist/workflows/target-assessment/lib/structured-llm.js +1 -0
- package/dist/workflows/target-assessment/phase5-persist.d.ts +3 -0
- package/dist/workflows/target-assessment/phase5-persist.js +8 -3
- package/dist/workflows/target-assessment/progress.d.ts +2 -1
- package/dist/workflows/target-assessment/progress.js +4 -3
- package/dist/workspace/filesystem.js +163 -74
- package/dist/workspace/paths.d.ts +2 -2
- package/dist/workspace/paths.js +2 -2
- package/package.json +95 -96
- package/dist/execution/filesystem-artifact-registry.d.ts +0 -57
- package/dist/execution/filesystem-artifact-registry.js +0 -92
- package/dist/lib/identifiers.d.ts +0 -8
- package/dist/lib/identifiers.js +0 -10
- package/dist/lib/mime-detect.d.ts +0 -17
- package/dist/lib/mime-detect.js +0 -29
- package/dist/lib/streamed-download.d.ts +0 -28
- package/dist/lib/streamed-download.js +0 -111
- package/dist/lib/trace-context.d.ts +0 -11
- package/dist/lib/trace-context.js +0 -49
- package/dist/providers/llm-capabilities.d.ts +0 -33
- package/dist/providers/llm-capabilities.js +0 -58
- package/dist/schemas/render-step-prompt.d.ts +0 -32
- package/dist/schemas/render-step-prompt.js +0 -76
- package/dist/tools/bio/find-approval-precedent.d.ts +0 -32
- package/dist/tools/bio/find-approval-precedent.js +0 -72
- package/dist/tools/bio/get-article-details.d.ts +0 -11
- package/dist/tools/bio/get-article-details.js +0 -20
- package/dist/tools/bio/get-article-full-text.d.ts +0 -19
- package/dist/tools/bio/get-article-full-text.js +0 -32
- package/dist/tools/bio/get-bioactivity.d.ts +0 -11
- package/dist/tools/bio/get-bioactivity.js +0 -23
- package/dist/tools/bio/get-drug-info.d.ts +0 -9
- package/dist/tools/bio/get-drug-info.js +0 -22
- package/dist/tools/bio/get-mechanism.d.ts +0 -8
- package/dist/tools/bio/get-mechanism.js +0 -18
- package/dist/tools/bio/get-pubchem-assays.d.ts +0 -22
- package/dist/tools/bio/get-pubchem-assays.js +0 -107
- package/dist/tools/bio/get-pubchem-cross-refs.d.ts +0 -15
- package/dist/tools/bio/get-pubchem-cross-refs.js +0 -68
- package/dist/tools/bio/get-target-safety.d.ts +0 -18
- package/dist/tools/bio/get-target-safety.js +0 -26
- package/dist/tools/bio/search-compounds.d.ts +0 -22
- package/dist/tools/bio/search-compounds.js +0 -160
- package/dist/tools/bio/search-ctx-chemical.d.ts +0 -50
- package/dist/tools/bio/search-ctx-chemical.js +0 -130
- package/dist/tools/bio/search-ctx-exposure.d.ts +0 -55
- package/dist/tools/bio/search-ctx-exposure.js +0 -176
- package/dist/tools/bio/search-ctx-hazard.d.ts +0 -111
- package/dist/tools/bio/search-ctx-hazard.js +0 -178
- package/dist/tools/bio/search-opentargets.d.ts +0 -11
- package/dist/tools/bio/search-opentargets.js +0 -25
- package/dist/tools/bio/search-pubchem-compound.d.ts +0 -26
- package/dist/tools/bio/search-pubchem-compound.js +0 -110
- package/dist/tools/bio/search-pubmed.d.ts +0 -17
- package/dist/tools/bio/search-pubmed.js +0 -31
- package/dist/tools/bio/search-targets.d.ts +0 -9
- package/dist/tools/bio/search-targets.js +0 -20
- package/dist/tools/bio/search-toxcast.d.ts +0 -36
- package/dist/tools/bio/search-toxcast.js +0 -141
- package/dist/workflows/register-workflows.d.ts +0 -51
- package/dist/workflows/register-workflows.js +0 -48
- package/dist/workflows/resume-execute-analysis.d.ts +0 -70
- package/dist/workflows/resume-execute-analysis.js +0 -80
- package/dist/workflows/target-assessment/correction-loop.d.ts +0 -45
- package/dist/workflows/target-assessment/correction-loop.js +0 -62
- package/dist/workflows/target-assessment/steps/phase0-resolve.d.ts +0 -71
- package/dist/workflows/target-assessment/steps/phase0-resolve.js +0 -21
- package/dist/workflows/target-assessment/steps/phase1-aggregate.d.ts +0 -1018
- package/dist/workflows/target-assessment/steps/phase1-aggregate.js +0 -25
- package/dist/workspace/mount-strategy.d.ts +0 -50
- package/dist/workspace/mount-strategy.js +0 -67
- package/dist/workspace/provenance-collector.d.ts +0 -34
- package/dist/workspace/provenance-collector.js +0 -16
- package/dist/workspace/workspace-paths.d.ts +0 -26
- package/dist/workspace/workspace-paths.js +0 -31
package/dist/loop/metrics.d.ts
CHANGED
|
@@ -1,21 +1,53 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Agent-loop OTel metrics → Middleware.io.
|
|
3
3
|
*
|
|
4
|
-
* Every `runAgent` completion records
|
|
4
|
+
* Every `runAgent` completion records, keyed by `agent_id`:
|
|
5
5
|
* - cortex.harness.agent.iterations — histogram of LLM iterations per run.
|
|
6
6
|
* A creeping distribution is a leading indicator of prompt drift.
|
|
7
7
|
* - cortex.harness.agent.cap_hits — counter, incremented once per run
|
|
8
8
|
* that exhausted `maxIterations` and took the forced wrap-up path.
|
|
9
|
+
* - cortex.harness.agent.input_tokens / .output_tokens — counters, summed
|
|
10
|
+
* over every LLM call the run made (the wrap-up included).
|
|
11
|
+
* - cortex.harness.agent.cache_read_tokens — counter, prefix tokens served
|
|
12
|
+
* from the prompt cache.
|
|
13
|
+
* - cortex.harness.agent.cache_write_tokens — counter, prefix tokens written
|
|
14
|
+
* into it (billed at a premium; only pays off once something reads it).
|
|
15
|
+
*
|
|
16
|
+
* The two cache counters are what make prompt caching *observable*: the hit
|
|
17
|
+
* rate for an agent type is `cache_read_tokens / input_tokens` (the harness's
|
|
18
|
+
* `inputTokens` is the total billed prefix, cache reads included). A flat-zero
|
|
19
|
+
* `cache_read_tokens` against a non-zero `cache_write_tokens` means every write
|
|
20
|
+
* is being thrown away — either a cache defeater is shifting the prefix (see
|
|
21
|
+
* `providers/prompt-cache.ts`) or the endpoint ignores cache directives
|
|
22
|
+
* outright, as the Claude Max OAuth path does.
|
|
23
|
+
*
|
|
24
|
+
* Token counters are recorded only from what a provider actually reports; a
|
|
25
|
+
* provider that reports no usage contributes nothing rather than zero.
|
|
9
26
|
*
|
|
10
27
|
* Instruments are created lazily on first use so the loop binds to whatever
|
|
11
28
|
* `MeterProvider` is registered globally at runtime (production: the OTLP
|
|
12
29
|
* exporter from `harness/lib/otel.ts`; tests: an in-memory reader).
|
|
13
30
|
*/
|
|
31
|
+
import type { ChatUsage } from "../providers/types.js";
|
|
32
|
+
/**
|
|
33
|
+
* Token usage summed over every LLM call one `runAgent` made. Each field stays
|
|
34
|
+
* `undefined` until some call actually reports it, so "the provider told us
|
|
35
|
+
* nothing" never masquerades as a measured zero.
|
|
36
|
+
*/
|
|
37
|
+
export interface AgentRunUsage {
|
|
38
|
+
inputTokens?: number;
|
|
39
|
+
outputTokens?: number;
|
|
40
|
+
cacheCreationInputTokens?: number;
|
|
41
|
+
cacheReadInputTokens?: number;
|
|
42
|
+
}
|
|
43
|
+
/** Fold one call's reported usage into a run's running total. */
|
|
44
|
+
export declare function addChatUsage(total: AgentRunUsage, usage: ChatUsage | undefined): void;
|
|
14
45
|
/** Record one completed `runAgent` invocation. */
|
|
15
46
|
export declare function recordAgentRun(run: {
|
|
16
47
|
readonly agentId: string;
|
|
17
48
|
readonly iterations: number;
|
|
18
49
|
readonly cappedOut: boolean;
|
|
50
|
+
readonly usage?: AgentRunUsage;
|
|
19
51
|
}): void;
|
|
20
52
|
/**
|
|
21
53
|
* Drop the memoized instruments so the next `recordAgentRun` rebinds to a
|
package/dist/loop/metrics.js
CHANGED
|
@@ -1,11 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Agent-loop OTel metrics → Middleware.io.
|
|
3
3
|
*
|
|
4
|
-
* Every `runAgent` completion records
|
|
4
|
+
* Every `runAgent` completion records, keyed by `agent_id`:
|
|
5
5
|
* - cortex.harness.agent.iterations — histogram of LLM iterations per run.
|
|
6
6
|
* A creeping distribution is a leading indicator of prompt drift.
|
|
7
7
|
* - cortex.harness.agent.cap_hits — counter, incremented once per run
|
|
8
8
|
* that exhausted `maxIterations` and took the forced wrap-up path.
|
|
9
|
+
* - cortex.harness.agent.input_tokens / .output_tokens — counters, summed
|
|
10
|
+
* over every LLM call the run made (the wrap-up included).
|
|
11
|
+
* - cortex.harness.agent.cache_read_tokens — counter, prefix tokens served
|
|
12
|
+
* from the prompt cache.
|
|
13
|
+
* - cortex.harness.agent.cache_write_tokens — counter, prefix tokens written
|
|
14
|
+
* into it (billed at a premium; only pays off once something reads it).
|
|
15
|
+
*
|
|
16
|
+
* The two cache counters are what make prompt caching *observable*: the hit
|
|
17
|
+
* rate for an agent type is `cache_read_tokens / input_tokens` (the harness's
|
|
18
|
+
* `inputTokens` is the total billed prefix, cache reads included). A flat-zero
|
|
19
|
+
* `cache_read_tokens` against a non-zero `cache_write_tokens` means every write
|
|
20
|
+
* is being thrown away — either a cache defeater is shifting the prefix (see
|
|
21
|
+
* `providers/prompt-cache.ts`) or the endpoint ignores cache directives
|
|
22
|
+
* outright, as the Claude Max OAuth path does.
|
|
23
|
+
*
|
|
24
|
+
* Token counters are recorded only from what a provider actually reports; a
|
|
25
|
+
* provider that reports no usage contributes nothing rather than zero.
|
|
9
26
|
*
|
|
10
27
|
* Instruments are created lazily on first use so the loop binds to whatever
|
|
11
28
|
* `MeterProvider` is registered globally at runtime (production: the OTLP
|
|
@@ -24,17 +41,54 @@ function getInstruments() {
|
|
|
24
41
|
capHits: meter.createCounter("cortex.harness.agent.cap_hits", {
|
|
25
42
|
description: "runAgent completions that exhausted maxIterations and took the " + "forced tool-less wrap-up path",
|
|
26
43
|
}),
|
|
44
|
+
inputTokens: meter.createCounter("cortex.harness.agent.input_tokens", {
|
|
45
|
+
description: "Input tokens billed across every LLM call in a runAgent completion, cache reads included",
|
|
46
|
+
unit: "{token}",
|
|
47
|
+
}),
|
|
48
|
+
outputTokens: meter.createCounter("cortex.harness.agent.output_tokens", {
|
|
49
|
+
description: "Output tokens generated across every LLM call in a runAgent completion",
|
|
50
|
+
unit: "{token}",
|
|
51
|
+
}),
|
|
52
|
+
cacheReadTokens: meter.createCounter("cortex.harness.agent.cache_read_tokens", {
|
|
53
|
+
description: "Prompt-prefix tokens served from the provider's prompt cache",
|
|
54
|
+
unit: "{token}",
|
|
55
|
+
}),
|
|
56
|
+
cacheWriteTokens: meter.createCounter("cortex.harness.agent.cache_write_tokens", {
|
|
57
|
+
description: "Prompt-prefix tokens written into the provider's prompt cache",
|
|
58
|
+
unit: "{token}",
|
|
59
|
+
}),
|
|
27
60
|
};
|
|
28
61
|
}
|
|
29
62
|
return instruments;
|
|
30
63
|
}
|
|
64
|
+
/** Fold one call's reported usage into a run's running total. */
|
|
65
|
+
export function addChatUsage(total, usage) {
|
|
66
|
+
if (usage === undefined)
|
|
67
|
+
return;
|
|
68
|
+
for (const key of ["inputTokens", "outputTokens", "cacheCreationInputTokens", "cacheReadInputTokens"]) {
|
|
69
|
+
const value = usage[key];
|
|
70
|
+
if (value !== undefined)
|
|
71
|
+
total[key] = (total[key] ?? 0) + value;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
31
74
|
/** Record one completed `runAgent` invocation. */
|
|
32
75
|
export function recordAgentRun(run) {
|
|
33
|
-
const { iterations, capHits } = getInstruments();
|
|
76
|
+
const { iterations, capHits, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens } = getInstruments();
|
|
34
77
|
const attributes = { agent_id: run.agentId };
|
|
35
78
|
iterations.record(run.iterations, attributes);
|
|
36
79
|
if (run.cappedOut)
|
|
37
80
|
capHits.add(1, attributes);
|
|
81
|
+
const usage = run.usage;
|
|
82
|
+
if (usage === undefined)
|
|
83
|
+
return;
|
|
84
|
+
if (usage.inputTokens !== undefined)
|
|
85
|
+
inputTokens.add(usage.inputTokens, attributes);
|
|
86
|
+
if (usage.outputTokens !== undefined)
|
|
87
|
+
outputTokens.add(usage.outputTokens, attributes);
|
|
88
|
+
if (usage.cacheReadInputTokens !== undefined)
|
|
89
|
+
cacheReadTokens.add(usage.cacheReadInputTokens, attributes);
|
|
90
|
+
if (usage.cacheCreationInputTokens !== undefined)
|
|
91
|
+
cacheWriteTokens.add(usage.cacheCreationInputTokens, attributes);
|
|
38
92
|
}
|
|
39
93
|
/**
|
|
40
94
|
* Drop the memoized instruments so the next `recordAgentRun` rebinds to a
|
package/dist/loop/run-agent.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type FinishReason } from "ai";
|
|
2
2
|
import type { AgentSession } from "../auth/types.js";
|
|
3
|
-
import type { AgentChat } from "../providers/types.js";
|
|
3
|
+
import type { AgentChat, PromptCachePolicy } from "../providers/types.js";
|
|
4
4
|
import type { AgentDefinition, EmitFn, LoopMessage, RunStep } from "./types.js";
|
|
5
5
|
export interface AgentFinish {
|
|
6
6
|
readonly reason: FinishReason | "max_iterations";
|
|
@@ -23,6 +23,18 @@ export interface RunAgentOptions {
|
|
|
23
23
|
readonly runStep: RunStep;
|
|
24
24
|
readonly formatStepName?: StepNameFormatter;
|
|
25
25
|
readonly isFatalLoopError?: (err: unknown) => boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Prompt-cache policy for every LLM call this run makes. Defaults to
|
|
28
|
+
* `DEFAULT_PROMPT_CACHE` (5m) — an agent loop always re-sends its prefix, so
|
|
29
|
+
* it breaks even by the second iteration. A host whose endpoint ignores or
|
|
30
|
+
* charges badly for cache directives passes `"off"`.
|
|
31
|
+
*
|
|
32
|
+
* The policy lives here, on the run, rather than on the provider, precisely
|
|
33
|
+
* so it applies to loops and *not* to the one-shot LLM calls made elsewhere
|
|
34
|
+
* (report generation, target-assessment steps): those would pay the
|
|
35
|
+
* cache-write premium for a cache nothing ever reads back.
|
|
36
|
+
*/
|
|
37
|
+
readonly promptCache?: PromptCachePolicy;
|
|
26
38
|
}
|
|
27
39
|
export declare function runAgent(agent: AgentDefinition, initial: readonly LoopMessage[], session: AgentSession, opts: RunAgentOptions): Promise<RunAgentResult>;
|
|
28
40
|
export declare function finalText(messages: readonly LoopMessage[]): string;
|
package/dist/loop/run-agent.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { jsonSchema, tool as aiTool } from "ai";
|
|
2
2
|
import { classifyProviderError } from "../providers/errors.js";
|
|
3
|
+
import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/prompt-cache.js";
|
|
3
4
|
import { resultStep } from "./run-step.js";
|
|
4
5
|
import { isToolError } from "../tools/define-tool.js";
|
|
5
|
-
import { recordAgentRun } from "./metrics.js";
|
|
6
|
+
import { addChatUsage, recordAgentRun } from "./metrics.js";
|
|
6
7
|
const TRUNCATED_PROSE_STEER = "Your previous reply was cut off at the output-token limit; continue concisely, or finish via your terminal tool.";
|
|
7
8
|
const TRUNCATED_TOOL_USE_ERROR = "Your previous tool call was cut off at the output-token limit and was not executed. Retry with a smaller payload, writing in incremental pieces.";
|
|
8
9
|
export const DEFAULT_STEP_NAME_FORMATTER = {
|
|
@@ -37,14 +38,21 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
37
38
|
});
|
|
38
39
|
let iterations = 0;
|
|
39
40
|
let truncationRecoveries = 0;
|
|
41
|
+
// Resolved once, not per iteration: an identical options object across every
|
|
42
|
+
// call is itself part of the cache contract — the request prefix has to be
|
|
43
|
+
// byte-identical to be read back.
|
|
44
|
+
const providerOptions = promptCacheProviderOptions(opts.promptCache ?? DEFAULT_PROMPT_CACHE);
|
|
45
|
+
const usage = {};
|
|
40
46
|
for (let i = 0; i < agent.maxIterations; i++) {
|
|
41
47
|
iterations = i + 1;
|
|
42
48
|
const request = {
|
|
43
49
|
system: agent.systemPrompt,
|
|
44
50
|
messages,
|
|
45
51
|
tools: toolDefs,
|
|
52
|
+
providerOptions,
|
|
46
53
|
};
|
|
47
54
|
const reply = await resultStep(runStep)(formatStepName.llm(i), () => provider.chat(request, session, signal));
|
|
55
|
+
addChatUsage(usage, reply.usage);
|
|
48
56
|
messages.push(reply.message);
|
|
49
57
|
const toolCalls = toolCallParts(reply.message);
|
|
50
58
|
if (reply.finishReason === "length") {
|
|
@@ -76,7 +84,7 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
76
84
|
}
|
|
77
85
|
if (reply.finishReason !== "tool-calls") {
|
|
78
86
|
await emit({ type: "iteration", source, index: i, final: true });
|
|
79
|
-
recordAgentRun({ agentId: agent.id, iterations, cappedOut: false });
|
|
87
|
+
recordAgentRun({ agentId: agent.id, iterations, cappedOut: false, usage });
|
|
80
88
|
return { messages, finish: { reason: reply.finishReason, cappedOut: false, truncationRecoveries } };
|
|
81
89
|
}
|
|
82
90
|
await emit({ type: "iteration", source, index: i, final: false });
|
|
@@ -96,10 +104,17 @@ export async function runAgent(agent, initial, session, opts) {
|
|
|
96
104
|
}
|
|
97
105
|
messages.push({ role: "tool", content: results });
|
|
98
106
|
}
|
|
99
|
-
|
|
107
|
+
// Cache defeater (known; not fixed here). Emptying the tool set changes the
|
|
108
|
+
// very front of the request prefix — tool definitions are cached ahead of
|
|
109
|
+
// system and history — so this call reads *nothing* back from the cache and
|
|
110
|
+
// rewrites the whole prefix from scratch. It still carries the cache options
|
|
111
|
+
// because it is the one call whose write is pure waste, and the
|
|
112
|
+
// cache_write_tokens counter is what makes that waste visible.
|
|
113
|
+
const wrapUp = await resultStep(runStep)(formatStepName.llm(agent.maxIterations), () => provider.chat({ system: agent.systemPrompt, messages, tools: {}, toolChoice: "none", providerOptions }, session, signal));
|
|
114
|
+
addChatUsage(usage, wrapUp.usage);
|
|
100
115
|
messages.push(wrapUp.message);
|
|
101
116
|
await emit({ type: "iteration", source, index: agent.maxIterations, final: true });
|
|
102
|
-
recordAgentRun({ agentId: agent.id, iterations, cappedOut: true });
|
|
117
|
+
recordAgentRun({ agentId: agent.id, iterations, cappedOut: true, usage });
|
|
103
118
|
return { messages, finish: { reason: "max_iterations", cappedOut: true, truncationRecoveries } };
|
|
104
119
|
}
|
|
105
120
|
function toolCallParts(message) {
|
|
@@ -63,7 +63,14 @@ export declare function buildRunCardData(pool: Pool, opts: {
|
|
|
63
63
|
runId?: string;
|
|
64
64
|
}): ResultAsync<RunCardData | null, DbError>;
|
|
65
65
|
/** Build the `data-presentation` payload from the `show_user` tool input.
|
|
66
|
-
* Self-contained — the card lives entirely in the input.
|
|
66
|
+
* Self-contained — the card lives entirely in the input.
|
|
67
|
+
*
|
|
68
|
+
* An `echart` spec passes through `normalizeEchartSpec` HERE rather than in the tool body, because
|
|
69
|
+
* this builder is the single construction site of a `PresentationContent`: normalizing in
|
|
70
|
+
* `show_user.execute` alone would leave the reconstruct-on-read path (which sees only the persisted
|
|
71
|
+
* raw `tool_use` input) rendering the un-normalized chart, breaking the byte-identical live/replay
|
|
72
|
+
* card this module exists to guarantee. The `id` stays keyed to the raw input, which both paths
|
|
73
|
+
* hold identically, so normalization cannot move a card's identity. */
|
|
67
74
|
export declare function buildPresentationCardData(input: Record<string, unknown>): PresentationCardData | null;
|
|
68
75
|
/** Max files a `show_file` group may reference; multiple render as a gallery. */
|
|
69
76
|
export declare const MAX_FILES = 10;
|
|
@@ -23,9 +23,13 @@ import { tryQuery } from "../lib/db-result.js";
|
|
|
23
23
|
import { tryFs } from "../lib/fs-result.js";
|
|
24
24
|
import { AnalysisPlanSchema } from "../schemas/workflow-state.js";
|
|
25
25
|
import { loadPlan } from "../state/index.js";
|
|
26
|
+
import { normalizeEchartSpec } from "../tools/display/normalize-echart-spec.js";
|
|
26
27
|
import { validatePath } from "../tools/lib/path-validation.js";
|
|
27
28
|
import { latestPreviewVersion, previewDir, PREVIEWS_ROOT } from "../workspace/paths.js";
|
|
28
29
|
const PREVIEW_META_FILE = "preview-meta.json";
|
|
30
|
+
function isRecord(value) {
|
|
31
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
32
|
+
}
|
|
29
33
|
/** Deterministic presentation id — must match `show_user`'s. */
|
|
30
34
|
function presentationId(input) {
|
|
31
35
|
const hash = createHash("sha256")
|
|
@@ -101,15 +105,28 @@ export function buildRunCardData(pool, opts) {
|
|
|
101
105
|
});
|
|
102
106
|
}
|
|
103
107
|
/** Build the `data-presentation` payload from the `show_user` tool input.
|
|
104
|
-
* Self-contained — the card lives entirely in the input.
|
|
108
|
+
* Self-contained — the card lives entirely in the input.
|
|
109
|
+
*
|
|
110
|
+
* An `echart` spec passes through `normalizeEchartSpec` HERE rather than in the tool body, because
|
|
111
|
+
* this builder is the single construction site of a `PresentationContent`: normalizing in
|
|
112
|
+
* `show_user.execute` alone would leave the reconstruct-on-read path (which sees only the persisted
|
|
113
|
+
* raw `tool_use` input) rendering the un-normalized chart, breaking the byte-identical live/replay
|
|
114
|
+
* card this module exists to guarantee. The `id` stays keyed to the raw input, which both paths
|
|
115
|
+
* hold identically, so normalization cannot move a card's identity. */
|
|
105
116
|
export function buildPresentationCardData(input) {
|
|
106
117
|
const { kind, title, ...rest } = input;
|
|
107
118
|
if (typeof kind !== "string")
|
|
108
119
|
return null;
|
|
120
|
+
const cardTitle = typeof title === "string" ? title : undefined;
|
|
121
|
+
const content = kind === "echart" && isRecord(rest.spec)
|
|
122
|
+
? { kind, ...rest, spec: normalizeEchartSpec(rest.spec, { title: cardTitle }) }
|
|
123
|
+
: // Every other kind (and an echart the model called without a `spec`) is carried through
|
|
124
|
+
// untouched — there is no layout to normalize, and no spec to invent.
|
|
125
|
+
{ kind, ...rest };
|
|
109
126
|
return {
|
|
110
127
|
id: presentationId(input),
|
|
111
|
-
...(
|
|
112
|
-
content
|
|
128
|
+
...(cardTitle !== undefined ? { title: cardTitle } : {}),
|
|
129
|
+
content,
|
|
113
130
|
};
|
|
114
131
|
}
|
|
115
132
|
/** Max files a `show_file` group may reference; multiple render as a gallery. */
|
|
@@ -19,9 +19,10 @@ export function createCardResolver(pool, analysisId, workspaceRoot) {
|
|
|
19
19
|
if (block.type !== "tool_use")
|
|
20
20
|
return null;
|
|
21
21
|
const input = (block.input ?? {});
|
|
22
|
-
// The
|
|
23
|
-
// `iterateReport` — accept
|
|
24
|
-
|
|
22
|
+
// The report tool is `submit_report`; legacy transcripts carry the
|
|
23
|
+
// former ids `iterate_report` / `iterateReport` — accept all three. The
|
|
24
|
+
// brief-carrying field is still `report`, so the card reads identically.
|
|
25
|
+
if (block.name === "submit_report" || block.name === "iterate_report" || block.name === "iterateReport") {
|
|
25
26
|
const report = (input.report ?? null);
|
|
26
27
|
const card = await buildPreviewCardData(workspaceRoot, {
|
|
27
28
|
previewId: typeof input.previewId === "string" ? input.previewId : undefined,
|
|
@@ -54,8 +54,11 @@ export interface ThreadHistory {
|
|
|
54
54
|
*/
|
|
55
55
|
appendTurn(threadId: string, messages: readonly ModelMessage[]): ResultAsync<void, DbError>;
|
|
56
56
|
/**
|
|
57
|
-
* Return
|
|
58
|
-
*
|
|
57
|
+
* Return a recent-turns window that fits `tokenBudget`, oldest-first,
|
|
58
|
+
* snapped to a valid AI SDK model-message sequence. The window START advances
|
|
59
|
+
* in whole `EVICTION_BLOCK_TURNS` blocks (see the constant) so the prompt-cache
|
|
60
|
+
* prefix stays byte-stable across appends; the window may therefore carry a
|
|
61
|
+
* little less than the budget would strictly allow, but never more.
|
|
59
62
|
*/
|
|
60
63
|
loadRecent(threadId: string, tokenBudget: number): ResultAsync<ModelMessage[], DbError>;
|
|
61
64
|
/**
|
|
@@ -72,6 +75,28 @@ export interface ThreadHistory {
|
|
|
72
75
|
* freshly-registered `MeterProvider`. Test-only.
|
|
73
76
|
*/
|
|
74
77
|
export declare function __resetThreadHistoryMetricsForTest(): void;
|
|
78
|
+
/**
|
|
79
|
+
* Chunked-eviction block size, in whole turns — the granularity at which
|
|
80
|
+
* `loadRecent`'s retained window may advance its START turn.
|
|
81
|
+
*
|
|
82
|
+
* `loadRecent` snaps its eviction count UP to a multiple of this block, so the
|
|
83
|
+
* window's first turn — hence `messages[0]` and the whole tools+system+history
|
|
84
|
+
* prompt-cache prefix — stays byte-identical across a run of appends and shifts
|
|
85
|
+
* only once per block instead of once per turn. That single shift is the only
|
|
86
|
+
* message-cache miss the block costs.
|
|
87
|
+
*
|
|
88
|
+
* The block is at once (a) the cache-miss cadence — one prefix shift per
|
|
89
|
+
* `EVICTION_BLOCK_TURNS` appends once a thread is over budget — and (b) the most
|
|
90
|
+
* extra context ever sacrificed: snapping up drops up to `EVICTION_BLOCK_TURNS -
|
|
91
|
+
* 1` oldest turns the budget alone would have kept, trading a little history for
|
|
92
|
+
* a still prefix. 4 holds the prefix for ~4 turns at a cost of at most 3 turns of
|
|
93
|
+
* headroom — negligible against a ~120k-token budget that spans far more turns,
|
|
94
|
+
* while cutting cache misses on a long thread roughly fourfold. A turn-count
|
|
95
|
+
* block (not a token block) is deliberate: a cache miss is triggered by the START
|
|
96
|
+
* turn moving, a per-turn event, so counting in turns makes the miss cadence
|
|
97
|
+
* exact and independent of how large any individual turn is.
|
|
98
|
+
*/
|
|
99
|
+
export declare const EVICTION_BLOCK_TURNS = 4;
|
|
75
100
|
/**
|
|
76
101
|
* Create a `ThreadHistory` bound to a Postgres pool — a factory closure
|
|
77
102
|
* capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
|
|
@@ -79,6 +79,28 @@ function getInstruments() {
|
|
|
79
79
|
export function __resetThreadHistoryMetricsForTest() {
|
|
80
80
|
instruments = undefined;
|
|
81
81
|
}
|
|
82
|
+
/**
|
|
83
|
+
* Chunked-eviction block size, in whole turns — the granularity at which
|
|
84
|
+
* `loadRecent`'s retained window may advance its START turn.
|
|
85
|
+
*
|
|
86
|
+
* `loadRecent` snaps its eviction count UP to a multiple of this block, so the
|
|
87
|
+
* window's first turn — hence `messages[0]` and the whole tools+system+history
|
|
88
|
+
* prompt-cache prefix — stays byte-identical across a run of appends and shifts
|
|
89
|
+
* only once per block instead of once per turn. That single shift is the only
|
|
90
|
+
* message-cache miss the block costs.
|
|
91
|
+
*
|
|
92
|
+
* The block is at once (a) the cache-miss cadence — one prefix shift per
|
|
93
|
+
* `EVICTION_BLOCK_TURNS` appends once a thread is over budget — and (b) the most
|
|
94
|
+
* extra context ever sacrificed: snapping up drops up to `EVICTION_BLOCK_TURNS -
|
|
95
|
+
* 1` oldest turns the budget alone would have kept, trading a little history for
|
|
96
|
+
* a still prefix. 4 holds the prefix for ~4 turns at a cost of at most 3 turns of
|
|
97
|
+
* headroom — negligible against a ~120k-token budget that spans far more turns,
|
|
98
|
+
* while cutting cache misses on a long thread roughly fourfold. A turn-count
|
|
99
|
+
* block (not a token block) is deliberate: a cache miss is triggered by the START
|
|
100
|
+
* turn moving, a per-turn event, so counting in turns makes the miss cadence
|
|
101
|
+
* exact and independent of how large any individual turn is.
|
|
102
|
+
*/
|
|
103
|
+
export const EVICTION_BLOCK_TURNS = 4;
|
|
82
104
|
/**
|
|
83
105
|
* Create a `ThreadHistory` bound to a Postgres pool — a factory closure
|
|
84
106
|
* capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
|
|
@@ -125,9 +147,22 @@ export function createThreadHistory(pool) {
|
|
|
125
147
|
const turns = groupTurns(parsed, (row) => isGenuineUserStart(row.message));
|
|
126
148
|
const turnTokens = turns.map((turn) => turn.reduce((sum, row) => sum + row.tokens, 0));
|
|
127
149
|
const threadTotal = turnTokens.reduce((sum, n) => sum + n, 0);
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
150
|
+
// Chunked eviction holds the prompt-cache prefix still. The budget alone
|
|
151
|
+
// evicts the OLDEST turns newest-first, so the window START — the stable
|
|
152
|
+
// head of the tools+system+history cache prefix — advances by ~one turn
|
|
153
|
+
// per appended turn; every turn then ships a shifted `messages[0]` and
|
|
154
|
+
// rewrites the whole message cache instead of reading it back (watch
|
|
155
|
+
// `cortex.harness.agent.cache_read_tokens` collapse on long threads).
|
|
156
|
+
// Instead, take the budget-minimal eviction and snap it UP to a whole
|
|
157
|
+
// `EVICTION_BLOCK_TURNS` block: `evicted` stays constant while the minimum
|
|
158
|
+
// sits inside a block and jumps by a block when it crosses, so
|
|
159
|
+
// `messages[0]` is byte-identical for a block of appends (the cache reads
|
|
160
|
+
// survive) and shifts only once per block. Snapping UP retains a subset of
|
|
161
|
+
// the budget-minimal window, so the kept tokens never exceed `tokenBudget`.
|
|
162
|
+
//
|
|
163
|
+
// Budget-minimal fill: walk newest-first, always keeping the most recent
|
|
164
|
+
// turn (a valid over-budget single turn beats an under-budget cut), then
|
|
165
|
+
// add older turns while they fit.
|
|
131
166
|
let included = 0;
|
|
132
167
|
let cumulative = 0;
|
|
133
168
|
for (let t = turns.length - 1; t >= 0; t--) {
|
|
@@ -136,13 +171,18 @@ export function createThreadHistory(pool) {
|
|
|
136
171
|
cumulative += turnTokens[t];
|
|
137
172
|
included++;
|
|
138
173
|
}
|
|
139
|
-
const
|
|
174
|
+
const minimalEvicted = turns.length - included;
|
|
175
|
+
// Snap the eviction count up to a block boundary, clamped below the last
|
|
176
|
+
// turn so the most recent turn always ships. Both terms of the min are
|
|
177
|
+
// >= `minimalEvicted`, so `turnsEvicted >= minimalEvicted`: the retained
|
|
178
|
+
// suffix is a subset of the budget-minimal one and stays within budget.
|
|
179
|
+
const turnsEvicted = turns.length === 0 ? 0 : Math.min(Math.ceil(minimalEvicted / EVICTION_BLOCK_TURNS) * EVICTION_BLOCK_TURNS, turns.length - 1);
|
|
140
180
|
const { totalTokens, turnsEvicted: turnsEvictedHist } = getInstruments();
|
|
141
181
|
const attributes = { eviction: turnsEvicted > 0 };
|
|
142
182
|
totalTokens.record(threadTotal, attributes);
|
|
143
183
|
turnsEvictedHist.record(turnsEvicted, attributes);
|
|
144
184
|
return turns
|
|
145
|
-
.slice(
|
|
185
|
+
.slice(turnsEvicted)
|
|
146
186
|
.flat()
|
|
147
187
|
.map((row) => row.message);
|
|
148
188
|
});
|
|
@@ -6,18 +6,48 @@
|
|
|
6
6
|
* to Markdown only when injected (see the harness-thread-store spec).
|
|
7
7
|
*
|
|
8
8
|
* Four sections. `goal`, `constraints`, `hypotheses` are analysis-flat;
|
|
9
|
-
* `findings` is
|
|
10
|
-
*
|
|
9
|
+
* `findings` is stored keyed by `runId`, so a finding stays attributable to
|
|
10
|
+
* the run that produced it. There is deliberately no `context` section —
|
|
11
11
|
* analysis context lives in `cortex_analysis_state.context` and is injected
|
|
12
12
|
* separately by the chat route.
|
|
13
13
|
*
|
|
14
14
|
* `updateSection` is a section-addressable read-modify-write: the named
|
|
15
15
|
* section is replaced or amended and the other three are left byte-identical.
|
|
16
|
+
*
|
|
17
|
+
* **Working memory is re-injected, in full, into the uncached tail of every
|
|
18
|
+
* turn** — so every stored character is re-paid on every later turn for the
|
|
19
|
+
* life of the analysis. Two consequences shape this module:
|
|
20
|
+
*
|
|
21
|
+
* - **Every section is bounded at write time** (`WORKING_MEMORY_LIMITS`). An
|
|
22
|
+
* over-cap write is *rejected* with a model-actionable message, never
|
|
23
|
+
* silently truncated: the agent must shorten the text or retire a stale
|
|
24
|
+
* entry. Every section — findings included — therefore supports
|
|
25
|
+
* add/revise/retire, so memory can shrink as well as grow.
|
|
26
|
+
* - **Memory holds references to runs, never run content.** A finding cites
|
|
27
|
+
* the `runId` that produced it and renders as one line in a single flat
|
|
28
|
+
* list; the run's detail stays retrievable via `inspect_run`.
|
|
16
29
|
*/
|
|
17
|
-
import type
|
|
30
|
+
import { type ResultAsync } from "neverthrow";
|
|
18
31
|
import type { Pool } from "pg";
|
|
19
32
|
import { z } from "zod";
|
|
20
33
|
import { type DbError } from "../lib/db-result.js";
|
|
34
|
+
/**
|
|
35
|
+
* The write-time bounds on working memory. `updateSection` enforces them, the
|
|
36
|
+
* `update_working_memory` input schema declares them to the model (importing
|
|
37
|
+
* these same numbers), and `renderWorkingMemory` uses them to bound its output.
|
|
38
|
+
*/
|
|
39
|
+
export declare const WORKING_MEMORY_LIMITS: {
|
|
40
|
+
/** Max characters in the goal. */
|
|
41
|
+
readonly goalChars: 500;
|
|
42
|
+
/** Max characters in one constraint / hypothesis / finding entry. */
|
|
43
|
+
readonly entryChars: 300;
|
|
44
|
+
/** Max constraint entries. */
|
|
45
|
+
readonly constraints: 20;
|
|
46
|
+
/** Max hypothesis entries. */
|
|
47
|
+
readonly hypotheses: 10;
|
|
48
|
+
/** Max finding entries, summed across every run. */
|
|
49
|
+
readonly findings: 30;
|
|
50
|
+
};
|
|
21
51
|
/** A binding rule — user-stated or agent-derived. The highest-value section. */
|
|
22
52
|
export declare const constraintSchema: z.ZodObject<{
|
|
23
53
|
id: z.ZodString;
|
|
@@ -43,6 +73,11 @@ export type Finding = z.infer<typeof findingSchema>;
|
|
|
43
73
|
/**
|
|
44
74
|
* The persisted `data` shape — one JSONB object per analysis. `findings` is
|
|
45
75
|
* keyed by `runId`; an analysis with no recorded findings has an empty map.
|
|
76
|
+
*
|
|
77
|
+
* Deliberately unbounded: the caps are a *write-time* rule, not a read-time
|
|
78
|
+
* one. A row written before the caps existed must still `load` — so this
|
|
79
|
+
* schema validates shape only, and `renderWorkingMemory` bounds what an
|
|
80
|
+
* oversized row is allowed to cost.
|
|
46
81
|
*/
|
|
47
82
|
export declare const workingMemorySchema: z.ZodObject<{
|
|
48
83
|
goal: z.ZodString;
|
|
@@ -66,6 +101,20 @@ export declare const workingMemorySchema: z.ZodObject<{
|
|
|
66
101
|
export type WorkingMemory = z.infer<typeof workingMemorySchema>;
|
|
67
102
|
/** The shape `load` returns for an analysis with no working-memory row. */
|
|
68
103
|
export declare function emptyWorkingMemory(): WorkingMemory;
|
|
104
|
+
/**
|
|
105
|
+
* A refused write: a cap was hit, or an id addresses nothing. Not a storage
|
|
106
|
+
* failure — it is an expected outcome the *model* can correct, so `message` is
|
|
107
|
+
* written for the model and says what to do next (shorten this, retire that).
|
|
108
|
+
* The tool surfaces it as an `is_error` tool result; the row is untouched.
|
|
109
|
+
*/
|
|
110
|
+
export interface WorkingMemoryRejection {
|
|
111
|
+
readonly type: "working_memory_rejected";
|
|
112
|
+
readonly message: string;
|
|
113
|
+
}
|
|
114
|
+
/** `updateSection`'s error channel: a storage failure, or a refused write. */
|
|
115
|
+
export type WorkingMemoryError = DbError | WorkingMemoryRejection;
|
|
116
|
+
/** Narrow a `WorkingMemoryError` to the model-correctable branch. */
|
|
117
|
+
export declare function isWorkingMemoryRejection(error: WorkingMemoryError): error is WorkingMemoryRejection;
|
|
69
118
|
/**
|
|
70
119
|
* A list-section amendment: append a new entry, revise an existing one's
|
|
71
120
|
* text, or retire a stale one. `revise`/`retire` address an entry by the
|
|
@@ -95,6 +144,23 @@ export type ConstraintOp = {
|
|
|
95
144
|
readonly op: "retire";
|
|
96
145
|
readonly id: string;
|
|
97
146
|
};
|
|
147
|
+
/**
|
|
148
|
+
* As `ListOp`, but `add` also carries the `runId` the finding is attributed
|
|
149
|
+
* to. `revise`/`retire` address the finding by its own id — the run holding it
|
|
150
|
+
* is found from that id, so the agent never has to re-cite the run.
|
|
151
|
+
*/
|
|
152
|
+
export type FindingOp = {
|
|
153
|
+
readonly op: "add";
|
|
154
|
+
readonly runId: string;
|
|
155
|
+
readonly text: string;
|
|
156
|
+
} | {
|
|
157
|
+
readonly op: "revise";
|
|
158
|
+
readonly id: string;
|
|
159
|
+
readonly text: string;
|
|
160
|
+
} | {
|
|
161
|
+
readonly op: "retire";
|
|
162
|
+
readonly id: string;
|
|
163
|
+
};
|
|
98
164
|
/** The `value` argument of `updateSection`, keyed by the `section` name. */
|
|
99
165
|
export interface SectionValue {
|
|
100
166
|
readonly goal: {
|
|
@@ -102,10 +168,7 @@ export interface SectionValue {
|
|
|
102
168
|
};
|
|
103
169
|
readonly constraint: ConstraintOp;
|
|
104
170
|
readonly hypothesis: ListOp;
|
|
105
|
-
readonly finding:
|
|
106
|
-
readonly runId: string;
|
|
107
|
-
readonly text: string;
|
|
108
|
-
};
|
|
171
|
+
readonly finding: FindingOp;
|
|
109
172
|
}
|
|
110
173
|
export type Section = keyof SectionValue;
|
|
111
174
|
/**
|
|
@@ -118,13 +181,33 @@ export interface WorkingMemoryStore {
|
|
|
118
181
|
load(analysisId: string): ResultAsync<WorkingMemory, DbError>;
|
|
119
182
|
/**
|
|
120
183
|
* Atomically read-modify-write one section. The other three sections are
|
|
121
|
-
* left byte-identical; the row is upserted if absent.
|
|
184
|
+
* left byte-identical; the row is upserted if absent. A write that breaks a
|
|
185
|
+
* cap or addresses an unknown id is refused — `err(WorkingMemoryRejection)`,
|
|
186
|
+
* and the row is left exactly as it was.
|
|
122
187
|
*/
|
|
123
|
-
updateSection<S extends Section>(analysisId: string, section: S, value: SectionValue[S]): ResultAsync<void,
|
|
188
|
+
updateSection<S extends Section>(analysisId: string, section: S, value: SectionValue[S]): ResultAsync<void, WorkingMemoryError>;
|
|
124
189
|
/** Serialize working memory to a Markdown document for injection. */
|
|
125
190
|
render(analysisId: string): ResultAsync<string, DbError>;
|
|
126
191
|
}
|
|
127
|
-
/**
|
|
192
|
+
/**
|
|
193
|
+
* Serialize working memory to the Markdown document injected each turn.
|
|
194
|
+
*
|
|
195
|
+
* Renders only what exists: an empty section is omitted entirely — no heading,
|
|
196
|
+
* no placeholder — and an entirely empty memory renders to the **empty
|
|
197
|
+
* string**, costing nothing. The chat route injects the render
|
|
198
|
+
* unconditionally, and the provider's outbound sanitizer drops a message whose
|
|
199
|
+
* content is `""` before the wire call (`providers/ai-sdk.ts`), so an empty
|
|
200
|
+
* memory reaches the model as no message at all.
|
|
201
|
+
*
|
|
202
|
+
* Findings render as ONE flat list, each line citing the run it came from —
|
|
203
|
+
* never a per-run heading block. Memory holds the reference; `inspect_run`
|
|
204
|
+
* holds the run.
|
|
205
|
+
*
|
|
206
|
+
* The output is bounded to `WORKING_MEMORY_LIMITS` no matter what the row
|
|
207
|
+
* holds. Writes are capped, so only a row written before the caps existed can
|
|
208
|
+
* exceed them — and it degrades to its newest entries rather than exploding
|
|
209
|
+
* the context window.
|
|
210
|
+
*/
|
|
128
211
|
export declare function renderWorkingMemory(wm: WorkingMemory): string;
|
|
129
212
|
/**
|
|
130
213
|
* Create a `WorkingMemoryStore` bound to a Postgres pool — a factory closure
|