@tangle-network/agent-eval 0.191.0 → 0.192.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +6 -6
- package/dist/analyst/index.js +3 -3
- package/dist/{benchmark-command-3tZJnD82.js → benchmark-command-C458yEFK.js} +7 -7
- package/dist/{benchmark-command-3tZJnD82.js.map → benchmark-command-C458yEFK.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +8 -9
- package/dist/{campaign-C8iWEF0M.js → campaign-BVQVBEfH.js} +12 -326
- package/dist/campaign-BVQVBEfH.js.map +1 -0
- package/dist/{campaign-evidence-BqjlTM3P.js → campaign-evidence-DcQiYhJI.js} +10 -45
- package/dist/campaign-evidence-DcQiYhJI.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +6 -6
- package/dist/contract/index.js +6 -6
- package/dist/{default-registry-DK_9aVJf.d.ts → default-registry-DdqA_-_o.d.ts} +3 -3
- package/dist/{default-registry-DK_9aVJf.d.ts.map → default-registry-DdqA_-_o.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BE43YTm4.js → define-agent-eval-DQuos3Aw.js} +8 -9
- package/dist/define-agent-eval-DQuos3Aw.js.map +1 -0
- package/dist/{define-agent-eval-P28y7mI1.d.ts → define-agent-eval-DcludNJS.d.ts} +13 -22
- package/dist/{define-agent-eval-P28y7mI1.d.ts.map → define-agent-eval-DcludNJS.d.ts.map} +1 -1
- package/dist/{dspy-rlm-engine-MfpwoEaj.js → dspy-rlm-engine-BX34vPVY.js} +2 -2
- package/dist/{dspy-rlm-engine-MfpwoEaj.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
- package/dist/{engine-vNiM02Ub.d.ts → engine-C1DIAEpn.d.ts} +2 -2
- package/dist/{engine-vNiM02Ub.d.ts.map → engine-C1DIAEpn.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.js +1 -1
- package/dist/{external-optimizer-process-xFTliuiK.js → external-optimizer-process-D-Sab4v3.js} +3 -62
- package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
- package/dist/{external-optimizer-subprocess-B234V9V0.js → external-optimizer-subprocess-kbqC-iWf.js} +64 -3
- package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
- package/dist/hosted/index.js +1 -1
- package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
- package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
- package/dist/{index-D-TN7ZRM.d.ts → index-DBoXFaz0.d.ts} +3 -3
- package/dist/{index-D-TN7ZRM.d.ts.map → index-DBoXFaz0.d.ts.map} +1 -1
- package/dist/{index-DH-MWH5f.d.ts → index-DNVzu9Vv.d.ts} +6 -6
- package/dist/{index-DH-MWH5f.d.ts.map → index-DNVzu9Vv.d.ts.map} +1 -1
- package/dist/{index-CzfnASj_.d.ts → index-DwQxdWve.d.ts} +338 -129
- package/dist/index-DwQxdWve.d.ts.map +1 -0
- package/dist/index.d.ts +9 -9
- package/dist/index.js +6 -6
- package/dist/jev.d.ts +1 -1
- package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
- package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-NJOHM0iA.js → llm-judge-Z6MkTkd0.js} +1819 -609
- package/dist/llm-judge-Z6MkTkd0.js.map +1 -0
- package/dist/{matrix-CgqdFm5N.d.ts → matrix-CQMMrEQ5.d.ts} +2 -2
- package/dist/{matrix-CgqdFm5N.d.ts.map → matrix-CQMMrEQ5.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-CY-L7CqS.js → produced-state-CVmCPQDJ.js} +3 -3
- package/dist/{produced-state-CY-L7CqS.js.map → produced-state-CVmCPQDJ.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/{search-ledger-CQ8Uy4FK.js → search-ledger-Ck7tok0h.js} +78 -12
- package/dist/search-ledger-Ck7tok0h.js.map +1 -0
- package/dist/{search-ledger-file-EvdDZRhv.js → search-ledger-file-DY0AXX3W.js} +2 -2
- package/dist/{search-ledger-file-EvdDZRhv.js.map → search-ledger-file-DY0AXX3W.js.map} +1 -1
- package/dist/{search-shipper-C55V7HE3.js → search-shipper-BG182p-U.js} +3 -3
- package/dist/{search-shipper-C55V7HE3.js.map → search-shipper-BG182p-U.js.map} +1 -1
- package/dist/{semantic-concept-judge-Bt7bXAAo.js → semantic-concept-judge-Bg-560H0.js} +3 -3
- package/dist/{semantic-concept-judge-Bt7bXAAo.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DwO3TOAV.js → skillopt-optimization-method-DoayuM6D.js} +6 -6
- package/dist/{skillopt-optimization-method-DwO3TOAV.js.map → skillopt-optimization-method-DoayuM6D.js.map} +1 -1
- package/dist/{statistical-heldout-BED3wBtX.d.ts → statistical-heldout-C1Ej5Ce8.d.ts} +2 -2
- package/dist/{statistical-heldout-BED3wBtX.d.ts.map → statistical-heldout-C1Ej5Ce8.d.ts.map} +1 -1
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts → store-tool-spans-ByLdU26j.d.ts} +3 -3
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts.map → store-tool-spans-ByLdU26j.d.ts.map} +1 -1
- package/dist/{tool-groups-D-mkeN0N.d.ts → tool-groups-DiBi16ns.d.ts} +2 -2
- package/dist/tool-groups-DiBi16ns.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-ClK-sHs8.d.ts → types-ByZslIlr.d.ts} +2 -2
- package/dist/{types-ClK-sHs8.d.ts.map → types-ByZslIlr.d.ts.map} +1 -1
- package/dist/{types-BmQRs661.d.ts → types-C1CAD70_.d.ts} +14 -15
- package/dist/types-C1CAD70_.d.ts.map +1 -0
- package/docs/campaign-proposers.md +14 -16
- package/docs/search-ledger.md +62 -1
- package/package.json +1 -1
- package/dist/campaign-C8iWEF0M.js.map +0 -1
- package/dist/campaign-evidence-BqjlTM3P.js.map +0 -1
- package/dist/define-agent-eval-BE43YTm4.js.map +0 -1
- package/dist/external-optimizer-process-xFTliuiK.js.map +0 -1
- package/dist/external-optimizer-subprocess-B234V9V0.js.map +0 -1
- package/dist/index-CzfnASj_.d.ts.map +0 -1
- package/dist/llm-judge-NJOHM0iA.js.map +0 -1
- package/dist/search-ledger-CQ8Uy4FK.js.map +0 -1
- package/dist/tool-groups-D-mkeN0N.d.ts.map +0 -1
- package/dist/types-BmQRs661.d.ts.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,43 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
6
6
|
|
|
7
7
|
## Unreleased
|
|
8
8
|
|
|
9
|
+
## [0.192.0] — 2026-09-25
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `runSearch` (`/campaign`) is the one search kernel: an event-driven loop with no generation barrier that separates where to expand (`SearchPolicy`), where to spend rollouts (`SearchAllocator`) and what to claim ([search ledger](./docs/search-ledger.md#run-a-search-the-kernel)).
|
|
14
|
+
A lane that frees up takes the next allocated cell (claim, then rung and root, then screen, then train); the policy proposes when no cell waits and the cap admits one proposal and its expected screens.
|
|
15
|
+
Its ports are an executor (`lanes`, `place`, `run`, `adopt`), a proposer, and an artifact codec.
|
|
16
|
+
Every reservation passes the ledger's admission rule before it is asked for: a hard lane holds its enforced per-cell maximum, an estimate lane 1.5 times the p99 of its settled cells once 20 settled (else its prior), and spend above a hold is recorded as overspend.
|
|
17
|
+
The ledger is the only checkpoint: rerun on an open ledger, the kernel records an interrupted proposal as failed at an unknown cost, finishes a recorded proposal from its stored output, and offers every unsettled cell to `executor.adopt` before it runs it.
|
|
18
|
+
Killed with SIGKILL 8 times at random ledger positions, a simulated search (7 nodes, 56 cells, 10 % environment faults, an $8 cap, estimate lanes) resumed to the same nodes, edges, cells, scores, decisions and close as an uninterrupted run; 10 attempts that finished before a kill were adopted, none finished twice, and committed spend ($2.55) stayed within the cap plus recorded overspend ($0.18).
|
|
19
|
+
A real `runOptimization` (runCampaign, CostLedger, cell cache, file ledger; 7 nodes, 28 cells) killed 6 times, once inside a proposal, returned the same winner and generations as an uninterrupted run and dispatched the same 28 cells to completion.
|
|
20
|
+
With 1.5 s cells on 4 slots, all 4 slots were busy for 93 % of the time any cell ran.
|
|
21
|
+
- `incumbent({ patience })` and `crowdedFrontierParent({ seed })` are `SearchPolicy`s; `uniform({ reps })` is the fixed-plan `SearchAllocator`.
|
|
22
|
+
- `SearchStateView.budget` gives the admission rule's room (`headroomUsd`, `claimReserveUsd`), and `SearchOperation` carries its recorded outcome and artifacts.
|
|
23
|
+
- `SearchRecorder.readBlob(ref)` reads a stored blob back and checks its digest and length.
|
|
24
|
+
- `openSearchLedger({ store })` keeps a ledger in an in-process text store (an in-memory `CampaignStorage`) for runs without a filesystem, with the file ledger's chain, parse and state rules.
|
|
25
|
+
- `scripts/search-sim.ts` runs the real kernel over a seeded synthetic objective, proposer and executor; `kill-resume` SIGKILLs it at random ledger positions and compares the resumed search with an uninterrupted one.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
|
|
29
|
+
- **Breaking:** `runOptimization` is a search on the kernel with `incumbent()` (or `policy`) and `uniform({ reps })`.
|
|
30
|
+
Its search ledger is always written (`<runDir>/search/ledger.jsonl`, held in `storage` unless that is the filesystem, or where `searchLedger` puts it), and `searchHistory` is always returned.
|
|
31
|
+
Running it again on the same run directory continues an interrupted search and reruns nothing that settled; after a closed search it starts the next one beside it.
|
|
32
|
+
A generation is one proposal of `populationSize` candidates; each candidate runs one cell at a time on lanes of `candidateConcurrency * maxConcurrency` slots.
|
|
33
|
+
`analyzeGeneration` runs just before the proposal it feeds, inside that proposal's operation, so its spend is booked there.
|
|
34
|
+
A candidate identical to a surface the search holds is a second edge into that node and is not measured again; it used to throw.
|
|
35
|
+
The kept surface is the policy's leader: the candidate that scored every scenario and beats the leader on the scenarios they share.
|
|
36
|
+
- **Breaking:** `selfImprove({ selectParent })` is `selfImprove({ policy })`, and `ProposeContext` gains `operator`.
|
|
37
|
+
- `CampaignStorage.kind` names a filesystem or memory storage.
|
|
38
|
+
- Migration: replace `selectParent: crowdedFrontierParent({ seed })` with `policy: crowdedFrontierParent({ seed })`; drop `selectionRankKey` (a domain metric belongs in the judge's composite, which the leader rule ranks); read a candidate's lineage from the search ledger's edges and its contrast from `estimateNode(state, nodeId, { against, split: 'train' })` instead of `parentSurfaceHash`, `parentComposite` and `observedDeltaFromParent`.
|
|
39
|
+
|
|
40
|
+
### Removed
|
|
41
|
+
|
|
42
|
+
- The generation loop body of `runOptimization`, its `compareRankKeys` promotion and the `selectionRankKey` option, `compareRankKeys`, `assertFiniteRankKey`, the `ParentSelector` and `ParentSelectionContext` types, and `OptimizationSearch`.
|
|
43
|
+
- `GenerationCandidate.ci95` (always null), `parentSurfaceHash`, `parentComposite` and the unpaired `observedDeltaFromParent`, with their `LoopProvenanceCandidate` copies and span attributes.
|
|
44
|
+
- The `runOptimization`, proposal-findings, parent-selection, loop-provenance-integrity and `selfImprove` unit tests. The proof is the search simulator's kill-resume run and a real `runOptimization` (runCampaign, CostLedger, cell cache, file ledger) killed and resumed against an uninterrupted run.
|
|
45
|
+
|
|
9
46
|
## [0.191.0] — 2026-09-25
|
|
10
47
|
|
|
11
48
|
### Changed
|
|
@@ -85,6 +122,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
85
122
|
- `knownSecrets` are cut out where they stand instead of replacing the whole string, so `redactText('cleanup failed: <key> was still set', { knownSecrets: [key] })` keeps `cleanup failed:` and `was still set`.
|
|
86
123
|
A whole-string base64 payload that holds a known secret at any byte offset is still replaced whole, and value shapes such as bearer tokens still replace the whole string.
|
|
87
124
|
|
|
125
|
+
|
|
88
126
|
### Removed
|
|
89
127
|
|
|
90
128
|
- The candidate-slot ledger: `SearchPlan`, candidate slots, `search-planned`, `search-plan-extended`, `candidate-registered`, `candidate-slot-closed`, `task-attempted`, `candidate-decided`, `search-completed`, `lineageNodeId`, `SearchLedgerReplay`, `SearchLedgerAudit`, `search-ledger-projector.ts`, and `recordCandidatePopulationSearch`.
|
package/dist/adapters/http.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { m as CostReceipt } from "../cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext, f as DispatchFn } from "../types-
|
|
3
|
-
import "../index-
|
|
2
|
+
import { R as Scenario, d as DispatchContext, f as DispatchFn } from "../types-C1CAD70_.js";
|
|
3
|
+
import "../index-DNVzu9Vv.js";
|
|
4
4
|
//#region src/adapters/http.d.ts
|
|
5
5
|
interface HttpDispatchOptions<TScenario extends Scenario, _TArtifact> {
|
|
6
6
|
/** Static endpoint URL. Mutually exclusive with `resolveUrl`. */
|
package/dist/analyst/index.d.ts
CHANGED
|
@@ -4,10 +4,10 @@ import { A as createChatClient, C as ChatResponse, D as MockTransportOpts, E as
|
|
|
4
4
|
import { A as runAnalystBenchmark, C as AnalystEvidenceResolutionError, D as AnalystLatencyDistribution, E as AnalystIssueExpectation, M as scoreAnalystFindings, O as RunAnalystBenchmarkOptions, S as AnalystEvidenceResolution, T as AnalystFindingScore, V as ExternalOptimizerModelCall, W as ExternalOptimizerModelExecutionObservation, _ as AnalystBenchmarkProvenance, b as AnalystBenchmarkSummary, d as AnalystBenchmarkDatasetRef, f as AnalystBenchmarkDescriptor, g as AnalystBenchmarkOutput, h as AnalystBenchmarkObservation, j as traceStoreEvidenceResolver, k as registryBenchmarkRunner, m as AnalystBenchmarkLabelState, p as AnalystBenchmarkError, q as ExternalOptimizerRunnerCommand, t as AgentProfile, u as AnalystBenchmarkCase, v as AnalystBenchmarkResult, w as AnalystEvidenceResolver, x as AnalystEvidenceExpectation, y as AnalystBenchmarkRunner } from "../agent-profile-DDfIjdyS.js";
|
|
5
5
|
import { a as ExactAnalystRunPolicySnapshot, c as ExactAnalystSnapshot, d as ExactExecutionComponentSnapshot, i as ExactAnalystRunEvent, l as ExactCapableAnalyst, n as ExactAnalystExecutionPlanSnapshot, o as ExactAnalystRunResult, r as ExactAnalystRunCompletion, s as ExactAnalystRunSummary, t as ExactAnalystBudgetSnapshot, u as ExactExecutionComponentIdentity } from "../exact-types-KGxBWH4b.js";
|
|
6
6
|
import { a as ExactAnalystBudgetPolicy, c as RegistryRunOpts, i as BudgetPolicy, n as AnalystRegistry, o as ExactAnalystRunExecutionError, r as AnalystRegistryOptions, s as ExactRegistryRunOpts, t as AnalystHooks } from "../registry-Bu3lNole.js";
|
|
7
|
-
import { a as createTraceAnalyst, c as BehavioralAnalystOptions, i as TraceAnalystDefinition, l as behavioralAnalyst, n as buildDefaultAnalystRegistry, o as renderPriorFindings, r as CreateTraceAnalystOptions, s as runTraceAnalyst, t as DefaultAnalystRegistryOptions, u as deriveEfficiencyFindings } from "../default-registry-
|
|
8
|
-
import { a as TraceAnalystLimits, c as RawAnalystEvidence, d as evidenceRefsFromRawFinding, f as parseRawFinding, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, o as resolveTraceAnalystLimits, r as TraceAnalysisEngineRequest, s as RAW_FINDING_SCHEMA_PROMPT, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, u as RawAnalystFindingSchema } from "../engine-
|
|
9
|
-
import { n as buildTraceToolsForGroup, t as TraceToolGroupName } from "../tool-groups-
|
|
10
|
-
import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, E as createChatTraceEngine, S as renderFindingSubject, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, b as findingSubjectGrammarPromptFor, c as emitControlIntegrityFindings, d as FindingsStore, f as PersistedFinding, g as FINDING_SUBJECT_SYNTAX, h as FINDING_SUBJECT_KINDS, i as IMPROVEMENT_KIND_SPEC, j as SemanticConceptJudgeOptions, l as DiffPolicy, m as diffFindings, n as KNOWLEDGE_POISONING_KIND_SPEC, o as CONTROL_INTEGRITY_ANALYST, p as defaultIsMaterial, r as KNOWLEDGE_GAP_KIND_SPEC, s as ControlIntegrityAnalyst, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine, x as parseFindingSubject, y as KIND_EXPECTED_SUBJECTS } from "../index-
|
|
7
|
+
import { a as createTraceAnalyst, c as BehavioralAnalystOptions, i as TraceAnalystDefinition, l as behavioralAnalyst, n as buildDefaultAnalystRegistry, o as renderPriorFindings, r as CreateTraceAnalystOptions, s as runTraceAnalyst, t as DefaultAnalystRegistryOptions, u as deriveEfficiencyFindings } from "../default-registry-DdqA_-_o.js";
|
|
8
|
+
import { a as TraceAnalystLimits, c as RawAnalystEvidence, d as evidenceRefsFromRawFinding, f as parseRawFinding, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, o as resolveTraceAnalystLimits, r as TraceAnalysisEngineRequest, s as RAW_FINDING_SCHEMA_PROMPT, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, u as RawAnalystFindingSchema } from "../engine-C1DIAEpn.js";
|
|
9
|
+
import { n as buildTraceToolsForGroup, t as TraceToolGroupName } from "../tool-groups-DiBi16ns.js";
|
|
10
|
+
import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, E as createChatTraceEngine, S as renderFindingSubject, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, b as findingSubjectGrammarPromptFor, c as emitControlIntegrityFindings, d as FindingsStore, f as PersistedFinding, g as FINDING_SUBJECT_SYNTAX, h as FINDING_SUBJECT_KINDS, i as IMPROVEMENT_KIND_SPEC, j as SemanticConceptJudgeOptions, l as DiffPolicy, m as diffFindings, n as KNOWLEDGE_POISONING_KIND_SPEC, o as CONTROL_INTEGRITY_ANALYST, p as defaultIsMaterial, r as KNOWLEDGE_GAP_KIND_SPEC, s as ControlIntegrityAnalyst, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine, x as parseFindingSubject, y as KIND_EXPECTED_SUBJECTS } from "../index-DBoXFaz0.js";
|
|
11
11
|
import { A as ReplyRowDecoded, C as primeProtocolSha256, D as DecodedReply, E as runPrimeExchange, M as decodeReplyRows, O as ReplyContract, S as normalizePrimeUsage, T as projectPrimeTrajectory, _ as buildPrimePrompt, a as PrimeProjectionOutcome, b as extractPrimeJsonObject, c as PrimeProtocolIdentity, d as PrimeRepairPromptSpec, f as PrimeRepairState, g as analystUsageReceiptFromPrimeUsage, h as PrimeTurnRecord, i as PrimeProjectionDelivery, j as ReplyRowRejection, k as ReplyEnvelope, l as PrimeRawUsage, m as PrimeRowDecoded, n as PrimeExchangeOutcome, o as PrimeProjectionSource, p as PrimeReplyContract, r as PrimeFailure, s as PrimePromptSpec, t as PrimeExchangeOptions, u as PrimeRejectedRow, v as buildPrimeRepairPrompt, w as primeReplyDefect, x as mergePrimeRawUsage, y as emptyPrimeRawUsage } from "../prime-protocol-T0i6xcAs.js";
|
|
12
12
|
import { i as nodeHttpPrimeBridgeTransport, n as PrimeBridgeTransportRequest, r as PrimeBridgeTransportResult, t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
|
|
13
13
|
import { z } from "zod";
|
|
@@ -1181,11 +1181,11 @@ declare function runAnalystBenchmarkCommand(argv: readonly string[], env?: NodeJ
|
|
|
1181
1181
|
declare const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = "sha256-canonical-source-manifest";
|
|
1182
1182
|
declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = "sha256-canonical-file-manifest";
|
|
1183
1183
|
declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES: readonly string[];
|
|
1184
|
-
declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "
|
|
1184
|
+
declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "091c759150ea8d7d8cf77f207b6b33992d605743f1d64f42b8e8b63207dade89";
|
|
1185
1185
|
declare const ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 = "1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b";
|
|
1186
1186
|
declare const ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 = "4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac";
|
|
1187
1187
|
declare const ANALYST_BENCHMARK_IMPLEMENTATION_FILES: readonly string[];
|
|
1188
|
-
declare const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "
|
|
1188
|
+
declare const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "00002cbbc8a749281287418f0abc03e4fc6a78ef3c1b4f21e33f2452629283da";
|
|
1189
1189
|
declare function analystBenchmarkImplementationDigest(): string;
|
|
1190
1190
|
declare function analystBenchmarkDependencyLockDigest(): string;
|
|
1191
1191
|
//#endregion
|
package/dist/analyst/index.js
CHANGED
|
@@ -4,11 +4,11 @@ import { B as FINDING_SUBJECT_SYNTAX, F as parseRawFinding, G as DEFAULT_TRACE_A
|
|
|
4
4
|
import { i as validateUsageSettlementTimeout, n as settleUsageReceiptFromCostLedger } from "../usage-receipt-Cx3xNeyB.js";
|
|
5
5
|
import { a as KNOWLEDGE_POISONING_KIND_SPEC, c as FAILURE_MODE_KIND_SPEC, d as emitControlIntegrityFindings, f as behavioralAnalyst, i as DEFAULT_TRACE_ANALYST_KINDS, l as CONTROL_INTEGRITY_ANALYST, n as AnalystRegistry, o as KNOWLEDGE_GAP_KIND_SPEC, p as deriveEfficiencyFindings, r as ExactAnalystRunExecutionError, s as IMPROVEMENT_KIND_SPEC, t as buildDefaultAnalystRegistry, u as ControlIntegrityAnalyst } from "../default-registry-xMLX160E.js";
|
|
6
6
|
import { i as makeProposalFinding, n as computeFindingId, r as makeFinding } from "../types-CW5b90JH.js";
|
|
7
|
-
import { t as createDspyRlmTraceEngine } from "../dspy-rlm-engine-
|
|
8
|
-
import { a as diffFindings, i as defaultIsMaterial, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "../semantic-concept-judge-
|
|
7
|
+
import { t as createDspyRlmTraceEngine } from "../dspy-rlm-engine-BX34vPVY.js";
|
|
8
|
+
import { a as diffFindings, i as defaultIsMaterial, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "../semantic-concept-judge-Bg-560H0.js";
|
|
9
9
|
import { t as createChatClient } from "../chat-client-B_ItsSBw.js";
|
|
10
10
|
import { a as scoreAnalystFindings, i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver, t as registryBenchmarkRunner } from "../benchmark-BeQJyrRC.js";
|
|
11
|
-
import { $ as ANALYST_BENCHMARK_MANIFEST_FILE, A as readAnalystInstructionsOverride, B as parseVerificationOutcome, C as emptyPublicBenchmarkRunner, D as summarizeCodeTraceCalibration, E as renderCodeTraceCalibrationMarkdown, F as publicBenchmarkRlmInstructions, G as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, H as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, I as publicBenchmarkSystemPrompt, J as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, L as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, M as MAX_INCORRECT_BLOCKS, N as MAX_INCORRECT_BLOCK_STEPS, O as analystInstructionsOverrideFromText, P as publicBenchmarkProtocolSha256, Q as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, R as appendVerificationArtifactsToOtlp, S as adaptPublicBenchmarkFindings, T as readAnalystBenchmarkArtifact, U as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, V as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, W as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, X as analystBenchmarkImplementationDigest, Y as analystBenchmarkDependencyLockDigest, Z as ANALYST_BENCHMARK_COST_LEDGER_FILE, _ as publicDirectAnalystDefinition, a as primeCodeTraceAnalystDefinition, at as codeTraceBenchCase, b as analystDefinitionAsymmetries, c as preparePublicAnalystBenchmark, ct as agentRxPredictionsToFindings, d as selectPublicBenchmarkRows, dt as normalizeBenchmarkLabel, et as ANALYST_BENCHMARK_OBSERVATIONS_FILE, f as createPublicBenchmarkRlmRunner, g as createPublicBenchmarkDirectRunner, h as runReplVariableAnalystDefinition, i as primeAnalystProtocolSha256, it as summarizeAgentRxCalibration, j as CODE_TRACE_BENCH_ANALYST_PROMPT, k as effectiveAnalystProtocolSha256, l as publicBenchmarkDistributions, lt as normalizeAgentRxCategory, m as rlmEngineLimits, n as renderAnalystBenchmarkMarkdown, nt as AGENT_RX_UPSTREAM_REVISION, o as runInlineAnalystDefinition, ot as codeTracerPredictionsToFindings, p as publicRlmAnalystDefinition, q as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, r as createPrimeBenchmarkRunner, rt as renderAgentRxCalibrationMarkdown, s as loadPublicBenchmarkRows, st as agentRxBenchmarkCase, t as runAnalystBenchmarkCommand, tt as compareAnalystRunners, u as publicBenchmarkSelectionReport, ut as roundAgentRxStep, v as runChunkedAnalystDefinition, w as expandCodeTraceFailureBlocks, x as analystDefinitionProtocolSha256, y as AnalystExpressivenessError, z as loadCodeTraceVerificationArtifacts } from "../benchmark-command-
|
|
11
|
+
import { $ as ANALYST_BENCHMARK_MANIFEST_FILE, A as readAnalystInstructionsOverride, B as parseVerificationOutcome, C as emptyPublicBenchmarkRunner, D as summarizeCodeTraceCalibration, E as renderCodeTraceCalibrationMarkdown, F as publicBenchmarkRlmInstructions, G as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, H as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, I as publicBenchmarkSystemPrompt, J as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, L as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, M as MAX_INCORRECT_BLOCKS, N as MAX_INCORRECT_BLOCK_STEPS, O as analystInstructionsOverrideFromText, P as publicBenchmarkProtocolSha256, Q as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, R as appendVerificationArtifactsToOtlp, S as adaptPublicBenchmarkFindings, T as readAnalystBenchmarkArtifact, U as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, V as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, W as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, X as analystBenchmarkImplementationDigest, Y as analystBenchmarkDependencyLockDigest, Z as ANALYST_BENCHMARK_COST_LEDGER_FILE, _ as publicDirectAnalystDefinition, a as primeCodeTraceAnalystDefinition, at as codeTraceBenchCase, b as analystDefinitionAsymmetries, c as preparePublicAnalystBenchmark, ct as agentRxPredictionsToFindings, d as selectPublicBenchmarkRows, dt as normalizeBenchmarkLabel, et as ANALYST_BENCHMARK_OBSERVATIONS_FILE, f as createPublicBenchmarkRlmRunner, g as createPublicBenchmarkDirectRunner, h as runReplVariableAnalystDefinition, i as primeAnalystProtocolSha256, it as summarizeAgentRxCalibration, j as CODE_TRACE_BENCH_ANALYST_PROMPT, k as effectiveAnalystProtocolSha256, l as publicBenchmarkDistributions, lt as normalizeAgentRxCategory, m as rlmEngineLimits, n as renderAnalystBenchmarkMarkdown, nt as AGENT_RX_UPSTREAM_REVISION, o as runInlineAnalystDefinition, ot as codeTracerPredictionsToFindings, p as publicRlmAnalystDefinition, q as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, r as createPrimeBenchmarkRunner, rt as renderAgentRxCalibrationMarkdown, s as loadPublicBenchmarkRows, st as agentRxBenchmarkCase, t as runAnalystBenchmarkCommand, tt as compareAnalystRunners, u as publicBenchmarkSelectionReport, ut as roundAgentRxStep, v as runChunkedAnalystDefinition, w as expandCodeTraceFailureBlocks, x as analystDefinitionProtocolSha256, y as AnalystExpressivenessError, z as loadCodeTraceVerificationArtifacts } from "../benchmark-command-C458yEFK.js";
|
|
12
12
|
import { a as extractPrimeJsonObject, c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, i as emptyPrimeRawUsage, l as primeReplyDefect, n as buildPrimePrompt, o as mergePrimeRawUsage, r as buildPrimeRepairPrompt, s as normalizePrimeUsage, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "../prime-protocol-DcAK9pC8.js";
|
|
13
13
|
import { t as nodeHttpPrimeBridgeTransport } from "../prime-bridge-transport-Z3cpKGoV.js";
|
|
14
14
|
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
@@ -3,18 +3,18 @@ import { a as hashCanonical, i as compareCodeUnits, o as jsonDocument, r as cano
|
|
|
3
3
|
import { r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
|
|
4
4
|
import { a as resolveModelPricing } from "./metrics-DOMpkoeb.js";
|
|
5
5
|
import { a as CostLedgerPersistenceError, i as CostLedger, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
|
|
6
|
-
import { d as
|
|
6
|
+
import { d as withLedgerFileLock, f as writeLedgerFileAtomically } from "./journal-DvKCvjty.js";
|
|
7
7
|
import "./ledger-core/index.js";
|
|
8
|
-
import {
|
|
8
|
+
import { C as fsCampaignStorage, S as createRunCostLedger, i as startExternalOptimizerModelProxy, r as runWithCleanup, x as acquireSingleRunLock, y as resolveExternalOptimizerProcessLimits } from "./external-optimizer-subprocess-kbqC-iWf.js";
|
|
9
9
|
import { f as redactText } from "./redact-B7rt9Zot.js";
|
|
10
10
|
import { M as RAW_FINDING_SCHEMA_PROMPT, N as RawAnalystFindingSchema, P as evidenceRefsFromRawFinding, g as TRACE_ANALYSIS_LIMITS, r as runTraceAnalyst } from "./kind-factory-CdPPHtGZ.js";
|
|
11
11
|
import { r as usageReceiptFromCostLedger } from "./usage-receipt-Cx3xNeyB.js";
|
|
12
12
|
import { a as callLlmJson, r as LlmResponseError, t as LlmCallError } from "./llm-client-KYQBh9C8.js";
|
|
13
13
|
import { r as makeFinding } from "./types-CW5b90JH.js";
|
|
14
|
-
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-
|
|
14
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-BX34vPVY.js";
|
|
15
15
|
import { i as otlpTextToTraceAnalysisStore, r as createOtlpBufferTraceStore, t as DEFAULT_MAX_TRACE_FILE_BYTES } from "./store-otlp-BPowgPm6.js";
|
|
16
16
|
import { i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver } from "./benchmark-BeQJyrRC.js";
|
|
17
|
-
import
|
|
17
|
+
import "./external-optimizer-process-D-Sab4v3.js";
|
|
18
18
|
import { t as assertEqualDeclarativeTerms } from "./equal-terms-Blx9Zk5I.js";
|
|
19
19
|
import { c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, n as buildPrimePrompt, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "./prime-protocol-DcAK9pC8.js";
|
|
20
20
|
import { t as nodeHttpPrimeBridgeTransport } from "./prime-bridge-transport-Z3cpKGoV.js";
|
|
@@ -1415,7 +1415,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
|
|
|
1415
1415
|
"package.json",
|
|
1416
1416
|
"pnpm-lock.yaml"
|
|
1417
1417
|
]);
|
|
1418
|
-
const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "
|
|
1418
|
+
const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "091c759150ea8d7d8cf77f207b6b33992d605743f1d64f42b8e8b63207dade89";
|
|
1419
1419
|
const ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 = "1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b";
|
|
1420
1420
|
const ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 = "4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac";
|
|
1421
1421
|
const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
|
|
@@ -1530,7 +1530,7 @@ const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
|
|
|
1530
1530
|
"src/trace/redact.ts",
|
|
1531
1531
|
"src/verdict-cache.ts"
|
|
1532
1532
|
]);
|
|
1533
|
-
const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "
|
|
1533
|
+
const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = "00002cbbc8a749281287418f0abc03e4fc6a78ef3c1b4f21e33f2452629283da";
|
|
1534
1534
|
function analystBenchmarkImplementationDigest() {
|
|
1535
1535
|
return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256;
|
|
1536
1536
|
}
|
|
@@ -6316,4 +6316,4 @@ function shellQuote(value) {
|
|
|
6316
6316
|
//#endregion
|
|
6317
6317
|
export { ANALYST_BENCHMARK_MANIFEST_FILE as $, readAnalystInstructionsOverride as A, parseVerificationOutcome as B, emptyPublicBenchmarkRunner as C, summarizeCodeTraceCalibration as D, renderCodeTraceCalibrationMarkdown as E, publicBenchmarkRlmInstructions as F, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as G, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as H, publicBenchmarkSystemPrompt as I, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as J, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as K, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as L, MAX_INCORRECT_BLOCKS as M, MAX_INCORRECT_BLOCK_STEPS as N, analystInstructionsOverrideFromText as O, publicBenchmarkProtocolSha256 as P, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as Q, appendVerificationArtifactsToOtlp as R, adaptPublicBenchmarkFindings as S, readAnalystBenchmarkArtifact as T, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as U, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as V, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as W, analystBenchmarkImplementationDigest as X, analystBenchmarkDependencyLockDigest as Y, ANALYST_BENCHMARK_COST_LEDGER_FILE as Z, publicDirectAnalystDefinition as _, primeCodeTraceAnalystDefinition as a, codeTraceBenchCase as at, analystDefinitionAsymmetries as b, preparePublicAnalystBenchmark as c, agentRxPredictionsToFindings as ct, selectPublicBenchmarkRows as d, normalizeBenchmarkLabel as dt, ANALYST_BENCHMARK_OBSERVATIONS_FILE as et, createPublicBenchmarkRlmRunner as f, createPublicBenchmarkDirectRunner as g, runReplVariableAnalystDefinition as h, primeAnalystProtocolSha256 as i, summarizeAgentRxCalibration as it, CODE_TRACE_BENCH_ANALYST_PROMPT as j, effectiveAnalystProtocolSha256 as k, publicBenchmarkDistributions as l, normalizeAgentRxCategory as lt, rlmEngineLimits as m, renderAnalystBenchmarkMarkdown as n, AGENT_RX_UPSTREAM_REVISION as nt, runInlineAnalystDefinition as o, codeTracerPredictionsToFindings as ot, publicRlmAnalystDefinition as p, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as q, createPrimeBenchmarkRunner as r, renderAgentRxCalibrationMarkdown as rt, loadPublicBenchmarkRows as s, agentRxBenchmarkCase as st, runAnalystBenchmarkCommand as t, compareAnalystRunners as tt, publicBenchmarkSelectionReport as u, roundAgentRxStep as ut, runChunkedAnalystDefinition as v, expandCodeTraceFailureBlocks as w, analystDefinitionProtocolSha256 as x, AnalystExpressivenessError as y, loadCodeTraceVerificationArtifacts as z };
|
|
6318
6318
|
|
|
6319
|
-
//# sourceMappingURL=benchmark-command-
|
|
6319
|
+
//# sourceMappingURL=benchmark-command-C458yEFK.js.map
|