@tangle-network/agent-eval 0.119.0 → 0.119.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/analyst/index.d.ts +30 -39
- package/dist/analyst/index.js +9 -5
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.js +5 -5
- package/dist/campaign/index.d.ts +28 -29
- package/dist/campaign/index.js +5 -5
- package/dist/{chunk-LMJ2TGWJ.js → chunk-JM2SKQMS.js} +2 -2
- package/dist/{chunk-RLCJQ6VZ.js → chunk-PSXJ32SR.js} +157 -3
- package/dist/chunk-PSXJ32SR.js.map +1 -0
- package/dist/{chunk-6QIM2EAP.js → chunk-PXD6ZFNY.js} +99 -1
- package/dist/chunk-PXD6ZFNY.js.map +1 -0
- package/dist/{chunk-GERDAIAL.js → chunk-QWMPPZ3X.js} +3 -3
- package/dist/{chunk-7A4LIMMY.js → chunk-S5TT5R3L.js} +229 -97
- package/dist/chunk-S5TT5R3L.js.map +1 -0
- package/dist/{chunk-ZYHJNKI3.js → chunk-ULOKLHIQ.js} +42 -5
- package/dist/chunk-ULOKLHIQ.js.map +1 -0
- package/dist/{chunk-FIUFRXP3.js → chunk-WW2A73HW.js} +49 -91
- package/dist/chunk-WW2A73HW.js.map +1 -0
- package/dist/{chunk-F6YUH3L4.js → chunk-XMBOU5W7.js} +12 -165
- package/dist/chunk-XMBOU5W7.js.map +1 -0
- package/dist/contract/index.d.ts +25 -16
- package/dist/contract/index.js +281 -5
- package/dist/contract/index.js.map +1 -1
- package/dist/index.d.ts +71 -68
- package/dist/index.js +31 -32
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/traces.d.ts +39 -25
- package/dist/traces.js +3 -5
- package/docs/trace-analysis.md +6 -2
- package/package.json +2 -2
- package/dist/chunk-6QIM2EAP.js.map +0 -1
- package/dist/chunk-7A4LIMMY.js.map +0 -1
- package/dist/chunk-F6YUH3L4.js.map +0 -1
- package/dist/chunk-FIUFRXP3.js.map +0 -1
- package/dist/chunk-RLCJQ6VZ.js.map +0 -1
- package/dist/chunk-ZYHJNKI3.js.map +0 -1
- /package/dist/{chunk-LMJ2TGWJ.js.map → chunk-JM2SKQMS.js.map} +0 -0
- /package/dist/{chunk-GERDAIAL.js.map → chunk-QWMPPZ3X.js.map} +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,12 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
+
## [0.119.1] — 2026-07-14 — portable improvement evidence
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `measuredComparisonFromSelfImproveResult()` converts paired held-out quality, cost, latency, uncertainty, power, decision, and provenance into the shared `AgentImprovementMeasuredComparison` contract.
|
|
12
|
+
|
|
7
13
|
## [0.119.0] — 2026-07-14 — chained, metered trace analysis
|
|
8
14
|
|
|
9
15
|
### Added
|
package/dist/analyst/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { TCloud } from '@tangle-network/tcloud';
|
|
2
|
-
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
|
+
import { AxAIArgs, AxAIService, AxFunction } from '@ax-llm/ax';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1706,6 +1706,8 @@ interface AnalystCost {
|
|
|
1706
1706
|
est_usd_per_run?: number;
|
|
1707
1707
|
/** Models the analyst expects to use (informational). */
|
|
1708
1708
|
models?: string[];
|
|
1709
|
+
/** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
|
|
1710
|
+
settlement_timeout_ms?: number;
|
|
1709
1711
|
}
|
|
1710
1712
|
interface AnalystRequirements {
|
|
1711
1713
|
/** Min number of shots / samples the analyst needs to produce signal. */
|
|
@@ -1729,7 +1731,7 @@ interface AnalystContext {
|
|
|
1729
1731
|
runId: string;
|
|
1730
1732
|
/** Stable correlation id so logs from a single registry.run() share a tag. */
|
|
1731
1733
|
correlationId: string;
|
|
1732
|
-
/**
|
|
1734
|
+
/** Enforced wall-clock deadline (epoch ms). */
|
|
1733
1735
|
deadlineMs?: number;
|
|
1734
1736
|
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
1735
1737
|
budgetUsd?: number;
|
|
@@ -1965,11 +1967,13 @@ interface CreateAnalystAiConfig {
|
|
|
1965
1967
|
apiKey: string;
|
|
1966
1968
|
/** OpenAI-compatible base URL — e.g. `https://router.tangle.tools/v1` or a
|
|
1967
1969
|
* cli-bridge loopback. */
|
|
1968
|
-
baseUrl
|
|
1969
|
-
/**
|
|
1970
|
+
baseUrl?: string;
|
|
1971
|
+
/** Additional headers required by the gateway, such as tenant or execution policy. */
|
|
1972
|
+
headers?: Record<string, string>;
|
|
1973
|
+
/** Model id forwarded to analyst calls. */
|
|
1970
1974
|
model: string;
|
|
1971
1975
|
/** Ax provider name. Defaults to the OpenAI-compatible client. */
|
|
1972
|
-
provider?: '
|
|
1976
|
+
provider?: AxAIArgs<unknown>['name'];
|
|
1973
1977
|
}
|
|
1974
1978
|
/**
|
|
1975
1979
|
* Construct the `AxAIService` an analyst kind calls through
|
|
@@ -2145,8 +2149,8 @@ declare function parseCanonicalRawFinding(row: unknown, log?: (msg: string, fiel
|
|
|
2145
2149
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
2146
2150
|
*
|
|
2147
2151
|
* A "kind" is a specialized analyst whose actor prompt, tool subset,
|
|
2148
|
-
* and Ax
|
|
2149
|
-
* classification, knowledge gap discovery, knowledge poisoning,
|
|
2152
|
+
* and bounded Ax subqueries target one failure-mode lens (failure-mode
|
|
2153
|
+
* classification, knowledge gap discovery, knowledge poisoning,
|
|
2150
2154
|
* self-improvement, ...). Kinds emit findings in the typed
|
|
2151
2155
|
* `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
|
|
2152
2156
|
* validates each row with Zod and lifts it into `AnalystFinding[]`.
|
|
@@ -2157,12 +2161,11 @@ declare function parseCanonicalRawFinding(row: unknown, log?: (msg: string, fiel
|
|
|
2157
2161
|
* - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
|
|
2158
2162
|
* A kind that never needs full-trace dumps can drop `viewTrace` /
|
|
2159
2163
|
* `viewSpans` and stay cheap.
|
|
2160
|
-
* - Each kind declares its
|
|
2161
|
-
*
|
|
2162
|
-
* (poisoning) usually stay at 0 since they have a tighter brief.
|
|
2164
|
+
* - Each kind declares its subquery + parallelism budget. Discovery-heavy
|
|
2165
|
+
* kinds can fan out more bounded semantic questions than narrow lenses.
|
|
2163
2166
|
*
|
|
2164
2167
|
* Optimizer hook: kinds may declare `goldens` — labeled examples used
|
|
2165
|
-
* by `
|
|
2168
|
+
* by `AxBootstrapFewShot` / `AxGEPA` to fit the actor
|
|
2166
2169
|
* description programmatically. Stored on the kind, not the registry,
|
|
2167
2170
|
* because the right metric is kind-specific.
|
|
2168
2171
|
*/
|
|
@@ -2182,20 +2185,18 @@ interface TraceAnalystKindSpec {
|
|
|
2182
2185
|
version: string;
|
|
2183
2186
|
/** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
|
|
2184
2187
|
actorDescription: string;
|
|
2185
|
-
/** Responder system prompt; falls back to a minimal "format the findings" instruction. */
|
|
2186
|
-
responderDescription?: string;
|
|
2187
2188
|
/** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
|
|
2188
2189
|
buildTools: (store: TraceAnalysisStore) => AxFunction[];
|
|
2189
|
-
/**
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2190
|
+
/** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
|
|
2191
|
+
subqueries?: {
|
|
2192
|
+
maxCalls: number;
|
|
2193
|
+
maxParallel?: number;
|
|
2193
2194
|
};
|
|
2194
2195
|
/** Actor turn cap. Default 12. */
|
|
2195
2196
|
maxTurns?: number;
|
|
2196
2197
|
/** Runtime char cap. Default 6000. */
|
|
2197
2198
|
maxRuntimeChars?: number;
|
|
2198
|
-
/** Maximum output tokens for every actor
|
|
2199
|
+
/** Maximum output tokens for every actor and subquery model call. Default 4096. */
|
|
2199
2200
|
maxOutputTokens?: number;
|
|
2200
2201
|
/** Cost classification surfaced in `registry.list()` and budget enforcement. */
|
|
2201
2202
|
cost: AnalystCost;
|
|
@@ -2348,7 +2349,7 @@ interface RegistryRunOpts {
|
|
|
2348
2349
|
skip?: string[];
|
|
2349
2350
|
/** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
|
|
2350
2351
|
budget?: BudgetPolicy;
|
|
2351
|
-
/**
|
|
2352
|
+
/** Active-work cap for the complete registry run. Model receipt settlement may follow. */
|
|
2352
2353
|
timeoutMs?: number;
|
|
2353
2354
|
/** Abort signal — forwarded into every analyst's context. */
|
|
2354
2355
|
signal?: AbortSignal;
|
|
@@ -2684,19 +2685,14 @@ declare function diffFindings(previous: PersistedFinding[], current: PersistedFi
|
|
|
2684
2685
|
* findings. The actor's job is *taxonomy + evidence*, not fix-design —
|
|
2685
2686
|
* that's the improvement-analyst's job.
|
|
2686
2687
|
*
|
|
2687
|
-
*
|
|
2688
|
-
*
|
|
2689
|
-
* into candidate clusters, each cluster spawns a focused investigator
|
|
2690
|
-
* that drills into representative traces, and a deeply-recursed
|
|
2691
|
-
* investigator may itself split a confounded mode into two sub-modes.
|
|
2692
|
-
* Each level fans out 4-way, so the analyst can investigate up to
|
|
2693
|
-
* ~16 leaf clusters before hitting the depth ceiling.
|
|
2688
|
+
* Eight bounded model subqueries let the actor compare candidate
|
|
2689
|
+
* clusters in parallel after it has loaded representative evidence.
|
|
2694
2690
|
*/
|
|
2695
2691
|
|
|
2696
2692
|
declare const FAILURE_MODE_KIND_SPEC: TraceAnalystKindSpec;
|
|
2697
2693
|
|
|
2698
2694
|
/**
|
|
2699
|
-
* Improvement analyst — actionable
|
|
2695
|
+
* Improvement analyst — actionable self-improvement findings.
|
|
2700
2696
|
*
|
|
2701
2697
|
* Brief: read findings from upstream analysts (failure-mode,
|
|
2702
2698
|
* knowledge-gap, knowledge-poisoning) AND the trace dataset itself,
|
|
@@ -2706,14 +2702,11 @@ declare const FAILURE_MODE_KIND_SPEC: TraceAnalystKindSpec;
|
|
|
2706
2702
|
* finding is one proposed edit with the locus, the diff, and the
|
|
2707
2703
|
* expected effect.
|
|
2708
2704
|
*
|
|
2709
|
-
* This is the
|
|
2705
|
+
* This is the self-improvement loop's last mile: the prior
|
|
2710
2706
|
* kinds describe *what's wrong*; this kind describes *what to change*.
|
|
2711
2707
|
*
|
|
2712
|
-
*
|
|
2713
|
-
*
|
|
2714
|
-
* fix directions (tighten prompt vs add tool vs adjust scaffolding),
|
|
2715
|
-
* and the actor should explore each with a focused subagent before
|
|
2716
|
-
* picking the highest-leverage one to recommend.
|
|
2708
|
+
* Eight bounded model subqueries let the actor compare competing fix
|
|
2709
|
+
* directions over the same cited evidence before recommending one.
|
|
2717
2710
|
*/
|
|
2718
2711
|
|
|
2719
2712
|
declare const IMPROVEMENT_KIND_SPEC: TraceAnalystKindSpec;
|
|
@@ -2740,8 +2733,8 @@ declare const IMPROVEMENT_KIND_SPEC: TraceAnalystKindSpec;
|
|
|
2740
2733
|
* knowledge-gap names the *information* whose absence (or staleness)
|
|
2741
2734
|
* caused the break. One failure-mode often maps to several gaps.
|
|
2742
2735
|
*
|
|
2743
|
-
*
|
|
2744
|
-
*
|
|
2736
|
+
* Five bounded model subqueries let the actor compare candidate gaps
|
|
2737
|
+
* across source layers after it has loaded the relevant excerpts.
|
|
2745
2738
|
*/
|
|
2746
2739
|
|
|
2747
2740
|
declare const KNOWLEDGE_GAP_KIND_SPEC: TraceAnalystKindSpec;
|
|
@@ -2760,10 +2753,8 @@ declare const KNOWLEDGE_GAP_KIND_SPEC: TraceAnalystKindSpec;
|
|
|
2760
2753
|
* surface as questions / self-correction; poisonings surface as
|
|
2761
2754
|
* confident-but-wrong actions that downstream evidence contradicts.
|
|
2762
2755
|
*
|
|
2763
|
-
*
|
|
2764
|
-
*
|
|
2765
|
-
* the agent acted on the false belief, one to confirm the belief
|
|
2766
|
-
* itself is actually false in ground truth.
|
|
2756
|
+
* Eight bounded model subqueries let the actor independently assess
|
|
2757
|
+
* the action and contradiction excerpts for candidate poisonings.
|
|
2767
2758
|
*/
|
|
2768
2759
|
|
|
2769
2760
|
declare const KNOWLEDGE_POISONING_KIND_SPEC: TraceAnalystKindSpec;
|
package/dist/analyst/index.js
CHANGED
|
@@ -9,13 +9,13 @@ import {
|
|
|
9
9
|
diffFindings,
|
|
10
10
|
emitSkillUsageFindings,
|
|
11
11
|
runSemanticConceptJudge
|
|
12
|
-
} from "../chunk-
|
|
12
|
+
} from "../chunk-JM2SKQMS.js";
|
|
13
13
|
import {
|
|
14
14
|
behavioralAnalyst,
|
|
15
15
|
buildDefaultAnalystRegistry,
|
|
16
16
|
createChatClient,
|
|
17
17
|
deriveEfficiencyFindings
|
|
18
|
-
} from "../chunk-
|
|
18
|
+
} from "../chunk-QWMPPZ3X.js";
|
|
19
19
|
import "../chunk-HHWE3POT.js";
|
|
20
20
|
import "../chunk-3YYRZDON.js";
|
|
21
21
|
import {
|
|
@@ -72,13 +72,13 @@ import {
|
|
|
72
72
|
validatePolicyEdit,
|
|
73
73
|
validatePolicyEditCandidateRecord,
|
|
74
74
|
validateUsageSettlementTimeout
|
|
75
|
-
} from "../chunk-
|
|
75
|
+
} from "../chunk-S5TT5R3L.js";
|
|
76
76
|
import "../chunk-NJC7U437.js";
|
|
77
77
|
import {
|
|
78
78
|
CostLedger
|
|
79
79
|
} from "../chunk-JHOJHHU7.js";
|
|
80
80
|
import "../chunk-VI2UW6B6.js";
|
|
81
|
-
import "../chunk-
|
|
81
|
+
import "../chunk-PXD6ZFNY.js";
|
|
82
82
|
import "../chunk-PC4UYEBM.js";
|
|
83
83
|
import "../chunk-XJYR7XFV.js";
|
|
84
84
|
import "../chunk-VSMTAMNK.js";
|
|
@@ -264,7 +264,11 @@ function createSemanticConceptJudgeAdapter(opts = {}) {
|
|
|
264
264
|
id,
|
|
265
265
|
description: "Runs the semantic-concept judge and surfaces missing / weak concepts as findings.",
|
|
266
266
|
inputKind: "custom",
|
|
267
|
-
cost: {
|
|
267
|
+
cost: {
|
|
268
|
+
kind: "llm",
|
|
269
|
+
models: opts.options?.model ? [opts.options.model] : void 0,
|
|
270
|
+
settlement_timeout_ms: settlementTimeoutMs
|
|
271
|
+
},
|
|
268
272
|
version: `${SEMANTIC_CONCEPT_JUDGE_VERSION}-adapter-${ADAPTER_REV}`,
|
|
269
273
|
async analyze(input, ctx) {
|
|
270
274
|
const costLedger = new CostLedger(ctx.budgetUsd);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/analyst/adapters.ts"],"sourcesContent":["/**\n * Adapter factories — lift each existing agent-eval primitive into the\n * Analyst contract without re-implementing it.\n *\n * Five primitives, five factories. Each one:\n * - Builds an Analyst with a stable id (caller chooses; defaults\n * given), a sensible default `inputKind`, a version derived from\n * the wrapped primitive's version + an adapter revision, and an\n * `analyze()` that calls the primitive and lifts its output to\n * AnalystFinding[] using `makeFinding()`.\n * - Maps severities: the existing `Severity` ('critical' | 'major' |\n * 'minor' | 'info') projects onto AnalystSeverity ('critical' |\n * 'high' | 'medium' | 'low' | 'info'); 'major' → 'high', 'minor' →\n * 'medium'. Domain analysts that want finer-grained mapping override.\n *\n * Adapters never own state. Calling the same factory twice with the\n * same primitive instance is safe.\n */\n\nimport { CostLedger } from '../cost-ledger'\nimport type {\n Finding as LayerFinding,\n Severity as LayerSeverity,\n MultiLayerVerifier,\n VerifyOptions,\n} from '../multi-layer-verifier'\nimport { RunCritic, type RunTrace } from '../run-critic'\nimport {\n runSemanticConceptJudge,\n SEMANTIC_CONCEPT_JUDGE_VERSION,\n type SemanticConceptJudgeInput,\n type SemanticConceptJudgeOptions,\n type SemanticConceptJudgeResult,\n} from '../semantic-concept-judge'\nimport type { JudgeFn, JudgeInput, JudgeScore, TCloud } from '../types'\nimport type { Analyst, AnalystFinding, AnalystSeverity } from './types'\nimport { makeFinding } from './types'\nimport { settleUsageReceiptFromCostLedger, validateUsageSettlementTimeout } from './usage-receipt'\n\nconst ADAPTER_REV = '1'\n\n// ── Severity bridges ───────────────────────────────────────────────\n\nexport function liftSeverity(s: LayerSeverity): AnalystSeverity {\n switch (s) {\n case 'critical':\n return 'critical'\n case 'major':\n return 'high'\n case 'minor':\n return 'medium'\n case 'info':\n return 'info'\n }\n}\n\n// ── 1. MultiLayerVerifier → Analyst ─────────────────────────────────\n\nexport interface VerifierAdapterOpts<Env> {\n id?: string\n area?: string\n verifier: MultiLayerVerifier<Env>\n /**\n * The verifier expects an `env` per run. Adapters take it from\n * `AnalystRunInputs.custom[<id>]` via the registry's 'custom' routing.\n */\n options?: Omit<VerifyOptions<Env>, 'env'>\n}\n\nexport function createVerifierAdapter<Env>(opts: VerifierAdapterOpts<Env>): Analyst<Env> {\n const id = opts.id ?? 'multi-layer-verifier'\n const area = opts.area ?? 'verification'\n return {\n id,\n description:\n \"Runs a MultiLayerVerifier and lifts each layer's findings into the analyst envelope.\",\n inputKind: 'custom',\n cost: { kind: 'deterministic' },\n version: `verifier-${ADAPTER_REV}`,\n async analyze(env, ctx) {\n const report = await opts.verifier.run({ env, ...opts.options })\n const out: AnalystFinding[] = []\n for (const layer of report.layers) {\n for (const finding of layer.findings) {\n out.push(liftLayerFinding(id, area, layer.layer, finding))\n }\n // Layer-level signal: a failed/error layer is itself a finding\n // even if it didn't emit per-finding rows.\n if (layer.status === 'fail' || layer.status === 'error' || layer.status === 'timeout') {\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: layer.layer,\n claim: `layer \"${layer.layer}\" ${layer.status}: ${layer.reason ?? 'no reason given'}`,\n severity:\n layer.status === 'error' ? 'high' : layer.status === 'timeout' ? 'medium' : 'high',\n confidence: 1,\n evidence_refs: [],\n metadata: {\n layer_status: layer.status,\n duration_ms: layer.durationMs,\n score: layer.score,\n diagnostics: layer.diagnostics,\n },\n }),\n )\n }\n }\n ctx.log?.('verifier complete', {\n layers: report.layers.length,\n blended: report.blendedScore,\n all_pass: report.allPass,\n })\n return out\n },\n }\n}\n\nfunction liftLayerFinding(\n analyst_id: string,\n area: string,\n layer: string,\n f: LayerFinding,\n): AnalystFinding {\n return makeFinding({\n analyst_id,\n area,\n subject: f.layer ?? layer,\n claim: f.message,\n severity: liftSeverity(f.severity),\n confidence: 0.85,\n evidence_refs: f.evidence\n ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }]\n : [],\n metadata: f.detail,\n })\n}\n\n// ── 2. RunCritic → Analyst ──────────────────────────────────────────\n\nexport interface RunCriticAdapterOpts {\n id?: string\n area?: string\n critic?: RunCritic\n /** Optional threshold below which a dimension is reported as a finding. Default 0.5. */\n threshold?: number\n}\n\nexport function createRunCriticAdapter(opts: RunCriticAdapterOpts = {}): Analyst<RunTrace> {\n const id = opts.id ?? 'run-critic'\n const area = opts.area ?? 'run-quality'\n const critic = opts.critic ?? new RunCritic()\n const threshold = opts.threshold ?? 0.5\n return {\n id,\n description:\n 'Scores a single run across success / grounding / drift / tool-quality and surfaces below-threshold dimensions.',\n inputKind: 'custom',\n cost: { kind: 'deterministic' },\n version: `run-critic-${ADAPTER_REV}`,\n async analyze(trace) {\n const score = critic.scoreTrace(trace)\n const out: AnalystFinding[] = []\n const dims: Array<[keyof typeof score, AnalystSeverity, string]> = [\n ['success', 'critical', 'run did not complete successfully'],\n ['goalProgress', 'high', 'goal progress is low'],\n ['repoGroundedness', 'high', 'output is poorly grounded in the repository'],\n ['toolUseQuality', 'medium', 'tool use quality is low'],\n ['patchQuality', 'medium', 'no real patch/edit evidence'],\n ['testReality', 'high', 'no real test/build evidence'],\n ['finalGate', 'critical', 'final gate is blocking'],\n ]\n for (const [dim, sev, msg] of dims) {\n const value = score[dim] as number\n if (typeof value === 'number' && value < threshold) {\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: dim,\n claim: msg,\n rationale: `${dim}=${value.toFixed(2)} below threshold ${threshold}`,\n severity: sev,\n confidence: 1,\n evidence_refs: [],\n metadata: { dimension: dim, value, threshold, run_id: trace.run.runId },\n }),\n )\n }\n }\n // Drift penalty is high → surface as a finding (inverse threshold).\n if (score.driftPenalty > 1 - threshold) {\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: 'drift',\n claim: 'agent output drifted from repository signal',\n rationale: `driftPenalty=${score.driftPenalty.toFixed(2)}`,\n severity: 'medium',\n confidence: 0.9,\n evidence_refs: [],\n metadata: { drift_penalty: score.driftPenalty, notes: score.notes },\n }),\n )\n }\n return out\n },\n }\n}\n\n// ── 3. JudgeFn → Analyst ────────────────────────────────────────────\n\nexport interface JudgeAdapterOpts {\n id?: string\n area?: string\n judge: JudgeFn\n /** TCloud handle the JudgeFn calls. */\n tcloud: TCloud\n /** Optional cost classification — most judges call an LLM. */\n cost?: Analyst['cost']\n /** Optional threshold below which a JudgeScore becomes a finding. Default 6 (on 0-10 scale). */\n threshold?: number\n}\n\nexport function createJudgeAdapter(opts: JudgeAdapterOpts): Analyst<JudgeInput> {\n const id = opts.id ?? 'judge'\n const area = opts.area ?? 'judge'\n const threshold = opts.threshold ?? 6\n return {\n id,\n description:\n 'Wraps an agent-eval JudgeFn into an analyst; below-threshold dimensions surface as findings.',\n inputKind: 'judge-input',\n cost: opts.cost ?? { kind: 'llm' },\n version: `judge-${ADAPTER_REV}`,\n async analyze(input) {\n const scores = await opts.judge(opts.tcloud, input)\n return scores\n .filter((s) => normalize10(s.score) < threshold)\n .map((s) => liftJudgeScore(id, area, s))\n },\n }\n}\n\nfunction normalize10(s: number): number {\n // JudgeScore convention is 0-10 but some judges emit 0-1. Coerce to 0-10.\n return s <= 1 ? s * 10 : s\n}\n\nfunction liftJudgeScore(analyst_id: string, area: string, s: JudgeScore): AnalystFinding {\n const score10 = normalize10(s.score)\n const severity: AnalystSeverity =\n score10 < 3 ? 'critical' : score10 < 5 ? 'high' : score10 < 7 ? 'medium' : 'low'\n return makeFinding({\n analyst_id,\n area,\n subject: s.dimension,\n claim: `${s.judgeName}/${s.dimension} scored ${score10.toFixed(1)}/10`,\n rationale: s.reasoning,\n severity,\n confidence: 0.8,\n evidence_refs: s.evidence\n ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: s.evidence }]\n : [],\n // Provenance: this finding IS a judge verdict (an acceptance score), not an\n // observation of behavior. The steer firewall (assertNoJudgeVerdict) rejects\n // it from steering — even when it cites an artifact above — because letting a\n // verdict steer the next attempt is the held-out judge leaking into the loop.\n derived_from_judge: true,\n metadata: { judge_name: s.judgeName, dimension: s.dimension, score_10: score10 },\n })\n}\n\n// ── 4. SemanticConceptJudge → Analyst ──────────────────────────────\n\nexport interface SemanticConceptJudgeAdapterOpts {\n id?: string\n area?: string\n /** Registry context owns cancellation and the per-analyst cost ledger. */\n options?: Omit<SemanticConceptJudgeOptions, 'costLedger' | 'signal'>\n /** Maximum post-cancellation wait for a provider receipt. Default 5 seconds. */\n settlementTimeoutMs?: number\n}\n\nexport function createSemanticConceptJudgeAdapter(\n opts: SemanticConceptJudgeAdapterOpts = {},\n): Analyst<SemanticConceptJudgeInput> {\n const id = opts.id ?? 'semantic-concept-judge'\n const area = opts.area ?? 'concept-coverage'\n const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs)\n return {\n id,\n description:\n 'Runs the semantic-concept judge and surfaces missing / weak concepts as findings.',\n inputKind: 'custom',\n cost: { kind: 'llm', models: opts.options?.model ? [opts.options.model] : undefined },\n version: `${SEMANTIC_CONCEPT_JUDGE_VERSION}-adapter-${ADAPTER_REV}`,\n async analyze(input, ctx) {\n const costLedger = new CostLedger(ctx.budgetUsd)\n let result: SemanticConceptJudgeResult\n try {\n result = await runSemanticConceptJudge(input, {\n ...opts.options,\n costLedger,\n signal: ctx.signal,\n })\n } finally {\n const usage = await settleUsageReceiptFromCostLedger(costLedger, {\n channel: 'judge',\n timeoutMs: settlementTimeoutMs,\n })\n if (!usage.settled) {\n ctx.log?.('semantic-concept judge provider settlement timed out', {\n pending_calls: usage.pendingCalls,\n timeout_ms: settlementTimeoutMs,\n })\n }\n ctx.recordUsage?.(usage.receipt)\n }\n if (!result.available) {\n return [\n makeFinding({\n analyst_id: id,\n area,\n claim: 'semantic-concept judge unavailable',\n rationale: result.error,\n severity: 'info',\n confidence: 1,\n evidence_refs: [],\n metadata: { reason: result.error },\n }),\n ]\n }\n const out: AnalystFinding[] = []\n for (const f of result.findings) {\n // Only surface gaps: missing concepts or low scores. Concepts at\n // 7+/10 with present=true are not findings — they're successes.\n if (f.present && f.score >= 7) continue\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: f.concept,\n claim: f.present\n ? `concept \"${f.concept}\" is weak (${f.score}/10)`\n : `concept \"${f.concept}\" is missing`,\n rationale: f.evidence,\n severity: liftSeverity(f.severity),\n confidence: 0.85,\n evidence_refs: [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }],\n metadata: {\n concept: f.concept,\n present: f.present,\n score_10: f.score,\n },\n }),\n )\n }\n return out\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,IAAM,cAAc;AAIb,SAAS,aAAa,GAAmC;AAC9D,UAAQ,GAAG;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAeO,SAAS,sBAA2B,MAA8C;AACvF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,YAAY,WAAW;AAAA,IAChC,MAAM,QAAQ,KAAK,KAAK;AACtB,YAAM,SAAS,MAAM,KAAK,SAAS,IAAI,EAAE,KAAK,GAAG,KAAK,QAAQ,CAAC;AAC/D,YAAM,MAAwB,CAAC;AAC/B,iBAAW,SAAS,OAAO,QAAQ;AACjC,mBAAW,WAAW,MAAM,UAAU;AACpC,cAAI,KAAK,iBAAiB,IAAI,MAAM,MAAM,OAAO,OAAO,CAAC;AAAA,QAC3D;AAGA,YAAI,MAAM,WAAW,UAAU,MAAM,WAAW,WAAW,MAAM,WAAW,WAAW;AACrF,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS,MAAM;AAAA,cACf,OAAO,UAAU,MAAM,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,UAAU,iBAAiB;AAAA,cACnF,UACE,MAAM,WAAW,UAAU,SAAS,MAAM,WAAW,YAAY,WAAW;AAAA,cAC9E,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU;AAAA,gBACR,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,gBACnB,OAAO,MAAM;AAAA,gBACb,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AACA,UAAI,MAAM,qBAAqB;AAAA,QAC7B,QAAQ,OAAO,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,UAAU,OAAO;AAAA,MACnB,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,SAAS,iBACP,YACA,MACA,OACA,GACgB;AAChB,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE,SAAS;AAAA,IACpB,OAAO,EAAE;AAAA,IACT,UAAU,aAAa,EAAE,QAAQ;AAAA,IACjC,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA,IACL,UAAU,EAAE;AAAA,EACd,CAAC;AACH;AAYO,SAAS,uBAAuB,OAA6B,CAAC,GAAsB;AACzF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,SAAS,KAAK,UAAU,IAAI,UAAU;AAC5C,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,cAAc,WAAW;AAAA,IAClC,MAAM,QAAQ,OAAO;AACnB,YAAM,QAAQ,OAAO,WAAW,KAAK;AACrC,YAAM,MAAwB,CAAC;AAC/B,YAAM,OAA6D;AAAA,QACjE,CAAC,WAAW,YAAY,mCAAmC;AAAA,QAC3D,CAAC,gBAAgB,QAAQ,sBAAsB;AAAA,QAC/C,CAAC,oBAAoB,QAAQ,6CAA6C;AAAA,QAC1E,CAAC,kBAAkB,UAAU,yBAAyB;AAAA,QACtD,CAAC,gBAAgB,UAAU,6BAA6B;AAAA,QACxD,CAAC,eAAe,QAAQ,6BAA6B;AAAA,QACrD,CAAC,aAAa,YAAY,wBAAwB;AAAA,MACpD;AACA,iBAAW,CAAC,KAAK,KAAK,GAAG,KAAK,MAAM;AAClC,cAAM,QAAQ,MAAM,GAAG;AACvB,YAAI,OAAO,UAAU,YAAY,QAAQ,WAAW;AAClD,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS;AAAA,cACT,OAAO;AAAA,cACP,WAAW,GAAG,GAAG,IAAI,MAAM,QAAQ,CAAC,CAAC,oBAAoB,SAAS;AAAA,cAClE,UAAU;AAAA,cACV,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU,EAAE,WAAW,KAAK,OAAO,WAAW,QAAQ,MAAM,IAAI,MAAM;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,UAAI,MAAM,eAAe,IAAI,WAAW;AACtC,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS;AAAA,YACT,OAAO;AAAA,YACP,WAAW,gBAAgB,MAAM,aAAa,QAAQ,CAAC,CAAC;AAAA,YACxD,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,eAAe,MAAM,cAAc,OAAO,MAAM,MAAM;AAAA,UACpE,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAgBO,SAAS,mBAAmB,MAA6C;AAC9E,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,KAAK,QAAQ,EAAE,MAAM,MAAM;AAAA,IACjC,SAAS,SAAS,WAAW;AAAA,IAC7B,MAAM,QAAQ,OAAO;AACnB,YAAM,SAAS,MAAM,KAAK,MAAM,KAAK,QAAQ,KAAK;AAClD,aAAO,OACJ,OAAO,CAAC,MAAM,YAAY,EAAE,KAAK,IAAI,SAAS,EAC9C,IAAI,CAAC,MAAM,eAAe,IAAI,MAAM,CAAC,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,SAAS,YAAY,GAAmB;AAEtC,SAAO,KAAK,IAAI,IAAI,KAAK;AAC3B;AAEA,SAAS,eAAe,YAAoB,MAAc,GAA+B;AACvF,QAAM,UAAU,YAAY,EAAE,KAAK;AACnC,QAAM,WACJ,UAAU,IAAI,aAAa,UAAU,IAAI,SAAS,UAAU,IAAI,WAAW;AAC7E,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE;AAAA,IACX,OAAO,GAAG,EAAE,SAAS,IAAI,EAAE,SAAS,WAAW,QAAQ,QAAQ,CAAC,CAAC;AAAA,IACjE,WAAW,EAAE;AAAA,IACb;AAAA,IACA,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,IAKL,oBAAoB;AAAA,IACpB,UAAU,EAAE,YAAY,EAAE,WAAW,WAAW,EAAE,WAAW,UAAU,QAAQ;AAAA,EACjF,CAAC;AACH;AAaO,SAAS,kCACd,OAAwC,CAAC,GACL;AACpC,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,sBAAsB,+BAA+B,KAAK,mBAAmB;AACnF,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,CAAC,KAAK,QAAQ,KAAK,IAAI,OAAU;AAAA,IACpF,SAAS,GAAG,8BAA8B,YAAY,WAAW;AAAA,IACjE,MAAM,QAAQ,OAAO,KAAK;AACxB,YAAM,aAAa,IAAI,WAAW,IAAI,SAAS;AAC/C,UAAI;AACJ,UAAI;AACF,iBAAS,MAAM,wBAAwB,OAAO;AAAA,UAC5C,GAAG,KAAK;AAAA,UACR;AAAA,UACA,QAAQ,IAAI;AAAA,QACd,CAAC;AAAA,MACH,UAAE;AACA,cAAM,QAAQ,MAAM,iCAAiC,YAAY;AAAA,UAC/D,SAAS;AAAA,UACT,WAAW;AAAA,QACb,CAAC;AACD,YAAI,CAAC,MAAM,SAAS;AAClB,cAAI,MAAM,wDAAwD;AAAA,YAChE,eAAe,MAAM;AAAA,YACrB,YAAY;AAAA,UACd,CAAC;AAAA,QACH;AACA,YAAI,cAAc,MAAM,OAAO;AAAA,MACjC;AACA,UAAI,CAAC,OAAO,WAAW;AACrB,eAAO;AAAA,UACL,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,OAAO;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,QAAQ,OAAO,MAAM;AAAA,UACnC,CAAC;AAAA,QACH;AAAA,MACF;AACA,YAAM,MAAwB,CAAC;AAC/B,iBAAW,KAAK,OAAO,UAAU;AAG/B,YAAI,EAAE,WAAW,EAAE,SAAS,EAAG;AAC/B,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS,EAAE;AAAA,YACX,OAAO,EAAE,UACL,YAAY,EAAE,OAAO,cAAc,EAAE,KAAK,SAC1C,YAAY,EAAE,OAAO;AAAA,YACzB,WAAW,EAAE;AAAA,YACb,UAAU,aAAa,EAAE,QAAQ;AAAA,YACjC,YAAY;AAAA,YACZ,eAAe,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC;AAAA,YACjF,UAAU;AAAA,cACR,SAAS,EAAE;AAAA,cACX,SAAS,EAAE;AAAA,cACX,UAAU,EAAE;AAAA,YACd;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/analyst/adapters.ts"],"sourcesContent":["/**\n * Adapter factories — lift each existing agent-eval primitive into the\n * Analyst contract without re-implementing it.\n *\n * Five primitives, five factories. Each one:\n * - Builds an Analyst with a stable id (caller chooses; defaults\n * given), a sensible default `inputKind`, a version derived from\n * the wrapped primitive's version + an adapter revision, and an\n * `analyze()` that calls the primitive and lifts its output to\n * AnalystFinding[] using `makeFinding()`.\n * - Maps severities: the existing `Severity` ('critical' | 'major' |\n * 'minor' | 'info') projects onto AnalystSeverity ('critical' |\n * 'high' | 'medium' | 'low' | 'info'); 'major' → 'high', 'minor' →\n * 'medium'. Domain analysts that want finer-grained mapping override.\n *\n * Adapters never own state. Calling the same factory twice with the\n * same primitive instance is safe.\n */\n\nimport { CostLedger } from '../cost-ledger'\nimport type {\n Finding as LayerFinding,\n Severity as LayerSeverity,\n MultiLayerVerifier,\n VerifyOptions,\n} from '../multi-layer-verifier'\nimport { RunCritic, type RunTrace } from '../run-critic'\nimport {\n runSemanticConceptJudge,\n SEMANTIC_CONCEPT_JUDGE_VERSION,\n type SemanticConceptJudgeInput,\n type SemanticConceptJudgeOptions,\n type SemanticConceptJudgeResult,\n} from '../semantic-concept-judge'\nimport type { JudgeFn, JudgeInput, JudgeScore, TCloud } from '../types'\nimport type { Analyst, AnalystFinding, AnalystSeverity } from './types'\nimport { makeFinding } from './types'\nimport { settleUsageReceiptFromCostLedger, validateUsageSettlementTimeout } from './usage-receipt'\n\nconst ADAPTER_REV = '1'\n\n// ── Severity bridges ───────────────────────────────────────────────\n\nexport function liftSeverity(s: LayerSeverity): AnalystSeverity {\n switch (s) {\n case 'critical':\n return 'critical'\n case 'major':\n return 'high'\n case 'minor':\n return 'medium'\n case 'info':\n return 'info'\n }\n}\n\n// ── 1. MultiLayerVerifier → Analyst ─────────────────────────────────\n\nexport interface VerifierAdapterOpts<Env> {\n id?: string\n area?: string\n verifier: MultiLayerVerifier<Env>\n /**\n * The verifier expects an `env` per run. Adapters take it from\n * `AnalystRunInputs.custom[<id>]` via the registry's 'custom' routing.\n */\n options?: Omit<VerifyOptions<Env>, 'env'>\n}\n\nexport function createVerifierAdapter<Env>(opts: VerifierAdapterOpts<Env>): Analyst<Env> {\n const id = opts.id ?? 'multi-layer-verifier'\n const area = opts.area ?? 'verification'\n return {\n id,\n description:\n \"Runs a MultiLayerVerifier and lifts each layer's findings into the analyst envelope.\",\n inputKind: 'custom',\n cost: { kind: 'deterministic' },\n version: `verifier-${ADAPTER_REV}`,\n async analyze(env, ctx) {\n const report = await opts.verifier.run({ env, ...opts.options })\n const out: AnalystFinding[] = []\n for (const layer of report.layers) {\n for (const finding of layer.findings) {\n out.push(liftLayerFinding(id, area, layer.layer, finding))\n }\n // Layer-level signal: a failed/error layer is itself a finding\n // even if it didn't emit per-finding rows.\n if (layer.status === 'fail' || layer.status === 'error' || layer.status === 'timeout') {\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: layer.layer,\n claim: `layer \"${layer.layer}\" ${layer.status}: ${layer.reason ?? 'no reason given'}`,\n severity:\n layer.status === 'error' ? 'high' : layer.status === 'timeout' ? 'medium' : 'high',\n confidence: 1,\n evidence_refs: [],\n metadata: {\n layer_status: layer.status,\n duration_ms: layer.durationMs,\n score: layer.score,\n diagnostics: layer.diagnostics,\n },\n }),\n )\n }\n }\n ctx.log?.('verifier complete', {\n layers: report.layers.length,\n blended: report.blendedScore,\n all_pass: report.allPass,\n })\n return out\n },\n }\n}\n\nfunction liftLayerFinding(\n analyst_id: string,\n area: string,\n layer: string,\n f: LayerFinding,\n): AnalystFinding {\n return makeFinding({\n analyst_id,\n area,\n subject: f.layer ?? layer,\n claim: f.message,\n severity: liftSeverity(f.severity),\n confidence: 0.85,\n evidence_refs: f.evidence\n ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }]\n : [],\n metadata: f.detail,\n })\n}\n\n// ── 2. RunCritic → Analyst ──────────────────────────────────────────\n\nexport interface RunCriticAdapterOpts {\n id?: string\n area?: string\n critic?: RunCritic\n /** Optional threshold below which a dimension is reported as a finding. Default 0.5. */\n threshold?: number\n}\n\nexport function createRunCriticAdapter(opts: RunCriticAdapterOpts = {}): Analyst<RunTrace> {\n const id = opts.id ?? 'run-critic'\n const area = opts.area ?? 'run-quality'\n const critic = opts.critic ?? new RunCritic()\n const threshold = opts.threshold ?? 0.5\n return {\n id,\n description:\n 'Scores a single run across success / grounding / drift / tool-quality and surfaces below-threshold dimensions.',\n inputKind: 'custom',\n cost: { kind: 'deterministic' },\n version: `run-critic-${ADAPTER_REV}`,\n async analyze(trace) {\n const score = critic.scoreTrace(trace)\n const out: AnalystFinding[] = []\n const dims: Array<[keyof typeof score, AnalystSeverity, string]> = [\n ['success', 'critical', 'run did not complete successfully'],\n ['goalProgress', 'high', 'goal progress is low'],\n ['repoGroundedness', 'high', 'output is poorly grounded in the repository'],\n ['toolUseQuality', 'medium', 'tool use quality is low'],\n ['patchQuality', 'medium', 'no real patch/edit evidence'],\n ['testReality', 'high', 'no real test/build evidence'],\n ['finalGate', 'critical', 'final gate is blocking'],\n ]\n for (const [dim, sev, msg] of dims) {\n const value = score[dim] as number\n if (typeof value === 'number' && value < threshold) {\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: dim,\n claim: msg,\n rationale: `${dim}=${value.toFixed(2)} below threshold ${threshold}`,\n severity: sev,\n confidence: 1,\n evidence_refs: [],\n metadata: { dimension: dim, value, threshold, run_id: trace.run.runId },\n }),\n )\n }\n }\n // Drift penalty is high → surface as a finding (inverse threshold).\n if (score.driftPenalty > 1 - threshold) {\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: 'drift',\n claim: 'agent output drifted from repository signal',\n rationale: `driftPenalty=${score.driftPenalty.toFixed(2)}`,\n severity: 'medium',\n confidence: 0.9,\n evidence_refs: [],\n metadata: { drift_penalty: score.driftPenalty, notes: score.notes },\n }),\n )\n }\n return out\n },\n }\n}\n\n// ── 3. JudgeFn → Analyst ────────────────────────────────────────────\n\nexport interface JudgeAdapterOpts {\n id?: string\n area?: string\n judge: JudgeFn\n /** TCloud handle the JudgeFn calls. */\n tcloud: TCloud\n /** Optional cost classification — most judges call an LLM. */\n cost?: Analyst['cost']\n /** Optional threshold below which a JudgeScore becomes a finding. Default 6 (on 0-10 scale). */\n threshold?: number\n}\n\nexport function createJudgeAdapter(opts: JudgeAdapterOpts): Analyst<JudgeInput> {\n const id = opts.id ?? 'judge'\n const area = opts.area ?? 'judge'\n const threshold = opts.threshold ?? 6\n return {\n id,\n description:\n 'Wraps an agent-eval JudgeFn into an analyst; below-threshold dimensions surface as findings.',\n inputKind: 'judge-input',\n cost: opts.cost ?? { kind: 'llm' },\n version: `judge-${ADAPTER_REV}`,\n async analyze(input) {\n const scores = await opts.judge(opts.tcloud, input)\n return scores\n .filter((s) => normalize10(s.score) < threshold)\n .map((s) => liftJudgeScore(id, area, s))\n },\n }\n}\n\nfunction normalize10(s: number): number {\n // JudgeScore convention is 0-10 but some judges emit 0-1. Coerce to 0-10.\n return s <= 1 ? s * 10 : s\n}\n\nfunction liftJudgeScore(analyst_id: string, area: string, s: JudgeScore): AnalystFinding {\n const score10 = normalize10(s.score)\n const severity: AnalystSeverity =\n score10 < 3 ? 'critical' : score10 < 5 ? 'high' : score10 < 7 ? 'medium' : 'low'\n return makeFinding({\n analyst_id,\n area,\n subject: s.dimension,\n claim: `${s.judgeName}/${s.dimension} scored ${score10.toFixed(1)}/10`,\n rationale: s.reasoning,\n severity,\n confidence: 0.8,\n evidence_refs: s.evidence\n ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: s.evidence }]\n : [],\n // Provenance: this finding IS a judge verdict (an acceptance score), not an\n // observation of behavior. The steer firewall (assertNoJudgeVerdict) rejects\n // it from steering — even when it cites an artifact above — because letting a\n // verdict steer the next attempt is the held-out judge leaking into the loop.\n derived_from_judge: true,\n metadata: { judge_name: s.judgeName, dimension: s.dimension, score_10: score10 },\n })\n}\n\n// ── 4. SemanticConceptJudge → Analyst ──────────────────────────────\n\nexport interface SemanticConceptJudgeAdapterOpts {\n id?: string\n area?: string\n /** Registry context owns cancellation and the per-analyst cost ledger. */\n options?: Omit<SemanticConceptJudgeOptions, 'costLedger' | 'signal'>\n /** Maximum post-cancellation wait for a provider receipt. Default 5 seconds. */\n settlementTimeoutMs?: number\n}\n\nexport function createSemanticConceptJudgeAdapter(\n opts: SemanticConceptJudgeAdapterOpts = {},\n): Analyst<SemanticConceptJudgeInput> {\n const id = opts.id ?? 'semantic-concept-judge'\n const area = opts.area ?? 'concept-coverage'\n const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs)\n return {\n id,\n description:\n 'Runs the semantic-concept judge and surfaces missing / weak concepts as findings.',\n inputKind: 'custom',\n cost: {\n kind: 'llm',\n models: opts.options?.model ? [opts.options.model] : undefined,\n settlement_timeout_ms: settlementTimeoutMs,\n },\n version: `${SEMANTIC_CONCEPT_JUDGE_VERSION}-adapter-${ADAPTER_REV}`,\n async analyze(input, ctx) {\n const costLedger = new CostLedger(ctx.budgetUsd)\n let result: SemanticConceptJudgeResult\n try {\n result = await runSemanticConceptJudge(input, {\n ...opts.options,\n costLedger,\n signal: ctx.signal,\n })\n } finally {\n const usage = await settleUsageReceiptFromCostLedger(costLedger, {\n channel: 'judge',\n timeoutMs: settlementTimeoutMs,\n })\n if (!usage.settled) {\n ctx.log?.('semantic-concept judge provider settlement timed out', {\n pending_calls: usage.pendingCalls,\n timeout_ms: settlementTimeoutMs,\n })\n }\n ctx.recordUsage?.(usage.receipt)\n }\n if (!result.available) {\n return [\n makeFinding({\n analyst_id: id,\n area,\n claim: 'semantic-concept judge unavailable',\n rationale: result.error,\n severity: 'info',\n confidence: 1,\n evidence_refs: [],\n metadata: { reason: result.error },\n }),\n ]\n }\n const out: AnalystFinding[] = []\n for (const f of result.findings) {\n // Only surface gaps: missing concepts or low scores. Concepts at\n // 7+/10 with present=true are not findings — they're successes.\n if (f.present && f.score >= 7) continue\n out.push(\n makeFinding({\n analyst_id: id,\n area,\n subject: f.concept,\n claim: f.present\n ? `concept \"${f.concept}\" is weak (${f.score}/10)`\n : `concept \"${f.concept}\" is missing`,\n rationale: f.evidence,\n severity: liftSeverity(f.severity),\n confidence: 0.85,\n evidence_refs: [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }],\n metadata: {\n concept: f.concept,\n present: f.present,\n score_10: f.score,\n },\n }),\n )\n }\n return out\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,IAAM,cAAc;AAIb,SAAS,aAAa,GAAmC;AAC9D,UAAQ,GAAG;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAeO,SAAS,sBAA2B,MAA8C;AACvF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,YAAY,WAAW;AAAA,IAChC,MAAM,QAAQ,KAAK,KAAK;AACtB,YAAM,SAAS,MAAM,KAAK,SAAS,IAAI,EAAE,KAAK,GAAG,KAAK,QAAQ,CAAC;AAC/D,YAAM,MAAwB,CAAC;AAC/B,iBAAW,SAAS,OAAO,QAAQ;AACjC,mBAAW,WAAW,MAAM,UAAU;AACpC,cAAI,KAAK,iBAAiB,IAAI,MAAM,MAAM,OAAO,OAAO,CAAC;AAAA,QAC3D;AAGA,YAAI,MAAM,WAAW,UAAU,MAAM,WAAW,WAAW,MAAM,WAAW,WAAW;AACrF,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS,MAAM;AAAA,cACf,OAAO,UAAU,MAAM,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,UAAU,iBAAiB;AAAA,cACnF,UACE,MAAM,WAAW,UAAU,SAAS,MAAM,WAAW,YAAY,WAAW;AAAA,cAC9E,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU;AAAA,gBACR,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,gBACnB,OAAO,MAAM;AAAA,gBACb,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AACA,UAAI,MAAM,qBAAqB;AAAA,QAC7B,QAAQ,OAAO,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,UAAU,OAAO;AAAA,MACnB,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,SAAS,iBACP,YACA,MACA,OACA,GACgB;AAChB,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE,SAAS;AAAA,IACpB,OAAO,EAAE;AAAA,IACT,UAAU,aAAa,EAAE,QAAQ;AAAA,IACjC,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA,IACL,UAAU,EAAE;AAAA,EACd,CAAC;AACH;AAYO,SAAS,uBAAuB,OAA6B,CAAC,GAAsB;AACzF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,SAAS,KAAK,UAAU,IAAI,UAAU;AAC5C,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,cAAc,WAAW;AAAA,IAClC,MAAM,QAAQ,OAAO;AACnB,YAAM,QAAQ,OAAO,WAAW,KAAK;AACrC,YAAM,MAAwB,CAAC;AAC/B,YAAM,OAA6D;AAAA,QACjE,CAAC,WAAW,YAAY,mCAAmC;AAAA,QAC3D,CAAC,gBAAgB,QAAQ,sBAAsB;AAAA,QAC/C,CAAC,oBAAoB,QAAQ,6CAA6C;AAAA,QAC1E,CAAC,kBAAkB,UAAU,yBAAyB;AAAA,QACtD,CAAC,gBAAgB,UAAU,6BAA6B;AAAA,QACxD,CAAC,eAAe,QAAQ,6BAA6B;AAAA,QACrD,CAAC,aAAa,YAAY,wBAAwB;AAAA,MACpD;AACA,iBAAW,CAAC,KAAK,KAAK,GAAG,KAAK,MAAM;AAClC,cAAM,QAAQ,MAAM,GAAG;AACvB,YAAI,OAAO,UAAU,YAAY,QAAQ,WAAW;AAClD,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS;AAAA,cACT,OAAO;AAAA,cACP,WAAW,GAAG,GAAG,IAAI,MAAM,QAAQ,CAAC,CAAC,oBAAoB,SAAS;AAAA,cAClE,UAAU;AAAA,cACV,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU,EAAE,WAAW,KAAK,OAAO,WAAW,QAAQ,MAAM,IAAI,MAAM;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,UAAI,MAAM,eAAe,IAAI,WAAW;AACtC,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS;AAAA,YACT,OAAO;AAAA,YACP,WAAW,gBAAgB,MAAM,aAAa,QAAQ,CAAC,CAAC;AAAA,YACxD,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,eAAe,MAAM,cAAc,OAAO,MAAM,MAAM;AAAA,UACpE,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAgBO,SAAS,mBAAmB,MAA6C;AAC9E,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,KAAK,QAAQ,EAAE,MAAM,MAAM;AAAA,IACjC,SAAS,SAAS,WAAW;AAAA,IAC7B,MAAM,QAAQ,OAAO;AACnB,YAAM,SAAS,MAAM,KAAK,MAAM,KAAK,QAAQ,KAAK;AAClD,aAAO,OACJ,OAAO,CAAC,MAAM,YAAY,EAAE,KAAK,IAAI,SAAS,EAC9C,IAAI,CAAC,MAAM,eAAe,IAAI,MAAM,CAAC,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,SAAS,YAAY,GAAmB;AAEtC,SAAO,KAAK,IAAI,IAAI,KAAK;AAC3B;AAEA,SAAS,eAAe,YAAoB,MAAc,GAA+B;AACvF,QAAM,UAAU,YAAY,EAAE,KAAK;AACnC,QAAM,WACJ,UAAU,IAAI,aAAa,UAAU,IAAI,SAAS,UAAU,IAAI,WAAW;AAC7E,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE;AAAA,IACX,OAAO,GAAG,EAAE,SAAS,IAAI,EAAE,SAAS,WAAW,QAAQ,QAAQ,CAAC,CAAC;AAAA,IACjE,WAAW,EAAE;AAAA,IACb;AAAA,IACA,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,IAKL,oBAAoB;AAAA,IACpB,UAAU,EAAE,YAAY,EAAE,WAAW,WAAW,EAAE,WAAW,UAAU,QAAQ;AAAA,EACjF,CAAC;AACH;AAaO,SAAS,kCACd,OAAwC,CAAC,GACL;AACpC,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,sBAAsB,+BAA+B,KAAK,mBAAmB;AACnF,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,QAAQ,KAAK,SAAS,QAAQ,CAAC,KAAK,QAAQ,KAAK,IAAI;AAAA,MACrD,uBAAuB;AAAA,IACzB;AAAA,IACA,SAAS,GAAG,8BAA8B,YAAY,WAAW;AAAA,IACjE,MAAM,QAAQ,OAAO,KAAK;AACxB,YAAM,aAAa,IAAI,WAAW,IAAI,SAAS;AAC/C,UAAI;AACJ,UAAI;AACF,iBAAS,MAAM,wBAAwB,OAAO;AAAA,UAC5C,GAAG,KAAK;AAAA,UACR;AAAA,UACA,QAAQ,IAAI;AAAA,QACd,CAAC;AAAA,MACH,UAAE;AACA,cAAM,QAAQ,MAAM,iCAAiC,YAAY;AAAA,UAC/D,SAAS;AAAA,UACT,WAAW;AAAA,QACb,CAAC;AACD,YAAI,CAAC,MAAM,SAAS;AAClB,cAAI,MAAM,wDAAwD;AAAA,YAChE,eAAe,MAAM;AAAA,YACrB,YAAY;AAAA,UACd,CAAC;AAAA,QACH;AACA,YAAI,cAAc,MAAM,OAAO;AAAA,MACjC;AACA,UAAI,CAAC,OAAO,WAAW;AACrB,eAAO;AAAA,UACL,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,OAAO;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,QAAQ,OAAO,MAAM;AAAA,UACnC,CAAC;AAAA,QACH;AAAA,MACF;AACA,YAAM,MAAwB,CAAC;AAC/B,iBAAW,KAAK,OAAO,UAAU;AAG/B,YAAI,EAAE,WAAW,EAAE,SAAS,EAAG;AAC/B,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS,EAAE;AAAA,YACX,OAAO,EAAE,UACL,YAAY,EAAE,OAAO,cAAc,EAAE,KAAK,SAC1C,YAAY,EAAE,OAAO;AAAA,YACzB,WAAW,EAAE;AAAA,YACb,UAAU,aAAa,EAAE,QAAQ;AAAA,YACjC,YAAY;AAAA,YACZ,eAAe,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC;AAAA,YACjF,UAAU;AAAA,cACR,SAAS,EAAE;AAAA,cACX,SAAS,EAAE;AAAA,cACX,UAAU,EAAE;AAAA,YACd;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;","names":[]}
|
package/dist/benchmarks/index.js
CHANGED
|
@@ -17,19 +17,19 @@ import {
|
|
|
17
17
|
runBenchmarkAdapter,
|
|
18
18
|
summarizeBenchmarkCampaign
|
|
19
19
|
} from "../chunk-D7AEXSM5.js";
|
|
20
|
-
import "../chunk-
|
|
21
|
-
import "../chunk-
|
|
20
|
+
import "../chunk-XMBOU5W7.js";
|
|
21
|
+
import "../chunk-PSXJ32SR.js";
|
|
22
22
|
import "../chunk-PAHNGS65.js";
|
|
23
23
|
import "../chunk-3YYRZDON.js";
|
|
24
|
-
import "../chunk-
|
|
24
|
+
import "../chunk-S5TT5R3L.js";
|
|
25
25
|
import "../chunk-ARU2PZFM.js";
|
|
26
26
|
import "../chunk-NJC7U437.js";
|
|
27
27
|
import "../chunk-PJQFMIOX.js";
|
|
28
28
|
import "../chunk-JHOJHHU7.js";
|
|
29
29
|
import "../chunk-VI2UW6B6.js";
|
|
30
|
-
import "../chunk-
|
|
30
|
+
import "../chunk-WW2A73HW.js";
|
|
31
31
|
import "../chunk-GGE4NNQT.js";
|
|
32
|
-
import "../chunk-
|
|
32
|
+
import "../chunk-PXD6ZFNY.js";
|
|
33
33
|
import "../chunk-PC4UYEBM.js";
|
|
34
34
|
import "../chunk-S3UZOQ5Y.js";
|
|
35
35
|
import "../chunk-MA6HLL3S.js";
|
package/dist/campaign/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
|
+
import { AxAIService, AxAgentActorTurnCallbackArgs, AxFunction, AxAIArgs } from '@ax-llm/ax';
|
|
3
3
|
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1279,12 +1279,14 @@ interface AnalystCost {
|
|
|
1279
1279
|
est_usd_per_run?: number;
|
|
1280
1280
|
/** Models the analyst expects to use (informational). */
|
|
1281
1281
|
models?: string[];
|
|
1282
|
+
/** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
|
|
1283
|
+
settlement_timeout_ms?: number;
|
|
1282
1284
|
}
|
|
1283
1285
|
interface AnalystContext {
|
|
1284
1286
|
runId: string;
|
|
1285
1287
|
/** Stable correlation id so logs from a single registry.run() share a tag. */
|
|
1286
1288
|
correlationId: string;
|
|
1287
|
-
/**
|
|
1289
|
+
/** Enforced wall-clock deadline (epoch ms). */
|
|
1288
1290
|
deadlineMs?: number;
|
|
1289
1291
|
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
1290
1292
|
budgetUsd?: number;
|
|
@@ -2157,11 +2159,11 @@ interface AnalyzeTracesInput {
|
|
|
2157
2159
|
question: string;
|
|
2158
2160
|
}
|
|
2159
2161
|
interface AnalyzeTracesResult {
|
|
2160
|
-
/** The
|
|
2162
|
+
/** The actor's submitted prose answer. */
|
|
2161
2163
|
answer: string;
|
|
2162
|
-
/** Bulleted findings
|
|
2164
|
+
/** Bulleted findings from the actor's structured completion. */
|
|
2163
2165
|
findings: string[];
|
|
2164
|
-
/** Per-
|
|
2166
|
+
/** Per-turn snapshots captured via `actorTurnCallback`. */
|
|
2165
2167
|
turns: AnalyzeTracesTurnSnapshot[];
|
|
2166
2168
|
/** Total turns the actor took. */
|
|
2167
2169
|
turnCount: number;
|
|
@@ -2187,13 +2189,14 @@ interface TraceAnalystChatMessage {
|
|
|
2187
2189
|
[key: string]: unknown;
|
|
2188
2190
|
}
|
|
2189
2191
|
interface AnalyzeTracesTurnSnapshot {
|
|
2192
|
+
stage: AxAgentActorTurnCallbackArgs['stage'];
|
|
2190
2193
|
turn: number;
|
|
2191
2194
|
isError: boolean;
|
|
2192
2195
|
/** The JS code the actor produced for this turn. */
|
|
2193
2196
|
code: string;
|
|
2194
2197
|
/** The formatted action-log entry the actor sees on the next turn. */
|
|
2195
2198
|
output: string;
|
|
2196
|
-
/** Provider thought (when `
|
|
2199
|
+
/** Provider thought (when `executorOptions.showThoughts` is true and the
|
|
2197
2200
|
* provider returns it). */
|
|
2198
2201
|
thought?: string;
|
|
2199
2202
|
}
|
|
@@ -2202,18 +2205,18 @@ interface AnalyzeTracesOptions {
|
|
|
2202
2205
|
source: string | TraceAnalysisStore;
|
|
2203
2206
|
/** Caller-provided AxAIService. */
|
|
2204
2207
|
ai: AxAIService;
|
|
2205
|
-
/** Model id forwarded to actor
|
|
2208
|
+
/** Model id forwarded to the actor. */
|
|
2206
2209
|
model?: string;
|
|
2207
|
-
/**
|
|
2208
|
-
|
|
2210
|
+
/** Maximum model subqueries. 0 disables model fan-out. Default 4. */
|
|
2211
|
+
maxSubqueries?: number;
|
|
2209
2212
|
/** Maximum actor turns. Default 12. */
|
|
2210
2213
|
maxTurns?: number;
|
|
2211
|
-
/** Maximum parallel
|
|
2212
|
-
|
|
2214
|
+
/** Maximum parallel model subqueries. Default 2. */
|
|
2215
|
+
maxParallelSubqueries?: number;
|
|
2216
|
+
/** Cancels in-flight model and tool work. */
|
|
2217
|
+
signal?: AbortSignal;
|
|
2213
2218
|
/** Override the actor description. */
|
|
2214
2219
|
actorDescription?: string;
|
|
2215
|
-
/** Override the subagent description. */
|
|
2216
|
-
subagentDescription?: string;
|
|
2217
2220
|
/** Per-turn observability hook. */
|
|
2218
2221
|
onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>;
|
|
2219
2222
|
/** Override max runtime characters per turn. Default 6000. */
|
|
@@ -2221,8 +2224,7 @@ interface AnalyzeTracesOptions {
|
|
|
2221
2224
|
/** When set, every turn's snapshot is appended to this JSONL file
|
|
2222
2225
|
* immediately. If the analyst crashes mid-loop (provider 503,
|
|
2223
2226
|
* network error, validator reject) the partial reasoning is still
|
|
2224
|
-
* on disk
|
|
2225
|
-
* evidence. */
|
|
2227
|
+
* on disk for diagnosis and recovery. */
|
|
2226
2228
|
progressLogPath?: string;
|
|
2227
2229
|
}
|
|
2228
2230
|
|
|
@@ -5912,8 +5914,8 @@ type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchem
|
|
|
5912
5914
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
5913
5915
|
*
|
|
5914
5916
|
* A "kind" is a specialized analyst whose actor prompt, tool subset,
|
|
5915
|
-
* and Ax
|
|
5916
|
-
* classification, knowledge gap discovery, knowledge poisoning,
|
|
5917
|
+
* and bounded Ax subqueries target one failure-mode lens (failure-mode
|
|
5918
|
+
* classification, knowledge gap discovery, knowledge poisoning,
|
|
5917
5919
|
* self-improvement, ...). Kinds emit findings in the typed
|
|
5918
5920
|
* `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
|
|
5919
5921
|
* validates each row with Zod and lifts it into `AnalystFinding[]`.
|
|
@@ -5924,12 +5926,11 @@ type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchem
|
|
|
5924
5926
|
* - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
|
|
5925
5927
|
* A kind that never needs full-trace dumps can drop `viewTrace` /
|
|
5926
5928
|
* `viewSpans` and stay cheap.
|
|
5927
|
-
* - Each kind declares its
|
|
5928
|
-
*
|
|
5929
|
-
* (poisoning) usually stay at 0 since they have a tighter brief.
|
|
5929
|
+
* - Each kind declares its subquery + parallelism budget. Discovery-heavy
|
|
5930
|
+
* kinds can fan out more bounded semantic questions than narrow lenses.
|
|
5930
5931
|
*
|
|
5931
5932
|
* Optimizer hook: kinds may declare `goldens` — labeled examples used
|
|
5932
|
-
* by `
|
|
5933
|
+
* by `AxBootstrapFewShot` / `AxGEPA` to fit the actor
|
|
5933
5934
|
* description programmatically. Stored on the kind, not the registry,
|
|
5934
5935
|
* because the right metric is kind-specific.
|
|
5935
5936
|
*/
|
|
@@ -5949,20 +5950,18 @@ interface TraceAnalystKindSpec {
|
|
|
5949
5950
|
version: string;
|
|
5950
5951
|
/** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
|
|
5951
5952
|
actorDescription: string;
|
|
5952
|
-
/** Responder system prompt; falls back to a minimal "format the findings" instruction. */
|
|
5953
|
-
responderDescription?: string;
|
|
5954
5953
|
/** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
|
|
5955
5954
|
buildTools: (store: TraceAnalysisStore) => AxFunction[];
|
|
5956
|
-
/**
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5955
|
+
/** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
|
|
5956
|
+
subqueries?: {
|
|
5957
|
+
maxCalls: number;
|
|
5958
|
+
maxParallel?: number;
|
|
5960
5959
|
};
|
|
5961
5960
|
/** Actor turn cap. Default 12. */
|
|
5962
5961
|
maxTurns?: number;
|
|
5963
5962
|
/** Runtime char cap. Default 6000. */
|
|
5964
5963
|
maxRuntimeChars?: number;
|
|
5965
|
-
/** Maximum output tokens for every actor
|
|
5964
|
+
/** Maximum output tokens for every actor and subquery model call. Default 4096. */
|
|
5966
5965
|
maxOutputTokens?: number;
|
|
5967
5966
|
/** Cost classification surfaced in `registry.list()` and budget enforcement. */
|
|
5968
5967
|
cost: AnalystCost;
|
|
@@ -6019,7 +6018,7 @@ interface TraceAnalystProposerOptions {
|
|
|
6019
6018
|
applyMaxTokens?: number;
|
|
6020
6019
|
/** Ax provider name. Default 'openai' — works for any OpenAI-compatible base
|
|
6021
6020
|
* via `apiURL`. Use 'deepseek' to hit DeepSeek's native provider. */
|
|
6022
|
-
provider?:
|
|
6021
|
+
provider?: AxAIArgs<unknown>['name'];
|
|
6023
6022
|
/** Which analyst kinds to run. Default = the full shipped suite. */
|
|
6024
6023
|
kinds?: readonly TraceAnalystKindSpec[];
|
|
6025
6024
|
/** Resolve the OTLP traces (JSONL string) the analyst should read for THIS
|
package/dist/campaign/index.js
CHANGED
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
userStoryScoreboard,
|
|
67
67
|
validateSearchLedgerEvent,
|
|
68
68
|
verifyCodeSurface
|
|
69
|
-
} from "../chunk-
|
|
69
|
+
} from "../chunk-XMBOU5W7.js";
|
|
70
70
|
import {
|
|
71
71
|
assertCodeSurfaceIdentity,
|
|
72
72
|
buildEvidenceVector,
|
|
@@ -103,7 +103,7 @@ import {
|
|
|
103
103
|
runOptimization,
|
|
104
104
|
surfaceContentHash,
|
|
105
105
|
surfaceHash
|
|
106
|
-
} from "../chunk-
|
|
106
|
+
} from "../chunk-PSXJ32SR.js";
|
|
107
107
|
import {
|
|
108
108
|
SearchLedgerConflictError,
|
|
109
109
|
SearchLedgerError,
|
|
@@ -120,15 +120,15 @@ import "../chunk-3YYRZDON.js";
|
|
|
120
120
|
import {
|
|
121
121
|
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
122
122
|
validatePolicyEditCandidateRecord
|
|
123
|
-
} from "../chunk-
|
|
123
|
+
} from "../chunk-S5TT5R3L.js";
|
|
124
124
|
import "../chunk-ARU2PZFM.js";
|
|
125
125
|
import "../chunk-NJC7U437.js";
|
|
126
126
|
import "../chunk-PJQFMIOX.js";
|
|
127
127
|
import "../chunk-JHOJHHU7.js";
|
|
128
128
|
import "../chunk-VI2UW6B6.js";
|
|
129
|
-
import "../chunk-
|
|
129
|
+
import "../chunk-WW2A73HW.js";
|
|
130
130
|
import "../chunk-GGE4NNQT.js";
|
|
131
|
-
import "../chunk-
|
|
131
|
+
import "../chunk-PXD6ZFNY.js";
|
|
132
132
|
import "../chunk-PC4UYEBM.js";
|
|
133
133
|
import "../chunk-S3UZOQ5Y.js";
|
|
134
134
|
import "../chunk-MA6HLL3S.js";
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
aggregateRunScore,
|
|
6
6
|
clamp01,
|
|
7
7
|
computeFindingId
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-S5TT5R3L.js";
|
|
9
9
|
import {
|
|
10
10
|
callLlmJson,
|
|
11
11
|
costReceiptFromLlm,
|
|
@@ -747,4 +747,4 @@ export {
|
|
|
747
747
|
runSemanticConceptJudge,
|
|
748
748
|
createSemanticConceptJudge
|
|
749
749
|
};
|
|
750
|
-
//# sourceMappingURL=chunk-
|
|
750
|
+
//# sourceMappingURL=chunk-JM2SKQMS.js.map
|