@tangle-network/agent-eval 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/analyst/index.d.ts +2772 -21
- package/dist/analyst/index.js +7 -6
- package/dist/analyst/index.js.map +1 -1
- package/dist/belief-state/index.d.ts +706 -10
- package/dist/belief-state/index.js +2 -1
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +958 -14
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.d.ts +449 -4
- package/dist/builder-eval/index.js +4 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +4276 -74
- package/dist/campaign/index.js +12 -10
- package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
- package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
- package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
- package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
- package/dist/chunk-DP3BBMYJ.js.map +1 -0
- package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
- package/dist/chunk-HKUCJ437.js.map +1 -0
- package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
- package/dist/chunk-KSDQVPLR.js +286 -0
- package/dist/chunk-KSDQVPLR.js.map +1 -0
- package/dist/chunk-MA6HLL3S.js +65 -0
- package/dist/chunk-MA6HLL3S.js.map +1 -0
- package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
- package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
- package/dist/chunk-PXE2VKMX.js +140 -0
- package/dist/chunk-PXE2VKMX.js.map +1 -0
- package/dist/{chunk-ZUXV7UWZ.js → chunk-Q442S5AS.js} +73 -19
- package/dist/{chunk-ZUXV7UWZ.js.map → chunk-Q442S5AS.js.map} +1 -1
- package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
- package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
- package/dist/chunk-QKEGNI5B.js.map +1 -0
- package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
- package/dist/chunk-S3UZOQ5Y.js.map +1 -0
- package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
- package/dist/chunk-SVH2ANFD.js.map +1 -0
- package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
- package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
- package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
- package/dist/chunk-WDHBCA3M.js +31 -0
- package/dist/chunk-WDHBCA3M.js.map +1 -0
- package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
- package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
- package/dist/chunk-ZET2UAYW.js.map +1 -0
- package/dist/contract/index.d.ts +4012 -38
- package/dist/contract/index.js +56 -29
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +1013 -9
- package/dist/control.js +4 -3
- package/dist/fuzz.d.ts +194 -4
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.d.ts +498 -17
- package/dist/index.d.ts +10982 -1286
- package/dist/index.js +97 -80
- package/dist/index.js.map +1 -1
- package/dist/matrix/index.d.ts +139 -4
- package/dist/meta-eval/index.d.ts +862 -15
- package/dist/meta-eval/index.js +2 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/index.d.ts +214 -14
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +392 -7
- package/dist/pipelines/index.js +5 -3
- package/dist/pipelines/index.js.map +1 -1
- package/dist/reporting.d.ts +1277 -17
- package/dist/rl.d.ts +2359 -28
- package/dist/rl.js +6 -5
- package/dist/rl.js.map +1 -1
- package/dist/storyboard/index.d.ts +86 -1
- package/dist/trace-attributes.d.ts +16 -0
- package/dist/trace-attributes.js +32 -0
- package/dist/trace-attributes.js.map +1 -0
- package/dist/traces.d.ts +1970 -697
- package/dist/traces.js +49 -32
- package/dist/wire/index.d.ts +655 -9
- package/docs/insight-report.md +44 -0
- package/package.json +9 -3
- package/dist/adversarial-B7loGVVX.d.ts +0 -19
- package/dist/analyst-C8HHvfJp.d.ts +0 -88
- package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
- package/dist/baseline-DKq3gJpP.d.ts +0 -141
- package/dist/calibration-C8MTS7cw.d.ts +0 -101
- package/dist/chunk-5UF54T55.js.map +0 -1
- package/dist/chunk-E4BUPP7Z.js.map +0 -1
- package/dist/chunk-FQNLDL4D.js.map +0 -1
- package/dist/chunk-LQUTGLOZ.js.map +0 -1
- package/dist/chunk-S2F4J57L.js.map +0 -1
- package/dist/chunk-YZPO4UHR.js.map +0 -1
- package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
- package/dist/control-6vuGfmDH.d.ts +0 -258
- package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
- package/dist/dataset-NENEzRgk.d.ts +0 -115
- package/dist/default-registry-DaK8b3fv.d.ts +0 -155
- package/dist/emitter-CjD7vUwv.d.ts +0 -122
- package/dist/errors-oeQrLqXC.d.ts +0 -74
- package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
- package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
- package/dist/gepa-eESocoDi.d.ts +0 -642
- package/dist/index-PdX4VnPA.d.ts +0 -423
- package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
- package/dist/integrity-DqlBiLyK.d.ts +0 -81
- package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
- package/dist/kind-factory-ClZmO25A.d.ts +0 -171
- package/dist/llm-client-qoDd18Qz.d.ts +0 -289
- package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
- package/dist/off-policy-DiwuKKg7.d.ts +0 -132
- package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
- package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
- package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
- package/dist/provenance-DpjwyseI.d.ts +0 -541
- package/dist/query-CF7PG61p.d.ts +0 -35
- package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
- package/dist/release-report-C8G2i5Xi.d.ts +0 -236
- package/dist/researcher-C8XyxQsu.d.ts +0 -387
- package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
- package/dist/run-record-BDH49H2E.d.ts +0 -360
- package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
- package/dist/schema-B3Q3l9Z_.d.ts +0 -201
- package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
- package/dist/sequential-5iSVfzl2.d.ts +0 -139
- package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
- package/dist/statistics-KUnG73jH.d.ts +0 -494
- package/dist/storage-DrX3v_5B.d.ts +0 -50
- package/dist/store-C1YxJDEK.d.ts +0 -248
- package/dist/store-DGqD0Pyo.d.ts +0 -116
- package/dist/summary-report-C5bKFfm-.d.ts +0 -445
- package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
- package/dist/types-BSw1rOUB.d.ts +0 -634
- package/dist/types-BUxNaJ8c.d.ts +0 -108
- package/dist/types-BkfcQnxV.d.ts +0 -313
- package/dist/verdict-C9MlYujm.d.ts +0 -35
- /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
- /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
- /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
- /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
- /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
- /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
- /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
- /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
- /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Off-policy evaluation primitives.
|
|
3
|
-
*
|
|
4
|
-
* Standard inverse-probability-weighted (IPS), self-normalized
|
|
5
|
-
* importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
|
|
6
|
-
* value of a *target* policy given trajectories collected under a
|
|
7
|
-
* *behavior* policy. This is the canonical RL eval task: "we have last
|
|
8
|
-
* week's runs, we changed the policy — how would the new one do without
|
|
9
|
-
* re-running?"
|
|
10
|
-
*
|
|
11
|
-
* The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
|
|
12
|
-
* & Joachims 2015 for SNIPS) but the *application* to LLM-agent
|
|
13
|
-
* evaluation needs care:
|
|
14
|
-
*
|
|
15
|
-
* - The "policy" is the (prompt, tool config, model snapshot) triple.
|
|
16
|
-
* Two policies have the same probability over an action *iff* their
|
|
17
|
-
* LLM call would emit the same token with the same probability —
|
|
18
|
-
* which is generally unknowable without the model log-probs.
|
|
19
|
-
* - For LLM agents, propensity scores must be supplied by the caller
|
|
20
|
-
* (logged in the trace, recovered from token log-probs, or estimated
|
|
21
|
-
* via a learned propensity model). We do NOT estimate propensity here.
|
|
22
|
-
* - Doubly-robust requires a Q-function (model-based reward predictor).
|
|
23
|
-
* We accept any callable; consumers pass either a tabular average,
|
|
24
|
-
* a regression fit, or a learned reward model.
|
|
25
|
-
*
|
|
26
|
-
* Bias / variance tradeoffs:
|
|
27
|
-
* - IPS: unbiased; high variance for small overlap, infinite variance
|
|
28
|
-
* when target has support outside behavior.
|
|
29
|
-
* - SNIPS: lower variance, slight bias; usually preferred in practice.
|
|
30
|
-
* - DR: doubly-robust — unbiased if either propensity OR Q-function is
|
|
31
|
-
* correct. Lowest practical variance when Q is decent. Use this.
|
|
32
|
-
*
|
|
33
|
-
* Caveat the panel will land: on the LLM-agent setting, propensity scores
|
|
34
|
-
* recovered from token log-probs are noisy, the action space is enormous,
|
|
35
|
-
* and overlap is often poor. These estimators are useful but not magic;
|
|
36
|
-
* complement with `replayCampaign` (exact replay where the request hashes
|
|
37
|
-
* match) for high-confidence answers and OPE for the gap.
|
|
38
|
-
*/
|
|
39
|
-
interface OffPolicyTrajectory {
|
|
40
|
-
/** Stable id, for traceability through the dataset. */
|
|
41
|
-
runId: string;
|
|
42
|
-
/** Reward observed under the behavior policy (the realized outcome). */
|
|
43
|
-
reward: number;
|
|
44
|
-
/**
|
|
45
|
-
* Behavior-policy probability of the action that was taken. For LLM
|
|
46
|
-
* agents this is typically `exp(sum(token_log_probs))` over the chosen
|
|
47
|
-
* trajectory. Must be in (0, 1].
|
|
48
|
-
*/
|
|
49
|
-
behaviorProb: number;
|
|
50
|
-
/**
|
|
51
|
-
* Target-policy probability of the same action. For replay-style
|
|
52
|
-
* counterfactual evaluation this is what the *new* policy would have
|
|
53
|
-
* assigned to the *old* trajectory. Must be in [0, 1].
|
|
54
|
-
*/
|
|
55
|
-
targetProb: number;
|
|
56
|
-
/**
|
|
57
|
-
* Optional model-based reward prediction at the same context. Used by
|
|
58
|
-
* `doublyRobust`. Set to `null` for IPS-only evaluation.
|
|
59
|
-
*/
|
|
60
|
-
qHat?: number | null;
|
|
61
|
-
}
|
|
62
|
-
interface OffPolicyEstimate {
|
|
63
|
-
/** Estimated value of the target policy. */
|
|
64
|
-
value: number;
|
|
65
|
-
/** Standard error of the estimate. */
|
|
66
|
-
standardError: number;
|
|
67
|
-
/** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */
|
|
68
|
-
effectiveSampleSize: number;
|
|
69
|
-
/** Number of trajectories used. */
|
|
70
|
-
n: number;
|
|
71
|
-
/**
|
|
72
|
-
* Diagnostic: maximum importance weight observed. Large values (>>10x
|
|
73
|
-
* mean) are a red flag — variance is dominated by a few outliers.
|
|
74
|
-
*/
|
|
75
|
-
maxImportanceWeight: number;
|
|
76
|
-
}
|
|
77
|
-
interface OffPolicyOptions {
|
|
78
|
-
/**
|
|
79
|
-
* Cap importance weights at this value (Ionides 2008 truncated IS) to
|
|
80
|
-
* trade unbiasedness for variance reduction. Default `Infinity` (no cap).
|
|
81
|
-
* Set e.g. `10` for stable estimates when the policies are close.
|
|
82
|
-
*/
|
|
83
|
-
weightCap?: number;
|
|
84
|
-
/** Reward clipping range. Default `[0, 1]`. */
|
|
85
|
-
rewardClip?: {
|
|
86
|
-
low: number;
|
|
87
|
-
high: number;
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
|
|
92
|
-
* of E[reward under target policy]. Variance scales with the spread of
|
|
93
|
-
* target/behavior ratios.
|
|
94
|
-
*/
|
|
95
|
-
declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
|
|
96
|
-
/**
|
|
97
|
-
* Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
|
|
98
|
-
* the cost of small bias (vanishing as N grows). The right default for
|
|
99
|
-
* LLM-agent evaluation where overlap is often poor.
|
|
100
|
-
*/
|
|
101
|
-
declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
|
|
102
|
-
/**
|
|
103
|
-
* Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
|
|
104
|
-
*
|
|
105
|
-
* V_DR = (1/N) * sum_i [ q_hat_i + (target_prob_i / behavior_prob_i) * (r_i - q_hat_i) ]
|
|
106
|
-
*
|
|
107
|
-
* Unbiased if EITHER:
|
|
108
|
-
* - the importance ratios are correct (IPS-style validity), OR
|
|
109
|
-
* - the Q-hat function is correct (model-based validity).
|
|
110
|
-
*
|
|
111
|
-
* In practice both are imperfect, but the residual bias is the *product*
|
|
112
|
-
* of both errors — much smaller than either alone. This is why DR is the
|
|
113
|
-
* default in production OPE pipelines.
|
|
114
|
-
*
|
|
115
|
-
* Requires `qHat` on every trajectory. If any are `null`, the estimator
|
|
116
|
-
* falls back to SNIPS for those entries (loud-fallback behavior; the
|
|
117
|
-
* report's `n` reflects the full set but `effectiveSampleSize` accounts
|
|
118
|
-
* for the lost variance reduction).
|
|
119
|
-
*/
|
|
120
|
-
declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
|
|
121
|
-
/**
|
|
122
|
-
* Convenience: run all three estimators and return them side-by-side.
|
|
123
|
-
* The recommended diagnostic — agreement across estimators is a much
|
|
124
|
-
* stronger signal than any single one.
|
|
125
|
-
*/
|
|
126
|
-
declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
|
|
127
|
-
ips: OffPolicyEstimate;
|
|
128
|
-
snips: OffPolicyEstimate;
|
|
129
|
-
dr: OffPolicyEstimate;
|
|
130
|
-
};
|
|
131
|
-
|
|
132
|
-
export { type OffPolicyEstimate as O, type OffPolicyOptions as a, type OffPolicyTrajectory as b, doublyRobust as d, inverseProbabilityWeighting as i, offPolicyEstimateAll as o, selfNormalizedImportanceWeighting as s };
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* OutcomeStore — deployment outcomes attached to Run IDs.
|
|
3
|
-
*
|
|
4
|
-
* Outcomes arrive asynchronously from production telemetry after the
|
|
5
|
-
* eval run completed: user ratings, retention flags, conversion events,
|
|
6
|
-
* revenue, support-ticket rate, anything a product team can measure.
|
|
7
|
-
* The store is a peer to TraceStore — separate lifecycle, same runId
|
|
8
|
-
* foreign key.
|
|
9
|
-
*
|
|
10
|
-
* The whole point of this module is to make the meta-eval correlation
|
|
11
|
-
* question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
|
|
12
|
-
*/
|
|
13
|
-
interface DeploymentOutcome {
|
|
14
|
-
runId: string;
|
|
15
|
-
capturedAt: number;
|
|
16
|
-
/** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
|
|
17
|
-
metrics: Record<string, number>;
|
|
18
|
-
/** Dimensions for stratified analysis — cohort, region, user_segment. */
|
|
19
|
-
labels?: Record<string, string>;
|
|
20
|
-
/** Free-form provenance (source system, pipeline version). */
|
|
21
|
-
source?: string;
|
|
22
|
-
}
|
|
23
|
-
interface OutcomeFilter {
|
|
24
|
-
runIds?: string[];
|
|
25
|
-
since?: number;
|
|
26
|
-
until?: number;
|
|
27
|
-
label?: {
|
|
28
|
-
key: string;
|
|
29
|
-
value: string;
|
|
30
|
-
};
|
|
31
|
-
source?: string;
|
|
32
|
-
}
|
|
33
|
-
interface OutcomeStore {
|
|
34
|
-
append(outcome: DeploymentOutcome): Promise<void>;
|
|
35
|
-
/** All outcomes attached to this run (a single run can have many — multiple
|
|
36
|
-
* capture windows over deployment time). */
|
|
37
|
-
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
38
|
-
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
39
|
-
}
|
|
40
|
-
declare class InMemoryOutcomeStore implements OutcomeStore {
|
|
41
|
-
private items;
|
|
42
|
-
append(outcome: DeploymentOutcome): Promise<void>;
|
|
43
|
-
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
44
|
-
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
45
|
-
}
|
|
46
|
-
interface FileSystemOutcomeStoreOptions {
|
|
47
|
-
dir: string;
|
|
48
|
-
maxBytes?: number;
|
|
49
|
-
}
|
|
50
|
-
declare class FileSystemOutcomeStore implements OutcomeStore {
|
|
51
|
-
private dir;
|
|
52
|
-
private maxBytes;
|
|
53
|
-
private memo?;
|
|
54
|
-
private loaded;
|
|
55
|
-
constructor(options: FileSystemOutcomeStoreOptions);
|
|
56
|
-
private ensureDir;
|
|
57
|
-
append(outcome: DeploymentOutcome): Promise<void>;
|
|
58
|
-
private load;
|
|
59
|
-
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
60
|
-
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export { type DeploymentOutcome as D, FileSystemOutcomeStore as F, InMemoryOutcomeStore as I, type OutcomeFilter as O, type FileSystemOutcomeStoreOptions as a, type OutcomeStore as b };
|
|
@@ -1,455 +0,0 @@
|
|
|
1
|
-
import { R as RunRecord, A as AgentProfileCell, f as AgentProfileJson } from './run-record-BDH49H2E.js';
|
|
2
|
-
import { V as ValidationError } from './errors-oeQrLqXC.js';
|
|
3
|
-
import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
|
|
4
|
-
import { a as JudgeInput } from './types-BkfcQnxV.js';
|
|
5
|
-
import { c as LlmCallRequest, d as LlmCallResult } from './llm-client-qoDd18Qz.js';
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* ChatClient — the single LLM abstraction analysts call.
|
|
9
|
-
*
|
|
10
|
-
* agent-eval already ships an `LlmClient` (OpenAI-compatible, retry,
|
|
11
|
-
* graceful JSON-schema degrade) and judges that talk to `TCloud`. Two
|
|
12
|
-
* mixed patterns force every analyst author to pick a transport, which
|
|
13
|
-
* couples analyst code to runtime concerns (cli-bridge vs router vs
|
|
14
|
-
* sandbox-sdk) it shouldn't know about.
|
|
15
|
-
*
|
|
16
|
-
* `ChatClient` is one interface every analyst takes via `AnalystContext.chat`.
|
|
17
|
-
* The operator decides at the registry boundary which transport binds
|
|
18
|
-
* to it. Analyst code stays transport-agnostic; swapping production
|
|
19
|
-
* (sandbox-sdk) for local dev (cli-bridge) or tests (mock) is a one-
|
|
20
|
-
* line factory call.
|
|
21
|
-
*
|
|
22
|
-
* Designed to coexist: existing `LlmClient` callers and existing
|
|
23
|
-
* `TCloud`-based judges keep working untouched. New analyst code uses
|
|
24
|
-
* `ChatClient`. When old call sites migrate, they pick up budgeting,
|
|
25
|
-
* cancellation, and unified telemetry for free.
|
|
26
|
-
*/
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Unified chat interface. Mirrors LlmCallRequest/Result so the OpenAI-
|
|
30
|
-
* compatible mental model stays. Two methods: a one-shot `chat()` and
|
|
31
|
-
* an `streamChat()` for future agentic loops (not yet exposed).
|
|
32
|
-
*/
|
|
33
|
-
interface ChatClient {
|
|
34
|
-
/** Display name of the bound transport — included in telemetry. */
|
|
35
|
-
readonly transport: ChatTransport;
|
|
36
|
-
/** Default model when caller omits — operators bind this per environment. */
|
|
37
|
-
readonly defaultModel?: string;
|
|
38
|
-
/** Total provider attempts this transport can make for one chat call. */
|
|
39
|
-
readonly maximumAttempts?: number;
|
|
40
|
-
/** Implementations must enforce `req.maxTokens` when it is present. */
|
|
41
|
-
chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>;
|
|
42
|
-
}
|
|
43
|
-
type ChatTransport = 'router' | 'sandbox-sdk' | 'cli-bridge' | 'direct-provider' | 'mock';
|
|
44
|
-
interface ChatRequest extends Omit<LlmCallRequest, 'model'> {
|
|
45
|
-
/** Optional — falls back to ChatClient.defaultModel. */
|
|
46
|
-
model?: string;
|
|
47
|
-
}
|
|
48
|
-
type ChatResponse = LlmCallResult;
|
|
49
|
-
interface ChatCallOpts {
|
|
50
|
-
/** Cancel the in-flight request. */
|
|
51
|
-
signal?: AbortSignal;
|
|
52
|
-
/** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */
|
|
53
|
-
maxCostUsd?: number;
|
|
54
|
-
/** Correlation tag carried into request headers when the transport allows. */
|
|
55
|
-
correlationId?: string;
|
|
56
|
-
/** Stable provider idempotency key for retries/redrives of one paid call. */
|
|
57
|
-
idempotencyKey?: string;
|
|
58
|
-
}
|
|
59
|
-
type CreateChatClientOpts = RouterTransportOpts | CliBridgeTransportOpts | DirectProviderTransportOpts | SandboxSdkTransportOpts | MockTransportOpts;
|
|
60
|
-
interface BaseTransportOpts {
|
|
61
|
-
defaultModel?: string;
|
|
62
|
-
/** Total provider attempts. Required for opaque transports used in capped runs. */
|
|
63
|
-
maximumAttempts?: number;
|
|
64
|
-
}
|
|
65
|
-
interface RouterTransportOpts extends BaseTransportOpts {
|
|
66
|
-
transport: 'router';
|
|
67
|
-
baseUrl?: string;
|
|
68
|
-
apiKey: string;
|
|
69
|
-
}
|
|
70
|
-
interface CliBridgeTransportOpts extends BaseTransportOpts {
|
|
71
|
-
transport: 'cli-bridge';
|
|
72
|
-
baseUrl?: string;
|
|
73
|
-
bearer?: string;
|
|
74
|
-
}
|
|
75
|
-
interface DirectProviderTransportOpts extends BaseTransportOpts {
|
|
76
|
-
transport: 'direct-provider';
|
|
77
|
-
baseUrl: string;
|
|
78
|
-
apiKey: string;
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Sandbox-SDK transport. Provided as a thin pass-through: the caller
|
|
82
|
-
* supplies a callable that mimics LlmClient.chat() against an already-
|
|
83
|
-
* configured Sandbox handle. We don't import the SDK here to keep
|
|
84
|
-
* agent-eval dep-free of @tangle-network/sandbox.
|
|
85
|
-
*/
|
|
86
|
-
interface SandboxSdkTransportOpts extends BaseTransportOpts {
|
|
87
|
-
transport: 'sandbox-sdk';
|
|
88
|
-
chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Mock transport for tests. The handler receives the request and returns
|
|
92
|
-
* whatever the test wants. No retries, no JSON-schema degrade.
|
|
93
|
-
*/
|
|
94
|
-
interface MockTransportOpts extends BaseTransportOpts {
|
|
95
|
-
transport: 'mock';
|
|
96
|
-
handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Build a ChatClient bound to a specific transport. The returned client
|
|
100
|
-
* is safe to share across analysts in a single registry run.
|
|
101
|
-
*/
|
|
102
|
-
declare function createChatClient(opts: CreateChatClientOpts): ChatClient;
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* Analyst contract — the missing orchestration layer over agent-eval's
|
|
106
|
-
* existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
|
|
107
|
-
* SemanticConceptJudge, JudgeFn, ...).
|
|
108
|
-
*
|
|
109
|
-
* Each existing primitive returns its own output shape. The Analyst
|
|
110
|
-
* contract is the single envelope every primitive lifts into, so a
|
|
111
|
-
* registry can run N analysts against a run and a single renderer can
|
|
112
|
-
* compose findings without knowing which analyzer produced them.
|
|
113
|
-
*
|
|
114
|
-
* The contract is intentionally domain-agnostic: nothing here knows
|
|
115
|
-
* about code, voice, RAG, or any particular agent stack. Analysts
|
|
116
|
-
* declare what INPUT KIND they need (a trace store, an artifact dir,
|
|
117
|
-
* a RunRecord, a JudgeInput, or `custom`), and the registry routes
|
|
118
|
-
* the matching input from `AnalystRunInputs`.
|
|
119
|
-
*/
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Unified envelope every analyst emits. Schema-versioned so renderers
|
|
123
|
-
* and time-series diffs survive future field additions.
|
|
124
|
-
*/
|
|
125
|
-
interface AnalystFinding {
|
|
126
|
-
schema_version: '1.0.0';
|
|
127
|
-
/**
|
|
128
|
-
* Stable hash over identity-defining fields (analyst_id + canonical
|
|
129
|
-
* claim + area + optional subject). Two findings from two runs that
|
|
130
|
-
* "are the same finding" share this id — that's what `diffFindings`
|
|
131
|
-
* uses to compute appeared/disappeared sets across runs.
|
|
132
|
-
*/
|
|
133
|
-
finding_id: string;
|
|
134
|
-
analyst_id: string;
|
|
135
|
-
produced_at: string;
|
|
136
|
-
severity: AnalystSeverity;
|
|
137
|
-
/**
|
|
138
|
-
* Coarse classification. Renderers group by this. Free-form so
|
|
139
|
-
* domain-specific analysts can introduce categories without a
|
|
140
|
-
* schema change ('agent-reasoning', 'verification', 'cost',
|
|
141
|
-
* 'tool-use', 'safety', 'latency', 'data-quality', ...).
|
|
142
|
-
*/
|
|
143
|
-
area: string;
|
|
144
|
-
claim: string;
|
|
145
|
-
rationale?: string;
|
|
146
|
-
evidence_refs: EvidenceRef[];
|
|
147
|
-
recommended_action?: string;
|
|
148
|
-
validation_plan?: string;
|
|
149
|
-
/** 0..1 — the analyst's own confidence. Not calibrated across analysts. */
|
|
150
|
-
confidence: number;
|
|
151
|
-
/**
|
|
152
|
-
* Optional subject the finding is about — leaf id, agent id, request
|
|
153
|
-
* id. Included in finding_id when present so per-subject findings
|
|
154
|
-
* diff cleanly across runs.
|
|
155
|
-
*/
|
|
156
|
-
subject?: string;
|
|
157
|
-
/** FIREWALL provenance (docs/learning-flywheel.md): true iff this finding was
|
|
158
|
-
* lifted from a JUDGE verdict (an acceptance score), not OBSERVED from the
|
|
159
|
-
* agent's behavior. A judge-derived finding must NEVER be admitted as a
|
|
160
|
-
* steering input — that is the held-out judge leaking into the loop. Set at
|
|
161
|
-
* the lift site (createJudgeAdapter); checked by `assertNoJudgeVerdict`.
|
|
162
|
-
* Provenance, not evidence presence, is the correct discriminator: an
|
|
163
|
-
* evidence-less trace-analyst observation legitimately steers, while a judge
|
|
164
|
-
* verdict that happens to cite an artifact must not. */
|
|
165
|
-
derived_from_judge?: boolean;
|
|
166
|
-
/** Analyst-private extras; renderers ignore unless they know the analyst. */
|
|
167
|
-
metadata?: Record<string, unknown>;
|
|
168
|
-
}
|
|
169
|
-
type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info';
|
|
170
|
-
interface EvidenceRef {
|
|
171
|
-
/**
|
|
172
|
-
* Where the evidence lives. `span` and `event` refer to OTLP trace
|
|
173
|
-
* elements; `artifact` to a file inside the run's artifact tree;
|
|
174
|
-
* `finding` to another AnalystFinding (cross-analyst chaining);
|
|
175
|
-
* `metric` to a named scalar reading the renderer knows how to read.
|
|
176
|
-
*/
|
|
177
|
-
kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
|
|
178
|
-
uri: string;
|
|
179
|
-
excerpt?: string;
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* The discriminator the registry uses to pass the right input.
|
|
183
|
-
* `custom` is the escape hatch — analysts that need something else
|
|
184
|
-
* (e.g. an embedding cache, a partner SDK handle) read it from
|
|
185
|
-
* `AnalystRunInputs.custom[<analyst id>]`.
|
|
186
|
-
*/
|
|
187
|
-
type AnalystInputKind = 'trace-store' | 'artifact-dir' | 'run-record' | 'judge-input' | 'custom';
|
|
188
|
-
interface AnalystCost {
|
|
189
|
-
/** `deterministic` analysts MUST NOT call the LLM. */
|
|
190
|
-
kind: 'deterministic' | 'llm';
|
|
191
|
-
/** Optional declared upper bound; the registry can enforce a budget. */
|
|
192
|
-
est_usd_per_run?: number;
|
|
193
|
-
/** Models the analyst expects to use (informational). */
|
|
194
|
-
models?: string[];
|
|
195
|
-
}
|
|
196
|
-
interface AnalystRequirements {
|
|
197
|
-
/** Min number of shots / samples the analyst needs to produce signal. */
|
|
198
|
-
min_shots?: number;
|
|
199
|
-
/** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */
|
|
200
|
-
capabilities?: string[];
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* What's passed to every analyst call. The registry resolves which
|
|
204
|
-
* field the analyst's `inputKind` selects and asserts it's present.
|
|
205
|
-
*/
|
|
206
|
-
interface AnalystRunInputs {
|
|
207
|
-
traceStore?: TraceAnalysisStore;
|
|
208
|
-
artifactDir?: string;
|
|
209
|
-
runRecord?: RunRecord;
|
|
210
|
-
judgeInput?: JudgeInput;
|
|
211
|
-
/** Keyed by analyst id; populated by callers that registered custom analysts. */
|
|
212
|
-
custom?: Record<string, unknown>;
|
|
213
|
-
}
|
|
214
|
-
interface AnalystContext {
|
|
215
|
-
runId: string;
|
|
216
|
-
/** Stable correlation id so logs from a single registry.run() share a tag. */
|
|
217
|
-
correlationId: string;
|
|
218
|
-
/** Wall-clock deadline (epoch ms). Analysts SHOULD honor for graceful cancel. */
|
|
219
|
-
deadlineMs?: number;
|
|
220
|
-
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
221
|
-
budgetUsd?: number;
|
|
222
|
-
/**
|
|
223
|
-
* Shared chat client. Analysts that call an LLM go through this so
|
|
224
|
-
* the operator picks transport (sandbox-sdk | router | cli-bridge |
|
|
225
|
-
* direct-provider | mock) at the registry boundary without touching
|
|
226
|
-
* analyst code.
|
|
227
|
-
*/
|
|
228
|
-
chat?: ChatClient;
|
|
229
|
-
/**
|
|
230
|
-
* Findings from a prior run the operator wants the analyst to see as
|
|
231
|
-
* retrieval context. Kinds that take advantage of cross-run memory
|
|
232
|
-
* (failure-mode "I saw this cluster last run", knowledge-gap "the wiki
|
|
233
|
-
* page I asked for is still missing") render these into the actor's
|
|
234
|
-
* working set. Filtering is the operator's job: pass the slice that
|
|
235
|
-
* matches the analyst's id, or pass everything and let the kind
|
|
236
|
-
* filter. Empty / absent means no cross-run context.
|
|
237
|
-
*/
|
|
238
|
-
priorFindings?: ReadonlyArray<AnalystFinding>;
|
|
239
|
-
/** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
|
|
240
|
-
tags?: Record<string, string>;
|
|
241
|
-
/** Logger callback — analysts SHOULD prefer this over console.* for testability. */
|
|
242
|
-
log?: (msg: string, fields?: Record<string, unknown>) => void;
|
|
243
|
-
/** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */
|
|
244
|
-
signal?: AbortSignal;
|
|
245
|
-
}
|
|
246
|
-
/**
|
|
247
|
-
* The minimal contract. Concrete analysts can refine `TInput` so
|
|
248
|
-
* implementations stay type-safe (e.g. a trace analyst's `TInput` is
|
|
249
|
-
* `TraceAnalysisStore`); the registry passes the right field from
|
|
250
|
-
* `AnalystRunInputs` based on `inputKind`.
|
|
251
|
-
*/
|
|
252
|
-
interface Analyst<TInput = unknown> {
|
|
253
|
-
/** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */
|
|
254
|
-
readonly id: string;
|
|
255
|
-
/** Human-readable. One sentence. */
|
|
256
|
-
readonly description: string;
|
|
257
|
-
readonly inputKind: AnalystInputKind;
|
|
258
|
-
readonly cost: AnalystCost;
|
|
259
|
-
readonly requires?: AnalystRequirements;
|
|
260
|
-
/** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */
|
|
261
|
-
readonly version: string;
|
|
262
|
-
analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
|
|
263
|
-
}
|
|
264
|
-
/**
|
|
265
|
-
* Compute the stable finding_id from the identity-defining fields.
|
|
266
|
-
* Default implementation hashes {analyst_id, area, subject, normalized claim}.
|
|
267
|
-
* Analysts that emit findings whose claim text varies per run (timestamps,
|
|
268
|
-
* counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,
|
|
269
|
-
* or (b) move the variable part into `rationale`/`metadata` and keep the
|
|
270
|
-
* `claim` static.
|
|
271
|
-
*/
|
|
272
|
-
declare function computeFindingId(input: {
|
|
273
|
-
analyst_id: string;
|
|
274
|
-
area: string;
|
|
275
|
-
subject?: string;
|
|
276
|
-
claim: string;
|
|
277
|
-
/** Override the claim for hashing — use when the displayed claim has run-specific bits. */
|
|
278
|
-
id_basis?: string;
|
|
279
|
-
}): string;
|
|
280
|
-
/**
|
|
281
|
-
* Convenience factory: produce a fully-formed AnalystFinding with the
|
|
282
|
-
* id computed automatically. Analyst code stays terse.
|
|
283
|
-
*/
|
|
284
|
-
declare function makeFinding(init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {
|
|
285
|
-
id_basis?: string;
|
|
286
|
-
produced_at?: string;
|
|
287
|
-
}): AnalystFinding;
|
|
288
|
-
interface AnalystRunSummary {
|
|
289
|
-
analyst_id: string;
|
|
290
|
-
status: 'ok' | 'skipped' | 'failed';
|
|
291
|
-
/** Why skipped — missing input, budget exceeded, capability unmet. */
|
|
292
|
-
reason?: string;
|
|
293
|
-
findings_count: number;
|
|
294
|
-
latency_ms: number;
|
|
295
|
-
cost_usd: number;
|
|
296
|
-
/** When `status='failed'`: the error class + message, never the full stack. */
|
|
297
|
-
error?: {
|
|
298
|
-
class: string;
|
|
299
|
-
message: string;
|
|
300
|
-
};
|
|
301
|
-
}
|
|
302
|
-
interface AnalystRunResult {
|
|
303
|
-
run_id: string;
|
|
304
|
-
correlation_id: string;
|
|
305
|
-
started_at: string;
|
|
306
|
-
ended_at: string;
|
|
307
|
-
findings: AnalystFinding[];
|
|
308
|
-
per_analyst: AnalystRunSummary[];
|
|
309
|
-
/** Total LLM cost in USD across all analysts in this registry.run(). */
|
|
310
|
-
total_cost_usd: number;
|
|
311
|
-
}
|
|
312
|
-
/**
|
|
313
|
-
* Events emitted by `AnalystRegistry.runStream(...)` in real time as
|
|
314
|
-
* the registry executes. UIs subscribe via `for await (const ev of
|
|
315
|
-
* registry.runStream(...))`; `registry.run(...)` is a thin collector
|
|
316
|
-
* over the same stream, so the two surfaces share their invariants.
|
|
317
|
-
*
|
|
318
|
-
* Per-finding events are intentionally omitted — analyzers are batch
|
|
319
|
-
* operations (an Ax actor returns the full `findings:json[]` at the
|
|
320
|
-
* end of the responder), so streaming inside one analyst would only
|
|
321
|
-
* emit partial JSON consumers can't render. The kind-completion event
|
|
322
|
-
* is the right granularity; subscribers wanting per-finding rendering
|
|
323
|
-
* iterate `event.findings` themselves.
|
|
324
|
-
*/
|
|
325
|
-
type AnalystRunEvent = {
|
|
326
|
-
type: 'run-started';
|
|
327
|
-
run_id: string;
|
|
328
|
-
correlation_id: string;
|
|
329
|
-
started_at: string;
|
|
330
|
-
/** The ordered list of analyst ids the registry will run. */
|
|
331
|
-
analyst_ids: ReadonlyArray<string>;
|
|
332
|
-
} | {
|
|
333
|
-
type: 'analyst-skipped';
|
|
334
|
-
summary: AnalystRunSummary;
|
|
335
|
-
} | {
|
|
336
|
-
type: 'analyst-started';
|
|
337
|
-
analyst_id: string;
|
|
338
|
-
started_at: string;
|
|
339
|
-
} | {
|
|
340
|
-
type: 'analyst-completed';
|
|
341
|
-
/** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */
|
|
342
|
-
summary: AnalystRunSummary;
|
|
343
|
-
findings: ReadonlyArray<AnalystFinding>;
|
|
344
|
-
} | {
|
|
345
|
-
type: 'run-completed';
|
|
346
|
-
result: AnalystRunResult;
|
|
347
|
-
};
|
|
348
|
-
|
|
349
|
-
type PolicyEditSchemaVersion = 'policy-edit/v1';
|
|
350
|
-
declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
|
|
351
|
-
type PolicyEditAxis = (typeof POLICY_EDIT_AXES)[number];
|
|
352
|
-
declare const POLICY_EDIT_TARGET_SURFACES: readonly ["prompt", "tool-contract", "runtime-config", "memory", "agent-profile", "code", "deployment"];
|
|
353
|
-
type PolicyEditTargetSurface = (typeof POLICY_EDIT_TARGET_SURFACES)[number];
|
|
354
|
-
type PolicyEditRisk = 'low' | 'medium' | 'high' | 'unknown';
|
|
355
|
-
type PolicyEditGainDirection = 'increase' | 'decrease';
|
|
356
|
-
type PolicyEditGainUnit = 'absolute' | 'relative' | 'percent' | 'score';
|
|
357
|
-
interface PolicyEditTarget {
|
|
358
|
-
surface: PolicyEditTargetSurface;
|
|
359
|
-
/** Stable path inside the target surface, for example `system-prompt:tools`
|
|
360
|
-
* or `budget.maxTurns`. */
|
|
361
|
-
path?: string;
|
|
362
|
-
/** Optional canonical deployment identity. Store the existing cell, not a
|
|
363
|
-
* local profile shape. */
|
|
364
|
-
agentProfileCell?: AgentProfileCell;
|
|
365
|
-
/** Human label when the path is not enough for a readable audit trail. */
|
|
366
|
-
label?: string;
|
|
367
|
-
}
|
|
368
|
-
type PolicyEditChange = {
|
|
369
|
-
kind: 'text';
|
|
370
|
-
mode: 'append' | 'prepend' | 'replace';
|
|
371
|
-
value: string;
|
|
372
|
-
/** Required when `mode === 'replace'`; exact match only. */
|
|
373
|
-
find?: string;
|
|
374
|
-
} | {
|
|
375
|
-
kind: 'json';
|
|
376
|
-
mode: 'set' | 'merge' | 'remove';
|
|
377
|
-
path: string;
|
|
378
|
-
value?: AgentProfileJson;
|
|
379
|
-
};
|
|
380
|
-
interface PolicyEditExpectedGain {
|
|
381
|
-
/** Metric this edit is expected to move, e.g. `holdout.composite`. */
|
|
382
|
-
metric: string;
|
|
383
|
-
direction: PolicyEditGainDirection;
|
|
384
|
-
/** Positive magnitude in the metric's native units. */
|
|
385
|
-
amount: number;
|
|
386
|
-
unit?: PolicyEditGainUnit;
|
|
387
|
-
rationale?: string;
|
|
388
|
-
}
|
|
389
|
-
interface PolicyEditSource {
|
|
390
|
-
findingIds: string[];
|
|
391
|
-
analystIds: string[];
|
|
392
|
-
evidenceRefs: EvidenceRef[];
|
|
393
|
-
/** Mirrors `AnalystFinding.derived_from_judge`; admission rejects it. */
|
|
394
|
-
derivedFromJudge?: boolean;
|
|
395
|
-
}
|
|
396
|
-
interface PolicyEdit {
|
|
397
|
-
schemaVersion: PolicyEditSchemaVersion;
|
|
398
|
-
editId: string;
|
|
399
|
-
axis: PolicyEditAxis;
|
|
400
|
-
target: PolicyEditTarget;
|
|
401
|
-
change: PolicyEditChange;
|
|
402
|
-
claim: string;
|
|
403
|
-
expectedGain: PolicyEditExpectedGain;
|
|
404
|
-
confidence: number;
|
|
405
|
-
risk: PolicyEditRisk;
|
|
406
|
-
source: PolicyEditSource;
|
|
407
|
-
rationale?: string;
|
|
408
|
-
validationPlan?: string;
|
|
409
|
-
metadata?: Record<string, unknown>;
|
|
410
|
-
}
|
|
411
|
-
declare const POLICY_EDIT_CANDIDATE_RECORD_SCHEMA: "tangle.policy-edit-candidate.v1";
|
|
412
|
-
/** JSON-safe attribution carried with a measured candidate and its scores. */
|
|
413
|
-
interface PolicyEditCandidateRecord {
|
|
414
|
-
schema: typeof POLICY_EDIT_CANDIDATE_RECORD_SCHEMA;
|
|
415
|
-
policyEdit: PolicyEdit;
|
|
416
|
-
}
|
|
417
|
-
type PolicyEditInit = Omit<PolicyEdit, 'schemaVersion' | 'editId'> & {
|
|
418
|
-
schemaVersion?: PolicyEditSchemaVersion;
|
|
419
|
-
editId?: string;
|
|
420
|
-
};
|
|
421
|
-
declare class PolicyEditValidationError extends ValidationError {
|
|
422
|
-
readonly path: string;
|
|
423
|
-
constructor(message: string, path?: string);
|
|
424
|
-
}
|
|
425
|
-
interface FindingToPolicyEditOptions {
|
|
426
|
-
expectedGain?: PolicyEditExpectedGain | ((finding: AnalystFinding) => PolicyEditExpectedGain | null | undefined);
|
|
427
|
-
risk?: PolicyEditRisk | ((finding: AnalystFinding) => PolicyEditRisk);
|
|
428
|
-
defaultAxis?: PolicyEditAxis;
|
|
429
|
-
defaultTargetSurface?: PolicyEditTargetSurface;
|
|
430
|
-
}
|
|
431
|
-
interface PolicyEditAdmissionOptions {
|
|
432
|
-
minScore?: number;
|
|
433
|
-
minExpectedGain?: number;
|
|
434
|
-
allowHighRisk?: boolean;
|
|
435
|
-
requireEvidence?: boolean;
|
|
436
|
-
}
|
|
437
|
-
interface PolicyEditAdmission {
|
|
438
|
-
edit: PolicyEdit;
|
|
439
|
-
decision: 'admit' | 'reject';
|
|
440
|
-
score: number;
|
|
441
|
-
reasons: string[];
|
|
442
|
-
}
|
|
443
|
-
declare function makePolicyEdit(init: PolicyEditInit): PolicyEdit;
|
|
444
|
-
declare function computePolicyEditId(edit: Omit<PolicyEdit, 'editId'> | PolicyEdit): string;
|
|
445
|
-
declare function validatePolicyEdit(input: unknown): PolicyEdit;
|
|
446
|
-
declare function makePolicyEditCandidateRecord(edit: PolicyEdit): PolicyEditCandidateRecord;
|
|
447
|
-
declare function validatePolicyEditCandidateRecord(input: unknown): PolicyEditCandidateRecord;
|
|
448
|
-
declare function isPolicyEdit(input: unknown): input is PolicyEdit;
|
|
449
|
-
declare function policyEditsFromFindings(findings: ReadonlyArray<AnalystFinding>, opts?: FindingToPolicyEditOptions): PolicyEdit[];
|
|
450
|
-
declare function policyEditFromFinding(finding: AnalystFinding, opts?: FindingToPolicyEditOptions): PolicyEdit | null;
|
|
451
|
-
declare function scorePolicyEditReadiness(edit: PolicyEdit, opts?: PolicyEditAdmissionOptions): number;
|
|
452
|
-
declare function admitPolicyEdit(edit: PolicyEdit, opts?: PolicyEditAdmissionOptions): PolicyEditAdmission;
|
|
453
|
-
declare function applyPolicyEditToSurface(surface: unknown, edit: PolicyEdit): unknown;
|
|
454
|
-
|
|
455
|
-
export { policyEditsFromFindings as $, type Analyst as A, type PolicyEditGainUnit as B, type ChatClient as C, type DirectProviderTransportOpts as D, type EvidenceRef as E, type FindingToPolicyEditOptions as F, type PolicyEditInit as G, type PolicyEditRisk as H, type PolicyEditSchemaVersion as I, type PolicyEditSource as J, type PolicyEditTarget as K, type PolicyEditTargetSurface as L, type MockTransportOpts as M, PolicyEditValidationError as N, admitPolicyEdit as O, type PolicyEditCandidateRecord as P, applyPolicyEditToSurface as Q, type RouterTransportOpts as R, type SandboxSdkTransportOpts as S, computeFindingId as T, computePolicyEditId as U, createChatClient as V, isPolicyEdit as W, makeFinding as X, makePolicyEdit as Y, makePolicyEditCandidateRecord as Z, policyEditFromFinding as _, type AnalystContext as a, scorePolicyEditReadiness as a0, validatePolicyEdit as a1, validatePolicyEditCandidateRecord as a2, type AnalystRunSummary as b, type AnalystFinding as c, type AnalystRunResult as d, type AnalystRunInputs as e, type AnalystRunEvent as f, type AnalystCost as g, type AnalystSeverity as h, type AnalystInputKind as i, type AnalystRequirements as j, type ChatCallOpts as k, type ChatRequest as l, type ChatResponse as m, type ChatTransport as n, type CliBridgeTransportOpts as o, type CreateChatClientOpts as p, POLICY_EDIT_AXES as q, POLICY_EDIT_CANDIDATE_RECORD_SCHEMA as r, POLICY_EDIT_TARGET_SURFACES as s, type PolicyEdit as t, type PolicyEditAdmission as u, type PolicyEditAdmissionOptions as v, type PolicyEditAxis as w, type PolicyEditChange as x, type PolicyEditExpectedGain as y, type PolicyEditGainDirection as z };
|