@tangle-network/agent-eval 0.93.0 → 0.94.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/otel.d.ts +4 -4
- package/dist/analyst/index.d.ts +10 -10
- package/dist/analyst/index.js +6 -6
- package/dist/{analyze-runs-DwCEkpO_.d.ts → analyze-runs-B6Ljo_dI.d.ts} +2 -2
- package/dist/{baseline-DE36-Np7.d.ts → baseline-Bbid3WoO.d.ts} +1 -1
- package/dist/belief-state/index.d.ts +2 -2
- package/dist/belief-state/index.js +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/{calibration-Cpr3WaX3.d.ts → calibration-BPmzuVPk.d.ts} +1 -1
- package/dist/campaign/index.d.ts +13 -13
- package/dist/campaign/index.js +8 -8
- package/dist/{chunk-TWS7AZEY.js → chunk-2K6UUZ7P.js} +2 -2
- package/dist/{chunk-Y47J2LJ3.js → chunk-2KNZHH3P.js} +5 -5
- package/dist/{chunk-LMZQ2Z4U.js → chunk-56GC6VYO.js} +126 -1
- package/dist/chunk-56GC6VYO.js.map +1 -0
- package/dist/{chunk-QG2OVF2D.js → chunk-CTBHKLEU.js} +99 -30
- package/dist/chunk-CTBHKLEU.js.map +1 -0
- package/dist/{chunk-CVVHBFGN.js → chunk-CWNP4DV4.js} +53 -5
- package/dist/chunk-CWNP4DV4.js.map +1 -0
- package/dist/{chunk-QS3RBQPI.js → chunk-D5KBVULY.js} +2 -2
- package/dist/{chunk-ZFIBGEOL.js → chunk-EGPMSBEZ.js} +4 -4
- package/dist/{chunk-GSH6QNNS.js → chunk-KW53MSA5.js} +101 -78
- package/dist/chunk-KW53MSA5.js.map +1 -0
- package/dist/{chunk-UMMZHCPB.js → chunk-MIFZUPEK.js} +7 -5
- package/dist/chunk-MIFZUPEK.js.map +1 -0
- package/dist/{chunk-XY4DDNEG.js → chunk-MPQWFX6Y.js} +2 -2
- package/dist/{chunk-L3JOU6XM.js → chunk-NACM5RS7.js} +3 -3
- package/dist/{chunk-D3V5B42D.js → chunk-Q5LIB7BC.js} +7 -4
- package/dist/{chunk-D3V5B42D.js.map → chunk-Q5LIB7BC.js.map} +1 -1
- package/dist/{chunk-QAY5UIJO.js → chunk-QMUEXQJS.js} +99 -57
- package/dist/chunk-QMUEXQJS.js.map +1 -0
- package/dist/{chunk-4FBZZIYD.js → chunk-QTMYW64F.js} +2 -2
- package/dist/{chunk-6SOJM3VR.js → chunk-SD2YFWQQ.js} +4 -4
- package/dist/{chunk-UHMJT4T7.js → chunk-TBDR6PAI.js} +2 -2
- package/dist/{chunk-47X6LRCE.js → chunk-UFSG7ACU.js} +10 -7
- package/dist/chunk-UFSG7ACU.js.map +1 -0
- package/dist/{chunk-CY6U5S3X.js → chunk-URWVKRCS.js} +2 -2
- package/dist/{chunk-T375SUOZ.js → chunk-ZOPLCJSY.js} +86 -31
- package/dist/chunk-ZOPLCJSY.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +17 -17
- package/dist/contract/index.js +11 -11
- package/dist/{control-_Qb7skHX.d.ts → control-D6qwHXIR.d.ts} +4 -4
- package/dist/{control-runtime-DuFBYg7A.d.ts → control-runtime-Acf9CGhw.d.ts} +2 -2
- package/dist/control.d.ts +5 -5
- package/dist/{corpus-BoR-041R.d.ts → corpus-B8A4BDR3.d.ts} +1 -1
- package/dist/{counterfactual-Dwibr5IW.d.ts → counterfactual-DlOz8PBx.d.ts} +3 -3
- package/dist/{default-registry-zoGHUQEH.d.ts → default-registry-6dhErQbs.d.ts} +2 -2
- package/dist/diagnose.d.ts +9 -9
- package/dist/diagnose.js +1 -1
- package/dist/{emitter-DEZwY14K.d.ts → emitter-C2rqGH_l.d.ts} +1 -1
- package/dist/{failure-cluster-CL7IVgkJ.d.ts → failure-cluster-DH9Flgcf.d.ts} +1 -1
- package/dist/{feedback-trajectory-D9OVLrg9.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +1 -1
- package/dist/governance/index.d.ts +2 -2
- package/dist/{harness-optimizer-EnEnQPsr.d.ts → harness-optimizer-mOl9XX_O.d.ts} +1 -1
- package/dist/hosted/index.d.ts +4 -4
- package/dist/index.d.ts +68 -46
- package/dist/index.js +261 -87
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-BBwvOh6x.d.ts → insight-report-DWl3z9tl.d.ts} +1 -1
- package/dist/{integrity-VJ9A7aST.d.ts → integrity-D2t12mMw.d.ts} +1 -1
- package/dist/{kind-factory-5b7xXXOr.d.ts → kind-factory-0BhLSI27.d.ts} +1 -1
- package/dist/knowledge/index.d.ts +3 -3
- package/dist/{llm-client-BeEcAokY.d.ts → llm-client-Bj7g0rqu.d.ts} +31 -0
- package/dist/meta-eval/index.d.ts +4 -4
- package/dist/meta-eval/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +6 -6
- package/dist/pipelines/index.js +40 -15
- package/dist/pipelines/index.js.map +1 -1
- package/dist/prm/index.d.ts +11 -7
- package/dist/prm/index.js +55 -12
- package/dist/prm/index.js.map +1 -1
- package/dist/{provenance-LnqRT0sS.d.ts → provenance-P-bCL2Fo.d.ts} +3 -3
- package/dist/{query-CqTxMwDw.d.ts → query-B7GGjRox.d.ts} +2 -1
- package/dist/{red-team-BXHil6c8.d.ts → red-team-BWdoyleI.d.ts} +1 -1
- package/dist/{release-report-euXIV_Sk.d.ts → release-report-BEbWmVYj.d.ts} +1 -1
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +3 -3
- package/dist/{researcher-DE6Gpnb4.d.ts → researcher-B0C2_fVO.d.ts} +5 -5
- package/dist/rl.d.ts +10 -10
- package/dist/rl.js +4 -4
- package/dist/{rubric-BOfxn4ja.d.ts → rubric-Cc6UHvUb.d.ts} +2 -2
- package/dist/{run-campaign-RDGAM5KJ.js → run-campaign-7WNXMDSN.js} +3 -3
- package/dist/{run-critic-BAIjX99r.d.ts → run-critic-CmMf05uV.d.ts} +1 -1
- package/dist/{run-improvement-loop-5z_l5zDz.d.ts → run-improvement-loop-DBahB8Ax.d.ts} +1 -1
- package/dist/{semantic-concept-judge-Dn8Z6KEG.d.ts → semantic-concept-judge-B9MgmBnM.d.ts} +3 -3
- package/dist/{statistics-C7PozGrZ.d.ts → statistics-CCJpTGOS.d.ts} +117 -2
- package/dist/{store-CKUAgsJz.d.ts → store-BcFXE6LG.d.ts} +16 -1
- package/dist/{summary-report-DGmUucwQ.d.ts → summary-report-BDOFevaT.d.ts} +1 -1
- package/dist/{test-graded-scenario-BdVaPyHT.d.ts → test-graded-scenario-DeODGLra.d.ts} +21 -2
- package/dist/traces.d.ts +19 -6
- package/dist/traces.js +4 -4
- package/dist/{trajectory-GEdXJCL5.d.ts → trajectory-2TkpSEVh.d.ts} +1 -1
- package/dist/{types-Croy5h7V.d.ts → types-C7DGg5ex.d.ts} +2 -0
- package/dist/{types-2VVIL04s.d.ts → types-Ce17tDlG.d.ts} +2 -2
- package/dist/wire/index.d.ts +4 -4
- package/dist/wire/index.js +2 -2
- package/dist/workflow/index.d.ts +13 -13
- package/dist/workflow/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-47X6LRCE.js.map +0 -1
- package/dist/chunk-CVVHBFGN.js.map +0 -1
- package/dist/chunk-GSH6QNNS.js.map +0 -1
- package/dist/chunk-LMZQ2Z4U.js.map +0 -1
- package/dist/chunk-QAY5UIJO.js.map +0 -1
- package/dist/chunk-QG2OVF2D.js.map +0 -1
- package/dist/chunk-T375SUOZ.js.map +0 -1
- package/dist/chunk-UMMZHCPB.js.map +0 -1
- /package/dist/{chunk-TWS7AZEY.js.map → chunk-2K6UUZ7P.js.map} +0 -0
- /package/dist/{chunk-Y47J2LJ3.js.map → chunk-2KNZHH3P.js.map} +0 -0
- /package/dist/{chunk-QS3RBQPI.js.map → chunk-D5KBVULY.js.map} +0 -0
- /package/dist/{chunk-ZFIBGEOL.js.map → chunk-EGPMSBEZ.js.map} +0 -0
- /package/dist/{chunk-XY4DDNEG.js.map → chunk-MPQWFX6Y.js.map} +0 -0
- /package/dist/{chunk-L3JOU6XM.js.map → chunk-NACM5RS7.js.map} +0 -0
- /package/dist/{chunk-4FBZZIYD.js.map → chunk-QTMYW64F.js.map} +0 -0
- /package/dist/{chunk-6SOJM3VR.js.map → chunk-SD2YFWQQ.js.map} +0 -0
- /package/dist/{chunk-UHMJT4T7.js.map → chunk-TBDR6PAI.js.map} +0 -0
- /package/dist/{chunk-CY6U5S3X.js.map → chunk-URWVKRCS.js.map} +0 -0
- /package/dist/{run-campaign-RDGAM5KJ.js.map → run-campaign-7WNXMDSN.js.map} +0 -0
package/dist/adapters/otel.d.ts
CHANGED
|
@@ -3,10 +3,10 @@ import '../types-BU-7W85F.js';
|
|
|
3
3
|
import '../run-record-e7vj1uZQ.js';
|
|
4
4
|
import '../errors-CzMUYo7b.js';
|
|
5
5
|
import '../schema-m0gsnbt3.js';
|
|
6
|
-
import '../insight-report-
|
|
7
|
-
import '../summary-report-
|
|
8
|
-
import '../failure-cluster-
|
|
9
|
-
import '../store-
|
|
6
|
+
import '../insight-report-DWl3z9tl.js';
|
|
7
|
+
import '../summary-report-BDOFevaT.js';
|
|
8
|
+
import '../failure-cluster-DH9Flgcf.js';
|
|
9
|
+
import '../store-BcFXE6LG.js';
|
|
10
10
|
import '../judge-calibration-0p2QcWNE.js';
|
|
11
11
|
|
|
12
12
|
/**
|
package/dist/analyst/index.d.ts
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
2
|
import { M as MultiLayerVerifier, V as VerifyOptions, S as Severity } from '../multi-layer-verifier-DUZXrPDA.js';
|
|
3
|
-
import { c as RunCritic, a as RunTrace } from '../run-critic-
|
|
4
|
-
import { S as SemanticConceptJudgeOptions, a as SemanticConceptJudgeInput, B as BehavioralMetrics } from '../semantic-concept-judge-
|
|
5
|
-
export { C as CreateAnalystAiConfig, D as DEFAULT_TRACE_ANALYST_KINDS, b as DiffPolicy, F as FAILURE_MODE_KIND_SPEC, c as FINDING_SUBJECT_GRAMMAR_PROMPT, d as FINDING_SUBJECT_KINDS, e as FindingSubject, f as FindingSubjectKind, g as FindingSubjectStringSchema, h as FindingsDiff, i as FindingsStore, I as IMPROVEMENT_KIND_SPEC, K as KIND_EXPECTED_SUBJECTS, j as KNOWLEDGE_GAP_KIND_SPEC, k as KNOWLEDGE_POISONING_KIND_SPEC, P as PersistedFinding, l as SKILL_USAGE_ANALYST, m as SkillUsageAnalyst, n as SkillUsageRecord, o as SkillUsageReport, p as SkillUsageScanConfig, q as buildSkillUsageReport, r as createAnalystAi, s as defaultIsMaterial, t as diffFindings, u as emitSkillUsageFindings, v as parseFindingSubject, w as renderFindingSubject } from '../semantic-concept-judge-
|
|
3
|
+
import { c as RunCritic, a as RunTrace } from '../run-critic-CmMf05uV.js';
|
|
4
|
+
import { S as SemanticConceptJudgeOptions, a as SemanticConceptJudgeInput, B as BehavioralMetrics } from '../semantic-concept-judge-B9MgmBnM.js';
|
|
5
|
+
export { C as CreateAnalystAiConfig, D as DEFAULT_TRACE_ANALYST_KINDS, b as DiffPolicy, F as FAILURE_MODE_KIND_SPEC, c as FINDING_SUBJECT_GRAMMAR_PROMPT, d as FINDING_SUBJECT_KINDS, e as FindingSubject, f as FindingSubjectKind, g as FindingSubjectStringSchema, h as FindingsDiff, i as FindingsStore, I as IMPROVEMENT_KIND_SPEC, K as KIND_EXPECTED_SUBJECTS, j as KNOWLEDGE_GAP_KIND_SPEC, k as KNOWLEDGE_POISONING_KIND_SPEC, P as PersistedFinding, l as SKILL_USAGE_ANALYST, m as SkillUsageAnalyst, n as SkillUsageRecord, o as SkillUsageReport, p as SkillUsageScanConfig, q as buildSkillUsageReport, r as createAnalystAi, s as defaultIsMaterial, t as diffFindings, u as emitSkillUsageFindings, v as parseFindingSubject, w as renderFindingSubject } from '../semantic-concept-judge-B9MgmBnM.js';
|
|
6
6
|
import { A as AnalyzeTracesOptions } from '../analyst-C8HHvfJp.js';
|
|
7
7
|
import { T as TraceAnalysisStore } from '../store-C1YxJDEK.js';
|
|
8
|
-
import { b as JudgeFn, a as JudgeInput } from '../types-
|
|
9
|
-
import { A as Analyst, h as AnalystSeverity, c as AnalystFinding } from '../types-
|
|
10
|
-
export { a as AnalystContext, g as AnalystCost, i as AnalystInputKind, j as AnalystRequirements, f as AnalystRunEvent, e as AnalystRunInputs, d as AnalystRunResult, b as AnalystRunSummary, k as ChatCallOpts, C as ChatClient, l as ChatRequest, m as ChatResponse, n as ChatTransport, o as CliBridgeTransportOpts, p as CreateChatClientOpts, D as DirectProviderTransportOpts, E as EvidenceRef, M as MockTransportOpts, R as RouterTransportOpts, S as SandboxSdkTransportOpts, q as computeFindingId, r as createChatClient, s as makeFinding } from '../types-
|
|
8
|
+
import { b as JudgeFn, a as JudgeInput } from '../types-C7DGg5ex.js';
|
|
9
|
+
import { A as Analyst, h as AnalystSeverity, c as AnalystFinding } from '../types-Ce17tDlG.js';
|
|
10
|
+
export { a as AnalystContext, g as AnalystCost, i as AnalystInputKind, j as AnalystRequirements, f as AnalystRunEvent, e as AnalystRunInputs, d as AnalystRunResult, b as AnalystRunSummary, k as ChatCallOpts, C as ChatClient, l as ChatRequest, m as ChatResponse, n as ChatTransport, o as CliBridgeTransportOpts, p as CreateChatClientOpts, D as DirectProviderTransportOpts, E as EvidenceRef, M as MockTransportOpts, R as RouterTransportOpts, S as SandboxSdkTransportOpts, q as computeFindingId, r as createChatClient, s as makeFinding } from '../types-Ce17tDlG.js';
|
|
11
11
|
import { TCloud } from '@tangle-network/tcloud';
|
|
12
|
-
export { a as AnalystHooks, A as AnalystRegistry, c as AnalystRegistryOptions, B as BudgetPolicy, D as DefaultAnalystRegistryOptions, R as RegistryRunOpts, b as buildDefaultAnalystRegistry } from '../default-registry-
|
|
13
|
-
export { A as ANALYST_SEVERITIES, C as CreateTraceAnalystKindOpts, R as RAW_FINDING_SCHEMA_PROMPT, a as RawAnalystFinding, b as RawAnalystFindingSchema, c as TraceAnalystGolden, T as TraceAnalystKindSpec, d as createTraceAnalystKind, p as parseRawFinding, r as renderPriorFindings } from '../kind-factory-
|
|
14
|
-
import { L as LlmClientOptions } from '../llm-client-
|
|
12
|
+
export { a as AnalystHooks, A as AnalystRegistry, c as AnalystRegistryOptions, B as BudgetPolicy, D as DefaultAnalystRegistryOptions, R as RegistryRunOpts, b as buildDefaultAnalystRegistry } from '../default-registry-6dhErQbs.js';
|
|
13
|
+
export { A as ANALYST_SEVERITIES, C as CreateTraceAnalystKindOpts, R as RAW_FINDING_SCHEMA_PROMPT, a as RawAnalystFinding, b as RawAnalystFindingSchema, c as TraceAnalystGolden, T as TraceAnalystKindSpec, d as createTraceAnalystKind, p as parseRawFinding, r as renderPriorFindings } from '../kind-factory-0BhLSI27.js';
|
|
14
|
+
import { L as LlmClientOptions } from '../llm-client-Bj7g0rqu.js';
|
|
15
15
|
import '../verdict-C9MlYujm.js';
|
|
16
16
|
import '../schema-m0gsnbt3.js';
|
|
17
|
-
import '../store-
|
|
17
|
+
import '../store-BcFXE6LG.js';
|
|
18
18
|
import 'zod';
|
|
19
19
|
import '../run-record-e7vj1uZQ.js';
|
|
20
20
|
import '../errors-CzMUYo7b.js';
|
package/dist/analyst/index.js
CHANGED
|
@@ -11,12 +11,12 @@ import {
|
|
|
11
11
|
diffFindings,
|
|
12
12
|
emitSkillUsageFindings,
|
|
13
13
|
runSemanticConceptJudge
|
|
14
|
-
} from "../chunk-
|
|
14
|
+
} from "../chunk-MPQWFX6Y.js";
|
|
15
15
|
import {
|
|
16
16
|
behavioralAnalyst,
|
|
17
17
|
buildDefaultAnalystRegistry,
|
|
18
18
|
deriveEfficiencyFindings
|
|
19
|
-
} from "../chunk-
|
|
19
|
+
} from "../chunk-MIFZUPEK.js";
|
|
20
20
|
import {
|
|
21
21
|
ANALYST_SEVERITIES,
|
|
22
22
|
AnalystRegistry,
|
|
@@ -41,16 +41,16 @@ import {
|
|
|
41
41
|
renderPriorFindings,
|
|
42
42
|
stripCodeFences,
|
|
43
43
|
structureFindings
|
|
44
|
-
} from "../chunk-
|
|
45
|
-
import "../chunk-
|
|
44
|
+
} from "../chunk-Q5LIB7BC.js";
|
|
45
|
+
import "../chunk-CWNP4DV4.js";
|
|
46
46
|
import {
|
|
47
47
|
computeFindingId,
|
|
48
48
|
makeFinding
|
|
49
49
|
} from "../chunk-45EEMHTC.js";
|
|
50
50
|
import {
|
|
51
51
|
analyzeTraces
|
|
52
|
-
} from "../chunk-
|
|
53
|
-
import "../chunk-
|
|
52
|
+
} from "../chunk-TBDR6PAI.js";
|
|
53
|
+
import "../chunk-QMUEXQJS.js";
|
|
54
54
|
import "../chunk-PC4UYEBM.js";
|
|
55
55
|
import "../chunk-3BFEG2F6.js";
|
|
56
56
|
import "../chunk-PZ5AY32C.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { A as AnalystRegistry } from './default-registry-
|
|
1
|
+
import { A as AnalystRegistry } from './default-registry-6dhErQbs.js';
|
|
2
2
|
import { a as DatasetScenario } from './dataset-BbGkaN2I.js';
|
|
3
3
|
import { R as RunRecord } from './run-record-e7vj1uZQ.js';
|
|
4
|
-
import { I as InsightReport } from './insight-report-
|
|
4
|
+
import { I as InsightReport } from './insight-report-DWl3z9tl.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* # `analyzeRuns()` — turn a set of agent runs into an actionable decision packet.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { c as CalibrationReport } from '../calibration-
|
|
1
|
+
import { c as CalibrationReport } from '../calibration-BPmzuVPk.js';
|
|
2
2
|
import { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory } from '../off-policy-DiwuKKg7.js';
|
|
3
3
|
import { d as CodeAgentSessionSource, a as CodeAgentSessionIntakeOptions, c as CodeAgentSessionMetrics, C as CodeAgentSessionDiagnostic } from '../code-agent-session-BO8nCnv3.js';
|
|
4
4
|
import { R as RunRecord, a as RunSplitTag } from '../run-record-e7vj1uZQ.js';
|
|
5
|
-
import { T as TraceStore } from '../store-
|
|
5
|
+
import { T as TraceStore } from '../store-BcFXE6LG.js';
|
|
6
6
|
import { R as RuntimeTrajectoryRecord, P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection } from '../runtime-trajectory-BDgfGZSr.js';
|
|
7
7
|
import '../schema-m0gsnbt3.js';
|
|
8
8
|
import '../outcome-store-rnXLEqSn.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult, T as TestGradedScenario, b as TestGradedRunResult } from '../test-graded-scenario-
|
|
2
|
-
import { T as TraceEmitter } from '../emitter-
|
|
1
|
+
import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult, T as TestGradedScenario, b as TestGradedRunResult } from '../test-graded-scenario-DeODGLra.js';
|
|
2
|
+
import { T as TraceEmitter } from '../emitter-C2rqGH_l.js';
|
|
3
3
|
import { R as Run } from '../schema-m0gsnbt3.js';
|
|
4
|
-
import { T as TraceStore } from '../store-
|
|
4
|
+
import { T as TraceStore } from '../store-BcFXE6LG.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* BuilderSession — ties a builder-of-builders workflow together.
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SandboxHarness,
|
|
3
3
|
runTestGradedScenario
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-ZOPLCJSY.js";
|
|
5
5
|
import {
|
|
6
6
|
judgeSpans
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-UFSG7ACU.js";
|
|
8
8
|
import "../chunk-5BKGXME7.js";
|
|
9
9
|
import {
|
|
10
10
|
TraceEmitter
|
package/dist/campaign/index.d.ts
CHANGED
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
import { A as AnalyzeTracesOptions, a as AnalyzeTracesInput, b as AnalyzeTracesResult } from '../analyst-C8HHvfJp.js';
|
|
2
2
|
import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, q as ProposeContext, m as GenerationRecord, G as Gate, J as JudgeScore, L as LabeledScenarioStore, r as LabeledScenarioWrite, s as LabeledScenarioSampleArgs, t as LabeledScenarioRecord, u as LabelTrust, v as LabeledScenarioSource, g as CampaignResult, i as CodeSurface } from '../types-BU-7W85F.js';
|
|
3
3
|
export { C as CampaignAggregates, d as CampaignArtifactWriter, e as CampaignCellResult, f as CampaignCostMeter, w as CampaignTokenUsage, h as CampaignTraceWriter, D as DispatchFn, j as GateContext, c as GateDecision, k as GateResult, l as GenerationCandidate, x as JudgeAggregate, n as JudgeDimension, o as Mutator, O as OptimizerConfig, P as ParetoParent, y as ProposedCandidate, R as RedactionStatus, z as ScenarioAggregate, p as SessionScript, T as TraceSpan, A as isProposedCandidate, B as labelTrustRank } from '../types-BU-7W85F.js';
|
|
4
|
-
import { b as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../run-improvement-loop-
|
|
5
|
-
export { e as GepaDriverConstraints, G as GepaDriverOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, a as RunImprovementLoopResult, R as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../run-improvement-loop-
|
|
6
|
-
export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryDriver, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-
|
|
7
|
-
import { L as LlmClientOptions } from '../llm-client-
|
|
8
|
-
import { T as TraceAnalystKindSpec } from '../kind-factory-
|
|
9
|
-
import { c as AnalystFinding } from '../types-
|
|
4
|
+
import { b as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../run-improvement-loop-DBahB8Ax.js';
|
|
5
|
+
export { e as GepaDriverConstraints, G as GepaDriverOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, a as RunImprovementLoopResult, R as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../run-improvement-loop-DBahB8Ax.js';
|
|
6
|
+
export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryDriver, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-P-bCL2Fo.js';
|
|
7
|
+
import { L as LlmClientOptions } from '../llm-client-Bj7g0rqu.js';
|
|
8
|
+
import { T as TraceAnalystKindSpec } from '../kind-factory-0BhLSI27.js';
|
|
9
|
+
import { c as AnalystFinding } from '../types-Ce17tDlG.js';
|
|
10
10
|
import { S as SignedManifest, A as AgentProfile, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-mAnCugl9.js';
|
|
11
|
-
import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-
|
|
11
|
+
import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-CCJpTGOS.js';
|
|
12
12
|
import { A as AgentEvalError } from '../errors-CzMUYo7b.js';
|
|
13
13
|
import { a as RunSplitTag, R as RunRecord } from '../run-record-e7vj1uZQ.js';
|
|
14
14
|
import '@ax-llm/ax';
|
|
15
15
|
import '../store-C1YxJDEK.js';
|
|
16
|
-
import '../red-team-
|
|
16
|
+
import '../red-team-BWdoyleI.js';
|
|
17
17
|
import '../dataset-BbGkaN2I.js';
|
|
18
|
-
import '../store-
|
|
18
|
+
import '../store-BcFXE6LG.js';
|
|
19
19
|
import '../schema-m0gsnbt3.js';
|
|
20
20
|
import '../pareto-E-pembql.js';
|
|
21
21
|
import '../hosted/index.js';
|
|
22
|
-
import '../insight-report-
|
|
23
|
-
import '../summary-report-
|
|
24
|
-
import '../failure-cluster-
|
|
22
|
+
import '../insight-report-DWl3z9tl.js';
|
|
23
|
+
import '../summary-report-BDOFevaT.js';
|
|
24
|
+
import '../failure-cluster-DH9Flgcf.js';
|
|
25
25
|
import '../judge-calibration-0p2QcWNE.js';
|
|
26
26
|
import '../raw-provider-sink-C46HDghv.js';
|
|
27
27
|
import 'zod';
|
|
28
|
-
import '../types-
|
|
28
|
+
import '../types-C7DGg5ex.js';
|
|
29
29
|
import '@tangle-network/tcloud';
|
|
30
30
|
import '../verdict-C9MlYujm.js';
|
|
31
31
|
|
package/dist/campaign/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
paretoPolicy,
|
|
11
11
|
paretoSignificanceGate,
|
|
12
12
|
runEval
|
|
13
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-SD2YFWQQ.js";
|
|
14
14
|
import {
|
|
15
15
|
agentProfileHash,
|
|
16
16
|
extractProducedState,
|
|
@@ -37,14 +37,14 @@ import {
|
|
|
37
37
|
runOptimization,
|
|
38
38
|
surfaceContentHash,
|
|
39
39
|
surfaceHash
|
|
40
|
-
} from "../chunk-
|
|
40
|
+
} from "../chunk-EGPMSBEZ.js";
|
|
41
41
|
import {
|
|
42
42
|
assertRealBackend,
|
|
43
43
|
fsCampaignStorage,
|
|
44
44
|
inMemoryCampaignStorage,
|
|
45
45
|
runCampaign,
|
|
46
46
|
summarizeBackendIntegrity
|
|
47
|
-
} from "../chunk-
|
|
47
|
+
} from "../chunk-2K6UUZ7P.js";
|
|
48
48
|
import {
|
|
49
49
|
estimateCost,
|
|
50
50
|
isModelPriced
|
|
@@ -53,24 +53,24 @@ import {
|
|
|
53
53
|
AnalystRegistry,
|
|
54
54
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
55
55
|
createTraceAnalystKind
|
|
56
|
-
} from "../chunk-
|
|
56
|
+
} from "../chunk-Q5LIB7BC.js";
|
|
57
57
|
import "../chunk-5HRORJQY.js";
|
|
58
58
|
import {
|
|
59
59
|
callLlm
|
|
60
|
-
} from "../chunk-
|
|
60
|
+
} from "../chunk-CWNP4DV4.js";
|
|
61
61
|
import "../chunk-45EEMHTC.js";
|
|
62
62
|
import {
|
|
63
63
|
eProcess,
|
|
64
64
|
mulberry32,
|
|
65
65
|
pairedBootstrap
|
|
66
|
-
} from "../chunk-
|
|
66
|
+
} from "../chunk-56GC6VYO.js";
|
|
67
67
|
import "../chunk-GGE4NNQT.js";
|
|
68
68
|
import {
|
|
69
69
|
analyzeTraces
|
|
70
|
-
} from "../chunk-
|
|
70
|
+
} from "../chunk-TBDR6PAI.js";
|
|
71
71
|
import {
|
|
72
72
|
OtlpFileTraceStore
|
|
73
|
-
} from "../chunk-
|
|
73
|
+
} from "../chunk-QMUEXQJS.js";
|
|
74
74
|
import "../chunk-PC4UYEBM.js";
|
|
75
75
|
import {
|
|
76
76
|
validateRunRecord
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
confidenceInterval
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-56GC6VYO.js";
|
|
4
4
|
import {
|
|
5
5
|
AgentEvalError
|
|
6
6
|
} from "./chunk-3BFEG2F6.js";
|
|
@@ -538,4 +538,4 @@ export {
|
|
|
538
538
|
inMemoryCampaignStorage,
|
|
539
539
|
runCampaign
|
|
540
540
|
};
|
|
541
|
-
//# sourceMappingURL=chunk-
|
|
541
|
+
//# sourceMappingURL=chunk-2K6UUZ7P.js.map
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import {
|
|
2
2
|
summarizeBackendIntegrity
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-2K6UUZ7P.js";
|
|
4
4
|
import {
|
|
5
5
|
paretoChart
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-URWVKRCS.js";
|
|
7
7
|
import {
|
|
8
8
|
cohensD,
|
|
9
9
|
pairedBootstrap,
|
|
10
10
|
pairedMde,
|
|
11
11
|
pairedTTest,
|
|
12
12
|
requiredSampleSize
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-56GC6VYO.js";
|
|
14
14
|
import {
|
|
15
15
|
llmSpans
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-UFSG7ACU.js";
|
|
17
17
|
|
|
18
18
|
// src/contamination-guard.ts
|
|
19
19
|
function checkCanaries(output, scenarios) {
|
|
@@ -856,4 +856,4 @@ export {
|
|
|
856
856
|
HoldoutAuditor,
|
|
857
857
|
analyzeRuns
|
|
858
858
|
};
|
|
859
|
-
//# sourceMappingURL=chunk-
|
|
859
|
+
//# sourceMappingURL=chunk-2KNZHH3P.js.map
|
|
@@ -756,6 +756,37 @@ function pairedMde(opts) {
|
|
|
756
756
|
const zBeta = zQuantile(power);
|
|
757
757
|
return (zAlpha + zBeta) / Math.sqrt(opts.nPaired);
|
|
758
758
|
}
|
|
759
|
+
function mcnemarRequiredN(opts) {
|
|
760
|
+
const { p10, p01 } = opts;
|
|
761
|
+
if (p10 < 0 || p01 < 0 || p10 + p01 > 1) {
|
|
762
|
+
throw new Error(`mcnemarRequiredN: require p10,p01 \u2265 0 and p10+p01 \u2264 1 (got ${p10}, ${p01})`);
|
|
763
|
+
}
|
|
764
|
+
const delta = p10 - p01;
|
|
765
|
+
if (delta === 0) return Infinity;
|
|
766
|
+
const alpha = opts.alpha ?? 0.05;
|
|
767
|
+
const power = opts.power ?? 0.8;
|
|
768
|
+
const twoSided = opts.twoSided ?? true;
|
|
769
|
+
const pDisc = p10 + p01;
|
|
770
|
+
const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
|
|
771
|
+
const zBeta = zQuantile(power);
|
|
772
|
+
const n = (zAlpha * Math.sqrt(pDisc) + zBeta * Math.sqrt(Math.max(0, pDisc - delta * delta))) ** 2 / (delta * delta);
|
|
773
|
+
return Math.ceil(n);
|
|
774
|
+
}
|
|
775
|
+
function mcnemarPower(opts) {
|
|
776
|
+
const { p10, p01, nPairs } = opts;
|
|
777
|
+
if (p10 < 0 || p01 < 0 || p10 + p01 > 1) {
|
|
778
|
+
throw new Error(`mcnemarPower: require p10,p01 \u2265 0 and p10+p01 \u2264 1 (got ${p10}, ${p01})`);
|
|
779
|
+
}
|
|
780
|
+
const alpha = opts.alpha ?? 0.05;
|
|
781
|
+
const twoSided = opts.twoSided ?? true;
|
|
782
|
+
const delta = p10 - p01;
|
|
783
|
+
if (delta === 0 || nPairs <= 0) return alpha;
|
|
784
|
+
const pDisc = p10 + p01;
|
|
785
|
+
const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
|
|
786
|
+
const denom = Math.sqrt(Math.max(1e-12, pDisc - delta * delta));
|
|
787
|
+
const zBeta = (Math.sqrt(nPairs) * Math.abs(delta) - zAlpha * Math.sqrt(pDisc)) / denom;
|
|
788
|
+
return Math.min(1, Math.max(0, normalCdf(zBeta)));
|
|
789
|
+
}
|
|
759
790
|
function bonferroni(pValues, alpha = 0.05) {
|
|
760
791
|
const k = pValues.length;
|
|
761
792
|
const adjusted = pValues.map((p) => Math.min(1, p * k));
|
|
@@ -829,6 +860,94 @@ function pairedBootstrap(before, after, opts = {}) {
|
|
|
829
860
|
resamples
|
|
830
861
|
};
|
|
831
862
|
}
|
|
863
|
+
function wilson(successes, n, confidence = 0.95) {
|
|
864
|
+
if (n <= 0) return { estimate: 0, lower: 0, upper: 0 };
|
|
865
|
+
if (successes < 0 || successes > n) {
|
|
866
|
+
throw new Error(`wilson: successes (${successes}) must be in [0, ${n}]`);
|
|
867
|
+
}
|
|
868
|
+
const z = zQuantile(1 - (1 - confidence) / 2);
|
|
869
|
+
const p = successes / n;
|
|
870
|
+
const z2 = z * z;
|
|
871
|
+
const denom = 1 + z2 / n;
|
|
872
|
+
const center = (p + z2 / (2 * n)) / denom;
|
|
873
|
+
const half = z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n) / denom;
|
|
874
|
+
return {
|
|
875
|
+
estimate: p,
|
|
876
|
+
lower: Math.max(0, center - half),
|
|
877
|
+
upper: Math.min(1, center + half)
|
|
878
|
+
};
|
|
879
|
+
}
|
|
880
|
+
function mcnemar(control, treatment) {
|
|
881
|
+
if (control.length !== treatment.length) {
|
|
882
|
+
throw new Error(`mcnemar: unequal sample sizes (${control.length} vs ${treatment.length})`);
|
|
883
|
+
}
|
|
884
|
+
const n = control.length;
|
|
885
|
+
let b = 0;
|
|
886
|
+
let c = 0;
|
|
887
|
+
for (let i = 0; i < n; i++) {
|
|
888
|
+
const ctrl = control[i] ? 1 : 0;
|
|
889
|
+
const treat = treatment[i] ? 1 : 0;
|
|
890
|
+
if (treat === 1 && ctrl === 0) b++;
|
|
891
|
+
else if (treat === 0 && ctrl === 1) c++;
|
|
892
|
+
}
|
|
893
|
+
const nDiscordant = b + c;
|
|
894
|
+
const statistic = nDiscordant === 0 ? 0 : (Math.abs(b - c) - 1) ** 2 / nDiscordant;
|
|
895
|
+
return { n, nDiscordant, b, c, statistic, pValue: binomialSignTwoSided(b, c) };
|
|
896
|
+
}
|
|
897
|
+
function pairedRiskDifference(control, treatment, confidence = 0.95) {
|
|
898
|
+
if (control.length !== treatment.length) {
|
|
899
|
+
throw new Error(
|
|
900
|
+
`pairedRiskDifference: unequal sample sizes (${control.length} vs ${treatment.length})`
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
const n = control.length;
|
|
904
|
+
if (n === 0) return { n: 0, b: 0, c: 0, riskDifference: 0, lower: 0, upper: 0, confidence };
|
|
905
|
+
let b = 0;
|
|
906
|
+
let c = 0;
|
|
907
|
+
for (let i = 0; i < n; i++) {
|
|
908
|
+
const ctrl = control[i] ? 1 : 0;
|
|
909
|
+
const treat = treatment[i] ? 1 : 0;
|
|
910
|
+
if (treat === 1 && ctrl === 0) b++;
|
|
911
|
+
else if (treat === 0 && ctrl === 1) c++;
|
|
912
|
+
}
|
|
913
|
+
const rd = (b - c) / n;
|
|
914
|
+
const variance = (b + c - (b - c) ** 2 / n) / (n * n);
|
|
915
|
+
const z = zQuantile(1 - (1 - confidence) / 2);
|
|
916
|
+
const half = z * Math.sqrt(Math.max(0, variance));
|
|
917
|
+
return {
|
|
918
|
+
n,
|
|
919
|
+
b,
|
|
920
|
+
c,
|
|
921
|
+
riskDifference: rd,
|
|
922
|
+
lower: Math.max(-1, rd - half),
|
|
923
|
+
upper: Math.min(1, rd + half),
|
|
924
|
+
confidence
|
|
925
|
+
};
|
|
926
|
+
}
|
|
927
|
+
function passAtK(n, c, k) {
|
|
928
|
+
if (!Number.isInteger(n) || !Number.isInteger(c) || !Number.isInteger(k)) {
|
|
929
|
+
throw new Error(`passAtK: n, c, k must be integers (got n=${n}, c=${c}, k=${k})`);
|
|
930
|
+
}
|
|
931
|
+
if (k < 1 || k > n || c < 0 || c > n) {
|
|
932
|
+
throw new Error(`passAtK: require 1 \u2264 k \u2264 n and 0 \u2264 c \u2264 n (got n=${n}, c=${c}, k=${k})`);
|
|
933
|
+
}
|
|
934
|
+
if (n - c < k) return 1;
|
|
935
|
+
let prob = 1;
|
|
936
|
+
for (let i = n - c + 1; i <= n; i++) prob *= 1 - k / i;
|
|
937
|
+
return 1 - prob;
|
|
938
|
+
}
|
|
939
|
+
function binomialSignTwoSided(b, c) {
|
|
940
|
+
const nd = b + c;
|
|
941
|
+
if (nd === 0) return 1;
|
|
942
|
+
const k = Math.min(b, c);
|
|
943
|
+
const logHalfN = nd * Math.log(0.5);
|
|
944
|
+
let tail = 0;
|
|
945
|
+
for (let i = 0; i <= k; i++) {
|
|
946
|
+
const logChoose = lnGamma(nd + 1) - lnGamma(i + 1) - lnGamma(nd - i + 1);
|
|
947
|
+
tail += Math.exp(logChoose + logHalfN);
|
|
948
|
+
}
|
|
949
|
+
return Math.min(1, 2 * tail);
|
|
950
|
+
}
|
|
832
951
|
function eProcess(opts = {}) {
|
|
833
952
|
const alpha = opts.alpha ?? 0.05;
|
|
834
953
|
const maxBet = opts.maxBet ?? 0.5;
|
|
@@ -968,10 +1087,16 @@ export {
|
|
|
968
1087
|
corpusInterRaterAgreementFromJudgeScores,
|
|
969
1088
|
requiredSampleSize,
|
|
970
1089
|
pairedMde,
|
|
1090
|
+
mcnemarRequiredN,
|
|
1091
|
+
mcnemarPower,
|
|
971
1092
|
bonferroni,
|
|
972
1093
|
benjaminiHochberg,
|
|
973
1094
|
pairedBootstrap,
|
|
1095
|
+
wilson,
|
|
1096
|
+
mcnemar,
|
|
1097
|
+
pairedRiskDifference,
|
|
1098
|
+
passAtK,
|
|
974
1099
|
eProcess,
|
|
975
1100
|
mulberry322 as mulberry32
|
|
976
1101
|
};
|
|
977
|
-
//# sourceMappingURL=chunk-
|
|
1102
|
+
//# sourceMappingURL=chunk-56GC6VYO.js.map
|