@tangle-network/agent-eval 0.220.2 → 0.221.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts → agent-profile-Bc7ADJY4.d.ts} +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts.map → agent-profile-Bc7ADJY4.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +77 -78
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +204 -25
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DqPJaJ4X.js → analyze-runs-CHmsykNJ.js} +4 -4
- package/dist/{analyze-runs-DqPJaJ4X.js.map → analyze-runs-CHmsykNJ.js.map} +1 -1
- package/dist/{analyze-runs-BWAHBDAw.d.ts → analyze-runs-D7In4c3z.d.ts} +2 -2
- package/dist/{analyze-runs-BWAHBDAw.d.ts.map → analyze-runs-D7In4c3z.d.ts.map} +1 -1
- package/dist/{benchmark-command-Dx0E4a6z.js → benchmark-command-DiGqnGUt.js} +13 -11
- package/dist/benchmark-command-DiGqnGUt.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/calibration-CTwxpJ-t.js +197 -0
- package/dist/calibration-CTwxpJ-t.js.map +1 -0
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-BVPC6ODQ.js → campaign-Bt6rbWIU.js} +9 -8
- package/dist/{campaign-BVPC6ODQ.js.map → campaign-Bt6rbWIU.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{completion-verifier-Cigap8CG.js → completion-verifier-YwoeujPl.js} +2 -2
- package/dist/completion-verifier-YwoeujPl.js.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +7 -7
- package/dist/decision-point-Co3u3W8k.js +831 -0
- package/dist/decision-point-Co3u3W8k.js.map +1 -0
- package/dist/default-registry-BMHqNR6_.d.ts +257 -0
- package/dist/default-registry-BMHqNR6_.d.ts.map +1 -0
- package/dist/{default-registry-ovrIBGmH.js → default-registry-BbFCkqvB.js} +607 -14
- package/dist/default-registry-BbFCkqvB.js.map +1 -0
- package/dist/{define-agent-eval-C2nFhHPu.d.ts → define-agent-eval-Bsoi2GUV.d.ts} +7 -7
- package/dist/{define-agent-eval-C2nFhHPu.d.ts.map → define-agent-eval-Bsoi2GUV.d.ts.map} +1 -1
- package/dist/{define-agent-eval-UQY3VbJ2.js → define-agent-eval-CJhECJNL.js} +4 -4
- package/dist/{define-agent-eval-UQY3VbJ2.js.map → define-agent-eval-CJhECJNL.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js → dspy-rlm-engine-BTy7u0jQ.js} +3 -3
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js.map → dspy-rlm-engine-BTy7u0jQ.js.map} +1 -1
- package/dist/{engine-BpGGw6cd.d.ts → engine-BF95CWzH.d.ts} +2 -2
- package/dist/{engine-BpGGw6cd.d.ts.map → engine-BF95CWzH.d.ts.map} +1 -1
- package/dist/{eval-campaign-BUMIxKTE.js → eval-campaign-DoX5vz3E.js} +2 -2
- package/dist/{eval-campaign-BUMIxKTE.js.map → eval-campaign-DoX5vz3E.js.map} +1 -1
- package/dist/{evaluation-CBdQt_kr.d.ts → evaluation-D1PGzkPR.d.ts} +3 -3
- package/dist/{evaluation-CBdQt_kr.d.ts.map → evaluation-D1PGzkPR.d.ts.map} +1 -1
- package/dist/evaluation-wdGyZ7jV.js +147 -0
- package/dist/evaluation-wdGyZ7jV.js.map +1 -0
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -136
- package/dist/{exact-types-BYFmYq44.d.ts → exact-types-CVn1QxFO.d.ts} +2 -2
- package/dist/{exact-types-BYFmYq44.d.ts.map → exact-types-CVn1QxFO.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js → external-optimizer-process-E5yY2SYS.js} +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js.map → external-optimizer-process-E5yY2SYS.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-D670Oxf2.js → external-optimizer-subprocess-DA02c_L6.js} +12 -19
- package/dist/{external-optimizer-subprocess-D670Oxf2.js.map → external-optimizer-subprocess-DA02c_L6.js.map} +1 -1
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts → feedback-trajectory-BZhVCWuz.d.ts} +3 -3
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts.map → feedback-trajectory-BZhVCWuz.d.ts.map} +1 -1
- package/dist/finding-composition-8n0fFgVI.js +131 -0
- package/dist/finding-composition-8n0fFgVI.js.map +1 -0
- package/dist/{findings-store-CvNo712L.js → findings-store-DRiFdoid.js} +4 -4
- package/dist/{findings-store-CvNo712L.js.map → findings-store-DRiFdoid.js.map} +1 -1
- package/dist/{index-CVFStrQD.d.ts → index-BJZ7s-sv.d.ts} +7 -58
- package/dist/index-BJZ7s-sv.d.ts.map +1 -0
- package/dist/{index-Q6DvdcwI.d.ts → index-BvX3CFNS.d.ts} +6 -6
- package/dist/{index-Q6DvdcwI.d.ts.map → index-BvX3CFNS.d.ts.map} +1 -1
- package/dist/index-DWd1gUwR.d.ts +1086 -0
- package/dist/index-DWd1gUwR.d.ts.map +1 -0
- package/dist/{index-DAXfpVf7.d.ts → index-y7i8l6Y_.d.ts} +8 -8
- package/dist/{index-DAXfpVf7.d.ts.map → index-y7i8l6Y_.d.ts.map} +1 -1
- package/dist/index.d.ts +18 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -15
- package/dist/{kind-factory-BhxbtUsQ.js → kind-factory-Dl0PmNQL.js} +2 -2
- package/dist/{kind-factory-BhxbtUsQ.js.map → kind-factory-Dl0PmNQL.js.map} +1 -1
- package/dist/{llm-judge-Cc8vDJh_.js → llm-judge-3uAeRawE.js} +4 -4
- package/dist/{llm-judge-Cc8vDJh_.js.map → llm-judge-3uAeRawE.js.map} +1 -1
- package/dist/{matrix-Cf-AGxGh.d.ts → matrix-BXo3PSqM.d.ts} +2 -2
- package/dist/{matrix-Cf-AGxGh.d.ts.map → matrix-BXo3PSqM.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -638
- package/dist/meta-eval/index.js +3 -193
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/outcome-observations-FLmRpgYa.js +78 -0
- package/dist/outcome-observations-FLmRpgYa.js.map +1 -0
- package/dist/{paired-arms-Ct4LTKc1.js → paired-arms-B2ttLc0_.js} +108 -31
- package/dist/paired-arms-B2ttLc0_.js.map +1 -0
- package/dist/{power-and-mde-N5nU4_Ly.js → power-and-mde-CQX-0weG.js} +2 -2
- package/dist/{power-and-mde-N5nU4_Ly.js.map → power-and-mde-CQX-0weG.js.map} +1 -1
- package/dist/{prime-protocol-CckHBxGu.d.ts → prime-protocol-BEaqBoQE.d.ts} +2 -2
- package/dist/{prime-protocol-CckHBxGu.d.ts.map → prime-protocol-BEaqBoQE.d.ts.map} +1 -1
- package/dist/{proposal-findings-E1aSC_cU.js → proposal-findings-D0_zkQ2l.js} +2 -2
- package/dist/{proposal-findings-E1aSC_cU.js.map → proposal-findings-D0_zkQ2l.js.map} +1 -1
- package/dist/{registry-B_yG-1jh.d.ts → registry-DDKaUrmM.d.ts} +3 -3
- package/dist/{registry-B_yG-1jh.d.ts.map → registry-DDKaUrmM.d.ts.map} +1 -1
- package/dist/reporting.d.ts +1 -1
- package/dist/reporting.js +3 -3
- package/dist/rl.d.ts +4 -5
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +5 -4
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-noY7uE03.d.ts → rubric-predictive-validity-Cd5SCtAf.d.ts} +58 -3
- package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-D3_oI72N.js → rubric-predictive-validity-CeKW2GQN.js} +3 -77
- package/dist/rubric-predictive-validity-CeKW2GQN.js.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts → search-kernel-C6__V0IS.d.ts} +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts.map → search-kernel-C6__V0IS.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-DLKqdhcR.js → skillopt-optimization-method-D21dH_Bh.js} +4 -4
- package/dist/{skillopt-optimization-method-DLKqdhcR.js.map → skillopt-optimization-method-D21dH_Bh.js.map} +1 -1
- package/dist/{statistical-heldout-3z3I-kJN.d.ts → statistical-heldout-pD1Yah-0.d.ts} +58 -8
- package/dist/statistical-heldout-pD1Yah-0.d.ts.map +1 -0
- package/dist/{store-otlp-dyWE3wAG.js → store-otlp-BVZfR168.js} +2 -2
- package/dist/{store-otlp-dyWE3wAG.js.map → store-otlp-BVZfR168.js.map} +1 -1
- package/dist/{store-tool-spans-DIUgZbzu.js → store-tool-spans-CjYiF4fF.js} +3 -3
- package/dist/{store-tool-spans-DIUgZbzu.js.map → store-tool-spans-CjYiF4fF.js.map} +1 -1
- package/dist/{store-tool-spans-DPwXHQkH.d.ts → store-tool-spans-DfnwWMFR.d.ts} +4 -4
- package/dist/{store-tool-spans-DPwXHQkH.d.ts.map → store-tool-spans-DfnwWMFR.d.ts.map} +1 -1
- package/dist/{summary-report-BsXJoYX5.js → summary-report-DHDWxe6p.js} +3 -3
- package/dist/{summary-report-BsXJoYX5.js.map → summary-report-DHDWxe6p.js.map} +1 -1
- package/dist/systemone-D3MrW63C.d.ts +400 -0
- package/dist/systemone-D3MrW63C.d.ts.map +1 -0
- package/dist/{systemone-protocol-DKoi1NdK.d.ts → systemone-protocol-Bl2EI2KQ.d.ts} +4 -2
- package/dist/{systemone-protocol-DKoi1NdK.d.ts.map → systemone-protocol-Bl2EI2KQ.d.ts.map} +1 -1
- package/dist/systemone-protocol.d.ts +2 -2
- package/dist/systemone-protocol.js +7 -7
- package/dist/systemone-protocol.js.map +1 -1
- package/dist/{systemone-review-CUBMuTLi.d.ts → systemone-review-C8DRiT_-.d.ts} +3 -3
- package/dist/{systemone-review-CUBMuTLi.d.ts.map → systemone-review-C8DRiT_-.d.ts.map} +1 -1
- package/dist/{systemone-review-DzdOJ8I6.js → systemone-review-DFGtIUcf.js} +2 -2
- package/dist/{systemone-review-DzdOJ8I6.js.map → systemone-review-DFGtIUcf.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -343
- package/dist/{tool-groups-LISbBWCg.d.ts → tool-groups-DK1bu5TD.d.ts} +3 -3
- package/dist/tool-groups-DK1bu5TD.d.ts.map +1 -0
- package/dist/{trace-questions-C42fYEE4.d.ts → trace-questions-D_OKYZ6u.d.ts} +3 -3
- package/dist/{trace-questions-C42fYEE4.d.ts.map → trace-questions-D_OKYZ6u.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +3 -3
- package/dist/{types-DqFMNTgK.js → types-CnWmj8NW.js} +19 -6
- package/dist/types-CnWmj8NW.js.map +1 -0
- package/dist/{types-CHfn1zg5.d.ts → types-DDKfH-N6.d.ts} +2 -2
- package/dist/{types-CHfn1zg5.d.ts.map → types-DDKfH-N6.d.ts.map} +1 -1
- package/dist/{types-8yQzNm8u.d.ts → types-DGImvHIr.d.ts} +150 -4
- package/dist/types-DGImvHIr.d.ts.map +1 -0
- package/dist/{types-D8GXK9Z5.d.ts → types-IERw_HxR.d.ts} +2 -2
- package/dist/{types-D8GXK9Z5.d.ts.map → types-IERw_HxR.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/behavioral-trace-questions.md +99 -0
- package/docs/experiment.md +27 -0
- package/docs/finding-layers.md +54 -1
- package/docs/systemone.md +107 -0
- package/package.json +4 -4
- package/dist/benchmark-command-Dx0E4a6z.js.map +0 -1
- package/dist/completion-verifier-Cigap8CG.js.map +0 -1
- package/dist/default-registry-Cwg_QN3d.d.ts +0 -70
- package/dist/default-registry-Cwg_QN3d.d.ts.map +0 -1
- package/dist/default-registry-ovrIBGmH.js.map +0 -1
- package/dist/evaluation.js.map +0 -1
- package/dist/index-CVFStrQD.d.ts.map +0 -1
- package/dist/judge-gate-Dc_mXQsL.d.ts +0 -453
- package/dist/judge-gate-Dc_mXQsL.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/outcome-store-CNt4iZ67.d.ts +0 -58
- package/dist/outcome-store-CNt4iZ67.d.ts.map +0 -1
- package/dist/paired-arms-Ct4LTKc1.js.map +0 -1
- package/dist/rubric-predictive-validity-D3_oI72N.js.map +0 -1
- package/dist/rubric-predictive-validity-noY7uE03.d.ts.map +0 -1
- package/dist/statistical-heldout-3z3I-kJN.d.ts.map +0 -1
- package/dist/systemone-DVA7t76V.d.ts +0 -135
- package/dist/systemone-DVA7t76V.d.ts.map +0 -1
- package/dist/systemone.js.map +0 -1
- package/dist/tool-groups-LISbBWCg.d.ts.map +0 -1
- package/dist/types-8yQzNm8u.d.ts.map +0 -1
- package/dist/types-DqFMNTgK.js.map +0 -1
package/dist/{rubric-predictive-validity-noY7uE03.d.ts → rubric-predictive-validity-Cd5SCtAf.d.ts}
RENAMED
|
@@ -1,5 +1,60 @@
|
|
|
1
1
|
import { a as RunRecord } from "./run-record-C3awiXPf.js";
|
|
2
|
-
|
|
2
|
+
//#region src/meta-eval/outcome-store.d.ts
|
|
3
|
+
/** Deployment observations join to evaluation records through their runId. */
|
|
4
|
+
interface DeploymentOutcome {
|
|
5
|
+
runId: string;
|
|
6
|
+
capturedAt: number;
|
|
7
|
+
/** Finite numeric outcomes; absence of a key means unmeasured. */
|
|
8
|
+
metrics: Record<string, number>;
|
|
9
|
+
labels?: Record<string, string>;
|
|
10
|
+
/** Source system or pipeline version. */
|
|
11
|
+
source?: string;
|
|
12
|
+
}
|
|
13
|
+
interface OutcomeFilter {
|
|
14
|
+
runIds?: string[];
|
|
15
|
+
since?: number;
|
|
16
|
+
until?: number;
|
|
17
|
+
label?: {
|
|
18
|
+
key: string;
|
|
19
|
+
value: string;
|
|
20
|
+
};
|
|
21
|
+
source?: string;
|
|
22
|
+
}
|
|
23
|
+
interface OutcomeStore {
|
|
24
|
+
append(outcome: DeploymentOutcome): Promise<void>;
|
|
25
|
+
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
26
|
+
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
27
|
+
}
|
|
28
|
+
declare class InMemoryOutcomeStore implements OutcomeStore {
|
|
29
|
+
private items;
|
|
30
|
+
append(outcome: DeploymentOutcome): Promise<void>;
|
|
31
|
+
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
32
|
+
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
33
|
+
}
|
|
34
|
+
interface FileSystemOutcomeStoreOptions {
|
|
35
|
+
dir: string;
|
|
36
|
+
/** Rotate before a write once the active file reaches this size. Default 32 MiB. */
|
|
37
|
+
maxBytes?: number;
|
|
38
|
+
}
|
|
39
|
+
/** Storage failures retain operation, path, source line, and the original cause. */
|
|
40
|
+
declare class OutcomeStoreError extends Error {
|
|
41
|
+
readonly operation: 'read' | 'decode' | 'write';
|
|
42
|
+
readonly path: string;
|
|
43
|
+
readonly line?: number | undefined;
|
|
44
|
+
constructor(operation: 'read' | 'decode' | 'write', path: string, cause: unknown, line?: number | undefined);
|
|
45
|
+
}
|
|
46
|
+
/** Local storage with serialized operations per instance; use one writer per directory. */
|
|
47
|
+
declare class FileSystemOutcomeStore implements OutcomeStore {
|
|
48
|
+
private readonly dir;
|
|
49
|
+
private readonly maxBytes;
|
|
50
|
+
private pending;
|
|
51
|
+
constructor(options: FileSystemOutcomeStoreOptions);
|
|
52
|
+
append(outcome: DeploymentOutcome): Promise<void>;
|
|
53
|
+
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
54
|
+
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
55
|
+
private serialize;
|
|
56
|
+
}
|
|
57
|
+
//#endregion
|
|
3
58
|
//#region src/meta-eval/outcome-observations.d.ts
|
|
4
59
|
type OutcomeReduction = 'latest' | 'mean' | 'max';
|
|
5
60
|
interface CorrelationInterval {
|
|
@@ -75,5 +130,5 @@ interface RubricPredictiveValidityReport {
|
|
|
75
130
|
}
|
|
76
131
|
declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
|
|
77
132
|
//#endregion
|
|
78
|
-
export { RubricPredictiveValidityReport as a, CorrelationInterval as c, RubricPredictiveValidityInput as i, OutcomeReduction as l, RubricOutcomeExclusion as n, RubricRanking as o, RubricOutcomePair as r, rubricPredictiveValidity as s, OutcomeMetricSpec as t };
|
|
79
|
-
//# sourceMappingURL=rubric-predictive-validity-
|
|
133
|
+
export { RubricPredictiveValidityReport as a, CorrelationInterval as c, FileSystemOutcomeStore as d, FileSystemOutcomeStoreOptions as f, OutcomeStoreError as g, OutcomeStore as h, RubricPredictiveValidityInput as i, OutcomeReduction as l, OutcomeFilter as m, RubricOutcomeExclusion as n, RubricRanking as o, InMemoryOutcomeStore as p, RubricOutcomePair as r, rubricPredictiveValidity as s, OutcomeMetricSpec as t, DeploymentOutcome as u };
|
|
134
|
+
//# sourceMappingURL=rubric-predictive-validity-Cd5SCtAf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rubric-predictive-validity-Cd5SCtAf.d.ts","names":[],"sources":["../src/meta-eval/outcome-store.ts","../src/meta-eval/outcome-observations.ts","../src/meta-eval/rubric-predictive-validity.ts"],"mappings":";;;UAIiB;EACf;EACA;;EAEA,SAAS;EACT,SAAS;;EAET;;UAGe;EACf;EACA;EACA;EACA;IAAU;IAAa;;EACvB;;UAGe;EACf,OAAO,SAAS,oBAAoB;EACpC,OAAO,gBAAgB,QAAQ;EAC/B,KAAK,SAAS,gBAAgB,QAAQ;;cAa3B,gCAAgC;UACnC;EAEF,OAAO,SAAS,oBAAoB;EAIpC,OAAO,gBAAgB,QAAQ;EAI/B,KAAK,SAAQ,gBAAqB,QAAQ;;UAOjC;EACf;;EAEA;;;cAIW,0BAA0B;WAEnB;WACA;WAEA;EAJlB,YACkB,wCACA,cAChB,gBACgB;;;cAUP,kCAAkC;mBAC5B;mBACA;UACT;EAER,YAAY,SAAS;EASf,OAAO,SAAS,oBAAoB;EA8BpC,OAAO,gBAAgB,QAAQ;EAI/B,KAAK,SAAQ,gBAAqB,QAAQ;UAqCxC;;;;KC/JE;UAEK;EACf;EACA;;;;UCWe;;EAEf;EACA;;UAGe;;EAEf,MAAM;EACN,UAAU;;EAEV,yBAAyB;;EAEzB;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,kBAAkB;EAClB;;EAEA;EACA;EACA,aAAa;EACb,cAAc;;EAEd;EACA;EACA,qBAAqB;;EAErB;;UAGe,sBAAsB,KAAK;;EAE1C;;UAGe;EACf;EACA;EACA,kBAAkB;;EAElB;EACA;;UAGe;EACf,gBAAgB;EAChB,OAAO;;EAEP,eAAe;;EAEf,QAAQ;;EAER;;EAEA;;EAEA;;iBAGoB,yBACpB,OAAO,gCACN,QAAQ"}
|
package/dist/{rubric-predictive-validity-D3_oI72N.js → rubric-predictive-validity-CeKW2GQN.js}
RENAMED
|
@@ -1,78 +1,4 @@
|
|
|
1
|
-
import { i as
|
|
2
|
-
import { a as spearmanR, r as pearsonR } from "./descriptive-CVNy48rR.js";
|
|
3
|
-
//#region src/meta-eval/outcome-observations.ts
|
|
4
|
-
/** Reduce observations of one named metric; unrelated keys cannot supply a value. */
|
|
5
|
-
function reduceOutcomeMetric(outcomes, metric, reduction) {
|
|
6
|
-
const observations = outcomes.flatMap((outcome) => {
|
|
7
|
-
const value = outcome.metrics[metric];
|
|
8
|
-
return Number.isFinite(outcome.capturedAt) && typeof value === "number" && Number.isFinite(value) ? [{
|
|
9
|
-
capturedAt: outcome.capturedAt,
|
|
10
|
-
value
|
|
11
|
-
}] : [];
|
|
12
|
-
});
|
|
13
|
-
if (observations.length === 0) return null;
|
|
14
|
-
if (reduction === "mean") return observations.reduce((sum, observation) => sum + observation.value / observations.length, 0);
|
|
15
|
-
if (reduction === "max") return Math.max(...observations.map((observation) => observation.value));
|
|
16
|
-
return observations.reduce((a, b) => b.capturedAt > a.capturedAt ? b : a).value;
|
|
17
|
-
}
|
|
18
|
-
function hasVariation(values) {
|
|
19
|
-
return values.some((value) => value !== values[0]);
|
|
20
|
-
}
|
|
21
|
-
function correlationSummary(xs, ys, iterations, seed) {
|
|
22
|
-
const rng = makeRng(seed, xs, ys);
|
|
23
|
-
const pearsons = [];
|
|
24
|
-
const spearmans = [];
|
|
25
|
-
for (let b = 0; b < iterations; b++) {
|
|
26
|
-
const rx = [];
|
|
27
|
-
const ry = [];
|
|
28
|
-
for (let i = 0; i < xs.length; i++) {
|
|
29
|
-
const index = Math.floor(rng() * xs.length);
|
|
30
|
-
rx.push(xs[index]);
|
|
31
|
-
ry.push(ys[index]);
|
|
32
|
-
}
|
|
33
|
-
if (!hasVariation(rx) || !hasVariation(ry)) continue;
|
|
34
|
-
const pearson = pearsonR(rx, ry);
|
|
35
|
-
const spearman = spearmanR(rx, ry);
|
|
36
|
-
if (Number.isFinite(pearson)) pearsons.push(pearson);
|
|
37
|
-
if (Number.isFinite(spearman)) spearmans.push(spearman);
|
|
38
|
-
}
|
|
39
|
-
return {
|
|
40
|
-
pearson: pearsonR(xs, ys),
|
|
41
|
-
spearman: spearmanR(xs, ys),
|
|
42
|
-
pearsonCi95: interval(pearsons),
|
|
43
|
-
spearmanCi95: interval(spearmans)
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
function validateObservationOptions(reduction, iterations, seed) {
|
|
47
|
-
if (![
|
|
48
|
-
"latest",
|
|
49
|
-
"mean",
|
|
50
|
-
"max"
|
|
51
|
-
].includes(reduction)) throw new Error("outcome reduction must be latest, mean, or max");
|
|
52
|
-
if (!Number.isSafeInteger(iterations) || iterations < 1) throw new Error("bootstrap iterations must be a positive safe integer");
|
|
53
|
-
if (seed !== void 0 && !Number.isFinite(seed)) throw new Error("bootstrap seed must be finite");
|
|
54
|
-
}
|
|
55
|
-
function assertUniqueObservationIds(ids, name) {
|
|
56
|
-
if (ids.some((id) => typeof id !== "string" || id.trim().length === 0)) throw new Error(`${name} must be a nonempty string`);
|
|
57
|
-
if (new Set(ids).size !== ids.length) throw new Error(`duplicate ${name} in outcome study`);
|
|
58
|
-
}
|
|
59
|
-
function validateOutcomeMetricSpecifications(metrics) {
|
|
60
|
-
if (!Array.isArray(metrics) || metrics.length === 0) throw new Error("outcomeMetrics must declare at least one outcome metric and direction");
|
|
61
|
-
for (const metric of metrics) {
|
|
62
|
-
if (!metric || typeof metric.id !== "string" || metric.id.trim().length === 0) throw new Error("each outcome metric must have a nonempty id and explicit direction");
|
|
63
|
-
if (metric.direction !== "higher-is-better" && metric.direction !== "lower-is-better") throw new Error(`outcome metric ${metric.id} must declare higher-is-better or lower-is-better`);
|
|
64
|
-
}
|
|
65
|
-
assertUniqueObservationIds(metrics.map((metric) => metric.id), "outcome metric");
|
|
66
|
-
}
|
|
67
|
-
function interval(values) {
|
|
68
|
-
if (values.length === 0) return null;
|
|
69
|
-
values.sort((a, b) => a - b);
|
|
70
|
-
return {
|
|
71
|
-
lower: Math.max(-1, values[Math.floor(.025 * values.length)]),
|
|
72
|
-
upper: Math.min(1, values[Math.min(values.length - 1, Math.floor(.975 * values.length))])
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
//#endregion
|
|
1
|
+
import { a as validateObservationOptions, i as reduceOutcomeMetric, n as correlationSummary, o as validateOutcomeMetricSpecifications, r as hasVariation, t as assertUniqueObservationIds } from "./outcome-observations-FLmRpgYa.js";
|
|
76
2
|
//#region src/meta-eval/rubric-predictive-validity.ts
|
|
77
3
|
async function rubricPredictiveValidity(input) {
|
|
78
4
|
const minSamples = input.minSamples ?? 8;
|
|
@@ -173,6 +99,6 @@ async function rubricPredictiveValidity(input) {
|
|
|
173
99
|
};
|
|
174
100
|
}
|
|
175
101
|
//#endregion
|
|
176
|
-
export {
|
|
102
|
+
export { rubricPredictiveValidity as t };
|
|
177
103
|
|
|
178
|
-
//# sourceMappingURL=rubric-predictive-validity-
|
|
104
|
+
//# sourceMappingURL=rubric-predictive-validity-CeKW2GQN.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rubric-predictive-validity-CeKW2GQN.js","names":[],"sources":["../src/meta-eval/rubric-predictive-validity.ts"],"sourcesContent":["/**\n * Join rubric scores to deployment outcomes and measure their association.\n * Higher rubric scores always mean better evaluated behavior.\n * Outcome directions are explicit because success rate and failure rate have opposite meanings.\n * These descriptive associations neither establish causation nor validate a change to rubric weights.\n */\n\nimport type { RunRecord } from '../run-record'\nimport {\n assertUniqueObservationIds,\n type CorrelationInterval,\n correlationSummary,\n hasVariation,\n reduceOutcomeMetric,\n validateObservationOptions,\n validateOutcomeMetricSpecifications,\n} from './outcome-observations'\nimport type { DeploymentOutcome, OutcomeStore } from './outcome-store'\n\nexport interface OutcomeMetricSpec {\n /** Exact key in DeploymentOutcome.metrics. */\n id: string\n direction: 'higher-is-better' | 'lower-is-better'\n}\n\nexport interface RubricPredictiveValidityInput {\n /** One record per independent run; rubric scores come from outcome.raw. */\n runs: RunRecord[]\n outcomes: OutcomeStore\n /** Declare desired directions before inspecting associations. */\n outcomeMetrics: readonly OutcomeMetricSpec[]\n /** Higher is better for each rubric. Omit to discover finite numeric outcome.raw keys. */\n rubrics?: readonly string[]\n /** Minimum joined runs for an estimate; an integer at least 3. Default 8. */\n minSamples?: number\n /** Bootstrap resamples for both correlation intervals. Default 500. */\n bootstrapResamples?: number\n /** Omit to derive a reproducible seed from the paired observations. */\n seed?: number\n /** Reduce finite observations of each named outcome within a run. Default latest. */\n reduction?: 'latest' | 'mean' | 'max'\n}\n\nexport interface RubricOutcomePair {\n rubric: string\n outcome: string\n outcomeDirection: OutcomeMetricSpec['direction']\n n: number\n /** Raw association with the recorded outcome, before direction alignment. */\n pearson: number\n spearman: number\n pearsonCi95: CorrelationInterval | null\n spearmanCi95: CorrelationInterval | null\n /** Positive values associate higher rubric scores with better outcomes. */\n alignedPearson: number\n alignedSpearman: number\n alignedSpearmanCi95: CorrelationInterval | null\n /** Descriptive buckets at aligned Spearman +/-0.4; no causal or release authority. */\n verdict: 'aligned' | 'inverse' | 'weak'\n}\n\nexport interface RubricRanking extends Omit<RubricOutcomePair, 'outcome'> {\n /** Outcome with the greatest direction-aligned Spearman for this rubric. */\n bestOutcome: string\n}\n\nexport interface RubricOutcomeExclusion {\n rubric: string\n outcome: string\n outcomeDirection: OutcomeMetricSpec['direction']\n /** Finite joined observations, including measured zeros. */\n n: number\n reason: 'insufficient_samples' | 'constant_rubric' | 'constant_outcome'\n}\n\nexport interface RubricPredictiveValidityReport {\n outcomeMetrics: OutcomeMetricSpec[]\n pairs: RubricOutcomePair[]\n /** All declared pairs lacking an estimate, with their usable observation count. */\n excludedPairs: RubricOutcomeExclusion[]\n /** Exploratory ordering by aligned Spearman; never use outcome selection as confirmatory evidence. */\n ranked: RubricRanking[]\n /** Runs contributing at least one finite pair, including pairs below minSamples. */\n joinedSamples: number\n /** Runs contributing no finite pair; joinedSamples + skippedRuns equals the input run count. */\n skippedRuns: number\n /** Declared rubrics with no finite score, distinct from too few outcomes or constant observations. */\n rubricsWithoutData: string[]\n}\n\nexport async function rubricPredictiveValidity(\n input: RubricPredictiveValidityInput,\n): Promise<RubricPredictiveValidityReport> {\n const minSamples = input.minSamples ?? 8\n const reduction = input.reduction ?? 'latest'\n const resamples = input.bootstrapResamples ?? 500\n const seed = input.seed\n if (!Number.isSafeInteger(minSamples) || minSamples < 3) {\n throw new Error('minSamples must be a safe integer at least 3')\n }\n validateObservationOptions(reduction, resamples, seed)\n validateOutcomeMetricSpecifications(input.outcomeMetrics)\n assertUniqueObservationIds(\n input.runs.map((run) => run.runId),\n 'runId',\n )\n\n const outcomeMetrics = input.outcomeMetrics.map((metric) => ({ ...metric }))\n const runs = input.runs.map((run) => ({ runId: run.runId, scores: { ...run.outcome.raw } }))\n const declaredRubrics = input.rubrics === undefined ? undefined : [...input.rubrics]\n if (declaredRubrics !== undefined) assertUniqueObservationIds(declaredRubrics, 'rubric')\n\n const outcomes = await input.outcomes.list()\n const outcomesByRun = new Map<string, DeploymentOutcome[]>()\n for (const outcome of outcomes) {\n const rows = outcomesByRun.get(outcome.runId) ?? []\n rows.push(outcome)\n outcomesByRun.set(outcome.runId, rows)\n }\n\n const observedRubrics = new Set<string>()\n for (const run of runs) {\n for (const [rubric, value] of Object.entries(run.scores)) {\n if (typeof value === 'number' && Number.isFinite(value)) observedRubrics.add(rubric)\n }\n }\n const rubrics = declaredRubrics ?? [...observedRubrics]\n const buckets = rubrics.flatMap((rubric) =>\n outcomeMetrics.map((outcome) => ({\n rubric,\n outcome,\n xs: [] as number[],\n ys: [] as number[],\n })),\n )\n\n let joined = 0\n for (const run of runs) {\n const rows = outcomesByRun.get(run.runId) ?? []\n let joinedThisRun = false\n for (const bucket of buckets) {\n const x = run.scores[bucket.rubric]\n if (typeof x !== 'number' || !Number.isFinite(x)) continue\n const y = reduceOutcomeMetric(rows, bucket.outcome.id, reduction)\n if (y === null) continue\n bucket.xs.push(x)\n bucket.ys.push(y)\n joinedThisRun = true\n }\n if (joinedThisRun) joined++\n }\n\n const pairs: RubricOutcomePair[] = []\n const excludedPairs: RubricOutcomeExclusion[] = []\n for (const bucket of buckets) {\n const identity = {\n rubric: bucket.rubric,\n outcome: bucket.outcome.id,\n outcomeDirection: bucket.outcome.direction,\n n: bucket.xs.length,\n }\n const reason =\n bucket.xs.length < minSamples\n ? 'insufficient_samples'\n : !hasVariation(bucket.xs)\n ? 'constant_rubric'\n : !hasVariation(bucket.ys)\n ? 'constant_outcome'\n : null\n if (reason !== null) {\n excludedPairs.push({ ...identity, reason })\n continue\n }\n const summary = correlationSummary(bucket.xs, bucket.ys, resamples, seed)\n const sign = bucket.outcome.direction === 'higher-is-better' ? 1 : -1\n const alignedSpearman = summary.spearman * sign\n const alignedSpearmanCi95 =\n summary.spearmanCi95 === null\n ? null\n : sign === 1\n ? summary.spearmanCi95\n : { lower: -summary.spearmanCi95.upper, upper: -summary.spearmanCi95.lower }\n pairs.push({\n ...identity,\n ...summary,\n alignedPearson: summary.pearson * sign,\n alignedSpearman,\n alignedSpearmanCi95,\n verdict: alignedSpearman >= 0.4 ? 'aligned' : alignedSpearman <= -0.4 ? 'inverse' : 'weak',\n })\n }\n\n const bestByRubric = new Map<string, RubricOutcomePair>()\n for (const pair of pairs) {\n const best = bestByRubric.get(pair.rubric)\n if (!best || pair.alignedSpearman > best.alignedSpearman) bestByRubric.set(pair.rubric, pair)\n }\n const ranked = [...bestByRubric.values()]\n .map(({ outcome, ...pair }) => ({ ...pair, bestOutcome: outcome }))\n .sort((a, b) => b.alignedSpearman - a.alignedSpearman)\n\n return {\n outcomeMetrics,\n pairs,\n excludedPairs,\n ranked,\n joinedSamples: joined,\n skippedRuns: runs.length - joined,\n rubricsWithoutData: rubrics.filter((rubric) => !observedRubrics.has(rubric)),\n }\n}\n"],"mappings":";;AA0FA,eAAsB,yBACpB,OACyC;CACzC,MAAM,aAAa,MAAM,cAAc;CACvC,MAAM,YAAY,MAAM,aAAa;CACrC,MAAM,YAAY,MAAM,sBAAsB;CAC9C,MAAM,OAAO,MAAM;CACnB,IAAI,CAAC,OAAO,cAAc,UAAU,KAAK,aAAa,GACpD,MAAM,IAAI,MAAM,8CAA8C;CAEhE,2BAA2B,WAAW,WAAW,IAAI;CACrD,oCAAoC,MAAM,cAAc;CACxD,2BACE,MAAM,KAAK,KAAK,QAAQ,IAAI,KAAK,GACjC,OACF;CAEA,MAAM,iBAAiB,MAAM,eAAe,KAAK,YAAY,EAAE,GAAG,OAAO,EAAE;CAC3E,MAAM,OAAO,MAAM,KAAK,KAAK,SAAS;EAAE,OAAO,IAAI;EAAO,QAAQ,EAAE,GAAG,IAAI,QAAQ,IAAI;CAAE,EAAE;CAC3F,MAAM,kBAAkB,MAAM,YAAY,KAAA,IAAY,KAAA,IAAY,CAAC,GAAG,MAAM,OAAO;CACnF,IAAI,oBAAoB,KAAA,GAAW,2BAA2B,iBAAiB,QAAQ;CAEvF,MAAM,WAAW,MAAM,MAAM,SAAS,KAAK;CAC3C,MAAM,gCAAgB,IAAI,IAAiC;CAC3D,KAAK,MAAM,WAAW,UAAU;EAC9B,MAAM,OAAO,cAAc,IAAI,QAAQ,KAAK,KAAK,CAAC;EAClD,KAAK,KAAK,OAAO;EACjB,cAAc,IAAI,QAAQ,OAAO,IAAI;CACvC;CAEA,MAAM,kCAAkB,IAAI,IAAY;CACxC,KAAK,MAAM,OAAO,MAChB,KAAK,MAAM,CAAC,QAAQ,UAAU,OAAO,QAAQ,IAAI,MAAM,GACrD,IAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAAG,gBAAgB,IAAI,MAAM;CAGvF,MAAM,UAAU,mBAAmB,CAAC,GAAG,eAAe;CACtD,MAAM,UAAU,QAAQ,SAAS,WAC/B,eAAe,KAAK,aAAa;EAC/B;EACA;EACA,IAAI,CAAC;EACL,IAAI,CAAC;CACP,EAAE,CACJ;CAEA,IAAI,SAAS;CACb,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,cAAc,IAAI,IAAI,KAAK,KAAK,CAAC;EAC9C,IAAI,gBAAgB;EACpB,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,IAAI,OAAO,OAAO;GAC5B,IAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,GAAG;GAClD,MAAM,IAAI,oBAAoB,MAAM,OAAO,QAAQ,IAAI,SAAS;GAChE,IAAI,MAAM,MAAM;GAChB,OAAO,GAAG,KAAK,CAAC;GAChB,OAAO,GAAG,KAAK,CAAC;GAChB,gBAAgB;EAClB;EACA,IAAI,eAAe;CACrB;CAEA,MAAM,QAA6B,CAAC;CACpC,MAAM,gBAA0C,CAAC;CACjD,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,WAAW;GACf,QAAQ,OAAO;GACf,SAAS,OAAO,QAAQ;GACxB,kBAAkB,OAAO,QAAQ;GACjC,GAAG,OAAO,GAAG;EACf;EACA,MAAM,SACJ,OAAO,GAAG,SAAS,aACf,yBACA,CAAC,aAAa,OAAO,EAAE,IACrB,oBACA,CAAC,aAAa,OAAO,EAAE,IACrB,qBACA;EACV,IAAI,WAAW,MAAM;GACnB,cAAc,KAAK;IAAE,GAAG;IAAU;GAAO,CAAC;GAC1C;EACF;EACA,MAAM,UAAU,mBAAmB,OAAO,IAAI,OAAO,IAAI,WAAW,IAAI;EACxE,MAAM,OAAO,OAAO,QAAQ,cAAc,qBAAqB,IAAI;EACnE,MAAM,kBAAkB,QAAQ,WAAW;EAC3C,MAAM,sBACJ,QAAQ,iBAAiB,OACrB,OACA,SAAS,IACP,QAAQ,eACR;GAAE,OAAO,CAAC,QAAQ,aAAa;GAAO,OAAO,CAAC,QAAQ,aAAa;EAAM;EACjF,MAAM,KAAK;GACT,GAAG;GACH,GAAG;GACH,gBAAgB,QAAQ,UAAU;GAClC;GACA;GACA,SAAS,mBAAmB,KAAM,YAAY,mBAAmB,MAAO,YAAY;EACtF,CAAC;CACH;CAEA,MAAM,+BAAe,IAAI,IAA+B;CACxD,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,OAAO,aAAa,IAAI,KAAK,MAAM;EACzC,IAAI,CAAC,QAAQ,KAAK,kBAAkB,KAAK,iBAAiB,aAAa,IAAI,KAAK,QAAQ,IAAI;CAC9F;CAKA,OAAO;EACL;EACA;EACA;EACA,QARa,CAAC,GAAG,aAAa,OAAO,CAAC,CAAC,CACtC,KAAK,EAAE,SAAS,GAAG,YAAY;GAAE,GAAG;GAAM,aAAa;EAAQ,EAAE,CAAC,CAClE,MAAM,GAAG,MAAM,EAAE,kBAAkB,EAAE,eAMjC;EACL,eAAe;EACf,aAAa,KAAK,SAAS;EAC3B,oBAAoB,QAAQ,QAAQ,WAAW,CAAC,gBAAgB,IAAI,MAAM,CAAC;CAC7E;AACF"}
|
package/dist/search/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { A as SearchLedgerHash, C as SearchEdgeOperator, G as SearchSplit, H as SearchProposerKind, I as SearchNodeStatus, L as SearchOpenedEvent, T as SearchEstimateMethod, W as SearchSourceRef, i as SearchArtifactRef, j as SearchModelIdentity, l as SearchCandidateSurface, m as SearchClaim, t as NodeEstimate, y as SearchCloseReason } from "../search-ledger-types-p2xKfiDJ.js";
|
|
2
|
-
import { X as SearchStateView, tt as SearchNode } from "../types-
|
|
3
|
-
import { $ as ExpansionOperator, I as SearchPolicy, R as SearchPolicyView, a as SearchArtifactCodec, at as OperatorYieldSignal, ct as LensSampleSummary, et as MIN_OUTCOMES_FOR_WEIGHT, h as SearchRunResult, i as RunSearchOptions, it as OperatorYieldRow, j as SearchUnitExtension, l as SearchLane, lt as LensSignal, nt as OperatorYieldData, ot as operatorYield, rt as OperatorYieldOptions, st as LensResult, tt as OperatorOutcomeCounts, ut as summarizeSamples } from "../search-kernel-
|
|
2
|
+
import { X as SearchStateView, tt as SearchNode } from "../types-DDKfH-N6.js";
|
|
3
|
+
import { $ as ExpansionOperator, I as SearchPolicy, R as SearchPolicyView, a as SearchArtifactCodec, at as OperatorYieldSignal, ct as LensSampleSummary, et as MIN_OUTCOMES_FOR_WEIGHT, h as SearchRunResult, i as RunSearchOptions, it as OperatorYieldRow, j as SearchUnitExtension, l as SearchLane, lt as LensSignal, nt as OperatorYieldData, ot as operatorYield, rt as OperatorYieldOptions, st as LensResult, tt as OperatorOutcomeCounts, ut as summarizeSamples } from "../search-kernel-C6__V0IS.js";
|
|
4
4
|
import { AgentProfileResourceRef } from "@tangle-network/agent-interface";
|
|
5
5
|
//#region src/search/lenses/edit-credit-contract.d.ts
|
|
6
6
|
interface EditCreditOptions {
|
|
@@ -3,7 +3,7 @@ import { a as RunRecord } from "./run-record-C3awiXPf.js";
|
|
|
3
3
|
import { a as PairedPromotionDecision } from "./paired-promotion-decision-CQMJ5xGN.js";
|
|
4
4
|
import { A as SearchLedgerHash, C as SearchEdgeOperator, E as SearchExecutionIdentity, G as SearchSplit, H as SearchProposerKind, I as SearchNodeStatus, L as SearchOpenedEvent, N as SearchNodeDecision, O as SearchLedgerEntry, P as SearchNodeRef, Q as SearchTaskOutcome, R as SearchOperationKind, S as SearchEdgeAttribution, T as SearchEstimateMethod, U as SearchReservation, V as SearchProposer, W as SearchSourceRef, Y as SearchSurfaceEvidence, Z as SearchTask, _ as SearchClaimPower, a as SearchAttemptAccounting, et as SearchTraceRef, i as SearchArtifactRef, j as SearchModelIdentity, k as SearchLedgerEvent, l as SearchCandidateSurface, m as SearchClaim, nt as EvaluationClaim, o as SearchAudit, p as SearchCellStage, t as NodeEstimate, tt as SearchUnknown, y as SearchCloseReason, z as SearchOperationRecordedEvent } from "./search-ledger-types-p2xKfiDJ.js";
|
|
5
5
|
import { C as LedgerTrustedHead, v as LedgerLineContext, w as LedgerTrustedHeadRemoval, x as LedgerTextStore } from "./index-CalAsXMR.js";
|
|
6
|
-
import { X as SearchStateView, at as SearchUnitScore, j as MutableSurface, rt as SearchScoredCell, tt as SearchNode } from "./types-
|
|
6
|
+
import { X as SearchStateView, at as SearchUnitScore, j as MutableSurface, rt as SearchScoredCell, tt as SearchNode } from "./types-DDKfH-N6.js";
|
|
7
7
|
//#region src/campaign/storage.d.ts
|
|
8
8
|
/**
|
|
9
9
|
* `CampaignStorage` — the filesystem seam `runCampaign` writes through
|
|
@@ -1192,4 +1192,4 @@ declare function searchDivergence(state: SearchStateView, nodeId: string): {
|
|
|
1192
1192
|
declare function runSearch<TArtifact>(options: RunSearchOptions<TArtifact>): Promise<SearchRunResult>;
|
|
1193
1193
|
//#endregion
|
|
1194
1194
|
export { ExpansionOperator as $, SearchRungDecision as A, SearchHistoryReceipt as At, beam as B, openSearchLedger as Bt, searchClaimDecision as C, surfaceNode as Ct, SearchAllocationView as D, SearchHistoryCoverage as Dt, AshaOptions as E, SearchHistoryAuditSummary as Et, SearchExpansion as F, FileSearchLedger as Ft, NodePosterior as G, createRunCostLedger as Gt, incumbent as H, replaySearchLedgerText as Ht, SearchPolicy as I, OpenSearchLedgerOptions as It, estimateNode as J, SEARCH_ESTIMATOR as K, fsCampaignStorage as Kt, SearchPolicyNode as L, SearchLedger as Lt, asha as M, createSearchHistoryReceipt as Mt, uniform as N, verifySearchHistoryArtifact as Nt, SearchAllocator as O, SearchHistoryCoverageRow as Ot, AideOptions as P, verifySearchHistoryReceipt as Pt, searchPosterior as Q, SearchPolicyView as R, SearchLedgerAppendResult as Rt, planSearchClaim as S, surfaceDiff as St, verifySearchClaim as T, SearchHistoryAdmissionOptions as Tt, incumbentWithOperatorBandit as U, validateSearchLedgerEvent as Ut, crowdedFrontierParent as V, parseSearchLedgerLine as Vt, EstimateNodeCellsInput as W, CampaignStorage as Wt, searchCellSetDigest as X, estimateNodeFromCells as Y, searchEstimateMethod as Z, SEARCH_CLAIM_RULE as _, SearchRecorderOptions as _t, SearchArtifactCodec as a, OperatorYieldSignal as at, SearchClaimVerification as b, developmentClaim as bt, SearchExecutor as c, LensSampleSummary as ct, SearchProposalRequest as d, AllocateSearchCellInput as dt, MIN_OUTCOMES_FOR_WEIGHT as et, SearchProposalResult as f, RecordSearchEdgeInput as ft, PlanSearchClaimInput as g, SearchRecorder as gt, SearchRunResult as h, SearchOpening as ht, RunSearchOptions as i, OperatorYieldRow as it, SearchUnitExtension as j, assertSearchHistoryMatchesState as jt, SearchCellPlan as k, SearchHistoryPolicy as kt, SearchLane as l, LensSignal as lt, SearchProposerPort as m, SearchLedgerBinding as mt, searchDivergence as n, OperatorYieldData as nt, SearchCellResult as o, operatorYield as ot, SearchProposedChild as p, RegisterSearchNodeInput as pt, SearchPosterior as q, inMemoryCampaignStorage as qt, searchPolicyView as r, OperatorYieldOptions as rt, SearchCellWork as s, LensResult as st, runSearch as t, OperatorOutcomeCounts as tt, SearchProposalBlob as u, summarizeSamples as ut, SearchClaimDecision as v, SearchRunIdentity as vt, searchClaimReserveUsd as w, CreateSearchHistoryReceiptInput as wt, decideSearchClaim as x, searchEdgeId as xt, SearchClaimPlan as y, SettleSearchCellInput as yt, aide as z, SearchLedgerTrustedHeadMode as zt };
|
|
1195
|
-
//# sourceMappingURL=search-kernel-
|
|
1195
|
+
//# sourceMappingURL=search-kernel-C6__V0IS.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-kernel-
|
|
1
|
+
{"version":3,"file":"search-kernel-C6__V0IS.d.ts","names":[],"sources":["../src/campaign/storage.ts","../src/campaign/search-ledger.ts","../src/campaign/search-history-receipt.ts","../src/campaign/search-ledger-recording.ts","../src/search/lenses/shared.ts","../src/search/lenses/operator-yield.ts","../src/campaign/estimate-node.ts","../src/campaign/search-policy.ts","../src/campaign/allocation.ts","../src/campaign/search-claim.ts","../src/campaign/search-kernel-contract.ts","../src/campaign/search-kernel-policy-view.ts","../src/campaign/search-kernel.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;UAoBiB;;;;WAIN;;EAET,UAAU;;EAEV,OAAO;;EAEP,KAAK;;EAEL,MAAM,cAAc,kBAAkB;;;EAGtC,OAAO,cAAc,iBAAiB;;;;;;;;;iBAUxB,qBAAqB;;;;iBA2CrB,2BAA2B;;iBAoC3B,oBAAoB;EAClC,SAAS;EACT;EACA;;EAEA;IACE;;;;;iBC0ZY,0BAA0B,iBAAiB;;;;;;;;;;;;;;KAqB/C;UAEK;EACf;EACA;EACA,cAAc;;;;;;;EAOd,QAAQ;;UAGO;EACf,OAAO;;EAEP;EACA,OAAO;;UAGQ;WACN;WACA;;WAEA;EACT,OAAO,OAAO,oBAAoB,QAAQ;;EAE1C,SAAS,QAAQ;;EAEjB,eAAe,QAAQ;;EAEvB,kBAAkB,QAAQ;;;EAG1B,oBAAoB,QAAQ;;;;iBAKd,iBAAiB,SAAS,0BAA0B;;;iBAOpD,uBACd,cACA,kBACA,iBACC;;;;;;;;iBAWa,sBACd,cACA,kBACA,SAAS,oBACR;;cAwDU,4BAA4B;WAC9B;WACA;WACA;mBACQ;mBACA;EAMjB,YAAY,cAAc,kBAAkB,cAAa;EAcnD,SAAS,QAAQ;EAIjB,OAAO,OAAO,oBAAoB,QAAQ;EAU1C,eAAe,QAAQ;EAIvB,kBAAkB,QAAQ;EAI1B,oBAAoB,QAAQ;;;;cCntB9B;cACA;;UAGW;WACN;WACA,UAAU;WACV,QAAQ;WACR,aAAa;WACb;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;;;;;;;;;;;UAaM;WACN,sBAAsB;WACtB;WACA,wBAAwB;WACxB,eAAe;;WAEf;;WAEA;;WAEA,QAAQ;;WAER,aAAa;WACb,SAAS;WACT;WACA;;UAGM;WACN;WACA;WACA;aAAmB;aAAuB;;WAC1C,UAAU,KAAK;;KAGd;UAEK;;;EAGf,sBAAsB;;EAEtB;;;;iBAsBc,2BACd,OAAO,kCACN;;;iBAWa,4BACd,SAAS,sBACT,SAAS,KAAK;UAuBC;WACN;WACA;WACA;WACA,kBAAkB;WAClB,UAAU;;WAEV;;UAGM;WACN,QAAQ;WACR;WACA,oBAAoB;;;iBAiBf,2BAA2B,SAAS,uBAAuB;;iBA+B3D,gCACd,SAAS,sBACT,OAAO;;;;UC9HQ;;EAEf,OAAO;;EAEP,UAAU;;EAEV,QAAQ;;EAER,OAAO;;EAEP;;EAEA,QAAQ;;;EAGR,QAAQ;;;UAIO;EACf,QAAQ;EACR,UAAU;;;iBAII,iBAAiB,uBAAuB;;KAW5C,gBAAgB,KAC1B;EAGA;IACE,OAAO;IACP,WAAW;IACX,MAAM;IACN;;;UAIa;EACf,QAAQ;;EAER;EACA,UAAU;;EAEV;;UAGe;EACf;EACA,UAAU;EACV,UAAU;;UAGK;EACf;;EAEA,SAAS,eAAe;EACxB,UAAU;EACV,aAAa;EACb,UAAU;;;EAGV;EACA;IAAc;IAAc,UAAU;;;EAEtC,oBAAoB;;EAEpB,OAAO,MAAM,oBAAoB;EACjC;;UAGe;EACf;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,cAAc;;UAGC;EACf;;EAEA;EACA,SAAS;EACT,YAAY;EACZ,UAAU;EACV;EACA;EACA;EACA;IAAc;IAAc;;EAC5B,kBAAkB;EAClB,WAAW;;EAEX,YAAY;EACZ,YAAY;;;iBAIE,aAAa,kBAAkB,qBAAqB;;cAQvD;WACF,QAAQ;mBACA;mBACA;mBACA;mBACA;UACT;UAED;;;;;SAuBM,KACX,SAAS,uBACT,SAAS,eACT,YAAW,sBACV,QAAQ;MAuCP;;EAKJ,SAAS,QAAQ;;EAKjB,KAAK,cAAc,gBAAgB;;;;;;EAsBnC,SAAS,KAAK;;EAkBR,aACJ,OAAO,0BACN;IAAU;IAAgB;;;EAmBvB,WAAW,OAAO,wBAAwB;EAmC1C,eAAe;IACnB;IACA,eAAe;IACf,cAAc;;;IAGd,YAAY;MACV;EAcE,gBACJ,OAAO,KAAK;IACV,YAAY;MAEb;;EAaG,aAAa,OAAO,0BAA0B;;EAyB9C,WAAW,OAAO,wBAAwB;EA2C1C,WAAW;IACf;IACA;MACE;;EAgBE,WAAW;IACf;IACA,UAAU;IACV,QAAQ;IACR;IACA;MACE;;;EAqBE,MAAM;IAAS,QAAQ;IAAmB,OAAO;MAAuB;;EAc9E,QAAQ;IAAS;IAAoB;MAAkB;UAIzC;;UAKN;;;iBAQM,YACd,UAAU,gBACV,SAAS,iBACR;;iBAUa,YACd,UAAU,gBACV,QAAQ,gBACR,OAAO,iBACN;;;;;;UCxhBc,WAAW;EAC1B;EACA,OAAO;;UAGQ,WAAW,OAAO;EACjC,MAAM;EACN,QAAQ,WAAW;;;;;;;;;;UAWJ;EACf;EACA;EACA;EACA;;;;;EAKA;;iBAGc,iBAAiB,4BAA4B,gBAAgB;;;;;;cCVhE;KACD,4BAA4B;;;;;cAM3B;UAKI;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,UAAU;;EAEV;;EAEA;;EAEA,UAAU;;;;EAIV;EACA,OAAO;;UAGQ;EACf;;EAEA;EACA,MAAM;;UAGS;EACf;;;;;KAMU,sBAAsB,OAAO;iBAEzB,cACd,OAAO,iBACP,UAAS;EACN,MAAM;EAAmB;IAAU;IAAc,OAAO;;;;;;cCvChD,kBAAkB;;;iBAOf,qBAAqB,gBAAgB;UAOpC;EACf;EACA;EACA,OAAO;EACP;;EAEA,oBAAoB;;EAEpB,uBAAuB;;;;;;;iBAQT,oBACd,OAAO,KAAK,uCACX;;;;;;;;;;;;;;;;;;;iBA+Ba,sBAAsB,OAAO,yBAAyB;;;;;;iBAiDtD,aACd,OAAO,iBACP,gBACA;EAAW;EAAiB,OAAO;IAClC;;UAmBc;EACf;;EAEA;;;EAGA;;EAEA;;UAGe;EACf,OAAO;EACP;;;;;;EAMA;;EAEA;;EAEA,OAAO;;;;;;;;;iBAUO,gBACd,OAAO,iBACP;EAAW,QAAQ;IAClB;;;;UClNc;WACN;WACA;WACA;;;WAGA;WACA;;WAEA;;WAEA;;WAEA;;;;EAIT,SAAS;;EAET,WAAW,0BAA0B;;EAErC,SAAS,gBAAgB,kBAAkB;;;EAG3C,kBAAkB;;;WAGT,WAAW;;;;;WAKX,iBAAiB;;;UAIX;WACN;WACA;;WAEA;;WAEA,UAAU;;;WAGV;;WAEA;;WAEA,QAAQ;;WAER;;;;WAIA;;;UAIM;EACf;EACA,UAAU,QAAQ;;EAElB;IAAa;IAAc,UAAU;;;UAGtB;;WAEN;;WAEA;;EAET,OAAO,MAAM,mBAAmB;;EAEhC,OAAO,MAAM;;;;;;;;;;;;;;iBAeC,UACd;EAAW;EAAmB;IAC7B;;;;;;;;;;;;;;;;;;;;;;;iBA+Ca,4BACd;EACE;EACA;EACA,eAAe,QAAQ,OAAO;IAE/B;;;;;;;;;;iBAyEa,sBAAsB;EAAW;IAAiB;UA4BjD;;;EAGf;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoCc,KAAK,UAAS,cAAmB;;;;;;;;;;;iBAmIjC,KAAK;EAAW;IAAkB;;;;UCxajC;EACf;EACA,OAAO;EACP;EACA,OAAO;;;UAIQ;WACN,OAAO;;;EAGhB,KAAK;;;UAIU;EACf;EACA;IAAY;IAAoB;;IAAmB;;;EAEnD,OAAO;EACP;EACA;;UAGe;;WAEN;;WAEA;;;EAGT,KAAK,OAAO,iBAAiB,gBAAgB,eAAe;;;EAG5D,WAAW,OAAO;;;EAGlB,QAAQ,MAAM,uBAAuB;;;EAGrC,MAAM,MAAM,sBAAsB,eAAe;;;;;;;;;iBAUnC,QAAQ;EAAW;IAAuB;UA8BzC;;;;EAIf;;;EAGA;;;EAGA;;EAEA;;;EAGA,SAAS;;;;;;;;;;UAWM;;WAEN;;;;;;EAMT,OAAO;IACL,OAAO;IACP;IACA;IACA;;IAEA;IACA;IACA;;;;;;;;;;;;;;;;;;;;;iBAsBY,KAAK,UAAS,cAAmB;;;;cCtFpC,mBAAmB;;;;;;;iBAchB,sBAAsB;EACpC;EACA;EACA;;;;;;UASe;EACf;EACA,MAAM;;EAEN,YAAY;;EAEZ,WAAW;IAAQ;IAAgB;;EACnC,OAAO;;;EAGP;EACA;;UAGe;EACf,YAAY;;EAEZ;;EAEA,WAAW;;;iBAIG,gBACd,OAAO,iBACP,OAAO,uBACN;;UAqGc;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;;;;;;iBAQc,kBACd,OAAO,iBACP,MAAM;EACH,OAAO;EAAa,WAAW;;;KA4HxB;EACN;;;;EAEA;EAAoB;;;;;EAGpB;EAAmB;;;;;;;;;;;;;;;;;;;iBAmBT,kBAAkB,OAAO,kBAAkB;;;;;;;;;;;;;;iBAiF3C,oBACd,OAAO,iBACP,iBACC;;;;UCtZc;EACf;;EAEA;;;EAGA;;;EAGA;;;UAIe,eAAe;EAC9B;EACA;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA;;EAEA;EACA,UAAU;EACV,QAAQ;;;;;KAME,mBAAmB,KAAK;;;UAInB,eAAe;EAC9B,kBAAkB;;EAElB,MAAM,MAAM;IAAmB;;EAC/B,IAAI,MAAM,eAAe,aAAa,QAAQ;;;;EAI9C,MAAM,MAAM,eAAe,aAAa,QAAQ;;;UAIjC,oBAAoB;;EAEnC,KAAK,UAAU,gBAAgB,UAAU,YAAY;;EAErD,KACE,UAAU,gBACV,QAAQ,WACR,OAAO,YACN,oBAAoB;;EAEvB,KAAK,UAAU,gBAAgB,MAAM,aAAa;;UAGnC,sBAAsB;EACrC;EACA;EACA,UAAU,QAAQ;;EAElB,SAAS;IAAgB;IAAgB,UAAU;;;EAEnD;EACA,QAAQ;;;UAIO,oBAAoB;EACnC,UAAU;EACV;;EAEA;;;EAGA,cAAc,SAAS;;UAGR,qBAAqB;EACpC,UAAU,cAAc,oBAAoB;;EAE5C;;EAEA,YAAY;EACZ,YAAY;;;UAIG,mBAAmB;WACzB;WACA,MAAM;WACN,QAAQ;WACR,WAAW;;WAEX;;;WAGA;EACT,QAAQ,SAAS,sBAAsB,aAAa,QAAQ,qBAAqB;;UAGlE,iBAAiB;;EAEhC,UAAU;;EAEV,MAAM;EACN,OAAO,oBAAoB;EAC3B,QAAQ;EACR,YAAY;EACZ,UAAU,mBAAmB;EAC7B,UAAU,eAAe;;;EAGzB,SAAS,UAAU;;EAEnB;;EAEA;;;;EAIA,SAAS;;EAET;;UAGe;;EAEf,OAAO;;;EAGP;EACA,QAAQ;;EAER,OAAO;;;;EAIP,mBAAmB;;;UAIJ;EACf;EACA,UAAU;EACV;EACA;IAAa;IAAc,UAAU;;EACrC,UAAU;IACR,MAAM;IACN,OAAO,MAAM,oBAAoB;IACjC;IACA;IACA,aAAa,SAAS;IACtB;;;;;;;;;;;;;iBChOY,iBACd,OAAO,iBACP;EACE;EACA;EACA;EACA;IAED;;;;;;;;iBA4Fa,iBACd,OAAO,iBACP;EACG,OAAO;EAAc;;;;;iBCnEJ,UAAU,WAC9B,SAAS,iBAAiB,aACzB,QAAQ"}
|
package/dist/{skillopt-optimization-method-DLKqdhcR.js → skillopt-optimization-method-D21dH_Bh.js}
RENAMED
|
@@ -3,17 +3,17 @@ import { t as mean$1 } from "./mean-BbA_JCzW.js";
|
|
|
3
3
|
import { t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
|
|
4
4
|
import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
|
|
5
5
|
import { n as contentHash, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
|
|
6
|
-
import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-
|
|
6
|
+
import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-3uAeRawE.js";
|
|
7
7
|
import { D as surfaceContentHash, I as assertCompleteCampaign, P as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-DunzcJGv.js";
|
|
8
8
|
import { a as mapConcurrent } from "./journal-C7om8Gum.js";
|
|
9
9
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
10
|
-
import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-
|
|
10
|
+
import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-DA02c_L6.js";
|
|
11
11
|
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
|
|
12
12
|
import { n as openSearchLedger } from "./search-ledger-CZRqtFbW.js";
|
|
13
13
|
import { t as acquireSingleRunLock } from "./single-run-lock-Dekd8_Wv.js";
|
|
14
14
|
import { n as isRecord } from "./json-BhdsJaOQ.js";
|
|
15
15
|
import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-DXuOTseU.js";
|
|
16
|
-
import { t as startExternalOptimizerCallback } from "./external-optimizer-process-
|
|
16
|
+
import { t as startExternalOptimizerCallback } from "./external-optimizer-process-E5yY2SYS.js";
|
|
17
17
|
import { createHash, randomBytes } from "node:crypto";
|
|
18
18
|
import { mkdir } from "node:fs/promises";
|
|
19
19
|
import { isDeepStrictEqual } from "node:util";
|
|
@@ -3245,4 +3245,4 @@ async function finishSkillOptRun(context, result, outputDir, modelProxy) {
|
|
|
3245
3245
|
//#endregion
|
|
3246
3246
|
export { importExternalEvaluations as a, composeGate as c, readExternalOptimizerObservationArtifact as d, externalSurface as i, externalTextOptimizationMethod as l, compareOptimizationMethods as n, importGepaPopulation as o, gepaOptimizationMethod as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
|
|
3247
3247
|
|
|
3248
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
3248
|
+
//# sourceMappingURL=skillopt-optimization-method-D21dH_Bh.js.map
|