@tangle-network/agent-eval 0.220.2 → 0.221.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts → agent-profile-Bc7ADJY4.d.ts} +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts.map → agent-profile-Bc7ADJY4.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +77 -78
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +204 -25
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DqPJaJ4X.js → analyze-runs-CHmsykNJ.js} +4 -4
- package/dist/{analyze-runs-DqPJaJ4X.js.map → analyze-runs-CHmsykNJ.js.map} +1 -1
- package/dist/{analyze-runs-BWAHBDAw.d.ts → analyze-runs-D7In4c3z.d.ts} +2 -2
- package/dist/{analyze-runs-BWAHBDAw.d.ts.map → analyze-runs-D7In4c3z.d.ts.map} +1 -1
- package/dist/{benchmark-command-Dx0E4a6z.js → benchmark-command-8XQfpUK3.js} +13 -11
- package/dist/benchmark-command-8XQfpUK3.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/calibration-CTwxpJ-t.js +197 -0
- package/dist/calibration-CTwxpJ-t.js.map +1 -0
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-BVPC6ODQ.js → campaign-Bt6rbWIU.js} +9 -8
- package/dist/{campaign-BVPC6ODQ.js.map → campaign-Bt6rbWIU.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{completion-verifier-Cigap8CG.js → completion-verifier-YwoeujPl.js} +2 -2
- package/dist/{completion-verifier-Cigap8CG.js.map → completion-verifier-YwoeujPl.js.map} +1 -1
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +7 -7
- package/dist/decision-point-Co3u3W8k.js +831 -0
- package/dist/decision-point-Co3u3W8k.js.map +1 -0
- package/dist/default-registry-BMHqNR6_.d.ts +257 -0
- package/dist/default-registry-BMHqNR6_.d.ts.map +1 -0
- package/dist/{default-registry-ovrIBGmH.js → default-registry-BbFCkqvB.js} +607 -14
- package/dist/default-registry-BbFCkqvB.js.map +1 -0
- package/dist/{define-agent-eval-C2nFhHPu.d.ts → define-agent-eval-Bsoi2GUV.d.ts} +7 -7
- package/dist/{define-agent-eval-C2nFhHPu.d.ts.map → define-agent-eval-Bsoi2GUV.d.ts.map} +1 -1
- package/dist/{define-agent-eval-UQY3VbJ2.js → define-agent-eval-CJhECJNL.js} +4 -4
- package/dist/{define-agent-eval-UQY3VbJ2.js.map → define-agent-eval-CJhECJNL.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js → dspy-rlm-engine-BTy7u0jQ.js} +3 -3
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js.map → dspy-rlm-engine-BTy7u0jQ.js.map} +1 -1
- package/dist/{engine-BpGGw6cd.d.ts → engine-BF95CWzH.d.ts} +2 -2
- package/dist/{engine-BpGGw6cd.d.ts.map → engine-BF95CWzH.d.ts.map} +1 -1
- package/dist/{eval-campaign-BUMIxKTE.js → eval-campaign-DoX5vz3E.js} +2 -2
- package/dist/{eval-campaign-BUMIxKTE.js.map → eval-campaign-DoX5vz3E.js.map} +1 -1
- package/dist/{evaluation-CBdQt_kr.d.ts → evaluation-D1PGzkPR.d.ts} +3 -3
- package/dist/{evaluation-CBdQt_kr.d.ts.map → evaluation-D1PGzkPR.d.ts.map} +1 -1
- package/dist/evaluation-wdGyZ7jV.js +147 -0
- package/dist/evaluation-wdGyZ7jV.js.map +1 -0
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -136
- package/dist/{exact-types-BYFmYq44.d.ts → exact-types-CVn1QxFO.d.ts} +2 -2
- package/dist/{exact-types-BYFmYq44.d.ts.map → exact-types-CVn1QxFO.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js → external-optimizer-process-E5yY2SYS.js} +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js.map → external-optimizer-process-E5yY2SYS.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-D670Oxf2.js → external-optimizer-subprocess-DA02c_L6.js} +12 -19
- package/dist/{external-optimizer-subprocess-D670Oxf2.js.map → external-optimizer-subprocess-DA02c_L6.js.map} +1 -1
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts → feedback-trajectory-BZhVCWuz.d.ts} +3 -3
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts.map → feedback-trajectory-BZhVCWuz.d.ts.map} +1 -1
- package/dist/finding-composition-8n0fFgVI.js +131 -0
- package/dist/finding-composition-8n0fFgVI.js.map +1 -0
- package/dist/{findings-store-CvNo712L.js → findings-store-DRiFdoid.js} +4 -4
- package/dist/{findings-store-CvNo712L.js.map → findings-store-DRiFdoid.js.map} +1 -1
- package/dist/{index-CVFStrQD.d.ts → index-BJZ7s-sv.d.ts} +7 -58
- package/dist/index-BJZ7s-sv.d.ts.map +1 -0
- package/dist/{index-Q6DvdcwI.d.ts → index-BvX3CFNS.d.ts} +6 -6
- package/dist/{index-Q6DvdcwI.d.ts.map → index-BvX3CFNS.d.ts.map} +1 -1
- package/dist/index-DWd1gUwR.d.ts +1086 -0
- package/dist/index-DWd1gUwR.d.ts.map +1 -0
- package/dist/{index-DAXfpVf7.d.ts → index-y7i8l6Y_.d.ts} +8 -8
- package/dist/{index-DAXfpVf7.d.ts.map → index-y7i8l6Y_.d.ts.map} +1 -1
- package/dist/index.d.ts +18 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -15
- package/dist/{kind-factory-BhxbtUsQ.js → kind-factory-Dl0PmNQL.js} +2 -2
- package/dist/{kind-factory-BhxbtUsQ.js.map → kind-factory-Dl0PmNQL.js.map} +1 -1
- package/dist/{llm-judge-Cc8vDJh_.js → llm-judge-3uAeRawE.js} +4 -4
- package/dist/{llm-judge-Cc8vDJh_.js.map → llm-judge-3uAeRawE.js.map} +1 -1
- package/dist/{matrix-Cf-AGxGh.d.ts → matrix-BXo3PSqM.d.ts} +2 -2
- package/dist/{matrix-Cf-AGxGh.d.ts.map → matrix-BXo3PSqM.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -638
- package/dist/meta-eval/index.js +3 -193
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/outcome-observations-FLmRpgYa.js +78 -0
- package/dist/outcome-observations-FLmRpgYa.js.map +1 -0
- package/dist/{paired-arms-Ct4LTKc1.js → paired-arms-B2ttLc0_.js} +108 -31
- package/dist/paired-arms-B2ttLc0_.js.map +1 -0
- package/dist/{power-and-mde-N5nU4_Ly.js → power-and-mde-CQX-0weG.js} +2 -2
- package/dist/{power-and-mde-N5nU4_Ly.js.map → power-and-mde-CQX-0weG.js.map} +1 -1
- package/dist/{prime-protocol-CckHBxGu.d.ts → prime-protocol-BEaqBoQE.d.ts} +2 -2
- package/dist/{prime-protocol-CckHBxGu.d.ts.map → prime-protocol-BEaqBoQE.d.ts.map} +1 -1
- package/dist/{proposal-findings-E1aSC_cU.js → proposal-findings-D0_zkQ2l.js} +2 -2
- package/dist/{proposal-findings-E1aSC_cU.js.map → proposal-findings-D0_zkQ2l.js.map} +1 -1
- package/dist/{registry-B_yG-1jh.d.ts → registry-DDKaUrmM.d.ts} +3 -3
- package/dist/{registry-B_yG-1jh.d.ts.map → registry-DDKaUrmM.d.ts.map} +1 -1
- package/dist/reporting.d.ts +1 -1
- package/dist/reporting.js +3 -3
- package/dist/rl.d.ts +4 -5
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +5 -4
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-noY7uE03.d.ts → rubric-predictive-validity-Cd5SCtAf.d.ts} +58 -3
- package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-D3_oI72N.js → rubric-predictive-validity-CeKW2GQN.js} +3 -77
- package/dist/rubric-predictive-validity-CeKW2GQN.js.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts → search-kernel-C6__V0IS.d.ts} +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts.map → search-kernel-C6__V0IS.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-DLKqdhcR.js → skillopt-optimization-method-D21dH_Bh.js} +4 -4
- package/dist/{skillopt-optimization-method-DLKqdhcR.js.map → skillopt-optimization-method-D21dH_Bh.js.map} +1 -1
- package/dist/{statistical-heldout-3z3I-kJN.d.ts → statistical-heldout-pD1Yah-0.d.ts} +58 -8
- package/dist/statistical-heldout-pD1Yah-0.d.ts.map +1 -0
- package/dist/{store-otlp-dyWE3wAG.js → store-otlp-BVZfR168.js} +2 -2
- package/dist/{store-otlp-dyWE3wAG.js.map → store-otlp-BVZfR168.js.map} +1 -1
- package/dist/{store-tool-spans-DIUgZbzu.js → store-tool-spans-CjYiF4fF.js} +3 -3
- package/dist/{store-tool-spans-DIUgZbzu.js.map → store-tool-spans-CjYiF4fF.js.map} +1 -1
- package/dist/{store-tool-spans-DPwXHQkH.d.ts → store-tool-spans-DfnwWMFR.d.ts} +4 -4
- package/dist/{store-tool-spans-DPwXHQkH.d.ts.map → store-tool-spans-DfnwWMFR.d.ts.map} +1 -1
- package/dist/{summary-report-BsXJoYX5.js → summary-report-DHDWxe6p.js} +3 -3
- package/dist/{summary-report-BsXJoYX5.js.map → summary-report-DHDWxe6p.js.map} +1 -1
- package/dist/systemone-D3MrW63C.d.ts +400 -0
- package/dist/systemone-D3MrW63C.d.ts.map +1 -0
- package/dist/{systemone-protocol-DKoi1NdK.d.ts → systemone-protocol-Bl2EI2KQ.d.ts} +4 -2
- package/dist/{systemone-protocol-DKoi1NdK.d.ts.map → systemone-protocol-Bl2EI2KQ.d.ts.map} +1 -1
- package/dist/systemone-protocol.d.ts +2 -2
- package/dist/systemone-protocol.js +7 -7
- package/dist/systemone-protocol.js.map +1 -1
- package/dist/{systemone-review-CUBMuTLi.d.ts → systemone-review-C8DRiT_-.d.ts} +3 -3
- package/dist/{systemone-review-CUBMuTLi.d.ts.map → systemone-review-C8DRiT_-.d.ts.map} +1 -1
- package/dist/{systemone-review-DzdOJ8I6.js → systemone-review-DFGtIUcf.js} +2 -2
- package/dist/{systemone-review-DzdOJ8I6.js.map → systemone-review-DFGtIUcf.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -343
- package/dist/{tool-groups-LISbBWCg.d.ts → tool-groups-DK1bu5TD.d.ts} +3 -3
- package/dist/tool-groups-DK1bu5TD.d.ts.map +1 -0
- package/dist/{trace-questions-C42fYEE4.d.ts → trace-questions-D_OKYZ6u.d.ts} +3 -3
- package/dist/{trace-questions-C42fYEE4.d.ts.map → trace-questions-D_OKYZ6u.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +3 -3
- package/dist/{types-DqFMNTgK.js → types-CnWmj8NW.js} +19 -6
- package/dist/types-CnWmj8NW.js.map +1 -0
- package/dist/{types-CHfn1zg5.d.ts → types-DDKfH-N6.d.ts} +2 -2
- package/dist/{types-CHfn1zg5.d.ts.map → types-DDKfH-N6.d.ts.map} +1 -1
- package/dist/{types-8yQzNm8u.d.ts → types-DGImvHIr.d.ts} +150 -4
- package/dist/types-DGImvHIr.d.ts.map +1 -0
- package/dist/{types-D8GXK9Z5.d.ts → types-IERw_HxR.d.ts} +2 -2
- package/dist/{types-D8GXK9Z5.d.ts.map → types-IERw_HxR.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/behavioral-trace-questions.md +99 -0
- package/docs/experiment.md +27 -0
- package/docs/finding-layers.md +54 -1
- package/docs/systemone.md +107 -0
- package/package.json +1 -1
- package/dist/benchmark-command-Dx0E4a6z.js.map +0 -1
- package/dist/default-registry-Cwg_QN3d.d.ts +0 -70
- package/dist/default-registry-Cwg_QN3d.d.ts.map +0 -1
- package/dist/default-registry-ovrIBGmH.js.map +0 -1
- package/dist/evaluation.js.map +0 -1
- package/dist/index-CVFStrQD.d.ts.map +0 -1
- package/dist/judge-gate-Dc_mXQsL.d.ts +0 -453
- package/dist/judge-gate-Dc_mXQsL.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/outcome-store-CNt4iZ67.d.ts +0 -58
- package/dist/outcome-store-CNt4iZ67.d.ts.map +0 -1
- package/dist/paired-arms-Ct4LTKc1.js.map +0 -1
- package/dist/rubric-predictive-validity-D3_oI72N.js.map +0 -1
- package/dist/rubric-predictive-validity-noY7uE03.d.ts.map +0 -1
- package/dist/statistical-heldout-3z3I-kJN.d.ts.map +0 -1
- package/dist/systemone-DVA7t76V.d.ts +0 -135
- package/dist/systemone-DVA7t76V.d.ts.map +0 -1
- package/dist/systemone.js.map +0 -1
- package/dist/tool-groups-LISbBWCg.d.ts.map +0 -1
- package/dist/types-8yQzNm8u.d.ts.map +0 -1
- package/dist/types-DqFMNTgK.js.map +0 -1
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
import { W as TraceAnalysisStore, n as AnalystContext } from "./types-8yQzNm8u.js";
|
|
2
|
-
import { l as ExactCapableAnalyst } from "./exact-types-BYFmYq44.js";
|
|
3
|
-
import { i as TraceAnalystLimits, r as TraceAnalysisEngineResult, s as RawAnalystFinding, t as TraceAnalysisEngine } from "./engine-BpGGw6cd.js";
|
|
4
|
-
import { t as TraceToolGroupName } from "./tool-groups-LISbBWCg.js";
|
|
5
|
-
import { i as AnalystRegistryOptions, t as AnalystRegistry } from "./registry-B_yG-1jh.js";
|
|
6
|
-
//#region src/analyst/behavioral-analyst.d.ts
|
|
7
|
-
interface BehavioralAnalystOptions {
|
|
8
|
-
/** Refuse larger unfiltered datasets instead of sampling them silently. Default 1,000. */
|
|
9
|
-
maxTraces?: number;
|
|
10
|
-
/** Evidence locations retained per finding. Counts still cover every analyzed trace. Default 20. */
|
|
11
|
-
maxEvidenceRefsPerFinding?: number;
|
|
12
|
-
}
|
|
13
|
-
/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
|
|
14
|
-
declare function behavioralAnalyst(options?: BehavioralAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
|
|
15
|
-
//#endregion
|
|
16
|
-
//#region src/analyst/kind-factory.d.ts
|
|
17
|
-
/** A named research question independent of the recursive engine that runs it. */
|
|
18
|
-
interface TraceAnalystDefinition {
|
|
19
|
-
id: string;
|
|
20
|
-
description: string;
|
|
21
|
-
area: string;
|
|
22
|
-
version: string;
|
|
23
|
-
/** User-facing question. A function may derive it from run context. */
|
|
24
|
-
question?: string | ((context: AnalystContext) => string);
|
|
25
|
-
/** Research policy, evidence rules, and domain-specific instructions. */
|
|
26
|
-
instructions: string;
|
|
27
|
-
/** Smallest trace-tool set that can answer the question. */
|
|
28
|
-
toolGroup: TraceToolGroupName;
|
|
29
|
-
/** Optional bounded context assembled deterministically before the investigation. */
|
|
30
|
-
prepareContext?: (store: TraceAnalysisStore, context: AnalystContext) => string | undefined | Promise<string | undefined>;
|
|
31
|
-
limits?: Partial<TraceAnalystLimits>;
|
|
32
|
-
postProcess?: (row: RawAnalystFinding, context: AnalystContext) => RawAnalystFinding | null;
|
|
33
|
-
minimumEvidenceCitations?: number;
|
|
34
|
-
requireStructuredFindings?: boolean;
|
|
35
|
-
}
|
|
36
|
-
interface CreateTraceAnalystOptions {
|
|
37
|
-
engine: TraceAnalysisEngine;
|
|
38
|
-
/** Override the definition's `version` (e.g. when an optimizer has fitted new instructions). */
|
|
39
|
-
versionSuffix?: string;
|
|
40
|
-
settlementTimeoutMs?: number;
|
|
41
|
-
}
|
|
42
|
-
/** Run one definition and retain its answer, findings, and full investigation record. */
|
|
43
|
-
declare function runTraceAnalyst(args: {
|
|
44
|
-
definition: TraceAnalystDefinition;
|
|
45
|
-
engine: TraceAnalysisEngine;
|
|
46
|
-
store: TraceAnalysisStore;
|
|
47
|
-
context: AnalystContext;
|
|
48
|
-
settlementTimeoutMs?: number;
|
|
49
|
-
}): Promise<TraceAnalysisEngineResult>;
|
|
50
|
-
/** Adapt a research definition to the common Analyst registry contract. */
|
|
51
|
-
declare function createTraceAnalyst(definition: TraceAnalystDefinition, options: CreateTraceAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
|
|
52
|
-
//#endregion
|
|
53
|
-
//#region src/analyst/default-registry.d.ts
|
|
54
|
-
interface DefaultAnalystRegistryOptions {
|
|
55
|
-
/**
|
|
56
|
-
* Recursive engine for model-backed trace analysts. When omitted, the
|
|
57
|
-
* registry contains only deterministic analysts. The engine's id, version,
|
|
58
|
-
* and model become the exact-run identity of every analyst it executes.
|
|
59
|
-
*/
|
|
60
|
-
engine?: TraceAnalysisEngine;
|
|
61
|
-
definitions?: readonly TraceAnalystDefinition[];
|
|
62
|
-
/** Set false to omit the deterministic behavioral analyst (default: include). */
|
|
63
|
-
includeBehavioral?: boolean;
|
|
64
|
-
behavioral?: BehavioralAnalystOptions;
|
|
65
|
-
registry?: AnalystRegistryOptions;
|
|
66
|
-
}
|
|
67
|
-
declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
|
|
68
|
-
//#endregion
|
|
69
|
-
export { createTraceAnalyst as a, behavioralAnalyst as c, TraceAnalystDefinition as i, buildDefaultAnalystRegistry as n, runTraceAnalyst as o, CreateTraceAnalystOptions as r, BehavioralAnalystOptions as s, DefaultAnalystRegistryOptions as t };
|
|
70
|
-
//# sourceMappingURL=default-registry-Cwg_QN3d.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"default-registry-Cwg_QN3d.d.ts","names":[],"sources":["../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;UA+BiB;;EAEf;;EAEA;;;iBAoGc,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgHI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;;;UCrKN;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
|