@tangle-network/agent-eval 0.148.0 → 0.150.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/README.md +5 -1
- package/dist/analyst/index.d.ts +14 -14
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +4 -4
- package/dist/{backend-integrity-Bz8nSrRE.d.ts → backend-integrity-DaeroaEs.d.ts} +2 -2
- package/dist/{backend-integrity-Bz8nSrRE.d.ts.map → backend-integrity-DaeroaEs.d.ts.map} +1 -1
- package/dist/{benchmark-Cn7ZMFNW.d.ts → benchmark-A0E-JZ1H.d.ts} +3 -3
- package/dist/{benchmark-Cn7ZMFNW.d.ts.map → benchmark-A0E-JZ1H.d.ts.map} +1 -1
- package/dist/{benchmark-command-Cz4fDmsk.js → benchmark-command-CqGA0SJQ.js} +8 -7
- package/dist/benchmark-command-CqGA0SJQ.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +9 -9
- package/dist/campaign/index.js +6 -6
- package/dist/{campaign-BcfXzmPM.js → campaign-CR6Yt5ZS.js} +80 -7
- package/dist/campaign-CR6Yt5ZS.js.map +1 -0
- package/dist/{capture-fetch-BAx3Gntt.d.ts → capture-fetch-Im0UN1i6.d.ts} +2 -2
- package/dist/{capture-fetch-BAx3Gntt.d.ts.map → capture-fetch-Im0UN1i6.d.ts.map} +1 -1
- package/dist/{chat-client-BX8Wh3fn.js → chat-client-sFiClniQ.js} +2 -2
- package/dist/{chat-client-BX8Wh3fn.js.map → chat-client-sFiClniQ.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{client-BV7icIfy.d.ts → client-w6vVJhSp.d.ts} +2 -2
- package/dist/{client-BV7icIfy.d.ts.map → client-w6vVJhSp.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +5 -5
- package/dist/{default-registry-BUtRMRKk.d.ts → default-registry-BhRNsP-D.d.ts} +6 -6
- package/dist/{default-registry-BUtRMRKk.d.ts.map → default-registry-BhRNsP-D.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CvZQW4u9.js → define-agent-eval-ChrtErmg.js} +3 -3
- package/dist/define-agent-eval-ChrtErmg.js.map +1 -0
- package/dist/{define-agent-eval-hXLUBKtb.d.ts → define-agent-eval-Cw3mkB4-.d.ts} +6 -7
- package/dist/define-agent-eval-Cw3mkB4-.d.ts.map +1 -0
- package/dist/{dspy-rlm-engine-BmtOl_kP.js → dspy-rlm-engine-Csi3CUAT.js} +2 -2
- package/dist/{dspy-rlm-engine-BmtOl_kP.js.map → dspy-rlm-engine-Csi3CUAT.js.map} +1 -1
- package/dist/{engine-CIy18RTX.d.ts → engine-CgQdUh-r.d.ts} +4 -4
- package/dist/{engine-CIy18RTX.d.ts.map → engine-CgQdUh-r.d.ts.map} +1 -1
- package/dist/{eval-campaign-bmZ6NIIP.js → eval-campaign-Czc2TcW5.js} +2 -2
- package/dist/{eval-campaign-bmZ6NIIP.js.map → eval-campaign-Czc2TcW5.js.map} +1 -1
- package/dist/{exact-types-rdKFzEnK.d.ts → exact-types-D9_T83Ol.d.ts} +2 -2
- package/dist/{exact-types-rdKFzEnK.d.ts.map → exact-types-D9_T83Ol.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +73 -4
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +175 -1
- package/dist/experiment/index.js.map +1 -1
- package/dist/{experiment-tracker-D9VHwRm-.d.ts → experiment-tracker-Cp9ougHs.d.ts} +2 -2
- package/dist/{experiment-tracker-D9VHwRm-.d.ts.map → experiment-tracker-Cp9ougHs.d.ts.map} +1 -1
- package/dist/{external-optimizer-contracts-CSjDLLmr.d.ts → external-optimizer-contracts-mbhR9Ldu.d.ts} +13 -6
- package/dist/{external-optimizer-contracts-CSjDLLmr.d.ts.map → external-optimizer-contracts-mbhR9Ldu.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-C4AyCcQd.js → external-optimizer-process-tDe72VRO.js} +4 -4
- package/dist/{external-optimizer-process-C4AyCcQd.js.map → external-optimizer-process-tDe72VRO.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DU1KM8yV.js → external-optimizer-subprocess-BepyNwtj.js} +341 -25
- package/dist/external-optimizer-subprocess-BepyNwtj.js.map +1 -0
- package/dist/{feedback-trajectory-niGeR6uJ.d.ts → feedback-trajectory-Cc9M2tNF.d.ts} +3 -3
- package/dist/{feedback-trajectory-niGeR6uJ.d.ts.map → feedback-trajectory-Cc9M2tNF.d.ts.map} +1 -1
- package/dist/fuzz.d.ts +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-gtjtfLEJ.d.ts → index-DL4MJuJL.d.ts} +8 -8
- package/dist/{index-gtjtfLEJ.d.ts.map → index-DL4MJuJL.d.ts.map} +1 -1
- package/dist/index-DjpxROrT.d.ts +1 -0
- package/dist/{index-Dzn8Q3C2.d.ts → index-dh0kp2GM.d.ts} +42 -11
- package/dist/index-dh0kp2GM.d.ts.map +1 -0
- package/dist/index.d.ts +21 -21
- package/dist/index.js +13 -20
- package/dist/index.js.map +1 -1
- package/dist/{integrity-CGfpTE5-.d.ts → integrity-D2MAlC_K.d.ts} +2 -2
- package/dist/{integrity-CGfpTE5-.d.ts.map → integrity-D2MAlC_K.d.ts.map} +1 -1
- package/dist/{llm-client-BMuxYoZy.js → llm-client-CxyJ8Oft.js} +15 -6
- package/dist/llm-client-CxyJ8Oft.js.map +1 -0
- package/dist/{llm-judge-DWq1Ptco.js → llm-judge-D7UuMI2t.js} +4 -3
- package/dist/llm-judge-D7UuMI2t.js.map +1 -0
- package/dist/{matrix-Bc111FKv.d.ts → matrix-D3hL68uV.d.ts} +2 -2
- package/dist/{matrix-Bc111FKv.d.ts.map → matrix-D3hL68uV.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/{multi-layer-verifier-DIguZc8Z.d.ts → multi-layer-verifier-BUaQ4C17.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DIguZc8Z.d.ts.map → multi-layer-verifier-BUaQ4C17.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-C0oJ4vr-.js → produced-state-WJs_enbZ.js} +3 -3
- package/dist/{produced-state-C0oJ4vr-.js.map → produced-state-WJs_enbZ.js.map} +1 -1
- package/dist/{promotion-policy-CewdjfCJ.d.ts → promotion-policy-DYRzXdTV.d.ts} +2 -2
- package/dist/{promotion-policy-CewdjfCJ.d.ts.map → promotion-policy-DYRzXdTV.d.ts.map} +1 -1
- package/dist/{provenance-DA-Pmyfv.d.ts → provenance-Bh9k3sjl.d.ts} +17 -6
- package/dist/{provenance-DA-Pmyfv.d.ts.map → provenance-Bh9k3sjl.d.ts.map} +1 -1
- package/dist/{registry-CaqJIDdN.d.ts → registry-DhpX9Bsg.d.ts} +4 -4
- package/dist/{registry-CaqJIDdN.d.ts.map → registry-DhpX9Bsg.d.ts.map} +1 -1
- package/dist/{researcher-BySlpla_.d.ts → researcher-CbCK-ppF.d.ts} +3 -3
- package/dist/{researcher-BySlpla_.d.ts.map → researcher-CbCK-ppF.d.ts.map} +1 -1
- package/dist/{reward-hacking-Bu8ev6PR.d.ts → reward-hacking-62tojkQd.d.ts} +2 -2
- package/dist/{reward-hacking-Bu8ev6PR.d.ts.map → reward-hacking-62tojkQd.d.ts.map} +1 -1
- package/dist/rl.d.ts +4 -4
- package/dist/rl.js +1 -1
- package/dist/{semantic-concept-judge-BI7Rrl5-.js → semantic-concept-judge-B7KPD57W.js} +2 -2
- package/dist/{semantic-concept-judge-BI7Rrl5-.js.map → semantic-concept-judge-B7KPD57W.js.map} +1 -1
- package/dist/{series-convergence-D2fsoJ2w.d.ts → series-convergence-VhTad5TM.d.ts} +2 -2
- package/dist/{series-convergence-D2fsoJ2w.d.ts.map → series-convergence-VhTad5TM.d.ts.map} +1 -1
- package/dist/{server-D9wQclzG.js → server-DNiUVirV.js} +2 -2
- package/dist/{server-D9wQclzG.js.map → server-DNiUVirV.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CrY0OG17.d.ts → skillopt-optimization-method-BtgL6PMH.d.ts} +20 -14
- package/dist/{skillopt-optimization-method-CrY0OG17.d.ts.map → skillopt-optimization-method-BtgL6PMH.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-Donmq4sq.js → skillopt-optimization-method-L8vi4h4m.js} +92 -14
- package/dist/skillopt-optimization-method-L8vi4h4m.js.map +1 -0
- package/dist/{statistical-heldout-uxSpFEjm.d.ts → statistical-heldout-Caxd30ZF.d.ts} +2 -2
- package/dist/{statistical-heldout-uxSpFEjm.d.ts.map → statistical-heldout-Caxd30ZF.d.ts.map} +1 -1
- package/dist/{store-tool-spans-D5FhM0_A.d.ts → store-tool-spans-BGg2QFk1.d.ts} +4 -4
- package/dist/{store-tool-spans-D5FhM0_A.d.ts.map → store-tool-spans-BGg2QFk1.d.ts.map} +1 -1
- package/dist/{tool-groups-zpufabP8.d.ts → tool-groups-BuhFx38z.d.ts} +3 -3
- package/dist/tool-groups-BuhFx38z.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +7 -7
- package/dist/{types-ColZlZtc.d.ts → types-B61iI2ZG.d.ts} +2 -2
- package/dist/{types-ColZlZtc.d.ts.map → types-B61iI2ZG.d.ts.map} +1 -1
- package/dist/{types-BohHewKK.d.ts → types-BNbhD_Vj.d.ts} +2 -2
- package/dist/{types-BohHewKK.d.ts.map → types-BNbhD_Vj.d.ts.map} +1 -1
- package/dist/{types-DOhGq4S1.d.ts → types-COAJy8Ku.d.ts} +15 -3
- package/dist/types-COAJy8Ku.d.ts.map +1 -0
- package/dist/{types-C1Bmeb8X.d.ts → types-COEjZMLY.d.ts} +3 -3
- package/dist/{types-C1Bmeb8X.d.ts.map → types-COEjZMLY.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +2 -2
- package/dist/wire/index.js +1 -1
- package/docs/campaign-proposers.md +60 -3
- package/docs/experiment.md +2 -0
- package/docs/feature-guide.md +1 -1
- package/docs/multishot-golden-records.md +3 -0
- package/package.json +4 -2
- package/dist/benchmark-command-Cz4fDmsk.js.map +0 -1
- package/dist/campaign-BcfXzmPM.js.map +0 -1
- package/dist/define-agent-eval-CvZQW4u9.js.map +0 -1
- package/dist/define-agent-eval-hXLUBKtb.d.ts.map +0 -1
- package/dist/external-optimizer-subprocess-DU1KM8yV.js.map +0 -1
- package/dist/index-CKblBxtr.d.ts +0 -1
- package/dist/index-Dzn8Q3C2.d.ts.map +0 -1
- package/dist/llm-client-BMuxYoZy.js.map +0 -1
- package/dist/llm-judge-DWq1Ptco.js.map +0 -1
- package/dist/skillopt-optimization-method-Donmq4sq.js.map +0 -1
- package/dist/tool-groups-zpufabP8.d.ts.map +0 -1
- package/dist/types-DOhGq4S1.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store-tool-spans-
|
|
1
|
+
{"version":3,"file":"store-tool-spans-BGg2QFk1.d.ts","names":[],"sources":["../src/trace/otel.ts","../src/trace/redact.ts","../src/trace-analyst/store-boundary.ts","../src/trace/otlp-flat.ts","../src/trace-analyst/analyst.ts","../src/trace-analyst/insights.ts","../src/trace-analyst/store-otlp.ts","../src/trace-analyst/store-tool-spans.ts"],"mappings":";;;;;;;;cAqBa;EAA0B;EAAoC;;UAE1D;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;IACV;IACA;MAAS;MAAsB;MAAmB;MAAsB;;;EAE1E,SAAS;IAAQ;IAAsB;IAAc,aAAa;;EAClE;IAAW;IAAc;;;UAGV;EACf;IAAY,YAAY;;EACxB,YAAY;IAAQ,cAAc;IAAuB,OAAO;;;UAGjD;EACf,eAAe;;;iBAIK,gBACpB,OAAO,YACP,eACA,gBAAe,4CACd,QAAQ;;;;;;;;;;;;;;;;UCvCM;EACf;EACA,SAAS;;EAET;;UAGe;EACf;EACA,QAAQ;;;cAIG,yBAAyB;cAezB;;;;;iBAMG,aACd,eACA,QAAO;EACJ;EAAgB,QAAQ;;;;;;;iBAuBb,YACd,gBACA,QAAO,iBACP,SAAQ;EACL;EAAgB,QAAQ;;;;;iBCjDb,gCACd,QAAQ,oBACR,UAAS,mCACR;;;KC5BS;UAEK;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE,MAAM;IACN;;EAEF;IAAY,YAAY;;EACxB,YAAY;EACZ,SAAS;IAAQ;IAAc;IAAuB,aAAa;;;;;UCNpD;EACf;EACA;EACA;EACA;;KAGU,sBAAsB;UAEjB;;EAEf,iBAAiB;;EAEjB,QAAQ;;EAER;;EAEA,YAAY;EACZ,SAAS,QAAQ;EACjB;EACA;EACA,aAAa;EACb;EACA,yBAAyB;EACzB,4BAA4B;EAC5B,eAAe,SAAS;EACxB,OAAO;EACP,OAAO,iBAAiB,SAAS;EACjC,SAAS;;;;;;;;iBASW,cACpB,OAAO,oBACP,SAAS,uBACR,QAAQ;;;UCtDM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf,OAAO;EACP,WAAW;EACX,QAAQ;EACR,SAAS;EACT;;UAGe;EACf,OAAO;EACP;EACA;EACA,WAAW;EACX,OAAO;EACP,UAAU;EACV,OAAO;EACP,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;iBA0BO,oBAAoB;iBAMpB,oBAAoB,OAAO;iBAuB3B,sBAAsB,qBAAqB;iBAS3C,0BAA0B,OAAO;iBAsBjC,0BAA0B,OAAO,0BAA0B;iBAsD3D,yBAAyB,OAAO,0BAA0B;iBAa1D,2BAA2B,SAAS,sBAAsB;iBA2D1D,4BAA4B;iBA6B5B,wBAAwB,OAAO;;;UC1K9B;;EAEf;;EAEA;;EAEA;;EAEA;;KAGG,gCAAgC;UAEpB,kCAAkC;;EAEjD;;;;;;;;EAQA;;uBAMa,kCAAkC;mBAC9B;mBACA;mBACA;mBACA;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;EAyBZ,SAAS,kBAAkB,UAAU,4BAA4B;EAOjE,SACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT;EAeG,YACJ,UAAU,qBACV,UAAU,4BACT,QAAQ;EAoDL,YACJ;IAAQ,UAAU;IAAqB;IAAe;KACtD,UAAU,4BACT,QAAQ;EAmBL,YACJ,UAAU,qBACV,UAAU,4BACT;EAMG,UACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT,QAAQ;EAuCL,UACJ;IAAQ;IAAkB;IAA6B;KACvD,UAAU,4BACT,QAAQ;EAsFL,YACJ;IAAQ;IAAkB;IAAuB;KACjD,UAAU,4BACT,QAAQ;EA4CL,WACJ;IAAQ;IAAkB;IAAiB;IAAuB;KAClE,UAAU,4BACT,QAAQ;;;EA6CL,cAAc,UAAU,4BAA4B;UAI5C;qBAmBK,WAAW,SAAS,cAAc,QAAQ;qBAE1C;UAEL;UAmBA;UAsHA;UA+DN;UAkBA;UAwDA;UAmBM;;cA+CH,2BAA2B;mBACrB;mBACA;EAEjB,YAAY,MAAM;;;YAYF,WAAW,SAAS,cAAc,QAAQ;YAkBhD;;;iBAwBI,6BACd,cACA,OAAM,uCACL;;;;cC/7BU,8BAA8B;EACzC;;;;;;iBASc,8BACd,gBAAgB,+BAChB,UAAS,uCACR"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { w as TraceAnalysisStore } from "./types-
|
|
2
|
-
import { _ as TraceAnalysisToolDescriptor } from "./engine-
|
|
1
|
+
import { w as TraceAnalysisStore } from "./types-BNbhD_Vj.js";
|
|
2
|
+
import { _ as TraceAnalysisToolDescriptor } from "./engine-CgQdUh-r.js";
|
|
3
3
|
//#region src/analyst/tool-groups.d.ts
|
|
4
4
|
/** Named tool sets. Kinds pass `tools: TRACE_TOOL_GROUPS.failureForensics` etc. */
|
|
5
5
|
type TraceToolGroupName =
|
|
@@ -25,4 +25,4 @@ type TraceToolGroupName =
|
|
|
25
25
|
declare function buildTraceToolsForGroup(group: TraceToolGroupName, store: TraceAnalysisStore): TraceAnalysisToolDescriptor[];
|
|
26
26
|
//#endregion
|
|
27
27
|
export { buildTraceToolsForGroup as n, TraceToolGroupName as t };
|
|
28
|
-
//# sourceMappingURL=tool-groups-
|
|
28
|
+
//# sourceMappingURL=tool-groups-BuhFx38z.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-groups-BuhFx38z.d.ts","names":[],"sources":["../src/analyst/tool-groups.ts"],"mappings":";;;;KAmBY;;;;;;;;;;;;;;;;;;;;iBAgDI,wBACd,OAAO,oBACP,OAAO,qBACN"}
|
|
@@ -2,7 +2,7 @@ import { r as CaptureIntegrityError } from "../errors-DEE6u6ot.js";
|
|
|
2
2
|
import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
|
|
3
3
|
import { b as CustomTokenPricing, c as CostLedgerHandle, p as CostProvenance } from "../cost-ledger-DbQdN3nO.js";
|
|
4
4
|
import { u as RunTokenUsage } from "../run-record-DVV82Gwh.js";
|
|
5
|
-
import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-
|
|
5
|
+
import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-CgQdUh-r.js";
|
|
6
6
|
import { t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
|
|
7
7
|
import { g as isRecordedTimeout, o as RecordedTrajectoryStep } from "../steps-AmkT-GIM.js";
|
|
8
8
|
//#region src/trace-repair/mini-swe-scaffold.d.ts
|
package/dist/traces.d.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { c as ValidationError, o as LimitExceededError, r as CaptureIntegrityError, s as NotFoundError, t as AgentEvalError } from "./errors-DEE6u6ot.js";
|
|
2
2
|
import { C as TraceEvent, D as isSandboxSpan, E as isRetrievalSpan, O as isToolSpan, S as ToolSpan, T as isLlmSpan, _ as Span, a as FAILURE_CLASSES, b as SpanStatus, c as JudgeSpan, d as RetrievalSpan, f as Run, g as SandboxSpan, h as RunStatus, i as EventKind, l as LlmSpan, m as RunOutcome, n as BudgetLedgerEntry, o as FailureClass, p as RunLayer, r as BudgetSpec, s as GenericSpan, t as Artifact, u as Message, v as SpanBase, w as isJudgeSpan, x as TRACE_SCHEMA_VERSION, y as SpanKind } from "./schema-BtVldJ3T.js";
|
|
3
3
|
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
|
|
4
|
-
import { a as ExtractedUsage, c as extractUsageFromResponse, i as ExtractUsageFromSseOptions, l as extractUsageFromSse, n as CaptureFetchOptions, o as SseUsageMode, r as captureFetchToRawSink, s as extractUsage, t as CaptureFetchContext } from "./capture-fetch-
|
|
5
|
-
import { _t as
|
|
4
|
+
import { a as ExtractedUsage, c as extractUsageFromResponse, i as ExtractUsageFromSseOptions, l as extractUsageFromSse, n as CaptureFetchOptions, o as SseUsageMode, r as captureFetchToRawSink, s as extractUsage, t as CaptureFetchContext } from "./capture-fetch-Im0UN1i6.js";
|
|
5
|
+
import { _t as defaultProviderRedactor, ct as FileSystemRawProviderSinkOptions, dt as NoopRawProviderSink, ft as ProviderRedactor, gt as RawProviderSinkFilter, ht as RawProviderSink, lt as InMemoryRawProviderSink, mt as RawProviderEvent, pt as RawProviderDirection, st as FileSystemRawProviderSink, ut as InMemoryRawProviderSinkOptions, vt as providerFromBaseUrl } from "./types-COAJy8Ku.js";
|
|
6
6
|
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-CT9YIIve.js";
|
|
7
7
|
import { a as TraceEmitterOptions, i as TraceEmitter, n as RunCompleteHookContext, o as llmSpanFromProvider, r as SpanHandle, t as RunCompleteHook } from "./emitter-DGQGoLyj.js";
|
|
8
|
-
import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-
|
|
9
|
-
import { A as createBoundedTraceAnalysisStore, B as OtlpSpan, C as scoreTraceInsightReadiness, D as AnalyzeTracesResult, E as AnalyzeTracesOptions, F as redactString, I as redactValue, L as OTEL_AGENT_EVAL_SCOPE, M as REDACTION_VERSION, N as RedactionReport, O as analyzeTraces, P as RedactionRule, R as OtlpExport, S as planTraceInsightQuestions, T as AnalyzeTracesInput, V as exportRunAsOtlp, _ as buildTraceInsightPrompt, a as ToolSpansToTraceAnalysisStoreOptions, b as domainEvidencePattern, c as TraceInsightFinding, d as TraceInsightQualityGate, f as TraceInsightQuestion, g as buildTraceInsightContext, h as TraceInsightTask, i as OtlpFileTraceStoreOptions, j as DEFAULT_REDACTION_RULES, k as OtlpFlatLine, l as TraceInsightPanelRole, m as TraceInsightSuite, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightReadiness, r as OtlpFileTraceStore, s as TraceInsightContext, t as ToolTraceMissingError, u as TraceInsightPromptInput, v as defaultTraceInsightPanel, w as tokenizeDomainWords, x as inferDomainKeywords, y as describeTraceInsightScope, z as OtlpResourceSpans } from "./store-tool-spans-
|
|
8
|
+
import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-D2MAlC_K.js";
|
|
9
|
+
import { A as createBoundedTraceAnalysisStore, B as OtlpSpan, C as scoreTraceInsightReadiness, D as AnalyzeTracesResult, E as AnalyzeTracesOptions, F as redactString, I as redactValue, L as OTEL_AGENT_EVAL_SCOPE, M as REDACTION_VERSION, N as RedactionReport, O as analyzeTraces, P as RedactionRule, R as OtlpExport, S as planTraceInsightQuestions, T as AnalyzeTracesInput, V as exportRunAsOtlp, _ as buildTraceInsightPrompt, a as ToolSpansToTraceAnalysisStoreOptions, b as domainEvidencePattern, c as TraceInsightFinding, d as TraceInsightQualityGate, f as TraceInsightQuestion, g as buildTraceInsightContext, h as TraceInsightTask, i as OtlpFileTraceStoreOptions, j as DEFAULT_REDACTION_RULES, k as OtlpFlatLine, l as TraceInsightPanelRole, m as TraceInsightSuite, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightReadiness, r as OtlpFileTraceStore, s as TraceInsightContext, t as ToolTraceMissingError, u as TraceInsightPromptInput, v as defaultTraceInsightPanel, w as tokenizeDomainWords, x as inferDomainKeywords, y as describeTraceInsightScope, z as OtlpResourceSpans } from "./store-tool-spans-BGg2QFk1.js";
|
|
10
10
|
import { C as TOOL_LATENCY_MS, D as asNumber, E as applyLlmSpanOtlpAttributes, O as contextInputTokens, S as TOOL_ARGS_CAPTURED, T as TOOL_NAME_ATTR_KEYS, _ as LlmSpanOtlpInput, a as LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, b as RUN_COST_ATTR_KEYS, c as LLM_COST_USD, d as LLM_MODEL_ATTR_KEYS, f as LLM_MODEL_NAME, g as LLM_REASONING_TOKEN_ATTR_KEYS, h as LLM_REASONING_TOKENS, i as LLM_CACHE_WRITE_TOKENS, k as firstNumberAttr, l as LLM_INPUT_TOKENS, m as LLM_OUTPUT_TOKEN_ATTR_KEYS, n as LLM_CACHED_TOKENS, o as LLM_CONTEXT_TOKENS, p as LLM_OUTPUT_TOKENS, r as LLM_CACHED_TOKEN_ATTR_KEYS, s as LLM_COST_ATTR_KEYS, t as INPUT_VALUE, u as LLM_INPUT_TOKEN_ATTR_KEYS, v as OPENINFERENCE_SPAN_KIND, w as TOOL_NAME, x as SPAN_KIND_ATTR_KEYS, y as OUTPUT_VALUE } from "./attribute-vocabulary-DLJ6303h.js";
|
|
11
11
|
import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-CJ_DX8vl.js";
|
|
12
|
-
import "./index-
|
|
13
|
-
import { A as SearchSpanResult, B as ViewSpansResult, C as TRACE_ANALYSIS_LIMITS, D as DatasetOverview, E as DEFAULT_TRACE_ANALYST_BUDGETS, F as TraceAnalystFilters, H as ViewTraceResult, I as TraceAnalystSpan, L as TraceAnalystSpanKind, M as SpanMatchRecord, N as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, O as ErrorCluster, P as TraceAnalystByteBudgets, R as TraceAnalystSpanStatus, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, V as ViewTraceOversized, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore, z as TraceAnalystTraceSummary } from "./types-
|
|
14
|
-
import { _ as TraceAnalysisToolDescriptor, g as TRACE_ANALYST_TOOL_NAMESPACE, h as BuildTraceAnalysisToolsOptions, v as buildTraceAnalysisToolDescriptors, y as traceAnalystFunctionGroup } from "./engine-
|
|
12
|
+
import "./index-DjpxROrT.js";
|
|
13
|
+
import { A as SearchSpanResult, B as ViewSpansResult, C as TRACE_ANALYSIS_LIMITS, D as DatasetOverview, E as DEFAULT_TRACE_ANALYST_BUDGETS, F as TraceAnalystFilters, H as ViewTraceResult, I as TraceAnalystSpan, L as TraceAnalystSpanKind, M as SpanMatchRecord, N as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, O as ErrorCluster, P as TraceAnalystByteBudgets, R as TraceAnalystSpanStatus, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, V as ViewTraceOversized, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore, z as TraceAnalystTraceSummary } from "./types-BNbhD_Vj.js";
|
|
14
|
+
import { _ as TraceAnalysisToolDescriptor, g as TRACE_ANALYST_TOOL_NAMESPACE, h as BuildTraceAnalysisToolsOptions, v as buildTraceAnalysisToolDescriptors, y as traceAnalystFunctionGroup } from "./engine-CgQdUh-r.js";
|
|
15
15
|
//#region src/trace/otlp-attributes.d.ts
|
|
16
16
|
type ToolSpanOtlpInput = Pick<ToolSpan, 'toolName' | 'args' | 'argsCaptured' | 'result' | 'latencyMs'>;
|
|
17
17
|
type OtlpSpanRole = 'AGENT' | 'CHAIN' | 'EVALUATOR' | 'GUARDRAIL' | 'LLM' | 'SPAN' | 'TOOL' | 'UNKNOWN';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { s as RunSplitTag } from "./run-record-DVV82Gwh.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext } from "./types-
|
|
2
|
+
import { R as Scenario, d as DispatchContext } from "./types-COEjZMLY.js";
|
|
3
3
|
//#region src/benchmarks/types.d.ts
|
|
4
4
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
5
5
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -90,4 +90,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
|
90
90
|
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
91
91
|
//#endregion
|
|
92
92
|
export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
|
|
93
|
-
//# sourceMappingURL=types-
|
|
93
|
+
//# sourceMappingURL=types-B61iI2ZG.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-B61iI2ZG.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
|
|
3
|
-
import { a as JudgeInput, p as ChatClient } from "./types-
|
|
3
|
+
import { a as JudgeInput, p as ChatClient } from "./types-COAJy8Ku.js";
|
|
4
4
|
import { RE2JS } from "re2js";
|
|
5
5
|
//#region src/trace-analyst/types.d.ts
|
|
6
6
|
/**
|
|
@@ -594,4 +594,4 @@ type AnalystRunEvent = {
|
|
|
594
594
|
};
|
|
595
595
|
//#endregion
|
|
596
596
|
export { SearchSpanResult as A, ViewSpansResult as B, TRACE_ANALYSIS_LIMITS as C, DatasetOverview as D, DEFAULT_TRACE_ANALYST_BUDGETS as E, TraceAnalystFilters as F, ViewTraceResult as H, TraceAnalystSpan as I, TraceAnalystSpanKind as L, SpanMatchRecord as M, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as N, ErrorCluster as O, TraceAnalystByteBudgets as P, TraceAnalystSpanStatus as R, BoundedTraceAnalysisStoreOptions as S, TraceAnalysisStoreContext as T, ViewTraceOversized as V, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, SearchTraceResult as j, QueryTracesPage as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, TraceAnalysisStore as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystTraceSummary as z };
|
|
597
|
-
//# sourceMappingURL=types-
|
|
597
|
+
//# sourceMappingURL=types-BNbhD_Vj.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-BNbhD_Vj.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KAgBY;KAUA;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC7MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;;;;;;;UAUM;EACf,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;;;;UC/DH;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;KAGD;;KAGA;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
|
@@ -219,6 +219,14 @@ declare class ModelSubstitutionError extends AgentEvalError {
|
|
|
219
219
|
readonly checks: ReadonlyArray<ServedModelCheck>;
|
|
220
220
|
constructor(message: string, checks: ReadonlyArray<ServedModelCheck>);
|
|
221
221
|
}
|
|
222
|
+
/**
|
|
223
|
+
* Consumer-facing name for the substitution policy a metered surface applies.
|
|
224
|
+
* `'exact'` rejects every substitution. `'allow-within-family'` accepts a
|
|
225
|
+
* different model of the same provider family; it keeps family-level claims
|
|
226
|
+
* valid and forfeits per-model claims. Maps to
|
|
227
|
+
* `AssertServedModelOptions.allowWithinFamily`.
|
|
228
|
+
*/
|
|
229
|
+
type ServedModelPolicy = 'exact' | 'allow-within-family';
|
|
222
230
|
interface AssertServedModelOptions {
|
|
223
231
|
/**
|
|
224
232
|
* Accept a different model of the same provider family (e.g. requested
|
|
@@ -383,7 +391,11 @@ interface LlmCallResult {
|
|
|
383
391
|
raw: Record<string, unknown>;
|
|
384
392
|
}
|
|
385
393
|
type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
|
|
386
|
-
/** Convert a provider result into the canonical paid-call receipt input.
|
|
394
|
+
/** Convert a provider result into the canonical paid-call receipt input.
|
|
395
|
+
* The receipt is JSON-clean: absent optional fields are omitted, never carried
|
|
396
|
+
* as explicit-undefined keys. Receipts cross JSON boundaries (the external
|
|
397
|
+
* optimizer's loopback proxy validates them with `assertJsonValue`), where an
|
|
398
|
+
* explicit-undefined value is rejected as non-serializable. */
|
|
387
399
|
declare function costReceiptFromLlm(result: LlmCallResult, customTokenPricing?: CustomTokenPricing): CostReceiptInput;
|
|
388
400
|
/** Structured-response failures retain their completed provider receipt. */
|
|
389
401
|
declare function costReceiptFromLlmError(error: Error, customTokenPricing?: CustomTokenPricing): CostReceiptInput | undefined;
|
|
@@ -835,5 +847,5 @@ interface CheckResult {
|
|
|
835
847
|
actual: string;
|
|
836
848
|
}
|
|
837
849
|
//#endregion
|
|
838
|
-
export {
|
|
839
|
-
//# sourceMappingURL=types-
|
|
850
|
+
export { assertServedModels as $, LlmClientOptions as A, isTransientLlmError as B, SandboxSdkTransportOpts as C, LlmCallRequest as D, LlmCallMetadata as E, assertLlmRoute as F, AssertServedModelOptions as G, probeLlm as H, callLlm as I, ServedModelCheck as J, ModelSubstitutionError as K, callLlmJson as L, LlmResponseError as M, LlmRouteRequirements as N, LlmCallResult as O, LlmUsage as P, assertServedModel as Q, costReceiptFromLlm as R, RouterTransportOpts as S, LlmCallError as T, stripFencedJson as U, maximumChargeForLlmRequest as V, AssertCrossFamilyServedOptions as W, ServedModelVerdict as X, ServedModelPolicy as Y, assertCrossFamilyServed as Z, CliBridgeTransportOpts as _, defaultProviderRedactor as _t, JudgeInput as a, assertCrossFamily as at, DirectProviderTransportOpts as b, PersonaConfig as c, FileSystemRawProviderSinkOptions as ct, Scenario as d, NoopRawProviderSink as dt, checkServedModel as et, ChatCallOpts as f, ProviderRedactor as ft, ChatTransport as g, RawProviderSinkFilter as gt, ChatResponse as h, RawProviderSink as ht, JudgeFn as i, JudgeFamily as it, LlmMessage as j, LlmClient as k, ProductClientConfig as l, InMemoryRawProviderSink as lt, ChatRequest as m, RawProviderEvent as mt, CompletionCriterion as n, AssertCrossFamilyOptions as nt, JudgeRubric as o, judgeFamily as ot, ChatClient as p, RawProviderDirection as pt, ServedCrossFamilyError as q, DriverState as r, CrossFamilyError as rt, JudgeScore as s, FileSystemRawProviderSink as st, CheckResult as t, servedModelAcceptable as tt, RouteMap as u, InMemoryRawProviderSinkOptions as ut, CreateChatClientOpts as v, providerFromBaseUrl as vt, createChatClient as w, MockTransportOpts as x, CustomTransportOpts as y, costReceiptFromLlmError as z };
|
|
851
|
+
//# sourceMappingURL=types-COAJy8Ku.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-COAJy8Ku.d.ts","names":[],"sources":["../src/trace/raw-provider-sink.ts","../src/judge-families.ts","../src/integrity/served-model.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA8BY;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;;iBAmB5C,wBAAwB,OAAO,mBAAmB;UA8CjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;;;;;;;;;;;KC5QxB;;;;;;;iBAkEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCnFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;iBA4Cc,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;;;;;;;;KAc9B;UAWK;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;UCnQc;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;KAI7C;UAEK;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;EACrC;EACA;;EAEA,WAAW;;EAEX;;;;;;iBAOc,2BACd,SAAS,KAAK,iFACd,UAAS,mBACR;UAgCc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;EAEA,KAAK;;KAGK,kBAAkB,KAAK;;;;;;iBAOnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC;;EAEf;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;EAEA;;EAEA,qBAAqB;;;;;;;EAOrB;;;;;;EAMA;;EAEA,WAAW;;EAEX,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBAyEhB,oBAAoB;;;;;;iBA6IpB,gBAAgB;;;;;;iBA6EV,QACpB,KAAK,gBACL,OAAM,mBACL,QAAQ;;;;;;;iBAoVW,YAAY,aAChC,KAAK,gBACL,OAAM,mBACL;EAAU,OAAO;EAAG,QAAQ;;UA6Ed;;;;;;;EAOf;;;;;EAKA,kBAAkB,eAAe;;EAEjC,kBAAkB,eAAe;;EAEjC;;;;;EAKA;;;;;;;;;;;iBAYc,eAAe,MAAM,kBAAkB,MAAK;;;;;;;;;;;;;;;;;;iBA6EtC,SACpB,eACA,OAAM;EAAqB;IAC1B;EACD;EACA;EACA;;EAEA;;EAEA;;;;;;;cAoCW;WACF;mBACQ;EAEjB,YAAY,OAAM;EAKlB,KAAK,KAAK,gBAAgB,MAAM,mBAAmB,QAAQ;EAK3D,SAAS,aACP,KAAK,gBACL,MAAM,mBACL;IAAU,OAAO;IAAG,QAAQ;;;;;;;;UC1qChB;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAQK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,sBACA,yBACA,8BACA,0BACA,sBACA;UAEM;EACR;;EAEA;;UAGe,4BAA4B;EAC3C;EACA;EACA;;UAGe,+BAA+B;EAC9C;EACA;EACA;;UAGe,oCAAoC;EACnD;EACA;EACA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;UAO1C,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UCjH7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAmBe;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UASe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAyBe;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;;;;;;;;;KAWT;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
|
|
3
|
-
import { E as LlmCallMetadata } from "./types-
|
|
4
|
-
import { _ as ProposalFinding } from "./types-
|
|
3
|
+
import { E as LlmCallMetadata } from "./types-COAJy8Ku.js";
|
|
4
|
+
import { _ as ProposalFinding } from "./types-BNbhD_Vj.js";
|
|
5
5
|
//#region src/campaign/types.d.ts
|
|
6
6
|
/** Stable identifier + kind tag for any scenario. Consumers
|
|
7
7
|
* extend with their per-domain payload (persona, task, requirement, ...). */
|
|
@@ -626,4 +626,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
626
626
|
}
|
|
627
627
|
//#endregion
|
|
628
628
|
export { LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, GateDecision as _, CampaignResult as a, GenerationRecord as b, CampaignTraceWriter as c, DispatchContext as d, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, GateCheckStatus as m, CampaignArtifactWriter as n, CampaignScenarioIdentity as o, Gate as p, CampaignCellResult as r, CampaignTokenUsage as s, CampaignAggregates as t, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
629
|
-
//# sourceMappingURL=types-
|
|
629
|
+
//# sourceMappingURL=types-COEjZMLY.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-COEjZMLY.d.ts","names":[],"sources":["../src/campaign/types.ts"],"mappings":";;;;;;;UAgCiB;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;EAIV;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;iBAKc,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;WACjC,gBAAgB;WAChB,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;UAGe;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { A as LlmClientOptions, N as LlmRouteRequirements } from "../types-
|
|
2
|
+
import { A as LlmClientOptions, N as LlmRouteRequirements } from "../types-COAJy8Ku.js";
|
|
3
3
|
import { s as TraceStore } from "../store-CT9YIIve.js";
|
|
4
|
-
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-
|
|
4
|
+
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-Cc9M2tNF.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { ServerType } from "@hono/node-server";
|
|
7
7
|
import { Hono } from "hono";
|
package/dist/wire/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { A as TraceEventSchema, C as HealthResponseSchema, D as RubricDimensionSchema, E as ListRubricsResponseSchema, F as hashRubric, M as TracesIngestResponseSchema, N as VersionResponseSchema, O as RubricInfoSchema, P as WIRE_VERSION, S as FeedbackTrajectorySchema, T as JudgeResultSchema, _ as ErrorResponseSchema, a as runRpcBatch, b as FeedbackIngestResponseSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as listBuiltinRubrics, h as getBuiltinRubric, i as dispatchRpc, j as TracesIngestRequestSchema, k as RubricSchema, l as handleFeedbackIngest, m as BUILTIN_RUBRICS, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FailureModeSchema, w as JudgeRequestSchema, x as FeedbackLabelSchema, y as FeedbackAttemptSchema } from "../server-
|
|
1
|
+
import { A as TraceEventSchema, C as HealthResponseSchema, D as RubricDimensionSchema, E as ListRubricsResponseSchema, F as hashRubric, M as TracesIngestResponseSchema, N as VersionResponseSchema, O as RubricInfoSchema, P as WIRE_VERSION, S as FeedbackTrajectorySchema, T as JudgeResultSchema, _ as ErrorResponseSchema, a as runRpcBatch, b as FeedbackIngestResponseSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as listBuiltinRubrics, h as getBuiltinRubric, i as dispatchRpc, j as TracesIngestRequestSchema, k as RubricSchema, l as handleFeedbackIngest, m as BUILTIN_RUBRICS, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FailureModeSchema, w as JudgeRequestSchema, x as FeedbackLabelSchema, y as FeedbackAttemptSchema } from "../server-DNiUVirV.js";
|
|
2
2
|
export { BUILTIN_RUBRICS, ErrorResponseSchema, FailureModeSchema, FeedbackAttemptSchema, FeedbackIngestResponseSchema, FeedbackLabelSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RubricDimensionSchema, RubricInfoSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
|
|
@@ -11,6 +11,15 @@ Use it for caller-defined logic, an agent-runtime worker, or declared parameter
|
|
|
11
11
|
Do not wrap a complete external optimizer in `SurfaceProposer`.
|
|
12
12
|
That would split its search state from its own selection behavior and make budgets harder to compare.
|
|
13
13
|
|
|
14
|
+
## Choose The Entry Point
|
|
15
|
+
|
|
16
|
+
An `OptimizationMethod` plugs into exactly two entry points.
|
|
17
|
+
`selfImprove()` from `/contract` is the improvement entry: one call gives the method disjoint train and selection partitions, re-scores the selected surface on a held-out split, and returns a `gateDecision`.
|
|
18
|
+
Use it when one surface must get better.
|
|
19
|
+
`compareOptimizationMethods()` from `/campaign` is the measurement entry: it gives every method equal inputs and scores the selected surfaces on final cases no method received.
|
|
20
|
+
Use it when two or more methods must be compared at equal budget.
|
|
21
|
+
Runnable versions: [`examples/self-improve-optimizer`](../examples/self-improve-optimizer/) and [`examples/compare-optimization-methods`](../examples/compare-optimization-methods/).
|
|
22
|
+
|
|
14
23
|
## Adapt A Third-Party Text Optimizer
|
|
15
24
|
|
|
16
25
|
`externalTextOptimizationMethod()` is the general adapter for a package that already owns text or component search.
|
|
@@ -300,10 +309,38 @@ const method = gepaOptimizationMethod({
|
|
|
300
309
|
|
|
301
310
|
Replace the rates with the exact rates charged by your endpoint.
|
|
302
311
|
If billed USD is unknown, omit `maxCostUsd`, `pricing`, and `maxProposerCostUsd`; the recorded cost remains unknown rather than becoming a guessed zero.
|
|
303
|
-
With `optimizer`, every recipe stage must use the standard `gepa` engine.
|
|
312
|
+
With `optimizer`, every recipe stage must use the standard `gepa` engine or a metered agent CLI engine (below).
|
|
304
313
|
Agent Eval receives no provider key, enforces the declared request and token budget, and records the execution owner's exact usage and opaque finite JSON evidence.
|
|
305
314
|
`maxProposerCostUsd` also limits each individual GEPA engine stage.
|
|
306
315
|
|
|
316
|
+
When no execution package owns the call, build the callback with `createOpenAiCompatibleExecutionOwner` from `/campaign`:
|
|
317
|
+
|
|
318
|
+
```ts
|
|
319
|
+
import { createOpenAiCompatibleExecutionOwner } from '@tangle-network/agent-eval/campaign'
|
|
320
|
+
|
|
321
|
+
const call = createOpenAiCompatibleExecutionOwner({
|
|
322
|
+
baseUrl: 'https://api.openai.com/v1',
|
|
323
|
+
apiKey: process.env.LLM_API_KEY!,
|
|
324
|
+
model: 'gpt-4.1-mini',
|
|
325
|
+
pricing: { inputUsdPerMillion: 0.4, outputUsdPerMillion: 1.6 },
|
|
326
|
+
})
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
It executes each admitted request against any OpenAI-compatible `/chat/completions` endpoint and returns the typed outcome with a JSON-clean receipt.
|
|
330
|
+
The credential stays inside the owner closure; the proxy still enforces every budget and identity check.
|
|
331
|
+
|
|
332
|
+
### Metered agent CLI engines
|
|
333
|
+
|
|
334
|
+
The `autoresearch` and `meta_harness` engines drive a `claude` CLI subprocess.
|
|
335
|
+
Set `optimizer.anthropicEndpoint: true` to admit them in proxied mode.
|
|
336
|
+
The loopback proxy then also serves `POST /v1/messages` (Anthropic Messages API) and the bridge child receives `ANTHROPIC_BASE_URL`, an ephemeral `ANTHROPIC_AUTH_TOKEN`, and `ANTHROPIC_MODEL` in its environment.
|
|
337
|
+
Every CLI call becomes one canonical execution-owner call with the same reservation, receipt, and budget pipeline as reflection traffic; the run fails if the receipt count differs from the admitted call count.
|
|
338
|
+
Each agent engine run must set `engineConfig.model` to `optimizer.model`, because the engines pass `--model` and that flag beats the injected environment.
|
|
339
|
+
The endpoint translates text conversations only; it refuses tool use, thinking, images, `top_p`, `top_k`, and `stop_sequences` with a loud Anthropic error envelope.
|
|
340
|
+
A budget refusal surfaces to the CLI as HTTP 402, which the CLI treats as terminal instead of retrying.
|
|
341
|
+
Agent sessions are chatty: size `budget.maxRequests` for tens of calls per engine run.
|
|
342
|
+
Without the flag, agent engines stay rejected in proxied mode.
|
|
343
|
+
|
|
307
344
|
Other official engines can still receive their own settings:
|
|
308
345
|
|
|
309
346
|
```ts
|
|
@@ -321,7 +358,10 @@ recipe: {
|
|
|
321
358
|
```
|
|
322
359
|
|
|
323
360
|
Their external model spend remains incomplete unless that engine reports it.
|
|
324
|
-
|
|
361
|
+
Supply provider API keys only through `runner.env`.
|
|
362
|
+
An exported shell variable never reaches the bridge child.
|
|
363
|
+
The spawn builds the child environment from a fixed allowlist of benign variables (PATH, HOME, locale, `PYTHONPATH`) plus `runner.env`, so the parent environment is stripped by construction.
|
|
364
|
+
When `optimizer` is set, `removeCredentialEnvironment` also deletes credential-shaped keys from `runner.env`; the child then receives only the loopback proxy URL and an ephemeral key inside the input JSON.
|
|
325
365
|
Do not place credentials in `engineConfig` because run settings are persisted.
|
|
326
366
|
|
|
327
367
|
`describeScenario()` controls the train and selection data sent to GEPA.
|
|
@@ -331,6 +371,22 @@ Neither callback can receive a final test case.
|
|
|
331
371
|
A direct standard GEPA run records `provenance.gepaCandidatePopulation`.
|
|
332
372
|
Pass that summary to `readGepaCandidatePopulationArtifact()` to verify and read every accepted candidate, its parent indices, and its selection scores.
|
|
333
373
|
Use `readExternalOptimizerObservationArtifact()` for every distinct callback submission, including proposals that GEPA rejected or the callback refused.
|
|
374
|
+
`provenance.evaluationCount` is the callback-metered evaluation total.
|
|
375
|
+
`provenance.upstreamReportedEvaluations` is GEPA's self-reported total; a difference means upstream skipped, cached, or double-counted work.
|
|
376
|
+
|
|
377
|
+
## Runtime Knobs
|
|
378
|
+
|
|
379
|
+
Each knob below has a default that works for small text campaigns and fails for agentic or slow-settling runs.
|
|
380
|
+
The table names the failure so you can set the knob before the run dies mid-spend.
|
|
381
|
+
|
|
382
|
+
| Knob | Default | What breaks when wrong | Where to set it |
|
|
383
|
+
|---|---|---|---|
|
|
384
|
+
| `timeoutMs` | 30 minutes | The whole bridge process tree is killed with `GEPA bridge exceeded 1800000ms`. An agentic run (40 evaluations over 45 s each) exceeds the default mid-spend. The same value bounds each callback POST and the runtime inspect pass. | `gepaOptimizationMethod({ timeoutMs })`, `skillOptOptimizationMethod({ timeoutMs })` |
|
|
385
|
+
| `dispatchShutdownTimeoutMs` | 5 seconds | A dispatch that cancels or settles paid calls slowly fails the cell with `CostAccountingIncompleteError` after the evaluations completed. | `runCampaign({ dispatchShutdownTimeoutMs })`; for comparisons, `compareOptimizationMethods` `optimizationRunOptions` |
|
|
386
|
+
| `servedModelPolicy` | `'exact'` | A router that substitutes a same-family model fails every proxied reflection call with a 502 `model substitution` error. `'allow-within-family'` accepts the substitute, keeps family-level claims, and forfeits per-model claims. | `optimizer.servedModelPolicy` |
|
|
387
|
+
| `reflection_lm_kwargs.num_retries` | litellm default (3) | Each failed reflection request retries 3 times inside litellm, so the proxy meters 4 request attempts per logical call and `budget.maxRequests` exhausts 4x early. Set `num_retries: 0`; the proxy already accounts each attempt. | `recipe.run.engineConfig.reflection.reflection_lm_kwargs` |
|
|
388
|
+
| `reflection_lm_kwargs.max_tokens` | `budget.maxOutputTokensPerRequest` | Every reflection request ships the full budget cap as `max_tokens`. A provider family with a lower completion cap rejects every call. A reasoning model also needs headroom for hidden reasoning tokens. Set a value at or below the smallest family cap; it must not exceed `budget.maxOutputTokensPerRequest`. | `recipe.run.engineConfig.reflection.reflection_lm_kwargs` |
|
|
389
|
+
| `maxProposerCostUsd` | unset | Without it, one engine stage can spend up to `optimizer.budget.maxCostUsd` or the campaign `costCeiling` before any limit fires. Supply it only when the execution owner can enforce billed USD. | `recipe.run.maxProposerCostUsd` |
|
|
334
390
|
|
|
335
391
|
## Install Official SkillOpt
|
|
336
392
|
|
|
@@ -464,7 +520,8 @@ Use `reps` when one surface needs repeated measurements.
|
|
|
464
520
|
- The same dispatch and judges score every method.
|
|
465
521
|
- Missing cost remains unknown.
|
|
466
522
|
- A method must declare bounded work before it starts.
|
|
467
|
-
- Credentials
|
|
523
|
+
- Credentials reach a bridge child only through `runner.env`; the child never inherits the parent process environment.
|
|
524
|
+
- The metered proxy path replaces provider credentials with a loopback URL and an ephemeral key.
|
|
468
525
|
- Resumed state must match every input that can change the result.
|
|
469
526
|
|
|
470
527
|
These rules make method comparisons inspectable without pretending different optimizers have identical internals.
|
package/docs/experiment.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
`@tangle-network/agent-eval/experiment` turns an experiment's registration into the object that runs it.
|
|
4
4
|
|
|
5
|
+
The registry of measured claims those experiments produce lives in [`evidence/`](../evidence/README.md); a sealed experiment's digest is the `experimentDigest` its registry record carries.
|
|
6
|
+
|
|
5
7
|
## The covenant
|
|
6
8
|
|
|
7
9
|
1. **The registered rule is the executed rule.**
|
package/docs/feature-guide.md
CHANGED
|
@@ -152,7 +152,7 @@ Store as `FeedbackTrajectory`, then derive:
|
|
|
152
152
|
| Area | Key exports | Best for | Notes |
|
|
153
153
|
| --- | --- | --- | --- |
|
|
154
154
|
| Judging | `llmJudge`, semantic judges, anti-slop, wire rubrics | Content, voice, semantic quality | Pair with objective checks when possible. |
|
|
155
|
-
| Verification | `MultiLayerVerifier`,
|
|
155
|
+
| Verification | `MultiLayerVerifier`, sandbox harness | Code and multi-step gates | Do not let semantic judges override failed builds. |
|
|
156
156
|
| Control | `runAgentControlLoop`, `objectiveEval`, `subjectiveEval` | Long-running agent tasks | Supports budgets, cost, stop policies, trace spans. |
|
|
157
157
|
| Propose/review | `runProposeReview` | Iterative artifact repair | Good for code, docs, plans, briefs. |
|
|
158
158
|
| Feedback data | `FeedbackTrajectory`, stores, converters | Human/environment labels | Domain adapters live in downstream repos. |
|
|
@@ -42,6 +42,9 @@ describe('my engine reproduces the multishot golden records', () => {
|
|
|
42
42
|
`checkMultishotGolden` runs every SHOT scenario in one call; the matrix scenarios have their own pair below.
|
|
43
43
|
It refuses an `only` id the catalog does not hold, so a stale id after a rename stops the check instead of greening a run of zero scenarios.
|
|
44
44
|
|
|
45
|
+
An external conversation engine plugs into the matrix through one seam: `RunMultishotMatrixOptions.runShot`.
|
|
46
|
+
The matrix runs that engine for every cell and reads its result through `MultishotCellOutput`, so the same golden checks grade any engine that implements the seam.
|
|
47
|
+
|
|
45
48
|
The matrix pair is `assertMultishotMatrixGoldenScenario` / `checkMultishotMatrixGoldenScenario`; both take a `runDir` the engine may write into, and both install a deterministic judge wire on `globalThis.fetch` for the duration of the run.
|
|
46
49
|
That wire is process-wide, so run matrix checks serially within one process and keep other fetch traffic out of it.
|
|
47
50
|
Both rules are enforced, not just documented: a second concurrent install throws, and the wire fails loud on any request it does not recognise rather than answering it.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.150.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -180,7 +180,9 @@
|
|
|
180
180
|
"check:model-ids": "node scripts/check-model-id-requests.mjs",
|
|
181
181
|
"check:analyst-benchmark": "node scripts/check-analyst-benchmark-implementation.mjs",
|
|
182
182
|
"openapi": "node dist/cli.js openapi --out dist/openapi.json",
|
|
183
|
-
"verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs"
|
|
183
|
+
"verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run evidence:check",
|
|
184
|
+
"evidence:render": "tsx scripts/render-evidence-index.ts",
|
|
185
|
+
"evidence:check": "tsx scripts/render-evidence-index.ts --check"
|
|
184
186
|
},
|
|
185
187
|
"dependencies": {
|
|
186
188
|
"@asteasolutions/zod-to-openapi": "^9.1.0",
|