@tangle-network/agent-eval 0.150.1 → 0.150.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/analyst/index.d.ts +3 -3
- package/dist/analyst/index.js +3 -3
- package/dist/{benchmark-command-CAFwbH0L.js → benchmark-command-BU1Las59.js} +6 -6
- package/dist/{benchmark-command-CAFwbH0L.js.map → benchmark-command-BU1Las59.js.map} +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-CN_7xJdV.js → campaign-la-gEYNz.js} +7 -7
- package/dist/{campaign-CN_7xJdV.js.map → campaign-la-gEYNz.js.map} +1 -1
- package/dist/{chat-client-2bVfrzhN.js → chat-client-Bvmxedyv.js} +2 -2
- package/dist/{chat-client-2bVfrzhN.js.map → chat-client-Bvmxedyv.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/contract/index.js +5 -5
- package/dist/{define-agent-eval-rqNyVhVV.js → define-agent-eval-C8V8sMqP.js} +7 -7
- package/dist/{define-agent-eval-rqNyVhVV.js.map → define-agent-eval-C8V8sMqP.js.map} +1 -1
- package/dist/{descriptive-B5MwKfbf.js → descriptive-jDOuI6mz.js} +22 -2
- package/dist/descriptive-jDOuI6mz.js.map +1 -0
- package/dist/{dspy-rlm-engine-DHI0WrUU.js → dspy-rlm-engine-CBYlPvNy.js} +2 -2
- package/dist/{dspy-rlm-engine-DHI0WrUU.js.map → dspy-rlm-engine-CBYlPvNy.js.map} +1 -1
- package/dist/{eval-campaign-C4jmuM-b.js → eval-campaign-CQuZrLR_.js} +2 -2
- package/dist/{eval-campaign-C4jmuM-b.js.map → eval-campaign-CQuZrLR_.js.map} +1 -1
- package/dist/{external-optimizer-process-Bhmzngf-.js → external-optimizer-process-x9oEXKsU.js} +2 -2
- package/dist/{external-optimizer-process-Bhmzngf-.js.map → external-optimizer-process-x9oEXKsU.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-Dn90UqN2.js → external-optimizer-subprocess-CKNb42oM.js} +7 -3
- package/dist/external-optimizer-subprocess-CKNb42oM.js.map +1 -0
- package/dist/{index-CTKpu9ry.d.ts → index-Aj3WO3_a.d.ts} +2 -2
- package/dist/{index-CTKpu9ry.d.ts.map → index-Aj3WO3_a.d.ts.map} +1 -1
- package/dist/index.d.ts +3 -72
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -11
- package/dist/{integrity-DysDBWDu.js → integrity-DL91tucI.js} +16 -3
- package/dist/integrity-DL91tucI.js.map +1 -0
- package/dist/{judge-calibration-DZkWrm5H.js → judge-calibration-zZjLz8hr.js} +2 -2
- package/dist/{judge-calibration-DZkWrm5H.js.map → judge-calibration-zZjLz8hr.js.map} +1 -1
- package/dist/{llm-judge-CVq33oz1.js → llm-judge-BqqMS8t7.js} +4 -4
- package/dist/{llm-judge-CVq33oz1.js.map → llm-judge-BqqMS8t7.js.map} +1 -1
- package/dist/meta-eval/index.js +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +1 -1
- package/dist/{produced-state-jfk8Du3b.js → produced-state-Bnq4FaDO.js} +2 -2
- package/dist/{produced-state-jfk8Du3b.js.map → produced-state-Bnq4FaDO.js.map} +1 -1
- package/dist/reporting.js +2 -2
- package/dist/{reward-hacking-DKI9T52l.js → reward-hacking-C0x0xihA.js} +2 -2
- package/dist/{reward-hacking-DKI9T52l.js.map → reward-hacking-C0x0xihA.js.map} +1 -1
- package/dist/rl.js +3 -3
- package/dist/{rubric-predictive-validity-Cwwyd7ah.js → rubric-predictive-validity-CzxLoZge.js} +2 -2
- package/dist/{rubric-predictive-validity-Cwwyd7ah.js.map → rubric-predictive-validity-CzxLoZge.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DLeUcK-K.js → skillopt-optimization-method-CPBlTcj5.js} +4 -4
- package/dist/{skillopt-optimization-method-DLeUcK-K.js.map → skillopt-optimization-method-CPBlTcj5.js.map} +1 -1
- package/dist/{summary-report-Blysd6Z2.js → summary-report-DW2bEpdB.js} +2 -2
- package/dist/{summary-report-Blysd6Z2.js.map → summary-report-DW2bEpdB.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +26 -4
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +102 -24
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{tool-waste-BDdBZG1F.js → tool-waste-C-VHSRwF.js} +3 -3
- package/dist/{tool-waste-BDdBZG1F.js.map → tool-waste-C-VHSRwF.js.map} +1 -1
- package/dist/{types-yLK8gXE9.d.ts → types-I5WwVzQ7.d.ts} +159 -9
- package/dist/types-I5WwVzQ7.d.ts.map +1 -0
- package/docs/adapters-observability.md +9 -23
- package/docs/campaign-proposers.md +13 -5
- package/docs/concepts.md +3 -4
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/descriptive-B5MwKfbf.js.map +0 -1
- package/dist/external-optimizer-subprocess-Dn90UqN2.js.map +0 -1
- package/dist/integrity-DysDBWDu.js.map +0 -1
- package/dist/types-yLK8gXE9.d.ts.map +0 -1
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { o as runRolloutReleaseCli } from "./hf-dataset-XggBupCr.js";
|
|
3
|
-
import { n as runAnalystBenchmarkCommand } from "./benchmark-command-
|
|
3
|
+
import { n as runAnalystBenchmarkCommand } from "./benchmark-command-BU1Las59.js";
|
|
4
4
|
import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-dIWwF3j_.js";
|
|
5
5
|
import { writeFileSync } from "node:fs";
|
|
6
6
|
//#region src/cli-config.ts
|
package/dist/contract/index.js
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import { s as ValidationError } from "../errors-Dngq5h35.js";
|
|
2
2
|
import { r as pairedBootstrap } from "../paired-tests-BHIhYVdu.js";
|
|
3
|
-
import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-
|
|
3
|
+
import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-C8V8sMqP.js";
|
|
4
4
|
import { a as parseRunRecordSafe } from "../run-record-D2lDdSAz.js";
|
|
5
|
-
import { $ as runEval, et as runCampaign, f as runImprovementLoop, q as defaultProductionGate, t as llmJudge, ut as campaignSplitDigest, x as compareOptimizationMethods } from "../llm-judge-
|
|
5
|
+
import { $ as runEval, et as runCampaign, f as runImprovementLoop, q as defaultProductionGate, t as llmJudge, ut as campaignSplitDigest, x as compareOptimizationMethods } from "../llm-judge-BqqMS8t7.js";
|
|
6
6
|
import { i as isModelPriced, n as estimateCost } from "../metrics-Qv-cpptD.js";
|
|
7
7
|
import { i as CostLedger } from "../cost-ledger-B1qx30B4.js";
|
|
8
8
|
import { d as mapConcurrentRange } from "../ledger-core-DTae9rv_.js";
|
|
9
|
-
import { S as inMemoryCampaignStorage, x as fsCampaignStorage } from "../external-optimizer-subprocess-
|
|
9
|
+
import { S as inMemoryCampaignStorage, x as fsCampaignStorage } from "../external-optimizer-subprocess-CKNb42oM.js";
|
|
10
10
|
import { a as heldoutSignificance, s as decidePairedPromotion } from "../power-preflight-DEw-uC7q.js";
|
|
11
11
|
import { r as makeProposalFinding } from "../types-BI4fT3HN.js";
|
|
12
12
|
import { $ as classifyOtlpSpanRole, et as isOtlpModelCall } from "../kind-factory-DmAa0h3K.js";
|
|
13
|
-
import { n as buildDefaultAnalystRegistry, t as createChatClient } from "../chat-client-
|
|
13
|
+
import { n as buildDefaultAnalystRegistry, t as createChatClient } from "../chat-client-Bvmxedyv.js";
|
|
14
14
|
import { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS } from "../trace-attributes.js";
|
|
15
15
|
import { t as extractUsage } from "../extract-usage-BrQ8mCLX.js";
|
|
16
16
|
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-ChBKlTd_.js";
|
|
17
17
|
import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "../task-failure-attributes--ZTP3tYO.js";
|
|
18
|
-
import { a as externalTextOptimizationMethod, c as REFERENCE_EQUIVALENCE_INPUT_LIMITS, d as runReferenceEquivalenceJudge, i as composeGate, l as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, r as heldOutGate, t as skillOptOptimizationMethod, u as createReferenceEquivalenceJudge } from "../skillopt-optimization-method-
|
|
18
|
+
import { a as externalTextOptimizationMethod, c as REFERENCE_EQUIVALENCE_INPUT_LIMITS, d as runReferenceEquivalenceJudge, i as composeGate, l as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, r as heldOutGate, t as skillOptOptimizationMethod, u as createReferenceEquivalenceJudge } from "../skillopt-optimization-method-CPBlTcj5.js";
|
|
19
19
|
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-xzA40Evo.js";
|
|
20
20
|
import { createHash } from "node:crypto";
|
|
21
21
|
import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
|
-
import { a as spearmanR, r as pearsonR } from "./descriptive-
|
|
3
|
-
import { r as continuousAgreement } from "./judge-calibration-
|
|
2
|
+
import { a as spearmanR, r as pearsonR } from "./descriptive-jDOuI6mz.js";
|
|
3
|
+
import { r as continuousAgreement } from "./judge-calibration-zZjLz8hr.js";
|
|
4
4
|
import { i as pairedCohensDz } from "./effect-sizes-DiH8MGOH.js";
|
|
5
5
|
import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-CHIrXJll.js";
|
|
6
6
|
import { r as pairRunRecords } from "./paired-arms-D-XRF_fy.js";
|
|
@@ -8,10 +8,10 @@ import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-BHIhYVdu.
|
|
|
8
8
|
import { r as welchsTTest } from "./baseline-BhPRQBVn.js";
|
|
9
9
|
import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
|
|
10
10
|
import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-D2lDdSAz.js";
|
|
11
|
-
import { r as paretoChart } from "./summary-report-
|
|
12
|
-
import { $ as runEval, A as surfaceHash, f as runImprovementLoop, k as surfaceContentHash, nt as resolveRunDir, o as emitLoopProvenance, q as defaultProductionGate, s as loopProvenanceArgsFromResult, y as assertOptimizationResult } from "./llm-judge-
|
|
13
|
-
import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-
|
|
14
|
-
import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-
|
|
11
|
+
import { r as paretoChart } from "./summary-report-DW2bEpdB.js";
|
|
12
|
+
import { $ as runEval, A as surfaceHash, f as runImprovementLoop, k as surfaceContentHash, nt as resolveRunDir, o as emitLoopProvenance, q as defaultProductionGate, s as loopProvenanceArgsFromResult, y as assertOptimizationResult } from "./llm-judge-BqqMS8t7.js";
|
|
13
|
+
import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-C0x0xihA.js";
|
|
14
|
+
import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-CKNb42oM.js";
|
|
15
15
|
import { t as powerPreflight } from "./power-preflight-DEw-uC7q.js";
|
|
16
16
|
import { t as createHostedClient } from "./client-LIuo-KPv.js";
|
|
17
17
|
//#region src/contamination-guard.ts
|
|
@@ -1491,4 +1491,4 @@ function requirePositiveInteger(value, field) {
|
|
|
1491
1491
|
//#endregion
|
|
1492
1492
|
export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
|
|
1493
1493
|
|
|
1494
|
-
//# sourceMappingURL=define-agent-eval-
|
|
1494
|
+
//# sourceMappingURL=define-agent-eval-C8V8sMqP.js.map
|