@tangle-network/agent-eval 0.144.7 → 0.144.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/{agent-profile-CgDTo40f.d.ts → agent-profile-DPi7IZg7.d.ts} +3 -3
- package/dist/{agent-profile-CgDTo40f.d.ts.map → agent-profile-DPi7IZg7.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +9 -9
- package/dist/analyst/index.js +1 -1
- package/dist/{analyze-runs-BNNK7irB.js → analyze-runs-C30yljDJ.js} +2 -2
- package/dist/{analyze-runs-BNNK7irB.js.map → analyze-runs-C30yljDJ.js.map} +1 -1
- package/dist/{benchmark-command-BCafwNrf.js → benchmark-command-Dtym4pA-.js} +2 -2
- package/dist/{benchmark-command-BCafwNrf.js.map → benchmark-command-Dtym4pA-.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-CDSolHq7.js → benchmarks-Ce4Ae5fX.js} +3 -3
- package/dist/{benchmarks-CDSolHq7.js.map → benchmarks-Ce4Ae5fX.js.map} +1 -1
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +2 -2
- package/dist/{campaign-Tdy3h62h.js → campaign-FG3thH2m.js} +4 -4
- package/dist/{campaign-Tdy3h62h.js.map → campaign-FG3thH2m.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-DjXROWpx.d.ts → client-C9gzZE59.d.ts} +4 -4
- package/dist/{client-DjXROWpx.d.ts.map → client-C9gzZE59.d.ts.map} +1 -1
- package/dist/{completion-verifier-foUCLif_.d.ts → completion-verifier-DJA5BhPb.d.ts} +3 -3
- package/dist/{completion-verifier-foUCLif_.d.ts.map → completion-verifier-DJA5BhPb.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +4 -4
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/control.js +1 -1
- package/dist/{default-registry-BZhStdjl.d.ts → default-registry-Bf8Woqmq.d.ts} +6 -6
- package/dist/{default-registry-BZhStdjl.d.ts.map → default-registry-Bf8Woqmq.d.ts.map} +1 -1
- package/dist/{engine-nB64f48I.d.ts → engine-3hL-XqwJ.d.ts} +7 -7
- package/dist/{engine-nB64f48I.d.ts.map → engine-3hL-XqwJ.d.ts.map} +1 -1
- package/dist/{eval-campaign-DNjCvAm-.js → eval-campaign-B_7wcnav.js} +2 -2
- package/dist/{eval-campaign-DNjCvAm-.js.map → eval-campaign-B_7wcnav.js.map} +1 -1
- package/dist/{exact-types-Djvzosly.d.ts → exact-types-DSFFpLLI.d.ts} +2 -2
- package/dist/{exact-types-Djvzosly.d.ts.map → exact-types-DSFFpLLI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/{feedback-trajectory-Rh280oXo.d.ts → feedback-trajectory-CacHpxsp.d.ts} +3 -3
- package/dist/{feedback-trajectory-Rh280oXo.d.ts.map → feedback-trajectory-CacHpxsp.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-CvXXlyz7.d.ts → index-BnEuDAK2.d.ts} +2 -2
- package/dist/{index-CvXXlyz7.d.ts.map → index-BnEuDAK2.d.ts.map} +1 -1
- package/dist/{index-Sh2I0DRc.d.ts → index-CBEMKO8d.d.ts} +9 -9
- package/dist/{index-Sh2I0DRc.d.ts.map → index-CBEMKO8d.d.ts.map} +1 -1
- package/dist/{index-C5HOo4ZF2.d.ts → index-CLKMJTJF2.d.ts} +5 -5
- package/dist/{index-C5HOo4ZF2.d.ts.map → index-CLKMJTJF2.d.ts.map} +1 -1
- package/dist/{index-CwDrUMe0.d.ts → index-CQTZ-4XN.d.ts} +2 -2
- package/dist/{index-CwDrUMe0.d.ts.map → index-CQTZ-4XN.d.ts.map} +1 -1
- package/dist/index.d.ts +22 -22
- package/dist/index.js +11 -11
- package/dist/{insight-report-C6h6F_4L.d.ts → insight-report-CgX_s0Ez.d.ts} +3 -3
- package/dist/{insight-report-C6h6F_4L.d.ts.map → insight-report-CgX_s0Ez.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/{mint-CGEkzPLf.js → mint-B2O60ACG.js} +2 -2
- package/dist/{mint-CGEkzPLf.js.map → mint-B2O60ACG.js.map} +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{promotion-policy-ChWhTDBH.d.ts → promotion-policy-Ckjhzg_4.d.ts} +2 -2
- package/dist/{promotion-policy-ChWhTDBH.d.ts.map → promotion-policy-Ckjhzg_4.d.ts.map} +1 -1
- package/dist/{propose-review-control-qgWLA6E9.js → propose-review-control-dSNPjFUH.js} +2 -2
- package/dist/{propose-review-control-qgWLA6E9.js.map → propose-review-control-dSNPjFUH.js.map} +1 -1
- package/dist/{release-report-Dy39cbFF.js → release-report-BUYmoKo2.js} +2 -2
- package/dist/{release-report-Dy39cbFF.js.map → release-report-BUYmoKo2.js.map} +1 -1
- package/dist/{release-report-CI8uisI1.d.ts → release-report-DA2BCu5a.d.ts} +3 -3
- package/dist/{release-report-CI8uisI1.d.ts.map → release-report-DA2BCu5a.d.ts.map} +1 -1
- package/dist/{replay-DyBLaKFc.js → replay-B3gACG_H.js} +2 -2
- package/dist/{replay-DyBLaKFc.js.map → replay-B3gACG_H.js.map} +1 -1
- package/dist/{replay-DFf-teiC.d.ts → replay-DbhZ4Ked.d.ts} +5 -5
- package/dist/{replay-DFf-teiC.d.ts.map → replay-DbhZ4Ked.d.ts.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +1 -1
- package/dist/{researcher-BoaxeCzP.d.ts → researcher-BchpD55R.d.ts} +3 -3
- package/dist/{researcher-BoaxeCzP.d.ts.map → researcher-BchpD55R.d.ts.map} +1 -1
- package/dist/{reward-hacking-B2tPL9a3.js → reward-hacking-BDToousL.js} +4 -3
- package/dist/reward-hacking-BDToousL.js.map +1 -0
- package/dist/{reward-hacking-Cf1PtEOz.d.ts → reward-hacking-BI0OMAlo.d.ts} +2 -2
- package/dist/{reward-hacking-Cf1PtEOz.d.ts.map → reward-hacking-BI0OMAlo.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +4 -4
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-2ECTXb2N.js → rollout-DRrksrcV.js} +2 -2
- package/dist/{rollout-2ECTXb2N.js.map → rollout-DRrksrcV.js.map} +1 -1
- package/dist/{rubric-predictive-validity-9qAwzkZm.d.ts → rubric-predictive-validity-DvkjPYCe.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-9qAwzkZm.d.ts.map → rubric-predictive-validity-DvkjPYCe.d.ts.map} +1 -1
- package/dist/{run-evidence-BDFFai9R.d.ts → run-evidence-Dhi3C81V.d.ts} +3 -3
- package/dist/{run-evidence-BDFFai9R.d.ts.map → run-evidence-Dhi3C81V.d.ts.map} +1 -1
- package/dist/{run-record-DqOw5X6_.js → run-record-BmSPWXJR.js} +2 -1
- package/dist/run-record-BmSPWXJR.js.map +1 -0
- package/dist/{run-record-DdSa93_W.d.ts → run-record-CF4Dwpxr.d.ts} +4 -1
- package/dist/run-record-CF4Dwpxr.d.ts.map +1 -0
- package/dist/{skill-usage-CJlWEUFt.d.ts → skill-usage-CO9OLRBx.d.ts} +6 -6
- package/dist/{skill-usage-CJlWEUFt.d.ts.map → skill-usage-CO9OLRBx.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-BO7NAl3b.d.ts → skillopt-optimization-method-D8wysvU1.d.ts} +6 -6
- package/dist/{skillopt-optimization-method-BO7NAl3b.d.ts.map → skillopt-optimization-method-D8wysvU1.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-CQwZ-ZX8.js → skillopt-optimization-method-Dtzp7QpP.js} +14 -4
- package/dist/skillopt-optimization-method-Dtzp7QpP.js.map +1 -0
- package/dist/{statistical-heldout-Dn9ruizm.d.ts → statistical-heldout-TQ-4CYiN.d.ts} +3 -3
- package/dist/{statistical-heldout-Dn9ruizm.d.ts.map → statistical-heldout-TQ-4CYiN.d.ts.map} +1 -1
- package/dist/{summary-report-DuUS_i7W.d.ts → summary-report-BNR7DWTj.d.ts} +2 -2
- package/dist/{summary-report-DuUS_i7W.d.ts.map → summary-report-BNR7DWTj.d.ts.map} +1 -1
- package/dist/{tool-groups-DIVBnJyl.d.ts → tool-groups-CZPGGlHf.d.ts} +3 -3
- package/dist/tool-groups-CZPGGlHf.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +4 -4
- package/dist/traces.js +1 -1
- package/dist/{types-DYuNHo9R.d.ts → types-BnjdJ70P.d.ts} +3 -3
- package/dist/{types-DYuNHo9R.d.ts.map → types-BnjdJ70P.d.ts.map} +1 -1
- package/dist/{types-DF_Udrp-.d.ts → types-BvDKaULh.d.ts} +2 -2
- package/dist/{types-DF_Udrp-.d.ts.map → types-BvDKaULh.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/campaign-proposers.md +5 -0
- package/docs/concepts.md +1 -0
- package/package.json +1 -1
- package/dist/reward-hacking-B2tPL9a3.js.map +0 -1
- package/dist/run-record-DdSa93_W.d.ts.map +0 -1
- package/dist/run-record-DqOw5X6_.js.map +0 -1
- package/dist/skillopt-optimization-method-CQwZ-ZX8.js.map +0 -1
- package/dist/tool-groups-DIVBnJyl.d.ts.map +0 -1
package/dist/{skillopt-optimization-method-CQwZ-ZX8.js → skillopt-optimization-method-Dtzp7QpP.js}
RENAMED
|
@@ -8,7 +8,7 @@ import { p as mapConcurrent } from "./ledger-core-DXZIqu17.js";
|
|
|
8
8
|
import { E as pairedBootstrap, H as weightedComposite, d as confidenceInterval } from "./statistics-ByxzSiOM.js";
|
|
9
9
|
import { c as heldoutSignificance, l as pairHoldout, s as dimensionRegressions } from "./promotion-policy-CrLrmys8.js";
|
|
10
10
|
import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
|
|
11
|
-
import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-
|
|
11
|
+
import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-BDToousL.js";
|
|
12
12
|
import { z } from "zod";
|
|
13
13
|
import { writeFileSync } from "node:fs";
|
|
14
14
|
import { basename, isAbsolute, join } from "node:path";
|
|
@@ -1809,6 +1809,7 @@ async function executeCell(args) {
|
|
|
1809
1809
|
const tokenUsage = {
|
|
1810
1810
|
input: agentCost.inputTokens,
|
|
1811
1811
|
output: agentCost.outputTokens,
|
|
1812
|
+
...agentCost.usageComplete ? {} : { tokensKnown: false },
|
|
1812
1813
|
...agentCost.reasoningTokens !== void 0 && agentCost.reasoningTokens > 0 ? { reasoning: agentCost.reasoningTokens } : {},
|
|
1813
1814
|
...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
|
|
1814
1815
|
...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
|
|
@@ -5207,6 +5208,7 @@ async function runOptimization(opts) {
|
|
|
5207
5208
|
const selectionRankKey = opts.selectionRankKey ?? ((campaign) => [campaignMeanComposite(campaign)]);
|
|
5208
5209
|
let winnerSurface = baselineSurface;
|
|
5209
5210
|
let winnerSurfaceHash = surfaceHash(baselineSurface);
|
|
5211
|
+
const admittedCandidateHashes = /* @__PURE__ */ new Set([winnerSurfaceHash]);
|
|
5210
5212
|
let winnerComposite = campaignMeanComposite(baselineCampaign);
|
|
5211
5213
|
let winnerRankKey = selectionRankKey(baselineCampaign);
|
|
5212
5214
|
assertFiniteRankKey(winnerRankKey, "selectionRankKey for baseline");
|
|
@@ -5255,11 +5257,19 @@ async function runOptimization(opts) {
|
|
|
5255
5257
|
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
5256
5258
|
const proposalSnapshot = immutableProposalSnapshot(proposed, "candidate outputs");
|
|
5257
5259
|
if (proposalSnapshot.length === 0) break;
|
|
5258
|
-
const
|
|
5260
|
+
const candidates = proposalSnapshot.map((p) => isProposedCandidate(p) ? p : {
|
|
5259
5261
|
surface: p,
|
|
5260
5262
|
label: "",
|
|
5261
5263
|
rationale: ""
|
|
5262
|
-
})
|
|
5264
|
+
});
|
|
5265
|
+
const generationHashes = /* @__PURE__ */ new Set();
|
|
5266
|
+
for (const { surface } of candidates) {
|
|
5267
|
+
const hash = surfaceHash(surface);
|
|
5268
|
+
if (admittedCandidateHashes.has(hash) || generationHashes.has(hash)) throw new Error(`runOptimization: duplicate candidate surface hash "${hash}" in generation ${gen}; candidate surfaces must be unique`);
|
|
5269
|
+
generationHashes.add(hash);
|
|
5270
|
+
}
|
|
5271
|
+
for (const hash of generationHashes) admittedCandidateHashes.add(hash);
|
|
5272
|
+
const surfaceResults = await mapConcurrent(candidates, candidateConcurrency, async ({ surface, label, rationale }, i) => {
|
|
5263
5273
|
const hash = surfaceHash(surface);
|
|
5264
5274
|
const campaign = await runCampaign({
|
|
5265
5275
|
...opts,
|
|
@@ -6520,4 +6530,4 @@ function skillOptOptimizationMethod(config) {
|
|
|
6520
6530
|
//#endregion
|
|
6521
6531
|
export { runReferenceEquivalenceJudge as $, componentSurfaceIdentityMaterial as A, planCampaignRun as B, combineComparisonCosts as C, assertCodeSurfaceIdentity as D, optimizationTokenUsageFromSummary as E, campaignMeanComposite as F, assertCampaignSplitIdentity as G, resolveRunDir as H, compareRankKeys as I, campaignSplitDigestFromIdentities as J, campaignScenarioIdentity as K, DEFAULT_MUTATION_PRIMITIVES as L, surfaceContentHash as M, surfaceHash as N, assertComponentSurface as O, campaignBreakdown as P, createReferenceEquivalenceJudge as Q, buildReflectionPrompt as R, assertOptimizationResult as S, costFromLedgerSummary as T, tangleTracesRoot as U, runCampaign as V, assertCampaignDesign as W, REFERENCE_EQUIVALENCE_INPUT_LIMITS as X, openAutoPr as Y, REFERENCE_EQUIVALENCE_JUDGE_VERSION as Z, heldOutGate as _, assertRealBackend as _t, emitLoopProvenance as a, Dataset as at, composeGate as b, JudgeParseError as bt, provenanceRecordPath as c, llmJudge as ct, runImprovementLoop as d, paretoFrontier as dt, DEFAULT_RED_TEAM_CORPUS as et, runOptimization as f, paretoFrontierWithCrowding as ft, gepaOptimizationMethod as g, assertRealAgentReceipts as gt, labelTrustRank as h, BackendIntegrityError as ht, canonicalDigest as i, toolNamesForRun as it, renderSurfaceDiff as j, codeSurfaceIdentityMaterial as k, provenanceSpansPath as l, crowdingDistance as lt, isProposedCandidate as m, recoverTruncatedJson as mt, buildLoopProvenanceRecord as n, redTeamReport as nt, loopProvenanceArgsFromResult as o, HoldoutLockedError as ot, runEval as p, scalarScore as pt, campaignSplitDigest as q, campaignMeasurementDigest as r, scoreRedTeamOutput as rt, loopProvenanceSpans as s, hashScenarios as st, skillOptOptimizationMethod as t, redTeamDataset as tt, verifyLoopProvenanceRecord as u, dominates as ut, defaultProductionGate as v, summarizeAgentReceiptIntegrity as vt, compareOptimizationMethods as w, externalTextOptimizationMethod as x, runCanaries as y, summarizeBackendIntegrity as yt, parseReflectionResponse as z };
|
|
6522
6532
|
|
|
6523
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
6533
|
+
//# sourceMappingURL=skillopt-optimization-method-Dtzp7QpP.js.map
|