@tangle-network/agent-runtime 0.281.1 → 0.282.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-C1ZAyQrQ.js → activation-Bi_oOB1-.js} +2 -2
- package/dist/{activation-C1ZAyQrQ.js.map → activation-Bi_oOB1-.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +1 -1
- package/dist/durable.d.ts +1 -1
- package/dist/{improvement-cycle-DCTCTmNl.js → improvement-cycle-8MUf5Xfp.js} +8 -1772
- package/dist/improvement-cycle-8MUf5Xfp.js.map +1 -0
- package/dist/{index-C4ygxavE.d.ts → index-CSyzK6Kd.d.ts} +125 -255
- package/dist/index.d.ts +2 -2
- package/dist/index.js +5 -5
- package/dist/intelligence.js +3 -2
- package/dist/intelligence.js.map +1 -1
- package/dist/kernel.d.ts +2 -2
- package/dist/kernel.js +3 -3
- package/dist/{loop-runner-bin-Co2eBFbb.js → loop-runner-bin-Cg78CKmf.js} +2 -2
- package/dist/{loop-runner-bin-Co2eBFbb.js.map → loop-runner-bin-Cg78CKmf.js.map} +1 -1
- package/dist/{loop-runner-bin-B9dTzFL3.d.ts → loop-runner-bin-Dtt8FdYO.d.ts} +2 -2
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +1 -1
- package/dist/{runtime-B_yRPBjj.js → runtime-Bid-kBes.js} +676 -675
- package/dist/runtime-Bid-kBes.js.map +1 -0
- package/dist/{shared-box-CZFdugL1.js → search-method-CbUyhdz5.js} +1799 -16
- package/dist/search-method-CbUyhdz5.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +9 -9
- package/dist/tui/index.d.ts +1 -1
- package/package.json +1 -1
- package/dist/improvement-cycle-DCTCTmNl.js.map +0 -1
- package/dist/runtime-B_yRPBjj.js.map +0 -1
- package/dist/shared-box-CZFdugL1.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { $i as detachedSnapshot, $t as isTraceAnalysisStore, Ai as readCommittedJsonLines, Ca as stringifySafe, Et as rollingDispatch, Gn as probeSandboxCapabilities, Gt as encodeResultBlob, Ht as InMemorySpawnJournal, Ia as armDeadlineTimer, Jn as readPromptOptions, Kt as insideCursorNamespace, M as withDriverExecutor, Mt as createExecutorRegistry, O as verdictFromJudgeScore, Oa as zeroSpend, Pt as createWorktreeCliExecutor, Qi as detachedFrozen, Sa as sleep, Un as runAgentRounds, Ut as SpawnEventIndex, Vn as defaultSelectWinner, Vt as InMemoryResultBlobStore, W as settledToIteration, Wn as createSandboxLineage, Wt as assertContentAddress, Xi as executableAgentProfileSnapshot, Xn as runBrainLoop, Yn as routerBrain, Zi as executableAgentSpecSnapshot, Zn as canonicalObservedModelParts, _ as CheckUnavailableError, cr as createPushTraceSource, ga as cloneSpend, ha as chargedTokens, ji as writeAllBytes, jt as createExecutor, ki as prepareJsonlAppend, kr as contentAddress, ma as addSpend, qn as assertBoxlessPromptOptions, r as createSupervisor, ua as notifySandboxEventObserver, va as hasCompleteCacheBreakdown, xa as randomSuffix, ya as isAbortError } from "./supervisor-UXgaPisu.js";
|
|
2
|
-
import { m as ValidationError, n as AnalystError, s as PlannerError } from "./errors-DodWX-cb.js";
|
|
3
|
-
import {
|
|
2
|
+
import { i as ConfigError, m as ValidationError, n as AnalystError, s as PlannerError } from "./errors-DodWX-cb.js";
|
|
3
|
+
import { Q as sample, X as refine, Z as runAgentic, c as reconcileInterruptedSearchCalls, ct as strategyAuthorMethod, et as ObservationError, it as profileChatClient, nt as observe, o as dedicatedLane, r as runtimeShipDecision, rt as renderReport, u as searchExecutor } from "./search-method-CbUyhdz5.js";
|
|
4
4
|
import { c as profileModelExecutionSettings, i as concreteModelId, l as profileProviderModel, o as enforceTokenLimits, t as assertExecutableAgentProfile } from "./model-policy-BbSCSak0.js";
|
|
5
5
|
import { C as canonicalCandidateDigest$1, D as immutableCandidateValue, k as sha256Bytes$1, m as captureMaterializedWorkspace } from "./workspace-archive-C9lgf77y.js";
|
|
6
6
|
import { i as notifyRuntimeHookEvent } from "./runtime-hooks-tXpAarhW.js";
|
|
@@ -16,13 +16,15 @@ import { CODING_HARNESSES, DEFAULT_TRACE_ANALYST_KINDS, InMemoryTraceStore, OUTP
|
|
|
16
16
|
import { createAgentRunOutcomeTracker } from "@tangle-network/sandbox/runtime";
|
|
17
17
|
import { createHash, randomUUID } from "node:crypto";
|
|
18
18
|
import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
|
|
19
|
-
import { appendFileSync, chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync,
|
|
19
|
+
import { appendFileSync, chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readlinkSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
20
20
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
21
21
|
import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
22
22
|
import { tmpdir } from "node:os";
|
|
23
23
|
import { isDeepStrictEqual, promisify } from "node:util";
|
|
24
|
+
import { redact } from "@tangle-network/agent-eval/traces";
|
|
24
25
|
import { collectAgentResponseText } from "@tangle-network/sandbox";
|
|
25
|
-
import {
|
|
26
|
+
import { SearchRecorder, asha, beam, createRunCostLedger, estimateNode, fsCampaignStorage, openSearchLedger, renderSearchSummary, runProfileMatrix, runSearch, searchClaimReserveUsd, searchModelIdentity, surfaceDiff } from "@tangle-network/agent-eval/campaign";
|
|
27
|
+
import { defineEvaluationClaim, summarizeEvaluationUnits } from "@tangle-network/agent-eval/experiment";
|
|
26
28
|
import { createInterface } from "node:readline";
|
|
27
29
|
import { pathToFileURL } from "node:url";
|
|
28
30
|
import { gzipSync } from "node:zlib";
|
|
@@ -59,7 +61,7 @@ function createTangleSandboxExactProcessProvider(client, options = {}) {
|
|
|
59
61
|
metadata,
|
|
60
62
|
idempotencyKey: requiredText(input.idempotencyKey, "exact-process idempotency key")
|
|
61
63
|
}, {
|
|
62
|
-
...input.provisionTimeoutMs === void 0 ? {} : { timeoutMs: positiveInteger(input.provisionTimeoutMs, "provision timeout") },
|
|
64
|
+
...input.provisionTimeoutMs === void 0 ? {} : { timeoutMs: positiveInteger$1(input.provisionTimeoutMs, "provision timeout") },
|
|
63
65
|
...input.signal === void 0 ? {} : { signal: input.signal }
|
|
64
66
|
});
|
|
65
67
|
try {
|
|
@@ -146,7 +148,7 @@ function sandboxAsExactProcessEnvironment(sandbox, provider) {
|
|
|
146
148
|
};
|
|
147
149
|
}
|
|
148
150
|
function sandboxProcessAsExactProcess(process) {
|
|
149
|
-
const pid = positiveInteger(process.pid, "Tangle Sandbox process pid");
|
|
151
|
+
const pid = positiveInteger$1(process.pid, "Tangle Sandbox process pid");
|
|
150
152
|
return {
|
|
151
153
|
pid,
|
|
152
154
|
async status() {
|
|
@@ -172,7 +174,7 @@ function sandboxProcessAsExactProcess(process) {
|
|
|
172
174
|
};
|
|
173
175
|
}
|
|
174
176
|
function exactProcessStatus(status, expectedPid) {
|
|
175
|
-
const pid = positiveInteger(status.pid, "Tangle Sandbox process pid");
|
|
177
|
+
const pid = positiveInteger$1(status.pid, "Tangle Sandbox process pid");
|
|
176
178
|
if (expectedPid !== void 0 && pid !== expectedPid) throw new Error("Tangle Sandbox process status changed process identity");
|
|
177
179
|
if (typeof status.running !== "boolean") throw new Error("Tangle Sandbox process status is missing running state");
|
|
178
180
|
if (!Number.isSafeInteger(status.exitCode)) throw new Error("Tangle Sandbox process status is missing an integer exit code");
|
|
@@ -272,8 +274,8 @@ function serializeEgressPolicy(policy) {
|
|
|
272
274
|
}
|
|
273
275
|
function sandboxResources(input) {
|
|
274
276
|
const cpuCores = positiveNumber(input.resources.cpu, "exact-process cpu");
|
|
275
|
-
const memoryMB = positiveInteger(input.resources.memoryMb, "exact-process memory");
|
|
276
|
-
const diskMb = positiveInteger(input.resources.diskMb, "exact-process disk");
|
|
277
|
+
const memoryMB = positiveInteger$1(input.resources.memoryMb, "exact-process memory");
|
|
278
|
+
const diskMb = positiveInteger$1(input.resources.diskMb, "exact-process disk");
|
|
277
279
|
if (diskMb % 1024 !== 0) throw new Error("Tangle Sandbox exact-process disk must be a whole GiB");
|
|
278
280
|
return {
|
|
279
281
|
cpuCores,
|
|
@@ -282,7 +284,7 @@ function sandboxResources(input) {
|
|
|
282
284
|
};
|
|
283
285
|
}
|
|
284
286
|
function sandboxLifetimeSeconds(maxLifetimeMs) {
|
|
285
|
-
const milliseconds = positiveInteger(maxLifetimeMs, "exact-process maximum lifetime");
|
|
287
|
+
const milliseconds = positiveInteger$1(maxLifetimeMs, "exact-process maximum lifetime");
|
|
286
288
|
if (milliseconds % 1e3 !== 0) throw new Error("Tangle Sandbox exact-process maximum lifetime must be a whole number of seconds");
|
|
287
289
|
return milliseconds / 1e3;
|
|
288
290
|
}
|
|
@@ -372,7 +374,7 @@ function positiveNumber(value, label) {
|
|
|
372
374
|
if (!Number.isFinite(value) || value <= 0) throw new Error(`${label} must be positive`);
|
|
373
375
|
return value;
|
|
374
376
|
}
|
|
375
|
-
function positiveInteger(value, label) {
|
|
377
|
+
function positiveInteger$1(value, label) {
|
|
376
378
|
if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`);
|
|
377
379
|
return value;
|
|
378
380
|
}
|
|
@@ -4894,188 +4896,6 @@ function requireSpend(rolled, id, root) {
|
|
|
4894
4896
|
return spend;
|
|
4895
4897
|
}
|
|
4896
4898
|
//#endregion
|
|
4897
|
-
//#region src/runtime/promotion-gate.ts
|
|
4898
|
-
/**
|
|
4899
|
-
* promotionGate — the statistical promotion decision over a holdout benchmark: does the
|
|
4900
|
-
* candidate strategy beat the incumbent on held-out tasks by a margin the task noise
|
|
4901
|
-
* cannot fake? The statistics come from `heldoutSignificance`, which chooses the
|
|
4902
|
-
* decision interval that matches the outcome shape, applies a minimum-evidence floor,
|
|
4903
|
-
* and requires the interval's lower bound to clear `deltaThreshold`. A raw h1>h0 point
|
|
4904
|
-
* comparison on m≈8 holdout tasks certifies false champions at near coin-flip rates.
|
|
4905
|
-
*/
|
|
4906
|
-
/** Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. */
|
|
4907
|
-
function promotionGate(opts) {
|
|
4908
|
-
const mode = opts.mode ?? "superiority";
|
|
4909
|
-
if (opts.candidate === opts.incumbent) return {
|
|
4910
|
-
promoted: false,
|
|
4911
|
-
reason: "identical-champion",
|
|
4912
|
-
mode,
|
|
4913
|
-
n: 0,
|
|
4914
|
-
lift: {
|
|
4915
|
-
mean: 0,
|
|
4916
|
-
median: 0,
|
|
4917
|
-
low: 0,
|
|
4918
|
-
high: 0
|
|
4919
|
-
}
|
|
4920
|
-
};
|
|
4921
|
-
const before = [];
|
|
4922
|
-
const after = [];
|
|
4923
|
-
const incUsd = [];
|
|
4924
|
-
const candUsd = [];
|
|
4925
|
-
const incMs = [];
|
|
4926
|
-
const candMs = [];
|
|
4927
|
-
const cellIds = [];
|
|
4928
|
-
/** Tasks where either arm's dollars were never measured — see the refusal below. */
|
|
4929
|
-
const costUnknownTasks = [];
|
|
4930
|
-
for (const row of opts.report.perTask) {
|
|
4931
|
-
const inc = row.cells?.[opts.incumbent];
|
|
4932
|
-
const cand = row.cells?.[opts.candidate];
|
|
4933
|
-
if (!inc || !cand) continue;
|
|
4934
|
-
before.push(inc.score);
|
|
4935
|
-
after.push(cand.score);
|
|
4936
|
-
incUsd.push(inc.usd);
|
|
4937
|
-
candUsd.push(cand.usd);
|
|
4938
|
-
if (inc.usdKnown === false || cand.usdKnown === false) costUnknownTasks.push(row.taskId);
|
|
4939
|
-
incMs.push(inc.ms);
|
|
4940
|
-
candMs.push(cand.ms);
|
|
4941
|
-
cellIds.push(row.taskId);
|
|
4942
|
-
}
|
|
4943
|
-
if (before.length === 0) throw new Error(`promotionGate: no holdout task carried cells for both "${opts.incumbent}" and "${opts.candidate}" — the report must come from a run that included both strategies`);
|
|
4944
|
-
const sig = heldoutSignificance({
|
|
4945
|
-
before,
|
|
4946
|
-
after,
|
|
4947
|
-
cellIds
|
|
4948
|
-
}, {
|
|
4949
|
-
deltaThreshold: opts.deltaThreshold ?? 0,
|
|
4950
|
-
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
4951
|
-
statistic: opts.statistic ?? "mean",
|
|
4952
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
4953
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
4954
|
-
});
|
|
4955
|
-
const lift = {
|
|
4956
|
-
mean: sig.bootstrap.mean,
|
|
4957
|
-
median: sig.bootstrap.median,
|
|
4958
|
-
low: sig.decision.low,
|
|
4959
|
-
high: sig.decision.high
|
|
4960
|
-
};
|
|
4961
|
-
const latSig = heldoutSignificance({
|
|
4962
|
-
before: incMs,
|
|
4963
|
-
after: candMs,
|
|
4964
|
-
cellIds
|
|
4965
|
-
}, {
|
|
4966
|
-
deltaThreshold: 0,
|
|
4967
|
-
minProductiveRuns: 1,
|
|
4968
|
-
statistic: opts.statistic ?? "mean",
|
|
4969
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
4970
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
4971
|
-
});
|
|
4972
|
-
const latency = {
|
|
4973
|
-
mean: latSig.bootstrap.mean,
|
|
4974
|
-
median: latSig.bootstrap.median,
|
|
4975
|
-
low: latSig.decision.low,
|
|
4976
|
-
high: latSig.decision.high
|
|
4977
|
-
};
|
|
4978
|
-
if (mode === "superiority") {
|
|
4979
|
-
if (sig.fewRuns) return {
|
|
4980
|
-
promoted: false,
|
|
4981
|
-
reason: "few-tasks",
|
|
4982
|
-
mode,
|
|
4983
|
-
n: sig.n,
|
|
4984
|
-
lift,
|
|
4985
|
-
latency
|
|
4986
|
-
};
|
|
4987
|
-
return sig.significant ? {
|
|
4988
|
-
promoted: true,
|
|
4989
|
-
reason: "significant",
|
|
4990
|
-
mode,
|
|
4991
|
-
n: sig.n,
|
|
4992
|
-
lift,
|
|
4993
|
-
latency
|
|
4994
|
-
} : {
|
|
4995
|
-
promoted: false,
|
|
4996
|
-
reason: "no-margin",
|
|
4997
|
-
mode,
|
|
4998
|
-
n: sig.n,
|
|
4999
|
-
lift,
|
|
5000
|
-
latency
|
|
5001
|
-
};
|
|
5002
|
-
}
|
|
5003
|
-
const tolerance = opts.scoreTolerance ?? .05;
|
|
5004
|
-
const scoreSig = heldoutSignificance({
|
|
5005
|
-
before,
|
|
5006
|
-
after,
|
|
5007
|
-
cellIds
|
|
5008
|
-
}, {
|
|
5009
|
-
deltaThreshold: -tolerance,
|
|
5010
|
-
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
5011
|
-
statistic: opts.statistic ?? "mean",
|
|
5012
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
5013
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
5014
|
-
});
|
|
5015
|
-
if (costUnknownTasks.length > 0) return {
|
|
5016
|
-
promoted: false,
|
|
5017
|
-
reason: "cost-unknown",
|
|
5018
|
-
mode,
|
|
5019
|
-
n: before.length,
|
|
5020
|
-
lift,
|
|
5021
|
-
costUnknownTasks: [...costUnknownTasks],
|
|
5022
|
-
latency
|
|
5023
|
-
};
|
|
5024
|
-
const costSig = heldoutSignificance({
|
|
5025
|
-
before: candUsd,
|
|
5026
|
-
after: incUsd,
|
|
5027
|
-
cellIds
|
|
5028
|
-
}, {
|
|
5029
|
-
deltaThreshold: 0,
|
|
5030
|
-
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
5031
|
-
statistic: opts.statistic ?? "mean",
|
|
5032
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
5033
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
5034
|
-
});
|
|
5035
|
-
const costSavings = {
|
|
5036
|
-
mean: costSig.bootstrap.mean,
|
|
5037
|
-
median: costSig.bootstrap.median,
|
|
5038
|
-
low: costSig.decision.low,
|
|
5039
|
-
high: costSig.decision.high
|
|
5040
|
-
};
|
|
5041
|
-
if (scoreSig.fewRuns) return {
|
|
5042
|
-
promoted: false,
|
|
5043
|
-
reason: "few-tasks",
|
|
5044
|
-
mode,
|
|
5045
|
-
n: scoreSig.n,
|
|
5046
|
-
lift,
|
|
5047
|
-
costSavings,
|
|
5048
|
-
latency
|
|
5049
|
-
};
|
|
5050
|
-
if (!scoreSig.significant) return {
|
|
5051
|
-
promoted: false,
|
|
5052
|
-
reason: "non-inferiority-unproven",
|
|
5053
|
-
mode,
|
|
5054
|
-
n: scoreSig.n,
|
|
5055
|
-
lift,
|
|
5056
|
-
costSavings,
|
|
5057
|
-
latency
|
|
5058
|
-
};
|
|
5059
|
-
if (!costSig.significant) return {
|
|
5060
|
-
promoted: false,
|
|
5061
|
-
reason: "not-cheaper",
|
|
5062
|
-
mode,
|
|
5063
|
-
n: scoreSig.n,
|
|
5064
|
-
lift,
|
|
5065
|
-
costSavings,
|
|
5066
|
-
latency
|
|
5067
|
-
};
|
|
5068
|
-
return {
|
|
5069
|
-
promoted: true,
|
|
5070
|
-
reason: "non-inferior-and-cheaper",
|
|
5071
|
-
mode,
|
|
5072
|
-
n: scoreSig.n,
|
|
5073
|
-
lift,
|
|
5074
|
-
costSavings,
|
|
5075
|
-
latency
|
|
5076
|
-
};
|
|
5077
|
-
}
|
|
5078
|
-
//#endregion
|
|
5079
4899
|
//#region src/runtime/run-benchmark.ts
|
|
5080
4900
|
/**
|
|
5081
4901
|
* runBenchmark — the packaged optimization suite. Define a domain by implementing an
|
|
@@ -5105,6 +4925,8 @@ async function pool(items, limit, fn) {
|
|
|
5105
4925
|
await Promise.all(workers);
|
|
5106
4926
|
return out;
|
|
5107
4927
|
}
|
|
4928
|
+
/** @internal Check each unique worker and analyst model once before any task runs, as
|
|
4929
|
+
* `BenchmarkConfig.modelPreflight` describes. `runStrategyEvolution` checks the same way. */
|
|
5108
4930
|
async function preflightModels(cfg) {
|
|
5109
4931
|
if (cfg.modelPreflight === false) return;
|
|
5110
4932
|
if (cfg.worker.complete && !cfg.modelPreflight) return;
|
|
@@ -5726,531 +5548,710 @@ const strategyAuthorSystemPrompt = "You are a senior researcher authoring optimi
|
|
|
5726
5548
|
function assertStrategyContract(code) {
|
|
5727
5549
|
assertAuthoredCode(code, { allowedImports: ["@tangle-network/agent-runtime/kernel"] });
|
|
5728
5550
|
}
|
|
5729
|
-
/**
|
|
5730
|
-
*
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
|
|
5739
|
-
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5551
|
+
/** Ask the author for a strategy module. A failed call throws; a reply without a fenced
|
|
5552
|
+
* module is `code: null`. With `fallbackProfile`, the fallback gets one attempt when the
|
|
5553
|
+
* primary call fails or its reply carries no module. */
|
|
5554
|
+
async function requestStrategySource(opts) {
|
|
5555
|
+
const turns = [];
|
|
5556
|
+
const attempt = async (profile) => {
|
|
5557
|
+
const res = await profileChatClient({
|
|
5558
|
+
profile,
|
|
5559
|
+
executor: opts.executor,
|
|
5560
|
+
context: "strategy author"
|
|
5561
|
+
}).chat({ messages: [{
|
|
5562
|
+
role: "user",
|
|
5563
|
+
content: `${opts.contract ?? strategyAuthorContract}\n\nBASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}) — the per-task losses are your gradient:\n${opts.lossesJson}\n\nAuthor ONE new strategy that you expect to beat the baselines on THIS environment at the same budget.\n${strategyAuthorMethod}\n\nOutput only the module code block.`
|
|
5564
|
+
}] }, { ...opts.signal ? { signal: opts.signal } : {} });
|
|
5565
|
+
turns.push({
|
|
5566
|
+
content: res.content,
|
|
5567
|
+
model: res.model,
|
|
5568
|
+
inputTokens: res.usage.promptTokens,
|
|
5569
|
+
outputTokens: res.usage.completionTokens,
|
|
5570
|
+
tokensKnown: res.usage.captured !== false,
|
|
5571
|
+
costUsd: res.costUsd
|
|
5572
|
+
});
|
|
5573
|
+
return res.content.match(/```(?:ts|typescript)?\s*\n([\s\S]*?)```/)?.[1] ?? null;
|
|
5574
|
+
};
|
|
5748
5575
|
try {
|
|
5749
|
-
code = await
|
|
5576
|
+
const code = await attempt(opts.profile);
|
|
5577
|
+
if (code !== null || !opts.fallbackProfile) return {
|
|
5578
|
+
code,
|
|
5579
|
+
turns
|
|
5580
|
+
};
|
|
5750
5581
|
} catch (primaryError) {
|
|
5751
5582
|
if (!opts.fallbackProfile) throw primaryError;
|
|
5752
|
-
code = await requestAuthoredCode(opts, opts.fallbackProfile);
|
|
5753
5583
|
}
|
|
5584
|
+
return {
|
|
5585
|
+
code: await attempt(opts.fallbackProfile),
|
|
5586
|
+
turns
|
|
5587
|
+
};
|
|
5588
|
+
}
|
|
5589
|
+
/** Lint an authored module, write it under `outDir` at a name its content fixes, and import
|
|
5590
|
+
* its default Strategy. The same source always loads from the same file. Throws when the
|
|
5591
|
+
* module breaks the contract or exports no Strategy. */
|
|
5592
|
+
async function loadAuthoredStrategy(code, outDir) {
|
|
5754
5593
|
assertStrategyContract(code);
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5594
|
+
const file = join(outDir, `strategy-${sha256Bytes$1(Buffer.from(code, "utf8")).slice(7)}.mts`);
|
|
5595
|
+
if (!existsSync(file)) {
|
|
5596
|
+
mkdirSync(outDir, { recursive: true });
|
|
5597
|
+
const partial = `${file}.${process.pid}.tmp`;
|
|
5598
|
+
writeFileSync(partial, code);
|
|
5599
|
+
renameSync(partial, file);
|
|
5600
|
+
}
|
|
5601
|
+
const mod = await import(pathToFileURL(file).href);
|
|
5759
5602
|
if (!mod.default || typeof mod.default.driver !== "function" || !mod.default.name) throw new Error(`authorStrategy: ${file} does not export a default Strategy`);
|
|
5760
5603
|
return {
|
|
5761
5604
|
strategy: mod.default,
|
|
5762
|
-
file
|
|
5605
|
+
file
|
|
5606
|
+
};
|
|
5607
|
+
}
|
|
5608
|
+
/** Author + load a strategy from losses. Throws when the author emits no loadable module;
|
|
5609
|
+
* with `fallbackProfile` set, the fallback gets one attempt first. */
|
|
5610
|
+
async function authorStrategy(opts) {
|
|
5611
|
+
const { code, turns } = await requestStrategySource(opts);
|
|
5612
|
+
if (code === null) throw new Error(`authorStrategy: no code block in the author's reply: ${(turns.at(-1)?.content ?? "").slice(0, 300)}`);
|
|
5613
|
+
return {
|
|
5614
|
+
...await loadAuthoredStrategy(code, opts.outDir),
|
|
5763
5615
|
code
|
|
5764
5616
|
};
|
|
5765
5617
|
}
|
|
5766
5618
|
//#endregion
|
|
5767
5619
|
//#region src/runtime/strategy-evolution.ts
|
|
5768
5620
|
/**
|
|
5769
|
-
* runStrategyEvolution
|
|
5770
|
-
* authors a POPULATION of candidate strategies from the current tournament's losses,
|
|
5771
|
-
* plays them against the incumbent at equal budget, and advances a champion; one final
|
|
5772
|
-
* promotion decision runs on a NEVER-BEFORE-USED holdout slice through `promotionGate`.
|
|
5621
|
+
* runStrategyEvolution: strategy search on Eval's search kernel (`runSearch`).
|
|
5773
5622
|
*
|
|
5774
|
-
*
|
|
5775
|
-
*
|
|
5776
|
-
*
|
|
5777
|
-
*
|
|
5778
|
-
*
|
|
5779
|
-
* candidates cannot win by overspending.
|
|
5780
|
-
* - Champion selection within the search is a SEARCH policy (configurable, default
|
|
5781
|
-
* cost-aware: ties on score go to the cheapest strategy — a scalar hides a strategy
|
|
5782
|
-
* that ties at half the cost). The promotion verdict never comes from search
|
|
5783
|
-
* selection; it comes from the gate on the fresh slice.
|
|
5784
|
-
* - Every authored artifact's description length (gzip bits) is recorded, so the
|
|
5785
|
-
* artifact-complexity-vs-holdout-gap relation is analyzable from any run's report.
|
|
5623
|
+
* Every node is a strategy: the caller's root, or a module an author model wrote from the
|
|
5624
|
+
* train results of the parent the policy chose. The default policy is `beam({ width: 2 })`
|
|
5625
|
+
* and the default allocator `asha()`, so a strategy is screened on a few selection tasks and
|
|
5626
|
+
* earns more tasks only by ranking well. Each cell is one `runAgentic` run of one strategy on
|
|
5627
|
+
* one task, scored by the environment's own check.
|
|
5786
5628
|
*
|
|
5787
|
-
*
|
|
5788
|
-
*
|
|
5789
|
-
*
|
|
5629
|
+
* Measurement follows the kernel's splits. The author reads train results only; the policy
|
|
5630
|
+
* and allocator rank on the private selection split; the claim runs the root and at most 3
|
|
5631
|
+
* finalists together on the sealed test split, once, with a power check and a Bonferroni
|
|
5632
|
+
* decision. The report is a projection of the ledger, and the ledger is the only checkpoint:
|
|
5633
|
+
* calling again with the same inputs continues an interrupted search and returns a finished
|
|
5634
|
+
* one without running anything.
|
|
5790
5635
|
*
|
|
5791
5636
|
* @experimental
|
|
5792
5637
|
*/
|
|
5793
|
-
|
|
5794
|
-
|
|
5795
|
-
|
|
5796
|
-
|
|
5797
|
-
|
|
5798
|
-
|
|
5799
|
-
|
|
5800
|
-
|
|
5801
|
-
|
|
5802
|
-
|
|
5803
|
-
})
|
|
5804
|
-
if (rows.length === 0) return null;
|
|
5805
|
-
const out = {};
|
|
5806
|
-
for (const name of fieldOrder) {
|
|
5807
|
-
const cells = rows.map((r) => r.cells?.[name]).filter((c) => !!c);
|
|
5808
|
-
out[name] = {
|
|
5809
|
-
score: cells.reduce((s, c) => s + c.score, 0) / cells.length,
|
|
5810
|
-
usd: cells.reduce((s, c) => s + c.usd, 0) / cells.length
|
|
5811
|
-
};
|
|
5812
|
-
}
|
|
5813
|
-
return out;
|
|
5814
|
-
}
|
|
5815
|
-
/** The champion pick over a means table. 'score' takes the best mean score (ties →
|
|
5816
|
-
* field order). 'costAware' treats scores within `epsilon` of the best as tied and
|
|
5817
|
-
* takes the cheapest — the (score, $) Pareto rule collapsed to one pick. */
|
|
5818
|
-
function pickChampion(means, fieldOrder, policy, epsilon) {
|
|
5819
|
-
const entries = fieldOrder.map((name) => ({
|
|
5820
|
-
name,
|
|
5821
|
-
summary: means[name]
|
|
5822
|
-
})).filter((e) => !!e.summary);
|
|
5823
|
-
if (entries.length === 0) throw new Error("pickChampion: the means table carries none of the field strategies");
|
|
5824
|
-
const best = Math.max(...entries.map((e) => e.summary.score));
|
|
5825
|
-
const pick = policy === "score" ? entries.find((e) => e.summary.score === best) : entries.filter((e) => e.summary.score >= best - epsilon).sort((a, b) => a.summary.usd - b.summary.usd || b.summary.score - a.summary.score)[0];
|
|
5826
|
-
if (!pick) throw new Error("pickChampion: empty pick (unreachable)");
|
|
5827
|
-
return {
|
|
5828
|
-
name: pick.name,
|
|
5829
|
-
score: pick.summary.score,
|
|
5830
|
-
usd: pick.summary.usd
|
|
5831
|
-
};
|
|
5832
|
-
}
|
|
5833
|
-
/** Search-side champion selection over a tournament report. */
|
|
5834
|
-
function selectChampion(report, fieldOrder, policy, epsilon) {
|
|
5835
|
-
return pickChampion(report.perStrategy, fieldOrder, policy, epsilon);
|
|
5836
|
-
}
|
|
5837
|
-
const fieldSummary = (archive) => archive.map((n) => `- ${n.name} (${n.source}, gen ${n.generation}, last score ${(n.score * 100).toFixed(0)}%)`).join("\n");
|
|
5838
|
-
/** The author-visible losses: EVERY train task in compact form (score/resolved/
|
|
5839
|
-
* progression per cell). A pretty-printed prefix slice would hide the tail tasks from
|
|
5840
|
-
* the author and bias which failure modes it can target; the hard cap stays only as a
|
|
5841
|
-
* guard against enormous fields. */
|
|
5842
|
-
const compactLosses = (report, detail) => {
|
|
5843
|
-
const r2 = (x) => Math.round(x * 100) / 100;
|
|
5844
|
-
const rows = report.perTask.map((row) => row.cells ? {
|
|
5845
|
-
task: row.taskId,
|
|
5846
|
-
...row.errors ? { errors: Object.fromEntries(Object.entries(row.errors).map(([n, msg]) => [n, msg.slice(0, 100)])) } : {},
|
|
5847
|
-
cells: Object.fromEntries(Object.entries(row.cells).map(([name, c]) => [name, detail === "binary" ? {
|
|
5848
|
-
resolved: c.resolved,
|
|
5849
|
-
usd: Math.round(c.usd * 1e4) / 1e4
|
|
5850
|
-
} : {
|
|
5851
|
-
score: r2(c.score),
|
|
5852
|
-
resolved: c.resolved,
|
|
5853
|
-
usd: Math.round(c.usd * 1e4) / 1e4,
|
|
5854
|
-
progression: (c.progression ?? []).map(r2)
|
|
5855
|
-
}]))
|
|
5856
|
-
} : {
|
|
5857
|
-
task: row.taskId,
|
|
5858
|
-
error: row.error?.slice(0, 80)
|
|
5859
|
-
});
|
|
5860
|
-
return JSON.stringify(rows).slice(0, 12e3);
|
|
5638
|
+
const STRATEGY_EVOLUTION_SOURCE = {
|
|
5639
|
+
uri: "npm:@tangle-network/agent-runtime#runStrategyEvolution",
|
|
5640
|
+
revision: canonicalCandidateDigest$1({
|
|
5641
|
+
name: "agent-runtime.strategy-evolution.2026-09",
|
|
5642
|
+
node: "the root strategy by name under the caller executionRef, or an authored module content-addressed by the sha256 of its source",
|
|
5643
|
+
edge: "a text diff from the parent module source to the child; unknown from the root, which has no source here",
|
|
5644
|
+
admission: "a module is refused when the reply carries none, it breaks the author contract, or it does not load a default Strategy",
|
|
5645
|
+
cell: "one runAgentic run on one task, metered as one paid call on the search cost ledger through the Runtime search executor; an environment fault during it is errored and retried, a thrown strategy is failed at score 0, anything else passed at its harness-verified score",
|
|
5646
|
+
proposer: "the author reads the strategy contract, the task tools, the train summary, the train results of every strategy and the parent source",
|
|
5647
|
+
ship: "the claim shipped, re-derives from the ledger, cost accounting is complete, and the shipped finalist test lower bound exceeds 0"
|
|
5648
|
+
})
|
|
5861
5649
|
};
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
function
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
|
|
5880
|
-
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
|
|
5885
|
-
|
|
5650
|
+
const NO_MODULE = "the author reply carried no fenced ts module";
|
|
5651
|
+
/** Evolve a strategy on the search kernel and claim once on the sealed test split. */
|
|
5652
|
+
async function runStrategyEvolution(cfg) {
|
|
5653
|
+
const root = cfg.root ?? refine;
|
|
5654
|
+
const budget = positiveInteger("budget", cfg.budget ?? 3);
|
|
5655
|
+
const concurrency = positiveInteger("concurrency", cfg.concurrency ?? 3);
|
|
5656
|
+
const maxExpansions = nonNegativeInteger("maxExpansions", cfg.maxExpansions ?? 4);
|
|
5657
|
+
const maxAttempts = positiveInteger("maxAttempts", cfg.maxAttempts ?? 3);
|
|
5658
|
+
const seed = nonNegativeInteger("seed", cfg.seed ?? 42);
|
|
5659
|
+
const cellUsd = dollars("cellUsd", cfg.cellUsd) ?? 0;
|
|
5660
|
+
const maxUsd = dollars("maxUsd", cfg.maxUsd);
|
|
5661
|
+
const policy = cfg.policy ?? beam({ width: 2 });
|
|
5662
|
+
const allocation = cfg.allocation ?? asha();
|
|
5663
|
+
if (cfg.deadline !== void 0 && !Number.isFinite(Date.parse(cfg.deadline))) throw new ConfigError("evolution: deadline must be an ISO time");
|
|
5664
|
+
if (!/^sha256:[0-9a-f]{64}$/.test(cfg.executionRef)) throw new ConfigError("evolution: executionRef must be a lowercase sha256:<64 hex> digest");
|
|
5665
|
+
if (!root.name.trim() || typeof root.driver !== "function") throw new ConfigError("evolution: root must be a named Strategy");
|
|
5666
|
+
const claim = defineEvaluationClaim(cfg.claim);
|
|
5667
|
+
if (claim.minimumEffect === void 0) throw new ConfigError("evolution: the claim needs minimumEffect for its power check");
|
|
5668
|
+
const taskIds = /* @__PURE__ */ new Set();
|
|
5669
|
+
const splitTasks = (label, tasks) => {
|
|
5670
|
+
if (!Array.isArray(tasks) || tasks.length === 0) throw new ConfigError(`evolution: ${label} needs at least one task`);
|
|
5671
|
+
for (const task of tasks) {
|
|
5672
|
+
if (typeof task.id !== "string" || !task.id.trim() || taskIds.has(task.id)) throw new ConfigError(`evolution: task IDs must be non-empty and unique across splits`);
|
|
5673
|
+
taskIds.add(task.id);
|
|
5886
5674
|
}
|
|
5675
|
+
return immutableCandidateValue(tasks);
|
|
5887
5676
|
};
|
|
5888
|
-
|
|
5889
|
-
|
|
5890
|
-
|
|
5891
|
-
const
|
|
5892
|
-
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
const
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
5921
|
-
generations,
|
|
5922
|
-
populationSize,
|
|
5923
|
-
baselines: baselines.map((baseline) => baseline.name),
|
|
5924
|
-
objective: cfg.objective ?? "score",
|
|
5925
|
-
scoreTolerance: cfg.scoreTolerance ?? .05,
|
|
5926
|
-
champion: policy,
|
|
5927
|
-
championEpsilon: epsilon,
|
|
5928
|
-
minPairedTasks: cfg.minPairedTasks ?? null,
|
|
5929
|
-
band: cfg.band ?? null,
|
|
5930
|
-
lossesDetail: cfg.lossesDetail ?? "exact",
|
|
5931
|
-
reproducerCheck: cfg.reproducerCheck ?? null,
|
|
5932
|
-
modelPreflight: cfg.modelPreflight !== false,
|
|
5933
|
-
modelPreflightTimeoutMs: cfg.modelPreflightTimeoutMs ?? null,
|
|
5934
|
-
worker: {
|
|
5677
|
+
const train = splitTasks("train", cfg.train);
|
|
5678
|
+
const selection = splitTasks("selection", cfg.selection);
|
|
5679
|
+
const test = splitTasks("test", cfg.test);
|
|
5680
|
+
const taskById = new Map([
|
|
5681
|
+
...train,
|
|
5682
|
+
...selection,
|
|
5683
|
+
...test
|
|
5684
|
+
].map((task) => [task.id, task]));
|
|
5685
|
+
const searchTasks = (tasks) => tasks.map((task) => ({
|
|
5686
|
+
taskId: task.id,
|
|
5687
|
+
unitId: summarizeEvaluationUnits(claim, [task]).units[0].id,
|
|
5688
|
+
source: {
|
|
5689
|
+
uri: `task://${task.id}`,
|
|
5690
|
+
revision: canonicalCandidateDigest$1(task)
|
|
5691
|
+
}
|
|
5692
|
+
}));
|
|
5693
|
+
const splits = {
|
|
5694
|
+
train: searchTasks(train),
|
|
5695
|
+
selection: searchTasks(selection),
|
|
5696
|
+
test: searchTasks(test)
|
|
5697
|
+
};
|
|
5698
|
+
const developmentUnits = new Set([...splits.train, ...splits.selection].map((task) => task.unitId));
|
|
5699
|
+
const workerProfile = cfg.worker.workerProfile;
|
|
5700
|
+
const analystProfile = cfg.worker.analystProfile ?? workerProfile;
|
|
5701
|
+
const workerModel = concreteModelId(workerProfile.model?.default);
|
|
5702
|
+
if (!workerModel) throw new ConfigError("evolution: worker AgentProfile.model.default must name an exact model");
|
|
5703
|
+
const authorModel = concreteModelId(cfg.author.profile.model?.default);
|
|
5704
|
+
if (!authorModel) throw new ConfigError("evolution: author AgentProfile.model.default must name an exact model");
|
|
5705
|
+
const workerSource = {
|
|
5706
|
+
uri: "npm:@tangle-network/agent-runtime#runAgentic",
|
|
5707
|
+
revision: canonicalCandidateDigest$1({
|
|
5708
|
+
worker: canonicalAgentProfileDigest(workerProfile),
|
|
5709
|
+
analyst: canonicalAgentProfileDigest(analystProfile),
|
|
5935
5710
|
routerBaseUrl: cfg.worker.routerBaseUrl,
|
|
5936
|
-
profile: cfg.worker.workerProfile,
|
|
5937
|
-
analystProfile: cfg.worker.analystProfile ?? cfg.worker.workerProfile,
|
|
5938
5711
|
corpusTags: cfg.worker.corpusTags ?? [],
|
|
5939
|
-
corpusReadback: cfg.worker.corpusReadback ?? null
|
|
5712
|
+
corpusReadback: cfg.worker.corpusReadback ?? null,
|
|
5713
|
+
budget
|
|
5714
|
+
})
|
|
5715
|
+
};
|
|
5716
|
+
const authorSource = {
|
|
5717
|
+
uri: `agent-profile:${cfg.author.profile.name}`,
|
|
5718
|
+
revision: canonicalCandidateDigest$1({
|
|
5719
|
+
profile: canonicalAgentProfileDigest(cfg.author.profile),
|
|
5720
|
+
fallback: cfg.author.fallbackProfile ? canonicalAgentProfileDigest(cfg.author.fallbackProfile) : null,
|
|
5721
|
+
backend: cfg.author.executor.backend,
|
|
5722
|
+
contract: strategyAuthorContract
|
|
5723
|
+
})
|
|
5724
|
+
};
|
|
5725
|
+
const environmentSource = {
|
|
5726
|
+
uri: `environment:${cfg.environment.name}`,
|
|
5727
|
+
revision: cfg.executionRef
|
|
5728
|
+
};
|
|
5729
|
+
const execution = {
|
|
5730
|
+
model: searchModelIdentity(workerModel, workerProfile.model?.provider ?? "unknown"),
|
|
5731
|
+
agent: workerSource,
|
|
5732
|
+
benchmark: environmentSource
|
|
5733
|
+
};
|
|
5734
|
+
const searchId = `evolution-${canonicalCandidateDigest$1({
|
|
5735
|
+
process: STRATEGY_EVOLUTION_SOURCE,
|
|
5736
|
+
executionRef: cfg.executionRef,
|
|
5737
|
+
root: root.name,
|
|
5738
|
+
worker: workerSource,
|
|
5739
|
+
author: authorSource,
|
|
5740
|
+
claim,
|
|
5741
|
+
splits,
|
|
5742
|
+
policy: policy.name,
|
|
5743
|
+
allocation: allocation.name,
|
|
5744
|
+
reps: allocation.reps,
|
|
5745
|
+
maxExpansions,
|
|
5746
|
+
concurrency,
|
|
5747
|
+
maxUsd,
|
|
5748
|
+
cellUsd,
|
|
5749
|
+
deadline: cfg.deadline ?? null,
|
|
5750
|
+
maxAttempts,
|
|
5751
|
+
seed
|
|
5752
|
+
}).slice(7, 31)}`;
|
|
5753
|
+
const searchDir = `${cfg.outDir}/${searchId}`;
|
|
5754
|
+
const modulesDir = `${searchDir}/strategies`;
|
|
5755
|
+
const ledgerPath = `${searchDir}/ledger.jsonl`;
|
|
5756
|
+
mkdirSync(searchDir, { recursive: true });
|
|
5757
|
+
const recorder = await SearchRecorder.open({ ledger: openSearchLedger({
|
|
5758
|
+
path: ledgerPath,
|
|
5759
|
+
searchId
|
|
5760
|
+
}) }, {
|
|
5761
|
+
subject: `strategy/${cfg.environment.name}`,
|
|
5762
|
+
process: {
|
|
5763
|
+
name: "strategy-evolution",
|
|
5764
|
+
executionRef: STRATEGY_EVOLUTION_SOURCE
|
|
5940
5765
|
},
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
5766
|
+
artifactKind: "code",
|
|
5767
|
+
objective: {
|
|
5768
|
+
metric: "score",
|
|
5769
|
+
direction: "maximize",
|
|
5770
|
+
judge: environmentSource,
|
|
5771
|
+
claim
|
|
5945
5772
|
},
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
...cfg.hooks ? { hooks: cfg.hooks } : {}
|
|
5980
|
-
});
|
|
5981
|
-
modelsPreflighted = true;
|
|
5982
|
-
return report;
|
|
5773
|
+
splits: {
|
|
5774
|
+
...splits,
|
|
5775
|
+
heldOutUnits: splits.test.every((task) => !developmentUnits.has(task.unitId))
|
|
5776
|
+
},
|
|
5777
|
+
policy: {
|
|
5778
|
+
expansion: policy.name,
|
|
5779
|
+
allocation: allocation.name,
|
|
5780
|
+
seed
|
|
5781
|
+
},
|
|
5782
|
+
budget: {
|
|
5783
|
+
maxUsd,
|
|
5784
|
+
maxCells: null,
|
|
5785
|
+
maxNodes: 1 + maxExpansions,
|
|
5786
|
+
deadline: cfg.deadline === void 0 ? null : new Date(cfg.deadline).toISOString(),
|
|
5787
|
+
maxConcurrency: concurrency,
|
|
5788
|
+
reservedClaimUsd: maxUsd === null ? 0 : searchClaimReserveUsd({
|
|
5789
|
+
testTasks: splits.test.length,
|
|
5790
|
+
reps: allocation.reps,
|
|
5791
|
+
cellUsd
|
|
5792
|
+
})
|
|
5793
|
+
},
|
|
5794
|
+
containment: null,
|
|
5795
|
+
derivedFrom: null,
|
|
5796
|
+
identity: execution
|
|
5797
|
+
});
|
|
5798
|
+
if (!(await recorder.state()).closed) await preflightModels({
|
|
5799
|
+
worker: cfg.worker,
|
|
5800
|
+
modelPreflight: cfg.modelPreflight,
|
|
5801
|
+
modelPreflightTimeoutMs: cfg.modelPreflightTimeoutMs
|
|
5802
|
+
});
|
|
5803
|
+
const rootNode = {
|
|
5804
|
+
kind: "root",
|
|
5805
|
+
name: root.name
|
|
5983
5806
|
};
|
|
5984
|
-
const
|
|
5985
|
-
|
|
5986
|
-
|
|
5987
|
-
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
5993
|
-
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
generations: generationRows,
|
|
6029
|
-
archive,
|
|
6030
|
-
trajectory
|
|
5807
|
+
const codec = {
|
|
5808
|
+
node(rec, node) {
|
|
5809
|
+
const artifact = rec.blob("strategy", { ...node });
|
|
5810
|
+
return {
|
|
5811
|
+
artifactDigest: node.kind === "root" ? canonicalCandidateDigest$1({
|
|
5812
|
+
kind: "root",
|
|
5813
|
+
name: node.name
|
|
5814
|
+
}) : sha256Bytes$1(Buffer.from(node.code, "utf8")),
|
|
5815
|
+
artifact,
|
|
5816
|
+
surfaces: [{
|
|
5817
|
+
surfaceId: "strategy",
|
|
5818
|
+
kind: "code",
|
|
5819
|
+
artifact
|
|
5820
|
+
}]
|
|
5821
|
+
};
|
|
5822
|
+
},
|
|
5823
|
+
diff(rec, parent, child) {
|
|
5824
|
+
if (parent.kind === "root" || child.kind === "root") return { unknown: "the root is a caller-supplied strategy with no source here" };
|
|
5825
|
+
return surfaceDiff(rec, parent.code, child.code);
|
|
5826
|
+
},
|
|
5827
|
+
load(rec, node) {
|
|
5828
|
+
const stored = rec.readBlob(node.artifact);
|
|
5829
|
+
if (stored.kind === "root") {
|
|
5830
|
+
if (stored.name !== root.name) throw new Error(`evolution: node ${node.nodeId} is root '${stored.name}', not '${root.name}'`);
|
|
5831
|
+
return rootNode;
|
|
5832
|
+
}
|
|
5833
|
+
if (stored.kind === "authored" && typeof stored.code === "string") return stored;
|
|
5834
|
+
throw new Error(`evolution: node ${node.nodeId} holds no strategy`);
|
|
5835
|
+
}
|
|
5836
|
+
};
|
|
5837
|
+
const loaded = /* @__PURE__ */ new Map();
|
|
5838
|
+
const strategyOf = (node) => {
|
|
5839
|
+
if (node.kind === "root") return Promise.resolve(root);
|
|
5840
|
+
let strategy = loaded.get(node.code);
|
|
5841
|
+
if (!strategy) {
|
|
5842
|
+
strategy = loadAuthoredStrategy(node.code, modulesDir).then((module) => module.strategy);
|
|
5843
|
+
loaded.set(node.code, strategy);
|
|
5844
|
+
}
|
|
5845
|
+
return strategy;
|
|
5846
|
+
};
|
|
5847
|
+
const storage = fsCampaignStorage();
|
|
5848
|
+
const costLedger = createRunCostLedger({
|
|
5849
|
+
storage,
|
|
5850
|
+
runDir: searchDir
|
|
6031
5851
|
});
|
|
6032
|
-
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
|
|
6036
|
-
|
|
6037
|
-
|
|
6038
|
-
|
|
6039
|
-
|
|
5852
|
+
const lanes = [dedicatedLane({
|
|
5853
|
+
name: "in-process",
|
|
5854
|
+
capacity: concurrency,
|
|
5855
|
+
cellUsd
|
|
5856
|
+
})];
|
|
5857
|
+
await reconcileInterruptedSearchCalls(costLedger, searchId, lanes);
|
|
5858
|
+
const executor = searchExecutor({
|
|
5859
|
+
lanes,
|
|
5860
|
+
profileOf: () => workerProfile,
|
|
5861
|
+
costLedger,
|
|
5862
|
+
identity: execution,
|
|
5863
|
+
storage,
|
|
5864
|
+
dir: searchDir,
|
|
5865
|
+
...cfg.workerSlots ? { workerSlots: cfg.workerSlots } : {},
|
|
5866
|
+
async runAttempt({ work, lane, costLedger: cellLedger }) {
|
|
5867
|
+
if (work.artifact.kind === "authored" && work.artifact.refusal !== null) throw new Error(`evolution: refused node ${work.nodeId} was given a cell`);
|
|
5868
|
+
const strategy = await strategyOf(work.artifact);
|
|
5869
|
+
const task = taskById.get(work.taskId);
|
|
5870
|
+
if (!task) throw new Error(`evolution: cell ${work.cellId} names unknown task ${work.taskId}`);
|
|
5871
|
+
const faults = [];
|
|
5872
|
+
const environment = faultRecording(cfg.environment, faults);
|
|
5873
|
+
const paid = await cellLedger.runPaidCall({
|
|
5874
|
+
channel: "agent",
|
|
5875
|
+
phase: `search.${work.stage}`,
|
|
5876
|
+
actor: `strategy:${strategy.name}`,
|
|
5877
|
+
model: workerModel,
|
|
5878
|
+
signal: work.signal,
|
|
5879
|
+
execute: () => runAgentic({
|
|
5880
|
+
...cfg.worker,
|
|
5881
|
+
surface: environment,
|
|
5882
|
+
task,
|
|
5883
|
+
strategy,
|
|
5884
|
+
budget,
|
|
5885
|
+
...cfg.hooks ? { hooks: cfg.hooks } : {}
|
|
5886
|
+
}),
|
|
5887
|
+
receipt: (run) => ({
|
|
5888
|
+
model: workerModel,
|
|
5889
|
+
inputTokens: run.tokens.input,
|
|
5890
|
+
outputTokens: run.tokens.output,
|
|
5891
|
+
...run.usdKnown ? { actualCostUsd: run.usd } : { costUnknown: true },
|
|
5892
|
+
...run.tokensKnown ? {} : { usageUnknown: true }
|
|
5893
|
+
}),
|
|
5894
|
+
receiptFromError: () => ({
|
|
5895
|
+
model: workerModel,
|
|
5896
|
+
inputTokens: 0,
|
|
5897
|
+
outputTokens: 0,
|
|
5898
|
+
costUnknown: true,
|
|
5899
|
+
usageUnknown: true
|
|
5900
|
+
})
|
|
5901
|
+
});
|
|
5902
|
+
const settled = {
|
|
5903
|
+
identity: execution,
|
|
5904
|
+
placement: {
|
|
5905
|
+
lane: lane.name,
|
|
5906
|
+
boxId: null
|
|
5907
|
+
},
|
|
5908
|
+
accounting: unknownAccounting("replaced by the attempt receipts"),
|
|
5909
|
+
...paid.succeeded ? { wallMs: paid.value.ms } : {}
|
|
5910
|
+
};
|
|
5911
|
+
if (faults.length > 0) return {
|
|
5912
|
+
...settled,
|
|
5913
|
+
...environmentFault(faults)
|
|
5914
|
+
};
|
|
5915
|
+
if (!paid.succeeded) return {
|
|
5916
|
+
...settled,
|
|
5917
|
+
outcome: {
|
|
5918
|
+
status: "failed",
|
|
5919
|
+
score: 0,
|
|
5920
|
+
metrics: {},
|
|
5921
|
+
failure: {
|
|
5922
|
+
code: "strategy-threw",
|
|
5923
|
+
message: shareText(paid.error)
|
|
5924
|
+
}
|
|
5925
|
+
}
|
|
5926
|
+
};
|
|
5927
|
+
const run = paid.value;
|
|
5928
|
+
return {
|
|
5929
|
+
...settled,
|
|
5930
|
+
outcome: {
|
|
5931
|
+
status: "passed",
|
|
5932
|
+
score: run.score,
|
|
5933
|
+
metrics: {
|
|
5934
|
+
resolved: run.resolved ? 1 : 0,
|
|
5935
|
+
shots: run.shots
|
|
5936
|
+
}
|
|
5937
|
+
}
|
|
5938
|
+
};
|
|
5939
|
+
}
|
|
5940
|
+
});
|
|
5941
|
+
let toolCatalog;
|
|
5942
|
+
const catalog = () => {
|
|
5943
|
+
toolCatalog ??= listToolCatalog(cfg.environment, train[0]);
|
|
5944
|
+
return toolCatalog;
|
|
5945
|
+
};
|
|
5946
|
+
const authorProvider = cfg.author.profile.model?.provider ?? "unknown";
|
|
5947
|
+
const result = await runSearch({
|
|
5948
|
+
recorder,
|
|
5949
|
+
root: rootNode,
|
|
5950
|
+
codec,
|
|
5951
|
+
policy,
|
|
5952
|
+
allocation,
|
|
5953
|
+
proposer: {
|
|
5954
|
+
name: "strategy-author",
|
|
5955
|
+
kind: "frontier-author",
|
|
5956
|
+
source: authorSource,
|
|
5957
|
+
execution: {
|
|
5958
|
+
kind: "model",
|
|
5959
|
+
model: searchModelIdentity(authorModel, authorProvider),
|
|
5960
|
+
source: authorSource
|
|
5961
|
+
},
|
|
5962
|
+
childrenPerProposal: 1,
|
|
5963
|
+
async propose(request) {
|
|
5964
|
+
const state = await recorder.state();
|
|
5965
|
+
const names = strategyNames(state, recorder, codec);
|
|
5966
|
+
const losses = JSON.stringify(strategyTournament(state, "train", names).rows.map((row) => ({
|
|
5967
|
+
task: row.taskId,
|
|
5968
|
+
...row.rep > 0 ? { rep: row.rep } : {},
|
|
5969
|
+
cells: row.cells
|
|
5970
|
+
}))).slice(0, 12e3);
|
|
5971
|
+
const summary = renderSearchSummary(state, { split: "train" });
|
|
5972
|
+
const parent = request.parents[0];
|
|
5973
|
+
const parentName = names.get(parent.nodeId) ?? parent.nodeId;
|
|
5974
|
+
const tried = [...names.values()].join(", ");
|
|
5975
|
+
const parentSource = parent.artifact.kind === "authored" ? `\n\nPARENT STRATEGY "${parentName}" (improve on it):\n${parent.artifact.code}` : `\n\nPARENT STRATEGY "${parentName}" is the built-in baseline; improve on it.`;
|
|
5976
|
+
const contract = `${strategyAuthorContract}\n\nEXAMPLE TOOLS FROM ONE TASK (tool sets VARY per task on this domain — a strategy MUST select tool names from await listTools(handle) at runtime; hardcoding these example names will zero your score on most tasks):\n${await catalog()}\n\nTHE SEARCH SO FAR (train split):\n${summary}\n\nSTRATEGIES ALREADY TRIED (author something MEANINGFULLY different — a new composition, not a rename): ${tried}${parentSource}`;
|
|
5977
|
+
const source = await requestStrategySource({
|
|
6040
5978
|
profile: cfg.author.profile,
|
|
6041
5979
|
executor: cfg.author.executor,
|
|
6042
5980
|
...cfg.author.fallbackProfile ? { fallbackProfile: cfg.author.fallbackProfile } : {},
|
|
6043
5981
|
contract,
|
|
6044
5982
|
environmentName: cfg.environment.name,
|
|
6045
|
-
lossesJson,
|
|
5983
|
+
lossesJson: losses,
|
|
6046
5984
|
budget,
|
|
6047
|
-
outDir:
|
|
6048
|
-
|
|
6049
|
-
const unique = byName.has(authored.strategy.name) ? `${authored.strategy.name}-g${g}c${i + 1}` : authored.strategy.name;
|
|
6050
|
-
const strategy = renameStrategy(authored.strategy, unique);
|
|
6051
|
-
byName.set(unique, strategy);
|
|
6052
|
-
codeByName.set(unique, authored.code);
|
|
6053
|
-
newStrategies.push(strategy);
|
|
6054
|
-
archive.push({
|
|
6055
|
-
name: unique,
|
|
6056
|
-
source: "authored",
|
|
6057
|
-
generation: g,
|
|
6058
|
-
parent: incumbent.name,
|
|
6059
|
-
gzipBits: gzipSync(Buffer.from(authored.code)).length * 8,
|
|
6060
|
-
file: authored.file,
|
|
6061
|
-
score: 0,
|
|
6062
|
-
usd: 0
|
|
5985
|
+
outDir: modulesDir,
|
|
5986
|
+
signal: request.signal
|
|
6063
5987
|
});
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
5988
|
+
let child = {
|
|
5989
|
+
kind: "authored",
|
|
5990
|
+
name: null,
|
|
5991
|
+
code: "",
|
|
5992
|
+
refusal: NO_MODULE
|
|
5993
|
+
};
|
|
5994
|
+
if (source.code !== null) try {
|
|
5995
|
+
const { strategy } = await loadAuthoredStrategy(source.code, modulesDir);
|
|
5996
|
+
child = {
|
|
5997
|
+
kind: "authored",
|
|
5998
|
+
name: strategy.name,
|
|
5999
|
+
code: source.code,
|
|
6000
|
+
refusal: null
|
|
6001
|
+
};
|
|
6002
|
+
} catch (error) {
|
|
6003
|
+
child = {
|
|
6004
|
+
kind: "authored",
|
|
6005
|
+
name: null,
|
|
6006
|
+
code: source.code,
|
|
6007
|
+
refusal: shareText(error)
|
|
6008
|
+
};
|
|
6009
|
+
}
|
|
6010
|
+
const answered = source.turns.at(-1);
|
|
6011
|
+
return {
|
|
6012
|
+
children: [{
|
|
6013
|
+
artifact: child,
|
|
6014
|
+
label: child.name ?? "refused module",
|
|
6015
|
+
rationale: `authored from the train results of ${parentName} and ${names.size - 1} other strategies`
|
|
6016
|
+
}],
|
|
6017
|
+
accounting: authorAccounting(source.turns),
|
|
6018
|
+
...answered ? { execution: {
|
|
6019
|
+
kind: "model",
|
|
6020
|
+
model: searchModelIdentity(answered.model, authorProvider),
|
|
6021
|
+
source: authorSource
|
|
6022
|
+
} } : {}
|
|
6023
|
+
};
|
|
6088
6024
|
}
|
|
6025
|
+
},
|
|
6026
|
+
executor,
|
|
6027
|
+
admit: (node) => node.kind === "authored" ? node.refusal : null,
|
|
6028
|
+
maxExpansions,
|
|
6029
|
+
maxAttempts,
|
|
6030
|
+
...cfg.signal ? { signal: cfg.signal } : {}
|
|
6031
|
+
});
|
|
6032
|
+
if (!result.claim || !result.claimVerification) throw new Error(`evolution: search ${searchId} closed without a claim`);
|
|
6033
|
+
const { state } = result;
|
|
6034
|
+
const names = strategyNames(state, recorder, codec);
|
|
6035
|
+
const rootId = state.rootNodeId;
|
|
6036
|
+
const strategies = state.nodes().map((node) => {
|
|
6037
|
+
const artifact = codec.load(recorder, node);
|
|
6038
|
+
const authored = artifact.kind === "authored" ? artifact : null;
|
|
6039
|
+
return {
|
|
6040
|
+
name: names.get(node.nodeId),
|
|
6041
|
+
nodeId: node.nodeId,
|
|
6042
|
+
parent: node.primaryParentId === null ? null : names.get(node.primaryParentId),
|
|
6043
|
+
source: artifact.kind,
|
|
6044
|
+
status: node.status,
|
|
6045
|
+
code: authored?.code ?? null,
|
|
6046
|
+
gzipBits: authored ? gzipSync(Buffer.from(authored.code)).length * 8 : null,
|
|
6047
|
+
refusal: authored?.refusal ?? null,
|
|
6048
|
+
selection: node.nodeId === rootId ? null : estimateNode(state, node.nodeId, {
|
|
6049
|
+
against: rootId,
|
|
6050
|
+
split: "selection"
|
|
6051
|
+
})
|
|
6052
|
+
};
|
|
6053
|
+
});
|
|
6054
|
+
const claimResult = result.claim;
|
|
6055
|
+
const shipped = claimResult.finalists.find((finalist) => finalist.nodeId === claimResult.selected && finalist.test !== null);
|
|
6056
|
+
const { spend } = state.audit;
|
|
6057
|
+
const { decision, reason } = runtimeShipDecision({
|
|
6058
|
+
claim: claimResult,
|
|
6059
|
+
verified: result.claimVerification.status === "verified",
|
|
6060
|
+
accountingComplete: spend.unknownCostCells === 0 && spend.unknownCostOperations === 0,
|
|
6061
|
+
lowerBound: shipped?.test?.interval[0] ?? null,
|
|
6062
|
+
minimumLift: 0
|
|
6063
|
+
});
|
|
6064
|
+
return {
|
|
6065
|
+
searchId,
|
|
6066
|
+
ledger: ledgerPath,
|
|
6067
|
+
closeReason: result.reason,
|
|
6068
|
+
strategies,
|
|
6069
|
+
selected: strategies.find((strategy) => strategy.nodeId === result.leader),
|
|
6070
|
+
claim: claimResult,
|
|
6071
|
+
claimVerification: result.claimVerification,
|
|
6072
|
+
decision,
|
|
6073
|
+
reason,
|
|
6074
|
+
tournament: {
|
|
6075
|
+
train: strategyTournament(state, "train", names),
|
|
6076
|
+
selection: strategyTournament(state, "selection", names),
|
|
6077
|
+
test: strategyTournament(state, "test", names)
|
|
6078
|
+
},
|
|
6079
|
+
spend: {
|
|
6080
|
+
knownUsd: spend.knownUsd,
|
|
6081
|
+
floorUsd: spend.floorUsd,
|
|
6082
|
+
unknownCostCells: spend.unknownCostCells,
|
|
6083
|
+
unknownCostOperations: spend.unknownCostOperations
|
|
6089
6084
|
}
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
|
|
6095
|
-
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
generations: generationRows,
|
|
6110
|
-
archive,
|
|
6111
|
-
trajectory
|
|
6112
|
-
});
|
|
6085
|
+
};
|
|
6086
|
+
}
|
|
6087
|
+
/**
|
|
6088
|
+
* The tournament on one split, projected from the ledger: each settled cell's final attempt,
|
|
6089
|
+
* by task and repeat, keyed by strategy name. `names` maps node ids to report names.
|
|
6090
|
+
*/
|
|
6091
|
+
function strategyTournament(state, split, names) {
|
|
6092
|
+
const rows = /* @__PURE__ */ new Map();
|
|
6093
|
+
const perStrategy = {};
|
|
6094
|
+
for (const nodeId of state.nodeIds()) {
|
|
6095
|
+
const name = names.get(nodeId);
|
|
6096
|
+
if (name === void 0) throw new Error(`strategyTournament: node ${nodeId} has no name`);
|
|
6097
|
+
perStrategy[name] = {
|
|
6098
|
+
cells: 0,
|
|
6099
|
+
scored: 0,
|
|
6100
|
+
score: null,
|
|
6101
|
+
usd: 0,
|
|
6102
|
+
usdKnown: true
|
|
6103
|
+
};
|
|
6113
6104
|
}
|
|
6114
|
-
|
|
6115
|
-
const
|
|
6116
|
-
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6120
|
-
|
|
6121
|
-
|
|
6122
|
-
|
|
6123
|
-
|
|
6124
|
-
|
|
6125
|
-
|
|
6126
|
-
|
|
6127
|
-
|
|
6105
|
+
const sums = /* @__PURE__ */ new Map();
|
|
6106
|
+
const cells = state.cells().filter((cell) => cell.split === split && cell.outcome !== null).sort((left, right) => compare(left.taskId, right.taskId) || left.rep - right.rep);
|
|
6107
|
+
for (const cell of cells) {
|
|
6108
|
+
const name = names.get(cell.nodeId);
|
|
6109
|
+
const key = `${cell.taskId}\u0000${cell.rep}`;
|
|
6110
|
+
let row = rows.get(key);
|
|
6111
|
+
if (!row) {
|
|
6112
|
+
row = {
|
|
6113
|
+
taskId: cell.taskId,
|
|
6114
|
+
unitId: cell.unitId,
|
|
6115
|
+
rep: cell.rep,
|
|
6116
|
+
cells: {}
|
|
6117
|
+
};
|
|
6118
|
+
rows.set(key, row);
|
|
6128
6119
|
}
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
6135
|
-
score: r.cells?.[reference.name]?.score ?? 0
|
|
6136
|
-
}));
|
|
6137
|
-
const inBandIds = new Set(refScores.filter((r) => r.score <= maxRef).map((r) => r.taskId));
|
|
6138
|
-
const kept = pool.filter((t) => inBandIds.has(t.id));
|
|
6139
|
-
if (kept.length < cfg.holdoutN) throw new Error(`evolution band: only ${kept.length}/${cfg.holdoutN} holdout tasks have headroom (pool ${cfg.band.holdoutPoolN}, reference "${reference.name}" ≤ ${maxRef}) — widen holdoutPoolN or raise maxRefScore`);
|
|
6140
|
-
holdoutTasks = kept.slice(0, cfg.holdoutN);
|
|
6141
|
-
bandInfo = {
|
|
6142
|
-
screened: refScores.length,
|
|
6143
|
-
inBand: kept.length,
|
|
6144
|
-
refScores
|
|
6120
|
+
const outcome = cell.outcome;
|
|
6121
|
+
row.cells[name] = {
|
|
6122
|
+
score: cell.score,
|
|
6123
|
+
outcome,
|
|
6124
|
+
usd: cell.spentUsd,
|
|
6125
|
+
usdKnown: cell.costKnown
|
|
6145
6126
|
};
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
holdout = await bench("holdout", holdoutTasks, finalists);
|
|
6155
|
-
verdict = promotionGate({
|
|
6156
|
-
report: holdout,
|
|
6157
|
-
incumbent: gen0Champion.name,
|
|
6158
|
-
candidate: incumbent.name,
|
|
6159
|
-
...cfg.objective === "cost" ? {
|
|
6160
|
-
mode: "non-inferiority",
|
|
6161
|
-
...cfg.scoreTolerance !== void 0 ? { scoreTolerance: cfg.scoreTolerance } : {}
|
|
6162
|
-
} : {},
|
|
6163
|
-
...cfg.minPairedTasks !== void 0 ? { minPairedTasks: cfg.minPairedTasks } : {}
|
|
6164
|
-
});
|
|
6165
|
-
save({
|
|
6166
|
-
gen0,
|
|
6167
|
-
gen0Champion,
|
|
6168
|
-
generations: generationRows,
|
|
6169
|
-
archive,
|
|
6170
|
-
trajectory,
|
|
6171
|
-
holdout,
|
|
6172
|
-
verdict,
|
|
6173
|
-
...bandInfo ? { band: bandInfo } : {}
|
|
6174
|
-
});
|
|
6127
|
+
const summary = perStrategy[name];
|
|
6128
|
+
summary.cells += 1;
|
|
6129
|
+
summary.usd += cell.spentUsd;
|
|
6130
|
+
summary.usdKnown &&= cell.costKnown;
|
|
6131
|
+
if (cell.score !== null) {
|
|
6132
|
+
summary.scored += 1;
|
|
6133
|
+
sums.set(name, (sums.get(name) ?? 0) + cell.score);
|
|
6134
|
+
}
|
|
6175
6135
|
}
|
|
6176
|
-
|
|
6177
|
-
|
|
6178
|
-
|
|
6179
|
-
|
|
6180
|
-
|
|
6181
|
-
|
|
6136
|
+
for (const [name, summary] of Object.entries(perStrategy)) summary.score = summary.scored === 0 ? null : sums.get(name) / summary.scored;
|
|
6137
|
+
return {
|
|
6138
|
+
split,
|
|
6139
|
+
rows: [...rows.values()],
|
|
6140
|
+
perStrategy
|
|
6141
|
+
};
|
|
6142
|
+
}
|
|
6143
|
+
/** Report names: each node's strategy name, suffixed `~<ordinal>` when an earlier node has it. */
|
|
6144
|
+
function strategyNames(state, recorder, codec) {
|
|
6145
|
+
const names = /* @__PURE__ */ new Map();
|
|
6146
|
+
const taken = /* @__PURE__ */ new Set();
|
|
6147
|
+
for (const node of state.nodes()) {
|
|
6148
|
+
const base = codec.load(recorder, node).name ?? "refused-module";
|
|
6149
|
+
const name = taken.has(base) ? `${base}~${node.ordinal}` : base;
|
|
6150
|
+
taken.add(name);
|
|
6151
|
+
names.set(node.nodeId, name);
|
|
6152
|
+
}
|
|
6153
|
+
return names;
|
|
6154
|
+
}
|
|
6155
|
+
/** The environment as one attempt sees it, recording each throw from `open`, `tools`,
|
|
6156
|
+
* `score` or `close`: those are the environment's faults. A `call` that throws is a tool
|
|
6157
|
+
* error the agent's loop handles. */
|
|
6158
|
+
function faultRecording(environment, faults) {
|
|
6159
|
+
const recorded = (name, method) => async (...args) => {
|
|
6182
6160
|
try {
|
|
6183
|
-
|
|
6184
|
-
|
|
6185
|
-
|
|
6186
|
-
|
|
6187
|
-
...cfg.author.profile.prompt,
|
|
6188
|
-
systemPrompt: `Summarize the optimization strategy implemented by this code in at most ${words} words. Describe the COMPOSITION (shots, critique, artifact handling, restarts, stopping) — not the code. Output only the summary.`
|
|
6189
|
-
}
|
|
6190
|
-
},
|
|
6191
|
-
executor: cfg.author.executor,
|
|
6192
|
-
context: "strategy reproducer summary"
|
|
6193
|
-
}).chat({ messages: [{
|
|
6194
|
-
role: "user",
|
|
6195
|
-
content: championCode
|
|
6196
|
-
}] })).content.trim();
|
|
6197
|
-
const reproduced = await authorStrategy({
|
|
6198
|
-
profile: cfg.author.profile,
|
|
6199
|
-
executor: cfg.author.executor,
|
|
6200
|
-
...cfg.author.fallbackProfile ? { fallbackProfile: cfg.author.fallbackProfile } : {},
|
|
6201
|
-
contract: `${strategyAuthorContract}\n\nIMPLEMENT EXACTLY THIS STRATEGY (a colleague's description — do not invent a different approach):\n${summary}`,
|
|
6202
|
-
environmentName: cfg.environment.name,
|
|
6203
|
-
lossesJson: "[]",
|
|
6204
|
-
budget,
|
|
6205
|
-
outDir: cfg.outDir
|
|
6206
|
-
});
|
|
6207
|
-
const reproStrategy = {
|
|
6208
|
-
name: `${incumbent.name}-reproduced`,
|
|
6209
|
-
driver: reproduced.strategy.driver
|
|
6210
|
-
};
|
|
6211
|
-
const reproducedHoldoutScore = (await bench("reproduce", holdoutTasks, [reproStrategy])).perStrategy[reproStrategy.name]?.score ?? 0;
|
|
6212
|
-
reproduction = {
|
|
6213
|
-
summary,
|
|
6214
|
-
reproducedName: reproStrategy.name,
|
|
6215
|
-
file: reproduced.file,
|
|
6216
|
-
championHoldoutScore,
|
|
6217
|
-
reproducedHoldoutScore,
|
|
6218
|
-
gap: championHoldoutScore - reproducedHoldoutScore,
|
|
6219
|
-
reproducible: reproducedHoldoutScore >= championHoldoutScore - tolerance
|
|
6220
|
-
};
|
|
6221
|
-
} catch (e) {
|
|
6222
|
-
reproduction = {
|
|
6223
|
-
summary: "",
|
|
6224
|
-
reproducedName: "",
|
|
6225
|
-
championHoldoutScore,
|
|
6226
|
-
reproducedHoldoutScore: 0,
|
|
6227
|
-
gap: championHoldoutScore,
|
|
6228
|
-
reproducible: false,
|
|
6229
|
-
error: e instanceof Error ? e.message.slice(0, 300) : String(e)
|
|
6230
|
-
};
|
|
6161
|
+
return await method(...args);
|
|
6162
|
+
} catch (error) {
|
|
6163
|
+
faults.push(`${name}: ${shareText(error)}`);
|
|
6164
|
+
throw error;
|
|
6231
6165
|
}
|
|
6232
|
-
}
|
|
6166
|
+
};
|
|
6233
6167
|
return {
|
|
6234
|
-
|
|
6235
|
-
|
|
6236
|
-
|
|
6237
|
-
|
|
6238
|
-
|
|
6239
|
-
|
|
6240
|
-
|
|
6241
|
-
|
|
6242
|
-
|
|
6243
|
-
|
|
6244
|
-
|
|
6245
|
-
}
|
|
6246
|
-
|
|
6247
|
-
|
|
6248
|
-
|
|
6249
|
-
|
|
6250
|
-
|
|
6251
|
-
|
|
6168
|
+
name: environment.name,
|
|
6169
|
+
open: recorded("open", (task) => environment.open(task)),
|
|
6170
|
+
tools: recorded("tools", (task, handle) => environment.tools(task, handle)),
|
|
6171
|
+
call: (handle, name, args) => environment.call(handle, name, args),
|
|
6172
|
+
score: recorded("score", (task, handle) => environment.score(task, handle)),
|
|
6173
|
+
close: recorded("close", (handle) => environment.close(handle))
|
|
6174
|
+
};
|
|
6175
|
+
}
|
|
6176
|
+
function environmentFault(faults) {
|
|
6177
|
+
return { outcome: {
|
|
6178
|
+
status: "errored",
|
|
6179
|
+
metrics: {},
|
|
6180
|
+
error: {
|
|
6181
|
+
code: "environment-fault",
|
|
6182
|
+
message: faults[0],
|
|
6183
|
+
retryable: true
|
|
6184
|
+
}
|
|
6185
|
+
} };
|
|
6186
|
+
}
|
|
6187
|
+
function authorAccounting(turns) {
|
|
6188
|
+
let usd = 0;
|
|
6189
|
+
let costKnown = true;
|
|
6190
|
+
for (const turn of turns) if (turn.costUsd === null) costKnown = false;
|
|
6191
|
+
else usd += turn.costUsd;
|
|
6192
|
+
return {
|
|
6193
|
+
tokens: turns.every((turn) => turn.tokensKnown) ? {
|
|
6194
|
+
status: "known",
|
|
6195
|
+
inputTokens: turns.reduce((sum, turn) => sum + turn.inputTokens, 0),
|
|
6196
|
+
outputTokens: turns.reduce((sum, turn) => sum + turn.outputTokens, 0),
|
|
6197
|
+
cachedTokens: 0
|
|
6198
|
+
} : {
|
|
6199
|
+
status: "unknown",
|
|
6200
|
+
reason: "an authoring call reported no token usage"
|
|
6201
|
+
},
|
|
6202
|
+
cost: costKnown ? {
|
|
6203
|
+
status: "known",
|
|
6204
|
+
usd,
|
|
6205
|
+
source: usd === 0 ? "free" : "provider"
|
|
6206
|
+
} : {
|
|
6207
|
+
status: "unknown",
|
|
6208
|
+
knownLowerBoundUsd: usd,
|
|
6209
|
+
reason: "an authoring call reported no billed cost"
|
|
6210
|
+
}
|
|
6211
|
+
};
|
|
6212
|
+
}
|
|
6213
|
+
function unknownAccounting(reason) {
|
|
6214
|
+
return {
|
|
6215
|
+
tokens: {
|
|
6216
|
+
status: "unknown",
|
|
6217
|
+
reason
|
|
6218
|
+
},
|
|
6219
|
+
cost: {
|
|
6220
|
+
status: "unknown",
|
|
6221
|
+
knownLowerBoundUsd: 0,
|
|
6222
|
+
reason
|
|
6223
|
+
}
|
|
6224
|
+
};
|
|
6225
|
+
}
|
|
6226
|
+
/** One probe round-trip lists the domain's tools so the author can focus shots, names and
|
|
6227
|
+
* descriptions only, never the implementations. */
|
|
6228
|
+
async function listToolCatalog(environment, task) {
|
|
6229
|
+
const handle = await environment.open(task);
|
|
6230
|
+
try {
|
|
6231
|
+
return (await environment.tools(task, handle)).map((tool) => `- ${tool.function.name}${tool.function.description ? ` — ${tool.function.description.slice(0, 120)}` : ""}`).join("\n");
|
|
6232
|
+
} finally {
|
|
6233
|
+
await environment.close(handle);
|
|
6252
6234
|
}
|
|
6253
|
-
|
|
6235
|
+
}
|
|
6236
|
+
/** Free text as the ledger stores it: redacted with the share profile and bounded. */
|
|
6237
|
+
function shareText(error) {
|
|
6238
|
+
return redact(error instanceof Error ? error.message : String(error), { profile: "share" }).value.slice(0, 300);
|
|
6239
|
+
}
|
|
6240
|
+
function compare(left, right) {
|
|
6241
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
6242
|
+
}
|
|
6243
|
+
function positiveInteger(name, value) {
|
|
6244
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new ConfigError(`evolution: ${name} must be a positive integer`);
|
|
6245
|
+
return value;
|
|
6246
|
+
}
|
|
6247
|
+
function nonNegativeInteger(name, value) {
|
|
6248
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new ConfigError(`evolution: ${name} must be a non-negative integer`);
|
|
6249
|
+
return value;
|
|
6250
|
+
}
|
|
6251
|
+
function dollars(name, value) {
|
|
6252
|
+
if (value === void 0) return null;
|
|
6253
|
+
if (!Number.isFinite(value) || value < 0) throw new ConfigError(`evolution: ${name} must be a finite non-negative number`);
|
|
6254
|
+
return value;
|
|
6254
6255
|
}
|
|
6255
6256
|
//#endregion
|
|
6256
6257
|
//#region src/runtime/supervise/chat-transport-executor.ts
|
|
@@ -8241,6 +8242,6 @@ function tail(s) {
|
|
|
8241
8242
|
return s.slice(-400);
|
|
8242
8243
|
}
|
|
8243
8244
|
//#endregion
|
|
8244
|
-
export {
|
|
8245
|
+
export { renderCorpusToInstructions as $, resolveMcpServerLaunch as $t, unsafeInProcessRunner as A, completionAuthorizes as At, SandboxRunAbortError as B, auditIntent as Bt, withUntrackedArtifacts as C, assertDeclaredCheck as Ct, patchDelivered as D, declaredCheckJudge as Dt, createSqlRunContext as E, declaredCheckDigest as Et, strategyTournament as F, pairwiseSignificance as Ft, trajectoryReport as G, SqlSpawnJournal as Gt, printBenchmarkReport as H, SqlRunOwnershipError as Ht, assertStrategyContract as I, renderLeaderboardHtml as It, builtinShapes as J, materializeLocalMcp as Jt, definePersona as K, McpSpawnFault as Kt, authorStrategy as L, renderLeaderboardMarkdown as Lt, chatWorkerSeam as M, sentinelCompletion as Mt, createChatSessionStore as N, stopSentinel as Nt, runCoderChecks as O, readDeclaredCheck as Ot, runStrategyEvolution as P, leaderboard as Pt, InMemoryCorpus as Q, mcpSecretEnvMetadataKey as Qt, strategyAuthorContract as R, renderLeaderboardSvg as Rt, copyUntrackedIntoClone as S, superviseDispatch as St, createFencedSqlRunContext as T, declaredCheckDeliverable as Tt, runBenchmark as U, openSqlRunStore as Ut, openSandboxRun as V, defaultAuditorInstruction as Vt, equalKOnCost as W, SqlResultBlobStore as Wt, registerShape as X, sanitizeMcpToolSchema as Xt, createShapeRegistry as Y, createMcpEnvironment as Yt, FileCorpus as Z, envKeyProvider as Zt, NOTE_MAX_CHARS as _, resolveSandboxClient as _t, localShell as a, selectValidWinner as at, composeWorkerEvidence as b, loopCampaignDispatch as bt, createVerifierEnvironment as c, assertTraceDerivedFindings as ct, harvestSurfaceDiffs as d, registryScopeAnalyst as dt, resolveSecretEnv as en, fanout as et, analystsFromRegistry as f, observationFromRegistry as ft, EVIDENCE_MAX_CHARS as g, defineLeaderboard as gt, worktreeFanout as h, harvestCorpus as ht, jjWorkspace as i, pipeline as it, chatTransportExecutor as j, deterministicCompletion as jt, codeModeSupervisorTools as k, runIsolatedCheck as kt, boxSurfaceReader as l, buildSteerContext as lt, superviseSurface as m, HarvestError as mt, makeFinding$1 as n, createTangleSandboxExactProcessProvider as nn, loopUntil as nt, runInWorkspace as o, verify as ot, failuresAnalyst as p, inProcessSandboxClient as pt, runPersonified as q, connectStdioMcp as qt, gitWorkspace as r, panel as rt, createWaterfallCollector as s, widen as st, computeFindingId$1 as t, secretEnvOfMcpServer as tn, flatWidenGate as tt, fsSurfaceReader as u, createScopeAnalyst as ut, VERIFY_TAIL_CHARS as v, localSandboxClient as vt, analyzeTrace as w, checkProgramDigest as wt, settledWorkerOut as x, loopDispatch as xt, closingWorkerNote as y, inlineSandboxClient as yt, strategyAuthorSystemPrompt as z, renderPairwiseMarkdown as zt };
|
|
8245
8246
|
|
|
8246
|
-
//# sourceMappingURL=runtime-
|
|
8247
|
+
//# sourceMappingURL=runtime-Bid-kBes.js.map
|