@tangle-network/agent-runtime 0.281.1 → 0.282.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-C1ZAyQrQ.js → activation-Bj8SMTOs.js} +3 -3
- package/dist/{activation-C1ZAyQrQ.js.map → activation-Bj8SMTOs.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +2 -2
- package/dist/{bundle-CvLGRVO-.js → bundle-BfW-32vB.js} +2 -2
- package/dist/{bundle-CvLGRVO-.js.map → bundle-BfW-32vB.js.map} +1 -1
- package/dist/candidate-execution/index.js +5 -5
- package/dist/{coordination-driver-B-HOpGN4.js → coordination-driver-BmkANXc5.js} +2 -2
- package/dist/{coordination-driver-B-HOpGN4.js.map → coordination-driver-BmkANXc5.js.map} +1 -1
- package/dist/{delegate-C1eIBN70.js → delegate-BfHKdHzV.js} +2 -2
- package/dist/{delegate-C1eIBN70.js.map → delegate-BfHKdHzV.js.map} +1 -1
- package/dist/durable.d.ts +1 -1
- package/dist/durable.js +3 -3
- package/dist/{graph-KjJaTmbO.js → graph-XmlRTlaI.js} +4 -4
- package/dist/{graph-KjJaTmbO.js.map → graph-XmlRTlaI.js.map} +1 -1
- package/dist/{improvement-cycle-DCTCTmNl.js → improvement-cycle-BUDfOR0s.js} +11 -1775
- package/dist/improvement-cycle-BUDfOR0s.js.map +1 -0
- package/dist/{index-C4ygxavE.d.ts → index-CkLB8mJF.d.ts} +135 -257
- package/dist/index.d.ts +2 -2
- package/dist/index.js +9 -9
- package/dist/intelligence.js +8 -7
- package/dist/intelligence.js.map +1 -1
- package/dist/kernel.d.ts +2 -2
- package/dist/kernel.js +9 -9
- package/dist/{loop-runner-bin-Co2eBFbb.js → loop-runner-bin-BhExTk_4.js} +3 -3
- package/dist/{loop-runner-bin-Co2eBFbb.js.map → loop-runner-bin-BhExTk_4.js.map} +1 -1
- package/dist/{loop-runner-bin-B9dTzFL3.d.ts → loop-runner-bin-CINywrHK.d.ts} +2 -2
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +4 -4
- package/dist/{prepare-DVlvGP7V.js → prepare-CxnyZ1fG.js} +3 -3
- package/dist/{prepare-DVlvGP7V.js.map → prepare-CxnyZ1fG.js.map} +1 -1
- package/dist/{profile-D3eXNBQV.js → profile-C4NDjZyB.js} +2 -2
- package/dist/{profile-D3eXNBQV.js.map → profile-C4NDjZyB.js.map} +1 -1
- package/dist/{protected-model-port-BrWgsCnl.js → protected-model-port-BSsgmdhI.js} +4 -4
- package/dist/{protected-model-port-BrWgsCnl.js.map → protected-model-port-BSsgmdhI.js.map} +1 -1
- package/dist/{provision-supervisor-CkN8RRjv.js → provision-supervisor-BNUmrHwC.js} +3 -3
- package/dist/{provision-supervisor-CkN8RRjv.js.map → provision-supervisor-BNUmrHwC.js.map} +1 -1
- package/dist/{runtime-B_yRPBjj.js → runtime-B3TTjtzE.js} +760 -696
- package/dist/runtime-B3TTjtzE.js.map +1 -0
- package/dist/{shared-box-CZFdugL1.js → search-method-q43Af9yu.js} +1799 -16
- package/dist/search-method-q43Af9yu.js.map +1 -0
- package/dist/{server-dOyG5Tw6.js → server-BnMcTE7J.js} +3 -3
- package/dist/{server-dOyG5Tw6.js.map → server-BnMcTE7J.js.map} +1 -1
- package/dist/{supervise-zt91I62x.js → supervise-DL_Hj1uk.js} +3 -3
- package/dist/{supervise-zt91I62x.js.map → supervise-DL_Hj1uk.js.map} +1 -1
- package/dist/{supervisor-UXgaPisu.js → supervisor-YAc4U8HQ.js} +2 -2
- package/dist/{supervisor-UXgaPisu.js.map → supervisor-YAc4U8HQ.js.map} +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +14 -14
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/dist/{workspace-archive-C9lgf77y.js → workspace-archive-MhowMCmp.js} +8 -6
- package/dist/workspace-archive-MhowMCmp.js.map +1 -0
- package/package.json +2 -2
- package/dist/improvement-cycle-DCTCTmNl.js.map +0 -1
- package/dist/runtime-B_yRPBjj.js.map +0 -1
- package/dist/shared-box-CZFdugL1.js.map +0 -1
- package/dist/workspace-archive-C9lgf77y.js.map +0 -1
|
@@ -1,28 +1,30 @@
|
|
|
1
|
-
import { $i as detachedSnapshot, $t as isTraceAnalysisStore, Ai as readCommittedJsonLines, Ca as stringifySafe, Et as rollingDispatch, Gn as probeSandboxCapabilities, Gt as encodeResultBlob, Ht as InMemorySpawnJournal, Ia as armDeadlineTimer, Jn as readPromptOptions, Kt as insideCursorNamespace, M as withDriverExecutor, Mt as createExecutorRegistry, O as verdictFromJudgeScore, Oa as zeroSpend, Pt as createWorktreeCliExecutor, Qi as detachedFrozen, Sa as sleep, Un as runAgentRounds, Ut as SpawnEventIndex, Vn as defaultSelectWinner, Vt as InMemoryResultBlobStore, W as settledToIteration, Wn as createSandboxLineage, Wt as assertContentAddress, Xi as executableAgentProfileSnapshot, Xn as runBrainLoop, Yn as routerBrain, Zi as executableAgentSpecSnapshot, Zn as canonicalObservedModelParts, _ as CheckUnavailableError, cr as createPushTraceSource, ga as cloneSpend, ha as chargedTokens, ji as writeAllBytes, jt as createExecutor, ki as prepareJsonlAppend, kr as contentAddress, ma as addSpend, qn as assertBoxlessPromptOptions, r as createSupervisor, ua as notifySandboxEventObserver, va as hasCompleteCacheBreakdown, xa as randomSuffix, ya as isAbortError } from "./supervisor-
|
|
2
|
-
import { m as ValidationError, n as AnalystError, s as PlannerError } from "./errors-DodWX-cb.js";
|
|
3
|
-
import {
|
|
1
|
+
import { $i as detachedSnapshot, $t as isTraceAnalysisStore, Ai as readCommittedJsonLines, Ca as stringifySafe, Et as rollingDispatch, Gn as probeSandboxCapabilities, Gt as encodeResultBlob, Ht as InMemorySpawnJournal, Ia as armDeadlineTimer, Jn as readPromptOptions, Kt as insideCursorNamespace, M as withDriverExecutor, Mt as createExecutorRegistry, O as verdictFromJudgeScore, Oa as zeroSpend, Pt as createWorktreeCliExecutor, Qi as detachedFrozen, Sa as sleep, Un as runAgentRounds, Ut as SpawnEventIndex, Vn as defaultSelectWinner, Vt as InMemoryResultBlobStore, W as settledToIteration, Wn as createSandboxLineage, Wt as assertContentAddress, Xi as executableAgentProfileSnapshot, Xn as runBrainLoop, Yn as routerBrain, Zi as executableAgentSpecSnapshot, Zn as canonicalObservedModelParts, _ as CheckUnavailableError, cr as createPushTraceSource, ga as cloneSpend, ha as chargedTokens, ji as writeAllBytes, jt as createExecutor, ki as prepareJsonlAppend, kr as contentAddress, ma as addSpend, qn as assertBoxlessPromptOptions, r as createSupervisor, ua as notifySandboxEventObserver, va as hasCompleteCacheBreakdown, xa as randomSuffix, ya as isAbortError } from "./supervisor-YAc4U8HQ.js";
|
|
2
|
+
import { i as ConfigError, m as ValidationError, n as AnalystError, s as PlannerError } from "./errors-DodWX-cb.js";
|
|
3
|
+
import { Q as sample, X as refine, Z as runAgentic, c as reconcileInterruptedSearchCalls, ct as strategyAuthorMethod, et as ObservationError, it as profileChatClient, nt as observe, o as dedicatedLane, r as runtimeShipDecision, rt as renderReport, u as searchExecutor } from "./search-method-q43Af9yu.js";
|
|
4
4
|
import { c as profileModelExecutionSettings, i as concreteModelId, l as profileProviderModel, o as enforceTokenLimits, t as assertExecutableAgentProfile } from "./model-policy-BbSCSak0.js";
|
|
5
|
-
import {
|
|
5
|
+
import { A as sha256Bytes$1, O as immutableCandidateValue, f as WorkspaceScanLimitError, h as captureMaterializedWorkspace, w as canonicalCandidateDigest$1 } from "./workspace-archive-MhowMCmp.js";
|
|
6
6
|
import { i as notifyRuntimeHookEvent } from "./runtime-hooks-tXpAarhW.js";
|
|
7
7
|
import { i as redactProtectedValue, r as redactProtectedReason } from "./protected-redaction-DylCC_ot.js";
|
|
8
|
-
import { S as foldCoordinationRecords, y as createInMemoryRunContext } from "./coordination-driver-
|
|
9
|
-
import { C as gateOnDeliverable, r as supervise, w as mapExecutorResult } from "./supervise-
|
|
8
|
+
import { S as foldCoordinationRecords, y as createInMemoryRunContext } from "./coordination-driver-BmkANXc5.js";
|
|
9
|
+
import { C as gateOnDeliverable, r as supervise, w as mapExecutorResult } from "./supervise-DL_Hj1uk.js";
|
|
10
10
|
import "./run-layout-vwd4SwX_.js";
|
|
11
|
-
import "./provision-supervisor-
|
|
12
|
-
import "./delegate-
|
|
13
|
-
import "./graph-
|
|
11
|
+
import "./provision-supervisor-BNUmrHwC.js";
|
|
12
|
+
import "./delegate-BfHKdHzV.js";
|
|
13
|
+
import "./graph-XmlRTlaI.js";
|
|
14
14
|
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, sha256Bytes, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
15
15
|
import { CODING_HARNESSES, DEFAULT_TRACE_ANALYST_KINDS, InMemoryTraceStore, OUTPUT_VALUE, benjaminiHochberg, buildTrajectory, computeFindingId as computeFindingId$1, confidenceInterval, createTraceAnalyst, expandProfileAxes, makeFinding as makeFinding$1, pairedBootstrap, paretoFrontier, wilcoxonSignedRank, wilson } from "@tangle-network/agent-eval";
|
|
16
16
|
import { createAgentRunOutcomeTracker } from "@tangle-network/sandbox/runtime";
|
|
17
17
|
import { createHash, randomUUID } from "node:crypto";
|
|
18
18
|
import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
|
|
19
|
-
import { appendFileSync, chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync,
|
|
19
|
+
import { appendFileSync, chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readlinkSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
20
20
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
21
|
-
import {
|
|
21
|
+
import { chmod, lstat, mkdir, mkdtemp, open, opendir, readFile, readdir, readlink, realpath, rm, stat, symlink, writeFile } from "node:fs/promises";
|
|
22
22
|
import { tmpdir } from "node:os";
|
|
23
23
|
import { isDeepStrictEqual, promisify } from "node:util";
|
|
24
|
+
import { redact } from "@tangle-network/agent-eval/traces";
|
|
24
25
|
import { collectAgentResponseText } from "@tangle-network/sandbox";
|
|
25
|
-
import {
|
|
26
|
+
import { SearchRecorder, asha, beam, createRunCostLedger, estimateNode, fsCampaignStorage, openSearchLedger, renderSearchSummary, runProfileMatrix, runSearch, searchClaimReserveUsd, searchModelIdentity, surfaceDiff } from "@tangle-network/agent-eval/campaign";
|
|
27
|
+
import { defineEvaluationClaim, summarizeEvaluationUnits } from "@tangle-network/agent-eval/experiment";
|
|
26
28
|
import { createInterface } from "node:readline";
|
|
27
29
|
import { pathToFileURL } from "node:url";
|
|
28
30
|
import { gzipSync } from "node:zlib";
|
|
@@ -59,7 +61,7 @@ function createTangleSandboxExactProcessProvider(client, options = {}) {
|
|
|
59
61
|
metadata,
|
|
60
62
|
idempotencyKey: requiredText(input.idempotencyKey, "exact-process idempotency key")
|
|
61
63
|
}, {
|
|
62
|
-
...input.provisionTimeoutMs === void 0 ? {} : { timeoutMs: positiveInteger(input.provisionTimeoutMs, "provision timeout") },
|
|
64
|
+
...input.provisionTimeoutMs === void 0 ? {} : { timeoutMs: positiveInteger$1(input.provisionTimeoutMs, "provision timeout") },
|
|
63
65
|
...input.signal === void 0 ? {} : { signal: input.signal }
|
|
64
66
|
});
|
|
65
67
|
try {
|
|
@@ -146,7 +148,7 @@ function sandboxAsExactProcessEnvironment(sandbox, provider) {
|
|
|
146
148
|
};
|
|
147
149
|
}
|
|
148
150
|
function sandboxProcessAsExactProcess(process) {
|
|
149
|
-
const pid = positiveInteger(process.pid, "Tangle Sandbox process pid");
|
|
151
|
+
const pid = positiveInteger$1(process.pid, "Tangle Sandbox process pid");
|
|
150
152
|
return {
|
|
151
153
|
pid,
|
|
152
154
|
async status() {
|
|
@@ -172,7 +174,7 @@ function sandboxProcessAsExactProcess(process) {
|
|
|
172
174
|
};
|
|
173
175
|
}
|
|
174
176
|
function exactProcessStatus(status, expectedPid) {
|
|
175
|
-
const pid = positiveInteger(status.pid, "Tangle Sandbox process pid");
|
|
177
|
+
const pid = positiveInteger$1(status.pid, "Tangle Sandbox process pid");
|
|
176
178
|
if (expectedPid !== void 0 && pid !== expectedPid) throw new Error("Tangle Sandbox process status changed process identity");
|
|
177
179
|
if (typeof status.running !== "boolean") throw new Error("Tangle Sandbox process status is missing running state");
|
|
178
180
|
if (!Number.isSafeInteger(status.exitCode)) throw new Error("Tangle Sandbox process status is missing an integer exit code");
|
|
@@ -272,8 +274,8 @@ function serializeEgressPolicy(policy) {
|
|
|
272
274
|
}
|
|
273
275
|
function sandboxResources(input) {
|
|
274
276
|
const cpuCores = positiveNumber(input.resources.cpu, "exact-process cpu");
|
|
275
|
-
const memoryMB = positiveInteger(input.resources.memoryMb, "exact-process memory");
|
|
276
|
-
const diskMb = positiveInteger(input.resources.diskMb, "exact-process disk");
|
|
277
|
+
const memoryMB = positiveInteger$1(input.resources.memoryMb, "exact-process memory");
|
|
278
|
+
const diskMb = positiveInteger$1(input.resources.diskMb, "exact-process disk");
|
|
277
279
|
if (diskMb % 1024 !== 0) throw new Error("Tangle Sandbox exact-process disk must be a whole GiB");
|
|
278
280
|
return {
|
|
279
281
|
cpuCores,
|
|
@@ -282,7 +284,7 @@ function sandboxResources(input) {
|
|
|
282
284
|
};
|
|
283
285
|
}
|
|
284
286
|
function sandboxLifetimeSeconds(maxLifetimeMs) {
|
|
285
|
-
const milliseconds = positiveInteger(maxLifetimeMs, "exact-process maximum lifetime");
|
|
287
|
+
const milliseconds = positiveInteger$1(maxLifetimeMs, "exact-process maximum lifetime");
|
|
286
288
|
if (milliseconds % 1e3 !== 0) throw new Error("Tangle Sandbox exact-process maximum lifetime must be a whole number of seconds");
|
|
287
289
|
return milliseconds / 1e3;
|
|
288
290
|
}
|
|
@@ -372,7 +374,7 @@ function positiveNumber(value, label) {
|
|
|
372
374
|
if (!Number.isFinite(value) || value <= 0) throw new Error(`${label} must be positive`);
|
|
373
375
|
return value;
|
|
374
376
|
}
|
|
375
|
-
function positiveInteger(value, label) {
|
|
377
|
+
function positiveInteger$1(value, label) {
|
|
376
378
|
if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`);
|
|
377
379
|
return value;
|
|
378
380
|
}
|
|
@@ -1804,7 +1806,8 @@ const contains = (parent, child) => {
|
|
|
1804
1806
|
* Bubblewrap requires /usr/bin/bwrap and permission to create Linux namespaces.
|
|
1805
1807
|
* Only trusted system toolchains, private proc/dev/tmp, and the writable copy are mounted.
|
|
1806
1808
|
* The canonical input path remains the working directory; copy writes are discarded.
|
|
1807
|
-
*
|
|
1809
|
+
* Time and output are bounded; input copy bytes and entries are bounded when requested.
|
|
1810
|
+
* The command's memory and generated files require a resource-limited placement.
|
|
1808
1811
|
* Callers must keep the input and trusted toolchains stable while preparing the check.
|
|
1809
1812
|
*
|
|
1810
1813
|
* A box check creates one fresh box with no owner secrets and blocked egress (or strict egress to
|
|
@@ -1817,7 +1820,7 @@ async function runIsolatedCheck(options) {
|
|
|
1817
1820
|
let result;
|
|
1818
1821
|
try {
|
|
1819
1822
|
if (process.platform !== "linux") throw new Error("Linux Bubblewrap namespaces are required");
|
|
1820
|
-
const { timeoutMs, maxOutputBytes } = checkLimits(options);
|
|
1823
|
+
const { timeoutMs, maxOutputBytes, maxInputBytes, maxInputEntries } = checkLimits(options);
|
|
1821
1824
|
if (options.signal?.aborted) return {
|
|
1822
1825
|
succeeded: false,
|
|
1823
1826
|
reason: "cancelled",
|
|
@@ -1840,11 +1843,7 @@ async function runIsolatedCheck(options) {
|
|
|
1840
1843
|
scratch = await mkdtemp(join(tmpdir(), "runtime-check-"));
|
|
1841
1844
|
if (contains(tree, scratch)) throw new Error("Temporary storage must be outside the input tree");
|
|
1842
1845
|
const copy = join(scratch, "tree");
|
|
1843
|
-
await
|
|
1844
|
-
recursive: true,
|
|
1845
|
-
dereference: false,
|
|
1846
|
-
verbatimSymlinks: true
|
|
1847
|
-
});
|
|
1846
|
+
await copyBoundedTree(tree, copy, maxInputBytes, maxInputEntries);
|
|
1848
1847
|
result = await execute([
|
|
1849
1848
|
"--unshare-all",
|
|
1850
1849
|
"--die-with-parent",
|
|
@@ -1884,7 +1883,7 @@ async function runIsolatedCheck(options) {
|
|
|
1884
1883
|
} catch (error) {
|
|
1885
1884
|
result = {
|
|
1886
1885
|
succeeded: false,
|
|
1887
|
-
reason: "refused",
|
|
1886
|
+
reason: error instanceof InputLimitError ? "input-limit" : "refused",
|
|
1888
1887
|
diagnostic: String(error)
|
|
1889
1888
|
};
|
|
1890
1889
|
}
|
|
@@ -1909,12 +1908,68 @@ async function runIsolatedCheck(options) {
|
|
|
1909
1908
|
function checkLimits(options) {
|
|
1910
1909
|
const timeoutMs = options.timeoutMs ?? 3e4;
|
|
1911
1910
|
const maxOutputBytes = options.maxOutputBytes ?? 1048576;
|
|
1912
|
-
|
|
1911
|
+
const maxInputBytes = options.maxInputBytes;
|
|
1912
|
+
const maxInputEntries = options.maxInputEntries;
|
|
1913
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147483647 || !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1 || maxInputBytes !== void 0 && (!Number.isSafeInteger(maxInputBytes) || maxInputBytes < 1) || maxInputEntries !== void 0 && (!Number.isSafeInteger(maxInputEntries) || maxInputEntries < 1)) throw new Error("Invalid execution limits");
|
|
1913
1914
|
if (!options.command.length || !options.command[0]) throw new Error("A command is required");
|
|
1914
1915
|
return {
|
|
1915
1916
|
timeoutMs,
|
|
1916
|
-
maxOutputBytes
|
|
1917
|
+
maxOutputBytes,
|
|
1918
|
+
maxInputBytes,
|
|
1919
|
+
maxInputEntries
|
|
1920
|
+
};
|
|
1921
|
+
}
|
|
1922
|
+
var InputLimitError = class extends Error {};
|
|
1923
|
+
/** Copy in bounded chunks, preserving symlink text without reading its target. */
|
|
1924
|
+
async function copyBoundedTree(source, destination, maxBytes, maxEntries) {
|
|
1925
|
+
let bytes = 0;
|
|
1926
|
+
let entries = 0;
|
|
1927
|
+
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
1928
|
+
const visit = async (from, to, isRoot = false) => {
|
|
1929
|
+
if (!isRoot && ++entries > (maxEntries ?? Number.MAX_SAFE_INTEGER)) throw new InputLimitError("Input tree exceeds maxInputEntries");
|
|
1930
|
+
const info = await lstat(from);
|
|
1931
|
+
if (info.isSymbolicLink()) await symlink(await readlink(from), to);
|
|
1932
|
+
else if (info.isDirectory()) {
|
|
1933
|
+
await mkdir(to, { mode: 448 });
|
|
1934
|
+
for await (const entry of await opendir(from)) await visit(join(from, entry.name), join(to, entry.name));
|
|
1935
|
+
await chmod(to, info.mode & 511);
|
|
1936
|
+
} else if (info.isFile()) {
|
|
1937
|
+
if (maxBytes !== void 0 && info.size > maxBytes - bytes) throw new InputLimitError("Input tree exceeds maxInputBytes");
|
|
1938
|
+
const input = await open(from, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
1939
|
+
try {
|
|
1940
|
+
const output = await open(to, "wx", info.mode & 511);
|
|
1941
|
+
try {
|
|
1942
|
+
while (true) {
|
|
1943
|
+
const { bytesRead } = await input.read(buffer, 0, maxBytes === void 0 ? buffer.length : Math.min(buffer.length, maxBytes - bytes + 1), null);
|
|
1944
|
+
if (bytesRead === 0) break;
|
|
1945
|
+
bytes += bytesRead;
|
|
1946
|
+
if (maxBytes !== void 0 && bytes > maxBytes) throw new InputLimitError("Input tree exceeds maxInputBytes");
|
|
1947
|
+
let written = 0;
|
|
1948
|
+
while (written < bytesRead) {
|
|
1949
|
+
const result = await output.write(buffer, written, bytesRead - written);
|
|
1950
|
+
if (result.bytesWritten === 0) throw new Error("Input copy made no write progress");
|
|
1951
|
+
written += result.bytesWritten;
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
} finally {
|
|
1955
|
+
await output.close();
|
|
1956
|
+
}
|
|
1957
|
+
await chmod(to, info.mode & 511);
|
|
1958
|
+
} finally {
|
|
1959
|
+
await input.close();
|
|
1960
|
+
}
|
|
1961
|
+
} else throw new Error(`Input tree contains an unsupported entry: ${from}`);
|
|
1962
|
+
};
|
|
1963
|
+
await visit(source, destination, true);
|
|
1964
|
+
}
|
|
1965
|
+
async function checkInputEntries(root, maxEntries) {
|
|
1966
|
+
let count = 0;
|
|
1967
|
+
const visit = async (path, isRoot = false) => {
|
|
1968
|
+
if (!isRoot && ++count > maxEntries) throw new InputLimitError("Input tree exceeds maxInputEntries");
|
|
1969
|
+
if (!(await lstat(path)).isDirectory()) return;
|
|
1970
|
+
for await (const entry of await opendir(path)) await visit(join(path, entry.name));
|
|
1917
1971
|
};
|
|
1972
|
+
await visit(root, true);
|
|
1918
1973
|
}
|
|
1919
1974
|
async function protectedTree(options) {
|
|
1920
1975
|
const workspace = await realpath(options.workspaceRoot);
|
|
@@ -1940,15 +1995,22 @@ async function runInBox(options, placement) {
|
|
|
1940
1995
|
let evidence;
|
|
1941
1996
|
let result;
|
|
1942
1997
|
try {
|
|
1943
|
-
const { timeoutMs, maxOutputBytes } = checkLimits(options);
|
|
1998
|
+
const { timeoutMs, maxOutputBytes, maxInputBytes, maxInputEntries } = checkLimits(options);
|
|
1944
1999
|
if (options.signal?.aborted) return {
|
|
1945
2000
|
succeeded: false,
|
|
1946
2001
|
reason: "cancelled",
|
|
1947
2002
|
diagnostic: "Check cancelled"
|
|
1948
2003
|
};
|
|
1949
2004
|
const { tree } = await protectedTree(options);
|
|
1950
|
-
|
|
2005
|
+
if (maxInputEntries !== void 0) await checkInputEntries(tree, maxInputEntries);
|
|
2006
|
+
const captured = await captureMaterializedWorkspace(tree, { limits: {
|
|
2007
|
+
maxFiles: maxInputEntries ?? Number.MAX_SAFE_INTEGER,
|
|
2008
|
+
maxFileBytes: maxInputBytes ?? Number.MAX_SAFE_INTEGER,
|
|
2009
|
+
maxTotalFileBytes: maxInputBytes ?? Number.MAX_SAFE_INTEGER
|
|
2010
|
+
} });
|
|
1951
2011
|
if (placement.builderAccounts.length === 0 || placement.builderAccounts.some((id) => !id)) throw new Error("builderAccounts must name the account of every key the judged run holds");
|
|
2012
|
+
const memoryMB = placement.resources?.memoryMB;
|
|
2013
|
+
if (memoryMB !== void 0 && (!Number.isSafeInteger(memoryMB) || memoryMB < 1 || placement.resources?.advisory === true || placement.resources?.advisoryFields?.includes("memoryMB"))) throw new Error("Check box requires an explicit finite resources.memoryMB limit");
|
|
1952
2014
|
const account = (await placement.client.getIdentity()).customerId;
|
|
1953
2015
|
if (!account) throw new Error("Sandbox returned no account for the check key");
|
|
1954
2016
|
if (placement.builderAccounts.includes(account)) throw new Error(`Check account ${account} is an account the judged run holds a key to, so the run can reach the check box`);
|
|
@@ -1965,6 +2027,8 @@ async function runInBox(options, placement) {
|
|
|
1965
2027
|
}, options.signal ? { signal: options.signal } : void 0);
|
|
1966
2028
|
const receipt = box.createReceipt();
|
|
1967
2029
|
if (receipt?.outcome !== "created" || receipt.ownerContext !== "isolated" || !Array.isArray(receipt.injectedSecrets) || receipt.injectedSecrets.length !== 0) throw new Error("Sandbox did not confirm a fresh box with no owner secrets");
|
|
2030
|
+
const grantedMemoryMB = box.resources?.memoryMB;
|
|
2031
|
+
if (memoryMB !== void 0 && (typeof grantedMemoryMB !== "number" || !Number.isSafeInteger(grantedMemoryMB) || grantedMemoryMB < 1 || grantedMemoryMB > memoryMB)) throw new Error("Sandbox did not confirm the requested finite memory cgroup limit");
|
|
1968
2032
|
const egress = (await box.egress.get()).policy;
|
|
1969
2033
|
const wanted = egressFor(placement);
|
|
1970
2034
|
if (egress.mode !== wanted.mode || wanted.mode === "strict" && (egress.includeImplicitDomains === true || [...egress.allowDomains ?? []].sort().join(",") !== [...wanted.allowDomains ?? []].sort().join(","))) throw new Error(`Check box egress is ${egress.mode} ${JSON.stringify(egress.allowDomains ?? [])}, not ${wanted.mode} ${JSON.stringify(wanted.allowDomains ?? [])}`);
|
|
@@ -1988,6 +2052,7 @@ async function runInBox(options, placement) {
|
|
|
1988
2052
|
evidence = {
|
|
1989
2053
|
sandboxId: box.id,
|
|
1990
2054
|
account,
|
|
2055
|
+
...memoryMB === void 0 ? {} : { memoryMB: grantedMemoryMB },
|
|
1991
2056
|
input: captured.manifest,
|
|
1992
2057
|
inputDigest: canonicalCandidateDigest(captured.manifest)
|
|
1993
2058
|
};
|
|
@@ -1995,7 +2060,7 @@ async function runInBox(options, placement) {
|
|
|
1995
2060
|
} catch (error) {
|
|
1996
2061
|
result = {
|
|
1997
2062
|
succeeded: false,
|
|
1998
|
-
reason: options.signal?.aborted ? "cancelled" : "refused",
|
|
2063
|
+
reason: options.signal?.aborted ? "cancelled" : error instanceof InputLimitError || error instanceof WorkspaceScanLimitError ? "input-limit" : "refused",
|
|
1999
2064
|
diagnostic: String(error)
|
|
2000
2065
|
};
|
|
2001
2066
|
}
|
|
@@ -4894,188 +4959,6 @@ function requireSpend(rolled, id, root) {
|
|
|
4894
4959
|
return spend;
|
|
4895
4960
|
}
|
|
4896
4961
|
//#endregion
|
|
4897
|
-
//#region src/runtime/promotion-gate.ts
|
|
4898
|
-
/**
|
|
4899
|
-
* promotionGate — the statistical promotion decision over a holdout benchmark: does the
|
|
4900
|
-
* candidate strategy beat the incumbent on held-out tasks by a margin the task noise
|
|
4901
|
-
* cannot fake? The statistics come from `heldoutSignificance`, which chooses the
|
|
4902
|
-
* decision interval that matches the outcome shape, applies a minimum-evidence floor,
|
|
4903
|
-
* and requires the interval's lower bound to clear `deltaThreshold`. A raw h1>h0 point
|
|
4904
|
-
* comparison on m≈8 holdout tasks certifies false champions at near coin-flip rates.
|
|
4905
|
-
*/
|
|
4906
|
-
/** Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. */
|
|
4907
|
-
function promotionGate(opts) {
|
|
4908
|
-
const mode = opts.mode ?? "superiority";
|
|
4909
|
-
if (opts.candidate === opts.incumbent) return {
|
|
4910
|
-
promoted: false,
|
|
4911
|
-
reason: "identical-champion",
|
|
4912
|
-
mode,
|
|
4913
|
-
n: 0,
|
|
4914
|
-
lift: {
|
|
4915
|
-
mean: 0,
|
|
4916
|
-
median: 0,
|
|
4917
|
-
low: 0,
|
|
4918
|
-
high: 0
|
|
4919
|
-
}
|
|
4920
|
-
};
|
|
4921
|
-
const before = [];
|
|
4922
|
-
const after = [];
|
|
4923
|
-
const incUsd = [];
|
|
4924
|
-
const candUsd = [];
|
|
4925
|
-
const incMs = [];
|
|
4926
|
-
const candMs = [];
|
|
4927
|
-
const cellIds = [];
|
|
4928
|
-
/** Tasks where either arm's dollars were never measured — see the refusal below. */
|
|
4929
|
-
const costUnknownTasks = [];
|
|
4930
|
-
for (const row of opts.report.perTask) {
|
|
4931
|
-
const inc = row.cells?.[opts.incumbent];
|
|
4932
|
-
const cand = row.cells?.[opts.candidate];
|
|
4933
|
-
if (!inc || !cand) continue;
|
|
4934
|
-
before.push(inc.score);
|
|
4935
|
-
after.push(cand.score);
|
|
4936
|
-
incUsd.push(inc.usd);
|
|
4937
|
-
candUsd.push(cand.usd);
|
|
4938
|
-
if (inc.usdKnown === false || cand.usdKnown === false) costUnknownTasks.push(row.taskId);
|
|
4939
|
-
incMs.push(inc.ms);
|
|
4940
|
-
candMs.push(cand.ms);
|
|
4941
|
-
cellIds.push(row.taskId);
|
|
4942
|
-
}
|
|
4943
|
-
if (before.length === 0) throw new Error(`promotionGate: no holdout task carried cells for both "${opts.incumbent}" and "${opts.candidate}" — the report must come from a run that included both strategies`);
|
|
4944
|
-
const sig = heldoutSignificance({
|
|
4945
|
-
before,
|
|
4946
|
-
after,
|
|
4947
|
-
cellIds
|
|
4948
|
-
}, {
|
|
4949
|
-
deltaThreshold: opts.deltaThreshold ?? 0,
|
|
4950
|
-
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
4951
|
-
statistic: opts.statistic ?? "mean",
|
|
4952
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
4953
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
4954
|
-
});
|
|
4955
|
-
const lift = {
|
|
4956
|
-
mean: sig.bootstrap.mean,
|
|
4957
|
-
median: sig.bootstrap.median,
|
|
4958
|
-
low: sig.decision.low,
|
|
4959
|
-
high: sig.decision.high
|
|
4960
|
-
};
|
|
4961
|
-
const latSig = heldoutSignificance({
|
|
4962
|
-
before: incMs,
|
|
4963
|
-
after: candMs,
|
|
4964
|
-
cellIds
|
|
4965
|
-
}, {
|
|
4966
|
-
deltaThreshold: 0,
|
|
4967
|
-
minProductiveRuns: 1,
|
|
4968
|
-
statistic: opts.statistic ?? "mean",
|
|
4969
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
4970
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
4971
|
-
});
|
|
4972
|
-
const latency = {
|
|
4973
|
-
mean: latSig.bootstrap.mean,
|
|
4974
|
-
median: latSig.bootstrap.median,
|
|
4975
|
-
low: latSig.decision.low,
|
|
4976
|
-
high: latSig.decision.high
|
|
4977
|
-
};
|
|
4978
|
-
if (mode === "superiority") {
|
|
4979
|
-
if (sig.fewRuns) return {
|
|
4980
|
-
promoted: false,
|
|
4981
|
-
reason: "few-tasks",
|
|
4982
|
-
mode,
|
|
4983
|
-
n: sig.n,
|
|
4984
|
-
lift,
|
|
4985
|
-
latency
|
|
4986
|
-
};
|
|
4987
|
-
return sig.significant ? {
|
|
4988
|
-
promoted: true,
|
|
4989
|
-
reason: "significant",
|
|
4990
|
-
mode,
|
|
4991
|
-
n: sig.n,
|
|
4992
|
-
lift,
|
|
4993
|
-
latency
|
|
4994
|
-
} : {
|
|
4995
|
-
promoted: false,
|
|
4996
|
-
reason: "no-margin",
|
|
4997
|
-
mode,
|
|
4998
|
-
n: sig.n,
|
|
4999
|
-
lift,
|
|
5000
|
-
latency
|
|
5001
|
-
};
|
|
5002
|
-
}
|
|
5003
|
-
const tolerance = opts.scoreTolerance ?? .05;
|
|
5004
|
-
const scoreSig = heldoutSignificance({
|
|
5005
|
-
before,
|
|
5006
|
-
after,
|
|
5007
|
-
cellIds
|
|
5008
|
-
}, {
|
|
5009
|
-
deltaThreshold: -tolerance,
|
|
5010
|
-
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
5011
|
-
statistic: opts.statistic ?? "mean",
|
|
5012
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
5013
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
5014
|
-
});
|
|
5015
|
-
if (costUnknownTasks.length > 0) return {
|
|
5016
|
-
promoted: false,
|
|
5017
|
-
reason: "cost-unknown",
|
|
5018
|
-
mode,
|
|
5019
|
-
n: before.length,
|
|
5020
|
-
lift,
|
|
5021
|
-
costUnknownTasks: [...costUnknownTasks],
|
|
5022
|
-
latency
|
|
5023
|
-
};
|
|
5024
|
-
const costSig = heldoutSignificance({
|
|
5025
|
-
before: candUsd,
|
|
5026
|
-
after: incUsd,
|
|
5027
|
-
cellIds
|
|
5028
|
-
}, {
|
|
5029
|
-
deltaThreshold: 0,
|
|
5030
|
-
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
5031
|
-
statistic: opts.statistic ?? "mean",
|
|
5032
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
5033
|
-
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
5034
|
-
});
|
|
5035
|
-
const costSavings = {
|
|
5036
|
-
mean: costSig.bootstrap.mean,
|
|
5037
|
-
median: costSig.bootstrap.median,
|
|
5038
|
-
low: costSig.decision.low,
|
|
5039
|
-
high: costSig.decision.high
|
|
5040
|
-
};
|
|
5041
|
-
if (scoreSig.fewRuns) return {
|
|
5042
|
-
promoted: false,
|
|
5043
|
-
reason: "few-tasks",
|
|
5044
|
-
mode,
|
|
5045
|
-
n: scoreSig.n,
|
|
5046
|
-
lift,
|
|
5047
|
-
costSavings,
|
|
5048
|
-
latency
|
|
5049
|
-
};
|
|
5050
|
-
if (!scoreSig.significant) return {
|
|
5051
|
-
promoted: false,
|
|
5052
|
-
reason: "non-inferiority-unproven",
|
|
5053
|
-
mode,
|
|
5054
|
-
n: scoreSig.n,
|
|
5055
|
-
lift,
|
|
5056
|
-
costSavings,
|
|
5057
|
-
latency
|
|
5058
|
-
};
|
|
5059
|
-
if (!costSig.significant) return {
|
|
5060
|
-
promoted: false,
|
|
5061
|
-
reason: "not-cheaper",
|
|
5062
|
-
mode,
|
|
5063
|
-
n: scoreSig.n,
|
|
5064
|
-
lift,
|
|
5065
|
-
costSavings,
|
|
5066
|
-
latency
|
|
5067
|
-
};
|
|
5068
|
-
return {
|
|
5069
|
-
promoted: true,
|
|
5070
|
-
reason: "non-inferior-and-cheaper",
|
|
5071
|
-
mode,
|
|
5072
|
-
n: scoreSig.n,
|
|
5073
|
-
lift,
|
|
5074
|
-
costSavings,
|
|
5075
|
-
latency
|
|
5076
|
-
};
|
|
5077
|
-
}
|
|
5078
|
-
//#endregion
|
|
5079
4962
|
//#region src/runtime/run-benchmark.ts
|
|
5080
4963
|
/**
|
|
5081
4964
|
* runBenchmark — the packaged optimization suite. Define a domain by implementing an
|
|
@@ -5105,6 +4988,8 @@ async function pool(items, limit, fn) {
|
|
|
5105
4988
|
await Promise.all(workers);
|
|
5106
4989
|
return out;
|
|
5107
4990
|
}
|
|
4991
|
+
/** @internal Check each unique worker and analyst model once before any task runs, as
|
|
4992
|
+
* `BenchmarkConfig.modelPreflight` describes. `runStrategyEvolution` checks the same way. */
|
|
5108
4993
|
async function preflightModels(cfg) {
|
|
5109
4994
|
if (cfg.modelPreflight === false) return;
|
|
5110
4995
|
if (cfg.worker.complete && !cfg.modelPreflight) return;
|
|
@@ -5726,531 +5611,710 @@ const strategyAuthorSystemPrompt = "You are a senior researcher authoring optimi
|
|
|
5726
5611
|
function assertStrategyContract(code) {
|
|
5727
5612
|
assertAuthoredCode(code, { allowedImports: ["@tangle-network/agent-runtime/kernel"] });
|
|
5728
5613
|
}
|
|
5729
|
-
/**
|
|
5730
|
-
*
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
|
|
5739
|
-
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5614
|
+
/** Ask the author for a strategy module. A failed call throws; a reply without a fenced
|
|
5615
|
+
* module is `code: null`. With `fallbackProfile`, the fallback gets one attempt when the
|
|
5616
|
+
* primary call fails or its reply carries no module. */
|
|
5617
|
+
async function requestStrategySource(opts) {
|
|
5618
|
+
const turns = [];
|
|
5619
|
+
const attempt = async (profile) => {
|
|
5620
|
+
const res = await profileChatClient({
|
|
5621
|
+
profile,
|
|
5622
|
+
executor: opts.executor,
|
|
5623
|
+
context: "strategy author"
|
|
5624
|
+
}).chat({ messages: [{
|
|
5625
|
+
role: "user",
|
|
5626
|
+
content: `${opts.contract ?? strategyAuthorContract}\n\nBASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}) — the per-task losses are your gradient:\n${opts.lossesJson}\n\nAuthor ONE new strategy that you expect to beat the baselines on THIS environment at the same budget.\n${strategyAuthorMethod}\n\nOutput only the module code block.`
|
|
5627
|
+
}] }, { ...opts.signal ? { signal: opts.signal } : {} });
|
|
5628
|
+
turns.push({
|
|
5629
|
+
content: res.content,
|
|
5630
|
+
model: res.model,
|
|
5631
|
+
inputTokens: res.usage.promptTokens,
|
|
5632
|
+
outputTokens: res.usage.completionTokens,
|
|
5633
|
+
tokensKnown: res.usage.captured !== false,
|
|
5634
|
+
costUsd: res.costUsd
|
|
5635
|
+
});
|
|
5636
|
+
return res.content.match(/```(?:ts|typescript)?\s*\n([\s\S]*?)```/)?.[1] ?? null;
|
|
5637
|
+
};
|
|
5748
5638
|
try {
|
|
5749
|
-
code = await
|
|
5639
|
+
const code = await attempt(opts.profile);
|
|
5640
|
+
if (code !== null || !opts.fallbackProfile) return {
|
|
5641
|
+
code,
|
|
5642
|
+
turns
|
|
5643
|
+
};
|
|
5750
5644
|
} catch (primaryError) {
|
|
5751
5645
|
if (!opts.fallbackProfile) throw primaryError;
|
|
5752
|
-
code = await requestAuthoredCode(opts, opts.fallbackProfile);
|
|
5753
5646
|
}
|
|
5647
|
+
return {
|
|
5648
|
+
code: await attempt(opts.fallbackProfile),
|
|
5649
|
+
turns
|
|
5650
|
+
};
|
|
5651
|
+
}
|
|
5652
|
+
/** Lint an authored module, write it under `outDir` at a name its content fixes, and import
|
|
5653
|
+
* its default Strategy. The same source always loads from the same file. Throws when the
|
|
5654
|
+
* module breaks the contract or exports no Strategy. */
|
|
5655
|
+
async function loadAuthoredStrategy(code, outDir) {
|
|
5754
5656
|
assertStrategyContract(code);
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5657
|
+
const file = join(outDir, `strategy-${sha256Bytes$1(Buffer.from(code, "utf8")).slice(7)}.mts`);
|
|
5658
|
+
if (!existsSync(file)) {
|
|
5659
|
+
mkdirSync(outDir, { recursive: true });
|
|
5660
|
+
const partial = `${file}.${process.pid}.tmp`;
|
|
5661
|
+
writeFileSync(partial, code);
|
|
5662
|
+
renameSync(partial, file);
|
|
5663
|
+
}
|
|
5664
|
+
const mod = await import(pathToFileURL(file).href);
|
|
5759
5665
|
if (!mod.default || typeof mod.default.driver !== "function" || !mod.default.name) throw new Error(`authorStrategy: ${file} does not export a default Strategy`);
|
|
5760
5666
|
return {
|
|
5761
5667
|
strategy: mod.default,
|
|
5762
|
-
file
|
|
5668
|
+
file
|
|
5669
|
+
};
|
|
5670
|
+
}
|
|
5671
|
+
/** Author + load a strategy from losses. Throws when the author emits no loadable module;
|
|
5672
|
+
* with `fallbackProfile` set, the fallback gets one attempt first. */
|
|
5673
|
+
async function authorStrategy(opts) {
|
|
5674
|
+
const { code, turns } = await requestStrategySource(opts);
|
|
5675
|
+
if (code === null) throw new Error(`authorStrategy: no code block in the author's reply: ${(turns.at(-1)?.content ?? "").slice(0, 300)}`);
|
|
5676
|
+
return {
|
|
5677
|
+
...await loadAuthoredStrategy(code, opts.outDir),
|
|
5763
5678
|
code
|
|
5764
5679
|
};
|
|
5765
5680
|
}
|
|
5766
5681
|
//#endregion
|
|
5767
5682
|
//#region src/runtime/strategy-evolution.ts
|
|
5768
5683
|
/**
|
|
5769
|
-
* runStrategyEvolution
|
|
5770
|
-
* authors a POPULATION of candidate strategies from the current tournament's losses,
|
|
5771
|
-
* plays them against the incumbent at equal budget, and advances a champion; one final
|
|
5772
|
-
* promotion decision runs on a NEVER-BEFORE-USED holdout slice through `promotionGate`.
|
|
5684
|
+
* runStrategyEvolution: strategy search on Eval's search kernel (`runSearch`).
|
|
5773
5685
|
*
|
|
5774
|
-
*
|
|
5775
|
-
*
|
|
5776
|
-
*
|
|
5777
|
-
*
|
|
5778
|
-
*
|
|
5779
|
-
* candidates cannot win by overspending.
|
|
5780
|
-
* - Champion selection within the search is a SEARCH policy (configurable, default
|
|
5781
|
-
* cost-aware: ties on score go to the cheapest strategy — a scalar hides a strategy
|
|
5782
|
-
* that ties at half the cost). The promotion verdict never comes from search
|
|
5783
|
-
* selection; it comes from the gate on the fresh slice.
|
|
5784
|
-
* - Every authored artifact's description length (gzip bits) is recorded, so the
|
|
5785
|
-
* artifact-complexity-vs-holdout-gap relation is analyzable from any run's report.
|
|
5686
|
+
* Every node is a strategy: the caller's root, or a module an author model wrote from the
|
|
5687
|
+
* train results of the parent the policy chose. The default policy is `beam({ width: 2 })`
|
|
5688
|
+
* and the default allocator `asha()`, so a strategy is screened on a few selection tasks and
|
|
5689
|
+
* earns more tasks only by ranking well. Each cell is one `runAgentic` run of one strategy on
|
|
5690
|
+
* one task, scored by the environment's own check.
|
|
5786
5691
|
*
|
|
5787
|
-
*
|
|
5788
|
-
*
|
|
5789
|
-
*
|
|
5692
|
+
* Measurement follows the kernel's splits. The author reads train results only; the policy
|
|
5693
|
+
* and allocator rank on the private selection split; the claim runs the root and at most 3
|
|
5694
|
+
* finalists together on the sealed test split, once, with a power check and a Bonferroni
|
|
5695
|
+
* decision. The report is a projection of the ledger, and the ledger is the only checkpoint:
|
|
5696
|
+
* calling again with the same inputs continues an interrupted search and returns a finished
|
|
5697
|
+
* one without running anything.
|
|
5790
5698
|
*
|
|
5791
5699
|
* @experimental
|
|
5792
5700
|
*/
|
|
5793
|
-
|
|
5794
|
-
|
|
5795
|
-
|
|
5796
|
-
|
|
5797
|
-
|
|
5798
|
-
|
|
5799
|
-
|
|
5800
|
-
|
|
5801
|
-
|
|
5802
|
-
|
|
5803
|
-
})
|
|
5804
|
-
if (rows.length === 0) return null;
|
|
5805
|
-
const out = {};
|
|
5806
|
-
for (const name of fieldOrder) {
|
|
5807
|
-
const cells = rows.map((r) => r.cells?.[name]).filter((c) => !!c);
|
|
5808
|
-
out[name] = {
|
|
5809
|
-
score: cells.reduce((s, c) => s + c.score, 0) / cells.length,
|
|
5810
|
-
usd: cells.reduce((s, c) => s + c.usd, 0) / cells.length
|
|
5811
|
-
};
|
|
5812
|
-
}
|
|
5813
|
-
return out;
|
|
5814
|
-
}
|
|
5815
|
-
/** The champion pick over a means table. 'score' takes the best mean score (ties →
|
|
5816
|
-
* field order). 'costAware' treats scores within `epsilon` of the best as tied and
|
|
5817
|
-
* takes the cheapest — the (score, $) Pareto rule collapsed to one pick. */
|
|
5818
|
-
function pickChampion(means, fieldOrder, policy, epsilon) {
|
|
5819
|
-
const entries = fieldOrder.map((name) => ({
|
|
5820
|
-
name,
|
|
5821
|
-
summary: means[name]
|
|
5822
|
-
})).filter((e) => !!e.summary);
|
|
5823
|
-
if (entries.length === 0) throw new Error("pickChampion: the means table carries none of the field strategies");
|
|
5824
|
-
const best = Math.max(...entries.map((e) => e.summary.score));
|
|
5825
|
-
const pick = policy === "score" ? entries.find((e) => e.summary.score === best) : entries.filter((e) => e.summary.score >= best - epsilon).sort((a, b) => a.summary.usd - b.summary.usd || b.summary.score - a.summary.score)[0];
|
|
5826
|
-
if (!pick) throw new Error("pickChampion: empty pick (unreachable)");
|
|
5827
|
-
return {
|
|
5828
|
-
name: pick.name,
|
|
5829
|
-
score: pick.summary.score,
|
|
5830
|
-
usd: pick.summary.usd
|
|
5831
|
-
};
|
|
5832
|
-
}
|
|
5833
|
-
/** Search-side champion selection over a tournament report. */
|
|
5834
|
-
function selectChampion(report, fieldOrder, policy, epsilon) {
|
|
5835
|
-
return pickChampion(report.perStrategy, fieldOrder, policy, epsilon);
|
|
5836
|
-
}
|
|
5837
|
-
const fieldSummary = (archive) => archive.map((n) => `- ${n.name} (${n.source}, gen ${n.generation}, last score ${(n.score * 100).toFixed(0)}%)`).join("\n");
|
|
5838
|
-
/** The author-visible losses: EVERY train task in compact form (score/resolved/
|
|
5839
|
-
* progression per cell). A pretty-printed prefix slice would hide the tail tasks from
|
|
5840
|
-
* the author and bias which failure modes it can target; the hard cap stays only as a
|
|
5841
|
-
* guard against enormous fields. */
|
|
5842
|
-
const compactLosses = (report, detail) => {
|
|
5843
|
-
const r2 = (x) => Math.round(x * 100) / 100;
|
|
5844
|
-
const rows = report.perTask.map((row) => row.cells ? {
|
|
5845
|
-
task: row.taskId,
|
|
5846
|
-
...row.errors ? { errors: Object.fromEntries(Object.entries(row.errors).map(([n, msg]) => [n, msg.slice(0, 100)])) } : {},
|
|
5847
|
-
cells: Object.fromEntries(Object.entries(row.cells).map(([name, c]) => [name, detail === "binary" ? {
|
|
5848
|
-
resolved: c.resolved,
|
|
5849
|
-
usd: Math.round(c.usd * 1e4) / 1e4
|
|
5850
|
-
} : {
|
|
5851
|
-
score: r2(c.score),
|
|
5852
|
-
resolved: c.resolved,
|
|
5853
|
-
usd: Math.round(c.usd * 1e4) / 1e4,
|
|
5854
|
-
progression: (c.progression ?? []).map(r2)
|
|
5855
|
-
}]))
|
|
5856
|
-
} : {
|
|
5857
|
-
task: row.taskId,
|
|
5858
|
-
error: row.error?.slice(0, 80)
|
|
5859
|
-
});
|
|
5860
|
-
return JSON.stringify(rows).slice(0, 12e3);
|
|
5701
|
+
const STRATEGY_EVOLUTION_SOURCE = {
|
|
5702
|
+
uri: "npm:@tangle-network/agent-runtime#runStrategyEvolution",
|
|
5703
|
+
revision: canonicalCandidateDigest$1({
|
|
5704
|
+
name: "agent-runtime.strategy-evolution.2026-09",
|
|
5705
|
+
node: "the root strategy by name under the caller executionRef, or an authored module content-addressed by the sha256 of its source",
|
|
5706
|
+
edge: "a text diff from the parent module source to the child; unknown from the root, which has no source here",
|
|
5707
|
+
admission: "a module is refused when the reply carries none, it breaks the author contract, or it does not load a default Strategy",
|
|
5708
|
+
cell: "one runAgentic run on one task, metered as one paid call on the search cost ledger through the Runtime search executor; an environment fault during it is errored and retried, a thrown strategy is failed at score 0, anything else passed at its harness-verified score",
|
|
5709
|
+
proposer: "the author reads the strategy contract, the task tools, the train summary, the train results of every strategy and the parent source",
|
|
5710
|
+
ship: "the claim shipped, re-derives from the ledger, cost accounting is complete, and the shipped finalist test lower bound exceeds 0"
|
|
5711
|
+
})
|
|
5861
5712
|
};
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
function
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
|
|
5880
|
-
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
|
|
5885
|
-
|
|
5713
|
+
const NO_MODULE = "the author reply carried no fenced ts module";
|
|
5714
|
+
/** Evolve a strategy on the search kernel and claim once on the sealed test split. */
|
|
5715
|
+
async function runStrategyEvolution(cfg) {
|
|
5716
|
+
const root = cfg.root ?? refine;
|
|
5717
|
+
const budget = positiveInteger("budget", cfg.budget ?? 3);
|
|
5718
|
+
const concurrency = positiveInteger("concurrency", cfg.concurrency ?? 3);
|
|
5719
|
+
const maxExpansions = nonNegativeInteger("maxExpansions", cfg.maxExpansions ?? 4);
|
|
5720
|
+
const maxAttempts = positiveInteger("maxAttempts", cfg.maxAttempts ?? 3);
|
|
5721
|
+
const seed = nonNegativeInteger("seed", cfg.seed ?? 42);
|
|
5722
|
+
const cellUsd = dollars("cellUsd", cfg.cellUsd) ?? 0;
|
|
5723
|
+
const maxUsd = dollars("maxUsd", cfg.maxUsd);
|
|
5724
|
+
const policy = cfg.policy ?? beam({ width: 2 });
|
|
5725
|
+
const allocation = cfg.allocation ?? asha();
|
|
5726
|
+
if (cfg.deadline !== void 0 && !Number.isFinite(Date.parse(cfg.deadline))) throw new ConfigError("evolution: deadline must be an ISO time");
|
|
5727
|
+
if (!/^sha256:[0-9a-f]{64}$/.test(cfg.executionRef)) throw new ConfigError("evolution: executionRef must be a lowercase sha256:<64 hex> digest");
|
|
5728
|
+
if (!root.name.trim() || typeof root.driver !== "function") throw new ConfigError("evolution: root must be a named Strategy");
|
|
5729
|
+
const claim = defineEvaluationClaim(cfg.claim);
|
|
5730
|
+
if (claim.minimumEffect === void 0) throw new ConfigError("evolution: the claim needs minimumEffect for its power check");
|
|
5731
|
+
const taskIds = /* @__PURE__ */ new Set();
|
|
5732
|
+
const splitTasks = (label, tasks) => {
|
|
5733
|
+
if (!Array.isArray(tasks) || tasks.length === 0) throw new ConfigError(`evolution: ${label} needs at least one task`);
|
|
5734
|
+
for (const task of tasks) {
|
|
5735
|
+
if (typeof task.id !== "string" || !task.id.trim() || taskIds.has(task.id)) throw new ConfigError(`evolution: task IDs must be non-empty and unique across splits`);
|
|
5736
|
+
taskIds.add(task.id);
|
|
5886
5737
|
}
|
|
5738
|
+
return immutableCandidateValue(tasks);
|
|
5887
5739
|
};
|
|
5888
|
-
|
|
5889
|
-
|
|
5890
|
-
|
|
5891
|
-
const
|
|
5892
|
-
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
const
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
5921
|
-
generations,
|
|
5922
|
-
populationSize,
|
|
5923
|
-
baselines: baselines.map((baseline) => baseline.name),
|
|
5924
|
-
objective: cfg.objective ?? "score",
|
|
5925
|
-
scoreTolerance: cfg.scoreTolerance ?? .05,
|
|
5926
|
-
champion: policy,
|
|
5927
|
-
championEpsilon: epsilon,
|
|
5928
|
-
minPairedTasks: cfg.minPairedTasks ?? null,
|
|
5929
|
-
band: cfg.band ?? null,
|
|
5930
|
-
lossesDetail: cfg.lossesDetail ?? "exact",
|
|
5931
|
-
reproducerCheck: cfg.reproducerCheck ?? null,
|
|
5932
|
-
modelPreflight: cfg.modelPreflight !== false,
|
|
5933
|
-
modelPreflightTimeoutMs: cfg.modelPreflightTimeoutMs ?? null,
|
|
5934
|
-
worker: {
|
|
5740
|
+
const train = splitTasks("train", cfg.train);
|
|
5741
|
+
const selection = splitTasks("selection", cfg.selection);
|
|
5742
|
+
const test = splitTasks("test", cfg.test);
|
|
5743
|
+
const taskById = new Map([
|
|
5744
|
+
...train,
|
|
5745
|
+
...selection,
|
|
5746
|
+
...test
|
|
5747
|
+
].map((task) => [task.id, task]));
|
|
5748
|
+
const searchTasks = (tasks) => tasks.map((task) => ({
|
|
5749
|
+
taskId: task.id,
|
|
5750
|
+
unitId: summarizeEvaluationUnits(claim, [task]).units[0].id,
|
|
5751
|
+
source: {
|
|
5752
|
+
uri: `task://${task.id}`,
|
|
5753
|
+
revision: canonicalCandidateDigest$1(task)
|
|
5754
|
+
}
|
|
5755
|
+
}));
|
|
5756
|
+
const splits = {
|
|
5757
|
+
train: searchTasks(train),
|
|
5758
|
+
selection: searchTasks(selection),
|
|
5759
|
+
test: searchTasks(test)
|
|
5760
|
+
};
|
|
5761
|
+
const developmentUnits = new Set([...splits.train, ...splits.selection].map((task) => task.unitId));
|
|
5762
|
+
const workerProfile = cfg.worker.workerProfile;
|
|
5763
|
+
const analystProfile = cfg.worker.analystProfile ?? workerProfile;
|
|
5764
|
+
const workerModel = concreteModelId(workerProfile.model?.default);
|
|
5765
|
+
if (!workerModel) throw new ConfigError("evolution: worker AgentProfile.model.default must name an exact model");
|
|
5766
|
+
const authorModel = concreteModelId(cfg.author.profile.model?.default);
|
|
5767
|
+
if (!authorModel) throw new ConfigError("evolution: author AgentProfile.model.default must name an exact model");
|
|
5768
|
+
const workerSource = {
|
|
5769
|
+
uri: "npm:@tangle-network/agent-runtime#runAgentic",
|
|
5770
|
+
revision: canonicalCandidateDigest$1({
|
|
5771
|
+
worker: canonicalAgentProfileDigest(workerProfile),
|
|
5772
|
+
analyst: canonicalAgentProfileDigest(analystProfile),
|
|
5935
5773
|
routerBaseUrl: cfg.worker.routerBaseUrl,
|
|
5936
|
-
profile: cfg.worker.workerProfile,
|
|
5937
|
-
analystProfile: cfg.worker.analystProfile ?? cfg.worker.workerProfile,
|
|
5938
5774
|
corpusTags: cfg.worker.corpusTags ?? [],
|
|
5939
|
-
corpusReadback: cfg.worker.corpusReadback ?? null
|
|
5775
|
+
corpusReadback: cfg.worker.corpusReadback ?? null,
|
|
5776
|
+
budget
|
|
5777
|
+
})
|
|
5778
|
+
};
|
|
5779
|
+
const authorSource = {
|
|
5780
|
+
uri: `agent-profile:${cfg.author.profile.name}`,
|
|
5781
|
+
revision: canonicalCandidateDigest$1({
|
|
5782
|
+
profile: canonicalAgentProfileDigest(cfg.author.profile),
|
|
5783
|
+
fallback: cfg.author.fallbackProfile ? canonicalAgentProfileDigest(cfg.author.fallbackProfile) : null,
|
|
5784
|
+
backend: cfg.author.executor.backend,
|
|
5785
|
+
contract: strategyAuthorContract
|
|
5786
|
+
})
|
|
5787
|
+
};
|
|
5788
|
+
const environmentSource = {
|
|
5789
|
+
uri: `environment:${cfg.environment.name}`,
|
|
5790
|
+
revision: cfg.executionRef
|
|
5791
|
+
};
|
|
5792
|
+
const execution = {
|
|
5793
|
+
model: searchModelIdentity(workerModel, workerProfile.model?.provider ?? "unknown"),
|
|
5794
|
+
agent: workerSource,
|
|
5795
|
+
benchmark: environmentSource
|
|
5796
|
+
};
|
|
5797
|
+
const searchId = `evolution-${canonicalCandidateDigest$1({
|
|
5798
|
+
process: STRATEGY_EVOLUTION_SOURCE,
|
|
5799
|
+
executionRef: cfg.executionRef,
|
|
5800
|
+
root: root.name,
|
|
5801
|
+
worker: workerSource,
|
|
5802
|
+
author: authorSource,
|
|
5803
|
+
claim,
|
|
5804
|
+
splits,
|
|
5805
|
+
policy: policy.name,
|
|
5806
|
+
allocation: allocation.name,
|
|
5807
|
+
reps: allocation.reps,
|
|
5808
|
+
maxExpansions,
|
|
5809
|
+
concurrency,
|
|
5810
|
+
maxUsd,
|
|
5811
|
+
cellUsd,
|
|
5812
|
+
deadline: cfg.deadline ?? null,
|
|
5813
|
+
maxAttempts,
|
|
5814
|
+
seed
|
|
5815
|
+
}).slice(7, 31)}`;
|
|
5816
|
+
const searchDir = `${cfg.outDir}/${searchId}`;
|
|
5817
|
+
const modulesDir = `${searchDir}/strategies`;
|
|
5818
|
+
const ledgerPath = `${searchDir}/ledger.jsonl`;
|
|
5819
|
+
mkdirSync(searchDir, { recursive: true });
|
|
5820
|
+
const recorder = await SearchRecorder.open({ ledger: openSearchLedger({
|
|
5821
|
+
path: ledgerPath,
|
|
5822
|
+
searchId
|
|
5823
|
+
}) }, {
|
|
5824
|
+
subject: `strategy/${cfg.environment.name}`,
|
|
5825
|
+
process: {
|
|
5826
|
+
name: "strategy-evolution",
|
|
5827
|
+
executionRef: STRATEGY_EVOLUTION_SOURCE
|
|
5940
5828
|
},
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
5829
|
+
artifactKind: "code",
|
|
5830
|
+
objective: {
|
|
5831
|
+
metric: "score",
|
|
5832
|
+
direction: "maximize",
|
|
5833
|
+
judge: environmentSource,
|
|
5834
|
+
claim
|
|
5945
5835
|
},
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
...cfg.hooks ? { hooks: cfg.hooks } : {}
|
|
5980
|
-
});
|
|
5981
|
-
modelsPreflighted = true;
|
|
5982
|
-
return report;
|
|
5836
|
+
splits: {
|
|
5837
|
+
...splits,
|
|
5838
|
+
heldOutUnits: splits.test.every((task) => !developmentUnits.has(task.unitId))
|
|
5839
|
+
},
|
|
5840
|
+
policy: {
|
|
5841
|
+
expansion: policy.name,
|
|
5842
|
+
allocation: allocation.name,
|
|
5843
|
+
seed
|
|
5844
|
+
},
|
|
5845
|
+
budget: {
|
|
5846
|
+
maxUsd,
|
|
5847
|
+
maxCells: null,
|
|
5848
|
+
maxNodes: 1 + maxExpansions,
|
|
5849
|
+
deadline: cfg.deadline === void 0 ? null : new Date(cfg.deadline).toISOString(),
|
|
5850
|
+
maxConcurrency: concurrency,
|
|
5851
|
+
reservedClaimUsd: maxUsd === null ? 0 : searchClaimReserveUsd({
|
|
5852
|
+
testTasks: splits.test.length,
|
|
5853
|
+
reps: allocation.reps,
|
|
5854
|
+
cellUsd
|
|
5855
|
+
})
|
|
5856
|
+
},
|
|
5857
|
+
containment: null,
|
|
5858
|
+
derivedFrom: null,
|
|
5859
|
+
identity: execution
|
|
5860
|
+
});
|
|
5861
|
+
if (!(await recorder.state()).closed) await preflightModels({
|
|
5862
|
+
worker: cfg.worker,
|
|
5863
|
+
modelPreflight: cfg.modelPreflight,
|
|
5864
|
+
modelPreflightTimeoutMs: cfg.modelPreflightTimeoutMs
|
|
5865
|
+
});
|
|
5866
|
+
const rootNode = {
|
|
5867
|
+
kind: "root",
|
|
5868
|
+
name: root.name
|
|
5983
5869
|
};
|
|
5984
|
-
const
|
|
5985
|
-
|
|
5986
|
-
|
|
5987
|
-
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
5993
|
-
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
generations: generationRows,
|
|
6029
|
-
archive,
|
|
6030
|
-
trajectory
|
|
5870
|
+
const codec = {
|
|
5871
|
+
node(rec, node) {
|
|
5872
|
+
const artifact = rec.blob("strategy", { ...node });
|
|
5873
|
+
return {
|
|
5874
|
+
artifactDigest: node.kind === "root" ? canonicalCandidateDigest$1({
|
|
5875
|
+
kind: "root",
|
|
5876
|
+
name: node.name
|
|
5877
|
+
}) : sha256Bytes$1(Buffer.from(node.code, "utf8")),
|
|
5878
|
+
artifact,
|
|
5879
|
+
surfaces: [{
|
|
5880
|
+
surfaceId: "strategy",
|
|
5881
|
+
kind: "code",
|
|
5882
|
+
artifact
|
|
5883
|
+
}]
|
|
5884
|
+
};
|
|
5885
|
+
},
|
|
5886
|
+
diff(rec, parent, child) {
|
|
5887
|
+
if (parent.kind === "root" || child.kind === "root") return { unknown: "the root is a caller-supplied strategy with no source here" };
|
|
5888
|
+
return surfaceDiff(rec, parent.code, child.code);
|
|
5889
|
+
},
|
|
5890
|
+
load(rec, node) {
|
|
5891
|
+
const stored = rec.readBlob(node.artifact);
|
|
5892
|
+
if (stored.kind === "root") {
|
|
5893
|
+
if (stored.name !== root.name) throw new Error(`evolution: node ${node.nodeId} is root '${stored.name}', not '${root.name}'`);
|
|
5894
|
+
return rootNode;
|
|
5895
|
+
}
|
|
5896
|
+
if (stored.kind === "authored" && typeof stored.code === "string") return stored;
|
|
5897
|
+
throw new Error(`evolution: node ${node.nodeId} holds no strategy`);
|
|
5898
|
+
}
|
|
5899
|
+
};
|
|
5900
|
+
const loaded = /* @__PURE__ */ new Map();
|
|
5901
|
+
const strategyOf = (node) => {
|
|
5902
|
+
if (node.kind === "root") return Promise.resolve(root);
|
|
5903
|
+
let strategy = loaded.get(node.code);
|
|
5904
|
+
if (!strategy) {
|
|
5905
|
+
strategy = loadAuthoredStrategy(node.code, modulesDir).then((module) => module.strategy);
|
|
5906
|
+
loaded.set(node.code, strategy);
|
|
5907
|
+
}
|
|
5908
|
+
return strategy;
|
|
5909
|
+
};
|
|
5910
|
+
const storage = fsCampaignStorage();
|
|
5911
|
+
const costLedger = createRunCostLedger({
|
|
5912
|
+
storage,
|
|
5913
|
+
runDir: searchDir
|
|
6031
5914
|
});
|
|
6032
|
-
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
|
|
6036
|
-
|
|
6037
|
-
|
|
6038
|
-
|
|
6039
|
-
|
|
5915
|
+
const lanes = [dedicatedLane({
|
|
5916
|
+
name: "in-process",
|
|
5917
|
+
capacity: concurrency,
|
|
5918
|
+
cellUsd
|
|
5919
|
+
})];
|
|
5920
|
+
await reconcileInterruptedSearchCalls(costLedger, searchId, lanes);
|
|
5921
|
+
const executor = searchExecutor({
|
|
5922
|
+
lanes,
|
|
5923
|
+
profileOf: () => workerProfile,
|
|
5924
|
+
costLedger,
|
|
5925
|
+
identity: execution,
|
|
5926
|
+
storage,
|
|
5927
|
+
dir: searchDir,
|
|
5928
|
+
...cfg.workerSlots ? { workerSlots: cfg.workerSlots } : {},
|
|
5929
|
+
async runAttempt({ work, lane, costLedger: cellLedger }) {
|
|
5930
|
+
if (work.artifact.kind === "authored" && work.artifact.refusal !== null) throw new Error(`evolution: refused node ${work.nodeId} was given a cell`);
|
|
5931
|
+
const strategy = await strategyOf(work.artifact);
|
|
5932
|
+
const task = taskById.get(work.taskId);
|
|
5933
|
+
if (!task) throw new Error(`evolution: cell ${work.cellId} names unknown task ${work.taskId}`);
|
|
5934
|
+
const faults = [];
|
|
5935
|
+
const environment = faultRecording(cfg.environment, faults);
|
|
5936
|
+
const paid = await cellLedger.runPaidCall({
|
|
5937
|
+
channel: "agent",
|
|
5938
|
+
phase: `search.${work.stage}`,
|
|
5939
|
+
actor: `strategy:${strategy.name}`,
|
|
5940
|
+
model: workerModel,
|
|
5941
|
+
signal: work.signal,
|
|
5942
|
+
execute: () => runAgentic({
|
|
5943
|
+
...cfg.worker,
|
|
5944
|
+
surface: environment,
|
|
5945
|
+
task,
|
|
5946
|
+
strategy,
|
|
5947
|
+
budget,
|
|
5948
|
+
...cfg.hooks ? { hooks: cfg.hooks } : {}
|
|
5949
|
+
}),
|
|
5950
|
+
receipt: (run) => ({
|
|
5951
|
+
model: workerModel,
|
|
5952
|
+
inputTokens: run.tokens.input,
|
|
5953
|
+
outputTokens: run.tokens.output,
|
|
5954
|
+
...run.usdKnown ? { actualCostUsd: run.usd } : { costUnknown: true },
|
|
5955
|
+
...run.tokensKnown ? {} : { usageUnknown: true }
|
|
5956
|
+
}),
|
|
5957
|
+
receiptFromError: () => ({
|
|
5958
|
+
model: workerModel,
|
|
5959
|
+
inputTokens: 0,
|
|
5960
|
+
outputTokens: 0,
|
|
5961
|
+
costUnknown: true,
|
|
5962
|
+
usageUnknown: true
|
|
5963
|
+
})
|
|
5964
|
+
});
|
|
5965
|
+
const settled = {
|
|
5966
|
+
identity: execution,
|
|
5967
|
+
placement: {
|
|
5968
|
+
lane: lane.name,
|
|
5969
|
+
boxId: null
|
|
5970
|
+
},
|
|
5971
|
+
accounting: unknownAccounting("replaced by the attempt receipts"),
|
|
5972
|
+
...paid.succeeded ? { wallMs: paid.value.ms } : {}
|
|
5973
|
+
};
|
|
5974
|
+
if (faults.length > 0) return {
|
|
5975
|
+
...settled,
|
|
5976
|
+
...environmentFault(faults)
|
|
5977
|
+
};
|
|
5978
|
+
if (!paid.succeeded) return {
|
|
5979
|
+
...settled,
|
|
5980
|
+
outcome: {
|
|
5981
|
+
status: "failed",
|
|
5982
|
+
score: 0,
|
|
5983
|
+
metrics: {},
|
|
5984
|
+
failure: {
|
|
5985
|
+
code: "strategy-threw",
|
|
5986
|
+
message: shareText(paid.error)
|
|
5987
|
+
}
|
|
5988
|
+
}
|
|
5989
|
+
};
|
|
5990
|
+
const run = paid.value;
|
|
5991
|
+
return {
|
|
5992
|
+
...settled,
|
|
5993
|
+
outcome: {
|
|
5994
|
+
status: "passed",
|
|
5995
|
+
score: run.score,
|
|
5996
|
+
metrics: {
|
|
5997
|
+
resolved: run.resolved ? 1 : 0,
|
|
5998
|
+
shots: run.shots
|
|
5999
|
+
}
|
|
6000
|
+
}
|
|
6001
|
+
};
|
|
6002
|
+
}
|
|
6003
|
+
});
|
|
6004
|
+
let toolCatalog;
|
|
6005
|
+
const catalog = () => {
|
|
6006
|
+
toolCatalog ??= listToolCatalog(cfg.environment, train[0]);
|
|
6007
|
+
return toolCatalog;
|
|
6008
|
+
};
|
|
6009
|
+
const authorProvider = cfg.author.profile.model?.provider ?? "unknown";
|
|
6010
|
+
const result = await runSearch({
|
|
6011
|
+
recorder,
|
|
6012
|
+
root: rootNode,
|
|
6013
|
+
codec,
|
|
6014
|
+
policy,
|
|
6015
|
+
allocation,
|
|
6016
|
+
proposer: {
|
|
6017
|
+
name: "strategy-author",
|
|
6018
|
+
kind: "frontier-author",
|
|
6019
|
+
source: authorSource,
|
|
6020
|
+
execution: {
|
|
6021
|
+
kind: "model",
|
|
6022
|
+
model: searchModelIdentity(authorModel, authorProvider),
|
|
6023
|
+
source: authorSource
|
|
6024
|
+
},
|
|
6025
|
+
childrenPerProposal: 1,
|
|
6026
|
+
async propose(request) {
|
|
6027
|
+
const state = await recorder.state();
|
|
6028
|
+
const names = strategyNames(state, recorder, codec);
|
|
6029
|
+
const losses = JSON.stringify(strategyTournament(state, "train", names).rows.map((row) => ({
|
|
6030
|
+
task: row.taskId,
|
|
6031
|
+
...row.rep > 0 ? { rep: row.rep } : {},
|
|
6032
|
+
cells: row.cells
|
|
6033
|
+
}))).slice(0, 12e3);
|
|
6034
|
+
const summary = renderSearchSummary(state, { split: "train" });
|
|
6035
|
+
const parent = request.parents[0];
|
|
6036
|
+
const parentName = names.get(parent.nodeId) ?? parent.nodeId;
|
|
6037
|
+
const tried = [...names.values()].join(", ");
|
|
6038
|
+
const parentSource = parent.artifact.kind === "authored" ? `\n\nPARENT STRATEGY "${parentName}" (improve on it):\n${parent.artifact.code}` : `\n\nPARENT STRATEGY "${parentName}" is the built-in baseline; improve on it.`;
|
|
6039
|
+
const contract = `${strategyAuthorContract}\n\nEXAMPLE TOOLS FROM ONE TASK (tool sets VARY per task on this domain — a strategy MUST select tool names from await listTools(handle) at runtime; hardcoding these example names will zero your score on most tasks):\n${await catalog()}\n\nTHE SEARCH SO FAR (train split):\n${summary}\n\nSTRATEGIES ALREADY TRIED (author something MEANINGFULLY different — a new composition, not a rename): ${tried}${parentSource}`;
|
|
6040
|
+
const source = await requestStrategySource({
|
|
6040
6041
|
profile: cfg.author.profile,
|
|
6041
6042
|
executor: cfg.author.executor,
|
|
6042
6043
|
...cfg.author.fallbackProfile ? { fallbackProfile: cfg.author.fallbackProfile } : {},
|
|
6043
6044
|
contract,
|
|
6044
6045
|
environmentName: cfg.environment.name,
|
|
6045
|
-
lossesJson,
|
|
6046
|
+
lossesJson: losses,
|
|
6046
6047
|
budget,
|
|
6047
|
-
outDir:
|
|
6048
|
-
|
|
6049
|
-
const unique = byName.has(authored.strategy.name) ? `${authored.strategy.name}-g${g}c${i + 1}` : authored.strategy.name;
|
|
6050
|
-
const strategy = renameStrategy(authored.strategy, unique);
|
|
6051
|
-
byName.set(unique, strategy);
|
|
6052
|
-
codeByName.set(unique, authored.code);
|
|
6053
|
-
newStrategies.push(strategy);
|
|
6054
|
-
archive.push({
|
|
6055
|
-
name: unique,
|
|
6056
|
-
source: "authored",
|
|
6057
|
-
generation: g,
|
|
6058
|
-
parent: incumbent.name,
|
|
6059
|
-
gzipBits: gzipSync(Buffer.from(authored.code)).length * 8,
|
|
6060
|
-
file: authored.file,
|
|
6061
|
-
score: 0,
|
|
6062
|
-
usd: 0
|
|
6048
|
+
outDir: modulesDir,
|
|
6049
|
+
signal: request.signal
|
|
6063
6050
|
});
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6051
|
+
let child = {
|
|
6052
|
+
kind: "authored",
|
|
6053
|
+
name: null,
|
|
6054
|
+
code: "",
|
|
6055
|
+
refusal: NO_MODULE
|
|
6056
|
+
};
|
|
6057
|
+
if (source.code !== null) try {
|
|
6058
|
+
const { strategy } = await loadAuthoredStrategy(source.code, modulesDir);
|
|
6059
|
+
child = {
|
|
6060
|
+
kind: "authored",
|
|
6061
|
+
name: strategy.name,
|
|
6062
|
+
code: source.code,
|
|
6063
|
+
refusal: null
|
|
6064
|
+
};
|
|
6065
|
+
} catch (error) {
|
|
6066
|
+
child = {
|
|
6067
|
+
kind: "authored",
|
|
6068
|
+
name: null,
|
|
6069
|
+
code: source.code,
|
|
6070
|
+
refusal: shareText(error)
|
|
6071
|
+
};
|
|
6072
|
+
}
|
|
6073
|
+
const answered = source.turns.at(-1);
|
|
6074
|
+
return {
|
|
6075
|
+
children: [{
|
|
6076
|
+
artifact: child,
|
|
6077
|
+
label: child.name ?? "refused module",
|
|
6078
|
+
rationale: `authored from the train results of ${parentName} and ${names.size - 1} other strategies`
|
|
6079
|
+
}],
|
|
6080
|
+
accounting: authorAccounting(source.turns),
|
|
6081
|
+
...answered ? { execution: {
|
|
6082
|
+
kind: "model",
|
|
6083
|
+
model: searchModelIdentity(answered.model, authorProvider),
|
|
6084
|
+
source: authorSource
|
|
6085
|
+
} } : {}
|
|
6086
|
+
};
|
|
6088
6087
|
}
|
|
6088
|
+
},
|
|
6089
|
+
executor,
|
|
6090
|
+
admit: (node) => node.kind === "authored" ? node.refusal : null,
|
|
6091
|
+
maxExpansions,
|
|
6092
|
+
maxAttempts,
|
|
6093
|
+
...cfg.signal ? { signal: cfg.signal } : {}
|
|
6094
|
+
});
|
|
6095
|
+
if (!result.claim || !result.claimVerification) throw new Error(`evolution: search ${searchId} closed without a claim`);
|
|
6096
|
+
const { state } = result;
|
|
6097
|
+
const names = strategyNames(state, recorder, codec);
|
|
6098
|
+
const rootId = state.rootNodeId;
|
|
6099
|
+
const strategies = state.nodes().map((node) => {
|
|
6100
|
+
const artifact = codec.load(recorder, node);
|
|
6101
|
+
const authored = artifact.kind === "authored" ? artifact : null;
|
|
6102
|
+
return {
|
|
6103
|
+
name: names.get(node.nodeId),
|
|
6104
|
+
nodeId: node.nodeId,
|
|
6105
|
+
parent: node.primaryParentId === null ? null : names.get(node.primaryParentId),
|
|
6106
|
+
source: artifact.kind,
|
|
6107
|
+
status: node.status,
|
|
6108
|
+
code: authored?.code ?? null,
|
|
6109
|
+
gzipBits: authored ? gzipSync(Buffer.from(authored.code)).length * 8 : null,
|
|
6110
|
+
refusal: authored?.refusal ?? null,
|
|
6111
|
+
selection: node.nodeId === rootId ? null : estimateNode(state, node.nodeId, {
|
|
6112
|
+
against: rootId,
|
|
6113
|
+
split: "selection"
|
|
6114
|
+
})
|
|
6115
|
+
};
|
|
6116
|
+
});
|
|
6117
|
+
const claimResult = result.claim;
|
|
6118
|
+
const shipped = claimResult.finalists.find((finalist) => finalist.nodeId === claimResult.selected && finalist.test !== null);
|
|
6119
|
+
const { spend } = state.audit;
|
|
6120
|
+
const { decision, reason } = runtimeShipDecision({
|
|
6121
|
+
claim: claimResult,
|
|
6122
|
+
verified: result.claimVerification.status === "verified",
|
|
6123
|
+
accountingComplete: spend.unknownCostCells === 0 && spend.unknownCostOperations === 0,
|
|
6124
|
+
lowerBound: shipped?.test?.interval[0] ?? null,
|
|
6125
|
+
minimumLift: 0
|
|
6126
|
+
});
|
|
6127
|
+
return {
|
|
6128
|
+
searchId,
|
|
6129
|
+
ledger: ledgerPath,
|
|
6130
|
+
closeReason: result.reason,
|
|
6131
|
+
strategies,
|
|
6132
|
+
selected: strategies.find((strategy) => strategy.nodeId === result.leader),
|
|
6133
|
+
claim: claimResult,
|
|
6134
|
+
claimVerification: result.claimVerification,
|
|
6135
|
+
decision,
|
|
6136
|
+
reason,
|
|
6137
|
+
tournament: {
|
|
6138
|
+
train: strategyTournament(state, "train", names),
|
|
6139
|
+
selection: strategyTournament(state, "selection", names),
|
|
6140
|
+
test: strategyTournament(state, "test", names)
|
|
6141
|
+
},
|
|
6142
|
+
spend: {
|
|
6143
|
+
knownUsd: spend.knownUsd,
|
|
6144
|
+
floorUsd: spend.floorUsd,
|
|
6145
|
+
unknownCostCells: spend.unknownCostCells,
|
|
6146
|
+
unknownCostOperations: spend.unknownCostOperations
|
|
6089
6147
|
}
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
|
|
6095
|
-
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
generations: generationRows,
|
|
6110
|
-
archive,
|
|
6111
|
-
trajectory
|
|
6112
|
-
});
|
|
6148
|
+
};
|
|
6149
|
+
}
|
|
6150
|
+
/**
|
|
6151
|
+
* The tournament on one split, projected from the ledger: each settled cell's final attempt,
|
|
6152
|
+
* by task and repeat, keyed by strategy name. `names` maps node ids to report names.
|
|
6153
|
+
*/
|
|
6154
|
+
function strategyTournament(state, split, names) {
|
|
6155
|
+
const rows = /* @__PURE__ */ new Map();
|
|
6156
|
+
const perStrategy = {};
|
|
6157
|
+
for (const nodeId of state.nodeIds()) {
|
|
6158
|
+
const name = names.get(nodeId);
|
|
6159
|
+
if (name === void 0) throw new Error(`strategyTournament: node ${nodeId} has no name`);
|
|
6160
|
+
perStrategy[name] = {
|
|
6161
|
+
cells: 0,
|
|
6162
|
+
scored: 0,
|
|
6163
|
+
score: null,
|
|
6164
|
+
usd: 0,
|
|
6165
|
+
usdKnown: true
|
|
6166
|
+
};
|
|
6113
6167
|
}
|
|
6114
|
-
|
|
6115
|
-
const
|
|
6116
|
-
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6120
|
-
|
|
6121
|
-
|
|
6122
|
-
|
|
6123
|
-
|
|
6124
|
-
|
|
6125
|
-
|
|
6126
|
-
|
|
6127
|
-
|
|
6168
|
+
const sums = /* @__PURE__ */ new Map();
|
|
6169
|
+
const cells = state.cells().filter((cell) => cell.split === split && cell.outcome !== null).sort((left, right) => compare(left.taskId, right.taskId) || left.rep - right.rep);
|
|
6170
|
+
for (const cell of cells) {
|
|
6171
|
+
const name = names.get(cell.nodeId);
|
|
6172
|
+
const key = `${cell.taskId}\u0000${cell.rep}`;
|
|
6173
|
+
let row = rows.get(key);
|
|
6174
|
+
if (!row) {
|
|
6175
|
+
row = {
|
|
6176
|
+
taskId: cell.taskId,
|
|
6177
|
+
unitId: cell.unitId,
|
|
6178
|
+
rep: cell.rep,
|
|
6179
|
+
cells: {}
|
|
6180
|
+
};
|
|
6181
|
+
rows.set(key, row);
|
|
6128
6182
|
}
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
6135
|
-
score: r.cells?.[reference.name]?.score ?? 0
|
|
6136
|
-
}));
|
|
6137
|
-
const inBandIds = new Set(refScores.filter((r) => r.score <= maxRef).map((r) => r.taskId));
|
|
6138
|
-
const kept = pool.filter((t) => inBandIds.has(t.id));
|
|
6139
|
-
if (kept.length < cfg.holdoutN) throw new Error(`evolution band: only ${kept.length}/${cfg.holdoutN} holdout tasks have headroom (pool ${cfg.band.holdoutPoolN}, reference "${reference.name}" ≤ ${maxRef}) — widen holdoutPoolN or raise maxRefScore`);
|
|
6140
|
-
holdoutTasks = kept.slice(0, cfg.holdoutN);
|
|
6141
|
-
bandInfo = {
|
|
6142
|
-
screened: refScores.length,
|
|
6143
|
-
inBand: kept.length,
|
|
6144
|
-
refScores
|
|
6183
|
+
const outcome = cell.outcome;
|
|
6184
|
+
row.cells[name] = {
|
|
6185
|
+
score: cell.score,
|
|
6186
|
+
outcome,
|
|
6187
|
+
usd: cell.spentUsd,
|
|
6188
|
+
usdKnown: cell.costKnown
|
|
6145
6189
|
};
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
|
|
6163
|
-
|
|
6164
|
-
|
|
6165
|
-
|
|
6166
|
-
|
|
6167
|
-
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
|
|
6171
|
-
holdout,
|
|
6172
|
-
verdict,
|
|
6173
|
-
...bandInfo ? { band: bandInfo } : {}
|
|
6174
|
-
});
|
|
6190
|
+
const summary = perStrategy[name];
|
|
6191
|
+
summary.cells += 1;
|
|
6192
|
+
summary.usd += cell.spentUsd;
|
|
6193
|
+
summary.usdKnown &&= cell.costKnown;
|
|
6194
|
+
if (cell.score !== null) {
|
|
6195
|
+
summary.scored += 1;
|
|
6196
|
+
sums.set(name, (sums.get(name) ?? 0) + cell.score);
|
|
6197
|
+
}
|
|
6198
|
+
}
|
|
6199
|
+
for (const [name, summary] of Object.entries(perStrategy)) summary.score = summary.scored === 0 ? null : sums.get(name) / summary.scored;
|
|
6200
|
+
return {
|
|
6201
|
+
split,
|
|
6202
|
+
rows: [...rows.values()],
|
|
6203
|
+
perStrategy
|
|
6204
|
+
};
|
|
6205
|
+
}
|
|
6206
|
+
/** Report names: each node's strategy name, suffixed `~<ordinal>` when an earlier node has it. */
|
|
6207
|
+
function strategyNames(state, recorder, codec) {
|
|
6208
|
+
const names = /* @__PURE__ */ new Map();
|
|
6209
|
+
const taken = /* @__PURE__ */ new Set();
|
|
6210
|
+
for (const node of state.nodes()) {
|
|
6211
|
+
const base = codec.load(recorder, node).name ?? "refused-module";
|
|
6212
|
+
const name = taken.has(base) ? `${base}~${node.ordinal}` : base;
|
|
6213
|
+
taken.add(name);
|
|
6214
|
+
names.set(node.nodeId, name);
|
|
6175
6215
|
}
|
|
6176
|
-
|
|
6177
|
-
|
|
6178
|
-
|
|
6179
|
-
|
|
6180
|
-
|
|
6181
|
-
|
|
6216
|
+
return names;
|
|
6217
|
+
}
|
|
6218
|
+
/** The environment as one attempt sees it, recording each throw from `open`, `tools`,
|
|
6219
|
+
* `score` or `close`: those are the environment's faults. A `call` that throws is a tool
|
|
6220
|
+
* error the agent's loop handles. */
|
|
6221
|
+
function faultRecording(environment, faults) {
|
|
6222
|
+
const recorded = (name, method) => async (...args) => {
|
|
6182
6223
|
try {
|
|
6183
|
-
|
|
6184
|
-
|
|
6185
|
-
|
|
6186
|
-
|
|
6187
|
-
...cfg.author.profile.prompt,
|
|
6188
|
-
systemPrompt: `Summarize the optimization strategy implemented by this code in at most ${words} words. Describe the COMPOSITION (shots, critique, artifact handling, restarts, stopping) — not the code. Output only the summary.`
|
|
6189
|
-
}
|
|
6190
|
-
},
|
|
6191
|
-
executor: cfg.author.executor,
|
|
6192
|
-
context: "strategy reproducer summary"
|
|
6193
|
-
}).chat({ messages: [{
|
|
6194
|
-
role: "user",
|
|
6195
|
-
content: championCode
|
|
6196
|
-
}] })).content.trim();
|
|
6197
|
-
const reproduced = await authorStrategy({
|
|
6198
|
-
profile: cfg.author.profile,
|
|
6199
|
-
executor: cfg.author.executor,
|
|
6200
|
-
...cfg.author.fallbackProfile ? { fallbackProfile: cfg.author.fallbackProfile } : {},
|
|
6201
|
-
contract: `${strategyAuthorContract}\n\nIMPLEMENT EXACTLY THIS STRATEGY (a colleague's description — do not invent a different approach):\n${summary}`,
|
|
6202
|
-
environmentName: cfg.environment.name,
|
|
6203
|
-
lossesJson: "[]",
|
|
6204
|
-
budget,
|
|
6205
|
-
outDir: cfg.outDir
|
|
6206
|
-
});
|
|
6207
|
-
const reproStrategy = {
|
|
6208
|
-
name: `${incumbent.name}-reproduced`,
|
|
6209
|
-
driver: reproduced.strategy.driver
|
|
6210
|
-
};
|
|
6211
|
-
const reproducedHoldoutScore = (await bench("reproduce", holdoutTasks, [reproStrategy])).perStrategy[reproStrategy.name]?.score ?? 0;
|
|
6212
|
-
reproduction = {
|
|
6213
|
-
summary,
|
|
6214
|
-
reproducedName: reproStrategy.name,
|
|
6215
|
-
file: reproduced.file,
|
|
6216
|
-
championHoldoutScore,
|
|
6217
|
-
reproducedHoldoutScore,
|
|
6218
|
-
gap: championHoldoutScore - reproducedHoldoutScore,
|
|
6219
|
-
reproducible: reproducedHoldoutScore >= championHoldoutScore - tolerance
|
|
6220
|
-
};
|
|
6221
|
-
} catch (e) {
|
|
6222
|
-
reproduction = {
|
|
6223
|
-
summary: "",
|
|
6224
|
-
reproducedName: "",
|
|
6225
|
-
championHoldoutScore,
|
|
6226
|
-
reproducedHoldoutScore: 0,
|
|
6227
|
-
gap: championHoldoutScore,
|
|
6228
|
-
reproducible: false,
|
|
6229
|
-
error: e instanceof Error ? e.message.slice(0, 300) : String(e)
|
|
6230
|
-
};
|
|
6224
|
+
return await method(...args);
|
|
6225
|
+
} catch (error) {
|
|
6226
|
+
faults.push(`${name}: ${shareText(error)}`);
|
|
6227
|
+
throw error;
|
|
6231
6228
|
}
|
|
6232
|
-
}
|
|
6229
|
+
};
|
|
6233
6230
|
return {
|
|
6234
|
-
|
|
6235
|
-
|
|
6236
|
-
|
|
6237
|
-
|
|
6238
|
-
|
|
6239
|
-
|
|
6240
|
-
|
|
6241
|
-
|
|
6242
|
-
|
|
6243
|
-
|
|
6244
|
-
|
|
6245
|
-
}
|
|
6246
|
-
|
|
6247
|
-
|
|
6248
|
-
|
|
6249
|
-
|
|
6250
|
-
|
|
6251
|
-
|
|
6231
|
+
name: environment.name,
|
|
6232
|
+
open: recorded("open", (task) => environment.open(task)),
|
|
6233
|
+
tools: recorded("tools", (task, handle) => environment.tools(task, handle)),
|
|
6234
|
+
call: (handle, name, args) => environment.call(handle, name, args),
|
|
6235
|
+
score: recorded("score", (task, handle) => environment.score(task, handle)),
|
|
6236
|
+
close: recorded("close", (handle) => environment.close(handle))
|
|
6237
|
+
};
|
|
6238
|
+
}
|
|
6239
|
+
function environmentFault(faults) {
|
|
6240
|
+
return { outcome: {
|
|
6241
|
+
status: "errored",
|
|
6242
|
+
metrics: {},
|
|
6243
|
+
error: {
|
|
6244
|
+
code: "environment-fault",
|
|
6245
|
+
message: faults[0],
|
|
6246
|
+
retryable: true
|
|
6247
|
+
}
|
|
6248
|
+
} };
|
|
6249
|
+
}
|
|
6250
|
+
function authorAccounting(turns) {
|
|
6251
|
+
let usd = 0;
|
|
6252
|
+
let costKnown = true;
|
|
6253
|
+
for (const turn of turns) if (turn.costUsd === null) costKnown = false;
|
|
6254
|
+
else usd += turn.costUsd;
|
|
6255
|
+
return {
|
|
6256
|
+
tokens: turns.every((turn) => turn.tokensKnown) ? {
|
|
6257
|
+
status: "known",
|
|
6258
|
+
inputTokens: turns.reduce((sum, turn) => sum + turn.inputTokens, 0),
|
|
6259
|
+
outputTokens: turns.reduce((sum, turn) => sum + turn.outputTokens, 0),
|
|
6260
|
+
cachedTokens: 0
|
|
6261
|
+
} : {
|
|
6262
|
+
status: "unknown",
|
|
6263
|
+
reason: "an authoring call reported no token usage"
|
|
6264
|
+
},
|
|
6265
|
+
cost: costKnown ? {
|
|
6266
|
+
status: "known",
|
|
6267
|
+
usd,
|
|
6268
|
+
source: usd === 0 ? "free" : "provider"
|
|
6269
|
+
} : {
|
|
6270
|
+
status: "unknown",
|
|
6271
|
+
knownLowerBoundUsd: usd,
|
|
6272
|
+
reason: "an authoring call reported no billed cost"
|
|
6273
|
+
}
|
|
6274
|
+
};
|
|
6275
|
+
}
|
|
6276
|
+
function unknownAccounting(reason) {
|
|
6277
|
+
return {
|
|
6278
|
+
tokens: {
|
|
6279
|
+
status: "unknown",
|
|
6280
|
+
reason
|
|
6281
|
+
},
|
|
6282
|
+
cost: {
|
|
6283
|
+
status: "unknown",
|
|
6284
|
+
knownLowerBoundUsd: 0,
|
|
6285
|
+
reason
|
|
6286
|
+
}
|
|
6287
|
+
};
|
|
6288
|
+
}
|
|
6289
|
+
/** One probe round-trip lists the domain's tools so the author can focus shots, names and
|
|
6290
|
+
* descriptions only, never the implementations. */
|
|
6291
|
+
async function listToolCatalog(environment, task) {
|
|
6292
|
+
const handle = await environment.open(task);
|
|
6293
|
+
try {
|
|
6294
|
+
return (await environment.tools(task, handle)).map((tool) => `- ${tool.function.name}${tool.function.description ? ` — ${tool.function.description.slice(0, 120)}` : ""}`).join("\n");
|
|
6295
|
+
} finally {
|
|
6296
|
+
await environment.close(handle);
|
|
6252
6297
|
}
|
|
6253
|
-
|
|
6298
|
+
}
|
|
6299
|
+
/** Free text as the ledger stores it: redacted with the share profile and bounded. */
|
|
6300
|
+
function shareText(error) {
|
|
6301
|
+
return redact(error instanceof Error ? error.message : String(error), { profile: "share" }).value.slice(0, 300);
|
|
6302
|
+
}
|
|
6303
|
+
function compare(left, right) {
|
|
6304
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
6305
|
+
}
|
|
6306
|
+
function positiveInteger(name, value) {
|
|
6307
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new ConfigError(`evolution: ${name} must be a positive integer`);
|
|
6308
|
+
return value;
|
|
6309
|
+
}
|
|
6310
|
+
function nonNegativeInteger(name, value) {
|
|
6311
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new ConfigError(`evolution: ${name} must be a non-negative integer`);
|
|
6312
|
+
return value;
|
|
6313
|
+
}
|
|
6314
|
+
function dollars(name, value) {
|
|
6315
|
+
if (value === void 0) return null;
|
|
6316
|
+
if (!Number.isFinite(value) || value < 0) throw new ConfigError(`evolution: ${name} must be a finite non-negative number`);
|
|
6317
|
+
return value;
|
|
6254
6318
|
}
|
|
6255
6319
|
//#endregion
|
|
6256
6320
|
//#region src/runtime/supervise/chat-transport-executor.ts
|
|
@@ -8241,6 +8305,6 @@ function tail(s) {
|
|
|
8241
8305
|
return s.slice(-400);
|
|
8242
8306
|
}
|
|
8243
8307
|
//#endregion
|
|
8244
|
-
export {
|
|
8308
|
+
export { renderCorpusToInstructions as $, resolveMcpServerLaunch as $t, unsafeInProcessRunner as A, completionAuthorizes as At, SandboxRunAbortError as B, auditIntent as Bt, withUntrackedArtifacts as C, assertDeclaredCheck as Ct, patchDelivered as D, declaredCheckJudge as Dt, createSqlRunContext as E, declaredCheckDigest as Et, strategyTournament as F, pairwiseSignificance as Ft, trajectoryReport as G, SqlSpawnJournal as Gt, printBenchmarkReport as H, SqlRunOwnershipError as Ht, assertStrategyContract as I, renderLeaderboardHtml as It, builtinShapes as J, materializeLocalMcp as Jt, definePersona as K, McpSpawnFault as Kt, authorStrategy as L, renderLeaderboardMarkdown as Lt, chatWorkerSeam as M, sentinelCompletion as Mt, createChatSessionStore as N, stopSentinel as Nt, runCoderChecks as O, readDeclaredCheck as Ot, runStrategyEvolution as P, leaderboard as Pt, InMemoryCorpus as Q, mcpSecretEnvMetadataKey as Qt, strategyAuthorContract as R, renderLeaderboardSvg as Rt, copyUntrackedIntoClone as S, superviseDispatch as St, createFencedSqlRunContext as T, declaredCheckDeliverable as Tt, runBenchmark as U, openSqlRunStore as Ut, openSandboxRun as V, defaultAuditorInstruction as Vt, equalKOnCost as W, SqlResultBlobStore as Wt, registerShape as X, sanitizeMcpToolSchema as Xt, createShapeRegistry as Y, createMcpEnvironment as Yt, FileCorpus as Z, envKeyProvider as Zt, NOTE_MAX_CHARS as _, resolveSandboxClient as _t, localShell as a, selectValidWinner as at, composeWorkerEvidence as b, loopCampaignDispatch as bt, createVerifierEnvironment as c, assertTraceDerivedFindings as ct, harvestSurfaceDiffs as d, registryScopeAnalyst as dt, resolveSecretEnv as en, fanout as et, analystsFromRegistry as f, observationFromRegistry as ft, EVIDENCE_MAX_CHARS as g, defineLeaderboard as gt, worktreeFanout as h, harvestCorpus as ht, jjWorkspace as i, pipeline as it, chatTransportExecutor as j, deterministicCompletion as jt, codeModeSupervisorTools as k, runIsolatedCheck as kt, boxSurfaceReader as l, buildSteerContext as lt, superviseSurface as m, HarvestError as mt, makeFinding$1 as n, createTangleSandboxExactProcessProvider as nn, loopUntil as nt, runInWorkspace as o, verify as ot, failuresAnalyst as p, inProcessSandboxClient as pt, runPersonified as q, connectStdioMcp as qt, gitWorkspace as r, panel as rt, createWaterfallCollector as s, widen as st, computeFindingId$1 as t, secretEnvOfMcpServer as tn, flatWidenGate as tt, fsSurfaceReader as u, createScopeAnalyst as ut, VERIFY_TAIL_CHARS as v, localSandboxClient as vt, analyzeTrace as w, checkProgramDigest as wt, settledWorkerOut as x, loopDispatch as xt, closingWorkerNote as y, inlineSandboxClient as yt, strategyAuthorSystemPrompt as z, renderPairwiseMarkdown as zt };
|
|
8245
8309
|
|
|
8246
|
-
//# sourceMappingURL=runtime-
|
|
8310
|
+
//# sourceMappingURL=runtime-B3TTjtzE.js.map
|