@tangle-network/agent-runtime 0.280.0 → 0.281.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-y-60DXF4.d.ts → activation-BcTWDNOD.d.ts} +141 -18
- package/dist/{activation-C97Cq4Q_.js → activation-C1ZAyQrQ.js} +2 -2
- package/dist/{activation-C97Cq4Q_.js.map → activation-C1ZAyQrQ.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +2 -2
- package/dist/{coordination-driver-CraAV-22.js → coordination-driver-B-HOpGN4.js} +2 -2
- package/dist/{coordination-driver-CraAV-22.js.map → coordination-driver-B-HOpGN4.js.map} +1 -1
- package/dist/{delegate-W1Ws_Wp0.js → delegate-C1eIBN70.js} +2 -2
- package/dist/{delegate-W1Ws_Wp0.js.map → delegate-C1eIBN70.js.map} +1 -1
- package/dist/durable.d.ts +2 -2
- package/dist/durable.js +794 -499
- package/dist/durable.js.map +1 -1
- package/dist/{graph-DlL_X4HX.js → graph-KjJaTmbO.js} +4 -4
- package/dist/{graph-DlL_X4HX.js.map → graph-KjJaTmbO.js.map} +1 -1
- package/dist/{improvement-cycle-DXMU-2Y4.js → improvement-cycle-DCTCTmNl.js} +761 -41
- package/dist/improvement-cycle-DCTCTmNl.js.map +1 -0
- package/dist/{index-Dm8SHDGW.d.ts → index-C4ygxavE.d.ts} +98 -144
- package/dist/index.d.ts +4 -4
- package/dist/index.js +8 -8
- package/dist/intelligence.d.ts +1 -1
- package/dist/intelligence.js +3 -3
- package/dist/kernel.d.ts +1 -1
- package/dist/kernel.js +8 -8
- package/dist/{loop-runner-bin-D6khQtoC.d.ts → loop-runner-bin-B9dTzFL3.d.ts} +2 -2
- package/dist/{loop-runner-bin-CT_-jSSg.js → loop-runner-bin-Co2eBFbb.js} +3 -3
- package/dist/{loop-runner-bin-CT_-jSSg.js.map → loop-runner-bin-Co2eBFbb.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +4 -4
- package/dist/platform.d.ts +34 -36
- package/dist/platform.js +122 -29
- package/dist/platform.js.map +1 -1
- package/dist/{provision-supervisor-BEmE95Mt.js → provision-supervisor-CkN8RRjv.js} +3 -3
- package/dist/{provision-supervisor-BEmE95Mt.js.map → provision-supervisor-CkN8RRjv.js.map} +1 -1
- package/dist/{runtime-Du7j-nDD.js → runtime-B_yRPBjj.js} +16 -966
- package/dist/runtime-B_yRPBjj.js.map +1 -0
- package/dist/{server-DZvGznAn.js → server-dOyG5Tw6.js} +3 -3
- package/dist/{server-DZvGznAn.js.map → server-dOyG5Tw6.js.map} +1 -1
- package/dist/{structural-rollout-Cl4TvomY.js → shared-box-CZFdugL1.js} +959 -9
- package/dist/shared-box-CZFdugL1.js.map +1 -0
- package/dist/{supervise-CWrLq2-F.js → supervise-zt91I62x.js} +3 -3
- package/dist/{supervise-CWrLq2-F.js.map → supervise-zt91I62x.js.map} +1 -1
- package/dist/{supervisor-BXXjVCbW.js → supervisor-UXgaPisu.js} +5 -15
- package/dist/{supervisor-BXXjVCbW.js.map → supervisor-UXgaPisu.js.map} +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +12 -12
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/package.json +4 -4
- package/skills/profile-authoring/SKILL.md +210 -0
- package/skills/supervise/SKILL.md +2 -2
- package/dist/improvement-cycle-DXMU-2Y4.js.map +0 -1
- package/dist/runtime-Du7j-nDD.js.map +0 -1
- package/dist/structural-rollout-Cl4TvomY.js.map +0 -1
- package/skills/supervise/references/profile-authoring.md +0 -19
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { $i as detachedSnapshot,
|
|
1
|
+
import { $i as detachedSnapshot, $r as runSettledCommand, Br as createOpenInferenceFileExporter, Cr as linkAbort, Gr as padSpanId, Ia as armDeadlineTimer, K as openSlotGroup, Kr as padTraceId, Ur as generateSpanId, Vr as createOtelExporter, di as terminateProcessTreeAndConfirm, jt as createExecutor, qr as toOtelAttributes, wr as runAbortable } from "./supervisor-UXgaPisu.js";
|
|
2
2
|
import { i as ConfigError } from "./errors-DodWX-cb.js";
|
|
3
|
-
import {
|
|
3
|
+
import { F as optimizerMethod, L as collectAgentTurn, R as streamAgentTurn, a as sharedBoxPlacement, f as defaultStructuralRolloutPolicy, i as isTransientSandboxFailure } from "./shared-box-CZFdugL1.js";
|
|
4
4
|
import { a as concreteProfileModel, t as assertExecutableAgentProfile } from "./model-policy-BbSCSak0.js";
|
|
5
5
|
import { A as verifyCanonicalCandidateDocument, C as canonicalCandidateDigest$1, D as immutableCandidateValue, O as omitTopLevelDigest, S as canonicalCandidateBytes$1, k as sha256Bytes$1, w as canonicalCandidateDocument } from "./workspace-archive-C9lgf77y.js";
|
|
6
6
|
import { i as syncDurableDirectory, r as publishExclusiveDurableFile } from "./durable-file-DWQA4ooo.js";
|
|
@@ -8,16 +8,17 @@ import { t as runAnalystLoop } from "./analyst-loop-DZw5QWT9.js";
|
|
|
8
8
|
import { c as executePreparedAgentCandidate, o as verifiedResourceTextByDigest, s as verifyAgentCandidateBundle, t as prepareAgentCandidateExecution } from "./prepare-DVlvGP7V.js";
|
|
9
9
|
import { a as assertCandidateProfileBinding, c as createAgentCandidateProfileActivation, d as omitUndefinedObjectFields, f as parseAgentCandidateProfileActivation, i as applyExactAgentProfileDiff, p as parseExactAgentProfile, r as agentCandidateProfileAsAgentProfile, u as materializeAgentCandidateProfilePlan } from "./profile-D3eXNBQV.js";
|
|
10
10
|
import { AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, agentImprovementActivationSchema, agentImprovementProposalSchema, agentImprovementReviewSchema, agentImprovementSourceMetadata, agentImprovementSourceSchema, agentProfileDiffSchema, agentProfileEnvironmentSchema, agentProfileImprovementArmSchema, agentProfileImprovementExecutionRefSchema, agentProfileModelHintsSchema, agentProfileSchema, agentProfileTrainingSchema, agentTrainingDatasetIdentitySchema, agentTrainingParametersSchema, agentTrainingReceiptSchema, agentTrainingTaskKey, agentTrainingTaskSchema, applyAgentProfileDiff, candidateExecutionEvidenceSchema, canonicalAgentProfileDigest, canonicalCandidateBytes, changedProfileImprovementSurfaces, defineAgentProfileDiff, diffAgentProfiles, numbersApproximatelyEqual, sha256DigestSchema, snapshotAgentProfile, trainedModelIdForArtifact } from "@tangle-network/agent-interface";
|
|
11
|
-
import { CostLedger, canonicalJson, makeProposalFinding } from "@tangle-network/agent-eval";
|
|
11
|
+
import { CostAccountingIncompleteError, CostCeilingReachedError, CostLedger, canonicalJson, costForTokenPricing, costForUsage, makeProposalFinding } from "@tangle-network/agent-eval";
|
|
12
12
|
import { createHash, randomUUID } from "node:crypto";
|
|
13
13
|
import { spawn, spawnSync } from "node:child_process";
|
|
14
|
-
import { constants, existsSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync } from "node:fs";
|
|
14
|
+
import { constants, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
15
15
|
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
|
16
16
|
import { chmod, mkdir, mkdtemp, open, realpath, rm, writeFile } from "node:fs/promises";
|
|
17
17
|
import { tmpdir } from "node:os";
|
|
18
|
+
import { ATTR, branchSpan } from "@tangle-network/agent-trace-contract";
|
|
18
19
|
import { redact } from "@tangle-network/agent-eval/traces";
|
|
19
20
|
import { assertProposalFindings } from "@tangle-network/agent-eval/analyst";
|
|
20
|
-
import { SearchRecorder, aide, asha, campaignCellSearchResult, campaignScenarioIdentity, campaignSplitDigestFromIdentities, compareOptimizationMethods, createRunCostLedger, fsCampaignStorage, gitWorktreeAdapter, isProposedCandidate, openSearchLedger, renderSearchSummary, resolveRunDir, runCampaign, runSearch, searchClaimReserveUsd, searchProposalExecution, searchProposerView, searchReceiptAccounting, verifyCodeSurface, verifySearchHistoryArtifact } from "@tangle-network/agent-eval/campaign";
|
|
21
|
+
import { SearchRecorder, aide, asha, campaignCellSearchResult, campaignScenarioIdentity, campaignSplitDigestFromIdentities, compareOptimizationMethods, createRunCostLedger, fsCampaignStorage, gitWorktreeAdapter, isProposedCandidate, openSearchLedger, renderSearchSummary, resolveRunDir, runCampaign, runSearch, searchClaimReserveUsd, searchNodeId, searchProposalExecution, searchProposerView, searchReceiptAccounting, verifyCodeSurface, verifySearchHistoryArtifact } from "@tangle-network/agent-eval/campaign";
|
|
21
22
|
import { measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, runAgentProfileImprovementExperiment, runCandidateExperiment, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateExperiment, selfImprove, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison } from "@tangle-network/agent-eval/contract";
|
|
22
23
|
import { defineEvaluationClaim, summarizeEvaluationUnits } from "@tangle-network/agent-eval/experiment";
|
|
23
24
|
//#region src/improvement/worktree-tree.ts
|
|
@@ -2044,6 +2045,651 @@ async function runMethodImprovement(profile, opts) {
|
|
|
2044
2045
|
};
|
|
2045
2046
|
}
|
|
2046
2047
|
//#endregion
|
|
2048
|
+
//#region src/improvement/search-executor.ts
|
|
2049
|
+
/**
|
|
2050
|
+
* The search kernel's executor port for cells Runtime runs.
|
|
2051
|
+
*
|
|
2052
|
+
* A lane is a pool of execution slots with one cost rule: a shared box, dedicated
|
|
2053
|
+
* environments, subscription seats, or the router. The kernel places each cell on a lane and
|
|
2054
|
+
* bounds each lane's running cells by its capacity. This executor adds what one search cannot
|
|
2055
|
+
* see or enforce on its own:
|
|
2056
|
+
*
|
|
2057
|
+
* - Fleet slots. A `WorkerSlots` allocator shared by concurrent searches bounds working cells
|
|
2058
|
+
* across all of them. A cell waits for a slot, and the wait is its `queueMs`.
|
|
2059
|
+
* - Fleet dollars. On a hard lane a cell holds its enforced maximum as a `BudgetPool` ticket
|
|
2060
|
+
* before it starts, waits while open tickets could return enough, and settles the ticket
|
|
2061
|
+
* with its measured spend. Estimate lanes take no ticket: they cannot bound a cell, and an
|
|
2062
|
+
* unknown cost would close a capped pool for every search that shares it.
|
|
2063
|
+
* - Hard caps. On a hard lane every paid call of the cell, the agent's and the judges', holds
|
|
2064
|
+
* its maximum against the lane's `cellUsd` and is refused when it would pass it, so the
|
|
2065
|
+
* kernel's hard reservation is a bound and not a hope.
|
|
2066
|
+
* - Adoption. A finished attempt is written under its run id before the kernel sees it, so a
|
|
2067
|
+
* kernel that restarts adopts the attempt instead of running it again. The run id is stable
|
|
2068
|
+
* across restarts; an agent that runs remote work keys it by `ctx.search.runId`, so a lost
|
|
2069
|
+
* response is fetched again rather than executed again.
|
|
2070
|
+
* - Accounting. Every paid call is tagged with its search and attempt, and an attempt's cost is
|
|
2071
|
+
* every call tagged with its run id, including calls a process that ended made for it. A
|
|
2072
|
+
* reopened search settles those interrupted calls first: from the provider through the lane's
|
|
2073
|
+
* `recoverReceipt`, else with an unknown cost, never as free.
|
|
2074
|
+
* - Environment faults. A `SearchEnvironmentFault`, or a platform failure a repeated request
|
|
2075
|
+
* can change, settles `errored` and retryable, and the kernel runs a fresh attempt.
|
|
2076
|
+
* - Traces. Each attempt is one trace whose root span carries `agent.branch.id` = the node id.
|
|
2077
|
+
* The spans the attempt's exporter wrote and dropped go into the cell's `traceRef`.
|
|
2078
|
+
*/
|
|
2079
|
+
function laneReceipts(options) {
|
|
2080
|
+
return options.recoverReceipt ? { recoverReceipt: options.recoverReceipt } : {};
|
|
2081
|
+
}
|
|
2082
|
+
const acceptEveryProfile = () => void 0;
|
|
2083
|
+
function laneName(name, fallback) {
|
|
2084
|
+
const resolved = name ?? fallback;
|
|
2085
|
+
if (!/^[A-Za-z0-9._-]{1,64}$/u.test(resolved)) throw new ConfigError(`search lane name '${resolved}' must be 1-64 of [A-Za-z0-9._-]`);
|
|
2086
|
+
return resolved;
|
|
2087
|
+
}
|
|
2088
|
+
function positiveInteger(value, name) {
|
|
2089
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new ConfigError(`${name} must be a positive integer`);
|
|
2090
|
+
return value;
|
|
2091
|
+
}
|
|
2092
|
+
function dollars(value, name) {
|
|
2093
|
+
if (!Number.isFinite(value) || value < 0) throw new ConfigError(`${name} must be a finite non-negative number of dollars`);
|
|
2094
|
+
return value;
|
|
2095
|
+
}
|
|
2096
|
+
/**
|
|
2097
|
+
* Cells as workers packed into shared Sandbox boxes, `workersPerBox` to a box.
|
|
2098
|
+
*
|
|
2099
|
+
* Capacity is `boxes × workersPerBox`. The workers of one box share its router key, which caps
|
|
2100
|
+
* no single worker, so the lane holds an estimate (`cellUsd` is the prior until 20 cells settle).
|
|
2101
|
+
* A profile the placement refuses (a repository writer, a replaced system prompt, MCP servers,
|
|
2102
|
+
* hooks, subagents, tool grants) needs another lane. The agent reaches the box through
|
|
2103
|
+
* `lane.placement.providerFor({ nodeId: ctx.search.runId })`, which names the attempt to the
|
|
2104
|
+
* router on every call.
|
|
2105
|
+
*/
|
|
2106
|
+
function sharedBoxLane(options) {
|
|
2107
|
+
const { name, boxes, cellUsd, recoverReceipt, ...placementOptions } = options;
|
|
2108
|
+
const placement = sharedBoxPlacement(placementOptions);
|
|
2109
|
+
const workersPerBox = placementOptions.workersPerBox ?? 8;
|
|
2110
|
+
return Object.freeze({
|
|
2111
|
+
name: laneName(name, "shared-box"),
|
|
2112
|
+
kind: "shared-box",
|
|
2113
|
+
capacity: positiveInteger(boxes, "sharedBoxLane(): boxes") * workersPerBox,
|
|
2114
|
+
costCap: "estimate",
|
|
2115
|
+
cellUsd: dollars(cellUsd, "sharedBoxLane(): cellUsd"),
|
|
2116
|
+
refusal: (profile) => placement.refusal(profile),
|
|
2117
|
+
...laneReceipts({ recoverReceipt }),
|
|
2118
|
+
placement
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
/**
|
|
2122
|
+
* Cells that each run in their own environment, `capacity` at once. The lane cannot bound a
|
|
2123
|
+
* cell's dollars, so it holds an estimate: `cellUsd` until 20 cells settle, then 1.5 × the p99
|
|
2124
|
+
* of its settled cells.
|
|
2125
|
+
*/
|
|
2126
|
+
function dedicatedLane(options) {
|
|
2127
|
+
return Object.freeze({
|
|
2128
|
+
name: laneName(options.name, "dedicated"),
|
|
2129
|
+
kind: "dedicated",
|
|
2130
|
+
capacity: positiveInteger(options.capacity, "dedicatedLane(): capacity"),
|
|
2131
|
+
costCap: "estimate",
|
|
2132
|
+
cellUsd: dollars(options.cellUsd, "dedicatedLane(): cellUsd"),
|
|
2133
|
+
refusal: acceptEveryProfile,
|
|
2134
|
+
...laneReceipts(options)
|
|
2135
|
+
});
|
|
2136
|
+
}
|
|
2137
|
+
/**
|
|
2138
|
+
* Cells on subscription seats, one cell a seat. A seat charges no dollar per call, so its own
|
|
2139
|
+
* work costs $0 and reports no tokens. `cellUsd` bounds the cell's priced calls, such as its
|
|
2140
|
+
* judges; every one of them must declare a maximum, and the lane refuses a call past it.
|
|
2141
|
+
*/
|
|
2142
|
+
function subscriptionLane(options) {
|
|
2143
|
+
return Object.freeze({
|
|
2144
|
+
name: laneName(options.name, "subscription"),
|
|
2145
|
+
kind: "subscription",
|
|
2146
|
+
capacity: positiveInteger(options.seats, "subscriptionLane(): seats"),
|
|
2147
|
+
costCap: "hard",
|
|
2148
|
+
cellUsd: dollars(options.cellUsd ?? 0, "subscriptionLane(): cellUsd"),
|
|
2149
|
+
refusal: acceptEveryProfile,
|
|
2150
|
+
...laneReceipts(options)
|
|
2151
|
+
});
|
|
2152
|
+
}
|
|
2153
|
+
/**
|
|
2154
|
+
* Cells whose model calls are priced router calls, `capacity` at once. Every paid call of a
|
|
2155
|
+
* cell must declare its maximum charge, and the lane refuses a call that would take the cell
|
|
2156
|
+
* past `cellUsd`, so `cellUsd` is each cell's hard maximum.
|
|
2157
|
+
*/
|
|
2158
|
+
function routerLane(options) {
|
|
2159
|
+
return Object.freeze({
|
|
2160
|
+
name: laneName(options.name, "router"),
|
|
2161
|
+
kind: "router",
|
|
2162
|
+
capacity: positiveInteger(options.capacity, "routerLane(): capacity"),
|
|
2163
|
+
costCap: "hard",
|
|
2164
|
+
cellUsd: dollars(options.cellUsd, "routerLane(): cellUsd"),
|
|
2165
|
+
refusal: acceptEveryProfile,
|
|
2166
|
+
...laneReceipts(options)
|
|
2167
|
+
});
|
|
2168
|
+
}
|
|
2169
|
+
/** Check a lane list: at least one lane, unique names, a hard lane's cap known. */
|
|
2170
|
+
function assertSearchLanes(lanes, context) {
|
|
2171
|
+
if (!Array.isArray(lanes) || lanes.length === 0) throw new ConfigError(`${context}: lanes needs at least one lane`);
|
|
2172
|
+
const names = /* @__PURE__ */ new Set();
|
|
2173
|
+
for (const lane of lanes) {
|
|
2174
|
+
if (!lane || typeof lane.refusal !== "function" || typeof lane.kind !== "string") throw new ConfigError(`${context}: every lane comes from sharedBoxLane, dedicatedLane, subscriptionLane or routerLane`);
|
|
2175
|
+
if (names.has(lane.name)) throw new ConfigError(`${context}: lane '${lane.name}' appears twice`);
|
|
2176
|
+
names.add(lane.name);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* The environment, not the candidate, ended the attempt: a box that died, a platform that
|
|
2181
|
+
* refused the run, a quota. The attempt settles `errored` and retryable, and the kernel runs a
|
|
2182
|
+
* fresh attempt up to its `maxAttempts`. Throw anything else for the agent's own failure.
|
|
2183
|
+
*/
|
|
2184
|
+
var SearchEnvironmentFault = class extends Error {
|
|
2185
|
+
constructor(message, options) {
|
|
2186
|
+
super(message, options);
|
|
2187
|
+
this.name = "SearchEnvironmentFault";
|
|
2188
|
+
}
|
|
2189
|
+
};
|
|
2190
|
+
/** Whether an attempt's error is the environment's: a `SearchEnvironmentFault`, or a platform,
|
|
2191
|
+
* gateway or network failure that a repeated request can change. */
|
|
2192
|
+
function isSearchEnvironmentFault(error) {
|
|
2193
|
+
return error instanceof SearchEnvironmentFault || isTransientSandboxFailure(error);
|
|
2194
|
+
}
|
|
2195
|
+
const USD_TOLERANCE = 1e-9;
|
|
2196
|
+
/** The kernel's executor port over Runtime lanes. */
|
|
2197
|
+
function searchExecutor(options) {
|
|
2198
|
+
assertSearchLanes(options.lanes, "searchExecutor()");
|
|
2199
|
+
const lanes = new Map(options.lanes.map((lane) => [lane.name, lane]));
|
|
2200
|
+
const placed = new Map(options.lanes.map((lane) => [lane.name, 0]));
|
|
2201
|
+
const slots = options.workerSlots ? openSlotGroup(options.workerSlots) : null;
|
|
2202
|
+
const attempts = attemptStore(options.storage, join(options.dir, "attempts"));
|
|
2203
|
+
const tracing = searchTracing(options);
|
|
2204
|
+
const laneOf = (work) => {
|
|
2205
|
+
const lane = lanes.get(work.lane);
|
|
2206
|
+
if (!lane) throw new Error(`search cell ${work.cellId} was placed on unknown lane '${work.lane}'`);
|
|
2207
|
+
return lane;
|
|
2208
|
+
};
|
|
2209
|
+
return {
|
|
2210
|
+
lanes: () => options.lanes,
|
|
2211
|
+
place(cell) {
|
|
2212
|
+
const profile = options.profileOf(cell.nodeId);
|
|
2213
|
+
let chosen;
|
|
2214
|
+
for (const lane of options.lanes) {
|
|
2215
|
+
if (profile && lane.refusal(profile) !== void 0) continue;
|
|
2216
|
+
if (chosen === void 0 || placed.get(lane.name) / lane.capacity < placed.get(chosen.name) / chosen.capacity) chosen = lane;
|
|
2217
|
+
}
|
|
2218
|
+
if (!chosen) throw new Error(`no search lane can run node ${cell.nodeId}; admission should have refused it`);
|
|
2219
|
+
placed.set(chosen.name, placed.get(chosen.name) + 1);
|
|
2220
|
+
return chosen.name;
|
|
2221
|
+
},
|
|
2222
|
+
async adopt(work) {
|
|
2223
|
+
return attempts.read(work.runId);
|
|
2224
|
+
},
|
|
2225
|
+
async run(work) {
|
|
2226
|
+
const lane = laneOf(work);
|
|
2227
|
+
const queuedAt = Date.now();
|
|
2228
|
+
let hold;
|
|
2229
|
+
try {
|
|
2230
|
+
hold = await holdFleet(lane, work, options.budgetPool, slots);
|
|
2231
|
+
} catch (error) {
|
|
2232
|
+
if (work.signal.aborted) return interrupted(work, lane, options.identity, error);
|
|
2233
|
+
throw error;
|
|
2234
|
+
}
|
|
2235
|
+
const queueMs = Date.now() - queuedAt;
|
|
2236
|
+
const trace = tracing.open(work, lane);
|
|
2237
|
+
let result;
|
|
2238
|
+
try {
|
|
2239
|
+
const context = Object.freeze({
|
|
2240
|
+
searchId: work.searchId,
|
|
2241
|
+
cellId: work.cellId,
|
|
2242
|
+
nodeId: work.nodeId,
|
|
2243
|
+
taskId: work.taskId,
|
|
2244
|
+
split: work.split,
|
|
2245
|
+
stage: work.stage,
|
|
2246
|
+
attempt: work.attempt,
|
|
2247
|
+
runId: work.runId,
|
|
2248
|
+
lane,
|
|
2249
|
+
trace: trace.ids
|
|
2250
|
+
});
|
|
2251
|
+
const tags = searchCallTags(work.searchId, {
|
|
2252
|
+
run: work.runId,
|
|
2253
|
+
lane: lane.name
|
|
2254
|
+
});
|
|
2255
|
+
const measured = await options.runAttempt({
|
|
2256
|
+
work,
|
|
2257
|
+
lane,
|
|
2258
|
+
context,
|
|
2259
|
+
costLedger: attemptLedger(options.costLedger, lane, tags),
|
|
2260
|
+
span: trace.span
|
|
2261
|
+
});
|
|
2262
|
+
result = {
|
|
2263
|
+
...measured,
|
|
2264
|
+
accounting: attemptAccounting(options.costLedger, lane, tags),
|
|
2265
|
+
queueMs,
|
|
2266
|
+
placement: {
|
|
2267
|
+
lane: lane.name,
|
|
2268
|
+
boxId: measured.placement?.boxId ?? null
|
|
2269
|
+
},
|
|
2270
|
+
traceRef: await trace.finish(measured)
|
|
2271
|
+
};
|
|
2272
|
+
} finally {
|
|
2273
|
+
hold.release(result);
|
|
2274
|
+
}
|
|
2275
|
+
if (!work.signal.aborted) attempts.write(work.runId, result);
|
|
2276
|
+
return result;
|
|
2277
|
+
}
|
|
2278
|
+
};
|
|
2279
|
+
}
|
|
2280
|
+
function interrupted(work, lane, identity, error) {
|
|
2281
|
+
return {
|
|
2282
|
+
outcome: {
|
|
2283
|
+
status: "errored",
|
|
2284
|
+
metrics: {},
|
|
2285
|
+
error: {
|
|
2286
|
+
code: "interrupted",
|
|
2287
|
+
message: `search cell ${work.runId} was interrupted before it ran: ${error instanceof Error ? error.message : String(error)}`,
|
|
2288
|
+
retryable: true
|
|
2289
|
+
}
|
|
2290
|
+
},
|
|
2291
|
+
accounting: {
|
|
2292
|
+
tokens: {
|
|
2293
|
+
status: "known",
|
|
2294
|
+
inputTokens: 0,
|
|
2295
|
+
outputTokens: 0,
|
|
2296
|
+
cachedTokens: 0
|
|
2297
|
+
},
|
|
2298
|
+
cost: {
|
|
2299
|
+
status: "known",
|
|
2300
|
+
usd: 0,
|
|
2301
|
+
source: "free"
|
|
2302
|
+
}
|
|
2303
|
+
},
|
|
2304
|
+
identity,
|
|
2305
|
+
placement: {
|
|
2306
|
+
lane: lane.name,
|
|
2307
|
+
boxId: null
|
|
2308
|
+
},
|
|
2309
|
+
traceRef: { unknown: "the attempt never started" }
|
|
2310
|
+
};
|
|
2311
|
+
}
|
|
2312
|
+
/** Take the attempt's fleet dollars, then its fleet slot. A cell waiting for budget holds no
|
|
2313
|
+
* slot, so it cannot hold a slot the cells it waits on need. */
|
|
2314
|
+
async function holdFleet(lane, work, pool, slots) {
|
|
2315
|
+
let ticket;
|
|
2316
|
+
if (pool && lane.costCap === "hard" && lane.cellUsd > 0) {
|
|
2317
|
+
const reserved = pool.reserve({
|
|
2318
|
+
maxUsd: lane.cellUsd,
|
|
2319
|
+
maxTokens: 0,
|
|
2320
|
+
maxIterations: 0
|
|
2321
|
+
}, {
|
|
2322
|
+
label: work.runId,
|
|
2323
|
+
stage: "admitted"
|
|
2324
|
+
}, { wait: true });
|
|
2325
|
+
if (!reserved.ok) throw new Error(`the fleet budget pool refused search cell ${work.runId} on lane ${lane.name}: ${reserved.reason}`);
|
|
2326
|
+
ticket = reserved.ticket;
|
|
2327
|
+
if (reserved.granted) try {
|
|
2328
|
+
await abortable(reserved.granted, work.signal);
|
|
2329
|
+
} catch (error) {
|
|
2330
|
+
pool.reconcile(ticket, noSpend());
|
|
2331
|
+
throw error;
|
|
2332
|
+
}
|
|
2333
|
+
pool.attribute(ticket, {
|
|
2334
|
+
label: work.runId,
|
|
2335
|
+
stage: "executing"
|
|
2336
|
+
});
|
|
2337
|
+
}
|
|
2338
|
+
const permit = slots?.acquire(0);
|
|
2339
|
+
if (permit) try {
|
|
2340
|
+
await abortable(permit.ready, work.signal);
|
|
2341
|
+
} catch (error) {
|
|
2342
|
+
permit.release();
|
|
2343
|
+
if (ticket) pool.reconcile(ticket, noSpend());
|
|
2344
|
+
throw error;
|
|
2345
|
+
}
|
|
2346
|
+
return { release(result) {
|
|
2347
|
+
permit?.release();
|
|
2348
|
+
if (!ticket) return;
|
|
2349
|
+
try {
|
|
2350
|
+
pool.reconcile(ticket, result ? spendOf(result) : unknownSpend());
|
|
2351
|
+
} catch {}
|
|
2352
|
+
} };
|
|
2353
|
+
}
|
|
2354
|
+
function abortable(promise, signal) {
|
|
2355
|
+
if (signal.aborted) return Promise.reject(signal.reason ?? /* @__PURE__ */ new Error("search cell aborted"));
|
|
2356
|
+
return new Promise((resolve, reject) => {
|
|
2357
|
+
const onAbort = () => reject(signal.reason ?? /* @__PURE__ */ new Error("search cell aborted"));
|
|
2358
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2359
|
+
promise.then((value) => {
|
|
2360
|
+
signal.removeEventListener("abort", onAbort);
|
|
2361
|
+
resolve(value);
|
|
2362
|
+
}, (error) => {
|
|
2363
|
+
signal.removeEventListener("abort", onAbort);
|
|
2364
|
+
reject(error);
|
|
2365
|
+
});
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
function noSpend() {
|
|
2369
|
+
return {
|
|
2370
|
+
iterations: 0,
|
|
2371
|
+
tokens: {
|
|
2372
|
+
input: 0,
|
|
2373
|
+
output: 0
|
|
2374
|
+
},
|
|
2375
|
+
usd: 0,
|
|
2376
|
+
ms: 0
|
|
2377
|
+
};
|
|
2378
|
+
}
|
|
2379
|
+
function unknownSpend() {
|
|
2380
|
+
return {
|
|
2381
|
+
...noSpend(),
|
|
2382
|
+
usdKnown: false,
|
|
2383
|
+
tokensKnown: false
|
|
2384
|
+
};
|
|
2385
|
+
}
|
|
2386
|
+
/** The attempt's spend as the budget pool reads it. Unknown dollars carry their proven floor. */
|
|
2387
|
+
function spendOf(result) {
|
|
2388
|
+
const { cost, tokens } = result.accounting;
|
|
2389
|
+
return {
|
|
2390
|
+
iterations: 0,
|
|
2391
|
+
tokens: tokens.status === "known" ? {
|
|
2392
|
+
input: tokens.inputTokens,
|
|
2393
|
+
output: tokens.outputTokens
|
|
2394
|
+
} : {
|
|
2395
|
+
input: 0,
|
|
2396
|
+
output: 0
|
|
2397
|
+
},
|
|
2398
|
+
...tokens.status === "known" ? {} : { tokensKnown: false },
|
|
2399
|
+
usd: cost.status === "known" ? cost.usd : cost.knownLowerBoundUsd,
|
|
2400
|
+
...cost.status === "known" ? {} : { usdKnown: false },
|
|
2401
|
+
ms: result.wallMs ?? 0
|
|
2402
|
+
};
|
|
2403
|
+
}
|
|
2404
|
+
/** The dollar maximum a paid call declares, or null when it declares no priced one. */
|
|
2405
|
+
function chargeMaximumUsd(maximum) {
|
|
2406
|
+
if (!maximum) return null;
|
|
2407
|
+
if ("externallyEnforcedMaximumUsd" in maximum) return maximum.externallyEnforcedMaximumUsd;
|
|
2408
|
+
if ("customTokenPricing" in maximum) return costForTokenPricing(maximum.customTokenPricing, maximum);
|
|
2409
|
+
const priced = costForUsage(maximum.model, maximum);
|
|
2410
|
+
return priced.costUnknown ? null : priced.costUsd;
|
|
2411
|
+
}
|
|
2412
|
+
const TAG_SEARCH = "searchId";
|
|
2413
|
+
const TAG_RUN = "searchRunId";
|
|
2414
|
+
const TAG_LANE = "searchLane";
|
|
2415
|
+
const TAG_OPERATION = "searchOperation";
|
|
2416
|
+
/** The cost-ledger tags of one search's calls: a cell attempt's, or a proposal operation's. */
|
|
2417
|
+
function searchCallTags(searchId, of) {
|
|
2418
|
+
return "run" in of ? {
|
|
2419
|
+
[TAG_SEARCH]: searchId,
|
|
2420
|
+
[TAG_RUN]: of.run,
|
|
2421
|
+
[TAG_LANE]: of.lane
|
|
2422
|
+
} : {
|
|
2423
|
+
[TAG_SEARCH]: searchId,
|
|
2424
|
+
[TAG_OPERATION]: of.operation
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
/** The search ledger with `tags` added to every paid call. */
|
|
2428
|
+
function taggedLedger(ledger, tags) {
|
|
2429
|
+
return ledgerView(ledger, (input) => ledger.runPaidCall({
|
|
2430
|
+
...input,
|
|
2431
|
+
tags: {
|
|
2432
|
+
...input.tags,
|
|
2433
|
+
...tags
|
|
2434
|
+
}
|
|
2435
|
+
}));
|
|
2436
|
+
}
|
|
2437
|
+
function ledgerView(ledger, runPaidCall) {
|
|
2438
|
+
return new Proxy(ledger, { get(target, property) {
|
|
2439
|
+
if (property === "runPaidCall") return runPaidCall;
|
|
2440
|
+
const value = Reflect.get(target, property, target);
|
|
2441
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
2442
|
+
} });
|
|
2443
|
+
}
|
|
2444
|
+
/**
|
|
2445
|
+
* The search ledger seen by one attempt: every paid call carries the attempt's tags. On a hard
|
|
2446
|
+
* lane every call also holds its declared maximum against the lane's `cellUsd` until its
|
|
2447
|
+
* receipt replaces the hold with the charge, and a call that would pass the cap, or declares no
|
|
2448
|
+
* priced maximum, is refused. A charge the provider left unknown keeps its maximum held.
|
|
2449
|
+
*/
|
|
2450
|
+
function attemptLedger(ledger, lane, tags) {
|
|
2451
|
+
const tagged = taggedLedger(ledger, tags);
|
|
2452
|
+
if (lane.costCap !== "hard") return tagged;
|
|
2453
|
+
let heldUsd = 0;
|
|
2454
|
+
const runPaidCall = async (input) => {
|
|
2455
|
+
const maximumUsd = chargeMaximumUsd(input.maximumCharge);
|
|
2456
|
+
if (maximumUsd === null) return {
|
|
2457
|
+
succeeded: false,
|
|
2458
|
+
error: new CostAccountingIncompleteError(`lane ${lane.name} caps each cell at $${lane.cellUsd}, so paid call '${input.actor}' during '${input.phase}' needs a priced maximumCharge`)
|
|
2459
|
+
};
|
|
2460
|
+
if (heldUsd + maximumUsd > lane.cellUsd + USD_TOLERANCE) return {
|
|
2461
|
+
succeeded: false,
|
|
2462
|
+
error: new CostCeilingReachedError(lane.cellUsd, heldUsd, maximumUsd, input.phase, input.actor)
|
|
2463
|
+
};
|
|
2464
|
+
heldUsd += maximumUsd;
|
|
2465
|
+
const result = await tagged.runPaidCall(input);
|
|
2466
|
+
const { receipt } = result;
|
|
2467
|
+
if (receipt && !receipt.costUnknown) heldUsd += receipt.costUsd - maximumUsd;
|
|
2468
|
+
return result;
|
|
2469
|
+
};
|
|
2470
|
+
return ledgerView(tagged, runPaidCall);
|
|
2471
|
+
}
|
|
2472
|
+
/**
|
|
2473
|
+
* An attempt's cost is every call tagged with its run id, a call a process that ended made for
|
|
2474
|
+
* it included; a call still pending makes the cost unknown with the settled calls as its floor.
|
|
2475
|
+
* A subscription seat reports no tokens, and its own work charges no dollar.
|
|
2476
|
+
*/
|
|
2477
|
+
function attemptAccounting(ledger, lane, tags) {
|
|
2478
|
+
const receipts = ledger.list({ tags });
|
|
2479
|
+
const accounting = searchReceiptAccounting(receipts);
|
|
2480
|
+
const pending = ledger.listPending?.({ tags }) ?? [];
|
|
2481
|
+
const floorUsd = accounting.cost.status === "known" ? accounting.cost.usd : accounting.cost.knownLowerBoundUsd;
|
|
2482
|
+
const cost = pending.length > 0 ? {
|
|
2483
|
+
status: "unknown",
|
|
2484
|
+
knownLowerBoundUsd: floorUsd,
|
|
2485
|
+
reason: `${pending.length} paid call(s) of the attempt had no receipt when it ended`
|
|
2486
|
+
} : accounting.cost.status === "known" && accounting.cost.usd > 0 && receipts.some((receipt) => receipt.actualCostUsd === void 0) ? {
|
|
2487
|
+
...accounting.cost,
|
|
2488
|
+
source: "pricing-table"
|
|
2489
|
+
} : accounting.cost;
|
|
2490
|
+
return {
|
|
2491
|
+
tokens: lane.kind === "subscription" ? {
|
|
2492
|
+
status: "unknown",
|
|
2493
|
+
reason: "a subscription seat reports no token usage"
|
|
2494
|
+
} : accounting.tokens,
|
|
2495
|
+
cost
|
|
2496
|
+
};
|
|
2497
|
+
}
|
|
2498
|
+
/**
|
|
2499
|
+
* Settle every paid call of `searchId` that a process which ended left pending, before the
|
|
2500
|
+
* reopened search makes a new one: a capped ledger refuses new paid work while one is
|
|
2501
|
+
* unresolved. A cell's call is asked of its lane's provider through `recoverReceipt`; any other
|
|
2502
|
+
* call, or one the provider cannot answer, settles failed with an unknown cost.
|
|
2503
|
+
*/
|
|
2504
|
+
async function reconcileInterruptedSearchCalls(ledger, searchId, lanes) {
|
|
2505
|
+
const interrupted = (ledger.listPending?.({ tags: { [TAG_SEARCH]: searchId } }) ?? []).filter((call) => call.state === "interrupted");
|
|
2506
|
+
let recovered = 0;
|
|
2507
|
+
for (const call of interrupted) {
|
|
2508
|
+
const receipt = await lanes.find((candidate) => candidate.name === call.tags?.[TAG_LANE])?.recoverReceipt?.(call) ?? null;
|
|
2509
|
+
if (receipt) {
|
|
2510
|
+
ledger.reconcile(call.callId, receipt);
|
|
2511
|
+
recovered += 1;
|
|
2512
|
+
} else ledger.reconcile(call.callId, {
|
|
2513
|
+
model: call.model,
|
|
2514
|
+
inputTokens: 0,
|
|
2515
|
+
outputTokens: 0,
|
|
2516
|
+
costUnknown: true,
|
|
2517
|
+
usageUnknown: true
|
|
2518
|
+
}, { error: "the process that made this call ended before its receipt" });
|
|
2519
|
+
}
|
|
2520
|
+
return {
|
|
2521
|
+
recovered,
|
|
2522
|
+
unknown: interrupted.length - recovered
|
|
2523
|
+
};
|
|
2524
|
+
}
|
|
2525
|
+
/** Finished attempts by run id: the record a restarted kernel adopts. */
|
|
2526
|
+
function attemptStore(storage, dir) {
|
|
2527
|
+
const pathOf = (runId) => join(dir, `${runId.replace(/[^A-Za-z0-9_-]/gu, ".")}.json`);
|
|
2528
|
+
storage.ensureDir(dir);
|
|
2529
|
+
return {
|
|
2530
|
+
read(runId) {
|
|
2531
|
+
const text = storage.read(pathOf(runId));
|
|
2532
|
+
if (text === void 0) return null;
|
|
2533
|
+
const record = JSON.parse(text);
|
|
2534
|
+
if (record.runId !== runId || !record.result) throw new Error(`attempt record ${pathOf(runId)} does not hold run ${runId}`);
|
|
2535
|
+
return record.result;
|
|
2536
|
+
},
|
|
2537
|
+
write(runId, result) {
|
|
2538
|
+
const text = `${JSON.stringify({
|
|
2539
|
+
runId,
|
|
2540
|
+
result
|
|
2541
|
+
})}\n`;
|
|
2542
|
+
const path = pathOf(runId);
|
|
2543
|
+
if (storage.kind !== "filesystem") {
|
|
2544
|
+
storage.write(path, text);
|
|
2545
|
+
return;
|
|
2546
|
+
}
|
|
2547
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
2548
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
2549
|
+
writeFileSync(temporary, text);
|
|
2550
|
+
renameSync(temporary, path);
|
|
2551
|
+
}
|
|
2552
|
+
};
|
|
2553
|
+
}
|
|
2554
|
+
const STATUS_OK = 1;
|
|
2555
|
+
const STATUS_ERROR = 2;
|
|
2556
|
+
const MAX_ATTRIBUTE_CHARS = 256;
|
|
2557
|
+
function searchTracing(options) {
|
|
2558
|
+
const mode = options.trace;
|
|
2559
|
+
const fileExporter = mode === void 0 && options.storage.kind === "filesystem" ? createOpenInferenceFileExporter(join(options.dir, "spans.otlp.jsonl")) : void 0;
|
|
2560
|
+
const off = (reason) => ({
|
|
2561
|
+
ids: null,
|
|
2562
|
+
span: () => ({
|
|
2563
|
+
setAttribute() {},
|
|
2564
|
+
end() {}
|
|
2565
|
+
}),
|
|
2566
|
+
finish: async () => ({ unknown: reason })
|
|
2567
|
+
});
|
|
2568
|
+
return { open(work, lane) {
|
|
2569
|
+
if (mode === "off") return off("tracing is off for this search");
|
|
2570
|
+
const exporter = mode === void 0 ? fileExporter : createOtelExporter(mode.otlp);
|
|
2571
|
+
if (!exporter) return off(mode === void 0 ? "the search storage is not the filesystem and no OTLP exporter was configured" : "the OTLP exporter has no endpoint");
|
|
2572
|
+
return attemptTrace(work, lane, exporter, mode !== void 0);
|
|
2573
|
+
} };
|
|
2574
|
+
}
|
|
2575
|
+
/**
|
|
2576
|
+
* One attempt's trace. Its ids derive from the run id, so an attempt run again after a restart
|
|
2577
|
+
* writes into the same trace. A shared file exporter writes each span synchronously, so the
|
|
2578
|
+
* attempt's written and dropped counts are the change in the exporter's own counts across each
|
|
2579
|
+
* of its spans; an attempt's own OTLP exporter is shut down at the end and read whole.
|
|
2580
|
+
*/
|
|
2581
|
+
function attemptTrace(work, lane, exporter, ownsExporter) {
|
|
2582
|
+
const traceId = padTraceId(`search-cell:${work.runId}`);
|
|
2583
|
+
const rootSpanId = padSpanId(`search-cell:${work.runId}`);
|
|
2584
|
+
const startMs = Date.now();
|
|
2585
|
+
let written = 0;
|
|
2586
|
+
let dropped = 0;
|
|
2587
|
+
const exportCounted = (span) => {
|
|
2588
|
+
if (ownsExporter) {
|
|
2589
|
+
exporter.exportSpan(span);
|
|
2590
|
+
return;
|
|
2591
|
+
}
|
|
2592
|
+
const before = exporter.stats();
|
|
2593
|
+
exporter.exportSpan(span);
|
|
2594
|
+
const after = exporter.stats();
|
|
2595
|
+
written += after.written - before.written;
|
|
2596
|
+
dropped += after.dropped - before.dropped;
|
|
2597
|
+
};
|
|
2598
|
+
return {
|
|
2599
|
+
ids: {
|
|
2600
|
+
traceId,
|
|
2601
|
+
spanId: rootSpanId
|
|
2602
|
+
},
|
|
2603
|
+
span(name, attributes = {}) {
|
|
2604
|
+
const opened = Date.now();
|
|
2605
|
+
const record = { ...attributes };
|
|
2606
|
+
let ended = false;
|
|
2607
|
+
return {
|
|
2608
|
+
setAttribute(key, value) {
|
|
2609
|
+
record[key] = value;
|
|
2610
|
+
},
|
|
2611
|
+
end(endAttributes = {}) {
|
|
2612
|
+
if (ended) return;
|
|
2613
|
+
ended = true;
|
|
2614
|
+
exportCounted(otelSpan({
|
|
2615
|
+
traceId,
|
|
2616
|
+
spanId: generateSpanId(),
|
|
2617
|
+
parentSpanId: rootSpanId,
|
|
2618
|
+
name,
|
|
2619
|
+
startMs: opened,
|
|
2620
|
+
endMs: Date.now(),
|
|
2621
|
+
attributes: {
|
|
2622
|
+
...record,
|
|
2623
|
+
...endAttributes
|
|
2624
|
+
},
|
|
2625
|
+
status: STATUS_OK
|
|
2626
|
+
}));
|
|
2627
|
+
}
|
|
2628
|
+
};
|
|
2629
|
+
},
|
|
2630
|
+
async finish(result) {
|
|
2631
|
+
const root = branchSpan({
|
|
2632
|
+
traceId,
|
|
2633
|
+
spanId: rootSpanId,
|
|
2634
|
+
name: "search.cell",
|
|
2635
|
+
startTime: new Date(startMs).toISOString(),
|
|
2636
|
+
branchId: work.nodeId,
|
|
2637
|
+
arm: lane.name,
|
|
2638
|
+
attributes: {
|
|
2639
|
+
"tangle.search.id": work.searchId,
|
|
2640
|
+
"tangle.search.cell_id": work.cellId,
|
|
2641
|
+
"tangle.search.run_id": work.runId,
|
|
2642
|
+
"tangle.search.task_id": work.taskId,
|
|
2643
|
+
"tangle.search.split": work.split,
|
|
2644
|
+
"tangle.search.stage": work.stage,
|
|
2645
|
+
"tangle.search.attempt": work.attempt,
|
|
2646
|
+
"tangle.search.lane": lane.name,
|
|
2647
|
+
"tangle.search.lane_kind": lane.kind,
|
|
2648
|
+
"tangle.search.outcome": result.outcome.status,
|
|
2649
|
+
..."score" in result.outcome ? { [ATTR.score]: result.outcome.score } : {}
|
|
2650
|
+
}
|
|
2651
|
+
});
|
|
2652
|
+
exportCounted(otelSpan({
|
|
2653
|
+
traceId,
|
|
2654
|
+
spanId: rootSpanId,
|
|
2655
|
+
name: root.name,
|
|
2656
|
+
startMs,
|
|
2657
|
+
endMs: Date.now(),
|
|
2658
|
+
attributes: root.attributes,
|
|
2659
|
+
status: result.outcome.status === "errored" ? STATUS_ERROR : STATUS_OK
|
|
2660
|
+
}));
|
|
2661
|
+
if (ownsExporter) {
|
|
2662
|
+
await exporter.shutdown();
|
|
2663
|
+
const stats = exporter.stats();
|
|
2664
|
+
written = stats.written;
|
|
2665
|
+
dropped = stats.dropped;
|
|
2666
|
+
}
|
|
2667
|
+
return {
|
|
2668
|
+
traceId,
|
|
2669
|
+
execRunId: null,
|
|
2670
|
+
spansWritten: written,
|
|
2671
|
+
spansDropped: dropped
|
|
2672
|
+
};
|
|
2673
|
+
}
|
|
2674
|
+
};
|
|
2675
|
+
}
|
|
2676
|
+
function otelSpan(input) {
|
|
2677
|
+
const attributes = {};
|
|
2678
|
+
for (const [key, value] of Object.entries(input.attributes)) if (typeof value === "number" || typeof value === "boolean") attributes[key] = value;
|
|
2679
|
+
else if (typeof value === "string") attributes[key] = value.slice(0, MAX_ATTRIBUTE_CHARS);
|
|
2680
|
+
return {
|
|
2681
|
+
traceId: input.traceId,
|
|
2682
|
+
spanId: input.spanId,
|
|
2683
|
+
...input.parentSpanId ? { parentSpanId: input.parentSpanId } : {},
|
|
2684
|
+
name: input.name,
|
|
2685
|
+
kind: 1,
|
|
2686
|
+
startTimeUnixNano: (BigInt(Math.floor(input.startMs)) * 1000000n).toString(),
|
|
2687
|
+
endTimeUnixNano: (BigInt(Math.floor(Math.max(input.startMs, input.endMs))) * 1000000n).toString(),
|
|
2688
|
+
attributes: toOtelAttributes(attributes),
|
|
2689
|
+
status: { code: input.status }
|
|
2690
|
+
};
|
|
2691
|
+
}
|
|
2692
|
+
//#endregion
|
|
2047
2693
|
//#region src/improvement/search-method.ts
|
|
2048
2694
|
/** Build Runtime's native search for `improve(profile, { method })`. */
|
|
2049
2695
|
function searchMethod(options) {
|
|
@@ -2063,6 +2709,7 @@ function searchMethod(options) {
|
|
|
2063
2709
|
if (options.deadline !== void 0 && !Number.isFinite(Date.parse(options.deadline))) throw new ConfigError("searchMethod(): deadline must be an ISO time");
|
|
2064
2710
|
const name = options.name ?? "search";
|
|
2065
2711
|
if (!name.trim() || name.trim() !== name) throw new ConfigError("searchMethod(): name must be a trimmed non-empty string");
|
|
2712
|
+
assertSearchLanes(options.lanes, "searchMethod()");
|
|
2066
2713
|
return Object.freeze({
|
|
2067
2714
|
kind: "search",
|
|
2068
2715
|
name,
|
|
@@ -2071,8 +2718,7 @@ function searchMethod(options) {
|
|
|
2071
2718
|
proposer,
|
|
2072
2719
|
maxExpansions: count("maxExpansions", options.maxExpansions, NaN, 0),
|
|
2073
2720
|
childrenPerProposal: count("childrenPerProposal", options.childrenPerProposal, 1, 1),
|
|
2074
|
-
|
|
2075
|
-
cellUsd: dollars("cellUsd", options.cellUsd) ?? 0,
|
|
2721
|
+
lanes: Object.freeze([...options.lanes]),
|
|
2076
2722
|
claimReserveUsd: dollars("claimReserveUsd", options.claimReserveUsd),
|
|
2077
2723
|
deadline: options.deadline === void 0 ? null : new Date(options.deadline).toISOString(),
|
|
2078
2724
|
maxAttempts: count("maxAttempts", options.maxAttempts, 3, 1)
|
|
@@ -2087,14 +2733,15 @@ const SEARCH_METHOD_SOURCE = {
|
|
|
2087
2733
|
name: "agent-runtime.search-method.2026-09",
|
|
2088
2734
|
node: "an exact AgentProfile materialized from the baseline and one surface, content-addressed by canonicalAgentProfileDigest; a surface that does not materialize is a node addressed by its surface and refused",
|
|
2089
2735
|
edge: "the Interface diffs from the parent profile to the child, stored when they reproduce the child digest",
|
|
2090
|
-
admission: "a candidate is refused when it does not materialize, declares training on a test task, fails the caller validator,
|
|
2091
|
-
cell: "a one-cell runCampaign of the materialized profile in a directory addressed by its digest; a cached cell is read back, not run again",
|
|
2736
|
+
admission: "a candidate is refused when it does not materialize, declares training on a test task, fails the caller validator, carries a credential or private value, or no lane can run its profile",
|
|
2737
|
+
cell: "a one-cell runCampaign of the materialized profile in a directory addressed by its digest, on a lane that accepts the profile; a cached cell is read back, not run again; a finished attempt is recorded by run id before it settles and adopted after a restart",
|
|
2738
|
+
lanes: "a hard lane refuses a paid call that declares no priced maximum or would take the cell past its cellUsd; an environment fault settles errored and retryable; a subscription seat reports no tokens and charges no dollars",
|
|
2092
2739
|
proposer: "a SurfaceProposer reads the parents, the train view and the train summary; paid calls it makes are its operation accounting",
|
|
2093
2740
|
ship: "the claim shipped, re-derives from the ledger, cost accounting is complete, and the shipped finalist test lower bound exceeds minimumLift"
|
|
2094
2741
|
})
|
|
2095
2742
|
};
|
|
2096
2743
|
async function runSearchImprovement(profile, opts) {
|
|
2097
|
-
const { method, claim: inputClaim, agent, trainScenarios, selectionScenarios, testScenarios, judges: _judges, surface: _surface, executionRef: _executionRef, validateCandidate: _validateCandidate, findings: _findings, skills: _skills, profileComponents: _profileComponents, minimumLift: _minimumLift, subject: _subject, costCeiling, costLedger: inputCostLedger, seed = 42, signal, ...campaignOptions } = opts;
|
|
2744
|
+
const { method, claim: inputClaim, agent, trainScenarios, selectionScenarios, testScenarios, judges: _judges, surface: _surface, executionRef: _executionRef, validateCandidate: _validateCandidate, findings: _findings, skills: _skills, profileComponents: _profileComponents, minimumLift: _minimumLift, subject: _subject, costCeiling, costLedger: inputCostLedger, seed = 42, signal, workerSlots, budgetPool, trace, ...campaignOptions } = opts;
|
|
2098
2745
|
if (!isImproveSearchMethod(method)) throw new ConfigError("improve(): a search needs method: searchMethod(...)");
|
|
2099
2746
|
const claim = defineEvaluationClaim(inputClaim);
|
|
2100
2747
|
if (claim.minimumEffect === void 0) throw new ConfigError("improve(): a search claim needs minimumEffect for its power check");
|
|
@@ -2103,15 +2750,16 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2103
2750
|
["selectionScenarios", selectionScenarios],
|
|
2104
2751
|
["testScenarios", testScenarios]
|
|
2105
2752
|
]) if (!Array.isArray(split) || split.length === 0) throw new ConfigError(`improve(): a search needs at least one of ${name}`);
|
|
2106
|
-
const { allocation, policy } = method;
|
|
2753
|
+
const { allocation, policy, lanes } = method;
|
|
2107
2754
|
const reps = allocation.reps;
|
|
2755
|
+
const concurrency = lanes.reduce((total, lane) => total + lane.capacity, 0);
|
|
2108
2756
|
const prepared = prepareProfileImprovement(profile, {
|
|
2109
2757
|
...opts,
|
|
2110
2758
|
seed,
|
|
2111
2759
|
reps,
|
|
2112
2760
|
optimizationRunOptions: {
|
|
2113
2761
|
reps,
|
|
2114
|
-
maxConcurrency:
|
|
2762
|
+
maxConcurrency: concurrency
|
|
2115
2763
|
}
|
|
2116
2764
|
});
|
|
2117
2765
|
const { evaluationRef } = prepared.identity;
|
|
@@ -2157,7 +2805,7 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2157
2805
|
const reservedClaimUsd = Math.max(method.claimReserveUsd ?? 0, searchClaimReserveUsd({
|
|
2158
2806
|
testTasks: splits.test.length,
|
|
2159
2807
|
reps,
|
|
2160
|
-
cellUsd:
|
|
2808
|
+
cellUsd: Math.max(...lanes.map((lane) => lane.cellUsd))
|
|
2161
2809
|
}));
|
|
2162
2810
|
const recorder = await SearchRecorder.open({
|
|
2163
2811
|
ledger: storage.kind === "filesystem" ? openSearchLedger({
|
|
@@ -2196,7 +2844,7 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2196
2844
|
maxCells: null,
|
|
2197
2845
|
maxNodes: 1 + method.maxExpansions * method.childrenPerProposal,
|
|
2198
2846
|
deadline: method.deadline,
|
|
2199
|
-
maxConcurrency:
|
|
2847
|
+
maxConcurrency: concurrency,
|
|
2200
2848
|
reservedClaimUsd: costCeiling === void 0 ? 0 : reservedClaimUsd
|
|
2201
2849
|
},
|
|
2202
2850
|
containment: null,
|
|
@@ -2219,7 +2867,32 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2219
2867
|
};
|
|
2220
2868
|
}
|
|
2221
2869
|
};
|
|
2222
|
-
const
|
|
2870
|
+
const profiles = /* @__PURE__ */ new Map();
|
|
2871
|
+
const profiled = profileCodec(prepared);
|
|
2872
|
+
const codec = {
|
|
2873
|
+
node(recorder, node) {
|
|
2874
|
+
const registered = profiled.node(recorder, node);
|
|
2875
|
+
profiles.set(searchNodeId(searchId, registered.artifactDigest), node.profile);
|
|
2876
|
+
return registered;
|
|
2877
|
+
},
|
|
2878
|
+
diff: profiled.diff,
|
|
2879
|
+
load(recorder, node) {
|
|
2880
|
+
const loaded = profiled.load(recorder, node);
|
|
2881
|
+
profiles.set(node.nodeId, loaded.profile);
|
|
2882
|
+
return loaded;
|
|
2883
|
+
}
|
|
2884
|
+
};
|
|
2885
|
+
for (const node of (await recorder.state()).nodes()) codec.load(recorder, node);
|
|
2886
|
+
await reconcileInterruptedSearchCalls(costLedger, searchId, lanes);
|
|
2887
|
+
const laneRefusal = (nodeProfile) => {
|
|
2888
|
+
const refusals = lanes.map((lane) => `${lane.name}: ${lane.refusal(nodeProfile)}`);
|
|
2889
|
+
return lanes.some((lane) => lane.refusal(nodeProfile) === void 0) ? null : `no lane can run the profile (${refusals.join("; ")})`;
|
|
2890
|
+
};
|
|
2891
|
+
const root = nodeOf(prepared.baselineSurface);
|
|
2892
|
+
if (root.profile) {
|
|
2893
|
+
const refused = laneRefusal(root.profile);
|
|
2894
|
+
if (refused) throw new ConfigError(`improve(): the baseline profile cannot run: ${refused}`);
|
|
2895
|
+
}
|
|
2223
2896
|
const scenarios = [
|
|
2224
2897
|
...trainScenarios,
|
|
2225
2898
|
...selectionScenarios,
|
|
@@ -2230,22 +2903,28 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2230
2903
|
if (scenarioIds.has(scenario.id)) throw new ConfigError(`improve(): scenario '${scenario.id}' appears in more than one split`);
|
|
2231
2904
|
scenarioIds.add(scenario.id);
|
|
2232
2905
|
}
|
|
2233
|
-
const
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2906
|
+
const executor = searchExecutor({
|
|
2907
|
+
lanes,
|
|
2908
|
+
profileOf(nodeId) {
|
|
2909
|
+
const known = profiles.get(nodeId);
|
|
2910
|
+
if (known === void 0) throw new Error(`improve(): node ${nodeId} was placed unregistered`);
|
|
2911
|
+
return known;
|
|
2912
|
+
},
|
|
2913
|
+
costLedger,
|
|
2914
|
+
identity: execution,
|
|
2915
|
+
storage,
|
|
2916
|
+
dir: searchDir,
|
|
2917
|
+
...workerSlots ? { workerSlots } : {},
|
|
2918
|
+
...budgetPool ? { budgetPool } : {},
|
|
2919
|
+
...trace ? { trace } : {},
|
|
2920
|
+
async runAttempt({ work, lane, context, costLedger: cellLedger, span }) {
|
|
2244
2921
|
const nodeProfile = work.artifact.profile;
|
|
2245
2922
|
if (!nodeProfile) throw new Error(`improve(): refused node ${work.nodeId} was given a cell`);
|
|
2246
2923
|
const profileDigest = canonicalAgentProfileDigest(nodeProfile);
|
|
2924
|
+
let fault;
|
|
2247
2925
|
const cell = (await runCampaign({
|
|
2248
2926
|
...campaignOptions,
|
|
2927
|
+
...lane.kind === "subscription" && campaignOptions.expectUsage === void 0 ? { expectUsage: "off" } : {},
|
|
2249
2928
|
scenarios,
|
|
2250
2929
|
judges: prepared.judges,
|
|
2251
2930
|
seed,
|
|
@@ -2253,21 +2932,35 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2253
2932
|
storage,
|
|
2254
2933
|
runDir: `${searchDir}/nodes/${profileDigest.slice(7)}`,
|
|
2255
2934
|
dispatchRef: `improve:${evaluationRef}:${profileDigest}`,
|
|
2256
|
-
dispatch
|
|
2935
|
+
async dispatch(scenario, ctx) {
|
|
2936
|
+
try {
|
|
2937
|
+
return await agent(nodeProfile, scenario, {
|
|
2938
|
+
...ctx,
|
|
2939
|
+
search: context
|
|
2940
|
+
});
|
|
2941
|
+
} catch (error) {
|
|
2942
|
+
if (isSearchEnvironmentFault(error)) fault = error;
|
|
2943
|
+
throw error;
|
|
2944
|
+
}
|
|
2945
|
+
},
|
|
2257
2946
|
cellFilter: ({ scenario, rep }) => scenario.id === work.taskId && rep === work.rep,
|
|
2258
2947
|
resumable: true,
|
|
2259
2948
|
maxConcurrency: 1,
|
|
2260
|
-
costLedger,
|
|
2949
|
+
costLedger: cellLedger,
|
|
2261
2950
|
costPhase: `search.${work.stage}`,
|
|
2951
|
+
buildTraceWriter: () => ({
|
|
2952
|
+
span,
|
|
2953
|
+
flush: async () => {}
|
|
2954
|
+
}),
|
|
2262
2955
|
signal: work.signal
|
|
2263
2956
|
})).cells[0];
|
|
2264
2957
|
if (!cell) throw new Error(`improve(): cell ${work.cellId} produced no campaign cell`);
|
|
2265
|
-
return campaignCellSearchResult(cell, {
|
|
2958
|
+
return faultResult(campaignCellSearchResult(cell, {
|
|
2266
2959
|
execution,
|
|
2267
|
-
lane:
|
|
2268
|
-
});
|
|
2960
|
+
lane: lane.name
|
|
2961
|
+
}), cell.errorStage === "dispatch" ? fault : void 0);
|
|
2269
2962
|
}
|
|
2270
|
-
};
|
|
2963
|
+
});
|
|
2271
2964
|
const { proposer } = method;
|
|
2272
2965
|
const proposerSource = {
|
|
2273
2966
|
uri: `proposer:${proposer.kind}`,
|
|
@@ -2288,7 +2981,7 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2288
2981
|
},
|
|
2289
2982
|
childrenPerProposal: method.childrenPerProposal,
|
|
2290
2983
|
async propose(request) {
|
|
2291
|
-
const
|
|
2984
|
+
const tags = searchCallTags(searchId, { operation: request.operationId });
|
|
2292
2985
|
const state = await recorder.state();
|
|
2293
2986
|
const parents = request.parents.map(({ nodeId, artifact }) => ({
|
|
2294
2987
|
nodeId,
|
|
@@ -2305,13 +2998,16 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2305
2998
|
populationSize: method.childrenPerProposal,
|
|
2306
2999
|
generation: request.expansion,
|
|
2307
3000
|
signal: request.signal,
|
|
2308
|
-
costLedger,
|
|
3001
|
+
costLedger: taggedLedger(costLedger, tags),
|
|
2309
3002
|
costPhase: proposalPhase
|
|
2310
3003
|
});
|
|
2311
3004
|
const decision = proposer.decide?.({ history: [] });
|
|
2312
3005
|
const proposed = decision?.stop ? [] : await proposer.propose(context);
|
|
2313
3006
|
if (!Array.isArray(proposed)) throw new ConfigError("improve(): the search proposer must return an array");
|
|
2314
|
-
const fresh = costLedger.list({
|
|
3007
|
+
const fresh = costLedger.list({
|
|
3008
|
+
phase: proposalPhase,
|
|
3009
|
+
tags
|
|
3010
|
+
});
|
|
2315
3011
|
return {
|
|
2316
3012
|
children: proposed.slice(0, method.childrenPerProposal).map((child) => isProposedCandidate(child) ? {
|
|
2317
3013
|
artifact: nodeOf(child.surface),
|
|
@@ -2337,12 +3033,13 @@ async function runSearchImprovement(profile, opts) {
|
|
|
2337
3033
|
return refusalText(error);
|
|
2338
3034
|
}
|
|
2339
3035
|
const paths = privateValuePaths([node.surface]);
|
|
2340
|
-
|
|
3036
|
+
if (paths.length > 0) return `the surface carries a credential or private value at ${paths.slice(0, 8).join(", ")}`;
|
|
3037
|
+
return laneRefusal(node.profile);
|
|
2341
3038
|
};
|
|
2342
3039
|
const startedAt = Date.now();
|
|
2343
3040
|
const result = await runSearch({
|
|
2344
3041
|
recorder,
|
|
2345
|
-
root
|
|
3042
|
+
root,
|
|
2346
3043
|
codec,
|
|
2347
3044
|
policy,
|
|
2348
3045
|
allocation,
|
|
@@ -2510,13 +3207,36 @@ function searchMethodDescriptor(method) {
|
|
|
2510
3207
|
proposer: method.proposer.kind,
|
|
2511
3208
|
maxExpansions: method.maxExpansions,
|
|
2512
3209
|
childrenPerProposal: method.childrenPerProposal,
|
|
2513
|
-
|
|
2514
|
-
|
|
3210
|
+
lanes: method.lanes.map(({ name, kind, capacity, costCap, cellUsd }) => ({
|
|
3211
|
+
name,
|
|
3212
|
+
kind,
|
|
3213
|
+
capacity,
|
|
3214
|
+
costCap,
|
|
3215
|
+
cellUsd
|
|
3216
|
+
})),
|
|
2515
3217
|
claimReserveUsd: method.claimReserveUsd,
|
|
2516
3218
|
deadline: method.deadline,
|
|
2517
3219
|
maxAttempts: method.maxAttempts
|
|
2518
3220
|
};
|
|
2519
3221
|
}
|
|
3222
|
+
/**
|
|
3223
|
+
* An environment fault the agent raised settles retryable. The campaign reports every error as
|
|
3224
|
+
* final because it applies its own retry policy; here the kernel owns retries.
|
|
3225
|
+
*/
|
|
3226
|
+
function faultResult(result, fault) {
|
|
3227
|
+
if (fault === void 0 || result.outcome.status !== "errored") return result;
|
|
3228
|
+
return {
|
|
3229
|
+
...result,
|
|
3230
|
+
outcome: {
|
|
3231
|
+
...result.outcome,
|
|
3232
|
+
error: {
|
|
3233
|
+
code: "environment-fault",
|
|
3234
|
+
message: refusalText(fault),
|
|
3235
|
+
retryable: true
|
|
3236
|
+
}
|
|
3237
|
+
}
|
|
3238
|
+
};
|
|
3239
|
+
}
|
|
2520
3240
|
/** A refusal as the ledger stores it: redacted with the share profile. */
|
|
2521
3241
|
function refusalText(error) {
|
|
2522
3242
|
return redact(error instanceof Error ? error.message : String(error), { profile: "share" }).value;
|
|
@@ -4140,6 +4860,6 @@ function sameOrderedValues(left, right) {
|
|
|
4140
4860
|
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
4141
4861
|
}
|
|
4142
4862
|
//#endregion
|
|
4143
|
-
export {
|
|
4863
|
+
export { applyRolloutPolicyToProfile as $, agentImprovementTargetInput as A, SearchEnvironmentFault as B, optimizationActivationReceiptFromMetadata as C, agentImprovementProfileSurfaceDigest as D, agentImprovementProfileDiffs as E, isAgentImprovementProfileSurface as F, subscriptionLane as G, isSearchEnvironmentFault as H, isAgentProfileMeasuredSurface as I, parseExecutionRef as J, assertCandidateValidator as K, improve as L, agentProfileImprovementStateDigest as M, assertProfileImprovementTargetsShareIdentity as N, agentImprovementProfileSurfaceInput as O, buildAgentImprovementActivationTargets as P, ROLLOUT_POLICY_EXTENSION as Q, createCommandProfileTrainer as R, createOptimizationActivationReceipt as S, AGENT_PROFILE_MEASURED_SURFACES as T, routerLane as U, dedicatedLane as V, sharedBoxLane as W, withMethodRuntimeControls as X, privateValuePaths as Y, rawTraceDistiller as Z, profilePolicyWithBudget as _, executeAgentCandidateExperimentCell as a, commandVerifier as at, profileTaskScenarioIdentity as b, requireSealedCandidateExperiment as c, verifyAgentImprovementActivation as d, normalizeRolloutPolicy as et, verifyAgentImprovementProposal as f, profileImprovementMetadata as g, createProfileImprovementCostLedger as h, createAgentImprovementProposal as i, agenticGenerator as it, agentImprovementTargetProfileDiffs as j, agentImprovementTargetDigest as k, reviewAgentImprovementProposal as l, verifyCandidateExecutionEvidence as m, createAgentImprovementActivation as n, serializeRolloutPolicy as nt, proposeAgentImprovement as o, defaultBuildPrompt as ot, verifyAgentImprovementReview as p, assertProfileTrainingIsHeldOut as q, createAgentImprovementMeasuredComparison as r, structuralRolloutPolicyFromProfile as rt, proposeAgentProfileImprovement as s, AgentCandidateExperimentCellExecutionError as t, parseRolloutPolicy as tt, runAgentCandidateExperiment as u, profilePreparationAccounting as v, AGENT_IMPROVEMENT_PROFILE_SURFACES as w, sealProfileImprovementBenchmark as x, profileStateDigest as y, searchMethod as z };
|
|
4144
4864
|
|
|
4145
|
-
//# sourceMappingURL=improvement-cycle-
|
|
4865
|
+
//# sourceMappingURL=improvement-cycle-DCTCTmNl.js.map
|