@tangle-network/agent-runtime 0.337.1 → 0.338.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-DMehb166.d.ts → activation-CsbNax2X.d.ts} +92 -4
- package/dist/activation-kbkAuSmY.js +889 -0
- package/dist/activation-kbkAuSmY.js.map +1 -0
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +3 -6
- package/dist/agent.js.map +1 -1
- package/dist/{bundle-BPGjESee.js → bundle-DM60AI2B.js} +2 -2
- package/dist/{bundle-BPGjESee.js.map → bundle-DM60AI2B.js.map} +1 -1
- package/dist/candidate-execution/index.d.ts +2 -2
- package/dist/candidate-execution/index.js +4 -5
- package/dist/candidate-execution/index.js.map +1 -1
- package/dist/content-digest-sBTLujlK.js +9 -0
- package/dist/content-digest-sBTLujlK.js.map +1 -0
- package/dist/{delegate-aC14Cww3.js → delegate-Se1YZKMb.js} +2 -2
- package/dist/{delegate-aC14Cww3.js.map → delegate-Se1YZKMb.js.map} +1 -1
- package/dist/{coordination-driver-Dg9a6qsm.js → driver-control-acknowledgers-T2a1Ouay.js} +4514 -4440
- package/dist/driver-control-acknowledgers-T2a1Ouay.js.map +1 -0
- package/dist/durable.d.ts +1 -1
- package/dist/durable.js +223 -227
- package/dist/durable.js.map +1 -1
- package/dist/{graph-Cfx711ES.js → graph-CY0wH838.js} +65 -18
- package/dist/graph-CY0wH838.js.map +1 -0
- package/dist/{graph-registry-DAI6KNXB.js → graph-registry-By0tUj4S.js} +16 -15
- package/dist/graph-registry-By0tUj4S.js.map +1 -0
- package/dist/{improvement-cycle-DHKcb_YW.js → improve-CWGBT13K.js} +75 -1220
- package/dist/improve-CWGBT13K.js.map +1 -0
- package/dist/index.d.ts +6 -6
- package/dist/index.js +10 -13
- package/dist/index.js.map +1 -1
- package/dist/{integrations-DopfCC2a.d.ts → integrations-6xttbPi9.d.ts} +3 -3
- package/dist/integrations-dFi-ALk7.js.map +1 -1
- package/dist/intelligence.d.ts +7 -94
- package/dist/intelligence.js +505 -13
- package/dist/intelligence.js.map +1 -1
- package/dist/kernel.d.ts +2 -2
- package/dist/kernel.js +13 -12
- package/dist/{loop-runner-bin-B86AiPBf.d.ts → loop-runner-bin-CFnrAHpl.d.ts} +2 -2
- package/dist/{loop-runner-bin-BSKBNC38.js → loop-runner-bin-CdPlhy9g.js} +3 -3
- package/dist/{loop-runner-bin-BSKBNC38.js.map → loop-runner-bin-CdPlhy9g.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +7 -3
- package/dist/mcp/index.js +6 -8
- package/dist/mcp/index.js.map +1 -1
- package/dist/{openai-tools-DHO2QS1q.d.ts → openai-tools-zh2I_uD7.d.ts} +2 -2
- package/dist/platform.d.ts +7 -2
- package/dist/platform.js +16 -8
- package/dist/platform.js.map +1 -1
- package/dist/{prepare-CER-ZkPp.js → prepare-Bt0IouPq.js} +1241 -586
- package/dist/prepare-Bt0IouPq.js.map +1 -0
- package/dist/{profile-gNUsHNIJ.js → profile-Do-5xO_D.js} +89 -3
- package/dist/profile-Do-5xO_D.js.map +1 -0
- package/dist/profiles.d.ts +1 -1
- package/dist/profiles.js +1 -3
- package/dist/profiles.js.map +1 -1
- package/dist/{protected-model-port-BhpQRTw-.d.ts → protected-model-port-DGxQ8_YM.d.ts} +3 -3
- package/dist/{provision-supervisor-72YFf-i_.js → provision-supervisor-0jMufp4h.js} +12 -9
- package/dist/provision-supervisor-0jMufp4h.js.map +1 -0
- package/dist/record-value-B6bwJxLa.js +12 -0
- package/dist/record-value-B6bwJxLa.js.map +1 -0
- package/dist/{run-layout-Cu8H7aeh.js → run-cancellation-BoW5Sg9Y.js} +174 -95
- package/dist/run-cancellation-BoW5Sg9Y.js.map +1 -0
- package/dist/{runtime-qFLOq1R9.js → runtime-D37zRSs7.js} +432 -684
- package/dist/runtime-D37zRSs7.js.map +1 -0
- package/dist/{sandbox-profile-compat-QTkLe1Z6.js → sandbox-profile-compat-xSoVtdWw.js} +690 -697
- package/dist/sandbox-profile-compat-xSoVtdWw.js.map +1 -0
- package/dist/search-executor-DmgUm5p7.js +1916 -0
- package/dist/search-executor-DmgUm5p7.js.map +1 -0
- package/dist/{server-_FldJ9Cl.js → server-DQHJszfk.js} +4 -4
- package/dist/server-DQHJszfk.js.map +1 -0
- package/dist/{search-executor-CGxvizHh.js → structural-rollout-DAvIE8mw.js} +2262 -3984
- package/dist/structural-rollout-DAvIE8mw.js.map +1 -0
- package/dist/{supervise-H4BTN6uA.js → supervise-WuPAwfPh.js} +7793 -5338
- package/dist/supervise-WuPAwfPh.js.map +1 -0
- package/dist/{supervisor-C27ne4yv.js → supervisor-CPG1I9UN.js} +27379 -27899
- package/dist/supervisor-CPG1I9UN.js.map +1 -0
- package/dist/testing.d.ts +226 -1
- package/dist/testing.js +13 -14
- package/dist/testing.js.map +1 -1
- package/dist/{top-app-BaMw1W2z.js → top-app-C55Wa7uy.js} +339 -337
- package/dist/top-app-C55Wa7uy.js.map +1 -0
- package/dist/tui/bin.js +1 -1
- package/dist/tui/index.d.ts +17 -15
- package/dist/tui/index.js +2 -2
- package/dist/{types-CupCBY-j.d.ts → types-CrNWif8-.d.ts} +17655 -17406
- package/dist/{workspace-streams-JLjh5DC6.d.ts → workspace-streams-BZZds1Fw.d.ts} +18 -16
- package/package.json +4 -4
- package/skills/agent-graphs/SKILL.md +10 -3
- package/skills/agent-graphs/references/authoring.md +17 -0
- package/skills/profile-authoring/SKILL.md +3 -3
- package/skills/supervise/SKILL.md +2 -2
- package/dist/activation-BgWVlVu2.js +0 -234
- package/dist/activation-BgWVlVu2.js.map +0 -1
- package/dist/coordination-driver-Dg9a6qsm.js.map +0 -1
- package/dist/graph-Cfx711ES.js.map +0 -1
- package/dist/graph-registry-DAI6KNXB.js.map +0 -1
- package/dist/improvement-cycle-DHKcb_YW.js.map +0 -1
- package/dist/prepare-CER-ZkPp.js.map +0 -1
- package/dist/profile-gNUsHNIJ.js.map +0 -1
- package/dist/protected-model-port-BuRsviSL.js +0 -766
- package/dist/protected-model-port-BuRsviSL.js.map +0 -1
- package/dist/provision-supervisor-72YFf-i_.js.map +0 -1
- package/dist/run-layout-Cu8H7aeh.js.map +0 -1
- package/dist/runtime-qFLOq1R9.js.map +0 -1
- package/dist/sandbox-profile-compat-QTkLe1Z6.js.map +0 -1
- package/dist/search-executor-CGxvizHh.js.map +0 -1
- package/dist/server-_FldJ9Cl.js.map +0 -1
- package/dist/supervise-H4BTN6uA.js.map +0 -1
- package/dist/supervisor-C27ne4yv.js.map +0 -1
- package/dist/top-app-BaMw1W2z.js.map +0 -1
|
@@ -0,0 +1,1916 @@
|
|
|
1
|
+
import { An as padSpanId, En as createOtelExporter, Kt as LEAF_CONTINUATION_TASK, La as promptFromAgentTurnInput, Mn as toOtelAttributes, On as generateSpanId, Ra as promptOptionsFromAgentTurnInput, Tn as createOpenInferenceFileExporter, jn as padTraceId, nn as openSlotGroup, ra as unavailableSignalOfFailure, ri as harnessInvocation, ta as unavailablePauseMs } from "./supervisor-CPG1I9UN.js";
|
|
2
|
+
import { i as ConfigError, m as ValidationError } from "./errors-DodWX-cb.js";
|
|
3
|
+
import { st as sleep, z as immutableCandidateValue } from "./sandbox-profile-compat-xSoVtdWw.js";
|
|
4
|
+
import { a as concreteProfileModel, l as profileProviderModel } from "./model-policy-CyCUwHed.js";
|
|
5
|
+
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest } from "@tangle-network/agent-interface";
|
|
6
|
+
import { CostAccountingIncompleteError, CostCeilingReachedError, costForTokenPricing, costForUsage } from "@tangle-network/agent-eval";
|
|
7
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
8
|
+
import { mkdirSync, renameSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { dirname, join } from "node:path";
|
|
10
|
+
import { materializeProfile } from "@tangle-network/agent-profile-materialize";
|
|
11
|
+
import { ATTR, branchSpan } from "@tangle-network/agent-trace-contract";
|
|
12
|
+
import { createRunCostLedger, fsCampaignStorage, openSearchLedger, searchClaimDecision, searchReceiptAccounting, verifySearchClaim } from "@tangle-network/agent-eval/campaign";
|
|
13
|
+
import { sealExperiment } from "@tangle-network/agent-eval/experiment";
|
|
14
|
+
import { harnessSystemPromptIntents } from "@tangle-network/agent-interface/harness-capabilities";
|
|
15
|
+
//#region src/improvement/search-decision.ts
|
|
16
|
+
/**
|
|
17
|
+
* Runtime's decision on a closed search, made from its records alone: the claim
|
|
18
|
+
* the ledger supports, the cost ledger's accounting, the ship rule, and, when
|
|
19
|
+
* the search ships, the profiles-table promotion that cites the claim.
|
|
20
|
+
*
|
|
21
|
+
* `improve()` applies it when a search closes. `decideSearchImprovement`
|
|
22
|
+
* applies it to a closed search's directory, so a search is decided again under
|
|
23
|
+
* the current rule without running or spending anything. The ship rule is not
|
|
24
|
+
* part of a search's identity: the same ledger re-decided after a rule change is
|
|
25
|
+
* the same search.
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* A search's cost as `ImproveCost`: the ledger's summary, plus the bound on the
|
|
29
|
+
* calls whose cost is unknown when each declared a maximum before it ran. Those
|
|
30
|
+
* are calls that failed with no receipt, or that a killed process left pending
|
|
31
|
+
* and a restart settled unknown. Each counts at its maximum, the same bound the
|
|
32
|
+
* ledger enforces its ceiling with.
|
|
33
|
+
*/
|
|
34
|
+
function improveSearchCost(ledger) {
|
|
35
|
+
const summary = ledger.summary();
|
|
36
|
+
const cost = {
|
|
37
|
+
totalCostUsd: summary.totalCostUsd,
|
|
38
|
+
accountingComplete: summary.accountingComplete,
|
|
39
|
+
incompleteReasons: [...summary.incompleteReasons]
|
|
40
|
+
};
|
|
41
|
+
if (summary.accountingComplete || summary.pendingCalls > 0) return cost;
|
|
42
|
+
let unknownCalls = 0;
|
|
43
|
+
let unknownMaximumUsd = 0;
|
|
44
|
+
for (const receipt of ledger.list()) if (receipt.costUnknown) {
|
|
45
|
+
if (receipt.maximumCostUsd === void 0) return cost;
|
|
46
|
+
unknownCalls += 1;
|
|
47
|
+
unknownMaximumUsd += receipt.maximumCostUsd;
|
|
48
|
+
} else if (receipt.usageUnknown || receipt.maximumCostUsd !== void 0 && receipt.costUsd > receipt.maximumCostUsd) return cost;
|
|
49
|
+
if (unknownCalls === 0) return cost;
|
|
50
|
+
return {
|
|
51
|
+
...cost,
|
|
52
|
+
costBound: {
|
|
53
|
+
unknownCalls,
|
|
54
|
+
unknownMaximumUsd,
|
|
55
|
+
totalMaximumUsd: summary.totalCostUsd + unknownMaximumUsd
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/** Runtime's ship rule over the kernel's statistical claim (design §6.5 step 5), shared by
|
|
60
|
+
* `searchMethod` and `runStrategyEvolution`. */
|
|
61
|
+
function runtimeShipDecision(input) {
|
|
62
|
+
const { claim, cost, costCeiling } = input;
|
|
63
|
+
if (claim.decision !== "ship") return {
|
|
64
|
+
decision: "hold",
|
|
65
|
+
reason: claim.reason
|
|
66
|
+
};
|
|
67
|
+
if (!input.verified) return {
|
|
68
|
+
decision: "hold",
|
|
69
|
+
reason: "the claim shipped, but it does not re-derive from the ledger under this rule revision"
|
|
70
|
+
};
|
|
71
|
+
const bound = cost.accountingComplete ? void 0 : cost.costBound;
|
|
72
|
+
if (!cost.accountingComplete && bound === void 0) return {
|
|
73
|
+
decision: "hold",
|
|
74
|
+
reason: "the claim shipped, but the search cost accounting is incomplete"
|
|
75
|
+
};
|
|
76
|
+
if (bound && costCeiling !== void 0 && bound.totalMaximumUsd > costCeiling) return {
|
|
77
|
+
decision: "hold",
|
|
78
|
+
reason: `the claim shipped, but ${bound.unknownCalls} call(s) of unknown cost, counted at their declared maximum, put the search at up to $${dollars$1(bound.totalMaximumUsd)}, above its $${dollars$1(costCeiling)} ceiling`
|
|
79
|
+
};
|
|
80
|
+
if (input.lowerBound === null || !(input.lowerBound > input.minimumLift)) return {
|
|
81
|
+
decision: "hold",
|
|
82
|
+
reason: `the claim shipped, but its test lower bound ${input.lowerBound} does not exceed minimumLift ${input.minimumLift}`
|
|
83
|
+
};
|
|
84
|
+
return {
|
|
85
|
+
decision: "ship",
|
|
86
|
+
reason: bound ? `${claim.reason}; ${bound.unknownCalls} call(s) ended with no receipt and count at their declared maximum, $${dollars$1(bound.unknownMaximumUsd)}, so the search cost at most $${dollars$1(bound.totalMaximumUsd)}${costCeiling === void 0 ? "" : ` of its $${dollars$1(costCeiling)} ceiling`}` : claim.reason
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Decide a closed search: the ship rule, and the promotion when it ships.
|
|
91
|
+
* `profileOf` returns the exact profile a node holds.
|
|
92
|
+
*/
|
|
93
|
+
async function decideClosedSearch(input) {
|
|
94
|
+
const { claim, state } = input;
|
|
95
|
+
const shipped = claim.finalists.find((finalist) => finalist.nodeId === claim.selected && finalist.test !== null);
|
|
96
|
+
const ruled = runtimeShipDecision({
|
|
97
|
+
claim,
|
|
98
|
+
verified: input.claimVerification.status === "verified",
|
|
99
|
+
cost: input.cost,
|
|
100
|
+
costCeiling: input.costCeiling,
|
|
101
|
+
lowerBound: shipped?.test?.interval[0] ?? null,
|
|
102
|
+
minimumLift: input.minimumLift
|
|
103
|
+
});
|
|
104
|
+
const lift = shipped?.test ? {
|
|
105
|
+
lift: shipped.test.delta,
|
|
106
|
+
liftInterval: {
|
|
107
|
+
low: shipped.test.interval[0],
|
|
108
|
+
high: shipped.test.interval[1]
|
|
109
|
+
}
|
|
110
|
+
} : {};
|
|
111
|
+
if (ruled.decision !== "ship") return {
|
|
112
|
+
...ruled,
|
|
113
|
+
...lift
|
|
114
|
+
};
|
|
115
|
+
const promotion = await searchPromotion(state, claim, input.profileOf);
|
|
116
|
+
if ("refused" in promotion) return {
|
|
117
|
+
decision: "hold",
|
|
118
|
+
reason: `${ruled.reason}; held: ${promotion.refused}`,
|
|
119
|
+
...lift
|
|
120
|
+
};
|
|
121
|
+
return {
|
|
122
|
+
...ruled,
|
|
123
|
+
...lift,
|
|
124
|
+
promotion
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The claim as a profiles-table promotion. The sealed experiment states the
|
|
129
|
+
* claim's design: the root as control and every finalist the claim tested as a
|
|
130
|
+
* treatment, each by its exact profile digest, fixed before any test cell ran.
|
|
131
|
+
* The decision is Eval's paired decision for the selected finalist, made again
|
|
132
|
+
* from the ledger by `searchClaimDecision`. The seal is dated by the search's
|
|
133
|
+
* close, so the same ledger always yields the same promotion.
|
|
134
|
+
*/
|
|
135
|
+
async function searchPromotion(state, claim, profileOf) {
|
|
136
|
+
const header = state.header;
|
|
137
|
+
const closed = state.closed;
|
|
138
|
+
const root = state.rootNodeId;
|
|
139
|
+
const selected = claim.selected;
|
|
140
|
+
if (!header || !closed || root === null || selected === null) return { refused: "the ledger holds no closed claim with a selected finalist" };
|
|
141
|
+
const decision = searchClaimDecision(state, selected);
|
|
142
|
+
if (decision === null || decision.promote !== true) return { refused: `Eval's paired decision for ${selected} does not promote` };
|
|
143
|
+
if (!("adequate" in claim.power)) return { refused: "the claim tested its finalists without a known power" };
|
|
144
|
+
const tested = claim.finalists.filter((finalist) => finalist.test !== null);
|
|
145
|
+
const estimator = claim.power.estimator;
|
|
146
|
+
let outcome;
|
|
147
|
+
if (estimator.kind === "binary") outcome = {
|
|
148
|
+
kind: "binary",
|
|
149
|
+
pass: `the per-unit test mean equals ${estimator.scale}`
|
|
150
|
+
};
|
|
151
|
+
else {
|
|
152
|
+
if ([root, ...tested.map((finalist) => finalist.nodeId)].flatMap((nodeId) => state.unitScores(nodeId, "test").map((unit) => unit.mean)).some((score) => score < 0 || score > 1)) return { refused: "a test score leaves the campaign's [0, 1] score scale" };
|
|
153
|
+
outcome = {
|
|
154
|
+
kind: "bounded-score",
|
|
155
|
+
min: 0,
|
|
156
|
+
max: 1,
|
|
157
|
+
orientation: header.objective.direction === "maximize" ? "higher-is-better" : "lower-is-better"
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
const arm = (nodeId, role) => ({
|
|
161
|
+
id: nodeId,
|
|
162
|
+
role,
|
|
163
|
+
profileDigest: canonicalAgentProfileDigest(profileOf(nodeId)),
|
|
164
|
+
pins: {
|
|
165
|
+
searchId: state.searchId,
|
|
166
|
+
nodeId
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
const testUnits = new Set(header.splits.test.tasks.map((task) => task.unitId)).size;
|
|
170
|
+
return {
|
|
171
|
+
experiment: await sealExperiment({
|
|
172
|
+
id: `${state.searchId}.claim`,
|
|
173
|
+
claim: header.objective.claim,
|
|
174
|
+
hypothesis: `Each finalist of search ${state.searchId} beats the root on its ${testUnits} sealed test units at confidence ${tested[0]?.test?.confidence ?? claim.confidence}, by Eval's paired decision (${claim.rule.uri} ${claim.rule.revision}); the claim fixed these arms before any test cell ran.`,
|
|
175
|
+
arms: [arm(root, "control"), ...tested.map((finalist) => arm(finalist.nodeId, "treatment"))],
|
|
176
|
+
outcome,
|
|
177
|
+
decision: {
|
|
178
|
+
kind: "report-only",
|
|
179
|
+
estimands: [],
|
|
180
|
+
intervals: [],
|
|
181
|
+
perRow: []
|
|
182
|
+
},
|
|
183
|
+
seed: header.policy.seed
|
|
184
|
+
}, { sealedAt: closed.occurredAt }),
|
|
185
|
+
decision
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Decide a closed search from its directory, under the current ship rule, with
|
|
190
|
+
* no model call: the claim verified from the ledger, the cost from its cost
|
|
191
|
+
* ledger, and the promotion when it ships. Use it to cite a finished search as
|
|
192
|
+
* a registry promotion, or to re-decide one that an earlier rule held.
|
|
193
|
+
*/
|
|
194
|
+
async function decideSearchImprovement(options) {
|
|
195
|
+
const storage = options.storage ?? fsCampaignStorage();
|
|
196
|
+
const minimumLift = options.minimumLift ?? 0;
|
|
197
|
+
if (!Number.isFinite(minimumLift) || minimumLift < 0) throw new ConfigError("decideSearchImprovement(): minimumLift must be a finite number of at least 0");
|
|
198
|
+
const ledgerPath = join(options.searchDir, "ledger.jsonl");
|
|
199
|
+
const text = storage.read(ledgerPath);
|
|
200
|
+
if (text === void 0) throw new ConfigError(`decideSearchImprovement(): no search ledger at ${ledgerPath}`);
|
|
201
|
+
const first = text.slice(0, text.indexOf("\n"));
|
|
202
|
+
const searchId = JSON.parse(first).searchId;
|
|
203
|
+
if (typeof searchId !== "string") throw new ConfigError(`decideSearchImprovement(): ${ledgerPath} names no searchId`);
|
|
204
|
+
const state = await (storage.kind === "filesystem" ? openSearchLedger({
|
|
205
|
+
path: ledgerPath,
|
|
206
|
+
searchId
|
|
207
|
+
}) : openSearchLedger({
|
|
208
|
+
path: ledgerPath,
|
|
209
|
+
searchId,
|
|
210
|
+
store: storage
|
|
211
|
+
})).state();
|
|
212
|
+
const claim = state.closed?.claim;
|
|
213
|
+
if (!state.closed || !claim) throw new ConfigError(`decideSearchImprovement(): search ${searchId} is ${state.closed ? "closed without a claim" : "still open"}`);
|
|
214
|
+
const costLedger = options.costLedger ?? createRunCostLedger({
|
|
215
|
+
storage,
|
|
216
|
+
runDir: options.searchDir,
|
|
217
|
+
ensureRunDir: false
|
|
218
|
+
});
|
|
219
|
+
const profileOf = searchProfileReader(state, storage, dirname(ledgerPath));
|
|
220
|
+
const cost = improveSearchCost(costLedger);
|
|
221
|
+
const claimVerification = verifySearchClaim(state);
|
|
222
|
+
const decided = await decideClosedSearch({
|
|
223
|
+
state,
|
|
224
|
+
claim,
|
|
225
|
+
claimVerification,
|
|
226
|
+
cost,
|
|
227
|
+
costCeiling: state.header?.budget.maxUsd ?? void 0,
|
|
228
|
+
minimumLift,
|
|
229
|
+
profileOf
|
|
230
|
+
});
|
|
231
|
+
const root = state.rootNodeId;
|
|
232
|
+
return {
|
|
233
|
+
searchId,
|
|
234
|
+
claim,
|
|
235
|
+
claimVerification,
|
|
236
|
+
cost,
|
|
237
|
+
...decided,
|
|
238
|
+
baseline: profileOf(root),
|
|
239
|
+
candidate: profileOf(claim.selected ?? root)
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
/** Each node's exact profile from the search's blob store, checked against
|
|
243
|
+
* the blob's content address and the node's profile digest. */
|
|
244
|
+
function searchProfileReader(state, storage, dir) {
|
|
245
|
+
const read = /* @__PURE__ */ new Map();
|
|
246
|
+
return (nodeId) => {
|
|
247
|
+
const known = read.get(nodeId);
|
|
248
|
+
if (known) return known;
|
|
249
|
+
const node = state.node(nodeId);
|
|
250
|
+
if (!node) throw new ConfigError(`decideSearchImprovement(): no node ${nodeId}`);
|
|
251
|
+
const stored = readBlob(storage, dir, node.artifact);
|
|
252
|
+
const parsed = agentProfileSchema.safeParse(stored.profile);
|
|
253
|
+
if (stored.kind !== "agent-profile" || !parsed.success) throw new ConfigError(`decideSearchImprovement(): node ${nodeId} holds no AgentProfile`);
|
|
254
|
+
const profile = immutableCandidateValue(parsed.data);
|
|
255
|
+
if (canonicalAgentProfileDigest(profile) !== node.artifactDigest) throw new ConfigError(`decideSearchImprovement(): node ${nodeId}'s profile does not match its digest ${node.artifactDigest}`);
|
|
256
|
+
read.set(nodeId, profile);
|
|
257
|
+
return profile;
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
/** A blob the search stored, read from the blob directory beside its ledger,
|
|
261
|
+
* whatever directory the ledger was written in, and checked against its digest. */
|
|
262
|
+
function readBlob(storage, dir, ref) {
|
|
263
|
+
const hex = ref.sha256.slice(7);
|
|
264
|
+
const path = join(dir, "blobs", `${hex}.json`);
|
|
265
|
+
const text = storage.read(path);
|
|
266
|
+
if (text === void 0) throw new ConfigError(`decideSearchImprovement(): missing blob ${path}`);
|
|
267
|
+
if (createHash("sha256").update(text, "utf8").digest("hex") !== hex) throw new ConfigError(`decideSearchImprovement(): blob ${path} does not match its digest`);
|
|
268
|
+
return JSON.parse(text);
|
|
269
|
+
}
|
|
270
|
+
function dollars$1(value) {
|
|
271
|
+
return value.toFixed(value < 1 ? 4 : 2);
|
|
272
|
+
}
|
|
273
|
+
//#endregion
|
|
274
|
+
//#region src/runtime/shared-box-retry.ts
|
|
275
|
+
/**
|
|
276
|
+
* Attempts for one Sandbox API call that failed on the way to the box.
|
|
277
|
+
*
|
|
278
|
+
* Every call re-verifies the API key with the platform, and a burst of workers makes that check
|
|
279
|
+
* time out: 8 of 64 workers failed with `502 platform_unavailable` (reason `platform_timeout`) in
|
|
280
|
+
* one burst on 2026-09-24, and 3 more lost their transcript enumeration the same way. The check
|
|
281
|
+
* refuses the request before it reaches the box, so repeating it is safe; five attempts over about
|
|
282
|
+
* fifteen seconds rode out every such failure measured.
|
|
283
|
+
*/
|
|
284
|
+
const TRANSIENT_ATTEMPTS = 5;
|
|
285
|
+
/** A failure that a repeated request can change: the platform, the gateway or the network. */
|
|
286
|
+
function isTransientSandboxFailure(error) {
|
|
287
|
+
if (error === null || typeof error !== "object") return false;
|
|
288
|
+
const record = error;
|
|
289
|
+
if (typeof record.status === "number" && [
|
|
290
|
+
408,
|
|
291
|
+
425,
|
|
292
|
+
429,
|
|
293
|
+
500,
|
|
294
|
+
502,
|
|
295
|
+
503,
|
|
296
|
+
504
|
|
297
|
+
].includes(record.status)) return true;
|
|
298
|
+
const text = `${String(record.code ?? "")} ${String(record.name ?? "")} ${String(record.message ?? "")}`;
|
|
299
|
+
return /platform_unavailable|platform_timeout|platform_unreachable|fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN|socket hang up|UND_ERR|bad gateway|gateway timeout/iu.test(text);
|
|
300
|
+
}
|
|
301
|
+
async function transientRetry(operation, options = {}) {
|
|
302
|
+
const delayMs = options.delayMs ?? 1e3;
|
|
303
|
+
for (let attempt = 1;; attempt += 1) try {
|
|
304
|
+
return await operation(attempt);
|
|
305
|
+
} catch (error) {
|
|
306
|
+
if (attempt >= TRANSIENT_ATTEMPTS || !isTransientSandboxFailure(error)) throw error;
|
|
307
|
+
options.signal?.throwIfAborted();
|
|
308
|
+
await sleep(delayMs * 2 ** (attempt - 1), options.signal);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* The Sandbox calls one worker makes, each repeated on a transient failure.
|
|
313
|
+
*
|
|
314
|
+
* A launch is repeated only after the pool confirms the box did not start it: a failed request
|
|
315
|
+
* may have reached the box before its answer was lost, so the repeat first looks for a process
|
|
316
|
+
* in the worker's own directory that this worker did not start yet, and adopts it. A process log
|
|
317
|
+
* that breaks mid-stream is read again from the start, because the box replays a process's
|
|
318
|
+
* buffered output, and the part already delivered is skipped.
|
|
319
|
+
*/
|
|
320
|
+
function resilientBox(box, delayMs) {
|
|
321
|
+
const retry = (operation) => transientRetry(operation, { delayMs });
|
|
322
|
+
return {
|
|
323
|
+
id: box.id,
|
|
324
|
+
exec: (command, options) => retry(() => box.exec(command, options)),
|
|
325
|
+
fs: {
|
|
326
|
+
read: (path) => retry(() => box.fs.read(path)),
|
|
327
|
+
write: (path, content) => retry(() => box.fs.write(path, content))
|
|
328
|
+
},
|
|
329
|
+
async spawnInDirectory(executable, args, options, known) {
|
|
330
|
+
return await transientRetry(async (attempt) => {
|
|
331
|
+
if (attempt > 1) {
|
|
332
|
+
const launched = (await box.process.list()).find((entry) => entry.cwd === options.cwd && entry.command === executable && !known.has(entry.pid));
|
|
333
|
+
const adopted = launched === void 0 ? null : await box.process.get(launched.pid);
|
|
334
|
+
if (adopted !== null) return adopted;
|
|
335
|
+
}
|
|
336
|
+
return await box.process.spawnExact(executable, args, options);
|
|
337
|
+
}, { delayMs });
|
|
338
|
+
},
|
|
339
|
+
async wait(process) {
|
|
340
|
+
return await retry(async () => {
|
|
341
|
+
try {
|
|
342
|
+
return await process.wait();
|
|
343
|
+
} catch (error) {
|
|
344
|
+
if (!isTransientSandboxFailure(error)) throw error;
|
|
345
|
+
const again = await box.process.get(process.pid);
|
|
346
|
+
if (again === null) throw error;
|
|
347
|
+
return await again.wait();
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
},
|
|
351
|
+
async *output(process, stream) {
|
|
352
|
+
let delivered = 0;
|
|
353
|
+
let source = process;
|
|
354
|
+
for (let attempt = 1;; attempt += 1) {
|
|
355
|
+
let seen = 0;
|
|
356
|
+
try {
|
|
357
|
+
for await (const chunk of source[stream]()) {
|
|
358
|
+
const fresh = seen + chunk.length <= delivered ? "" : chunk.slice(Math.max(0, delivered - seen));
|
|
359
|
+
seen += chunk.length;
|
|
360
|
+
if (fresh.length > 0) {
|
|
361
|
+
delivered += fresh.length;
|
|
362
|
+
yield fresh;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
return;
|
|
366
|
+
} catch (error) {
|
|
367
|
+
if (attempt >= TRANSIENT_ATTEMPTS || !isTransientSandboxFailure(error)) throw error;
|
|
368
|
+
await sleep(delayMs * 2 ** (attempt - 1));
|
|
369
|
+
source = await retry(() => box.process.get(process.pid)) ?? source;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
},
|
|
373
|
+
delete: () => retry(() => box.delete())
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
//#endregion
|
|
377
|
+
//#region src/runtime/shared-box-worker.ts
|
|
378
|
+
const HARNESS = "opencode";
|
|
379
|
+
const SHARED_PROVIDER_NAME = "tangle-shared-box";
|
|
380
|
+
const DEFAULT_EXEC_TIMEOUT_MS = 12e4;
|
|
381
|
+
/** The request header the router records as a usage row's `clientName`. */
|
|
382
|
+
const ROUTER_CLIENT_HEADER = "x-tangle-client";
|
|
383
|
+
/**
|
|
384
|
+
* The router client name of the worker that runs supervised node `nodeId`.
|
|
385
|
+
*
|
|
386
|
+
* The node id is already in the spawn journal, so a keeper joins a router row to a node without
|
|
387
|
+
* a new record: the row whose `clientName` is `agent-runtime-node/<nodeId>` is that node's spend.
|
|
388
|
+
*/
|
|
389
|
+
function sharedWorkerClientName(nodeId) {
|
|
390
|
+
if (!/^[!-~]+$/u.test(nodeId)) throw new ValidationError("sharedWorkerClientName: a node id must be visible ASCII");
|
|
391
|
+
return `agent-runtime-node/${nodeId}`;
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Why a shared box cannot carry `profile`, or `undefined` when it can.
|
|
395
|
+
*
|
|
396
|
+
* A shared box carries a profile whose whole behavior reaches an opencode process through its own
|
|
397
|
+
* working directory, its own configuration and the box's own model credential. Everything else
|
|
398
|
+
* keeps a dedicated box: another harness, a model the box credential cannot reach, a replaced
|
|
399
|
+
* system prompt, MCP servers, connections, hooks, subagents, extensions, tool grants, and any
|
|
400
|
+
* resource the materializer does not lower into the worker's directory.
|
|
401
|
+
*/
|
|
402
|
+
function sharedBoxRefusal(profile) {
|
|
403
|
+
if (profile.model?.metadata?.credentialSource !== void 0) return "model.metadata.credentialSource requires a dedicated provider";
|
|
404
|
+
if (profile.harness !== "opencode") return `harness ${String(profile.harness)} is not ${HARNESS}`;
|
|
405
|
+
if (concreteProfileModel(profile) === void 0) return "profile names no concrete model";
|
|
406
|
+
if (profile.model?.provider === void 0) return "profile names no model provider";
|
|
407
|
+
if (profile.model?.reasoningEffort !== void 0) {
|
|
408
|
+
if (tryInvocation(profile) === void 0) return "model.reasoningEffort has no opencode variant";
|
|
409
|
+
}
|
|
410
|
+
if (profile.prompt?.systemPrompt !== void 0) return "prompt.systemPrompt replaces a system prompt";
|
|
411
|
+
for (const field of [
|
|
412
|
+
"mcp",
|
|
413
|
+
"connections",
|
|
414
|
+
"hooks",
|
|
415
|
+
"subagents",
|
|
416
|
+
"extensions",
|
|
417
|
+
"confidential",
|
|
418
|
+
"modes"
|
|
419
|
+
]) if (profile[field] !== void 0) return `profile declares ${field}`;
|
|
420
|
+
if (profile.tools !== void 0 && Object.keys(profile.tools).length > 0) return "profile grants tools";
|
|
421
|
+
const resources = profile.resources;
|
|
422
|
+
if (resources !== void 0) {
|
|
423
|
+
for (const field of [
|
|
424
|
+
"skills",
|
|
425
|
+
"agents",
|
|
426
|
+
"commands",
|
|
427
|
+
"tools"
|
|
428
|
+
]) if (resources[field] !== void 0) return `profile declares resources.${field}`;
|
|
429
|
+
for (const file of resources.files ?? []) if (file.resource.kind !== "inline") return `resource file ${file.path} is not inline`;
|
|
430
|
+
if (resources.instructions !== void 0 && typeof resources.instructions !== "string" && resources.instructions.kind !== "inline") return "resources.instructions is not inline";
|
|
431
|
+
}
|
|
432
|
+
let plan;
|
|
433
|
+
try {
|
|
434
|
+
plan = materializeProfile(profile, HARNESS);
|
|
435
|
+
} catch (error) {
|
|
436
|
+
return `opencode materialization failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
437
|
+
}
|
|
438
|
+
if (plan.unsupported.length > 0) return `opencode cannot carry ${plan.unsupported.map((row) => row.dimension).join(", ")}`;
|
|
439
|
+
if (Object.keys(plan.env).length > 0 || plan.flags.length > 0) return "profile needs launch environment or flags";
|
|
440
|
+
for (const file of plan.files) {
|
|
441
|
+
if (file.root === "agent") return `profile writes ${file.relPath} into the harness home`;
|
|
442
|
+
if ((file.secretSlots?.length ?? 0) > 0) return `profile file ${file.relPath} carries a secret`;
|
|
443
|
+
if (isUnsafeRelativePath(file.relPath)) return `profile file ${file.relPath} leaves its directory`;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
function tryInvocation(profile) {
|
|
447
|
+
try {
|
|
448
|
+
return harnessInvocation(HARNESS, invocationProfile(profile), "probe").args;
|
|
449
|
+
} catch {
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* The profile the invocation table reads. The prompt reaches opencode through its configuration
|
|
455
|
+
* and the model through the provider this placement declares, so only the reasoning effort is
|
|
456
|
+
* left for the table to lower into `--variant`.
|
|
457
|
+
*/
|
|
458
|
+
function invocationProfile(profile) {
|
|
459
|
+
const { prompt: _prompt, model, ...rest } = profile;
|
|
460
|
+
return {
|
|
461
|
+
...rest,
|
|
462
|
+
...model?.reasoningEffort === void 0 ? {} : { model: { reasoningEffort: model.reasoningEffort } }
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
function isUnsafeRelativePath(path) {
|
|
466
|
+
return path.length === 0 || path.startsWith("/") || path.split("/").some((segment) => segment === ".." || segment === "");
|
|
467
|
+
}
|
|
468
|
+
function sharedBoxCapabilities() {
|
|
469
|
+
return {
|
|
470
|
+
profile: {
|
|
471
|
+
namedProfiles: false,
|
|
472
|
+
systemPrompt: {
|
|
473
|
+
...harnessSystemPromptIntents(HARNESS),
|
|
474
|
+
replace: false
|
|
475
|
+
},
|
|
476
|
+
instructions: true,
|
|
477
|
+
tools: false,
|
|
478
|
+
permissions: true,
|
|
479
|
+
mcp: false,
|
|
480
|
+
subagents: false,
|
|
481
|
+
resources: {
|
|
482
|
+
files: true,
|
|
483
|
+
instructions: true
|
|
484
|
+
},
|
|
485
|
+
runtimeUpdate: false,
|
|
486
|
+
validation: false
|
|
487
|
+
},
|
|
488
|
+
streaming: {
|
|
489
|
+
live: true,
|
|
490
|
+
replay: false,
|
|
491
|
+
detach: false,
|
|
492
|
+
turnIdempotency: false
|
|
493
|
+
},
|
|
494
|
+
sessions: {
|
|
495
|
+
continue: false,
|
|
496
|
+
list: false,
|
|
497
|
+
messages: false
|
|
498
|
+
},
|
|
499
|
+
workspace: {
|
|
500
|
+
read: true,
|
|
501
|
+
write: true,
|
|
502
|
+
exec: true,
|
|
503
|
+
git: false,
|
|
504
|
+
upload: false,
|
|
505
|
+
download: false
|
|
506
|
+
},
|
|
507
|
+
branching: {
|
|
508
|
+
checkpoint: false,
|
|
509
|
+
fork: false
|
|
510
|
+
},
|
|
511
|
+
placement: true,
|
|
512
|
+
usage: true,
|
|
513
|
+
confidential: false
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
function parseSharedWorkerProfile(input) {
|
|
517
|
+
const parsed = agentProfileSchema.safeParse(input.profile);
|
|
518
|
+
if (!parsed.success) throw new ValidationError("sharedBoxPlacement: an exact AgentProfile is required at create");
|
|
519
|
+
return parsed.data;
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* A worker create carries only what a slot in a shared box can honor. Box-level settings belong
|
|
523
|
+
* to the placement's own box options, and a per-worker request for them is refused rather than
|
|
524
|
+
* dropped: a worker that asked for an environment variable, a repository or its own resources
|
|
525
|
+
* would otherwise run without them and report nothing.
|
|
526
|
+
*/
|
|
527
|
+
function assertWorkerCreateInput(input) {
|
|
528
|
+
for (const key of [
|
|
529
|
+
"idempotencyKey",
|
|
530
|
+
"workspace",
|
|
531
|
+
"resources",
|
|
532
|
+
"runtimeAttachments",
|
|
533
|
+
"requestedId",
|
|
534
|
+
"egress",
|
|
535
|
+
"billingOwner"
|
|
536
|
+
]) if (input[key] !== void 0) throw new ValidationError(`sharedBoxPlacement: a shared worker cannot set ${key}`);
|
|
537
|
+
if (input.env !== void 0 && Object.keys(input.env).length > 0) throw new ValidationError("sharedBoxPlacement: a shared worker cannot set env");
|
|
538
|
+
if (input.secrets !== void 0 && Object.keys(input.secrets).length > 0) throw new ValidationError("sharedBoxPlacement: a shared worker cannot request secrets");
|
|
539
|
+
if (input.backend !== void 0 && input.backend !== "opencode") throw new ValidationError(`sharedBoxPlacement: backend ${input.backend} is not ${HARNESS}`);
|
|
540
|
+
for (const key of Object.keys(input.providerOptions ?? {})) if (key !== "sandboxCreateOptions") throw new ValidationError(`sharedBoxPlacement: a shared worker cannot set providerOptions.${key}`);
|
|
541
|
+
const box = input.providerOptions?.sandboxCreateOptions;
|
|
542
|
+
if (box === void 0) return;
|
|
543
|
+
if (box === null || typeof box !== "object" || Array.isArray(box)) throw new ValidationError("sharedBoxPlacement: sandboxCreateOptions must be an object");
|
|
544
|
+
for (const key of Object.keys(box)) if (![
|
|
545
|
+
"backend",
|
|
546
|
+
"metadata",
|
|
547
|
+
"name"
|
|
548
|
+
].includes(key)) throw new ValidationError(`sharedBoxPlacement: a shared worker cannot set ${key}`);
|
|
549
|
+
const projected = box;
|
|
550
|
+
if (projected.name !== void 0 && projected.name !== input.name) throw new ValidationError("sharedBoxPlacement: projected name must match the worker name");
|
|
551
|
+
if (projected.metadata !== void 0 && (input.metadata === void 0 || canonicalCandidateDigest(projected.metadata) !== canonicalCandidateDigest(input.metadata))) throw new ValidationError("sharedBoxPlacement: projected metadata must match worker metadata");
|
|
552
|
+
const backend = projected.backend;
|
|
553
|
+
if (backend?.type !== void 0 && backend.type !== "opencode") throw new ValidationError(`sharedBoxPlacement: backend ${backend.type} is not ${HARNESS}`);
|
|
554
|
+
for (const key of Object.keys(backend ?? {})) if (key !== "type" && key !== "profile") throw new ValidationError(`sharedBoxPlacement: a shared worker cannot set backend.${key}`);
|
|
555
|
+
if (backend?.profile !== void 0 && canonicalAgentProfileDigest(agentProfileSchema.parse(backend.profile)) !== canonicalAgentProfileDigest(parseSharedWorkerProfile(input))) throw new ValidationError("sharedBoxPlacement: projected profile must match the exact worker profile");
|
|
556
|
+
}
|
|
557
|
+
/** The environment one worker's processes see: its own directory, HOME and XDG roots. */
|
|
558
|
+
function workerEnv(paths) {
|
|
559
|
+
return [
|
|
560
|
+
"OPENCODE_DISABLE_PROJECT_CONFIG=1",
|
|
561
|
+
`OPENCODE_CONFIG_DIR=${paths.dir}/.opencode`,
|
|
562
|
+
`PWD=${paths.dir}`,
|
|
563
|
+
`HOME=${paths.home}`,
|
|
564
|
+
`XDG_DATA_HOME=${paths.home}/.local/share`,
|
|
565
|
+
`XDG_CONFIG_HOME=${paths.home}/.config`,
|
|
566
|
+
`XDG_STATE_HOME=${paths.home}/.local/state`,
|
|
567
|
+
`XDG_CACHE_HOME=${paths.cache}`
|
|
568
|
+
];
|
|
569
|
+
}
|
|
570
|
+
async function createWorker(lease, root, createProfile, placement) {
|
|
571
|
+
const { clientName, unavailablePause } = placement;
|
|
572
|
+
const box = resilientBox(lease.box, placement.retryDelayMs);
|
|
573
|
+
const workerId = `w-${randomUUID()}`;
|
|
574
|
+
const paths = {
|
|
575
|
+
dir: `${root}/${workerId}`,
|
|
576
|
+
home: `${root}/${workerId}/.home`,
|
|
577
|
+
cache: `${root}/.cache`
|
|
578
|
+
};
|
|
579
|
+
const made = await box.exec(`mkdir -p '${paths.home}' '${paths.cache}'`, { timeoutMs: DEFAULT_EXEC_TIMEOUT_MS });
|
|
580
|
+
if ((made.exitCode ?? 1) !== 0) throw new Error(`sharedBoxPlacement: could not create ${paths.dir}: ${made.stderr ?? ""}`);
|
|
581
|
+
const env = workerEnv(paths);
|
|
582
|
+
let running;
|
|
583
|
+
let turnActive = false;
|
|
584
|
+
let released = false;
|
|
585
|
+
const started = /* @__PURE__ */ new Set();
|
|
586
|
+
const run = async (args, timeoutMs, extraEnv) => {
|
|
587
|
+
const process = await box.spawnInDirectory("/usr/bin/env", [...env, ...args], {
|
|
588
|
+
cwd: paths.dir,
|
|
589
|
+
...extraEnv === void 0 ? {} : { env: extraEnv },
|
|
590
|
+
timeoutMs
|
|
591
|
+
}, started);
|
|
592
|
+
started.add(process.pid);
|
|
593
|
+
return process;
|
|
594
|
+
};
|
|
595
|
+
const collect = async (process) => {
|
|
596
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
597
|
+
drain(box.output(process, "stdout")),
|
|
598
|
+
drain(box.output(process, "stderr")),
|
|
599
|
+
box.wait(process)
|
|
600
|
+
]);
|
|
601
|
+
return {
|
|
602
|
+
exitCode,
|
|
603
|
+
stdout,
|
|
604
|
+
stderr
|
|
605
|
+
};
|
|
606
|
+
};
|
|
607
|
+
const resolvePath = (path) => {
|
|
608
|
+
if (path.startsWith("/")) return path;
|
|
609
|
+
if (isUnsafeRelativePath(path)) throw new ValidationError(`sharedBoxPlacement: path ${path} leaves the worker directory`);
|
|
610
|
+
return `${paths.dir}/${path}`;
|
|
611
|
+
};
|
|
612
|
+
/**
|
|
613
|
+
* One turn: one or more opencode runs of the same session. The first runs `message`; a later run
|
|
614
|
+
* continues the session after the model provider refused a run for capacity.
|
|
615
|
+
*/
|
|
616
|
+
async function* turn(message, turnProfile, options) {
|
|
617
|
+
const signal = options?.signal;
|
|
618
|
+
const timeoutMs = options?.timeoutMs ?? 0;
|
|
619
|
+
const turnDeadline = timeoutMs > 0 ? Date.now() + timeoutMs : void 0;
|
|
620
|
+
let sessionId;
|
|
621
|
+
const subagents = /* @__PURE__ */ new Set();
|
|
622
|
+
let refusalsInARow = 0;
|
|
623
|
+
let prompt = message;
|
|
624
|
+
for (let invocation = 1;; invocation += 1) {
|
|
625
|
+
const launch = await materializeWorker(box, paths, turnProfile, prompt, options?.backend?.model, clientName, sessionId);
|
|
626
|
+
signal?.throwIfAborted();
|
|
627
|
+
const remainingMs = turnDeadline === void 0 ? 0 : Math.max(1, turnDeadline - Date.now());
|
|
628
|
+
const process = await run(launch.args, remainingMs, { OPENCODE_CONFIG_CONTENT: launch.config });
|
|
629
|
+
running = process;
|
|
630
|
+
const onAbort = () => {
|
|
631
|
+
process.kill("SIGKILL", { tree: true }).catch(() => void 0);
|
|
632
|
+
};
|
|
633
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
634
|
+
const stderr = drain(box.output(process, "stderr"));
|
|
635
|
+
const lines = lineReader();
|
|
636
|
+
const parsed = createOpencodeEventParser(launch.model);
|
|
637
|
+
let exitCode;
|
|
638
|
+
let errorText;
|
|
639
|
+
try {
|
|
640
|
+
for await (const chunk of box.output(process, "stdout")) for (const line of lines.push(chunk)) yield* parsed.line(line);
|
|
641
|
+
for (const line of lines.end()) yield* parsed.line(line);
|
|
642
|
+
exitCode = await box.wait(process);
|
|
643
|
+
errorText = (await stderr).slice(-4e3);
|
|
644
|
+
} finally {
|
|
645
|
+
signal?.removeEventListener("abort", onAbort);
|
|
646
|
+
running = void 0;
|
|
647
|
+
}
|
|
648
|
+
sessionId = parsed.sessionId() ?? sessionId;
|
|
649
|
+
for (const child of parsed.subagentSessionIds()) subagents.add(child);
|
|
650
|
+
const failure = parsed.failure(exitCode, errorText);
|
|
651
|
+
const refusal = failure === void 0 || unavailablePause === false || signal?.aborted ? void 0 : unavailableSignalOfFailure({ error: failure });
|
|
652
|
+
refusalsInARow = parsed.madeProgress() ? 1 : refusalsInARow + 1;
|
|
653
|
+
const pauseMs = refusal === void 0 || unavailablePause === false ? 0 : unavailablePauseMs(refusalsInARow, unavailablePause);
|
|
654
|
+
if (refusal === void 0 || turnDeadline !== void 0 && Date.now() + pauseMs >= turnDeadline) {
|
|
655
|
+
if (sessionId !== void 0) await exportSession(run, collect, paths, sessionId);
|
|
656
|
+
for (const child of subagents) if (child !== sessionId) await exportSession(run, collect, paths, child);
|
|
657
|
+
signal?.throwIfAborted();
|
|
658
|
+
yield* parsed.finish(exitCode, errorText, invocation);
|
|
659
|
+
return;
|
|
660
|
+
}
|
|
661
|
+
await sleep(pauseMs, signal);
|
|
662
|
+
signal?.throwIfAborted();
|
|
663
|
+
if (released) throw new ValidationError("sharedBoxPlacement: this worker was released");
|
|
664
|
+
prompt = sessionId === void 0 ? message : LEAF_CONTINUATION_TASK;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
return {
|
|
668
|
+
id: `${box.id}/${workerId}`,
|
|
669
|
+
provider: SHARED_PROVIDER_NAME,
|
|
670
|
+
name: placement.name ?? workerId,
|
|
671
|
+
creation: "created",
|
|
672
|
+
capabilities: sharedBoxCapabilities(),
|
|
673
|
+
async status() {
|
|
674
|
+
return released ? "stopped" : "running";
|
|
675
|
+
},
|
|
676
|
+
metadata: {
|
|
677
|
+
...placement.metadata,
|
|
678
|
+
sharedBoxId: box.id,
|
|
679
|
+
workerId,
|
|
680
|
+
workerDir: paths.dir
|
|
681
|
+
},
|
|
682
|
+
async *stream(input) {
|
|
683
|
+
const message = promptFromAgentTurnInput(input);
|
|
684
|
+
if (typeof message !== "string") throw new ValidationError("sharedBoxPlacement: a shared worker requires a text prompt");
|
|
685
|
+
const options = promptOptionsFromAgentTurnInput(input);
|
|
686
|
+
if (released) throw new ValidationError("sharedBoxPlacement: this worker was released");
|
|
687
|
+
if (turnActive) throw new ValidationError("sharedBoxPlacement: a worker runs one turn at a time");
|
|
688
|
+
const turnProfile = options?.backend?.profile === void 0 ? createProfile : agentProfileSchema.parse(options.backend.profile);
|
|
689
|
+
if (canonicalAgentProfileDigest(turnProfile) !== canonicalAgentProfileDigest(createProfile) && sharedBoxRefusal(turnProfile) !== void 0) throw new ValidationError(`sharedBoxPlacement: ${sharedBoxRefusal(turnProfile)}`);
|
|
690
|
+
if (options?.backend?.type !== void 0 && options.backend.type !== "opencode") throw new ValidationError(`sharedBoxPlacement: backend ${options.backend.type} is not ${HARNESS}`);
|
|
691
|
+
turnActive = true;
|
|
692
|
+
try {
|
|
693
|
+
yield* turn(message, turnProfile, options);
|
|
694
|
+
} finally {
|
|
695
|
+
turnActive = false;
|
|
696
|
+
}
|
|
697
|
+
},
|
|
698
|
+
async exec(command, options) {
|
|
699
|
+
const process = await run([
|
|
700
|
+
"/bin/sh",
|
|
701
|
+
"-c",
|
|
702
|
+
command
|
|
703
|
+
], options?.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS);
|
|
704
|
+
return await collect(process);
|
|
705
|
+
},
|
|
706
|
+
async read(path) {
|
|
707
|
+
return await box.fs.read(resolvePath(path));
|
|
708
|
+
},
|
|
709
|
+
async write(path, content) {
|
|
710
|
+
await box.fs.write(resolvePath(path), content);
|
|
711
|
+
},
|
|
712
|
+
async placement() {
|
|
713
|
+
return {
|
|
714
|
+
kind: "sandbox",
|
|
715
|
+
sandboxId: box.id
|
|
716
|
+
};
|
|
717
|
+
},
|
|
718
|
+
async refresh() {},
|
|
719
|
+
async destroy() {
|
|
720
|
+
if (released) return;
|
|
721
|
+
released = true;
|
|
722
|
+
try {
|
|
723
|
+
await running?.kill("SIGKILL", { tree: true });
|
|
724
|
+
await box.exec(`rm -rf '${paths.dir}'`, { timeoutMs: DEFAULT_EXEC_TIMEOUT_MS });
|
|
725
|
+
} finally {
|
|
726
|
+
await lease.release();
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
async function drain(stream) {
|
|
732
|
+
let text = "";
|
|
733
|
+
for await (const chunk of stream) text += chunk;
|
|
734
|
+
return text;
|
|
735
|
+
}
|
|
736
|
+
function lineReader() {
|
|
737
|
+
let pending = "";
|
|
738
|
+
return {
|
|
739
|
+
push(chunk) {
|
|
740
|
+
pending += chunk;
|
|
741
|
+
const lines = pending.split("\n");
|
|
742
|
+
pending = lines.pop() ?? "";
|
|
743
|
+
return lines.filter((line) => line.trim().length > 0);
|
|
744
|
+
},
|
|
745
|
+
end() {
|
|
746
|
+
const rest = pending.trim();
|
|
747
|
+
pending = "";
|
|
748
|
+
return rest.length > 0 ? [rest] : [];
|
|
749
|
+
}
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* Write the worker's materialized profile into its directory and build its launch.
|
|
754
|
+
*
|
|
755
|
+
* The files come from the same materializer the Sandbox sidecar uses, so a shared worker gets the
|
|
756
|
+
* same instruction file, permissions and resource files a dedicated box would. The generated
|
|
757
|
+
* `opencode.json` is passed as `OPENCODE_CONFIG_CONTENT` with its instruction paths made absolute,
|
|
758
|
+
* because opencode resolves an instruction path in that variable against nothing.
|
|
759
|
+
*/
|
|
760
|
+
async function materializeWorker(box, paths, profile, task, turnModel, clientName, continueSession) {
|
|
761
|
+
const refusal = sharedBoxRefusal(profile);
|
|
762
|
+
if (refusal !== void 0) throw new ValidationError(`sharedBoxPlacement: ${refusal}`);
|
|
763
|
+
if (turnModel?.cliAuth !== void 0 || turnModel?.authFiles !== void 0 || turnModel?.authMode === "oauth") throw new ValidationError("sharedBoxPlacement: a shared worker cannot carry native credentials");
|
|
764
|
+
const plan = materializeProfile(profile, HARNESS);
|
|
765
|
+
let generated = {};
|
|
766
|
+
for (const file of plan.files) {
|
|
767
|
+
if (file.relPath === "opencode.json" && file.source === "generated") {
|
|
768
|
+
generated = JSON.parse(file.content);
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
if (continueSession === void 0) await box.fs.write(`${paths.dir}/${file.relPath}`, file.content);
|
|
772
|
+
}
|
|
773
|
+
const instructions = Array.isArray(generated.instructions) ? generated.instructions.map((entry) => typeof entry === "string" && !entry.startsWith("/") ? `${paths.dir}/${entry}` : entry) : void 0;
|
|
774
|
+
const providerId = profile.model.provider;
|
|
775
|
+
const modelId = profileProviderModel(profile);
|
|
776
|
+
const apiKeyEnv = typeof turnModel?.apiKeyEnv === "string" ? turnModel.apiKeyEnv : "OPENCODE_MODEL_API_KEY";
|
|
777
|
+
const baseURL = typeof turnModel?.baseUrl === "string" ? turnModel.baseUrl : "{env:OPENCODE_MODEL_BASE_URL}";
|
|
778
|
+
const config = {
|
|
779
|
+
$schema: "https://opencode.ai/config.json",
|
|
780
|
+
...generated,
|
|
781
|
+
...instructions === void 0 ? {} : { instructions },
|
|
782
|
+
permission: {
|
|
783
|
+
external_directory: {
|
|
784
|
+
"*": "deny",
|
|
785
|
+
"/tmp/**": "allow"
|
|
786
|
+
},
|
|
787
|
+
...generated.permission ?? {}
|
|
788
|
+
},
|
|
789
|
+
provider: { [providerId]: {
|
|
790
|
+
npm: "@ai-sdk/openai-compatible",
|
|
791
|
+
name: providerId,
|
|
792
|
+
options: {
|
|
793
|
+
baseURL,
|
|
794
|
+
apiKey: `{env:${apiKeyEnv}}`,
|
|
795
|
+
...clientName === void 0 ? {} : { headers: { [ROUTER_CLIENT_HEADER]: clientName } }
|
|
796
|
+
},
|
|
797
|
+
models: { [modelId]: { name: modelId } }
|
|
798
|
+
} }
|
|
799
|
+
};
|
|
800
|
+
const invocation = harnessInvocation(HARNESS, invocationProfile(profile), task, { dangerouslySkipPermissions: true });
|
|
801
|
+
const model = `${providerId}/${modelId}`;
|
|
802
|
+
if (continueSession !== void 0 && !/^[A-Za-z0-9_-]+$/u.test(continueSession)) throw new ValidationError(`sharedBoxPlacement: session id ${continueSession} is not opencode's`);
|
|
803
|
+
return {
|
|
804
|
+
args: [
|
|
805
|
+
invocation.command,
|
|
806
|
+
...invocation.args,
|
|
807
|
+
...continueSession === void 0 ? [] : ["--session", continueSession],
|
|
808
|
+
"--format",
|
|
809
|
+
"json",
|
|
810
|
+
"-m",
|
|
811
|
+
model
|
|
812
|
+
],
|
|
813
|
+
config: JSON.stringify(config),
|
|
814
|
+
model
|
|
815
|
+
};
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Save the harness's own record of the session where the transcript capture reads it.
|
|
819
|
+
*
|
|
820
|
+
* Current opencode keeps sessions in a SQLite database, which the capture cannot carry as text.
|
|
821
|
+
* `opencode export` writes the same session as JSON.
|
|
822
|
+
*/
|
|
823
|
+
async function exportSession(run, collect, paths, sessionId) {
|
|
824
|
+
if (!SESSION_ID.test(sessionId)) return;
|
|
825
|
+
await collect(await run([
|
|
826
|
+
"/bin/sh",
|
|
827
|
+
"-c",
|
|
828
|
+
"mkdir -p \"$(dirname \"$2\")\" && exec opencode export \"$1\" > \"$2\"",
|
|
829
|
+
"export-session",
|
|
830
|
+
sessionId,
|
|
831
|
+
`${paths.home}/.local/share/opencode/export/${sessionId}.json`
|
|
832
|
+
], DEFAULT_EXEC_TIMEOUT_MS));
|
|
833
|
+
}
|
|
834
|
+
const SESSION_ID = /^[A-Za-z0-9_-]+$/u;
|
|
835
|
+
/** A subagent's session id in a `task` result (`<task id="ses_…">`) or failure (`task_id: ses_…`). */
|
|
836
|
+
const TASK_SESSION = /(?:<task id="|task_id: )(ses_[A-Za-z0-9]+)/u;
|
|
837
|
+
/**
|
|
838
|
+
* The session a harness subagent ran in, named by its parent's `task` tool part.
|
|
839
|
+
*
|
|
840
|
+
* opencode starts each subagent in a child session and refuses a nested one unless the
|
|
841
|
+
* configuration raises `subagent_depth` above 1, so under the default the parent's parts name
|
|
842
|
+
* every subagent session. A running part carries the id in `state.metadata.sessionId`; a finished
|
|
843
|
+
* one also carries it in its output, and a failed one in its error.
|
|
844
|
+
*/
|
|
845
|
+
function subagentSessionOf(part) {
|
|
846
|
+
if (part.type !== "tool" || part.tool !== "task") return void 0;
|
|
847
|
+
const state = part.state;
|
|
848
|
+
if (state === void 0 || state === null || typeof state !== "object") return void 0;
|
|
849
|
+
const { metadata, output, error } = state;
|
|
850
|
+
const named = metadata !== null && typeof metadata === "object" ? metadata.sessionId : void 0;
|
|
851
|
+
if (typeof named === "string" && SESSION_ID.test(named)) return named;
|
|
852
|
+
for (const text of [output, error]) {
|
|
853
|
+
const match = typeof text === "string" ? TASK_SESSION.exec(text) : null;
|
|
854
|
+
if (match !== null) return match[1];
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Translate opencode's JSON event lines into events the provider executor reads.
|
|
859
|
+
*
|
|
860
|
+
* Every part becomes `message.part.updated`, which is how the sidecar forwards opencode parts, so
|
|
861
|
+
* the existing trace and progress readers decode them unchanged. Each `step-finish` part also
|
|
862
|
+
* becomes one canonical `llm_call` receipt with its token counts. opencode reports no dollar
|
|
863
|
+
* receipt for a custom provider, so every receipt says the cost is unknown and Runtime prices the
|
|
864
|
+
* tokens as an estimate instead of a bill.
|
|
865
|
+
*/
|
|
866
|
+
function createOpencodeEventParser(model) {
|
|
867
|
+
let sessionId;
|
|
868
|
+
let finalMessageId;
|
|
869
|
+
const textByMessage = /* @__PURE__ */ new Map();
|
|
870
|
+
const subagentSessions = /* @__PURE__ */ new Set();
|
|
871
|
+
const errors = [];
|
|
872
|
+
let malformed = 0;
|
|
873
|
+
let spentTokens = false;
|
|
874
|
+
const failureOf = (exitCode, stderr) => {
|
|
875
|
+
if (exitCode !== 0) {
|
|
876
|
+
const said = [...errors, stderr.trim().slice(-1e3)].filter((text) => text.length > 0);
|
|
877
|
+
return `opencode exited ${exitCode}${said.length > 0 ? `: ${said.join("; ")}` : ""}`;
|
|
878
|
+
}
|
|
879
|
+
return errors.length > 0 && finalMessageId === void 0 ? errors.join("; ") : void 0;
|
|
880
|
+
};
|
|
881
|
+
return {
|
|
882
|
+
sessionId: () => sessionId,
|
|
883
|
+
subagentSessionIds: () => [...subagentSessions],
|
|
884
|
+
/** Why this run failed, or `undefined` when it answered. */
|
|
885
|
+
failure: failureOf,
|
|
886
|
+
/** Whether this run spent any model tokens before it ended. */
|
|
887
|
+
madeProgress: () => spentTokens,
|
|
888
|
+
*line(raw) {
|
|
889
|
+
let event;
|
|
890
|
+
try {
|
|
891
|
+
event = JSON.parse(raw);
|
|
892
|
+
} catch {
|
|
893
|
+
malformed += 1;
|
|
894
|
+
return;
|
|
895
|
+
}
|
|
896
|
+
if (typeof event.sessionID === "string") sessionId ??= event.sessionID;
|
|
897
|
+
if (event.type === "error") {
|
|
898
|
+
const message = errorMessage(event.error);
|
|
899
|
+
errors.push(message);
|
|
900
|
+
yield {
|
|
901
|
+
type: "error",
|
|
902
|
+
data: { message }
|
|
903
|
+
};
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
const part = event.part;
|
|
907
|
+
if (part === void 0 || part === null || typeof part !== "object") return;
|
|
908
|
+
const subagent = subagentSessionOf(part);
|
|
909
|
+
if (subagent !== void 0) subagentSessions.add(subagent);
|
|
910
|
+
yield {
|
|
911
|
+
type: "message.part.updated",
|
|
912
|
+
data: { part }
|
|
913
|
+
};
|
|
914
|
+
const messageId = typeof part.messageID === "string" ? part.messageID : void 0;
|
|
915
|
+
if (part.type === "text" && messageId !== void 0 && typeof part.text === "string") {
|
|
916
|
+
const texts = textByMessage.get(messageId) ?? [];
|
|
917
|
+
texts.push(part.text);
|
|
918
|
+
textByMessage.set(messageId, texts);
|
|
919
|
+
}
|
|
920
|
+
if (part.type === "step-finish") {
|
|
921
|
+
if (messageId !== void 0) finalMessageId = messageId;
|
|
922
|
+
const receipt = stepReceipt(part, model);
|
|
923
|
+
if (receipt !== void 0) {
|
|
924
|
+
const data = receipt.data;
|
|
925
|
+
if ((data?.tokensIn ?? 0) > 0 || (data?.tokensOut ?? 0) > 0) spentTokens = true;
|
|
926
|
+
yield receipt;
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
},
|
|
930
|
+
/** The turn's terminal frames, from its last run. `runs` counts the runs of the session. */
|
|
931
|
+
*finish(exitCode, stderr, runs = 1) {
|
|
932
|
+
const finalText = (finalMessageId === void 0 ? void 0 : textByMessage.get(finalMessageId)?.join("")) ?? "";
|
|
933
|
+
const failure = failureOf(exitCode, stderr);
|
|
934
|
+
const counted = {
|
|
935
|
+
...malformed > 0 ? { malformedLines: malformed } : {},
|
|
936
|
+
...runs > 1 ? { sessionRuns: runs } : {}
|
|
937
|
+
};
|
|
938
|
+
if (failure !== void 0) {
|
|
939
|
+
yield {
|
|
940
|
+
type: "result",
|
|
941
|
+
data: {
|
|
942
|
+
success: false,
|
|
943
|
+
status: "failed",
|
|
944
|
+
error: failure,
|
|
945
|
+
finalText,
|
|
946
|
+
...counted
|
|
947
|
+
}
|
|
948
|
+
};
|
|
949
|
+
yield {
|
|
950
|
+
type: "done",
|
|
951
|
+
data: { outcome: { type: "failed" } }
|
|
952
|
+
};
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
yield {
|
|
956
|
+
type: "result",
|
|
957
|
+
data: {
|
|
958
|
+
success: true,
|
|
959
|
+
finalText,
|
|
960
|
+
...counted
|
|
961
|
+
}
|
|
962
|
+
};
|
|
963
|
+
yield {
|
|
964
|
+
type: "done",
|
|
965
|
+
data: { outcome: { type: "completed" } }
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
};
|
|
969
|
+
}
|
|
970
|
+
function stepReceipt(part, model) {
|
|
971
|
+
const tokens = part.tokens;
|
|
972
|
+
if (tokens === void 0 || tokens === null || typeof tokens !== "object") return void 0;
|
|
973
|
+
const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : void 0;
|
|
974
|
+
const cache = tokens.cache ?? {};
|
|
975
|
+
const input = count(tokens.input);
|
|
976
|
+
const output = count(tokens.output);
|
|
977
|
+
const reasoning = count(tokens.reasoning) ?? 0;
|
|
978
|
+
const cacheRead = count(cache.read) ?? 0;
|
|
979
|
+
const cacheWrite = count(cache.write) ?? 0;
|
|
980
|
+
const known = input !== void 0 && output !== void 0;
|
|
981
|
+
return {
|
|
982
|
+
type: "llm_call",
|
|
983
|
+
...typeof part.id === "string" ? { id: part.id } : {},
|
|
984
|
+
usageMode: "delta",
|
|
985
|
+
...known ? { usage: {
|
|
986
|
+
inputTokens: input + cacheRead + cacheWrite,
|
|
987
|
+
outputTokens: output + reasoning,
|
|
988
|
+
...reasoning > 0 ? { reasoningTokens: reasoning } : {}
|
|
989
|
+
} } : {},
|
|
990
|
+
data: {
|
|
991
|
+
usageMode: "delta",
|
|
992
|
+
model,
|
|
993
|
+
...known ? {
|
|
994
|
+
tokensIn: input + cacheRead + cacheWrite,
|
|
995
|
+
tokensOut: output + reasoning,
|
|
996
|
+
...reasoning > 0 ? { reasoningTokens: reasoning } : {}
|
|
997
|
+
} : { tokensKnown: false },
|
|
998
|
+
costKnown: false
|
|
999
|
+
}
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
1002
|
+
function errorMessage(error) {
|
|
1003
|
+
if (typeof error === "string") return error;
|
|
1004
|
+
if (error !== null && typeof error === "object") {
|
|
1005
|
+
const record = error;
|
|
1006
|
+
const data = record.data;
|
|
1007
|
+
const text = typeof data?.message === "string" ? data.message : typeof record.message === "string" ? record.message : typeof record.name === "string" ? record.name : "opencode reported an error";
|
|
1008
|
+
const status = data?.statusCode;
|
|
1009
|
+
return typeof status === "number" && Number.isSafeInteger(status) ? `${text} (status code ${status})` : text;
|
|
1010
|
+
}
|
|
1011
|
+
return "opencode reported an error";
|
|
1012
|
+
}
|
|
1013
|
+
//#endregion
|
|
1014
|
+
//#region src/runtime/shared-box.ts
|
|
1015
|
+
/**
|
|
1016
|
+
* Shared-box placement: many harness workers in one Sandbox box, each as its own process.
|
|
1017
|
+
*
|
|
1018
|
+
* The default provider placement gives every worker its own Sandbox box. A box costs a create
|
|
1019
|
+
* call, a sidecar, an egress proxy, a preview link and a memory reservation sized for the box,
|
|
1020
|
+
* while one opencode worker uses about half a gigabyte. This module packs workers into a small
|
|
1021
|
+
* pool of boxes instead. Each worker runs `opencode run` as a separate process with its own
|
|
1022
|
+
* working directory and its own HOME, so each worker has its own harness session, its own
|
|
1023
|
+
* materialized profile and its own transcript files.
|
|
1024
|
+
*
|
|
1025
|
+
* Why processes and not sidecar sessions: the sidecar serves every session in one box from one
|
|
1026
|
+
* workspace and writes each session's profile into the same files. Measured 2026-09-24 on
|
|
1027
|
+
* sandbox-ff52a989e7c9: eight concurrent sessions with different instructions, and seven of eight
|
|
1028
|
+
* answered with another session's instructions. A process with its own directory, its own HOME and
|
|
1029
|
+
* `OPENCODE_DISABLE_PROJECT_CONFIG` reads only its own configuration: sixteen of sixteen concurrent
|
|
1030
|
+
* workers answered with their own instructions and wrote only into their own directory.
|
|
1031
|
+
*
|
|
1032
|
+
* The boundary is cooperative, not a security boundary. Every worker runs as the same Linux user,
|
|
1033
|
+
* so a worker can read a co-tenant's files through its shell. A profile that writes a repository,
|
|
1034
|
+
* runs untrusted code, holds a credential of its own, or must stay blind to its siblings keeps a
|
|
1035
|
+
* dedicated box: {@link SharedBoxPlacement.refusal} names the profile features this placement
|
|
1036
|
+
* cannot carry, and Runtime then uses the dedicated provider.
|
|
1037
|
+
*
|
|
1038
|
+
* A worker whose model provider refuses a turn for capacity keeps its process's session and
|
|
1039
|
+
* directory, pauses by the driver's rule, and continues the same opencode session, the way a
|
|
1040
|
+
* dedicated leaf does (`ProviderExecutorOptions.unavailablePause`). Measured 2026-09-24: 15 of 23
|
|
1041
|
+
* down children of play anomaly-referee-v3d ended on `provider_quota_exhausted`, and opencode
|
|
1042
|
+
* itself gives up on a refused model after about 70 seconds of its own retries.
|
|
1043
|
+
*
|
|
1044
|
+
* The workers of one box share the box's router key. Each worker names itself to the router with
|
|
1045
|
+
* the `x-tangle-client` header, which the router stores as `clientName` on every usage row, so a
|
|
1046
|
+
* keeper can price one worker from the key's per-client rows (`tangle-admin router-spend --key
|
|
1047
|
+
* <box key>`). A request without the header stays charged to the box.
|
|
1048
|
+
*
|
|
1049
|
+
* `workersPerBox` is the one structural cap, and it protects the box. A Tangle box has a fixed
|
|
1050
|
+
* 512-task pids limit. An idle box uses about 118 tasks and one opencode worker about 34, so
|
|
1051
|
+
* twelve workers reach the limit. At that limit the sidecar's own process spawn fails with EAGAIN
|
|
1052
|
+
* and the sidecar restarts, which ends every worker in the box (measured on the same box: 28
|
|
1053
|
+
* workers crashed the sidecar twice; 16 workers crashed it once).
|
|
1054
|
+
*/
|
|
1055
|
+
/**
|
|
1056
|
+
* The most workers one default box carries at once.
|
|
1057
|
+
*
|
|
1058
|
+
* It keeps a worker count that the 512-task pids limit of a Tangle box can hold with headroom
|
|
1059
|
+
* for the sidecar's own processes and the workers' tool shells: 118 idle tasks plus 8 workers at
|
|
1060
|
+
* about 34 tasks each is 390. Memory is not the binding limit: 8 workers used about 4 GB in a
|
|
1061
|
+
* 16 GB box. Raise it only for a box whose pids limit was raised with it.
|
|
1062
|
+
*/
|
|
1063
|
+
const DEFAULT_SHARED_BOX_WORKERS = 8;
|
|
1064
|
+
/** The box shape the default placement creates: memory for 8 workers at about 0.5 GB each plus
|
|
1065
|
+
* the sidecar, and 4 cores, because the workers wait on the model most of the time. */
|
|
1066
|
+
const DEFAULT_SHARED_BOX_RESOURCES = Object.freeze({
|
|
1067
|
+
cpuCores: 4,
|
|
1068
|
+
memoryMB: 8192
|
|
1069
|
+
});
|
|
1070
|
+
const DEFAULT_WORKER_ROOT = "/home/agent/workers";
|
|
1071
|
+
/**
|
|
1072
|
+
* Place accepted workers as processes in a pool of shared Sandbox boxes.
|
|
1073
|
+
*
|
|
1074
|
+
* The pool fills a box before it creates the next one and deletes a box when its last worker
|
|
1075
|
+
* releases it. Runtime reaches this placement through `createExecutor({ backend: 'provider',
|
|
1076
|
+
* shared })`: a profile that {@link SharedBoxPlacement.refusal} accepts runs here, and every other
|
|
1077
|
+
* profile keeps the dedicated provider.
|
|
1078
|
+
*/
|
|
1079
|
+
function sharedBoxPlacement(options) {
|
|
1080
|
+
const workersPerBox = options.workersPerBox ?? 8;
|
|
1081
|
+
if (!Number.isSafeInteger(workersPerBox) || workersPerBox < 1) throw new ValidationError("sharedBoxPlacement: workersPerBox must be a positive integer");
|
|
1082
|
+
const workerRoot = options.workerRoot ?? DEFAULT_WORKER_ROOT;
|
|
1083
|
+
if (!workerRoot.startsWith("/") || isUnsafeRelativePath(workerRoot.slice(1))) throw new ValidationError("sharedBoxPlacement: workerRoot must be an absolute, normalized path");
|
|
1084
|
+
const boxOptions = {
|
|
1085
|
+
resources: { ...DEFAULT_SHARED_BOX_RESOURCES },
|
|
1086
|
+
...options.box ?? {}
|
|
1087
|
+
};
|
|
1088
|
+
if (boxOptions.backend !== void 0) throw new ValidationError("sharedBoxPlacement: box.backend is owned by Runtime");
|
|
1089
|
+
const identity = {
|
|
1090
|
+
id: "shared-box",
|
|
1091
|
+
digest: canonicalCandidateDigest({
|
|
1092
|
+
kind: "shared-box-placement.v1",
|
|
1093
|
+
harness: HARNESS,
|
|
1094
|
+
workersPerBox,
|
|
1095
|
+
workerRoot,
|
|
1096
|
+
box: publicBoxOptions(boxOptions)
|
|
1097
|
+
})
|
|
1098
|
+
};
|
|
1099
|
+
const retryDelayMs = options.retryDelayMs ?? 1e3;
|
|
1100
|
+
const pool = createBoxPool(options.client, boxOptions, workersPerBox, retryDelayMs);
|
|
1101
|
+
const providerFor = (worker = {}) => {
|
|
1102
|
+
const clientName = worker.nodeId === void 0 ? void 0 : sharedWorkerClientName(worker.nodeId);
|
|
1103
|
+
return {
|
|
1104
|
+
name: SHARED_PROVIDER_NAME,
|
|
1105
|
+
capabilities: sharedBoxCapabilities,
|
|
1106
|
+
async create(input) {
|
|
1107
|
+
const profile = parseSharedWorkerProfile(input);
|
|
1108
|
+
const refusal = sharedBoxRefusal(profile);
|
|
1109
|
+
if (refusal !== void 0) throw new ValidationError(`sharedBoxPlacement: ${refusal}`);
|
|
1110
|
+
assertWorkerCreateInput(input);
|
|
1111
|
+
input.signal?.throwIfAborted();
|
|
1112
|
+
const lease = await pool.acquire();
|
|
1113
|
+
try {
|
|
1114
|
+
input.signal?.throwIfAborted();
|
|
1115
|
+
return await createWorker(lease, workerRoot, profile, {
|
|
1116
|
+
retryDelayMs,
|
|
1117
|
+
clientName,
|
|
1118
|
+
name: input.name,
|
|
1119
|
+
metadata: input.metadata,
|
|
1120
|
+
unavailablePause: options.unavailablePause ?? {}
|
|
1121
|
+
});
|
|
1122
|
+
} catch (error) {
|
|
1123
|
+
await lease.release();
|
|
1124
|
+
throw error;
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
};
|
|
1128
|
+
};
|
|
1129
|
+
return {
|
|
1130
|
+
providerFor,
|
|
1131
|
+
identity,
|
|
1132
|
+
refusal: sharedBoxRefusal,
|
|
1133
|
+
stats: () => pool.stats(),
|
|
1134
|
+
close: () => pool.close()
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
function publicBoxOptions(box) {
|
|
1138
|
+
const { env, secrets, ...rest } = box;
|
|
1139
|
+
return {
|
|
1140
|
+
...rest,
|
|
1141
|
+
...env === void 0 ? {} : { environmentVariableNames: Object.keys(env).sort() },
|
|
1142
|
+
...secrets === void 0 ? {} : { secrets }
|
|
1143
|
+
};
|
|
1144
|
+
}
|
|
1145
|
+
function createBoxPool(client, boxOptions, workersPerBox, retryDelayMs) {
|
|
1146
|
+
const poolId = randomUUID();
|
|
1147
|
+
const boxes = [];
|
|
1148
|
+
const history = [];
|
|
1149
|
+
let seq = 0;
|
|
1150
|
+
let placed = 0;
|
|
1151
|
+
let peak = 0;
|
|
1152
|
+
let closing = false;
|
|
1153
|
+
const retire = async (entry) => {
|
|
1154
|
+
if (entry.retired) return void 0;
|
|
1155
|
+
entry.retired = true;
|
|
1156
|
+
const index = boxes.indexOf(entry);
|
|
1157
|
+
if (index >= 0) boxes.splice(index, 1);
|
|
1158
|
+
let handle;
|
|
1159
|
+
try {
|
|
1160
|
+
handle = await entry.ready;
|
|
1161
|
+
} catch {
|
|
1162
|
+
return;
|
|
1163
|
+
}
|
|
1164
|
+
try {
|
|
1165
|
+
await transientRetry(() => handle.delete(), { delayMs: retryDelayMs });
|
|
1166
|
+
entry.deleted = true;
|
|
1167
|
+
return {
|
|
1168
|
+
boxId: handle.id,
|
|
1169
|
+
deleted: true
|
|
1170
|
+
};
|
|
1171
|
+
} catch (error) {
|
|
1172
|
+
return {
|
|
1173
|
+
boxId: handle.id,
|
|
1174
|
+
deleted: false,
|
|
1175
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1178
|
+
};
|
|
1179
|
+
return {
|
|
1180
|
+
async acquire() {
|
|
1181
|
+
if (closing) throw new ValidationError("sharedBoxPlacement: the placement is closed");
|
|
1182
|
+
let entry = boxes.find((candidate) => !candidate.retired && candidate.workers < workersPerBox);
|
|
1183
|
+
if (entry === void 0) {
|
|
1184
|
+
const boxSeq = seq++;
|
|
1185
|
+
const request = {
|
|
1186
|
+
...boxOptions,
|
|
1187
|
+
name: boxOptions.name ?? `shared-box-${poolId.slice(0, 8)}-${boxSeq}`,
|
|
1188
|
+
idempotencyKey: `shared-box-${poolId}-${boxSeq}`,
|
|
1189
|
+
backend: {
|
|
1190
|
+
type: HARNESS,
|
|
1191
|
+
profile: {
|
|
1192
|
+
name: "shared-box",
|
|
1193
|
+
harness: HARNESS
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
};
|
|
1197
|
+
const requestedAt = Date.now();
|
|
1198
|
+
const created = {
|
|
1199
|
+
seq: boxSeq,
|
|
1200
|
+
ready: transientRetry(() => client.create(request), { delayMs: retryDelayMs }).then((box) => box),
|
|
1201
|
+
workers: 0,
|
|
1202
|
+
served: 0,
|
|
1203
|
+
retired: false,
|
|
1204
|
+
deleted: false
|
|
1205
|
+
};
|
|
1206
|
+
created.ready.then((box) => {
|
|
1207
|
+
created.id = box.id;
|
|
1208
|
+
created.createMs = Date.now() - requestedAt;
|
|
1209
|
+
}, () => {
|
|
1210
|
+
const index = boxes.indexOf(created);
|
|
1211
|
+
if (index >= 0) boxes.splice(index, 1);
|
|
1212
|
+
created.retired = true;
|
|
1213
|
+
});
|
|
1214
|
+
boxes.push(created);
|
|
1215
|
+
history.push(created);
|
|
1216
|
+
entry = created;
|
|
1217
|
+
}
|
|
1218
|
+
const leased = entry;
|
|
1219
|
+
leased.workers += 1;
|
|
1220
|
+
leased.served += 1;
|
|
1221
|
+
placed += 1;
|
|
1222
|
+
peak = Math.max(peak, leased.workers);
|
|
1223
|
+
let box;
|
|
1224
|
+
try {
|
|
1225
|
+
box = await leased.ready;
|
|
1226
|
+
} catch (error) {
|
|
1227
|
+
leased.workers -= 1;
|
|
1228
|
+
throw error;
|
|
1229
|
+
}
|
|
1230
|
+
let released = false;
|
|
1231
|
+
return {
|
|
1232
|
+
box,
|
|
1233
|
+
async release() {
|
|
1234
|
+
if (released) return;
|
|
1235
|
+
released = true;
|
|
1236
|
+
leased.workers -= 1;
|
|
1237
|
+
if (leased.workers === 0 && !closing) await retire(leased);
|
|
1238
|
+
}
|
|
1239
|
+
};
|
|
1240
|
+
},
|
|
1241
|
+
stats() {
|
|
1242
|
+
return {
|
|
1243
|
+
boxesCreated: history.length,
|
|
1244
|
+
boxesLive: history.filter((entry) => !entry.retired).length,
|
|
1245
|
+
workersPlaced: placed,
|
|
1246
|
+
workersLive: history.reduce((sum, entry) => sum + entry.workers, 0),
|
|
1247
|
+
peakWorkersPerBox: peak,
|
|
1248
|
+
boxes: history.filter((entry) => entry.id !== void 0).map((entry) => ({
|
|
1249
|
+
id: entry.id,
|
|
1250
|
+
workersServed: entry.served,
|
|
1251
|
+
deleted: entry.deleted,
|
|
1252
|
+
createMs: entry.createMs ?? 0
|
|
1253
|
+
}))
|
|
1254
|
+
};
|
|
1255
|
+
},
|
|
1256
|
+
async close() {
|
|
1257
|
+
closing = true;
|
|
1258
|
+
return (await Promise.all(history.map((entry) => retire(entry)))).filter((receipt) => receipt !== void 0);
|
|
1259
|
+
}
|
|
1260
|
+
};
|
|
1261
|
+
}
|
|
1262
|
+
//#endregion
|
|
1263
|
+
//#region src/improvement/search-executor.ts
|
|
1264
|
+
/**
|
|
1265
|
+
* The search kernel's executor port for cells Runtime runs.
|
|
1266
|
+
*
|
|
1267
|
+
* A lane is a pool of execution slots with one cost rule: a shared box, dedicated
|
|
1268
|
+
* environments, subscription seats, or the router. The kernel places each cell on a lane and
|
|
1269
|
+
* bounds each lane's running cells by its capacity. This executor adds what one search cannot
|
|
1270
|
+
* see or enforce on its own:
|
|
1271
|
+
*
|
|
1272
|
+
* - Fleet slots. A `WorkerSlots` allocator shared by concurrent searches bounds working cells
|
|
1273
|
+
* across all of them. A cell waits for a slot, and the wait is its `queueMs`.
|
|
1274
|
+
* - Fleet dollars. On a hard lane a cell holds its enforced maximum as a `BudgetPool` ticket
|
|
1275
|
+
* before it starts, waits while open tickets could return enough, and settles the ticket
|
|
1276
|
+
* with its measured spend. Estimate lanes take no ticket: they cannot bound a cell, and an
|
|
1277
|
+
* unknown cost would close a capped pool for every search that shares it.
|
|
1278
|
+
* - Hard caps. On a hard lane every paid call of the cell, the agent's and the judges', holds
|
|
1279
|
+
* its maximum against the lane's `cellUsd` and is refused when it would pass it, so the
|
|
1280
|
+
* kernel's hard reservation is a bound and not a hope.
|
|
1281
|
+
* - Adoption. A finished attempt is written under its run id before the kernel sees it, so a
|
|
1282
|
+
* kernel that restarts adopts the attempt instead of running it again. The run id is stable
|
|
1283
|
+
* across restarts; an agent that runs remote work keys it by `ctx.search.runId`, so a lost
|
|
1284
|
+
* response is fetched again rather than executed again.
|
|
1285
|
+
* - Accounting. Every paid call is tagged with its search and attempt, and an attempt's cost is
|
|
1286
|
+
* every call tagged with its run id, including calls a process that ended made for it. A
|
|
1287
|
+
* reopened search settles those interrupted calls first: from the provider through the lane's
|
|
1288
|
+
* `recoverReceipt`, else with an unknown cost, never as free.
|
|
1289
|
+
* - Environment faults. A `SearchEnvironmentFault`, or a platform failure a repeated request
|
|
1290
|
+
* can change, settles `errored` and retryable, and the kernel runs a fresh attempt.
|
|
1291
|
+
* - Traces. Each attempt is one trace whose root span carries `agent.branch.id` = the node id.
|
|
1292
|
+
* The spans the attempt's exporter wrote and dropped go into the cell's `traceRef`.
|
|
1293
|
+
*/
|
|
1294
|
+
function laneReceipts(options) {
|
|
1295
|
+
return options.recoverReceipt ? { recoverReceipt: options.recoverReceipt } : {};
|
|
1296
|
+
}
|
|
1297
|
+
const acceptEveryProfile = () => void 0;
|
|
1298
|
+
function laneName(name, fallback) {
|
|
1299
|
+
const resolved = name ?? fallback;
|
|
1300
|
+
if (!/^[A-Za-z0-9._-]{1,64}$/u.test(resolved)) throw new ConfigError(`search lane name '${resolved}' must be 1-64 of [A-Za-z0-9._-]`);
|
|
1301
|
+
return resolved;
|
|
1302
|
+
}
|
|
1303
|
+
function positiveInteger(value, name) {
|
|
1304
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new ConfigError(`${name} must be a positive integer`);
|
|
1305
|
+
return value;
|
|
1306
|
+
}
|
|
1307
|
+
function dollars(value, name) {
|
|
1308
|
+
if (!Number.isFinite(value) || value < 0) throw new ConfigError(`${name} must be a finite non-negative number of dollars`);
|
|
1309
|
+
return value;
|
|
1310
|
+
}
|
|
1311
|
+
/**
|
|
1312
|
+
* Cells as workers packed into shared Sandbox boxes, `workersPerBox` to a box.
|
|
1313
|
+
*
|
|
1314
|
+
* Capacity is `boxes × workersPerBox`. The workers of one box share its router key, which caps
|
|
1315
|
+
* no single worker, so the lane holds an estimate (`cellUsd` is the prior until 20 cells settle).
|
|
1316
|
+
* A profile the placement refuses (a repository writer, a replaced system prompt, MCP servers,
|
|
1317
|
+
* hooks, subagents, tool grants) needs another lane. The agent reaches the box through
|
|
1318
|
+
* `lane.placement.providerFor({ nodeId: ctx.search.runId })`, which names the attempt to the
|
|
1319
|
+
* router on every call.
|
|
1320
|
+
*/
|
|
1321
|
+
function sharedBoxLane(options) {
|
|
1322
|
+
const { name, boxes, cellUsd, recoverReceipt, ...placementOptions } = options;
|
|
1323
|
+
const placement = sharedBoxPlacement(placementOptions);
|
|
1324
|
+
const workersPerBox = placementOptions.workersPerBox ?? 8;
|
|
1325
|
+
return Object.freeze({
|
|
1326
|
+
name: laneName(name, "shared-box"),
|
|
1327
|
+
kind: "shared-box",
|
|
1328
|
+
capacity: positiveInteger(boxes, "sharedBoxLane(): boxes") * workersPerBox,
|
|
1329
|
+
costCap: "estimate",
|
|
1330
|
+
cellUsd: dollars(cellUsd, "sharedBoxLane(): cellUsd"),
|
|
1331
|
+
refusal: (profile) => placement.refusal(profile),
|
|
1332
|
+
...laneReceipts({ recoverReceipt }),
|
|
1333
|
+
placement
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
/**
|
|
1337
|
+
* Cells that each run in their own environment, `capacity` at once. The lane cannot bound a
|
|
1338
|
+
* cell's dollars, so it holds an estimate: `cellUsd` until 20 cells settle, then 1.5 × the p99
|
|
1339
|
+
* of its settled cells.
|
|
1340
|
+
*/
|
|
1341
|
+
function dedicatedLane(options) {
|
|
1342
|
+
return Object.freeze({
|
|
1343
|
+
name: laneName(options.name, "dedicated"),
|
|
1344
|
+
kind: "dedicated",
|
|
1345
|
+
capacity: positiveInteger(options.capacity, "dedicatedLane(): capacity"),
|
|
1346
|
+
costCap: "estimate",
|
|
1347
|
+
cellUsd: dollars(options.cellUsd, "dedicatedLane(): cellUsd"),
|
|
1348
|
+
refusal: acceptEveryProfile,
|
|
1349
|
+
...laneReceipts(options)
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
1352
|
+
/**
|
|
1353
|
+
* Cells on subscription seats, one cell a seat. A seat charges no dollar per call, so its own
|
|
1354
|
+
* work costs $0 and reports no tokens. `cellUsd` bounds the cell's priced calls, such as its
|
|
1355
|
+
* judges; every one of them must declare a maximum, and the lane refuses a call past it.
|
|
1356
|
+
*/
|
|
1357
|
+
function subscriptionLane(options) {
|
|
1358
|
+
return Object.freeze({
|
|
1359
|
+
name: laneName(options.name, "subscription"),
|
|
1360
|
+
kind: "subscription",
|
|
1361
|
+
capacity: positiveInteger(options.seats, "subscriptionLane(): seats"),
|
|
1362
|
+
costCap: "hard",
|
|
1363
|
+
cellUsd: dollars(options.cellUsd ?? 0, "subscriptionLane(): cellUsd"),
|
|
1364
|
+
refusal: acceptEveryProfile,
|
|
1365
|
+
...laneReceipts(options)
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
/**
|
|
1369
|
+
* Cells whose model calls are priced router calls, `capacity` at once. Every paid call of a
|
|
1370
|
+
* cell must declare its maximum charge, and the lane refuses a call that would take the cell
|
|
1371
|
+
* past `cellUsd`, so `cellUsd` is each cell's hard maximum.
|
|
1372
|
+
*/
|
|
1373
|
+
function routerLane(options) {
|
|
1374
|
+
return Object.freeze({
|
|
1375
|
+
name: laneName(options.name, "router"),
|
|
1376
|
+
kind: "router",
|
|
1377
|
+
capacity: positiveInteger(options.capacity, "routerLane(): capacity"),
|
|
1378
|
+
costCap: "hard",
|
|
1379
|
+
cellUsd: dollars(options.cellUsd, "routerLane(): cellUsd"),
|
|
1380
|
+
refusal: acceptEveryProfile,
|
|
1381
|
+
...laneReceipts(options)
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
/** Check a lane list: at least one lane, unique names, a hard lane's cap known. */
|
|
1385
|
+
function assertSearchLanes(lanes, context) {
|
|
1386
|
+
if (!Array.isArray(lanes) || lanes.length === 0) throw new ConfigError(`${context}: lanes needs at least one lane`);
|
|
1387
|
+
const names = /* @__PURE__ */ new Set();
|
|
1388
|
+
for (const lane of lanes) {
|
|
1389
|
+
if (!lane || typeof lane.refusal !== "function" || typeof lane.kind !== "string") throw new ConfigError(`${context}: every lane comes from sharedBoxLane, dedicatedLane, subscriptionLane or routerLane`);
|
|
1390
|
+
if (names.has(lane.name)) throw new ConfigError(`${context}: lane '${lane.name}' appears twice`);
|
|
1391
|
+
names.add(lane.name);
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* The environment, not the candidate, ended the attempt: a box that died, a platform that
|
|
1396
|
+
* refused the run, a quota. The attempt settles `errored` and retryable, and the kernel runs a
|
|
1397
|
+
* fresh attempt up to its `maxAttempts`. Throw anything else for the agent's own failure.
|
|
1398
|
+
*/
|
|
1399
|
+
var SearchEnvironmentFault = class extends Error {
|
|
1400
|
+
constructor(message, options) {
|
|
1401
|
+
super(message, options);
|
|
1402
|
+
this.name = "SearchEnvironmentFault";
|
|
1403
|
+
}
|
|
1404
|
+
};
|
|
1405
|
+
/** Whether an attempt's error is the environment's: a `SearchEnvironmentFault`, or a platform,
|
|
1406
|
+
* gateway or network failure that a repeated request can change. */
|
|
1407
|
+
function isSearchEnvironmentFault(error) {
|
|
1408
|
+
return error instanceof SearchEnvironmentFault || isTransientSandboxFailure(error);
|
|
1409
|
+
}
|
|
1410
|
+
const USD_TOLERANCE = 1e-9;
|
|
1411
|
+
/** The kernel's executor port over Runtime lanes. */
|
|
1412
|
+
function searchExecutor(options) {
|
|
1413
|
+
assertSearchLanes(options.lanes, "searchExecutor()");
|
|
1414
|
+
const lanes = new Map(options.lanes.map((lane) => [lane.name, lane]));
|
|
1415
|
+
const placed = new Map(options.lanes.map((lane) => [lane.name, 0]));
|
|
1416
|
+
const slots = options.workerSlots ? openSlotGroup(options.workerSlots) : null;
|
|
1417
|
+
const attempts = attemptStore(options.storage, join(options.dir, "attempts"));
|
|
1418
|
+
const tracing = searchTracing(options);
|
|
1419
|
+
const laneOf = (work) => {
|
|
1420
|
+
const lane = lanes.get(work.lane);
|
|
1421
|
+
if (!lane) throw new Error(`search cell ${work.cellId} was placed on unknown lane '${work.lane}'`);
|
|
1422
|
+
return lane;
|
|
1423
|
+
};
|
|
1424
|
+
return {
|
|
1425
|
+
lanes: () => options.lanes,
|
|
1426
|
+
place(cell) {
|
|
1427
|
+
const profile = options.profileOf(cell.nodeId);
|
|
1428
|
+
let chosen;
|
|
1429
|
+
for (const lane of options.lanes) {
|
|
1430
|
+
if (profile && lane.refusal(profile) !== void 0) continue;
|
|
1431
|
+
if (chosen === void 0 || placed.get(lane.name) / lane.capacity < placed.get(chosen.name) / chosen.capacity) chosen = lane;
|
|
1432
|
+
}
|
|
1433
|
+
if (!chosen) throw new Error(`no search lane can run node ${cell.nodeId}; admission should have refused it`);
|
|
1434
|
+
placed.set(chosen.name, placed.get(chosen.name) + 1);
|
|
1435
|
+
return chosen.name;
|
|
1436
|
+
},
|
|
1437
|
+
async adopt(work) {
|
|
1438
|
+
return attempts.read(work.runId);
|
|
1439
|
+
},
|
|
1440
|
+
async run(work) {
|
|
1441
|
+
const lane = laneOf(work);
|
|
1442
|
+
const queuedAt = Date.now();
|
|
1443
|
+
let hold;
|
|
1444
|
+
try {
|
|
1445
|
+
hold = await holdFleet(lane, work, options.budgetPool, slots);
|
|
1446
|
+
} catch (error) {
|
|
1447
|
+
if (work.signal.aborted) return interrupted(work, lane, options.identity, error);
|
|
1448
|
+
throw error;
|
|
1449
|
+
}
|
|
1450
|
+
const queueMs = Date.now() - queuedAt;
|
|
1451
|
+
const trace = tracing.open(work, lane);
|
|
1452
|
+
let result;
|
|
1453
|
+
try {
|
|
1454
|
+
const context = Object.freeze({
|
|
1455
|
+
searchId: work.searchId,
|
|
1456
|
+
cellId: work.cellId,
|
|
1457
|
+
nodeId: work.nodeId,
|
|
1458
|
+
taskId: work.taskId,
|
|
1459
|
+
split: work.split,
|
|
1460
|
+
stage: work.stage,
|
|
1461
|
+
attempt: work.attempt,
|
|
1462
|
+
runId: work.runId,
|
|
1463
|
+
lane,
|
|
1464
|
+
trace: trace.ids
|
|
1465
|
+
});
|
|
1466
|
+
const tags = searchCallTags(work.searchId, {
|
|
1467
|
+
run: work.runId,
|
|
1468
|
+
lane: lane.name
|
|
1469
|
+
});
|
|
1470
|
+
const { recordCell, ...measured } = await options.runAttempt({
|
|
1471
|
+
work,
|
|
1472
|
+
lane,
|
|
1473
|
+
context,
|
|
1474
|
+
costLedger: attemptLedger(options.costLedger, lane, tags),
|
|
1475
|
+
span: trace.span
|
|
1476
|
+
});
|
|
1477
|
+
const traceRef = await trace.finish(measured);
|
|
1478
|
+
const execRunId = measured.traceRef && "execRunId" in measured.traceRef ? measured.traceRef.execRunId : null;
|
|
1479
|
+
result = {
|
|
1480
|
+
...measured,
|
|
1481
|
+
accounting: attemptAccounting(options.costLedger, lane, tags),
|
|
1482
|
+
queueMs,
|
|
1483
|
+
placement: {
|
|
1484
|
+
lane: lane.name,
|
|
1485
|
+
boxId: measured.placement?.boxId ?? null
|
|
1486
|
+
},
|
|
1487
|
+
traceRef: execRunId && "traceId" in traceRef ? {
|
|
1488
|
+
...traceRef,
|
|
1489
|
+
execRunId
|
|
1490
|
+
} : traceRef
|
|
1491
|
+
};
|
|
1492
|
+
if (recordCell && result.outcome.status !== "errored") result.runRecord = await recordCell(result);
|
|
1493
|
+
} finally {
|
|
1494
|
+
hold.release(result);
|
|
1495
|
+
}
|
|
1496
|
+
if (!work.signal.aborted) attempts.write(work.runId, result);
|
|
1497
|
+
return result;
|
|
1498
|
+
}
|
|
1499
|
+
};
|
|
1500
|
+
}
|
|
1501
|
+
function interrupted(work, lane, identity, error) {
|
|
1502
|
+
return {
|
|
1503
|
+
outcome: {
|
|
1504
|
+
status: "errored",
|
|
1505
|
+
metrics: {},
|
|
1506
|
+
error: {
|
|
1507
|
+
code: "interrupted",
|
|
1508
|
+
message: `search cell ${work.runId} was interrupted before it ran: ${error instanceof Error ? error.message : String(error)}`,
|
|
1509
|
+
retryable: true
|
|
1510
|
+
}
|
|
1511
|
+
},
|
|
1512
|
+
accounting: {
|
|
1513
|
+
tokens: {
|
|
1514
|
+
status: "known",
|
|
1515
|
+
inputTokens: 0,
|
|
1516
|
+
outputTokens: 0,
|
|
1517
|
+
cachedTokens: 0
|
|
1518
|
+
},
|
|
1519
|
+
cost: {
|
|
1520
|
+
status: "known",
|
|
1521
|
+
usd: 0,
|
|
1522
|
+
source: "free"
|
|
1523
|
+
}
|
|
1524
|
+
},
|
|
1525
|
+
identity,
|
|
1526
|
+
placement: {
|
|
1527
|
+
lane: lane.name,
|
|
1528
|
+
boxId: null
|
|
1529
|
+
},
|
|
1530
|
+
traceRef: { unknown: "the attempt never started" }
|
|
1531
|
+
};
|
|
1532
|
+
}
|
|
1533
|
+
/** Take the attempt's fleet dollars, then its fleet slot. A cell waiting for budget holds no
|
|
1534
|
+
* slot, so it cannot hold a slot the cells it waits on need. */
|
|
1535
|
+
async function holdFleet(lane, work, pool, slots) {
|
|
1536
|
+
let ticket;
|
|
1537
|
+
if (pool && lane.costCap === "hard" && lane.cellUsd > 0) {
|
|
1538
|
+
const reserved = pool.reserve({
|
|
1539
|
+
maxUsd: lane.cellUsd,
|
|
1540
|
+
maxTokens: 0,
|
|
1541
|
+
maxIterations: 0
|
|
1542
|
+
}, {
|
|
1543
|
+
label: work.runId,
|
|
1544
|
+
stage: "admitted"
|
|
1545
|
+
}, { wait: true });
|
|
1546
|
+
if (!reserved.ok) throw new Error(`the fleet budget pool refused search cell ${work.runId} on lane ${lane.name}: ${reserved.reason}`);
|
|
1547
|
+
ticket = reserved.ticket;
|
|
1548
|
+
if (reserved.granted) try {
|
|
1549
|
+
await abortable(reserved.granted, work.signal);
|
|
1550
|
+
} catch (error) {
|
|
1551
|
+
pool.reconcile(ticket, noSpend());
|
|
1552
|
+
throw error;
|
|
1553
|
+
}
|
|
1554
|
+
pool.attribute(ticket, {
|
|
1555
|
+
label: work.runId,
|
|
1556
|
+
stage: "executing"
|
|
1557
|
+
});
|
|
1558
|
+
}
|
|
1559
|
+
const permit = slots?.acquire(0);
|
|
1560
|
+
if (permit) try {
|
|
1561
|
+
await abortable(permit.ready, work.signal);
|
|
1562
|
+
} catch (error) {
|
|
1563
|
+
permit.release();
|
|
1564
|
+
if (ticket) pool.reconcile(ticket, noSpend());
|
|
1565
|
+
throw error;
|
|
1566
|
+
}
|
|
1567
|
+
return { release(result) {
|
|
1568
|
+
permit?.release();
|
|
1569
|
+
if (!ticket) return;
|
|
1570
|
+
try {
|
|
1571
|
+
pool.reconcile(ticket, result ? spendOf(result) : unknownSpend());
|
|
1572
|
+
} catch {}
|
|
1573
|
+
} };
|
|
1574
|
+
}
|
|
1575
|
+
function abortable(promise, signal) {
|
|
1576
|
+
if (signal.aborted) return Promise.reject(signal.reason ?? /* @__PURE__ */ new Error("search cell aborted"));
|
|
1577
|
+
return new Promise((resolve, reject) => {
|
|
1578
|
+
const onAbort = () => reject(signal.reason ?? /* @__PURE__ */ new Error("search cell aborted"));
|
|
1579
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1580
|
+
promise.then((value) => {
|
|
1581
|
+
signal.removeEventListener("abort", onAbort);
|
|
1582
|
+
resolve(value);
|
|
1583
|
+
}, (error) => {
|
|
1584
|
+
signal.removeEventListener("abort", onAbort);
|
|
1585
|
+
reject(error);
|
|
1586
|
+
});
|
|
1587
|
+
});
|
|
1588
|
+
}
|
|
1589
|
+
function noSpend() {
|
|
1590
|
+
return {
|
|
1591
|
+
iterations: 0,
|
|
1592
|
+
tokens: {
|
|
1593
|
+
input: 0,
|
|
1594
|
+
output: 0
|
|
1595
|
+
},
|
|
1596
|
+
usd: 0,
|
|
1597
|
+
ms: 0
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1600
|
+
function unknownSpend() {
|
|
1601
|
+
return {
|
|
1602
|
+
...noSpend(),
|
|
1603
|
+
usdKnown: false,
|
|
1604
|
+
tokensKnown: false
|
|
1605
|
+
};
|
|
1606
|
+
}
|
|
1607
|
+
/** The attempt's spend as the budget pool reads it. Unknown dollars carry their proven floor. */
|
|
1608
|
+
function spendOf(result) {
|
|
1609
|
+
const { cost, tokens } = result.accounting;
|
|
1610
|
+
return {
|
|
1611
|
+
iterations: 0,
|
|
1612
|
+
tokens: tokens.status === "known" ? {
|
|
1613
|
+
input: tokens.inputTokens,
|
|
1614
|
+
output: tokens.outputTokens
|
|
1615
|
+
} : {
|
|
1616
|
+
input: 0,
|
|
1617
|
+
output: 0
|
|
1618
|
+
},
|
|
1619
|
+
...tokens.status === "known" ? {} : { tokensKnown: false },
|
|
1620
|
+
usd: cost.status === "known" ? cost.usd : cost.knownLowerBoundUsd,
|
|
1621
|
+
...cost.status === "known" ? {} : { usdKnown: false },
|
|
1622
|
+
ms: result.wallMs ?? 0
|
|
1623
|
+
};
|
|
1624
|
+
}
|
|
1625
|
+
/** The dollar maximum a paid call declares, or null when it declares no priced one. */
|
|
1626
|
+
function chargeMaximumUsd(maximum) {
|
|
1627
|
+
if (!maximum) return null;
|
|
1628
|
+
if ("externallyEnforcedMaximumUsd" in maximum) return maximum.externallyEnforcedMaximumUsd;
|
|
1629
|
+
if ("customTokenPricing" in maximum) return costForTokenPricing(maximum.customTokenPricing, maximum);
|
|
1630
|
+
const priced = costForUsage(maximum.model, maximum);
|
|
1631
|
+
return priced.costUnknown ? null : priced.costUsd;
|
|
1632
|
+
}
|
|
1633
|
+
const TAG_SEARCH = "searchId";
|
|
1634
|
+
const TAG_RUN = "searchRunId";
|
|
1635
|
+
const TAG_LANE = "searchLane";
|
|
1636
|
+
const TAG_OPERATION = "searchOperation";
|
|
1637
|
+
/** The cost-ledger tags of one search's calls: a cell attempt's, or a proposal operation's. */
|
|
1638
|
+
function searchCallTags(searchId, of) {
|
|
1639
|
+
return "run" in of ? {
|
|
1640
|
+
[TAG_SEARCH]: searchId,
|
|
1641
|
+
[TAG_RUN]: of.run,
|
|
1642
|
+
[TAG_LANE]: of.lane
|
|
1643
|
+
} : {
|
|
1644
|
+
[TAG_SEARCH]: searchId,
|
|
1645
|
+
[TAG_OPERATION]: of.operation
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1648
|
+
/** The search ledger with `tags` added to every paid call. */
|
|
1649
|
+
function taggedLedger(ledger, tags) {
|
|
1650
|
+
return ledgerView(ledger, (input) => ledger.runPaidCall({
|
|
1651
|
+
...input,
|
|
1652
|
+
tags: {
|
|
1653
|
+
...input.tags,
|
|
1654
|
+
...tags
|
|
1655
|
+
}
|
|
1656
|
+
}));
|
|
1657
|
+
}
|
|
1658
|
+
function ledgerView(ledger, runPaidCall) {
|
|
1659
|
+
return new Proxy(ledger, { get(target, property) {
|
|
1660
|
+
if (property === "runPaidCall") return runPaidCall;
|
|
1661
|
+
const value = Reflect.get(target, property, target);
|
|
1662
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
1663
|
+
} });
|
|
1664
|
+
}
|
|
1665
|
+
/**
|
|
1666
|
+
* The search ledger seen by one attempt: every paid call carries the attempt's tags. On a hard
|
|
1667
|
+
* lane every call also holds its declared maximum against the lane's `cellUsd` until its
|
|
1668
|
+
* receipt replaces the hold with the charge, and a call that would pass the cap, or declares no
|
|
1669
|
+
* priced maximum, is refused. A charge the provider left unknown keeps its maximum held.
|
|
1670
|
+
*/
|
|
1671
|
+
function attemptLedger(ledger, lane, tags) {
|
|
1672
|
+
const tagged = taggedLedger(ledger, tags);
|
|
1673
|
+
if (lane.costCap !== "hard") return tagged;
|
|
1674
|
+
let heldUsd = 0;
|
|
1675
|
+
const runPaidCall = async (input) => {
|
|
1676
|
+
const maximumUsd = chargeMaximumUsd(input.maximumCharge);
|
|
1677
|
+
if (maximumUsd === null) return {
|
|
1678
|
+
succeeded: false,
|
|
1679
|
+
error: new CostAccountingIncompleteError(`lane ${lane.name} caps each cell at $${lane.cellUsd}, so paid call '${input.actor}' during '${input.phase}' needs a priced maximumCharge`)
|
|
1680
|
+
};
|
|
1681
|
+
if (heldUsd + maximumUsd > lane.cellUsd + USD_TOLERANCE) return {
|
|
1682
|
+
succeeded: false,
|
|
1683
|
+
error: new CostCeilingReachedError(lane.cellUsd, heldUsd, maximumUsd, input.phase, input.actor)
|
|
1684
|
+
};
|
|
1685
|
+
heldUsd += maximumUsd;
|
|
1686
|
+
const result = await tagged.runPaidCall(input);
|
|
1687
|
+
const { receipt } = result;
|
|
1688
|
+
if (receipt && !receipt.costUnknown) heldUsd += receipt.costUsd - maximumUsd;
|
|
1689
|
+
return result;
|
|
1690
|
+
};
|
|
1691
|
+
return ledgerView(tagged, runPaidCall);
|
|
1692
|
+
}
|
|
1693
|
+
/**
|
|
1694
|
+
* An attempt's cost is every call tagged with its run id, a call a process that ended made for
|
|
1695
|
+
* it included; a call still pending makes the cost unknown with the settled calls as its floor.
|
|
1696
|
+
* A subscription seat reports no tokens, and its own work charges no dollar.
|
|
1697
|
+
*/
|
|
1698
|
+
function attemptAccounting(ledger, lane, tags) {
|
|
1699
|
+
const receipts = ledger.list({ tags });
|
|
1700
|
+
const accounting = searchReceiptAccounting(receipts);
|
|
1701
|
+
const pending = ledger.listPending?.({ tags }) ?? [];
|
|
1702
|
+
const floorUsd = accounting.cost.status === "known" ? accounting.cost.usd : accounting.cost.knownLowerBoundUsd;
|
|
1703
|
+
const cost = pending.length > 0 ? {
|
|
1704
|
+
status: "unknown",
|
|
1705
|
+
knownLowerBoundUsd: floorUsd,
|
|
1706
|
+
reason: `${pending.length} paid call(s) of the attempt had no receipt when it ended`
|
|
1707
|
+
} : accounting.cost.status === "known" && accounting.cost.usd > 0 && receipts.some((receipt) => receipt.actualCostUsd === void 0) ? {
|
|
1708
|
+
...accounting.cost,
|
|
1709
|
+
source: "pricing-table"
|
|
1710
|
+
} : accounting.cost;
|
|
1711
|
+
return {
|
|
1712
|
+
tokens: lane.kind === "subscription" ? {
|
|
1713
|
+
status: "unknown",
|
|
1714
|
+
reason: "a subscription seat reports no token usage"
|
|
1715
|
+
} : accounting.tokens,
|
|
1716
|
+
cost
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
/**
|
|
1720
|
+
* Settle every paid call of `searchId` that a process which ended left pending, before the
|
|
1721
|
+
* reopened search makes a new one: a capped ledger refuses new paid work while one is
|
|
1722
|
+
* unresolved. A cell's call is asked of its lane's provider through `recoverReceipt`; any other
|
|
1723
|
+
* call, or one the provider cannot answer, settles failed with an unknown cost.
|
|
1724
|
+
*/
|
|
1725
|
+
async function reconcileInterruptedSearchCalls(ledger, searchId, lanes) {
|
|
1726
|
+
const interrupted = (ledger.listPending?.({ tags: { [TAG_SEARCH]: searchId } }) ?? []).filter((call) => call.state === "interrupted");
|
|
1727
|
+
let recovered = 0;
|
|
1728
|
+
for (const call of interrupted) {
|
|
1729
|
+
const receipt = await lanes.find((candidate) => candidate.name === call.tags?.[TAG_LANE])?.recoverReceipt?.(call) ?? null;
|
|
1730
|
+
if (receipt) {
|
|
1731
|
+
ledger.reconcile(call.callId, receipt);
|
|
1732
|
+
recovered += 1;
|
|
1733
|
+
} else ledger.reconcile(call.callId, {
|
|
1734
|
+
model: call.model,
|
|
1735
|
+
inputTokens: 0,
|
|
1736
|
+
outputTokens: 0,
|
|
1737
|
+
costUnknown: true,
|
|
1738
|
+
usageUnknown: true
|
|
1739
|
+
}, { error: "the process that made this call ended before its receipt" });
|
|
1740
|
+
}
|
|
1741
|
+
return {
|
|
1742
|
+
recovered,
|
|
1743
|
+
unknown: interrupted.length - recovered
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
1746
|
+
/** Finished attempts by run id: the record a restarted kernel adopts. */
|
|
1747
|
+
function attemptStore(storage, dir) {
|
|
1748
|
+
const pathOf = (runId) => join(dir, `${runId.replace(/[^A-Za-z0-9_-]/gu, ".")}.json`);
|
|
1749
|
+
storage.ensureDir(dir);
|
|
1750
|
+
return {
|
|
1751
|
+
read(runId) {
|
|
1752
|
+
const text = storage.read(pathOf(runId));
|
|
1753
|
+
if (text === void 0) return null;
|
|
1754
|
+
const record = JSON.parse(text);
|
|
1755
|
+
if (record.runId !== runId || !record.result) throw new Error(`attempt record ${pathOf(runId)} does not hold run ${runId}`);
|
|
1756
|
+
return record.result;
|
|
1757
|
+
},
|
|
1758
|
+
write(runId, result) {
|
|
1759
|
+
const text = `${JSON.stringify({
|
|
1760
|
+
runId,
|
|
1761
|
+
result
|
|
1762
|
+
})}\n`;
|
|
1763
|
+
const path = pathOf(runId);
|
|
1764
|
+
if (storage.kind !== "filesystem") {
|
|
1765
|
+
storage.write(path, text);
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
1768
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1769
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
1770
|
+
writeFileSync(temporary, text);
|
|
1771
|
+
renameSync(temporary, path);
|
|
1772
|
+
}
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
const STATUS_OK = 1;
|
|
1776
|
+
const STATUS_ERROR = 2;
|
|
1777
|
+
const MAX_ATTRIBUTE_CHARS = 256;
|
|
1778
|
+
function searchTracing(options) {
|
|
1779
|
+
const mode = options.trace;
|
|
1780
|
+
const fileExporter = mode === void 0 && options.storage.kind === "filesystem" ? createOpenInferenceFileExporter(join(options.dir, "spans.otlp.jsonl")) : void 0;
|
|
1781
|
+
const off = (reason) => ({
|
|
1782
|
+
ids: null,
|
|
1783
|
+
span: () => ({
|
|
1784
|
+
setAttribute() {},
|
|
1785
|
+
end() {}
|
|
1786
|
+
}),
|
|
1787
|
+
finish: async () => ({ unknown: reason })
|
|
1788
|
+
});
|
|
1789
|
+
return { open(work, lane) {
|
|
1790
|
+
if (mode === "off") return off("tracing is off for this search");
|
|
1791
|
+
const exporter = mode === void 0 ? fileExporter : createOtelExporter(mode.otlp);
|
|
1792
|
+
if (!exporter) return off(mode === void 0 ? "the search storage is not the filesystem and no OTLP exporter was configured" : "the OTLP exporter has no endpoint");
|
|
1793
|
+
return attemptTrace(work, lane, exporter, mode !== void 0);
|
|
1794
|
+
} };
|
|
1795
|
+
}
|
|
1796
|
+
/**
|
|
1797
|
+
* One attempt's trace. Its ids derive from the run id, so an attempt run again after a restart
|
|
1798
|
+
* writes into the same trace. A shared file exporter writes each span synchronously, so the
|
|
1799
|
+
* attempt's written and dropped counts are the change in the exporter's own counts across each
|
|
1800
|
+
* of its spans; an attempt's own OTLP exporter is shut down at the end and read whole.
|
|
1801
|
+
*/
|
|
1802
|
+
function attemptTrace(work, lane, exporter, ownsExporter) {
|
|
1803
|
+
const traceId = padTraceId(`search-cell:${work.runId}`);
|
|
1804
|
+
const rootSpanId = padSpanId(`search-cell:${work.runId}`);
|
|
1805
|
+
const startMs = Date.now();
|
|
1806
|
+
let written = 0;
|
|
1807
|
+
let dropped = 0;
|
|
1808
|
+
const exportCounted = (span) => {
|
|
1809
|
+
if (ownsExporter) {
|
|
1810
|
+
exporter.exportSpan(span);
|
|
1811
|
+
return;
|
|
1812
|
+
}
|
|
1813
|
+
const before = exporter.stats();
|
|
1814
|
+
exporter.exportSpan(span);
|
|
1815
|
+
const after = exporter.stats();
|
|
1816
|
+
written += after.written - before.written;
|
|
1817
|
+
dropped += after.dropped - before.dropped;
|
|
1818
|
+
};
|
|
1819
|
+
return {
|
|
1820
|
+
ids: {
|
|
1821
|
+
traceId,
|
|
1822
|
+
spanId: rootSpanId
|
|
1823
|
+
},
|
|
1824
|
+
span(name, attributes = {}) {
|
|
1825
|
+
const opened = Date.now();
|
|
1826
|
+
const record = { ...attributes };
|
|
1827
|
+
let ended = false;
|
|
1828
|
+
return {
|
|
1829
|
+
setAttribute(key, value) {
|
|
1830
|
+
record[key] = value;
|
|
1831
|
+
},
|
|
1832
|
+
end(endAttributes = {}) {
|
|
1833
|
+
if (ended) return;
|
|
1834
|
+
ended = true;
|
|
1835
|
+
exportCounted(otelSpan({
|
|
1836
|
+
traceId,
|
|
1837
|
+
spanId: generateSpanId(),
|
|
1838
|
+
parentSpanId: rootSpanId,
|
|
1839
|
+
name,
|
|
1840
|
+
startMs: opened,
|
|
1841
|
+
endMs: Date.now(),
|
|
1842
|
+
attributes: {
|
|
1843
|
+
...record,
|
|
1844
|
+
...endAttributes
|
|
1845
|
+
},
|
|
1846
|
+
status: STATUS_OK
|
|
1847
|
+
}));
|
|
1848
|
+
}
|
|
1849
|
+
};
|
|
1850
|
+
},
|
|
1851
|
+
async finish(result) {
|
|
1852
|
+
const root = branchSpan({
|
|
1853
|
+
traceId,
|
|
1854
|
+
spanId: rootSpanId,
|
|
1855
|
+
name: "search.cell",
|
|
1856
|
+
startTime: new Date(startMs).toISOString(),
|
|
1857
|
+
branchId: work.nodeId,
|
|
1858
|
+
arm: lane.name,
|
|
1859
|
+
attributes: {
|
|
1860
|
+
"tangle.search.id": work.searchId,
|
|
1861
|
+
"tangle.search.cell_id": work.cellId,
|
|
1862
|
+
"tangle.search.run_id": work.runId,
|
|
1863
|
+
"tangle.search.task_id": work.taskId,
|
|
1864
|
+
"tangle.search.split": work.split,
|
|
1865
|
+
"tangle.search.stage": work.stage,
|
|
1866
|
+
"tangle.search.attempt": work.attempt,
|
|
1867
|
+
"tangle.search.lane": lane.name,
|
|
1868
|
+
"tangle.search.lane_kind": lane.kind,
|
|
1869
|
+
"tangle.search.outcome": result.outcome.status,
|
|
1870
|
+
..."score" in result.outcome ? { [ATTR.score]: result.outcome.score } : {}
|
|
1871
|
+
}
|
|
1872
|
+
});
|
|
1873
|
+
exportCounted(otelSpan({
|
|
1874
|
+
traceId,
|
|
1875
|
+
spanId: rootSpanId,
|
|
1876
|
+
name: root.name,
|
|
1877
|
+
startMs,
|
|
1878
|
+
endMs: Date.now(),
|
|
1879
|
+
attributes: root.attributes,
|
|
1880
|
+
status: result.outcome.status === "errored" ? STATUS_ERROR : STATUS_OK
|
|
1881
|
+
}));
|
|
1882
|
+
if (ownsExporter) {
|
|
1883
|
+
await exporter.shutdown();
|
|
1884
|
+
const stats = exporter.stats();
|
|
1885
|
+
written = stats.written;
|
|
1886
|
+
dropped = stats.dropped;
|
|
1887
|
+
}
|
|
1888
|
+
return {
|
|
1889
|
+
traceId,
|
|
1890
|
+
execRunId: null,
|
|
1891
|
+
spansWritten: written,
|
|
1892
|
+
spansDropped: dropped
|
|
1893
|
+
};
|
|
1894
|
+
}
|
|
1895
|
+
};
|
|
1896
|
+
}
|
|
1897
|
+
function otelSpan(input) {
|
|
1898
|
+
const attributes = {};
|
|
1899
|
+
for (const [key, value] of Object.entries(input.attributes)) if (typeof value === "number" || typeof value === "boolean") attributes[key] = value;
|
|
1900
|
+
else if (typeof value === "string") attributes[key] = value.slice(0, MAX_ATTRIBUTE_CHARS);
|
|
1901
|
+
return {
|
|
1902
|
+
traceId: input.traceId,
|
|
1903
|
+
spanId: input.spanId,
|
|
1904
|
+
...input.parentSpanId ? { parentSpanId: input.parentSpanId } : {},
|
|
1905
|
+
name: input.name,
|
|
1906
|
+
kind: 1,
|
|
1907
|
+
startTimeUnixNano: (BigInt(Math.floor(input.startMs)) * 1000000n).toString(),
|
|
1908
|
+
endTimeUnixNano: (BigInt(Math.floor(Math.max(input.startMs, input.endMs))) * 1000000n).toString(),
|
|
1909
|
+
attributes: toOtelAttributes(attributes),
|
|
1910
|
+
status: { code: input.status }
|
|
1911
|
+
};
|
|
1912
|
+
}
|
|
1913
|
+
//#endregion
|
|
1914
|
+
export { sharedWorkerClientName as _, reconcileInterruptedSearchCalls as a, improveSearchCost as b, searchExecutor as c, taggedLedger as d, DEFAULT_SHARED_BOX_RESOURCES as f, sharedBoxRefusal as g, ROUTER_CLIENT_HEADER as h, isSearchEnvironmentFault as i, sharedBoxLane as l, sharedBoxPlacement as m, assertSearchLanes as n, routerLane as o, DEFAULT_SHARED_BOX_WORKERS as p, dedicatedLane as r, searchCallTags as s, SearchEnvironmentFault as t, subscriptionLane as u, decideClosedSearch as v, runtimeShipDecision as x, decideSearchImprovement as y };
|
|
1915
|
+
|
|
1916
|
+
//# sourceMappingURL=search-executor-DmgUm5p7.js.map
|