@cat-factory/server 0.290.1 → 0.292.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/CompositeAgentExecutor.d.ts +3 -3
- package/dist/agents/CompositeAgentExecutor.d.ts.map +1 -1
- package/dist/agents/CompositeAgentExecutor.js +4 -4
- package/dist/agents/CompositeAgentExecutor.js.map +1 -1
- package/dist/agents/ContainerAgentExecutor.d.ts +13 -93
- package/dist/agents/ContainerAgentExecutor.d.ts.map +1 -1
- package/dist/agents/ContainerAgentExecutor.js +31 -226
- package/dist/agents/ContainerAgentExecutor.js.map +1 -1
- package/dist/agents/RunnerJobClient.d.ts +20 -1
- package/dist/agents/RunnerJobClient.d.ts.map +1 -1
- package/dist/agents/RunnerJobClient.js +18 -0
- package/dist/agents/RunnerJobClient.js.map +1 -1
- package/dist/agents/containerJobAccounting.d.ts +113 -0
- package/dist/agents/containerJobAccounting.d.ts.map +1 -0
- package/dist/agents/containerJobAccounting.js +191 -0
- package/dist/agents/containerJobAccounting.js.map +1 -0
- package/dist/agents/containerJobAddressing.d.ts +57 -0
- package/dist/agents/containerJobAddressing.d.ts.map +1 -0
- package/dist/agents/containerJobAddressing.js +88 -0
- package/dist/agents/containerJobAddressing.js.map +1 -0
- package/package.json +6 -6
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AgentExecutor, type AgentJobHandle, type AgentJobUpdate, type AgentRunContext, type AgentRunResult, type AsyncAgentExecutor } from '@cat-factory/kernel';
|
|
1
|
+
import { type AgentExecutor, type AgentJobHandle, type AgentJobUpdate, type AgentRunContext, type AgentRunResult, type AsyncAgentExecutor, type RunReclaimTarget } from '@cat-factory/kernel';
|
|
2
2
|
import type { DispatchToolServers } from '@cat-factory/contracts';
|
|
3
3
|
import { type AgentKindRegistry } from '@cat-factory/agents';
|
|
4
4
|
export declare class CompositeAgentExecutor implements AsyncAgentExecutor {
|
|
@@ -41,10 +41,10 @@ export declare class CompositeAgentExecutor implements AsyncAgentExecutor {
|
|
|
41
41
|
pollJob(handle: AgentJobHandle): Promise<AgentJobUpdate>;
|
|
42
42
|
/**
|
|
43
43
|
* Best-effort container reclaim. The engine narrows the composite (not the inner
|
|
44
|
-
* container executor) when stopping a run, so the composite must forward
|
|
44
|
+
* container executor) when stopping a run, so the composite must forward the reclaim
|
|
45
45
|
* to the container — otherwise the Layer-2 reclaim silently no-ops and leaks a
|
|
46
46
|
* warm instance. Delegates only when a container that supports it is wired.
|
|
47
47
|
*/
|
|
48
|
-
|
|
48
|
+
reclaimRun(target: RunReclaimTarget): Promise<void>;
|
|
49
49
|
}
|
|
50
50
|
//# sourceMappingURL=CompositeAgentExecutor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CompositeAgentExecutor.d.ts","sourceRoot":"","sources":["../../src/agents/CompositeAgentExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,kBAAkB,
|
|
1
|
+
{"version":3,"file":"CompositeAgentExecutor.d.ts","sourceRoot":"","sources":["../../src/agents/CompositeAgentExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AACjE,OAAO,EACL,KAAK,iBAAiB,EAGvB,MAAM,qBAAqB,CAAA;AAwB5B,qBAAa,sBAAuB,YAAW,kBAAkB;IAK7D,OAAO,CAAC,QAAQ,CAAC,MAAM;IAGvB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAP5B,qGAAqG;IACrG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;IAE5C,YACmB,MAAM,EAAE,aAAa,EAGrB,SAAS,EAAE,aAAa,GAAG,IAAI,EAGhD,QAAQ,GAAE,iBAA8C,EAGzD;IAED;;;;OAIG;IACH,OAAO,CAAC,IAAI;IAqBZ,GAAG,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAErD;IAED;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAGlE;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAGrF;IAED;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,CAIvD;IAED,+EAA+E;IAC/E,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAG3C;IAED,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAM1D;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAMvD;IAED;;;;;OAKG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAIxD;CACF"}
|
|
@@ -118,13 +118,13 @@ export class CompositeAgentExecutor {
|
|
|
118
118
|
}
|
|
119
119
|
/**
|
|
120
120
|
* Best-effort container reclaim. The engine narrows the composite (not the inner
|
|
121
|
-
* container executor) when stopping a run, so the composite must forward
|
|
121
|
+
* container executor) when stopping a run, so the composite must forward the reclaim
|
|
122
122
|
* to the container — otherwise the Layer-2 reclaim silently no-ops and leaks a
|
|
123
123
|
* warm instance. Delegates only when a container that supports it is wired.
|
|
124
124
|
*/
|
|
125
|
-
async
|
|
126
|
-
if (this.container && isAsyncAgentExecutor(this.container) && this.container.
|
|
127
|
-
await this.container.
|
|
125
|
+
async reclaimRun(target) {
|
|
126
|
+
if (this.container && isAsyncAgentExecutor(this.container) && this.container.reclaimRun) {
|
|
127
|
+
await this.container.reclaimRun(target);
|
|
128
128
|
}
|
|
129
129
|
}
|
|
130
130
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CompositeAgentExecutor.js","sourceRoot":"","sources":["../../src/agents/CompositeAgentExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,oBAAoB,
|
|
1
|
+
{"version":3,"file":"CompositeAgentExecutor.js","sourceRoot":"","sources":["../../src/agents/CompositeAgentExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,oBAAoB,GAErB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,wBAAwB,EACxB,eAAe,GAChB,MAAM,qBAAqB,CAAA;AAE5B,gFAAgF;AAChF,+EAA+E;AAC/E,2EAA2E;AAC3E,kFAAkF;AAClF,gFAAgF;AAChF,oFAAoF;AACpF,kFAAkF;AAClF,mEAAmE;AACnE,gFAAgF;AAChF,sFAAsF;AACtF,6EAA6E;AAC7E,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,kFAAkF;AAClF,kFAAkF;AAClF,gFAAgF;AAChF,EAAE;AACF,6EAA6E;AAC7E,yEAAyE;AACzE,qEAAqE;AAErE,MAAM,OAAO,sBAAsB;IAKd,MAAM;IAGN,SAAS;IAP5B,qGAAqG;IACpF,QAAQ,CAAmB;IAE5C,YACmB,MAAqB;IACtC,+EAA+E;IAC/E,oEAAoE;IACnD,SAA+B;IAChD,oFAAoF;IACpF,uEAAuE;IACvE,QAAQ,GAAsB,wBAAwB,EAAE;sBANvC,MAAM;yBAGN,SAAS;QAK1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,CAAC;IAED;;;;OAIG;IACK,IAAI,CAAC,OAAwB;QACnC,8EAA8E;QAC9E,2EAA2E;QAC3E,mFAAmF;QACnF,wEAAwE;QACxE,qFAAqF;QACrF,oFAAoF;QACpF,uEAAuE;QACvE,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC,MAAM,CAAA;QAC1E,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,eAAe,OAAO,CAAC,SAAS,iDAAiD;gBAC/E,kFAAkF;gBAClF,8EAA8E;gBAC9E,+EAA+E;gBAC/E,0BAA0B,CAC7B,CAAA;QACH,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED,GAAG,CAAC,OAAwB;QAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,OAAwB;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACnC,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACvE,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,OAAwB;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACnC,OAAO,QAAQ,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IAC7E,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,OAAwB;QACnC,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QACrF,OAAO,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACzE,CAAC;IAED,+EAA+E;IAC/E,SAAS,CAAC,OAAwB;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACnC,OAAO,oBAAoB,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IACtE,CAAC;IAED,QAAQ,CAAC,OAAwB;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACnC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qCAAqC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;QAC5E,CAAC;QACD,OAAO,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;IACnC,CAAC;IAED,OAAO,CAAC,MAAsB;QAC5B,qEAAqE;QACrE,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QACnE,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACvC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,MAAwB;QACvC,IAAI,IAAI,CAAC,SAAS,IAAI,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC;YACxF,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;QACzC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AgentContextRecorder, type AgentJobHandle, type AgentJobUpdate, type AgentRunContext, type AgentRunResult, type AsyncAgentExecutor, type HarnessKind, type LlmTraceSink, type Logger, type ModelFlavor, type ModelRef, type McpOAuthTokenSource, type OperationalMetrics, type StoreAgentContextGate, type SubscriptionQuotaTarget, type SubscriptionVendor, type TestSecretEntry, type ToolSecretResolver, type WebSearchAvailability } from '@cat-factory/kernel';
|
|
1
|
+
import { type AgentContextRecorder, type AgentJobHandle, type AgentJobUpdate, type AgentRunContext, type AgentRunResult, type AsyncAgentExecutor, type HarnessKind, type LlmTraceSink, type Logger, type ModelFlavor, type ModelRef, type McpOAuthTokenSource, type OperationalMetrics, type RunReclaimTarget, type StoreAgentContextGate, type SubscriptionQuotaTarget, type SubscriptionVendor, type TestSecretEntry, type ToolSecretResolver, type WebSearchAvailability } from '@cat-factory/kernel';
|
|
2
2
|
import { type AgentKindRegistry, type AgentRouting } from '@cat-factory/agents';
|
|
3
3
|
import { type RecordToolCalls } from './toolTrajectory.js';
|
|
4
4
|
import { type HarnessCallsRecordInput } from '@cat-factory/orchestration';
|
|
@@ -256,44 +256,11 @@ export declare class ContainerAgentExecutor implements AsyncAgentExecutor {
|
|
|
256
256
|
/** Shared backend-polymorphic dispatch/poll/release plumbing (see RunnerJobClient). */
|
|
257
257
|
private readonly jobs;
|
|
258
258
|
/**
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
* whose surrounding upsert/emit throws) would replay and double-count, unfairly
|
|
263
|
-
* penalising the token in the usage-aware rotation. Recording once per job id
|
|
264
|
-
* guards that. Best-effort + bounded: cleared wholesale past a cap, and it cannot
|
|
265
|
-
* survive a cold isolate replay — a re-record there is the documented, benign
|
|
266
|
-
* worst case (one extra job's tokens on one row), never silent over-counting.
|
|
267
|
-
*/
|
|
268
|
-
private readonly recordedUsageJobs;
|
|
269
|
-
/**
|
|
270
|
-
* Job ids whose per-call telemetry (`llm_call_metrics`) has already been recorded.
|
|
271
|
-
* Separate from {@link recordedUsageJobs} because the two recorders are independently
|
|
272
|
-
* wired and gated (telemetry records even for a personal subscription that leases no
|
|
273
|
-
* pooled token id). Same replay-safety rationale + bound as the usage guard.
|
|
274
|
-
*/
|
|
275
|
-
private readonly recordedCallMetricJobs;
|
|
276
|
-
/**
|
|
277
|
-
* Per-job set of harness call `seq`s this process already recorded from the LIVE poll drain,
|
|
278
|
-
* so the terminal write can skip them. Without it the terminal pass re-walks the job's WHOLE
|
|
279
|
-
* list, and each already-stored call costs a chain-tip read plus an ignored insert — hundreds
|
|
280
|
-
* of pointless round-trips at the end of a long run (and, on the Worker, hundreds of
|
|
281
|
-
* subrequests inside one Workflow step).
|
|
282
|
-
*
|
|
283
|
-
* A set of the seqs actually recorded, not a high-water mark: a drained batch whose write
|
|
284
|
-
* failed is swallowed (telemetry never fails a run), and those calls must still be picked up
|
|
285
|
-
* by the terminal write rather than skipped as "already done". Dropped once the job records
|
|
286
|
-
* terminally, and bounded by the same wholesale clear as {@link recordedCallMetricJobs} — a
|
|
287
|
-
* lost entry only costs the redundant write it was there to avoid.
|
|
288
|
-
*/
|
|
289
|
-
private readonly recordedCallSeqs;
|
|
290
|
-
/**
|
|
291
|
-
* Job ids whose subscription usage has already been folded into the modeled quota
|
|
292
|
-
* cycle. Separate from {@link recordedUsageJobs} because quota tracking counts BOTH
|
|
293
|
-
* pooled and personal runs (not gated on a pooled token id). Same replay-safety
|
|
294
|
-
* rationale + bound as the usage guard.
|
|
259
|
+
* Where a settled subscription job's tokens are recorded: per-call telemetry, the leased pool
|
|
260
|
+
* token's rotation counters, the modeled quota cycle. It owns the at-most-once guards those
|
|
261
|
+
* three need (see `containerJobAccounting.ts`).
|
|
295
262
|
*/
|
|
296
|
-
private readonly
|
|
263
|
+
private readonly accounting;
|
|
297
264
|
/** Resolves which model + subscription path a step runs on (routing policy). */
|
|
298
265
|
private readonly modelRouter;
|
|
299
266
|
/** The app-owned agent-kind registry the job-body builders read (custom-kind prompts/tuning). */
|
|
@@ -311,63 +278,16 @@ export declare class ContainerAgentExecutor implements AsyncAgentExecutor {
|
|
|
311
278
|
/** Poll a dispatched job for its state, mapping the runner view into an update. */
|
|
312
279
|
pollJob(handle: AgentJobHandle): Promise<AgentJobUpdate>;
|
|
313
280
|
/**
|
|
314
|
-
*
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
318
|
-
* Called from two places, deliberately: on EVERY poll for the calls the harness drained
|
|
319
|
-
* since the last one (so a run's telemetry lands while it runs, and survives the run dying),
|
|
320
|
-
* and once on the terminal state for the complete list (so a transport that forwards no live
|
|
321
|
-
* drain, or a run whose last window never reached us, still records everything). The calls
|
|
322
|
-
* carry a stable per-job `seq`, so the ids the recorder mints are the same in both channels
|
|
323
|
-
* and the second write of an already-recorded call is a no-op at the store.
|
|
281
|
+
* Reclaim every runner resource a run holds: resolve the same transport its jobs dispatched
|
|
282
|
+
* to (by workspace) and release EACH container the run opened — for the Cloudflare backend
|
|
283
|
+
* that SIGKILLs them instead of letting them idle out.
|
|
324
284
|
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
/**
|
|
330
|
-
* Remember which `seq`s of a job landed, so {@link recordHarnessCallsOnce} can skip them.
|
|
331
|
-
* Only called after the batch was written. A call with no `seq` (an older harness image,
|
|
332
|
-
* which streams nothing) is not tracked: there the terminal list is the only channel.
|
|
333
|
-
*/
|
|
334
|
-
private markCallSeqsRecorded;
|
|
335
|
-
/**
|
|
336
|
-
* The TERMINAL record of a job's full call list (see {@link recordHarnessCalls}). An
|
|
337
|
-
* in-memory once-per-job guard skips the redundant walk within this process; the recorder
|
|
338
|
-
* additionally mints deterministic per-call ids, so even a durable-driver replay in a fresh
|
|
339
|
-
* isolate (empty guard) re-records idempotently rather than duplicating rows.
|
|
340
|
-
*
|
|
341
|
-
* Calls this process already recorded from the live drain are filtered out first: the store
|
|
342
|
-
* would ignore them anyway, but only after a chain-tip read + an insert each, which on a long
|
|
343
|
-
* run is hundreds of round-trips for no new rows. Anything the drain never delivered (a lost
|
|
344
|
-
* poll response, a transport that forwards no drain, a replay in a fresh isolate) still goes
|
|
345
|
-
* through.
|
|
346
|
-
*/
|
|
347
|
-
private recordHarnessCallsOnce;
|
|
348
|
-
/**
|
|
349
|
-
* Attribute a subscription harness's reported usage to its leased pool token
|
|
350
|
-
* (usage-aware rotation) and the telemetry sink. Best-effort: a missing usage
|
|
351
|
-
* signal or unconfigured recorder is a no-op; recorded at most once per job id
|
|
352
|
-
* so a retried/replayed poll can't double-count (see `recordedUsageJobs`).
|
|
353
|
-
*/
|
|
354
|
-
private recordSubscriptionUsageOnce;
|
|
355
|
-
/**
|
|
356
|
-
* Fold the SAME subscription usage into the modeled quota-cycle counters (Part B), for
|
|
357
|
-
* BOTH pooled and personal runs. A subscription run is the one reporting per-call
|
|
358
|
-
* metrics (Pi is proxy-metered and has none), and the handle's provider is the vendor
|
|
359
|
-
* slug. Scope = the leased pool token when present, else the run initiator (personal).
|
|
360
|
-
* Best-effort, once per job id so a replayed poll can't double-count.
|
|
361
|
-
*/
|
|
362
|
-
private recordSubscriptionQuotaUsageOnce;
|
|
363
|
-
/**
|
|
364
|
-
* Stop a running job and reclaim its backing runner: resolve the same transport
|
|
365
|
-
* the job dispatched to (by workspace) and `release` it — for the Cloudflare
|
|
366
|
-
* backend this SIGKILLs the per-run container instead of letting it idle out.
|
|
367
|
-
* Best-effort/idempotent: a transport without `release`, or an already-gone job,
|
|
368
|
-
* is a no-op.
|
|
285
|
+
* "Each" rather than "the": a run whose step declared a different executor image (the
|
|
286
|
+
* browser-carrying `tester-ui`) holds a second container beside the ordinary one, and
|
|
287
|
+
* reclaiming one ref leaves the other running. This resolves WHICH images from the kinds the
|
|
288
|
+
* run dispatched, because only the registry knows; the fan-out itself is the job client's.
|
|
369
289
|
*/
|
|
370
|
-
|
|
290
|
+
reclaimRun(target: RunReclaimTarget): Promise<void>;
|
|
371
291
|
/**
|
|
372
292
|
* Synchronous convenience for non-durable callers (and tests): dispatch then
|
|
373
293
|
* poll inline until the job finishes. The durable driver does not use this — it
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ContainerAgentExecutor.d.ts","sourceRoot":"","sources":["../../src/agents/ContainerAgentExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,kBAAkB,
|
|
1
|
+
{"version":3,"file":"ContainerAgentExecutor.d.ts","sourceRoot":"","sources":["../../src/agents/ContainerAgentExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,MAAM,EACX,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EAKvB,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,qBAAqB,EAC3B,MAAM,qBAAqB,CAAA;AAS5B,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,YAAY,EAMlB,MAAM,qBAAqB,CAAA;AAoB5B,OAAO,EAAE,KAAK,eAAe,EAAkB,MAAM,qBAAqB,CAAA;AAC1E,OAAO,EAAgB,KAAK,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACvF,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAA;AACvF,OAAO,EAAmB,KAAK,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AASnF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAUhE,YAAY,EAAE,sBAAsB,EAAE,CAAA;AAItC,OAAO,KAAK,EACV,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EAErB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,oBAAoB,CAAA;AAE3B,YAAY,EACV,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAEpD,0EAA0E;AAC1E,UAAU,uBAAuB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,gFAAgF;AAChF,KAAK,sBAAsB,GAAG,CAC5B,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,kBAAkB,KACvB,OAAO,CAAC,uBAAuB,CAAC,CAAA;AAErC;;;;;;GAMG;AACH,KAAK,8BAA8B,GAAG,CACpC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,KACvB,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAA;AAEhC,kFAAkF;AAClF,KAAK,uBAAuB,GAAG,CAC7B,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACjD,OAAO,CAAC,IAAI,CAAC,CAAA;AAElB;;;;;GAKG;AACH,KAAK,4BAA4B,GAAG,CAClC,MAAM,EAAE,uBAAuB,EAC/B,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACjD,OAAO,CAAC,IAAI,CAAC,CAAA;AAElB;;;;;;;GAOG;AACH,KAAK,kBAAkB,GAAG,CAAC,KAAK,EAAE,uBAAuB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;AAE3E,MAAM,WAAW,kCAAkC;IACjD,6FAA6F;IAC7F,gBAAgB,EAAE,sBAAsB,CAAA;IACxC,yEAAyE;IACzE,YAAY,EAAE,YAAY,CAAA;IAC1B,2FAA2F;IAC3F,iBAAiB,EAAE,CACjB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,kBAAkB,CAAC,EAAE,SAAS,WAAW,EAAE,KACxC,QAAQ,GAAG,SAAS,CAAA;IACzB;;;OAGG;IACH,4BAA4B,CAAC,EAAE,CAC7B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IAChC,2DAA2D;IAC3D,iBAAiB,EAAE,iBAAiB,CAAA;IACpC;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IACvC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CAAA;IAC9E,iFAAiF;IACjF,qBAAqB,EAAE,qBAAqB,CAAA;IAC5C;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,gFAAgF;IAChF,cAAc,EAAE,uBAAuB,CAAA;IACvC;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,sBAAsB,CAAA;IAC/C;;;;OAIG;IACH,8BAA8B,CAAC,EAAE,8BAA8B,CAAA;IAC/D,gGAAgG;IAChG,uBAAuB,CAAC,EAAE,uBAAuB,CAAA;IACjD;;;;;OAKG;IACH,4BAA4B,CAAC,EAAE,4BAA4B,CAAA;IAC3D;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IACvC,iFAAiF;IACjF,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,YAAY,CAAC,EAAE,qBAAqB,CAAA;IACpC;;;;;;;;;;;OAWG;IACH,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,kBAAkB,GAAG,SAAS,KAAK,OAAO,CAAA;IAC7F;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC5F;;;;;;OAMG;IACH,uBAAuB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC1F;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;IACrC;;;;;;;;;OASG;IACH,4BAA4B,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAA;IACtF;;;;;;;OAOG;IACH,wBAAwB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAA;IACrF;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;IACzF;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IACvC;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,mBAAmB,CAAA;IAC5C;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IACvC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B;;;;;;OAMG;IACH,yBAAyB,CAAC,EAAE,oBAAoB,CAAA;IAChD;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;CACtC;AAKD;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,kBAAkB;IAiBnD,OAAO,CAAC,QAAQ,CAAC,IAAI;IAhBjC,uFAAuF;IACvF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IAEtC;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IAEnD,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IAEzC,iGAAiG;IACjG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IAErD,YAA6B,IAAI,EAAE,kCAAkC,EAWpE;IAED,qFAAqF;IACrF,SAAS,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAE5C;IAED;;;;;OAKG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAyDhE;IAED,mFAAmF;IAC7E,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAyF7D;IAED;;;;;;;;;OASG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAMxD;IAED;;;;OAIG;IACG,GAAG,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAW3D;IAED;;;;OAIG;IACG,YAAY,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAG5D;IAED;;;;;;;OAOG;IACG,YAAY,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,CAI7D;IAED;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAevB,uFAAuF;IACvF,OAAO,CAAC,UAAU;IAalB;;;;;;;;;;;;;;;OAeG;YACW,sBAAsB;IAiCpC,qFAAqF;YACvE,YAAY;IAsP1B;;;;;;;;;;;;;OAaG;YACW,mBAAmB;IAwBjC;;;;OAIG;YACW,WAAW;CAiH1B"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import {} from '@cat-factory/kernel';
|
|
2
|
-
import { ConflictError, CredentialRequiredError, VCS_DOC_URLS, SUBSCRIPTION_VENDORS, isIndividualVendor,
|
|
2
|
+
import { ConflictError, CredentialRequiredError, VCS_DOC_URLS, SUBSCRIPTION_VENDORS, isIndividualVendor, } from '@cat-factory/kernel';
|
|
3
3
|
import { resolveAprioriWorkingBranch, resolveInstanceTypeId } from '@cat-factory/contracts';
|
|
4
4
|
import { agentTuningFor, withComplexityAllowance, defaultAgentKindRegistry, isReadOnlyAgentKind, webResearchGuidanceFor, } from '@cat-factory/agents';
|
|
5
5
|
import { ModelRouter } from './ModelRouter.js';
|
|
@@ -14,53 +14,11 @@ import { recordAgentContextSnapshot } from './agentContextRecord.js';
|
|
|
14
14
|
import { drainToolCalls } from './toolTrajectory.js';
|
|
15
15
|
import { isTesterKind } from '@cat-factory/orchestration';
|
|
16
16
|
import { RunnerJobClient } from './RunnerJobClient.js';
|
|
17
|
+
import { imageVariantFor, providerOf, refForHandle, runImageVariants, stepJobId, } from './containerJobAddressing.js';
|
|
18
|
+
import { ContainerJobAccounting } from './containerJobAccounting.js';
|
|
17
19
|
import { buildCommonBody, dispatchDesignImageDelivery, buildRepoSpec, githubRepoOrigin, resolveAuxiliaryRepos, } from './containerAgentBody.js';
|
|
18
20
|
import { jobTokenRepoIds } from './repoTargeting.js';
|
|
19
21
|
export { jobTokenRepoIds } from './repoTargeting.js';
|
|
20
|
-
/**
|
|
21
|
-
* The repo spec every container job body carries: clone coordinates plus, for a
|
|
22
|
-
* monorepo service, the subdirectory the harness should run the agent within. Built
|
|
23
|
-
* here once so the (six) agent-kind job bodies can't drift on which repo fields they
|
|
24
|
-
* forward.
|
|
25
|
-
*/
|
|
26
|
-
/**
|
|
27
|
-
* The harness job id for one dispatch: the run (execution) id, the agent kind, and — past the
|
|
28
|
-
* first job of that kind in the run — the `dispatchEpoch`. A run executes a sequence of steps that
|
|
29
|
-
* all share the one per-run container, so each job needs an id that is UNIQUE WITHIN THE RUN: the
|
|
30
|
-
* harness keys its per-kind job registries by it, and two jobs sharing an id alias there (the bug
|
|
31
|
-
* where an `architect` /explore poll read back the `spec-writer`'s /spec result). The run itself is
|
|
32
|
-
* addressed separately by the execution id (the {@link RunnerJobRef.runId}).
|
|
33
|
-
*
|
|
34
|
-
* The epoch is what makes that uniqueness total, because the engine dispatches one kind more than
|
|
35
|
-
* once per run in two ways: a step RE-dispatched (a Tester re-test after a fixer round, a gate's
|
|
36
|
-
* helper retry, a companion's rework round, an eviction recovery) and the same helper kind
|
|
37
|
-
* escalated off DIFFERENT steps (`fixer` serves four gates). The harness re-attaches to an EXISTING
|
|
38
|
-
* job id rather than re-running (replay idempotency), and a container-reusing transport — a warm
|
|
39
|
-
* local pool, a self-hosted runner pool — keeps that registry alive across rounds, since reclaiming
|
|
40
|
-
* a pooled member does NOT destroy it. So a reused id replays a completed job: the Tester that
|
|
41
|
-
* appeared to "pass regardless" and never re-tested, an eviction recovery landing back on the job
|
|
42
|
-
* whose runner just died. `dispatchEpochFor` counts the run's prior dispatches of the kind, so the
|
|
43
|
-
* id names the n-th job of that kind and the run's first keeps the unsuffixed shape.
|
|
44
|
-
*/
|
|
45
|
-
function stepJobId(executionId, agentKind, dispatchEpoch = 0) {
|
|
46
|
-
const base = `${executionId}-${agentKind}`;
|
|
47
|
-
return dispatchEpoch > 0 ? `${base}-${dispatchEpoch}` : base;
|
|
48
|
-
}
|
|
49
|
-
/** The provider slug from a handle's `provider:model` string (fallback when the handle omits `provider`). */
|
|
50
|
-
function providerOf(model) {
|
|
51
|
-
if (!model)
|
|
52
|
-
return 'unknown';
|
|
53
|
-
const colon = model.indexOf(':');
|
|
54
|
-
return colon > 0 ? model.slice(0, colon) : model;
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* The {@link RunnerJobRef} a job handle addresses: the run (for the per-run container)
|
|
58
|
-
* plus the per-step job id. Falls back to the job id as the run id for a handle minted
|
|
59
|
-
* before run ids were carried (or a single-job flow where the two coincide).
|
|
60
|
-
*/
|
|
61
|
-
function refForHandle(handle) {
|
|
62
|
-
return { runId: handle.runId ?? handle.jobId, jobId: handle.jobId };
|
|
63
|
-
}
|
|
64
22
|
/** Poll cadence for the non-durable `run()` fallback (the durable driver sleeps between polls itself). */
|
|
65
23
|
const RUN_POLL_INTERVAL_MS = 5_000;
|
|
66
24
|
/**
|
|
@@ -82,44 +40,11 @@ export class ContainerAgentExecutor {
|
|
|
82
40
|
/** Shared backend-polymorphic dispatch/poll/release plumbing (see RunnerJobClient). */
|
|
83
41
|
jobs;
|
|
84
42
|
/**
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
* whose surrounding upsert/emit throws) would replay and double-count, unfairly
|
|
89
|
-
* penalising the token in the usage-aware rotation. Recording once per job id
|
|
90
|
-
* guards that. Best-effort + bounded: cleared wholesale past a cap, and it cannot
|
|
91
|
-
* survive a cold isolate replay — a re-record there is the documented, benign
|
|
92
|
-
* worst case (one extra job's tokens on one row), never silent over-counting.
|
|
93
|
-
*/
|
|
94
|
-
recordedUsageJobs = new Set();
|
|
95
|
-
/**
|
|
96
|
-
* Job ids whose per-call telemetry (`llm_call_metrics`) has already been recorded.
|
|
97
|
-
* Separate from {@link recordedUsageJobs} because the two recorders are independently
|
|
98
|
-
* wired and gated (telemetry records even for a personal subscription that leases no
|
|
99
|
-
* pooled token id). Same replay-safety rationale + bound as the usage guard.
|
|
100
|
-
*/
|
|
101
|
-
recordedCallMetricJobs = new Set();
|
|
102
|
-
/**
|
|
103
|
-
* Per-job set of harness call `seq`s this process already recorded from the LIVE poll drain,
|
|
104
|
-
* so the terminal write can skip them. Without it the terminal pass re-walks the job's WHOLE
|
|
105
|
-
* list, and each already-stored call costs a chain-tip read plus an ignored insert — hundreds
|
|
106
|
-
* of pointless round-trips at the end of a long run (and, on the Worker, hundreds of
|
|
107
|
-
* subrequests inside one Workflow step).
|
|
108
|
-
*
|
|
109
|
-
* A set of the seqs actually recorded, not a high-water mark: a drained batch whose write
|
|
110
|
-
* failed is swallowed (telemetry never fails a run), and those calls must still be picked up
|
|
111
|
-
* by the terminal write rather than skipped as "already done". Dropped once the job records
|
|
112
|
-
* terminally, and bounded by the same wholesale clear as {@link recordedCallMetricJobs} — a
|
|
113
|
-
* lost entry only costs the redundant write it was there to avoid.
|
|
114
|
-
*/
|
|
115
|
-
recordedCallSeqs = new Map();
|
|
116
|
-
/**
|
|
117
|
-
* Job ids whose subscription usage has already been folded into the modeled quota
|
|
118
|
-
* cycle. Separate from {@link recordedUsageJobs} because quota tracking counts BOTH
|
|
119
|
-
* pooled and personal runs (not gated on a pooled token id). Same replay-safety
|
|
120
|
-
* rationale + bound as the usage guard.
|
|
43
|
+
* Where a settled subscription job's tokens are recorded: per-call telemetry, the leased pool
|
|
44
|
+
* token's rotation counters, the modeled quota cycle. It owns the at-most-once guards those
|
|
45
|
+
* three need (see `containerJobAccounting.ts`).
|
|
121
46
|
*/
|
|
122
|
-
|
|
47
|
+
accounting;
|
|
123
48
|
/** Resolves which model + subscription path a step runs on (routing policy). */
|
|
124
49
|
modelRouter;
|
|
125
50
|
/** The app-owned agent-kind registry the job-body builders read (custom-kind prompts/tuning). */
|
|
@@ -127,6 +52,7 @@ export class ContainerAgentExecutor {
|
|
|
127
52
|
constructor(deps) {
|
|
128
53
|
this.deps = deps;
|
|
129
54
|
this.jobs = new RunnerJobClient(deps.resolveTransport);
|
|
55
|
+
this.accounting = new ContainerJobAccounting(deps);
|
|
130
56
|
this.agentKindRegistry = deps.agentKindRegistry ?? defaultAgentKindRegistry();
|
|
131
57
|
this.modelRouter = new ModelRouter({
|
|
132
58
|
agentRouting: deps.agentRouting,
|
|
@@ -153,7 +79,10 @@ export class ContainerAgentExecutor {
|
|
|
153
79
|
// run's container never collide in the harness's per-kind job registries; the run
|
|
154
80
|
// itself is addressed by the execution id, so its container is reclaimed as a unit.
|
|
155
81
|
const jobId = body.jobId;
|
|
156
|
-
|
|
82
|
+
// The image variant rides the ref, not just the dispatch options: a per-run container
|
|
83
|
+
// backend has to address THIS step's container again on every poll and on release.
|
|
84
|
+
const image = imageVariantFor(context.agentKind, this.agentKindRegistry);
|
|
85
|
+
const ref = { runId: executionId, jobId, ...(image ? { image } : {}) };
|
|
157
86
|
const jobLog = containerJobLog(this.deps.logger, { workspaceId, executionId, jobId, agentKind: context.agentKind }, this.deps.operationalMetrics);
|
|
158
87
|
// Dispatch, log the transition, and refuse a run whose body carries a capability this runner
|
|
159
88
|
// image told us it cannot serve. See `containerAgentDispatch.ts`.
|
|
@@ -209,7 +138,7 @@ export class ContainerAgentExecutor {
|
|
|
209
138
|
// Logged and re-thrown; the lifecycle is unchanged.
|
|
210
139
|
let view;
|
|
211
140
|
try {
|
|
212
|
-
view = await this.jobs.poll(handle.workspaceId, refForHandle(handle));
|
|
141
|
+
view = await this.jobs.poll(handle.workspaceId, refForHandle(handle, this.agentKindRegistry));
|
|
213
142
|
}
|
|
214
143
|
catch (error) {
|
|
215
144
|
jobLog.pollFailed(error);
|
|
@@ -225,7 +154,7 @@ export class ContainerAgentExecutor {
|
|
|
225
154
|
// batching to the end meant a killed run reported zero calls no matter how many tokens it
|
|
226
155
|
// had spent — the exact run an operator most needs to see. Idempotent via the calls' `seq`,
|
|
227
156
|
// so the terminal write below re-offers them harmlessly.
|
|
228
|
-
await this.
|
|
157
|
+
await this.accounting.recordCalls(handle, view.callMetrics);
|
|
229
158
|
// Forward-looking items the Coder streamed since the last poll (drain-on-read): surfaced
|
|
230
159
|
// on both running and done so a final burst on the completion poll isn't lost. Normalise
|
|
231
160
|
// the transport's optional `detail` to the engine's `StreamedFollowUp` shape.
|
|
@@ -246,7 +175,7 @@ export class ContainerAgentExecutor {
|
|
|
246
175
|
// conflicts) is exactly what an operator needs to inspect — so record it before the
|
|
247
176
|
// terminal returns below, on every terminal state.
|
|
248
177
|
const result = view.result ?? {};
|
|
249
|
-
await this.
|
|
178
|
+
await this.accounting.recordCallsOnce(handle, result);
|
|
250
179
|
if (view.state === 'failed') {
|
|
251
180
|
jobLog.settled('failed', settleFailureFields(view));
|
|
252
181
|
return { state: 'failed', error: view.error ?? 'Implementation job failed', ...failureMeta };
|
|
@@ -260,8 +189,8 @@ export class ContainerAgentExecutor {
|
|
|
260
189
|
// Best-effort subscription usage attribution, split into their own methods so `pollJob` stays
|
|
261
190
|
// within the complexity budget: the pool-token usage feedback + telemetry sink, and the
|
|
262
191
|
// modeled quota-cycle counters. Both are idempotent (once per job id) and behaviour-neutral.
|
|
263
|
-
await this.
|
|
264
|
-
await this.
|
|
192
|
+
await this.accounting.recordPooledUsageOnce(handle, result);
|
|
193
|
+
await this.accounting.recordQuotaUsageOnce(handle, result);
|
|
265
194
|
const runResult = toRunResult(result, handle.agentKind, this.agentKindRegistry);
|
|
266
195
|
// The poll site can't resolve the model ref, but the dispatch captured its label
|
|
267
196
|
// (`handle.model`, already used for `recordHarnessCalls`). Fold it onto the result so the
|
|
@@ -286,142 +215,21 @@ export class ContainerAgentExecutor {
|
|
|
286
215
|
return buildDoneUpdate(view, runResult, followUps);
|
|
287
216
|
}
|
|
288
217
|
/**
|
|
289
|
-
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
218
|
+
* Reclaim every runner resource a run holds: resolve the same transport its jobs dispatched
|
|
219
|
+
* to (by workspace) and release EACH container the run opened — for the Cloudflare backend
|
|
220
|
+
* that SIGKILLs them instead of letting them idle out.
|
|
292
221
|
*
|
|
293
|
-
*
|
|
294
|
-
*
|
|
295
|
-
*
|
|
296
|
-
*
|
|
297
|
-
* carry a stable per-job `seq`, so the ids the recorder mints are the same in both channels
|
|
298
|
-
* and the second write of an already-recorded call is a no-op at the store.
|
|
299
|
-
*
|
|
300
|
-
* Best-effort: an unwired recorder or an empty batch is a no-op, and a failure is swallowed
|
|
301
|
-
* — telemetry is observability, never a reason to fail (or fail to complete) a run.
|
|
222
|
+
* "Each" rather than "the": a run whose step declared a different executor image (the
|
|
223
|
+
* browser-carrying `tester-ui`) holds a second container beside the ordinary one, and
|
|
224
|
+
* reclaiming one ref leaves the other running. This resolves WHICH images from the kinds the
|
|
225
|
+
* run dispatched, because only the registry knows; the fan-out itself is the job client's.
|
|
302
226
|
*/
|
|
303
|
-
async
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
workspaceId: handle.workspaceId,
|
|
310
|
-
executionId: handle.runId ?? null,
|
|
311
|
-
agentKind: handle.agentKind ?? 'agent',
|
|
312
|
-
provider: handle.provider ?? providerOf(handle.model),
|
|
313
|
-
model: handle.model ?? '',
|
|
314
|
-
jobId: handle.jobId,
|
|
315
|
-
calls,
|
|
316
|
-
});
|
|
317
|
-
this.markCallSeqsRecorded(handle.jobId, calls);
|
|
318
|
-
}
|
|
319
|
-
catch {
|
|
320
|
-
// Swallowed: telemetry is observability, never a reason to fail (or fail to
|
|
321
|
-
// complete) a run.
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
/**
|
|
325
|
-
* Remember which `seq`s of a job landed, so {@link recordHarnessCallsOnce} can skip them.
|
|
326
|
-
* Only called after the batch was written. A call with no `seq` (an older harness image,
|
|
327
|
-
* which streams nothing) is not tracked: there the terminal list is the only channel.
|
|
328
|
-
*/
|
|
329
|
-
markCallSeqsRecorded(jobId, calls) {
|
|
330
|
-
const seqs = calls.map((c) => c.seq).filter((seq) => seq !== undefined);
|
|
331
|
-
if (seqs.length === 0)
|
|
332
|
-
return;
|
|
333
|
-
if (this.recordedCallSeqs.size >= 10_000)
|
|
334
|
-
this.recordedCallSeqs.clear();
|
|
335
|
-
const known = this.recordedCallSeqs.get(jobId) ?? new Set();
|
|
336
|
-
for (const seq of seqs)
|
|
337
|
-
known.add(seq);
|
|
338
|
-
this.recordedCallSeqs.set(jobId, known);
|
|
339
|
-
}
|
|
340
|
-
/**
|
|
341
|
-
* The TERMINAL record of a job's full call list (see {@link recordHarnessCalls}). An
|
|
342
|
-
* in-memory once-per-job guard skips the redundant walk within this process; the recorder
|
|
343
|
-
* additionally mints deterministic per-call ids, so even a durable-driver replay in a fresh
|
|
344
|
-
* isolate (empty guard) re-records idempotently rather than duplicating rows.
|
|
345
|
-
*
|
|
346
|
-
* Calls this process already recorded from the live drain are filtered out first: the store
|
|
347
|
-
* would ignore them anyway, but only after a chain-tip read + an insert each, which on a long
|
|
348
|
-
* run is hundreds of round-trips for no new rows. Anything the drain never delivered (a lost
|
|
349
|
-
* poll response, a transport that forwards no drain, a replay in a fresh isolate) still goes
|
|
350
|
-
* through.
|
|
351
|
-
*/
|
|
352
|
-
async recordHarnessCallsOnce(handle, result) {
|
|
353
|
-
if (this.recordedCallMetricJobs.has(handle.jobId))
|
|
354
|
-
return;
|
|
355
|
-
const recorded = this.recordedCallSeqs.get(handle.jobId);
|
|
356
|
-
const pending = recorded
|
|
357
|
-
? result.callMetrics?.filter((c) => c.seq === undefined || !recorded.has(c.seq))
|
|
358
|
-
: result.callMetrics;
|
|
359
|
-
await this.recordHarnessCalls(handle, pending);
|
|
360
|
-
if (this.recordedCallMetricJobs.size >= 10_000)
|
|
361
|
-
this.recordedCallMetricJobs.clear();
|
|
362
|
-
this.recordedCallMetricJobs.add(handle.jobId);
|
|
363
|
-
// The job is settled: its per-seq bookkeeping can go (the once-per-job guard covers a
|
|
364
|
-
// repeat poll of the same terminal state from here on).
|
|
365
|
-
this.recordedCallSeqs.delete(handle.jobId);
|
|
366
|
-
}
|
|
367
|
-
/**
|
|
368
|
-
* Attribute a subscription harness's reported usage to its leased pool token
|
|
369
|
-
* (usage-aware rotation) and the telemetry sink. Best-effort: a missing usage
|
|
370
|
-
* signal or unconfigured recorder is a no-op; recorded at most once per job id
|
|
371
|
-
* so a retried/replayed poll can't double-count (see `recordedUsageJobs`).
|
|
372
|
-
*/
|
|
373
|
-
async recordSubscriptionUsageOnce(handle, result) {
|
|
374
|
-
if (handle.subscriptionTokenId &&
|
|
375
|
-
handle.workspaceId &&
|
|
376
|
-
result.usage &&
|
|
377
|
-
this.deps.recordSubscriptionUsage &&
|
|
378
|
-
!this.recordedUsageJobs.has(handle.jobId)) {
|
|
379
|
-
await this.deps.recordSubscriptionUsage(handle.workspaceId, handle.subscriptionTokenId, result.usage);
|
|
380
|
-
// Mark only AFTER a successful write: a failed record is left to retry rather
|
|
381
|
-
// than silently dropped. Bound the set so a long-lived process can't grow it
|
|
382
|
-
// unboundedly (clearing only risks a benign re-record on a later retry).
|
|
383
|
-
if (this.recordedUsageJobs.size >= 10_000)
|
|
384
|
-
this.recordedUsageJobs.clear();
|
|
385
|
-
this.recordedUsageJobs.add(handle.jobId);
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
/**
|
|
389
|
-
* Fold the SAME subscription usage into the modeled quota-cycle counters (Part B), for
|
|
390
|
-
* BOTH pooled and personal runs. A subscription run is the one reporting per-call
|
|
391
|
-
* metrics (Pi is proxy-metered and has none), and the handle's provider is the vendor
|
|
392
|
-
* slug. Scope = the leased pool token when present, else the run initiator (personal).
|
|
393
|
-
* Best-effort, once per job id so a replayed poll can't double-count.
|
|
394
|
-
*/
|
|
395
|
-
async recordSubscriptionQuotaUsageOnce(handle, result) {
|
|
396
|
-
const quotaVendor = handle.provider ?? providerOf(handle.model);
|
|
397
|
-
if (result.callMetrics &&
|
|
398
|
-
result.callMetrics.length > 0 &&
|
|
399
|
-
result.usage &&
|
|
400
|
-
this.deps.recordSubscriptionQuotaUsage &&
|
|
401
|
-
isSubscriptionVendor(quotaVendor) &&
|
|
402
|
-
!this.recordedQuotaJobs.has(handle.jobId)) {
|
|
403
|
-
const target = handle.subscriptionTokenId
|
|
404
|
-
? { scope: 'pooled', scopeId: handle.subscriptionTokenId, vendor: quotaVendor }
|
|
405
|
-
: handle.initiatedByUserId
|
|
406
|
-
? { scope: 'user', scopeId: handle.initiatedByUserId, vendor: quotaVendor }
|
|
407
|
-
: null;
|
|
408
|
-
if (target) {
|
|
409
|
-
await this.deps.recordSubscriptionQuotaUsage(target, result.usage);
|
|
410
|
-
if (this.recordedQuotaJobs.size >= 10_000)
|
|
411
|
-
this.recordedQuotaJobs.clear();
|
|
412
|
-
this.recordedQuotaJobs.add(handle.jobId);
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
/**
|
|
417
|
-
* Stop a running job and reclaim its backing runner: resolve the same transport
|
|
418
|
-
* the job dispatched to (by workspace) and `release` it — for the Cloudflare
|
|
419
|
-
* backend this SIGKILLs the per-run container instead of letting it idle out.
|
|
420
|
-
* Best-effort/idempotent: a transport without `release`, or an already-gone job,
|
|
421
|
-
* is a no-op.
|
|
422
|
-
*/
|
|
423
|
-
async stopJob(handle) {
|
|
424
|
-
await this.jobs.release(handle.workspaceId, refForHandle(handle));
|
|
227
|
+
async reclaimRun(target) {
|
|
228
|
+
await this.jobs.releaseRun(target.workspaceId, {
|
|
229
|
+
runId: target.runId,
|
|
230
|
+
jobId: target.jobId,
|
|
231
|
+
images: runImageVariants(target.agentKinds, this.agentKindRegistry),
|
|
232
|
+
});
|
|
425
233
|
}
|
|
426
234
|
/**
|
|
427
235
|
* Synchronous convenience for non-durable callers (and tests): dispatch then
|
|
@@ -474,10 +282,7 @@ export class ContainerAgentExecutor {
|
|
|
474
282
|
dispatchOptions(context) {
|
|
475
283
|
const provider = context.service?.cloudProvider;
|
|
476
284
|
const size = context.service?.instanceSize;
|
|
477
|
-
|
|
478
|
-
// Playwright + browser image); absent ⇒ the default harness image, so the browser never
|
|
479
|
-
// bloats every other kind's cold start.
|
|
480
|
-
const image = this.agentKindRegistry.agentStep(context.agentKind)?.image;
|
|
285
|
+
const image = imageVariantFor(context.agentKind, this.agentKindRegistry);
|
|
481
286
|
if (!provider && !size && !image)
|
|
482
287
|
return undefined;
|
|
483
288
|
return {
|