@crewhaus/runtime-core 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +138 -0
- package/dist/alert-watchdog.js +290 -0
- package/dist/incident-collector.d.ts +42 -0
- package/dist/incident-collector.js +125 -0
- package/dist/index.d.ts +286 -4
- package/dist/index.js +733 -30
- package/dist/janitor.d.ts +88 -0
- package/dist/janitor.js +287 -0
- package/dist/observability.d.ts +178 -2
- package/dist/observability.js +466 -2
- package/dist/project-memory.d.ts +17 -1
- package/dist/project-memory.js +42 -0
- package/dist/slo-monitor.d.ts +221 -0
- package/dist/slo-monitor.js +397 -0
- package/package.json +40 -39
package/dist/index.js
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
|
+
import { dirname as pathDirname } from "node:path";
|
|
1
2
|
import * as readline from "node:readline/promises";
|
|
2
3
|
import { createAbortTree } from "@crewhaus/abort-controller";
|
|
3
4
|
import { collectFinalMessage, consumeStream, } from "@crewhaus/adapter-anthropic";
|
|
4
|
-
import { setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
|
|
5
|
+
import { classifyBoundary, setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
|
|
5
6
|
import { wrap as wrapWithCircuitBreaker } from "@crewhaus/circuit-breaker";
|
|
6
7
|
import { autoCompact } from "@crewhaus/compaction-autocompact";
|
|
7
8
|
import { snip } from "@crewhaus/compaction-snip";
|
|
9
|
+
import { DEFAULT_PRICING, computeCostMicros, createCostTracker, resolvePricing, } from "@crewhaus/cost-tracker";
|
|
8
10
|
import { classifyEgress, summarizeEgress, } from "@crewhaus/egress-classifier";
|
|
9
11
|
import { ConfigError, RuntimeError } from "@crewhaus/errors";
|
|
10
12
|
import { openEventLog } from "@crewhaus/event-log";
|
|
11
13
|
import { aggregateDecisions, runHooks } from "@crewhaus/hooks-engine";
|
|
12
|
-
import { parseModelString, resolveModel } from "@crewhaus/model-router";
|
|
14
|
+
import { createFailoverChain, createPolicyRouter, createTierRouter, parseModelString, resolveModel, } from "@crewhaus/model-router";
|
|
13
15
|
import { BUILTIN_DEFAULT_RULES, emptyRuleSet, evaluateJustification, evaluateWithReason, ruleBasedJustificationJudge, } from "@crewhaus/permission-engine";
|
|
14
16
|
import { manage as manageCacheMarkers } from "@crewhaus/prompt-cache-manager";
|
|
15
17
|
import { buildRedactionNotice, classifyText, llmClassifierEnabled, } from "@crewhaus/prompt-injection-detector";
|
|
16
18
|
import { advanceState, initialRecoveryState, recover, } from "@crewhaus/recovery-engine";
|
|
19
|
+
import { computeReward, openScoreboard, } from "@crewhaus/routing-store";
|
|
17
20
|
import { createRunContext, tagContent } from "@crewhaus/run-context";
|
|
18
21
|
import { createSessionStore } from "@crewhaus/session-store";
|
|
19
22
|
import { formatSkillsForPrompt } from "@crewhaus/skills-registry";
|
|
@@ -30,7 +33,8 @@ import { TraceEventBus } from "@crewhaus/trace-event-bus";
|
|
|
30
33
|
import { initialState, transition, } from "@crewhaus/turn-state-machine";
|
|
31
34
|
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
32
35
|
import { createCliMarkdownRenderer, isCliMarkdownEnabled, } from "./cli-markdown";
|
|
33
|
-
import {
|
|
36
|
+
import { attachIncidentCollector } from "./incident-collector";
|
|
37
|
+
import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, } from "./observability";
|
|
34
38
|
import { loadProjectMemory } from "./project-memory";
|
|
35
39
|
import { createCliOutput, isSpinnerEnabled } from "./spinner";
|
|
36
40
|
/**
|
|
@@ -105,14 +109,59 @@ const DEFAULT_CONTEXT_LIMIT = 200_000;
|
|
|
105
109
|
// agentic turn reaches it, low enough to stop a runaway/injected tool loop
|
|
106
110
|
// before it burns a provider quota.
|
|
107
111
|
const DEFAULT_MAX_TOOL_ITERATIONS = 500;
|
|
112
|
+
// Upper bound on how many concurrency-safe tool calls in one turn run at
|
|
113
|
+
// once. Bounds parallel sub-agent (`Task`) fan-out so a wide dispatch
|
|
114
|
+
// can't open unbounded model connections and trip a provider rate limit;
|
|
115
|
+
// also caps parallel reads, whose per-call cost is negligible. Configurable
|
|
116
|
+
// via `runChatLoop({ maxConcurrentTools })`.
|
|
117
|
+
const DEFAULT_MAX_CONCURRENT_TOOLS = 4;
|
|
108
118
|
const DEFAULT_COMPACTION_THRESHOLD = 0.85;
|
|
109
119
|
const DEFAULT_SNIP_KEEP_HEAD = 4;
|
|
110
120
|
const DEFAULT_SNIP_KEEP_TAIL = 20;
|
|
121
|
+
/**
|
|
122
|
+
* Run `items` through `fn` with at most `limit` in flight at once,
|
|
123
|
+
* preserving input order in the returned results. A pool of `limit`
|
|
124
|
+
* workers pull from a shared cursor, so a slow item never blocks a fast
|
|
125
|
+
* one behind it (unlike fixed-size `Promise.all` chunks). `limit <= 0`
|
|
126
|
+
* is clamped to 1; a limit larger than the batch just runs everything.
|
|
127
|
+
* Exported for unit testing of the concurrency bound.
|
|
128
|
+
*/
|
|
129
|
+
export async function mapWithConcurrency(items, limit, fn) {
|
|
130
|
+
const results = new Array(items.length);
|
|
131
|
+
const bound = Math.max(1, Math.min(limit, items.length));
|
|
132
|
+
let cursor = 0;
|
|
133
|
+
const worker = async () => {
|
|
134
|
+
while (true) {
|
|
135
|
+
const i = cursor;
|
|
136
|
+
cursor += 1;
|
|
137
|
+
if (i >= items.length)
|
|
138
|
+
return;
|
|
139
|
+
results[i] = await fn(items[i], i);
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
await Promise.all(Array.from({ length: bound }, () => worker()));
|
|
143
|
+
return results;
|
|
144
|
+
}
|
|
111
145
|
// Section 17 — `resolveAuth` / `createAnthropicClient` / `ResolvedAuth` /
|
|
112
146
|
// `OAUTH_BETAS` / `CLAUDE_CODE_HEADERS` / `CLAUDE_CODE_SYSTEM_PREFIX`
|
|
113
147
|
// moved into `@crewhaus/adapter-anthropic`. Re-exported here so legacy
|
|
114
148
|
// importers (eval-judge fallback, downstream tests) keep compiling.
|
|
115
149
|
export { CLAUDE_CODE_HEADERS, CLAUDE_CODE_SYSTEM_PREFIX, OAUTH_BETAS, createAnthropicClient, resolveAuth, } from "@crewhaus/adapter-anthropic";
|
|
150
|
+
// Ops item 36 — boot-time self-heal janitor for daemon shapes. Lives in its
|
|
151
|
+
// own module (`./janitor`) but is re-exported here because emitted bundles
|
|
152
|
+
// only import the package root.
|
|
153
|
+
export { createJanitor } from "./janitor";
|
|
154
|
+
// Ops item 31 — alert watchdog seams re-exported so the CLI/codegen can build
|
|
155
|
+
// the durable + off-box alert sink (audit append + settings.json alert hook /
|
|
156
|
+
// webhook) it passes as `RunChatLoopOptions.alertSink`.
|
|
157
|
+
export { attachDefaultSubscribers, } from "./observability";
|
|
158
|
+
// Ops item 37 — SLO monitor seams re-exported so the CLI/codegen can build the
|
|
159
|
+
// injected mitigation ladder (audit / pause-intake / rollback) it passes as
|
|
160
|
+
// `RunChatLoopOptions.sloSink`, and the doctor probe can share the breach types.
|
|
161
|
+
export { DEFAULT_SLO_WINDOW_MS, MIN_SLO_SAMPLES, SloWindow, attachSloMonitor, detectSloBreaches, } from "./slo-monitor";
|
|
162
|
+
// Ops item 32 — incident collector seams re-exported so the CLI can share the
|
|
163
|
+
// raw-capture shape + trigger classifier with `incident collect`.
|
|
164
|
+
export { attachIncidentCollector, classifyIncidentTrigger, } from "./incident-collector";
|
|
116
165
|
/**
|
|
117
166
|
* Reconcile a message history by dropping "orphan" `tool_use` blocks — a
|
|
118
167
|
* `tool_use` with no answering `tool_result` anywhere later in the
|
|
@@ -289,6 +338,19 @@ export function defaultSinkScope(toolName) {
|
|
|
289
338
|
return "external-dynamic";
|
|
290
339
|
return "external-configured";
|
|
291
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Neutralize a boundary-block closing delimiter embedded in untrusted content
|
|
343
|
+
* so a poisoned line can't terminate its wrapper block early and inject
|
|
344
|
+
* trailing instructions. Replaces `</tag>` (any casing/whitespace) with a
|
|
345
|
+
* visually-inert `<\/tag>` so the content survives for the model to read but
|
|
346
|
+
* no longer parses as the real closing tag. Used for the recalled-memory block
|
|
347
|
+
* (#53) — recalled memories may carry content shaped by untrusted tool output.
|
|
348
|
+
*/
|
|
349
|
+
function escapeBoundaryDelimiter(text, tag) {
|
|
350
|
+
// Match `</tag>` tolerantly (leading whitespace inside the tag, any casing).
|
|
351
|
+
const re = new RegExp(`</\\s*${tag}\\s*>`, "gi");
|
|
352
|
+
return text.replace(re, `<\\/${tag}>`);
|
|
353
|
+
}
|
|
292
354
|
/**
|
|
293
355
|
* Strip the model-string provider grammar down to the wire model id
|
|
294
356
|
* (`"bedrock/us.anthropic.claude-..."` → `"us.anthropic.claude-..."`)
|
|
@@ -305,6 +367,23 @@ function bestEffortWireModelId(modelString) {
|
|
|
305
367
|
return modelString;
|
|
306
368
|
}
|
|
307
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* A short, stable fingerprint of a `model_pool` config, stamped on every
|
|
372
|
+
* `model_route` event as `policyVersion` so a learned decision can be tied back
|
|
373
|
+
* to the exact policy that made it. Deterministic (djb2 over the candidate
|
|
374
|
+
* roster + policy + objective) — no clock, no randomness.
|
|
375
|
+
*/
|
|
376
|
+
function poolFingerprint(pool) {
|
|
377
|
+
const canonical = JSON.stringify({
|
|
378
|
+
p: pool.policy,
|
|
379
|
+
c: pool.candidates.map((c) => [c.model, [...c.tags].sort()]),
|
|
380
|
+
o: pool.objective ?? null,
|
|
381
|
+
});
|
|
382
|
+
let hash = 5381;
|
|
383
|
+
for (let i = 0; i < canonical.length; i++)
|
|
384
|
+
hash = ((hash << 5) + hash + canonical.charCodeAt(i)) >>> 0;
|
|
385
|
+
return `pool-${hash.toString(36)}`;
|
|
386
|
+
}
|
|
308
387
|
export async function runChatLoop(opts) {
|
|
309
388
|
// Pillar 3 — make the model-backed Layer-3 classifier reachable at EVERY
|
|
310
389
|
// trust boundary (MCP / sub-agent / channel / federation / skill /
|
|
@@ -333,25 +412,65 @@ export async function runChatLoop(opts) {
|
|
|
333
412
|
providerId: opts._adapter.providerId,
|
|
334
413
|
}
|
|
335
414
|
: await resolveModel(opts.model);
|
|
336
|
-
// Section 27
|
|
337
|
-
// `
|
|
338
|
-
//
|
|
339
|
-
//
|
|
415
|
+
// Section 27 / item 22 — resilience wrapping around the primary adapter.
|
|
416
|
+
// - `modelFallbacks` non-empty → a model-router failover chain: every
|
|
417
|
+
// candidate (primary first) breaker-wrapped with the `circuitBreaker`
|
|
418
|
+
// tuning; calls route to the first candidate whose breaker admits
|
|
419
|
+
// traffic; `model_failover` trace events surface routing changes.
|
|
420
|
+
// - `circuitBreaker` alone → the single-adapter breaker wrap.
|
|
421
|
+
// The chain publishes through a LATE-BOUND bus getter: the run's real bus
|
|
422
|
+
// is minted below (createRunContext), so `observabilityBus` is re-pointed
|
|
423
|
+
// at it once it exists — a `crewhaus run` without a caller-supplied
|
|
424
|
+
// runContext still surfaces model_failover + circuit_state_changed.
|
|
340
425
|
const baseAdapter = primaryResolution.adapter;
|
|
341
426
|
let breakerWrap;
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
427
|
+
let failoverChain;
|
|
428
|
+
let observabilityBus = opts.runContext?.eventBus;
|
|
429
|
+
const modelFallbacks = opts.modelFallbacks ?? [];
|
|
430
|
+
let adapter;
|
|
431
|
+
if (modelFallbacks.length > 0) {
|
|
432
|
+
// Seed the chain with the already-resolved primary (honours `_adapter`
|
|
433
|
+
// injection and avoids a second resolveModel round-trip) plus any
|
|
434
|
+
// test-injected fallback adapters.
|
|
435
|
+
const injectedAdapters = new Map();
|
|
436
|
+
injectedAdapters.set(opts.model, baseAdapter);
|
|
437
|
+
for (const [modelString, injected] of opts._failoverAdapters ?? []) {
|
|
438
|
+
injectedAdapters.set(modelString, injected);
|
|
439
|
+
}
|
|
440
|
+
failoverChain = await createFailoverChain({
|
|
441
|
+
model: opts.model,
|
|
442
|
+
fallbacks: modelFallbacks,
|
|
443
|
+
...(opts.circuitBreaker !== undefined ? { breaker: opts.circuitBreaker } : {}),
|
|
444
|
+
getBus: () => observabilityBus,
|
|
445
|
+
adapters: injectedAdapters,
|
|
446
|
+
});
|
|
447
|
+
// Doctor-style boot report: fallbacks that failed credential/package
|
|
448
|
+
// resolution are listed once, loudly, on stderr — and stay in the chain
|
|
449
|
+
// (they are re-tried when routing actually reaches them; see item 22).
|
|
450
|
+
for (const warning of failoverChain.warnings()) {
|
|
451
|
+
process.stderr.write(`[failover] ${warning}\n`);
|
|
452
|
+
}
|
|
453
|
+
adapter = failoverChain;
|
|
454
|
+
}
|
|
455
|
+
else if (opts.circuitBreaker !== undefined) {
|
|
345
456
|
breakerWrap = wrapWithCircuitBreaker(baseAdapter, {
|
|
346
457
|
...opts.circuitBreaker,
|
|
347
458
|
adapterName: baseAdapter.providerId,
|
|
348
459
|
bus: opts.runContext?.eventBus,
|
|
349
460
|
});
|
|
350
|
-
|
|
351
|
-
}
|
|
461
|
+
adapter = breakerWrap;
|
|
462
|
+
}
|
|
463
|
+
else {
|
|
464
|
+
adapter = baseAdapter;
|
|
465
|
+
}
|
|
352
466
|
void breakerWrap; // exposed via runtime stats once gateway/eval consumers land
|
|
353
|
-
|
|
354
|
-
|
|
467
|
+
// `let` (item 27): a `budget: { on_exceed: degrade }` breach re-resolves
|
|
468
|
+
// the primary model in place, so these become the degraded model's
|
|
469
|
+
// identity. `degradedSpecModel` overrides `opts.model` as the specModel
|
|
470
|
+
// stamped on model_request/response once a degrade has happened.
|
|
471
|
+
let providerId = primaryResolution.providerId;
|
|
472
|
+
let wireModelId = primaryResolution.modelId;
|
|
473
|
+
let degradedSpecModel;
|
|
355
474
|
// Section 17 — feature gate: a spec that declares tools cannot run on an
|
|
356
475
|
// adapter that doesn't speak tool use (e.g. Bedrock Llama/Mistral
|
|
357
476
|
// families). Fail with a clear ConfigError naming the model instead of
|
|
@@ -371,9 +490,41 @@ export async function runChatLoop(opts) {
|
|
|
371
490
|
compactionWireModelId = c.modelId;
|
|
372
491
|
}
|
|
373
492
|
else {
|
|
493
|
+
// Item 22 v1 scope note: with no explicit `compactionModel`, compaction
|
|
494
|
+
// reuses the primary adapter — which IS the failover chain when
|
|
495
|
+
// `modelFallbacks` is set, so compaction inherits the chain by
|
|
496
|
+
// construction. An explicit `compactionModel` keeps its own single
|
|
497
|
+
// adapter (no chain), as do judge/grader slots.
|
|
374
498
|
compactionAdapter = adapter;
|
|
375
499
|
compactionWireModelId = wireModelId;
|
|
376
500
|
}
|
|
501
|
+
// Item 26 — two-tier turn-difficulty router. When `modelTiers` is set, BOTH
|
|
502
|
+
// tier adapters resolve HERE at boot (adapters bind once; the per-turn pick
|
|
503
|
+
// just selects between the two already-resolved adapters — the same
|
|
504
|
+
// discipline as compaction's second adapter and the failover chain's
|
|
505
|
+
// multiple candidates). The `_tierAdapters` map is a test seam mirroring
|
|
506
|
+
// `_failoverAdapters`.
|
|
507
|
+
let tierRouter;
|
|
508
|
+
if (opts.modelTiers !== undefined) {
|
|
509
|
+
const resolveTier = async (modelString) => {
|
|
510
|
+
const injected = opts._tierAdapters?.get(modelString);
|
|
511
|
+
if (injected !== undefined) {
|
|
512
|
+
return { adapter: injected, modelId: bestEffortWireModelId(modelString), modelString };
|
|
513
|
+
}
|
|
514
|
+
const r = await resolveModel(modelString);
|
|
515
|
+
return { adapter: r.adapter, modelId: r.modelId, modelString };
|
|
516
|
+
};
|
|
517
|
+
const fast = await resolveTier(opts.modelTiers.fast);
|
|
518
|
+
const dflt = await resolveTier(opts.modelTiers.default);
|
|
519
|
+
tierRouter = createTierRouter({
|
|
520
|
+
fast,
|
|
521
|
+
default: dflt,
|
|
522
|
+
...(opts.modelTiers.routing !== undefined ? { config: opts.modelTiers.routing } : {}),
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
// Item 26 — tool_use blocks the PREVIOUS turn produced (a tier-routing
|
|
526
|
+
// signal). Persists across `runOneTurn` calls; 0 on the first turn.
|
|
527
|
+
let priorTurnToolUseCount = 0;
|
|
377
528
|
// Default model max OUTPUT tokens for one turn. Callers thread the spec's
|
|
378
529
|
// `agent.max_tokens` here when set (the CLI target's codegen + apps/cli);
|
|
379
530
|
// this fallback applies when the spec is silent. Kept comfortably above the
|
|
@@ -408,6 +559,80 @@ export async function runChatLoop(opts) {
|
|
|
408
559
|
// Housekeeping side-effect: evicts any session whose mtime is older than
|
|
409
560
|
// the TTL (default 30 days). The returned list is intentionally discarded.
|
|
410
561
|
await sessionStore.list();
|
|
562
|
+
// Adaptive model routing (spec `agent.model_pool`). Resolve every candidate
|
|
563
|
+
// adapter ONCE at boot (mirroring the tier router), open the durable reward
|
|
564
|
+
// scoreboard beside the sessions dir, and build the PolicyRouter. The
|
|
565
|
+
// scoreboard is opened whenever a pool is present — not just for `learned` —
|
|
566
|
+
// so `route status` has data and a later switch to `learned` inherits the
|
|
567
|
+
// accumulated history. `recordPoolOutcome`/`poolCostUsd` fold each turn's
|
|
568
|
+
// observed result back in; they no-op when no pool is active.
|
|
569
|
+
let poolRouter;
|
|
570
|
+
let scoreboard;
|
|
571
|
+
let rewardConfig;
|
|
572
|
+
let poolPolicyVersion;
|
|
573
|
+
if (opts.modelPool !== undefined) {
|
|
574
|
+
const pool = opts.modelPool;
|
|
575
|
+
const candidates = [];
|
|
576
|
+
for (const c of pool.candidates) {
|
|
577
|
+
const injected = opts._poolAdapters?.get(c.model);
|
|
578
|
+
if (injected !== undefined) {
|
|
579
|
+
candidates.push({
|
|
580
|
+
adapter: injected,
|
|
581
|
+
modelId: bestEffortWireModelId(c.model),
|
|
582
|
+
modelString: c.model,
|
|
583
|
+
tags: c.tags,
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
else {
|
|
587
|
+
const r = await resolveModel(c.model);
|
|
588
|
+
candidates.push({
|
|
589
|
+
adapter: r.adapter,
|
|
590
|
+
modelId: r.modelId,
|
|
591
|
+
modelString: c.model,
|
|
592
|
+
tags: c.tags,
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
const routingRoot = sessionRootDir !== undefined ? pathDirname(sessionRootDir) : ".crewhaus";
|
|
597
|
+
scoreboard = opts._scoreboard ?? openScoreboard(routingRoot);
|
|
598
|
+
const sb = scoreboard;
|
|
599
|
+
rewardConfig = {
|
|
600
|
+
...(pool.objective !== undefined ? { objective: pool.objective } : {}),
|
|
601
|
+
...(pool.learning?.costRefUsd !== undefined ? { costRefUsd: pool.learning.costRefUsd } : {}),
|
|
602
|
+
...(pool.learning?.latencyRefMs !== undefined
|
|
603
|
+
? { latencyRefMs: pool.learning.latencyRefMs }
|
|
604
|
+
: {}),
|
|
605
|
+
};
|
|
606
|
+
poolPolicyVersion = poolFingerprint(pool);
|
|
607
|
+
poolRouter = createPolicyRouter({
|
|
608
|
+
candidates,
|
|
609
|
+
policy: pool.policy,
|
|
610
|
+
...(pool.routing !== undefined ? { routing: pool.routing } : {}),
|
|
611
|
+
...(pool.learning?.minSamplesPerArm !== undefined
|
|
612
|
+
? { learning: { minSamplesPerArm: pool.learning.minSamplesPerArm } }
|
|
613
|
+
: {}),
|
|
614
|
+
// The learned policy reads the live scoreboard; static/heuristic ignore it.
|
|
615
|
+
...(pool.policy === "learned" ? { score: (rk, m) => sb.score(rk, m) } : {}),
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
// Fold one turn's observed outcome into the scoreboard (no-op without a pool).
|
|
619
|
+
const recordPoolOutcome = (turn, obs) => {
|
|
620
|
+
// Capture into consts so the narrowing survives (they are closed-over lets).
|
|
621
|
+
const sb = scoreboard;
|
|
622
|
+
const rc = rewardConfig;
|
|
623
|
+
if (turn === undefined || sb === undefined || rc === undefined)
|
|
624
|
+
return;
|
|
625
|
+
sb.record(turn.routeKey, turn.modelString, computeReward(obs, rc), obs);
|
|
626
|
+
};
|
|
627
|
+
// Per-turn USD cost from token usage + the static pricing table. Undefined
|
|
628
|
+
// when the served model isn't on the table (the reward then drops the cost
|
|
629
|
+
// term and reweights over quality + latency).
|
|
630
|
+
const poolCostUsd = (provider, wireModelId, usage) => {
|
|
631
|
+
const row = resolvePricing(DEFAULT_PRICING, provider, wireModelId);
|
|
632
|
+
if (row === undefined)
|
|
633
|
+
return undefined;
|
|
634
|
+
return (computeCostMicros(row, usage.input, usage.output, usage.cacheRead ?? 0, usage.cacheCreate ?? 0) / 1_000_000);
|
|
635
|
+
};
|
|
411
636
|
let sessionId;
|
|
412
637
|
let resumedMessages;
|
|
413
638
|
if (opts.resume) {
|
|
@@ -443,7 +668,117 @@ export async function runChatLoop(opts) {
|
|
|
443
668
|
// run with `OTEL_EXPORTER_OTLP_ENDPOINT` set automatically exports.
|
|
444
669
|
const bus = runContext.eventBus;
|
|
445
670
|
void TraceEventBus; // keep type import alive for future direct constructions
|
|
446
|
-
|
|
671
|
+
// Item 22 — re-point the failover chain's late-bound bus getter at the
|
|
672
|
+
// run's real bus (see the chain construction above): the chain and its
|
|
673
|
+
// per-candidate breakers publish model_failover / circuit_state_changed
|
|
674
|
+
// through this reference from the first stream() call onward.
|
|
675
|
+
observabilityBus = bus;
|
|
676
|
+
const subscribers = await attachDefaultSubscribers(bus, runContext, process.env, {
|
|
677
|
+
...(opts.alertSink !== undefined ? { alertSink: opts.alertSink } : {}),
|
|
678
|
+
...(opts.alertMetricsDir !== undefined ? { metricsDir: opts.alertMetricsDir } : {}),
|
|
679
|
+
...(opts.sloTargets !== undefined ? { sloTargets: opts.sloTargets } : {}),
|
|
680
|
+
...(opts.sloSink !== undefined ? { sloSink: opts.sloSink } : {}),
|
|
681
|
+
});
|
|
682
|
+
// Ops item 32 — auto-assemble an incident bundle on the first failure-class
|
|
683
|
+
// trigger (circuit → open, egress-blocked, justification-deny storm). Gated
|
|
684
|
+
// by CREWHAUS_INCIDENTS; writes a raw capture (ring buffer + trigger meta)
|
|
685
|
+
// that `crewhaus incident collect --session <id>` turns into a full bundle.
|
|
686
|
+
const incidentCollector = attachIncidentCollector(bus, runContext, process.env, {
|
|
687
|
+
...(opts.incidentsDir !== undefined ? { incidentsDir: opts.incidentsDir } : {}),
|
|
688
|
+
...(opts.incidentSpec !== undefined ? { spec: opts.incidentSpec } : {}),
|
|
689
|
+
});
|
|
690
|
+
// Item 22 — surface each failover on stderr so an operator watching a
|
|
691
|
+
// plain (non-CREWHAUS_TRACE) run still sees provider switches. NOTE: the
|
|
692
|
+
// advisor-events session-log subscriber pattern has NOT landed on main
|
|
693
|
+
// (packages/event-log has no advisor kind) — when it does, failovers
|
|
694
|
+
// should ALSO persist into the session JSONL through it; until then the
|
|
695
|
+
// trace event + this stderr note are the observable surfaces.
|
|
696
|
+
let failoverNoteUnsubscribe;
|
|
697
|
+
if (failoverChain !== undefined) {
|
|
698
|
+
failoverNoteUnsubscribe = bus.subscribe((event) => {
|
|
699
|
+
if (event.kind !== "model_failover")
|
|
700
|
+
return;
|
|
701
|
+
process.stderr.write(`[failover] ${event.from} → ${event.to} (${event.reason})\n`);
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
// Item 27 — run-level spend cap. An ALWAYS-ON cost meter (independent of
|
|
705
|
+
// the CREWHAUS_COST_TRACKING env flag) accrues per-response spend priced
|
|
706
|
+
// off the wire model that actually served — `suppressEvents` keeps it off
|
|
707
|
+
// the trace surface so it never double-prints beside an env-attached
|
|
708
|
+
// cost-tracker. The pre-turn check (`enforceBudget`, below) reads
|
|
709
|
+
// `budgetMeter.getRunCost(runId).totalUsdMicros`. `budgetDegraded` gates
|
|
710
|
+
// the ladder to a single rung: once we degrade, we never degrade again —
|
|
711
|
+
// a second breach stops the run.
|
|
712
|
+
const budgetMeter = opts.budget !== undefined ? createCostTracker(bus, { suppressEvents: true }) : undefined;
|
|
713
|
+
let budgetDegraded = false;
|
|
714
|
+
/**
|
|
715
|
+
* Item 27 — pre-turn budget gate. Returns "stop" when the run must end
|
|
716
|
+
* before the next turn opens; "continue" otherwise. On a `degrade` breach
|
|
717
|
+
* it re-resolves the primary model to the cheaper rung IN PLACE (mutating
|
|
718
|
+
* `adapter`, `providerId`, `wireModelId`) and emits a `model_failover`
|
|
719
|
+
* (reason `budget_degrade`) — so the very next model call, and all cost
|
|
720
|
+
* accrual after it, use the degraded model. The current turn always
|
|
721
|
+
* completes; this only gates the NEXT turn, so an in-flight tool loop is
|
|
722
|
+
* never severed. Never throws — a re-resolution failure logs and stops.
|
|
723
|
+
*/
|
|
724
|
+
const enforceBudget = async () => {
|
|
725
|
+
if (opts.budget === undefined || budgetMeter === undefined)
|
|
726
|
+
return "continue";
|
|
727
|
+
const spent = budgetMeter.getRunCost(bus.runId).totalUsdMicros;
|
|
728
|
+
if (spent < opts.budget.usdMicros)
|
|
729
|
+
return "continue";
|
|
730
|
+
const spentUsd = (spent / 1_000_000).toFixed(4);
|
|
731
|
+
const capUsd = (opts.budget.usdMicros / 1_000_000).toFixed(4);
|
|
732
|
+
if (opts.budget.onExceed.kind === "stop" || budgetDegraded) {
|
|
733
|
+
const why = budgetDegraded ? "degraded model also reached the cap" : "run budget reached";
|
|
734
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — ${why}; ending the run.\n`);
|
|
735
|
+
return "stop";
|
|
736
|
+
}
|
|
737
|
+
// degrade: re-resolve the primary model to the cheaper rung, once.
|
|
738
|
+
const target = opts.budget.onExceed.model;
|
|
739
|
+
try {
|
|
740
|
+
const degraded = opts._budgetDegradeAdapter !== undefined
|
|
741
|
+
? {
|
|
742
|
+
adapter: opts._budgetDegradeAdapter,
|
|
743
|
+
modelId: bestEffortWireModelId(target),
|
|
744
|
+
providerId: opts._budgetDegradeAdapter.providerId,
|
|
745
|
+
}
|
|
746
|
+
: await resolveModel(target);
|
|
747
|
+
const from = opts.model;
|
|
748
|
+
adapter =
|
|
749
|
+
opts.circuitBreaker !== undefined
|
|
750
|
+
? wrapWithCircuitBreaker(degraded.adapter, {
|
|
751
|
+
...opts.circuitBreaker,
|
|
752
|
+
adapterName: degraded.adapter.providerId,
|
|
753
|
+
bus,
|
|
754
|
+
})
|
|
755
|
+
: degraded.adapter;
|
|
756
|
+
// The degraded model is now the serving one; keep the trace/pricing
|
|
757
|
+
// identity coherent for the model_request/response events below.
|
|
758
|
+
providerId = degraded.providerId;
|
|
759
|
+
wireModelId = degraded.modelId;
|
|
760
|
+
degradedSpecModel = target;
|
|
761
|
+
// A degrade takes over from the failover chain: subsequent calls hit
|
|
762
|
+
// the degraded adapter directly (v1 keeps one rung, no re-entry to the
|
|
763
|
+
// chain). Null it so the request/response stamping stops consulting it.
|
|
764
|
+
failoverChain = undefined;
|
|
765
|
+
budgetDegraded = true;
|
|
766
|
+
bus.publish({
|
|
767
|
+
...bus.envelope(),
|
|
768
|
+
kind: "model_failover",
|
|
769
|
+
from,
|
|
770
|
+
to: target,
|
|
771
|
+
reason: "budget_degrade",
|
|
772
|
+
});
|
|
773
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrading ${from} → ${target}.\n`);
|
|
774
|
+
return "continue";
|
|
775
|
+
}
|
|
776
|
+
catch (err) {
|
|
777
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
778
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrade to "${target}" failed (${message}); ending the run.\n`);
|
|
779
|
+
return "stop";
|
|
780
|
+
}
|
|
781
|
+
};
|
|
447
782
|
// Working-indicator spinner. Auto-enables only for an interactive REPL on a
|
|
448
783
|
// TTY (see `isSpinnerEnabled`); `singleTurn`/injected-`input`/piped runs stay
|
|
449
784
|
// byte-identical to before. `out.write` is the spinner-aware stdout sink:
|
|
@@ -505,6 +840,11 @@ export async function runChatLoop(opts) {
|
|
|
505
840
|
inputTokens: ev.inputTokens,
|
|
506
841
|
outputTokens: ev.outputTokens,
|
|
507
842
|
cachedReadTokens: ev.cachedReadTokens,
|
|
843
|
+
// Optional on the event (older emitters may omit it) — persist
|
|
844
|
+
// only when present so old-log parsing stays additive-safe.
|
|
845
|
+
...(ev.cacheCreationTokens !== undefined
|
|
846
|
+
? { cacheCreationTokens: ev.cacheCreationTokens }
|
|
847
|
+
: {}),
|
|
508
848
|
costUsdMicros: ev.costUsdMicros,
|
|
509
849
|
...(ev.tenantId !== undefined ? { tenantId: ev.tenantId } : {}),
|
|
510
850
|
},
|
|
@@ -516,6 +856,18 @@ export async function runChatLoop(opts) {
|
|
|
516
856
|
});
|
|
517
857
|
});
|
|
518
858
|
}
|
|
859
|
+
// Advisor groundwork (item 14) — persist the trace-bus-only advisory
|
|
860
|
+
// signals (recovery actions, per-call tool stats, resolved permission
|
|
861
|
+
// decisions, model stop reasons) into the same session JSONL the
|
|
862
|
+
// transcript already writes to, so `crewhaus advise` can mine sessions
|
|
863
|
+
// offline. DEFAULT-ON (the lines are tiny and they are the advisor's
|
|
864
|
+
// food); disable with CREWHAUS_ADVISOR_EVENTS=0. See observability.ts.
|
|
865
|
+
const advisorPersist = attachAdvisorPersistence(bus, eventLog, runContext);
|
|
866
|
+
// Ops item 38 — persist the trace-bus-only `mcp_call_end` events into the
|
|
867
|
+
// session JSONL as the durable `mcp_stats` kind so `crewhaus mcp doctor` can
|
|
868
|
+
// score per-server MCP health offline. Shares the advisor's DEFAULT-ON gate
|
|
869
|
+
// (CREWHAUS_ADVISOR_EVENTS=0 disables both). See observability.ts.
|
|
870
|
+
const mcpStatsPersist = attachMcpStatsPersistence(bus, eventLog, runContext);
|
|
519
871
|
// Per-run state container — coordination surface for hooks/skills/tools
|
|
520
872
|
// landing in Section 11+. Section 10 only ships the plumbing; the
|
|
521
873
|
// underscore prefix marks the intentional non-consumer.
|
|
@@ -557,6 +909,21 @@ export async function runChatLoop(opts) {
|
|
|
557
909
|
return { allowed: true };
|
|
558
910
|
}
|
|
559
911
|
}
|
|
912
|
+
// Feature #53 — auto-capture at teardown. Runs in the same finally where the
|
|
913
|
+
// `stop` hook fires, but is NOT a credential-stripped hook: the caller-
|
|
914
|
+
// supplied `onCapture` (which writes to the memory-store from the CLI's full
|
|
915
|
+
// environment) needs the process env, exactly like the feedback teardown.
|
|
916
|
+
// Best-effort — a capture failure never turns a clean exit into an error.
|
|
917
|
+
const maybeAutoCapture = async () => {
|
|
918
|
+
if (opts.memory?.autoCapture !== true || opts.memory.onCapture === undefined)
|
|
919
|
+
return;
|
|
920
|
+
try {
|
|
921
|
+
await opts.memory.onCapture(runContext.turnNumber, sessionId);
|
|
922
|
+
}
|
|
923
|
+
catch (err) {
|
|
924
|
+
runContext.logger.warn("memory auto-capture failed", { error: err.message });
|
|
925
|
+
}
|
|
926
|
+
};
|
|
560
927
|
// Section 11 — `session-start` fires once after the persistence + run
|
|
561
928
|
// context boot is complete. Observational only (no deny/block effect).
|
|
562
929
|
await fireHook("session-start", {
|
|
@@ -594,7 +961,9 @@ export async function runChatLoop(opts) {
|
|
|
594
961
|
// CLAUDE.md auto-load behaviour. Pillar 3 compliance: each file's content
|
|
595
962
|
// is classified via boundary-classifier with origin "user" inside
|
|
596
963
|
// loadProjectMemory() (the user's repo is developer-trusted).
|
|
597
|
-
const projectMemory = await loadProjectMemory(
|
|
964
|
+
const projectMemory = await loadProjectMemory(opts.preferenceFiles !== undefined && opts.preferenceFiles.length > 0
|
|
965
|
+
? { extraFiles: opts.preferenceFiles }
|
|
966
|
+
: {});
|
|
598
967
|
const projectMemoryBlock = projectMemory.prompt.length > 0
|
|
599
968
|
? [
|
|
600
969
|
{
|
|
@@ -609,12 +978,50 @@ export async function runChatLoop(opts) {
|
|
|
609
978
|
.map((f) => f.filename)
|
|
610
979
|
.join(", ")}\n`);
|
|
611
980
|
}
|
|
981
|
+
// Feature #53 — auto-recall: at session start, pull the top-K relevant
|
|
982
|
+
// cross-session memories and inject them as a system block, mirroring the
|
|
983
|
+
// project-memory auto-load above. The caller supplies the `recall` seam
|
|
984
|
+
// (runtime-core stays independent of memory-store). Best-effort: a recall
|
|
985
|
+
// failure never aborts the run.
|
|
986
|
+
let memoryRecallBlock = [];
|
|
987
|
+
if (opts.memory?.autoRecall === true && opts.memory.recall !== undefined) {
|
|
988
|
+
try {
|
|
989
|
+
const seed = opts.memory.recallSeed ?? opts.instructions;
|
|
990
|
+
const k = opts.memory.recallK ?? 5;
|
|
991
|
+
const lines = await opts.memory.recall(seed, k);
|
|
992
|
+
if (lines.length > 0) {
|
|
993
|
+
// Pillar 3 — a recalled memory can embed content shaped by untrusted
|
|
994
|
+
// tool output from an earlier session, so it is NOT verbatim-safe like
|
|
995
|
+
// the developer's own repo files. Two defenses, mirroring how the rest
|
|
996
|
+
// of the security fabric treats injected content:
|
|
997
|
+
// 1. Delimiter safety — neutralize any `</recalled_memory>` closing
|
|
998
|
+
// tag inside a recalled line so a poisoned memory can't break out
|
|
999
|
+
// of its block and inject trailing instructions.
|
|
1000
|
+
// 2. Classification — run the assembled block through the SAME
|
|
1001
|
+
// `classifyBoundary(text, { origin: "user" })` best-effort path
|
|
1002
|
+
// `loadProjectMemory()` uses (pass policy: classify + emit trace
|
|
1003
|
+
// events without mutating), so recalled content flows through the
|
|
1004
|
+
// boundary classifier exactly like every other prompt-bound source.
|
|
1005
|
+
const body = lines
|
|
1006
|
+
.map((l) => `- ${escapeBoundaryDelimiter(l, "recalled_memory")}`)
|
|
1007
|
+
.join("\n");
|
|
1008
|
+
const text = `<recalled_memory>\nRelevant facts remembered from earlier sessions:\n${body}\n</recalled_memory>`;
|
|
1009
|
+
await classifyBoundary(text, { origin: "user" }).catch(() => undefined);
|
|
1010
|
+
memoryRecallBlock = [{ type: "text", text, cache_control: { type: "ephemeral" } }];
|
|
1011
|
+
process.stdout.write(`[memory] recalled ${lines.length} memory(ies) into the prompt\n`);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
catch (err) {
|
|
1015
|
+
runContext.logger.warn("memory auto-recall failed", { error: err.message });
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
612
1018
|
// Section 17 — runtime-core no longer prepends the Claude Code OAuth
|
|
613
1019
|
// prefix; `adapter-anthropic` handles that internally so each adapter
|
|
614
1020
|
// owns its provider-specific auth-shape requirements.
|
|
615
1021
|
let systemBlocks = [
|
|
616
1022
|
userInstructions,
|
|
617
1023
|
...projectMemoryBlock,
|
|
1024
|
+
...memoryRecallBlock,
|
|
618
1025
|
...skillsBlock,
|
|
619
1026
|
];
|
|
620
1027
|
// Section 27 — rotate cache markers on the system block array if the
|
|
@@ -814,6 +1221,21 @@ export async function runChatLoop(opts) {
|
|
|
814
1221
|
`tool denied: \`${tu.name}\` defaulted to "ask" and single-turn mode has no interactive surface to prompt on. ` +
|
|
815
1222
|
`Add an explicit rule to permissions.rules in your spec, e.g. \`{ type: alwaysAllow, pattern: ${tu.name} }\`, or run in REPL mode where "ask" can prompt.`;
|
|
816
1223
|
}
|
|
1224
|
+
// Advisor groundwork (item 14) — event the ask RESOLUTION. The publish
|
|
1225
|
+
// above fired BEFORE the approval prompt (decision "ask", no outcome);
|
|
1226
|
+
// this one fires after `askApproval` resolves — or after the
|
|
1227
|
+
// single-turn fallback collapses the ask to a deny — so offline advice
|
|
1228
|
+
// mining can measure how each tool's prompts are actually answered.
|
|
1229
|
+
// The advisor persistence subscriber (observability.ts) keys on
|
|
1230
|
+
// `askOutcome` to persist exactly this resolved form.
|
|
1231
|
+
bus.publish({
|
|
1232
|
+
...bus.envelope(),
|
|
1233
|
+
kind: "permission_decision",
|
|
1234
|
+
toolName: tu.name,
|
|
1235
|
+
decision: "ask",
|
|
1236
|
+
mode: permissionMode,
|
|
1237
|
+
askOutcome: approved ? "approved" : "denied",
|
|
1238
|
+
});
|
|
817
1239
|
}
|
|
818
1240
|
if (!approved) {
|
|
819
1241
|
return finish({
|
|
@@ -952,6 +1374,36 @@ export async function runChatLoop(opts) {
|
|
|
952
1374
|
originsFound: egress.originsFound,
|
|
953
1375
|
matchCount: egress.matchCount,
|
|
954
1376
|
});
|
|
1377
|
+
// Item 20 — emit the durable `egress_decision` audit record for every
|
|
1378
|
+
// non-pass verdict so egress history is triageable offline (the
|
|
1379
|
+
// trace-bus event above is ephemeral). Only the lineage SUMMARY is
|
|
1380
|
+
// stored — never the raw outbound payload (which is sensitive and
|
|
1381
|
+
// often carries the very tagged content that tripped the classifier),
|
|
1382
|
+
// matching the payload shape `@crewhaus/audit-log` documents for this
|
|
1383
|
+
// kind. Best-effort but surfaced: an I/O failure on the audit trail is
|
|
1384
|
+
// logged and swallowed so a full audit disk does not crash a governed
|
|
1385
|
+
// run, mirroring the justification sink above. Absent sink → no-op.
|
|
1386
|
+
if (opts.egressAuditSink !== undefined) {
|
|
1387
|
+
try {
|
|
1388
|
+
await opts.egressAuditSink.append({
|
|
1389
|
+
kind: "egress_decision",
|
|
1390
|
+
payload: {
|
|
1391
|
+
sinkId: egress.sinkId,
|
|
1392
|
+
sinkScope: egress.sinkScope,
|
|
1393
|
+
verdict: egress.verdict,
|
|
1394
|
+
originsFound: egress.originsFound,
|
|
1395
|
+
matchCount: egress.matchCount,
|
|
1396
|
+
},
|
|
1397
|
+
});
|
|
1398
|
+
}
|
|
1399
|
+
catch (err) {
|
|
1400
|
+
runContext.logger.warn("egress audit append failed", {
|
|
1401
|
+
toolUseId: tu.id,
|
|
1402
|
+
toolName: tu.name,
|
|
1403
|
+
error: err.message,
|
|
1404
|
+
});
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
955
1407
|
}
|
|
956
1408
|
if (egress.verdict === "block") {
|
|
957
1409
|
return finish({
|
|
@@ -1181,10 +1633,15 @@ export async function runChatLoop(opts) {
|
|
|
1181
1633
|
* they line up with the assistant turn's tool_use blocks.
|
|
1182
1634
|
*/
|
|
1183
1635
|
async function runToolBatch(toolUses) {
|
|
1184
|
-
|
|
1636
|
+
// Pass the full catalog so per-call classifiers (Task) can resolve a
|
|
1637
|
+
// dispatch's child tool set; without it, classifier-based tools stay
|
|
1638
|
+
// serial (fail-closed).
|
|
1639
|
+
const partition = partitionToolCalls(toolUses, (n) => toolByName.get(n), tools);
|
|
1640
|
+
const maxConcurrentTools = opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS;
|
|
1185
1641
|
runContext.logger.debug("tool partition", {
|
|
1186
1642
|
concurrent: partition.concurrent.map((b) => b.length),
|
|
1187
1643
|
serial: partition.serial.length,
|
|
1644
|
+
maxConcurrentTools,
|
|
1188
1645
|
});
|
|
1189
1646
|
// Map each tool_use's identity to its slot in the original order so
|
|
1190
1647
|
// results can be placed back in order regardless of the
|
|
@@ -1196,7 +1653,7 @@ export async function runChatLoop(opts) {
|
|
|
1196
1653
|
toolUses.forEach((tu, idx) => indexByBlock.set(tu, idx));
|
|
1197
1654
|
const results = new Array(toolUses.length);
|
|
1198
1655
|
for (const batch of partition.concurrent) {
|
|
1199
|
-
const settled = await
|
|
1656
|
+
const settled = await mapWithConcurrency(batch, maxConcurrentTools, (tu) => executeOneToolUse(tu));
|
|
1200
1657
|
batch.forEach((tu, i) => {
|
|
1201
1658
|
// biome-ignore lint/style/noNonNullAssertion: every block came from toolUses, so its index is registered.
|
|
1202
1659
|
results[indexByBlock.get(tu)] = settled[i];
|
|
@@ -1243,6 +1700,18 @@ export async function runChatLoop(opts) {
|
|
|
1243
1700
|
let recovery = initialRecoveryState;
|
|
1244
1701
|
const maxToolIterations = opts.maxToolIterations ?? DEFAULT_MAX_TOOL_ITERATIONS;
|
|
1245
1702
|
let toolIterations = 0;
|
|
1703
|
+
// Item 26 — tool_use blocks THIS turn produced (fed to the NEXT turn's
|
|
1704
|
+
// tier decision) and the misroute-recovery escalation latch (a fast-tier
|
|
1705
|
+
// failure forces `default` for the rest of this turn). `servedFastTier`
|
|
1706
|
+
// records whether the LAST attempt streamed on the fast tier, so the
|
|
1707
|
+
// error catch only escalates a genuine fast-tier misroute.
|
|
1708
|
+
let thisTurnToolUseCount = 0;
|
|
1709
|
+
let escalateTier = false;
|
|
1710
|
+
let servedFastTier = false;
|
|
1711
|
+
// Adaptive model routing — once a pool candidate fails, stick with the
|
|
1712
|
+
// strongest candidate for the rest of the run (misroute recovery, mirroring
|
|
1713
|
+
// the tier router's fast→default escalation latch).
|
|
1714
|
+
let escalatePool = false;
|
|
1246
1715
|
while (state.kind !== "Done") {
|
|
1247
1716
|
if (turnAbort.signal.aborted) {
|
|
1248
1717
|
state = transition(state, { kind: "Aborted" });
|
|
@@ -1280,6 +1749,10 @@ export async function runChatLoop(opts) {
|
|
|
1280
1749
|
// gap before the model responds now shows live progress. Cleared in
|
|
1281
1750
|
// the catch below on any model-call error.
|
|
1282
1751
|
out.spinner.start("thinking", { prefix: "agent> " });
|
|
1752
|
+
// Adaptive model routing — declared OUTSIDE the try so the catch can
|
|
1753
|
+
// read the chosen candidate + start time to record a failure reward.
|
|
1754
|
+
let poolTurn;
|
|
1755
|
+
let modelCallStartMs = performance.now();
|
|
1283
1756
|
try {
|
|
1284
1757
|
// Section 27 — rate-limit the model call before opening the
|
|
1285
1758
|
// stream. The keys are caller-configured (typically
|
|
@@ -1294,17 +1767,107 @@ export async function runChatLoop(opts) {
|
|
|
1294
1767
|
// → `us.anthropic.…`) so cost-tracker pricing lookups and the
|
|
1295
1768
|
// OTel `gen_ai.request.model` attribute match; the original
|
|
1296
1769
|
// spec string rides along as `specModel` when it differs.
|
|
1770
|
+
// Item 22 — with a failover chain the serving candidate may not
|
|
1771
|
+
// be the primary: stamp the request with the chain's routing
|
|
1772
|
+
// plan (best-effort; the response event below uses lastServed(),
|
|
1773
|
+
// which is exact, so cost-tracker pricing always keys on the
|
|
1774
|
+
// model that actually served).
|
|
1775
|
+
const plannedCandidate = failoverChain?.plan();
|
|
1776
|
+
let reqWireModelId = plannedCandidate?.modelId ?? wireModelId;
|
|
1777
|
+
let reqProviderId = plannedCandidate?.providerId ?? providerId;
|
|
1778
|
+
// Item 27 — after a budget degrade, `failoverChain` is null and
|
|
1779
|
+
// `degradedSpecModel` is the running spec model.
|
|
1780
|
+
let reqSpecModel = plannedCandidate?.modelString ?? degradedSpecModel ?? opts.model;
|
|
1781
|
+
// The adapter this turn streams through: the primary/chain by
|
|
1782
|
+
// default, or a tier adapter when the two-tier router is active.
|
|
1783
|
+
let turnAdapter = adapter;
|
|
1784
|
+
// Adaptive model routing — the PolicyRouter picks a pool candidate
|
|
1785
|
+
// this turn (static / heuristic / learned) from the SAME
|
|
1786
|
+
// deterministic signals as the tier router. `escalatePool` forces
|
|
1787
|
+
// the strongest candidate after a prior pool failure (misroute
|
|
1788
|
+
// recovery). Publish a `model_route` event and stream through the
|
|
1789
|
+
// chosen candidate; the outcome is folded into the reward
|
|
1790
|
+
// scoreboard post-turn. Takes precedence over the tier router (the
|
|
1791
|
+
// two are mutually exclusive in the spec).
|
|
1792
|
+
if (poolRouter !== undefined) {
|
|
1793
|
+
const base = poolRouter.route({
|
|
1794
|
+
contextTokens: estimateTokens(messages),
|
|
1795
|
+
toolsInPlay: (anthropicTools?.length ?? 0) > 0,
|
|
1796
|
+
turnIndex: Math.max(0, runContext.turnNumber - 1),
|
|
1797
|
+
priorTurnToolUseCount,
|
|
1798
|
+
});
|
|
1799
|
+
const decision = escalatePool
|
|
1800
|
+
? {
|
|
1801
|
+
...base,
|
|
1802
|
+
candidate: poolRouter.escalation(),
|
|
1803
|
+
reason: `escalated to strongest candidate after a pool failure (was: ${base.reason})`,
|
|
1804
|
+
explored: false,
|
|
1805
|
+
}
|
|
1806
|
+
: base;
|
|
1807
|
+
turnAdapter = decision.candidate.adapter;
|
|
1808
|
+
reqWireModelId = decision.candidate.modelId;
|
|
1809
|
+
reqProviderId = decision.candidate.adapter.providerId;
|
|
1810
|
+
reqSpecModel = decision.candidate.modelString;
|
|
1811
|
+
poolTurn = {
|
|
1812
|
+
modelString: decision.candidate.modelString,
|
|
1813
|
+
routeKey: decision.routeKey,
|
|
1814
|
+
};
|
|
1815
|
+
bus.publish({
|
|
1816
|
+
...bus.envelope(),
|
|
1817
|
+
kind: "model_route",
|
|
1818
|
+
routeKey: decision.routeKey,
|
|
1819
|
+
model: decision.candidate.modelId,
|
|
1820
|
+
policy: decision.policy,
|
|
1821
|
+
reason: decision.reason,
|
|
1822
|
+
...(decision.explored ? { explored: true } : {}),
|
|
1823
|
+
...(poolPolicyVersion !== undefined ? { policyVersion: poolPolicyVersion } : {}),
|
|
1824
|
+
});
|
|
1825
|
+
}
|
|
1826
|
+
else if (tierRouter !== undefined) {
|
|
1827
|
+
// Item 26 — two-tier turn-difficulty router. Pick a tier from
|
|
1828
|
+
// DETERMINISTIC per-turn signals (context size, tools-in-play,
|
|
1829
|
+
// turn index, prior-turn tool_use density); a fast-tier misroute
|
|
1830
|
+
// recovery forces `default` (see the catch below). Publish the
|
|
1831
|
+
// decision as a `model_tier_route` event, then stream through the
|
|
1832
|
+
// chosen tier's already-resolved adapter.
|
|
1833
|
+
const decision = escalateTier
|
|
1834
|
+
? { tier: "default", reason: "escalated after fast-tier failure" }
|
|
1835
|
+
: tierRouter.route({
|
|
1836
|
+
contextTokens: estimateTokens(messages),
|
|
1837
|
+
toolsInPlay: (anthropicTools?.length ?? 0) > 0,
|
|
1838
|
+
// `turnNumber` is 1-based (incremented before the turn
|
|
1839
|
+
// runs); the router wants a 0-based index so its
|
|
1840
|
+
// first-turn check keys on the opening turn.
|
|
1841
|
+
turnIndex: Math.max(0, runContext.turnNumber - 1),
|
|
1842
|
+
priorTurnToolUseCount,
|
|
1843
|
+
});
|
|
1844
|
+
const chosen = tierRouter.tier(decision.tier);
|
|
1845
|
+
turnAdapter = chosen.adapter;
|
|
1846
|
+
reqWireModelId = chosen.modelId;
|
|
1847
|
+
reqProviderId = chosen.adapter.providerId;
|
|
1848
|
+
reqSpecModel = chosen.modelString;
|
|
1849
|
+
servedFastTier = decision.tier === "fast";
|
|
1850
|
+
bus.publish({
|
|
1851
|
+
...bus.envelope(),
|
|
1852
|
+
kind: "model_tier_route",
|
|
1853
|
+
tier: decision.tier,
|
|
1854
|
+
model: chosen.modelId,
|
|
1855
|
+
reason: decision.reason,
|
|
1856
|
+
...(escalateTier ? { escalated: true } : {}),
|
|
1857
|
+
});
|
|
1858
|
+
}
|
|
1297
1859
|
bus.publish({
|
|
1298
1860
|
...modelStartEnv,
|
|
1299
1861
|
kind: "model_request",
|
|
1300
|
-
model:
|
|
1301
|
-
...(
|
|
1302
|
-
provider:
|
|
1862
|
+
model: reqWireModelId,
|
|
1863
|
+
...(reqSpecModel !== reqWireModelId ? { specModel: reqSpecModel } : {}),
|
|
1864
|
+
provider: reqProviderId,
|
|
1303
1865
|
messageCount: messages.length,
|
|
1304
1866
|
toolCount: anthropicTools?.length ?? 0,
|
|
1305
1867
|
streaming: opts.streaming === true,
|
|
1306
1868
|
});
|
|
1307
1869
|
const t0Model = performance.now();
|
|
1870
|
+
modelCallStartMs = t0Model;
|
|
1308
1871
|
let streamChunkIndex = 0;
|
|
1309
1872
|
// Section 17 — `adapter.stream(req)` returns
|
|
1310
1873
|
// `AsyncIterable<StreamEvent>`; we accumulate via
|
|
@@ -1320,8 +1883,11 @@ export async function runChatLoop(opts) {
|
|
|
1320
1883
|
// bookkeeping is wire-compatible for the block kinds we
|
|
1321
1884
|
// actually consume (text / image / tool_use / tool_result
|
|
1322
1885
|
// / thinking).
|
|
1323
|
-
|
|
1324
|
-
|
|
1886
|
+
// Item 22 — the failover chain rewrites `model` per serving
|
|
1887
|
+
// candidate internally; the planned id here is the single-adapter
|
|
1888
|
+
// value and the chain's best-effort prediction otherwise.
|
|
1889
|
+
const reqStream = turnAdapter.stream({
|
|
1890
|
+
model: reqWireModelId,
|
|
1325
1891
|
system: systemBlocks.map((b) => ({
|
|
1326
1892
|
type: "text",
|
|
1327
1893
|
text: b.text,
|
|
@@ -1356,6 +1922,10 @@ export async function runChatLoop(opts) {
|
|
|
1356
1922
|
toolByName,
|
|
1357
1923
|
abortSignal: turnAbort.signal,
|
|
1358
1924
|
onTextDelta,
|
|
1925
|
+
// Honor the same concurrency bound as the non-streaming
|
|
1926
|
+
// path — otherwise `maxConcurrentTools` would be silently
|
|
1927
|
+
// ignored under `streaming: true`.
|
|
1928
|
+
maxConcurrent: opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS,
|
|
1359
1929
|
runTool: (block) => executeOneToolUse({
|
|
1360
1930
|
id: block.id,
|
|
1361
1931
|
name: block.name,
|
|
@@ -1388,19 +1958,41 @@ export async function runChatLoop(opts) {
|
|
|
1388
1958
|
await logEvent("assistant_message", { content: committedStreamContent });
|
|
1389
1959
|
}
|
|
1390
1960
|
terminalContent = [...committedStreamContent];
|
|
1961
|
+
// Item 22 — lastServed() is exact: the candidate that actually
|
|
1962
|
+
// streamed this response. Cost-tracker prices off this pair.
|
|
1963
|
+
const servedStreaming = failoverChain?.lastServed();
|
|
1964
|
+
// With a pool, the served identity is the chosen candidate (no
|
|
1965
|
+
// failover chain when pooling), so cost-tracker prices on it.
|
|
1966
|
+
const respModelIdS = poolTurn !== undefined ? reqWireModelId : (servedStreaming?.modelId ?? wireModelId);
|
|
1967
|
+
const respSpecModelS = poolTurn !== undefined
|
|
1968
|
+
? poolTurn.modelString
|
|
1969
|
+
: (servedStreaming?.modelString ?? degradedSpecModel ?? opts.model);
|
|
1970
|
+
const respProviderS = poolTurn !== undefined
|
|
1971
|
+
? reqProviderId
|
|
1972
|
+
: (servedStreaming?.providerId ?? providerId);
|
|
1391
1973
|
bus.publish({
|
|
1392
1974
|
...bus.envelope(),
|
|
1393
1975
|
spanId: modelStartEnv.spanId,
|
|
1394
1976
|
kind: "model_response",
|
|
1395
1977
|
// Wire model id, NOT the spec string — cost-tracker resolves
|
|
1396
1978
|
// pricing on this field (see the model_request publish above).
|
|
1397
|
-
model:
|
|
1398
|
-
...(
|
|
1399
|
-
provider:
|
|
1979
|
+
model: respModelIdS,
|
|
1980
|
+
...(respSpecModelS !== respModelIdS ? { specModel: respSpecModelS } : {}),
|
|
1981
|
+
provider: respProviderS,
|
|
1400
1982
|
stopReason: stopReason ?? "end_turn",
|
|
1401
1983
|
usage,
|
|
1402
1984
|
durationMs: performance.now() - t0Model,
|
|
1403
1985
|
});
|
|
1986
|
+
// Adaptive model routing — fold this successful turn into the
|
|
1987
|
+
// reward scoreboard so the learned policy improves next run.
|
|
1988
|
+
if (poolTurn !== undefined) {
|
|
1989
|
+
const costUsd = poolCostUsd(respProviderS, respModelIdS, usage);
|
|
1990
|
+
recordPoolOutcome(poolTurn, {
|
|
1991
|
+
success: true,
|
|
1992
|
+
latencyMs: performance.now() - t0Model,
|
|
1993
|
+
...(costUsd !== undefined ? { costUsd } : {}),
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1404
1996
|
await fireHook("post-model", {
|
|
1405
1997
|
streaming: true,
|
|
1406
1998
|
contentBlocks: finalContent.length,
|
|
@@ -1410,6 +2002,8 @@ export async function runChatLoop(opts) {
|
|
|
1410
2002
|
state = transition(state, { kind: "ModelReturnedText" });
|
|
1411
2003
|
break;
|
|
1412
2004
|
}
|
|
2005
|
+
// Item 26 — accrue this turn's tool_use count (next-turn tier signal).
|
|
2006
|
+
thisTurnToolUseCount += toolUses.length;
|
|
1413
2007
|
for (const tu of toolUses) {
|
|
1414
2008
|
toolUseHistory.push({ id: tu.id, name: tu.name, input: tu.input });
|
|
1415
2009
|
}
|
|
@@ -1447,19 +2041,38 @@ export async function runChatLoop(opts) {
|
|
|
1447
2041
|
},
|
|
1448
2042
|
});
|
|
1449
2043
|
endTurn();
|
|
2044
|
+
// Item 22 — lastServed() is exact: the candidate that actually
|
|
2045
|
+
// streamed this response. Cost-tracker prices off this pair.
|
|
2046
|
+
const servedCandidate = failoverChain?.lastServed();
|
|
2047
|
+
// With a pool, the served identity is the chosen candidate.
|
|
2048
|
+
const respWireModelId = poolTurn !== undefined ? reqWireModelId : (servedCandidate?.modelId ?? wireModelId);
|
|
2049
|
+
const respSpecModel = poolTurn !== undefined
|
|
2050
|
+
? poolTurn.modelString
|
|
2051
|
+
: (servedCandidate?.modelString ?? degradedSpecModel ?? opts.model);
|
|
2052
|
+
const respProvider = poolTurn !== undefined ? reqProviderId : (servedCandidate?.providerId ?? providerId);
|
|
1450
2053
|
bus.publish({
|
|
1451
2054
|
...bus.envelope(),
|
|
1452
2055
|
spanId: modelStartEnv.spanId,
|
|
1453
2056
|
kind: "model_response",
|
|
1454
2057
|
// Wire model id, NOT the spec string — cost-tracker resolves
|
|
1455
2058
|
// pricing on this field (see the model_request publish above).
|
|
1456
|
-
model:
|
|
1457
|
-
...(
|
|
1458
|
-
provider:
|
|
2059
|
+
model: respWireModelId,
|
|
2060
|
+
...(respSpecModel !== respWireModelId ? { specModel: respSpecModel } : {}),
|
|
2061
|
+
provider: respProvider,
|
|
1459
2062
|
stopReason: final.stopReason,
|
|
1460
2063
|
usage: final.usage,
|
|
1461
2064
|
durationMs: performance.now() - t0Model,
|
|
1462
2065
|
});
|
|
2066
|
+
// Adaptive model routing — fold this successful turn into the
|
|
2067
|
+
// reward scoreboard so the learned policy improves next run.
|
|
2068
|
+
if (poolTurn !== undefined) {
|
|
2069
|
+
const costUsd = poolCostUsd(respProvider, respWireModelId, final.usage);
|
|
2070
|
+
recordPoolOutcome(poolTurn, {
|
|
2071
|
+
success: true,
|
|
2072
|
+
latencyMs: performance.now() - t0Model,
|
|
2073
|
+
...(costUsd !== undefined ? { costUsd } : {}),
|
|
2074
|
+
});
|
|
2075
|
+
}
|
|
1463
2076
|
// Persist the assistant turn. On a clean stop the FULL content-block
|
|
1464
2077
|
// array is committed so subsequent `tool_result` references resolve.
|
|
1465
2078
|
//
|
|
@@ -1529,6 +2142,28 @@ export async function runChatLoop(opts) {
|
|
|
1529
2142
|
break;
|
|
1530
2143
|
}
|
|
1531
2144
|
lastErrorForRecovery = err;
|
|
2145
|
+
// Item 26 — a fast-tier turn that FAILED is a MISROUTE: latch the
|
|
2146
|
+
// escalation so the recovery retry re-runs on the `default` tier
|
|
2147
|
+
// (the loop re-enters NeedModel via the recovery ladder, which
|
|
2148
|
+
// then picks `default`). Composes with the existing recovery
|
|
2149
|
+
// actions — a retry/continue now streams on the sturdier tier.
|
|
2150
|
+
if (tierRouter !== undefined && servedFastTier) {
|
|
2151
|
+
escalateTier = true;
|
|
2152
|
+
}
|
|
2153
|
+
// Adaptive model routing — record the failed candidate as a
|
|
2154
|
+
// negative reward (drops the cost term: no usage on a throw) and
|
|
2155
|
+
// latch the escalation so the recovery retry re-runs on the
|
|
2156
|
+
// strongest candidate, unless the failed candidate WAS already the
|
|
2157
|
+
// strongest (then escalation buys nothing).
|
|
2158
|
+
if (poolRouter !== undefined && poolTurn !== undefined) {
|
|
2159
|
+
recordPoolOutcome(poolTurn, {
|
|
2160
|
+
success: false,
|
|
2161
|
+
latencyMs: performance.now() - modelCallStartMs,
|
|
2162
|
+
});
|
|
2163
|
+
if (poolTurn.modelString !== poolRouter.escalation().modelString) {
|
|
2164
|
+
escalatePool = true;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
1532
2167
|
const errObj = err;
|
|
1533
2168
|
const errorName = typeof errObj.name === "string" ? errObj.name : "Error";
|
|
1534
2169
|
const errorMessage = typeof errObj.message === "string" ? errObj.message : String(err);
|
|
@@ -1556,6 +2191,9 @@ export async function runChatLoop(opts) {
|
|
|
1556
2191
|
state = transition(state, { kind: "Aborted" });
|
|
1557
2192
|
break;
|
|
1558
2193
|
}
|
|
2194
|
+
// Item 26 — accrue this turn's tool_use count (a next-turn tier
|
|
2195
|
+
// signal): a dense tool turn marks an active multi-step task.
|
|
2196
|
+
thisTurnToolUseCount += state.toolUses.length;
|
|
1559
2197
|
const toolResults = await runToolBatch(state.toolUses);
|
|
1560
2198
|
messages.push({ role: "user", content: toolResults });
|
|
1561
2199
|
await logEvent("user_message", { content: toolResults });
|
|
@@ -1576,7 +2214,10 @@ export async function runChatLoop(opts) {
|
|
|
1576
2214
|
// emitting `BudgetExceeded` from `runOneTurn`, restore a
|
|
1577
2215
|
// `NeedCompaction` → `CompactionDone` arm (and cover it) here.
|
|
1578
2216
|
case "NeedRecovery": {
|
|
1579
|
-
|
|
2217
|
+
// Item 23 / Section 55 — consult the spec's failure_taxonomy first
|
|
2218
|
+
// (named classes take precedence over the built-in flow), so a
|
|
2219
|
+
// matched `switch-model` (or retry/compact/…) entry wins.
|
|
2220
|
+
const action = recover(lastErrorForRecovery, recovery, opts.failureTaxonomy);
|
|
1580
2221
|
recovery = advanceState(recovery, action);
|
|
1581
2222
|
runContext.logger.info("recovery.action", {
|
|
1582
2223
|
kind: action.kind,
|
|
@@ -1675,6 +2316,32 @@ export async function runChatLoop(opts) {
|
|
|
1675
2316
|
state = transition(state, { kind: "RecoveryDone" });
|
|
1676
2317
|
break;
|
|
1677
2318
|
}
|
|
2319
|
+
case "switch-model": {
|
|
2320
|
+
// Item 23 — a `switch-model` failure_taxonomy verdict: abandon
|
|
2321
|
+
// the active provider candidate FOR THIS TURN and re-issue the
|
|
2322
|
+
// same request onto the next failover candidate. Mechanism:
|
|
2323
|
+
// force the active candidate's breaker open so the chain
|
|
2324
|
+
// reroutes on the next `stream()`; the chain publishes the
|
|
2325
|
+
// `model_failover` (reason breaker_open) itself. No message
|
|
2326
|
+
// mutation and no backoff sleep — a switch is instant. Without
|
|
2327
|
+
// a failover chain (no `modelFallbacks`) there is nothing to
|
|
2328
|
+
// switch to: this degrades to a plain re-issue against the same
|
|
2329
|
+
// adapter, which the per-turn budget bounds. RecoveryDone loops
|
|
2330
|
+
// back to NeedModel, which re-streams.
|
|
2331
|
+
const tripped = failoverChain?.tripActive("switch-model recovery");
|
|
2332
|
+
if (tripped !== undefined) {
|
|
2333
|
+
out.spinner.start("switching model");
|
|
2334
|
+
out.spinner.stop();
|
|
2335
|
+
runContext.logger.info("recovery.switch-model tripped candidate", {
|
|
2336
|
+
from: tripped,
|
|
2337
|
+
});
|
|
2338
|
+
}
|
|
2339
|
+
else {
|
|
2340
|
+
runContext.logger.warn("recovery.switch-model: no failover chain — re-issuing on the same model");
|
|
2341
|
+
}
|
|
2342
|
+
state = transition(state, { kind: "RecoveryDone" });
|
|
2343
|
+
break;
|
|
2344
|
+
}
|
|
1678
2345
|
case "fail":
|
|
1679
2346
|
throw new RuntimeError(`recovery failed: ${action.reason}`);
|
|
1680
2347
|
}
|
|
@@ -1682,6 +2349,9 @@ export async function runChatLoop(opts) {
|
|
|
1682
2349
|
}
|
|
1683
2350
|
}
|
|
1684
2351
|
}
|
|
2352
|
+
// Item 26 — hand this turn's tool_use count to the next turn's tier
|
|
2353
|
+
// decision (the prior-tool-density signal).
|
|
2354
|
+
priorTurnToolUseCount = thisTurnToolUseCount;
|
|
1685
2355
|
return { terminalContent };
|
|
1686
2356
|
}
|
|
1687
2357
|
// Single-shot path used by the workflow target and (Section 12) the
|
|
@@ -1768,9 +2438,16 @@ export async function runChatLoop(opts) {
|
|
|
1768
2438
|
turnCount: runContext.turnNumber,
|
|
1769
2439
|
reason: "complete",
|
|
1770
2440
|
});
|
|
2441
|
+
await maybeAutoCapture();
|
|
1771
2442
|
await subscribers.flushAll();
|
|
1772
2443
|
await subscribers.shutdownAll();
|
|
1773
2444
|
costPersistUnsubscribe?.();
|
|
2445
|
+
failoverNoteUnsubscribe?.();
|
|
2446
|
+
incidentCollector?.unsubscribe();
|
|
2447
|
+
budgetMeter?.unsubscribe();
|
|
2448
|
+
advisorPersist?.unsubscribe();
|
|
2449
|
+
mcpStatsPersist?.unsubscribe();
|
|
2450
|
+
printAdvisorDigest(advisorPersist);
|
|
1774
2451
|
await sessionStore
|
|
1775
2452
|
.update(sessionId, { lastTurnIndex: runContext.turnNumber })
|
|
1776
2453
|
.catch((err) => {
|
|
@@ -1909,6 +2586,13 @@ export async function runChatLoop(opts) {
|
|
|
1909
2586
|
}
|
|
1910
2587
|
}
|
|
1911
2588
|
}
|
|
2589
|
+
// Item 27 — run-level spend cap. Enforced HERE (a real turn is about
|
|
2590
|
+
// to run, input is not exit/blank) so an idle prompt never trips it.
|
|
2591
|
+
// On a `stop` breach we break before committing the input — the turn
|
|
2592
|
+
// never runs. On a `degrade` breach `enforceBudget` has already
|
|
2593
|
+
// swapped the primary adapter, so this turn runs on the cheaper model.
|
|
2594
|
+
if ((await enforceBudget()) === "stop")
|
|
2595
|
+
break;
|
|
1912
2596
|
messages.push({ role: "user", content: effectiveInput });
|
|
1913
2597
|
await logEvent("user_message", { content: effectiveInput });
|
|
1914
2598
|
runContext.turnNumber += 1;
|
|
@@ -1979,9 +2663,16 @@ export async function runChatLoop(opts) {
|
|
|
1979
2663
|
turnCount: runContext.turnNumber,
|
|
1980
2664
|
reason: runAbort.signal.aborted ? "abort" : "exit",
|
|
1981
2665
|
});
|
|
2666
|
+
await maybeAutoCapture();
|
|
1982
2667
|
await subscribers.flushAll();
|
|
1983
2668
|
await subscribers.shutdownAll();
|
|
1984
2669
|
costPersistUnsubscribe?.();
|
|
2670
|
+
failoverNoteUnsubscribe?.();
|
|
2671
|
+
incidentCollector?.unsubscribe();
|
|
2672
|
+
budgetMeter?.unsubscribe();
|
|
2673
|
+
advisorPersist?.unsubscribe();
|
|
2674
|
+
mcpStatsPersist?.unsubscribe();
|
|
2675
|
+
printAdvisorDigest(advisorPersist);
|
|
1985
2676
|
if (shouldInstallSigint) {
|
|
1986
2677
|
process.removeListener("SIGINT", sigintHandler);
|
|
1987
2678
|
}
|
|
@@ -1995,6 +2686,18 @@ export async function runChatLoop(opts) {
|
|
|
1995
2686
|
}
|
|
1996
2687
|
return "";
|
|
1997
2688
|
}
|
|
2689
|
+
/**
|
|
2690
|
+
* Item 14 in-run digest — one stderr line at session end when the advisor
|
|
2691
|
+
* persistence subscriber's cheap tally tripped a threshold, pointing at
|
|
2692
|
+
* `crewhaus advise --session <id>` for the full report. stderr (not stdout)
|
|
2693
|
+
* so piped/captured agent output is never polluted; silent on healthy runs
|
|
2694
|
+
* and when the subscriber is disabled (CREWHAUS_ADVISOR_EVENTS=0).
|
|
2695
|
+
*/
|
|
2696
|
+
function printAdvisorDigest(advisorPersist) {
|
|
2697
|
+
const line = advisorPersist?.digestLine();
|
|
2698
|
+
if (line !== undefined)
|
|
2699
|
+
process.stderr.write(`${line}\n`);
|
|
2700
|
+
}
|
|
1998
2701
|
function isAbortError(err) {
|
|
1999
2702
|
// Section 17 — runtime-core no longer imports the Anthropic SDK at
|
|
2000
2703
|
// runtime, so we name-match instead of `instanceof`. The provider
|