@crewhaus/runtime-core 0.1.8 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,19 +1,22 @@
1
+ import { dirname as pathDirname } from "node:path";
1
2
  import * as readline from "node:readline/promises";
2
3
  import { createAbortTree } from "@crewhaus/abort-controller";
3
4
  import { collectFinalMessage, consumeStream, } from "@crewhaus/adapter-anthropic";
4
- import { setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
5
+ import { classifyBoundary, setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
5
6
  import { wrap as wrapWithCircuitBreaker } from "@crewhaus/circuit-breaker";
6
7
  import { autoCompact } from "@crewhaus/compaction-autocompact";
7
8
  import { snip } from "@crewhaus/compaction-snip";
9
+ import { DEFAULT_PRICING, computeCostMicros, createCostTracker, resolvePricing, } from "@crewhaus/cost-tracker";
8
10
  import { classifyEgress, summarizeEgress, } from "@crewhaus/egress-classifier";
9
11
  import { ConfigError, RuntimeError } from "@crewhaus/errors";
10
12
  import { openEventLog } from "@crewhaus/event-log";
11
13
  import { aggregateDecisions, runHooks } from "@crewhaus/hooks-engine";
12
- import { parseModelString, resolveModel } from "@crewhaus/model-router";
14
+ import { createFailoverChain, createPolicyRouter, createTierRouter, parseModelString, resolveModel, } from "@crewhaus/model-router";
13
15
  import { BUILTIN_DEFAULT_RULES, emptyRuleSet, evaluateJustification, evaluateWithReason, ruleBasedJustificationJudge, } from "@crewhaus/permission-engine";
14
16
  import { manage as manageCacheMarkers } from "@crewhaus/prompt-cache-manager";
15
17
  import { buildRedactionNotice, classifyText, llmClassifierEnabled, } from "@crewhaus/prompt-injection-detector";
16
18
  import { advanceState, initialRecoveryState, recover, } from "@crewhaus/recovery-engine";
19
+ import { computeReward, openScoreboard, } from "@crewhaus/routing-store";
17
20
  import { createRunContext, tagContent } from "@crewhaus/run-context";
18
21
  import { createSessionStore } from "@crewhaus/session-store";
19
22
  import { formatSkillsForPrompt } from "@crewhaus/skills-registry";
@@ -30,7 +33,8 @@ import { TraceEventBus } from "@crewhaus/trace-event-bus";
30
33
  import { initialState, transition, } from "@crewhaus/turn-state-machine";
31
34
  import { zodToJsonSchema } from "zod-to-json-schema";
32
35
  import { createCliMarkdownRenderer, isCliMarkdownEnabled, } from "./cli-markdown";
33
- import { attachDefaultSubscribers } from "./observability";
36
+ import { attachIncidentCollector } from "./incident-collector";
37
+ import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, } from "./observability";
34
38
  import { loadProjectMemory } from "./project-memory";
35
39
  import { createCliOutput, isSpinnerEnabled } from "./spinner";
36
40
  /**
@@ -105,14 +109,59 @@ const DEFAULT_CONTEXT_LIMIT = 200_000;
105
109
  // agentic turn reaches it, low enough to stop a runaway/injected tool loop
106
110
  // before it burns a provider quota.
107
111
  const DEFAULT_MAX_TOOL_ITERATIONS = 500;
112
+ // Upper bound on how many concurrency-safe tool calls in one turn run at
113
+ // once. Bounds parallel sub-agent (`Task`) fan-out so a wide dispatch
114
+ // can't open unbounded model connections and trip a provider rate limit;
115
+ // also caps parallel reads, whose per-call cost is negligible. Configurable
116
+ // via `runChatLoop({ maxConcurrentTools })`.
117
+ const DEFAULT_MAX_CONCURRENT_TOOLS = 4;
108
118
  const DEFAULT_COMPACTION_THRESHOLD = 0.85;
109
119
  const DEFAULT_SNIP_KEEP_HEAD = 4;
110
120
  const DEFAULT_SNIP_KEEP_TAIL = 20;
121
+ /**
122
+ * Run `items` through `fn` with at most `limit` in flight at once,
123
+ * preserving input order in the returned results. A pool of `limit`
124
+ * workers pull from a shared cursor, so a slow item never blocks a fast
125
+ * one behind it (unlike fixed-size `Promise.all` chunks). `limit <= 0`
126
+ * is clamped to 1; a limit larger than the batch just runs everything.
127
+ * Exported for unit testing of the concurrency bound.
128
+ */
129
+ export async function mapWithConcurrency(items, limit, fn) {
130
+ const results = new Array(items.length);
131
+ const bound = Math.max(1, Math.min(limit, items.length));
132
+ let cursor = 0;
133
+ const worker = async () => {
134
+ while (true) {
135
+ const i = cursor;
136
+ cursor += 1;
137
+ if (i >= items.length)
138
+ return;
139
+ results[i] = await fn(items[i], i);
140
+ }
141
+ };
142
+ await Promise.all(Array.from({ length: bound }, () => worker()));
143
+ return results;
144
+ }
111
145
  // Section 17 — `resolveAuth` / `createAnthropicClient` / `ResolvedAuth` /
112
146
  // `OAUTH_BETAS` / `CLAUDE_CODE_HEADERS` / `CLAUDE_CODE_SYSTEM_PREFIX`
113
147
  // moved into `@crewhaus/adapter-anthropic`. Re-exported here so legacy
114
148
  // importers (eval-judge fallback, downstream tests) keep compiling.
115
149
  export { CLAUDE_CODE_HEADERS, CLAUDE_CODE_SYSTEM_PREFIX, OAUTH_BETAS, createAnthropicClient, resolveAuth, } from "@crewhaus/adapter-anthropic";
150
+ // Ops item 36 — boot-time self-heal janitor for daemon shapes. Lives in its
151
+ // own module (`./janitor`) but is re-exported here because emitted bundles
152
+ // only import the package root.
153
+ export { createJanitor } from "./janitor";
154
+ // Ops item 31 — alert watchdog seams re-exported so the CLI/codegen can build
155
+ // the durable + off-box alert sink (audit append + settings.json alert hook /
156
+ // webhook) it passes as `RunChatLoopOptions.alertSink`.
157
+ export { attachDefaultSubscribers, } from "./observability";
158
+ // Ops item 37 — SLO monitor seams re-exported so the CLI/codegen can build the
159
+ // injected mitigation ladder (audit / pause-intake / rollback) it passes as
160
+ // `RunChatLoopOptions.sloSink`, and the doctor probe can share the breach types.
161
+ export { DEFAULT_SLO_WINDOW_MS, MIN_SLO_SAMPLES, SloWindow, attachSloMonitor, detectSloBreaches, } from "./slo-monitor";
162
+ // Ops item 32 — incident collector seams re-exported so the CLI can share the
163
+ // raw-capture shape + trigger classifier with `incident collect`.
164
+ export { attachIncidentCollector, classifyIncidentTrigger, } from "./incident-collector";
116
165
  /**
117
166
  * Reconcile a message history by dropping "orphan" `tool_use` blocks — a
118
167
  * `tool_use` with no answering `tool_result` anywhere later in the
@@ -289,6 +338,19 @@ export function defaultSinkScope(toolName) {
289
338
  return "external-dynamic";
290
339
  return "external-configured";
291
340
  }
341
+ /**
342
+ * Neutralize a boundary-block closing delimiter embedded in untrusted content
343
+ * so a poisoned line can't terminate its wrapper block early and inject
344
+ * trailing instructions. Replaces `</tag>` (any casing/whitespace) with a
345
+ * visually-inert `<\/tag>` so the content survives for the model to read but
346
+ * no longer parses as the real closing tag. Used for the recalled-memory block
347
+ * (#53) — recalled memories may carry content shaped by untrusted tool output.
348
+ */
349
+ function escapeBoundaryDelimiter(text, tag) {
350
+ // Match `</tag>` tolerantly (leading whitespace inside the tag, any casing).
351
+ const re = new RegExp(`</\\s*${tag}\\s*>`, "gi");
352
+ return text.replace(re, `<\\/${tag}>`);
353
+ }
292
354
  /**
293
355
  * Strip the model-string provider grammar down to the wire model id
294
356
  * (`"bedrock/us.anthropic.claude-..."` → `"us.anthropic.claude-..."`)
@@ -305,6 +367,23 @@ function bestEffortWireModelId(modelString) {
305
367
  return modelString;
306
368
  }
307
369
  }
370
+ /**
371
+ * A short, stable fingerprint of a `model_pool` config, stamped on every
372
+ * `model_route` event as `policyVersion` so a learned decision can be tied back
373
+ * to the exact policy that made it. Deterministic (djb2 over the candidate
374
+ * roster + policy + objective) — no clock, no randomness.
375
+ */
376
+ function poolFingerprint(pool) {
377
+ const canonical = JSON.stringify({
378
+ p: pool.policy,
379
+ c: pool.candidates.map((c) => [c.model, [...c.tags].sort()]),
380
+ o: pool.objective ?? null,
381
+ });
382
+ let hash = 5381;
383
+ for (let i = 0; i < canonical.length; i++)
384
+ hash = ((hash << 5) + hash + canonical.charCodeAt(i)) >>> 0;
385
+ return `pool-${hash.toString(36)}`;
386
+ }
308
387
  export async function runChatLoop(opts) {
309
388
  // Pillar 3 — make the model-backed Layer-3 classifier reachable at EVERY
310
389
  // trust boundary (MCP / sub-agent / channel / federation / skill /
@@ -333,25 +412,65 @@ export async function runChatLoop(opts) {
333
412
  providerId: opts._adapter.providerId,
334
413
  }
335
414
  : await resolveModel(opts.model);
336
- // Section 27 — wrap the primary adapter in a circuit breaker when
337
- // `circuitBreaker` opts are set (or by default in production codegen).
338
- // The breaker is hung off the run-context's bus so state-change events
339
- // surface for audit-log + structured-event-printer + OTel.
415
+ // Section 27 / item 22 — resilience wrapping around the primary adapter.
416
+ // - `modelFallbacks` non-empty → a model-router failover chain: every
417
+ // candidate (primary first) breaker-wrapped with the `circuitBreaker`
418
+ // tuning; calls route to the first candidate whose breaker admits
419
+ // traffic; `model_failover` trace events surface routing changes.
420
+ // - `circuitBreaker` alone → the single-adapter breaker wrap.
421
+ // The chain publishes through a LATE-BOUND bus getter: the run's real bus
422
+ // is minted below (createRunContext), so `observabilityBus` is re-pointed
423
+ // at it once it exists — a `crewhaus run` without a caller-supplied
424
+ // runContext still surfaces model_failover + circuit_state_changed.
340
425
  const baseAdapter = primaryResolution.adapter;
341
426
  let breakerWrap;
342
- const adapter = (() => {
343
- if (opts.circuitBreaker === undefined)
344
- return baseAdapter;
427
+ let failoverChain;
428
+ let observabilityBus = opts.runContext?.eventBus;
429
+ const modelFallbacks = opts.modelFallbacks ?? [];
430
+ let adapter;
431
+ if (modelFallbacks.length > 0) {
432
+ // Seed the chain with the already-resolved primary (honours `_adapter`
433
+ // injection and avoids a second resolveModel round-trip) plus any
434
+ // test-injected fallback adapters.
435
+ const injectedAdapters = new Map();
436
+ injectedAdapters.set(opts.model, baseAdapter);
437
+ for (const [modelString, injected] of opts._failoverAdapters ?? []) {
438
+ injectedAdapters.set(modelString, injected);
439
+ }
440
+ failoverChain = await createFailoverChain({
441
+ model: opts.model,
442
+ fallbacks: modelFallbacks,
443
+ ...(opts.circuitBreaker !== undefined ? { breaker: opts.circuitBreaker } : {}),
444
+ getBus: () => observabilityBus,
445
+ adapters: injectedAdapters,
446
+ });
447
+ // Doctor-style boot report: fallbacks that failed credential/package
448
+ // resolution are listed once, loudly, on stderr — and stay in the chain
449
+ // (they are re-tried when routing actually reaches them; see item 22).
450
+ for (const warning of failoverChain.warnings()) {
451
+ process.stderr.write(`[failover] ${warning}\n`);
452
+ }
453
+ adapter = failoverChain;
454
+ }
455
+ else if (opts.circuitBreaker !== undefined) {
345
456
  breakerWrap = wrapWithCircuitBreaker(baseAdapter, {
346
457
  ...opts.circuitBreaker,
347
458
  adapterName: baseAdapter.providerId,
348
459
  bus: opts.runContext?.eventBus,
349
460
  });
350
- return breakerWrap;
351
- })();
461
+ adapter = breakerWrap;
462
+ }
463
+ else {
464
+ adapter = baseAdapter;
465
+ }
352
466
  void breakerWrap; // exposed via runtime stats once gateway/eval consumers land
353
- const providerId = primaryResolution.providerId;
354
- const wireModelId = primaryResolution.modelId;
467
+ // `let` (item 27): a `budget: { on_exceed: degrade }` breach re-resolves
468
+ // the primary model in place, so these become the degraded model's
469
+ // identity. `degradedSpecModel` overrides `opts.model` as the specModel
470
+ // stamped on model_request/response once a degrade has happened.
471
+ let providerId = primaryResolution.providerId;
472
+ let wireModelId = primaryResolution.modelId;
473
+ let degradedSpecModel;
355
474
  // Section 17 — feature gate: a spec that declares tools cannot run on an
356
475
  // adapter that doesn't speak tool use (e.g. Bedrock Llama/Mistral
357
476
  // families). Fail with a clear ConfigError naming the model instead of
@@ -371,9 +490,41 @@ export async function runChatLoop(opts) {
371
490
  compactionWireModelId = c.modelId;
372
491
  }
373
492
  else {
493
+ // Item 22 v1 scope note: with no explicit `compactionModel`, compaction
494
+ // reuses the primary adapter — which IS the failover chain when
495
+ // `modelFallbacks` is set, so compaction inherits the chain by
496
+ // construction. An explicit `compactionModel` keeps its own single
497
+ // adapter (no chain), as do judge/grader slots.
374
498
  compactionAdapter = adapter;
375
499
  compactionWireModelId = wireModelId;
376
500
  }
501
+ // Item 26 — two-tier turn-difficulty router. When `modelTiers` is set, BOTH
502
+ // tier adapters resolve HERE at boot (adapters bind once; the per-turn pick
503
+ // just selects between the two already-resolved adapters — the same
504
+ // discipline as compaction's second adapter and the failover chain's
505
+ // multiple candidates). The `_tierAdapters` map is a test seam mirroring
506
+ // `_failoverAdapters`.
507
+ let tierRouter;
508
+ if (opts.modelTiers !== undefined) {
509
+ const resolveTier = async (modelString) => {
510
+ const injected = opts._tierAdapters?.get(modelString);
511
+ if (injected !== undefined) {
512
+ return { adapter: injected, modelId: bestEffortWireModelId(modelString), modelString };
513
+ }
514
+ const r = await resolveModel(modelString);
515
+ return { adapter: r.adapter, modelId: r.modelId, modelString };
516
+ };
517
+ const fast = await resolveTier(opts.modelTiers.fast);
518
+ const dflt = await resolveTier(opts.modelTiers.default);
519
+ tierRouter = createTierRouter({
520
+ fast,
521
+ default: dflt,
522
+ ...(opts.modelTiers.routing !== undefined ? { config: opts.modelTiers.routing } : {}),
523
+ });
524
+ }
525
+ // Item 26 — tool_use blocks the PREVIOUS turn produced (a tier-routing
526
+ // signal). Persists across `runOneTurn` calls; 0 on the first turn.
527
+ let priorTurnToolUseCount = 0;
377
528
  // Default model max OUTPUT tokens for one turn. Callers thread the spec's
378
529
  // `agent.max_tokens` here when set (the CLI target's codegen + apps/cli);
379
530
  // this fallback applies when the spec is silent. Kept comfortably above the
@@ -408,6 +559,80 @@ export async function runChatLoop(opts) {
408
559
  // Housekeeping side-effect: evicts any session whose mtime is older than
409
560
  // the TTL (default 30 days). The returned list is intentionally discarded.
410
561
  await sessionStore.list();
562
+ // Adaptive model routing (spec `agent.model_pool`). Resolve every candidate
563
+ // adapter ONCE at boot (mirroring the tier router), open the durable reward
564
+ // scoreboard beside the sessions dir, and build the PolicyRouter. The
565
+ // scoreboard is opened whenever a pool is present — not just for `learned` —
566
+ // so `route status` has data and a later switch to `learned` inherits the
567
+ // accumulated history. `recordPoolOutcome`/`poolCostUsd` fold each turn's
568
+ // observed result back in; they no-op when no pool is active.
569
+ let poolRouter;
570
+ let scoreboard;
571
+ let rewardConfig;
572
+ let poolPolicyVersion;
573
+ if (opts.modelPool !== undefined) {
574
+ const pool = opts.modelPool;
575
+ const candidates = [];
576
+ for (const c of pool.candidates) {
577
+ const injected = opts._poolAdapters?.get(c.model);
578
+ if (injected !== undefined) {
579
+ candidates.push({
580
+ adapter: injected,
581
+ modelId: bestEffortWireModelId(c.model),
582
+ modelString: c.model,
583
+ tags: c.tags,
584
+ });
585
+ }
586
+ else {
587
+ const r = await resolveModel(c.model);
588
+ candidates.push({
589
+ adapter: r.adapter,
590
+ modelId: r.modelId,
591
+ modelString: c.model,
592
+ tags: c.tags,
593
+ });
594
+ }
595
+ }
596
+ const routingRoot = sessionRootDir !== undefined ? pathDirname(sessionRootDir) : ".crewhaus";
597
+ scoreboard = opts._scoreboard ?? openScoreboard(routingRoot);
598
+ const sb = scoreboard;
599
+ rewardConfig = {
600
+ ...(pool.objective !== undefined ? { objective: pool.objective } : {}),
601
+ ...(pool.learning?.costRefUsd !== undefined ? { costRefUsd: pool.learning.costRefUsd } : {}),
602
+ ...(pool.learning?.latencyRefMs !== undefined
603
+ ? { latencyRefMs: pool.learning.latencyRefMs }
604
+ : {}),
605
+ };
606
+ poolPolicyVersion = poolFingerprint(pool);
607
+ poolRouter = createPolicyRouter({
608
+ candidates,
609
+ policy: pool.policy,
610
+ ...(pool.routing !== undefined ? { routing: pool.routing } : {}),
611
+ ...(pool.learning?.minSamplesPerArm !== undefined
612
+ ? { learning: { minSamplesPerArm: pool.learning.minSamplesPerArm } }
613
+ : {}),
614
+ // The learned policy reads the live scoreboard; static/heuristic ignore it.
615
+ ...(pool.policy === "learned" ? { score: (rk, m) => sb.score(rk, m) } : {}),
616
+ });
617
+ }
618
+ // Fold one turn's observed outcome into the scoreboard (no-op without a pool).
619
+ const recordPoolOutcome = (turn, obs) => {
620
+ // Capture into consts so the narrowing survives (they are closed-over lets).
621
+ const sb = scoreboard;
622
+ const rc = rewardConfig;
623
+ if (turn === undefined || sb === undefined || rc === undefined)
624
+ return;
625
+ sb.record(turn.routeKey, turn.modelString, computeReward(obs, rc), obs);
626
+ };
627
+ // Per-turn USD cost from token usage + the static pricing table. Undefined
628
+ // when the served model isn't on the table (the reward then drops the cost
629
+ // term and reweights over quality + latency).
630
+ const poolCostUsd = (provider, wireModelId, usage) => {
631
+ const row = resolvePricing(DEFAULT_PRICING, provider, wireModelId);
632
+ if (row === undefined)
633
+ return undefined;
634
+ return (computeCostMicros(row, usage.input, usage.output, usage.cacheRead ?? 0, usage.cacheCreate ?? 0) / 1_000_000);
635
+ };
411
636
  let sessionId;
412
637
  let resumedMessages;
413
638
  if (opts.resume) {
@@ -443,7 +668,117 @@ export async function runChatLoop(opts) {
443
668
  // run with `OTEL_EXPORTER_OTLP_ENDPOINT` set automatically exports.
444
669
  const bus = runContext.eventBus;
445
670
  void TraceEventBus; // keep type import alive for future direct constructions
446
- const subscribers = await attachDefaultSubscribers(bus, runContext);
671
+ // Item 22 — re-point the failover chain's late-bound bus getter at the
672
+ // run's real bus (see the chain construction above): the chain and its
673
+ // per-candidate breakers publish model_failover / circuit_state_changed
674
+ // through this reference from the first stream() call onward.
675
+ observabilityBus = bus;
676
+ const subscribers = await attachDefaultSubscribers(bus, runContext, process.env, {
677
+ ...(opts.alertSink !== undefined ? { alertSink: opts.alertSink } : {}),
678
+ ...(opts.alertMetricsDir !== undefined ? { metricsDir: opts.alertMetricsDir } : {}),
679
+ ...(opts.sloTargets !== undefined ? { sloTargets: opts.sloTargets } : {}),
680
+ ...(opts.sloSink !== undefined ? { sloSink: opts.sloSink } : {}),
681
+ });
682
+ // Ops item 32 — auto-assemble an incident bundle on the first failure-class
683
+ // trigger (circuit → open, egress-blocked, justification-deny storm). Gated
684
+ // by CREWHAUS_INCIDENTS; writes a raw capture (ring buffer + trigger meta)
685
+ // that `crewhaus incident collect --session <id>` turns into a full bundle.
686
+ const incidentCollector = attachIncidentCollector(bus, runContext, process.env, {
687
+ ...(opts.incidentsDir !== undefined ? { incidentsDir: opts.incidentsDir } : {}),
688
+ ...(opts.incidentSpec !== undefined ? { spec: opts.incidentSpec } : {}),
689
+ });
690
+ // Item 22 — surface each failover on stderr so an operator watching a
691
+ // plain (non-CREWHAUS_TRACE) run still sees provider switches. NOTE: the
692
+ // advisor-events session-log subscriber pattern has NOT landed on main
693
+ // (packages/event-log has no advisor kind) — when it does, failovers
694
+ // should ALSO persist into the session JSONL through it; until then the
695
+ // trace event + this stderr note are the observable surfaces.
696
+ let failoverNoteUnsubscribe;
697
+ if (failoverChain !== undefined) {
698
+ failoverNoteUnsubscribe = bus.subscribe((event) => {
699
+ if (event.kind !== "model_failover")
700
+ return;
701
+ process.stderr.write(`[failover] ${event.from} → ${event.to} (${event.reason})\n`);
702
+ });
703
+ }
704
+ // Item 27 — run-level spend cap. An ALWAYS-ON cost meter (independent of
705
+ // the CREWHAUS_COST_TRACKING env flag) accrues per-response spend priced
706
+ // off the wire model that actually served — `suppressEvents` keeps it off
707
+ // the trace surface so it never double-prints beside an env-attached
708
+ // cost-tracker. The pre-turn check (`enforceBudget`, below) reads
709
+ // `budgetMeter.getRunCost(runId).totalUsdMicros`. `budgetDegraded` gates
710
+ // the ladder to a single rung: once we degrade, we never degrade again —
711
+ // a second breach stops the run.
712
+ const budgetMeter = opts.budget !== undefined ? createCostTracker(bus, { suppressEvents: true }) : undefined;
713
+ let budgetDegraded = false;
714
+ /**
715
+ * Item 27 — pre-turn budget gate. Returns "stop" when the run must end
716
+ * before the next turn opens; "continue" otherwise. On a `degrade` breach
717
+ * it re-resolves the primary model to the cheaper rung IN PLACE (mutating
718
+ * `adapter`, `providerId`, `wireModelId`) and emits a `model_failover`
719
+ * (reason `budget_degrade`) — so the very next model call, and all cost
720
+ * accrual after it, use the degraded model. The current turn always
721
+ * completes; this only gates the NEXT turn, so an in-flight tool loop is
722
+ * never severed. Never throws — a re-resolution failure logs and stops.
723
+ */
724
+ const enforceBudget = async () => {
725
+ if (opts.budget === undefined || budgetMeter === undefined)
726
+ return "continue";
727
+ const spent = budgetMeter.getRunCost(bus.runId).totalUsdMicros;
728
+ if (spent < opts.budget.usdMicros)
729
+ return "continue";
730
+ const spentUsd = (spent / 1_000_000).toFixed(4);
731
+ const capUsd = (opts.budget.usdMicros / 1_000_000).toFixed(4);
732
+ if (opts.budget.onExceed.kind === "stop" || budgetDegraded) {
733
+ const why = budgetDegraded ? "degraded model also reached the cap" : "run budget reached";
734
+ process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — ${why}; ending the run.\n`);
735
+ return "stop";
736
+ }
737
+ // degrade: re-resolve the primary model to the cheaper rung, once.
738
+ const target = opts.budget.onExceed.model;
739
+ try {
740
+ const degraded = opts._budgetDegradeAdapter !== undefined
741
+ ? {
742
+ adapter: opts._budgetDegradeAdapter,
743
+ modelId: bestEffortWireModelId(target),
744
+ providerId: opts._budgetDegradeAdapter.providerId,
745
+ }
746
+ : await resolveModel(target);
747
+ const from = opts.model;
748
+ adapter =
749
+ opts.circuitBreaker !== undefined
750
+ ? wrapWithCircuitBreaker(degraded.adapter, {
751
+ ...opts.circuitBreaker,
752
+ adapterName: degraded.adapter.providerId,
753
+ bus,
754
+ })
755
+ : degraded.adapter;
756
+ // The degraded model is now the serving one; keep the trace/pricing
757
+ // identity coherent for the model_request/response events below.
758
+ providerId = degraded.providerId;
759
+ wireModelId = degraded.modelId;
760
+ degradedSpecModel = target;
761
+ // A degrade takes over from the failover chain: subsequent calls hit
762
+ // the degraded adapter directly (v1 keeps one rung, no re-entry to the
763
+ // chain). Null it so the request/response stamping stops consulting it.
764
+ failoverChain = undefined;
765
+ budgetDegraded = true;
766
+ bus.publish({
767
+ ...bus.envelope(),
768
+ kind: "model_failover",
769
+ from,
770
+ to: target,
771
+ reason: "budget_degrade",
772
+ });
773
+ process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrading ${from} → ${target}.\n`);
774
+ return "continue";
775
+ }
776
+ catch (err) {
777
+ const message = err instanceof Error ? err.message : String(err);
778
+ process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrade to "${target}" failed (${message}); ending the run.\n`);
779
+ return "stop";
780
+ }
781
+ };
447
782
  // Working-indicator spinner. Auto-enables only for an interactive REPL on a
448
783
  // TTY (see `isSpinnerEnabled`); `singleTurn`/injected-`input`/piped runs stay
449
784
  // byte-identical to before. `out.write` is the spinner-aware stdout sink:
@@ -505,6 +840,11 @@ export async function runChatLoop(opts) {
505
840
  inputTokens: ev.inputTokens,
506
841
  outputTokens: ev.outputTokens,
507
842
  cachedReadTokens: ev.cachedReadTokens,
843
+ // Optional on the event (older emitters may omit it) — persist
844
+ // only when present so old-log parsing stays additive-safe.
845
+ ...(ev.cacheCreationTokens !== undefined
846
+ ? { cacheCreationTokens: ev.cacheCreationTokens }
847
+ : {}),
508
848
  costUsdMicros: ev.costUsdMicros,
509
849
  ...(ev.tenantId !== undefined ? { tenantId: ev.tenantId } : {}),
510
850
  },
@@ -516,6 +856,18 @@ export async function runChatLoop(opts) {
516
856
  });
517
857
  });
518
858
  }
859
+ // Advisor groundwork (item 14) — persist the trace-bus-only advisory
860
+ // signals (recovery actions, per-call tool stats, resolved permission
861
+ // decisions, model stop reasons) into the same session JSONL the
862
+ // transcript already writes to, so `crewhaus advise` can mine sessions
863
+ // offline. DEFAULT-ON (the lines are tiny and they are the advisor's
864
+ // food); disable with CREWHAUS_ADVISOR_EVENTS=0. See observability.ts.
865
+ const advisorPersist = attachAdvisorPersistence(bus, eventLog, runContext);
866
+ // Ops item 38 — persist the trace-bus-only `mcp_call_end` events into the
867
+ // session JSONL as the durable `mcp_stats` kind so `crewhaus mcp doctor` can
868
+ // score per-server MCP health offline. Shares the advisor's DEFAULT-ON gate
869
+ // (CREWHAUS_ADVISOR_EVENTS=0 disables both). See observability.ts.
870
+ const mcpStatsPersist = attachMcpStatsPersistence(bus, eventLog, runContext);
519
871
  // Per-run state container — coordination surface for hooks/skills/tools
520
872
  // landing in Section 11+. Section 10 only ships the plumbing; the
521
873
  // underscore prefix marks the intentional non-consumer.
@@ -557,6 +909,21 @@ export async function runChatLoop(opts) {
557
909
  return { allowed: true };
558
910
  }
559
911
  }
912
+ // Feature #53 — auto-capture at teardown. Runs in the same finally where the
913
+ // `stop` hook fires, but is NOT a credential-stripped hook: the caller-
914
+ // supplied `onCapture` (which writes to the memory-store from the CLI's full
915
+ // environment) needs the process env, exactly like the feedback teardown.
916
+ // Best-effort — a capture failure never turns a clean exit into an error.
917
+ const maybeAutoCapture = async () => {
918
+ if (opts.memory?.autoCapture !== true || opts.memory.onCapture === undefined)
919
+ return;
920
+ try {
921
+ await opts.memory.onCapture(runContext.turnNumber, sessionId);
922
+ }
923
+ catch (err) {
924
+ runContext.logger.warn("memory auto-capture failed", { error: err.message });
925
+ }
926
+ };
560
927
  // Section 11 — `session-start` fires once after the persistence + run
561
928
  // context boot is complete. Observational only (no deny/block effect).
562
929
  await fireHook("session-start", {
@@ -594,7 +961,9 @@ export async function runChatLoop(opts) {
594
961
  // CLAUDE.md auto-load behaviour. Pillar 3 compliance: each file's content
595
962
  // is classified via boundary-classifier with origin "user" inside
596
963
  // loadProjectMemory() (the user's repo is developer-trusted).
597
- const projectMemory = await loadProjectMemory();
964
+ const projectMemory = await loadProjectMemory(opts.preferenceFiles !== undefined && opts.preferenceFiles.length > 0
965
+ ? { extraFiles: opts.preferenceFiles }
966
+ : {});
598
967
  const projectMemoryBlock = projectMemory.prompt.length > 0
599
968
  ? [
600
969
  {
@@ -609,12 +978,50 @@ export async function runChatLoop(opts) {
609
978
  .map((f) => f.filename)
610
979
  .join(", ")}\n`);
611
980
  }
981
+ // Feature #53 — auto-recall: at session start, pull the top-K relevant
982
+ // cross-session memories and inject them as a system block, mirroring the
983
+ // project-memory auto-load above. The caller supplies the `recall` seam
984
+ // (runtime-core stays independent of memory-store). Best-effort: a recall
985
+ // failure never aborts the run.
986
+ let memoryRecallBlock = [];
987
+ if (opts.memory?.autoRecall === true && opts.memory.recall !== undefined) {
988
+ try {
989
+ const seed = opts.memory.recallSeed ?? opts.instructions;
990
+ const k = opts.memory.recallK ?? 5;
991
+ const lines = await opts.memory.recall(seed, k);
992
+ if (lines.length > 0) {
993
+ // Pillar 3 — a recalled memory can embed content shaped by untrusted
994
+ // tool output from an earlier session, so it is NOT verbatim-safe like
995
+ // the developer's own repo files. Two defenses, mirroring how the rest
996
+ // of the security fabric treats injected content:
997
+ // 1. Delimiter safety — neutralize any `</recalled_memory>` closing
998
+ // tag inside a recalled line so a poisoned memory can't break out
999
+ // of its block and inject trailing instructions.
1000
+ // 2. Classification — run the assembled block through the SAME
1001
+ // `classifyBoundary(text, { origin: "user" })` best-effort path
1002
+ // `loadProjectMemory()` uses (pass policy: classify + emit trace
1003
+ // events without mutating), so recalled content flows through the
1004
+ // boundary classifier exactly like every other prompt-bound source.
1005
+ const body = lines
1006
+ .map((l) => `- ${escapeBoundaryDelimiter(l, "recalled_memory")}`)
1007
+ .join("\n");
1008
+ const text = `<recalled_memory>\nRelevant facts remembered from earlier sessions:\n${body}\n</recalled_memory>`;
1009
+ await classifyBoundary(text, { origin: "user" }).catch(() => undefined);
1010
+ memoryRecallBlock = [{ type: "text", text, cache_control: { type: "ephemeral" } }];
1011
+ process.stdout.write(`[memory] recalled ${lines.length} memory(ies) into the prompt\n`);
1012
+ }
1013
+ }
1014
+ catch (err) {
1015
+ runContext.logger.warn("memory auto-recall failed", { error: err.message });
1016
+ }
1017
+ }
612
1018
  // Section 17 — runtime-core no longer prepends the Claude Code OAuth
613
1019
  // prefix; `adapter-anthropic` handles that internally so each adapter
614
1020
  // owns its provider-specific auth-shape requirements.
615
1021
  let systemBlocks = [
616
1022
  userInstructions,
617
1023
  ...projectMemoryBlock,
1024
+ ...memoryRecallBlock,
618
1025
  ...skillsBlock,
619
1026
  ];
620
1027
  // Section 27 — rotate cache markers on the system block array if the
@@ -814,6 +1221,21 @@ export async function runChatLoop(opts) {
814
1221
  `tool denied: \`${tu.name}\` defaulted to "ask" and single-turn mode has no interactive surface to prompt on. ` +
815
1222
  `Add an explicit rule to permissions.rules in your spec, e.g. \`{ type: alwaysAllow, pattern: ${tu.name} }\`, or run in REPL mode where "ask" can prompt.`;
816
1223
  }
1224
+ // Advisor groundwork (item 14) — event the ask RESOLUTION. The publish
1225
+ // above fired BEFORE the approval prompt (decision "ask", no outcome);
1226
+ // this one fires after `askApproval` resolves — or after the
1227
+ // single-turn fallback collapses the ask to a deny — so offline advice
1228
+ // mining can measure how each tool's prompts are actually answered.
1229
+ // The advisor persistence subscriber (observability.ts) keys on
1230
+ // `askOutcome` to persist exactly this resolved form.
1231
+ bus.publish({
1232
+ ...bus.envelope(),
1233
+ kind: "permission_decision",
1234
+ toolName: tu.name,
1235
+ decision: "ask",
1236
+ mode: permissionMode,
1237
+ askOutcome: approved ? "approved" : "denied",
1238
+ });
817
1239
  }
818
1240
  if (!approved) {
819
1241
  return finish({
@@ -952,6 +1374,36 @@ export async function runChatLoop(opts) {
952
1374
  originsFound: egress.originsFound,
953
1375
  matchCount: egress.matchCount,
954
1376
  });
1377
+ // Item 20 — emit the durable `egress_decision` audit record for every
1378
+ // non-pass verdict so egress history is triageable offline (the
1379
+ // trace-bus event above is ephemeral). Only the lineage SUMMARY is
1380
+ // stored — never the raw outbound payload (which is sensitive and
1381
+ // often carries the very tagged content that tripped the classifier),
1382
+ // matching the payload shape `@crewhaus/audit-log` documents for this
1383
+ // kind. Best-effort but surfaced: an I/O failure on the audit trail is
1384
+ // logged and swallowed so a full audit disk does not crash a governed
1385
+ // run, mirroring the justification sink above. Absent sink → no-op.
1386
+ if (opts.egressAuditSink !== undefined) {
1387
+ try {
1388
+ await opts.egressAuditSink.append({
1389
+ kind: "egress_decision",
1390
+ payload: {
1391
+ sinkId: egress.sinkId,
1392
+ sinkScope: egress.sinkScope,
1393
+ verdict: egress.verdict,
1394
+ originsFound: egress.originsFound,
1395
+ matchCount: egress.matchCount,
1396
+ },
1397
+ });
1398
+ }
1399
+ catch (err) {
1400
+ runContext.logger.warn("egress audit append failed", {
1401
+ toolUseId: tu.id,
1402
+ toolName: tu.name,
1403
+ error: err.message,
1404
+ });
1405
+ }
1406
+ }
955
1407
  }
956
1408
  if (egress.verdict === "block") {
957
1409
  return finish({
@@ -1181,10 +1633,15 @@ export async function runChatLoop(opts) {
1181
1633
  * they line up with the assistant turn's tool_use blocks.
1182
1634
  */
1183
1635
  async function runToolBatch(toolUses) {
1184
- const partition = partitionToolCalls(toolUses, (n) => toolByName.get(n));
1636
+ // Pass the full catalog so per-call classifiers (Task) can resolve a
1637
+ // dispatch's child tool set; without it, classifier-based tools stay
1638
+ // serial (fail-closed).
1639
+ const partition = partitionToolCalls(toolUses, (n) => toolByName.get(n), tools);
1640
+ const maxConcurrentTools = opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS;
1185
1641
  runContext.logger.debug("tool partition", {
1186
1642
  concurrent: partition.concurrent.map((b) => b.length),
1187
1643
  serial: partition.serial.length,
1644
+ maxConcurrentTools,
1188
1645
  });
1189
1646
  // Map each tool_use's identity to its slot in the original order so
1190
1647
  // results can be placed back in order regardless of the
@@ -1196,7 +1653,7 @@ export async function runChatLoop(opts) {
1196
1653
  toolUses.forEach((tu, idx) => indexByBlock.set(tu, idx));
1197
1654
  const results = new Array(toolUses.length);
1198
1655
  for (const batch of partition.concurrent) {
1199
- const settled = await Promise.all(batch.map((tu) => executeOneToolUse(tu)));
1656
+ const settled = await mapWithConcurrency(batch, maxConcurrentTools, (tu) => executeOneToolUse(tu));
1200
1657
  batch.forEach((tu, i) => {
1201
1658
  // biome-ignore lint/style/noNonNullAssertion: every block came from toolUses, so its index is registered.
1202
1659
  results[indexByBlock.get(tu)] = settled[i];
@@ -1243,6 +1700,18 @@ export async function runChatLoop(opts) {
1243
1700
  let recovery = initialRecoveryState;
1244
1701
  const maxToolIterations = opts.maxToolIterations ?? DEFAULT_MAX_TOOL_ITERATIONS;
1245
1702
  let toolIterations = 0;
1703
+ // Item 26 — tool_use blocks THIS turn produced (fed to the NEXT turn's
1704
+ // tier decision) and the misroute-recovery escalation latch (a fast-tier
1705
+ // failure forces `default` for the rest of this turn). `servedFastTier`
1706
+ // records whether the LAST attempt streamed on the fast tier, so the
1707
+ // error catch only escalates a genuine fast-tier misroute.
1708
+ let thisTurnToolUseCount = 0;
1709
+ let escalateTier = false;
1710
+ let servedFastTier = false;
1711
+ // Adaptive model routing — once a pool candidate fails, stick with the
1712
+ // strongest candidate for the rest of the run (misroute recovery, mirroring
1713
+ // the tier router's fast→default escalation latch).
1714
+ let escalatePool = false;
1246
1715
  while (state.kind !== "Done") {
1247
1716
  if (turnAbort.signal.aborted) {
1248
1717
  state = transition(state, { kind: "Aborted" });
@@ -1280,6 +1749,10 @@ export async function runChatLoop(opts) {
1280
1749
  // gap before the model responds now shows live progress. Cleared in
1281
1750
  // the catch below on any model-call error.
1282
1751
  out.spinner.start("thinking", { prefix: "agent> " });
1752
+ // Adaptive model routing — declared OUTSIDE the try so the catch can
1753
+ // read the chosen candidate + start time to record a failure reward.
1754
+ let poolTurn;
1755
+ let modelCallStartMs = performance.now();
1283
1756
  try {
1284
1757
  // Section 27 — rate-limit the model call before opening the
1285
1758
  // stream. The keys are caller-configured (typically
@@ -1294,17 +1767,107 @@ export async function runChatLoop(opts) {
1294
1767
  // → `us.anthropic.…`) so cost-tracker pricing lookups and the
1295
1768
  // OTel `gen_ai.request.model` attribute match; the original
1296
1769
  // spec string rides along as `specModel` when it differs.
1770
+ // Item 22 — with a failover chain the serving candidate may not
1771
+ // be the primary: stamp the request with the chain's routing
1772
+ // plan (best-effort; the response event below uses lastServed(),
1773
+ // which is exact, so cost-tracker pricing always keys on the
1774
+ // model that actually served).
1775
+ const plannedCandidate = failoverChain?.plan();
1776
+ let reqWireModelId = plannedCandidate?.modelId ?? wireModelId;
1777
+ let reqProviderId = plannedCandidate?.providerId ?? providerId;
1778
+ // Item 27 — after a budget degrade, `failoverChain` is null and
1779
+ // `degradedSpecModel` is the running spec model.
1780
+ let reqSpecModel = plannedCandidate?.modelString ?? degradedSpecModel ?? opts.model;
1781
+ // The adapter this turn streams through: the primary/chain by
1782
+ // default, or a tier adapter when the two-tier router is active.
1783
+ let turnAdapter = adapter;
1784
+ // Adaptive model routing — the PolicyRouter picks a pool candidate
1785
+ // this turn (static / heuristic / learned) from the SAME
1786
+ // deterministic signals as the tier router. `escalatePool` forces
1787
+ // the strongest candidate after a prior pool failure (misroute
1788
+ // recovery). Publish a `model_route` event and stream through the
1789
+ // chosen candidate; the outcome is folded into the reward
1790
+ // scoreboard post-turn. Takes precedence over the tier router (the
1791
+ // two are mutually exclusive in the spec).
1792
+ if (poolRouter !== undefined) {
1793
+ const base = poolRouter.route({
1794
+ contextTokens: estimateTokens(messages),
1795
+ toolsInPlay: (anthropicTools?.length ?? 0) > 0,
1796
+ turnIndex: Math.max(0, runContext.turnNumber - 1),
1797
+ priorTurnToolUseCount,
1798
+ });
1799
+ const decision = escalatePool
1800
+ ? {
1801
+ ...base,
1802
+ candidate: poolRouter.escalation(),
1803
+ reason: `escalated to strongest candidate after a pool failure (was: ${base.reason})`,
1804
+ explored: false,
1805
+ }
1806
+ : base;
1807
+ turnAdapter = decision.candidate.adapter;
1808
+ reqWireModelId = decision.candidate.modelId;
1809
+ reqProviderId = decision.candidate.adapter.providerId;
1810
+ reqSpecModel = decision.candidate.modelString;
1811
+ poolTurn = {
1812
+ modelString: decision.candidate.modelString,
1813
+ routeKey: decision.routeKey,
1814
+ };
1815
+ bus.publish({
1816
+ ...bus.envelope(),
1817
+ kind: "model_route",
1818
+ routeKey: decision.routeKey,
1819
+ model: decision.candidate.modelId,
1820
+ policy: decision.policy,
1821
+ reason: decision.reason,
1822
+ ...(decision.explored ? { explored: true } : {}),
1823
+ ...(poolPolicyVersion !== undefined ? { policyVersion: poolPolicyVersion } : {}),
1824
+ });
1825
+ }
1826
+ else if (tierRouter !== undefined) {
1827
+ // Item 26 — two-tier turn-difficulty router. Pick a tier from
1828
+ // DETERMINISTIC per-turn signals (context size, tools-in-play,
1829
+ // turn index, prior-turn tool_use density); a fast-tier misroute
1830
+ // recovery forces `default` (see the catch below). Publish the
1831
+ // decision as a `model_tier_route` event, then stream through the
1832
+ // chosen tier's already-resolved adapter.
1833
+ const decision = escalateTier
1834
+ ? { tier: "default", reason: "escalated after fast-tier failure" }
1835
+ : tierRouter.route({
1836
+ contextTokens: estimateTokens(messages),
1837
+ toolsInPlay: (anthropicTools?.length ?? 0) > 0,
1838
+ // `turnNumber` is 1-based (incremented before the turn
1839
+ // runs); the router wants a 0-based index so its
1840
+ // first-turn check keys on the opening turn.
1841
+ turnIndex: Math.max(0, runContext.turnNumber - 1),
1842
+ priorTurnToolUseCount,
1843
+ });
1844
+ const chosen = tierRouter.tier(decision.tier);
1845
+ turnAdapter = chosen.adapter;
1846
+ reqWireModelId = chosen.modelId;
1847
+ reqProviderId = chosen.adapter.providerId;
1848
+ reqSpecModel = chosen.modelString;
1849
+ servedFastTier = decision.tier === "fast";
1850
+ bus.publish({
1851
+ ...bus.envelope(),
1852
+ kind: "model_tier_route",
1853
+ tier: decision.tier,
1854
+ model: chosen.modelId,
1855
+ reason: decision.reason,
1856
+ ...(escalateTier ? { escalated: true } : {}),
1857
+ });
1858
+ }
1297
1859
  bus.publish({
1298
1860
  ...modelStartEnv,
1299
1861
  kind: "model_request",
1300
- model: wireModelId,
1301
- ...(opts.model !== wireModelId ? { specModel: opts.model } : {}),
1302
- provider: providerId,
1862
+ model: reqWireModelId,
1863
+ ...(reqSpecModel !== reqWireModelId ? { specModel: reqSpecModel } : {}),
1864
+ provider: reqProviderId,
1303
1865
  messageCount: messages.length,
1304
1866
  toolCount: anthropicTools?.length ?? 0,
1305
1867
  streaming: opts.streaming === true,
1306
1868
  });
1307
1869
  const t0Model = performance.now();
1870
+ modelCallStartMs = t0Model;
1308
1871
  let streamChunkIndex = 0;
1309
1872
  // Section 17 — `adapter.stream(req)` returns
1310
1873
  // `AsyncIterable<StreamEvent>`; we accumulate via
@@ -1320,8 +1883,11 @@ export async function runChatLoop(opts) {
1320
1883
  // bookkeeping is wire-compatible for the block kinds we
1321
1884
  // actually consume (text / image / tool_use / tool_result
1322
1885
  // / thinking).
1323
- const reqStream = adapter.stream({
1324
- model: wireModelId,
1886
+ // Item 22 — the failover chain rewrites `model` per serving
1887
+ // candidate internally; the planned id here is the single-adapter
1888
+ // value and the chain's best-effort prediction otherwise.
1889
+ const reqStream = turnAdapter.stream({
1890
+ model: reqWireModelId,
1325
1891
  system: systemBlocks.map((b) => ({
1326
1892
  type: "text",
1327
1893
  text: b.text,
@@ -1356,6 +1922,10 @@ export async function runChatLoop(opts) {
1356
1922
  toolByName,
1357
1923
  abortSignal: turnAbort.signal,
1358
1924
  onTextDelta,
1925
+ // Honor the same concurrency bound as the non-streaming
1926
+ // path — otherwise `maxConcurrentTools` would be silently
1927
+ // ignored under `streaming: true`.
1928
+ maxConcurrent: opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS,
1359
1929
  runTool: (block) => executeOneToolUse({
1360
1930
  id: block.id,
1361
1931
  name: block.name,
@@ -1388,19 +1958,41 @@ export async function runChatLoop(opts) {
1388
1958
  await logEvent("assistant_message", { content: committedStreamContent });
1389
1959
  }
1390
1960
  terminalContent = [...committedStreamContent];
1961
+ // Item 22 — lastServed() is exact: the candidate that actually
1962
+ // streamed this response. Cost-tracker prices off this pair.
1963
+ const servedStreaming = failoverChain?.lastServed();
1964
+ // With a pool, the served identity is the chosen candidate (no
1965
+ // failover chain when pooling), so cost-tracker prices on it.
1966
+ const respModelIdS = poolTurn !== undefined ? reqWireModelId : (servedStreaming?.modelId ?? wireModelId);
1967
+ const respSpecModelS = poolTurn !== undefined
1968
+ ? poolTurn.modelString
1969
+ : (servedStreaming?.modelString ?? degradedSpecModel ?? opts.model);
1970
+ const respProviderS = poolTurn !== undefined
1971
+ ? reqProviderId
1972
+ : (servedStreaming?.providerId ?? providerId);
1391
1973
  bus.publish({
1392
1974
  ...bus.envelope(),
1393
1975
  spanId: modelStartEnv.spanId,
1394
1976
  kind: "model_response",
1395
1977
  // Wire model id, NOT the spec string — cost-tracker resolves
1396
1978
  // pricing on this field (see the model_request publish above).
1397
- model: wireModelId,
1398
- ...(opts.model !== wireModelId ? { specModel: opts.model } : {}),
1399
- provider: providerId,
1979
+ model: respModelIdS,
1980
+ ...(respSpecModelS !== respModelIdS ? { specModel: respSpecModelS } : {}),
1981
+ provider: respProviderS,
1400
1982
  stopReason: stopReason ?? "end_turn",
1401
1983
  usage,
1402
1984
  durationMs: performance.now() - t0Model,
1403
1985
  });
1986
+ // Adaptive model routing — fold this successful turn into the
1987
+ // reward scoreboard so the learned policy improves next run.
1988
+ if (poolTurn !== undefined) {
1989
+ const costUsd = poolCostUsd(respProviderS, respModelIdS, usage);
1990
+ recordPoolOutcome(poolTurn, {
1991
+ success: true,
1992
+ latencyMs: performance.now() - t0Model,
1993
+ ...(costUsd !== undefined ? { costUsd } : {}),
1994
+ });
1995
+ }
1404
1996
  await fireHook("post-model", {
1405
1997
  streaming: true,
1406
1998
  contentBlocks: finalContent.length,
@@ -1410,6 +2002,8 @@ export async function runChatLoop(opts) {
1410
2002
  state = transition(state, { kind: "ModelReturnedText" });
1411
2003
  break;
1412
2004
  }
2005
+ // Item 26 — accrue this turn's tool_use count (next-turn tier signal).
2006
+ thisTurnToolUseCount += toolUses.length;
1413
2007
  for (const tu of toolUses) {
1414
2008
  toolUseHistory.push({ id: tu.id, name: tu.name, input: tu.input });
1415
2009
  }
@@ -1447,19 +2041,38 @@ export async function runChatLoop(opts) {
1447
2041
  },
1448
2042
  });
1449
2043
  endTurn();
2044
+ // Item 22 — lastServed() is exact: the candidate that actually
2045
+ // streamed this response. Cost-tracker prices off this pair.
2046
+ const servedCandidate = failoverChain?.lastServed();
2047
+ // With a pool, the served identity is the chosen candidate.
2048
+ const respWireModelId = poolTurn !== undefined ? reqWireModelId : (servedCandidate?.modelId ?? wireModelId);
2049
+ const respSpecModel = poolTurn !== undefined
2050
+ ? poolTurn.modelString
2051
+ : (servedCandidate?.modelString ?? degradedSpecModel ?? opts.model);
2052
+ const respProvider = poolTurn !== undefined ? reqProviderId : (servedCandidate?.providerId ?? providerId);
1450
2053
  bus.publish({
1451
2054
  ...bus.envelope(),
1452
2055
  spanId: modelStartEnv.spanId,
1453
2056
  kind: "model_response",
1454
2057
  // Wire model id, NOT the spec string — cost-tracker resolves
1455
2058
  // pricing on this field (see the model_request publish above).
1456
- model: wireModelId,
1457
- ...(opts.model !== wireModelId ? { specModel: opts.model } : {}),
1458
- provider: providerId,
2059
+ model: respWireModelId,
2060
+ ...(respSpecModel !== respWireModelId ? { specModel: respSpecModel } : {}),
2061
+ provider: respProvider,
1459
2062
  stopReason: final.stopReason,
1460
2063
  usage: final.usage,
1461
2064
  durationMs: performance.now() - t0Model,
1462
2065
  });
2066
+ // Adaptive model routing — fold this successful turn into the
2067
+ // reward scoreboard so the learned policy improves next run.
2068
+ if (poolTurn !== undefined) {
2069
+ const costUsd = poolCostUsd(respProvider, respWireModelId, final.usage);
2070
+ recordPoolOutcome(poolTurn, {
2071
+ success: true,
2072
+ latencyMs: performance.now() - t0Model,
2073
+ ...(costUsd !== undefined ? { costUsd } : {}),
2074
+ });
2075
+ }
1463
2076
  // Persist the assistant turn. On a clean stop the FULL content-block
1464
2077
  // array is committed so subsequent `tool_result` references resolve.
1465
2078
  //
@@ -1529,6 +2142,28 @@ export async function runChatLoop(opts) {
1529
2142
  break;
1530
2143
  }
1531
2144
  lastErrorForRecovery = err;
2145
+ // Item 26 — a fast-tier turn that FAILED is a MISROUTE: latch the
2146
+ // escalation so the recovery retry re-runs on the `default` tier
2147
+ // (the loop re-enters NeedModel via the recovery ladder, which
2148
+ // then picks `default`). Composes with the existing recovery
2149
+ // actions — a retry/continue now streams on the sturdier tier.
2150
+ if (tierRouter !== undefined && servedFastTier) {
2151
+ escalateTier = true;
2152
+ }
2153
+ // Adaptive model routing — record the failed candidate as a
2154
+ // negative reward (drops the cost term: no usage on a throw) and
2155
+ // latch the escalation so the recovery retry re-runs on the
2156
+ // strongest candidate, unless the failed candidate WAS already the
2157
+ // strongest (then escalation buys nothing).
2158
+ if (poolRouter !== undefined && poolTurn !== undefined) {
2159
+ recordPoolOutcome(poolTurn, {
2160
+ success: false,
2161
+ latencyMs: performance.now() - modelCallStartMs,
2162
+ });
2163
+ if (poolTurn.modelString !== poolRouter.escalation().modelString) {
2164
+ escalatePool = true;
2165
+ }
2166
+ }
1532
2167
  const errObj = err;
1533
2168
  const errorName = typeof errObj.name === "string" ? errObj.name : "Error";
1534
2169
  const errorMessage = typeof errObj.message === "string" ? errObj.message : String(err);
@@ -1556,6 +2191,9 @@ export async function runChatLoop(opts) {
1556
2191
  state = transition(state, { kind: "Aborted" });
1557
2192
  break;
1558
2193
  }
2194
+ // Item 26 — accrue this turn's tool_use count (a next-turn tier
2195
+ // signal): a dense tool turn marks an active multi-step task.
2196
+ thisTurnToolUseCount += state.toolUses.length;
1559
2197
  const toolResults = await runToolBatch(state.toolUses);
1560
2198
  messages.push({ role: "user", content: toolResults });
1561
2199
  await logEvent("user_message", { content: toolResults });
@@ -1576,7 +2214,10 @@ export async function runChatLoop(opts) {
1576
2214
  // emitting `BudgetExceeded` from `runOneTurn`, restore a
1577
2215
  // `NeedCompaction` → `CompactionDone` arm (and cover it) here.
1578
2216
  case "NeedRecovery": {
1579
- const action = recover(lastErrorForRecovery, recovery);
2217
+ // Item 23 / Section 55 — consult the spec's failure_taxonomy first
2218
+ // (named classes take precedence over the built-in flow), so a
2219
+ // matched `switch-model` (or retry/compact/…) entry wins.
2220
+ const action = recover(lastErrorForRecovery, recovery, opts.failureTaxonomy);
1580
2221
  recovery = advanceState(recovery, action);
1581
2222
  runContext.logger.info("recovery.action", {
1582
2223
  kind: action.kind,
@@ -1675,6 +2316,32 @@ export async function runChatLoop(opts) {
1675
2316
  state = transition(state, { kind: "RecoveryDone" });
1676
2317
  break;
1677
2318
  }
2319
+ case "switch-model": {
2320
+ // Item 23 — a `switch-model` failure_taxonomy verdict: abandon
2321
+ // the active provider candidate FOR THIS TURN and re-issue the
2322
+ // same request onto the next failover candidate. Mechanism:
2323
+ // force the active candidate's breaker open so the chain
2324
+ // reroutes on the next `stream()`; the chain publishes the
2325
+ // `model_failover` (reason breaker_open) itself. No message
2326
+ // mutation and no backoff sleep — a switch is instant. Without
2327
+ // a failover chain (no `modelFallbacks`) there is nothing to
2328
+ // switch to: this degrades to a plain re-issue against the same
2329
+ // adapter, which the per-turn budget bounds. RecoveryDone loops
2330
+ // back to NeedModel, which re-streams.
2331
+ const tripped = failoverChain?.tripActive("switch-model recovery");
2332
+ if (tripped !== undefined) {
2333
+ out.spinner.start("switching model");
2334
+ out.spinner.stop();
2335
+ runContext.logger.info("recovery.switch-model tripped candidate", {
2336
+ from: tripped,
2337
+ });
2338
+ }
2339
+ else {
2340
+ runContext.logger.warn("recovery.switch-model: no failover chain — re-issuing on the same model");
2341
+ }
2342
+ state = transition(state, { kind: "RecoveryDone" });
2343
+ break;
2344
+ }
1678
2345
  case "fail":
1679
2346
  throw new RuntimeError(`recovery failed: ${action.reason}`);
1680
2347
  }
@@ -1682,6 +2349,9 @@ export async function runChatLoop(opts) {
1682
2349
  }
1683
2350
  }
1684
2351
  }
2352
+ // Item 26 — hand this turn's tool_use count to the next turn's tier
2353
+ // decision (the prior-tool-density signal).
2354
+ priorTurnToolUseCount = thisTurnToolUseCount;
1685
2355
  return { terminalContent };
1686
2356
  }
1687
2357
  // Single-shot path used by the workflow target and (Section 12) the
@@ -1768,9 +2438,16 @@ export async function runChatLoop(opts) {
1768
2438
  turnCount: runContext.turnNumber,
1769
2439
  reason: "complete",
1770
2440
  });
2441
+ await maybeAutoCapture();
1771
2442
  await subscribers.flushAll();
1772
2443
  await subscribers.shutdownAll();
1773
2444
  costPersistUnsubscribe?.();
2445
+ failoverNoteUnsubscribe?.();
2446
+ incidentCollector?.unsubscribe();
2447
+ budgetMeter?.unsubscribe();
2448
+ advisorPersist?.unsubscribe();
2449
+ mcpStatsPersist?.unsubscribe();
2450
+ printAdvisorDigest(advisorPersist);
1774
2451
  await sessionStore
1775
2452
  .update(sessionId, { lastTurnIndex: runContext.turnNumber })
1776
2453
  .catch((err) => {
@@ -1909,6 +2586,13 @@ export async function runChatLoop(opts) {
1909
2586
  }
1910
2587
  }
1911
2588
  }
2589
+ // Item 27 — run-level spend cap. Enforced HERE (a real turn is about
2590
+ // to run, input is not exit/blank) so an idle prompt never trips it.
2591
+ // On a `stop` breach we break before committing the input — the turn
2592
+ // never runs. On a `degrade` breach `enforceBudget` has already
2593
+ // swapped the primary adapter, so this turn runs on the cheaper model.
2594
+ if ((await enforceBudget()) === "stop")
2595
+ break;
1912
2596
  messages.push({ role: "user", content: effectiveInput });
1913
2597
  await logEvent("user_message", { content: effectiveInput });
1914
2598
  runContext.turnNumber += 1;
@@ -1979,9 +2663,16 @@ export async function runChatLoop(opts) {
1979
2663
  turnCount: runContext.turnNumber,
1980
2664
  reason: runAbort.signal.aborted ? "abort" : "exit",
1981
2665
  });
2666
+ await maybeAutoCapture();
1982
2667
  await subscribers.flushAll();
1983
2668
  await subscribers.shutdownAll();
1984
2669
  costPersistUnsubscribe?.();
2670
+ failoverNoteUnsubscribe?.();
2671
+ incidentCollector?.unsubscribe();
2672
+ budgetMeter?.unsubscribe();
2673
+ advisorPersist?.unsubscribe();
2674
+ mcpStatsPersist?.unsubscribe();
2675
+ printAdvisorDigest(advisorPersist);
1985
2676
  if (shouldInstallSigint) {
1986
2677
  process.removeListener("SIGINT", sigintHandler);
1987
2678
  }
@@ -1995,6 +2686,18 @@ export async function runChatLoop(opts) {
1995
2686
  }
1996
2687
  return "";
1997
2688
  }
2689
+ /**
2690
+ * Item 14 in-run digest — one stderr line at session end when the advisor
2691
+ * persistence subscriber's cheap tally tripped a threshold, pointing at
2692
+ * `crewhaus advise --session <id>` for the full report. stderr (not stdout)
2693
+ * so piped/captured agent output is never polluted; silent on healthy runs
2694
+ * and when the subscriber is disabled (CREWHAUS_ADVISOR_EVENTS=0).
2695
+ */
2696
+ function printAdvisorDigest(advisorPersist) {
2697
+ const line = advisorPersist?.digestLine();
2698
+ if (line !== undefined)
2699
+ process.stderr.write(`${line}\n`);
2700
+ }
1998
2701
  function isAbortError(err) {
1999
2702
  // Section 17 — runtime-core no longer imports the Anthropic SDK at
2000
2703
  // runtime, so we name-match instead of `instanceof`. The provider