@crewhaus/runtime-core 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,15 +1,16 @@
1
1
  import * as readline from "node:readline/promises";
2
2
  import { createAbortTree } from "@crewhaus/abort-controller";
3
3
  import { collectFinalMessage, consumeStream, } from "@crewhaus/adapter-anthropic";
4
- import { setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
4
+ import { classifyBoundary, setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
5
5
  import { wrap as wrapWithCircuitBreaker } from "@crewhaus/circuit-breaker";
6
6
  import { autoCompact } from "@crewhaus/compaction-autocompact";
7
7
  import { snip } from "@crewhaus/compaction-snip";
8
+ import { createCostTracker } from "@crewhaus/cost-tracker";
8
9
  import { classifyEgress, summarizeEgress, } from "@crewhaus/egress-classifier";
9
10
  import { ConfigError, RuntimeError } from "@crewhaus/errors";
10
11
  import { openEventLog } from "@crewhaus/event-log";
11
12
  import { aggregateDecisions, runHooks } from "@crewhaus/hooks-engine";
12
- import { parseModelString, resolveModel } from "@crewhaus/model-router";
13
+ import { createFailoverChain, createTierRouter, parseModelString, resolveModel, } from "@crewhaus/model-router";
13
14
  import { BUILTIN_DEFAULT_RULES, emptyRuleSet, evaluateJustification, evaluateWithReason, ruleBasedJustificationJudge, } from "@crewhaus/permission-engine";
14
15
  import { manage as manageCacheMarkers } from "@crewhaus/prompt-cache-manager";
15
16
  import { buildRedactionNotice, classifyText, llmClassifierEnabled, } from "@crewhaus/prompt-injection-detector";
@@ -30,7 +31,8 @@ import { TraceEventBus } from "@crewhaus/trace-event-bus";
30
31
  import { initialState, transition, } from "@crewhaus/turn-state-machine";
31
32
  import { zodToJsonSchema } from "zod-to-json-schema";
32
33
  import { createCliMarkdownRenderer, isCliMarkdownEnabled, } from "./cli-markdown";
33
- import { attachDefaultSubscribers } from "./observability";
34
+ import { attachIncidentCollector } from "./incident-collector";
35
+ import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, } from "./observability";
34
36
  import { loadProjectMemory } from "./project-memory";
35
37
  import { createCliOutput, isSpinnerEnabled } from "./spinner";
36
38
  /**
@@ -113,6 +115,21 @@ const DEFAULT_SNIP_KEEP_TAIL = 20;
113
115
  // moved into `@crewhaus/adapter-anthropic`. Re-exported here so legacy
114
116
  // importers (eval-judge fallback, downstream tests) keep compiling.
115
117
  export { CLAUDE_CODE_HEADERS, CLAUDE_CODE_SYSTEM_PREFIX, OAUTH_BETAS, createAnthropicClient, resolveAuth, } from "@crewhaus/adapter-anthropic";
118
+ // Ops item 36 — boot-time self-heal janitor for daemon shapes. Lives in its
119
+ // own module (`./janitor`) but is re-exported here because emitted bundles
120
+ // only import the package root.
121
+ export { createJanitor } from "./janitor";
122
+ // Ops item 31 — alert watchdog seams re-exported so the CLI/codegen can build
123
+ // the durable + off-box alert sink (audit append + settings.json alert hook /
124
+ // webhook) it passes as `RunChatLoopOptions.alertSink`.
125
+ export { attachDefaultSubscribers, } from "./observability";
126
+ // Ops item 37 — SLO monitor seams re-exported so the CLI/codegen can build the
127
+ // injected mitigation ladder (audit / pause-intake / rollback) it passes as
128
+ // `RunChatLoopOptions.sloSink`, and the doctor probe can share the breach types.
129
+ export { DEFAULT_SLO_WINDOW_MS, MIN_SLO_SAMPLES, SloWindow, attachSloMonitor, detectSloBreaches, } from "./slo-monitor";
130
+ // Ops item 32 — incident collector seams re-exported so the CLI can share the
131
+ // raw-capture shape + trigger classifier with `incident collect`.
132
+ export { attachIncidentCollector, classifyIncidentTrigger, } from "./incident-collector";
116
133
  /**
117
134
  * Reconcile a message history by dropping "orphan" `tool_use` blocks — a
118
135
  * `tool_use` with no answering `tool_result` anywhere later in the
@@ -289,6 +306,19 @@ export function defaultSinkScope(toolName) {
289
306
  return "external-dynamic";
290
307
  return "external-configured";
291
308
  }
309
+ /**
310
+ * Neutralize a boundary-block closing delimiter embedded in untrusted content
311
+ * so a poisoned line can't terminate its wrapper block early and inject
312
+ * trailing instructions. Replaces `</tag>` (any casing/whitespace) with a
313
+ * visually-inert `<\/tag>` so the content survives for the model to read but
314
+ * no longer parses as the real closing tag. Used for the recalled-memory block
315
+ * (#53) — recalled memories may carry content shaped by untrusted tool output.
316
+ */
317
+ function escapeBoundaryDelimiter(text, tag) {
318
+ // Match `</tag>` tolerantly (leading whitespace inside the tag, any casing).
319
+ const re = new RegExp(`</\\s*${tag}\\s*>`, "gi");
320
+ return text.replace(re, `<\\/${tag}>`);
321
+ }
292
322
  /**
293
323
  * Strip the model-string provider grammar down to the wire model id
294
324
  * (`"bedrock/us.anthropic.claude-..."` → `"us.anthropic.claude-..."`)
@@ -333,25 +363,65 @@ export async function runChatLoop(opts) {
333
363
  providerId: opts._adapter.providerId,
334
364
  }
335
365
  : await resolveModel(opts.model);
336
- // Section 27 — wrap the primary adapter in a circuit breaker when
337
- // `circuitBreaker` opts are set (or by default in production codegen).
338
- // The breaker is hung off the run-context's bus so state-change events
339
- // surface for audit-log + structured-event-printer + OTel.
366
+ // Section 27 / item 22 — resilience wrapping around the primary adapter.
367
+ // - `modelFallbacks` non-empty → a model-router failover chain: every
368
+ // candidate (primary first) breaker-wrapped with the `circuitBreaker`
369
+ // tuning; calls route to the first candidate whose breaker admits
370
+ // traffic; `model_failover` trace events surface routing changes.
371
+ // - `circuitBreaker` alone → the single-adapter breaker wrap.
372
+ // The chain publishes through a LATE-BOUND bus getter: the run's real bus
373
+ // is minted below (createRunContext), so `observabilityBus` is re-pointed
374
+ // at it once it exists — a `crewhaus run` without a caller-supplied
375
+ // runContext still surfaces model_failover + circuit_state_changed.
340
376
  const baseAdapter = primaryResolution.adapter;
341
377
  let breakerWrap;
342
- const adapter = (() => {
343
- if (opts.circuitBreaker === undefined)
344
- return baseAdapter;
378
+ let failoverChain;
379
+ let observabilityBus = opts.runContext?.eventBus;
380
+ const modelFallbacks = opts.modelFallbacks ?? [];
381
+ let adapter;
382
+ if (modelFallbacks.length > 0) {
383
+ // Seed the chain with the already-resolved primary (honours `_adapter`
384
+ // injection and avoids a second resolveModel round-trip) plus any
385
+ // test-injected fallback adapters.
386
+ const injectedAdapters = new Map();
387
+ injectedAdapters.set(opts.model, baseAdapter);
388
+ for (const [modelString, injected] of opts._failoverAdapters ?? []) {
389
+ injectedAdapters.set(modelString, injected);
390
+ }
391
+ failoverChain = await createFailoverChain({
392
+ model: opts.model,
393
+ fallbacks: modelFallbacks,
394
+ ...(opts.circuitBreaker !== undefined ? { breaker: opts.circuitBreaker } : {}),
395
+ getBus: () => observabilityBus,
396
+ adapters: injectedAdapters,
397
+ });
398
+ // Doctor-style boot report: fallbacks that failed credential/package
399
+ // resolution are listed once, loudly, on stderr — and stay in the chain
400
+ // (they are re-tried when routing actually reaches them; see item 22).
401
+ for (const warning of failoverChain.warnings()) {
402
+ process.stderr.write(`[failover] ${warning}\n`);
403
+ }
404
+ adapter = failoverChain;
405
+ }
406
+ else if (opts.circuitBreaker !== undefined) {
345
407
  breakerWrap = wrapWithCircuitBreaker(baseAdapter, {
346
408
  ...opts.circuitBreaker,
347
409
  adapterName: baseAdapter.providerId,
348
410
  bus: opts.runContext?.eventBus,
349
411
  });
350
- return breakerWrap;
351
- })();
412
+ adapter = breakerWrap;
413
+ }
414
+ else {
415
+ adapter = baseAdapter;
416
+ }
352
417
  void breakerWrap; // exposed via runtime stats once gateway/eval consumers land
353
- const providerId = primaryResolution.providerId;
354
- const wireModelId = primaryResolution.modelId;
418
+ // `let` (item 27): a `budget: { on_exceed: degrade }` breach re-resolves
419
+ // the primary model in place, so these become the degraded model's
420
+ // identity. `degradedSpecModel` overrides `opts.model` as the specModel
421
+ // stamped on model_request/response once a degrade has happened.
422
+ let providerId = primaryResolution.providerId;
423
+ let wireModelId = primaryResolution.modelId;
424
+ let degradedSpecModel;
355
425
  // Section 17 — feature gate: a spec that declares tools cannot run on an
356
426
  // adapter that doesn't speak tool use (e.g. Bedrock Llama/Mistral
357
427
  // families). Fail with a clear ConfigError naming the model instead of
@@ -371,9 +441,41 @@ export async function runChatLoop(opts) {
371
441
  compactionWireModelId = c.modelId;
372
442
  }
373
443
  else {
444
+ // Item 22 v1 scope note: with no explicit `compactionModel`, compaction
445
+ // reuses the primary adapter — which IS the failover chain when
446
+ // `modelFallbacks` is set, so compaction inherits the chain by
447
+ // construction. An explicit `compactionModel` keeps its own single
448
+ // adapter (no chain), as do judge/grader slots.
374
449
  compactionAdapter = adapter;
375
450
  compactionWireModelId = wireModelId;
376
451
  }
452
+ // Item 26 — two-tier turn-difficulty router. When `modelTiers` is set, BOTH
453
+ // tier adapters resolve HERE at boot (adapters bind once; the per-turn pick
454
+ // just selects between the two already-resolved adapters — the same
455
+ // discipline as compaction's second adapter and the failover chain's
456
+ // multiple candidates). The `_tierAdapters` map is a test seam mirroring
457
+ // `_failoverAdapters`.
458
+ let tierRouter;
459
+ if (opts.modelTiers !== undefined) {
460
+ const resolveTier = async (modelString) => {
461
+ const injected = opts._tierAdapters?.get(modelString);
462
+ if (injected !== undefined) {
463
+ return { adapter: injected, modelId: bestEffortWireModelId(modelString), modelString };
464
+ }
465
+ const r = await resolveModel(modelString);
466
+ return { adapter: r.adapter, modelId: r.modelId, modelString };
467
+ };
468
+ const fast = await resolveTier(opts.modelTiers.fast);
469
+ const dflt = await resolveTier(opts.modelTiers.default);
470
+ tierRouter = createTierRouter({
471
+ fast,
472
+ default: dflt,
473
+ ...(opts.modelTiers.routing !== undefined ? { config: opts.modelTiers.routing } : {}),
474
+ });
475
+ }
476
+ // Item 26 — tool_use blocks the PREVIOUS turn produced (a tier-routing
477
+ // signal). Persists across `runOneTurn` calls; 0 on the first turn.
478
+ let priorTurnToolUseCount = 0;
377
479
  // Default model max OUTPUT tokens for one turn. Callers thread the spec's
378
480
  // `agent.max_tokens` here when set (the CLI target's codegen + apps/cli);
379
481
  // this fallback applies when the spec is silent. Kept comfortably above the
@@ -443,7 +545,117 @@ export async function runChatLoop(opts) {
443
545
  // run with `OTEL_EXPORTER_OTLP_ENDPOINT` set automatically exports.
444
546
  const bus = runContext.eventBus;
445
547
  void TraceEventBus; // keep type import alive for future direct constructions
446
- const subscribers = await attachDefaultSubscribers(bus, runContext);
548
+ // Item 22 — re-point the failover chain's late-bound bus getter at the
549
+ // run's real bus (see the chain construction above): the chain and its
550
+ // per-candidate breakers publish model_failover / circuit_state_changed
551
+ // through this reference from the first stream() call onward.
552
+ observabilityBus = bus;
553
+ const subscribers = await attachDefaultSubscribers(bus, runContext, process.env, {
554
+ ...(opts.alertSink !== undefined ? { alertSink: opts.alertSink } : {}),
555
+ ...(opts.alertMetricsDir !== undefined ? { metricsDir: opts.alertMetricsDir } : {}),
556
+ ...(opts.sloTargets !== undefined ? { sloTargets: opts.sloTargets } : {}),
557
+ ...(opts.sloSink !== undefined ? { sloSink: opts.sloSink } : {}),
558
+ });
559
+ // Ops item 32 — auto-assemble an incident bundle on the first failure-class
560
+ // trigger (circuit → open, egress-blocked, justification-deny storm). Gated
561
+ // by CREWHAUS_INCIDENTS; writes a raw capture (ring buffer + trigger meta)
562
+ // that `crewhaus incident collect --session <id>` turns into a full bundle.
563
+ const incidentCollector = attachIncidentCollector(bus, runContext, process.env, {
564
+ ...(opts.incidentsDir !== undefined ? { incidentsDir: opts.incidentsDir } : {}),
565
+ ...(opts.incidentSpec !== undefined ? { spec: opts.incidentSpec } : {}),
566
+ });
567
+ // Item 22 — surface each failover on stderr so an operator watching a
568
+ // plain (non-CREWHAUS_TRACE) run still sees provider switches. NOTE: the
569
+ // advisor-events session-log subscriber pattern has NOT landed on main
570
+ // (packages/event-log has no advisor kind) — when it does, failovers
571
+ // should ALSO persist into the session JSONL through it; until then the
572
+ // trace event + this stderr note are the observable surfaces.
573
+ let failoverNoteUnsubscribe;
574
+ if (failoverChain !== undefined) {
575
+ failoverNoteUnsubscribe = bus.subscribe((event) => {
576
+ if (event.kind !== "model_failover")
577
+ return;
578
+ process.stderr.write(`[failover] ${event.from} → ${event.to} (${event.reason})\n`);
579
+ });
580
+ }
581
+ // Item 27 — run-level spend cap. An ALWAYS-ON cost meter (independent of
582
+ // the CREWHAUS_COST_TRACKING env flag) accrues per-response spend priced
583
+ // off the wire model that actually served — `suppressEvents` keeps it off
584
+ // the trace surface so it never double-prints beside an env-attached
585
+ // cost-tracker. The pre-turn check (`enforceBudget`, below) reads
586
+ // `budgetMeter.getRunCost(runId).totalUsdMicros`. `budgetDegraded` gates
587
+ // the ladder to a single rung: once we degrade, we never degrade again —
588
+ // a second breach stops the run.
589
+ const budgetMeter = opts.budget !== undefined ? createCostTracker(bus, { suppressEvents: true }) : undefined;
590
+ let budgetDegraded = false;
591
+ /**
592
+ * Item 27 — pre-turn budget gate. Returns "stop" when the run must end
593
+ * before the next turn opens; "continue" otherwise. On a `degrade` breach
594
+ * it re-resolves the primary model to the cheaper rung IN PLACE (mutating
595
+ * `adapter`, `providerId`, `wireModelId`) and emits a `model_failover`
596
+ * (reason `budget_degrade`) — so the very next model call, and all cost
597
+ * accrual after it, use the degraded model. The current turn always
598
+ * completes; this only gates the NEXT turn, so an in-flight tool loop is
599
+ * never severed. Never throws — a re-resolution failure logs and stops.
600
+ */
601
+ const enforceBudget = async () => {
602
+ if (opts.budget === undefined || budgetMeter === undefined)
603
+ return "continue";
604
+ const spent = budgetMeter.getRunCost(bus.runId).totalUsdMicros;
605
+ if (spent < opts.budget.usdMicros)
606
+ return "continue";
607
+ const spentUsd = (spent / 1_000_000).toFixed(4);
608
+ const capUsd = (opts.budget.usdMicros / 1_000_000).toFixed(4);
609
+ if (opts.budget.onExceed.kind === "stop" || budgetDegraded) {
610
+ const why = budgetDegraded ? "degraded model also reached the cap" : "run budget reached";
611
+ process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — ${why}; ending the run.\n`);
612
+ return "stop";
613
+ }
614
+ // degrade: re-resolve the primary model to the cheaper rung, once.
615
+ const target = opts.budget.onExceed.model;
616
+ try {
617
+ const degraded = opts._budgetDegradeAdapter !== undefined
618
+ ? {
619
+ adapter: opts._budgetDegradeAdapter,
620
+ modelId: bestEffortWireModelId(target),
621
+ providerId: opts._budgetDegradeAdapter.providerId,
622
+ }
623
+ : await resolveModel(target);
624
+ const from = opts.model;
625
+ adapter =
626
+ opts.circuitBreaker !== undefined
627
+ ? wrapWithCircuitBreaker(degraded.adapter, {
628
+ ...opts.circuitBreaker,
629
+ adapterName: degraded.adapter.providerId,
630
+ bus,
631
+ })
632
+ : degraded.adapter;
633
+ // The degraded model is now the serving one; keep the trace/pricing
634
+ // identity coherent for the model_request/response events below.
635
+ providerId = degraded.providerId;
636
+ wireModelId = degraded.modelId;
637
+ degradedSpecModel = target;
638
+ // A degrade takes over from the failover chain: subsequent calls hit
639
+ // the degraded adapter directly (v1 keeps one rung, no re-entry to the
640
+ // chain). Null it so the request/response stamping stops consulting it.
641
+ failoverChain = undefined;
642
+ budgetDegraded = true;
643
+ bus.publish({
644
+ ...bus.envelope(),
645
+ kind: "model_failover",
646
+ from,
647
+ to: target,
648
+ reason: "budget_degrade",
649
+ });
650
+ process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrading ${from} → ${target}.\n`);
651
+ return "continue";
652
+ }
653
+ catch (err) {
654
+ const message = err instanceof Error ? err.message : String(err);
655
+ process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrade to "${target}" failed (${message}); ending the run.\n`);
656
+ return "stop";
657
+ }
658
+ };
447
659
  // Working-indicator spinner. Auto-enables only for an interactive REPL on a
448
660
  // TTY (see `isSpinnerEnabled`); `singleTurn`/injected-`input`/piped runs stay
449
661
  // byte-identical to before. `out.write` is the spinner-aware stdout sink:
@@ -505,6 +717,11 @@ export async function runChatLoop(opts) {
505
717
  inputTokens: ev.inputTokens,
506
718
  outputTokens: ev.outputTokens,
507
719
  cachedReadTokens: ev.cachedReadTokens,
720
+ // Optional on the event (older emitters may omit it) — persist
721
+ // only when present so old-log parsing stays additive-safe.
722
+ ...(ev.cacheCreationTokens !== undefined
723
+ ? { cacheCreationTokens: ev.cacheCreationTokens }
724
+ : {}),
508
725
  costUsdMicros: ev.costUsdMicros,
509
726
  ...(ev.tenantId !== undefined ? { tenantId: ev.tenantId } : {}),
510
727
  },
@@ -516,6 +733,18 @@ export async function runChatLoop(opts) {
516
733
  });
517
734
  });
518
735
  }
736
+ // Advisor groundwork (item 14) — persist the trace-bus-only advisory
737
+ // signals (recovery actions, per-call tool stats, resolved permission
738
+ // decisions, model stop reasons) into the same session JSONL the
739
+ // transcript already writes to, so `crewhaus advise` can mine sessions
740
+ // offline. DEFAULT-ON (the lines are tiny and they are the advisor's
741
+ // food); disable with CREWHAUS_ADVISOR_EVENTS=0. See observability.ts.
742
+ const advisorPersist = attachAdvisorPersistence(bus, eventLog, runContext);
743
+ // Ops item 38 — persist the trace-bus-only `mcp_call_end` events into the
744
+ // session JSONL as the durable `mcp_stats` kind so `crewhaus mcp doctor` can
745
+ // score per-server MCP health offline. Shares the advisor's DEFAULT-ON gate
746
+ // (CREWHAUS_ADVISOR_EVENTS=0 disables both). See observability.ts.
747
+ const mcpStatsPersist = attachMcpStatsPersistence(bus, eventLog, runContext);
519
748
  // Per-run state container — coordination surface for hooks/skills/tools
520
749
  // landing in Section 11+. Section 10 only ships the plumbing; the
521
750
  // underscore prefix marks the intentional non-consumer.
@@ -557,6 +786,21 @@ export async function runChatLoop(opts) {
557
786
  return { allowed: true };
558
787
  }
559
788
  }
789
+ // Feature #53 — auto-capture at teardown. Runs in the same finally where the
790
+ // `stop` hook fires, but is NOT a credential-stripped hook: the caller-
791
+ // supplied `onCapture` (which writes to the memory-store from the CLI's full
792
+ // environment) needs the process env, exactly like the feedback teardown.
793
+ // Best-effort — a capture failure never turns a clean exit into an error.
794
+ const maybeAutoCapture = async () => {
795
+ if (opts.memory?.autoCapture !== true || opts.memory.onCapture === undefined)
796
+ return;
797
+ try {
798
+ await opts.memory.onCapture(runContext.turnNumber, sessionId);
799
+ }
800
+ catch (err) {
801
+ runContext.logger.warn("memory auto-capture failed", { error: err.message });
802
+ }
803
+ };
560
804
  // Section 11 — `session-start` fires once after the persistence + run
561
805
  // context boot is complete. Observational only (no deny/block effect).
562
806
  await fireHook("session-start", {
@@ -594,7 +838,9 @@ export async function runChatLoop(opts) {
594
838
  // CLAUDE.md auto-load behaviour. Pillar 3 compliance: each file's content
595
839
  // is classified via boundary-classifier with origin "user" inside
596
840
  // loadProjectMemory() (the user's repo is developer-trusted).
597
- const projectMemory = await loadProjectMemory();
841
+ const projectMemory = await loadProjectMemory(opts.preferenceFiles !== undefined && opts.preferenceFiles.length > 0
842
+ ? { extraFiles: opts.preferenceFiles }
843
+ : {});
598
844
  const projectMemoryBlock = projectMemory.prompt.length > 0
599
845
  ? [
600
846
  {
@@ -609,12 +855,50 @@ export async function runChatLoop(opts) {
609
855
  .map((f) => f.filename)
610
856
  .join(", ")}\n`);
611
857
  }
858
+ // Feature #53 — auto-recall: at session start, pull the top-K relevant
859
+ // cross-session memories and inject them as a system block, mirroring the
860
+ // project-memory auto-load above. The caller supplies the `recall` seam
861
+ // (runtime-core stays independent of memory-store). Best-effort: a recall
862
+ // failure never aborts the run.
863
+ let memoryRecallBlock = [];
864
+ if (opts.memory?.autoRecall === true && opts.memory.recall !== undefined) {
865
+ try {
866
+ const seed = opts.memory.recallSeed ?? opts.instructions;
867
+ const k = opts.memory.recallK ?? 5;
868
+ const lines = await opts.memory.recall(seed, k);
869
+ if (lines.length > 0) {
870
+ // Pillar 3 — a recalled memory can embed content shaped by untrusted
871
+ // tool output from an earlier session, so it is NOT verbatim-safe like
872
+ // the developer's own repo files. Two defenses, mirroring how the rest
873
+ // of the security fabric treats injected content:
874
+ // 1. Delimiter safety — neutralize any `</recalled_memory>` closing
875
+ // tag inside a recalled line so a poisoned memory can't break out
876
+ // of its block and inject trailing instructions.
877
+ // 2. Classification — run the assembled block through the SAME
878
+ // `classifyBoundary(text, { origin: "user" })` best-effort path
879
+ // `loadProjectMemory()` uses (pass policy: classify + emit trace
880
+ // events without mutating), so recalled content flows through the
881
+ // boundary classifier exactly like every other prompt-bound source.
882
+ const body = lines
883
+ .map((l) => `- ${escapeBoundaryDelimiter(l, "recalled_memory")}`)
884
+ .join("\n");
885
+ const text = `<recalled_memory>\nRelevant facts remembered from earlier sessions:\n${body}\n</recalled_memory>`;
886
+ await classifyBoundary(text, { origin: "user" }).catch(() => undefined);
887
+ memoryRecallBlock = [{ type: "text", text, cache_control: { type: "ephemeral" } }];
888
+ process.stdout.write(`[memory] recalled ${lines.length} memory(ies) into the prompt\n`);
889
+ }
890
+ }
891
+ catch (err) {
892
+ runContext.logger.warn("memory auto-recall failed", { error: err.message });
893
+ }
894
+ }
612
895
  // Section 17 — runtime-core no longer prepends the Claude Code OAuth
613
896
  // prefix; `adapter-anthropic` handles that internally so each adapter
614
897
  // owns its provider-specific auth-shape requirements.
615
898
  let systemBlocks = [
616
899
  userInstructions,
617
900
  ...projectMemoryBlock,
901
+ ...memoryRecallBlock,
618
902
  ...skillsBlock,
619
903
  ];
620
904
  // Section 27 — rotate cache markers on the system block array if the
@@ -814,6 +1098,21 @@ export async function runChatLoop(opts) {
814
1098
  `tool denied: \`${tu.name}\` defaulted to "ask" and single-turn mode has no interactive surface to prompt on. ` +
815
1099
  `Add an explicit rule to permissions.rules in your spec, e.g. \`{ type: alwaysAllow, pattern: ${tu.name} }\`, or run in REPL mode where "ask" can prompt.`;
816
1100
  }
1101
+ // Advisor groundwork (item 14) — event the ask RESOLUTION. The publish
1102
+ // above fired BEFORE the approval prompt (decision "ask", no outcome);
1103
+ // this one fires after `askApproval` resolves — or after the
1104
+ // single-turn fallback collapses the ask to a deny — so offline advice
1105
+ // mining can measure how each tool's prompts are actually answered.
1106
+ // The advisor persistence subscriber (observability.ts) keys on
1107
+ // `askOutcome` to persist exactly this resolved form.
1108
+ bus.publish({
1109
+ ...bus.envelope(),
1110
+ kind: "permission_decision",
1111
+ toolName: tu.name,
1112
+ decision: "ask",
1113
+ mode: permissionMode,
1114
+ askOutcome: approved ? "approved" : "denied",
1115
+ });
817
1116
  }
818
1117
  if (!approved) {
819
1118
  return finish({
@@ -952,6 +1251,36 @@ export async function runChatLoop(opts) {
952
1251
  originsFound: egress.originsFound,
953
1252
  matchCount: egress.matchCount,
954
1253
  });
1254
+ // Item 20 — emit the durable `egress_decision` audit record for every
1255
+ // non-pass verdict so egress history is triageable offline (the
1256
+ // trace-bus event above is ephemeral). Only the lineage SUMMARY is
1257
+ // stored — never the raw outbound payload (which is sensitive and
1258
+ // often carries the very tagged content that tripped the classifier),
1259
+ // matching the payload shape `@crewhaus/audit-log` documents for this
1260
+ // kind. Best-effort but surfaced: an I/O failure on the audit trail is
1261
+ // logged and swallowed so a full audit disk does not crash a governed
1262
+ // run, mirroring the justification sink above. Absent sink → no-op.
1263
+ if (opts.egressAuditSink !== undefined) {
1264
+ try {
1265
+ await opts.egressAuditSink.append({
1266
+ kind: "egress_decision",
1267
+ payload: {
1268
+ sinkId: egress.sinkId,
1269
+ sinkScope: egress.sinkScope,
1270
+ verdict: egress.verdict,
1271
+ originsFound: egress.originsFound,
1272
+ matchCount: egress.matchCount,
1273
+ },
1274
+ });
1275
+ }
1276
+ catch (err) {
1277
+ runContext.logger.warn("egress audit append failed", {
1278
+ toolUseId: tu.id,
1279
+ toolName: tu.name,
1280
+ error: err.message,
1281
+ });
1282
+ }
1283
+ }
955
1284
  }
956
1285
  if (egress.verdict === "block") {
957
1286
  return finish({
@@ -1243,6 +1572,14 @@ export async function runChatLoop(opts) {
1243
1572
  let recovery = initialRecoveryState;
1244
1573
  const maxToolIterations = opts.maxToolIterations ?? DEFAULT_MAX_TOOL_ITERATIONS;
1245
1574
  let toolIterations = 0;
1575
+ // Item 26 — tool_use blocks THIS turn produced (fed to the NEXT turn's
1576
+ // tier decision) and the misroute-recovery escalation latch (a fast-tier
1577
+ // failure forces `default` for the rest of this turn). `servedFastTier`
1578
+ // records whether the LAST attempt streamed on the fast tier, so the
1579
+ // error catch only escalates a genuine fast-tier misroute.
1580
+ let thisTurnToolUseCount = 0;
1581
+ let escalateTier = false;
1582
+ let servedFastTier = false;
1246
1583
  while (state.kind !== "Done") {
1247
1584
  if (turnAbort.signal.aborted) {
1248
1585
  state = transition(state, { kind: "Aborted" });
@@ -1294,12 +1631,59 @@ export async function runChatLoop(opts) {
1294
1631
  // → `us.anthropic.…`) so cost-tracker pricing lookups and the
1295
1632
  // OTel `gen_ai.request.model` attribute match; the original
1296
1633
  // spec string rides along as `specModel` when it differs.
1634
+ // Item 22 — with a failover chain the serving candidate may not
1635
+ // be the primary: stamp the request with the chain's routing
1636
+ // plan (best-effort; the response event below uses lastServed(),
1637
+ // which is exact, so cost-tracker pricing always keys on the
1638
+ // model that actually served).
1639
+ const plannedCandidate = failoverChain?.plan();
1640
+ let reqWireModelId = plannedCandidate?.modelId ?? wireModelId;
1641
+ let reqProviderId = plannedCandidate?.providerId ?? providerId;
1642
+ // Item 27 — after a budget degrade, `failoverChain` is null and
1643
+ // `degradedSpecModel` is the running spec model.
1644
+ let reqSpecModel = plannedCandidate?.modelString ?? degradedSpecModel ?? opts.model;
1645
+ // The adapter this turn streams through: the primary/chain by
1646
+ // default, or a tier adapter when the two-tier router is active.
1647
+ let turnAdapter = adapter;
1648
+ // Item 26 — two-tier turn-difficulty router. Pick a tier from
1649
+ // DETERMINISTIC per-turn signals (context size, tools-in-play,
1650
+ // turn index, prior-turn tool_use density); a fast-tier misroute
1651
+ // recovery forces `default` (see the catch below). Publish the
1652
+ // decision as a `model_tier_route` event, then stream through the
1653
+ // chosen tier's already-resolved adapter.
1654
+ if (tierRouter !== undefined) {
1655
+ const decision = escalateTier
1656
+ ? { tier: "default", reason: "escalated after fast-tier failure" }
1657
+ : tierRouter.route({
1658
+ contextTokens: estimateTokens(messages),
1659
+ toolsInPlay: (anthropicTools?.length ?? 0) > 0,
1660
+ // `turnNumber` is 1-based (incremented before the turn
1661
+ // runs); the router wants a 0-based index so its
1662
+ // first-turn check keys on the opening turn.
1663
+ turnIndex: Math.max(0, runContext.turnNumber - 1),
1664
+ priorTurnToolUseCount,
1665
+ });
1666
+ const chosen = tierRouter.tier(decision.tier);
1667
+ turnAdapter = chosen.adapter;
1668
+ reqWireModelId = chosen.modelId;
1669
+ reqProviderId = chosen.adapter.providerId;
1670
+ reqSpecModel = chosen.modelString;
1671
+ servedFastTier = decision.tier === "fast";
1672
+ bus.publish({
1673
+ ...bus.envelope(),
1674
+ kind: "model_tier_route",
1675
+ tier: decision.tier,
1676
+ model: chosen.modelId,
1677
+ reason: decision.reason,
1678
+ ...(escalateTier ? { escalated: true } : {}),
1679
+ });
1680
+ }
1297
1681
  bus.publish({
1298
1682
  ...modelStartEnv,
1299
1683
  kind: "model_request",
1300
- model: wireModelId,
1301
- ...(opts.model !== wireModelId ? { specModel: opts.model } : {}),
1302
- provider: providerId,
1684
+ model: reqWireModelId,
1685
+ ...(reqSpecModel !== reqWireModelId ? { specModel: reqSpecModel } : {}),
1686
+ provider: reqProviderId,
1303
1687
  messageCount: messages.length,
1304
1688
  toolCount: anthropicTools?.length ?? 0,
1305
1689
  streaming: opts.streaming === true,
@@ -1320,8 +1704,11 @@ export async function runChatLoop(opts) {
1320
1704
  // bookkeeping is wire-compatible for the block kinds we
1321
1705
  // actually consume (text / image / tool_use / tool_result
1322
1706
  // / thinking).
1323
- const reqStream = adapter.stream({
1324
- model: wireModelId,
1707
+ // Item 22 — the failover chain rewrites `model` per serving
1708
+ // candidate internally; the planned id here is the single-adapter
1709
+ // value and the chain's best-effort prediction otherwise.
1710
+ const reqStream = turnAdapter.stream({
1711
+ model: reqWireModelId,
1325
1712
  system: systemBlocks.map((b) => ({
1326
1713
  type: "text",
1327
1714
  text: b.text,
@@ -1388,15 +1775,20 @@ export async function runChatLoop(opts) {
1388
1775
  await logEvent("assistant_message", { content: committedStreamContent });
1389
1776
  }
1390
1777
  terminalContent = [...committedStreamContent];
1778
+ // Item 22 — lastServed() is exact: the candidate that actually
1779
+ // streamed this response. Cost-tracker prices off this pair.
1780
+ const servedStreaming = failoverChain?.lastServed();
1781
+ const respModelIdS = servedStreaming?.modelId ?? wireModelId;
1782
+ const respSpecModelS = servedStreaming?.modelString ?? degradedSpecModel ?? opts.model;
1391
1783
  bus.publish({
1392
1784
  ...bus.envelope(),
1393
1785
  spanId: modelStartEnv.spanId,
1394
1786
  kind: "model_response",
1395
1787
  // Wire model id, NOT the spec string — cost-tracker resolves
1396
1788
  // pricing on this field (see the model_request publish above).
1397
- model: wireModelId,
1398
- ...(opts.model !== wireModelId ? { specModel: opts.model } : {}),
1399
- provider: providerId,
1789
+ model: respModelIdS,
1790
+ ...(respSpecModelS !== respModelIdS ? { specModel: respSpecModelS } : {}),
1791
+ provider: servedStreaming?.providerId ?? providerId,
1400
1792
  stopReason: stopReason ?? "end_turn",
1401
1793
  usage,
1402
1794
  durationMs: performance.now() - t0Model,
@@ -1410,6 +1802,8 @@ export async function runChatLoop(opts) {
1410
1802
  state = transition(state, { kind: "ModelReturnedText" });
1411
1803
  break;
1412
1804
  }
1805
+ // Item 26 — accrue this turn's tool_use count (next-turn tier signal).
1806
+ thisTurnToolUseCount += toolUses.length;
1413
1807
  for (const tu of toolUses) {
1414
1808
  toolUseHistory.push({ id: tu.id, name: tu.name, input: tu.input });
1415
1809
  }
@@ -1447,15 +1841,20 @@ export async function runChatLoop(opts) {
1447
1841
  },
1448
1842
  });
1449
1843
  endTurn();
1844
+ // Item 22 — lastServed() is exact: the candidate that actually
1845
+ // streamed this response. Cost-tracker prices off this pair.
1846
+ const servedCandidate = failoverChain?.lastServed();
1847
+ const respWireModelId = servedCandidate?.modelId ?? wireModelId;
1848
+ const respSpecModel = servedCandidate?.modelString ?? degradedSpecModel ?? opts.model;
1450
1849
  bus.publish({
1451
1850
  ...bus.envelope(),
1452
1851
  spanId: modelStartEnv.spanId,
1453
1852
  kind: "model_response",
1454
1853
  // Wire model id, NOT the spec string — cost-tracker resolves
1455
1854
  // pricing on this field (see the model_request publish above).
1456
- model: wireModelId,
1457
- ...(opts.model !== wireModelId ? { specModel: opts.model } : {}),
1458
- provider: providerId,
1855
+ model: respWireModelId,
1856
+ ...(respSpecModel !== respWireModelId ? { specModel: respSpecModel } : {}),
1857
+ provider: servedCandidate?.providerId ?? providerId,
1459
1858
  stopReason: final.stopReason,
1460
1859
  usage: final.usage,
1461
1860
  durationMs: performance.now() - t0Model,
@@ -1529,6 +1928,14 @@ export async function runChatLoop(opts) {
1529
1928
  break;
1530
1929
  }
1531
1930
  lastErrorForRecovery = err;
1931
+ // Item 26 — a fast-tier turn that FAILED is a MISROUTE: latch the
1932
+ // escalation so the recovery retry re-runs on the `default` tier
1933
+ // (the loop re-enters NeedModel via the recovery ladder, which
1934
+ // then picks `default`). Composes with the existing recovery
1935
+ // actions — a retry/continue now streams on the sturdier tier.
1936
+ if (tierRouter !== undefined && servedFastTier) {
1937
+ escalateTier = true;
1938
+ }
1532
1939
  const errObj = err;
1533
1940
  const errorName = typeof errObj.name === "string" ? errObj.name : "Error";
1534
1941
  const errorMessage = typeof errObj.message === "string" ? errObj.message : String(err);
@@ -1556,6 +1963,9 @@ export async function runChatLoop(opts) {
1556
1963
  state = transition(state, { kind: "Aborted" });
1557
1964
  break;
1558
1965
  }
1966
+ // Item 26 — accrue this turn's tool_use count (a next-turn tier
1967
+ // signal): a dense tool turn marks an active multi-step task.
1968
+ thisTurnToolUseCount += state.toolUses.length;
1559
1969
  const toolResults = await runToolBatch(state.toolUses);
1560
1970
  messages.push({ role: "user", content: toolResults });
1561
1971
  await logEvent("user_message", { content: toolResults });
@@ -1576,7 +1986,10 @@ export async function runChatLoop(opts) {
1576
1986
  // emitting `BudgetExceeded` from `runOneTurn`, restore a
1577
1987
  // `NeedCompaction` → `CompactionDone` arm (and cover it) here.
1578
1988
  case "NeedRecovery": {
1579
- const action = recover(lastErrorForRecovery, recovery);
1989
+ // Item 23 / Section 55 — consult the spec's failure_taxonomy first
1990
+ // (named classes take precedence over the built-in flow), so a
1991
+ // matched `switch-model` (or retry/compact/…) entry wins.
1992
+ const action = recover(lastErrorForRecovery, recovery, opts.failureTaxonomy);
1580
1993
  recovery = advanceState(recovery, action);
1581
1994
  runContext.logger.info("recovery.action", {
1582
1995
  kind: action.kind,
@@ -1675,6 +2088,32 @@ export async function runChatLoop(opts) {
1675
2088
  state = transition(state, { kind: "RecoveryDone" });
1676
2089
  break;
1677
2090
  }
2091
+ case "switch-model": {
2092
+ // Item 23 — a `switch-model` failure_taxonomy verdict: abandon
2093
+ // the active provider candidate FOR THIS TURN and re-issue the
2094
+ // same request onto the next failover candidate. Mechanism:
2095
+ // force the active candidate's breaker open so the chain
2096
+ // reroutes on the next `stream()`; the chain publishes the
2097
+ // `model_failover` (reason breaker_open) itself. No message
2098
+ // mutation and no backoff sleep — a switch is instant. Without
2099
+ // a failover chain (no `modelFallbacks`) there is nothing to
2100
+ // switch to: this degrades to a plain re-issue against the same
2101
+ // adapter, which the per-turn budget bounds. RecoveryDone loops
2102
+ // back to NeedModel, which re-streams.
2103
+ const tripped = failoverChain?.tripActive("switch-model recovery");
2104
+ if (tripped !== undefined) {
2105
+ out.spinner.start("switching model");
2106
+ out.spinner.stop();
2107
+ runContext.logger.info("recovery.switch-model tripped candidate", {
2108
+ from: tripped,
2109
+ });
2110
+ }
2111
+ else {
2112
+ runContext.logger.warn("recovery.switch-model: no failover chain — re-issuing on the same model");
2113
+ }
2114
+ state = transition(state, { kind: "RecoveryDone" });
2115
+ break;
2116
+ }
1678
2117
  case "fail":
1679
2118
  throw new RuntimeError(`recovery failed: ${action.reason}`);
1680
2119
  }
@@ -1682,6 +2121,9 @@ export async function runChatLoop(opts) {
1682
2121
  }
1683
2122
  }
1684
2123
  }
2124
+ // Item 26 — hand this turn's tool_use count to the next turn's tier
2125
+ // decision (the prior-tool-density signal).
2126
+ priorTurnToolUseCount = thisTurnToolUseCount;
1685
2127
  return { terminalContent };
1686
2128
  }
1687
2129
  // Single-shot path used by the workflow target and (Section 12) the
@@ -1768,9 +2210,16 @@ export async function runChatLoop(opts) {
1768
2210
  turnCount: runContext.turnNumber,
1769
2211
  reason: "complete",
1770
2212
  });
2213
+ await maybeAutoCapture();
1771
2214
  await subscribers.flushAll();
1772
2215
  await subscribers.shutdownAll();
1773
2216
  costPersistUnsubscribe?.();
2217
+ failoverNoteUnsubscribe?.();
2218
+ incidentCollector?.unsubscribe();
2219
+ budgetMeter?.unsubscribe();
2220
+ advisorPersist?.unsubscribe();
2221
+ mcpStatsPersist?.unsubscribe();
2222
+ printAdvisorDigest(advisorPersist);
1774
2223
  await sessionStore
1775
2224
  .update(sessionId, { lastTurnIndex: runContext.turnNumber })
1776
2225
  .catch((err) => {
@@ -1909,6 +2358,13 @@ export async function runChatLoop(opts) {
1909
2358
  }
1910
2359
  }
1911
2360
  }
2361
+ // Item 27 — run-level spend cap. Enforced HERE (a real turn is about
2362
+ // to run, input is not exit/blank) so an idle prompt never trips it.
2363
+ // On a `stop` breach we break before committing the input — the turn
2364
+ // never runs. On a `degrade` breach `enforceBudget` has already
2365
+ // swapped the primary adapter, so this turn runs on the cheaper model.
2366
+ if ((await enforceBudget()) === "stop")
2367
+ break;
1912
2368
  messages.push({ role: "user", content: effectiveInput });
1913
2369
  await logEvent("user_message", { content: effectiveInput });
1914
2370
  runContext.turnNumber += 1;
@@ -1979,9 +2435,16 @@ export async function runChatLoop(opts) {
1979
2435
  turnCount: runContext.turnNumber,
1980
2436
  reason: runAbort.signal.aborted ? "abort" : "exit",
1981
2437
  });
2438
+ await maybeAutoCapture();
1982
2439
  await subscribers.flushAll();
1983
2440
  await subscribers.shutdownAll();
1984
2441
  costPersistUnsubscribe?.();
2442
+ failoverNoteUnsubscribe?.();
2443
+ incidentCollector?.unsubscribe();
2444
+ budgetMeter?.unsubscribe();
2445
+ advisorPersist?.unsubscribe();
2446
+ mcpStatsPersist?.unsubscribe();
2447
+ printAdvisorDigest(advisorPersist);
1985
2448
  if (shouldInstallSigint) {
1986
2449
  process.removeListener("SIGINT", sigintHandler);
1987
2450
  }
@@ -1995,6 +2458,18 @@ export async function runChatLoop(opts) {
1995
2458
  }
1996
2459
  return "";
1997
2460
  }
2461
+ /**
2462
+ * Item 14 in-run digest — one stderr line at session end when the advisor
2463
+ * persistence subscriber's cheap tally tripped a threshold, pointing at
2464
+ * `crewhaus advise --session <id>` for the full report. stderr (not stdout)
2465
+ * so piped/captured agent output is never polluted; silent on healthy runs
2466
+ * and when the subscriber is disabled (CREWHAUS_ADVISOR_EVENTS=0).
2467
+ */
2468
+ function printAdvisorDigest(advisorPersist) {
2469
+ const line = advisorPersist?.digestLine();
2470
+ if (line !== undefined)
2471
+ process.stderr.write(`${line}\n`);
2472
+ }
1998
2473
  function isAbortError(err) {
1999
2474
  // Section 17 — runtime-core no longer imports the Anthropic SDK at
2000
2475
  // runtime, so we name-match instead of `instanceof`. The provider