@crewhaus/runtime-core 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +138 -0
- package/dist/alert-watchdog.js +290 -0
- package/dist/incident-collector.d.ts +42 -0
- package/dist/incident-collector.js +125 -0
- package/dist/index.d.ts +222 -4
- package/dist/index.js +512 -32
- package/dist/janitor.d.ts +88 -0
- package/dist/janitor.js +287 -0
- package/dist/observability.d.ts +178 -2
- package/dist/observability.js +466 -2
- package/dist/project-memory.d.ts +17 -1
- package/dist/project-memory.js +42 -0
- package/dist/slo-monitor.d.ts +221 -0
- package/dist/slo-monitor.js +397 -0
- package/package.json +39 -39
package/dist/index.js
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import * as readline from "node:readline/promises";
|
|
2
2
|
import { createAbortTree } from "@crewhaus/abort-controller";
|
|
3
3
|
import { collectFinalMessage, consumeStream, } from "@crewhaus/adapter-anthropic";
|
|
4
|
-
import { setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
|
|
4
|
+
import { classifyBoundary, setDefaultBoundaryLlmClassifier } from "@crewhaus/boundary-classifier";
|
|
5
5
|
import { wrap as wrapWithCircuitBreaker } from "@crewhaus/circuit-breaker";
|
|
6
6
|
import { autoCompact } from "@crewhaus/compaction-autocompact";
|
|
7
7
|
import { snip } from "@crewhaus/compaction-snip";
|
|
8
|
+
import { createCostTracker } from "@crewhaus/cost-tracker";
|
|
8
9
|
import { classifyEgress, summarizeEgress, } from "@crewhaus/egress-classifier";
|
|
9
10
|
import { ConfigError, RuntimeError } from "@crewhaus/errors";
|
|
10
11
|
import { openEventLog } from "@crewhaus/event-log";
|
|
11
12
|
import { aggregateDecisions, runHooks } from "@crewhaus/hooks-engine";
|
|
12
|
-
import { parseModelString, resolveModel } from "@crewhaus/model-router";
|
|
13
|
+
import { createFailoverChain, createTierRouter, parseModelString, resolveModel, } from "@crewhaus/model-router";
|
|
13
14
|
import { BUILTIN_DEFAULT_RULES, emptyRuleSet, evaluateJustification, evaluateWithReason, ruleBasedJustificationJudge, } from "@crewhaus/permission-engine";
|
|
14
15
|
import { manage as manageCacheMarkers } from "@crewhaus/prompt-cache-manager";
|
|
15
16
|
import { buildRedactionNotice, classifyText, llmClassifierEnabled, } from "@crewhaus/prompt-injection-detector";
|
|
@@ -30,7 +31,8 @@ import { TraceEventBus } from "@crewhaus/trace-event-bus";
|
|
|
30
31
|
import { initialState, transition, } from "@crewhaus/turn-state-machine";
|
|
31
32
|
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
32
33
|
import { createCliMarkdownRenderer, isCliMarkdownEnabled, } from "./cli-markdown";
|
|
33
|
-
import {
|
|
34
|
+
import { attachIncidentCollector } from "./incident-collector";
|
|
35
|
+
import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, } from "./observability";
|
|
34
36
|
import { loadProjectMemory } from "./project-memory";
|
|
35
37
|
import { createCliOutput, isSpinnerEnabled } from "./spinner";
|
|
36
38
|
/**
|
|
@@ -113,6 +115,21 @@ const DEFAULT_SNIP_KEEP_TAIL = 20;
|
|
|
113
115
|
// moved into `@crewhaus/adapter-anthropic`. Re-exported here so legacy
|
|
114
116
|
// importers (eval-judge fallback, downstream tests) keep compiling.
|
|
115
117
|
export { CLAUDE_CODE_HEADERS, CLAUDE_CODE_SYSTEM_PREFIX, OAUTH_BETAS, createAnthropicClient, resolveAuth, } from "@crewhaus/adapter-anthropic";
|
|
118
|
+
// Ops item 36 — boot-time self-heal janitor for daemon shapes. Lives in its
|
|
119
|
+
// own module (`./janitor`) but is re-exported here because emitted bundles
|
|
120
|
+
// only import the package root.
|
|
121
|
+
export { createJanitor } from "./janitor";
|
|
122
|
+
// Ops item 31 — alert watchdog seams re-exported so the CLI/codegen can build
|
|
123
|
+
// the durable + off-box alert sink (audit append + settings.json alert hook /
|
|
124
|
+
// webhook) it passes as `RunChatLoopOptions.alertSink`.
|
|
125
|
+
export { attachDefaultSubscribers, } from "./observability";
|
|
126
|
+
// Ops item 37 — SLO monitor seams re-exported so the CLI/codegen can build the
|
|
127
|
+
// injected mitigation ladder (audit / pause-intake / rollback) it passes as
|
|
128
|
+
// `RunChatLoopOptions.sloSink`, and the doctor probe can share the breach types.
|
|
129
|
+
export { DEFAULT_SLO_WINDOW_MS, MIN_SLO_SAMPLES, SloWindow, attachSloMonitor, detectSloBreaches, } from "./slo-monitor";
|
|
130
|
+
// Ops item 32 — incident collector seams re-exported so the CLI can share the
|
|
131
|
+
// raw-capture shape + trigger classifier with `incident collect`.
|
|
132
|
+
export { attachIncidentCollector, classifyIncidentTrigger, } from "./incident-collector";
|
|
116
133
|
/**
|
|
117
134
|
* Reconcile a message history by dropping "orphan" `tool_use` blocks — a
|
|
118
135
|
* `tool_use` with no answering `tool_result` anywhere later in the
|
|
@@ -289,6 +306,19 @@ export function defaultSinkScope(toolName) {
|
|
|
289
306
|
return "external-dynamic";
|
|
290
307
|
return "external-configured";
|
|
291
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* Neutralize a boundary-block closing delimiter embedded in untrusted content
|
|
311
|
+
* so a poisoned line can't terminate its wrapper block early and inject
|
|
312
|
+
* trailing instructions. Replaces `</tag>` (any casing/whitespace) with a
|
|
313
|
+
* visually-inert `<\/tag>` so the content survives for the model to read but
|
|
314
|
+
* no longer parses as the real closing tag. Used for the recalled-memory block
|
|
315
|
+
* (#53) — recalled memories may carry content shaped by untrusted tool output.
|
|
316
|
+
*/
|
|
317
|
+
function escapeBoundaryDelimiter(text, tag) {
|
|
318
|
+
// Match `</tag>` tolerantly (leading whitespace inside the tag, any casing).
|
|
319
|
+
const re = new RegExp(`</\\s*${tag}\\s*>`, "gi");
|
|
320
|
+
return text.replace(re, `<\\/${tag}>`);
|
|
321
|
+
}
|
|
292
322
|
/**
|
|
293
323
|
* Strip the model-string provider grammar down to the wire model id
|
|
294
324
|
* (`"bedrock/us.anthropic.claude-..."` → `"us.anthropic.claude-..."`)
|
|
@@ -333,25 +363,65 @@ export async function runChatLoop(opts) {
|
|
|
333
363
|
providerId: opts._adapter.providerId,
|
|
334
364
|
}
|
|
335
365
|
: await resolveModel(opts.model);
|
|
336
|
-
// Section 27
|
|
337
|
-
// `
|
|
338
|
-
//
|
|
339
|
-
//
|
|
366
|
+
// Section 27 / item 22 — resilience wrapping around the primary adapter.
|
|
367
|
+
// - `modelFallbacks` non-empty → a model-router failover chain: every
|
|
368
|
+
// candidate (primary first) breaker-wrapped with the `circuitBreaker`
|
|
369
|
+
// tuning; calls route to the first candidate whose breaker admits
|
|
370
|
+
// traffic; `model_failover` trace events surface routing changes.
|
|
371
|
+
// - `circuitBreaker` alone → the single-adapter breaker wrap.
|
|
372
|
+
// The chain publishes through a LATE-BOUND bus getter: the run's real bus
|
|
373
|
+
// is minted below (createRunContext), so `observabilityBus` is re-pointed
|
|
374
|
+
// at it once it exists — a `crewhaus run` without a caller-supplied
|
|
375
|
+
// runContext still surfaces model_failover + circuit_state_changed.
|
|
340
376
|
const baseAdapter = primaryResolution.adapter;
|
|
341
377
|
let breakerWrap;
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
378
|
+
let failoverChain;
|
|
379
|
+
let observabilityBus = opts.runContext?.eventBus;
|
|
380
|
+
const modelFallbacks = opts.modelFallbacks ?? [];
|
|
381
|
+
let adapter;
|
|
382
|
+
if (modelFallbacks.length > 0) {
|
|
383
|
+
// Seed the chain with the already-resolved primary (honours `_adapter`
|
|
384
|
+
// injection and avoids a second resolveModel round-trip) plus any
|
|
385
|
+
// test-injected fallback adapters.
|
|
386
|
+
const injectedAdapters = new Map();
|
|
387
|
+
injectedAdapters.set(opts.model, baseAdapter);
|
|
388
|
+
for (const [modelString, injected] of opts._failoverAdapters ?? []) {
|
|
389
|
+
injectedAdapters.set(modelString, injected);
|
|
390
|
+
}
|
|
391
|
+
failoverChain = await createFailoverChain({
|
|
392
|
+
model: opts.model,
|
|
393
|
+
fallbacks: modelFallbacks,
|
|
394
|
+
...(opts.circuitBreaker !== undefined ? { breaker: opts.circuitBreaker } : {}),
|
|
395
|
+
getBus: () => observabilityBus,
|
|
396
|
+
adapters: injectedAdapters,
|
|
397
|
+
});
|
|
398
|
+
// Doctor-style boot report: fallbacks that failed credential/package
|
|
399
|
+
// resolution are listed once, loudly, on stderr — and stay in the chain
|
|
400
|
+
// (they are re-tried when routing actually reaches them; see item 22).
|
|
401
|
+
for (const warning of failoverChain.warnings()) {
|
|
402
|
+
process.stderr.write(`[failover] ${warning}\n`);
|
|
403
|
+
}
|
|
404
|
+
adapter = failoverChain;
|
|
405
|
+
}
|
|
406
|
+
else if (opts.circuitBreaker !== undefined) {
|
|
345
407
|
breakerWrap = wrapWithCircuitBreaker(baseAdapter, {
|
|
346
408
|
...opts.circuitBreaker,
|
|
347
409
|
adapterName: baseAdapter.providerId,
|
|
348
410
|
bus: opts.runContext?.eventBus,
|
|
349
411
|
});
|
|
350
|
-
|
|
351
|
-
}
|
|
412
|
+
adapter = breakerWrap;
|
|
413
|
+
}
|
|
414
|
+
else {
|
|
415
|
+
adapter = baseAdapter;
|
|
416
|
+
}
|
|
352
417
|
void breakerWrap; // exposed via runtime stats once gateway/eval consumers land
|
|
353
|
-
|
|
354
|
-
|
|
418
|
+
// `let` (item 27): a `budget: { on_exceed: degrade }` breach re-resolves
|
|
419
|
+
// the primary model in place, so these become the degraded model's
|
|
420
|
+
// identity. `degradedSpecModel` overrides `opts.model` as the specModel
|
|
421
|
+
// stamped on model_request/response once a degrade has happened.
|
|
422
|
+
let providerId = primaryResolution.providerId;
|
|
423
|
+
let wireModelId = primaryResolution.modelId;
|
|
424
|
+
let degradedSpecModel;
|
|
355
425
|
// Section 17 — feature gate: a spec that declares tools cannot run on an
|
|
356
426
|
// adapter that doesn't speak tool use (e.g. Bedrock Llama/Mistral
|
|
357
427
|
// families). Fail with a clear ConfigError naming the model instead of
|
|
@@ -371,9 +441,41 @@ export async function runChatLoop(opts) {
|
|
|
371
441
|
compactionWireModelId = c.modelId;
|
|
372
442
|
}
|
|
373
443
|
else {
|
|
444
|
+
// Item 22 v1 scope note: with no explicit `compactionModel`, compaction
|
|
445
|
+
// reuses the primary adapter — which IS the failover chain when
|
|
446
|
+
// `modelFallbacks` is set, so compaction inherits the chain by
|
|
447
|
+
// construction. An explicit `compactionModel` keeps its own single
|
|
448
|
+
// adapter (no chain), as do judge/grader slots.
|
|
374
449
|
compactionAdapter = adapter;
|
|
375
450
|
compactionWireModelId = wireModelId;
|
|
376
451
|
}
|
|
452
|
+
// Item 26 — two-tier turn-difficulty router. When `modelTiers` is set, BOTH
|
|
453
|
+
// tier adapters resolve HERE at boot (adapters bind once; the per-turn pick
|
|
454
|
+
// just selects between the two already-resolved adapters — the same
|
|
455
|
+
// discipline as compaction's second adapter and the failover chain's
|
|
456
|
+
// multiple candidates). The `_tierAdapters` map is a test seam mirroring
|
|
457
|
+
// `_failoverAdapters`.
|
|
458
|
+
let tierRouter;
|
|
459
|
+
if (opts.modelTiers !== undefined) {
|
|
460
|
+
const resolveTier = async (modelString) => {
|
|
461
|
+
const injected = opts._tierAdapters?.get(modelString);
|
|
462
|
+
if (injected !== undefined) {
|
|
463
|
+
return { adapter: injected, modelId: bestEffortWireModelId(modelString), modelString };
|
|
464
|
+
}
|
|
465
|
+
const r = await resolveModel(modelString);
|
|
466
|
+
return { adapter: r.adapter, modelId: r.modelId, modelString };
|
|
467
|
+
};
|
|
468
|
+
const fast = await resolveTier(opts.modelTiers.fast);
|
|
469
|
+
const dflt = await resolveTier(opts.modelTiers.default);
|
|
470
|
+
tierRouter = createTierRouter({
|
|
471
|
+
fast,
|
|
472
|
+
default: dflt,
|
|
473
|
+
...(opts.modelTiers.routing !== undefined ? { config: opts.modelTiers.routing } : {}),
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
// Item 26 — tool_use blocks the PREVIOUS turn produced (a tier-routing
|
|
477
|
+
// signal). Persists across `runOneTurn` calls; 0 on the first turn.
|
|
478
|
+
let priorTurnToolUseCount = 0;
|
|
377
479
|
// Default model max OUTPUT tokens for one turn. Callers thread the spec's
|
|
378
480
|
// `agent.max_tokens` here when set (the CLI target's codegen + apps/cli);
|
|
379
481
|
// this fallback applies when the spec is silent. Kept comfortably above the
|
|
@@ -443,7 +545,117 @@ export async function runChatLoop(opts) {
|
|
|
443
545
|
// run with `OTEL_EXPORTER_OTLP_ENDPOINT` set automatically exports.
|
|
444
546
|
const bus = runContext.eventBus;
|
|
445
547
|
void TraceEventBus; // keep type import alive for future direct constructions
|
|
446
|
-
|
|
548
|
+
// Item 22 — re-point the failover chain's late-bound bus getter at the
|
|
549
|
+
// run's real bus (see the chain construction above): the chain and its
|
|
550
|
+
// per-candidate breakers publish model_failover / circuit_state_changed
|
|
551
|
+
// through this reference from the first stream() call onward.
|
|
552
|
+
observabilityBus = bus;
|
|
553
|
+
const subscribers = await attachDefaultSubscribers(bus, runContext, process.env, {
|
|
554
|
+
...(opts.alertSink !== undefined ? { alertSink: opts.alertSink } : {}),
|
|
555
|
+
...(opts.alertMetricsDir !== undefined ? { metricsDir: opts.alertMetricsDir } : {}),
|
|
556
|
+
...(opts.sloTargets !== undefined ? { sloTargets: opts.sloTargets } : {}),
|
|
557
|
+
...(opts.sloSink !== undefined ? { sloSink: opts.sloSink } : {}),
|
|
558
|
+
});
|
|
559
|
+
// Ops item 32 — auto-assemble an incident bundle on the first failure-class
|
|
560
|
+
// trigger (circuit → open, egress-blocked, justification-deny storm). Gated
|
|
561
|
+
// by CREWHAUS_INCIDENTS; writes a raw capture (ring buffer + trigger meta)
|
|
562
|
+
// that `crewhaus incident collect --session <id>` turns into a full bundle.
|
|
563
|
+
const incidentCollector = attachIncidentCollector(bus, runContext, process.env, {
|
|
564
|
+
...(opts.incidentsDir !== undefined ? { incidentsDir: opts.incidentsDir } : {}),
|
|
565
|
+
...(opts.incidentSpec !== undefined ? { spec: opts.incidentSpec } : {}),
|
|
566
|
+
});
|
|
567
|
+
// Item 22 — surface each failover on stderr so an operator watching a
|
|
568
|
+
// plain (non-CREWHAUS_TRACE) run still sees provider switches. NOTE: the
|
|
569
|
+
// advisor-events session-log subscriber pattern has NOT landed on main
|
|
570
|
+
// (packages/event-log has no advisor kind) — when it does, failovers
|
|
571
|
+
// should ALSO persist into the session JSONL through it; until then the
|
|
572
|
+
// trace event + this stderr note are the observable surfaces.
|
|
573
|
+
let failoverNoteUnsubscribe;
|
|
574
|
+
if (failoverChain !== undefined) {
|
|
575
|
+
failoverNoteUnsubscribe = bus.subscribe((event) => {
|
|
576
|
+
if (event.kind !== "model_failover")
|
|
577
|
+
return;
|
|
578
|
+
process.stderr.write(`[failover] ${event.from} → ${event.to} (${event.reason})\n`);
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
// Item 27 — run-level spend cap. An ALWAYS-ON cost meter (independent of
|
|
582
|
+
// the CREWHAUS_COST_TRACKING env flag) accrues per-response spend priced
|
|
583
|
+
// off the wire model that actually served — `suppressEvents` keeps it off
|
|
584
|
+
// the trace surface so it never double-prints beside an env-attached
|
|
585
|
+
// cost-tracker. The pre-turn check (`enforceBudget`, below) reads
|
|
586
|
+
// `budgetMeter.getRunCost(runId).totalUsdMicros`. `budgetDegraded` gates
|
|
587
|
+
// the ladder to a single rung: once we degrade, we never degrade again —
|
|
588
|
+
// a second breach stops the run.
|
|
589
|
+
const budgetMeter = opts.budget !== undefined ? createCostTracker(bus, { suppressEvents: true }) : undefined;
|
|
590
|
+
let budgetDegraded = false;
|
|
591
|
+
/**
|
|
592
|
+
* Item 27 — pre-turn budget gate. Returns "stop" when the run must end
|
|
593
|
+
* before the next turn opens; "continue" otherwise. On a `degrade` breach
|
|
594
|
+
* it re-resolves the primary model to the cheaper rung IN PLACE (mutating
|
|
595
|
+
* `adapter`, `providerId`, `wireModelId`) and emits a `model_failover`
|
|
596
|
+
* (reason `budget_degrade`) — so the very next model call, and all cost
|
|
597
|
+
* accrual after it, use the degraded model. The current turn always
|
|
598
|
+
* completes; this only gates the NEXT turn, so an in-flight tool loop is
|
|
599
|
+
* never severed. Never throws — a re-resolution failure logs and stops.
|
|
600
|
+
*/
|
|
601
|
+
const enforceBudget = async () => {
|
|
602
|
+
if (opts.budget === undefined || budgetMeter === undefined)
|
|
603
|
+
return "continue";
|
|
604
|
+
const spent = budgetMeter.getRunCost(bus.runId).totalUsdMicros;
|
|
605
|
+
if (spent < opts.budget.usdMicros)
|
|
606
|
+
return "continue";
|
|
607
|
+
const spentUsd = (spent / 1_000_000).toFixed(4);
|
|
608
|
+
const capUsd = (opts.budget.usdMicros / 1_000_000).toFixed(4);
|
|
609
|
+
if (opts.budget.onExceed.kind === "stop" || budgetDegraded) {
|
|
610
|
+
const why = budgetDegraded ? "degraded model also reached the cap" : "run budget reached";
|
|
611
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — ${why}; ending the run.\n`);
|
|
612
|
+
return "stop";
|
|
613
|
+
}
|
|
614
|
+
// degrade: re-resolve the primary model to the cheaper rung, once.
|
|
615
|
+
const target = opts.budget.onExceed.model;
|
|
616
|
+
try {
|
|
617
|
+
const degraded = opts._budgetDegradeAdapter !== undefined
|
|
618
|
+
? {
|
|
619
|
+
adapter: opts._budgetDegradeAdapter,
|
|
620
|
+
modelId: bestEffortWireModelId(target),
|
|
621
|
+
providerId: opts._budgetDegradeAdapter.providerId,
|
|
622
|
+
}
|
|
623
|
+
: await resolveModel(target);
|
|
624
|
+
const from = opts.model;
|
|
625
|
+
adapter =
|
|
626
|
+
opts.circuitBreaker !== undefined
|
|
627
|
+
? wrapWithCircuitBreaker(degraded.adapter, {
|
|
628
|
+
...opts.circuitBreaker,
|
|
629
|
+
adapterName: degraded.adapter.providerId,
|
|
630
|
+
bus,
|
|
631
|
+
})
|
|
632
|
+
: degraded.adapter;
|
|
633
|
+
// The degraded model is now the serving one; keep the trace/pricing
|
|
634
|
+
// identity coherent for the model_request/response events below.
|
|
635
|
+
providerId = degraded.providerId;
|
|
636
|
+
wireModelId = degraded.modelId;
|
|
637
|
+
degradedSpecModel = target;
|
|
638
|
+
// A degrade takes over from the failover chain: subsequent calls hit
|
|
639
|
+
// the degraded adapter directly (v1 keeps one rung, no re-entry to the
|
|
640
|
+
// chain). Null it so the request/response stamping stops consulting it.
|
|
641
|
+
failoverChain = undefined;
|
|
642
|
+
budgetDegraded = true;
|
|
643
|
+
bus.publish({
|
|
644
|
+
...bus.envelope(),
|
|
645
|
+
kind: "model_failover",
|
|
646
|
+
from,
|
|
647
|
+
to: target,
|
|
648
|
+
reason: "budget_degrade",
|
|
649
|
+
});
|
|
650
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrading ${from} → ${target}.\n`);
|
|
651
|
+
return "continue";
|
|
652
|
+
}
|
|
653
|
+
catch (err) {
|
|
654
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
655
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrade to "${target}" failed (${message}); ending the run.\n`);
|
|
656
|
+
return "stop";
|
|
657
|
+
}
|
|
658
|
+
};
|
|
447
659
|
// Working-indicator spinner. Auto-enables only for an interactive REPL on a
|
|
448
660
|
// TTY (see `isSpinnerEnabled`); `singleTurn`/injected-`input`/piped runs stay
|
|
449
661
|
// byte-identical to before. `out.write` is the spinner-aware stdout sink:
|
|
@@ -505,6 +717,11 @@ export async function runChatLoop(opts) {
|
|
|
505
717
|
inputTokens: ev.inputTokens,
|
|
506
718
|
outputTokens: ev.outputTokens,
|
|
507
719
|
cachedReadTokens: ev.cachedReadTokens,
|
|
720
|
+
// Optional on the event (older emitters may omit it) — persist
|
|
721
|
+
// only when present so old-log parsing stays additive-safe.
|
|
722
|
+
...(ev.cacheCreationTokens !== undefined
|
|
723
|
+
? { cacheCreationTokens: ev.cacheCreationTokens }
|
|
724
|
+
: {}),
|
|
508
725
|
costUsdMicros: ev.costUsdMicros,
|
|
509
726
|
...(ev.tenantId !== undefined ? { tenantId: ev.tenantId } : {}),
|
|
510
727
|
},
|
|
@@ -516,6 +733,18 @@ export async function runChatLoop(opts) {
|
|
|
516
733
|
});
|
|
517
734
|
});
|
|
518
735
|
}
|
|
736
|
+
// Advisor groundwork (item 14) — persist the trace-bus-only advisory
|
|
737
|
+
// signals (recovery actions, per-call tool stats, resolved permission
|
|
738
|
+
// decisions, model stop reasons) into the same session JSONL the
|
|
739
|
+
// transcript already writes to, so `crewhaus advise` can mine sessions
|
|
740
|
+
// offline. DEFAULT-ON (the lines are tiny and they are the advisor's
|
|
741
|
+
// food); disable with CREWHAUS_ADVISOR_EVENTS=0. See observability.ts.
|
|
742
|
+
const advisorPersist = attachAdvisorPersistence(bus, eventLog, runContext);
|
|
743
|
+
// Ops item 38 — persist the trace-bus-only `mcp_call_end` events into the
|
|
744
|
+
// session JSONL as the durable `mcp_stats` kind so `crewhaus mcp doctor` can
|
|
745
|
+
// score per-server MCP health offline. Shares the advisor's DEFAULT-ON gate
|
|
746
|
+
// (CREWHAUS_ADVISOR_EVENTS=0 disables both). See observability.ts.
|
|
747
|
+
const mcpStatsPersist = attachMcpStatsPersistence(bus, eventLog, runContext);
|
|
519
748
|
// Per-run state container — coordination surface for hooks/skills/tools
|
|
520
749
|
// landing in Section 11+. Section 10 only ships the plumbing; the
|
|
521
750
|
// underscore prefix marks the intentional non-consumer.
|
|
@@ -557,6 +786,21 @@ export async function runChatLoop(opts) {
|
|
|
557
786
|
return { allowed: true };
|
|
558
787
|
}
|
|
559
788
|
}
|
|
789
|
+
// Feature #53 — auto-capture at teardown. Runs in the same finally where the
|
|
790
|
+
// `stop` hook fires, but is NOT a credential-stripped hook: the caller-
|
|
791
|
+
// supplied `onCapture` (which writes to the memory-store from the CLI's full
|
|
792
|
+
// environment) needs the process env, exactly like the feedback teardown.
|
|
793
|
+
// Best-effort — a capture failure never turns a clean exit into an error.
|
|
794
|
+
const maybeAutoCapture = async () => {
|
|
795
|
+
if (opts.memory?.autoCapture !== true || opts.memory.onCapture === undefined)
|
|
796
|
+
return;
|
|
797
|
+
try {
|
|
798
|
+
await opts.memory.onCapture(runContext.turnNumber, sessionId);
|
|
799
|
+
}
|
|
800
|
+
catch (err) {
|
|
801
|
+
runContext.logger.warn("memory auto-capture failed", { error: err.message });
|
|
802
|
+
}
|
|
803
|
+
};
|
|
560
804
|
// Section 11 — `session-start` fires once after the persistence + run
|
|
561
805
|
// context boot is complete. Observational only (no deny/block effect).
|
|
562
806
|
await fireHook("session-start", {
|
|
@@ -594,7 +838,9 @@ export async function runChatLoop(opts) {
|
|
|
594
838
|
// CLAUDE.md auto-load behaviour. Pillar 3 compliance: each file's content
|
|
595
839
|
// is classified via boundary-classifier with origin "user" inside
|
|
596
840
|
// loadProjectMemory() (the user's repo is developer-trusted).
|
|
597
|
-
const projectMemory = await loadProjectMemory(
|
|
841
|
+
const projectMemory = await loadProjectMemory(opts.preferenceFiles !== undefined && opts.preferenceFiles.length > 0
|
|
842
|
+
? { extraFiles: opts.preferenceFiles }
|
|
843
|
+
: {});
|
|
598
844
|
const projectMemoryBlock = projectMemory.prompt.length > 0
|
|
599
845
|
? [
|
|
600
846
|
{
|
|
@@ -609,12 +855,50 @@ export async function runChatLoop(opts) {
|
|
|
609
855
|
.map((f) => f.filename)
|
|
610
856
|
.join(", ")}\n`);
|
|
611
857
|
}
|
|
858
|
+
// Feature #53 — auto-recall: at session start, pull the top-K relevant
|
|
859
|
+
// cross-session memories and inject them as a system block, mirroring the
|
|
860
|
+
// project-memory auto-load above. The caller supplies the `recall` seam
|
|
861
|
+
// (runtime-core stays independent of memory-store). Best-effort: a recall
|
|
862
|
+
// failure never aborts the run.
|
|
863
|
+
let memoryRecallBlock = [];
|
|
864
|
+
if (opts.memory?.autoRecall === true && opts.memory.recall !== undefined) {
|
|
865
|
+
try {
|
|
866
|
+
const seed = opts.memory.recallSeed ?? opts.instructions;
|
|
867
|
+
const k = opts.memory.recallK ?? 5;
|
|
868
|
+
const lines = await opts.memory.recall(seed, k);
|
|
869
|
+
if (lines.length > 0) {
|
|
870
|
+
// Pillar 3 — a recalled memory can embed content shaped by untrusted
|
|
871
|
+
// tool output from an earlier session, so it is NOT verbatim-safe like
|
|
872
|
+
// the developer's own repo files. Two defenses, mirroring how the rest
|
|
873
|
+
// of the security fabric treats injected content:
|
|
874
|
+
// 1. Delimiter safety — neutralize any `</recalled_memory>` closing
|
|
875
|
+
// tag inside a recalled line so a poisoned memory can't break out
|
|
876
|
+
// of its block and inject trailing instructions.
|
|
877
|
+
// 2. Classification — run the assembled block through the SAME
|
|
878
|
+
// `classifyBoundary(text, { origin: "user" })` best-effort path
|
|
879
|
+
// `loadProjectMemory()` uses (pass policy: classify + emit trace
|
|
880
|
+
// events without mutating), so recalled content flows through the
|
|
881
|
+
// boundary classifier exactly like every other prompt-bound source.
|
|
882
|
+
const body = lines
|
|
883
|
+
.map((l) => `- ${escapeBoundaryDelimiter(l, "recalled_memory")}`)
|
|
884
|
+
.join("\n");
|
|
885
|
+
const text = `<recalled_memory>\nRelevant facts remembered from earlier sessions:\n${body}\n</recalled_memory>`;
|
|
886
|
+
await classifyBoundary(text, { origin: "user" }).catch(() => undefined);
|
|
887
|
+
memoryRecallBlock = [{ type: "text", text, cache_control: { type: "ephemeral" } }];
|
|
888
|
+
process.stdout.write(`[memory] recalled ${lines.length} memory(ies) into the prompt\n`);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
catch (err) {
|
|
892
|
+
runContext.logger.warn("memory auto-recall failed", { error: err.message });
|
|
893
|
+
}
|
|
894
|
+
}
|
|
612
895
|
// Section 17 — runtime-core no longer prepends the Claude Code OAuth
|
|
613
896
|
// prefix; `adapter-anthropic` handles that internally so each adapter
|
|
614
897
|
// owns its provider-specific auth-shape requirements.
|
|
615
898
|
let systemBlocks = [
|
|
616
899
|
userInstructions,
|
|
617
900
|
...projectMemoryBlock,
|
|
901
|
+
...memoryRecallBlock,
|
|
618
902
|
...skillsBlock,
|
|
619
903
|
];
|
|
620
904
|
// Section 27 — rotate cache markers on the system block array if the
|
|
@@ -814,6 +1098,21 @@ export async function runChatLoop(opts) {
|
|
|
814
1098
|
`tool denied: \`${tu.name}\` defaulted to "ask" and single-turn mode has no interactive surface to prompt on. ` +
|
|
815
1099
|
`Add an explicit rule to permissions.rules in your spec, e.g. \`{ type: alwaysAllow, pattern: ${tu.name} }\`, or run in REPL mode where "ask" can prompt.`;
|
|
816
1100
|
}
|
|
1101
|
+
// Advisor groundwork (item 14) — event the ask RESOLUTION. The publish
|
|
1102
|
+
// above fired BEFORE the approval prompt (decision "ask", no outcome);
|
|
1103
|
+
// this one fires after `askApproval` resolves — or after the
|
|
1104
|
+
// single-turn fallback collapses the ask to a deny — so offline advice
|
|
1105
|
+
// mining can measure how each tool's prompts are actually answered.
|
|
1106
|
+
// The advisor persistence subscriber (observability.ts) keys on
|
|
1107
|
+
// `askOutcome` to persist exactly this resolved form.
|
|
1108
|
+
bus.publish({
|
|
1109
|
+
...bus.envelope(),
|
|
1110
|
+
kind: "permission_decision",
|
|
1111
|
+
toolName: tu.name,
|
|
1112
|
+
decision: "ask",
|
|
1113
|
+
mode: permissionMode,
|
|
1114
|
+
askOutcome: approved ? "approved" : "denied",
|
|
1115
|
+
});
|
|
817
1116
|
}
|
|
818
1117
|
if (!approved) {
|
|
819
1118
|
return finish({
|
|
@@ -952,6 +1251,36 @@ export async function runChatLoop(opts) {
|
|
|
952
1251
|
originsFound: egress.originsFound,
|
|
953
1252
|
matchCount: egress.matchCount,
|
|
954
1253
|
});
|
|
1254
|
+
// Item 20 — emit the durable `egress_decision` audit record for every
|
|
1255
|
+
// non-pass verdict so egress history is triageable offline (the
|
|
1256
|
+
// trace-bus event above is ephemeral). Only the lineage SUMMARY is
|
|
1257
|
+
// stored — never the raw outbound payload (which is sensitive and
|
|
1258
|
+
// often carries the very tagged content that tripped the classifier),
|
|
1259
|
+
// matching the payload shape `@crewhaus/audit-log` documents for this
|
|
1260
|
+
// kind. Best-effort but surfaced: an I/O failure on the audit trail is
|
|
1261
|
+
// logged and swallowed so a full audit disk does not crash a governed
|
|
1262
|
+
// run, mirroring the justification sink above. Absent sink → no-op.
|
|
1263
|
+
if (opts.egressAuditSink !== undefined) {
|
|
1264
|
+
try {
|
|
1265
|
+
await opts.egressAuditSink.append({
|
|
1266
|
+
kind: "egress_decision",
|
|
1267
|
+
payload: {
|
|
1268
|
+
sinkId: egress.sinkId,
|
|
1269
|
+
sinkScope: egress.sinkScope,
|
|
1270
|
+
verdict: egress.verdict,
|
|
1271
|
+
originsFound: egress.originsFound,
|
|
1272
|
+
matchCount: egress.matchCount,
|
|
1273
|
+
},
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
catch (err) {
|
|
1277
|
+
runContext.logger.warn("egress audit append failed", {
|
|
1278
|
+
toolUseId: tu.id,
|
|
1279
|
+
toolName: tu.name,
|
|
1280
|
+
error: err.message,
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
955
1284
|
}
|
|
956
1285
|
if (egress.verdict === "block") {
|
|
957
1286
|
return finish({
|
|
@@ -1243,6 +1572,14 @@ export async function runChatLoop(opts) {
|
|
|
1243
1572
|
let recovery = initialRecoveryState;
|
|
1244
1573
|
const maxToolIterations = opts.maxToolIterations ?? DEFAULT_MAX_TOOL_ITERATIONS;
|
|
1245
1574
|
let toolIterations = 0;
|
|
1575
|
+
// Item 26 — tool_use blocks THIS turn produced (fed to the NEXT turn's
|
|
1576
|
+
// tier decision) and the misroute-recovery escalation latch (a fast-tier
|
|
1577
|
+
// failure forces `default` for the rest of this turn). `servedFastTier`
|
|
1578
|
+
// records whether the LAST attempt streamed on the fast tier, so the
|
|
1579
|
+
// error catch only escalates a genuine fast-tier misroute.
|
|
1580
|
+
let thisTurnToolUseCount = 0;
|
|
1581
|
+
let escalateTier = false;
|
|
1582
|
+
let servedFastTier = false;
|
|
1246
1583
|
while (state.kind !== "Done") {
|
|
1247
1584
|
if (turnAbort.signal.aborted) {
|
|
1248
1585
|
state = transition(state, { kind: "Aborted" });
|
|
@@ -1294,12 +1631,59 @@ export async function runChatLoop(opts) {
|
|
|
1294
1631
|
// → `us.anthropic.…`) so cost-tracker pricing lookups and the
|
|
1295
1632
|
// OTel `gen_ai.request.model` attribute match; the original
|
|
1296
1633
|
// spec string rides along as `specModel` when it differs.
|
|
1634
|
+
// Item 22 — with a failover chain the serving candidate may not
|
|
1635
|
+
// be the primary: stamp the request with the chain's routing
|
|
1636
|
+
// plan (best-effort; the response event below uses lastServed(),
|
|
1637
|
+
// which is exact, so cost-tracker pricing always keys on the
|
|
1638
|
+
// model that actually served).
|
|
1639
|
+
const plannedCandidate = failoverChain?.plan();
|
|
1640
|
+
let reqWireModelId = plannedCandidate?.modelId ?? wireModelId;
|
|
1641
|
+
let reqProviderId = plannedCandidate?.providerId ?? providerId;
|
|
1642
|
+
// Item 27 — after a budget degrade, `failoverChain` is null and
|
|
1643
|
+
// `degradedSpecModel` is the running spec model.
|
|
1644
|
+
let reqSpecModel = plannedCandidate?.modelString ?? degradedSpecModel ?? opts.model;
|
|
1645
|
+
// The adapter this turn streams through: the primary/chain by
|
|
1646
|
+
// default, or a tier adapter when the two-tier router is active.
|
|
1647
|
+
let turnAdapter = adapter;
|
|
1648
|
+
// Item 26 — two-tier turn-difficulty router. Pick a tier from
|
|
1649
|
+
// DETERMINISTIC per-turn signals (context size, tools-in-play,
|
|
1650
|
+
// turn index, prior-turn tool_use density); a fast-tier misroute
|
|
1651
|
+
// recovery forces `default` (see the catch below). Publish the
|
|
1652
|
+
// decision as a `model_tier_route` event, then stream through the
|
|
1653
|
+
// chosen tier's already-resolved adapter.
|
|
1654
|
+
if (tierRouter !== undefined) {
|
|
1655
|
+
const decision = escalateTier
|
|
1656
|
+
? { tier: "default", reason: "escalated after fast-tier failure" }
|
|
1657
|
+
: tierRouter.route({
|
|
1658
|
+
contextTokens: estimateTokens(messages),
|
|
1659
|
+
toolsInPlay: (anthropicTools?.length ?? 0) > 0,
|
|
1660
|
+
// `turnNumber` is 1-based (incremented before the turn
|
|
1661
|
+
// runs); the router wants a 0-based index so its
|
|
1662
|
+
// first-turn check keys on the opening turn.
|
|
1663
|
+
turnIndex: Math.max(0, runContext.turnNumber - 1),
|
|
1664
|
+
priorTurnToolUseCount,
|
|
1665
|
+
});
|
|
1666
|
+
const chosen = tierRouter.tier(decision.tier);
|
|
1667
|
+
turnAdapter = chosen.adapter;
|
|
1668
|
+
reqWireModelId = chosen.modelId;
|
|
1669
|
+
reqProviderId = chosen.adapter.providerId;
|
|
1670
|
+
reqSpecModel = chosen.modelString;
|
|
1671
|
+
servedFastTier = decision.tier === "fast";
|
|
1672
|
+
bus.publish({
|
|
1673
|
+
...bus.envelope(),
|
|
1674
|
+
kind: "model_tier_route",
|
|
1675
|
+
tier: decision.tier,
|
|
1676
|
+
model: chosen.modelId,
|
|
1677
|
+
reason: decision.reason,
|
|
1678
|
+
...(escalateTier ? { escalated: true } : {}),
|
|
1679
|
+
});
|
|
1680
|
+
}
|
|
1297
1681
|
bus.publish({
|
|
1298
1682
|
...modelStartEnv,
|
|
1299
1683
|
kind: "model_request",
|
|
1300
|
-
model:
|
|
1301
|
-
...(
|
|
1302
|
-
provider:
|
|
1684
|
+
model: reqWireModelId,
|
|
1685
|
+
...(reqSpecModel !== reqWireModelId ? { specModel: reqSpecModel } : {}),
|
|
1686
|
+
provider: reqProviderId,
|
|
1303
1687
|
messageCount: messages.length,
|
|
1304
1688
|
toolCount: anthropicTools?.length ?? 0,
|
|
1305
1689
|
streaming: opts.streaming === true,
|
|
@@ -1320,8 +1704,11 @@ export async function runChatLoop(opts) {
|
|
|
1320
1704
|
// bookkeeping is wire-compatible for the block kinds we
|
|
1321
1705
|
// actually consume (text / image / tool_use / tool_result
|
|
1322
1706
|
// / thinking).
|
|
1323
|
-
|
|
1324
|
-
|
|
1707
|
+
// Item 22 — the failover chain rewrites `model` per serving
|
|
1708
|
+
// candidate internally; the planned id here is the single-adapter
|
|
1709
|
+
// value and the chain's best-effort prediction otherwise.
|
|
1710
|
+
const reqStream = turnAdapter.stream({
|
|
1711
|
+
model: reqWireModelId,
|
|
1325
1712
|
system: systemBlocks.map((b) => ({
|
|
1326
1713
|
type: "text",
|
|
1327
1714
|
text: b.text,
|
|
@@ -1388,15 +1775,20 @@ export async function runChatLoop(opts) {
|
|
|
1388
1775
|
await logEvent("assistant_message", { content: committedStreamContent });
|
|
1389
1776
|
}
|
|
1390
1777
|
terminalContent = [...committedStreamContent];
|
|
1778
|
+
// Item 22 — lastServed() is exact: the candidate that actually
|
|
1779
|
+
// streamed this response. Cost-tracker prices off this pair.
|
|
1780
|
+
const servedStreaming = failoverChain?.lastServed();
|
|
1781
|
+
const respModelIdS = servedStreaming?.modelId ?? wireModelId;
|
|
1782
|
+
const respSpecModelS = servedStreaming?.modelString ?? degradedSpecModel ?? opts.model;
|
|
1391
1783
|
bus.publish({
|
|
1392
1784
|
...bus.envelope(),
|
|
1393
1785
|
spanId: modelStartEnv.spanId,
|
|
1394
1786
|
kind: "model_response",
|
|
1395
1787
|
// Wire model id, NOT the spec string — cost-tracker resolves
|
|
1396
1788
|
// pricing on this field (see the model_request publish above).
|
|
1397
|
-
model:
|
|
1398
|
-
...(
|
|
1399
|
-
provider: providerId,
|
|
1789
|
+
model: respModelIdS,
|
|
1790
|
+
...(respSpecModelS !== respModelIdS ? { specModel: respSpecModelS } : {}),
|
|
1791
|
+
provider: servedStreaming?.providerId ?? providerId,
|
|
1400
1792
|
stopReason: stopReason ?? "end_turn",
|
|
1401
1793
|
usage,
|
|
1402
1794
|
durationMs: performance.now() - t0Model,
|
|
@@ -1410,6 +1802,8 @@ export async function runChatLoop(opts) {
|
|
|
1410
1802
|
state = transition(state, { kind: "ModelReturnedText" });
|
|
1411
1803
|
break;
|
|
1412
1804
|
}
|
|
1805
|
+
// Item 26 — accrue this turn's tool_use count (next-turn tier signal).
|
|
1806
|
+
thisTurnToolUseCount += toolUses.length;
|
|
1413
1807
|
for (const tu of toolUses) {
|
|
1414
1808
|
toolUseHistory.push({ id: tu.id, name: tu.name, input: tu.input });
|
|
1415
1809
|
}
|
|
@@ -1418,7 +1812,7 @@ export async function runChatLoop(opts) {
|
|
|
1418
1812
|
const warning = maybeBuildLoopWarning();
|
|
1419
1813
|
if (warning !== null) {
|
|
1420
1814
|
messages.push(warning);
|
|
1421
|
-
await logEvent("user_message", { content: warning.content });
|
|
1815
|
+
await logEvent("user_message", { content: warning.content, synthetic: true });
|
|
1422
1816
|
}
|
|
1423
1817
|
// Synthesize the two transitions the state machine expects:
|
|
1424
1818
|
// tools have already executed during the stream so we
|
|
@@ -1447,15 +1841,20 @@ export async function runChatLoop(opts) {
|
|
|
1447
1841
|
},
|
|
1448
1842
|
});
|
|
1449
1843
|
endTurn();
|
|
1844
|
+
// Item 22 — lastServed() is exact: the candidate that actually
|
|
1845
|
+
// streamed this response. Cost-tracker prices off this pair.
|
|
1846
|
+
const servedCandidate = failoverChain?.lastServed();
|
|
1847
|
+
const respWireModelId = servedCandidate?.modelId ?? wireModelId;
|
|
1848
|
+
const respSpecModel = servedCandidate?.modelString ?? degradedSpecModel ?? opts.model;
|
|
1450
1849
|
bus.publish({
|
|
1451
1850
|
...bus.envelope(),
|
|
1452
1851
|
spanId: modelStartEnv.spanId,
|
|
1453
1852
|
kind: "model_response",
|
|
1454
1853
|
// Wire model id, NOT the spec string — cost-tracker resolves
|
|
1455
1854
|
// pricing on this field (see the model_request publish above).
|
|
1456
|
-
model:
|
|
1457
|
-
...(
|
|
1458
|
-
provider: providerId,
|
|
1855
|
+
model: respWireModelId,
|
|
1856
|
+
...(respSpecModel !== respWireModelId ? { specModel: respSpecModel } : {}),
|
|
1857
|
+
provider: servedCandidate?.providerId ?? providerId,
|
|
1459
1858
|
stopReason: final.stopReason,
|
|
1460
1859
|
usage: final.usage,
|
|
1461
1860
|
durationMs: performance.now() - t0Model,
|
|
@@ -1529,6 +1928,14 @@ export async function runChatLoop(opts) {
|
|
|
1529
1928
|
break;
|
|
1530
1929
|
}
|
|
1531
1930
|
lastErrorForRecovery = err;
|
|
1931
|
+
// Item 26 — a fast-tier turn that FAILED is a MISROUTE: latch the
|
|
1932
|
+
// escalation so the recovery retry re-runs on the `default` tier
|
|
1933
|
+
// (the loop re-enters NeedModel via the recovery ladder, which
|
|
1934
|
+
// then picks `default`). Composes with the existing recovery
|
|
1935
|
+
// actions — a retry/continue now streams on the sturdier tier.
|
|
1936
|
+
if (tierRouter !== undefined && servedFastTier) {
|
|
1937
|
+
escalateTier = true;
|
|
1938
|
+
}
|
|
1532
1939
|
const errObj = err;
|
|
1533
1940
|
const errorName = typeof errObj.name === "string" ? errObj.name : "Error";
|
|
1534
1941
|
const errorMessage = typeof errObj.message === "string" ? errObj.message : String(err);
|
|
@@ -1556,13 +1963,16 @@ export async function runChatLoop(opts) {
|
|
|
1556
1963
|
state = transition(state, { kind: "Aborted" });
|
|
1557
1964
|
break;
|
|
1558
1965
|
}
|
|
1966
|
+
// Item 26 — accrue this turn's tool_use count (a next-turn tier
|
|
1967
|
+
// signal): a dense tool turn marks an active multi-step task.
|
|
1968
|
+
thisTurnToolUseCount += state.toolUses.length;
|
|
1559
1969
|
const toolResults = await runToolBatch(state.toolUses);
|
|
1560
1970
|
messages.push({ role: "user", content: toolResults });
|
|
1561
1971
|
await logEvent("user_message", { content: toolResults });
|
|
1562
1972
|
const warning = maybeBuildLoopWarning();
|
|
1563
1973
|
if (warning !== null) {
|
|
1564
1974
|
messages.push(warning);
|
|
1565
|
-
await logEvent("user_message", { content: warning.content });
|
|
1975
|
+
await logEvent("user_message", { content: warning.content, synthetic: true });
|
|
1566
1976
|
}
|
|
1567
1977
|
state = transition(state, { kind: "ToolsExecuted" });
|
|
1568
1978
|
break;
|
|
@@ -1576,7 +1986,10 @@ export async function runChatLoop(opts) {
|
|
|
1576
1986
|
// emitting `BudgetExceeded` from `runOneTurn`, restore a
|
|
1577
1987
|
// `NeedCompaction` → `CompactionDone` arm (and cover it) here.
|
|
1578
1988
|
case "NeedRecovery": {
|
|
1579
|
-
|
|
1989
|
+
// Item 23 / Section 55 — consult the spec's failure_taxonomy first
|
|
1990
|
+
// (named classes take precedence over the built-in flow), so a
|
|
1991
|
+
// matched `switch-model` (or retry/compact/…) entry wins.
|
|
1992
|
+
const action = recover(lastErrorForRecovery, recovery, opts.failureTaxonomy);
|
|
1580
1993
|
recovery = advanceState(recovery, action);
|
|
1581
1994
|
runContext.logger.info("recovery.action", {
|
|
1582
1995
|
kind: action.kind,
|
|
@@ -1635,9 +2048,14 @@ export async function runChatLoop(opts) {
|
|
|
1635
2048
|
state = transition(state, { kind: "RecoveryDone" });
|
|
1636
2049
|
break;
|
|
1637
2050
|
case "continue": {
|
|
2051
|
+
// Runtime-injected user_message events (this nudge, the tombstone
|
|
2052
|
+
// retry, and loop warnings) are marked `synthetic: true` in the
|
|
2053
|
+
// event log. They are NOT human turns, so they do not increment
|
|
2054
|
+
// runContext.turnNumber — and `crewhaus distill`'s deriveTurns
|
|
2055
|
+
// skips them so its turn ordinal matches the runtime + web UI.
|
|
1638
2056
|
const continueMsg = "Please continue from where you left off.";
|
|
1639
2057
|
messages.push({ role: "user", content: continueMsg });
|
|
1640
|
-
await logEvent("user_message", { content: continueMsg });
|
|
2058
|
+
await logEvent("user_message", { content: continueMsg, synthetic: true });
|
|
1641
2059
|
state = transition(state, { kind: "RecoveryDone" });
|
|
1642
2060
|
break;
|
|
1643
2061
|
}
|
|
@@ -1666,7 +2084,33 @@ export async function runChatLoop(opts) {
|
|
|
1666
2084
|
}
|
|
1667
2085
|
const tombstoneMsg = "[previous assistant turn was rejected as invalid; please retry]";
|
|
1668
2086
|
messages.push({ role: "user", content: tombstoneMsg });
|
|
1669
|
-
await logEvent("user_message", { content: tombstoneMsg });
|
|
2087
|
+
await logEvent("user_message", { content: tombstoneMsg, synthetic: true });
|
|
2088
|
+
state = transition(state, { kind: "RecoveryDone" });
|
|
2089
|
+
break;
|
|
2090
|
+
}
|
|
2091
|
+
case "switch-model": {
|
|
2092
|
+
// Item 23 — a `switch-model` failure_taxonomy verdict: abandon
|
|
2093
|
+
// the active provider candidate FOR THIS TURN and re-issue the
|
|
2094
|
+
// same request onto the next failover candidate. Mechanism:
|
|
2095
|
+
// force the active candidate's breaker open so the chain
|
|
2096
|
+
// reroutes on the next `stream()`; the chain publishes the
|
|
2097
|
+
// `model_failover` (reason breaker_open) itself. No message
|
|
2098
|
+
// mutation and no backoff sleep — a switch is instant. Without
|
|
2099
|
+
// a failover chain (no `modelFallbacks`) there is nothing to
|
|
2100
|
+
// switch to: this degrades to a plain re-issue against the same
|
|
2101
|
+
// adapter, which the per-turn budget bounds. RecoveryDone loops
|
|
2102
|
+
// back to NeedModel, which re-streams.
|
|
2103
|
+
const tripped = failoverChain?.tripActive("switch-model recovery");
|
|
2104
|
+
if (tripped !== undefined) {
|
|
2105
|
+
out.spinner.start("switching model");
|
|
2106
|
+
out.spinner.stop();
|
|
2107
|
+
runContext.logger.info("recovery.switch-model tripped candidate", {
|
|
2108
|
+
from: tripped,
|
|
2109
|
+
});
|
|
2110
|
+
}
|
|
2111
|
+
else {
|
|
2112
|
+
runContext.logger.warn("recovery.switch-model: no failover chain — re-issuing on the same model");
|
|
2113
|
+
}
|
|
1670
2114
|
state = transition(state, { kind: "RecoveryDone" });
|
|
1671
2115
|
break;
|
|
1672
2116
|
}
|
|
@@ -1677,6 +2121,9 @@ export async function runChatLoop(opts) {
|
|
|
1677
2121
|
}
|
|
1678
2122
|
}
|
|
1679
2123
|
}
|
|
2124
|
+
// Item 26 — hand this turn's tool_use count to the next turn's tier
|
|
2125
|
+
// decision (the prior-tool-density signal).
|
|
2126
|
+
priorTurnToolUseCount = thisTurnToolUseCount;
|
|
1680
2127
|
return { terminalContent };
|
|
1681
2128
|
}
|
|
1682
2129
|
// Single-shot path used by the workflow target and (Section 12) the
|
|
@@ -1763,9 +2210,16 @@ export async function runChatLoop(opts) {
|
|
|
1763
2210
|
turnCount: runContext.turnNumber,
|
|
1764
2211
|
reason: "complete",
|
|
1765
2212
|
});
|
|
2213
|
+
await maybeAutoCapture();
|
|
1766
2214
|
await subscribers.flushAll();
|
|
1767
2215
|
await subscribers.shutdownAll();
|
|
1768
2216
|
costPersistUnsubscribe?.();
|
|
2217
|
+
failoverNoteUnsubscribe?.();
|
|
2218
|
+
incidentCollector?.unsubscribe();
|
|
2219
|
+
budgetMeter?.unsubscribe();
|
|
2220
|
+
advisorPersist?.unsubscribe();
|
|
2221
|
+
mcpStatsPersist?.unsubscribe();
|
|
2222
|
+
printAdvisorDigest(advisorPersist);
|
|
1769
2223
|
await sessionStore
|
|
1770
2224
|
.update(sessionId, { lastTurnIndex: runContext.turnNumber })
|
|
1771
2225
|
.catch((err) => {
|
|
@@ -1904,6 +2358,13 @@ export async function runChatLoop(opts) {
|
|
|
1904
2358
|
}
|
|
1905
2359
|
}
|
|
1906
2360
|
}
|
|
2361
|
+
// Item 27 — run-level spend cap. Enforced HERE (a real turn is about
|
|
2362
|
+
// to run, input is not exit/blank) so an idle prompt never trips it.
|
|
2363
|
+
// On a `stop` breach we break before committing the input — the turn
|
|
2364
|
+
// never runs. On a `degrade` breach `enforceBudget` has already
|
|
2365
|
+
// swapped the primary adapter, so this turn runs on the cheaper model.
|
|
2366
|
+
if ((await enforceBudget()) === "stop")
|
|
2367
|
+
break;
|
|
1907
2368
|
messages.push({ role: "user", content: effectiveInput });
|
|
1908
2369
|
await logEvent("user_message", { content: effectiveInput });
|
|
1909
2370
|
runContext.turnNumber += 1;
|
|
@@ -1974,9 +2435,16 @@ export async function runChatLoop(opts) {
|
|
|
1974
2435
|
turnCount: runContext.turnNumber,
|
|
1975
2436
|
reason: runAbort.signal.aborted ? "abort" : "exit",
|
|
1976
2437
|
});
|
|
2438
|
+
await maybeAutoCapture();
|
|
1977
2439
|
await subscribers.flushAll();
|
|
1978
2440
|
await subscribers.shutdownAll();
|
|
1979
2441
|
costPersistUnsubscribe?.();
|
|
2442
|
+
failoverNoteUnsubscribe?.();
|
|
2443
|
+
incidentCollector?.unsubscribe();
|
|
2444
|
+
budgetMeter?.unsubscribe();
|
|
2445
|
+
advisorPersist?.unsubscribe();
|
|
2446
|
+
mcpStatsPersist?.unsubscribe();
|
|
2447
|
+
printAdvisorDigest(advisorPersist);
|
|
1980
2448
|
if (shouldInstallSigint) {
|
|
1981
2449
|
process.removeListener("SIGINT", sigintHandler);
|
|
1982
2450
|
}
|
|
@@ -1990,6 +2458,18 @@ export async function runChatLoop(opts) {
|
|
|
1990
2458
|
}
|
|
1991
2459
|
return "";
|
|
1992
2460
|
}
|
|
2461
|
+
/**
|
|
2462
|
+
* Item 14 in-run digest — one stderr line at session end when the advisor
|
|
2463
|
+
* persistence subscriber's cheap tally tripped a threshold, pointing at
|
|
2464
|
+
* `crewhaus advise --session <id>` for the full report. stderr (not stdout)
|
|
2465
|
+
* so piped/captured agent output is never polluted; silent on healthy runs
|
|
2466
|
+
* and when the subscriber is disabled (CREWHAUS_ADVISOR_EVENTS=0).
|
|
2467
|
+
*/
|
|
2468
|
+
function printAdvisorDigest(advisorPersist) {
|
|
2469
|
+
const line = advisorPersist?.digestLine();
|
|
2470
|
+
if (line !== undefined)
|
|
2471
|
+
process.stderr.write(`${line}\n`);
|
|
2472
|
+
}
|
|
1993
2473
|
function isAbortError(err) {
|
|
1994
2474
|
// Section 17 — runtime-core no longer imports the Anthropic SDK at
|
|
1995
2475
|
// runtime, so we name-match instead of `instanceof`. The provider
|