@crewhaus/runtime-core 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +53 -0
- package/dist/alert-watchdog.js +88 -7
- package/dist/ephemeral.d.ts +6 -0
- package/dist/ephemeral.js +88 -0
- package/dist/index.d.ts +795 -21
- package/dist/index.js +2573 -208
- package/dist/model-latency.d.ts +43 -0
- package/dist/model-latency.js +61 -0
- package/dist/observability.d.ts +47 -2
- package/dist/observability.js +104 -3
- package/dist/pre-route.d.ts +149 -0
- package/dist/pre-route.js +271 -0
- package/dist/slo-monitor.d.ts +20 -0
- package/dist/slo-monitor.js +45 -1
- package/package.json +42 -40
package/dist/index.js
CHANGED
|
@@ -8,24 +8,26 @@ import { wrap as wrapWithCircuitBreaker } from "@crewhaus/circuit-breaker";
|
|
|
8
8
|
import { autoCompact } from "@crewhaus/compaction-autocompact";
|
|
9
9
|
import { DEFAULT_DEDUPE_THRESHOLD, curate as curateItems, } from "@crewhaus/compaction-curator";
|
|
10
10
|
import { snip } from "@crewhaus/compaction-snip";
|
|
11
|
-
import { DEFAULT_PRICING, computeCostMicros, createCostTracker, resolvePricing, } from "@crewhaus/cost-tracker";
|
|
11
|
+
import { DEFAULT_CAPABILITIES, DEFAULT_PRICING, computeCostMicros, createCostTracker, resolveCapabilities, resolvePricing, sumRoleCost, } from "@crewhaus/cost-tracker";
|
|
12
12
|
import { classifyEgress, summarizeEgress, } from "@crewhaus/egress-classifier";
|
|
13
13
|
import { ConfigError, EXIT_CODES, RunFailedError, RuntimeError, isRunFailedError, } from "@crewhaus/errors";
|
|
14
14
|
import { DEFAULT_ROOT_DIR, openEventLog } from "@crewhaus/event-log";
|
|
15
15
|
import { aggregateDecisions, runHooks } from "@crewhaus/hooks-engine";
|
|
16
|
-
import {
|
|
16
|
+
import { buildAdvertisement, buildRequestParams, deriveSignalRecord, loadPriors, parseModelDirective, planFingerprint, priorsFingerprint, seededScoreLookup, stripDirectiveToken, toolConfigFor, } from "@crewhaus/model-plan";
|
|
17
|
+
import { createFailoverChain, createPolicyRouter, createTierRouter, parseModelString, poolCandidateArmId, resolveModel, unscopedRouteKey, } from "@crewhaus/model-router";
|
|
17
18
|
import { BUILTIN_DEFAULT_RULES, emptyRuleSet, evaluateJustification, evaluateWithReason, loadSettingsRules, ruleBasedJustificationJudge, } from "@crewhaus/permission-engine";
|
|
18
19
|
import { manage as manageCacheMarkers } from "@crewhaus/prompt-cache-manager";
|
|
19
20
|
import { buildRedactionNotice, classifyText, llmClassifierEnabled, } from "@crewhaus/prompt-injection-detector";
|
|
20
21
|
import { createRateLimiter } from "@crewhaus/rate-limiter";
|
|
21
|
-
import { advanceState, initialRecoveryState, recover, } from "@crewhaus/recovery-engine";
|
|
22
|
-
import { computeReward, openScoreboard, } from "@crewhaus/routing-store";
|
|
22
|
+
import { BUILTIN_FAILURE_CLASSES, advanceState, initialRecoveryState, recover, } from "@crewhaus/recovery-engine";
|
|
23
|
+
import { SHADOW_LANE_PRIMARY_ARM, SHADOW_LANE_SHADOW_ARM, computeReward, freezeScoreboard, openScoreboard, readRouteFreeze, readRoutingPriorsRaw, routingPriorsPath, shadowLaneQuality, shadowRouteKey, } from "@crewhaus/routing-store";
|
|
23
24
|
import { createRunContext, tagContent } from "@crewhaus/run-context";
|
|
24
|
-
import { createSessionStore, evictExpiredApprovals, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
25
|
+
import { createSessionStore, evictExpiredApprovals, generateApprovalId, hashApprovalInput, mergeSessionModels, resolveSessionPin, } from "@crewhaus/session-store";
|
|
25
26
|
import { formatSkillsForPrompt } from "@crewhaus/skills-registry";
|
|
26
27
|
import { expand as expandSlash } from "@crewhaus/slash-commands";
|
|
27
28
|
import { createStore } from "@crewhaus/state-store";
|
|
28
29
|
import { executeStreaming } from "@crewhaus/streaming-tool-executor";
|
|
30
|
+
import { narrowRuleSet } from "@crewhaus/sub-agent-permission-inheritance";
|
|
29
31
|
import { currentTenantContext } from "@crewhaus/tenancy";
|
|
30
32
|
import { TokenBudget, estimateTokens } from "@crewhaus/token-budget";
|
|
31
33
|
import { stripJustificationField, withJustificationField } from "@crewhaus/tool-catalog";
|
|
@@ -33,14 +35,17 @@ import { executeTool } from "@crewhaus/tool-executor";
|
|
|
33
35
|
import { detectLoop } from "@crewhaus/tool-loop-detection";
|
|
34
36
|
import { partitionToolCalls } from "@crewhaus/tool-orchestrator";
|
|
35
37
|
import { storeAndPreview } from "@crewhaus/tool-result-store";
|
|
36
|
-
import { TraceEventBus, } from "@crewhaus/trace-event-bus";
|
|
38
|
+
import { AUXILIARY_MODEL_ROLES, TraceEventBus, } from "@crewhaus/trace-event-bus";
|
|
37
39
|
import { initialState, transition, } from "@crewhaus/turn-state-machine";
|
|
38
40
|
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
39
41
|
import { createCliMarkdownRenderer, isCliMarkdownEnabled, } from "./cli-markdown";
|
|
42
|
+
import { createMemoryEventLog, createMemorySessionStore } from "./ephemeral";
|
|
40
43
|
import { FEEDBACK_EVENT_KIND, runExitRating } from "./exit-rating";
|
|
41
44
|
import { attachIncidentCollector } from "./incident-collector";
|
|
42
45
|
import { LIST_TOOLS_NAME, buildListToolsTool } from "./list-tools";
|
|
43
|
-
import {
|
|
46
|
+
import { createSpanUnionTracker, modelLatencyMs } from "./model-latency";
|
|
47
|
+
import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, attachRoutingPersistence, attachWatchmeCapture, } from "./observability";
|
|
48
|
+
import { describeHintEvidence, isSyntheticMessage, latestHumanUserMessage, markSynthetic, messageText, preRoute, stripDirectiveFromMessage, } from "./pre-route";
|
|
44
49
|
import { loadProjectMemory } from "./project-memory";
|
|
45
50
|
import { createCliOutput, isSpinnerEnabled } from "./spinner";
|
|
46
51
|
/**
|
|
@@ -236,6 +241,23 @@ export const CONTINUITY_LEDGER_MAX_CHARS = 16 * 1024;
|
|
|
236
241
|
* first consumer.
|
|
237
242
|
*/
|
|
238
243
|
export const PLAN_DIRTY_STATE_KEY = "plan.dirty";
|
|
244
|
+
/**
|
|
245
|
+
* 0.6.0 (design §6.2) — the default `budget.judge_share`: the fraction of
|
|
246
|
+
* `budget.usd` the auxiliary roles (`AUXILIARY_MODEL_ROLES` from
|
|
247
|
+
* `@crewhaus/trace-event-bus`) may spend before the runtime raises the
|
|
248
|
+
* `judge_share_exhausted` signal. Applied when the spec's `budget` block
|
|
249
|
+
* omits `judge_share`.
|
|
250
|
+
*/
|
|
251
|
+
export const DEFAULT_JUDGE_SHARE = 0.3;
|
|
252
|
+
/**
|
|
253
|
+
* 0.6.0 (design §6.2, §7.12) — the budget meters whose `judge_share` notice
|
|
254
|
+
* has already printed. Keyed on the METER rather than the loop so a
|
|
255
|
+
* caller-supplied run-spanning meter (the workflow bundle's `__budgetMeter`,
|
|
256
|
+
* shared by every step's `runChatLoop` call) prints the notice once per RUN,
|
|
257
|
+
* while a per-loop meter is one entry that dies with its loop. A WeakSet, so
|
|
258
|
+
* nothing recorded here outlives the meter it describes.
|
|
259
|
+
*/
|
|
260
|
+
const judgeShareNoticedMeters = new WeakSet();
|
|
239
261
|
/** compaction-snip's marker message — filtered out of eviction records (it
|
|
240
262
|
* is runtime scaffolding, not conversation content). */
|
|
241
263
|
const SNIP_MARKER_RE = /^\[Context compacted: \d+ messages removed\]$/;
|
|
@@ -409,6 +431,10 @@ export { JANITOR_BUILTIN_STEPS } from "./janitor";
|
|
|
409
431
|
// `loadOrCreateAgentIdentity()` at boot and threads the resulting `agentId`
|
|
410
432
|
// into `RunChatLoopOptions.agentId` (and onto any bus it constructs itself).
|
|
411
433
|
export { AGENT_IDENTITY_SCHEMA_VERSION, DEFAULT_IDENTITY_DIR, IDENTITY_FILENAME, agentFingerprint, loadOrCreateAgentIdentity, } from "./identity";
|
|
434
|
+
// 0.6.0 §7.2 / §7.11 — the preRoute decision phase (pure) and the synthetic-
|
|
435
|
+
// message marker it relies on. Re-exported so `route explain`'s offline replay
|
|
436
|
+
// and tests can reproduce a decision from the persisted line alone.
|
|
437
|
+
export { describeHintEvidence, isSyntheticMessage, latestHumanUserMessage, markSynthetic, messageHasImages, messageText, preRoute, stripDirectiveFromMessage, } from "./pre-route";
|
|
412
438
|
// Loop contract 0.4 (Batch C, G11) — the pending-approval persistence seam is
|
|
413
439
|
// owned by `@crewhaus/session-store`; re-exported here so the CLI/codegen can
|
|
414
440
|
// build the store + input hash it passes as `RunChatLoopOptions.approvals`
|
|
@@ -600,7 +626,24 @@ export async function replayMessageHistory(log) {
|
|
|
600
626
|
continue;
|
|
601
627
|
if (ev.kind === "user_message") {
|
|
602
628
|
const p = ev.payload;
|
|
603
|
-
|
|
629
|
+
// 0.6.0 §7.2.1 — honour the logged `synthetic: true` flag on replay so
|
|
630
|
+
// a resumed transcript's runtime-injected user messages stay marked
|
|
631
|
+
// (the rules and the classifier read only HUMAN text).
|
|
632
|
+
let replayed = { role: "user", content: p.content };
|
|
633
|
+
if (p.directive === true) {
|
|
634
|
+
// §7.2.1 — the persisted line keeps the `/model …` token; the REQUEST
|
|
635
|
+
// copy never carries it, on the first turn or on any resumed one. A
|
|
636
|
+
// directive-only input ran no turn (the acknowledgement was the reply,
|
|
637
|
+
// no assistant message follows), so replaying it would leave two
|
|
638
|
+
// adjacent user messages — it is dropped from the request copy.
|
|
639
|
+
const rest = stripDirectiveToken(messageText(replayed));
|
|
640
|
+
const hasOtherBlocks = typeof replayed.content !== "string" &&
|
|
641
|
+
replayed.content.some((block) => block.type !== "text");
|
|
642
|
+
if (rest.length === 0 && !hasOtherBlocks)
|
|
643
|
+
continue;
|
|
644
|
+
replayed = stripDirectiveFromMessage(replayed, rest);
|
|
645
|
+
}
|
|
646
|
+
messages.push(p.synthetic === true ? markSynthetic(replayed) : replayed);
|
|
604
647
|
}
|
|
605
648
|
else if (ev.kind === "assistant_message") {
|
|
606
649
|
const p = ev.payload;
|
|
@@ -613,31 +656,20 @@ export async function replayMessageHistory(log) {
|
|
|
613
656
|
// request. Drop such orphans so `--resume` self-heals. No-op on healthy logs.
|
|
614
657
|
return sanitizeOrphanToolUses(messages).messages;
|
|
615
658
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
* managed daemon wraps every gateway request in `withTenant(...)`, so when a
|
|
631
|
-
* tenant context is active we use that tenant's path-rebased `sessionRoot`
|
|
632
|
-
* and never fall back to the process-global default — one tenant's prompts
|
|
633
|
-
* and transcripts can no longer land in another tenant's directory (#142).
|
|
634
|
-
*
|
|
635
|
-
* Precedence: an explicit per-call `optsRoot` (set only by trusted callers
|
|
636
|
-
* such as a target emitter or a test) wins; then the active tenant's
|
|
637
|
-
* `sessionRoot`; then `CREWHAUS_SESSION_DIR`; otherwise `undefined` (which
|
|
638
|
-
* lets `session-store` apply its own non-tenant default). The env var and the
|
|
639
|
-
* global default are only consulted OUTSIDE any tenant scope.
|
|
640
|
-
*/
|
|
659
|
+
export async function attachRunEventSink(runContext, opts = {}) {
|
|
660
|
+
const rootDir = resolveSessionRootDir(opts.sessionRootDir);
|
|
661
|
+
const log = await openEventLog(runContext.sessionId, { rootDir });
|
|
662
|
+
const attached = attachRoutingPersistence(runContext.eventBus, log, runContext, {
|
|
663
|
+
kinds: opts.kinds ?? ["judge_verdict"],
|
|
664
|
+
});
|
|
665
|
+
return {
|
|
666
|
+
sessionId: runContext.sessionId,
|
|
667
|
+
async close() {
|
|
668
|
+
attached.unsubscribe();
|
|
669
|
+
await log.close();
|
|
670
|
+
},
|
|
671
|
+
};
|
|
672
|
+
}
|
|
641
673
|
export function resolveSessionRootDir(optsRoot) {
|
|
642
674
|
if (optsRoot !== undefined)
|
|
643
675
|
return optsRoot;
|
|
@@ -722,19 +754,113 @@ function bestEffortWireModelId(modelString) {
|
|
|
722
754
|
/**
|
|
723
755
|
* A short, stable fingerprint of a `model_pool` config, stamped on every
|
|
724
756
|
* `model_route` event as `policyVersion` so a learned decision can be tied back
|
|
725
|
-
* to the exact policy that made it. Deterministic
|
|
726
|
-
*
|
|
757
|
+
* to the exact policy that made it. Deterministic — no clock, no randomness.
|
|
758
|
+
*
|
|
759
|
+
* 0.6.0 §4.4 — hashes the FULL pool (every per-candidate setting, rules,
|
|
760
|
+
* strategy, reward, scope …) through `@crewhaus/model-plan`'s canonical
|
|
761
|
+
* `planFingerprint`, not just `{policy, model, tags, objective}`: a profile's
|
|
762
|
+
* `max_tokens` or `tools` edit must change `policyVersion`, or a learned
|
|
763
|
+
* decision could be attributed to a policy that no longer exists.
|
|
764
|
+
*/
|
|
765
|
+
function poolFingerprint(pool, priorsDigest) {
|
|
766
|
+
// §7.11 — the accepted priors file is part of the policy: fold its digest
|
|
767
|
+
// in only when one was loaded, so a pool without priors keeps its version.
|
|
768
|
+
return `pool-${planFingerprint(priorsDigest === undefined ? pool : { pool, priors: priorsDigest })}`;
|
|
769
|
+
}
|
|
770
|
+
/**
|
|
771
|
+
* The auto-registered loop tools that survive a profile's `tools: []` (§5.2):
|
|
772
|
+
* the runtime-bookkeeping family `BUILTIN_BOOKKEEPING_RULES` already allows
|
|
773
|
+
* (`Skill`, `ListTools`, the Focus / Plan / Goal tools), `Task`, and the
|
|
774
|
+
* memory tools. A judge profile that could not call `ListTools` would
|
|
775
|
+
* deadlock a headless run, so these are dropped only by an explicit
|
|
776
|
+
* `permissions.deny`. Hybrid tools (`Consult` / `Escalate`) join the list at
|
|
777
|
+
* boot from `opts.hybridTools`.
|
|
778
|
+
*/
|
|
779
|
+
export const RETAINED_LOOP_TOOL_NAMES = [
|
|
780
|
+
LIST_TOOLS_NAME,
|
|
781
|
+
"Skill",
|
|
782
|
+
"Task",
|
|
783
|
+
"FocusRead",
|
|
784
|
+
"FocusWrite",
|
|
785
|
+
"PlanRead",
|
|
786
|
+
"PlanUpdate",
|
|
787
|
+
"PlanComplete",
|
|
788
|
+
"GoalWrite",
|
|
789
|
+
"GoalUpdate",
|
|
790
|
+
"GoalList",
|
|
791
|
+
"Remember",
|
|
792
|
+
"Recall",
|
|
793
|
+
"MemoryForget",
|
|
794
|
+
"MemoryClear",
|
|
795
|
+
];
|
|
796
|
+
/**
|
|
797
|
+
* 0.6.0 §4.2 — the SINGLE-MODEL degrade rung's plan. `budget.on_exceed:
|
|
798
|
+
* degrade` re-resolves the primary model to a cheaper rung mid-run; when
|
|
799
|
+
* that rung's slot resolved through a `models:` profile, the profile's
|
|
800
|
+
* pinned request params must serve from then on — otherwise the cheap rung
|
|
801
|
+
* would keep answering with the expensive model's thinking budget and output
|
|
802
|
+
* ceiling, which is the opposite of what a degrade is for.
|
|
803
|
+
*
|
|
804
|
+
* Everything except the model identity and the params is INHERITED from the
|
|
805
|
+
* primary plan: a degrade narrows spend, never the toolset, the permission
|
|
806
|
+
* rules or the rate buckets. Returns `undefined` when the rung pinned no
|
|
807
|
+
* params, so a degrade without a profile leaves the serving plan exactly as
|
|
808
|
+
* it was (byte-identical to a pre-0.6.0 run).
|
|
809
|
+
*
|
|
810
|
+
* Exclusivity (§4.1) is the pool's rule verbatim: a rung that pins
|
|
811
|
+
* `temperature` drops the run's `thinking`, and one that pins `thinking`
|
|
812
|
+
* drops the run's `temperature` — the two never ride one request.
|
|
727
813
|
*/
|
|
728
|
-
function
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
814
|
+
function degradePlanFrom(base, specModel, wireModelId, params) {
|
|
815
|
+
if (params === undefined)
|
|
816
|
+
return undefined;
|
|
817
|
+
const baseThinking = base.params.reasoningEffort !== undefined
|
|
818
|
+
? { effort: base.params.reasoningEffort }
|
|
819
|
+
: base.params.thinking !== undefined
|
|
820
|
+
? { budgetTokens: base.params.thinking.budgetTokens }
|
|
821
|
+
: undefined;
|
|
822
|
+
const keepThinking = params.temperature !== undefined && params.thinking === undefined ? undefined : baseThinking;
|
|
823
|
+
const keepTemperature = params.thinking !== undefined && params.temperature === undefined
|
|
824
|
+
? undefined
|
|
825
|
+
: base.params.temperature;
|
|
826
|
+
const merged = buildRequestParams(params, {
|
|
827
|
+
maxTokens: base.params.maxTokens,
|
|
828
|
+
...(keepThinking !== undefined ? { thinking: keepThinking } : {}),
|
|
829
|
+
...(keepTemperature !== undefined ? { temperature: keepTemperature } : {}),
|
|
733
830
|
});
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
831
|
+
return {
|
|
832
|
+
...base,
|
|
833
|
+
armId: specModel,
|
|
834
|
+
modelString: specModel,
|
|
835
|
+
wireModelId,
|
|
836
|
+
params: merged,
|
|
837
|
+
paramsFingerprint: planFingerprint(merged),
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
/**
|
|
841
|
+
* Does ANY user message in the request transcript carry an image block
|
|
842
|
+
* (directly or inside a `tool_result`)? Transcript-wide on purpose: the
|
|
843
|
+
* request sends the whole `messages` array, so an image seeded two turns
|
|
844
|
+
* back, or one a sighted candidate's `ReadImage` returned earlier in this
|
|
845
|
+
* turn, still reaches the provider on every later call — a `vision: false`
|
|
846
|
+
* candidate would 400 or silently drop it just the same. Recomputed per
|
|
847
|
+
* call from the live array, so compaction that summarises the image away
|
|
848
|
+
* re-admits the candidate naturally. Early-exits on the first hit.
|
|
849
|
+
*/
|
|
850
|
+
function transcriptHasImages(messages) {
|
|
851
|
+
for (const m of messages) {
|
|
852
|
+
if (m.role !== "user" || typeof m.content === "string")
|
|
853
|
+
continue;
|
|
854
|
+
for (const block of m.content) {
|
|
855
|
+
if (block.type === "image")
|
|
856
|
+
return true;
|
|
857
|
+
if (block.type === "tool_result" && Array.isArray(block.content)) {
|
|
858
|
+
if (block.content.some((b) => b.type === "image"))
|
|
859
|
+
return true;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
return false;
|
|
738
864
|
}
|
|
739
865
|
/** Read the branded timeout reason off an aborted signal, or undefined when
|
|
740
866
|
* the signal is not aborted / was aborted for another cause (SIGINT, EOF). */
|
|
@@ -804,7 +930,9 @@ export async function runChatLoop(opts) {
|
|
|
804
930
|
// Item 3 (G32) — resolve the advertised tool set ONCE, up front, folding in
|
|
805
931
|
// any plugin-contributed tools so the tool-capable-model feature gate and the
|
|
806
932
|
// per-turn advertisement below read the same list.
|
|
807
|
-
|
|
933
|
+
// 0.6.0 — then the hybrid-strategy tools (`Consult` / `Escalate`) the
|
|
934
|
+
// composition root registered, same first-party-wins posture.
|
|
935
|
+
const mergedTools = mergeEffectiveTools(mergeEffectiveTools(opts.tools ?? [], opts.plugins?.tools), opts.hybridTools);
|
|
808
936
|
// #405 — the runtime's own toolset-introspection tool rides every
|
|
809
937
|
// TOOL-CARRYING loop. Late-bound: it is part of the list it reports, so it
|
|
810
938
|
// reads the advertised summaries assigned after that list is built below.
|
|
@@ -956,6 +1084,9 @@ export async function runChatLoop(opts) {
|
|
|
956
1084
|
// Item 26 — tool_use blocks the PREVIOUS turn produced (a tier-routing
|
|
957
1085
|
// signal). Persists across `runOneTurn` calls; 0 on the first turn.
|
|
958
1086
|
let priorTurnToolUseCount = 0;
|
|
1087
|
+
// 0.6.0 §7.2.2 — the names behind that count (the `toolNamesLastTurn`
|
|
1088
|
+
// routing signal a rule can read).
|
|
1089
|
+
let priorTurnToolNames = [];
|
|
959
1090
|
// Item 4 / G28 — the provider's ACTUAL input_tokens (input + cache read +
|
|
960
1091
|
// cache create) from the MOST RECENT `model_response`. Updated after every
|
|
961
1092
|
// main-turn model call; feeds the pre-turn compaction trigger and the
|
|
@@ -963,6 +1094,10 @@ export async function runChatLoop(opts) {
|
|
|
963
1094
|
// chars/4 heuristic. `undefined` before the first response — the heuristic
|
|
964
1095
|
// is used only pre-first-call.
|
|
965
1096
|
let lastModelInputTokens;
|
|
1097
|
+
// 0.6.0 §8.1 — the WIRE model id of the most recent main-turn
|
|
1098
|
+
// `model_response`, i.e. the arm that produced the text an in-loop
|
|
1099
|
+
// evaluation grades; stamped as `eval_graded.model`.
|
|
1100
|
+
let lastResponseWireModel;
|
|
966
1101
|
/** The context-size signal for routing/compaction: the provider's real
|
|
967
1102
|
* last-call input_tokens once available, else the chars/4 estimate. */
|
|
968
1103
|
const contextTokenSignal = (msgs) => lastModelInputTokens ?? estimateTokens(msgs);
|
|
@@ -1023,11 +1158,27 @@ export async function runChatLoop(opts) {
|
|
|
1023
1158
|
// Persistence boot: session create/resume + event log open. Runs first
|
|
1024
1159
|
// so the run context's sessionId — possibly carried over from a prior
|
|
1025
1160
|
// run — is the one the rest of the loop logs against.
|
|
1161
|
+
// 0.6.0 §16 Q6 — `persistSession: false` swaps BOTH stores for in-memory
|
|
1162
|
+
// twins and skips every disk sweep below: a nested side call leaves no
|
|
1163
|
+
// file behind. There is then nothing to resume from, so `resume` is refused.
|
|
1164
|
+
const persistSession = opts.persistSession !== false;
|
|
1165
|
+
if (!persistSession && opts.resume !== undefined) {
|
|
1166
|
+
throw new RuntimeError("runChatLoop: `persistSession: false` cannot be combined with `resume` — an in-memory run has no session file to resume from");
|
|
1167
|
+
}
|
|
1168
|
+
// 0.6.0 §7.6 — the committee replaces the host's ONE turn; on a REPL loop
|
|
1169
|
+
// it would replace every turn the user is waiting on. The spec rejects it
|
|
1170
|
+
// on cli / channel / managed; the loop refuses it here as the second gate.
|
|
1171
|
+
if (opts.sideCalls?.committee !== undefined && opts.singleTurn !== true) {
|
|
1172
|
+
throw new RuntimeError("runChatLoop: `sideCalls.committee` is legal in singleTurn mode only (a workflow step, a graph node, a crew role) — a REPL or per-message turn is one the user is waiting on");
|
|
1173
|
+
}
|
|
1026
1174
|
const sessionRootDir = resolveSessionRootDir(opts.sessionRootDir);
|
|
1027
|
-
const sessionStore =
|
|
1175
|
+
const sessionStore = persistSession
|
|
1176
|
+
? createSessionStore({ rootDir: sessionRootDir })
|
|
1177
|
+
: createMemorySessionStore();
|
|
1028
1178
|
// Housekeeping side-effect: evicts any session whose mtime is older than
|
|
1029
1179
|
// the TTL (default 30 days). The returned list is intentionally discarded.
|
|
1030
|
-
|
|
1180
|
+
if (persistSession)
|
|
1181
|
+
await sessionStore.list();
|
|
1031
1182
|
// Loop contract 0.4 (G11) — the same sweep for the approvals log sitting
|
|
1032
1183
|
// beside those sessions. It has no other unattended pruner: only `list()`
|
|
1033
1184
|
// compacts it, and `list()`'s only production callers are the human-invoked
|
|
@@ -1040,9 +1191,11 @@ export async function runChatLoop(opts) {
|
|
|
1040
1191
|
// `opts.approvals`) so a harness that has since switched to `ask_mode: deny`
|
|
1041
1192
|
// still gets its old parks pruned; it costs one ENOENT when the file does
|
|
1042
1193
|
// not exist. Best-effort, exactly like the session sweep above.
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1194
|
+
if (persistSession) {
|
|
1195
|
+
await evictExpiredApprovals({
|
|
1196
|
+
...(sessionRootDir !== undefined ? { rootDir: sessionRootDir } : {}),
|
|
1197
|
+
}).catch(() => undefined);
|
|
1198
|
+
}
|
|
1046
1199
|
// Adaptive model routing (spec `agent.model_pool`). Resolve every candidate
|
|
1047
1200
|
// adapter ONCE at boot (mirroring the tier router), open the durable reward
|
|
1048
1201
|
// scoreboard beside the sessions dir, and build the PolicyRouter. The
|
|
@@ -1054,31 +1207,136 @@ export async function runChatLoop(opts) {
|
|
|
1054
1207
|
let scoreboard;
|
|
1055
1208
|
let rewardConfig;
|
|
1056
1209
|
let poolPolicyVersion;
|
|
1210
|
+
const poolCandidateConfigs = new Map();
|
|
1211
|
+
// §7.9 — the scoped routeKey prefix: declared on the pool, else the
|
|
1212
|
+
// caller's toolset scope (the crew orchestrator / workflow / graph stamp the
|
|
1213
|
+
// former; a cli agent has neither and stays unscoped).
|
|
1214
|
+
const poolScope = opts.modelPool?.scope ?? opts.toolsetScope;
|
|
1057
1215
|
if (opts.modelPool !== undefined) {
|
|
1058
1216
|
const pool = opts.modelPool;
|
|
1059
1217
|
const candidates = [];
|
|
1060
1218
|
for (const c of pool.candidates) {
|
|
1219
|
+
// 0.6.0 §7.1 — a withdrawn candidate never becomes an arm.
|
|
1220
|
+
if (c.enabled === false)
|
|
1221
|
+
continue;
|
|
1061
1222
|
const injected = opts._poolAdapters?.get(c.model);
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1223
|
+
// §7.9 — the ARM IDENTITY (what the scoreboard keys on and the preRoute
|
|
1224
|
+
// hint names the candidate by): the profile name when it is a profile,
|
|
1225
|
+
// else the model string.
|
|
1226
|
+
const armId = c.profile ?? c.model;
|
|
1227
|
+
const base = injected !== undefined
|
|
1228
|
+
? { adapter: injected, modelId: bestEffortWireModelId(c.model) }
|
|
1229
|
+
: await (async () => {
|
|
1230
|
+
const r = await resolveModel(c.model);
|
|
1231
|
+
return { adapter: r.adapter, modelId: r.modelId };
|
|
1232
|
+
})();
|
|
1233
|
+
let resolved;
|
|
1234
|
+
if (c.fallbacks !== undefined && c.fallbacks.length > 0) {
|
|
1235
|
+
// §4.4 — a per-profile failover chain: the candidate's adapter IS the
|
|
1236
|
+
// chain (itself a ProviderAdapter, every member breaker-wrapped inside
|
|
1237
|
+
// it). Seeded with the already-resolved primary plus any injected
|
|
1238
|
+
// pool adapters, mirroring the run-level chain above. The chain
|
|
1239
|
+
// rewrites `model` internally, so `lastServed()` names the member
|
|
1240
|
+
// that actually served — what both attribution sites read.
|
|
1241
|
+
const chainAdapters = new Map();
|
|
1242
|
+
chainAdapters.set(c.model, base.adapter);
|
|
1243
|
+
for (const [modelString, a] of opts._poolAdapters ?? []) {
|
|
1244
|
+
if (!chainAdapters.has(modelString))
|
|
1245
|
+
chainAdapters.set(modelString, a);
|
|
1246
|
+
}
|
|
1247
|
+
const chain = await createFailoverChain({
|
|
1248
|
+
model: c.model,
|
|
1249
|
+
fallbacks: c.fallbacks,
|
|
1250
|
+
...(c.circuitBreaker !== undefined ? { breaker: c.circuitBreaker } : {}),
|
|
1251
|
+
getBus: () => observabilityBus,
|
|
1252
|
+
adapters: chainAdapters,
|
|
1253
|
+
});
|
|
1254
|
+
for (const warning of chain.warnings()) {
|
|
1255
|
+
process.stderr.write(`[model_pool] ${armId}: ${warning}\n`);
|
|
1256
|
+
}
|
|
1257
|
+
resolved = {
|
|
1258
|
+
adapter: chain,
|
|
1259
|
+
modelId: base.modelId,
|
|
1066
1260
|
modelString: c.model,
|
|
1067
1261
|
tags: c.tags,
|
|
1068
|
-
|
|
1262
|
+
armId,
|
|
1263
|
+
// The chain is "open" only when every member's breaker is (a chain
|
|
1264
|
+
// with one closed member can still serve).
|
|
1265
|
+
breaker: {
|
|
1266
|
+
state: () => {
|
|
1267
|
+
const members = chain.candidates();
|
|
1268
|
+
return members.length > 0 && members.every((m) => m.breakerState === "open")
|
|
1269
|
+
? "open"
|
|
1270
|
+
: "closed";
|
|
1271
|
+
},
|
|
1272
|
+
},
|
|
1273
|
+
lastServed: () => chain.lastServed(),
|
|
1274
|
+
};
|
|
1069
1275
|
}
|
|
1070
1276
|
else {
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1277
|
+
// §4.4 — every candidate is breaker-wrapped; `state() !== "open"` is
|
|
1278
|
+
// the eligibility filter (preRoute's N1 check and the router both
|
|
1279
|
+
// read it). The profile's `circuit_breaker` tunes it; package
|
|
1280
|
+
// defaults apply otherwise.
|
|
1281
|
+
const wrapped = wrapWithCircuitBreaker(base.adapter, {
|
|
1282
|
+
...(c.circuitBreaker ?? {}),
|
|
1283
|
+
adapterName: armId,
|
|
1284
|
+
bus: opts.runContext?.eventBus,
|
|
1285
|
+
});
|
|
1286
|
+
resolved = {
|
|
1287
|
+
adapter: wrapped,
|
|
1288
|
+
modelId: base.modelId,
|
|
1075
1289
|
modelString: c.model,
|
|
1076
1290
|
tags: c.tags,
|
|
1077
|
-
|
|
1291
|
+
armId,
|
|
1292
|
+
breaker: wrapped,
|
|
1293
|
+
};
|
|
1078
1294
|
}
|
|
1295
|
+
candidates.push(resolved);
|
|
1296
|
+
poolCandidateConfigs.set(resolved, c);
|
|
1079
1297
|
}
|
|
1080
1298
|
const routingRoot = sessionRootDir !== undefined ? pathDirname(sessionRootDir) : ".crewhaus";
|
|
1081
|
-
|
|
1299
|
+
const rewardBlock = pool.reward;
|
|
1300
|
+
// §6.3 item 4 — the per-arm LINEAGE: a PROFILED arm's lines are stamped
|
|
1301
|
+
// with a fingerprint of its profile + the pool's quality source + the
|
|
1302
|
+
// grader kind; a line stamped under a different lineage is history from a
|
|
1303
|
+
// profile that changed under the same arm id and is skipped on load
|
|
1304
|
+
// (`reset_on_profile_change`, default true). Unprofiled arms (arm id =
|
|
1305
|
+
// model string) are not stamped, so a bare 0.5.x pool writes 0.5.x lines.
|
|
1306
|
+
// 0.6.0 §6.2 (PR 13) — the JUDGE MODEL now folds in too. PR 10 could not
|
|
1307
|
+
// see it: the judge was closed over by `evaluation.evaluate` and nothing
|
|
1308
|
+
// at this seam knew which model graded. `RunEvaluation.judgeModel` states
|
|
1309
|
+
// it declaratively (emitted for `llm_judge` graders by the three
|
|
1310
|
+
// `renderEvaluation` copies and the interpreter), so re-pointing the
|
|
1311
|
+
// judge at a different model starts a NEW lineage rather than averaging
|
|
1312
|
+
// two instruments' verdicts into one arm's quality. Absent for
|
|
1313
|
+
// deterministic graders and for hand-built IR — a `contains` grader has
|
|
1314
|
+
// no judge — and `planFingerprint` drops `undefined` members, so a
|
|
1315
|
+
// judge-less pool's lineage is byte-identical to PR 10's.
|
|
1316
|
+
const lineage = {};
|
|
1317
|
+
for (const candidate of candidates) {
|
|
1318
|
+
const cfg = poolCandidateConfigs.get(candidate);
|
|
1319
|
+
if (cfg?.profile === undefined)
|
|
1320
|
+
continue;
|
|
1321
|
+
lineage[cfg.profile] = planFingerprint({
|
|
1322
|
+
candidate: cfg,
|
|
1323
|
+
qualitySource: rewardBlock?.qualitySource ?? "none",
|
|
1324
|
+
grader: opts.evaluation?.graderType,
|
|
1325
|
+
judge: opts.evaluation?.judgeModel,
|
|
1326
|
+
});
|
|
1327
|
+
}
|
|
1328
|
+
const liveScoreboard = opts._scoreboard ??
|
|
1329
|
+
openScoreboard(routingRoot, {
|
|
1330
|
+
...(Object.keys(lineage).length > 0 ? { lineage } : {}),
|
|
1331
|
+
...(rewardBlock?.resetOnProfileChange !== undefined
|
|
1332
|
+
? { resetOnProfileChange: rewardBlock.resetOnProfileChange }
|
|
1333
|
+
: {}),
|
|
1334
|
+
});
|
|
1335
|
+
// §10.1 — `crewhaus route freeze <policyVersion>`: the kill switch. While
|
|
1336
|
+
// the marker exists the scoreboard is read-only for this run and every
|
|
1337
|
+
// decision reports the FROZEN policyVersion.
|
|
1338
|
+
const routeFreeze = readRouteFreeze(routingRoot, (detail) => process.stderr.write(`[model_pool] route freeze marker ignored: ${detail}\n`));
|
|
1339
|
+
scoreboard = routeFreeze !== undefined ? freezeScoreboard(liveScoreboard) : liveScoreboard;
|
|
1082
1340
|
const sb = scoreboard;
|
|
1083
1341
|
rewardConfig = {
|
|
1084
1342
|
...(pool.objective !== undefined ? { objective: pool.objective } : {}),
|
|
@@ -1087,10 +1345,85 @@ export async function runChatLoop(opts) {
|
|
|
1087
1345
|
? { latencyRefMs: pool.learning.latencyRefMs }
|
|
1088
1346
|
: {}),
|
|
1089
1347
|
};
|
|
1090
|
-
|
|
1348
|
+
// §7.11 N2 — eval-seeded priors. Read beside the arms, validated against
|
|
1349
|
+
// the ROSTER fingerprint, and composed over the live lookup so a seeded
|
|
1350
|
+
// arm skips warm-up. Any problem is a boot warning and cold warm-up —
|
|
1351
|
+
// never a silently wrong prior. The accepted file's digest is folded into
|
|
1352
|
+
// `policyVersion`, so a new leaderboard is a new policy.
|
|
1353
|
+
let liveLookup = (rk, m) => sb.score(rk, m);
|
|
1354
|
+
let priorsDigest;
|
|
1355
|
+
if (rewardBlock?.priors === "eval") {
|
|
1356
|
+
const raw = readRoutingPriorsRaw(routingRoot);
|
|
1357
|
+
const warnPriors = (detail) => {
|
|
1358
|
+
process.stderr.write(`[model_pool] reward.priors: eval — priors ignored (${detail}); the learned policy warms up cold.\n`);
|
|
1359
|
+
};
|
|
1360
|
+
if (raw === undefined) {
|
|
1361
|
+
warnPriors(`no ${routingPriorsPath(routingRoot)} — run eval leaderboard --export-priors`);
|
|
1362
|
+
}
|
|
1363
|
+
else if (!raw.ok) {
|
|
1364
|
+
warnPriors(raw.error);
|
|
1365
|
+
}
|
|
1366
|
+
else {
|
|
1367
|
+
const loaded = loadPriors(raw.raw, {
|
|
1368
|
+
expectFingerprint: priorsFingerprint(pool.candidates),
|
|
1369
|
+
});
|
|
1370
|
+
if (loaded.ok) {
|
|
1371
|
+
priorsDigest = loaded.priors.digest;
|
|
1372
|
+
liveLookup = seededScoreLookup(liveLookup, loaded.priors);
|
|
1373
|
+
}
|
|
1374
|
+
else {
|
|
1375
|
+
warnPriors(`${loaded.reason}: ${loaded.detail}`);
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
const computedPolicyVersion = poolFingerprint(pool, priorsDigest);
|
|
1380
|
+
if (routeFreeze !== undefined) {
|
|
1381
|
+
poolPolicyVersion = routeFreeze.policyVersion;
|
|
1382
|
+
process.stderr.write(`[model_pool] routing is FROZEN at policyVersion ${routeFreeze.policyVersion}${routeFreeze.policyVersion !== computedPolicyVersion
|
|
1383
|
+
? ` (the current pool would be ${computedPolicyVersion})`
|
|
1384
|
+
: ""} — no new observations are recorded; \`crewhaus route freeze --clear\` lifts it.\n`);
|
|
1385
|
+
}
|
|
1386
|
+
else {
|
|
1387
|
+
poolPolicyVersion = computedPolicyVersion;
|
|
1388
|
+
}
|
|
1389
|
+
// 0.6.0 §7.2.3 — under `classifier` the router itself stays the heuristic
|
|
1390
|
+
// selector: the classifier's verdict arrives as the preRoute hint's forced
|
|
1391
|
+
// arm, and the heuristic is the documented fallback when it does not.
|
|
1392
|
+
const routerPolicy = pool.policy === "classifier" ? "heuristic" : pool.policy;
|
|
1393
|
+
// §7.10 — `reward.floor.arm` is a ROLE SLOT (a candidate tag, a
|
|
1394
|
+
// `$profile` the compiler has already lowered to its bare name, or a
|
|
1395
|
+
// model string); the router compares the floor arm to arm IDS by strict
|
|
1396
|
+
// equality, so resolve the slot against the booted roster HERE. An
|
|
1397
|
+
// unresolvable floor is a boot error, never a silently `unavailable`
|
|
1398
|
+
// check — the floor is the acceptance guarantee (§1) that a learned
|
|
1399
|
+
// policy cannot exploit below a pinned arm.
|
|
1400
|
+
let routerFloor;
|
|
1401
|
+
if (rewardBlock?.floor !== undefined) {
|
|
1402
|
+
const declared = rewardBlock.floor.arm;
|
|
1403
|
+
if (declared === undefined) {
|
|
1404
|
+
routerFloor = rewardBlock.floor;
|
|
1405
|
+
}
|
|
1406
|
+
else {
|
|
1407
|
+
const wanted = declared.startsWith("$") ? declared.slice(1) : declared;
|
|
1408
|
+
const floorCandidate = candidates.find((c) => c.modelString === declared) ??
|
|
1409
|
+
candidates.find((c) => poolCandidateConfigs.get(c)?.profile === wanted) ??
|
|
1410
|
+
candidates.find((c) => c.tags.includes(wanted));
|
|
1411
|
+
if (floorCandidate === undefined) {
|
|
1412
|
+
const roster = candidates
|
|
1413
|
+
.map((c) => poolCandidateConfigs.get(c)?.profile ?? c.modelString)
|
|
1414
|
+
.join(", ");
|
|
1415
|
+
throw new RuntimeError(`model_pool.reward.floor.arm "${declared}" is not a roster arm (declared: ${roster}) — name a candidate's model, its profile or a tag`);
|
|
1416
|
+
}
|
|
1417
|
+
routerFloor = { ...rewardBlock.floor, arm: poolCandidateArmId(floorCandidate) };
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1091
1420
|
poolRouter = createPolicyRouter({
|
|
1092
1421
|
candidates,
|
|
1093
|
-
policy:
|
|
1422
|
+
policy: routerPolicy,
|
|
1423
|
+
// §7.9 — scoped arms (`<scope>/<band>`, backing off to the band while
|
|
1424
|
+
// under-sampled); §7.10 — the floor the learned policy exploits above.
|
|
1425
|
+
...(poolScope !== undefined ? { scope: poolScope } : {}),
|
|
1426
|
+
...(routerFloor !== undefined ? { floor: routerFloor } : {}),
|
|
1094
1427
|
...(pool.routing !== undefined ? { routing: pool.routing } : {}),
|
|
1095
1428
|
...(pool.learning !== undefined
|
|
1096
1429
|
? {
|
|
@@ -1105,30 +1438,148 @@ export async function runChatLoop(opts) {
|
|
|
1105
1438
|
},
|
|
1106
1439
|
}
|
|
1107
1440
|
: {}),
|
|
1108
|
-
// The learned policy reads the live scoreboard
|
|
1109
|
-
|
|
1441
|
+
// The learned policy reads the live scoreboard (seeded by the priors
|
|
1442
|
+
// when declared); static/heuristic ignore it.
|
|
1443
|
+
...(pool.policy === "learned" ? { score: liveLookup } : {}),
|
|
1110
1444
|
});
|
|
1111
1445
|
}
|
|
1446
|
+
// 0.6.0 §7.12 — under a pool, `budget.on_exceed: degrade` is an ELIGIBILITY
|
|
1447
|
+
// restriction, not an adapter swap (the pool branch overwrote `turnAdapter`
|
|
1448
|
+
// on every call, so the swap was dead config). Pre-resolve the degrade
|
|
1449
|
+
// model as one more `PoolCandidate` at boot: bound to the roster member
|
|
1450
|
+
// when the spec names one, else an extra always-eligible rung (the
|
|
1451
|
+
// compiler warned `budget-degrade-outside-pool`). Resolution is
|
|
1452
|
+
// best-effort — a rung that cannot resolve is reported once here and the
|
|
1453
|
+
// breach then ends the run, exactly as the lazy path did before.
|
|
1454
|
+
let budgetDegradeRung;
|
|
1455
|
+
let budgetDegradeRungError;
|
|
1456
|
+
if (poolRouter !== undefined && opts.budget?.onExceed.kind === "degrade") {
|
|
1457
|
+
const target = opts.budget.onExceed.model;
|
|
1458
|
+
const member = poolRouter.candidates().find((c) => c.modelString === target);
|
|
1459
|
+
if (member !== undefined) {
|
|
1460
|
+
budgetDegradeRung = member;
|
|
1461
|
+
}
|
|
1462
|
+
else if (opts._budgetDegradeAdapter !== undefined) {
|
|
1463
|
+
budgetDegradeRung = {
|
|
1464
|
+
adapter: opts._budgetDegradeAdapter,
|
|
1465
|
+
modelId: bestEffortWireModelId(target),
|
|
1466
|
+
modelString: target,
|
|
1467
|
+
tags: [],
|
|
1468
|
+
};
|
|
1469
|
+
}
|
|
1470
|
+
else {
|
|
1471
|
+
try {
|
|
1472
|
+
const r = await resolveModel(target);
|
|
1473
|
+
budgetDegradeRung = {
|
|
1474
|
+
adapter: r.adapter,
|
|
1475
|
+
modelId: r.modelId,
|
|
1476
|
+
modelString: target,
|
|
1477
|
+
tags: [],
|
|
1478
|
+
};
|
|
1479
|
+
}
|
|
1480
|
+
catch (err) {
|
|
1481
|
+
budgetDegradeRungError = err instanceof Error ? err.message : String(err);
|
|
1482
|
+
process.stderr.write(`[budget] degrade model "${target}" could not be resolved at boot (${budgetDegradeRungError}) — a budget breach will end the run.\n`);
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
// 0.6.0 §6.3 — a judged quality reaches the REWARD only under
|
|
1487
|
+
// `reward.quality_source: in_loop`; otherwise it is persisted on the `v:2`
|
|
1488
|
+
// line but the reward is computed without it (byte-identical to 0.5.x).
|
|
1489
|
+
const qualityIntoReward = opts.modelPool?.reward?.qualitySource === "in_loop";
|
|
1490
|
+
// 0.6.0 §7.8 — the LAST successfully served pool call of the current turn
|
|
1491
|
+
// (arm, band, latency, cost): what the shadow lane grades against and
|
|
1492
|
+
// records beside the shadow arm. Undefined without a pool.
|
|
1493
|
+
let lastPoolOutcome;
|
|
1494
|
+
/** §7.9 — the routing provenance every arm line carries (`pv` / `sc` / `h`). */
|
|
1495
|
+
const routeProvenance = () => ({
|
|
1496
|
+
...(poolPolicyVersion !== undefined ? { policyVersion: poolPolicyVersion } : {}),
|
|
1497
|
+
...(poolScope !== undefined ? { scope: poolScope } : {}),
|
|
1498
|
+
...(opts.sessionName !== undefined ? { harness: opts.sessionName } : {}),
|
|
1499
|
+
});
|
|
1112
1500
|
// Fold one turn's observed outcome into the scoreboard (no-op without a pool).
|
|
1501
|
+
// The arm is `turn.armId` (the profile name for a profiled candidate, §7.9),
|
|
1502
|
+
// falling back to the model string for the strategy arm and bare candidates.
|
|
1113
1503
|
const recordPoolOutcome = (turn, obs) => {
|
|
1114
1504
|
// Capture into consts so the narrowing survives (they are closed-over lets).
|
|
1115
1505
|
const sb = scoreboard;
|
|
1116
1506
|
const rc = rewardConfig;
|
|
1117
1507
|
if (turn === undefined || sb === undefined || rc === undefined)
|
|
1118
1508
|
return;
|
|
1119
|
-
|
|
1509
|
+
const arm = turn.armId ?? turn.modelString;
|
|
1510
|
+
if (obs.success) {
|
|
1511
|
+
lastPoolOutcome = {
|
|
1512
|
+
modelString: turn.modelString,
|
|
1513
|
+
armId: arm,
|
|
1514
|
+
routeKey: turn.routeKey,
|
|
1515
|
+
...(obs.latencyMs !== undefined ? { latencyMs: obs.latencyMs } : {}),
|
|
1516
|
+
...(obs.costUsd !== undefined ? { costUsd: obs.costUsd } : {}),
|
|
1517
|
+
};
|
|
1518
|
+
}
|
|
1519
|
+
// §6.3 item 1 / §7.13 — under `in_loop` an OMITTED quality is
|
|
1520
|
+
// unrepresentable: `computeReward` reads it as a perfect 1.0, so a
|
|
1521
|
+
// successful observation the judge never scored (a direct-served strong
|
|
1522
|
+
// rung past the judge share, an aborted turn, a stage whose grader threw)
|
|
1523
|
+
// is counted `ungraded` instead of recorded. Failed calls still record —
|
|
1524
|
+
// the reward short-circuits on `!success` before reading quality.
|
|
1525
|
+
if (qualityIntoReward && obs.success && obs.quality === undefined) {
|
|
1526
|
+
sb.ungraded(turn.routeKey, arm);
|
|
1527
|
+
return;
|
|
1528
|
+
}
|
|
1529
|
+
const { quality: _quality, ...withoutQuality } = obs;
|
|
1530
|
+
sb.record(turn.routeKey, arm, computeReward(qualityIntoReward ? obs : withoutQuality, rc), {
|
|
1531
|
+
...obs,
|
|
1532
|
+
...routeProvenance(),
|
|
1533
|
+
});
|
|
1534
|
+
};
|
|
1535
|
+
/** 0.6.0 §7.3 — resolve a roster arm by spec model string, profile name (`$` optional) or tag. */
|
|
1536
|
+
const resolveRosterArm = (value) => {
|
|
1537
|
+
if (poolRouter === undefined)
|
|
1538
|
+
return undefined;
|
|
1539
|
+
const roster = poolRouter.candidates();
|
|
1540
|
+
const wanted = value.startsWith("$") ? value.slice(1) : value;
|
|
1541
|
+
return (roster.find((c) => c.modelString === value) ??
|
|
1542
|
+
roster.find((c) => poolCandidateConfigs.get(c)?.profile === wanted) ??
|
|
1543
|
+
roster.find((c) => c.tags.includes(wanted)));
|
|
1120
1544
|
};
|
|
1545
|
+
// 0.6.0 §7.3 — `forcedCandidate`: the workflow / graph retry closure's
|
|
1546
|
+
// escalation rung, resolved against the booted roster ONCE. Refused loudly
|
|
1547
|
+
// when nothing can honour it — forcing is only meaningful over a roster.
|
|
1548
|
+
let forcedRunCandidate;
|
|
1549
|
+
if (opts.forcedCandidate !== undefined) {
|
|
1550
|
+
if (poolRouter === undefined) {
|
|
1551
|
+
throw new RuntimeError(`forcedCandidate "${opts.forcedCandidate}" requires a model_pool — this run declares none, so there is no roster arm to force the turn onto`);
|
|
1552
|
+
}
|
|
1553
|
+
forcedRunCandidate = resolveRosterArm(opts.forcedCandidate);
|
|
1554
|
+
if (forcedRunCandidate === undefined) {
|
|
1555
|
+
const declared = poolRouter
|
|
1556
|
+
.candidates()
|
|
1557
|
+
.map((c) => poolCandidateConfigs.get(c)?.profile ?? c.modelString)
|
|
1558
|
+
.join(", ");
|
|
1559
|
+
throw new RuntimeError(`forcedCandidate "${opts.forcedCandidate}" is not a roster arm (declared: ${declared}) — name a candidate's model, its profile or a tag`);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1121
1562
|
// Per-turn USD cost from token usage + the static pricing table. Undefined
|
|
1122
1563
|
// when the served model isn't on the table (the reward then drops the cost
|
|
1123
1564
|
// term and reweights over quality + latency).
|
|
1124
1565
|
const poolCostUsd = (provider, wireModelId, usage) => {
|
|
1125
|
-
|
|
1566
|
+
// §7.9 — the INSTALLED feed when the host injected one, else the table.
|
|
1567
|
+
const row = resolvePricing(opts.pricing ?? DEFAULT_PRICING, provider, wireModelId);
|
|
1126
1568
|
if (row === undefined)
|
|
1127
1569
|
return undefined;
|
|
1128
1570
|
return (computeCostMicros(row, usage.input, usage.output, usage.cacheRead ?? 0, usage.cacheCreate ?? 0) / 1_000_000);
|
|
1129
1571
|
};
|
|
1130
1572
|
let sessionId;
|
|
1131
1573
|
let resumedMessages;
|
|
1574
|
+
// 0.6.0 (design §8.1) — the session's `models` provenance: every wire model
|
|
1575
|
+
// that served a main-turn response, first-seen order. Seeded from the
|
|
1576
|
+
// resumed session and folded on every `model_response` below
|
|
1577
|
+
// (`recordServedModel`), so `sessions tail` can name the arms a pooled
|
|
1578
|
+
// conversation actually used without replaying its event log.
|
|
1579
|
+
let sessionModels;
|
|
1580
|
+
// 0.6.0 §7.2.1 — the resumed session's `/model` pin, read back UNTRUSTED
|
|
1581
|
+
// (type + roster check against the pool once it is resolved below).
|
|
1582
|
+
let resumedPin;
|
|
1132
1583
|
// #405 — the toolset recorded by this session's LAST run (undefined when
|
|
1133
1584
|
// the session predates the record, or on a fresh session).
|
|
1134
1585
|
let resumedToolNames;
|
|
@@ -1137,12 +1588,21 @@ export async function runChatLoop(opts) {
|
|
|
1137
1588
|
// re-evictions happens at fold time): a resumed session recovers every
|
|
1138
1589
|
// externalized user requirement without trusting any summary.
|
|
1139
1590
|
const resumedLedgerSeed = [];
|
|
1591
|
+
// 0.6.0 §7.12 — `budget.scope: "session"`: the spend the session log
|
|
1592
|
+
// already persisted (top-level `cost_accrual` lines, per-call only — the
|
|
1593
|
+
// persist subscriber skips `summary` aggregates) seeds this run's meter, so
|
|
1594
|
+
// the cap bounds the conversation rather than one resumed run. Stays 0
|
|
1595
|
+
// under the default `run` scope.
|
|
1596
|
+
let resumedCostUsdMicros = 0;
|
|
1597
|
+
const budgetSessionScoped = opts.budget?.scope === "session";
|
|
1140
1598
|
if (opts.resume) {
|
|
1141
1599
|
const existing = await sessionStore.get(opts.resume.sessionId);
|
|
1142
1600
|
if (existing === null) {
|
|
1143
1601
|
throw new RuntimeError(`cannot --resume "${opts.resume.sessionId}": session not found in ${sessionRootDir ?? ".crewhaus/sessions"}`);
|
|
1144
1602
|
}
|
|
1145
1603
|
sessionId = existing.id;
|
|
1604
|
+
sessionModels = existing.models;
|
|
1605
|
+
resumedPin = existing.pin;
|
|
1146
1606
|
const replayLog = await openEventLog(sessionId, { rootDir: sessionRootDir });
|
|
1147
1607
|
resumedMessages = await replayMessageHistory(replayLog);
|
|
1148
1608
|
// One pass gathers both resume-time reads: the continuity ledger seed
|
|
@@ -1164,6 +1624,22 @@ export async function runChatLoop(opts) {
|
|
|
1164
1624
|
nestDepth = Math.max(0, nestDepth - 1);
|
|
1165
1625
|
continue;
|
|
1166
1626
|
}
|
|
1627
|
+
if (ev.kind === "cost_accrual") {
|
|
1628
|
+
if (!budgetSessionScoped)
|
|
1629
|
+
continue;
|
|
1630
|
+
const p = ev.payload;
|
|
1631
|
+
// 0.6.0 §10.2 — a child's role-bearing roll-up sits INSIDE its
|
|
1632
|
+
// sub_agent bracket but was published on THIS bus and counted by this
|
|
1633
|
+
// run's meter, so it seeds the resumed meter too; a peer's per-call
|
|
1634
|
+
// lines inside a bracket stay skipped as before.
|
|
1635
|
+
const childRollup = p.summary === true && typeof p.role === "string";
|
|
1636
|
+
if (nestDepth > 0 && !childRollup)
|
|
1637
|
+
continue;
|
|
1638
|
+
if (typeof p.costUsdMicros === "number" && Number.isFinite(p.costUsdMicros)) {
|
|
1639
|
+
resumedCostUsdMicros += Math.max(0, p.costUsdMicros);
|
|
1640
|
+
}
|
|
1641
|
+
continue;
|
|
1642
|
+
}
|
|
1167
1643
|
if (ev.kind === "toolset") {
|
|
1168
1644
|
if (nestDepth > 0)
|
|
1169
1645
|
continue;
|
|
@@ -1263,11 +1739,11 @@ export async function runChatLoop(opts) {
|
|
|
1263
1739
|
...(opts.incidentSpec !== undefined ? { spec: opts.incidentSpec } : {}),
|
|
1264
1740
|
});
|
|
1265
1741
|
// Item 22 — surface each failover on stderr so an operator watching a
|
|
1266
|
-
// plain (non-CREWHAUS_TRACE) run still sees provider switches.
|
|
1267
|
-
//
|
|
1268
|
-
//
|
|
1269
|
-
//
|
|
1270
|
-
//
|
|
1742
|
+
// plain (non-CREWHAUS_TRACE) run still sees provider switches. The durable
|
|
1743
|
+
// half landed in 0.6.0 (design §8.1): `attachRoutingPersistence` below
|
|
1744
|
+
// mirrors every `model_failover` into the session JSONL as the event-log
|
|
1745
|
+
// kind of the same name, so this note is the live surface and the JSONL
|
|
1746
|
+
// line is the offline one.
|
|
1271
1747
|
let failoverNoteUnsubscribe;
|
|
1272
1748
|
if (failoverChain !== undefined) {
|
|
1273
1749
|
failoverNoteUnsubscribe = bus.subscribe((event) => {
|
|
@@ -1283,25 +1759,132 @@ export async function runChatLoop(opts) {
|
|
|
1283
1759
|
// cost-tracker. The pre-turn check (`enforceBudget`, below) reads
|
|
1284
1760
|
// `budgetMeter.getRunCost(runId).totalUsdMicros`. `budgetDegraded` gates
|
|
1285
1761
|
// the ladder to a single rung: once we degrade, we never degrade again —
|
|
1286
|
-
//
|
|
1287
|
-
|
|
1762
|
+
// the run stops at the next turn boundary.
|
|
1763
|
+
// 0.6.0 — `budget.scope: "session"` needs the per-call spend ON DISK so the
|
|
1764
|
+
// next resume can seed from it. When an env-attached cost-tracker already
|
|
1765
|
+
// publishes `cost_accrual` (persisted below) the meter stays silent as
|
|
1766
|
+
// before; otherwise the meter itself publishes, and the persist subscriber
|
|
1767
|
+
// below is armed for it. `run`-scoped budgets are byte-identical to 0.5.x.
|
|
1768
|
+
const budgetSessionPersist = budgetSessionScoped && subscribers.costTracker === undefined;
|
|
1769
|
+
// 0.6.0 §7.12 — a caller-supplied meter (the workflow bundle's run-spanning
|
|
1770
|
+
// tracker) is used as-is and outlives this loop; otherwise the per-loop
|
|
1771
|
+
// meter, exactly as before.
|
|
1772
|
+
const ownBudgetMeter = opts.budgetMeter === undefined && opts.budget !== undefined
|
|
1773
|
+
? createCostTracker(bus, { suppressEvents: !budgetSessionPersist })
|
|
1774
|
+
: undefined;
|
|
1775
|
+
const budgetMeter = opts.budgetMeter ?? ownBudgetMeter;
|
|
1776
|
+
// 0.6.0 §6.2 — `budget.judge_share`: the sub-cap on AUXILIARY-role spend
|
|
1777
|
+
// (judge, compaction, guide, classifier, consult, committee, shadow) inside
|
|
1778
|
+
// the run cap. The meter already splits priced spend by `role`, so the
|
|
1779
|
+
// share is a read over `byRole`. Exhaustion is an accounting signal here —
|
|
1780
|
+
// one stderr notice per meter, plus `reason: "judge_share_exhausted"` on
|
|
1781
|
+
// every `eval_graded` published while the share stands exhausted — never a
|
|
1782
|
+
// stop (the total cap is the stop); the cascade behaviour consuming it
|
|
1783
|
+
// lands with the hybrid strategies. The read rides EVERY budget gate (see
|
|
1784
|
+
// `enforceBudget`), not only the in-loop grade, so a run without
|
|
1785
|
+
// `evaluation:` still surfaces the signal.
|
|
1786
|
+
const judgeShareMicros = opts.budget !== undefined
|
|
1787
|
+
? Math.round(opts.budget.usdMicros * (opts.budget.judgeShare ?? DEFAULT_JUDGE_SHARE))
|
|
1788
|
+
: undefined;
|
|
1789
|
+
const auxiliarySpentMicros = () => budgetMeter === undefined
|
|
1790
|
+
? 0
|
|
1791
|
+
: sumRoleCost(budgetMeter.getRunCost(bus.runId), AUXILIARY_MODEL_ROLES);
|
|
1792
|
+
/** True once auxiliary spend has reached the share. Re-read on demand so a
|
|
1793
|
+
* reader (every budget gate and the eval gate today, the cascade later)
|
|
1794
|
+
* sees the live state; the notice prints once per meter. */
|
|
1795
|
+
const judgeShareExhausted = () => {
|
|
1796
|
+
if (judgeShareMicros === undefined || budgetMeter === undefined)
|
|
1797
|
+
return false;
|
|
1798
|
+
const aux = auxiliarySpentMicros();
|
|
1799
|
+
if (aux < judgeShareMicros)
|
|
1800
|
+
return false;
|
|
1801
|
+
if (!judgeShareNoticedMeters.has(budgetMeter)) {
|
|
1802
|
+
judgeShareNoticedMeters.add(budgetMeter);
|
|
1803
|
+
const share = opts.budget?.judgeShare ?? DEFAULT_JUDGE_SHARE;
|
|
1804
|
+
process.stderr.write(`[budget] auxiliary spend $${(aux / 1_000_000).toFixed(4)} ≥ judge_share $${(judgeShareMicros / 1_000_000).toFixed(4)} (${share} of the $${((opts.budget?.usdMicros ?? 0) / 1_000_000).toFixed(4)} cap) — judge_share_exhausted; judging continues under the total cap.\n`);
|
|
1805
|
+
}
|
|
1806
|
+
return true;
|
|
1807
|
+
};
|
|
1288
1808
|
let budgetDegraded = false;
|
|
1289
1809
|
/**
|
|
1290
|
-
*
|
|
1291
|
-
*
|
|
1292
|
-
*
|
|
1293
|
-
*
|
|
1294
|
-
*
|
|
1295
|
-
*
|
|
1296
|
-
|
|
1297
|
-
|
|
1810
|
+
* 0.6.0 §4.2 — the SINGLE-MODEL degrade rung's plan, built once when the
|
|
1811
|
+
* rung takes over (`enforceBudget`'s non-pool branch) and read by the
|
|
1812
|
+
* per-call plan selection from then on, so the degrade profile's pinned
|
|
1813
|
+
* request params serve instead of the primary's. Undefined until a degrade
|
|
1814
|
+
* fires — and always undefined under a pool, where degrade is an
|
|
1815
|
+
* ELIGIBILITY restriction and the rung's own candidate plan already serves.
|
|
1816
|
+
*/
|
|
1817
|
+
let degradedPlan;
|
|
1818
|
+
// 0.6.0 — the turn in which the degrade fired. Spend never falls back
|
|
1819
|
+
// under the cap, so "a later breach on the degraded rung" cannot be an
|
|
1820
|
+
// accrual boundary (the rung's first priced response would trip the very
|
|
1821
|
+
// next per-call gate and sever the turn — degrade would collapse to one
|
|
1822
|
+
// model call). It is a TURN boundary: every gate of this turn lets the
|
|
1823
|
+
// rung serve; the first gate of the next turn stops the run.
|
|
1824
|
+
let budgetDegradedAtTurn = 0;
|
|
1825
|
+
/**
|
|
1826
|
+
* 0.6.0 §7.12 — the READ-ONLY twin of `enforceBudget` for the post-answer
|
|
1827
|
+
* shadow: true when the total cap is reached and this is not the turn a
|
|
1828
|
+
* `degrade` fired in (the rung serves every remaining call of that turn).
|
|
1829
|
+
* The shadow cannot call `enforceBudget` itself — a `degrade` breach is a
|
|
1830
|
+
* TURN-BOUNDARY latch, and firing it from the audition lane after the
|
|
1831
|
+
* answer would consume the degraded turn (the next pre-turn gate would
|
|
1832
|
+
* then stop instead of serving one degraded turn). Auxiliary spend past
|
|
1833
|
+
* the cap is skipped, never opened.
|
|
1834
|
+
*/
|
|
1835
|
+
const totalCapReached = (turn) => {
|
|
1836
|
+
if (opts.budget === undefined || budgetMeter === undefined)
|
|
1837
|
+
return false;
|
|
1838
|
+
const spent = resumedCostUsdMicros + budgetMeter.getRunCost(bus.runId).totalUsdMicros;
|
|
1839
|
+
if (spent < opts.budget.usdMicros)
|
|
1840
|
+
return false;
|
|
1841
|
+
return !(budgetDegraded && turn === budgetDegradedAtTurn);
|
|
1842
|
+
};
|
|
1843
|
+
/**
|
|
1844
|
+
* Item 27 / 0.6.0 §7.12 — the budget gate. Returns "stop" when the run
|
|
1845
|
+
* must end; "continue" otherwise. Called from TWO sites: the REPL's
|
|
1846
|
+
* pre-turn check (unchanged — an idle run ends cleanly) and, since 0.6.0,
|
|
1847
|
+
* the per-model-call gate at the top of `NeedModel`, so a tool loop that
|
|
1848
|
+
* keeps calling the model is stopped at the cap on every host. `spent`
|
|
1849
|
+
* is this run's metered spend plus, under `scope: "session"`, the spend
|
|
1850
|
+
* the session log persisted before this resume. `turn` is the 1-based
|
|
1851
|
+
* number of the turn being gated: the per-call gate passes the current
|
|
1852
|
+
* `runContext.turnNumber`; the pre-turn gate runs BEFORE the increment,
|
|
1853
|
+
* so it passes `turnNumber + 1` (the turn about to open).
|
|
1854
|
+
*
|
|
1855
|
+
* On a `degrade` breach (once), the rung serves the rest of `turn`:
|
|
1856
|
+
* - single-model / `modelTiers` runs re-resolve the primary to the
|
|
1857
|
+
* cheaper rung IN PLACE (mutating `adapter`, `providerId`,
|
|
1858
|
+
* `wireModelId`; the tier router is bypassed from here on so the swap
|
|
1859
|
+
* actually serves) and emit `model_failover` (reason
|
|
1860
|
+
* `budget_degrade`) — the event is published only on this path, where
|
|
1861
|
+
* the swap really takes effect;
|
|
1862
|
+
* - `modelPool` runs latch `budgetDegraded` and leave the adapter alone:
|
|
1863
|
+
* the pool branch then forces the pre-resolved `budgetDegradeRung`
|
|
1864
|
+
* (`model_route` policy `forced`, reason `budget_degrade`). No
|
|
1865
|
+
* `model_failover` is published — nothing failed over.
|
|
1866
|
+
* Every gate of that turn returns "continue" however far past the cap the
|
|
1867
|
+
* rung accrues; the first gate of the next turn stops (on the REPL that is
|
|
1868
|
+
* the pre-turn gate — a clean exit 0; a singleTurn run has no next turn,
|
|
1869
|
+
* so the rung serves the rest of the run).
|
|
1870
|
+
* Never throws — a re-resolution failure logs and stops.
|
|
1298
1871
|
*/
|
|
1299
|
-
const enforceBudget = async () => {
|
|
1872
|
+
const enforceBudget = async (turn) => {
|
|
1300
1873
|
if (opts.budget === undefined || budgetMeter === undefined)
|
|
1301
1874
|
return "continue";
|
|
1302
|
-
|
|
1875
|
+
// 0.6.0 §6.2 — the judge_share read rides every gate, so the one-time
|
|
1876
|
+
// notice fires on ANY budgeted run once auxiliary spend crosses the
|
|
1877
|
+
// share — a compaction-only run, or a workflow whose `kind: judge` gates
|
|
1878
|
+
// run between steps on the shared meter — not only when an in-loop
|
|
1879
|
+
// `evaluation:` grade happens to read it. Accounting only: the share
|
|
1880
|
+
// never stops or degrades; the total cap below does.
|
|
1881
|
+
judgeShareExhausted();
|
|
1882
|
+
const spent = resumedCostUsdMicros + budgetMeter.getRunCost(bus.runId).totalUsdMicros;
|
|
1303
1883
|
if (spent < opts.budget.usdMicros)
|
|
1304
1884
|
return "continue";
|
|
1885
|
+
// Degraded in THIS turn: the rung serves every remaining call of it.
|
|
1886
|
+
if (budgetDegraded && turn === budgetDegradedAtTurn)
|
|
1887
|
+
return "continue";
|
|
1305
1888
|
const spentUsd = (spent / 1_000_000).toFixed(4);
|
|
1306
1889
|
const capUsd = (opts.budget.usdMicros / 1_000_000).toFixed(4);
|
|
1307
1890
|
if (opts.budget.onExceed.kind === "stop" || budgetDegraded) {
|
|
@@ -1311,6 +1894,17 @@ export async function runChatLoop(opts) {
|
|
|
1311
1894
|
}
|
|
1312
1895
|
// degrade: re-resolve the primary model to the cheaper rung, once.
|
|
1313
1896
|
const target = opts.budget.onExceed.model;
|
|
1897
|
+
if (poolRouter !== undefined) {
|
|
1898
|
+
// Pool-aware degrade — restrict eligibility to the rung; never swap.
|
|
1899
|
+
if (budgetDegradeRung === undefined) {
|
|
1900
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — degrade to "${target}" failed (${budgetDegradeRungError ?? "rung unresolved"}); ending the run.\n`);
|
|
1901
|
+
return "stop";
|
|
1902
|
+
}
|
|
1903
|
+
budgetDegraded = true;
|
|
1904
|
+
budgetDegradedAtTurn = turn;
|
|
1905
|
+
process.stderr.write(`[budget] $${spentUsd} spent ≥ $${capUsd} cap — restricting model_pool to ${target} (budget_degrade).\n`);
|
|
1906
|
+
return "continue";
|
|
1907
|
+
}
|
|
1314
1908
|
try {
|
|
1315
1909
|
const degraded = opts._budgetDegradeAdapter !== undefined
|
|
1316
1910
|
? {
|
|
@@ -1333,11 +1927,18 @@ export async function runChatLoop(opts) {
|
|
|
1333
1927
|
providerId = degraded.providerId;
|
|
1334
1928
|
wireModelId = degraded.modelId;
|
|
1335
1929
|
degradedSpecModel = target;
|
|
1930
|
+
// 0.6.0 §4.2 — the degrade rung serves on its OWN plan from here on:
|
|
1931
|
+
// the profile's pinned params replace the primary's, exactly as a pool
|
|
1932
|
+
// candidate's would. Built once (a degrade never re-enters), off the
|
|
1933
|
+
// primary plan so the toolset, permissions and rate buckets stay the
|
|
1934
|
+
// run's — only the model identity and the request params move.
|
|
1935
|
+
degradedPlan = degradePlanFrom(primaryPlan, target, degraded.modelId, opts.budget.onExceed.params);
|
|
1336
1936
|
// A degrade takes over from the failover chain: subsequent calls hit
|
|
1337
1937
|
// the degraded adapter directly (v1 keeps one rung, no re-entry to the
|
|
1338
1938
|
// chain). Null it so the request/response stamping stops consulting it.
|
|
1339
1939
|
failoverChain = undefined;
|
|
1340
1940
|
budgetDegraded = true;
|
|
1941
|
+
budgetDegradedAtTurn = turn;
|
|
1341
1942
|
bus.publish({
|
|
1342
1943
|
...bus.envelope(),
|
|
1343
1944
|
kind: "model_failover",
|
|
@@ -1393,7 +1994,9 @@ export async function runChatLoop(opts) {
|
|
|
1393
1994
|
mdRenderer.end();
|
|
1394
1995
|
out.write("\n");
|
|
1395
1996
|
};
|
|
1396
|
-
const eventLog =
|
|
1997
|
+
const eventLog = persistSession
|
|
1998
|
+
? await openEventLog(sessionId, { rootDir: sessionRootDir })
|
|
1999
|
+
: createMemoryEventLog();
|
|
1397
2000
|
// #405 — reconcile the session's recorded toolset with THIS run's.
|
|
1398
2001
|
//
|
|
1399
2002
|
// The failure this closes: a session parked on build N enumerates its
|
|
@@ -1411,6 +2014,51 @@ export async function runChatLoop(opts) {
|
|
|
1411
2014
|
// turn. Sessions from before this record exist get no marker on their first
|
|
1412
2015
|
// resume — there is nothing to compare against — and are covered from then
|
|
1413
2016
|
// on.
|
|
2017
|
+
// 0.6.0 §4.4 / §5 — the subset-only advertisement per pool candidate:
|
|
2018
|
+
// shape tools → profile `tools` subset → capability filter (a tool whose
|
|
2019
|
+
// `requiresModelFeatures` the candidate cannot satisfy is dropped), with the
|
|
2020
|
+
// auto-registered loop tools and the hybrid pair surviving `tools: []`
|
|
2021
|
+
// unless the profile's `permissions.deny` names them. Computed once here —
|
|
2022
|
+
// it depends only on the effective tool list, the candidate's profile and
|
|
2023
|
+
// its adapter's features — and consumed twice: by the `toolset` record just
|
|
2024
|
+
// below and by the plan table further down.
|
|
2025
|
+
const retainedToolNames = [
|
|
2026
|
+
...RETAINED_LOOP_TOOL_NAMES,
|
|
2027
|
+
...(opts.hybridTools ?? []).map((t) => t.name),
|
|
2028
|
+
];
|
|
2029
|
+
const candidateFeatures = (candidate, cfg) => {
|
|
2030
|
+
const declared = cfg?.capabilities;
|
|
2031
|
+
if (declared === undefined)
|
|
2032
|
+
return candidate.adapter.features;
|
|
2033
|
+
return {
|
|
2034
|
+
...candidate.adapter.features,
|
|
2035
|
+
...(declared.tool_use !== undefined ? { tool_use: declared.tool_use } : {}),
|
|
2036
|
+
...(declared.vision !== undefined ? { vision: declared.vision } : {}),
|
|
2037
|
+
...(declared.thinking !== undefined ? { thinking: declared.thinking } : {}),
|
|
2038
|
+
...(declared.web_search !== undefined ? { web_search: declared.web_search } : {}),
|
|
2039
|
+
...(declared.caching !== undefined ? { caching: declared.caching } : {}),
|
|
2040
|
+
};
|
|
2041
|
+
};
|
|
2042
|
+
const candidateAdvertisements = new Map();
|
|
2043
|
+
for (const candidate of poolRouter?.candidates() ?? []) {
|
|
2044
|
+
const cfg = poolCandidateConfigs.get(candidate);
|
|
2045
|
+
candidateAdvertisements.set(candidate, buildAdvertisement(effectiveTools, {
|
|
2046
|
+
...(cfg?.tools !== undefined ? { tools: cfg.tools } : {}),
|
|
2047
|
+
...(cfg?.permissions !== undefined ? { permissions: cfg.permissions } : {}),
|
|
2048
|
+
}, {
|
|
2049
|
+
capabilities: {
|
|
2050
|
+
features: candidateFeatures(candidate, cfg),
|
|
2051
|
+
...(cfg?.capabilities?.contextWindow !== undefined
|
|
2052
|
+
? { contextWindow: cfg.capabilities.contextWindow }
|
|
2053
|
+
: {}),
|
|
2054
|
+
...(cfg?.capabilities?.maxOutputTokens !== undefined
|
|
2055
|
+
? { maxOutputTokens: cfg.capabilities.maxOutputTokens }
|
|
2056
|
+
: {}),
|
|
2057
|
+
},
|
|
2058
|
+
retain: retainedToolNames,
|
|
2059
|
+
}));
|
|
2060
|
+
}
|
|
2061
|
+
const armIdOf = (candidate) => poolCandidateConfigs.get(candidate)?.profile ?? candidate.modelString;
|
|
1414
2062
|
{
|
|
1415
2063
|
const currentToolNames = [...effectiveTools.map((t) => t.name)].sort();
|
|
1416
2064
|
const prior = resumedToolNames === undefined ? undefined : [...resumedToolNames].sort();
|
|
@@ -1422,18 +2070,31 @@ export async function runChatLoop(opts) {
|
|
|
1422
2070
|
const removed = prior.filter((n) => !currentToolNames.includes(n));
|
|
1423
2071
|
const delta = [...added.map((n) => `+${n}`), ...removed.map((n) => `−${n}`)].join(", ");
|
|
1424
2072
|
const marker = `[system] The toolset changed while this session was idle: ${delta}. The tool schema attached to this request is the authoritative list — any toolset enumerated earlier in this conversation is stale. Call ListTools to verify.`;
|
|
1425
|
-
resumedMessages.push({ role: "user", content: marker });
|
|
2073
|
+
resumedMessages.push(markSynthetic({ role: "user", content: marker }));
|
|
1426
2074
|
await eventLog.append({
|
|
1427
2075
|
kind: "user_message",
|
|
1428
2076
|
payload: { content: marker, synthetic: true, toolset: { added, removed } },
|
|
1429
2077
|
});
|
|
1430
2078
|
}
|
|
1431
2079
|
if (prior === undefined || changed) {
|
|
2080
|
+
// 0.6.0 §5.3 — the record is keyed on the UNION the run advertises, so a
|
|
2081
|
+
// pool whose candidates see different subsets never fires "toolset
|
|
2082
|
+
// changed" on resume just because a different candidate served. The
|
|
2083
|
+
// per-candidate subsets ride along as `candidates` (arm id → sorted
|
|
2084
|
+
// names) only when at least one candidate is narrower than the union,
|
|
2085
|
+
// so a pool of bare candidates writes the pre-0.6.0 line byte-for-byte.
|
|
2086
|
+
const narrowed = {};
|
|
2087
|
+
for (const [candidate, ad] of candidateAdvertisements) {
|
|
2088
|
+
if (ad.names.size === currentToolNames.length)
|
|
2089
|
+
continue;
|
|
2090
|
+
narrowed[armIdOf(candidate)] = [...ad.names].sort();
|
|
2091
|
+
}
|
|
1432
2092
|
await eventLog.append({
|
|
1433
2093
|
kind: "toolset",
|
|
1434
2094
|
payload: {
|
|
1435
2095
|
toolNames: currentToolNames,
|
|
1436
2096
|
...(opts.toolsetScope !== undefined ? { scope: opts.toolsetScope } : {}),
|
|
2097
|
+
...(Object.keys(narrowed).length > 0 ? { candidates: narrowed } : {}),
|
|
1437
2098
|
},
|
|
1438
2099
|
});
|
|
1439
2100
|
}
|
|
@@ -1442,17 +2103,20 @@ export async function runChatLoop(opts) {
|
|
|
1442
2103
|
// the session JSONL so `crewhaus cost-summary --session <id>` can sum spend
|
|
1443
2104
|
// after the run. Gated on the same switch that attaches the cost-tracker
|
|
1444
2105
|
// (CREWHAUS_COST_TRACKING): no tracker → no accruals to persist, and the
|
|
1445
|
-
// message-only transcript stays unchanged by default. Per-call accruals
|
|
1446
|
-
//
|
|
1447
|
-
//
|
|
1448
|
-
//
|
|
2106
|
+
// message-only transcript stays unchanged by default. Per-call accruals plus
|
|
2107
|
+
// (0.6.0 §10.2) the ROLE-bearing `summary` roll-ups a nested run re-publishes
|
|
2108
|
+
// on this bus (a sub-agent's `role: "subagent"` total — spend this log has no
|
|
2109
|
+
// per-call lines for); the FR-003 terminal role-less `summary` aggregate is
|
|
2110
|
+
// still skipped so a run total never double-counts. `append()` writes
|
|
2111
|
+
// synchronously, and a persist failure must never abort a turn, so the
|
|
2112
|
+
// result is logged rather than thrown.
|
|
1449
2113
|
let costPersistUnsubscribe;
|
|
1450
|
-
if (subscribers.costTracker !== undefined) {
|
|
2114
|
+
if (subscribers.costTracker !== undefined || budgetSessionPersist) {
|
|
1451
2115
|
costPersistUnsubscribe = bus.subscribe((event) => {
|
|
1452
2116
|
if (event.kind !== "cost_accrual")
|
|
1453
2117
|
return;
|
|
1454
2118
|
const ev = event;
|
|
1455
|
-
if (ev.summary === true)
|
|
2119
|
+
if (ev.summary === true && ev.role === undefined)
|
|
1456
2120
|
return;
|
|
1457
2121
|
void eventLog
|
|
1458
2122
|
.append({
|
|
@@ -1460,6 +2124,7 @@ export async function runChatLoop(opts) {
|
|
|
1460
2124
|
payload: {
|
|
1461
2125
|
provider: ev.provider,
|
|
1462
2126
|
modelId: ev.modelId,
|
|
2127
|
+
...(ev.summary === true ? { summary: true } : {}),
|
|
1463
2128
|
...(ev.specModel !== undefined ? { specModel: ev.specModel } : {}),
|
|
1464
2129
|
inputTokens: ev.inputTokens,
|
|
1465
2130
|
outputTokens: ev.outputTokens,
|
|
@@ -1471,6 +2136,15 @@ export async function runChatLoop(opts) {
|
|
|
1471
2136
|
: {}),
|
|
1472
2137
|
costUsdMicros: ev.costUsdMicros,
|
|
1473
2138
|
...(ev.tenantId !== undefined ? { tenantId: ev.tenantId } : {}),
|
|
2139
|
+
// 0.6.0 (design §8.1) — attribution persists only when present,
|
|
2140
|
+
// so an unattributed run's cost lines stay byte-identical.
|
|
2141
|
+
...(ev.role !== undefined ? { role: ev.role } : {}),
|
|
2142
|
+
...(ev.stage !== undefined ? { stage: ev.stage } : {}),
|
|
2143
|
+
...(ev.profile !== undefined ? { profile: ev.profile } : {}),
|
|
2144
|
+
...(ev.paramsFingerprint !== undefined
|
|
2145
|
+
? { paramsFingerprint: ev.paramsFingerprint }
|
|
2146
|
+
: {}),
|
|
2147
|
+
...(ev.effectiveParams !== undefined ? { effectiveParams: ev.effectiveParams } : {}),
|
|
1474
2148
|
},
|
|
1475
2149
|
})
|
|
1476
2150
|
.catch((err) => {
|
|
@@ -1492,11 +2166,37 @@ export async function runChatLoop(opts) {
|
|
|
1492
2166
|
// score per-server MCP health offline. Shares the advisor's DEFAULT-ON gate
|
|
1493
2167
|
// (CREWHAUS_ADVISOR_EVENTS=0 disables both). See observability.ts.
|
|
1494
2168
|
const mcpStatsPersist = attachMcpStatsPersistence(bus, eventLog, runContext);
|
|
2169
|
+
// 0.6.0 (design §8.1) — persist the routing decisions that were bus-only
|
|
2170
|
+
// (`model_failover` from the chain or a budget degrade, and the new
|
|
2171
|
+
// `model_stage` / `model_directive` / `judge_verdict` kinds) into the same
|
|
2172
|
+
// session JSONL as the `model_route` line, ungated, so `route explain` and
|
|
2173
|
+
// `--resume` read them offline. See observability.ts.
|
|
2174
|
+
const routingPersist = attachRoutingPersistence(bus, eventLog, runContext);
|
|
2175
|
+
// 0.6.0 (design §8.1) — the attribution stamped on every model_request /
|
|
2176
|
+
// model_response this loop publishes: pass-through of the caller's
|
|
2177
|
+
// `modelRole` / `modelStage` (a nested consult/guide/committee/shadow loop
|
|
2178
|
+
// or a sub-agent child); empty — and therefore byte-identical — on the
|
|
2179
|
+
// main loop, whose absent role reads as "primary".
|
|
2180
|
+
// 0.6.0 §7.3 — a cascade turn overrides the role / stage per ATTEMPT
|
|
2181
|
+
// (`draft` on the graded draft, `escalation` on the forced re-run and on the
|
|
2182
|
+
// rest of a turn after a self-`Escalate`); `runOneTurn` sets it on entry
|
|
2183
|
+
// and clears it on return, so the loop's own defaults stand otherwise.
|
|
2184
|
+
let turnAttributionOverride = {};
|
|
2185
|
+
const modelAttribution = () => {
|
|
2186
|
+
const role = turnAttributionOverride.role ?? opts.modelRole;
|
|
2187
|
+
const stage = turnAttributionOverride.stage ?? opts.modelStage;
|
|
2188
|
+
return {
|
|
2189
|
+
...(role !== undefined ? { role } : {}),
|
|
2190
|
+
...(stage !== undefined ? { stage } : {}),
|
|
2191
|
+
};
|
|
2192
|
+
};
|
|
1495
2193
|
// "Watch me" — live capture tap (CREWHAUS_WATCHME-gated, off by default):
|
|
1496
2194
|
// append the bus-only TraceEvent kinds to the `sessions/<id>.events.jsonl`
|
|
1497
2195
|
// sibling that `sessions export` already reads. Mirrored/ephemeral kinds are
|
|
1498
2196
|
// skipped, so the sibling stays metadata-grade. See observability.ts.
|
|
1499
|
-
const watchmeCapture =
|
|
2197
|
+
const watchmeCapture = persistSession
|
|
2198
|
+
? attachWatchmeCapture(bus, sessionRootDir ?? DEFAULT_ROOT_DIR, sessionId)
|
|
2199
|
+
: undefined;
|
|
1500
2200
|
// Per-run state container — coordination surface for hooks/skills/tools.
|
|
1501
2201
|
// Shipped as plumbing in Section 10; v0.3.0 Goal 1 (§2.5) lands its first
|
|
1502
2202
|
// consumer: plan-mutating tools set `PLAN_DIRTY_STATE_KEY` here (via the
|
|
@@ -1745,9 +2445,20 @@ export async function runChatLoop(opts) {
|
|
|
1745
2445
|
};
|
|
1746
2446
|
/** The extra `maybeCompact` args (Item 1 curation + Item 4 real tokens),
|
|
1747
2447
|
* recomputed per call so `realInputTokens` reflects the newest response. */
|
|
2448
|
+
// 0.6.0 §6.2 — the compaction slot's SPEC string, for the `specModel`
|
|
2449
|
+
// attribution on its metered side-calls (the wire id is
|
|
2450
|
+
// `compactionWireModelId`). With no explicit `compactionModel` the slot IS
|
|
2451
|
+
// the agent's own model.
|
|
2452
|
+
const compactionSpecModel = opts.compactionModel ?? opts.model;
|
|
1748
2453
|
const compactionExtras = () => ({
|
|
1749
2454
|
...(lastModelInputTokens !== undefined ? { realInputTokens: lastModelInputTokens } : {}),
|
|
1750
2455
|
...(curateConfig !== undefined ? { curate: curateConfig, onCurate: publishCurate } : {}),
|
|
2456
|
+
// 0.6.0 §6.2 — the pre-turn autocompact summary publishes on the run bus
|
|
2457
|
+
// with role "compaction", so it is priced and counted toward `budget`.
|
|
2458
|
+
bus,
|
|
2459
|
+
compactionSpecModel,
|
|
2460
|
+
// 0.6.0 §4.2 — the compaction profile's pinned request params.
|
|
2461
|
+
...(opts.compactionParams !== undefined ? { compactionParams: opts.compactionParams } : {}),
|
|
1751
2462
|
});
|
|
1752
2463
|
const permissionMode = opts.permissionMode ?? "default";
|
|
1753
2464
|
// #383 — merge the harness's `.crewhaus/settings.json` rules into the
|
|
@@ -1968,6 +2679,9 @@ export async function runChatLoop(opts) {
|
|
|
1968
2679
|
// with the gate flag reflecting the ADVERTISED contract (a runtime-injected
|
|
1969
2680
|
// justification is part of that contract even though the tool's own schema
|
|
1970
2681
|
// never declared it).
|
|
2682
|
+
// 0.6.0 §4.4 — re-pointed at the SERVING candidate's summaries on every
|
|
2683
|
+
// plan selection below, so `ListTools` under a narrowed profile lists
|
|
2684
|
+
// exactly that profile's tools and never misreports.
|
|
1971
2685
|
listToolsSummaries = advertisedTools.map(({ tool, advertised }) => ({
|
|
1972
2686
|
name: tool.name,
|
|
1973
2687
|
description: tool.description,
|
|
@@ -2036,6 +2750,302 @@ export async function runChatLoop(opts) {
|
|
|
2036
2750
|
}
|
|
2037
2751
|
const hasToolRateBucket = (toolName) => opts.rateLimits !== undefined &&
|
|
2038
2752
|
(opts.rateLimits[toolName] !== undefined || opts.rateLimits["*"] !== undefined);
|
|
2753
|
+
// -------------------------------------------------------------------------
|
|
2754
|
+
// 0.6.0 §4.4 — the per-candidate plan table. One plan per enabled pool
|
|
2755
|
+
// candidate (plus the degrade rung when it sits outside the roster), built
|
|
2756
|
+
// ONCE here from the run-level seams above; `primaryPlan` is the
|
|
2757
|
+
// single-model path's plan and reproduces today's boot constants exactly.
|
|
2758
|
+
// `servingPlan` is the turn-scoped selection every hot path below reads —
|
|
2759
|
+
// set once per model call after the routing if-else chain.
|
|
2760
|
+
// -------------------------------------------------------------------------
|
|
2761
|
+
const namesFingerprint = (names) => planFingerprint([...names].sort());
|
|
2762
|
+
const buildPlan = (candidate, cfg) => {
|
|
2763
|
+
const fromPool = candidate !== undefined && cfg !== undefined;
|
|
2764
|
+
const ad = candidate !== undefined ? candidateAdvertisements.get(candidate) : undefined;
|
|
2765
|
+
const planAdvertised = ad !== undefined
|
|
2766
|
+
? advertisedTools.filter(({ tool }) => ad.names.has(tool.name))
|
|
2767
|
+
: advertisedTools;
|
|
2768
|
+
const planAnthropicTools = ad === undefined
|
|
2769
|
+
? anthropicTools
|
|
2770
|
+
: planAdvertised.length > 0
|
|
2771
|
+
? planAdvertised.map(({ tool, advertised }) => ({
|
|
2772
|
+
name: tool.name,
|
|
2773
|
+
description: tool.description,
|
|
2774
|
+
input_schema: advertised.inputSchema,
|
|
2775
|
+
}))
|
|
2776
|
+
: undefined;
|
|
2777
|
+
// Exclusivity (§4.1): `temperature` and `thinking` never ride one
|
|
2778
|
+
// request. A candidate that pins one drops the RUN's other; a candidate
|
|
2779
|
+
// pinning neither inherits both run-level values as before.
|
|
2780
|
+
const baseThinking = cfg?.temperature !== undefined && cfg.thinking === undefined ? undefined : opts.thinking;
|
|
2781
|
+
const baseTemperature = cfg?.thinking !== undefined && cfg.temperature === undefined ? undefined : opts.temperature;
|
|
2782
|
+
const params = fromPool
|
|
2783
|
+
? buildRequestParams(cfg, {
|
|
2784
|
+
maxTokens,
|
|
2785
|
+
...(baseThinking !== undefined ? { thinking: baseThinking } : {}),
|
|
2786
|
+
...(baseTemperature !== undefined ? { temperature: baseTemperature } : {}),
|
|
2787
|
+
}, cfg.capabilities?.maxOutputTokens !== undefined
|
|
2788
|
+
? { maxOutputTokens: cfg.capabilities.maxOutputTokens }
|
|
2789
|
+
: {})
|
|
2790
|
+
: {
|
|
2791
|
+
maxTokens,
|
|
2792
|
+
effectiveMaxTokens,
|
|
2793
|
+
...(thinkingRequest !== undefined ? thinkingRequest : {}),
|
|
2794
|
+
...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
|
|
2795
|
+
};
|
|
2796
|
+
const deny = cfg?.permissions?.deny ?? [];
|
|
2797
|
+
const ask = cfg?.permissions?.ask ?? [];
|
|
2798
|
+
const rateEntries = Object.entries(cfg?.rateLimits ?? {});
|
|
2799
|
+
const armId = fromPool ? (cfg.profile ?? cfg.model) : (candidate?.modelString ?? opts.model);
|
|
2800
|
+
let planLimiter;
|
|
2801
|
+
if (rateEntries.length > 0) {
|
|
2802
|
+
// Bucket ids `tool:<name>@<armId>` (the `"*"` entry becomes the
|
|
2803
|
+
// limiter's per-dimension default, so every unlisted tool the profile
|
|
2804
|
+
// caps still gets its OWN bucket keyed on `<name>@<armId>`).
|
|
2805
|
+
const buckets = new Map();
|
|
2806
|
+
for (const [name, limit] of rateEntries) {
|
|
2807
|
+
buckets.set(name === "*" ? "tool:*" : `tool:${name}@${armId}`, {
|
|
2808
|
+
kind: "token-bucket",
|
|
2809
|
+
capacity: limit.burst ?? limit.rpm,
|
|
2810
|
+
refillPerSec: limit.rpm / 60,
|
|
2811
|
+
});
|
|
2812
|
+
}
|
|
2813
|
+
planLimiter = createRateLimiter({ buckets });
|
|
2814
|
+
}
|
|
2815
|
+
const planRateLimits = cfg?.rateLimits;
|
|
2816
|
+
const names = new Set(planAdvertised.map(({ tool }) => tool.name));
|
|
2817
|
+
return {
|
|
2818
|
+
armId,
|
|
2819
|
+
...(cfg?.profile !== undefined ? { profile: cfg.profile } : {}),
|
|
2820
|
+
modelString: candidate?.modelString ?? opts.model,
|
|
2821
|
+
wireModelId: candidate?.modelId ?? wireModelId,
|
|
2822
|
+
fromPool,
|
|
2823
|
+
params,
|
|
2824
|
+
paramsFingerprint: planFingerprint(params),
|
|
2825
|
+
...(cfg?.modelCallTimeoutMs !== undefined
|
|
2826
|
+
? { modelCallTimeoutMs: cfg.modelCallTimeoutMs }
|
|
2827
|
+
: {}),
|
|
2828
|
+
anthropicTools: planAnthropicTools,
|
|
2829
|
+
advertisedNames: names,
|
|
2830
|
+
listToolsSummaries: planAdvertised.map(({ tool, advertised }) => ({
|
|
2831
|
+
name: tool.name,
|
|
2832
|
+
description: tool.description,
|
|
2833
|
+
readOnly: tool.readOnly,
|
|
2834
|
+
destructive: tool.destructive,
|
|
2835
|
+
gated: tool.requireJustification === true || advertised.justificationInjected === true,
|
|
2836
|
+
})),
|
|
2837
|
+
toolByName: new Map(planAdvertised.map(({ tool }) => [tool.name, tool])),
|
|
2838
|
+
toolsetFingerprint: namesFingerprint(names),
|
|
2839
|
+
excludedTools: ad?.excluded.map((e) => e.name) ?? [],
|
|
2840
|
+
permissionRules: fromPool ? narrowRuleSet(permissionRules, deny, ask) : permissionRules,
|
|
2841
|
+
...(planLimiter !== undefined ? { toolRateLimiter: planLimiter } : {}),
|
|
2842
|
+
hasToolRateBucket: (toolName) => planRateLimits !== undefined &&
|
|
2843
|
+
(planRateLimits[toolName] !== undefined || planRateLimits["*"] !== undefined),
|
|
2844
|
+
...(cfg?.toolConfigs !== undefined ? { toolConfigs: cfg.toolConfigs } : {}),
|
|
2845
|
+
...(cfg?.overlay !== undefined
|
|
2846
|
+
? { overlayBlock: { type: "text", text: cfg.overlay } }
|
|
2847
|
+
: {}),
|
|
2848
|
+
cacheMode: cfg?.caching === "off" ? "off" : "prefer",
|
|
2849
|
+
features: candidate !== undefined ? candidateFeatures(candidate, cfg) : adapter.features,
|
|
2850
|
+
...(cfg?.costCapUsdMicros !== undefined ? { costCapUsdMicros: cfg.costCapUsdMicros } : {}),
|
|
2851
|
+
spentUsdMicros: 0,
|
|
2852
|
+
};
|
|
2853
|
+
};
|
|
2854
|
+
const primaryPlan = buildPlan(undefined, undefined);
|
|
2855
|
+
const planByCandidate = new Map();
|
|
2856
|
+
for (const candidate of poolRouter?.candidates() ?? []) {
|
|
2857
|
+
planByCandidate.set(candidate, buildPlan(candidate, poolCandidateConfigs.get(candidate)));
|
|
2858
|
+
}
|
|
2859
|
+
if (budgetDegradeRung !== undefined && !planByCandidate.has(budgetDegradeRung)) {
|
|
2860
|
+
planByCandidate.set(budgetDegradeRung, buildPlan(budgetDegradeRung, undefined));
|
|
2861
|
+
}
|
|
2862
|
+
const planFor = (candidate) => planByCandidate.get(candidate) ?? buildPlan(candidate, poolCandidateConfigs.get(candidate));
|
|
2863
|
+
let servingPlan = primaryPlan;
|
|
2864
|
+
// 0.6.0 §4.4 / §7.7 — the arm that SERVED the last model call, as projected
|
|
2865
|
+
// onto the bridge for children (`ParentRunHandle.routing.served`). Set beside
|
|
2866
|
+
// `servingPlan` on every model call; until the first call it is the primary.
|
|
2867
|
+
// A child inherits it only behind `inheritRouting: true` — `bridge.model`
|
|
2868
|
+
// stays the declared primary.
|
|
2869
|
+
let servedArm = {
|
|
2870
|
+
model: primaryPlan.modelString,
|
|
2871
|
+
wireModelId: primaryPlan.wireModelId,
|
|
2872
|
+
armId: primaryPlan.armId,
|
|
2873
|
+
fromPool: false,
|
|
2874
|
+
};
|
|
2875
|
+
for (const plan of planByCandidate.values()) {
|
|
2876
|
+
if (plan.excludedTools.length === 0 && plan.permissionRules === permissionRules)
|
|
2877
|
+
continue;
|
|
2878
|
+
runContext.logger.info("model_pool candidate plan", {
|
|
2879
|
+
arm: plan.armId,
|
|
2880
|
+
model: plan.modelString,
|
|
2881
|
+
advertised: [...plan.advertisedNames],
|
|
2882
|
+
excluded: plan.excludedTools,
|
|
2883
|
+
narrowedPermissions: plan.permissionRules !== permissionRules,
|
|
2884
|
+
});
|
|
2885
|
+
}
|
|
2886
|
+
/** The `model` / `profile` attribution stamped on tool + permission events while a pool candidate serves. */
|
|
2887
|
+
const toolAttribution = () => servingPlan.fromPool
|
|
2888
|
+
? {
|
|
2889
|
+
model: servingPlan.wireModelId,
|
|
2890
|
+
...(servingPlan.profile !== undefined ? { profile: servingPlan.profile } : {}),
|
|
2891
|
+
}
|
|
2892
|
+
: {};
|
|
2893
|
+
/** The `role` / `profile` / `paramsFingerprint` attribution on model events while a pool candidate serves. */
|
|
2894
|
+
const planAttribution = (plan) => plan.fromPool
|
|
2895
|
+
? {
|
|
2896
|
+
role: turnAttributionOverride.role ?? opts.modelRole ?? "primary",
|
|
2897
|
+
...(plan.profile !== undefined ? { profile: plan.profile } : {}),
|
|
2898
|
+
paramsFingerprint: plan.paramsFingerprint,
|
|
2899
|
+
}
|
|
2900
|
+
: {};
|
|
2901
|
+
/**
|
|
2902
|
+
* 0.6.0 §7.3 / §8.1 — publish one hybrid-strategy stage transition (the
|
|
2903
|
+
* shape metrics / alerts count: an `escalation`-role stage with `outcome:
|
|
2904
|
+
* "started"` is ONE escalation — `isEscalationEvent`). A roster stage (the
|
|
2905
|
+
* cascade rung, a self-escalation) names its `candidate` and the profile is
|
|
2906
|
+
* read off the pool; a side-call stage (guide / shadow / committee, PR 9d)
|
|
2907
|
+
* names the wire `model` and its `profile` directly. Durable through the
|
|
2908
|
+
* loop's `attachRoutingPersistence` mirror into the session JSONL, so no
|
|
2909
|
+
* caller needs its own `logEvent`.
|
|
2910
|
+
*/
|
|
2911
|
+
const publishStage = (stage) => {
|
|
2912
|
+
const profile = stage.profile ??
|
|
2913
|
+
(stage.candidate !== undefined
|
|
2914
|
+
? poolCandidateConfigs.get(stage.candidate)?.profile
|
|
2915
|
+
: undefined);
|
|
2916
|
+
bus.publish({
|
|
2917
|
+
...bus.envelope(),
|
|
2918
|
+
kind: "model_stage",
|
|
2919
|
+
stage: stage.stage,
|
|
2920
|
+
strategy: stage.strategy,
|
|
2921
|
+
role: stage.role,
|
|
2922
|
+
model: stage.candidate?.modelId ?? stage.model ?? "",
|
|
2923
|
+
...(profile !== undefined ? { profile } : {}),
|
|
2924
|
+
outcome: stage.outcome,
|
|
2925
|
+
...(stage.cause !== undefined ? { cause: stage.cause } : {}),
|
|
2926
|
+
...(stage.costUsdMicros !== undefined ? { costUsdMicros: stage.costUsdMicros } : {}),
|
|
2927
|
+
});
|
|
2928
|
+
};
|
|
2929
|
+
/** Fold a served wire model into the session's `models` provenance (one update per new model). */
|
|
2930
|
+
const recordServedModel = async (served) => {
|
|
2931
|
+
const merged = mergeSessionModels(sessionModels, served);
|
|
2932
|
+
if (merged === sessionModels)
|
|
2933
|
+
return;
|
|
2934
|
+
sessionModels = merged;
|
|
2935
|
+
await sessionStore.update(sessionId, { models: merged }).catch((err) => {
|
|
2936
|
+
runContext.logger.warn("session-store: models update failed", {
|
|
2937
|
+
error: err.message,
|
|
2938
|
+
});
|
|
2939
|
+
});
|
|
2940
|
+
};
|
|
2941
|
+
// -------------------------------------------------------------------------
|
|
2942
|
+
// 0.6.0 §7.2.1 — user-directed routing: `/model <$profile|tag>` and
|
|
2943
|
+
// `/model auto`, parsed at the two typed INPUT seams (never from the
|
|
2944
|
+
// transcript) and sticky per session through the additive `pin` field.
|
|
2945
|
+
// -------------------------------------------------------------------------
|
|
2946
|
+
const directivesOn = poolRouter !== undefined && opts.modelPool?.directives === true;
|
|
2947
|
+
/** The roster a directive may name: arm id (profile or model string), model string, tags. */
|
|
2948
|
+
const directiveRoster = (poolRouter?.candidates() ?? []).map((c) => ({
|
|
2949
|
+
armId: armIdOf(c),
|
|
2950
|
+
model: c.modelString,
|
|
2951
|
+
tags: c.tags,
|
|
2952
|
+
}));
|
|
2953
|
+
/** The run's live pin — an arm id on the roster, or none (automatic routing). */
|
|
2954
|
+
let directivePin;
|
|
2955
|
+
/** Publish + persist one `model_directive` (the typed event `route explain` reconstructs the pin from). */
|
|
2956
|
+
const recordDirective = async (ev) => {
|
|
2957
|
+
bus.publish({ ...bus.envelope(), kind: "model_directive", ...ev });
|
|
2958
|
+
await logEvent("model_directive", ev);
|
|
2959
|
+
};
|
|
2960
|
+
const writePin = async (pin) => {
|
|
2961
|
+
await sessionStore.update(sessionId, { pin }).catch((err) => {
|
|
2962
|
+
runContext.logger.warn("session-store: pin update failed", {
|
|
2963
|
+
error: err.message,
|
|
2964
|
+
});
|
|
2965
|
+
});
|
|
2966
|
+
};
|
|
2967
|
+
// A resumed session's pin is UNTRUSTED input: honoured only when directives
|
|
2968
|
+
// are on for this harness AND it names a roster arm; otherwise routing is
|
|
2969
|
+
// automatic and the reason is recorded.
|
|
2970
|
+
if (resumedPin !== undefined && poolRouter !== undefined) {
|
|
2971
|
+
const restored = resolveSessionPin({ pin: resumedPin }, directiveRoster.map((a) => a.armId));
|
|
2972
|
+
if (!directivesOn) {
|
|
2973
|
+
await recordDirective({
|
|
2974
|
+
source: "session",
|
|
2975
|
+
requested: String(resumedPin),
|
|
2976
|
+
accepted: false,
|
|
2977
|
+
reason: "session pin ignored — model_pool.directives is off for this harness",
|
|
2978
|
+
});
|
|
2979
|
+
}
|
|
2980
|
+
else if (restored.pin !== undefined) {
|
|
2981
|
+
directivePin = restored.pin;
|
|
2982
|
+
await recordDirective({
|
|
2983
|
+
source: "session",
|
|
2984
|
+
requested: String(resumedPin),
|
|
2985
|
+
resolved: restored.pin,
|
|
2986
|
+
accepted: true,
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2989
|
+
else {
|
|
2990
|
+
await recordDirective({
|
|
2991
|
+
source: "session",
|
|
2992
|
+
requested: String(resumedPin),
|
|
2993
|
+
accepted: false,
|
|
2994
|
+
...(restored.reason !== undefined ? { reason: restored.reason } : {}),
|
|
2995
|
+
});
|
|
2996
|
+
}
|
|
2997
|
+
}
|
|
2998
|
+
/**
|
|
2999
|
+
* Parse a typed input for a `/model …` directive and apply it. `undefined`
|
|
3000
|
+
* when the input carries none (or the run has no pool — then it is prose).
|
|
3001
|
+
* Otherwise the pin is written / cleared, a `model_directive` is recorded,
|
|
3002
|
+
* and the caller gets the input with the token removed plus a one-line
|
|
3003
|
+
* acknowledgement to print. A target outside the roster is REFUSED with
|
|
3004
|
+
* the roster listed; with `directives: false` the token is left in the
|
|
3005
|
+
* text as prose and only the refusal is recorded.
|
|
3006
|
+
*/
|
|
3007
|
+
const applyDirective = async (text, source) => {
|
|
3008
|
+
if (poolRouter === undefined)
|
|
3009
|
+
return undefined;
|
|
3010
|
+
const d = parseModelDirective(text, directiveRoster);
|
|
3011
|
+
if (d === undefined)
|
|
3012
|
+
return undefined;
|
|
3013
|
+
if (!directivesOn) {
|
|
3014
|
+
const reason = "per-message /model directives are off for this harness (model_pool.directives: false)";
|
|
3015
|
+
await recordDirective({ source, requested: d.requested, accepted: false, reason });
|
|
3016
|
+
return { rest: text, ack: reason, stripped: false };
|
|
3017
|
+
}
|
|
3018
|
+
if (d.kind === "clear") {
|
|
3019
|
+
directivePin = undefined;
|
|
3020
|
+
await writePin(null);
|
|
3021
|
+
await recordDirective({ source, requested: "auto", accepted: true });
|
|
3022
|
+
return {
|
|
3023
|
+
rest: d.rest,
|
|
3024
|
+
ack: "model pin cleared — routing is automatic again",
|
|
3025
|
+
stripped: true,
|
|
3026
|
+
};
|
|
3027
|
+
}
|
|
3028
|
+
if (!d.accepted) {
|
|
3029
|
+
await recordDirective({
|
|
3030
|
+
source,
|
|
3031
|
+
requested: d.requested,
|
|
3032
|
+
accepted: false,
|
|
3033
|
+
...(d.reason !== undefined ? { reason: d.reason } : {}),
|
|
3034
|
+
});
|
|
3035
|
+
return { rest: d.rest, ack: d.reason ?? "directive refused", stripped: true };
|
|
3036
|
+
}
|
|
3037
|
+
const resolved = d.resolved;
|
|
3038
|
+
if (resolved === undefined)
|
|
3039
|
+
return undefined;
|
|
3040
|
+
directivePin = resolved;
|
|
3041
|
+
await writePin(resolved);
|
|
3042
|
+
await recordDirective({ source, requested: d.requested, resolved, accepted: true });
|
|
3043
|
+
return {
|
|
3044
|
+
rest: d.rest,
|
|
3045
|
+
ack: `model pinned to "${resolved}" for this session — "/model auto" clears it`,
|
|
3046
|
+
stripped: true,
|
|
3047
|
+
};
|
|
3048
|
+
};
|
|
2039
3049
|
// Root abort tree for the whole run; each turn gets a child.
|
|
2040
3050
|
const runAbort = createAbortTree(runContext.abortSignal);
|
|
2041
3051
|
// Per-turn abort tree, replaced at the start of each turn so a SIGINT
|
|
@@ -2072,9 +3082,12 @@ export async function runChatLoop(opts) {
|
|
|
2072
3082
|
};
|
|
2073
3083
|
let modelCallTimer;
|
|
2074
3084
|
const armModelCallTimer = () => {
|
|
2075
|
-
|
|
3085
|
+
// 0.6.0 §4.4 — the serving candidate's `limits.model_call_timeout_ms`
|
|
3086
|
+
// wins over the run's when it declares one.
|
|
3087
|
+
const limitMs = servingPlan.modelCallTimeoutMs ?? opts.modelCallTimeoutMs;
|
|
3088
|
+
if (limitMs === undefined || limitMs <= 0)
|
|
2076
3089
|
return;
|
|
2077
|
-
modelCallTimer = armTimer(turnAbort, "model_call",
|
|
3090
|
+
modelCallTimer = armTimer(turnAbort, "model_call", limitMs);
|
|
2078
3091
|
};
|
|
2079
3092
|
const clearModelCallTimer = () => {
|
|
2080
3093
|
if (modelCallTimer !== undefined)
|
|
@@ -2129,7 +3142,7 @@ export async function runChatLoop(opts) {
|
|
|
2129
3142
|
// Response cost in USD-micros, or undefined when the served model isn't on the
|
|
2130
3143
|
// pricing table (the split is then not computable — no attribution stamped).
|
|
2131
3144
|
const responseCostMicros = (provider, wireModelId, usage) => {
|
|
2132
|
-
const row = resolvePricing(DEFAULT_PRICING, provider, wireModelId);
|
|
3145
|
+
const row = resolvePricing(opts.pricing ?? DEFAULT_PRICING, provider, wireModelId);
|
|
2133
3146
|
if (row === undefined)
|
|
2134
3147
|
return undefined;
|
|
2135
3148
|
return computeCostMicros(row, usage.input, usage.output, usage.cacheRead ?? 0, usage.cacheCreate ?? 0);
|
|
@@ -2187,11 +3200,16 @@ export async function runChatLoop(opts) {
|
|
|
2187
3200
|
const attributedCostUsdMicros = toolCostAttribution.get(tu.id);
|
|
2188
3201
|
if (attributedCostUsdMicros !== undefined)
|
|
2189
3202
|
toolCostAttribution.delete(tu.id);
|
|
3203
|
+
// 0.6.0 §5.6 — the serving model / profile ride every tool and permission
|
|
3204
|
+
// event of this call (stamped only while a pool candidate serves, so a
|
|
3205
|
+
// single-model run's events are byte-identical).
|
|
3206
|
+
const callAttribution = toolAttribution();
|
|
2190
3207
|
await logEvent("tool_use", {
|
|
2191
3208
|
id: tu.id,
|
|
2192
3209
|
name: tu.name,
|
|
2193
3210
|
input: tu.input,
|
|
2194
3211
|
...(attributedCostUsdMicros !== undefined ? { attributedCostUsdMicros } : {}),
|
|
3212
|
+
...callAttribution,
|
|
2195
3213
|
});
|
|
2196
3214
|
const inputBytes = Buffer.byteLength(JSON.stringify(tu.input ?? null), "utf8");
|
|
2197
3215
|
const toolStartEnvelope = bus.envelope();
|
|
@@ -2201,6 +3219,7 @@ export async function runChatLoop(opts) {
|
|
|
2201
3219
|
toolUseId: tu.id,
|
|
2202
3220
|
toolName: tu.name,
|
|
2203
3221
|
inputBytes,
|
|
3222
|
+
...callAttribution,
|
|
2204
3223
|
});
|
|
2205
3224
|
const t0Tool = performance.now();
|
|
2206
3225
|
const tool = toolByName.get(tu.name);
|
|
@@ -2248,6 +3267,7 @@ export async function runChatLoop(opts) {
|
|
|
2248
3267
|
isError: result.is_error === true,
|
|
2249
3268
|
outputBytes,
|
|
2250
3269
|
durationMs: performance.now() - t0Tool,
|
|
3270
|
+
...callAttribution,
|
|
2251
3271
|
});
|
|
2252
3272
|
// Section 11 — fire post-tool after the result is captured. Decision
|
|
2253
3273
|
// is observational here: a deny on post-tool logs a warning but does
|
|
@@ -2274,6 +3294,24 @@ export async function runChatLoop(opts) {
|
|
|
2274
3294
|
is_error: true,
|
|
2275
3295
|
});
|
|
2276
3296
|
}
|
|
3297
|
+
// 0.6.0 §4.4 item 3 — the dispatch SUBSET gate, the security-load-bearing
|
|
3298
|
+
// consumer of the plan. A subset in the advertisement is not a subset in
|
|
3299
|
+
// execution: `toolByName` is the run-wide UNION, and a model can (and,
|
|
3300
|
+
// steered by injected content, does) call a tool it was never shown. So
|
|
3301
|
+
// a name outside the serving candidate's advertisement fails HERE — with
|
|
3302
|
+
// an `is_error` result naming the profile — before permission, rate-limit
|
|
3303
|
+
// and justification, which keep operating on the union.
|
|
3304
|
+
if (!servingPlan.advertisedNames.has(tu.name)) {
|
|
3305
|
+
const who = servingPlan.profile !== undefined
|
|
3306
|
+
? `profile "${servingPlan.profile}" (${servingPlan.modelString})`
|
|
3307
|
+
: `model "${servingPlan.modelString}"`;
|
|
3308
|
+
return finish({
|
|
3309
|
+
type: "tool_result",
|
|
3310
|
+
tool_use_id: tu.id,
|
|
3311
|
+
content: `tool "${tu.name}" is not available to the serving model ${who}: it was not advertised on this request (the candidate's toolset is a subset of the run's). Call ListTools to see the tools this model can use, or proceed without it.`,
|
|
3312
|
+
is_error: true,
|
|
3313
|
+
});
|
|
3314
|
+
}
|
|
2277
3315
|
// Section 11 — pre-tool hook: short-circuit with the hook's reason
|
|
2278
3316
|
// when any matching hook returns deny/block.
|
|
2279
3317
|
const preHook = await fireHook("pre-tool", { id: tu.id, name: tu.name, input: tu.input });
|
|
@@ -2292,13 +3330,17 @@ export async function runChatLoop(opts) {
|
|
|
2292
3330
|
// this; surfaces that record what the model DID keep `tu.input`. See the
|
|
2293
3331
|
// `justificationInjectedTools` comment at loop start for the full split.
|
|
2294
3332
|
let operativeInput = operativeToolInput(tu.name, tu.input);
|
|
3333
|
+
// 0.6.0 §4.4 — the SERVING candidate's rule set: the run's rules narrowed
|
|
3334
|
+
// by the profile's `permissions.deny` / `.ask` through `narrowRuleSet`
|
|
3335
|
+
// (a decision-level meet — a profile can only tighten). The primary plan
|
|
3336
|
+
// carries the run's rules by reference, so a single-model run is unchanged.
|
|
2295
3337
|
const decisionDetails = evaluateWithReason({
|
|
2296
3338
|
toolName: tu.name,
|
|
2297
3339
|
input: operativeInput,
|
|
2298
3340
|
readOnly: tool.readOnly,
|
|
2299
3341
|
destructive: tool.destructive,
|
|
2300
3342
|
requiresSandbox: tool.requiresSandbox,
|
|
2301
|
-
}, permissionMode, permissionRules, { sandboxAvailable: opts.sandboxAvailable === true });
|
|
3343
|
+
}, permissionMode, servingPlan.permissionRules, { sandboxAvailable: opts.sandboxAvailable === true });
|
|
2302
3344
|
const decision = decisionDetails.decision;
|
|
2303
3345
|
bus.publish({
|
|
2304
3346
|
...bus.envelope(),
|
|
@@ -2307,6 +3349,7 @@ export async function runChatLoop(opts) {
|
|
|
2307
3349
|
decision,
|
|
2308
3350
|
mode: permissionMode,
|
|
2309
3351
|
...(decisionDetails.reason !== undefined ? { reason: decisionDetails.reason } : {}),
|
|
3352
|
+
...callAttribution,
|
|
2310
3353
|
});
|
|
2311
3354
|
let approved = decision === "allow";
|
|
2312
3355
|
let denialMessage;
|
|
@@ -2392,13 +3435,16 @@ export async function runChatLoop(opts) {
|
|
|
2392
3435
|
// rules are read fresh each loop and an operator may have tightened
|
|
2393
3436
|
// them since — and a grant is permission from a human, not an
|
|
2394
3437
|
// exemption from the spec. A rule that now DENIES wins.
|
|
3438
|
+
// §10.1 — approval replay across candidates uses the SERVING
|
|
3439
|
+
// candidate's rule set: a grant parked while the strong arm
|
|
3440
|
+
// served does not run under a narrower arm's denies.
|
|
2395
3441
|
const recheck = evaluateWithReason({
|
|
2396
3442
|
toolName: tu.name,
|
|
2397
3443
|
input: approvedOperative,
|
|
2398
3444
|
readOnly: tool.readOnly,
|
|
2399
3445
|
destructive: tool.destructive,
|
|
2400
3446
|
requiresSandbox: tool.requiresSandbox,
|
|
2401
|
-
}, permissionMode, permissionRules, { sandboxAvailable: opts.sandboxAvailable === true });
|
|
3447
|
+
}, permissionMode, servingPlan.permissionRules, { sandboxAvailable: opts.sandboxAvailable === true });
|
|
2402
3448
|
if (recheck.decision === "deny") {
|
|
2403
3449
|
approved = false;
|
|
2404
3450
|
denialMessage =
|
|
@@ -2443,6 +3489,7 @@ export async function runChatLoop(opts) {
|
|
|
2443
3489
|
decision: "allow",
|
|
2444
3490
|
mode: permissionMode,
|
|
2445
3491
|
reason: `approval ${replayedFrom.id} replayed: executing the APPROVED input, not the regenerated one`,
|
|
3492
|
+
...callAttribution,
|
|
2446
3493
|
});
|
|
2447
3494
|
runContext.logger.info("tool approval replayed", {
|
|
2448
3495
|
approvalId: replayedFrom.id,
|
|
@@ -2538,6 +3585,7 @@ export async function runChatLoop(opts) {
|
|
|
2538
3585
|
decision: "ask",
|
|
2539
3586
|
mode: permissionMode,
|
|
2540
3587
|
askOutcome: approved ? "approved" : "denied",
|
|
3588
|
+
...callAttribution,
|
|
2541
3589
|
});
|
|
2542
3590
|
}
|
|
2543
3591
|
if (!approved) {
|
|
@@ -2583,6 +3631,7 @@ export async function runChatLoop(opts) {
|
|
|
2583
3631
|
...(verdict.confidence !== undefined
|
|
2584
3632
|
? { justificationConfidence: verdict.confidence }
|
|
2585
3633
|
: {}),
|
|
3634
|
+
...callAttribution,
|
|
2586
3635
|
});
|
|
2587
3636
|
runContext.logger.info("justification evaluated", {
|
|
2588
3637
|
toolUseId: tu.id,
|
|
@@ -2670,6 +3719,7 @@ export async function runChatLoop(opts) {
|
|
|
2670
3719
|
mode: permissionMode,
|
|
2671
3720
|
outcome: egressOutcome,
|
|
2672
3721
|
...(egress.originsFound.length > 0 ? { reason: `egress: ${summarizeEgress(egress)}` } : {}),
|
|
3722
|
+
...callAttribution,
|
|
2673
3723
|
});
|
|
2674
3724
|
if (egress.verdict !== "pass") {
|
|
2675
3725
|
runContext.logger.warn("egress classification non-clean", {
|
|
@@ -2728,9 +3778,19 @@ export async function runChatLoop(opts) {
|
|
|
2728
3778
|
// keys, so the guard is what keeps unlisted tools ungated). Exhausting
|
|
2729
3779
|
// the limiter's wait budget fails just THIS call with an `is_error`
|
|
2730
3780
|
// result the model can adapt to — never the run.
|
|
2731
|
-
|
|
3781
|
+
// 0.6.0 §4.4 — the serving candidate's own bucket (`tool:<name>@<armId>`)
|
|
3782
|
+
// wins when its profile declares `rate_limits` for the tool; otherwise
|
|
3783
|
+
// the run-level `tool:<name>` bucket applies exactly as before.
|
|
3784
|
+
const planLimiter = servingPlan.toolRateLimiter;
|
|
3785
|
+
const planGated = planLimiter !== undefined && servingPlan.hasToolRateBucket(tu.name);
|
|
3786
|
+
if (planGated || (toolRateLimiter !== undefined && hasToolRateBucket(tu.name))) {
|
|
2732
3787
|
try {
|
|
2733
|
-
|
|
3788
|
+
if (planGated && planLimiter !== undefined) {
|
|
3789
|
+
await planLimiter.acquire([{ dimension: "tool", id: `${tu.name}@${servingPlan.armId}` }], 1);
|
|
3790
|
+
}
|
|
3791
|
+
else if (toolRateLimiter !== undefined) {
|
|
3792
|
+
await toolRateLimiter.acquire([{ dimension: "tool", id: tu.name }], 1);
|
|
3793
|
+
}
|
|
2734
3794
|
}
|
|
2735
3795
|
catch (err) {
|
|
2736
3796
|
runContext.logger.warn("tool rate limit exhausted", {
|
|
@@ -2756,15 +3816,42 @@ export async function runChatLoop(opts) {
|
|
|
2756
3816
|
// coarse "tool" origin. The Task-tool-only fields (`spawnSubAgent`,
|
|
2757
3817
|
// `crewMailbox`, `subAgents`) stay conditional — they're omitted when not
|
|
2758
3818
|
// wired, and the Task tool already checks for undefined before using them.
|
|
3819
|
+
//
|
|
3820
|
+
// 0.6.0 §4.4 — the bridge is built from the SERVING plan, not the run:
|
|
3821
|
+
// `permissionRules` is the candidate's narrowed meet and `tools` is the
|
|
3822
|
+
// run catalog intersected with the candidate's advertisement. `Task`
|
|
3823
|
+
// survives a profile's `tools: []` (RETAINED_LOOP_TOOL_NAMES), and
|
|
3824
|
+
// tool-task derives the child's catalog (`buildChildCatalog(bridge.tools)`)
|
|
3825
|
+
// and rules (`resolveChildPermissions({ rules: bridge.permissionRules })`)
|
|
3826
|
+
// from exactly these two fields — so a `$fast` candidate denied `Bash`
|
|
3827
|
+
// must not be able to reach it by spawning a child that inherits the
|
|
3828
|
+
// run's un-narrowed catalog and rules ("a subset in the advertisement is
|
|
3829
|
+
// not a subset in execution" applies to the one retained tool too).
|
|
3830
|
+
// `resolveChildPermissions` narrows from what it is handed, never widens.
|
|
3831
|
+
// Without a pool `servingPlan` is `primaryPlan`, whose rules ARE the run's
|
|
3832
|
+
// and whose advertisement is the whole catalog, so the bridge is
|
|
3833
|
+
// byte-identical to today's. (Full child routing — a child served by its
|
|
3834
|
+
// own candidate — is PR 11's.)
|
|
3835
|
+
const bridgeTools = servingPlan.fromPool
|
|
3836
|
+
? tools.filter((t) => servingPlan.advertisedNames.has(t.name))
|
|
3837
|
+
: tools;
|
|
2759
3838
|
const bridge = {
|
|
2760
3839
|
runContext,
|
|
2761
3840
|
eventLog,
|
|
2762
3841
|
permissionMode,
|
|
2763
|
-
permissionRules,
|
|
2764
|
-
tools,
|
|
3842
|
+
permissionRules: servingPlan.permissionRules,
|
|
3843
|
+
tools: bridgeTools,
|
|
2765
3844
|
model: opts.model,
|
|
2766
3845
|
maxTokens,
|
|
2767
3846
|
...(sessionRootDir !== undefined ? { sessionRootDir } : {}),
|
|
3847
|
+
// 0.6.0 §4.4 / §10.2 — the routing projection for children: the arm
|
|
3848
|
+
// that served this call (a child runs on it only behind
|
|
3849
|
+
// `inheritRouting: true`) and the run cap (`budgetShare` takes a share
|
|
3850
|
+
// of it). `model` above stays `opts.model`, the declared primary.
|
|
3851
|
+
routing: {
|
|
3852
|
+
served: servedArm,
|
|
3853
|
+
...(opts.budget !== undefined ? { budgetUsdMicros: opts.budget.usdMicros } : {}),
|
|
3854
|
+
},
|
|
2768
3855
|
hooks,
|
|
2769
3856
|
// v0.3.0 §2.5 — the per-run state-store, so plan-mutating tools can
|
|
2770
3857
|
// set PLAN_DIRTY_STATE_KEY and the loop re-renders the plan tail
|
|
@@ -2793,9 +3880,23 @@ export async function runChatLoop(opts) {
|
|
|
2793
3880
|
// every gate (permission/justification/egress) so it never overlaps the
|
|
2794
3881
|
// approval prompt, and torn down in `.finally` on success or throw.
|
|
2795
3882
|
beginToolWork(tu.name);
|
|
3883
|
+
// 0.6.0 §4.4 — the serving model and this tool's per-candidate
|
|
3884
|
+
// `tool_config` block, threaded through tool-executor into the tool's
|
|
3885
|
+
// `ToolExecuteContext.model` / `.toolConfig`. Both presence-gated.
|
|
3886
|
+
const planToolConfig = toolConfigFor(servingPlan.toolConfigs, tu.name);
|
|
3887
|
+
const planModel = servingPlan.fromPool
|
|
3888
|
+
? {
|
|
3889
|
+
armId: servingPlan.armId,
|
|
3890
|
+
wireModelId: servingPlan.wireModelId,
|
|
3891
|
+
specModel: servingPlan.modelString,
|
|
3892
|
+
...(servingPlan.profile !== undefined ? { profile: servingPlan.profile } : {}),
|
|
3893
|
+
}
|
|
3894
|
+
: undefined;
|
|
2796
3895
|
const raw = await executeTool(tool, operativeInput, {
|
|
2797
3896
|
toolUseId: tu.id,
|
|
2798
3897
|
signal: toolAbort.signal,
|
|
3898
|
+
...(planModel !== undefined ? { model: planModel } : {}),
|
|
3899
|
+
...(planToolConfig !== undefined ? { toolConfig: planToolConfig } : {}),
|
|
2799
3900
|
// #160 follow-up — the bridge (built on every run, above) carries
|
|
2800
3901
|
// `runContext`, which boundary-site tools (tool-mcp, skills-registry)
|
|
2801
3902
|
// read to tag their external content under the precise "mcp"/"skill"
|
|
@@ -2986,7 +4087,12 @@ export async function runChatLoop(opts) {
|
|
|
2986
4087
|
// Pass the full catalog so per-call classifiers (Task) can resolve a
|
|
2987
4088
|
// dispatch's child tool set; without it, classifier-based tools stay
|
|
2988
4089
|
// serial (fail-closed).
|
|
2989
|
-
|
|
4090
|
+
// 0.6.0 §4.4 — partition over the SERVING candidate's advertised tools; a
|
|
4091
|
+
// name outside them resolves to nothing here and is refused by the
|
|
4092
|
+
// dispatch gate inside `executeOneToolUse`.
|
|
4093
|
+
const partition = partitionToolCalls(toolUses, (n) => servingPlan.toolByName.get(n), [
|
|
4094
|
+
...servingPlan.toolByName.values(),
|
|
4095
|
+
]);
|
|
2990
4096
|
const maxConcurrentTools = opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS;
|
|
2991
4097
|
runContext.logger.debug("tool partition", {
|
|
2992
4098
|
concurrent: partition.concurrent.map((b) => b.length),
|
|
@@ -3130,9 +4236,304 @@ export async function runChatLoop(opts) {
|
|
|
3130
4236
|
await publishRunFailed(report);
|
|
3131
4237
|
throw new RunFailedError(report);
|
|
3132
4238
|
};
|
|
3133
|
-
|
|
4239
|
+
// -------------------------------------------------------------------------
|
|
4240
|
+
// 0.6.0 PR 9d — the side-call lifecycle (guide / shadow / committee). The
|
|
4241
|
+
// closures in `opts.sideCalls` make the model calls; this block decides
|
|
4242
|
+
// WHEN they run, what their text does, and publishes one `model_stage`
|
|
4243
|
+
// event per lifecycle transition (`started` | `done` | `failed` |
|
|
4244
|
+
// `skipped`, §8.1). `attachRoutingPersistence` mirrors every `model_stage`
|
|
4245
|
+
// into the session JSONL, so nothing here needs its own `logEvent`.
|
|
4246
|
+
// -------------------------------------------------------------------------
|
|
4247
|
+
/**
|
|
4248
|
+
* §7.12 / §14(2) — the classified `crewhaus_budget` stop, shared by the
|
|
4249
|
+
* per-model-call gate inside `runOneTurn` and the committee gate (a
|
|
4250
|
+
* committee REPLACES the primary turn, so nothing after it would gate the
|
|
4251
|
+
* run). `where` names the call the cap stopped.
|
|
4252
|
+
*/
|
|
4253
|
+
const failBudgetStop = async (where) => {
|
|
4254
|
+
const spentMicros = resumedCostUsdMicros + (budgetMeter?.getRunCost(bus.runId).totalUsdMicros ?? 0);
|
|
4255
|
+
const report = {
|
|
4256
|
+
class: BUILTIN_FAILURE_CLASSES.crewhaus_budget.class,
|
|
4257
|
+
title: BUILTIN_FAILURE_CLASSES.crewhaus_budget.title,
|
|
4258
|
+
detail: `$${(spentMicros / 1_000_000).toFixed(4)} spent ≥ $${((opts.budget?.usdMicros ?? 0) / 1_000_000).toFixed(4)} cap before ${where}${budgetDegraded ? " (degraded rung also reached the cap)" : ""}`,
|
|
4259
|
+
remediation: BUILTIN_FAILURE_CLASSES.crewhaus_budget.remediation,
|
|
4260
|
+
exitCode: BUILTIN_FAILURE_CLASSES.crewhaus_budget.exitCode,
|
|
4261
|
+
};
|
|
4262
|
+
await publishRunFailed(report);
|
|
4263
|
+
throw new RunFailedError(report);
|
|
4264
|
+
};
|
|
4265
|
+
const sideCallContext = (messages, signal) => ({
|
|
4266
|
+
runContext,
|
|
4267
|
+
bus,
|
|
4268
|
+
turnNumber: runContext.turnNumber,
|
|
4269
|
+
instructions: opts.instructions,
|
|
4270
|
+
// A snapshot: the turn keeps appending to the live array after the side
|
|
4271
|
+
// call returns (and the shadow runs while it does).
|
|
4272
|
+
messages: [...messages],
|
|
4273
|
+
signal,
|
|
4274
|
+
...(poolScope !== undefined ? { scope: poolScope } : {}),
|
|
4275
|
+
// §7.12 — a multi-call closure re-reads the total cap between its calls.
|
|
4276
|
+
...(opts.budget !== undefined
|
|
4277
|
+
? { budgetGate: () => enforceBudget(runContext.turnNumber) }
|
|
4278
|
+
: {}),
|
|
4279
|
+
});
|
|
4280
|
+
// §7.4 — the guide block. Runs ONCE per strategy turn (keyed on the turn
|
|
4281
|
+
// number, so an evaluation retry or a cascade stage inside the same turn
|
|
4282
|
+
// never re-plans), before the turn's first model call. `first_turn` keeps
|
|
4283
|
+
// its block for the rest of the run (plan-execute); `turn` replaces it each
|
|
4284
|
+
// turn — and a `turn` guide that fails, returns nothing, is skipped or is
|
|
4285
|
+
// boundary-blocked leaves the turn UNGUIDED (§7.13): the previous turn's
|
|
4286
|
+
// block is dropped before the call, never re-injected beside a stage line
|
|
4287
|
+
// that says the guide did not run. Volatile by construction: never
|
|
4288
|
+
// persisted, never replayed on resume.
|
|
4289
|
+
let guideBlocks = [];
|
|
4290
|
+
let guideRanForTurn = 0;
|
|
4291
|
+
const maybeRunGuide = async (messages) => {
|
|
4292
|
+
const guide = opts.sideCalls?.guide;
|
|
4293
|
+
if (guide === undefined)
|
|
4294
|
+
return;
|
|
4295
|
+
if (guideRanForTurn === runContext.turnNumber)
|
|
4296
|
+
return;
|
|
4297
|
+
guideRanForTurn = runContext.turnNumber;
|
|
4298
|
+
if (guide.every === "turn")
|
|
4299
|
+
guideBlocks = [];
|
|
4300
|
+
if (guide.every === "first_turn" && runContext.turnNumber !== 1)
|
|
4301
|
+
return;
|
|
4302
|
+
const stage = {
|
|
4303
|
+
stage: "guide",
|
|
4304
|
+
strategy: "guide",
|
|
4305
|
+
role: "guide",
|
|
4306
|
+
model: guide.model,
|
|
4307
|
+
...(guide.profile !== undefined ? { profile: guide.profile } : {}),
|
|
4308
|
+
};
|
|
4309
|
+
// §6.2 — the guide is auxiliary spend: past the judge_share it is skipped
|
|
4310
|
+
// (never silently — the stage line says so) and the turn runs unguided.
|
|
4311
|
+
if (judgeShareExhausted()) {
|
|
4312
|
+
publishStage({ ...stage, outcome: "skipped", cause: "judge_share_exhausted" });
|
|
4313
|
+
return;
|
|
4314
|
+
}
|
|
4315
|
+
// §7.12 — the total cap gates every model call, the guide's included.
|
|
4316
|
+
// Same gate, same turn as the primary's first `NeedModel` gate a moment
|
|
4317
|
+
// later, so a `degrade` fired here is the one that gate would have fired;
|
|
4318
|
+
// on "stop" the guide is skipped and that gate ends the run.
|
|
4319
|
+
if ((await enforceBudget(runContext.turnNumber)) === "stop") {
|
|
4320
|
+
publishStage({ ...stage, outcome: "skipped", cause: "budget" });
|
|
4321
|
+
return;
|
|
4322
|
+
}
|
|
4323
|
+
publishStage({ ...stage, outcome: "started" });
|
|
4324
|
+
out.spinner.start("guide");
|
|
4325
|
+
let reply;
|
|
4326
|
+
try {
|
|
4327
|
+
reply = await guide.run(sideCallContext(messages, turnAbort.signal));
|
|
4328
|
+
}
|
|
4329
|
+
catch (err) {
|
|
4330
|
+
// §7.13 — a guide failure is non-fatal: the turn proceeds unguided.
|
|
4331
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4332
|
+
runContext.logger.warn("guide side call failed — turn proceeds unguided", { message });
|
|
4333
|
+
publishStage({ ...stage, outcome: "failed", cause: message });
|
|
4334
|
+
return;
|
|
4335
|
+
}
|
|
4336
|
+
finally {
|
|
4337
|
+
out.spinner.stop();
|
|
4338
|
+
}
|
|
4339
|
+
const text = reply.text?.trim() ?? "";
|
|
4340
|
+
if (text.length === 0) {
|
|
4341
|
+
publishStage({ ...stage, outcome: "skipped", cause: reply.cause ?? "empty" });
|
|
4342
|
+
return;
|
|
4343
|
+
}
|
|
4344
|
+
// Pillar 3 — the guide text enters the parent's own system region, so it
|
|
4345
|
+
// is classified HERE at TrustOrigin "consult" (the same tier as a Consult
|
|
4346
|
+
// reply) and lineage-tagged for the egress fabric. A malicious verdict
|
|
4347
|
+
// drops the block outright rather than injecting a redaction notice
|
|
4348
|
+
// into the system prompt.
|
|
4349
|
+
const boundary = await classifyBoundary(text, { origin: "consult" });
|
|
4350
|
+
if (boundary.action === "redact") {
|
|
4351
|
+
runContext.logger.warn("guide text blocked by the boundary classifier — turn proceeds unguided");
|
|
4352
|
+
publishStage({ ...stage, outcome: "failed", cause: "boundary-blocked" });
|
|
4353
|
+
return;
|
|
4354
|
+
}
|
|
4355
|
+
tagContent(runContext, text, "consult");
|
|
4356
|
+
guideBlocks = [{ type: "text", text: `<guide>\n${text}\n</guide>` }];
|
|
4357
|
+
publishStage({ ...stage, outcome: "done" });
|
|
4358
|
+
// Plan-execute: the plan also lands in the continuity plan store when the
|
|
4359
|
+
// seam is wired, and the `<current_plan>` tail re-renders from it.
|
|
4360
|
+
if (guide.every === "first_turn" && opts.continuity?.savePlan !== undefined) {
|
|
4361
|
+
try {
|
|
4362
|
+
await opts.continuity.savePlan(text);
|
|
4363
|
+
runState.set({ [PLAN_DIRTY_STATE_KEY]: true });
|
|
4364
|
+
}
|
|
4365
|
+
catch (err) {
|
|
4366
|
+
runContext.logger.warn("guide plan write to the continuity store failed", {
|
|
4367
|
+
error: err.message,
|
|
4368
|
+
});
|
|
4369
|
+
}
|
|
4370
|
+
}
|
|
4371
|
+
};
|
|
4372
|
+
// §7.8 — the shadow (audition) lane. Scheduled AFTER the turn's answer is
|
|
4373
|
+
// final, so nothing the shadow does can change what the user gets; runs
|
|
4374
|
+
// on a deterministic sampled share of turns (seeded by the pool seed and
|
|
4375
|
+
// the turn number, so a replay samples the same turns); one shadow at a
|
|
4376
|
+
// time; awaited at teardown so a singleTurn host never drops it.
|
|
4377
|
+
let shadowInFlight;
|
|
4378
|
+
let shadowFailures = 0;
|
|
4379
|
+
const sampleDraw = (turn) => {
|
|
4380
|
+
// FNV-1a over `<seed>|shadow|<turn>` → [0, 1). Deterministic per run.
|
|
4381
|
+
let h = 0x811c9dc5;
|
|
4382
|
+
for (const ch of `${poolSeed}|shadow|${turn}`) {
|
|
4383
|
+
h ^= ch.charCodeAt(0);
|
|
4384
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
4385
|
+
}
|
|
4386
|
+
return h / 0x1_0000_0000;
|
|
4387
|
+
};
|
|
4388
|
+
const maybeScheduleShadow = (messagesBefore, primaryText) => {
|
|
4389
|
+
const shadow = opts.sideCalls?.shadow;
|
|
4390
|
+
if (shadow === undefined)
|
|
4391
|
+
return;
|
|
4392
|
+
const turn = runContext.turnNumber;
|
|
4393
|
+
const served = lastPoolOutcome;
|
|
4394
|
+
const stage = {
|
|
4395
|
+
stage: "shadow",
|
|
4396
|
+
strategy: "shadow",
|
|
4397
|
+
role: "shadow",
|
|
4398
|
+
model: shadow.candidate,
|
|
4399
|
+
...(shadow.profile !== undefined ? { profile: shadow.profile } : {}),
|
|
4400
|
+
};
|
|
4401
|
+
// §7.9 — the shadow arm records under its ARM ID (the profile name when
|
|
4402
|
+
// the shadow candidate is a profile, else its model string), like every
|
|
4403
|
+
// live arm.
|
|
4404
|
+
const shadowArm = shadow.profile ?? shadow.candidate;
|
|
4405
|
+
if (served !== undefined &&
|
|
4406
|
+
(served.armId === shadowArm || served.modelString === shadow.candidate)) {
|
|
4407
|
+
publishStage({ ...stage, outcome: "skipped", cause: "same-as-primary" });
|
|
4408
|
+
return;
|
|
4409
|
+
}
|
|
4410
|
+
if (sampleDraw(turn) >= shadow.sampleRate) {
|
|
4411
|
+
publishStage({ ...stage, outcome: "skipped", cause: "sample_rate" });
|
|
4412
|
+
return;
|
|
4413
|
+
}
|
|
4414
|
+
if (judgeShareExhausted()) {
|
|
4415
|
+
publishStage({ ...stage, outcome: "skipped", cause: "judge_share_exhausted" });
|
|
4416
|
+
return;
|
|
4417
|
+
}
|
|
4418
|
+
// §7.12 — the total cap gates the audition lane too (read-only: see
|
|
4419
|
+
// `totalCapReached` for why the shadow never fires `enforceBudget`).
|
|
4420
|
+
if (totalCapReached(turn)) {
|
|
4421
|
+
publishStage({ ...stage, outcome: "skipped", cause: "budget" });
|
|
4422
|
+
return;
|
|
4423
|
+
}
|
|
4424
|
+
if (primaryText.trim().length === 0) {
|
|
4425
|
+
publishStage({ ...stage, outcome: "skipped", cause: "empty-primary" });
|
|
4426
|
+
return;
|
|
4427
|
+
}
|
|
4428
|
+
const primaryModel = lastResponseWireModel ?? wireModelId;
|
|
4429
|
+
const seed = [...messagesBefore];
|
|
4430
|
+
publishStage({ ...stage, outcome: "started" });
|
|
4431
|
+
const prior = shadowInFlight ?? Promise.resolve();
|
|
4432
|
+
shadowInFlight = prior.then(async () => {
|
|
4433
|
+
try {
|
|
4434
|
+
const v = await shadow.run({
|
|
4435
|
+
...sideCallContext(seed, runAbort.signal),
|
|
4436
|
+
turnNumber: turn,
|
|
4437
|
+
primaryText,
|
|
4438
|
+
primaryModel,
|
|
4439
|
+
});
|
|
4440
|
+
const shadowCost = poolCostUsd(v.provider, v.model, v.usage);
|
|
4441
|
+
// Both arms record into the observe-only shadow lane under the
|
|
4442
|
+
// primary's band: the router never reads `shadow:` keys, so this
|
|
4443
|
+
// steers nothing until `route promote` folds it (PR 14).
|
|
4444
|
+
const sb = scoreboard;
|
|
4445
|
+
const rc = rewardConfig;
|
|
4446
|
+
if (served !== undefined && sb !== undefined && rc !== undefined) {
|
|
4447
|
+
// The primary's key is already `<scope>/<band>` under a scoped pool
|
|
4448
|
+
// (§7.9); the lane re-scopes the bare band as `shadow:<scope>/<band>`.
|
|
4449
|
+
const key = shadowRouteKey(unscopedRouteKey(served.routeKey), poolScope);
|
|
4450
|
+
const q = shadowLaneQuality(v.verdict);
|
|
4451
|
+
// §6.3 / §7.10 — each side of the audition is STAMPED, because the
|
|
4452
|
+
// two lines are not the same kind of evidence: the shadow arm never
|
|
4453
|
+
// served, while the primary already recorded this very turn live
|
|
4454
|
+
// through `recordPoolOutcome`. `route promote` folds the shadow side
|
|
4455
|
+
// and skips the primary one on this stamp; without it the promotion
|
|
4456
|
+
// could not tell them apart even in principle.
|
|
4457
|
+
const shadowObs = {
|
|
4458
|
+
success: true,
|
|
4459
|
+
latencyMs: v.latencyMs,
|
|
4460
|
+
...(shadowCost !== undefined ? { costUsd: shadowCost } : {}),
|
|
4461
|
+
quality: q.shadow,
|
|
4462
|
+
attributedTo: SHADOW_LANE_SHADOW_ARM,
|
|
4463
|
+
};
|
|
4464
|
+
const primaryObs = {
|
|
4465
|
+
success: true,
|
|
4466
|
+
...(served.latencyMs !== undefined ? { latencyMs: served.latencyMs } : {}),
|
|
4467
|
+
...(served.costUsd !== undefined ? { costUsd: served.costUsd } : {}),
|
|
4468
|
+
quality: q.primary,
|
|
4469
|
+
attributedTo: SHADOW_LANE_PRIMARY_ARM,
|
|
4470
|
+
};
|
|
4471
|
+
sb.record(key, shadowArm, computeReward(shadowObs, rc), {
|
|
4472
|
+
...shadowObs,
|
|
4473
|
+
...routeProvenance(),
|
|
4474
|
+
});
|
|
4475
|
+
sb.record(key, served.armId, computeReward(primaryObs, rc), {
|
|
4476
|
+
...primaryObs,
|
|
4477
|
+
...routeProvenance(),
|
|
4478
|
+
});
|
|
4479
|
+
}
|
|
4480
|
+
publishStage({
|
|
4481
|
+
...stage,
|
|
4482
|
+
model: v.model,
|
|
4483
|
+
outcome: "done",
|
|
4484
|
+
cause: v.agreed ? v.verdict : `${v.verdict} (orders disagreed)`,
|
|
4485
|
+
...(shadowCost !== undefined ? { costUsdMicros: Math.round(shadowCost * 1e6) } : {}),
|
|
4486
|
+
});
|
|
4487
|
+
}
|
|
4488
|
+
catch (err) {
|
|
4489
|
+
// §7.13 — a shadow failure is silently dropped and counted.
|
|
4490
|
+
shadowFailures += 1;
|
|
4491
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4492
|
+
runContext.logger.warn("shadow side call failed — dropped", { message, shadowFailures });
|
|
4493
|
+
publishStage({ ...stage, outcome: "failed", cause: message });
|
|
4494
|
+
}
|
|
4495
|
+
});
|
|
4496
|
+
};
|
|
4497
|
+
const awaitShadow = async () => {
|
|
4498
|
+
if (shadowInFlight === undefined)
|
|
4499
|
+
return;
|
|
4500
|
+
try {
|
|
4501
|
+
await shadowInFlight;
|
|
4502
|
+
}
|
|
4503
|
+
catch {
|
|
4504
|
+
// Every failure is caught inside the chain; this only guards teardown.
|
|
4505
|
+
}
|
|
4506
|
+
};
|
|
4507
|
+
async function runOneTurn(messages, turnOpts = {}) {
|
|
4508
|
+
// 0.6.0 §7.4 — the guide runs once per strategy turn, before the turn's
|
|
4509
|
+
// first model call (a no-op without `sideCalls.guide`).
|
|
4510
|
+
await maybeRunGuide(messages);
|
|
3134
4511
|
let state = initialState;
|
|
3135
4512
|
let terminalContent = [];
|
|
4513
|
+
turnAttributionOverride = {
|
|
4514
|
+
...(turnOpts.role !== undefined ? { role: turnOpts.role } : {}),
|
|
4515
|
+
...(turnOpts.stage !== undefined ? { stage: turnOpts.stage } : {}),
|
|
4516
|
+
...(turnOpts.strategy !== undefined ? { strategy: turnOpts.strategy } : {}),
|
|
4517
|
+
};
|
|
4518
|
+
const pendingObservations = [];
|
|
4519
|
+
let attemptCostUsd = 0;
|
|
4520
|
+
let attemptServedModel;
|
|
4521
|
+
let attemptServedProfile;
|
|
4522
|
+
/** §7.10 — defer a successful pool observation to the strategy-turn boundary, or fold it now. */
|
|
4523
|
+
const foldOutcome = (turn, obs) => {
|
|
4524
|
+
if (turnOpts.defer === true && obs.success) {
|
|
4525
|
+
pendingObservations.push({
|
|
4526
|
+
turn: {
|
|
4527
|
+
modelString: turn.modelString,
|
|
4528
|
+
routeKey: turn.routeKey,
|
|
4529
|
+
...(turn.armId !== undefined ? { armId: turn.armId } : {}),
|
|
4530
|
+
},
|
|
4531
|
+
obs,
|
|
4532
|
+
});
|
|
4533
|
+
return;
|
|
4534
|
+
}
|
|
4535
|
+
recordPoolOutcome(turn, obs);
|
|
4536
|
+
};
|
|
3136
4537
|
// G02 — aggregate token usage across every MAIN-turn model call this
|
|
3137
4538
|
// turn makes (each tool iteration streams once). Handed to the in-loop
|
|
3138
4539
|
// evaluator so a judge can weigh cost/verbosity; side-calls (compaction,
|
|
@@ -3158,12 +4559,22 @@ export async function runChatLoop(opts) {
|
|
|
3158
4559
|
// records whether the LAST attempt streamed on the fast tier, so the
|
|
3159
4560
|
// error catch only escalates a genuine fast-tier misroute.
|
|
3160
4561
|
let thisTurnToolUseCount = 0;
|
|
4562
|
+
const thisTurnToolNames = [];
|
|
3161
4563
|
let escalateTier = false;
|
|
3162
4564
|
let servedFastTier = false;
|
|
3163
4565
|
// Adaptive model routing — once a pool candidate fails, stick with the
|
|
3164
4566
|
// strongest candidate for the rest of the run (misroute recovery, mirroring
|
|
3165
4567
|
// the tier router's fast→default escalation latch).
|
|
3166
4568
|
let escalatePool = false;
|
|
4569
|
+
// 0.6.0 §7.2.4 — the model's own `Escalate` request, once consumed: the
|
|
4570
|
+
// latch above is armed and the decision below names its target + receipt.
|
|
4571
|
+
let selfEscalation;
|
|
4572
|
+
// 0.6.0 §7.2.3 — the classifier is budgeted as ONE model call per TURN:
|
|
4573
|
+
// every inner model call of a tool loop re-runs `preRoute` over the same
|
|
4574
|
+
// human message, so the verdict is memoised on that message object for
|
|
4575
|
+
// the rest of the turn (a new turn, or a new human message, classifies
|
|
4576
|
+
// afresh). Persisted on every route line either way.
|
|
4577
|
+
let turnClassifierVerdict;
|
|
3167
4578
|
while (state.kind !== "Done") {
|
|
3168
4579
|
if (turnAbort.signal.aborted) {
|
|
3169
4580
|
state = transition(state, { kind: "Aborted" });
|
|
@@ -3171,6 +4582,22 @@ export async function runChatLoop(opts) {
|
|
|
3171
4582
|
}
|
|
3172
4583
|
switch (state.kind) {
|
|
3173
4584
|
case "NeedModel": {
|
|
4585
|
+
// 0.6.0 §7.12 — the per-model-call budget gate: EVERY model call,
|
|
4586
|
+
// tool iterations included, on every host (the singleTurn path
|
|
4587
|
+
// never reached the REPL's pre-turn check, so a workflow step,
|
|
4588
|
+
// channel message or managed request was uncapped, and a runaway
|
|
4589
|
+
// tool loop could not be stopped by `budget` anywhere). A breach
|
|
4590
|
+
// here is the classified `crewhaus_budget` stop: it lands BEFORE
|
|
4591
|
+
// the request opens, so every tool result already obtained is
|
|
4592
|
+
// committed and no `tool_use` is left unanswered — the transcript
|
|
4593
|
+
// stays resumable. A `degrade` breach latches the rung, and the
|
|
4594
|
+
// call below AND every remaining call of this turn serve on it —
|
|
4595
|
+
// the second breach is the next turn boundary, so a degraded tool
|
|
4596
|
+
// loop finishes and a single-turn host gets one complete degraded
|
|
4597
|
+
// reply. (Sanctioned change §14(2).)
|
|
4598
|
+
if ((await enforceBudget(runContext.turnNumber)) === "stop") {
|
|
4599
|
+
await failBudgetStop(`model call ${toolIterations + 1} of turn ${runContext.turnNumber}`);
|
|
4600
|
+
}
|
|
3174
4601
|
out.write("agent> ");
|
|
3175
4602
|
// Section 11 — pre-model hook. Deny short-circuits the turn with
|
|
3176
4603
|
// a synthetic "[blocked by hook]" assistant message so the loop
|
|
@@ -3181,6 +4608,15 @@ export async function runChatLoop(opts) {
|
|
|
3181
4608
|
streaming: opts.streaming === true,
|
|
3182
4609
|
messageCount: messages.length,
|
|
3183
4610
|
});
|
|
4611
|
+
// 0.6.0 §7.4 — the hook's `mutate.systemAppend` is HONOURED: a
|
|
4612
|
+
// string is appended to this call's system prompt in the volatile
|
|
4613
|
+
// region (after the cache-marked prefix, beside the candidate
|
|
4614
|
+
// overlay), so the same seam a guide model would use is open to a
|
|
4615
|
+
// hook. Accepted-then-discarded before this release.
|
|
4616
|
+
const hookSystemAppend = typeof preModel.mutate?.["systemAppend"] === "string" &&
|
|
4617
|
+
preModel.mutate["systemAppend"].length > 0
|
|
4618
|
+
? preModel.mutate["systemAppend"]
|
|
4619
|
+
: undefined;
|
|
3184
4620
|
if (!preModel.allowed) {
|
|
3185
4621
|
const blockedText = `[blocked by hook] ${preModel.reason ?? "denied"}`;
|
|
3186
4622
|
out.write(`${blockedText}\n`);
|
|
@@ -3241,34 +4677,342 @@ export async function runChatLoop(opts) {
|
|
|
3241
4677
|
// chosen candidate; the outcome is folded into the reward
|
|
3242
4678
|
// scoreboard post-turn. Takes precedence over the tier router (the
|
|
3243
4679
|
// two are mutually exclusive in the spec).
|
|
4680
|
+
// 0.6.0 §7.2.4 — the `Escalate` tool's request (recorded during the
|
|
4681
|
+
// preceding tool iteration) is consumed HERE, exactly once, with the
|
|
4682
|
+
// transcript length snapshotted onto the receipt. With a pool it
|
|
4683
|
+
// arms the escalation latch so the rest of this turn is served by
|
|
4684
|
+
// the request's target; without one there is nothing to escalate
|
|
4685
|
+
// to — the request is consumed and logged, never silently kept.
|
|
4686
|
+
if (opts.escalation !== undefined && opts.escalation.pending() !== undefined) {
|
|
4687
|
+
const consumed = opts.escalation.consume({ transcriptLength: messages.length });
|
|
4688
|
+
if (consumed !== undefined) {
|
|
4689
|
+
if (poolRouter !== undefined) {
|
|
4690
|
+
escalatePool = true;
|
|
4691
|
+
selfEscalation = consumed;
|
|
4692
|
+
// §7.2.4 — the self-escalation is a hybrid stage: one
|
|
4693
|
+
// `started` line (what metrics count as the escalation),
|
|
4694
|
+
// and the rest of the turn carries `role: "escalation"`.
|
|
4695
|
+
turnAttributionOverride = {
|
|
4696
|
+
...turnAttributionOverride,
|
|
4697
|
+
role: "escalation",
|
|
4698
|
+
stage: "escalate",
|
|
4699
|
+
strategy: "model_directed",
|
|
4700
|
+
};
|
|
4701
|
+
publishStage({
|
|
4702
|
+
stage: "escalate",
|
|
4703
|
+
strategy: "model_directed",
|
|
4704
|
+
role: "escalation",
|
|
4705
|
+
candidate: poolRouter
|
|
4706
|
+
.candidates()
|
|
4707
|
+
.find((c) => c.modelString === consumed.target.modelString) ??
|
|
4708
|
+
poolRouter.escalation(),
|
|
4709
|
+
outcome: "started",
|
|
4710
|
+
cause: "self",
|
|
4711
|
+
});
|
|
4712
|
+
}
|
|
4713
|
+
else {
|
|
4714
|
+
runContext.logger.warn("Escalate requested but no model_pool is active — nothing to escalate to", { receipt: consumed.receipt, reason: consumed.reason });
|
|
4715
|
+
}
|
|
4716
|
+
}
|
|
4717
|
+
}
|
|
3244
4718
|
if (poolRouter !== undefined) {
|
|
3245
|
-
|
|
4719
|
+
// 0.6.0 §7.2 — the preRoute decision phase. The deterministic
|
|
4720
|
+
// signals (the tier router's four, plus the 0.6.0 additions:
|
|
4721
|
+
// the latest HUMAN user text — input only, never persisted —
|
|
4722
|
+
// image presence, last turn's tool names, the budget ratio and
|
|
4723
|
+
// the host's channel hint) feed the forced → directive → rules →
|
|
4724
|
+
// classifier → eligibility ladder; the resulting hint goes INTO
|
|
4725
|
+
// the synchronous `route()` and is persisted on the route line.
|
|
4726
|
+
const humanMessage = latestHumanUserMessage(messages);
|
|
4727
|
+
const humanText = humanMessage !== undefined ? messageText(humanMessage) : undefined;
|
|
4728
|
+
const budgetSpentRatio = opts.budget !== undefined && budgetMeter !== undefined && opts.budget.usdMicros > 0
|
|
4729
|
+
? Math.min(1, (resumedCostUsdMicros + budgetMeter.getRunCost(bus.runId).totalUsdMicros) /
|
|
4730
|
+
opts.budget.usdMicros)
|
|
4731
|
+
: undefined;
|
|
4732
|
+
const turnHasImages = transcriptHasImages(messages);
|
|
4733
|
+
const toolsInPlay = (anthropicTools?.length ?? 0) > 0;
|
|
4734
|
+
const routeSignals = {
|
|
3246
4735
|
contextTokens: contextTokenSignal(messages),
|
|
3247
|
-
toolsInPlay
|
|
4736
|
+
toolsInPlay,
|
|
3248
4737
|
turnIndex: Math.max(0, runContext.turnNumber - 1),
|
|
3249
4738
|
priorTurnToolUseCount,
|
|
3250
|
-
|
|
4739
|
+
...(humanText !== undefined && humanText.length > 0 ? { userText: humanText } : {}),
|
|
4740
|
+
hasImages: turnHasImages,
|
|
4741
|
+
...(priorTurnToolNames.length > 0 ? { toolNamesLastTurn: priorTurnToolNames } : {}),
|
|
4742
|
+
...(budgetSpentRatio !== undefined ? { budgetSpentRatio } : {}),
|
|
4743
|
+
...(opts.channelHint !== undefined ? { channelHint: opts.channelHint } : {}),
|
|
4744
|
+
};
|
|
4745
|
+
const roster = poolRouter.candidates();
|
|
4746
|
+
// N1 — each arm's eligibility inputs: the plan's (declared-
|
|
4747
|
+
// override-folded) features, the declared or table-known
|
|
4748
|
+
// context window / output cap, its own `requires`, and the
|
|
4749
|
+
// per-profile cost cap against what this run has spent on it.
|
|
4750
|
+
const preRouteArmOf = (c) => {
|
|
4751
|
+
const plan = planFor(c);
|
|
4752
|
+
const cfg = poolCandidateConfigs.get(c);
|
|
4753
|
+
const table = resolveCapabilities(DEFAULT_CAPABILITIES, c.adapter.providerId, c.modelId);
|
|
4754
|
+
const contextWindow = cfg?.capabilities?.contextWindow ?? table?.contextWindow;
|
|
4755
|
+
const maxOutputTokens = cfg?.capabilities?.maxOutputTokens ?? table?.maxOutputTokens;
|
|
4756
|
+
const breakerState = c.breaker?.state();
|
|
4757
|
+
return {
|
|
4758
|
+
armId: armIdOf(c),
|
|
4759
|
+
modelString: c.modelString,
|
|
4760
|
+
tags: c.tags,
|
|
4761
|
+
...(breakerState !== undefined ? { breakerState } : {}),
|
|
4762
|
+
capabilities: {
|
|
4763
|
+
features: plan.features,
|
|
4764
|
+
...(contextWindow !== undefined ? { contextWindow } : {}),
|
|
4765
|
+
...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}),
|
|
4766
|
+
},
|
|
4767
|
+
...(cfg?.requires !== undefined ? { requires: cfg.requires } : {}),
|
|
4768
|
+
...(plan.costCapUsdMicros !== undefined
|
|
4769
|
+
? {
|
|
4770
|
+
costCapUsdMicros: plan.costCapUsdMicros,
|
|
4771
|
+
spentUsdMicros: plan.spentUsdMicros,
|
|
4772
|
+
}
|
|
4773
|
+
: {}),
|
|
4774
|
+
};
|
|
4775
|
+
};
|
|
4776
|
+
const preRouteArms = roster.map(preRouteArmOf);
|
|
4777
|
+
// The forced lane (§7.12 / §7.2.4 / §7.3) — a budget degrade
|
|
4778
|
+
// restricts the pool to its rung for the rest of the turn it
|
|
4779
|
+
// fired in (outranking everything below: escalating to the
|
|
4780
|
+
// strongest would defeat the cap); the cascade's escalation rung
|
|
4781
|
+
// (`runOneTurn(messages, { force })`, or a `forcedCandidate`
|
|
4782
|
+
// run) is pinned for the whole turn, outranking the misroute
|
|
4783
|
+
// latch; the escalation latch — a prior pool failure or the
|
|
4784
|
+
// model's own `Escalate` — names its target. All three reach
|
|
4785
|
+
// `preRoute` as the forced source, so a `/model` pin can never
|
|
4786
|
+
// disarm them and the N1 check covers the forced arm too
|
|
4787
|
+
// (§7.11). An off-roster rung hands `preRoute` its own
|
|
4788
|
+
// eligibility inputs.
|
|
4789
|
+
const cascadeForce = turnOpts.force;
|
|
4790
|
+
const cascadeForceReason = turnOpts.forceReason ?? "forced onto the cascade's escalation rung";
|
|
4791
|
+
const escalationTarget = cascadeForce !== undefined
|
|
4792
|
+
? cascadeForce
|
|
4793
|
+
: escalatePool
|
|
4794
|
+
? ((selfEscalation !== undefined
|
|
4795
|
+
? roster.find((c) => c.modelString === selfEscalation?.target.modelString)
|
|
4796
|
+
: undefined) ?? poolRouter.escalation())
|
|
4797
|
+
: undefined;
|
|
4798
|
+
const forced = budgetDegraded && budgetDegradeRung !== undefined
|
|
4799
|
+
? {
|
|
4800
|
+
...(roster.includes(budgetDegradeRung)
|
|
4801
|
+
? { armId: armIdOf(budgetDegradeRung) }
|
|
4802
|
+
: { candidate: preRouteArmOf(budgetDegradeRung) }),
|
|
4803
|
+
lane: "budget",
|
|
4804
|
+
reason: "budget_degrade",
|
|
4805
|
+
}
|
|
4806
|
+
: escalationTarget !== undefined
|
|
4807
|
+
? {
|
|
4808
|
+
...(roster.includes(escalationTarget)
|
|
4809
|
+
? { armId: armIdOf(escalationTarget) }
|
|
4810
|
+
: { candidate: preRouteArmOf(escalationTarget) }),
|
|
4811
|
+
lane: "escalation",
|
|
4812
|
+
reason: cascadeForce !== undefined
|
|
4813
|
+
? cascadeForceReason
|
|
4814
|
+
: selfEscalation !== undefined
|
|
4815
|
+
? `Escalate receipt ${selfEscalation.receipt}`
|
|
4816
|
+
: "pool failure",
|
|
4817
|
+
}
|
|
4818
|
+
: undefined;
|
|
4819
|
+
const pool = opts.modelPool;
|
|
4820
|
+
const classifierLabels = Object.keys(pool?.classifier?.labels ?? {});
|
|
4821
|
+
const routeClassifier = opts.routeClassifier;
|
|
4822
|
+
const classifierSkipReason = pool?.policy === "classifier" && judgeShareExhausted()
|
|
4823
|
+
? "judge_share_exhausted"
|
|
4824
|
+
: undefined;
|
|
4825
|
+
const pre = await preRoute({
|
|
4826
|
+
roster: preRouteArms,
|
|
4827
|
+
signals: routeSignals,
|
|
4828
|
+
turn: {
|
|
4829
|
+
hasImages: turnHasImages,
|
|
4830
|
+
contextTokens: routeSignals.contextTokens,
|
|
4831
|
+
toolsInPlay,
|
|
4832
|
+
},
|
|
4833
|
+
policy: pool?.policy ?? "static",
|
|
4834
|
+
...(forced !== undefined ? { forced } : {}),
|
|
4835
|
+
...(directivePin !== undefined ? { pin: directivePin } : {}),
|
|
4836
|
+
...(pool?.rules !== undefined ? { rules: pool.rules } : {}),
|
|
4837
|
+
...(pool?.policy === "classifier" && routeClassifier !== undefined
|
|
4838
|
+
? {
|
|
4839
|
+
classifier: async () => {
|
|
4840
|
+
// Once per turn: a later inner call of the same turn
|
|
4841
|
+
// (a tool iteration) reuses the verdict for the same
|
|
4842
|
+
// human message instead of re-classifying it.
|
|
4843
|
+
if (turnClassifierVerdict !== undefined &&
|
|
4844
|
+
turnClassifierVerdict.message === humanMessage) {
|
|
4845
|
+
return turnClassifierVerdict.verdict;
|
|
4846
|
+
}
|
|
4847
|
+
// The classifier is its own metered model call: it
|
|
4848
|
+
// takes the run's model-call limiter like the main
|
|
4849
|
+
// call (§7.2.3) and publishes on the run bus under
|
|
4850
|
+
// `role: "classifier"` so `judge_share` counts it.
|
|
4851
|
+
if (opts.rateLimiter &&
|
|
4852
|
+
opts.rateLimitKeys &&
|
|
4853
|
+
opts.rateLimitKeys.length > 0) {
|
|
4854
|
+
await opts.rateLimiter.acquire(opts.rateLimitKeys, 1);
|
|
4855
|
+
}
|
|
4856
|
+
const verdict = await routeClassifier({
|
|
4857
|
+
userText: humanText ?? "",
|
|
4858
|
+
labels: classifierLabels,
|
|
4859
|
+
bus,
|
|
4860
|
+
signal: turnAbort.signal,
|
|
4861
|
+
});
|
|
4862
|
+
turnClassifierVerdict = { message: humanMessage, verdict };
|
|
4863
|
+
return verdict;
|
|
4864
|
+
},
|
|
4865
|
+
classifierLabels,
|
|
4866
|
+
}
|
|
4867
|
+
: {}),
|
|
4868
|
+
...(classifierSkipReason !== undefined ? { classifierSkipReason } : {}),
|
|
4869
|
+
...(pool?.routing?.strongTag !== undefined
|
|
4870
|
+
? { strongTag: pool.routing.strongTag }
|
|
4871
|
+
: {}),
|
|
4872
|
+
});
|
|
4873
|
+
const base = poolRouter.route(routeSignals, poolSeed,
|
|
3251
4874
|
// Monotonic exploration sequence: the transcript length keeps
|
|
3252
4875
|
// advancing across `--resume` (and a channel-bot's resume-per-
|
|
3253
4876
|
// message), so ε-greedy draws a fresh coin per decision instead
|
|
3254
4877
|
// of freezing on the reset-to-0 turnIndex. Deterministic on
|
|
3255
4878
|
// replay (the same transcript reconstructs the same length).
|
|
3256
|
-
messages.length);
|
|
3257
|
-
|
|
4879
|
+
messages.length, pre.hint);
|
|
4880
|
+
// The loop substitutes the forced lanes itself (the degrade rung
|
|
4881
|
+
// may sit outside the roster); everything else the router
|
|
4882
|
+
// already applied from the hint. The band (`routeKey`) is kept
|
|
4883
|
+
// so the arm is still keyed on the turn's difficulty. When
|
|
4884
|
+
// `preRoute` found the forced arm ineligible, its result names
|
|
4885
|
+
// the eligible roster arm served in its place (§7.11).
|
|
4886
|
+
const forcedCandidate = forced === undefined
|
|
4887
|
+
? undefined
|
|
4888
|
+
: ((pre.forcedIneligible?.served !== undefined
|
|
4889
|
+
? roster.find((c) => armIdOf(c) === pre.forcedIneligible?.served)
|
|
4890
|
+
: undefined) ??
|
|
4891
|
+
(forced.lane === "budget" ? budgetDegradeRung : escalationTarget));
|
|
4892
|
+
let decision = forced?.lane === "budget" && forcedCandidate !== undefined
|
|
3258
4893
|
? {
|
|
3259
4894
|
...base,
|
|
3260
|
-
candidate:
|
|
3261
|
-
reason:
|
|
4895
|
+
candidate: forcedCandidate,
|
|
4896
|
+
reason: "budget_degrade",
|
|
4897
|
+
policy: "forced",
|
|
3262
4898
|
explored: false,
|
|
3263
4899
|
}
|
|
3264
|
-
:
|
|
4900
|
+
: forced?.lane === "escalation" && forcedCandidate !== undefined
|
|
4901
|
+
? cascadeForce !== undefined
|
|
4902
|
+
? {
|
|
4903
|
+
// 0.6.0 §7.3 — the cascade's escalation rung (or a
|
|
4904
|
+
// `forcedCandidate` run): pinned for the whole turn,
|
|
4905
|
+
// outranking the misroute latch, the directive pin
|
|
4906
|
+
// and the policy (`preRoute` may have swapped in the
|
|
4907
|
+
// strongest eligible arm when the rung failed N1).
|
|
4908
|
+
...base,
|
|
4909
|
+
candidate: forcedCandidate,
|
|
4910
|
+
reason: cascadeForceReason,
|
|
4911
|
+
policy: "forced",
|
|
4912
|
+
explored: false,
|
|
4913
|
+
}
|
|
4914
|
+
: {
|
|
4915
|
+
...base,
|
|
4916
|
+
candidate: forcedCandidate,
|
|
4917
|
+
policy: "escalation",
|
|
4918
|
+
reason: selfEscalation !== undefined
|
|
4919
|
+
? `escalated on the model's own Escalate request (receipt ${selfEscalation.receipt}: ${selfEscalation.reason}; was: ${base.reason})`
|
|
4920
|
+
: `escalated to strongest candidate after a pool failure (was: ${base.reason})`,
|
|
4921
|
+
explored: false,
|
|
4922
|
+
}
|
|
4923
|
+
: base;
|
|
4924
|
+
// §7.13 — an empty eligible set: the pick stands, the reason
|
|
4925
|
+
// says so, and a warning fires (the primary is always the
|
|
4926
|
+
// router's fallback here, so the run never halts on it). Under
|
|
4927
|
+
// a forced lane the same applies when the forced arm itself is
|
|
4928
|
+
// ineligible and no eligible arm remained to serve instead.
|
|
4929
|
+
const exclusionNotes = pre.excluded.map((e) => `${e.armId} ineligible (${e.reason})`);
|
|
4930
|
+
const reasonNotes = [...pre.notes, ...exclusionNotes];
|
|
4931
|
+
const nothingEligible = forced === undefined
|
|
4932
|
+
? pre.eligible.length === 0
|
|
4933
|
+
: pre.forcedIneligible !== undefined && pre.forcedIneligible.served === undefined;
|
|
4934
|
+
if (nothingEligible) {
|
|
4935
|
+
reasonNotes.push(`no-eligible-candidate (served ${armIdOf(decision.candidate)})`);
|
|
4936
|
+
runContext.logger.warn("model_pool: no eligible candidate for this turn", {
|
|
4937
|
+
picked: armIdOf(decision.candidate),
|
|
4938
|
+
excluded: pre.excluded,
|
|
4939
|
+
});
|
|
4940
|
+
}
|
|
4941
|
+
if (reasonNotes.length > 0) {
|
|
4942
|
+
decision = { ...decision, reason: `${decision.reason}; ${reasonNotes.join("; ")}` };
|
|
4943
|
+
}
|
|
4944
|
+
const poolPlan = planFor(decision.candidate);
|
|
3265
4945
|
turnAdapter = decision.candidate.adapter;
|
|
3266
4946
|
reqWireModelId = decision.candidate.modelId;
|
|
3267
4947
|
reqProviderId = decision.candidate.adapter.providerId;
|
|
3268
4948
|
reqSpecModel = decision.candidate.modelString;
|
|
3269
4949
|
poolTurn = {
|
|
3270
4950
|
modelString: decision.candidate.modelString,
|
|
4951
|
+
armId: armIdOf(decision.candidate),
|
|
3271
4952
|
routeKey: decision.routeKey,
|
|
4953
|
+
plan: poolPlan,
|
|
4954
|
+
candidate: decision.candidate,
|
|
4955
|
+
};
|
|
4956
|
+
// §5.3 / §8.1 — the routing attribution every route line gains:
|
|
4957
|
+
// the candidate's profile, its advertised-toolset fingerprint,
|
|
4958
|
+
// the eligible arms, the preRoute hint, the matched rule, the
|
|
4959
|
+
// classifier verdict and the pool's scope (each absent when the
|
|
4960
|
+
// pool declares nothing of the kind, so a bare pool's route
|
|
4961
|
+
// events keep their pre-0.6.0 shape — `eligible` and `hint` are
|
|
4962
|
+
// the two always present, since the phase always runs).
|
|
4963
|
+
const signalRecord = deriveSignalRecord(routeSignals);
|
|
4964
|
+
const { userTextHash, ...signalRest } = signalRecord;
|
|
4965
|
+
// §7.9 — the hybrid stage the decision serves. Present only on a
|
|
4966
|
+
// cascade / self-escalation attempt or a nested side call, so a
|
|
4967
|
+
// bare pool's route lines keep their pre-0.6.0 shape — and it is
|
|
4968
|
+
// what lets `watchme report --feed-routing` fold one delayed
|
|
4969
|
+
// quality onto EACH stage instead of the turn's first route.
|
|
4970
|
+
const routeStage = turnAttributionOverride.stage ?? opts.modelStage;
|
|
4971
|
+
const routeStrategy = turnAttributionOverride.strategy;
|
|
4972
|
+
const routeAttribution = {
|
|
4973
|
+
...(routeStage !== undefined ? { stage: routeStage } : {}),
|
|
4974
|
+
...(routeStrategy !== undefined ? { strategy: routeStrategy } : {}),
|
|
4975
|
+
...(poolPlan.profile !== undefined ? { profile: poolPlan.profile } : {}),
|
|
4976
|
+
...(poolPlan.fromPool ? { toolsetFingerprint: poolPlan.toolsetFingerprint } : {}),
|
|
4977
|
+
eligible: pre.eligible,
|
|
4978
|
+
hint: {
|
|
4979
|
+
source: pre.hint.source,
|
|
4980
|
+
...(pre.hint.forcedArm !== undefined ? { forcedArm: pre.hint.forcedArm } : {}),
|
|
4981
|
+
...(pre.hint.excludedArms !== undefined
|
|
4982
|
+
? { excludedArms: pre.hint.excludedArms }
|
|
4983
|
+
: {}),
|
|
4984
|
+
...(pre.hint.routeKeySuffix !== undefined
|
|
4985
|
+
? { routeKeySuffix: pre.hint.routeKeySuffix }
|
|
4986
|
+
: {}),
|
|
4987
|
+
...(Object.keys(pre.hint.evidence).length > 0
|
|
4988
|
+
? { evidence: describeHintEvidence(pre.hint.evidence) }
|
|
4989
|
+
: {}),
|
|
4990
|
+
},
|
|
4991
|
+
...(pre.ruleId !== undefined ? { ruleId: pre.ruleId } : {}),
|
|
4992
|
+
...(pre.classifierVerdict !== undefined
|
|
4993
|
+
? { classifierVerdict: pre.classifierVerdict }
|
|
4994
|
+
: {}),
|
|
4995
|
+
...(poolScope !== undefined ? { scope: poolScope } : {}),
|
|
4996
|
+
};
|
|
4997
|
+
// Derived values only — never the user's text (§7.2.2).
|
|
4998
|
+
const persistedSignals = {
|
|
4999
|
+
...signalRest,
|
|
5000
|
+
...(userTextHash !== undefined ? { textHash: userTextHash } : {}),
|
|
5001
|
+
};
|
|
5002
|
+
// 0.6.0 (design §8.1) — `specModel` (when it differs from the
|
|
5003
|
+
// wire id) closes the wire/spec identity split: scoreboard arms
|
|
5004
|
+
// key on the SPEC string, so this is what lets `route explain`
|
|
5005
|
+
// name the same arm `route status` shows.
|
|
5006
|
+
const routeSpecModel = decision.candidate.modelString !== decision.candidate.modelId
|
|
5007
|
+
? { specModel: decision.candidate.modelString }
|
|
5008
|
+
: {};
|
|
5009
|
+
// §7.10 / §7.9 — the floor verdict and the scoped-arm backoff
|
|
5010
|
+
// (each present only when the learned policy produced one).
|
|
5011
|
+
const learnedAttribution = {
|
|
5012
|
+
...(decision.floor !== undefined ? { floor: decision.floor } : {}),
|
|
5013
|
+
...(decision.backedOffTo !== undefined
|
|
5014
|
+
? { backedOffTo: decision.backedOffTo }
|
|
5015
|
+
: {}),
|
|
3272
5016
|
};
|
|
3273
5017
|
bus.publish({
|
|
3274
5018
|
...bus.envelope(),
|
|
@@ -3279,6 +5023,10 @@ export async function runChatLoop(opts) {
|
|
|
3279
5023
|
reason: decision.reason,
|
|
3280
5024
|
...(decision.explored ? { explored: true } : {}),
|
|
3281
5025
|
...(poolPolicyVersion !== undefined ? { policyVersion: poolPolicyVersion } : {}),
|
|
5026
|
+
...routeSpecModel,
|
|
5027
|
+
signals: persistedSignals,
|
|
5028
|
+
...routeAttribution,
|
|
5029
|
+
...learnedAttribution,
|
|
3282
5030
|
});
|
|
3283
5031
|
// Persist the decision so `crewhaus route explain <session>` can
|
|
3284
5032
|
// replay per-turn routing after the fact. Non-conversational, so
|
|
@@ -3291,10 +5039,18 @@ export async function runChatLoop(opts) {
|
|
|
3291
5039
|
reason: decision.reason,
|
|
3292
5040
|
explored: decision.explored,
|
|
3293
5041
|
...(poolPolicyVersion !== undefined ? { policyVersion: poolPolicyVersion } : {}),
|
|
5042
|
+
...routeSpecModel,
|
|
5043
|
+
signals: persistedSignals,
|
|
5044
|
+
...routeAttribution,
|
|
5045
|
+
...learnedAttribution,
|
|
3294
5046
|
});
|
|
3295
5047
|
}
|
|
3296
|
-
else if (tierRouter !== undefined) {
|
|
5048
|
+
else if (tierRouter !== undefined && !budgetDegraded) {
|
|
3297
5049
|
// Item 26 — two-tier turn-difficulty router. Pick a tier from
|
|
5050
|
+
// (0.6.0 §7.12 — bypassed once a budget degrade has swapped
|
|
5051
|
+
// `adapter`: both tiers collapse to the rung, which the default
|
|
5052
|
+
// path below now serves, so the `model_failover` the swap
|
|
5053
|
+
// published is true.)
|
|
3298
5054
|
// DETERMINISTIC per-turn signals (context size, tools-in-play,
|
|
3299
5055
|
// turn index, prior-turn tool_use density); a fast-tier misroute
|
|
3300
5056
|
// recovery forces `default` (see the catch below). Publish the
|
|
@@ -3325,17 +5081,39 @@ export async function runChatLoop(opts) {
|
|
|
3325
5081
|
reason: decision.reason,
|
|
3326
5082
|
...(escalateTier ? { escalated: true } : {}),
|
|
3327
5083
|
});
|
|
5084
|
+
// 0.6.0 (design §8.1) — the tier twin of the durable
|
|
5085
|
+
// `model_route` line: persisted beside the publish so `route
|
|
5086
|
+
// explain` covers tiers too. Non-conversational, so
|
|
5087
|
+
// `replayMessageHistory` ignores it and `--resume` is unaffected.
|
|
5088
|
+
await logEvent("model_tier_route", {
|
|
5089
|
+
turnNumber: runContext.turnNumber,
|
|
5090
|
+
tier: decision.tier,
|
|
5091
|
+
model: chosen.modelId,
|
|
5092
|
+
reason: decision.reason,
|
|
5093
|
+
...(escalateTier ? { escalated: true } : {}),
|
|
5094
|
+
});
|
|
3328
5095
|
}
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
5096
|
+
// 0.6.0 §4.4 — SELECT the plan: once per model call, after the
|
|
5097
|
+
// whole routing if-else chain (pool / tier / default / degraded
|
|
5098
|
+
// primary) and before the request is built. The pool branch chose
|
|
5099
|
+
// its candidate's plan; every other branch serves on the primary
|
|
5100
|
+
// plan, whose settings ARE the run's — so nothing below changes
|
|
5101
|
+
// for a single-model run. `servingPlan` is what the dispatch
|
|
5102
|
+
// gate, the permission / rate / tool_config seams and `ListTools`
|
|
5103
|
+
// read for every tool call this model call produces.
|
|
5104
|
+
const plan = poolTurn?.plan ?? degradedPlan ?? primaryPlan;
|
|
5105
|
+
servingPlan = plan;
|
|
5106
|
+
// The served arm for children: the pool candidate's identity when
|
|
5107
|
+
// one serves, else whatever this call is going to (a tier, the
|
|
5108
|
+
// degrade rung, or the primary — `reqSpecModel` is that model).
|
|
5109
|
+
servedArm = {
|
|
5110
|
+
model: reqSpecModel,
|
|
5111
|
+
wireModelId: reqWireModelId,
|
|
5112
|
+
...(plan.profile !== undefined ? { profile: plan.profile } : {}),
|
|
5113
|
+
armId: plan.fromPool ? plan.armId : reqSpecModel,
|
|
5114
|
+
fromPool: plan.fromPool,
|
|
5115
|
+
};
|
|
5116
|
+
listToolsSummaries = plan.listToolsSummaries;
|
|
3339
5117
|
const t0Model = performance.now();
|
|
3340
5118
|
modelCallStartMs = t0Model;
|
|
3341
5119
|
let streamChunkIndex = 0;
|
|
@@ -3362,50 +5140,105 @@ export async function runChatLoop(opts) {
|
|
|
3362
5140
|
// `[]` when the seam is absent — the request payload is then
|
|
3363
5141
|
// byte-identical to a pre-0.3.0 runtime.
|
|
3364
5142
|
const continuityTail = await buildContinuityTail();
|
|
3365
|
-
// G10 — arm the per-model-call watchdog for exactly the stream's
|
|
3366
|
-
// lifetime: cleared right after each drain below and (throw
|
|
3367
|
-
// paths) at the top of the catch. G01 — every MAIN-turn request
|
|
3368
|
-
// (and only these: compaction/judge side-calls build their own
|
|
3369
|
-
// requests) carries the resolved thinking fields, and the token
|
|
3370
|
-
// ceiling is the thinking-aware `effectiveMaxTokens`.
|
|
3371
|
-
armModelCallTimer();
|
|
3372
5143
|
// Item 9 / G79 — cache the settled transcript prefix when the
|
|
3373
5144
|
// serving adapter supports explicit caching. A request-local copy;
|
|
3374
5145
|
// the persisted `messages` array never carries the marker.
|
|
3375
|
-
|
|
5146
|
+
// 0.6.0 §4.4 — a candidate declared `caching: off` carries no
|
|
5147
|
+
// marker anywhere (system markers are stripped below too).
|
|
5148
|
+
const keepCacheMarkers = plan.cacheMode !== "off";
|
|
5149
|
+
const reqMessages = turnAdapter.features.caching === "explicit" && keepCacheMarkers
|
|
3376
5150
|
? withMessageCacheBreakpoint(messages)
|
|
3377
5151
|
: messages;
|
|
3378
|
-
|
|
5152
|
+
// 0.6.0 §4.4 item 1 — the request is built from the SERVING plan
|
|
5153
|
+
// and hoisted into a const so `model_request` can echo what the
|
|
5154
|
+
// adapter will actually send (`effectiveParams`, PR 5's SPI).
|
|
5155
|
+
// G01 — every MAIN-turn request (and only these: compaction /
|
|
5156
|
+
// judge side-calls build their own requests) carries the plan's
|
|
5157
|
+
// resolved thinking fields, and the token ceiling is the
|
|
5158
|
+
// thinking-aware `effectiveMaxTokens` — per candidate now.
|
|
5159
|
+
const req = {
|
|
3379
5160
|
model: reqWireModelId,
|
|
3380
5161
|
system: [
|
|
3381
5162
|
...systemBlocks.map((b) => ({
|
|
3382
5163
|
type: "text",
|
|
3383
5164
|
text: b.text,
|
|
3384
|
-
...(b.cache_control !== undefined
|
|
5165
|
+
...(b.cache_control !== undefined && keepCacheMarkers
|
|
5166
|
+
? { cache_control: b.cache_control }
|
|
5167
|
+
: {}),
|
|
3385
5168
|
})),
|
|
3386
5169
|
// Item 2 / G21 — the volatile recalled-memory block: rebuilt
|
|
3387
5170
|
// per-turn in per-turn recall mode, NO cache marker, sits in
|
|
3388
5171
|
// the mutable tail so a swap never busts the frozen prefix.
|
|
3389
5172
|
...volatileRecallBlocks.map((b) => ({ type: "text", text: b.text })),
|
|
3390
5173
|
...continuityTail.map((b) => ({ type: "text", text: b.text })),
|
|
5174
|
+
// 0.6.0 §7.4 — the `<guide>` block: volatile like the tail it
|
|
5175
|
+
// follows (after the cache marker, so a per-turn guide re-writes
|
|
5176
|
+
// only the transcript cache entry, never the frozen prefix),
|
|
5177
|
+
// and before the candidate overlay.
|
|
5178
|
+
...guideBlocks.map((b) => ({ type: "text", text: b.text })),
|
|
5179
|
+
// 0.6.0 §4.4 — the candidate's instructions overlay: appended
|
|
5180
|
+
// in the VOLATILE region after the continuity tail, never
|
|
5181
|
+
// before the cache marker, so switching candidates re-writes
|
|
5182
|
+
// only the tail and the frozen system prefix stays cached.
|
|
5183
|
+
...(plan.overlayBlock !== undefined
|
|
5184
|
+
? [{ type: "text", text: plan.overlayBlock.text }]
|
|
5185
|
+
: []),
|
|
5186
|
+
// 0.6.0 §7.4 — the `pre-model` hook's `mutate.systemAppend`,
|
|
5187
|
+
// volatile like the overlay (never cache-marked).
|
|
5188
|
+
...(hookSystemAppend !== undefined
|
|
5189
|
+
? [{ type: "text", text: hookSystemAppend }]
|
|
5190
|
+
: []),
|
|
3391
5191
|
],
|
|
3392
5192
|
messages: reqMessages,
|
|
3393
|
-
tools: anthropicTools !== undefined
|
|
3394
|
-
? anthropicTools.map((t) => ({
|
|
5193
|
+
tools: plan.anthropicTools !== undefined
|
|
5194
|
+
? plan.anthropicTools.map((t) => ({
|
|
3395
5195
|
name: t.name,
|
|
3396
5196
|
description: t.description ?? "",
|
|
3397
5197
|
input_schema: t.input_schema,
|
|
3398
5198
|
}))
|
|
3399
5199
|
: undefined,
|
|
3400
|
-
maxTokens: effectiveMaxTokens,
|
|
3401
|
-
...(
|
|
5200
|
+
maxTokens: plan.params.effectiveMaxTokens,
|
|
5201
|
+
...(plan.params.thinking !== undefined ? { thinking: plan.params.thinking } : {}),
|
|
5202
|
+
...(plan.params.reasoningEffort !== undefined
|
|
5203
|
+
? { reasoningEffort: plan.params.reasoningEffort }
|
|
5204
|
+
: {}),
|
|
5205
|
+
...(plan.params.temperature !== undefined
|
|
5206
|
+
? { temperature: plan.params.temperature }
|
|
5207
|
+
: {}),
|
|
3402
5208
|
signal: turnAbort.signal,
|
|
5209
|
+
};
|
|
5210
|
+
// The adapter's own echo of what it will send after its parameter
|
|
5211
|
+
// gating (Claude 5 drops `temperature`, …) — set only when the
|
|
5212
|
+
// serving adapter implements the projection and can project it
|
|
5213
|
+
// (a wrapper reports `undefined` for "cannot project").
|
|
5214
|
+
const effectiveParams = turnAdapter.effectiveParams?.(req);
|
|
5215
|
+
bus.publish({
|
|
5216
|
+
...modelStartEnv,
|
|
5217
|
+
kind: "model_request",
|
|
5218
|
+
model: reqWireModelId,
|
|
5219
|
+
...(reqSpecModel !== reqWireModelId ? { specModel: reqSpecModel } : {}),
|
|
5220
|
+
provider: reqProviderId,
|
|
5221
|
+
messageCount: messages.length,
|
|
5222
|
+
toolCount: plan.anthropicTools?.length ?? 0,
|
|
5223
|
+
streaming: opts.streaming === true,
|
|
5224
|
+
...modelAttribution(),
|
|
5225
|
+
...planAttribution(plan),
|
|
5226
|
+
...(effectiveParams !== undefined ? { effectiveParams } : {}),
|
|
3403
5227
|
});
|
|
5228
|
+
// G10 — arm the per-model-call watchdog for exactly the stream's
|
|
5229
|
+
// lifetime: cleared right after each drain below and (throw
|
|
5230
|
+
// paths) at the top of the catch.
|
|
5231
|
+
armModelCallTimer();
|
|
5232
|
+
const reqStream = turnAdapter.stream(req);
|
|
3404
5233
|
if (opts.streaming) {
|
|
3405
5234
|
// Streaming path: dispatch tools mid-stream via the
|
|
3406
5235
|
// refactored streaming-tool-executor (now consumes
|
|
3407
5236
|
// AsyncIterable<StreamEvent> directly).
|
|
3408
5237
|
void modelStartEnv;
|
|
5238
|
+
// 0.6.0 §7.9 — union of the mid-stream tool spans, so the pool
|
|
5239
|
+
// reward's latency term below is the MODEL's latency, not the
|
|
5240
|
+
// tools'. Wall time (`durationMs`, `turn_end`) is untouched.
|
|
5241
|
+
const toolSpans = createSpanUnionTracker();
|
|
3409
5242
|
const onTextDelta = (chunk) => {
|
|
3410
5243
|
writeText(chunk);
|
|
3411
5244
|
bus.publish({
|
|
@@ -3416,18 +5249,29 @@ export async function runChatLoop(opts) {
|
|
|
3416
5249
|
}, { ephemeral: true });
|
|
3417
5250
|
};
|
|
3418
5251
|
const { finalContent, toolResults, stopReason, usage } = await executeStreaming(reqStream, {
|
|
3419
|
-
|
|
5252
|
+
// 0.6.0 §4.4 — the serving candidate's advertised tools; a
|
|
5253
|
+
// name outside them still reaches `runTool`, where the
|
|
5254
|
+
// dispatch gate refuses it naming the profile.
|
|
5255
|
+
toolByName: plan.toolByName,
|
|
3420
5256
|
abortSignal: turnAbort.signal,
|
|
3421
5257
|
onTextDelta,
|
|
3422
5258
|
// Honor the same concurrency bound as the non-streaming
|
|
3423
5259
|
// path — otherwise `maxConcurrentTools` would be silently
|
|
3424
5260
|
// ignored under `streaming: true`.
|
|
3425
5261
|
maxConcurrent: opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS,
|
|
3426
|
-
runTool: (block) =>
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
5262
|
+
runTool: async (block) => {
|
|
5263
|
+
toolSpans.enter();
|
|
5264
|
+
try {
|
|
5265
|
+
return await executeOneToolUse({
|
|
5266
|
+
id: block.id,
|
|
5267
|
+
name: block.name,
|
|
5268
|
+
input: block.input,
|
|
5269
|
+
});
|
|
5270
|
+
}
|
|
5271
|
+
finally {
|
|
5272
|
+
toolSpans.exit();
|
|
5273
|
+
}
|
|
5274
|
+
},
|
|
3431
5275
|
onEvent: (e) => {
|
|
3432
5276
|
runContext.logger.debug("streaming-tool", { ...e });
|
|
3433
5277
|
},
|
|
@@ -3460,14 +5304,20 @@ export async function runChatLoop(opts) {
|
|
|
3460
5304
|
// Item 22 — lastServed() is exact: the candidate that actually
|
|
3461
5305
|
// streamed this response. Cost-tracker prices off this pair.
|
|
3462
5306
|
const servedStreaming = failoverChain?.lastServed();
|
|
3463
|
-
// With a pool, the served identity is the chosen candidate
|
|
3464
|
-
//
|
|
3465
|
-
|
|
5307
|
+
// With a pool, the served identity is the chosen candidate — or,
|
|
5308
|
+
// when the candidate is a per-profile failover chain (§4.4), the
|
|
5309
|
+
// chain member `lastServed()` names, so cost-tracker prices the
|
|
5310
|
+
// member that streamed and never bills a fallback's tokens to
|
|
5311
|
+
// the profile's primary. The ARM stays the candidate.
|
|
5312
|
+
const poolServedS = poolTurn?.candidate.lastServed?.();
|
|
5313
|
+
const respModelIdS = poolTurn !== undefined
|
|
5314
|
+
? (poolServedS?.modelId ?? reqWireModelId)
|
|
5315
|
+
: (servedStreaming?.modelId ?? wireModelId);
|
|
3466
5316
|
const respSpecModelS = poolTurn !== undefined
|
|
3467
|
-
? poolTurn.modelString
|
|
5317
|
+
? (poolServedS?.modelString ?? poolTurn.modelString)
|
|
3468
5318
|
: (servedStreaming?.modelString ?? degradedSpecModel ?? opts.model);
|
|
3469
5319
|
const respProviderS = poolTurn !== undefined
|
|
3470
|
-
? reqProviderId
|
|
5320
|
+
? (poolServedS?.providerId ?? reqProviderId)
|
|
3471
5321
|
: (servedStreaming?.providerId ?? providerId);
|
|
3472
5322
|
bus.publish({
|
|
3473
5323
|
...bus.envelope(),
|
|
@@ -3481,7 +5331,10 @@ export async function runChatLoop(opts) {
|
|
|
3481
5331
|
stopReason: stopReason ?? "end_turn",
|
|
3482
5332
|
usage,
|
|
3483
5333
|
durationMs: performance.now() - t0Model,
|
|
5334
|
+
...modelAttribution(),
|
|
5335
|
+
...planAttribution(plan),
|
|
3484
5336
|
});
|
|
5337
|
+
await recordServedModel(respModelIdS);
|
|
3485
5338
|
// Item 4 / G28 — record the provider's real input_tokens for the
|
|
3486
5339
|
// next pre-turn compaction trigger + routing signal. A response
|
|
3487
5340
|
// that reports NO usage (sum 0) means "unknown", not "empty":
|
|
@@ -3491,13 +5344,23 @@ export async function runChatLoop(opts) {
|
|
|
3491
5344
|
const reportedInputTokens = responseInputTokens(usage);
|
|
3492
5345
|
if (reportedInputTokens > 0)
|
|
3493
5346
|
lastModelInputTokens = reportedInputTokens;
|
|
5347
|
+
lastResponseWireModel = respModelIdS;
|
|
5348
|
+
attemptServedModel = respModelIdS;
|
|
3494
5349
|
// Adaptive model routing — fold this successful turn into the
|
|
3495
5350
|
// reward scoreboard so the learned policy improves next run.
|
|
3496
5351
|
if (poolTurn !== undefined) {
|
|
3497
5352
|
const costUsd = poolCostUsd(respProviderS, respModelIdS, usage);
|
|
3498
|
-
|
|
5353
|
+
// §4.4 — the arm's own spend, read by the per-profile cost cap.
|
|
5354
|
+
if (costUsd !== undefined)
|
|
5355
|
+
poolTurn.plan.spentUsdMicros += Math.round(costUsd * 1e6);
|
|
5356
|
+
if (costUsd !== undefined)
|
|
5357
|
+
attemptCostUsd += costUsd;
|
|
5358
|
+
attemptServedProfile = poolTurn.plan.profile;
|
|
5359
|
+
foldOutcome(poolTurn, {
|
|
3499
5360
|
success: true,
|
|
3500
|
-
|
|
5361
|
+
// §7.9 — model latency excluding the runTool spans (the
|
|
5362
|
+
// non-streaming fold below is already tool-free).
|
|
5363
|
+
latencyMs: modelLatencyMs(t0Model, performance.now(), toolSpans.busyMs()),
|
|
3501
5364
|
...(costUsd !== undefined ? { costUsd } : {}),
|
|
3502
5365
|
});
|
|
3503
5366
|
}
|
|
@@ -3512,6 +5375,8 @@ export async function runChatLoop(opts) {
|
|
|
3512
5375
|
}
|
|
3513
5376
|
// Item 26 — accrue this turn's tool_use count (next-turn tier signal).
|
|
3514
5377
|
thisTurnToolUseCount += toolUses.length;
|
|
5378
|
+
for (const tu of toolUses)
|
|
5379
|
+
thisTurnToolNames.push(tu.name);
|
|
3515
5380
|
for (const tu of toolUses) {
|
|
3516
5381
|
// (#386) — loop detection compares OPERATIVE inputs: a
|
|
3517
5382
|
// runtime-injected justification is re-worded every call, so
|
|
@@ -3527,7 +5392,7 @@ export async function runChatLoop(opts) {
|
|
|
3527
5392
|
await logEvent("user_message", { content: [...toolResults] });
|
|
3528
5393
|
const loopOutcome = evaluateToolLoop();
|
|
3529
5394
|
if (loopOutcome.kind === "message") {
|
|
3530
|
-
messages.push(loopOutcome.message);
|
|
5395
|
+
messages.push(markSynthetic(loopOutcome.message));
|
|
3531
5396
|
await logEvent("user_message", {
|
|
3532
5397
|
content: loopOutcome.message.content,
|
|
3533
5398
|
synthetic: true,
|
|
@@ -3572,12 +5437,19 @@ export async function runChatLoop(opts) {
|
|
|
3572
5437
|
// Item 22 — lastServed() is exact: the candidate that actually
|
|
3573
5438
|
// streamed this response. Cost-tracker prices off this pair.
|
|
3574
5439
|
const servedCandidate = failoverChain?.lastServed();
|
|
3575
|
-
// With a pool, the served identity is the chosen candidate
|
|
3576
|
-
|
|
5440
|
+
// With a pool, the served identity is the chosen candidate — or the
|
|
5441
|
+
// member its per-profile chain actually served (§4.4, see the
|
|
5442
|
+
// streaming site above).
|
|
5443
|
+
const poolServed = poolTurn?.candidate.lastServed?.();
|
|
5444
|
+
const respWireModelId = poolTurn !== undefined
|
|
5445
|
+
? (poolServed?.modelId ?? reqWireModelId)
|
|
5446
|
+
: (servedCandidate?.modelId ?? wireModelId);
|
|
3577
5447
|
const respSpecModel = poolTurn !== undefined
|
|
3578
|
-
? poolTurn.modelString
|
|
5448
|
+
? (poolServed?.modelString ?? poolTurn.modelString)
|
|
3579
5449
|
: (servedCandidate?.modelString ?? degradedSpecModel ?? opts.model);
|
|
3580
|
-
const respProvider = poolTurn !== undefined
|
|
5450
|
+
const respProvider = poolTurn !== undefined
|
|
5451
|
+
? (poolServed?.providerId ?? reqProviderId)
|
|
5452
|
+
: (servedCandidate?.providerId ?? providerId);
|
|
3581
5453
|
bus.publish({
|
|
3582
5454
|
...bus.envelope(),
|
|
3583
5455
|
spanId: modelStartEnv.spanId,
|
|
@@ -3590,7 +5462,10 @@ export async function runChatLoop(opts) {
|
|
|
3590
5462
|
stopReason: final.stopReason,
|
|
3591
5463
|
usage: final.usage,
|
|
3592
5464
|
durationMs: performance.now() - t0Model,
|
|
5465
|
+
...modelAttribution(),
|
|
5466
|
+
...planAttribution(plan),
|
|
3593
5467
|
});
|
|
5468
|
+
await recordServedModel(respWireModelId);
|
|
3594
5469
|
// Item 4 / G28 — record the provider's real input_tokens for the
|
|
3595
5470
|
// next pre-turn compaction trigger + routing signal. A response that
|
|
3596
5471
|
// reports NO usage (sum 0) means "unknown", not "empty": keep the
|
|
@@ -3600,11 +5475,19 @@ export async function runChatLoop(opts) {
|
|
|
3600
5475
|
const reportedInputTokens = responseInputTokens(final.usage);
|
|
3601
5476
|
if (reportedInputTokens > 0)
|
|
3602
5477
|
lastModelInputTokens = reportedInputTokens;
|
|
5478
|
+
lastResponseWireModel = respWireModelId;
|
|
5479
|
+
attemptServedModel = respWireModelId;
|
|
3603
5480
|
// Adaptive model routing — fold this successful turn into the
|
|
3604
5481
|
// reward scoreboard so the learned policy improves next run.
|
|
3605
5482
|
if (poolTurn !== undefined) {
|
|
3606
5483
|
const costUsd = poolCostUsd(respProvider, respWireModelId, final.usage);
|
|
3607
|
-
|
|
5484
|
+
// §4.4 — the arm's own spend, read by the per-profile cost cap.
|
|
5485
|
+
if (costUsd !== undefined)
|
|
5486
|
+
poolTurn.plan.spentUsdMicros += Math.round(costUsd * 1e6);
|
|
5487
|
+
if (costUsd !== undefined)
|
|
5488
|
+
attemptCostUsd += costUsd;
|
|
5489
|
+
attemptServedProfile = poolTurn.plan.profile;
|
|
5490
|
+
foldOutcome(poolTurn, {
|
|
3608
5491
|
success: true,
|
|
3609
5492
|
latencyMs: performance.now() - t0Model,
|
|
3610
5493
|
...(costUsd !== undefined ? { costUsd } : {}),
|
|
@@ -3749,6 +5632,8 @@ export async function runChatLoop(opts) {
|
|
|
3749
5632
|
// Item 26 — accrue this turn's tool_use count (a next-turn tier
|
|
3750
5633
|
// signal): a dense tool turn marks an active multi-step task.
|
|
3751
5634
|
thisTurnToolUseCount += state.toolUses.length;
|
|
5635
|
+
for (const tu of state.toolUses)
|
|
5636
|
+
thisTurnToolNames.push(tu.name);
|
|
3752
5637
|
let toolResults;
|
|
3753
5638
|
try {
|
|
3754
5639
|
toolResults = await runToolBatch(state.toolUses);
|
|
@@ -3772,7 +5657,7 @@ export async function runChatLoop(opts) {
|
|
|
3772
5657
|
await logEvent("user_message", { content: toolResults });
|
|
3773
5658
|
const loopOutcome = evaluateToolLoop();
|
|
3774
5659
|
if (loopOutcome.kind === "message") {
|
|
3775
|
-
messages.push(loopOutcome.message);
|
|
5660
|
+
messages.push(markSynthetic(loopOutcome.message));
|
|
3776
5661
|
await logEvent("user_message", {
|
|
3777
5662
|
content: loopOutcome.message.content,
|
|
3778
5663
|
synthetic: true,
|
|
@@ -3838,6 +5723,13 @@ export async function runChatLoop(opts) {
|
|
|
3838
5723
|
logger: runContext.logger,
|
|
3839
5724
|
onEvict: externalizeEvicted,
|
|
3840
5725
|
getLedgerText: ledgerAnchorText,
|
|
5726
|
+
// 0.6.0 §6.2 — the reactive summary is metered too.
|
|
5727
|
+
bus,
|
|
5728
|
+
compactionSpecModel,
|
|
5729
|
+
// 0.6.0 §4.2 — and carries the profile's pinned params.
|
|
5730
|
+
...(opts.compactionParams !== undefined
|
|
5731
|
+
? { compactionParams: opts.compactionParams }
|
|
5732
|
+
: {}),
|
|
3841
5733
|
}).finally(() => out.spinner.stop());
|
|
3842
5734
|
// §2.3 — persist the summary text alongside the counts (the
|
|
3843
5735
|
// reactive path always ends in autocompact's [marker, summary]).
|
|
@@ -3882,7 +5774,7 @@ export async function runChatLoop(opts) {
|
|
|
3882
5774
|
// runContext.turnNumber — and `crewhaus distill`'s deriveTurns
|
|
3883
5775
|
// skips them so its turn ordinal matches the runtime + web UI.
|
|
3884
5776
|
const continueMsg = "Please continue from where you left off.";
|
|
3885
|
-
messages.push({ role: "user", content: continueMsg });
|
|
5777
|
+
messages.push(markSynthetic({ role: "user", content: continueMsg }));
|
|
3886
5778
|
await logEvent("user_message", { content: continueMsg, synthetic: true });
|
|
3887
5779
|
state = transition(state, { kind: "RecoveryDone" });
|
|
3888
5780
|
break;
|
|
@@ -3911,7 +5803,7 @@ export async function runChatLoop(opts) {
|
|
|
3911
5803
|
});
|
|
3912
5804
|
}
|
|
3913
5805
|
const tombstoneMsg = "[previous assistant turn was rejected as invalid; please retry]";
|
|
3914
|
-
messages.push({ role: "user", content: tombstoneMsg });
|
|
5806
|
+
messages.push(markSynthetic({ role: "user", content: tombstoneMsg }));
|
|
3915
5807
|
await logEvent("user_message", { content: tombstoneMsg, synthetic: true });
|
|
3916
5808
|
state = transition(state, { kind: "RecoveryDone" });
|
|
3917
5809
|
break;
|
|
@@ -3948,12 +5840,16 @@ export async function runChatLoop(opts) {
|
|
|
3948
5840
|
// leaves a structured `run_failed` behind, then throw the
|
|
3949
5841
|
// byte-identical pre-0.3.0 RuntimeError (die()'s one-liner
|
|
3950
5842
|
// for non-RunFailedError CrewhausErrors is unchanged).
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
5843
|
+
// §7.13 — a `fallible` turn (the cascade's escalation stage)
|
|
5844
|
+
// has a caller that recovers with the draft: no `run_failed`.
|
|
5845
|
+
if (turnOpts.fallible !== true) {
|
|
5846
|
+
await publishRunFailed({
|
|
5847
|
+
class: "unknown",
|
|
5848
|
+
title: "recovery failed",
|
|
5849
|
+
detail: action.reason,
|
|
5850
|
+
exitCode: EXIT_CODES.generic,
|
|
5851
|
+
});
|
|
5852
|
+
}
|
|
3957
5853
|
throw new RuntimeError(`recovery failed: ${action.reason}`);
|
|
3958
5854
|
}
|
|
3959
5855
|
case "halt":
|
|
@@ -3975,6 +5871,7 @@ export async function runChatLoop(opts) {
|
|
|
3975
5871
|
// Item 26 — hand this turn's tool_use count to the next turn's tier
|
|
3976
5872
|
// decision (the prior-tool-density signal).
|
|
3977
5873
|
priorTurnToolUseCount = thisTurnToolUseCount;
|
|
5874
|
+
priorTurnToolNames = [...new Set(thisTurnToolNames)];
|
|
3978
5875
|
// G10 — the turn wound down through the Aborted transition; if a
|
|
3979
5876
|
// wall-clock timer (not a SIGINT) is what aborted it, classify the stop.
|
|
3980
5877
|
// The abort tree propagates reasons parent→child, so a run-deadline
|
|
@@ -3994,7 +5891,15 @@ export async function runChatLoop(opts) {
|
|
|
3994
5891
|
message: `turn aborted: the configured ${TIMEOUT_SPEC_KEYS[timedOut.crewhausTimeout]} (${timedOut.limitMs}ms) elapsed`,
|
|
3995
5892
|
});
|
|
3996
5893
|
}
|
|
3997
|
-
|
|
5894
|
+
turnAttributionOverride = {};
|
|
5895
|
+
return {
|
|
5896
|
+
terminalContent,
|
|
5897
|
+
usage: turnUsage,
|
|
5898
|
+
observations: pendingObservations,
|
|
5899
|
+
costUsd: attemptCostUsd,
|
|
5900
|
+
...(attemptServedModel !== undefined ? { servedModel: attemptServedModel } : {}),
|
|
5901
|
+
...(attemptServedProfile !== undefined ? { profile: attemptServedProfile } : {}),
|
|
5902
|
+
};
|
|
3998
5903
|
}
|
|
3999
5904
|
/** Concatenated text of a terminal content-block array (the same
|
|
4000
5905
|
* projection the `singleTurn` return value uses). */
|
|
@@ -4002,50 +5907,236 @@ export async function runChatLoop(opts) {
|
|
|
4002
5907
|
.filter((b) => b.type === "text")
|
|
4003
5908
|
.map((b) => b.text)
|
|
4004
5909
|
.join("");
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
5910
|
+
let warnedEscalateWithoutPool = false;
|
|
5911
|
+
const resolveCascade = () => {
|
|
5912
|
+
const evaluation = opts.evaluation;
|
|
5913
|
+
if (evaluation === undefined || evaluation.onFail !== "escalate")
|
|
5914
|
+
return undefined;
|
|
5915
|
+
if (poolRouter === undefined) {
|
|
5916
|
+
if (!warnedEscalateWithoutPool) {
|
|
5917
|
+
warnedEscalateWithoutPool = true;
|
|
5918
|
+
runContext.logger.warn("evaluation.on_fail: escalate needs a model_pool to escalate to — no pool is active, so a failing grade leaves the answer standing (note semantics)");
|
|
5919
|
+
}
|
|
5920
|
+
return undefined;
|
|
5921
|
+
}
|
|
5922
|
+
const declared = evaluation.escalateTo ?? opts.modelPool?.strategy?.cascade?.escalateTo;
|
|
5923
|
+
const resolved = declared !== undefined ? resolveRosterArm(declared) : undefined;
|
|
5924
|
+
if (declared !== undefined && resolved === undefined) {
|
|
5925
|
+
runContext.logger.warn("evaluation.escalateTo names no roster arm — escalating to the strongest candidate", { escalateTo: declared });
|
|
5926
|
+
}
|
|
5927
|
+
return {
|
|
5928
|
+
target: resolved ?? poolRouter.escalation(),
|
|
5929
|
+
cleanPrompt: evaluation.cleanPrompt ?? opts.modelPool?.strategy?.cascade?.cleanPrompt ?? false,
|
|
5930
|
+
maxEscalations: Math.max(0, Math.floor(evaluation.maxEscalations ?? opts.modelPool?.strategy?.maxEscalations ?? 1)),
|
|
5931
|
+
};
|
|
5932
|
+
};
|
|
5933
|
+
/** §7.10 — fold a cascade turn's deferred observations, then the strategy-arm line. */
|
|
5934
|
+
const flushCascade = (stages, final, wallMs) => {
|
|
5935
|
+
if (scoreboard === undefined)
|
|
5936
|
+
return;
|
|
5937
|
+
const draft = stages.find((st) => st.stage === "draft");
|
|
5938
|
+
const escalation = stages.find((st) => st.stage === "escalate");
|
|
5939
|
+
// The draft's counterfactual: did it score at least as well as the
|
|
5940
|
+
// escalation the judge then graded? (`true` ⇒ escalating bought nothing.)
|
|
5941
|
+
const wouldPass = draft?.score !== undefined && escalation?.score !== undefined
|
|
5942
|
+
? draft.score >= escalation.score
|
|
5943
|
+
: undefined;
|
|
5944
|
+
for (const st of stages) {
|
|
5945
|
+
for (const { turn, obs } of st.observations) {
|
|
5946
|
+
recordPoolOutcome(turn, {
|
|
5947
|
+
...obs,
|
|
5948
|
+
...(st.score !== undefined ? { quality: st.score } : {}),
|
|
5949
|
+
stage: st.stage,
|
|
5950
|
+
strategy: "cascade",
|
|
5951
|
+
attributedTo: st.attributedTo,
|
|
5952
|
+
...(st.stage === "draft" && wouldPass !== undefined ? { wouldPass } : {}),
|
|
5953
|
+
});
|
|
5954
|
+
}
|
|
5955
|
+
}
|
|
5956
|
+
const routeKey = stages.flatMap((st) => st.observations)[0]?.turn.routeKey;
|
|
5957
|
+
if (routeKey === undefined)
|
|
5958
|
+
return;
|
|
5959
|
+
// §7.10 / §7.13 — the strategy line folds the turn's FINAL quality. A
|
|
5960
|
+
// successful turn whose standing answer was never graded (its grader
|
|
5961
|
+
// threw, or the strong rung served directly past the judge share) has no
|
|
5962
|
+
// final quality to fold: no strategy line, rather than one an
|
|
5963
|
+
// `in_loop` reader would score as a perfect 1.0.
|
|
5964
|
+
if (final.success && final.score === undefined)
|
|
5965
|
+
return;
|
|
5966
|
+
const costUsd = stages.reduce((acc, st) => acc + st.costUsd, 0);
|
|
5967
|
+
recordPoolOutcome({ modelString: "strategy:cascade", routeKey }, {
|
|
5968
|
+
success: final.success,
|
|
5969
|
+
latencyMs: wallMs,
|
|
5970
|
+
...(costUsd > 0 ? { costUsd } : {}),
|
|
5971
|
+
...(final.score !== undefined ? { quality: final.score } : {}),
|
|
5972
|
+
strategy: "cascade",
|
|
5973
|
+
attributedTo: "strategy",
|
|
5974
|
+
});
|
|
5975
|
+
};
|
|
5976
|
+
/** §7.13 — the grader threw: the attempt's arms served but cannot be graded. */
|
|
5977
|
+
const discardUngraded = (attempt) => {
|
|
5978
|
+
for (const { turn } of attempt.observations) {
|
|
5979
|
+
scoreboard?.ungraded(turn.routeKey, turn.armId ?? turn.modelString);
|
|
5980
|
+
}
|
|
5981
|
+
};
|
|
5982
|
+
const admitGraderText = async (text, field) => {
|
|
5983
|
+
const trimmed = text?.trim() ?? "";
|
|
5984
|
+
if (trimmed.length === 0)
|
|
5985
|
+
return { kind: "empty" };
|
|
5986
|
+
const boundary = await classifyBoundary(trimmed, { origin: "consult" });
|
|
5987
|
+
if (boundary.action === "redact") {
|
|
5988
|
+
runContext.logger.warn("evaluation: grader text blocked at the transcript boundary — nudging with the score alone", { field, rules: [...new Set(boundary.verdict.hits.map((h) => h.rule))] });
|
|
5989
|
+
return { kind: "blocked" };
|
|
5990
|
+
}
|
|
5991
|
+
tagContent(runContext, trimmed, "consult");
|
|
5992
|
+
return { kind: "admitted", text: trimmed };
|
|
5993
|
+
};
|
|
5994
|
+
/** Run one turn on a forced rung as an `escalate` stage (started → done | failed). */
|
|
5995
|
+
const runEscalationStage = async (messages, target, cause, forceReason, defer, fallible = false) => {
|
|
5996
|
+
publishStage({
|
|
5997
|
+
stage: "escalate",
|
|
5998
|
+
strategy: "cascade",
|
|
5999
|
+
role: "escalation",
|
|
6000
|
+
candidate: target,
|
|
6001
|
+
outcome: "started",
|
|
6002
|
+
...(cause !== undefined ? { cause } : {}),
|
|
6003
|
+
});
|
|
6004
|
+
let served;
|
|
6005
|
+
try {
|
|
6006
|
+
served = await runOneTurn(messages, {
|
|
6007
|
+
force: target,
|
|
6008
|
+
forceReason,
|
|
6009
|
+
role: "escalation",
|
|
6010
|
+
stage: "escalate",
|
|
6011
|
+
strategy: "cascade",
|
|
6012
|
+
defer,
|
|
6013
|
+
...(fallible ? { fallible } : {}),
|
|
6014
|
+
});
|
|
6015
|
+
}
|
|
6016
|
+
catch (err) {
|
|
6017
|
+
publishStage({
|
|
6018
|
+
stage: "escalate",
|
|
6019
|
+
strategy: "cascade",
|
|
6020
|
+
role: "escalation",
|
|
6021
|
+
candidate: target,
|
|
6022
|
+
outcome: "failed",
|
|
6023
|
+
cause: err instanceof Error ? err.message : String(err),
|
|
6024
|
+
});
|
|
6025
|
+
throw err;
|
|
6026
|
+
}
|
|
6027
|
+
publishStage({
|
|
6028
|
+
stage: "escalate",
|
|
6029
|
+
strategy: "cascade",
|
|
6030
|
+
role: "escalation",
|
|
6031
|
+
candidate: target,
|
|
6032
|
+
outcome: "done",
|
|
6033
|
+
...(served.costUsd > 0 ? { costUsdMicros: Math.round(served.costUsd * 1e6) } : {}),
|
|
6034
|
+
});
|
|
6035
|
+
return served;
|
|
6036
|
+
};
|
|
4024
6037
|
async function runEvaluatedTurn(messages) {
|
|
4025
6038
|
const evaluation = opts.evaluation;
|
|
4026
|
-
|
|
6039
|
+
const cascade = resolveCascade();
|
|
6040
|
+
const turnStartMs = performance.now();
|
|
6041
|
+
// The pre-draft snapshot is a COPY of the messages, not a length: the
|
|
6042
|
+
// draft turn can rewrite `messages` in place (reactive compaction leaves
|
|
6043
|
+
// `[marker, summary]`, the tombstone reconciliation rebuilds it), so a
|
|
6044
|
+
// remembered length would extend a shorter array with holes instead of
|
|
6045
|
+
// reproducing the prompt the strong rung is promised under `cleanPrompt`.
|
|
6046
|
+
const preDraft = messages.slice();
|
|
6047
|
+
// §6.3 item 2 — under `in_loop` EVERY evaluated turn defers its
|
|
6048
|
+
// successful observations so the grade can be stamped on them; without
|
|
6049
|
+
// a judge the line count is unchanged and nothing is deferred.
|
|
6050
|
+
const deferForGrade = qualityIntoReward && evaluation !== undefined;
|
|
6051
|
+
// §6.2 — past the judge share a cascade serves the strong rung DIRECTLY:
|
|
6052
|
+
// the draft it cannot afford to grade is not drafted, the judge is not
|
|
6053
|
+
// skipped silently — the route reason and the stage cause both say why.
|
|
6054
|
+
if (cascade !== undefined && forcedRunCandidate === undefined && judgeShareExhausted()) {
|
|
6055
|
+
const served = await runEscalationStage(messages, cascade.target, "judge_share_exhausted", "judge_share_exhausted", true);
|
|
6056
|
+
// §6.3 — folded through the same quality gate as every cascade line:
|
|
6057
|
+
// the calls are un-judged, so under `in_loop` the strong arm is counted
|
|
6058
|
+
// `ungraded` rather than credited a perfect 1.0 for the rest of the
|
|
6059
|
+
// run; no strategy line (there is no final quality to fold).
|
|
6060
|
+
flushCascade([
|
|
6061
|
+
{
|
|
6062
|
+
stage: "escalate",
|
|
6063
|
+
attributedTo: "escalation",
|
|
6064
|
+
observations: served.observations,
|
|
6065
|
+
costUsd: served.costUsd,
|
|
6066
|
+
},
|
|
6067
|
+
], { success: true }, performance.now() - turnStartMs);
|
|
6068
|
+
return served;
|
|
6069
|
+
}
|
|
6070
|
+
let attempt = forcedRunCandidate !== undefined
|
|
6071
|
+
? await runEscalationStage(messages, forcedRunCandidate, "retry_previous", `forced onto ${armIdOf(forcedRunCandidate)} (forcedCandidate)`, cascade !== undefined || deferForGrade)
|
|
6072
|
+
: await runOneTurn(messages, cascade !== undefined
|
|
6073
|
+
? { role: "draft", stage: "draft", strategy: "cascade", defer: true }
|
|
6074
|
+
: deferForGrade
|
|
6075
|
+
? { defer: true }
|
|
6076
|
+
: {});
|
|
4027
6077
|
if (evaluation === undefined)
|
|
4028
6078
|
return attempt;
|
|
6079
|
+
/** Fold a non-cascade attempt's deferred lines with the grade it earned (none ⇒ the §6.3 gate). */
|
|
6080
|
+
const foldGraded = (a, quality) => {
|
|
6081
|
+
for (const { turn, obs } of a.observations) {
|
|
6082
|
+
recordPoolOutcome(turn, quality !== undefined ? { ...obs, quality } : obs);
|
|
6083
|
+
}
|
|
6084
|
+
};
|
|
4029
6085
|
const maxRetries = Math.max(0, Math.floor(evaluation.maxRetries));
|
|
6086
|
+
const stages = [];
|
|
6087
|
+
let current;
|
|
6088
|
+
if (cascade !== undefined) {
|
|
6089
|
+
current = {
|
|
6090
|
+
stage: "draft",
|
|
6091
|
+
attributedTo: "draft",
|
|
6092
|
+
observations: attempt.observations,
|
|
6093
|
+
costUsd: attempt.costUsd,
|
|
6094
|
+
};
|
|
6095
|
+
stages.push(current);
|
|
6096
|
+
}
|
|
6097
|
+
let escalations = 0;
|
|
6098
|
+
const gradedStages = () => stages.filter((st) => st.score !== undefined);
|
|
4030
6099
|
for (let retryIndex = 0;; retryIndex += 1) {
|
|
4031
6100
|
// A turn the abort tree stopped (SIGINT, turn/model-call/run timer)
|
|
4032
6101
|
// did not COMPLETE — grading its partial output would be noise, and
|
|
4033
6102
|
// a retry would fight the very signal that stopped it.
|
|
4034
|
-
if (turnAbort.signal.aborted || runAbort.signal.aborted)
|
|
6103
|
+
if (turnAbort.signal.aborted || runAbort.signal.aborted) {
|
|
6104
|
+
// An abort is not a judge outage: the deferred lines fold without a
|
|
6105
|
+
// quality (exactly what an un-deferred call would have recorded —
|
|
6106
|
+
// under `in_loop` the §6.3 gate counts them `ungraded`).
|
|
6107
|
+
if (cascade !== undefined) {
|
|
6108
|
+
flushCascade(stages, { success: false }, performance.now() - turnStartMs);
|
|
6109
|
+
}
|
|
6110
|
+
else {
|
|
6111
|
+
foldGraded(attempt);
|
|
6112
|
+
}
|
|
4035
6113
|
return attempt;
|
|
6114
|
+
}
|
|
4036
6115
|
let graded;
|
|
4037
6116
|
out.spinner.start("evaluating");
|
|
4038
6117
|
try {
|
|
4039
6118
|
graded = await evaluation.evaluate({
|
|
4040
6119
|
finalText: terminalText(attempt.terminalContent),
|
|
4041
6120
|
messages,
|
|
6121
|
+
// The runtime owns the synthetic marker; a transcript judge that
|
|
6122
|
+
// cannot see it grades its own retry nudge as a user turn.
|
|
6123
|
+
isSynthetic: isSyntheticMessage,
|
|
4042
6124
|
usage: attempt.usage,
|
|
6125
|
+
// 0.6.0 §6.2 — the judge publishes its model calls on THIS bus, so
|
|
6126
|
+
// the budget meter above (and any attached cost-tracker) sees them.
|
|
6127
|
+
bus,
|
|
6128
|
+
...(attempt.servedModel !== undefined ? { servedModel: attempt.servedModel } : {}),
|
|
6129
|
+
...(attempt.profile !== undefined ? { profile: attempt.profile } : {}),
|
|
6130
|
+
...(current !== undefined ? { stage: current.stage } : {}),
|
|
4043
6131
|
});
|
|
4044
6132
|
}
|
|
4045
6133
|
catch (err) {
|
|
4046
6134
|
// Grading INFRASTRUCTURE failure (judge model down, evaluator bug)
|
|
4047
6135
|
// is not a verdict. Fail open: record it and let the turn stand —
|
|
4048
|
-
// a flaky judge must not kill an otherwise healthy run.
|
|
6136
|
+
// a flaky judge must not kill an otherwise healthy run. §7.13 — a
|
|
6137
|
+
// cascade records NO arm line for the ungraded attempt (an omitted
|
|
6138
|
+
// quality would read as a perfect 1.0 through the reward) and counts
|
|
6139
|
+
// it as `ungraded`; stages already graded still fold.
|
|
4049
6140
|
const errObj = err;
|
|
4050
6141
|
const name = typeof errObj.name === "string" ? errObj.name : "EvaluationError";
|
|
4051
6142
|
const message = typeof errObj.message === "string" ? errObj.message : String(err);
|
|
@@ -4054,6 +6145,13 @@ export async function runChatLoop(opts) {
|
|
|
4054
6145
|
message,
|
|
4055
6146
|
});
|
|
4056
6147
|
await logEvent("error", { name, message: `evaluation grader failed: ${message}` });
|
|
6148
|
+
// A no-op for an un-deferred attempt (its lines already folded).
|
|
6149
|
+
discardUngraded(attempt);
|
|
6150
|
+
if (cascade !== undefined) {
|
|
6151
|
+
// Stages already graded still fold; with no final quality the
|
|
6152
|
+
// strategy line is withheld (see `flushCascade`).
|
|
6153
|
+
flushCascade(gradedStages(), { success: true }, performance.now() - turnStartMs);
|
|
6154
|
+
}
|
|
4057
6155
|
return attempt;
|
|
4058
6156
|
}
|
|
4059
6157
|
finally {
|
|
@@ -4064,6 +6162,22 @@ export async function runChatLoop(opts) {
|
|
|
4064
6162
|
const score = Number.isFinite(graded.score) ? graded.score : 0;
|
|
4065
6163
|
const rationale = graded.rationale;
|
|
4066
6164
|
const verdict = score >= evaluation.threshold ? "pass" : "fail";
|
|
6165
|
+
if (current !== undefined)
|
|
6166
|
+
current.score = score;
|
|
6167
|
+
// §6.3 item 2 — a non-cascade attempt folds with ITS grade right here
|
|
6168
|
+
// (each retry attempt is graded on its own); a cascade folds at the
|
|
6169
|
+
// strategy-turn boundary through `flushCascade`.
|
|
6170
|
+
if (cascade === undefined)
|
|
6171
|
+
foldGraded(attempt, score);
|
|
6172
|
+
// 0.6.0 §6.2 — read AFTER the grade so the judge call that just ran is
|
|
6173
|
+
// in the meter: the signal says "this grade was produced past the share".
|
|
6174
|
+
const shareExhausted = judgeShareExhausted();
|
|
6175
|
+
// §7.3 — decided BEFORE the publish so the failing grade names the rung
|
|
6176
|
+
// the turn is handed to (`escalatedTo`).
|
|
6177
|
+
const willEscalate = verdict === "fail" &&
|
|
6178
|
+
evaluation.onFail === "escalate" &&
|
|
6179
|
+
cascade !== undefined &&
|
|
6180
|
+
escalations < cascade.maxEscalations;
|
|
4067
6181
|
bus.publish({
|
|
4068
6182
|
...bus.envelope(),
|
|
4069
6183
|
kind: "eval_graded",
|
|
@@ -4076,9 +6190,36 @@ export async function runChatLoop(opts) {
|
|
|
4076
6190
|
// (`dataset mine`'s eval-fail signal) cannot distinguish a ladder that
|
|
4077
6191
|
// was fully spent from one cut short, and over-claims "exhausted".
|
|
4078
6192
|
maxRetries,
|
|
6193
|
+
// 0.6.0 §8.1 — attribution: the arm that produced the graded text, the
|
|
6194
|
+
// judge that scored it and what that judge call cost (grader-reported,
|
|
6195
|
+
// absent for deterministic graders), and the judge-share signal.
|
|
6196
|
+
...(attempt.servedModel !== undefined
|
|
6197
|
+
? { model: attempt.servedModel }
|
|
6198
|
+
: lastResponseWireModel !== undefined
|
|
6199
|
+
? { model: lastResponseWireModel }
|
|
6200
|
+
: {}),
|
|
6201
|
+
...(attempt.profile !== undefined ? { profile: attempt.profile } : {}),
|
|
6202
|
+
...(graded.judge !== undefined ? { judgeModel: graded.judge.model } : {}),
|
|
6203
|
+
...(graded.judge?.costUsdMicros !== undefined
|
|
6204
|
+
? { judgeCostUsdMicros: graded.judge.costUsdMicros }
|
|
6205
|
+
: {}),
|
|
6206
|
+
...(willEscalate && cascade !== undefined
|
|
6207
|
+
? { escalatedTo: cascade.target.modelString }
|
|
6208
|
+
: {}),
|
|
6209
|
+
...(shareExhausted ? { reason: "judge_share_exhausted" } : {}),
|
|
4079
6210
|
});
|
|
4080
|
-
|
|
6211
|
+
const standing = () => {
|
|
6212
|
+
if (cascade !== undefined) {
|
|
6213
|
+
flushCascade(stages, { success: verdict === "pass", score }, performance.now() - turnStartMs);
|
|
6214
|
+
}
|
|
4081
6215
|
return attempt;
|
|
6216
|
+
};
|
|
6217
|
+
if (verdict === "pass" || evaluation.onFail === "note")
|
|
6218
|
+
return standing();
|
|
6219
|
+
// `escalate` without a pool has nothing to escalate to (warned once at
|
|
6220
|
+
// resolve time): the verdict stands, note semantics.
|
|
6221
|
+
if (evaluation.onFail === "escalate" && cascade === undefined)
|
|
6222
|
+
return standing();
|
|
4082
6223
|
const rationaleNote = rationale !== undefined && rationale.trim().length > 0 ? ` — ${rationale.trim()}` : "";
|
|
4083
6224
|
if (evaluation.onFail === "halt") {
|
|
4084
6225
|
const detail = `the ${evaluation.graderType} grader scored the final answer ${score.toFixed(2)}, below the required ${evaluation.threshold} threshold${rationaleNote}`;
|
|
@@ -4089,22 +6230,168 @@ export async function runChatLoop(opts) {
|
|
|
4089
6230
|
remediation: "improve the agent (instructions/model), lower evaluation.threshold, or soften evaluation.on_fail to retry/note",
|
|
4090
6231
|
exitCode: EXIT_CODES.evaluation,
|
|
4091
6232
|
};
|
|
6233
|
+
standing();
|
|
4092
6234
|
await publishRunFailed(report);
|
|
4093
6235
|
throw new RunFailedError(report);
|
|
4094
6236
|
}
|
|
6237
|
+
// §7.3 draft-verify — the grader's own correction (a strong verifier's
|
|
6238
|
+
// `edits`) replaces the default rationale nudge; either way it is
|
|
6239
|
+
// APPENDED as a synthetic user message, never written into the draft.
|
|
6240
|
+
// Pillar 3 — whichever grader text is about to enter the transcript is
|
|
6241
|
+
// admitted through the `consult` boundary first; blocked ⇒ score-only.
|
|
6242
|
+
const admittedCorrection = await admitGraderText(graded.correction, "correction");
|
|
6243
|
+
const admittedRationale = admittedCorrection.kind === "empty"
|
|
6244
|
+
? await admitGraderText(rationale, "rationale")
|
|
6245
|
+
: { kind: "empty" };
|
|
6246
|
+
const feedback = admittedRationale.kind === "admitted" ? ` Grader feedback: ${admittedRationale.text}` : "";
|
|
6247
|
+
const correction = admittedCorrection.kind === "admitted"
|
|
6248
|
+
? `[evaluation failed: scored ${score.toFixed(2)}, threshold ${evaluation.threshold}]\nRequired correction: ${admittedCorrection.text}\nRevise your previous answer to apply the correction, then give the corrected answer in full.`
|
|
6249
|
+
: `[evaluation failed: scored ${score.toFixed(2)}, threshold ${evaluation.threshold}]${feedback}\nPlease revise your previous answer to address the feedback, then give the corrected answer in full.`;
|
|
6250
|
+
if (evaluation.onFail === "escalate" && cascade !== undefined) {
|
|
6251
|
+
if (!willEscalate) {
|
|
6252
|
+
// §7.13 — `max_escalations` reached: the last rung's answer stands.
|
|
6253
|
+
publishStage({
|
|
6254
|
+
stage: "escalate",
|
|
6255
|
+
strategy: "cascade",
|
|
6256
|
+
role: "escalation",
|
|
6257
|
+
candidate: cascade.target,
|
|
6258
|
+
outcome: "skipped",
|
|
6259
|
+
cause: "max_escalations",
|
|
6260
|
+
});
|
|
6261
|
+
return standing();
|
|
6262
|
+
}
|
|
6263
|
+
escalations += 1;
|
|
6264
|
+
// The transcript as graded — restored should the escalation rung be
|
|
6265
|
+
// unavailable (§7.13: the draft answer stands).
|
|
6266
|
+
const draftTranscript = messages.slice();
|
|
6267
|
+
if (cascade.cleanPrompt) {
|
|
6268
|
+
// The pre-draft snapshot: the strong rung answers the ORIGINAL
|
|
6269
|
+
// prompt, never seeing the rejected draft. The session log keeps
|
|
6270
|
+
// the draft's lines (they are already on disk); the durable
|
|
6271
|
+
// `model_stage` line marks the cut for `route explain`.
|
|
6272
|
+
messages.splice(0, messages.length, ...preDraft);
|
|
6273
|
+
}
|
|
6274
|
+
else {
|
|
6275
|
+
messages.push(markSynthetic({ role: "user", content: correction }));
|
|
6276
|
+
await logEvent("user_message", { content: correction, synthetic: true });
|
|
6277
|
+
}
|
|
6278
|
+
const stage = {
|
|
6279
|
+
stage: "escalate",
|
|
6280
|
+
attributedTo: "escalation",
|
|
6281
|
+
observations: [],
|
|
6282
|
+
costUsd: 0,
|
|
6283
|
+
};
|
|
6284
|
+
try {
|
|
6285
|
+
attempt = await runEscalationStage(messages, cascade.target, undefined, `cascade: escalated after a failing grade (${score.toFixed(2)} < ${evaluation.threshold})`, true, true);
|
|
6286
|
+
}
|
|
6287
|
+
catch (err) {
|
|
6288
|
+
// A classified halt (billing, auth, a spent rate-limit budget, a
|
|
6289
|
+
// fired run deadline) or an abort is terminal for the RUN, not for
|
|
6290
|
+
// the rung: the graded stages fold and the error propagates.
|
|
6291
|
+
if (isAbortError(err) || isRunFailedError(err)) {
|
|
6292
|
+
flushCascade(gradedStages(), { success: false }, performance.now() - turnStartMs);
|
|
6293
|
+
throw err;
|
|
6294
|
+
}
|
|
6295
|
+
// §7.13 — "cascade escalation target unavailable": fall back to
|
|
6296
|
+
// the draft answer with its failing grade attached; `on_fail`
|
|
6297
|
+
// behaves as `note`. The `model_stage{failed}` line already marks
|
|
6298
|
+
// the attempt; the draft transcript is restored (a `cleanPrompt`
|
|
6299
|
+
// cut re-appends the draft, the appended correction is withdrawn)
|
|
6300
|
+
// and the draft's arms fold with the grade they earned.
|
|
6301
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
6302
|
+
runContext.logger.warn("cascade: escalation rung unavailable — the draft answer stands with its failing grade (note semantics)", { escalateTo: cascade.target.modelString, message });
|
|
6303
|
+
messages.splice(0, messages.length, ...draftTranscript);
|
|
6304
|
+
return standing();
|
|
6305
|
+
}
|
|
6306
|
+
stage.observations = attempt.observations;
|
|
6307
|
+
stage.costUsd = attempt.costUsd;
|
|
6308
|
+
stages.push(stage);
|
|
6309
|
+
current = stage;
|
|
6310
|
+
continue;
|
|
6311
|
+
}
|
|
4095
6312
|
// onFail: "retry" — bounded by the resolved cap; when spent, the last
|
|
4096
6313
|
// attempt stands (the failing eval_graded trail tells the story).
|
|
4097
6314
|
if (retryIndex >= maxRetries)
|
|
4098
6315
|
return attempt;
|
|
4099
|
-
|
|
4100
|
-
? ` Grader feedback: ${rationale.trim()}`
|
|
4101
|
-
: "";
|
|
4102
|
-
const correction = `[evaluation failed: scored ${score.toFixed(2)}, threshold ${evaluation.threshold}]${feedback}\nPlease revise your previous answer to address the feedback, then give the corrected answer in full.`;
|
|
4103
|
-
messages.push({ role: "user", content: correction });
|
|
6316
|
+
messages.push(markSynthetic({ role: "user", content: correction }));
|
|
4104
6317
|
await logEvent("user_message", { content: correction, synthetic: true });
|
|
4105
|
-
attempt = await runOneTurn(messages);
|
|
6318
|
+
attempt = await runOneTurn(messages, deferForGrade ? { defer: true } : {});
|
|
4106
6319
|
}
|
|
4107
6320
|
}
|
|
6321
|
+
/**
|
|
6322
|
+
* 0.6.0 §7.6 — a committee turn on a single-turn host. N tool-less nested
|
|
6323
|
+
* member calls, a judge pick (order-controlled), and the strong tie-breaker
|
|
6324
|
+
* only on disagreement — all inside the injected closure; this wrapper owns
|
|
6325
|
+
* the lifecycle: the `judge_share` gate, the `model_stage` events, the
|
|
6326
|
+
* transcript / event-log append of the served text, and the fall-back to
|
|
6327
|
+
* the plain evaluated turn when the committee cannot decide at all
|
|
6328
|
+
* (§7.13). Without `sideCalls.committee` this IS `runEvaluatedTurn`.
|
|
6329
|
+
*/
|
|
6330
|
+
// §7.10 (PR 10 decision) — committee MEMBER arms do NOT fold into the live
|
|
6331
|
+
// scoreboard, and neither do shadow member arms until `route promote`
|
|
6332
|
+
// (PR 14). The committee's judge PICKS a winner: that is a pairwise
|
|
6333
|
+
// preference between members, not a per-member judged score on the same
|
|
6334
|
+
// instrument the floor and the live arms read, and "agreement" is an
|
|
6335
|
+
// acceptor signal that is never a reward (§7.6). So a committee turn
|
|
6336
|
+
// records no `model_route` and no arm line — its members carry cost only,
|
|
6337
|
+
// like the judge, guide, classifier and consult calls (§7.10). The winner's
|
|
6338
|
+
// identity rides the committee `model_stage` line for attribution.
|
|
6339
|
+
async function runCommitteeOrTurn(messages) {
|
|
6340
|
+
const committee = opts.sideCalls?.committee;
|
|
6341
|
+
if (committee === undefined)
|
|
6342
|
+
return runEvaluatedTurn(messages);
|
|
6343
|
+
const stage = {
|
|
6344
|
+
stage: "committee",
|
|
6345
|
+
strategy: "committee",
|
|
6346
|
+
role: "committee",
|
|
6347
|
+
model: committee.judge,
|
|
6348
|
+
};
|
|
6349
|
+
if (judgeShareExhausted()) {
|
|
6350
|
+
publishStage({ ...stage, outcome: "skipped", cause: "judge_share_exhausted" });
|
|
6351
|
+
return runEvaluatedTurn(messages);
|
|
6352
|
+
}
|
|
6353
|
+
// §7.12 / §14(2) — the committee bypasses `runOneTurn`, so the total cap
|
|
6354
|
+
// is gated HERE: a "stop" is the classified `crewhaus_budget` failure the
|
|
6355
|
+
// per-model-call gate would have raised (a committee replaces the primary
|
|
6356
|
+
// turn — no later gate would see the breach); a `degrade` latches the
|
|
6357
|
+
// rung for this turn and the members run. The closure re-reads the gate
|
|
6358
|
+
// between its members through `SideCallTurnContext.budgetGate`.
|
|
6359
|
+
if ((await enforceBudget(runContext.turnNumber)) === "stop") {
|
|
6360
|
+
publishStage({ ...stage, outcome: "skipped", cause: "budget" });
|
|
6361
|
+
await failBudgetStop(`the committee of turn ${runContext.turnNumber}`);
|
|
6362
|
+
}
|
|
6363
|
+
publishStage({ ...stage, outcome: "started" });
|
|
6364
|
+
out.spinner.start("committee");
|
|
6365
|
+
let verdict;
|
|
6366
|
+
try {
|
|
6367
|
+
verdict = await committee.run(sideCallContext(messages, turnAbort.signal));
|
|
6368
|
+
}
|
|
6369
|
+
catch (err) {
|
|
6370
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
6371
|
+
runContext.logger.warn("committee side call failed — falling back to the primary turn", {
|
|
6372
|
+
message,
|
|
6373
|
+
});
|
|
6374
|
+
publishStage({ ...stage, outcome: "failed", cause: message });
|
|
6375
|
+
out.spinner.stop();
|
|
6376
|
+
return runEvaluatedTurn(messages);
|
|
6377
|
+
}
|
|
6378
|
+
out.spinner.stop();
|
|
6379
|
+
const block = { type: "text", text: verdict.text, citations: null };
|
|
6380
|
+
messages.push({ role: "assistant", content: [block] });
|
|
6381
|
+
await logEvent("assistant_message", { content: [block] });
|
|
6382
|
+
out.write(`agent> ${verdict.text}\n`);
|
|
6383
|
+
lastResponseWireModel = verdict.winner.model;
|
|
6384
|
+
await recordServedModel(verdict.winner.model);
|
|
6385
|
+
publishStage({
|
|
6386
|
+
...stage,
|
|
6387
|
+
model: verdict.winner.model,
|
|
6388
|
+
...(verdict.winner.profile !== undefined ? { profile: verdict.winner.profile } : {}),
|
|
6389
|
+
outcome: "done",
|
|
6390
|
+
cause: verdict.cause ??
|
|
6391
|
+
(verdict.escalated ? "escalated" : verdict.agreed ? "agreed" : "disagreement"),
|
|
6392
|
+
});
|
|
6393
|
+
return { terminalContent: [block], usage: verdict.usage };
|
|
6394
|
+
}
|
|
4108
6395
|
// Single-shot path used by the workflow target and (Section 12) the
|
|
4109
6396
|
// channel-bot session router. Runs one user→assistant turn (with the tool
|
|
4110
6397
|
// inner-loop until Done), persists the transcript, returns the terminal
|
|
@@ -4122,14 +6409,29 @@ export async function runChatLoop(opts) {
|
|
|
4122
6409
|
// seed IS the entire turn and is logged through.
|
|
4123
6410
|
const replayed = resumedMessages ?? [];
|
|
4124
6411
|
let messages = [...replayed, ...seed];
|
|
6412
|
+
// 0.6.0 §7.2.1 — the single-turn directive seam: ONLY the last seed
|
|
6413
|
+
// message (the new inbound input) is parsed, never the replayed prefix.
|
|
6414
|
+
const seedDirective = await applyDirective(messageText(last), "seed");
|
|
4125
6415
|
for (const m of seed) {
|
|
4126
6416
|
if (m.role === "user") {
|
|
4127
|
-
await logEvent("user_message", {
|
|
6417
|
+
await logEvent("user_message", {
|
|
6418
|
+
content: m.content,
|
|
6419
|
+
...(m === last && seedDirective?.stripped === true ? { directive: true } : {}),
|
|
6420
|
+
});
|
|
4128
6421
|
}
|
|
4129
6422
|
else if (m.role === "assistant") {
|
|
4130
6423
|
await logEvent("assistant_message", { content: m.content });
|
|
4131
6424
|
}
|
|
4132
6425
|
}
|
|
6426
|
+
if (seedDirective?.stripped === true) {
|
|
6427
|
+
const hasOtherBlocks = typeof last.content !== "string" && last.content.some((b) => b.type !== "text");
|
|
6428
|
+
if (seedDirective.rest.length === 0 && !hasOtherBlocks) {
|
|
6429
|
+
// A directive-only message: the acknowledgement IS the reply; no
|
|
6430
|
+
// model call, no turn.
|
|
6431
|
+
return seedDirective.ack;
|
|
6432
|
+
}
|
|
6433
|
+
messages[messages.length - 1] = stripDirectiveFromMessage(last, seedDirective.rest);
|
|
6434
|
+
}
|
|
4133
6435
|
runContext.turnNumber += 1;
|
|
4134
6436
|
bus.setTurnNumber(runContext.turnNumber);
|
|
4135
6437
|
const t0SingleTurn = performance.now();
|
|
@@ -4184,8 +6486,11 @@ export async function runChatLoop(opts) {
|
|
|
4184
6486
|
// attempts inside the same armed window: a retried turn is still ONE
|
|
4185
6487
|
// turn against `turn_timeout_ms`.)
|
|
4186
6488
|
armTurnTimer();
|
|
4187
|
-
|
|
6489
|
+
// 0.6.0 §7.8 — the shadow re-runs exactly what the primary answered.
|
|
6490
|
+
const shadowSeed = [...messages];
|
|
6491
|
+
const { terminalContent } = await runCommitteeOrTurn(messages);
|
|
4188
6492
|
clearTurnTimer();
|
|
6493
|
+
maybeScheduleShadow(shadowSeed, terminalText(terminalContent));
|
|
4189
6494
|
bus.publish({
|
|
4190
6495
|
...bus.envelope(),
|
|
4191
6496
|
kind: "turn_end",
|
|
@@ -4195,6 +6500,20 @@ export async function runChatLoop(opts) {
|
|
|
4195
6500
|
return terminalText(terminalContent);
|
|
4196
6501
|
}
|
|
4197
6502
|
finally {
|
|
6503
|
+
// 0.6.0 §7.8 — the single turn IS the run. A per-message host
|
|
6504
|
+
// (`sideCallDrain`) takes the in-flight shadow and the served text
|
|
6505
|
+
// returns at once — the shadow is non-blocking there as on the REPL;
|
|
6506
|
+
// a one-shot host lets it finish before the bus and stores tear down
|
|
6507
|
+
// (its verdict is what the audition lane exists to record; the served
|
|
6508
|
+
// text is already final either way).
|
|
6509
|
+
if (opts.sideCallDrain !== undefined) {
|
|
6510
|
+
if (shadowInFlight !== undefined) {
|
|
6511
|
+
opts.sideCallDrain(shadowInFlight.then(() => undefined, () => undefined));
|
|
6512
|
+
}
|
|
6513
|
+
}
|
|
6514
|
+
else {
|
|
6515
|
+
await awaitShadow();
|
|
6516
|
+
}
|
|
4198
6517
|
clearAllTimers();
|
|
4199
6518
|
out.spinner.stop();
|
|
4200
6519
|
await fireHook("stop", {
|
|
@@ -4209,8 +6528,9 @@ export async function runChatLoop(opts) {
|
|
|
4209
6528
|
await subscribers.shutdownAll();
|
|
4210
6529
|
costPersistUnsubscribe?.();
|
|
4211
6530
|
failoverNoteUnsubscribe?.();
|
|
6531
|
+
routingPersist.unsubscribe();
|
|
4212
6532
|
incidentCollector?.unsubscribe();
|
|
4213
|
-
|
|
6533
|
+
ownBudgetMeter?.unsubscribe();
|
|
4214
6534
|
advisorPersist?.unsubscribe();
|
|
4215
6535
|
mcpStatsPersist?.unsubscribe();
|
|
4216
6536
|
watchmeCapture?.unsubscribe();
|
|
@@ -4372,12 +6692,39 @@ export async function runChatLoop(opts) {
|
|
|
4372
6692
|
// Item 27 — run-level spend cap. Enforced HERE (a real turn is about
|
|
4373
6693
|
// to run, input is not exit/blank) so an idle prompt never trips it.
|
|
4374
6694
|
// On a `stop` breach we break before committing the input — the turn
|
|
4375
|
-
// never runs. On a `degrade`
|
|
4376
|
-
// swapped the primary adapter,
|
|
4377
|
-
|
|
6695
|
+
// never runs, and the REPL ends cleanly (exit 0). On a `degrade`
|
|
6696
|
+
// breach `enforceBudget` has already swapped the primary adapter (or,
|
|
6697
|
+
// under a pool, latched the forced rung), so this turn runs on the
|
|
6698
|
+
// cheaper model. (0.6.0 §7.12 — a `stop` breach MID-turn is caught by
|
|
6699
|
+
// the per-model-call gate inside `runOneTurn`, which ends the run with
|
|
6700
|
+
// the classified `crewhaus_budget` failure instead; a `degrade` that
|
|
6701
|
+
// fired mid-turn finished that turn on the rung, and THIS gate is the
|
|
6702
|
+
// one that then ends the run cleanly.) `turnNumber` is incremented
|
|
6703
|
+
// just below, so the turn being gated is `turnNumber + 1`.
|
|
6704
|
+
if ((await enforceBudget(runContext.turnNumber + 1)) === "stop")
|
|
4378
6705
|
break;
|
|
4379
|
-
|
|
4380
|
-
|
|
6706
|
+
// 0.6.0 §7.2.1 — the `/model …` directive seam: parsed HERE, from the
|
|
6707
|
+
// typed input, before it joins the transcript. The persisted
|
|
6708
|
+
// `user_message` keeps the token (so replay reproduces the session);
|
|
6709
|
+
// the request copy carries the rest. A directive-only input runs no
|
|
6710
|
+
// turn — the acknowledgement is the reply.
|
|
6711
|
+
let requestInput = effectiveInput;
|
|
6712
|
+
const directive = await applyDirective(effectiveInput, "repl");
|
|
6713
|
+
if (directive !== undefined) {
|
|
6714
|
+
out.write(`[model] ${directive.ack}\n`);
|
|
6715
|
+
if (directive.stripped) {
|
|
6716
|
+
if (directive.rest.length === 0) {
|
|
6717
|
+
await logEvent("user_message", { content: effectiveInput, directive: true });
|
|
6718
|
+
continue;
|
|
6719
|
+
}
|
|
6720
|
+
requestInput = directive.rest;
|
|
6721
|
+
}
|
|
6722
|
+
}
|
|
6723
|
+
messages.push({ role: "user", content: requestInput });
|
|
6724
|
+
await logEvent("user_message", {
|
|
6725
|
+
content: effectiveInput,
|
|
6726
|
+
...(directive?.stripped === true ? { directive: true } : {}),
|
|
6727
|
+
});
|
|
4381
6728
|
runContext.turnNumber += 1;
|
|
4382
6729
|
bus.setTurnNumber(runContext.turnNumber);
|
|
4383
6730
|
const t0Turn = performance.now();
|
|
@@ -4428,11 +6775,16 @@ export async function runChatLoop(opts) {
|
|
|
4428
6775
|
// RunFailedError, which the catch rethrows.
|
|
4429
6776
|
armTurnTimer();
|
|
4430
6777
|
try {
|
|
6778
|
+
// 0.6.0 §7.8 — the shadow re-runs exactly what the primary answered.
|
|
6779
|
+
const shadowSeed = [...messages];
|
|
4431
6780
|
const { terminalContent } = await runEvaluatedTurn(messages);
|
|
4432
6781
|
// Item 1 — the exit rating prompt's "worth rating" floor: count only
|
|
4433
6782
|
// turns that ended in a real assistant answer.
|
|
4434
6783
|
if (terminalText(terminalContent).trim() !== "")
|
|
4435
6784
|
assistantAnswers += 1;
|
|
6785
|
+
// Non-blocking: scheduled after the answer is final, runs while the
|
|
6786
|
+
// REPL waits for the next input, awaited at teardown.
|
|
6787
|
+
maybeScheduleShadow(shadowSeed, terminalText(terminalContent));
|
|
4436
6788
|
}
|
|
4437
6789
|
catch (err) {
|
|
4438
6790
|
// Defensive: tear down any animation a thrown turn left spinning.
|
|
@@ -4472,6 +6824,8 @@ export async function runChatLoop(opts) {
|
|
|
4472
6824
|
replExitedCleanly = !runAbort.signal.aborted;
|
|
4473
6825
|
}
|
|
4474
6826
|
finally {
|
|
6827
|
+
// 0.6.0 §7.8 — let an in-flight shadow finish before teardown.
|
|
6828
|
+
await awaitShadow();
|
|
4475
6829
|
clearAllTimers();
|
|
4476
6830
|
out.spinner.stop();
|
|
4477
6831
|
await fireHook("stop", {
|
|
@@ -4486,8 +6840,9 @@ export async function runChatLoop(opts) {
|
|
|
4486
6840
|
await subscribers.shutdownAll();
|
|
4487
6841
|
costPersistUnsubscribe?.();
|
|
4488
6842
|
failoverNoteUnsubscribe?.();
|
|
6843
|
+
routingPersist.unsubscribe();
|
|
4489
6844
|
incidentCollector?.unsubscribe();
|
|
4490
|
-
|
|
6845
|
+
ownBudgetMeter?.unsubscribe();
|
|
4491
6846
|
advisorPersist?.unsubscribe();
|
|
4492
6847
|
mcpStatsPersist?.unsubscribe();
|
|
4493
6848
|
watchmeCapture?.unsubscribe();
|
|
@@ -4825,7 +7180,7 @@ function summaryTextOf(tuple) {
|
|
|
4825
7180
|
* the post-stage deltas; absent (pre-first-call) → the pure heuristic.
|
|
4826
7181
|
*/
|
|
4827
7182
|
async function maybeCompact(args, onCompaction) {
|
|
4828
|
-
const { messages, adapter, model, contextLimit, compactionThreshold, snipKeepHead, snipKeepTail, logger, onEvict, getLedgerText, realInputTokens, curate: curateCfg, onCurate, } = args;
|
|
7183
|
+
const { messages, adapter, model, contextLimit, compactionThreshold, snipKeepHead, snipKeepTail, logger, onEvict, getLedgerText, realInputTokens, curate: curateCfg, onCurate, bus, compactionSpecModel, compactionParams, } = args;
|
|
4829
7184
|
// Item 4 / G28 — real-token calibration. The initial array's real size is
|
|
4830
7185
|
// `realInputTokens` (the provider's count); every later stage subtracts the
|
|
4831
7186
|
// ESTIMATED tokens it freed. Pre-first-call (`realInputTokens` undefined) →
|
|
@@ -4904,7 +7259,12 @@ async function maybeCompact(args, onCompaction) {
|
|
|
4904
7259
|
if (onEvict !== undefined && snipped.length > 0)
|
|
4905
7260
|
await onEvict(snipped);
|
|
4906
7261
|
const ledgerText = getLedgerText?.();
|
|
4907
|
-
const after = await autoCompact(snipped, adapter, model,
|
|
7262
|
+
const after = await autoCompact(snipped, adapter, model, {
|
|
7263
|
+
...(ledgerText !== undefined ? { ledgerText } : {}),
|
|
7264
|
+
...(bus !== undefined ? { bus } : {}),
|
|
7265
|
+
...(compactionSpecModel !== undefined ? { specModel: compactionSpecModel } : {}),
|
|
7266
|
+
...(compactionParams !== undefined ? { params: compactionParams } : {}),
|
|
7267
|
+
});
|
|
4908
7268
|
if (onCompaction !== undefined) {
|
|
4909
7269
|
const summary = summaryTextOf(after);
|
|
4910
7270
|
await onCompaction({
|
|
@@ -4923,7 +7283,7 @@ async function maybeCompact(args, onCompaction) {
|
|
|
4923
7283
|
* largest blob).
|
|
4924
7284
|
*/
|
|
4925
7285
|
async function forceCompact(args) {
|
|
4926
|
-
const { messages, adapter, model, snipKeepHead, snipKeepTail, logger, onEvict, getLedgerText } = args;
|
|
7286
|
+
const { messages, adapter, model, snipKeepHead, snipKeepTail, logger, onEvict, getLedgerText, bus, compactionSpecModel, compactionParams, } = args;
|
|
4927
7287
|
const snipped = snip(messages, snipKeepHead, snipKeepTail);
|
|
4928
7288
|
// §2.3 — same externalize-before-drop contract as the pre-turn ladder:
|
|
4929
7289
|
// the snip diff first, then the full remaining history autocompact is
|
|
@@ -4937,5 +7297,10 @@ async function forceCompact(args) {
|
|
|
4937
7297
|
if (onEvict !== undefined && snipped.length > 0)
|
|
4938
7298
|
await onEvict(snipped);
|
|
4939
7299
|
const ledgerText = getLedgerText?.();
|
|
4940
|
-
return await autoCompact(snipped, adapter, model,
|
|
7300
|
+
return await autoCompact(snipped, adapter, model, {
|
|
7301
|
+
...(ledgerText !== undefined ? { ledgerText } : {}),
|
|
7302
|
+
...(bus !== undefined ? { bus } : {}),
|
|
7303
|
+
...(compactionSpecModel !== undefined ? { specModel: compactionSpecModel } : {}),
|
|
7304
|
+
...(compactionParams !== undefined ? { params: compactionParams } : {}),
|
|
7305
|
+
});
|
|
4941
7306
|
}
|