@crewhaus/runtime-core 0.5.8 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +53 -0
- package/dist/alert-watchdog.js +88 -7
- package/dist/ephemeral.d.ts +6 -0
- package/dist/ephemeral.js +88 -0
- package/dist/index.d.ts +795 -21
- package/dist/index.js +2575 -210
- package/dist/model-latency.d.ts +43 -0
- package/dist/model-latency.js +61 -0
- package/dist/observability.d.ts +47 -2
- package/dist/observability.js +110 -8
- package/dist/pre-route.d.ts +149 -0
- package/dist/pre-route.js +271 -0
- package/dist/slo-monitor.d.ts +20 -0
- package/dist/slo-monitor.js +45 -1
- package/package.json +42 -40
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type Anthropic from "@anthropic-ai/sdk";
|
|
2
|
-
import { type ProviderAdapter } from "@crewhaus/adapter-anthropic";
|
|
2
|
+
import { type ProviderAdapter, type ProviderFeatures, type ProviderId } from "@crewhaus/adapter-anthropic";
|
|
3
3
|
import type { CrewMailbox, SpawnSubAgentFn, SubAgentDefinition } from "@crewhaus/agent-context-isolation";
|
|
4
4
|
import { type EmbedderFn } from "@crewhaus/compaction-curator";
|
|
5
|
+
import { type CostTracker, type PricingTable } from "@crewhaus/cost-tracker";
|
|
5
6
|
import { type EgressMatcher, type SinkScope } from "@crewhaus/egress-classifier";
|
|
6
7
|
import { type FailureReport } from "@crewhaus/errors";
|
|
7
8
|
import { type EventLog } from "@crewhaus/event-log";
|
|
8
9
|
import { type HookDef } from "@crewhaus/hooks-engine";
|
|
10
|
+
import { type ModelProfile, type ModelThinking, type RequestParams, type RouteRule } from "@crewhaus/model-plan";
|
|
9
11
|
import { type JustificationJudge, type PermissionMode, type RuleSet } from "@crewhaus/permission-engine";
|
|
10
12
|
import { type RateLimiter } from "@crewhaus/rate-limiter";
|
|
11
13
|
import { type NamedFailureClass } from "@crewhaus/recovery-engine";
|
|
@@ -15,9 +17,11 @@ import { type PendingApproval, type PendingApprovalStore } from "@crewhaus/sessi
|
|
|
15
17
|
import { type SkillRef } from "@crewhaus/skills-registry";
|
|
16
18
|
import { type SlashCommand } from "@crewhaus/slash-commands";
|
|
17
19
|
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
18
|
-
import { type ModelUsage } from "@crewhaus/trace-event-bus";
|
|
20
|
+
import { type ModelRole, type ModelUsage, TraceEventBus } from "@crewhaus/trace-event-bus";
|
|
19
21
|
import { type ExitRatingFeedback } from "./exit-rating";
|
|
20
|
-
import { type
|
|
22
|
+
import { type AdvertisedToolSummary } from "./list-tools";
|
|
23
|
+
import { type AlertSink, type RoutingPersistedKind } from "./observability";
|
|
24
|
+
import { type PreRouteClassifierVerdict } from "./pre-route";
|
|
21
25
|
import type { SloMitigationSink, SloTargets } from "./slo-monitor";
|
|
22
26
|
/**
|
|
23
27
|
* Hard cap (chars) on the RENDERED mutable tail — the `<current_plan>` +
|
|
@@ -44,6 +48,14 @@ export declare const CONTINUITY_LEDGER_MAX_CHARS: number;
|
|
|
44
48
|
* first consumer.
|
|
45
49
|
*/
|
|
46
50
|
export declare const PLAN_DIRTY_STATE_KEY = "plan.dirty";
|
|
51
|
+
/**
|
|
52
|
+
* 0.6.0 (design §6.2) — the default `budget.judge_share`: the fraction of
|
|
53
|
+
* `budget.usd` the auxiliary roles (`AUXILIARY_MODEL_ROLES` from
|
|
54
|
+
* `@crewhaus/trace-event-bus`) may spend before the runtime raises the
|
|
55
|
+
* `judge_share_exhausted` signal. Applied when the spec's `budget` block
|
|
56
|
+
* omits `judge_share`.
|
|
57
|
+
*/
|
|
58
|
+
export declare const DEFAULT_JUDGE_SHARE = 0.3;
|
|
47
59
|
/** Role attribution on an evicted-content record (§2.3): `"user"` for user
|
|
48
60
|
* text, `"assistant"` for assistant text, `"tool"` for tool_result content
|
|
49
61
|
* (which the API carries inside user-role messages). */
|
|
@@ -124,6 +136,8 @@ export { createJanitor } from "./janitor";
|
|
|
124
136
|
export { JANITOR_BUILTIN_STEPS } from "./janitor";
|
|
125
137
|
export type { CreateJanitorOptions, Janitor, JanitorBuiltinStepName, JanitorReservationStore, JanitorRunResult, JanitorStep, JanitorStepName, JanitorStepOutcome, JanitorStepResult, JanitorStepStatus, } from "./janitor";
|
|
126
138
|
export { AGENT_IDENTITY_SCHEMA_VERSION, DEFAULT_IDENTITY_DIR, IDENTITY_FILENAME, agentFingerprint, loadOrCreateAgentIdentity, } from "./identity";
|
|
139
|
+
export { describeHintEvidence, isSyntheticMessage, latestHumanUserMessage, markSynthetic, messageHasImages, messageText, preRoute, stripDirectiveFromMessage, } from "./pre-route";
|
|
140
|
+
export type { PreRouteArm, PreRouteClassifierVerdict, PreRouteContext, PreRouteForced, PreRouteResult, } from "./pre-route";
|
|
127
141
|
export type { AgentIdentityFile } from "./identity";
|
|
128
142
|
export { type ApprovalDecision, type PendingApproval, type PendingApprovalStore, type PendingApprovalStoreOptions, createPendingApprovalStore, evictExpiredApprovals, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
129
143
|
/**
|
|
@@ -275,6 +289,17 @@ export type EvaluationTurn = {
|
|
|
275
289
|
* evaluators must not mutate it; the runtime owns the array).
|
|
276
290
|
*/
|
|
277
291
|
readonly messages: ReadonlyArray<Anthropic.MessageParam>;
|
|
292
|
+
/**
|
|
293
|
+
* 0.6.0 §6.2 — membership test for the runtime-INJECTED messages inside
|
|
294
|
+
* `messages` (retry nudges, cascade corrections, continue/tombstone
|
|
295
|
+
* prompts, the resumed-toolset marker). The marker lives in a
|
|
296
|
+
* module-private WeakSet, so nothing on the message object itself carries
|
|
297
|
+
* it and a grader in another package cannot recover it. A `target:
|
|
298
|
+
* "transcript"` judge MUST skip them: on attempt 2+ the retry nudge quotes
|
|
299
|
+
* the judge's own previous rationale, and grading that as a user
|
|
300
|
+
* instruction is the measurement-integrity hazard §7.2.1 exists to prevent.
|
|
301
|
+
*/
|
|
302
|
+
readonly isSynthetic?: (message: Anthropic.MessageParam) => boolean;
|
|
278
303
|
/**
|
|
279
304
|
* Aggregate token usage across every MAIN-turn model call this attempt
|
|
280
305
|
* made (all tool iterations included; compaction/judge side-calls are
|
|
@@ -283,6 +308,23 @@ export type EvaluationTurn = {
|
|
|
283
308
|
* invocation sees the usage of exactly the attempt it is grading.
|
|
284
309
|
*/
|
|
285
310
|
readonly usage: ModelUsage;
|
|
311
|
+
/**
|
|
312
|
+
* 0.6.0 (design §6.2) — the RUN's trace bus. The injected grader hands it
|
|
313
|
+
* to its judge (`judge({ …, bus })`) so the judge's model calls publish
|
|
314
|
+
* `model_request` / `model_response` with `role: "judge"` on the same bus
|
|
315
|
+
* the run's cost meter subscribes to — that is what makes judge spend
|
|
316
|
+
* priced, visible in `cost-summary`, and counted toward `budget.usd`
|
|
317
|
+
* (bounded by `budget.judge_share`). Deterministic graders ignore it.
|
|
318
|
+
*/
|
|
319
|
+
readonly bus: TraceEventBus;
|
|
320
|
+
/** 0.6.0 §7.3 — the WIRE model id that produced `finalText` (this attempt's
|
|
321
|
+
* last main-turn response), when known. */
|
|
322
|
+
readonly servedModel?: string;
|
|
323
|
+
/** 0.6.0 §7.3 — the `models:` profile of the serving pool candidate, when any. */
|
|
324
|
+
readonly profile?: string;
|
|
325
|
+
/** 0.6.0 §7.3 — the cascade stage this attempt is (`"draft"` / `"escalate"`);
|
|
326
|
+
* absent outside a cascade. */
|
|
327
|
+
readonly stage?: string;
|
|
286
328
|
};
|
|
287
329
|
/** The injected grader's verdict for one graded attempt. */
|
|
288
330
|
export type EvaluationResult = {
|
|
@@ -291,6 +333,43 @@ export type EvaluationResult = {
|
|
|
291
333
|
readonly score: number;
|
|
292
334
|
/** Grader explanation; appended to the retry nudge when `onFail: "retry"`. */
|
|
293
335
|
readonly rationale?: string;
|
|
336
|
+
/**
|
|
337
|
+
* 0.6.0 (design §6.2, §8.1) — the judge that produced the score, when the
|
|
338
|
+
* grader is model-backed: the WIRE model id and the call's priced spend
|
|
339
|
+
* (absent when the model has no pricing row). Stamped verbatim onto the
|
|
340
|
+
* `eval_graded` event as `judgeModel` / `judgeCostUsdMicros`. Deterministic
|
|
341
|
+
* graders leave it undefined.
|
|
342
|
+
*/
|
|
343
|
+
readonly judge?: {
|
|
344
|
+
readonly model: string;
|
|
345
|
+
readonly costUsdMicros?: number;
|
|
346
|
+
};
|
|
347
|
+
/**
|
|
348
|
+
* 0.6.0 §7.3 — draft-verify: the correction a strong verifier (`{ok, edits}`,
|
|
349
|
+
* `@crewhaus/eval-judge`'s `verifyDraft`) wants folded into the answer. On a
|
|
350
|
+
* failing verdict it REPLACES the default rationale nudge as the synthetic
|
|
351
|
+
* user message appended before the re-run (`retry`) or the escalation
|
|
352
|
+
* (`escalate` without `clean_prompt`). It is never written into the draft
|
|
353
|
+
* assistant message — a draft carrying `tool_use` blocks would orphan its
|
|
354
|
+
* `tool_result` pairs (the `isCuratableMessage` constraint) — and under
|
|
355
|
+
* `clean_prompt` it is not needed: the strong rung re-answers the original
|
|
356
|
+
* prompt from the pre-draft snapshot.
|
|
357
|
+
*/
|
|
358
|
+
readonly correction?: string;
|
|
359
|
+
};
|
|
360
|
+
/**
|
|
361
|
+
* 0.6.0 §4.2 — the request parameters a `models:` profile pins on an
|
|
362
|
+
* AUXILIARY model slot (judge / compaction / degrade / security / watchme /
|
|
363
|
+
* grounding), lowered by the compiler as `IrModelParams`. Structurally the
|
|
364
|
+
* profile half of `@crewhaus/model-plan`'s `buildRequestParams`, so a
|
|
365
|
+
* profile's `max_tokens` / `thinking` / `temperature` mean the same thing on
|
|
366
|
+
* an auxiliary slot as on a serving one. Absent everywhere → byte-identical
|
|
367
|
+
* to a pre-0.6.0 run.
|
|
368
|
+
*/
|
|
369
|
+
export type AuxRequestParams = {
|
|
370
|
+
readonly thinking?: ModelThinking;
|
|
371
|
+
readonly maxTokens?: number;
|
|
372
|
+
readonly temperature?: number;
|
|
294
373
|
};
|
|
295
374
|
/**
|
|
296
375
|
* Loop contract 0.4 (G02) — the in-loop evaluation seam (spec
|
|
@@ -311,21 +390,288 @@ export type RunEvaluation = {
|
|
|
311
390
|
/** Passing bar: `score >= threshold` passes. (The compiler resolves the
|
|
312
391
|
* spec default: 0.7 for llm_judge; deterministic graders use 1.) */
|
|
313
392
|
readonly threshold: number;
|
|
314
|
-
/**
|
|
315
|
-
|
|
393
|
+
/**
|
|
394
|
+
* What a failing verdict does — see the option docblock. 0.6.0 §7.3 adds
|
|
395
|
+
* `"escalate"` (spec `on_fail: escalate`), the CASCADE: the graded attempt
|
|
396
|
+
* is the cheap DRAFT (its calls carry `role: "draft"`, `stage: "draft"`);
|
|
397
|
+
* a failing grade re-runs the turn on the escalation rung through
|
|
398
|
+
* `runOneTurn(messages, { force })` (`policy: "forced"`, `role:
|
|
399
|
+
* "escalation"`, a `model_stage{stage: "escalate"}` pair around it), from
|
|
400
|
+
* the PRE-DRAFT snapshot when `cleanPrompt` (the draft is truncated out of
|
|
401
|
+
* the request) or with the draft kept plus the synthetic correction
|
|
402
|
+
* otherwise; the escalation is re-graded, bounded by `maxEscalations`
|
|
403
|
+
* (default 1 — a spent cap leaves the last rung's answer standing with a
|
|
404
|
+
* `model_stage{outcome: "skipped", cause: "max_escalations"}`). The pool
|
|
405
|
+
* outcomes of a cascade turn are folded at the TURN boundary, after the
|
|
406
|
+
* grade, each member-arm line carrying its stage's judged quality
|
|
407
|
+
* (§7.10); a grader that throws records no line and increments the arm's
|
|
408
|
+
* `ungraded` counter. Past `budget.judge_share` the cascade serves the
|
|
409
|
+
* strong rung DIRECTLY (`reason: "judge_share_exhausted"` on the route,
|
|
410
|
+
* `cause: "judge_share_exhausted"` on the stage) instead of spending more
|
|
411
|
+
* on judging. Without a `modelPool` there is nothing to escalate to: the
|
|
412
|
+
* verdict is recorded and the draft stands (`note` semantics), with one
|
|
413
|
+
* warning.
|
|
414
|
+
*/
|
|
415
|
+
readonly onFail: "retry" | "halt" | "note" | "escalate";
|
|
316
416
|
/** Hard cap on evaluation-triggered re-runs (>= 0; resolved default 1). */
|
|
317
417
|
readonly maxRetries: number;
|
|
418
|
+
/**
|
|
419
|
+
* 0.6.0 §7.3 — under `onFail: "escalate"`, the roster arm the failing turn
|
|
420
|
+
* is re-run on: a candidate's spec model string, its `models:` profile
|
|
421
|
+
* name (with or without the `$` sigil) or a tag (the first candidate
|
|
422
|
+
* carrying it). The compiler resolves it (`strategy.cascade.escalate_to`,
|
|
423
|
+
* else the pool's strongest); absent → the pool's `strategy.cascade.
|
|
424
|
+
* escalateTo`, else the router's strongest candidate. Never a model outside
|
|
425
|
+
* the roster: an unknown value falls back to the strongest, logged.
|
|
426
|
+
*/
|
|
427
|
+
readonly escalateTo?: string;
|
|
428
|
+
/**
|
|
429
|
+
* 0.6.0 §7.3 — truncate the request to the pre-draft snapshot before the
|
|
430
|
+
* escalation (`strategy.cascade.clean_prompt`); absent → the pool blob's
|
|
431
|
+
* value, else false (the draft stays, followed by the synthetic correction).
|
|
432
|
+
*/
|
|
433
|
+
readonly cleanPrompt?: boolean;
|
|
434
|
+
/** 0.6.0 §7.2 — escalations per turn (`strategy.max_escalations`); absent →
|
|
435
|
+
* the pool blob's value, else 1. */
|
|
436
|
+
readonly maxEscalations?: number;
|
|
318
437
|
/**
|
|
319
438
|
* Which grader kind produced the score — stamped verbatim onto every
|
|
320
439
|
* `eval_graded` trace event (the event field is required, and only the
|
|
321
440
|
* caller knows what it built `evaluate` from).
|
|
322
441
|
*/
|
|
323
442
|
readonly graderType: "llm_judge" | "contains" | "regex";
|
|
443
|
+
/**
|
|
444
|
+
* 0.6.0 §6.2 (PR 13, the PR-10 carry-over) — the JUDGE MODEL `evaluate`
|
|
445
|
+
* calls, when it calls one. Purely declarative: the closure still owns the
|
|
446
|
+
* call, this states which model it makes it with, because the identity of
|
|
447
|
+
* the judge is part of the instrument a learned arm's quality was measured
|
|
448
|
+
* on. Under `reward.quality_source`, the per-arm lineage folds it beside
|
|
449
|
+
* `graderType` (§6.3 item 4 — `reset_on_profile_change`), so re-pointing
|
|
450
|
+
* `evaluation.grader.model` at a different judge starts a NEW lineage
|
|
451
|
+
* instead of averaging two instruments' verdicts into one arm.
|
|
452
|
+
*
|
|
453
|
+
* Emitted by the three `renderEvaluation` copies and the interpreter's
|
|
454
|
+
* `buildRunEvaluation` for `llm_judge` graders only — a `contains`/`regex`
|
|
455
|
+
* grader has no judge, and their bundles stay byte-identical. 0.6.0 §6.2
|
|
456
|
+
* (PR 13b): under a declared `judges` PANEL this is every panelist joined
|
|
457
|
+
* with `+`, because two panels are two instruments and a per-arm lineage
|
|
458
|
+
* keyed on this string must re-baseline rather than average them.
|
|
459
|
+
*/
|
|
460
|
+
readonly judgeModel?: string;
|
|
461
|
+
};
|
|
462
|
+
/**
|
|
463
|
+
* 0.6.0 §7.2.4 — the seam between the `Escalate` tool and the loop (the
|
|
464
|
+
* structural twin of `@crewhaus/tool-consult`'s `EscalationLatch`; kept
|
|
465
|
+
* structural so runtime-core does not depend on the tool package). The tool
|
|
466
|
+
* records a request when the serving model asks for a stronger sibling; the
|
|
467
|
+
* loop consumes it exactly once, at its next model call, snapshotting the
|
|
468
|
+
* transcript length onto the record.
|
|
469
|
+
*/
|
|
470
|
+
export type HybridEscalationLatch = {
|
|
471
|
+
/** The accepted, not-yet-consumed request, if any. */
|
|
472
|
+
pending(): HybridEscalationRequest | undefined;
|
|
473
|
+
/** Take the pending request (clearing it), stamping the loop's snapshot. */
|
|
474
|
+
consume(snapshot: {
|
|
475
|
+
readonly transcriptLength: number;
|
|
476
|
+
}): HybridEscalationRequest | undefined;
|
|
477
|
+
};
|
|
478
|
+
export type HybridEscalationRequest = {
|
|
479
|
+
readonly receipt: number;
|
|
480
|
+
readonly reason: string;
|
|
481
|
+
/** The roster candidate the escalation targets (its spec model string). */
|
|
482
|
+
readonly target: {
|
|
483
|
+
readonly modelString: string;
|
|
484
|
+
readonly profile?: string;
|
|
485
|
+
};
|
|
486
|
+
readonly turnNumber: number;
|
|
487
|
+
};
|
|
488
|
+
/**
|
|
489
|
+
* 0.6.0 §7.4 / §7.6 / §7.8 (PR 9d) — the hybrid SIDE CALLS: closures the
|
|
490
|
+
* composition root (`@crewhaus/model-service`'s `wireModels`) builds from
|
|
491
|
+
* `model_pool.strategy` and hands the loop. Each runs as a nested single-turn
|
|
492
|
+
* `runChatLoop` on its OWN child `RunContext` and child bus — the side-call
|
|
493
|
+
* isolation contract (§7.6, §10.2): never a parallel `runOneTurn`, never the
|
|
494
|
+
* parent's turn counter, `turnAbort`, `lastErrorForRecovery` or watchdog
|
|
495
|
+
* slots — with the child's model events re-published on the PARENT bus so
|
|
496
|
+
* the parent's meter counts them under `budget.judge_share`, and with
|
|
497
|
+
* `persistSession: false` so no child session file is written (§16 Q6).
|
|
498
|
+
* runtime-core owns the LIFECYCLE (when a side call runs, what its text
|
|
499
|
+
* does, the `model_stage` events, the scoreboard fold); the closures own the
|
|
500
|
+
* model calls. Absent → byte-identical to a pre-0.6.0 runtime.
|
|
501
|
+
*/
|
|
502
|
+
export type SideCallTurnContext = {
|
|
503
|
+
/** The PARENT run's context — the closure mints its child from it. */
|
|
504
|
+
readonly runContext: RunContext;
|
|
505
|
+
/** The parent bus — the closure re-publishes its child's model events here. */
|
|
506
|
+
readonly bus: TraceEventBus;
|
|
507
|
+
readonly turnNumber: number;
|
|
508
|
+
/** The run's user instructions (the system prompt a side call may reuse). */
|
|
509
|
+
readonly instructions: string;
|
|
510
|
+
/**
|
|
511
|
+
* The transcript as of the side call — for a guide, before the turn's first
|
|
512
|
+
* model call; for a shadow, exactly the messages the primary answered (the
|
|
513
|
+
* turn's input last). A snapshot the closure may keep; the runtime's live
|
|
514
|
+
* array is never handed out.
|
|
515
|
+
*/
|
|
516
|
+
readonly messages: ReadonlyArray<Anthropic.MessageParam>;
|
|
517
|
+
/** Aborting this signal aborts the side call (the turn's for guide and
|
|
518
|
+
* committee, the run's for the post-response shadow). */
|
|
519
|
+
readonly signal: AbortSignal;
|
|
520
|
+
/** The pool's scope (step / node / role name), when any. */
|
|
521
|
+
readonly scope?: string;
|
|
522
|
+
/**
|
|
523
|
+
* §7.12 / §14(2) — the run's budget gate (`enforceBudget` for the current
|
|
524
|
+
* turn), so a closure that makes SEVERAL model calls (the committee's
|
|
525
|
+
* members and tie-breaker) can re-read the total cap between them: `"stop"`
|
|
526
|
+
* means the cap is reached — exclude every call not yet made (cause
|
|
527
|
+
* `budget`) rather than open it. Absent when the loop has no `budget`.
|
|
528
|
+
* The single-call closures (guide, shadow) are gated by the loop itself
|
|
529
|
+
* before they start.
|
|
530
|
+
*/
|
|
531
|
+
readonly budgetGate?: () => Promise<"continue" | "stop">;
|
|
532
|
+
};
|
|
533
|
+
/** What a guide call produced: text to inject, or `null` with a `cause`. */
|
|
534
|
+
export type GuideReply = {
|
|
535
|
+
readonly text: string | null;
|
|
536
|
+
readonly cause?: string;
|
|
537
|
+
};
|
|
538
|
+
/**
|
|
539
|
+
* §7.4 — the guide (plan-execute): a bounded single-turn call on the guide
|
|
540
|
+
* model whose text the loop appends in the VOLATILE system region as a
|
|
541
|
+
* `<guide>` block (after the continuity tail, before the candidate overlay),
|
|
542
|
+
* classified at TrustOrigin `consult` and lineage-tagged. `every: "first_turn"`
|
|
543
|
+
* IS plan-execute — the strong model plans once, the block stays for the rest
|
|
544
|
+
* of the run, and the plan is written to the continuity plan store when
|
|
545
|
+
* `continuity.savePlan` is wired; `every: "turn"` re-plans every turn (and
|
|
546
|
+
* re-writes the transcript cache entry every turn — the documented cost).
|
|
547
|
+
*/
|
|
548
|
+
export type GuideSideCall = {
|
|
549
|
+
readonly every: "first_turn" | "turn";
|
|
550
|
+
/** Spec model string of the guide (for `model_stage`). */
|
|
551
|
+
readonly model: string;
|
|
552
|
+
readonly profile?: string;
|
|
553
|
+
readonly run: (ctx: SideCallTurnContext) => Promise<GuideReply>;
|
|
554
|
+
};
|
|
555
|
+
/** The blind, order-swapped pairwise verdict of one shadow run. */
|
|
556
|
+
export type ShadowVerdict = {
|
|
557
|
+
readonly verdict: "shadow" | "primary" | "tie";
|
|
558
|
+
/** True when both presentation orders agreed (a disagreement is a tie). */
|
|
559
|
+
readonly agreed: boolean;
|
|
560
|
+
/** Wire model id that served the shadow call, and its provider. */
|
|
561
|
+
readonly model: string;
|
|
562
|
+
readonly provider: ProviderId;
|
|
563
|
+
readonly usage: ModelUsage;
|
|
564
|
+
readonly latencyMs: number;
|
|
565
|
+
readonly judgeModel?: string;
|
|
566
|
+
};
|
|
567
|
+
/**
|
|
568
|
+
* §7.8 — the audition lane: the same request re-run on the shadow candidate
|
|
569
|
+
* AFTER the primary has answered, on a sampled share of turns, graded blind
|
|
570
|
+
* against the primary; both arms record into `shadow:<scope>/<band>`. The
|
|
571
|
+
* shadow's text never reaches the transcript — the closure returns only the
|
|
572
|
+
* verdict.
|
|
573
|
+
*/
|
|
574
|
+
export type ShadowSideCall = {
|
|
575
|
+
/** Spec model string of the shadow candidate — its arm id. */
|
|
576
|
+
readonly candidate: string;
|
|
577
|
+
readonly profile?: string;
|
|
578
|
+
/** Share of turns that run the shadow, in [0, 1]. */
|
|
579
|
+
readonly sampleRate: number;
|
|
580
|
+
readonly run: (ctx: SideCallTurnContext & {
|
|
581
|
+
readonly primaryText: string;
|
|
582
|
+
readonly primaryModel: string;
|
|
583
|
+
}) => Promise<ShadowVerdict>;
|
|
584
|
+
};
|
|
585
|
+
export type CommitteeMemberOutcome = {
|
|
586
|
+
readonly modelString: string;
|
|
587
|
+
readonly profile?: string;
|
|
588
|
+
/** Wire model id that served (absent when the member failed before serving). */
|
|
589
|
+
readonly model?: string;
|
|
590
|
+
/** `skipped`: the member never ran — the budget gate stopped the committee
|
|
591
|
+
* before its turn came (`cause: "budget"`). */
|
|
592
|
+
readonly outcome: "done" | "failed" | "skipped";
|
|
593
|
+
readonly latencyMs: number;
|
|
594
|
+
readonly usage?: ModelUsage;
|
|
595
|
+
/** The member's judged quality (1 won, 0 lost, 0.5 unresolved), when judged. */
|
|
596
|
+
readonly quality?: number;
|
|
597
|
+
readonly cause?: string;
|
|
324
598
|
};
|
|
599
|
+
/** The committee's decision for one single-turn host turn. */
|
|
600
|
+
export type CommitteeVerdict = {
|
|
601
|
+
/** The served text — the judged winner's, or the tie-breaker's. */
|
|
602
|
+
readonly text: string;
|
|
603
|
+
readonly winner: {
|
|
604
|
+
readonly modelString: string;
|
|
605
|
+
readonly model: string;
|
|
606
|
+
readonly profile?: string;
|
|
607
|
+
};
|
|
608
|
+
/** True when the judge's two presentation orders agreed on the winner. */
|
|
609
|
+
readonly agreed: boolean;
|
|
610
|
+
/** True when the disagreement tie-breaker produced the served text. */
|
|
611
|
+
readonly escalated: boolean;
|
|
612
|
+
/** How the served text was chosen (`"agreed"`, `"escalated"`,
|
|
613
|
+
* `"disagreement"`, `"single-member"`, `"judge-failed"`, …); stamped on
|
|
614
|
+
* the committee's `model_stage` line. */
|
|
615
|
+
readonly cause?: string;
|
|
616
|
+
readonly members: ReadonlyArray<CommitteeMemberOutcome>;
|
|
617
|
+
/** Usage summed over every member call and the tie-breaker. */
|
|
618
|
+
readonly usage: ModelUsage;
|
|
619
|
+
};
|
|
620
|
+
/**
|
|
621
|
+
* §7.6 — the committee, legal on SINGLE-TURN hosts only (a workflow step, a
|
|
622
|
+
* graph node, a crew role): N tool-less nested side calls, then `decide:
|
|
623
|
+
* judge` picks; `escalateOnDisagreement` runs the strong tie-breaker only
|
|
624
|
+
* when the judge's orders disagree — agreement is an acceptor signal, never a
|
|
625
|
+
* reward. Replaces the host's primary turn: the served text is the verdict.
|
|
626
|
+
*/
|
|
627
|
+
export type CommitteeSideCall = {
|
|
628
|
+
readonly members: ReadonlyArray<{
|
|
629
|
+
readonly modelString: string;
|
|
630
|
+
readonly profile?: string;
|
|
631
|
+
}>;
|
|
632
|
+
/** Spec model string of the deciding judge (for `model_stage`). */
|
|
633
|
+
readonly judge: string;
|
|
634
|
+
readonly escalateOnDisagreement?: {
|
|
635
|
+
readonly modelString: string;
|
|
636
|
+
readonly profile?: string;
|
|
637
|
+
};
|
|
638
|
+
readonly run: (ctx: SideCallTurnContext) => Promise<CommitteeVerdict>;
|
|
639
|
+
};
|
|
640
|
+
export type HybridSideCalls = {
|
|
641
|
+
readonly guide?: GuideSideCall;
|
|
642
|
+
readonly shadow?: ShadowSideCall;
|
|
643
|
+
readonly committee?: CommitteeSideCall;
|
|
644
|
+
};
|
|
645
|
+
/**
|
|
646
|
+
* 0.6.0 §7.2.3 — the injected `policy: classifier` label call. Built by
|
|
647
|
+
* `@crewhaus/model-service`'s `wireModels` over `@crewhaus/eval-judge`'s
|
|
648
|
+
* `classifyRouteLabel` (an enum-constrained forced-tool call metered with
|
|
649
|
+
* `role: "classifier"`); runtime-core only calls it, inside `preRoute`, with
|
|
650
|
+
* the latest HUMAN user text, and falls back to the heuristic policy on any
|
|
651
|
+
* throw (`reason: "classifier failed"`). Publish on `bus` so the run's
|
|
652
|
+
* budget meter counts the call under `judge_share`.
|
|
653
|
+
*/
|
|
654
|
+
export type RouteClassifierFn = (input: {
|
|
655
|
+
readonly userText: string;
|
|
656
|
+
/** The pool's declared tags — the closed vocabulary the verdict must come from. */
|
|
657
|
+
readonly labels: readonly string[];
|
|
658
|
+
readonly bus: TraceEventBus;
|
|
659
|
+
readonly signal: AbortSignal;
|
|
660
|
+
}) => Promise<PreRouteClassifierVerdict>;
|
|
325
661
|
export type RunChatLoopOptions = {
|
|
326
662
|
model: string;
|
|
327
663
|
instructions: string;
|
|
328
664
|
maxTokens?: number;
|
|
665
|
+
/**
|
|
666
|
+
* 0.6.0 §4.1 — sampling temperature for every MAIN-turn request (spec
|
|
667
|
+
* `agent.temperature`, or the primary's `$profile`). Exclusive with
|
|
668
|
+
* `thinking` on one request — the compiler rejects the pair on a slot, and
|
|
669
|
+
* a pool candidate that pins one drops the run's other (see
|
|
670
|
+
* {@link CandidatePlan}). Absent → the request carries no temperature,
|
|
671
|
+
* byte-identical to a pre-0.6.0 runtime (`ProviderRequest.temperature`
|
|
672
|
+
* already existed; nothing set it on main-turn calls before).
|
|
673
|
+
*/
|
|
674
|
+
temperature?: number;
|
|
329
675
|
/**
|
|
330
676
|
* Section 17 — optional override for the model used by
|
|
331
677
|
* `compaction-autocompact`. When undefined, compaction reuses
|
|
@@ -335,6 +681,15 @@ export type RunChatLoopOptions = {
|
|
|
335
681
|
* elsewhere.
|
|
336
682
|
*/
|
|
337
683
|
compactionModel?: string;
|
|
684
|
+
/**
|
|
685
|
+
* 0.6.0 §4.2 — the request params a `models:` profile pins on the
|
|
686
|
+
* COMPACTION slot (`compaction.model: $summariser`), lowered as
|
|
687
|
+
* `IrCompaction.params` and handed verbatim to
|
|
688
|
+
* `@crewhaus/compaction-autocompact`, which folds them over the
|
|
689
|
+
* summariser's own ceiling through the shared `buildRequestParams`. Absent
|
|
690
|
+
* → the summariser's defaults, byte-identical to a pre-0.6.0 run.
|
|
691
|
+
*/
|
|
692
|
+
compactionParams?: AuxRequestParams;
|
|
338
693
|
/**
|
|
339
694
|
* Test injection backdoor: pre-built ProviderAdapter that bypasses
|
|
340
695
|
* the model-router. When set, `model` is still the user-facing
|
|
@@ -389,6 +744,73 @@ export type RunChatLoopOptions = {
|
|
|
389
744
|
plugins?: {
|
|
390
745
|
readonly tools?: ReadonlyArray<RegisteredTool>;
|
|
391
746
|
};
|
|
747
|
+
/**
|
|
748
|
+
* 0.6.0 §7.2.4 / §5.2 — the hybrid-strategy tools the composition root
|
|
749
|
+
* (`@crewhaus/model-service`'s `wireModels`) registers when the pool declares
|
|
750
|
+
* `strategy.model_directed: true`: `Consult` and `Escalate` from
|
|
751
|
+
* `@crewhaus/tool-consult`. APPENDED to the effective tool list after the
|
|
752
|
+
* caller's own tools and the plugin tools, first-party winning any name
|
|
753
|
+
* collision (the plugin posture), and advertised like every other tool —
|
|
754
|
+
* they survive a profile's `tools: []` like the auto-registered loop tools
|
|
755
|
+
* and are dropped per profile through `permissions.deny`. Absent → the run
|
|
756
|
+
* is byte-identical to a pre-0.6.0 runtime.
|
|
757
|
+
*/
|
|
758
|
+
hybridTools?: ReadonlyArray<RegisteredTool>;
|
|
759
|
+
/**
|
|
760
|
+
* 0.6.0 §7.2.4 — the `Escalate` tool's latch (built by `wireModels` beside
|
|
761
|
+
* the tool). When the serving model calls `Escalate`, the loop consumes the
|
|
762
|
+
* request at its next model call: it arms the pool's escalation latch (the
|
|
763
|
+
* same one a failed cheap pick arms) so the rest of the turn is served by
|
|
764
|
+
* the request's target when that is a roster candidate, else the strongest
|
|
765
|
+
* candidate, publishes `model_stage{stage: "escalate", strategy:
|
|
766
|
+
* "model_directed", role: "escalation", cause: "self"}` and stamps the
|
|
767
|
+
* remaining calls of the turn `role: "escalation"`. The turn CONTINUES on
|
|
768
|
+
* the target rather than being re-run from a snapshot: the transcript
|
|
769
|
+
* already carries the `Escalate` tool_use / tool_result pair, and a
|
|
770
|
+
* truncating re-run would orphan it (the `isCuratableMessage` constraint);
|
|
771
|
+
* the cascade's clean-prompt re-run belongs to `evaluation.onFail:
|
|
772
|
+
* "escalate"`. Meaningful only with `modelPool`; without one a request is
|
|
773
|
+
* consumed and logged, never served.
|
|
774
|
+
*/
|
|
775
|
+
escalation?: HybridEscalationLatch;
|
|
776
|
+
/**
|
|
777
|
+
* 0.6.0 §7.3 — force EVERY main-turn model call of this run onto one roster
|
|
778
|
+
* arm (a candidate's spec model string, its profile name, or a tag): the
|
|
779
|
+
* seam the workflow and graph `retry_previous` closures use when the gating
|
|
780
|
+
* judge declares `escalate_to`. The run publishes the same
|
|
781
|
+
* `model_stage{stage: "escalate", strategy: "cascade", role: "escalation"}`
|
|
782
|
+
* pair the in-loop cascade does around the forced turn, its `model_route`
|
|
783
|
+
* carries `policy: "forced"`, and its model events carry `role:
|
|
784
|
+
* "escalation"`. Requires a `modelPool`; a value outside the roster or a
|
|
785
|
+
* pool-less run throws at boot rather than silently serving the primary.
|
|
786
|
+
*/
|
|
787
|
+
forcedCandidate?: string;
|
|
788
|
+
/**
|
|
789
|
+
* 0.6.0 §7.4 / §7.6 / §7.8 — the guide / shadow / committee side calls
|
|
790
|
+
* (see {@link HybridSideCalls}), built by `wireModels` from the pool's
|
|
791
|
+
* `strategy` block. `committee` is legal in `singleTurn` mode only (a
|
|
792
|
+
* workflow step, a graph node, a crew role) — the spec rejects it on REPL
|
|
793
|
+
* hosts and the loop refuses it at boot as the second line of defence.
|
|
794
|
+
* Absent → byte-identical to a pre-0.6.0 runtime.
|
|
795
|
+
*/
|
|
796
|
+
sideCalls?: HybridSideCalls;
|
|
797
|
+
/**
|
|
798
|
+
* §7.8 — the shadow is NON-BLOCKING: on the REPL it runs while the loop
|
|
799
|
+
* waits for the next line; on a `singleTurn` host the turn IS the run, so
|
|
800
|
+
* without this seam the loop awaits an in-flight shadow at teardown and
|
|
801
|
+
* the served text is not returned until the audition lane (the shadow call
|
|
802
|
+
* plus two judge calls) has settled. A host that serves MANY single turns
|
|
803
|
+
* from one process — `crewhaus serve`, a channel bot — supplies this: the
|
|
804
|
+
* loop hands it the shadow's settled promise (never rejects) and returns
|
|
805
|
+
* the served text at once; the host awaits every drained promise before it
|
|
806
|
+
* exits (serve does so in its cleanup). A one-shot host (a workflow step, a
|
|
807
|
+
* graph node, a crew role) leaves it undefined and keeps the inline await,
|
|
808
|
+
* so its process never exits under a running shadow. Stage lines the
|
|
809
|
+
* drained shadow publishes after the loop's teardown reach the parent
|
|
810
|
+
* bus's live subscribers only (the run's session mirror is closed by then);
|
|
811
|
+
* the scoreboard record — what the lane exists for — is unaffected.
|
|
812
|
+
*/
|
|
813
|
+
sideCallDrain?: (settled: Promise<void>) => void;
|
|
392
814
|
/** Per-run identity, abort signal, and logger. Defaults to a fresh `createRunContext()`. */
|
|
393
815
|
runContext?: RunContext;
|
|
394
816
|
/** Context-window limit in tokens. Defaults to 200_000 (Claude opus/sonnet 4.x). */
|
|
@@ -477,6 +899,17 @@ export type RunChatLoopOptions = {
|
|
|
477
899
|
* event-log `.jsonl` files live. Defaults to `.crewhaus/sessions`.
|
|
478
900
|
*/
|
|
479
901
|
sessionRootDir?: string;
|
|
902
|
+
/**
|
|
903
|
+
* 0.6.0 §16 Q6 — `false` keeps this run entirely IN MEMORY: no session
|
|
904
|
+
* `.json`, no event-log `.jsonl`, no watch-me sidecar, no TTL sweep of the
|
|
905
|
+
* sessions dir or the approvals log. Set by the nested single-turn side
|
|
906
|
+
* calls a hybrid strategy runs (consult, guide, shadow, committee members)
|
|
907
|
+
* so a run never writes one child session file per member per turn — their
|
|
908
|
+
* spend and stage transitions are re-published on the PARENT bus and land
|
|
909
|
+
* in the parent's log. Cannot be combined with `resume`. Default `true`,
|
|
910
|
+
* byte-identical to before.
|
|
911
|
+
*/
|
|
912
|
+
persistSession?: boolean;
|
|
480
913
|
/**
|
|
481
914
|
* Human-readable label for new sessions; persisted to the session JSON.
|
|
482
915
|
* Typically the spec `name`. Defaults to `(unnamed)`.
|
|
@@ -497,6 +930,24 @@ export type RunChatLoopOptions = {
|
|
|
497
930
|
* against their own unscoped history exactly as before.
|
|
498
931
|
*/
|
|
499
932
|
toolsetScope?: string;
|
|
933
|
+
/**
|
|
934
|
+
* 0.6.0 (design §8.1) — the ROLE every model call this loop publishes is
|
|
935
|
+
* attributed with (`model_request` / `model_response`, and therefore the
|
|
936
|
+
* `cost_accrual` cost-tracker derives and the `model_meta` session line).
|
|
937
|
+
* Left unset by the main agent loop — an absent role reads as `"primary"`
|
|
938
|
+
* — and set by the nested single-turn loops a hybrid strategy runs
|
|
939
|
+
* (`"consult"`, `"guide"`, `"committee"`, `"shadow"`, …) and by sub-agent
|
|
940
|
+
* children (`"subagent"`), so a budget meter or cost fold on the run can
|
|
941
|
+
* split spend by purpose. Pass-through only: the runtime stamps what it is
|
|
942
|
+
* handed and never infers a role.
|
|
943
|
+
*/
|
|
944
|
+
modelRole?: ModelRole;
|
|
945
|
+
/**
|
|
946
|
+
* 0.6.0 (design §8.1) — the hybrid-strategy STAGE name every model call
|
|
947
|
+
* this loop publishes belongs to (`"draft"`, `"verify"`, `"plan"`, …).
|
|
948
|
+
* Pass-through like `modelRole`; absent on the main loop.
|
|
949
|
+
*/
|
|
950
|
+
modelStage?: string;
|
|
500
951
|
/**
|
|
501
952
|
* Resume an existing session: load its metadata, replay its event log
|
|
502
953
|
* into the message history (only `user_message` + `assistant_message`
|
|
@@ -663,6 +1114,14 @@ export type RunChatLoopOptions = {
|
|
|
663
1114
|
readonly ledger?: boolean;
|
|
664
1115
|
/** Deterministic teardown handoff write. */
|
|
665
1116
|
readonly onHandoff?: (input: HandoffInput) => Promise<void>;
|
|
1117
|
+
/**
|
|
1118
|
+
* 0.6.0 §7.4 — write a plan the GUIDE produced (`strategy.guide` with
|
|
1119
|
+
* `every: first_turn`) into the continuity plan store, so plan-execute
|
|
1120
|
+
* survives the run: the strong model plans once, the store re-renders it
|
|
1121
|
+
* as `<current_plan>` on every later call and on resume. Best-effort — a
|
|
1122
|
+
* store failure leaves the volatile `<guide>` block in place and warns.
|
|
1123
|
+
*/
|
|
1124
|
+
readonly savePlan?: (planText: string) => Promise<void>;
|
|
666
1125
|
};
|
|
667
1126
|
/**
|
|
668
1127
|
* Item #56 — absolute paths to per-user preference files injected at run
|
|
@@ -890,11 +1349,95 @@ export type RunChatLoopOptions = {
|
|
|
890
1349
|
* pool takes precedence if all are somehow set. Absent → single-model path.
|
|
891
1350
|
*/
|
|
892
1351
|
modelPool?: {
|
|
893
|
-
|
|
894
|
-
|
|
1352
|
+
/**
|
|
1353
|
+
* 0.6.0 §4.2 / §4.4 — each candidate is a full {@link ModelProfile}
|
|
1354
|
+
* (`@crewhaus/model-plan`, structurally the IR's `IrModelPoolCandidate`)
|
|
1355
|
+
* plus its routing identity `tags`. At boot the loop builds ONE
|
|
1356
|
+
* {@link CandidatePlan} per enabled candidate from these fields —
|
|
1357
|
+
* request params (`thinking` / `maxTokens` / `temperature` /
|
|
1358
|
+
* `limits.model_call_timeout_ms`), the subset-only tool advertisement
|
|
1359
|
+
* (`tools`, capability-filtered), the narrowed permission rule set
|
|
1360
|
+
* (`permissions.deny` / `.ask`), per-candidate rate buckets
|
|
1361
|
+
* (`rate_limits`), per-call `tool_config`, the volatile `overlay`,
|
|
1362
|
+
* `caching: off`, and the per-run spend cap (`cost`) — and selects that
|
|
1363
|
+
* plan on every model call the candidate serves. A bare `{ model, tags }`
|
|
1364
|
+
* candidate serves on the run's own settings, exactly as before.
|
|
1365
|
+
* `fallbacks` / `circuitBreaker` / `requires` are carried for PR 10's
|
|
1366
|
+
* per-candidate chains and breakers and the preRoute eligibility phase.
|
|
1367
|
+
*/
|
|
1368
|
+
readonly candidates: ReadonlyArray<Omit<ModelProfile, "capabilities"> & {
|
|
895
1369
|
readonly tags: readonly string[];
|
|
1370
|
+
/**
|
|
1371
|
+
* The IR's FLAT declared-capability override (`IrModelCapabilities`:
|
|
1372
|
+
* the four feature booleans, `caching`, `contextWindow`,
|
|
1373
|
+
* `maxOutputTokens`) as the pool blob carries it — the plan table
|
|
1374
|
+
* folds it over the adapter's `features`. (`@crewhaus/model-plan`'s
|
|
1375
|
+
* `CandidateCapabilities` nests the features; the blob does not.)
|
|
1376
|
+
*/
|
|
1377
|
+
readonly capabilities?: {
|
|
1378
|
+
readonly tool_use?: boolean;
|
|
1379
|
+
readonly vision?: boolean;
|
|
1380
|
+
readonly thinking?: boolean;
|
|
1381
|
+
readonly web_search?: boolean;
|
|
1382
|
+
readonly caching?: "explicit" | "automatic" | false;
|
|
1383
|
+
readonly contextWindow?: number;
|
|
1384
|
+
readonly maxOutputTokens?: number;
|
|
1385
|
+
};
|
|
1386
|
+
/**
|
|
1387
|
+
* 0.6.0 §7.1 — `false` withdraws the candidate from routing without
|
|
1388
|
+
* deleting its learned history: it is skipped at boot. The compiler
|
|
1389
|
+
* and the `modelPlanIntegrity` pass guarantee at least one candidate
|
|
1390
|
+
* stays routable.
|
|
1391
|
+
*/
|
|
1392
|
+
readonly enabled?: false;
|
|
896
1393
|
}>;
|
|
897
|
-
|
|
1394
|
+
/**
|
|
1395
|
+
* 0.6.0 §7.9 — the scoped `routeKey` prefix: the step / role / node /
|
|
1396
|
+
* sub-agent the pool belongs to. The compiler never stamps it (that would
|
|
1397
|
+
* change every pre-0.6.0 pooled blob); the host that knows the scope
|
|
1398
|
+
* does — the crew orchestrator (role name), the workflow and graph
|
|
1399
|
+
* emitters (step / node name). Absent → the caller's `toolsetScope`, else
|
|
1400
|
+
* unscoped. Stamped on `model_route` as `scope`; the routing store keys
|
|
1401
|
+
* arms by it from PR 10 on.
|
|
1402
|
+
*/
|
|
1403
|
+
readonly scope?: string;
|
|
1404
|
+
/**
|
|
1405
|
+
* 0.6.0 §7.2.3 — `classifier` runs the injected {@link RouteClassifierFn}
|
|
1406
|
+
* (`routeClassifier`) inside the `preRoute` phase on every model call and
|
|
1407
|
+
* serves the arm its label names; without a wired classifier, past the
|
|
1408
|
+
* `judge_share`, or on any classifier error the turn routes as
|
|
1409
|
+
* `heuristic` with `reason: "classifier failed"` — the documented
|
|
1410
|
+
* fallback, never a stop.
|
|
1411
|
+
*/
|
|
1412
|
+
readonly policy: "static" | "heuristic" | "learned" | "classifier";
|
|
1413
|
+
/**
|
|
1414
|
+
* 0.6.0 §7.2.1 — per-message `/model <$profile|tag>` / `/model auto`
|
|
1415
|
+
* steering, parsed at the typed INPUT seams (the REPL input, the
|
|
1416
|
+
* single-turn seed message — never the transcript) and sticky per
|
|
1417
|
+
* session through the additive session-metadata `pin`. Default OFF on
|
|
1418
|
+
* every shape: a channel shape has no sender allowlist and inbound text
|
|
1419
|
+
* is untrusted.
|
|
1420
|
+
*/
|
|
1421
|
+
readonly directives?: boolean;
|
|
1422
|
+
/**
|
|
1423
|
+
* 0.6.0 §7.2.2 — rule-directed routing, first match wins, evaluated in
|
|
1424
|
+
* `preRoute` before the policy over the turn's `RouteSignals`
|
|
1425
|
+
* (`@crewhaus/model-plan`'s `evaluateRules`). The matched `ruleId` is
|
|
1426
|
+
* persisted on the `model_route` line. Carried verbatim from the IR.
|
|
1427
|
+
*/
|
|
1428
|
+
readonly rules?: readonly RouteRule[];
|
|
1429
|
+
/**
|
|
1430
|
+
* 0.6.0 §7.2.3 — the `policy: classifier` block (model, its declared
|
|
1431
|
+
* labels — one per routable tag — and the label call's output cap).
|
|
1432
|
+
* Consumed by the composition root that builds `routeClassifier`; the
|
|
1433
|
+
* loop reads `labels` as the closed verdict vocabulary.
|
|
1434
|
+
*/
|
|
1435
|
+
readonly classifier?: {
|
|
1436
|
+
readonly model: string;
|
|
1437
|
+
readonly modelProfile?: string;
|
|
1438
|
+
readonly labels: Readonly<Record<string, string>>;
|
|
1439
|
+
readonly maxTokens?: number;
|
|
1440
|
+
};
|
|
898
1441
|
readonly objective?: {
|
|
899
1442
|
readonly quality?: number;
|
|
900
1443
|
readonly cost?: number;
|
|
@@ -916,16 +1459,93 @@ export type RunChatLoopOptions = {
|
|
|
916
1459
|
readonly seed?: string;
|
|
917
1460
|
readonly bandit?: "epsilon-greedy" | "thompson";
|
|
918
1461
|
};
|
|
1462
|
+
/**
|
|
1463
|
+
* 0.6.0 §7.3 — the hybrid strategy block (structurally the IR's
|
|
1464
|
+
* `IrModelPoolStrategy`; the pool blob carries it verbatim). The loop reads
|
|
1465
|
+
* `cascade` (`escalateTo` / `cleanPrompt`, the fallbacks for the
|
|
1466
|
+
* `evaluation` option's own fields) and `maxEscalations`; `guide` /
|
|
1467
|
+
* `shadow` / `committee` are consumed by the side-call closures (PR 9d),
|
|
1468
|
+
* `modelDirected` by the composition root that registers the tools.
|
|
1469
|
+
*/
|
|
1470
|
+
readonly strategy?: {
|
|
1471
|
+
readonly cascade?: {
|
|
1472
|
+
readonly draft: string;
|
|
1473
|
+
readonly escalateTo: string;
|
|
1474
|
+
readonly cleanPrompt?: boolean;
|
|
1475
|
+
};
|
|
1476
|
+
readonly modelDirected?: boolean;
|
|
1477
|
+
readonly maxEscalations?: number;
|
|
1478
|
+
};
|
|
1479
|
+
/**
|
|
1480
|
+
* 0.6.0 §6.3 / §7.10 / §7.11 — the reward block. `qualitySource: "in_loop"`
|
|
1481
|
+
* lets a cascade turn's judged quality reach the REWARD (`computeReward`);
|
|
1482
|
+
* under the default `none` the quality is still persisted on the `v:2`
|
|
1483
|
+
* arm line (`route status` shows it) but the reward is computed without
|
|
1484
|
+
* it — byte-identical to a 0.5.x reward. `priors: "eval"` seeds the
|
|
1485
|
+
* learned policy from `<root>/routing/priors.json` (N2 — a malformed or
|
|
1486
|
+
* stale file is ignored with a boot warning). `floor` bounds learned
|
|
1487
|
+
* exploitation to arms whose judged-quality lower bound clears the floor
|
|
1488
|
+
* arm's mean (`reason: "floor-blocked"` otherwise). `resetOnProfileChange`
|
|
1489
|
+
* (default true) discards a profiled arm's lines recorded under a
|
|
1490
|
+
* different profile / quality-source lineage.
|
|
1491
|
+
*/
|
|
1492
|
+
readonly reward?: {
|
|
1493
|
+
readonly qualitySource?: "none" | "in_loop" | "shadow" | "promoted";
|
|
1494
|
+
readonly priors?: "none" | "eval";
|
|
1495
|
+
readonly floor?: {
|
|
1496
|
+
readonly arm?: string;
|
|
1497
|
+
readonly confidence?: number;
|
|
1498
|
+
readonly tolerance?: number;
|
|
1499
|
+
};
|
|
1500
|
+
readonly resetOnProfileChange?: boolean;
|
|
1501
|
+
};
|
|
919
1502
|
};
|
|
1503
|
+
/**
|
|
1504
|
+
* 0.6.0 §7.9 — the pricing table the pool's reward and the per-tool cost
|
|
1505
|
+
* attribution price with: the INSTALLED feed (`crewhaus pricing sync`,
|
|
1506
|
+
* `loadUserPricing` on the CLI) injected by the host, so the learned
|
|
1507
|
+
* policy's cost term and `cost_accrual` agree. Absent → the compiled-in
|
|
1508
|
+
* `DEFAULT_PRICING`, exactly as before.
|
|
1509
|
+
*/
|
|
1510
|
+
pricing?: PricingTable;
|
|
1511
|
+
/**
|
|
1512
|
+
* 0.6.0 §7.2.3 — the `policy: classifier` label call (see
|
|
1513
|
+
* {@link RouteClassifierFn}). Wired by `@crewhaus/model-service` when the
|
|
1514
|
+
* pool declares `policy: classifier` + `classifier:`; meaningful only with
|
|
1515
|
+
* such a pool. Absent under `policy: classifier` → every turn routes
|
|
1516
|
+
* heuristically with `reason: "classifier failed: no classifier wired"`.
|
|
1517
|
+
*/
|
|
1518
|
+
routeClassifier?: RouteClassifierFn;
|
|
1519
|
+
/**
|
|
1520
|
+
* 0.6.0 §7.2.2 — a host-supplied routing hint for the `channel` rule
|
|
1521
|
+
* condition (a Slack channel name, a Telegram chat kind, …). Carried on
|
|
1522
|
+
* `RouteSignals.channelHint` and persisted on the route line; never
|
|
1523
|
+
* user-controlled text.
|
|
1524
|
+
*/
|
|
1525
|
+
channelHint?: string;
|
|
920
1526
|
/**
|
|
921
1527
|
* Test injection for the pool candidate adapters, keyed by their spec model
|
|
922
|
-
* string (mirrors `_tierAdapters`). Production callers leave it undefined
|
|
1528
|
+
* string (mirrors `_tierAdapters`). Production callers leave it undefined —
|
|
1529
|
+
* including a routed `crewhaus eval`, which injects only `_scoreboard` and
|
|
1530
|
+
* lets every candidate resolve through the normal router, so the eval
|
|
1531
|
+
* measures the real providers.
|
|
923
1532
|
*/
|
|
924
1533
|
_poolAdapters?: ReadonlyMap<string, ProviderAdapter>;
|
|
925
1534
|
/**
|
|
926
|
-
*
|
|
927
|
-
* undefined (the runtime opens a file-backed scoreboard
|
|
928
|
-
* dir)
|
|
1535
|
+
* The reward scoreboard the pool learns into. Production `run` / bundle
|
|
1536
|
+
* callers leave it undefined (the runtime opens a file-backed scoreboard
|
|
1537
|
+
* beside the sessions dir) and tests inject an in-memory or tmp-dir
|
|
1538
|
+
* instance to assert learning.
|
|
1539
|
+
*
|
|
1540
|
+
* 0.6.0 §6.1 — `@crewhaus/eval-runner` is now a PRODUCTION consumer of this
|
|
1541
|
+
* seam, not only a test one: a routed eval (`crewhaus eval --routing`)
|
|
1542
|
+
* injects a FROZEN arm snapshot here, whose `score()` answers from
|
|
1543
|
+
* statistics read once at run start and whose `record()` / `ungraded()` /
|
|
1544
|
+
* `compact()` are no-op sinks. That is what makes a routed measurement
|
|
1545
|
+
* reproducible (samples run four-wide, so a shared mutable scoreboard would
|
|
1546
|
+
* race across them) AND safe (a measurement run can never move the
|
|
1547
|
+
* harness's learned policy). The snapshot's digest is the eval's instrument
|
|
1548
|
+
* identity — see `packages/eval-runner/src/routing.ts`.
|
|
929
1549
|
*/
|
|
930
1550
|
_scoreboard?: Scoreboard;
|
|
931
1551
|
/**
|
|
@@ -980,17 +1600,65 @@ export type RunChatLoopOptions = {
|
|
|
980
1600
|
* the optimizer's `BudgetMeter` to normal runs (`crewhaus run
|
|
981
1601
|
* --budget-usd` / spec `budget: { usd, on_exceed }`). An always-on cost
|
|
982
1602
|
* meter (independent of `CREWHAUS_COST_TRACKING`) accrues per-response
|
|
983
|
-
* spend priced off the WIRE model that actually served, and
|
|
984
|
-
*
|
|
985
|
-
*
|
|
986
|
-
*
|
|
987
|
-
*
|
|
988
|
-
*
|
|
1603
|
+
* spend priced off the WIRE model that actually served, and the cap is
|
|
1604
|
+
* enforced at two gates:
|
|
1605
|
+
* - the REPL's PRE-TURN check — beside the token-budget/compaction
|
|
1606
|
+
* check — before the next user turn opens (an idle run that has hit
|
|
1607
|
+
* the cap ends cleanly, exit 0, with a `[budget]` notice); and
|
|
1608
|
+
* - 0.6.0 §7.12: a PER-MODEL-CALL gate at the top of the tool loop's
|
|
1609
|
+
* `NeedModel` arm, on every host (singleTurn included — the workflow
|
|
1610
|
+
* step, channel message and managed request paths never reached the
|
|
1611
|
+
* REPL check, so the cap was inert there). A breach mid-turn ends the
|
|
1612
|
+
* run with the classified `crewhaus_budget` failure (`run_failed`
|
|
1613
|
+
* published + logged, `RunFailedError` thrown, exit
|
|
1614
|
+
* {@link EXIT_CODES.crewhaus_budget}). This is a stated change to the
|
|
1615
|
+
* pre-0.6.0 "an in-flight tool loop is never severed" contract: the
|
|
1616
|
+
* gate sits BEFORE the model call, so every tool result already
|
|
1617
|
+
* obtained is committed to the transcript and no `tool_use` is left
|
|
1618
|
+
* unanswered — the run stops at a request boundary, never mid-tool.
|
|
1619
|
+
* - `on_exceed.kind: "stop"` ends the run at whichever gate trips.
|
|
989
1620
|
* - `on_exceed.kind: "degrade"` re-resolves the primary model to
|
|
990
1621
|
* `on_exceed.model` ONCE (the cheaper rung of the ladder) and
|
|
991
1622
|
* continues; a `model_failover` event (reason `budget_degrade`)
|
|
992
|
-
* records the switch
|
|
993
|
-
*
|
|
1623
|
+
* records the switch — published ONLY when the adapter swap actually
|
|
1624
|
+
* takes effect (single-model and `modelTiers` runs; under tiers the
|
|
1625
|
+
* rung replaces both tiers). Under a `modelPool` the rung is
|
|
1626
|
+
* pre-resolved at boot as one extra `PoolCandidate` (bound to the
|
|
1627
|
+
* roster member when `on_exceed.model` names one) and, on a breach,
|
|
1628
|
+
* becomes the FORCED candidate — `model_route` policy `forced`,
|
|
1629
|
+
* reason `budget_degrade` — instead of swapping the closed-over
|
|
1630
|
+
* adapter the pool overwrote on every call (dead config before
|
|
1631
|
+
* 0.6.0). The second breach is defined at a TURN boundary, not an
|
|
1632
|
+
* accrual boundary: the rung serves EVERY remaining model call of the
|
|
1633
|
+
* turn in which the degrade fired (a mid-turn degrade finishes its
|
|
1634
|
+
* tool loop; a single-turn host gets one complete degraded reply),
|
|
1635
|
+
* and the first gate of the NEXT turn — the REPL's pre-turn check —
|
|
1636
|
+
* ends the run cleanly (exit 0). The ladder has one rung in v1.
|
|
1637
|
+
* - `scope` (0.6.0, default `"run"`): `"session"` seeds the meter on
|
|
1638
|
+
* `resume` from the `cost_accrual` lines persisted in the session log
|
|
1639
|
+
* (and persists them for this run even without
|
|
1640
|
+
* `CREWHAUS_COST_TRACKING`), so a channel/managed cap bounds the
|
|
1641
|
+
* conversation, not one inbound message.
|
|
1642
|
+
* - `judgeShare` (0.6.0 §6.2, default {@link DEFAULT_JUDGE_SHARE} = 0.3):
|
|
1643
|
+
* the fraction of `usdMicros` the AUXILIARY roles
|
|
1644
|
+
* (`AUXILIARY_MODEL_ROLES` — judge, compaction, guide, classifier,
|
|
1645
|
+
* consult, committee, shadow) may spend between them. Those calls now
|
|
1646
|
+
* ride the run bus with a `role`, so the always-on meter counts them
|
|
1647
|
+
* toward the cap like any other call; the share is the sub-cap inside
|
|
1648
|
+
* it. Crossing it is an ACCOUNTING signal in this release: the share is
|
|
1649
|
+
* re-read at every budget gate (`enforceBudget` — the per-model-call
|
|
1650
|
+
* gate and the REPL's pre-turn gate) and after every in-loop grade, so
|
|
1651
|
+
* once auxiliary spend reaches it the runtime prints one `[budget]`
|
|
1652
|
+
* notice per METER (once per run on a shared `budgetMeter`) whether or
|
|
1653
|
+
* not the run declares `evaluation:` — a compaction-only run, or a
|
|
1654
|
+
* workflow whose `kind: judge` gates run between steps, raises it from
|
|
1655
|
+
* the next per-call gate — and every `eval_graded` published while the
|
|
1656
|
+
* share stands exhausted carries `reason: "judge_share_exhausted"` (the
|
|
1657
|
+
* workflow bundle stamps the same `reason` on `judge_verdict` from its
|
|
1658
|
+
* shared meter). The judge keeps judging under the TOTAL cap (the
|
|
1659
|
+
* in-loop retry path is unchanged); the cascade that consumes the
|
|
1660
|
+
* signal — serving its strong rung directly instead of spending more on
|
|
1661
|
+
* judging — lands with the hybrid strategies.
|
|
994
1662
|
* `usdMicros` is the ceiling in USD-micros (1 USD = 1_000_000). Absent →
|
|
995
1663
|
* no cap (zero behaviour change for existing callers). A model the
|
|
996
1664
|
* pricing table can't price accrues $0, so an all-unpriced run degrades
|
|
@@ -1000,11 +1668,35 @@ export type RunChatLoopOptions = {
|
|
|
1000
1668
|
readonly usdMicros: number;
|
|
1001
1669
|
readonly onExceed: {
|
|
1002
1670
|
readonly kind: "stop";
|
|
1003
|
-
}
|
|
1671
|
+
}
|
|
1672
|
+
/** 0.6.0 §4.2 — `params` is the degrade profile's pinned request
|
|
1673
|
+
* params: once the rung takes over, the serving plan's params ARE
|
|
1674
|
+
* the profile's, so a `$cheap` degrade target gets its own
|
|
1675
|
+
* `max_tokens` / `thinking` / `temperature` and not the primary's. */
|
|
1676
|
+
| {
|
|
1004
1677
|
readonly kind: "degrade";
|
|
1005
1678
|
readonly model: string;
|
|
1679
|
+
readonly params?: AuxRequestParams;
|
|
1006
1680
|
};
|
|
1681
|
+
readonly scope?: "run" | "session";
|
|
1682
|
+
readonly judgeShare?: number;
|
|
1007
1683
|
};
|
|
1684
|
+
/**
|
|
1685
|
+
* 0.6.0 §7.12 — a caller-owned, RUN-spanning budget meter. The loop's own
|
|
1686
|
+
* meter is created per `runChatLoop` call and reads `getRunCost(bus.runId)`,
|
|
1687
|
+
* so a shape that runs several loops per run (the workflow target: one
|
|
1688
|
+
* `singleTurn` loop per step) had `budget.usd` bounding each STEP — N steps
|
|
1689
|
+
* could spend N× the cap. The compiled workflow bundle now creates ONE
|
|
1690
|
+
* `createCostTracker(__runContext.eventBus, { suppressEvents: true })` on
|
|
1691
|
+
* the shared run context and hands it to every step through this option
|
|
1692
|
+
* (together with `runContext`, so every step publishes on the meter's
|
|
1693
|
+
* bus under one `runId`); the gates then read the accumulated run spend.
|
|
1694
|
+
* When supplied, the loop neither creates its own meter nor unsubscribes
|
|
1695
|
+
* this one — its lifetime is the caller's. Absent (every other shape) →
|
|
1696
|
+
* the per-loop meter, byte-identical to before. Meaningful only alongside
|
|
1697
|
+
* `budget`.
|
|
1698
|
+
*/
|
|
1699
|
+
budgetMeter?: CostTracker;
|
|
1008
1700
|
/**
|
|
1009
1701
|
* Loop contract 0.4 (G10) — whole-run wall-clock ceiling in ms (spec
|
|
1010
1702
|
* `limits.deadline_ms`). Armed once at loop start; on fire it aborts the
|
|
@@ -1219,6 +1911,28 @@ export type RunChatLoopOptions = {
|
|
|
1219
1911
|
* lets `session-store` apply its own non-tenant default). The env var and the
|
|
1220
1912
|
* global default are only consulted OUTSIDE any tenant scope.
|
|
1221
1913
|
*/
|
|
1914
|
+
/**
|
|
1915
|
+
* 0.6.0 §7.3 / §8.1 (PR 9c) — a durable sink for the routing kinds a HOST
|
|
1916
|
+
* publishes on a shared `RunContext` bus BETWEEN loops, when no `runChatLoop`
|
|
1917
|
+
* is live to mirror them into the session JSONL: the workflow / graph bundles'
|
|
1918
|
+
* `kind: judge` gates publish `judge_verdict` between steps / nodes. Every
|
|
1919
|
+
* step and node runs on the same `RunContext`, so they all persist into the
|
|
1920
|
+
* session log named by its `sessionId`; this sink appends the between-loop
|
|
1921
|
+
* lines to that same file (`<sessionRootDir>/<sessionId>.jsonl`), so `route
|
|
1922
|
+
* explain`, `sessions tail` and Hangar read them from disk. Defaults to
|
|
1923
|
+
* `judge_verdict` only: the `model_stage` / `model_failover` /
|
|
1924
|
+
* `model_directive` lines are published INSIDE a loop, whose own
|
|
1925
|
+
* `attachRoutingPersistence` already writes them to the same file — a second
|
|
1926
|
+
* subscriber there would duplicate every line. `close()` unsubscribes.
|
|
1927
|
+
*/
|
|
1928
|
+
export type RunEventSink = {
|
|
1929
|
+
readonly sessionId: string;
|
|
1930
|
+
close(): Promise<void>;
|
|
1931
|
+
};
|
|
1932
|
+
export declare function attachRunEventSink(runContext: RunContext, opts?: {
|
|
1933
|
+
readonly sessionRootDir?: string;
|
|
1934
|
+
readonly kinds?: ReadonlyArray<RoutingPersistedKind>;
|
|
1935
|
+
}): Promise<RunEventSink>;
|
|
1222
1936
|
export declare function resolveSessionRootDir(optsRoot: string | undefined): string | undefined;
|
|
1223
1937
|
/**
|
|
1224
1938
|
* Tenant-scoped root for the tool-result spill store, mirroring
|
|
@@ -1238,6 +1952,66 @@ export declare function resolveToolResultRoot(): string | undefined;
|
|
|
1238
1952
|
* to mark federation-joined or other runtime sinks dynamic too.
|
|
1239
1953
|
*/
|
|
1240
1954
|
export declare function defaultSinkScope(toolName: string): SinkScope;
|
|
1955
|
+
/**
|
|
1956
|
+
* Everything the loop needs to serve ONE model — computed once per pool
|
|
1957
|
+
* candidate at boot (plus one `primaryPlan` for the single-model path and
|
|
1958
|
+
* any degrade rung) and SELECTED once per model call, after the routing
|
|
1959
|
+
* if-else chain and before the request is built. Four hot paths read it: the
|
|
1960
|
+
* request build (params, tools, overlay, cache mode), the dispatch gate in
|
|
1961
|
+
* `executeOneToolUse` (`advertisedNames`, `permissionRules`, rate buckets,
|
|
1962
|
+
* `toolConfigs`), `ListTools` (`listToolsSummaries`) and reporting
|
|
1963
|
+
* (`model_request.toolCount`, `model_route.toolsetFingerprint`).
|
|
1964
|
+
*
|
|
1965
|
+
* `fromPool` is false only for the primary plan (and a non-roster degrade
|
|
1966
|
+
* rung): those serve on the run's settings with NOTHING narrowed, so a run
|
|
1967
|
+
* without a pool — or a pool of bare `{model, tags}` candidates — produces
|
|
1968
|
+
* byte-identical requests and events.
|
|
1969
|
+
*/
|
|
1970
|
+
export type CandidatePlan = {
|
|
1971
|
+
/** Scoreboard arm identity: the `models:` profile name, else the spec model string (§7.9). */
|
|
1972
|
+
readonly armId: string;
|
|
1973
|
+
readonly profile?: string;
|
|
1974
|
+
readonly modelString: string;
|
|
1975
|
+
readonly wireModelId: string;
|
|
1976
|
+
readonly fromPool: boolean;
|
|
1977
|
+
readonly params: RequestParams;
|
|
1978
|
+
readonly paramsFingerprint: string;
|
|
1979
|
+
readonly modelCallTimeoutMs?: number;
|
|
1980
|
+
/** The subset advertised to this candidate, in the shape's declared order (justification contract injected). */
|
|
1981
|
+
readonly anthropicTools: Anthropic.Tool[] | undefined;
|
|
1982
|
+
/** `anthropicTools`' names — the dispatch gate's allow list. */
|
|
1983
|
+
readonly advertisedNames: ReadonlySet<string>;
|
|
1984
|
+
readonly listToolsSummaries: ReadonlyArray<AdvertisedToolSummary>;
|
|
1985
|
+
/** Advertised tools by name — what the batch / streaming executors partition over. */
|
|
1986
|
+
readonly toolByName: Map<string, RegisteredTool>;
|
|
1987
|
+
readonly toolsetFingerprint: string;
|
|
1988
|
+
/** Tool names the profile / capability filter removed, for the boot log. */
|
|
1989
|
+
readonly excludedTools: ReadonlyArray<string>;
|
|
1990
|
+
readonly permissionRules: RuleSet;
|
|
1991
|
+
/** Per-candidate `rate_limits` buckets (`tool:<name>@<armId>`); absent → the run's limiter. */
|
|
1992
|
+
readonly toolRateLimiter?: RateLimiter;
|
|
1993
|
+
readonly hasToolRateBucket: (toolName: string) => boolean;
|
|
1994
|
+
readonly toolConfigs?: Readonly<Record<string, unknown>>;
|
|
1995
|
+
/** The per-model instructions overlay, appended in the VOLATILE region (never cache-marked). */
|
|
1996
|
+
readonly overlayBlock?: Anthropic.TextBlockParam;
|
|
1997
|
+
/** `off` strips every `cache_control` marker (system AND transcript) for this candidate. */
|
|
1998
|
+
readonly cacheMode: "prefer" | "off";
|
|
1999
|
+
/** The adapter's features, overridden by a declared `capabilities:` block. */
|
|
2000
|
+
readonly features: ProviderFeatures;
|
|
2001
|
+
readonly costCapUsdMicros?: number;
|
|
2002
|
+
/** Priced spend attributed to this arm in this run (micro-USD); read by the cost-cap gate. */
|
|
2003
|
+
spentUsdMicros: number;
|
|
2004
|
+
};
|
|
2005
|
+
/**
|
|
2006
|
+
* The auto-registered loop tools that survive a profile's `tools: []` (§5.2):
|
|
2007
|
+
* the runtime-bookkeeping family `BUILTIN_BOOKKEEPING_RULES` already allows
|
|
2008
|
+
* (`Skill`, `ListTools`, the Focus / Plan / Goal tools), `Task`, and the
|
|
2009
|
+
* memory tools. A judge profile that could not call `ListTools` would
|
|
2010
|
+
* deadlock a headless run, so these are dropped only by an explicit
|
|
2011
|
+
* `permissions.deny`. Hybrid tools (`Consult` / `Escalate`) join the list at
|
|
2012
|
+
* boot from `opts.hybridTools`.
|
|
2013
|
+
*/
|
|
2014
|
+
export declare const RETAINED_LOOP_TOOL_NAMES: ReadonlyArray<string>;
|
|
1241
2015
|
/** Which configured limit fired. `run` = `limits.deadline_ms`,
|
|
1242
2016
|
* `turn` = `limits.turn_timeout_ms`, `model_call` = `limits.model_call_timeout_ms`. */
|
|
1243
2017
|
export type TimeoutScope = "run" | "turn" | "model_call";
|