@north-light/crouter 0.3.278 → 0.3.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/api/dto/model-config.d.ts +25 -0
  2. package/dist/api/dto/model-config.js +1 -0
  3. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +11 -3
  4. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +11 -3
  5. package/dist/clients/attach/chrome/status-line.js +1 -22
  6. package/dist/clients/attach/render/chat-view.d.ts +11 -0
  7. package/dist/clients/attach/render/chat-view.js +23 -4
  8. package/dist/clients/attach/render/tool-calls.d.ts +6 -1
  9. package/dist/clients/attach/render/tool-calls.js +47 -3
  10. package/dist/clients/attach/session/bindings.d.ts +1 -1
  11. package/dist/clients/attach/session/bindings.js +1 -1
  12. package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
  13. package/dist/clients/attach/session/chrome-refresh.js +3 -3
  14. package/dist/clients/attach/viewer.js +557 -557
  15. package/dist/commands/node/bash.js +5 -8
  16. package/dist/core/__tests__/helpers/fleet.d.ts +1 -1
  17. package/dist/core/__tests__/helpers/fleet.js +5 -0
  18. package/dist/core/__tests__/helpers/harness.js +5 -1
  19. package/dist/core/model-config.d.ts +20 -0
  20. package/dist/core/model-config.js +107 -0
  21. package/dist/core/model-routes.d.ts +5 -0
  22. package/dist/core/model-routes.js +5 -5
  23. package/dist/core/runtime/busy.d.ts +13 -0
  24. package/dist/core/runtime/busy.js +43 -1
  25. package/dist/core/runtime/revive.js +2 -0
  26. package/dist/daemon/api/__tests__/model-config.test.d.ts +1 -0
  27. package/dist/daemon/api/__tests__/model-config.test.js +73 -0
  28. package/dist/daemon/api/handlers/model-config.d.ts +2 -0
  29. package/dist/daemon/api/handlers/model-config.js +44 -0
  30. package/dist/daemon/api/server.js +2 -0
  31. package/dist/daemon/fleet.d.ts +3 -0
  32. package/dist/daemon/fleet.js +35 -5
  33. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +14 -0
  34. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +18 -0
  35. package/package.json +1 -1
  36. package/runtime.lock.json +2 -2
@@ -2,7 +2,6 @@
2
2
  import { envNodeId } from '../../shared/env.js';
3
3
  import { defineBranch, defineLeaf } from '../../core/command.js';
4
4
  import { InputError } from '../../core/io.js';
5
- import { displayMessage } from '../../core/runtime/tmux-chrome.js';
6
5
  import { contextDir } from '../../core/canvas/paths.js';
7
6
  import { existsSync } from 'node:fs';
8
7
  import { activeBackgroundBashJobs, backgroundRunningBashJobs, bashJobCommandLine, bashJobPaths, clearJobDeadline, formatBashElapsed, isSafeBashJobId, jobStartedAtMs, readJobPgid, stopBackgroundBashJob, tailJobLog, writeJobDeadlineMs } from '../../core/bash-jobs.js';
@@ -41,19 +40,17 @@ const nodeBashBackground = defineLeaf({
41
40
  effects: ['Signals every foreground bash command currently running for the node to return from its tool call immediately while its detached supervisor keeps the command running.', 'Each handed-off command retains its log under the node context jobs directory and sends the node an urgent inbox message when it exits, which steers the node mid-turn.'],
42
41
  },
43
42
  run: async (input) => {
44
- const { id, pane } = resolveBashTarget(input, 'no node found for the bash handoff');
43
+ const { id } = resolveBashTarget(input, 'no node found for the bash handoff');
45
44
  if ((await getNodeOrNull(id)) === null) {
46
45
  throw new InputError({ error: 'not_found', message: `no node: ${id}`, next: 'List nodes with `crtr node inspect list`.' });
47
46
  }
48
47
  const jobs = backgroundRunningBashJobs(contextDir(id));
49
48
  const now = Date.now();
50
49
  const elapsed = jobs.map((job) => ({ job_id: job.jobId, elapsed_ms: now - (jobStartedAtMs(job.jobId) ?? now) }));
51
- const toast = jobs.length === 0
52
- ? 'crtr: no running bash command'
53
- : jobs.length === 1
54
- ? `crtr: bash handed off · ${formatBashElapsed(elapsed[0].elapsed_ms)}`
55
- : `crtr: bash handed off (${jobs.length})`;
56
- displayMessage(toast, pane);
50
+ // No tmux toast. A tmux status-line message commandeers the whole client's
51
+ // status bar for seconds over an action the user just took deliberately;
52
+ // the viewer reports its own handoff in its own status line, which is where
53
+ // someone watching that pane is already looking.
57
54
  return { node_id: id, jobs: elapsed };
58
55
  },
59
56
  render: (result) => {
@@ -9,4 +9,4 @@ export interface TestFleet {
9
9
  * process's fleet. Call in `before`/`beforeEach`; `bindFleet` throws on a
10
10
  * conflicting rebind, so the previous binding is reset first (same idiom as
11
11
  * resetEventSourceForTests). Pass `now` to inject the throttle clock. */
12
- export declare function bindTestFleet(opts?: Partial<Pick<DaemonFleetOpts, 'epoch' | 'now'>>): TestFleet;
12
+ export declare function bindTestFleet(opts?: Partial<Pick<DaemonFleetOpts, 'epoch' | 'now' | 'probeOnline'>>): TestFleet;
@@ -30,6 +30,11 @@ export function bindTestFleet(opts = {}) {
30
30
  });
31
31
  },
32
32
  ...(opts.now !== undefined ? { now: opts.now } : {}),
33
+ // Respawn policy consults network reachability — an offline box parks a
34
+ // node rather than spending its respawn budget. Default it online so tests
35
+ // stay deterministic off the network; pass `probeOnline` to drive the
36
+ // outage branch.
37
+ probeOnline: opts.probeOnline ?? (async () => true),
33
38
  });
34
39
  bindFleet(fleet);
35
40
  return {
@@ -252,7 +252,11 @@ export async function createHarness(opts = {}) {
252
252
  await tail;
253
253
  } while (tail !== laneTail);
254
254
  }
255
- const fleet = new DaemonFleet({ epoch: 'harness-epoch', enqueue: enqueueLaneJob });
255
+ // Respawn policy probes network reachability (an offline box parks a node
256
+ // instead of spending its respawn budget). Reachability is an environment
257
+ // condition, not runtime behavior under test — pin it online so a sandboxed
258
+ // or offline runner exercises the same recovery path a connected one does.
259
+ const fleet = new DaemonFleet({ epoch: 'harness-epoch', enqueue: enqueueLaneJob, probeOnline: async () => true });
256
260
  bindFleet(fleet);
257
261
  // Neutralize ensureDaemon (spawn.ts calls it on every `node new`): write a
258
262
  // fake-live-daemon pidfile pointing at OUR (always-alive) pid, so
@@ -0,0 +1,20 @@
1
+ import type { ModelRoutingConfig, ModelStrength } from '../types.js';
2
+ export type CoreModelRoute = {
3
+ family: string;
4
+ credentialSource: string;
5
+ providerId: string;
6
+ models: Record<ModelStrength, string>;
7
+ };
8
+ export type CoreKindAssignment = {
9
+ model: string;
10
+ orchestratorModel?: string;
11
+ };
12
+ export type CoreModelConfig = {
13
+ modelRoutes: Record<string, CoreModelRoute>;
14
+ modelRouting: ModelRoutingConfig;
15
+ kinds: Record<string, CoreKindAssignment>;
16
+ };
17
+ /** Validate the complete Core-owned payload before its single user-scope write. */
18
+ export declare function validateCoreModelConfig(value: unknown): CoreModelConfig;
19
+ /** Write Core-owned fields atomically while retaining unrelated config and kind fields. */
20
+ export declare function replaceCoreModelConfig(config: CoreModelConfig): void;
@@ -0,0 +1,107 @@
1
+ import { updateRawConfigAtomically } from './config.js';
2
+ const strengths = ['ultra', 'strong', 'medium', 'light'];
3
+ const logicalModel = /^[^/\s]+\/(ultra|strong|medium|light)$/;
4
+ function object(value, name) {
5
+ if (value === null || typeof value !== 'object' || Array.isArray(value))
6
+ throw new Error(`${name} must be an object`);
7
+ return value;
8
+ }
9
+ function nonempty(value, name) {
10
+ if (typeof value !== 'string' || value.trim() === '')
11
+ throw new Error(`${name} must be a non-empty string`);
12
+ return value;
13
+ }
14
+ /** Validate the complete Core-owned payload before its single user-scope write. */
15
+ export function validateCoreModelConfig(value) {
16
+ const raw = object(value, 'model config');
17
+ const routes = object(raw.modelRoutes, 'modelRoutes');
18
+ const modelRoutes = {};
19
+ for (const [id, value] of Object.entries(routes)) {
20
+ nonempty(id, 'route id');
21
+ const route = object(value, `modelRoutes.${id}`);
22
+ const providerId = nonempty(route.providerId, `modelRoutes.${id}.providerId`);
23
+ const models = object(route.models, `modelRoutes.${id}.models`);
24
+ const complete = {};
25
+ for (const strength of strengths) {
26
+ const spec = nonempty(models[strength], `modelRoutes.${id}.models.${strength}`);
27
+ const slash = spec.indexOf('/');
28
+ if (slash <= 0 || slash === spec.length - 1 || spec.slice(0, slash) !== providerId) {
29
+ throw new Error(`modelRoutes.${id}.models.${strength} must be a ${providerId}/id ref`);
30
+ }
31
+ complete[strength] = spec;
32
+ }
33
+ for (const strength of Object.keys(models)) {
34
+ if (!strengths.includes(strength))
35
+ throw new Error(`modelRoutes.${id}.models.${strength} is not a model strength`);
36
+ }
37
+ modelRoutes[id] = {
38
+ family: nonempty(route.family, `modelRoutes.${id}.family`),
39
+ credentialSource: nonempty(route.credentialSource, `modelRoutes.${id}.credentialSource`),
40
+ providerId,
41
+ models: complete,
42
+ };
43
+ }
44
+ const modelRouting = object(raw.modelRouting, 'modelRouting');
45
+ for (const field of ['credentialSourceOrder', 'familyOrder']) {
46
+ if (modelRouting[field] !== undefined && (!Array.isArray(modelRouting[field]) || !modelRouting[field].every((item) => typeof item === 'string' && item.trim() !== ''))) {
47
+ throw new Error(`modelRouting.${field} must be an array of non-empty strings`);
48
+ }
49
+ }
50
+ if (modelRouting.strengthFallback !== undefined && (!Array.isArray(modelRouting.strengthFallback) || !modelRouting.strengthFallback.every((item) => strengths.includes(item)))) {
51
+ throw new Error('modelRouting.strengthFallback must be an array of model strengths');
52
+ }
53
+ const families = new Set(Object.values(modelRoutes).map((route) => route.family));
54
+ const credentialSources = new Set(Object.values(modelRoutes).map((route) => route.credentialSource));
55
+ for (const family of modelRouting.familyOrder ?? []) {
56
+ if (!families.has(family))
57
+ throw new Error(`modelRouting.familyOrder names unknown route family ${family}`);
58
+ }
59
+ for (const source of modelRouting.credentialSourceOrder ?? []) {
60
+ if (!credentialSources.has(source))
61
+ throw new Error(`modelRouting.credentialSourceOrder names unknown credential source ${source}`);
62
+ }
63
+ const rawKinds = object(raw.kinds, 'kinds');
64
+ const kinds = {};
65
+ for (const [kind, value] of Object.entries(rawKinds)) {
66
+ nonempty(kind, 'kind');
67
+ const assignment = object(value, `kinds.${kind}`);
68
+ const model = nonempty(assignment.model, `kinds.${kind}.model`);
69
+ if (!logicalModel.test(model))
70
+ throw new Error(`kinds.${kind}.model must be family/strength`);
71
+ if (!families.has(model.slice(0, model.lastIndexOf('/'))))
72
+ throw new Error(`kinds.${kind}.model names unknown route family`);
73
+ const orchestratorModel = assignment.orchestratorModel === undefined ? undefined : nonempty(assignment.orchestratorModel, `kinds.${kind}.orchestratorModel`);
74
+ if (orchestratorModel !== undefined && !logicalModel.test(orchestratorModel))
75
+ throw new Error(`kinds.${kind}.orchestratorModel must be family/strength`);
76
+ if (orchestratorModel !== undefined && !families.has(orchestratorModel.slice(0, orchestratorModel.lastIndexOf('/'))))
77
+ throw new Error(`kinds.${kind}.orchestratorModel names unknown route family`);
78
+ kinds[kind] = { model, ...(orchestratorModel === undefined ? {} : { orchestratorModel }) };
79
+ }
80
+ return { modelRoutes, modelRouting: structuredClone(modelRouting), kinds };
81
+ }
82
+ /** Write Core-owned fields atomically while retaining unrelated config and kind fields. */
83
+ export function replaceCoreModelConfig(config) {
84
+ updateRawConfigAtomically('user', (latest) => {
85
+ const currentKinds = latest.kinds !== null && typeof latest.kinds === 'object' && !Array.isArray(latest.kinds)
86
+ ? latest.kinds
87
+ : {};
88
+ const kinds = Object.fromEntries(Object.entries(currentKinds).map(([kind, value]) => {
89
+ const previous = value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
90
+ const { model: _model, orchestratorModel: _orchestratorModel, ...unowned } = previous;
91
+ return [kind, unowned];
92
+ }));
93
+ for (const [kind, assignment] of Object.entries(config.kinds)) {
94
+ const previous = kinds[kind] !== null && typeof kinds[kind] === 'object' && !Array.isArray(kinds[kind])
95
+ ? kinds[kind]
96
+ : {};
97
+ // Materialize Core's inheritance here: omission cannot clear a lower scope field.
98
+ kinds[kind] = { ...previous, model: assignment.model, orchestratorModel: assignment.orchestratorModel ?? assignment.model };
99
+ }
100
+ return {
101
+ ...latest,
102
+ modelRoutes: structuredClone(config.modelRoutes),
103
+ modelRouting: structuredClone(config.modelRouting),
104
+ kinds,
105
+ };
106
+ });
107
+ }
@@ -59,6 +59,11 @@ export interface UnregisteredConcreteModel {
59
59
  candidate: RoutedCandidate;
60
60
  reason: 'unregistered';
61
61
  }
62
+ export declare function parseModelSpec(spec: string): {
63
+ providerId: string;
64
+ modelId: string;
65
+ thinkingLevel?: string;
66
+ } | undefined;
62
67
  /** Expand config-derived route order only. Availability never affects rank. */
63
68
  export declare function expandModelCandidates(request: ModelRequest, cwd?: string, profileId?: string | null): RoutedCandidate[];
64
69
  /** Identify the one permanently invalid recipe shape: a concrete/exact request
@@ -2,7 +2,7 @@ import { readMergedLaunchConfig } from './config.js';
2
2
  import { isManagedProvider, readRotationConfig, readSubscriptionPool, resolveProviderCandidates, } from './subscription-state.js';
3
3
  const STRENGTHS = ['ultra', 'strong', 'medium', 'light'];
4
4
  const THINKING_SUFFIXES = new Set(['off', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max']);
5
- function parseSpec(spec) {
5
+ export function parseModelSpec(spec) {
6
6
  const slash = spec.indexOf('/');
7
7
  if (slash <= 0 || slash === spec.length - 1)
8
8
  return undefined;
@@ -43,13 +43,13 @@ export function expandModelCandidates(request, cwd, profileId) {
43
43
  const declarationFamilies = routeEntries.map(([, route]) => route.family).filter((value, index, all) => all.indexOf(value) === index);
44
44
  const declarationSources = routeEntries.map(([, route]) => route.credentialSource).filter((value, index, all) => all.indexOf(value) === index);
45
45
  if (request.kind === 'exact') {
46
- const parsed = parseSpec(request.spec);
46
+ const parsed = parseModelSpec(request.spec);
47
47
  return parsed ? [{ routeId: 'exact', credentialSource: 'exact', family: parsed.providerId, strength: 'strong', ...parsed, rank: 0 }] : [];
48
48
  }
49
- const concrete = request.kind === 'concrete' ? parseSpec(request.spec) : undefined;
49
+ const concrete = request.kind === 'concrete' ? parseModelSpec(request.spec) : undefined;
50
50
  const logical = request.kind === 'logical' ? request : undefined;
51
51
  const matchesConcrete = (spec, exactThinking) => {
52
- const parsed = parseSpec(spec);
52
+ const parsed = parseModelSpec(spec);
53
53
  return parsed !== undefined
54
54
  && concrete !== undefined
55
55
  && parsed.providerId === concrete.providerId
@@ -79,7 +79,7 @@ export function expandModelCandidates(request, cwd, profileId) {
79
79
  const emitted = [];
80
80
  const seen = new Set();
81
81
  const append = (routeId, route, strength, spec) => {
82
- const parsed = parseSpec(spec);
82
+ const parsed = parseModelSpec(spec);
83
83
  if (!parsed)
84
84
  return;
85
85
  if (concrete && parsed.modelId !== concrete.modelId)
@@ -1,3 +1,7 @@
1
+ export type TransportPoisonVerdict = {
2
+ consecutivePasses: number;
3
+ routes: string[];
4
+ };
1
5
  /** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
2
6
  * prior clean-abort certificate stale before it can affect a later crash. */
3
7
  export declare function markBusy(nodeId: string): void;
@@ -10,6 +14,15 @@ export declare function clearBusy(nodeId: string): void;
10
14
  export declare function markCleanAbort(nodeId: string): void;
11
15
  /** Remove a consumed clean-abort certificate as a broker boots. */
12
16
  export declare function clearCleanAbort(nodeId: string): void;
17
+ /** Record the provider-rotation verdict immediately before it terminates this broker.
18
+ * It survives the broker exit so the fleet can surface the cause if this launch
19
+ * exhausts respawn recovery. */
20
+ export declare function recordTransportPoisonVerdict(nodeId: string, verdict: TransportPoisonVerdict): void;
21
+ /** A verdict belongs to one broker launch. Clear it before every replacement so
22
+ * a later unrelated crash loop cannot inherit an old transport diagnosis. */
23
+ export declare function clearTransportPoisonVerdict(nodeId: string): void;
24
+ /** Read only a complete, structurally valid provider-rotation verdict. */
25
+ export declare function readTransportPoisonVerdict(nodeId: string): TransportPoisonVerdict | null;
13
26
  /** Whether the dead broker left a persisted clean-abort turn witness. */
14
27
  export declare function hasCleanAbort(nodeId: string): boolean;
15
28
  /** Is the node currently inside a turn? AND this with `pidAlive` at the call
@@ -13,7 +13,7 @@
13
13
  // read site, so a stale marker (process crashed mid-turn without firing
14
14
  // agent_end) is harmless — the dead pid fails the AND and the node is reaped.
15
15
  // No db migration, atomic touch/unlink, best-effort (never throws).
16
- import { existsSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
16
+ import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
17
17
  import { join } from 'node:path';
18
18
  import { jobDir } from '../canvas/paths.js';
19
19
  function busyPath(nodeId) {
@@ -22,6 +22,9 @@ function busyPath(nodeId) {
22
22
  function cleanAbortPath(nodeId) {
23
23
  return join(jobDir(nodeId), 'clean-abort');
24
24
  }
25
+ function transportPoisonPath(nodeId) {
26
+ return join(jobDir(nodeId), 'transport-poison.json');
27
+ }
25
28
  /** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
26
29
  * prior clean-abort certificate stale before it can affect a later crash. */
27
30
  export function markBusy(nodeId) {
@@ -65,6 +68,45 @@ export function clearCleanAbort(nodeId) {
65
68
  /* best-effort */
66
69
  }
67
70
  }
71
+ /** Record the provider-rotation verdict immediately before it terminates this broker.
72
+ * It survives the broker exit so the fleet can surface the cause if this launch
73
+ * exhausts respawn recovery. */
74
+ export function recordTransportPoisonVerdict(nodeId, verdict) {
75
+ try {
76
+ mkdirSync(jobDir(nodeId), { recursive: true });
77
+ writeFileSync(transportPoisonPath(nodeId), JSON.stringify(verdict));
78
+ }
79
+ catch {
80
+ /* best-effort: no marker means the crash report remains explicitly unknown */
81
+ }
82
+ }
83
+ /** A verdict belongs to one broker launch. Clear it before every replacement so
84
+ * a later unrelated crash loop cannot inherit an old transport diagnosis. */
85
+ export function clearTransportPoisonVerdict(nodeId) {
86
+ try {
87
+ rmSync(transportPoisonPath(nodeId), { force: true });
88
+ }
89
+ catch {
90
+ /* best-effort */
91
+ }
92
+ }
93
+ /** Read only a complete, structurally valid provider-rotation verdict. */
94
+ export function readTransportPoisonVerdict(nodeId) {
95
+ try {
96
+ const value = JSON.parse(readFileSync(transportPoisonPath(nodeId), 'utf8'));
97
+ if (typeof value !== 'object' || value === null)
98
+ return null;
99
+ const verdict = value;
100
+ const consecutivePasses = verdict.consecutivePasses;
101
+ const routes = verdict.routes;
102
+ if (typeof consecutivePasses !== 'number' || !Number.isInteger(consecutivePasses) || consecutivePasses < 1 || !Array.isArray(routes) || routes.length === 0 || !routes.every((route) => typeof route === 'string' && route !== ''))
103
+ return null;
104
+ return { consecutivePasses, routes };
105
+ }
106
+ catch {
107
+ return null;
108
+ }
109
+ }
68
110
  /** Whether the dead broker left a persisted clean-abort turn witness. */
69
111
  export function hasCleanAbort(nodeId) {
70
112
  return existsSync(cleanAbortPath(nodeId));
@@ -38,6 +38,7 @@ import { encodeKickoffOrigin, KICKOFF_ORIGIN_ENV } from './stamp/protocol.js';
38
38
  import { headlessBrokerHost } from './host.js';
39
39
  import { boundFleet, brokerCapReached, brokerThresholdsForDaemon } from './fleet.js';
40
40
  import { clearFault, beginBootFaultAttempt } from './fault.js';
41
+ import { clearTransportPoisonVerdict } from './busy.js';
41
42
  import { clearContextExposureState, initializeForkContextExposure, } from '../substrate/injected-store.js';
42
43
  import { rootOfSpine } from './nodes.js';
43
44
  import { isReviewCompanionBound } from '../review/companion.js';
@@ -468,6 +469,7 @@ async function launchRevive(nodeId, meta, opts) {
468
469
  if (opts.recovery !== true)
469
470
  cancelCronsOnWake(nodeId);
470
471
  ensureNodeDirs(nodeId);
472
+ clearTransportPoisonVerdict(nodeId);
471
473
  const lifecycleHooks = discoverLifecycleHookRegistry(meta.cwd, meta.profile_id).plans.get('node:start') ?? [];
472
474
  await invokeLifecycleHooks(lifecycleHooks, {
473
475
  node: {
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,73 @@
1
+ import { after, afterEach, beforeEach, test } from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
4
+ import { tmpdir } from 'node:os';
5
+ import { join } from 'node:path';
6
+ import { ApiError } from '../../../api/errors.js';
7
+ import { resetScopeCache } from '../../../core/scope.js';
8
+ import { modelConfigRoutes } from '../handlers/model-config.js';
9
+ const previousHome = process.env.HOME;
10
+ const previousAgentDir = process.env['PI_CODING_AGENT_DIR'];
11
+ let home = '';
12
+ const payload = {
13
+ config_version: 'core-doc-digest',
14
+ modelRoutes: {
15
+ northlight: {
16
+ family: 'cloud', credentialSource: 'northlight', providerId: 'northlight',
17
+ models: { ultra: 'northlight/u', strong: 'northlight/s:high', medium: 'northlight/m', light: 'northlight/l' },
18
+ },
19
+ },
20
+ modelRouting: { familyOrder: ['cloud'], strengthFallback: ['strong', 'medium'] },
21
+ kinds: { developer: { model: 'cloud/medium' } },
22
+ };
23
+ beforeEach(() => {
24
+ home = mkdtempSync(join(tmpdir(), 'crtr-model-config-'));
25
+ process.env.HOME = home;
26
+ const agentDir = join(home, 'agent');
27
+ mkdirSync(agentDir);
28
+ process.env['PI_CODING_AGENT_DIR'] = agentDir;
29
+ writeFileSync(join(agentDir, 'models.json'), JSON.stringify({
30
+ providers: { northlight: { baseUrl: 'http://127.0.0.1:1', api: 'openai-completions', apiKey: 'test', models: ['u', 's', 'm', 'l'].map((id) => ({ id, name: id })) } },
31
+ }));
32
+ resetScopeCache();
33
+ });
34
+ afterEach(() => {
35
+ rmSync(home, { recursive: true, force: true });
36
+ resetScopeCache();
37
+ });
38
+ after(() => {
39
+ if (previousHome === undefined)
40
+ delete process.env.HOME;
41
+ else
42
+ process.env.HOME = previousHome;
43
+ if (previousAgentDir === undefined)
44
+ delete process.env['PI_CODING_AGENT_DIR'];
45
+ else
46
+ process.env['PI_CODING_AGENT_DIR'] = previousAgentDir;
47
+ resetScopeCache();
48
+ });
49
+ function putRoute() {
50
+ const route = modelConfigRoutes.find((entry) => entry.method === 'PUT' && entry.pattern === '/v1/model-config');
51
+ assert.ok(route, 'PUT /v1/model-config is registered');
52
+ return route;
53
+ }
54
+ function context(body) {
55
+ return { method: 'PUT', path: '/v1/model-config', params: {}, query: new URLSearchParams(), body };
56
+ }
57
+ test('PUT atomically replaces Core-owned blocks without touching unrelated config', async () => {
58
+ const configPath = join(home, '.crouter', 'config.json');
59
+ mkdirSync(join(home, '.crouter'));
60
+ writeFileSync(configPath, JSON.stringify({ completion_bell: false, kinds: { developer: { tools: ['bash'], model: 'old/strong', orchestratorModel: 'old/ultra' } } }));
61
+ const result = await putRoute().handler(context(payload));
62
+ assert.deepEqual(result, { status: 200, body: { config_version: 'core-doc-digest', delivered: true } });
63
+ const written = JSON.parse(readFileSync(configPath, 'utf8'));
64
+ assert.equal(written.completion_bell, false);
65
+ assert.deepEqual(written.kinds.developer, { tools: ['bash'], model: 'cloud/medium', orchestratorModel: 'cloud/medium' });
66
+ assert.deepEqual(written.modelRoutes, payload.modelRoutes);
67
+ });
68
+ test('PUT rejects invalid payloads and absent baked refs before writing', async () => {
69
+ const configPath = join(home, '.crouter', 'config.json');
70
+ await assert.rejects(async () => putRoute().handler(context({ ...payload, modelRoutes: { ...payload.modelRoutes, northlight: { ...payload.modelRoutes.northlight, models: { strong: 'northlight/s' } } } })), (error) => error instanceof ApiError && error.status === 400 && error.code === 'invalid_request');
71
+ await assert.rejects(async () => putRoute().handler(context({ ...payload, modelRoutes: { ...payload.modelRoutes, northlight: { ...payload.modelRoutes.northlight, models: { ...payload.modelRoutes.northlight.models, ultra: 'northlight/missing' } } } })), (error) => error instanceof ApiError && error.status === 422 && error.code === 'unprocessable');
72
+ assert.throws(() => readFileSync(configPath, 'utf8'), { code: 'ENOENT' });
73
+ });
@@ -0,0 +1,2 @@
1
+ import type { RouteTable } from '../router.js';
2
+ export declare const modelConfigRoutes: RouteTable;
@@ -0,0 +1,44 @@
1
+ import { ApiError } from '../../../api/errors.js';
2
+ import { validateCoreModelConfig, replaceCoreModelConfig } from '../../../core/model-config.js';
3
+ import { parseModelSpec } from '../../../core/model-routes.js';
4
+ import { openModelRegistry } from '../../../core/runtime/model-registry.js';
5
+ function parseRequest(body) {
6
+ if (body === null || typeof body !== 'object' || Array.isArray(body))
7
+ throw new ApiError(400, 'invalid_request', 'model config must be an object');
8
+ const request = body;
9
+ if (typeof request.config_version !== 'string' || request.config_version.trim() === '') {
10
+ throw new ApiError(400, 'invalid_request', 'config_version must be a non-empty string');
11
+ }
12
+ try {
13
+ return { configVersion: request.config_version, config: validateCoreModelConfig(request) };
14
+ }
15
+ catch (error) {
16
+ throw new ApiError(400, 'invalid_request', error instanceof Error ? error.message : String(error));
17
+ }
18
+ }
19
+ /** Validate route refs against this image's baked descriptor catalog before disk changes. */
20
+ async function validateCatalog(config) {
21
+ const registry = await openModelRegistry();
22
+ for (const route of Object.values(config.modelRoutes)) {
23
+ for (const spec of Object.values(route.models)) {
24
+ const model = parseModelSpec(spec);
25
+ if (model === undefined || registry.find(model.providerId, model.modelId) === undefined) {
26
+ throw new ApiError(422, 'unprocessable', `route ref ${spec} is absent from the baked model registry`);
27
+ }
28
+ }
29
+ }
30
+ }
31
+ async function handlePut(ctx) {
32
+ const { configVersion, config } = parseRequest(ctx.body);
33
+ await validateCatalog(config);
34
+ try {
35
+ replaceCoreModelConfig(config);
36
+ }
37
+ catch (error) {
38
+ throw new ApiError(500, 'model_config_write_failed', error instanceof Error ? error.message : String(error));
39
+ }
40
+ return { status: 200, body: { config_version: configVersion, delivered: true } };
41
+ }
42
+ export const modelConfigRoutes = [
43
+ { method: 'PUT', pattern: '/v1/model-config', handler: handlePut },
44
+ ];
@@ -36,6 +36,7 @@ import { feedbackCommentRoutes } from './handlers/feedback-comments.js';
36
36
  import { memoryRoutes } from './handlers/memory.js';
37
37
  import { messageRoutes } from './handlers/messages.js';
38
38
  import { modelAuthRoutes } from './handlers/modelauth.js';
39
+ import { modelConfigRoutes } from './handlers/model-config.js';
39
40
  import { nodeRoutes } from './handlers/nodes.js';
40
41
  import { profileRoutes } from './handlers/profiles.js';
41
42
  import { prospectiveChatInventoryRoutes } from './handlers/prospective-chat-inventory.js';
@@ -67,6 +68,7 @@ function buildRouter() {
67
68
  .registerAll(cronRoutes)
68
69
  .registerAll(profileRoutes)
69
70
  .registerAll(modelAuthRoutes)
71
+ .registerAll(modelConfigRoutes)
70
72
  .registerAll(humanRoutes)
71
73
  .registerAll(reviewRoutes)
72
74
  .registerAll(reviewCommentRoutes)
@@ -52,6 +52,9 @@ export interface DaemonFleetOpts {
52
52
  enqueue: (job: () => void | Promise<void>) => void;
53
53
  /** Injectable clock for tests. */
54
54
  now?: () => number;
55
+ /** Injectable network-reachability probe. Every respawn decision consults it,
56
+ * so tests drive the offline branch without touching the real network. */
57
+ probeOnline?: () => Promise<boolean>;
55
58
  }
56
59
  export declare class DaemonFleet implements FleetRegistry {
57
60
  #private;
@@ -16,9 +16,10 @@ import { fullName } from '../core/canvas/labels.js';
16
16
  import { isSafeNodeId } from '../core/canvas/paths.js';
17
17
  import { transition } from '../core/runtime/lifecycle.js';
18
18
  import { fanDoctrineWake } from '../core/runtime/close.js';
19
- import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
19
+ import { hasCleanAbort, isBusy, readTransportPoisonVerdict } from '../core/runtime/busy.js';
20
20
  import { hasPendingWake } from './reconcilers/live-obligation.js';
21
- import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
21
+ import { clearRespawnPolicy, deferRespawnWhileOffline, MAX_RESPAWN_ATTEMPTS, OFFLINE_RESPAWN_RETRY_MS, recordRespawnPolicy, respawnBackoffMs, } from './reconcilers/node-lifecycle/respawn-policy.js';
22
+ import { probeOnline as probeOnlineReal } from '../core/runtime/connectivity.js';
22
23
  import { reviveNode } from '../core/runtime/revive.js';
23
24
  import { pushUrgent } from '../core/feed/feed.js';
24
25
  import { emitEvent } from '../core/events/emit.js';
@@ -42,9 +43,14 @@ export async function surfaceBootFailure(meta) {
42
43
  await pushUrgent(meta.node_id, body, { from: meta.node_id });
43
44
  }
44
45
  async function surfaceCrashLoop(meta) {
46
+ const transportPoison = readTransportPoisonVerdict(meta.node_id);
47
+ const observedCause = transportPoison === null
48
+ ? ''
49
+ : `\n\nProvider rotation observed ${transportPoison.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${transportPoison.routes.join(', ')}). Check network and provider transport before reviving.`;
45
50
  const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
46
- `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.\n\n` +
47
- `Fix the crash cause, then revive the node if the work still needs doing.`;
51
+ `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.` +
52
+ observedCause +
53
+ `\n\nFix the crash cause, then revive the node if the work still needs doing.`;
48
54
  await pushUrgent(meta.node_id, body, { from: meta.node_id });
49
55
  }
50
56
  /** Surface a recovery refusal that happened before a broker could launch. */
@@ -121,6 +127,7 @@ export class DaemonFleet {
121
127
  #epoch;
122
128
  #enqueue;
123
129
  #now;
130
+ #probeOnline;
124
131
  #map = new Map();
125
132
  /** Slots claimed by launches that have not registered a broker yet. */
126
133
  #reservations = new Set();
@@ -129,6 +136,7 @@ export class DaemonFleet {
129
136
  this.#epoch = opts.epoch;
130
137
  this.#enqueue = opts.enqueue;
131
138
  this.#now = opts.now ?? Date.now;
139
+ this.#probeOnline = opts.probeOnline ?? (() => probeOnlineReal());
132
140
  }
133
141
  epoch() {
134
142
  return this.#epoch;
@@ -292,8 +300,30 @@ export class DaemonFleet {
292
300
  const row = getRow(nodeId);
293
301
  if (row === null)
294
302
  return;
295
- const assess = row.respawn_not_before == null;
296
303
  const priorFailures = row.respawn_failures ?? 0;
304
+ // A box with no network fails every relaunch for a reason that has nothing
305
+ // to do with this node: with no route reachable, the broker's provider queue
306
+ // exhausts with no HTTP response at all, it declares its own transport
307
+ // poisoned, and it exits for a fresh process that fails identically. Six of
308
+ // those and the node is dead a few minutes into an outage, holding an
309
+ // unfinished turn that only a human revive resumes. Park instead of counting
310
+ // — the outage is a condition to wait out, not a failure sequence to pace.
311
+ if (!(await this.#probeOnline())) {
312
+ deferRespawnWhileOffline(nodeId, priorFailures, this.#now());
313
+ emitEvent({
314
+ level: 'info',
315
+ event: 'broker.respawn.offline_deferred',
316
+ ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
317
+ fields: {
318
+ ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
319
+ action,
320
+ retry_in_ms: OFFLINE_RESPAWN_RETRY_MS,
321
+ consecutive_failures: priorFailures,
322
+ },
323
+ });
324
+ return;
325
+ }
326
+ const assess = row.respawn_not_before == null;
297
327
  const attempt = assess
298
328
  ? recordRespawnPolicy(row, this.#now())
299
329
  : { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
@@ -1,5 +1,11 @@
1
1
  import { type NodeRow } from '../../../core/canvas/index.js';
2
2
  export declare const MAX_RESPAWN_ATTEMPTS = 6;
3
+ /** How long an offline box waits before re-testing the network and relaunching.
4
+ * Flat, not exponential: this is not a failure sequence being paced, it is a
5
+ * wait for a condition outside the node — the same "my wifi is back" gate the
6
+ * broker's connection-fault retry uses, sampled less often because each sample
7
+ * here can cost a whole broker launch. */
8
+ export declare const OFFLINE_RESPAWN_RETRY_MS = 30000;
3
9
  export declare function respawnBackoffMs(consecutiveFailures: number): number;
4
10
  export interface RespawnAttempt {
5
11
  failures: number;
@@ -11,6 +17,14 @@ export declare function isInBackoff(row: NodeRow, now: number): boolean;
11
17
  /** The only respawn-failure policy: progress since launch clears the sequence;
12
18
  * every other missing-engine observation advances it and records its backoff. */
13
19
  export declare function decideRespawnAttempt(row: Pick<NodeRow, 'last_settled_at' | 'launched_at' | 'respawn_failures'>, now: number): RespawnAttempt;
20
+ /** Hold a relaunch while the box has no network, WITHOUT advancing the failure
21
+ * sequence. An outage makes every launch fail identically, so counting those
22
+ * launches spends the whole respawn budget on a condition the node did not
23
+ * cause and cannot fix — the node terminalizes minutes into an outage and only
24
+ * a human revive brings it back. Re-arming instead parks it: the row stays
25
+ * active, the tick re-enters this policy after the window, and the first
26
+ * launch made once the network returns resumes the interrupted turn. */
27
+ export declare function deferRespawnWhileOffline(nodeId: string, failures: number, now: number): RespawnAttempt;
14
28
  /** Persist the policy decision through the daemon-only canvas writer. */
15
29
  export declare function recordRespawnPolicy(row: NodeRow, now: number): RespawnAttempt;
16
30
  export declare function clearRespawnPolicy(nodeId: string): void;
@@ -2,6 +2,12 @@ import { recordRespawnAttempt } from '../../../core/canvas/index.js';
2
2
  export const MAX_RESPAWN_ATTEMPTS = 6;
3
3
  const RESPAWN_BACKOFF_BASE_MS = 5_000;
4
4
  const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
5
+ /** How long an offline box waits before re-testing the network and relaunching.
6
+ * Flat, not exponential: this is not a failure sequence being paced, it is a
7
+ * wait for a condition outside the node — the same "my wifi is back" gate the
8
+ * broker's connection-fault retry uses, sampled less often because each sample
9
+ * here can cost a whole broker launch. */
10
+ export const OFFLINE_RESPAWN_RETRY_MS = 30_000;
5
11
  export function respawnBackoffMs(consecutiveFailures) {
6
12
  if (consecutiveFailures <= 0)
7
13
  return 0;
@@ -30,6 +36,18 @@ export function decideRespawnAttempt(row, now) {
30
36
  exhausted: failures >= MAX_RESPAWN_ATTEMPTS,
31
37
  };
32
38
  }
39
+ /** Hold a relaunch while the box has no network, WITHOUT advancing the failure
40
+ * sequence. An outage makes every launch fail identically, so counting those
41
+ * launches spends the whole respawn budget on a condition the node did not
42
+ * cause and cannot fix — the node terminalizes minutes into an outage and only
43
+ * a human revive brings it back. Re-arming instead parks it: the row stays
44
+ * active, the tick re-enters this policy after the window, and the first
45
+ * launch made once the network returns resumes the interrupted turn. */
46
+ export function deferRespawnWhileOffline(nodeId, failures, now) {
47
+ const notBefore = new Date(now + OFFLINE_RESPAWN_RETRY_MS).toISOString();
48
+ recordRespawnAttempt(nodeId, { failures, notBefore });
49
+ return { failures, notBefore, exhausted: false };
50
+ }
33
51
  /** Persist the policy decision through the daemon-only canvas writer. */
34
52
  export function recordRespawnPolicy(row, now) {
35
53
  const attempt = decideRespawnAttempt(row, now);