@north-light/crouter 0.3.278 → 0.3.280
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/dto/model-config.d.ts +25 -0
- package/dist/api/dto/model-config.js +1 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +11 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +11 -3
- package/dist/clients/attach/chrome/status-line.js +1 -22
- package/dist/clients/attach/render/chat-view.d.ts +11 -0
- package/dist/clients/attach/render/chat-view.js +23 -4
- package/dist/clients/attach/render/tool-calls.d.ts +6 -1
- package/dist/clients/attach/render/tool-calls.js +47 -3
- package/dist/clients/attach/session/bindings.d.ts +1 -1
- package/dist/clients/attach/session/bindings.js +1 -1
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +3 -3
- package/dist/clients/attach/viewer.js +557 -557
- package/dist/commands/node/bash.js +5 -8
- package/dist/core/__tests__/helpers/fleet.d.ts +1 -1
- package/dist/core/__tests__/helpers/fleet.js +5 -0
- package/dist/core/__tests__/helpers/harness.js +5 -1
- package/dist/core/model-config.d.ts +20 -0
- package/dist/core/model-config.js +107 -0
- package/dist/core/model-routes.d.ts +5 -0
- package/dist/core/model-routes.js +5 -5
- package/dist/core/runtime/busy.d.ts +13 -0
- package/dist/core/runtime/busy.js +43 -1
- package/dist/core/runtime/revive.js +2 -0
- package/dist/daemon/api/__tests__/model-config.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/model-config.test.js +73 -0
- package/dist/daemon/api/handlers/model-config.d.ts +2 -0
- package/dist/daemon/api/handlers/model-config.js +44 -0
- package/dist/daemon/api/server.js +2 -0
- package/dist/daemon/fleet.d.ts +3 -0
- package/dist/daemon/fleet.js +35 -5
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +14 -0
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +18 -0
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
import { envNodeId } from '../../shared/env.js';
|
|
3
3
|
import { defineBranch, defineLeaf } from '../../core/command.js';
|
|
4
4
|
import { InputError } from '../../core/io.js';
|
|
5
|
-
import { displayMessage } from '../../core/runtime/tmux-chrome.js';
|
|
6
5
|
import { contextDir } from '../../core/canvas/paths.js';
|
|
7
6
|
import { existsSync } from 'node:fs';
|
|
8
7
|
import { activeBackgroundBashJobs, backgroundRunningBashJobs, bashJobCommandLine, bashJobPaths, clearJobDeadline, formatBashElapsed, isSafeBashJobId, jobStartedAtMs, readJobPgid, stopBackgroundBashJob, tailJobLog, writeJobDeadlineMs } from '../../core/bash-jobs.js';
|
|
@@ -41,19 +40,17 @@ const nodeBashBackground = defineLeaf({
|
|
|
41
40
|
effects: ['Signals every foreground bash command currently running for the node to return from its tool call immediately while its detached supervisor keeps the command running.', 'Each handed-off command retains its log under the node context jobs directory and sends the node an urgent inbox message when it exits, which steers the node mid-turn.'],
|
|
42
41
|
},
|
|
43
42
|
run: async (input) => {
|
|
44
|
-
const { id
|
|
43
|
+
const { id } = resolveBashTarget(input, 'no node found for the bash handoff');
|
|
45
44
|
if ((await getNodeOrNull(id)) === null) {
|
|
46
45
|
throw new InputError({ error: 'not_found', message: `no node: ${id}`, next: 'List nodes with `crtr node inspect list`.' });
|
|
47
46
|
}
|
|
48
47
|
const jobs = backgroundRunningBashJobs(contextDir(id));
|
|
49
48
|
const now = Date.now();
|
|
50
49
|
const elapsed = jobs.map((job) => ({ job_id: job.jobId, elapsed_ms: now - (jobStartedAtMs(job.jobId) ?? now) }));
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
: `crtr: bash handed off (${jobs.length})`;
|
|
56
|
-
displayMessage(toast, pane);
|
|
50
|
+
// No tmux toast. A tmux status-line message commandeers the whole client's
|
|
51
|
+
// status bar for seconds over an action the user just took deliberately;
|
|
52
|
+
// the viewer reports its own handoff in its own status line, which is where
|
|
53
|
+
// someone watching that pane is already looking.
|
|
57
54
|
return { node_id: id, jobs: elapsed };
|
|
58
55
|
},
|
|
59
56
|
render: (result) => {
|
|
@@ -9,4 +9,4 @@ export interface TestFleet {
|
|
|
9
9
|
* process's fleet. Call in `before`/`beforeEach`; `bindFleet` throws on a
|
|
10
10
|
* conflicting rebind, so the previous binding is reset first (same idiom as
|
|
11
11
|
* resetEventSourceForTests). Pass `now` to inject the throttle clock. */
|
|
12
|
-
export declare function bindTestFleet(opts?: Partial<Pick<DaemonFleetOpts, 'epoch' | 'now'>>): TestFleet;
|
|
12
|
+
export declare function bindTestFleet(opts?: Partial<Pick<DaemonFleetOpts, 'epoch' | 'now' | 'probeOnline'>>): TestFleet;
|
|
@@ -30,6 +30,11 @@ export function bindTestFleet(opts = {}) {
|
|
|
30
30
|
});
|
|
31
31
|
},
|
|
32
32
|
...(opts.now !== undefined ? { now: opts.now } : {}),
|
|
33
|
+
// Respawn policy consults network reachability — an offline box parks a
|
|
34
|
+
// node rather than spending its respawn budget. Default it online so tests
|
|
35
|
+
// stay deterministic off the network; pass `probeOnline` to drive the
|
|
36
|
+
// outage branch.
|
|
37
|
+
probeOnline: opts.probeOnline ?? (async () => true),
|
|
33
38
|
});
|
|
34
39
|
bindFleet(fleet);
|
|
35
40
|
return {
|
|
@@ -252,7 +252,11 @@ export async function createHarness(opts = {}) {
|
|
|
252
252
|
await tail;
|
|
253
253
|
} while (tail !== laneTail);
|
|
254
254
|
}
|
|
255
|
-
|
|
255
|
+
// Respawn policy probes network reachability (an offline box parks a node
|
|
256
|
+
// instead of spending its respawn budget). Reachability is an environment
|
|
257
|
+
// condition, not runtime behavior under test — pin it online so a sandboxed
|
|
258
|
+
// or offline runner exercises the same recovery path a connected one does.
|
|
259
|
+
const fleet = new DaemonFleet({ epoch: 'harness-epoch', enqueue: enqueueLaneJob, probeOnline: async () => true });
|
|
256
260
|
bindFleet(fleet);
|
|
257
261
|
// Neutralize ensureDaemon (spawn.ts calls it on every `node new`): write a
|
|
258
262
|
// fake-live-daemon pidfile pointing at OUR (always-alive) pid, so
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ModelRoutingConfig, ModelStrength } from '../types.js';
|
|
2
|
+
export type CoreModelRoute = {
|
|
3
|
+
family: string;
|
|
4
|
+
credentialSource: string;
|
|
5
|
+
providerId: string;
|
|
6
|
+
models: Record<ModelStrength, string>;
|
|
7
|
+
};
|
|
8
|
+
export type CoreKindAssignment = {
|
|
9
|
+
model: string;
|
|
10
|
+
orchestratorModel?: string;
|
|
11
|
+
};
|
|
12
|
+
export type CoreModelConfig = {
|
|
13
|
+
modelRoutes: Record<string, CoreModelRoute>;
|
|
14
|
+
modelRouting: ModelRoutingConfig;
|
|
15
|
+
kinds: Record<string, CoreKindAssignment>;
|
|
16
|
+
};
|
|
17
|
+
/** Validate the complete Core-owned payload before its single user-scope write. */
|
|
18
|
+
export declare function validateCoreModelConfig(value: unknown): CoreModelConfig;
|
|
19
|
+
/** Write Core-owned fields atomically while retaining unrelated config and kind fields. */
|
|
20
|
+
export declare function replaceCoreModelConfig(config: CoreModelConfig): void;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { updateRawConfigAtomically } from './config.js';
|
|
2
|
+
const strengths = ['ultra', 'strong', 'medium', 'light'];
|
|
3
|
+
const logicalModel = /^[^/\s]+\/(ultra|strong|medium|light)$/;
|
|
4
|
+
function object(value, name) {
|
|
5
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value))
|
|
6
|
+
throw new Error(`${name} must be an object`);
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
function nonempty(value, name) {
|
|
10
|
+
if (typeof value !== 'string' || value.trim() === '')
|
|
11
|
+
throw new Error(`${name} must be a non-empty string`);
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
/** Validate the complete Core-owned payload before its single user-scope write. */
|
|
15
|
+
export function validateCoreModelConfig(value) {
|
|
16
|
+
const raw = object(value, 'model config');
|
|
17
|
+
const routes = object(raw.modelRoutes, 'modelRoutes');
|
|
18
|
+
const modelRoutes = {};
|
|
19
|
+
for (const [id, value] of Object.entries(routes)) {
|
|
20
|
+
nonempty(id, 'route id');
|
|
21
|
+
const route = object(value, `modelRoutes.${id}`);
|
|
22
|
+
const providerId = nonempty(route.providerId, `modelRoutes.${id}.providerId`);
|
|
23
|
+
const models = object(route.models, `modelRoutes.${id}.models`);
|
|
24
|
+
const complete = {};
|
|
25
|
+
for (const strength of strengths) {
|
|
26
|
+
const spec = nonempty(models[strength], `modelRoutes.${id}.models.${strength}`);
|
|
27
|
+
const slash = spec.indexOf('/');
|
|
28
|
+
if (slash <= 0 || slash === spec.length - 1 || spec.slice(0, slash) !== providerId) {
|
|
29
|
+
throw new Error(`modelRoutes.${id}.models.${strength} must be a ${providerId}/id ref`);
|
|
30
|
+
}
|
|
31
|
+
complete[strength] = spec;
|
|
32
|
+
}
|
|
33
|
+
for (const strength of Object.keys(models)) {
|
|
34
|
+
if (!strengths.includes(strength))
|
|
35
|
+
throw new Error(`modelRoutes.${id}.models.${strength} is not a model strength`);
|
|
36
|
+
}
|
|
37
|
+
modelRoutes[id] = {
|
|
38
|
+
family: nonempty(route.family, `modelRoutes.${id}.family`),
|
|
39
|
+
credentialSource: nonempty(route.credentialSource, `modelRoutes.${id}.credentialSource`),
|
|
40
|
+
providerId,
|
|
41
|
+
models: complete,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const modelRouting = object(raw.modelRouting, 'modelRouting');
|
|
45
|
+
for (const field of ['credentialSourceOrder', 'familyOrder']) {
|
|
46
|
+
if (modelRouting[field] !== undefined && (!Array.isArray(modelRouting[field]) || !modelRouting[field].every((item) => typeof item === 'string' && item.trim() !== ''))) {
|
|
47
|
+
throw new Error(`modelRouting.${field} must be an array of non-empty strings`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (modelRouting.strengthFallback !== undefined && (!Array.isArray(modelRouting.strengthFallback) || !modelRouting.strengthFallback.every((item) => strengths.includes(item)))) {
|
|
51
|
+
throw new Error('modelRouting.strengthFallback must be an array of model strengths');
|
|
52
|
+
}
|
|
53
|
+
const families = new Set(Object.values(modelRoutes).map((route) => route.family));
|
|
54
|
+
const credentialSources = new Set(Object.values(modelRoutes).map((route) => route.credentialSource));
|
|
55
|
+
for (const family of modelRouting.familyOrder ?? []) {
|
|
56
|
+
if (!families.has(family))
|
|
57
|
+
throw new Error(`modelRouting.familyOrder names unknown route family ${family}`);
|
|
58
|
+
}
|
|
59
|
+
for (const source of modelRouting.credentialSourceOrder ?? []) {
|
|
60
|
+
if (!credentialSources.has(source))
|
|
61
|
+
throw new Error(`modelRouting.credentialSourceOrder names unknown credential source ${source}`);
|
|
62
|
+
}
|
|
63
|
+
const rawKinds = object(raw.kinds, 'kinds');
|
|
64
|
+
const kinds = {};
|
|
65
|
+
for (const [kind, value] of Object.entries(rawKinds)) {
|
|
66
|
+
nonempty(kind, 'kind');
|
|
67
|
+
const assignment = object(value, `kinds.${kind}`);
|
|
68
|
+
const model = nonempty(assignment.model, `kinds.${kind}.model`);
|
|
69
|
+
if (!logicalModel.test(model))
|
|
70
|
+
throw new Error(`kinds.${kind}.model must be family/strength`);
|
|
71
|
+
if (!families.has(model.slice(0, model.lastIndexOf('/'))))
|
|
72
|
+
throw new Error(`kinds.${kind}.model names unknown route family`);
|
|
73
|
+
const orchestratorModel = assignment.orchestratorModel === undefined ? undefined : nonempty(assignment.orchestratorModel, `kinds.${kind}.orchestratorModel`);
|
|
74
|
+
if (orchestratorModel !== undefined && !logicalModel.test(orchestratorModel))
|
|
75
|
+
throw new Error(`kinds.${kind}.orchestratorModel must be family/strength`);
|
|
76
|
+
if (orchestratorModel !== undefined && !families.has(orchestratorModel.slice(0, orchestratorModel.lastIndexOf('/'))))
|
|
77
|
+
throw new Error(`kinds.${kind}.orchestratorModel names unknown route family`);
|
|
78
|
+
kinds[kind] = { model, ...(orchestratorModel === undefined ? {} : { orchestratorModel }) };
|
|
79
|
+
}
|
|
80
|
+
return { modelRoutes, modelRouting: structuredClone(modelRouting), kinds };
|
|
81
|
+
}
|
|
82
|
+
/** Write Core-owned fields atomically while retaining unrelated config and kind fields. */
|
|
83
|
+
export function replaceCoreModelConfig(config) {
|
|
84
|
+
updateRawConfigAtomically('user', (latest) => {
|
|
85
|
+
const currentKinds = latest.kinds !== null && typeof latest.kinds === 'object' && !Array.isArray(latest.kinds)
|
|
86
|
+
? latest.kinds
|
|
87
|
+
: {};
|
|
88
|
+
const kinds = Object.fromEntries(Object.entries(currentKinds).map(([kind, value]) => {
|
|
89
|
+
const previous = value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
90
|
+
const { model: _model, orchestratorModel: _orchestratorModel, ...unowned } = previous;
|
|
91
|
+
return [kind, unowned];
|
|
92
|
+
}));
|
|
93
|
+
for (const [kind, assignment] of Object.entries(config.kinds)) {
|
|
94
|
+
const previous = kinds[kind] !== null && typeof kinds[kind] === 'object' && !Array.isArray(kinds[kind])
|
|
95
|
+
? kinds[kind]
|
|
96
|
+
: {};
|
|
97
|
+
// Materialize Core's inheritance here: omission cannot clear a lower scope field.
|
|
98
|
+
kinds[kind] = { ...previous, model: assignment.model, orchestratorModel: assignment.orchestratorModel ?? assignment.model };
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
...latest,
|
|
102
|
+
modelRoutes: structuredClone(config.modelRoutes),
|
|
103
|
+
modelRouting: structuredClone(config.modelRouting),
|
|
104
|
+
kinds,
|
|
105
|
+
};
|
|
106
|
+
});
|
|
107
|
+
}
|
|
@@ -59,6 +59,11 @@ export interface UnregisteredConcreteModel {
|
|
|
59
59
|
candidate: RoutedCandidate;
|
|
60
60
|
reason: 'unregistered';
|
|
61
61
|
}
|
|
62
|
+
export declare function parseModelSpec(spec: string): {
|
|
63
|
+
providerId: string;
|
|
64
|
+
modelId: string;
|
|
65
|
+
thinkingLevel?: string;
|
|
66
|
+
} | undefined;
|
|
62
67
|
/** Expand config-derived route order only. Availability never affects rank. */
|
|
63
68
|
export declare function expandModelCandidates(request: ModelRequest, cwd?: string, profileId?: string | null): RoutedCandidate[];
|
|
64
69
|
/** Identify the one permanently invalid recipe shape: a concrete/exact request
|
|
@@ -2,7 +2,7 @@ import { readMergedLaunchConfig } from './config.js';
|
|
|
2
2
|
import { isManagedProvider, readRotationConfig, readSubscriptionPool, resolveProviderCandidates, } from './subscription-state.js';
|
|
3
3
|
const STRENGTHS = ['ultra', 'strong', 'medium', 'light'];
|
|
4
4
|
const THINKING_SUFFIXES = new Set(['off', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max']);
|
|
5
|
-
function
|
|
5
|
+
export function parseModelSpec(spec) {
|
|
6
6
|
const slash = spec.indexOf('/');
|
|
7
7
|
if (slash <= 0 || slash === spec.length - 1)
|
|
8
8
|
return undefined;
|
|
@@ -43,13 +43,13 @@ export function expandModelCandidates(request, cwd, profileId) {
|
|
|
43
43
|
const declarationFamilies = routeEntries.map(([, route]) => route.family).filter((value, index, all) => all.indexOf(value) === index);
|
|
44
44
|
const declarationSources = routeEntries.map(([, route]) => route.credentialSource).filter((value, index, all) => all.indexOf(value) === index);
|
|
45
45
|
if (request.kind === 'exact') {
|
|
46
|
-
const parsed =
|
|
46
|
+
const parsed = parseModelSpec(request.spec);
|
|
47
47
|
return parsed ? [{ routeId: 'exact', credentialSource: 'exact', family: parsed.providerId, strength: 'strong', ...parsed, rank: 0 }] : [];
|
|
48
48
|
}
|
|
49
|
-
const concrete = request.kind === 'concrete' ?
|
|
49
|
+
const concrete = request.kind === 'concrete' ? parseModelSpec(request.spec) : undefined;
|
|
50
50
|
const logical = request.kind === 'logical' ? request : undefined;
|
|
51
51
|
const matchesConcrete = (spec, exactThinking) => {
|
|
52
|
-
const parsed =
|
|
52
|
+
const parsed = parseModelSpec(spec);
|
|
53
53
|
return parsed !== undefined
|
|
54
54
|
&& concrete !== undefined
|
|
55
55
|
&& parsed.providerId === concrete.providerId
|
|
@@ -79,7 +79,7 @@ export function expandModelCandidates(request, cwd, profileId) {
|
|
|
79
79
|
const emitted = [];
|
|
80
80
|
const seen = new Set();
|
|
81
81
|
const append = (routeId, route, strength, spec) => {
|
|
82
|
-
const parsed =
|
|
82
|
+
const parsed = parseModelSpec(spec);
|
|
83
83
|
if (!parsed)
|
|
84
84
|
return;
|
|
85
85
|
if (concrete && parsed.modelId !== concrete.modelId)
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
export type TransportPoisonVerdict = {
|
|
2
|
+
consecutivePasses: number;
|
|
3
|
+
routes: string[];
|
|
4
|
+
};
|
|
1
5
|
/** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
|
|
2
6
|
* prior clean-abort certificate stale before it can affect a later crash. */
|
|
3
7
|
export declare function markBusy(nodeId: string): void;
|
|
@@ -10,6 +14,15 @@ export declare function clearBusy(nodeId: string): void;
|
|
|
10
14
|
export declare function markCleanAbort(nodeId: string): void;
|
|
11
15
|
/** Remove a consumed clean-abort certificate as a broker boots. */
|
|
12
16
|
export declare function clearCleanAbort(nodeId: string): void;
|
|
17
|
+
/** Record the provider-rotation verdict immediately before it terminates this broker.
|
|
18
|
+
* It survives the broker exit so the fleet can surface the cause if this launch
|
|
19
|
+
* exhausts respawn recovery. */
|
|
20
|
+
export declare function recordTransportPoisonVerdict(nodeId: string, verdict: TransportPoisonVerdict): void;
|
|
21
|
+
/** A verdict belongs to one broker launch. Clear it before every replacement so
|
|
22
|
+
* a later unrelated crash loop cannot inherit an old transport diagnosis. */
|
|
23
|
+
export declare function clearTransportPoisonVerdict(nodeId: string): void;
|
|
24
|
+
/** Read only a complete, structurally valid provider-rotation verdict. */
|
|
25
|
+
export declare function readTransportPoisonVerdict(nodeId: string): TransportPoisonVerdict | null;
|
|
13
26
|
/** Whether the dead broker left a persisted clean-abort turn witness. */
|
|
14
27
|
export declare function hasCleanAbort(nodeId: string): boolean;
|
|
15
28
|
/** Is the node currently inside a turn? AND this with `pidAlive` at the call
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// read site, so a stale marker (process crashed mid-turn without firing
|
|
14
14
|
// agent_end) is harmless — the dead pid fails the AND and the node is reaped.
|
|
15
15
|
// No db migration, atomic touch/unlink, best-effort (never throws).
|
|
16
|
-
import { existsSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
|
|
16
|
+
import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
|
|
17
17
|
import { join } from 'node:path';
|
|
18
18
|
import { jobDir } from '../canvas/paths.js';
|
|
19
19
|
function busyPath(nodeId) {
|
|
@@ -22,6 +22,9 @@ function busyPath(nodeId) {
|
|
|
22
22
|
function cleanAbortPath(nodeId) {
|
|
23
23
|
return join(jobDir(nodeId), 'clean-abort');
|
|
24
24
|
}
|
|
25
|
+
function transportPoisonPath(nodeId) {
|
|
26
|
+
return join(jobDir(nodeId), 'transport-poison.json');
|
|
27
|
+
}
|
|
25
28
|
/** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
|
|
26
29
|
* prior clean-abort certificate stale before it can affect a later crash. */
|
|
27
30
|
export function markBusy(nodeId) {
|
|
@@ -65,6 +68,45 @@ export function clearCleanAbort(nodeId) {
|
|
|
65
68
|
/* best-effort */
|
|
66
69
|
}
|
|
67
70
|
}
|
|
71
|
+
/** Record the provider-rotation verdict immediately before it terminates this broker.
|
|
72
|
+
* It survives the broker exit so the fleet can surface the cause if this launch
|
|
73
|
+
* exhausts respawn recovery. */
|
|
74
|
+
export function recordTransportPoisonVerdict(nodeId, verdict) {
|
|
75
|
+
try {
|
|
76
|
+
mkdirSync(jobDir(nodeId), { recursive: true });
|
|
77
|
+
writeFileSync(transportPoisonPath(nodeId), JSON.stringify(verdict));
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
/* best-effort: no marker means the crash report remains explicitly unknown */
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** A verdict belongs to one broker launch. Clear it before every replacement so
|
|
84
|
+
* a later unrelated crash loop cannot inherit an old transport diagnosis. */
|
|
85
|
+
export function clearTransportPoisonVerdict(nodeId) {
|
|
86
|
+
try {
|
|
87
|
+
rmSync(transportPoisonPath(nodeId), { force: true });
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
/* best-effort */
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Read only a complete, structurally valid provider-rotation verdict. */
|
|
94
|
+
export function readTransportPoisonVerdict(nodeId) {
|
|
95
|
+
try {
|
|
96
|
+
const value = JSON.parse(readFileSync(transportPoisonPath(nodeId), 'utf8'));
|
|
97
|
+
if (typeof value !== 'object' || value === null)
|
|
98
|
+
return null;
|
|
99
|
+
const verdict = value;
|
|
100
|
+
const consecutivePasses = verdict.consecutivePasses;
|
|
101
|
+
const routes = verdict.routes;
|
|
102
|
+
if (typeof consecutivePasses !== 'number' || !Number.isInteger(consecutivePasses) || consecutivePasses < 1 || !Array.isArray(routes) || routes.length === 0 || !routes.every((route) => typeof route === 'string' && route !== ''))
|
|
103
|
+
return null;
|
|
104
|
+
return { consecutivePasses, routes };
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
68
110
|
/** Whether the dead broker left a persisted clean-abort turn witness. */
|
|
69
111
|
export function hasCleanAbort(nodeId) {
|
|
70
112
|
return existsSync(cleanAbortPath(nodeId));
|
|
@@ -38,6 +38,7 @@ import { encodeKickoffOrigin, KICKOFF_ORIGIN_ENV } from './stamp/protocol.js';
|
|
|
38
38
|
import { headlessBrokerHost } from './host.js';
|
|
39
39
|
import { boundFleet, brokerCapReached, brokerThresholdsForDaemon } from './fleet.js';
|
|
40
40
|
import { clearFault, beginBootFaultAttempt } from './fault.js';
|
|
41
|
+
import { clearTransportPoisonVerdict } from './busy.js';
|
|
41
42
|
import { clearContextExposureState, initializeForkContextExposure, } from '../substrate/injected-store.js';
|
|
42
43
|
import { rootOfSpine } from './nodes.js';
|
|
43
44
|
import { isReviewCompanionBound } from '../review/companion.js';
|
|
@@ -468,6 +469,7 @@ async function launchRevive(nodeId, meta, opts) {
|
|
|
468
469
|
if (opts.recovery !== true)
|
|
469
470
|
cancelCronsOnWake(nodeId);
|
|
470
471
|
ensureNodeDirs(nodeId);
|
|
472
|
+
clearTransportPoisonVerdict(nodeId);
|
|
471
473
|
const lifecycleHooks = discoverLifecycleHookRegistry(meta.cwd, meta.profile_id).plans.get('node:start') ?? [];
|
|
472
474
|
await invokeLifecycleHooks(lifecycleHooks, {
|
|
473
475
|
node: {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { after, afterEach, beforeEach, test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { ApiError } from '../../../api/errors.js';
|
|
7
|
+
import { resetScopeCache } from '../../../core/scope.js';
|
|
8
|
+
import { modelConfigRoutes } from '../handlers/model-config.js';
|
|
9
|
+
const previousHome = process.env.HOME;
|
|
10
|
+
const previousAgentDir = process.env['PI_CODING_AGENT_DIR'];
|
|
11
|
+
let home = '';
|
|
12
|
+
const payload = {
|
|
13
|
+
config_version: 'core-doc-digest',
|
|
14
|
+
modelRoutes: {
|
|
15
|
+
northlight: {
|
|
16
|
+
family: 'cloud', credentialSource: 'northlight', providerId: 'northlight',
|
|
17
|
+
models: { ultra: 'northlight/u', strong: 'northlight/s:high', medium: 'northlight/m', light: 'northlight/l' },
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
modelRouting: { familyOrder: ['cloud'], strengthFallback: ['strong', 'medium'] },
|
|
21
|
+
kinds: { developer: { model: 'cloud/medium' } },
|
|
22
|
+
};
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
home = mkdtempSync(join(tmpdir(), 'crtr-model-config-'));
|
|
25
|
+
process.env.HOME = home;
|
|
26
|
+
const agentDir = join(home, 'agent');
|
|
27
|
+
mkdirSync(agentDir);
|
|
28
|
+
process.env['PI_CODING_AGENT_DIR'] = agentDir;
|
|
29
|
+
writeFileSync(join(agentDir, 'models.json'), JSON.stringify({
|
|
30
|
+
providers: { northlight: { baseUrl: 'http://127.0.0.1:1', api: 'openai-completions', apiKey: 'test', models: ['u', 's', 'm', 'l'].map((id) => ({ id, name: id })) } },
|
|
31
|
+
}));
|
|
32
|
+
resetScopeCache();
|
|
33
|
+
});
|
|
34
|
+
afterEach(() => {
|
|
35
|
+
rmSync(home, { recursive: true, force: true });
|
|
36
|
+
resetScopeCache();
|
|
37
|
+
});
|
|
38
|
+
after(() => {
|
|
39
|
+
if (previousHome === undefined)
|
|
40
|
+
delete process.env.HOME;
|
|
41
|
+
else
|
|
42
|
+
process.env.HOME = previousHome;
|
|
43
|
+
if (previousAgentDir === undefined)
|
|
44
|
+
delete process.env['PI_CODING_AGENT_DIR'];
|
|
45
|
+
else
|
|
46
|
+
process.env['PI_CODING_AGENT_DIR'] = previousAgentDir;
|
|
47
|
+
resetScopeCache();
|
|
48
|
+
});
|
|
49
|
+
function putRoute() {
|
|
50
|
+
const route = modelConfigRoutes.find((entry) => entry.method === 'PUT' && entry.pattern === '/v1/model-config');
|
|
51
|
+
assert.ok(route, 'PUT /v1/model-config is registered');
|
|
52
|
+
return route;
|
|
53
|
+
}
|
|
54
|
+
function context(body) {
|
|
55
|
+
return { method: 'PUT', path: '/v1/model-config', params: {}, query: new URLSearchParams(), body };
|
|
56
|
+
}
|
|
57
|
+
test('PUT atomically replaces Core-owned blocks without touching unrelated config', async () => {
|
|
58
|
+
const configPath = join(home, '.crouter', 'config.json');
|
|
59
|
+
mkdirSync(join(home, '.crouter'));
|
|
60
|
+
writeFileSync(configPath, JSON.stringify({ completion_bell: false, kinds: { developer: { tools: ['bash'], model: 'old/strong', orchestratorModel: 'old/ultra' } } }));
|
|
61
|
+
const result = await putRoute().handler(context(payload));
|
|
62
|
+
assert.deepEqual(result, { status: 200, body: { config_version: 'core-doc-digest', delivered: true } });
|
|
63
|
+
const written = JSON.parse(readFileSync(configPath, 'utf8'));
|
|
64
|
+
assert.equal(written.completion_bell, false);
|
|
65
|
+
assert.deepEqual(written.kinds.developer, { tools: ['bash'], model: 'cloud/medium', orchestratorModel: 'cloud/medium' });
|
|
66
|
+
assert.deepEqual(written.modelRoutes, payload.modelRoutes);
|
|
67
|
+
});
|
|
68
|
+
test('PUT rejects invalid payloads and absent baked refs before writing', async () => {
|
|
69
|
+
const configPath = join(home, '.crouter', 'config.json');
|
|
70
|
+
await assert.rejects(async () => putRoute().handler(context({ ...payload, modelRoutes: { ...payload.modelRoutes, northlight: { ...payload.modelRoutes.northlight, models: { strong: 'northlight/s' } } } })), (error) => error instanceof ApiError && error.status === 400 && error.code === 'invalid_request');
|
|
71
|
+
await assert.rejects(async () => putRoute().handler(context({ ...payload, modelRoutes: { ...payload.modelRoutes, northlight: { ...payload.modelRoutes.northlight, models: { ...payload.modelRoutes.northlight.models, ultra: 'northlight/missing' } } } })), (error) => error instanceof ApiError && error.status === 422 && error.code === 'unprocessable');
|
|
72
|
+
assert.throws(() => readFileSync(configPath, 'utf8'), { code: 'ENOENT' });
|
|
73
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { ApiError } from '../../../api/errors.js';
|
|
2
|
+
import { validateCoreModelConfig, replaceCoreModelConfig } from '../../../core/model-config.js';
|
|
3
|
+
import { parseModelSpec } from '../../../core/model-routes.js';
|
|
4
|
+
import { openModelRegistry } from '../../../core/runtime/model-registry.js';
|
|
5
|
+
function parseRequest(body) {
|
|
6
|
+
if (body === null || typeof body !== 'object' || Array.isArray(body))
|
|
7
|
+
throw new ApiError(400, 'invalid_request', 'model config must be an object');
|
|
8
|
+
const request = body;
|
|
9
|
+
if (typeof request.config_version !== 'string' || request.config_version.trim() === '') {
|
|
10
|
+
throw new ApiError(400, 'invalid_request', 'config_version must be a non-empty string');
|
|
11
|
+
}
|
|
12
|
+
try {
|
|
13
|
+
return { configVersion: request.config_version, config: validateCoreModelConfig(request) };
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
throw new ApiError(400, 'invalid_request', error instanceof Error ? error.message : String(error));
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/** Validate route refs against this image's baked descriptor catalog before disk changes. */
|
|
20
|
+
async function validateCatalog(config) {
|
|
21
|
+
const registry = await openModelRegistry();
|
|
22
|
+
for (const route of Object.values(config.modelRoutes)) {
|
|
23
|
+
for (const spec of Object.values(route.models)) {
|
|
24
|
+
const model = parseModelSpec(spec);
|
|
25
|
+
if (model === undefined || registry.find(model.providerId, model.modelId) === undefined) {
|
|
26
|
+
throw new ApiError(422, 'unprocessable', `route ref ${spec} is absent from the baked model registry`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async function handlePut(ctx) {
|
|
32
|
+
const { configVersion, config } = parseRequest(ctx.body);
|
|
33
|
+
await validateCatalog(config);
|
|
34
|
+
try {
|
|
35
|
+
replaceCoreModelConfig(config);
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
throw new ApiError(500, 'model_config_write_failed', error instanceof Error ? error.message : String(error));
|
|
39
|
+
}
|
|
40
|
+
return { status: 200, body: { config_version: configVersion, delivered: true } };
|
|
41
|
+
}
|
|
42
|
+
export const modelConfigRoutes = [
|
|
43
|
+
{ method: 'PUT', pattern: '/v1/model-config', handler: handlePut },
|
|
44
|
+
];
|
|
@@ -36,6 +36,7 @@ import { feedbackCommentRoutes } from './handlers/feedback-comments.js';
|
|
|
36
36
|
import { memoryRoutes } from './handlers/memory.js';
|
|
37
37
|
import { messageRoutes } from './handlers/messages.js';
|
|
38
38
|
import { modelAuthRoutes } from './handlers/modelauth.js';
|
|
39
|
+
import { modelConfigRoutes } from './handlers/model-config.js';
|
|
39
40
|
import { nodeRoutes } from './handlers/nodes.js';
|
|
40
41
|
import { profileRoutes } from './handlers/profiles.js';
|
|
41
42
|
import { prospectiveChatInventoryRoutes } from './handlers/prospective-chat-inventory.js';
|
|
@@ -67,6 +68,7 @@ function buildRouter() {
|
|
|
67
68
|
.registerAll(cronRoutes)
|
|
68
69
|
.registerAll(profileRoutes)
|
|
69
70
|
.registerAll(modelAuthRoutes)
|
|
71
|
+
.registerAll(modelConfigRoutes)
|
|
70
72
|
.registerAll(humanRoutes)
|
|
71
73
|
.registerAll(reviewRoutes)
|
|
72
74
|
.registerAll(reviewCommentRoutes)
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -52,6 +52,9 @@ export interface DaemonFleetOpts {
|
|
|
52
52
|
enqueue: (job: () => void | Promise<void>) => void;
|
|
53
53
|
/** Injectable clock for tests. */
|
|
54
54
|
now?: () => number;
|
|
55
|
+
/** Injectable network-reachability probe. Every respawn decision consults it,
|
|
56
|
+
* so tests drive the offline branch without touching the real network. */
|
|
57
|
+
probeOnline?: () => Promise<boolean>;
|
|
55
58
|
}
|
|
56
59
|
export declare class DaemonFleet implements FleetRegistry {
|
|
57
60
|
#private;
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -16,9 +16,10 @@ import { fullName } from '../core/canvas/labels.js';
|
|
|
16
16
|
import { isSafeNodeId } from '../core/canvas/paths.js';
|
|
17
17
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
18
18
|
import { fanDoctrineWake } from '../core/runtime/close.js';
|
|
19
|
-
import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
|
|
19
|
+
import { hasCleanAbort, isBusy, readTransportPoisonVerdict } from '../core/runtime/busy.js';
|
|
20
20
|
import { hasPendingWake } from './reconcilers/live-obligation.js';
|
|
21
|
-
import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
|
|
21
|
+
import { clearRespawnPolicy, deferRespawnWhileOffline, MAX_RESPAWN_ATTEMPTS, OFFLINE_RESPAWN_RETRY_MS, recordRespawnPolicy, respawnBackoffMs, } from './reconcilers/node-lifecycle/respawn-policy.js';
|
|
22
|
+
import { probeOnline as probeOnlineReal } from '../core/runtime/connectivity.js';
|
|
22
23
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
23
24
|
import { pushUrgent } from '../core/feed/feed.js';
|
|
24
25
|
import { emitEvent } from '../core/events/emit.js';
|
|
@@ -42,9 +43,14 @@ export async function surfaceBootFailure(meta) {
|
|
|
42
43
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
43
44
|
}
|
|
44
45
|
async function surfaceCrashLoop(meta) {
|
|
46
|
+
const transportPoison = readTransportPoisonVerdict(meta.node_id);
|
|
47
|
+
const observedCause = transportPoison === null
|
|
48
|
+
? ''
|
|
49
|
+
: `\n\nProvider rotation observed ${transportPoison.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${transportPoison.routes.join(', ')}). Check network and provider transport before reviving.`;
|
|
45
50
|
const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
|
|
46
|
-
`Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying
|
|
47
|
-
|
|
51
|
+
`Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.` +
|
|
52
|
+
observedCause +
|
|
53
|
+
`\n\nFix the crash cause, then revive the node if the work still needs doing.`;
|
|
48
54
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
49
55
|
}
|
|
50
56
|
/** Surface a recovery refusal that happened before a broker could launch. */
|
|
@@ -121,6 +127,7 @@ export class DaemonFleet {
|
|
|
121
127
|
#epoch;
|
|
122
128
|
#enqueue;
|
|
123
129
|
#now;
|
|
130
|
+
#probeOnline;
|
|
124
131
|
#map = new Map();
|
|
125
132
|
/** Slots claimed by launches that have not registered a broker yet. */
|
|
126
133
|
#reservations = new Set();
|
|
@@ -129,6 +136,7 @@ export class DaemonFleet {
|
|
|
129
136
|
this.#epoch = opts.epoch;
|
|
130
137
|
this.#enqueue = opts.enqueue;
|
|
131
138
|
this.#now = opts.now ?? Date.now;
|
|
139
|
+
this.#probeOnline = opts.probeOnline ?? (() => probeOnlineReal());
|
|
132
140
|
}
|
|
133
141
|
epoch() {
|
|
134
142
|
return this.#epoch;
|
|
@@ -292,8 +300,30 @@ export class DaemonFleet {
|
|
|
292
300
|
const row = getRow(nodeId);
|
|
293
301
|
if (row === null)
|
|
294
302
|
return;
|
|
295
|
-
const assess = row.respawn_not_before == null;
|
|
296
303
|
const priorFailures = row.respawn_failures ?? 0;
|
|
304
|
+
// A box with no network fails every relaunch for a reason that has nothing
|
|
305
|
+
// to do with this node: with no route reachable, the broker's provider queue
|
|
306
|
+
// exhausts with no HTTP response at all, it declares its own transport
|
|
307
|
+
// poisoned, and it exits for a fresh process that fails identically. Six of
|
|
308
|
+
// those and the node is dead a few minutes into an outage, holding an
|
|
309
|
+
// unfinished turn that only a human revive resumes. Park instead of counting
|
|
310
|
+
// — the outage is a condition to wait out, not a failure sequence to pace.
|
|
311
|
+
if (!(await this.#probeOnline())) {
|
|
312
|
+
deferRespawnWhileOffline(nodeId, priorFailures, this.#now());
|
|
313
|
+
emitEvent({
|
|
314
|
+
level: 'info',
|
|
315
|
+
event: 'broker.respawn.offline_deferred',
|
|
316
|
+
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
317
|
+
fields: {
|
|
318
|
+
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
319
|
+
action,
|
|
320
|
+
retry_in_ms: OFFLINE_RESPAWN_RETRY_MS,
|
|
321
|
+
consecutive_failures: priorFailures,
|
|
322
|
+
},
|
|
323
|
+
});
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
const assess = row.respawn_not_before == null;
|
|
297
327
|
const attempt = assess
|
|
298
328
|
? recordRespawnPolicy(row, this.#now())
|
|
299
329
|
: { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { type NodeRow } from '../../../core/canvas/index.js';
|
|
2
2
|
export declare const MAX_RESPAWN_ATTEMPTS = 6;
|
|
3
|
+
/** How long an offline box waits before re-testing the network and relaunching.
|
|
4
|
+
* Flat, not exponential: this is not a failure sequence being paced, it is a
|
|
5
|
+
* wait for a condition outside the node — the same "my wifi is back" gate the
|
|
6
|
+
* broker's connection-fault retry uses, sampled less often because each sample
|
|
7
|
+
* here can cost a whole broker launch. */
|
|
8
|
+
export declare const OFFLINE_RESPAWN_RETRY_MS = 30000;
|
|
3
9
|
export declare function respawnBackoffMs(consecutiveFailures: number): number;
|
|
4
10
|
export interface RespawnAttempt {
|
|
5
11
|
failures: number;
|
|
@@ -11,6 +17,14 @@ export declare function isInBackoff(row: NodeRow, now: number): boolean;
|
|
|
11
17
|
/** The only respawn-failure policy: progress since launch clears the sequence;
|
|
12
18
|
* every other missing-engine observation advances it and records its backoff. */
|
|
13
19
|
export declare function decideRespawnAttempt(row: Pick<NodeRow, 'last_settled_at' | 'launched_at' | 'respawn_failures'>, now: number): RespawnAttempt;
|
|
20
|
+
/** Hold a relaunch while the box has no network, WITHOUT advancing the failure
|
|
21
|
+
* sequence. An outage makes every launch fail identically, so counting those
|
|
22
|
+
* launches spends the whole respawn budget on a condition the node did not
|
|
23
|
+
* cause and cannot fix — the node terminalizes minutes into an outage and only
|
|
24
|
+
* a human revive brings it back. Re-arming instead parks it: the row stays
|
|
25
|
+
* active, the tick re-enters this policy after the window, and the first
|
|
26
|
+
* launch made once the network returns resumes the interrupted turn. */
|
|
27
|
+
export declare function deferRespawnWhileOffline(nodeId: string, failures: number, now: number): RespawnAttempt;
|
|
14
28
|
/** Persist the policy decision through the daemon-only canvas writer. */
|
|
15
29
|
export declare function recordRespawnPolicy(row: NodeRow, now: number): RespawnAttempt;
|
|
16
30
|
export declare function clearRespawnPolicy(nodeId: string): void;
|
|
@@ -2,6 +2,12 @@ import { recordRespawnAttempt } from '../../../core/canvas/index.js';
|
|
|
2
2
|
export const MAX_RESPAWN_ATTEMPTS = 6;
|
|
3
3
|
const RESPAWN_BACKOFF_BASE_MS = 5_000;
|
|
4
4
|
const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
|
|
5
|
+
/** How long an offline box waits before re-testing the network and relaunching.
|
|
6
|
+
* Flat, not exponential: this is not a failure sequence being paced, it is a
|
|
7
|
+
* wait for a condition outside the node — the same "my wifi is back" gate the
|
|
8
|
+
* broker's connection-fault retry uses, sampled less often because each sample
|
|
9
|
+
* here can cost a whole broker launch. */
|
|
10
|
+
export const OFFLINE_RESPAWN_RETRY_MS = 30_000;
|
|
5
11
|
export function respawnBackoffMs(consecutiveFailures) {
|
|
6
12
|
if (consecutiveFailures <= 0)
|
|
7
13
|
return 0;
|
|
@@ -30,6 +36,18 @@ export function decideRespawnAttempt(row, now) {
|
|
|
30
36
|
exhausted: failures >= MAX_RESPAWN_ATTEMPTS,
|
|
31
37
|
};
|
|
32
38
|
}
|
|
39
|
+
/** Hold a relaunch while the box has no network, WITHOUT advancing the failure
|
|
40
|
+
* sequence. An outage makes every launch fail identically, so counting those
|
|
41
|
+
* launches spends the whole respawn budget on a condition the node did not
|
|
42
|
+
* cause and cannot fix — the node terminalizes minutes into an outage and only
|
|
43
|
+
* a human revive brings it back. Re-arming instead parks it: the row stays
|
|
44
|
+
* active, the tick re-enters this policy after the window, and the first
|
|
45
|
+
* launch made once the network returns resumes the interrupted turn. */
|
|
46
|
+
export function deferRespawnWhileOffline(nodeId, failures, now) {
|
|
47
|
+
const notBefore = new Date(now + OFFLINE_RESPAWN_RETRY_MS).toISOString();
|
|
48
|
+
recordRespawnAttempt(nodeId, { failures, notBefore });
|
|
49
|
+
return { failures, notBefore, exhausted: false };
|
|
50
|
+
}
|
|
33
51
|
/** Persist the policy decision through the daemon-only canvas writer. */
|
|
34
52
|
export function recordRespawnPolicy(row, now) {
|
|
35
53
|
const attempt = decideRespawnAttempt(row, now);
|