@north-light/crouter 0.3.277 → 0.3.279
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +11 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +11 -3
- package/dist/clients/attach/chrome/status-line.js +1 -22
- package/dist/clients/attach/input/capabilities.js +2 -1
- package/dist/clients/attach/render/chat-view.d.ts +11 -0
- package/dist/clients/attach/render/chat-view.js +23 -4
- package/dist/clients/attach/render/tool-calls.d.ts +6 -1
- package/dist/clients/attach/render/tool-calls.js +47 -3
- package/dist/clients/attach/session/bindings.d.ts +1 -1
- package/dist/clients/attach/session/bindings.js +1 -1
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +3 -3
- package/dist/clients/attach/slash/dispatch.js +42 -19
- package/dist/clients/attach/viewer.js +564 -564
- package/dist/commands/canvas-browse.js +2 -2
- package/dist/commands/node/bash.js +5 -8
- package/dist/core/__tests__/helpers/fleet.d.ts +1 -1
- package/dist/core/__tests__/helpers/fleet.js +5 -0
- package/dist/core/__tests__/helpers/harness.js +5 -1
- package/dist/core/canvas/browse/app.d.ts +1 -1
- package/dist/core/canvas/browse/app.js +32 -7
- package/dist/core/canvas/browse/render.d.ts +17 -2
- package/dist/core/canvas/browse/render.js +342 -242
- package/dist/core/canvas/render-source.d.ts +26 -0
- package/dist/core/canvas/render-source.js +73 -0
- package/dist/core/keybindings/catalog.d.ts +2 -2
- package/dist/core/keybindings/catalog.js +1 -0
- package/dist/core/runtime/busy.d.ts +13 -0
- package/dist/core/runtime/busy.js +43 -1
- package/dist/core/runtime/revive.js +2 -0
- package/dist/core/runtime/tmux-bindings.js +1 -1
- package/dist/daemon/fleet.d.ts +3 -0
- package/dist/daemon/fleet.js +35 -5
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +14 -0
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +18 -0
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -103,6 +103,15 @@ export interface DashboardRow {
|
|
|
103
103
|
/** Lazy-enrichment guard (internal): true once loadPreview has read goal + the
|
|
104
104
|
* session (prompts + lastAssistant). Idempotency marker for the selected row. */
|
|
105
105
|
previewLoaded?: boolean;
|
|
106
|
+
/** The node's conversation as ORDERED user/assistant turns — what the preview panel
|
|
107
|
+
* paints as a chat transcript. Populated LAZILY by loadTranscript for the SELECTED
|
|
108
|
+
* row only (never on the background warm pass): one transcript per row is a few KB,
|
|
109
|
+
* and holding one for every node on a multi-thousand-node canvas is what `prompts`
|
|
110
|
+
* and `lastAssistant` exist to avoid. Undefined until asked for; an empty array
|
|
111
|
+
* means "read, and the node has no session turns". */
|
|
112
|
+
transcript?: TranscriptTurn[];
|
|
113
|
+
/** Lazy guard (internal): true once loadTranscript has read the session for this row. */
|
|
114
|
+
transcriptLoaded?: boolean;
|
|
106
115
|
/** The source `NodeRow` this row was built from (dashboardRowFromNodeRow only).
|
|
107
116
|
* Carries every column listNodes already fetched — the runtime fields
|
|
108
117
|
* (host_kind/parent/waiting_for/final_report/finalized_at/pi_pid) plus the
|
|
@@ -132,6 +141,17 @@ interface SessionParts {
|
|
|
132
141
|
lastAssistant?: string;
|
|
133
142
|
}
|
|
134
143
|
export declare function readSessionParts(sessionFile: string | null | undefined): SessionParts;
|
|
144
|
+
/** One conversation turn as the preview panel paints it. */
|
|
145
|
+
export interface TranscriptTurn {
|
|
146
|
+
role: 'user' | 'assistant';
|
|
147
|
+
text: string;
|
|
148
|
+
}
|
|
149
|
+
/** The node's conversation as ordered user/assistant turns, newest-last, capped to
|
|
150
|
+
* the last TRANSCRIPT_TURNS. Generated context cards are dropped (they are runtime
|
|
151
|
+
* scaffolding, not something the user or the agent said) and so are assistant turns
|
|
152
|
+
* with no text (tool-only turns), so the result reads as a chat rather than a log.
|
|
153
|
+
* One file read; never throws; empty for a node that was never revived. */
|
|
154
|
+
export declare function readTranscript(sessionFile: string | null | undefined): TranscriptTurn[];
|
|
135
155
|
/** Did this node's engine ever PRODUCE something — an assistant message with real
|
|
136
156
|
* text, a tool call, or non-empty reasoning? The signal that a node did work and
|
|
137
157
|
* is not an empty shell. A never-revived node (no session file) is trivially
|
|
@@ -174,4 +194,10 @@ export declare function enrichRowsFromSource(_source: CanvasSource, rows: Dashbo
|
|
|
174
194
|
/** goal (initial-prompt.md) and session parts are local disk reads — suppressed
|
|
175
195
|
* to undefined for a remote source. */
|
|
176
196
|
export declare function loadPreviewFromSource(source: CanvasSource, row: DashboardRow): Promise<DashboardRow>;
|
|
197
|
+
/** Fold the ordered conversation onto `row` — the preview panel's chat body. Kept
|
|
198
|
+
* OUT of loadPreviewFromSource because that runs over every row on the background
|
|
199
|
+
* warm pass, and a per-row transcript across a whole canvas is exactly the memory
|
|
200
|
+
* cost the capped `prompts` string avoids. Call it for the selected row only.
|
|
201
|
+
* Remote sources have no local session file, so the transcript stays empty. */
|
|
202
|
+
export declare function loadTranscriptFromSource(source: CanvasSource, row: DashboardRow): Promise<DashboardRow>;
|
|
177
203
|
export {};
|
|
@@ -307,6 +307,61 @@ export function readSessionParts(sessionFile) {
|
|
|
307
307
|
}
|
|
308
308
|
return { prompts, lastAssistant };
|
|
309
309
|
}
|
|
310
|
+
/** How many trailing turns the transcript keeps. A preview panel can paint a few
|
|
311
|
+
* dozen lines, and search only ever windows to a match, so the whole history is
|
|
312
|
+
* never needed — but the window has to be wide enough that a match found in
|
|
313
|
+
* `prompts` (which spans the whole session) is usually still present here. */
|
|
314
|
+
const TRANSCRIPT_TURNS = 120;
|
|
315
|
+
const TRANSCRIPT_MSG_CAP = 4096;
|
|
316
|
+
/** The node's conversation as ordered user/assistant turns, newest-last, capped to
|
|
317
|
+
* the last TRANSCRIPT_TURNS. Generated context cards are dropped (they are runtime
|
|
318
|
+
* scaffolding, not something the user or the agent said) and so are assistant turns
|
|
319
|
+
* with no text (tool-only turns), so the result reads as a chat rather than a log.
|
|
320
|
+
* One file read; never throws; empty for a node that was never revived. */
|
|
321
|
+
export function readTranscript(sessionFile) {
|
|
322
|
+
if (sessionFile === undefined || sessionFile === null || sessionFile === '')
|
|
323
|
+
return [];
|
|
324
|
+
let lines;
|
|
325
|
+
try {
|
|
326
|
+
if (!existsSync(sessionFile))
|
|
327
|
+
return [];
|
|
328
|
+
lines = readFileSync(sessionFile, 'utf8').split('\n');
|
|
329
|
+
}
|
|
330
|
+
catch {
|
|
331
|
+
return [];
|
|
332
|
+
}
|
|
333
|
+
const turns = [];
|
|
334
|
+
for (const line of lines) {
|
|
335
|
+
// Same cheap prefilter as readSessionParts: skip every line that cannot be a
|
|
336
|
+
// user or assistant message before paying for JSON.parse.
|
|
337
|
+
if (line === '')
|
|
338
|
+
continue;
|
|
339
|
+
const isUser = line.indexOf('"role":"user"') !== -1 || line.indexOf('"role": "user"') !== -1;
|
|
340
|
+
const isAssistant = !isUser
|
|
341
|
+
&& (line.indexOf('"role":"assistant"') !== -1 || line.indexOf('"role": "assistant"') !== -1);
|
|
342
|
+
if (!isUser && !isAssistant)
|
|
343
|
+
continue;
|
|
344
|
+
let rec;
|
|
345
|
+
try {
|
|
346
|
+
rec = JSON.parse(line);
|
|
347
|
+
}
|
|
348
|
+
catch {
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
if (rec.type !== 'message')
|
|
352
|
+
continue;
|
|
353
|
+
const role = rec.message?.role;
|
|
354
|
+
if (role !== 'user' && role !== 'assistant')
|
|
355
|
+
continue;
|
|
356
|
+
const text = extractUserText(rec.message?.content);
|
|
357
|
+
if (text === '')
|
|
358
|
+
continue;
|
|
359
|
+
if (parseCard({ role, content: text }) !== null)
|
|
360
|
+
continue;
|
|
361
|
+
turns.push({ role, text: text.length > TRANSCRIPT_MSG_CAP ? text.slice(0, TRANSCRIPT_MSG_CAP) : text });
|
|
362
|
+
}
|
|
363
|
+
return turns.length > TRANSCRIPT_TURNS ? turns.slice(turns.length - TRANSCRIPT_TURNS) : turns;
|
|
364
|
+
}
|
|
310
365
|
/** Did this node's engine ever PRODUCE something — an assistant message with real
|
|
311
366
|
* text, a tool call, or non-empty reasoning? The signal that a node did work and
|
|
312
367
|
* is not an empty shell. A never-revived node (no session file) is trivially
|
|
@@ -550,3 +605,21 @@ export async function loadPreviewFromSource(source, row) {
|
|
|
550
605
|
row.previewLoaded = true;
|
|
551
606
|
return row;
|
|
552
607
|
}
|
|
608
|
+
/** Fold the ordered conversation onto `row` — the preview panel's chat body. Kept
|
|
609
|
+
* OUT of loadPreviewFromSource because that runs over every row on the background
|
|
610
|
+
* warm pass, and a per-row transcript across a whole canvas is exactly the memory
|
|
611
|
+
* cost the capped `prompts` string avoids. Call it for the selected row only.
|
|
612
|
+
* Remote sources have no local session file, so the transcript stays empty. */
|
|
613
|
+
export async function loadTranscriptFromSource(source, row) {
|
|
614
|
+
if (row.transcriptLoaded === true)
|
|
615
|
+
return row;
|
|
616
|
+
if (source instanceof RemoteCanvasSource) {
|
|
617
|
+
row.transcript = [];
|
|
618
|
+
row.transcriptLoaded = true;
|
|
619
|
+
return row;
|
|
620
|
+
}
|
|
621
|
+
const meta = await source.getNode(row.node_id);
|
|
622
|
+
row.transcript = readTranscript(meta?.pi_session_file);
|
|
623
|
+
row.transcriptLoaded = true;
|
|
624
|
+
return row;
|
|
625
|
+
}
|
|
@@ -3,13 +3,13 @@ import type { AttachBindingId, AttachMenuBindingId } from './attach-control.js';
|
|
|
3
3
|
type TmuxMenuAction = 'open' | 'attach' | 'show' | 'view-settings' | 'fold-tools' | 'metadata' | 'promote' | 'demote-mode' | 'resume' | 'fork' | 'demote' | 'make-resident' | 'detach' | 'close-subtree' | 'background-bash' | 'file-review' | 'pin' | 'reload-viewer' | 'recap' | 'context' | 'context-admin' | 'providers' | 'plugins' | 'graph' | 'focus-manager' | 'copy-id' | 'issues';
|
|
4
4
|
type TmuxMenuEntryAction = Exclude<TmuxMenuAction, 'open' | 'attach'> | `issue.${'idea' | 'bug'}`;
|
|
5
5
|
type GraphAction = 'down' | 'up' | 'first' | 'last' | 'collapse-or-manager' | 'expand-or-child' | 'focus' | 'revive' | 'focus-manager' | 'close-subtree' | 'dismiss';
|
|
6
|
-
type BrowseAction = 'quit' | 'down' | 'up' | 'first' | 'last' | 'expand-or-child' | 'collapse-or-parent' | 'toggle-fold' | 'tab-next' | 'tab-previous' | `tab.${1 | 2 | 3 | 4 | 5}` | 'close-subtree' | 'confirm-close' | 'sort' | 'scope' | 'profile' | 'residents' | 'preview' | 'pin' | 'search' | 'revive-now' | 'focus' | 'focus-background' | 'yank-id';
|
|
6
|
+
type BrowseAction = 'quit' | 'down' | 'up' | 'first' | 'last' | 'expand-or-child' | 'collapse-or-parent' | 'toggle-fold' | 'tab-next' | 'tab-previous' | `tab.${1 | 2 | 3 | 4 | 5}` | 'close-subtree' | 'confirm-close' | 'sort' | 'scope' | 'profile' | 'residents' | 'preview' | 'report' | 'pin' | 'search' | 'revive-now' | 'focus' | 'focus-background' | 'yank-id';
|
|
7
7
|
type ProfileMenuAction = 'up' | 'down' | 'select' | 'accept-default' | 'abort' | `slot.${1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9}` | 'create' | 'search-existing' | 'use-root' | 'set-default' | 'add-directory' | 'unpin' | 'manage';
|
|
8
8
|
type SetupAction = 'cancel' | 'tab-next' | 'tab-previous' | 'open' | 'list-down' | 'list-up' | 'toggle' | 'details' | 'advance' | 'cell-previous' | 'cell-next';
|
|
9
9
|
type SetupKeybindingAction = 'edit' | 'commit-edit' | 'cancel-edit' | 'unbind' | 'reset-row' | 'reset-all' | 'save' | 'discard' | 'page-up' | 'page-down' | 'filter' | 'filter-delete' | 'filter-clear';
|
|
10
10
|
type SetupModelsAction = 'open' | 'commit' | 'cancel' | 'scope-next';
|
|
11
11
|
export type BindingId = `crtr.tmux.menu.${TmuxMenuAction}` | `crtr.tmux.menu.issue.${'idea' | 'bug'}` | AttachMenuBindingId | `crtr.tmux.menu-global.${TmuxMenuEntryAction}` | `crtr.tmux.node.${'next' | 'previous'}` | 'crtr.tmux.inbox.toggle' | `crtr.attach.scroll.${'up' | 'down' | 'page-up' | 'page-down' | 'live'}` | 'crtr.attach.mouse.toggle' | 'crtr.attach.copy.message' | 'crtr.attach.help.toggle' | `crtr.attach.search.${'next' | 'previous' | 'edit' | 'close'}` | AttachBindingId | `crtr.graph.${GraphAction}` | `crtr.graph.confirm.${'accept' | 'cancel'}` | `crtr.inspector.${'force-quit' | 'down' | 'up' | 'section-next' | 'section-prev' | 'page-down' | 'page-up' | 'toggle-detail' | 'cancel-item' | 'open-log' | 'confirm' | 'dismiss' | 'retarget' | 'refresh' | 'quit'}` | `crtr.browse.${BrowseAction}` | `crtr.browse.search.${'cancel' | 'commit' | 'delete' | 'quit'}` | `crtr.profile.menu.${ProfileMenuAction}` | `crtr.profile.search.${'up' | 'down' | 'pick' | 'back' | 'delete' | 'abort'}` | `crtr.setup.${SetupAction}` | `crtr.setup.confirm.${'accept' | 'back'}` | `crtr.setup.keybindings.${SetupKeybindingAction}` | `crtr.setup.models.${SetupModelsAction}`;
|
|
12
|
-
export declare const BINDING_CATALOG: readonly [BindingDefinition<"crtr.tmux.menu.open">, BindingDefinition<"crtr.tmux.menu.attach">, ...(BindingDefinition<BindingId> | BindingDefinition<"crtr.tmux.node.next"> | BindingDefinition<"crtr.tmux.node.previous"> | BindingDefinition<"crtr.tmux.inbox.toggle"> | BindingDefinition<"crtr.attach.scroll.up"> | BindingDefinition<"crtr.attach.scroll.down"> | BindingDefinition<"crtr.attach.scroll.page-up"> | BindingDefinition<"crtr.attach.scroll.page-down"> | BindingDefinition<"crtr.attach.scroll.live"> | BindingDefinition<"crtr.attach.mouse.toggle"> | BindingDefinition<"crtr.attach.copy.message"> | BindingDefinition<"crtr.attach.search.next"> | BindingDefinition<"crtr.attach.search.previous"> | BindingDefinition<"crtr.attach.search.edit"> | BindingDefinition<"crtr.attach.search.close"> | BindingDefinition<"crtr.graph.down"> | BindingDefinition<"crtr.graph.up"> | BindingDefinition<"crtr.graph.first"> | BindingDefinition<"crtr.graph.last"> | BindingDefinition<"crtr.graph.collapse-or-manager"> | BindingDefinition<"crtr.graph.expand-or-child"> | BindingDefinition<"crtr.graph.focus"> | BindingDefinition<"crtr.graph.revive"> | BindingDefinition<"crtr.graph.focus-manager"> | BindingDefinition<"crtr.graph.close-subtree"> | BindingDefinition<"crtr.graph.dismiss"> | BindingDefinition<"crtr.graph.confirm.accept"> | BindingDefinition<"crtr.graph.confirm.cancel"> | BindingDefinition<"crtr.inspector.force-quit"> | BindingDefinition<"crtr.inspector.down"> | BindingDefinition<"crtr.inspector.up"> | BindingDefinition<"crtr.inspector.section-next"> | BindingDefinition<"crtr.inspector.section-prev"> | BindingDefinition<"crtr.inspector.page-down"> | BindingDefinition<"crtr.inspector.page-up"> | BindingDefinition<"crtr.inspector.toggle-detail"> | BindingDefinition<"crtr.inspector.cancel-item"> | BindingDefinition<"crtr.inspector.open-log"> | BindingDefinition<"crtr.inspector.confirm"> | BindingDefinition<"crtr.inspector.dismiss"> | BindingDefinition<"crtr.inspector.retarget"> | BindingDefinition<"crtr.inspector.refresh"> | BindingDefinition<"crtr.inspector.quit"> | BindingDefinition<"crtr.browse.quit"> | BindingDefinition<"crtr.browse.down"> | BindingDefinition<"crtr.browse.up"> | BindingDefinition<"crtr.browse.first"> | BindingDefinition<"crtr.browse.last"> | BindingDefinition<"crtr.browse.expand-or-child"> | BindingDefinition<"crtr.browse.collapse-or-parent"> | BindingDefinition<"crtr.browse.toggle-fold"> | BindingDefinition<"crtr.browse.tab-next"> | BindingDefinition<"crtr.browse.tab-previous"> | BindingDefinition<"crtr.browse.tab.2" | "crtr.browse.tab.1" | "crtr.browse.tab.3" | "crtr.browse.tab.4" | "crtr.browse.tab.5"> | BindingDefinition<"crtr.browse.close-subtree"> | BindingDefinition<"crtr.browse.confirm-close"> | BindingDefinition<"crtr.browse.sort"> | BindingDefinition<"crtr.browse.scope"> | BindingDefinition<"crtr.browse.profile"> | BindingDefinition<"crtr.browse.residents"> | BindingDefinition<"crtr.browse.preview"> | BindingDefinition<"crtr.browse.pin"> | BindingDefinition<"crtr.browse.search"> | BindingDefinition<"crtr.browse.revive-now"> | BindingDefinition<"crtr.browse.focus"> | BindingDefinition<"crtr.browse.focus-background"> | BindingDefinition<"crtr.browse.yank-id"> | BindingDefinition<"crtr.browse.search.cancel"> | BindingDefinition<"crtr.browse.search.commit"> | BindingDefinition<"crtr.browse.search.delete"> | BindingDefinition<"crtr.browse.search.quit"> | BindingDefinition<"crtr.profile.menu.up"> | BindingDefinition<"crtr.profile.menu.down"> | BindingDefinition<"crtr.profile.menu.select"> | BindingDefinition<"crtr.profile.menu.accept-default"> | BindingDefinition<"crtr.profile.menu.abort"> | BindingDefinition<"crtr.profile.menu.slot.2" | "crtr.profile.menu.slot.1" | "crtr.profile.menu.slot.3" | "crtr.profile.menu.slot.4" | "crtr.profile.menu.slot.5" | "crtr.profile.menu.slot.6" | "crtr.profile.menu.slot.7" | "crtr.profile.menu.slot.8" | "crtr.profile.menu.slot.9">)[], BindingDefinition<"crtr.profile.menu.create">, BindingDefinition<"crtr.profile.menu.search-existing">, BindingDefinition<"crtr.profile.menu.use-root">, BindingDefinition<"crtr.profile.menu.set-default">, BindingDefinition<"crtr.profile.menu.add-directory">, BindingDefinition<"crtr.profile.menu.unpin">, BindingDefinition<"crtr.profile.menu.manage">, BindingDefinition<"crtr.profile.search.up">, BindingDefinition<"crtr.profile.search.down">, BindingDefinition<"crtr.profile.search.pick">, BindingDefinition<"crtr.profile.search.back">, BindingDefinition<"crtr.profile.search.delete">, BindingDefinition<"crtr.profile.search.abort">, BindingDefinition<"crtr.setup.cancel">, BindingDefinition<"crtr.setup.tab-next">, BindingDefinition<"crtr.setup.tab-previous">, BindingDefinition<"crtr.setup.open">, BindingDefinition<"crtr.setup.list-down">, BindingDefinition<"crtr.setup.list-up">, BindingDefinition<"crtr.setup.toggle">, BindingDefinition<"crtr.setup.details">, BindingDefinition<"crtr.setup.advance">, BindingDefinition<"crtr.setup.confirm.accept">, BindingDefinition<"crtr.setup.confirm.back">, BindingDefinition<"crtr.setup.keybindings.edit">, BindingDefinition<"crtr.setup.keybindings.commit-edit">, BindingDefinition<"crtr.setup.keybindings.cancel-edit">, BindingDefinition<"crtr.setup.keybindings.unbind">, BindingDefinition<"crtr.setup.keybindings.reset-row">, BindingDefinition<"crtr.setup.keybindings.reset-all">, BindingDefinition<"crtr.setup.keybindings.save">, BindingDefinition<"crtr.setup.keybindings.discard">, BindingDefinition<"crtr.setup.keybindings.page-up">, BindingDefinition<"crtr.setup.keybindings.page-down">, BindingDefinition<"crtr.setup.keybindings.filter">, BindingDefinition<"crtr.setup.keybindings.filter-delete">, BindingDefinition<"crtr.setup.keybindings.filter-clear">, BindingDefinition<"crtr.setup.cell-previous">, BindingDefinition<"crtr.setup.cell-next">, BindingDefinition<"crtr.setup.models.open">, BindingDefinition<"crtr.setup.models.scope-next">, BindingDefinition<"crtr.setup.models.commit">, BindingDefinition<"crtr.setup.models.cancel">];
|
|
12
|
+
export declare const BINDING_CATALOG: readonly [BindingDefinition<"crtr.tmux.menu.open">, BindingDefinition<"crtr.tmux.menu.attach">, ...(BindingDefinition<BindingId> | BindingDefinition<"crtr.tmux.node.next"> | BindingDefinition<"crtr.tmux.node.previous"> | BindingDefinition<"crtr.tmux.inbox.toggle"> | BindingDefinition<"crtr.attach.scroll.up"> | BindingDefinition<"crtr.attach.scroll.down"> | BindingDefinition<"crtr.attach.scroll.page-up"> | BindingDefinition<"crtr.attach.scroll.page-down"> | BindingDefinition<"crtr.attach.scroll.live"> | BindingDefinition<"crtr.attach.mouse.toggle"> | BindingDefinition<"crtr.attach.copy.message"> | BindingDefinition<"crtr.attach.search.next"> | BindingDefinition<"crtr.attach.search.previous"> | BindingDefinition<"crtr.attach.search.edit"> | BindingDefinition<"crtr.attach.search.close"> | BindingDefinition<"crtr.graph.down"> | BindingDefinition<"crtr.graph.up"> | BindingDefinition<"crtr.graph.first"> | BindingDefinition<"crtr.graph.last"> | BindingDefinition<"crtr.graph.collapse-or-manager"> | BindingDefinition<"crtr.graph.expand-or-child"> | BindingDefinition<"crtr.graph.focus"> | BindingDefinition<"crtr.graph.revive"> | BindingDefinition<"crtr.graph.focus-manager"> | BindingDefinition<"crtr.graph.close-subtree"> | BindingDefinition<"crtr.graph.dismiss"> | BindingDefinition<"crtr.graph.confirm.accept"> | BindingDefinition<"crtr.graph.confirm.cancel"> | BindingDefinition<"crtr.inspector.force-quit"> | BindingDefinition<"crtr.inspector.down"> | BindingDefinition<"crtr.inspector.up"> | BindingDefinition<"crtr.inspector.section-next"> | BindingDefinition<"crtr.inspector.section-prev"> | BindingDefinition<"crtr.inspector.page-down"> | BindingDefinition<"crtr.inspector.page-up"> | BindingDefinition<"crtr.inspector.toggle-detail"> | BindingDefinition<"crtr.inspector.cancel-item"> | BindingDefinition<"crtr.inspector.open-log"> | BindingDefinition<"crtr.inspector.confirm"> | BindingDefinition<"crtr.inspector.dismiss"> | BindingDefinition<"crtr.inspector.retarget"> | BindingDefinition<"crtr.inspector.refresh"> | BindingDefinition<"crtr.inspector.quit"> | BindingDefinition<"crtr.browse.quit"> | BindingDefinition<"crtr.browse.down"> | BindingDefinition<"crtr.browse.up"> | BindingDefinition<"crtr.browse.first"> | BindingDefinition<"crtr.browse.last"> | BindingDefinition<"crtr.browse.expand-or-child"> | BindingDefinition<"crtr.browse.collapse-or-parent"> | BindingDefinition<"crtr.browse.toggle-fold"> | BindingDefinition<"crtr.browse.tab-next"> | BindingDefinition<"crtr.browse.tab-previous"> | BindingDefinition<"crtr.browse.tab.2" | "crtr.browse.tab.1" | "crtr.browse.tab.3" | "crtr.browse.tab.4" | "crtr.browse.tab.5"> | BindingDefinition<"crtr.browse.close-subtree"> | BindingDefinition<"crtr.browse.confirm-close"> | BindingDefinition<"crtr.browse.sort"> | BindingDefinition<"crtr.browse.scope"> | BindingDefinition<"crtr.browse.profile"> | BindingDefinition<"crtr.browse.residents"> | BindingDefinition<"crtr.browse.preview"> | BindingDefinition<"crtr.browse.report"> | BindingDefinition<"crtr.browse.pin"> | BindingDefinition<"crtr.browse.search"> | BindingDefinition<"crtr.browse.revive-now"> | BindingDefinition<"crtr.browse.focus"> | BindingDefinition<"crtr.browse.focus-background"> | BindingDefinition<"crtr.browse.yank-id"> | BindingDefinition<"crtr.browse.search.cancel"> | BindingDefinition<"crtr.browse.search.commit"> | BindingDefinition<"crtr.browse.search.delete"> | BindingDefinition<"crtr.browse.search.quit"> | BindingDefinition<"crtr.profile.menu.up"> | BindingDefinition<"crtr.profile.menu.down"> | BindingDefinition<"crtr.profile.menu.select"> | BindingDefinition<"crtr.profile.menu.accept-default"> | BindingDefinition<"crtr.profile.menu.abort"> | BindingDefinition<"crtr.profile.menu.slot.2" | "crtr.profile.menu.slot.1" | "crtr.profile.menu.slot.3" | "crtr.profile.menu.slot.4" | "crtr.profile.menu.slot.5" | "crtr.profile.menu.slot.6" | "crtr.profile.menu.slot.7" | "crtr.profile.menu.slot.8" | "crtr.profile.menu.slot.9">)[], BindingDefinition<"crtr.profile.menu.create">, BindingDefinition<"crtr.profile.menu.search-existing">, BindingDefinition<"crtr.profile.menu.use-root">, BindingDefinition<"crtr.profile.menu.set-default">, BindingDefinition<"crtr.profile.menu.add-directory">, BindingDefinition<"crtr.profile.menu.unpin">, BindingDefinition<"crtr.profile.menu.manage">, BindingDefinition<"crtr.profile.search.up">, BindingDefinition<"crtr.profile.search.down">, BindingDefinition<"crtr.profile.search.pick">, BindingDefinition<"crtr.profile.search.back">, BindingDefinition<"crtr.profile.search.delete">, BindingDefinition<"crtr.profile.search.abort">, BindingDefinition<"crtr.setup.cancel">, BindingDefinition<"crtr.setup.tab-next">, BindingDefinition<"crtr.setup.tab-previous">, BindingDefinition<"crtr.setup.open">, BindingDefinition<"crtr.setup.list-down">, BindingDefinition<"crtr.setup.list-up">, BindingDefinition<"crtr.setup.toggle">, BindingDefinition<"crtr.setup.details">, BindingDefinition<"crtr.setup.advance">, BindingDefinition<"crtr.setup.confirm.accept">, BindingDefinition<"crtr.setup.confirm.back">, BindingDefinition<"crtr.setup.keybindings.edit">, BindingDefinition<"crtr.setup.keybindings.commit-edit">, BindingDefinition<"crtr.setup.keybindings.cancel-edit">, BindingDefinition<"crtr.setup.keybindings.unbind">, BindingDefinition<"crtr.setup.keybindings.reset-row">, BindingDefinition<"crtr.setup.keybindings.reset-all">, BindingDefinition<"crtr.setup.keybindings.save">, BindingDefinition<"crtr.setup.keybindings.discard">, BindingDefinition<"crtr.setup.keybindings.page-up">, BindingDefinition<"crtr.setup.keybindings.page-down">, BindingDefinition<"crtr.setup.keybindings.filter">, BindingDefinition<"crtr.setup.keybindings.filter-delete">, BindingDefinition<"crtr.setup.keybindings.filter-clear">, BindingDefinition<"crtr.setup.cell-previous">, BindingDefinition<"crtr.setup.cell-next">, BindingDefinition<"crtr.setup.models.open">, BindingDefinition<"crtr.setup.models.scope-next">, BindingDefinition<"crtr.setup.models.commit">, BindingDefinition<"crtr.setup.models.cancel">];
|
|
13
13
|
export declare const BINDING_IDS: ReadonlySet<string>;
|
|
14
14
|
/** True for an action whose direct gesture is handled inside an attach viewer.
|
|
15
15
|
* Tmux menu selectors and root shortcuts are chrome, not pane input. */
|
|
@@ -148,6 +148,7 @@ function browseBindings() {
|
|
|
148
148
|
b('crtr.browse.profile', 'Browse and pickers', 'Toggle profile', 'Toggle the browse profile scope.', ['shift+p'], 'terminal', [nav]),
|
|
149
149
|
b('crtr.browse.residents', 'Browse and pickers', 'Toggle residents', 'Toggle resident display.', ['r'], 'terminal', [nav]),
|
|
150
150
|
b('crtr.browse.preview', 'Browse and pickers', 'Toggle preview', 'Toggle the preview panel.', ['v'], 'terminal', [nav]),
|
|
151
|
+
b('crtr.browse.report', 'Browse and pickers', 'Toggle report', 'Swap the preview panel between the conversation and the node\'s newest report.', ['shift+v'], 'terminal', [nav]),
|
|
151
152
|
b('crtr.browse.pin', 'Browse and pickers', 'Pin node', 'Star or unstar the selected node.', ['p'], 'terminal', [nav]),
|
|
152
153
|
b('crtr.browse.search', 'Browse and pickers', 'Search', 'Open browse search.', ['/'], 'terminal', [nav]),
|
|
153
154
|
b('crtr.browse.revive-now', 'Browse and pickers', 'Revive now', 'Revive the selected node now.', ['shift+k'], 'terminal', [nav]),
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
export type TransportPoisonVerdict = {
|
|
2
|
+
consecutivePasses: number;
|
|
3
|
+
routes: string[];
|
|
4
|
+
};
|
|
1
5
|
/** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
|
|
2
6
|
* prior clean-abort certificate stale before it can affect a later crash. */
|
|
3
7
|
export declare function markBusy(nodeId: string): void;
|
|
@@ -10,6 +14,15 @@ export declare function clearBusy(nodeId: string): void;
|
|
|
10
14
|
export declare function markCleanAbort(nodeId: string): void;
|
|
11
15
|
/** Remove a consumed clean-abort certificate as a broker boots. */
|
|
12
16
|
export declare function clearCleanAbort(nodeId: string): void;
|
|
17
|
+
/** Record the provider-rotation verdict immediately before it terminates this broker.
|
|
18
|
+
* It survives the broker exit so the fleet can surface the cause if this launch
|
|
19
|
+
* exhausts respawn recovery. */
|
|
20
|
+
export declare function recordTransportPoisonVerdict(nodeId: string, verdict: TransportPoisonVerdict): void;
|
|
21
|
+
/** A verdict belongs to one broker launch. Clear it before every replacement so
|
|
22
|
+
* a later unrelated crash loop cannot inherit an old transport diagnosis. */
|
|
23
|
+
export declare function clearTransportPoisonVerdict(nodeId: string): void;
|
|
24
|
+
/** Read only a complete, structurally valid provider-rotation verdict. */
|
|
25
|
+
export declare function readTransportPoisonVerdict(nodeId: string): TransportPoisonVerdict | null;
|
|
13
26
|
/** Whether the dead broker left a persisted clean-abort turn witness. */
|
|
14
27
|
export declare function hasCleanAbort(nodeId: string): boolean;
|
|
15
28
|
/** Is the node currently inside a turn? AND this with `pidAlive` at the call
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// read site, so a stale marker (process crashed mid-turn without firing
|
|
14
14
|
// agent_end) is harmless — the dead pid fails the AND and the node is reaped.
|
|
15
15
|
// No db migration, atomic touch/unlink, best-effort (never throws).
|
|
16
|
-
import { existsSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
|
|
16
|
+
import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
|
|
17
17
|
import { join } from 'node:path';
|
|
18
18
|
import { jobDir } from '../canvas/paths.js';
|
|
19
19
|
function busyPath(nodeId) {
|
|
@@ -22,6 +22,9 @@ function busyPath(nodeId) {
|
|
|
22
22
|
function cleanAbortPath(nodeId) {
|
|
23
23
|
return join(jobDir(nodeId), 'clean-abort');
|
|
24
24
|
}
|
|
25
|
+
function transportPoisonPath(nodeId) {
|
|
26
|
+
return join(jobDir(nodeId), 'transport-poison.json');
|
|
27
|
+
}
|
|
25
28
|
/** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
|
|
26
29
|
* prior clean-abort certificate stale before it can affect a later crash. */
|
|
27
30
|
export function markBusy(nodeId) {
|
|
@@ -65,6 +68,45 @@ export function clearCleanAbort(nodeId) {
|
|
|
65
68
|
/* best-effort */
|
|
66
69
|
}
|
|
67
70
|
}
|
|
71
|
+
/** Record the provider-rotation verdict immediately before it terminates this broker.
|
|
72
|
+
* It survives the broker exit so the fleet can surface the cause if this launch
|
|
73
|
+
* exhausts respawn recovery. */
|
|
74
|
+
export function recordTransportPoisonVerdict(nodeId, verdict) {
|
|
75
|
+
try {
|
|
76
|
+
mkdirSync(jobDir(nodeId), { recursive: true });
|
|
77
|
+
writeFileSync(transportPoisonPath(nodeId), JSON.stringify(verdict));
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
/* best-effort: no marker means the crash report remains explicitly unknown */
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** A verdict belongs to one broker launch. Clear it before every replacement so
|
|
84
|
+
* a later unrelated crash loop cannot inherit an old transport diagnosis. */
|
|
85
|
+
export function clearTransportPoisonVerdict(nodeId) {
|
|
86
|
+
try {
|
|
87
|
+
rmSync(transportPoisonPath(nodeId), { force: true });
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
/* best-effort */
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Read only a complete, structurally valid provider-rotation verdict. */
|
|
94
|
+
export function readTransportPoisonVerdict(nodeId) {
|
|
95
|
+
try {
|
|
96
|
+
const value = JSON.parse(readFileSync(transportPoisonPath(nodeId), 'utf8'));
|
|
97
|
+
if (typeof value !== 'object' || value === null)
|
|
98
|
+
return null;
|
|
99
|
+
const verdict = value;
|
|
100
|
+
const consecutivePasses = verdict.consecutivePasses;
|
|
101
|
+
const routes = verdict.routes;
|
|
102
|
+
if (typeof consecutivePasses !== 'number' || !Number.isInteger(consecutivePasses) || consecutivePasses < 1 || !Array.isArray(routes) || routes.length === 0 || !routes.every((route) => typeof route === 'string' && route !== ''))
|
|
103
|
+
return null;
|
|
104
|
+
return { consecutivePasses, routes };
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
68
110
|
/** Whether the dead broker left a persisted clean-abort turn witness. */
|
|
69
111
|
export function hasCleanAbort(nodeId) {
|
|
70
112
|
return existsSync(cleanAbortPath(nodeId));
|
|
@@ -38,6 +38,7 @@ import { encodeKickoffOrigin, KICKOFF_ORIGIN_ENV } from './stamp/protocol.js';
|
|
|
38
38
|
import { headlessBrokerHost } from './host.js';
|
|
39
39
|
import { boundFleet, brokerCapReached, brokerThresholdsForDaemon } from './fleet.js';
|
|
40
40
|
import { clearFault, beginBootFaultAttempt } from './fault.js';
|
|
41
|
+
import { clearTransportPoisonVerdict } from './busy.js';
|
|
41
42
|
import { clearContextExposureState, initializeForkContextExposure, } from '../substrate/injected-store.js';
|
|
42
43
|
import { rootOfSpine } from './nodes.js';
|
|
43
44
|
import { isReviewCompanionBound } from '../review/companion.js';
|
|
@@ -468,6 +469,7 @@ async function launchRevive(nodeId, meta, opts) {
|
|
|
468
469
|
if (opts.recovery !== true)
|
|
469
470
|
cancelCronsOnWake(nodeId);
|
|
470
471
|
ensureNodeDirs(nodeId);
|
|
472
|
+
clearTransportPoisonVerdict(nodeId);
|
|
471
473
|
const lifecycleHooks = discoverLifecycleHookRegistry(meta.cwd, meta.profile_id).plans.get('node:start') ?? [];
|
|
472
474
|
await invokeLifecycleHooks(lifecycleHooks, {
|
|
473
475
|
node: {
|
|
@@ -15,7 +15,7 @@ const TMUX_BINDING_MANIFEST = '@crtr_keybindings_manifest';
|
|
|
15
15
|
const ROOT_TABLE = 'root';
|
|
16
16
|
const LEAF_MENU_ACTIONS = {
|
|
17
17
|
'crtr.tmux.menu.promote': { description: 'promote to orchestrator', action: { kind: 'keys', keys: '/promote' } },
|
|
18
|
-
'crtr.tmux.menu.resume': { description: 'find node', action: { kind: 'keys', keys: '/resume
|
|
18
|
+
'crtr.tmux.menu.resume': { description: 'find node', action: { kind: 'keys', keys: '/resume' } },
|
|
19
19
|
'crtr.tmux.menu.fork': { description: 'fork into idle root', action: { kind: 'direct', run: 'node fork --pane {pane}' } },
|
|
20
20
|
// Mode demote is the mirror of `/promote`: `node config` persists the change;
|
|
21
21
|
// the broker appends its pending context before the next provider response.
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -52,6 +52,9 @@ export interface DaemonFleetOpts {
|
|
|
52
52
|
enqueue: (job: () => void | Promise<void>) => void;
|
|
53
53
|
/** Injectable clock for tests. */
|
|
54
54
|
now?: () => number;
|
|
55
|
+
/** Injectable network-reachability probe. Every respawn decision consults it,
|
|
56
|
+
* so tests drive the offline branch without touching the real network. */
|
|
57
|
+
probeOnline?: () => Promise<boolean>;
|
|
55
58
|
}
|
|
56
59
|
export declare class DaemonFleet implements FleetRegistry {
|
|
57
60
|
#private;
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -16,9 +16,10 @@ import { fullName } from '../core/canvas/labels.js';
|
|
|
16
16
|
import { isSafeNodeId } from '../core/canvas/paths.js';
|
|
17
17
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
18
18
|
import { fanDoctrineWake } from '../core/runtime/close.js';
|
|
19
|
-
import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
|
|
19
|
+
import { hasCleanAbort, isBusy, readTransportPoisonVerdict } from '../core/runtime/busy.js';
|
|
20
20
|
import { hasPendingWake } from './reconcilers/live-obligation.js';
|
|
21
|
-
import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
|
|
21
|
+
import { clearRespawnPolicy, deferRespawnWhileOffline, MAX_RESPAWN_ATTEMPTS, OFFLINE_RESPAWN_RETRY_MS, recordRespawnPolicy, respawnBackoffMs, } from './reconcilers/node-lifecycle/respawn-policy.js';
|
|
22
|
+
import { probeOnline as probeOnlineReal } from '../core/runtime/connectivity.js';
|
|
22
23
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
23
24
|
import { pushUrgent } from '../core/feed/feed.js';
|
|
24
25
|
import { emitEvent } from '../core/events/emit.js';
|
|
@@ -42,9 +43,14 @@ export async function surfaceBootFailure(meta) {
|
|
|
42
43
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
43
44
|
}
|
|
44
45
|
async function surfaceCrashLoop(meta) {
|
|
46
|
+
const transportPoison = readTransportPoisonVerdict(meta.node_id);
|
|
47
|
+
const observedCause = transportPoison === null
|
|
48
|
+
? ''
|
|
49
|
+
: `\n\nProvider rotation observed ${transportPoison.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${transportPoison.routes.join(', ')}). Check network and provider transport before reviving.`;
|
|
45
50
|
const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
|
|
46
|
-
`Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying
|
|
47
|
-
|
|
51
|
+
`Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.` +
|
|
52
|
+
observedCause +
|
|
53
|
+
`\n\nFix the crash cause, then revive the node if the work still needs doing.`;
|
|
48
54
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
49
55
|
}
|
|
50
56
|
/** Surface a recovery refusal that happened before a broker could launch. */
|
|
@@ -121,6 +127,7 @@ export class DaemonFleet {
|
|
|
121
127
|
#epoch;
|
|
122
128
|
#enqueue;
|
|
123
129
|
#now;
|
|
130
|
+
#probeOnline;
|
|
124
131
|
#map = new Map();
|
|
125
132
|
/** Slots claimed by launches that have not registered a broker yet. */
|
|
126
133
|
#reservations = new Set();
|
|
@@ -129,6 +136,7 @@ export class DaemonFleet {
|
|
|
129
136
|
this.#epoch = opts.epoch;
|
|
130
137
|
this.#enqueue = opts.enqueue;
|
|
131
138
|
this.#now = opts.now ?? Date.now;
|
|
139
|
+
this.#probeOnline = opts.probeOnline ?? (() => probeOnlineReal());
|
|
132
140
|
}
|
|
133
141
|
epoch() {
|
|
134
142
|
return this.#epoch;
|
|
@@ -292,8 +300,30 @@ export class DaemonFleet {
|
|
|
292
300
|
const row = getRow(nodeId);
|
|
293
301
|
if (row === null)
|
|
294
302
|
return;
|
|
295
|
-
const assess = row.respawn_not_before == null;
|
|
296
303
|
const priorFailures = row.respawn_failures ?? 0;
|
|
304
|
+
// A box with no network fails every relaunch for a reason that has nothing
|
|
305
|
+
// to do with this node: with no route reachable, the broker's provider queue
|
|
306
|
+
// exhausts with no HTTP response at all, it declares its own transport
|
|
307
|
+
// poisoned, and it exits for a fresh process that fails identically. Six of
|
|
308
|
+
// those and the node is dead a few minutes into an outage, holding an
|
|
309
|
+
// unfinished turn that only a human revive resumes. Park instead of counting
|
|
310
|
+
// — the outage is a condition to wait out, not a failure sequence to pace.
|
|
311
|
+
if (!(await this.#probeOnline())) {
|
|
312
|
+
deferRespawnWhileOffline(nodeId, priorFailures, this.#now());
|
|
313
|
+
emitEvent({
|
|
314
|
+
level: 'info',
|
|
315
|
+
event: 'broker.respawn.offline_deferred',
|
|
316
|
+
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
317
|
+
fields: {
|
|
318
|
+
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
319
|
+
action,
|
|
320
|
+
retry_in_ms: OFFLINE_RESPAWN_RETRY_MS,
|
|
321
|
+
consecutive_failures: priorFailures,
|
|
322
|
+
},
|
|
323
|
+
});
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
const assess = row.respawn_not_before == null;
|
|
297
327
|
const attempt = assess
|
|
298
328
|
? recordRespawnPolicy(row, this.#now())
|
|
299
329
|
: { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { type NodeRow } from '../../../core/canvas/index.js';
|
|
2
2
|
export declare const MAX_RESPAWN_ATTEMPTS = 6;
|
|
3
|
+
/** How long an offline box waits before re-testing the network and relaunching.
|
|
4
|
+
* Flat, not exponential: this is not a failure sequence being paced, it is a
|
|
5
|
+
* wait for a condition outside the node — the same "my wifi is back" gate the
|
|
6
|
+
* broker's connection-fault retry uses, sampled less often because each sample
|
|
7
|
+
* here can cost a whole broker launch. */
|
|
8
|
+
export declare const OFFLINE_RESPAWN_RETRY_MS = 30000;
|
|
3
9
|
export declare function respawnBackoffMs(consecutiveFailures: number): number;
|
|
4
10
|
export interface RespawnAttempt {
|
|
5
11
|
failures: number;
|
|
@@ -11,6 +17,14 @@ export declare function isInBackoff(row: NodeRow, now: number): boolean;
|
|
|
11
17
|
/** The only respawn-failure policy: progress since launch clears the sequence;
|
|
12
18
|
* every other missing-engine observation advances it and records its backoff. */
|
|
13
19
|
export declare function decideRespawnAttempt(row: Pick<NodeRow, 'last_settled_at' | 'launched_at' | 'respawn_failures'>, now: number): RespawnAttempt;
|
|
20
|
+
/** Hold a relaunch while the box has no network, WITHOUT advancing the failure
|
|
21
|
+
* sequence. An outage makes every launch fail identically, so counting those
|
|
22
|
+
* launches spends the whole respawn budget on a condition the node did not
|
|
23
|
+
* cause and cannot fix — the node terminalizes minutes into an outage and only
|
|
24
|
+
* a human revive brings it back. Re-arming instead parks it: the row stays
|
|
25
|
+
* active, the tick re-enters this policy after the window, and the first
|
|
26
|
+
* launch made once the network returns resumes the interrupted turn. */
|
|
27
|
+
export declare function deferRespawnWhileOffline(nodeId: string, failures: number, now: number): RespawnAttempt;
|
|
14
28
|
/** Persist the policy decision through the daemon-only canvas writer. */
|
|
15
29
|
export declare function recordRespawnPolicy(row: NodeRow, now: number): RespawnAttempt;
|
|
16
30
|
export declare function clearRespawnPolicy(nodeId: string): void;
|
|
@@ -2,6 +2,12 @@ import { recordRespawnAttempt } from '../../../core/canvas/index.js';
|
|
|
2
2
|
export const MAX_RESPAWN_ATTEMPTS = 6;
|
|
3
3
|
const RESPAWN_BACKOFF_BASE_MS = 5_000;
|
|
4
4
|
const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
|
|
5
|
+
/** How long an offline box waits before re-testing the network and relaunching.
|
|
6
|
+
* Flat, not exponential: this is not a failure sequence being paced, it is a
|
|
7
|
+
* wait for a condition outside the node — the same "my wifi is back" gate the
|
|
8
|
+
* broker's connection-fault retry uses, sampled less often because each sample
|
|
9
|
+
* here can cost a whole broker launch. */
|
|
10
|
+
export const OFFLINE_RESPAWN_RETRY_MS = 30_000;
|
|
5
11
|
export function respawnBackoffMs(consecutiveFailures) {
|
|
6
12
|
if (consecutiveFailures <= 0)
|
|
7
13
|
return 0;
|
|
@@ -30,6 +36,18 @@ export function decideRespawnAttempt(row, now) {
|
|
|
30
36
|
exhausted: failures >= MAX_RESPAWN_ATTEMPTS,
|
|
31
37
|
};
|
|
32
38
|
}
|
|
39
|
+
/** Hold a relaunch while the box has no network, WITHOUT advancing the failure
|
|
40
|
+
* sequence. An outage makes every launch fail identically, so counting those
|
|
41
|
+
* launches spends the whole respawn budget on a condition the node did not
|
|
42
|
+
* cause and cannot fix — the node terminalizes minutes into an outage and only
|
|
43
|
+
* a human revive brings it back. Re-arming instead parks it: the row stays
|
|
44
|
+
* active, the tick re-enters this policy after the window, and the first
|
|
45
|
+
* launch made once the network returns resumes the interrupted turn. */
|
|
46
|
+
export function deferRespawnWhileOffline(nodeId, failures, now) {
|
|
47
|
+
const notBefore = new Date(now + OFFLINE_RESPAWN_RETRY_MS).toISOString();
|
|
48
|
+
recordRespawnAttempt(nodeId, { failures, notBefore });
|
|
49
|
+
return { failures, notBefore, exhausted: false };
|
|
50
|
+
}
|
|
33
51
|
/** Persist the policy decision through the daemon-only canvas writer. */
|
|
34
52
|
export function recordRespawnPolicy(row, now) {
|
|
35
53
|
const attempt = decideRespawnAttempt(row, now);
|
package/package.json
CHANGED
package/runtime.lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/crouter",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.279",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@north-light/crouter",
|
|
9
|
-
"version": "0.3.
|
|
9
|
+
"version": "0.3.279",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "MIT",
|
|
12
12
|
"dependencies": {
|