@crouton-kit/crouter 0.3.78 → 0.3.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-root.d.ts +12 -4
- package/dist/build-root.js +25 -6
- package/dist/builtin-memory/crouter-development/plugins.md +82 -5
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +1228 -12
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +3 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +2 -787
- package/dist/clients/attach/__tests__/attach-chrome-remote.test.js +8 -3
- package/dist/clients/attach/__tests__/attach-keybindings.test.d.ts +1 -0
- package/dist/clients/attach/__tests__/attach-keybindings.test.js +113 -0
- package/dist/clients/attach/__tests__/mermaid-render.test.js +9 -1
- package/dist/clients/attach/attach-cmd.d.ts +9 -1
- package/dist/clients/attach/attach-cmd.js +849 -803
- package/dist/clients/attach/auth-pickers.d.ts +0 -12
- package/dist/clients/attach/auth-pickers.js +64 -15
- package/dist/clients/attach/graph-overlay.d.ts +12 -2
- package/dist/clients/attach/graph-overlay.js +83 -33
- package/dist/commands/human/queue.js +3 -4
- package/dist/commands/pkg/plugin-inspect.js +22 -1
- package/dist/commands/pkg/plugin-manage.js +31 -9
- package/dist/commands/sys/__tests__/setup-core.test.js +158 -1
- package/dist/commands/sys/doctor.js +42 -4
- package/dist/commands/sys/setup-core.d.ts +37 -0
- package/dist/commands/sys/setup-core.js +138 -1
- package/dist/commands/sys/setup.d.ts +88 -0
- package/dist/commands/sys/setup.js +915 -171
- package/dist/commands/view-pick.d.ts +4 -0
- package/dist/commands/view-pick.js +17 -7
- package/dist/core/__tests__/canvas-inbox-watcher.test.js +34 -9
- package/dist/core/__tests__/command-plugins-surfaces.test.d.ts +1 -0
- package/dist/core/__tests__/command-plugins-surfaces.test.js +298 -0
- package/dist/core/__tests__/command-plugins.test.d.ts +1 -0
- package/dist/core/__tests__/command-plugins.test.js +444 -0
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +6 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +9 -1
- package/dist/core/__tests__/preview-registry-sync.test.js +30 -1
- package/dist/core/__tests__/scope-crouter-home-fence.test.d.ts +1 -0
- package/dist/core/__tests__/scope-crouter-home-fence.test.js +55 -0
- package/dist/core/canvas/browse/app.d.ts +6 -0
- package/dist/core/canvas/browse/app.js +71 -41
- package/dist/core/command-plugins/adapter.d.ts +15 -0
- package/dist/core/command-plugins/adapter.js +145 -0
- package/dist/core/command-plugins/compose.d.ts +5 -0
- package/dist/core/command-plugins/compose.js +56 -0
- package/dist/core/command-plugins/discovery.d.ts +104 -0
- package/dist/core/command-plugins/discovery.js +565 -0
- package/dist/core/config.d.ts +2 -5
- package/dist/core/keybindings/__tests__/bespoke-consumers.test.d.ts +1 -0
- package/dist/core/keybindings/__tests__/bespoke-consumers.test.js +40 -0
- package/dist/core/keybindings/__tests__/resolve.test.js +2 -2
- package/dist/core/keybindings/catalog.d.ts +3 -3
- package/dist/core/keybindings/catalog.js +2 -1
- package/dist/core/profiles/select.d.ts +6 -0
- package/dist/core/profiles/select.js +86 -52
- package/dist/core/provider-management.d.ts +12 -0
- package/dist/core/provider-management.js +24 -0
- package/dist/core/runtime/broker.js +7 -6
- package/dist/core/runtime/pi-vendored.d.ts +8 -0
- package/dist/core/runtime/pi-vendored.js +14 -0
- package/dist/core/runtime/recap.d.ts +1 -1
- package/dist/core/runtime/recap.js +50 -25
- package/dist/core/runtime/session-list-cache.d.ts +10 -0
- package/dist/core/runtime/session-list-cache.js +94 -26
- package/dist/core/runtime/spawn.js +1 -13
- package/dist/core/runtime/tmux.js +2 -1
- package/dist/core/scope.js +27 -4
- package/dist/core/subscription-state.d.ts +90 -0
- package/dist/core/subscription-state.js +762 -0
- package/dist/daemon/crtrd.js +253 -12
- package/dist/pi-extensions/canvas-recap.js +43 -17
- package/dist/types.d.ts +6 -13
- package/dist/types.js +0 -3
- package/package.json +7 -3
package/dist/daemon/crtrd.js
CHANGED
|
@@ -139,6 +139,160 @@ function clearStrandedRelaunchState(id) {
|
|
|
139
139
|
strandedRelaunchAttempts.delete(id);
|
|
140
140
|
strandedRelaunchLastAttempt.delete(id);
|
|
141
141
|
}
|
|
142
|
+
// Idle-release/inbox retry cap (2026-07-10 diagnosis, round 2): the SAME
|
|
143
|
+
// livelock shape as the stranded-relaunch case above, but on the SEPARATE
|
|
144
|
+
// pass-2 path that resumes a dormant idle-released node the instant its inbox
|
|
145
|
+
// gains an unseen entry. That path called reviveNode() unconditionally every
|
|
146
|
+
// poll with no grace, no backoff, and no attempt cap.
|
|
147
|
+
//
|
|
148
|
+
// ROUND 1 keyed the retry counter on row-state residency (pid liveness),
|
|
149
|
+
// cleared the instant the pid-alive branch observed a live engine. That
|
|
150
|
+
// caught a broker that never boots at all — but the RECURRING incident's
|
|
151
|
+
// actual failure is TURN-level, not launch-level: the revive succeeds (broker
|
|
152
|
+
// launches, records a pid — clearing the round-1 counter), the turn then
|
|
153
|
+
// faults against the provider, the engine exits with the inbox CURSOR
|
|
154
|
+
// UNADVANCED, the node re-enters idle/idle-release, pass-2 sees the same
|
|
155
|
+
// unseen entries again, and — because the counter was just cleared by the
|
|
156
|
+
// transiently-alive engine — starts over at attempt 1/6 every single cycle.
|
|
157
|
+
// crtrd.err showed exactly this: hundreds of `attempt 1/6` lines, never
|
|
158
|
+
// `2/6`, interleaved with `fatal-fault <id> (other, pi→provider)`.
|
|
159
|
+
//
|
|
160
|
+
// ROUND 2 fix: key the counter on INBOX-CURSOR PROGRESS, not row-state
|
|
161
|
+
// residency or pid liveness. `readCursor(id)` is recorded as the state's
|
|
162
|
+
// baseline the first time a stuck delivery is observed; a transiently-alive
|
|
163
|
+
// engine does NOT reset it (there is no pid-alive clear for this state
|
|
164
|
+
// anymore — see handleNodeLiveness). Only an ADVANCED cursor on the NEXT
|
|
165
|
+
// observation — proof the wake actually delivered something — resets the
|
|
166
|
+
// counter (via `progressed` below); an unchanged cursor is the SAME stuck
|
|
167
|
+
// delivery and increments/backs off exactly like the stranded-relaunch path.
|
|
168
|
+
const IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS = 6;
|
|
169
|
+
const IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS = 10 * 60_000; // 10 minutes
|
|
170
|
+
const idleReleaseRelaunchState = new Map();
|
|
171
|
+
function idleReleaseRelaunchBackoffMs(attempts) {
|
|
172
|
+
return Math.min(REVIVE_GRACE_MS * 2 ** attempts, IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS);
|
|
173
|
+
}
|
|
174
|
+
/** Cleared ONLY when the node row itself is gone, or once the counter gives
|
|
175
|
+
* up and terminalizes the node — NEVER on a transiently-alive engine (that
|
|
176
|
+
* was round 1's bug) and never on a bare idle-release status flap (the
|
|
177
|
+
* fault-recovery cycle flaps status EVERY episode, so clearing on that would
|
|
178
|
+
* reproduce the exact same livelock one level up). */
|
|
179
|
+
function clearIdleReleaseRelaunchState(id) {
|
|
180
|
+
idleReleaseRelaunchState.delete(id);
|
|
181
|
+
}
|
|
182
|
+
// ---------------------------------------------------------------------------
|
|
183
|
+
// Concurrency cap (2026-07-10 incident): nothing previously bounded how many
|
|
184
|
+
// live brokers the daemon's automatic supervision sweep would launch. Each pi
|
|
185
|
+
// broker is tens-to-hundreds of MB resident and grows with context; on one
|
|
186
|
+
// machine 223 accumulated, exhausted ~19GB of swap, and froze the host — all
|
|
187
|
+
// parked idle in kevent, not doing work, just wedged. Cap the FIVE reviveNode
|
|
188
|
+
// call sites in THIS file (the daemon's automatic sweep) at a sane per-host
|
|
189
|
+
// default; override via CRTR_MAX_LIVE_BROKERS for a beefier host. Deliberately
|
|
190
|
+
// does NOT apply to `node lifecycle revive` / `node focus` — those call
|
|
191
|
+
// reviveNode directly from their own command files, never through this file,
|
|
192
|
+
// so an explicit human ask for a specific node is never blocked by this cap.
|
|
193
|
+
const DEFAULT_MAX_LIVE_BROKERS = 64;
|
|
194
|
+
function maxLiveBrokers() {
|
|
195
|
+
const raw = process.env['CRTR_MAX_LIVE_BROKERS'];
|
|
196
|
+
if (raw === undefined || raw === '')
|
|
197
|
+
return DEFAULT_MAX_LIVE_BROKERS;
|
|
198
|
+
const n = Number(raw);
|
|
199
|
+
return Number.isFinite(n) && n >= 1 ? Math.floor(n) : DEFAULT_MAX_LIVE_BROKERS;
|
|
200
|
+
}
|
|
201
|
+
// Row-census blind spot (2026-07-10 incident, round 2): the cap's `count`
|
|
202
|
+
// originally seeded ONLY from rows whose recorded `pi_pid` reads confirmed-
|
|
203
|
+
// alive. A broker that spawns but never re-records its pid (the exact
|
|
204
|
+
// stranded-relaunch shape REVIVE_GRACE_MS/strandedRelaunch* exist to recover
|
|
205
|
+
// from) is INVISIBLE to that census — the row still shows pid=null or a stale
|
|
206
|
+
// dead pid. So the row census stayed near-zero while ~190 such brokers piled
|
|
207
|
+
// up as real OS processes, every 2s tick believed it had headroom under
|
|
208
|
+
// max=16, and kept launching straight past the cap.
|
|
209
|
+
//
|
|
210
|
+
// Fix: also census the ACTUAL OS processes. Every broker is launched as
|
|
211
|
+
// `<hostExecPath> <.../broker-cli.js> <nodeId>` (host.ts's one launch()
|
|
212
|
+
// call site) — `broker-cli.js` is a stable, distinctive substring of every
|
|
213
|
+
// broker's full command line and nothing else's. One `ps` shell-out per tick
|
|
214
|
+
// (same primitive/cost class as the wedge detector's own `ps -ax` sample
|
|
215
|
+
// just below) counts them directly, with no dependency on any canvas-row
|
|
216
|
+
// state at all. `count` is then max(rowCensus, osCensus) — see superviseTick
|
|
217
|
+
// — so a failed/unavailable probe (null here) can only ever make the daemon
|
|
218
|
+
// MORE permissive than the row census alone, never less; it is never treated
|
|
219
|
+
// as "zero live brokers" and can only widen the effective cap toward the row
|
|
220
|
+
// census, matching the "never undercount, fail toward the row census" brief.
|
|
221
|
+
function countLiveBrokerProcesses() {
|
|
222
|
+
try {
|
|
223
|
+
const r = spawnSync('ps', ['-ax', '-o', 'command='], { encoding: 'utf8', timeout: 2000 });
|
|
224
|
+
if (r.status !== 0 || typeof r.stdout !== 'string')
|
|
225
|
+
return null;
|
|
226
|
+
let count = 0;
|
|
227
|
+
for (const line of r.stdout.split('\n')) {
|
|
228
|
+
if (line.includes('broker-cli.js'))
|
|
229
|
+
count++;
|
|
230
|
+
}
|
|
231
|
+
return count;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
// Over-cap surfacing (2026-07-10 incident follow-up): the daemon's own launch
|
|
238
|
+
// sites are all capacity-gated, so the OS census EXCEEDING the cap means
|
|
239
|
+
// brokers are leaking or arriving out-of-band (explicit `node lifecycle
|
|
240
|
+
// revive`/`node focus` runs, the ungated fresh_revive clock pass, or a
|
|
241
|
+
// stranded-spawn pile-up). That condition was invisible in crtrd.err unless
|
|
242
|
+
// someone went looking — and the last time it went unnoticed it ended in swap
|
|
243
|
+
// exhaustion and a frozen host. Surface it where the human actually is: a
|
|
244
|
+
// throttled `tmux display-message` on every attached client (tmux is crtr's
|
|
245
|
+
// one guaranteed viewer surface). No tmux server / no clients → silent no-op;
|
|
246
|
+
// the stderr line still records the condition either way.
|
|
247
|
+
const OVER_CAP_WARN_THROTTLE_MS = 5 * 60_000;
|
|
248
|
+
let lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
|
|
249
|
+
function warnOverCapInTmux(count, max) {
|
|
250
|
+
const now = Date.now();
|
|
251
|
+
if (now - lastOverCapWarnAt < OVER_CAP_WARN_THROTTLE_MS)
|
|
252
|
+
return;
|
|
253
|
+
lastOverCapWarnAt = now;
|
|
254
|
+
process.stderr.write(`[crtrd] over-cap: ${count} live brokers exceed max ${max} — surfacing to tmux clients\n`);
|
|
255
|
+
const msg = `⚠ crtrd: ${count} live brokers (cap ${max}) — possible broker leak. ps -ax | grep broker-cli.js`;
|
|
256
|
+
try {
|
|
257
|
+
const clients = spawnSync('tmux', ['list-clients', '-F', '#{client_name}'], {
|
|
258
|
+
encoding: 'utf8',
|
|
259
|
+
timeout: 2000,
|
|
260
|
+
});
|
|
261
|
+
if (clients.status !== 0 || typeof clients.stdout !== 'string')
|
|
262
|
+
return;
|
|
263
|
+
for (const client of clients.stdout.split('\n')) {
|
|
264
|
+
if (client === '')
|
|
265
|
+
continue;
|
|
266
|
+
// -d 10000 holds the message for 10s (tmux ≥3.2); if this tmux predates
|
|
267
|
+
// -d, fall back to the default display-time rather than not warning.
|
|
268
|
+
const r = spawnSync('tmux', ['display-message', '-d', '10000', '-c', client, msg], { encoding: 'utf8', timeout: 2000 });
|
|
269
|
+
if (r.status !== 0) {
|
|
270
|
+
spawnSync('tmux', ['display-message', '-c', client, msg], {
|
|
271
|
+
encoding: 'utf8',
|
|
272
|
+
timeout: 2000,
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
/* no tmux server — the stderr line above still records the condition */
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
// Throttle for the "at capacity" log line: without it, every deferred revive
|
|
282
|
+
// on every 2s poll would spam crtrd.err as badly as the livelock this cap
|
|
283
|
+
// exists to prevent.
|
|
284
|
+
const CAPACITY_LOG_THROTTLE_MS = 30_000;
|
|
285
|
+
let lastCapacityLogAt = Number.NEGATIVE_INFINITY;
|
|
286
|
+
function hasBrokerCapacity(capacity) {
|
|
287
|
+
if (capacity.count < capacity.max)
|
|
288
|
+
return true;
|
|
289
|
+
const now = Date.now();
|
|
290
|
+
if (now - lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
|
|
291
|
+
lastCapacityLogAt = now;
|
|
292
|
+
process.stderr.write(`[crtrd] at broker capacity (${capacity.count}/${capacity.max}) — deferring automatic revives to next poll\n`);
|
|
293
|
+
}
|
|
294
|
+
return false;
|
|
295
|
+
}
|
|
142
296
|
// §H refresh-authority grace: how long a node may sit with intent='refresh', its
|
|
143
297
|
// turn OVER (busy marker absent) and its engine still ALIVE before the daemon
|
|
144
298
|
// concludes the refresh stalled and force-kills the engine. The healthy window is
|
|
@@ -697,7 +851,7 @@ export function retryResumeMode(meta) {
|
|
|
697
851
|
* dead engine's last launch attempt was itself a cycling revive that never
|
|
698
852
|
* reached session_start (cycle_pending) — then retry it AS a cycle instead
|
|
699
853
|
* (retryResumeMode). */
|
|
700
|
-
async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
854
|
+
async function handleNodeLiveness(row, now, revivedThisTick, capacity) {
|
|
701
855
|
const id = row.node_id;
|
|
702
856
|
const pid = row.pi_pid;
|
|
703
857
|
// The engine is live → nothing pending; clear any boot/grace timer. THEN
|
|
@@ -725,6 +879,11 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
|
725
879
|
if (pid != null && recordedPidLiveness(pid, row.pi_pid_identity) === 'alive') {
|
|
726
880
|
unhealthySince.delete(id);
|
|
727
881
|
clearStrandedRelaunchState(id); // a live engine means the stranding resolved
|
|
882
|
+
// Deliberately NOT clearing idleReleaseRelaunchState here (2026-07-10,
|
|
883
|
+
// round 2): a TRANSIENTLY-alive engine — one that boots, then faults its
|
|
884
|
+
// turn and dies again with the inbox cursor unadvanced — is exactly the
|
|
885
|
+
// recurrence this state now guards against. Only cursor PROGRESS (checked
|
|
886
|
+
// in pass 2, keyed on readCursor) may reset that counter.
|
|
728
887
|
handleYieldStall(row, pid, now);
|
|
729
888
|
handleWedgeDetection(id, pid, now);
|
|
730
889
|
handleFatalFault(id, now);
|
|
@@ -800,10 +959,13 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
|
800
959
|
const backoff = strandedRelaunchBackoffMs(attempts);
|
|
801
960
|
if (lastAttempt !== undefined && now - lastAttempt < backoff)
|
|
802
961
|
return; // backing off before the next attempt
|
|
962
|
+
if (!hasBrokerCapacity(capacity))
|
|
963
|
+
return; // deferred, not attempted — costs no retry budget
|
|
803
964
|
strandedRelaunchAttempts.set(id, attempts + 1);
|
|
804
965
|
strandedRelaunchLastAttempt.set(id, now);
|
|
805
966
|
process.stderr.write(`[crtrd] revive ${id} (stranded relaunch — pid never re-recorded, attempt ${attempts + 1}/${STRANDED_RELAUNCH_MAX_ATTEMPTS})\n`);
|
|
806
967
|
reviveNode(id, { resume: retryResumeMode(meta) });
|
|
968
|
+
capacity.count++;
|
|
807
969
|
revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
|
|
808
970
|
return;
|
|
809
971
|
}
|
|
@@ -824,9 +986,13 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
|
824
986
|
yieldTermAt.delete(id);
|
|
825
987
|
if (row.intent === 'refresh') {
|
|
826
988
|
// Clean yield — or the §H force-kill above just landed — → respawn FRESH,
|
|
827
|
-
// immediately (no grace wait)
|
|
989
|
+
// immediately (no grace wait), UNLESS the daemon is already at its live-
|
|
990
|
+
// broker cap — then defer to the next poll rather than launching over it.
|
|
991
|
+
if (!hasBrokerCapacity(capacity))
|
|
992
|
+
return;
|
|
828
993
|
process.stderr.write(`[crtrd] revive ${id} (refresh-yield)\n`);
|
|
829
994
|
reviveNode(id, { resume: false });
|
|
995
|
+
capacity.count++;
|
|
830
996
|
revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
|
|
831
997
|
return;
|
|
832
998
|
}
|
|
@@ -883,8 +1049,11 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
|
883
1049
|
}
|
|
884
1050
|
return;
|
|
885
1051
|
}
|
|
1052
|
+
if (!hasBrokerCapacity(capacity))
|
|
1053
|
+
return; // deferred — retried plainly next poll
|
|
886
1054
|
process.stderr.write(`[crtrd] revive ${id} (engine dead, intent=${String(row.intent)})\n`);
|
|
887
1055
|
reviveNode(id, { resume: retryResumeMode(meta) });
|
|
1056
|
+
capacity.count++;
|
|
888
1057
|
revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
|
|
889
1058
|
}
|
|
890
1059
|
/** Fail loud for a drifted/broken trigger (design §6.6/Q5): wake the ARMER
|
|
@@ -1137,6 +1306,24 @@ export async function superviseTick(now = Date.now()) {
|
|
|
1137
1306
|
// Node-ids revived in pass 1 + pass 2 THIS tick — the third pass's bare branch
|
|
1138
1307
|
// skips them so it never launches a second pi on the same .jsonl (Maj-4).
|
|
1139
1308
|
const revivedThisTick = new Set();
|
|
1309
|
+
// Concurrency-cap budget for this tick (2026-07-10 incident, round 2 — see
|
|
1310
|
+
// countLiveBrokerProcesses's doc for the row-census blind spot this closes).
|
|
1311
|
+
// Seed from max(rowCensus, osCensus): the row census alone undercounts any
|
|
1312
|
+
// broker that spawned but never re-recorded its pid, so it can never be the
|
|
1313
|
+
// sole source of truth for a HARD cap; `?? 0` on a failed/unavailable OS
|
|
1314
|
+
// probe degrades exactly to the row census (never lower — a null probe must
|
|
1315
|
+
// never be read as "zero live brokers"). Threaded by reference through both
|
|
1316
|
+
// automatic-revive passes below so a burst of due revives in one tick can't
|
|
1317
|
+
// collectively launch past the cap.
|
|
1318
|
+
const rowLiveBrokerCount = rows.filter((r) => r.pi_pid != null && recordedPidLiveness(r.pi_pid, r.pi_pid_identity) === 'alive').length;
|
|
1319
|
+
const capacity = {
|
|
1320
|
+
count: Math.max(rowLiveBrokerCount, countLiveBrokerProcesses() ?? 0),
|
|
1321
|
+
max: maxLiveBrokers(),
|
|
1322
|
+
};
|
|
1323
|
+
// Already OVER the cap at tick start — the daemon can't have done this
|
|
1324
|
+
// (every launch site below is gated), so tell the human about the leak.
|
|
1325
|
+
if (capacity.count > capacity.max)
|
|
1326
|
+
warnOverCapInTmux(capacity.count, capacity.max);
|
|
1140
1327
|
for (const row of rows) {
|
|
1141
1328
|
try {
|
|
1142
1329
|
// Every node is broker-hosted (host_kind always 'broker'; pane/window/
|
|
@@ -1144,7 +1331,7 @@ export async function superviseTick(now = Date.now()) {
|
|
|
1144
1331
|
// pane, if any, is just a viewer — the daemon never consults it and a viewer
|
|
1145
1332
|
// closing is not a node death. There is no inline-root carve-out and no
|
|
1146
1333
|
// pane-gone reaping: one liveness path for all.
|
|
1147
|
-
await handleNodeLiveness(row, now, revivedThisTick);
|
|
1334
|
+
await handleNodeLiveness(row, now, revivedThisTick, capacity);
|
|
1148
1335
|
}
|
|
1149
1336
|
catch (err) {
|
|
1150
1337
|
// One bad node must never kill the loop.
|
|
@@ -1155,13 +1342,32 @@ export async function superviseTick(now = Date.now()) {
|
|
|
1155
1342
|
// The in-process inbox-watcher dies with pi, so the daemon owns wake-on-message
|
|
1156
1343
|
// for dormant nodes. readCursor is the cursor the watcher persisted before
|
|
1157
1344
|
// exit; any entry past it is undelivered work — resume the node to handle it.
|
|
1345
|
+
//
|
|
1346
|
+
// Backoff + attempt cap, keyed on CURSOR PROGRESS (2026-07-10 incident, round
|
|
1347
|
+
// 2 — see the idleReleaseRelaunchState doc above for the full failure-mode
|
|
1348
|
+
// writeup): a node whose delivery keeps failing — whether the broker never
|
|
1349
|
+
// boots, OR it boots and then faults its turn before advancing the cursor —
|
|
1350
|
+
// used to be relaunched here every poll (round 1) or every fault-recovery
|
|
1351
|
+
// cycle (round 1's fix, still exploitable), forever. The counter now only
|
|
1352
|
+
// resets when readCursor(id) has actually moved since the last time this
|
|
1353
|
+
// node's stuck delivery was considered — proof the wake accomplished
|
|
1354
|
+
// something — never on a bare status flap or a transiently-alive engine.
|
|
1355
|
+
// Capacity-checked too, so a burst of due idle-release wakes can't stampede
|
|
1356
|
+
// past the live-broker cap either.
|
|
1158
1357
|
for (const row of rows) {
|
|
1358
|
+
const id = row.node_id;
|
|
1159
1359
|
try {
|
|
1160
1360
|
// Re-read the ROW for fresh runtime (the first pass may have mutated it);
|
|
1161
1361
|
// no meta needed — status/intent live in the row.
|
|
1162
|
-
const r = getRow(
|
|
1163
|
-
if (r === null)
|
|
1362
|
+
const r = getRow(id);
|
|
1363
|
+
if (r === null) {
|
|
1364
|
+
clearIdleReleaseRelaunchState(id); // node gone — nothing left to track
|
|
1164
1365
|
continue;
|
|
1366
|
+
}
|
|
1367
|
+
// NOT idle-release right now (e.g. mid fault-recovery, briefly active) —
|
|
1368
|
+
// leave idleReleaseRelaunchState UNTOUCHED. Clearing here would let the
|
|
1369
|
+
// fault-recovery cycle (idle-release → briefly alive → fault → idle-
|
|
1370
|
+
// release again) reset the counter every episode — the exact round-2 bug.
|
|
1165
1371
|
if (r.status !== 'idle' || r.intent !== 'idle-release')
|
|
1166
1372
|
continue;
|
|
1167
1373
|
// The in-process inbox-watcher only owns delivery while the engine is LIVE.
|
|
@@ -1169,18 +1375,53 @@ export async function superviseTick(now = Date.now()) {
|
|
|
1169
1375
|
// has no watcher and the daemon must wake it. Gate the skip on engine
|
|
1170
1376
|
// liveness alone — viewer panes are irrelevant here. Identity-aware: a
|
|
1171
1377
|
// bare isPidAlive would misread a reused pid as "still alive" and skip
|
|
1172
|
-
// the revive forever, stranding the node on its unseen inbox entry.
|
|
1378
|
+
// the revive forever, stranding the node on its unseen inbox entry. Same
|
|
1379
|
+
// no-clear rule as above — a transiently-alive engine proves nothing about
|
|
1380
|
+
// delivery progress.
|
|
1173
1381
|
if (r.pi_pid != null && isRecordedPidAlive(r.pi_pid, r.pi_pid_identity))
|
|
1174
1382
|
continue;
|
|
1175
|
-
const
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1383
|
+
const cursorNow = readCursor(id);
|
|
1384
|
+
const entries = readInboxSince(id, cursorNow);
|
|
1385
|
+
if (entries.length === 0) {
|
|
1386
|
+
clearIdleReleaseRelaunchState(id); // fully delivered — no stuck episode to track
|
|
1387
|
+
continue;
|
|
1388
|
+
}
|
|
1389
|
+
const prior = idleReleaseRelaunchState.get(id);
|
|
1390
|
+
// Cursor progress since the LAST time this episode was considered (or no
|
|
1391
|
+
// prior episode at all) — the ONLY thing allowed to reset the counter.
|
|
1392
|
+
const progressed = prior === undefined || prior.cursor !== cursorNow;
|
|
1393
|
+
const attempts = progressed ? 0 : prior.attempts;
|
|
1394
|
+
if (attempts >= IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS) {
|
|
1395
|
+
clearIdleReleaseRelaunchState(id);
|
|
1396
|
+
const meta = getNode(id);
|
|
1397
|
+
process.stderr.write(`[crtrd] giving up on ${id} after ${attempts} idle-release relaunch attempts (no inbox-cursor progress)\n`);
|
|
1398
|
+
transition(id, 'crash');
|
|
1399
|
+
if (meta !== null) {
|
|
1400
|
+
try {
|
|
1401
|
+
fanDoctrineWake(id, subscribersOf(id), `Child crashed — ${fullName(meta)} (${id}) failed to relaunch after ${attempts} attempts while trying to ` +
|
|
1402
|
+
`deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays ` +
|
|
1403
|
+
`dead until you revive it — \`crtr node lifecycle revive ${id}\`.`, { reason: 'child-crashed', child: id });
|
|
1404
|
+
}
|
|
1405
|
+
catch {
|
|
1406
|
+
/* best-effort, mirrors every other fan-out in this file */
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
continue;
|
|
1180
1410
|
}
|
|
1411
|
+
const lastAttempt = progressed ? undefined : prior?.lastAttempt;
|
|
1412
|
+
const backoff = idleReleaseRelaunchBackoffMs(attempts);
|
|
1413
|
+
if (lastAttempt !== undefined && now - lastAttempt < backoff)
|
|
1414
|
+
continue; // backing off before the next attempt
|
|
1415
|
+
if (!hasBrokerCapacity(capacity))
|
|
1416
|
+
continue; // deferred, not attempted — costs no retry budget
|
|
1417
|
+
idleReleaseRelaunchState.set(id, { cursor: cursorNow, attempts: attempts + 1, lastAttempt: now });
|
|
1418
|
+
process.stderr.write(`[crtrd] revive ${id} (idle-release, inbox, attempt ${attempts + 1}/${IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS})\n`);
|
|
1419
|
+
reviveNode(id, { resume: true });
|
|
1420
|
+
capacity.count++;
|
|
1421
|
+
revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
|
|
1181
1422
|
}
|
|
1182
1423
|
catch (err) {
|
|
1183
|
-
process.stderr.write(`[crtrd] error polling inbox ${
|
|
1424
|
+
process.stderr.write(`[crtrd] error polling inbox ${id}: ${err.message}\n`);
|
|
1184
1425
|
}
|
|
1185
1426
|
}
|
|
1186
1427
|
// Third pass: FIRE DUE CLOCK TRIGGERS (the triggers engine, migration v9).
|
|
@@ -14,10 +14,13 @@
|
|
|
14
14
|
// • a single low-rate timer (no busy-loop) checks elapsed-since-last-message
|
|
15
15
|
// and, on crossing IDLE_MS, generates + shows the recap once.
|
|
16
16
|
//
|
|
17
|
-
// Brief input =
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
// and
|
|
17
|
+
// Brief input = the FIRST 5 and MOST RECENT 5 user/assistant messages (literal
|
|
18
|
+
// text only — the first messages carry the original request, the last carry
|
|
19
|
+
// where it left off), plus the node's roadmap file when one exists. Tool calls,
|
|
20
|
+
// tool results, thinking blocks, and extension-injected custom messages are
|
|
21
|
+
// excluded (only `message` entries with role user/assistant, and only their
|
|
22
|
+
// `text` content blocks). Long messages are truncated per-message so the
|
|
23
|
+
// opening request is never crowded out by a recent wall of text.
|
|
21
24
|
//
|
|
22
25
|
// Placement: key `crtr-recap`, placement `aboveEditor`.
|
|
23
26
|
//
|
|
@@ -33,6 +36,7 @@
|
|
|
33
36
|
// Plain TS-with-types — no imports from @earendil-works/* so this compiles
|
|
34
37
|
// inside crouter's own tsc build without a dep on the pi packages.
|
|
35
38
|
import { generateRecap } from '../core/runtime/recap.js';
|
|
39
|
+
import { readRoadmap } from '../core/runtime/roadmap.js';
|
|
36
40
|
// ---------------------------------------------------------------------------
|
|
37
41
|
// Tuning constants
|
|
38
42
|
// ---------------------------------------------------------------------------
|
|
@@ -45,10 +49,12 @@ function idleMs() {
|
|
|
45
49
|
}
|
|
46
50
|
/** How often the single low-rate timer checks elapsed-since-last-message. */
|
|
47
51
|
const POLL_MS = 7_000;
|
|
48
|
-
/**
|
|
49
|
-
*
|
|
50
|
-
|
|
51
|
-
|
|
52
|
+
/** How many messages from each end of the conversation feed the brief model —
|
|
53
|
+
* the first N carry the original request, the last N carry where it left off. */
|
|
54
|
+
const CONVO_EDGE_MESSAGES = 5;
|
|
55
|
+
/** Per-message char cap — keeps one giant paste from crowding out the rest
|
|
56
|
+
* (10 messages × this ≈ the total budget the model needs for the gist). */
|
|
57
|
+
const MESSAGE_CHAR_BUDGET = 1_200;
|
|
52
58
|
const WIDGET_KEY = 'crtr-recap';
|
|
53
59
|
// ---------------------------------------------------------------------------
|
|
54
60
|
// Module-level state — persists across /reload so the timer doesn't stack.
|
|
@@ -73,9 +79,11 @@ function messageText(msg) {
|
|
|
73
79
|
.join('\n')
|
|
74
80
|
.trim();
|
|
75
81
|
}
|
|
76
|
-
/** Concatenate the
|
|
77
|
-
* `User:`/`Agent:` transcript
|
|
78
|
-
*
|
|
82
|
+
/** Concatenate the FIRST and MOST-RECENT CONVO_EDGE_MESSAGES user/assistant
|
|
83
|
+
* messages into a single `User:`/`Agent:` transcript (each message capped at
|
|
84
|
+
* MESSAGE_CHAR_BUDGET chars), with an omission marker where the middle was
|
|
85
|
+
* dropped — the opening messages carry the original request the brief exists
|
|
86
|
+
* to surface. Excludes tool calls/results (role toolResult, plus toolCall/
|
|
79
87
|
* thinking content blocks dropped by messageText) and extension-injected
|
|
80
88
|
* custom messages (entry.type !== 'message'). Returns '' when there is nothing
|
|
81
89
|
* to summarize. */
|
|
@@ -94,15 +102,23 @@ function buildConversation(sm) {
|
|
|
94
102
|
const role = e.message.role;
|
|
95
103
|
if (role !== 'user' && role !== 'assistant')
|
|
96
104
|
continue; // drop toolResult etc.
|
|
97
|
-
|
|
105
|
+
let text = messageText(e.message);
|
|
98
106
|
if (text === '')
|
|
99
107
|
continue;
|
|
108
|
+
if (text.length > MESSAGE_CHAR_BUDGET)
|
|
109
|
+
text = `${text.slice(0, MESSAGE_CHAR_BUDGET)} …[truncated]`;
|
|
100
110
|
parts.push(`${role === 'user' ? 'User' : 'Agent'}: ${text}`);
|
|
101
111
|
}
|
|
102
112
|
if (parts.length === 0)
|
|
103
113
|
return '';
|
|
104
|
-
|
|
105
|
-
|
|
114
|
+
if (parts.length <= CONVO_EDGE_MESSAGES * 2)
|
|
115
|
+
return parts.join('\n\n');
|
|
116
|
+
const omitted = parts.length - CONVO_EDGE_MESSAGES * 2;
|
|
117
|
+
return [
|
|
118
|
+
...parts.slice(0, CONVO_EDGE_MESSAGES),
|
|
119
|
+
`[… ${omitted} message${omitted === 1 ? '' : 's'} omitted …]`,
|
|
120
|
+
...parts.slice(parts.length - CONVO_EDGE_MESSAGES),
|
|
121
|
+
].join('\n\n');
|
|
106
122
|
}
|
|
107
123
|
// ---------------------------------------------------------------------------
|
|
108
124
|
// Extension
|
|
@@ -146,12 +162,17 @@ export function registerCanvasRecap(pi) {
|
|
|
146
162
|
}
|
|
147
163
|
catch { /* best-effort */ }
|
|
148
164
|
};
|
|
149
|
-
/** Render the briefing into the viewer's low-contrast above-editor slot.
|
|
165
|
+
/** Render the briefing into the viewer's low-contrast above-editor slot.
|
|
166
|
+
* The briefing arrives as the two-line `Goal:`/`State:` format — one widget
|
|
167
|
+
* line each (the viewer wraps long lines itself). */
|
|
150
168
|
const showRecap = (briefing) => {
|
|
151
169
|
if (!canPaint() || briefing === '')
|
|
152
170
|
return;
|
|
171
|
+
const lines = briefing.split('\n').filter((l) => l.trim() !== '');
|
|
172
|
+
if (lines.length === 0)
|
|
173
|
+
return;
|
|
153
174
|
try {
|
|
154
|
-
ui.setWidget(WIDGET_KEY,
|
|
175
|
+
ui.setWidget(WIDGET_KEY, lines, { placement: 'aboveEditor' });
|
|
155
176
|
}
|
|
156
177
|
catch { /* best-effort */ }
|
|
157
178
|
};
|
|
@@ -208,6 +229,11 @@ export function registerCanvasRecap(pi) {
|
|
|
208
229
|
// (now-stale) result instead of painting over the live conversation.
|
|
209
230
|
const startedAt = lastMessageAt;
|
|
210
231
|
generating = true;
|
|
232
|
+
let roadmap;
|
|
233
|
+
try {
|
|
234
|
+
roadmap = readRoadmap(nodeId) ?? undefined;
|
|
235
|
+
}
|
|
236
|
+
catch { /* best-effort */ }
|
|
211
237
|
generateRecap(convo, (briefing) => {
|
|
212
238
|
generating = false;
|
|
213
239
|
if (lastMessageAt !== startedAt)
|
|
@@ -216,7 +242,7 @@ export function registerCanvasRecap(pi) {
|
|
|
216
242
|
return;
|
|
217
243
|
recapShown = true;
|
|
218
244
|
showRecap(briefing);
|
|
219
|
-
}, liveModel);
|
|
245
|
+
}, liveModel, roadmap);
|
|
220
246
|
// generateRecap is silent on failure (never calls back), so clear the
|
|
221
247
|
// in-flight flag after the timeout window even if no callback fires.
|
|
222
248
|
setTimeout(() => { generating = false; }, 30_000).unref?.();
|
package/dist/types.d.ts
CHANGED
|
@@ -20,6 +20,12 @@ export interface PluginManifest {
|
|
|
20
20
|
source?: string;
|
|
21
21
|
owner?: OwnerRef;
|
|
22
22
|
kinds?: string[];
|
|
23
|
+
/** Plugin-root-relative path to one declarative command manifest
|
|
24
|
+
* (`commands.json`). When present on an installed, enabled plugin, that
|
|
25
|
+
* manifest contributes top-level command branches to the `crtr` tree —
|
|
26
|
+
* crtr owns parse/help/render/errors and direct-spawns the plugin's one
|
|
27
|
+
* executable per leaf invocation. See `src/core/command-plugins/`. */
|
|
28
|
+
commands?: string;
|
|
23
29
|
}
|
|
24
30
|
export interface MarketplacePluginEntry {
|
|
25
31
|
name: string;
|
|
@@ -72,18 +78,6 @@ export interface CanvasMenuItem {
|
|
|
72
78
|
action: CanvasAction;
|
|
73
79
|
confirm?: boolean;
|
|
74
80
|
}
|
|
75
|
-
export interface CanvasBind {
|
|
76
|
-
run: string;
|
|
77
|
-
confirm?: boolean;
|
|
78
|
-
desc?: string;
|
|
79
|
-
}
|
|
80
|
-
export interface CanvasNavConfig {
|
|
81
|
-
prefixKey?: string;
|
|
82
|
-
prefixBinds: Record<string, CanvasAction & {
|
|
83
|
-
desc?: string;
|
|
84
|
-
}>;
|
|
85
|
-
graphBinds: Record<string, CanvasBind>;
|
|
86
|
-
}
|
|
87
81
|
export type ModelStrength = 'ultra' | 'strong' | 'medium' | 'light';
|
|
88
82
|
export type ModelProvider = 'anthropic' | 'openai';
|
|
89
83
|
export interface ModelLaddersConfig {
|
|
@@ -257,6 +251,5 @@ export declare function defaultRemoteCanvasConfig(): RemoteCanvasConfig;
|
|
|
257
251
|
* ancestor) — `product/teardown` is included explicitly below for parity
|
|
258
252
|
* even though its value equals the default. */
|
|
259
253
|
export declare function defaultKindsConfig(): Record<string, KindConfig>;
|
|
260
|
-
export declare function defaultCanvasNavConfig(): CanvasNavConfig;
|
|
261
254
|
export declare function defaultModelLaddersConfig(): ModelLaddersConfig;
|
|
262
255
|
export declare function defaultScopeState(): ScopeState;
|
package/dist/types.js
CHANGED
|
@@ -119,9 +119,6 @@ export function defaultKindsConfig() {
|
|
|
119
119
|
},
|
|
120
120
|
};
|
|
121
121
|
}
|
|
122
|
-
export function defaultCanvasNavConfig() {
|
|
123
|
-
throw new Error('runtime consumer must resolve semantic crouter keybindings');
|
|
124
|
-
}
|
|
125
122
|
export function defaultModelLaddersConfig() {
|
|
126
123
|
return {
|
|
127
124
|
anthropic: {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crouton-kit/crouter",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.79",
|
|
4
4
|
"description": "crtr — agent runtime with memory, plugins, and marketplaces",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -30,7 +30,6 @@
|
|
|
30
30
|
"scripts/postinstall.mjs"
|
|
31
31
|
],
|
|
32
32
|
"scripts": {
|
|
33
|
-
"prebuild": "rm -rf node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-tui node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-ai node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-agent-core",
|
|
34
33
|
"build": "rm -rf dist && tsc && cp -R src/builtin-views dist/builtin-views && cp -R src/builtin-memory dist/builtin-memory && cp -R src/builtin-pi-packages dist/builtin-pi-packages && npm run build:attach && vite build --config src/clients/web/web-client/vite.config.ts",
|
|
35
34
|
"lint:web-px": "node scripts/lint-web-px.mjs",
|
|
36
35
|
"build:attach": "esbuild src/clients/attach/attach-cmd.ts --bundle --minify --format=esm --platform=node --target=node22 --alias:@earendil-works/pi-tui=$PWD/node_modules/@earendil-works/pi-tui --external:@crouton-kit/humanloop --outfile=dist/clients/attach/attach-cmd.js --log-level=warning --banner:js=\"import{createRequire as __cr}from'node:module';const require=__cr(import.meta.url);\"",
|
|
@@ -44,13 +43,18 @@
|
|
|
44
43
|
"test:full:live": "node --import tsx/esm --test-concurrency=1 --test src/core/__tests__/live-mutation-verbs.test.ts",
|
|
45
44
|
"test:full:heavy": "node --import tsx/esm --test-concurrency=1 --test 'src/core/__tests__/full/*.test.ts'"
|
|
46
45
|
},
|
|
46
|
+
"overrides": {
|
|
47
|
+
"@earendil-works/pi-ai": "0.80.6",
|
|
48
|
+
"@earendil-works/pi-agent-core": "0.80.6",
|
|
49
|
+
"@earendil-works/pi-tui": "0.80.6"
|
|
50
|
+
},
|
|
47
51
|
"repository": {
|
|
48
52
|
"type": "git",
|
|
49
53
|
"url": "git+https://github.com/vallum-security/crouter.git"
|
|
50
54
|
},
|
|
51
55
|
"license": "MIT",
|
|
52
56
|
"dependencies": {
|
|
53
|
-
"@crouton-kit/humanloop": "^0.3.
|
|
57
|
+
"@crouton-kit/humanloop": "^0.3.34",
|
|
54
58
|
"@earendil-works/pi-agent-core": "0.80.6",
|
|
55
59
|
"@earendil-works/pi-ai": "0.80.6",
|
|
56
60
|
"@earendil-works/pi-coding-agent": "0.80.6",
|