@north-light/crouter 0.3.250 → 0.3.251

package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js

@@ -13,6 +13,9 @@ import registerCanvasInboxWatcher from '../../pi-extensions/canvas-inbox-watcher
 import { appendInbox, readCursor } from '../feed/inbox.js';
 import { drainBearings } from '../runtime/kickoff.js';
 import { appendSituationalContext } from '../runtime/situational-context.js';
+import { createEngineDriver } from '../runtime/broker/engine-drive.js';
+import { createTurnAdmissionGate } from '../runtime/broker/turn-admission.js';
+import { heldDeferredInboxPromptJoin } from '../runtime/broker/held-deferred-inbox.js';
 import { createNode, getNode, recordPid, setIntent } from '../canvas/canvas.js';
 import { createProfile, pauseProfile, resumeProfile } from '../profiles/manifest.js';
 import { closeDb } from '../canvas/db.js';
@@ -275,6 +278,98 @@ describe('canvas inbox watcher — hold + idle delivery', () => {
         assert.equal(pi.injected.length, 1);
         assert.equal(pi.injected[0].deliverAs, undefined, 'idle → sendUserMessage triggers a turn, no deliverAs');
     });
+    test('the broker prompt boundary prepends held deferred mail before the human body', async () => {
+        freshNode('node-deferred-engine-prompt');
+        const pi = makeFakePi();
+        disposers.push(registerWatcher(pi));
+        await wait(TICK_MS + 100);
+        const entry = appendInbox('node-deferred-engine-prompt', {
+            from: 'child-4',
+            tier: 'deferred',
+            kind: 'update',
+            label: 'held update',
+        });
+        await wait(SETTLE_MS);
+        const order = [];
+        const session = {
+            isStreaming: false,
+            sendCustomMessage: async ({ content }) => { order.push(content); },
+            prompt: async (text, options) => {
+                order.push(text);
+                options.preflightResult?.(true);
+            },
+        };
+        const drive = createEngineDriver({
+            registry: { sendFrame: () => undefined },
+            replies: {},
+            session: () => session,
+            dispatches: { track: (promise) => promise },
+            memoryRefs: { guidanceFor: () => null },
+            authReloadGate: { inFlight: () => undefined },
+            rebind: { installedGeneration: () => 0 },
+            projection: {},
+            toolGroups: {},
+            turnAdmission: createTurnAdmissionGate(),
+        });
+        await drive({}, { type: 'prompt', text: 'where are we?' });
+        assert.equal(order.length, 2, 'one held card and the one human prompt share the admitted operation');
+        assert.match(order[0], /held update/, 'the held inbox card is first');
+        assert.equal(order[1], 'where are we?', 'the human body follows the held card');
+        assert.equal(readCursor('node-deferred-engine-prompt'), undefined, 'the joined handoff waits for settlement');
+        await pi.fire('agent_start', { type: 'agent_start' }, { isIdle: () => false });
+        await settle(pi);
+        assert.equal(readCursor('node-deferred-engine-prompt'), entry.entry_id, 'agent_settled commits the broker-joined handoff');
+        const staleFollowUpEntry = appendInbox('node-deferred-engine-prompt', {
+            from: 'child-4',
+            tier: 'deferred',
+            kind: 'update',
+            label: 'held after a stale client snapshot',
+        });
+        await wait(SETTLE_MS);
+        const beforeFollowUp = order.length;
+        await drive({}, { type: 'follow_up', text: 'continue from there' });
+        assert.equal(order.length, beforeFollowUp + 2, 'an idle-routed follow_up shares the same prompt boundary');
+        assert.match(order.at(-2), /held after a stale client snapshot/, 'the held card still precedes the stale follow_up body');
+        assert.equal(order.at(-1), 'continue from there');
+        await pi.fire('agent_start', { type: 'agent_start' }, { isIdle: () => false });
+        await settle(pi);
+        assert.equal(readCursor('node-deferred-engine-prompt'), staleFollowUpEntry.entry_id, 'the m-C prompt settles its joined handoff too');
+    });
+    test('a held deferred batch is prepended to an admitted human prompt and replays if that prompt reloads before settlement', async () => {
+        freshNode('node-deferred-human-prompt');
+        const pi = makeFakePi();
+        disposers.push(registerWatcher(pi));
+        await wait(TICK_MS + 100);
+        const entry = appendInbox('node-deferred-human-prompt', {
+            from: 'child-4',
+            tier: 'deferred',
+            kind: 'update',
+            label: 'held update',
+        });
+        await wait(SETTLE_MS);
+        assert.equal(pi.injected.length, 0, 'deferred-only mail remains held while no turn starts');
+        const firstPromptPrefix = [];
+        await heldDeferredInboxPromptJoin()?.({
+            sendCustomMessage: async ({ content }) => { firstPromptPrefix.push(content); },
+        });
+        assert.equal(firstPromptPrefix.length, 1, 'the held card is sent before the human prompt starts');
+        assert.match(firstPromptPrefix[0], /held update/);
+        assert.equal(readCursor('node-deferred-human-prompt'), undefined, 'the pre-prompt handoff is not committed');
+        await shutdown(pi);
+        assert.equal(readCursor('node-deferred-human-prompt'), undefined, 'a reload before agent_settled leaves the held entry replayable');
+        const replacement = makeFakePi();
+        disposers.push(registerWatcher(replacement));
+        await wait(SETTLE_MS);
+        const replayedPrefix = [];
+        await heldDeferredInboxPromptJoin()?.({
+            sendCustomMessage: async ({ content }) => { replayedPrefix.push(content); },
+        });
+        assert.equal(replayedPrefix.length, 1, 'the replacement prompt receives the uncommitted held card again');
+        assert.match(replayedPrefix[0], /held update/);
+        await replacement.fire('agent_start', { type: 'agent_start' }, { isIdle: () => false });
+        await settle(replacement);
+        assert.equal(readCursor('node-deferred-human-prompt'), entry.entry_id, 'only agent_settled commits the prompt-joined handoff');
+    });
     // Pause makes the profile inert while its broker is still alive: the entry
     // must be RETAINED, not consumed. A critical entry is the sharpest case —
     // its route aborts the live turn, exactly the punch-through pause forbids.

package/dist/core/__tests__/fixtures/fake-engine.js

@@ -494,12 +494,19 @@ class FakeSession {
      *  a resolvable `/name` token must not crash on a missing method. */
     customMessages = [];
     sendCustomMessage(message, options) {
-        this.customMessages.push({
+        const customMessage = {
             customType: message.customType,
             content: message.content,
             display: message.display,
             deliverAs: options?.deliverAs,
-        });
+        };
+        this.customMessages.push(customMessage);
+        try {
+            appendFileSync(join(this.dir, 'fake-pi.custom-messages.jsonl'), JSON.stringify(customMessage) + '\n');
+        }
+        catch {
+            /* best effort test proof */
+        }
         return Promise.resolve();
     }
     abort() {

package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js

@@ -0,0 +1,63 @@
+// Run with: npm run build && node --test --test-timeout=180000 dist/core/__tests__/seam/held-deferred-human-prompt.test.js
+//
+// Regression: Pi loads canvas extensions through Jiti with moduleCache:false,
+// while broker frame dispatch is native ESM. Deferred mail held by the Jiti
+// watcher must still join the next human prompt driven by native broker code.
+import { after, before, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { appendInbox, readCursor } from '../../feed/inbox.js';
+import { updateNode } from '../../canvas/index.js';
+import { reviveNode } from '../../runtime/revive.js';
+import { createHeadlessHarness } from '../helpers/harness.js';
+import { createAttachKit, delay, frameHas, tok } from '../helpers/broker-clients.js';
+let h;
+let root;
+const kit = createAttachKit(() => h);
+before(async () => {
+    h = await createHeadlessHarness({ sessionPrefix: 'crtr-held-deferred-prompt' });
+    root = h.spawnRoot('held deferred prompt seam root');
+});
+after(async () => {
+    kit.closeAll();
+    if (h !== undefined)
+        await h.dispose();
+});
+test('a Jiti-loaded watcher prepends its held deferred batch to a native broker human prompt', { timeout: 30_000 }, async () => {
+    const nodeId = h.fabricateBrokerNode({
+        id: 'held-deferred-human-prompt',
+        parent: root,
+        lifecycle: 'resident',
+        status: 'idle',
+    });
+    updateNode(nodeId, {
+        launch: {
+            extensions: [],
+            tools: [],
+            systemPrompt: '',
+            env: {
+                CRTR_WATCHER_TICK_MS: '20',
+                CRTR_WATCHER_DEBOUNCE_MS: '25',
+            },
+        },
+    });
+    reviveNode(nodeId, { resume: true, capacity: 'freeze' });
+    await h.awaitBoot(nodeId);
+    const entry = appendInbox(nodeId, {
+        from: 'child',
+        tier: 'deferred',
+        kind: 'update',
+        label: 'held Jiti deferred update',
+    });
+    await delay(150);
+    const customPath = join(h.home, 'nodes', nodeId, 'fake-pi.custom-messages.jsonl');
+    assert.equal(existsSync(customPath), false, 'deferred-only mail stays held and does not wake the broker');
+    const controller = await kit.attach(nodeId, 'controller', 'held-deferred-prompt-controller');
+    const prompt = tok('HELD-DEFERRED-HUMAN-PROMPT');
+    controller.send({ type: 'prompt', text: prompt });
+    await controller.waitFrame((frame) => frame.type === 'agent_end' && frameHas(frame, prompt), 'the human prompt turn');
+    const customMessages = readFileSync(customPath, 'utf8');
+    assert.match(customMessages, /held Jiti deferred update/, 'the Jiti extension instance joined its held batch before the native broker prompt');
+    assert.equal(readCursor(nodeId), entry.entry_id, 'agent_settled commits the joined handoff');
+});

package/dist/core/runtime/broker/engine-drive.js

@@ -1,6 +1,7 @@
 import { emitEvent } from '../../events/emit.js';
 import { operationIdContext } from '../../events/operation-id.js';
 import { promptWithAdvertisedCommandInvocation } from '../advertised-command-invocation.js';
+import { heldDeferredInboxPromptJoin } from './held-deferred-inbox.js';
 import { chooseGuidanceDeliveryMode, promptWithNextTurnGuidance, resolveEngineRoute, resolveSteerCall, } from './engine-routing.js';
 export function createEngineDriver(deps) {
     const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, turnAdmission, } = deps;
@@ -44,14 +45,24 @@ export function createEngineDriver(deps) {
             // false. Admission waits for that preflight, then routes against the
             // state it actually established instead of its stale snapshot.
             const route = resolveEngineRoute(frame, session().isStreaming);
+            const promptOptions = route.call === 'prompt' ? route.options : undefined;
             const guidance = memoryRefs.guidanceFor(frame.text);
+            const prompt = async () => {
+                // Both human surfaces converge at this admitted boundary. The watcher
+                // claims held deferred mail before the prompt body, but its cursor stays
+                // uncommitted until the resulting turn's agent_settled event.
+                if (route.call === 'prompt' && !session().isStreaming) {
+                    await heldDeferredInboxPromptJoin()?.(session());
+                }
+                await session().prompt(frame.text, { ...promptOptions, preflightResult: release });
+            };
             if (guidance === null) {
                 if (route.call === 'prompt') {
                     // A routed prompt STARTS a turn — including an idle follow_up
                     // escalated by m-C — so it gets a fresh operation id. A true
                     // (mid-stream) followUp rides the current turn's operation, like
                     // steer.
-                    return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release })).catch(relayError))));
+                    return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt).catch(relayError))));
                 }
                 // C1: pi's followUp() takes images as a POSITIONAL 2nd arg (`followUp(text, images?)`).
                 release();
@@ -67,9 +78,9 @@ export function createEngineDriver(deps) {
                     // ATOMICALLY clears its own pending entry if prompt() settles
                     // without ever reaching that drain (an early throw/return leaves
                     // nothing stranded to leak into a later, unrelated turn).
-                    return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release })))).catch(relayError)));
+                    return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt))).catch(relayError)));
                 }
-                return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release }))
+                return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt)
                     .catch(relayError)
                     .then(() => sendGuidance(guidance, deliverAs)))));
             }

package/dist/core/runtime/broker/held-deferred-inbox.d.ts

@@ -0,0 +1,9 @@
+export declare const HELD_DEFERRED_INBOX_PROMPT_JOIN: unique symbol;
+export type HeldDeferredInboxPromptJoin = (session: {
+    sendCustomMessage: (message: {
+        customType: string;
+        content: string;
+        display: boolean;
+    }) => Promise<void>;
+}) => Promise<void>;
+export declare function heldDeferredInboxPromptJoin(): HeldDeferredInboxPromptJoin | undefined;

package/dist/core/runtime/broker/held-deferred-inbox.js

@@ -0,0 +1,8 @@
+// The inbox watcher is a Pi extension loaded through Jiti, while the broker is
+// native ESM. They therefore cannot share a module singleton. This process-global
+// symbol names the watcher-owned join function across that loader boundary.
+export const HELD_DEFERRED_INBOX_PROMPT_JOIN = Symbol.for('@crouton-kit/crtr:held-deferred-inbox-prompt-join');
+export function heldDeferredInboxPromptJoin() {
+    const join = globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN];
+    return typeof join === 'function' ? join : undefined;
+}

package/dist/pi-extensions/canvas-inbox-watcher.js

@@ -10,7 +10,10 @@
 // Context sends never trigger a turn: the visible human delivery is the sole
 // trigger, so the agent cannot run on context alone before hearing the person.
 // A context-only batch keeps its wake by triggering on its LAST card while idle,
-// and rides the running turn while streaming.
+// and rides the running turn while streaming. A deferred-only idle batch stays
+// held until either durable mail wakes a turn or the broker prepends it to an
+// already-admitted live human prompt; that bridge still commits only at
+// agent_settled.
 import { envNodeId, envProfileId, envWatcherDebounceMs, envWatcherTickMs } from '../shared/env.js';
 import { coalesceBrokerInbox, entryCards, readBrokerCanceledEntryIds, readBrokerCursor, readBrokerInboxSince, reportRefNodeId, } from '../core/runtime/broker/inbox.js';
 import { advanceBrokerInboxCursor, brokerExtensionState } from '../core/runtime/broker/daemon-ops.js';
@@ -20,6 +23,7 @@ import { errorClassFromError } from '../core/events/errors.js';
 import { operationIdContext } from '../core/events/operation-id.js';
 import { situationalContextEnvelope, SITUATIONAL_CONTEXT_CUSTOM_TYPE } from '../core/runtime/situational-context.js';
 import { isProfilePaused } from '../core/profiles/manifest.js';
+import { HELD_DEFERRED_INBOX_PROMPT_JOIN, } from '../core/runtime/broker/held-deferred-inbox.js';
 const MAX_PENDING_HANDOFFS = 64;
 const MAX_SOURCE_OPERATION_IDS = 16;
 let currentWatcher;
@@ -188,6 +192,7 @@ class CanvasInboxWatcher {
     lastArrival = 0;
     pendingHandoffs = [];
     pendingCommit;
+    promptJoinInFlight = false;
     seeded = false;
     disposed = false;
     disposalReason;
@@ -250,7 +255,7 @@ class CanvasInboxWatcher {
                 // every queued continuation has drained, so hasPendingMessages() must
                 // not veto the commit here. Await crtrd before changing the in-memory
                 // durable cursor; a failed commit remains replayable after process exit.
-                const result = await this.settle(nodeId, ctx, false);
+                const result = await this.settle(nodeId, ctx, false, true);
                 if (result === 'refresh_held')
                     this.dispose('refresh', false);
             }
@@ -383,6 +388,7 @@ class CanvasInboxWatcher {
             this.reportNodes.clear();
             this.reportNodesAsked.clear();
             this.pendingCommit = undefined;
+            this.promptJoinInFlight = false;
             this.nodeId = undefined;
             this.lastCtx = undefined;
             this.lastArrival = 0;
@@ -531,6 +537,8 @@ class CanvasInboxWatcher {
             this.retryPendingCommitOnTick(nodeId);
             return;
         }
+        if (this.promptJoinInFlight)
+            return;
         // A paused profile is inert: read nothing, send nothing, leave the cursor
         // where it is, so every queued entry — critical included, since aborting the
         // live turn is exactly the punch-through pause forbids — is still there to
@@ -599,6 +607,9 @@ class CanvasInboxWatcher {
         const hiddenIds = new Set(hiddenEntries.map((entry) => entry.entry_id));
         const visibleIds = new Set(visibleEntries.map((entry) => entry.entry_id));
         const routedEntries = entries.filter((entry) => hiddenIds.has(entry.entry_id) || visibleIds.has(entry.entry_id));
+        // Deferred-only idle mail remains in this buffer until a non-deferred inbox
+        // entry starts delivery, or a live human prompt claims it through
+        // prependHeldDeferredInbox(). Neither path wakes the node on its own.
         const deferredHold = this.isIdle()
             && routedEntries.length > 0
             && routedEntries.every((entry) => entry.tier === 'deferred');
@@ -826,6 +837,83 @@ class CanvasInboxWatcher {
         this.requeueFromBatch(batch, execution.requeueEntryIds);
         return 'attempted';
     }
+    /** Claim the deferred-only idle batch for the human prompt already admitted
+     * by the broker. Each rendered delivery is persisted before that prompt, but
+     * no cursor may advance until the prompt's agent_settled event proves the
+     * turn consumed it. */
+    async prependHeldDeferredInbox(session) {
+        if (!this.canMutate() || this.nodeId === undefined || profilePaused())
+            return;
+        const plan = this.ordinaryPlan();
+        if (!plan.deferredHold)
+            return;
+        const deliveries = plan.visibleEntries.length === 0
+            ? []
+            : this.digest(plan.visibleEntries, 'poll');
+        const handoffCount = Number(plan.hiddenEntries.length > 0) + deliveries.length;
+        if (this.pendingHandoffs.length + handoffCount > MAX_PENDING_HANDOFFS)
+            return;
+        const batch = [...this.buffer];
+        this.promptJoinInFlight = true;
+        try {
+            if (plan.hiddenEntries.length > 0) {
+                await this.sendPromptJoinDelivery(session, 'hidden', plan.hiddenEntries, this.contextCards(this.nodeId, plan.hiddenEntries));
+            }
+            for (const delivery of deliveries) {
+                await this.sendPromptJoinDelivery(session, 'visible', delivery.entries, [...delivery.cards, delivery.text]);
+            }
+            if (!this.canMutate())
+                return;
+            this.buffer.splice(0, batch.length);
+        }
+        finally {
+            this.promptJoinInFlight = false;
+        }
+    }
+    async sendPromptJoinDelivery(session, channel, entries, messages) {
+        let facts;
+        try {
+            facts = deliveryFacts(entries, channel);
+        }
+        catch (error) {
+            this.raiseReadFatal('poll', error);
+        }
+        let operationId;
+        try {
+            await operationIdContext.fresh(async (freshOperationId) => {
+                operationId = freshOperationId;
+                emitBestEffort({
+                    level: 'info',
+                    event: 'inbox.delivery.started',
+                    operation_id: freshOperationId,
+                    fields: deliveryFields(facts),
+                });
+                for (const content of messages) {
+                    await session.sendCustomMessage({
+                        customType: SITUATIONAL_CONTEXT_CUSTOM_TYPE,
+                        content,
+                        display: true,
+                    });
+                }
+            });
+        }
+        catch (error) {
+            if (operationId === undefined)
+                this.raiseReadFatal('poll', error);
+            emitDeliveryFailure(operationId, facts, 'send', error);
+            throw error;
+        }
+        if (operationId === undefined) {
+            this.raiseReadFatal('poll', new Error(`${channel} prompt-join delivery returned without an operation id`));
+        }
+        const handoff = { operationId, ...facts, settleAtAgentSettled: true };
+        if (!this.canMutate()) {
+            if (this.disposed && this.disposalReason !== undefined)
+                emitDeliveryAbandoned(handoff, this.disposalReason);
+            return;
+        }
+        this.pendingHandoffs.push(handoff);
+    }
     coveredEntryIds(channel) {
         const covered = new Set();
         for (const handoff of this.pendingHandoffs) {
@@ -883,7 +971,7 @@ class CanvasInboxWatcher {
             emitDeliveryCompleted(handoff);
         return 'settled';
     }
-    async settle(nodeId, ctx, holdPendingMessages) {
+    async settle(nodeId, ctx, holdPendingMessages, atAgentSettled = false) {
         if (holdPendingMessages) {
             try {
                 if (ctx?.hasPendingMessages?.() === true)
@@ -903,9 +991,17 @@ class CanvasInboxWatcher {
             for (const handoff of alreadyDurable)
                 emitDeliveryCompleted(handoff);
         }
+        const agentSettledBlock = atAgentSettled
+            ? undefined
+            : this.pendingHandoffs
+                .filter((handoff) => handoff.settleAtAgentSettled === true)
+                .map((handoff) => physical.throughIndexes.get(handoff))
+                .reduce((first, index) => first === undefined ? index : Math.min(first, index), undefined);
         const eligible = this.pendingHandoffs.filter((handoff) => {
             const throughIndex = physical.throughIndexes.get(handoff);
-            return physical.blockedIndex === undefined || throughIndex < physical.blockedIndex;
+            return (handoff.settleAtAgentSettled !== true || atAgentSettled)
+                && (physical.blockedIndex === undefined || throughIndex < physical.blockedIndex)
+                && (agentSettledBlock === undefined || throughIndex < agentSettledBlock);
         });
         if (eligible.length === 0)
             return 'settled';
@@ -1006,6 +1102,11 @@ function installWatcher(pi, cadence) {
     currentWatcher?.dispose('reload', false);
     const watcher = new CanvasInboxWatcher(pi, cadence, ++nextGeneration);
     currentWatcher = watcher;
+    // Pi loads this extension through Jiti while the broker driver is native ESM,
+    // so their module-level singletons differ. Register the bound watcher on the
+    // process-global key that the broker reads immediately before a human prompt.
+    const promptJoin = (session) => watcher.prependHeldDeferredInbox(session);
+    globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN] = promptJoin;
     try {
         watcher.start();
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.250",
+  "version": "0.3.251",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.250",
+  "version": "0.3.251",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/crouter",
-      "version": "0.3.250",
+      "version": "0.3.251",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {