@animalabs/agent-framework 0.30.0 → 0.31.0

package/dist/src/tune-out/coordinator.js

@@ -49,6 +49,54 @@ function writeIdOf(m) {
     const writeId = m.metadata?.deferredWriteId;
     return typeof writeId === 'string' ? writeId : undefined;
 }
+function keysOf(rows) {
+    const keys = { ids: [], writeIds: [] };
+    for (const m of rows) {
+        const writeId = m.writeId ?? writeIdOf(m);
+        if (writeId !== undefined)
+            keys.writeIds.push(writeId);
+        else if (m.id !== undefined)
+            keys.ids.push(m.id);
+    }
+    return keys;
+}
+function keyedBy(keys) {
+    const ids = new Set(keys.flatMap((k) => k.ids));
+    const writeIds = new Set(keys.flatMap((k) => k.writeIds));
+    return (m) => {
+        const writeId = m.writeId ?? writeIdOf(m);
+        return writeId !== undefined ? writeIds.has(writeId) : m.id !== undefined && ids.has(m.id);
+    };
+}
+const isStrings = (value) => Array.isArray(value) && value.every((v) => typeof v === 'string');
+const isRowKeys = (value) => typeof value === 'object' && value !== null && isStrings(value.ids) && isStrings(value.writeIds);
+function isReleaseRecord(value) {
+    if (!isRowKeys(value))
+        return false;
+    const record = value;
+    return typeof record.total === 'number' && typeof record.truncated === 'number' && isRowKeys(record.report);
+}
+function releaseRecordsOf(rows, epochId) {
+    const records = [];
+    for (const m of rows) {
+        const meta = m.metadata;
+        if (meta?.kind !== 'tune-out-cancel' || meta.epochId !== epochId)
+            continue;
+        if (isReleaseRecord(meta.released) && !records.some((r) => JSON.stringify(r) === JSON.stringify(meta.released))) {
+            records.push(meta.released);
+        }
+    }
+    return records;
+}
+function truncationNote(records) {
+    const truncated = records.reduce((n, r) => n + r.truncated, 0);
+    if (truncated === 0)
+        return '';
+    const total = records.reduce((n, r) => n + r.total, 0);
+    return records.length === 1
+        ? ` ${truncated} of ${total} diverted messages, the oldest, weren't delivered raw to the resident.`
+        : ` ${truncated} of ${total} diverted messages across ${records.length} dumps, the oldest in each dump that left any out, weren't delivered raw to the resident.`;
+}
 /**
  * Whether a held message is one of this channel's: by its channel id, and by
  * its server where it records one, since two servers can register the same
@@ -61,7 +109,7 @@ function inChannelOf(serverId, channelId) {
         return (meta?.channelId ?? channelId) === channelId && (meta?.serverId ?? serverId) === serverId;
     };
 }
-/** How long after an end time whose cancel threw the cancel is tried again. */
+/** How long after an end time's or a wake budget's cancel threw the cancel is tried again. */
 const EXPIRY_RETRY_MS = 60_000;
 /** Ends a backlog line cut to fit in one of the residents' messages. */
 const CUT_MARK = ` [${NOTICE_TAG.cutToFit}; the whole message is in the channel's history]`;
@@ -157,8 +205,6 @@ export class TuneOutCoordinator {
     lateForks = new WeakSet();
     /** The all-except scope's durable record, cached (undefined = not read yet). */
     scopeCache;
-    /** The epoch whose release this process delivered and whose hold is not yet recorded as cleared. */
-    scopeReleased;
     /** Timer key for the scope's cadence and expiry. */
     static SCOPE_KEY = '\0all-except';
     constructor(channelRegistry, serverRegistry, hooks) {
@@ -207,26 +253,59 @@ export class TuneOutCoordinator {
             // An epoch whose deadline passed while the process was down cancels
             // immediately — the resident gets their backlog on the schedule they
             // committed to, not whenever the host happened to restart.
-            if (params.expiresAtMs !== undefined && params.expiresAtMs <= Date.now()) {
-                this.cancel(serverId, channelId, 'duration', 'duration elapsed');
-                continue;
+            const inForce = () => this.channelRegistry.getTuneOutState(serverId, channelId)?.params.epochId === params.epochId;
+            const expired = params.expiresAtMs !== undefined && params.expiresAtMs <= Date.now();
+            if (expired) {
+                try {
+                    this.cancel(serverId, channelId, 'duration', 'duration elapsed');
+                    continue;
+                }
+                catch (err) {
+                    if (!inForce()) {
+                        console.error(`[tune-out] ${this.shown(serverId, channelId)} reached its end time while the process was down and its cancel ended it, but a step after the end failed:`, err);
+                        continue;
+                    }
+                    console.error(`[tune-out] ${this.shown(serverId, channelId)} reached its end time while the process was down, and its cancel failed, so it stays tuned out and its cancel is tried again in ${EXPIRY_RETRY_MS / 1000} s:`, err);
+                }
             }
             this.startCadence(serverId, channelId, params.cadenceSeconds);
-            this.armExpiry(serverId, channelId, params);
+            const wakeCount = this.channelRegistry.getTuneOutState(serverId, channelId)?.wakeCount ?? 0;
+            if (wakeCount > params.maxWakes) {
+                this.cancelPastBudget(serverId, channelId, params.maxWakes, 'start');
+                if (!inForce())
+                    continue;
+            }
+            else {
+                this.armExpiry(serverId, channelId, expired ? { ...params, expiresAtMs: Date.now() + EXPIRY_RETRY_MS } : params);
+            }
             if (oldestAnchor === null || params.startedAtSequence < oldestAnchor) {
                 oldestAnchor = params.startedAtSequence;
             }
         }
         const scope = this.getAllExcept();
         if (scope) {
+            let inForce = true;
             if (scope.params.expiresAtMs <= Date.now()) {
-                this.cancelAllExcept('duration', 'duration elapsed');
+                try {
+                    this.cancelAllExcept('duration', 'duration elapsed');
+                    inForce = false;
+                }
+                catch (err) {
+                    if (this.getAllExcept()?.params.epochId !== scope.params.epochId) {
+                        console.error('[tune-out] the all-except hold reached its end time while the process was down and its release ended it, but a step after the end failed:', err);
+                        inForce = false;
+                    }
+                    else {
+                        console.error(`[tune-out] the all-except hold reached its end time while the process was down, and its release failed, so it stays in force and its release is tried again in ${EXPIRY_RETRY_MS / 1000} s:`, err);
+                        this.armScopeTimers({ ...scope.params, expiresAtMs: Date.now() + EXPIRY_RETRY_MS });
+                    }
+                }
             }
             else {
                 this.armScopeTimers(scope.params);
-                if (oldestAnchor === null || scope.params.startedAtSequence < oldestAnchor) {
-                    oldestAnchor = scope.params.startedAtSequence;
-                }
+            }
+            if (inForce && (oldestAnchor === null || scope.params.startedAtSequence < oldestAnchor)) {
+                oldestAnchor = scope.params.startedAtSequence;
             }
         }
         if (oldestAnchor !== null) {
@@ -418,41 +497,28 @@ export class TuneOutCoordinator {
     hasReader() {
         return this.readerAvailable();
     }
-    /**
-     * End a tune-out: durable flip, system-framed backlog dump into the
-     * resident's window (capped), lifecycle notice, one wake for the
-     * resident. `subconsciousNote` is the subconscious's own text (from its
-     * cancel_tuneout call) — delivered verbatim under its name, never
-     * synthesized here.
-     */
     cancel(serverId, channelId, source, reason, subconsciousNote, 
     /** `reader: 'forks'`: the reader fork whose note this is. */
     noteAuthor, 
     /** The lifecycle state the channel returns to (a tool close ends in 'closed'). */
     nextState = 'open') {
-        const ended = this.channelRegistry.cancelTuneOut(serverId, channelId, nextState, source);
-        if (!ended) {
+        const ending = this.channelRegistry.getTuneOutState(serverId, channelId);
+        if (!ending) {
             // Its reader is the resident's tune_out: cancel_tuneout finds the channel first.
             return { ok: false, error: this.notTunedOut(this.quotedShown(serverId, channelId), 'you') };
         }
-        this.clearTimers(serverId, channelId);
-        this.divertedSinceInvocation.delete(this.key(serverId, channelId));
-        // Backlog: every message held under this epoch. The stored ones come in
-        // store order, then those still in the deferred queue (held mid-turn,
-        // not yet written), oldest first. That's arrival order, except when a
-        // held message stays queued while a later one is stored: a store write
-        // refused at a turn-start flush leaves it queued, and that turn's
-        // assembly can then store later coalesced messages directly.
-        const epochId = ended.params.epochId;
+        const epochId = ending.params.epochId;
         const ofEpoch = (m) => m.metadata?.tuneOut?.epochId === epochId;
-        const held = (this.hooks.getDeferredMessages?.() ?? []).filter(ofEpoch);
-        const backlog = [
-            ...this.hooks.getStoredMessages().filter(ofEpoch),
-            ...held,
-        ];
+        const storedRows = this.hooks.getStoredMessages();
+        const queuedRows = this.hooks.getDeferredMessages?.() ?? [];
+        const earlier = releaseRecordsOf([...storedRows, ...queuedRows], epochId);
+        const dumpedEarlier = keyedBy(earlier);
+        const unreleased = (m) => ofEpoch(m) && !dumpedEarlier(m);
+        const held = queuedRows.filter(unreleased);
+        const backlog = [...storedRows.filter(unreleased), ...held];
         // Whole messages: a cap below one shows none, where slice(-0.5) would
         // start at 0 and take them all.
-        const cap = Math.floor(ended.params.backlogCap);
+        const cap = Math.floor(ending.params.backlogCap);
         const authorOf = (m) => suppliedValue(m.metadata?.author?.name ?? m.participant);
         // A line names its author as the resident knows them and marks what a
         // text line can't carry (`[image]`), so the dump says what it leaves out.
@@ -462,7 +528,7 @@ export class TuneOutCoordinator {
         const historyTool = this.hooks.historyTool ? this.hooks.historyTool() : 'fetch_history';
         // The channel as the resident and the subconscious are shown it.
         const name = this.shown(serverId, channelId);
-        const lifecycle = `[${NOTICE_TAG.tuneOut}: ${name} cancelled — ${reason}, ${ended.wakeCount} wake${ended.wakeCount === 1 ? '' : 's'}]`;
+        const lifecycle = `[${NOTICE_TAG.tuneOut}: ${name} cancelled — ${reason}, ${ending.wakeCount} wake${ending.wakeCount === 1 ? '' : 's'}]`;
         const render = (lines, truncated) => {
             const attrs = [
                 `channel=${this.quotedShown(serverId, channelId)}`,
@@ -533,10 +599,26 @@ export class TuneOutCoordinator {
             ...heldLeftOut,
             ...held.slice(heldLeftOut.length).filter((m) => m.content.some((b) => b.type !== 'text') || (newestCut && m === shown[0])),
         ];
-        // Host-framed system artifacts — never voiced as anyone.
-        this.hooks.addMessage('user', [{ type: 'text', text: render(lines, truncated) }], { system: true, kind: 'tune-out-cancel', channelId, epochId });
-        // The epoch is already ended durably; the release must be as durable.
-        this.hooks.commitDeliveries?.();
+        const records = [...earlier];
+        if (earlier.length === 0 || backlog.length > 0) {
+            const record = { ...keysOf(backlog), total: backlog.length, truncated, report: keysOf(heldForReport) };
+            this.hooks.addMessage('user', [{ type: 'text', text: render(lines, truncated) }], { system: true, kind: 'tune-out-cancel', channelId, epochId, released: record });
+            records.push(record);
+        }
+        if (this.hooks.commitDeliveries?.() === false) {
+            throw new Error(`the backlog of ${this.quotedShown(serverId, channelId)} could not be made durable, so its tune-out stays in force`);
+        }
+        this.channelRegistry.cancelTuneOut(serverId, channelId, nextState, source);
+        this.clearTimers(serverId, channelId);
+        this.divertedSinceInvocation.delete(this.key(serverId, channelId));
+        const reportedEarlier = keyedBy(earlier.map((r) => r.report));
+        const release = {
+            truncated: records.reduce((n, r) => n + r.truncated, 0),
+            heldForReport: [
+                ...[...storedRows, ...queuedRows].filter((m) => ofEpoch(m) && reportedEarlier(m)),
+                ...(records.length > earlier.length ? heldForReport : []),
+            ],
+        };
         // The subconscious's own words, under its own name, verbatim.
         const subName = this.hooks.subconsciousName();
         if (subconsciousNote && subName) {
@@ -564,17 +646,15 @@ export class TuneOutCoordinator {
         // and when it wrote no note and the dump left any message out, since
         // nothing else then covers it. Otherwise its note speaks for what it saw.
         const selfCancel = source === 'subconscious';
-        if (subName && (!selfCancel || heldForReport.length > 0 || (!subconsciousNote && truncated > 0))) {
-            const truncNote = truncated > 0
-                ? ` ${truncated} of ${backlog.length} diverted messages, the oldest, weren't delivered raw to the resident.`
-                : '';
-            const heldNote = heldForReport.length > 0
-                ? ` Still held at the cancel, so your view may not have them yet: the ${heldForReport.length} message${heldForReport.length === 1 ? '' : 's'} after this one.`
+        if (subName && (!selfCancel || release.heldForReport.length > 0 || (!subconsciousNote && release.truncated > 0))) {
+            const truncNote = truncationNote(records);
+            const heldNote = release.heldForReport.length > 0
+                ? ` Still held at the cancel, so your view may not have them yet: the ${release.heldForReport.length} message${release.heldForReport.length === 1 ? '' : 's'} after this one.`
                 : '';
             this.hooks.addMessage('user', [{ type: 'text', text: `[${NOTICE_TAG.tuneOutCancelled}: ${this.shown(serverId, channelId)} — ${reason}.${truncNote}${heldNote}]` }], { system: true, kind: 'tune-out-cancel-report', channelId }, subName);
             // Each held message is a row of its own, so a per-message cap cuts each
             // one on its own, its author's label included, not the batch's newest.
-            for (const m of heldForReport) {
+            for (const m of release.heldForReport) {
                 this.hooks.addMessage('user', [{ type: 'text', text: `${authorOf(m)}:` }, ...m.content], { system: true, kind: 'tune-out-cancel-held', channelId }, subName);
             }
             this.hooks.requestInference(subName, `tune-out cancelled: ${name}`, 'tune-out');
@@ -592,9 +672,7 @@ export class TuneOutCoordinator {
             const rest = this.unhandedBacklog(this.handOffKey(epochId, serverId, channelId), stored, queued);
             const unread = rest.backlog.length + rest.queued.length;
             if (!selfCancel || unread > 0) {
-                const truncNote = truncated > 0
-                    ? ` ${truncated} of ${backlog.length} diverted messages, the oldest, weren't delivered raw to the resident.`
-                    : '';
+                const truncNote = truncationNote(records);
                 const lateNote = selfCancel
                     ? ` ${unread === 1 ? 'This message' : `These ${unread} messages`} arrived after the reader that cancelled was handed its own.`
                     : '';
@@ -607,7 +685,7 @@ export class TuneOutCoordinator {
         // there that its reader hasn't looked at are counted again, as the
         // channel's own tune-out's count stood in for theirs.
         this.rebuildTally(serverId, channelId);
-        this.hooks.emitTrace({ type: 'tune-out:cancelled', serverId, channelId, epochId, reason, wakeCount: ended.wakeCount });
+        this.hooks.emitTrace({ type: 'tune-out:cancelled', serverId, channelId, epochId, reason, wakeCount: ending.wakeCount });
         return { ok: true };
     }
     // ==========================================================================
@@ -814,20 +892,20 @@ export class TuneOutCoordinator {
         }
         return counts;
     }
-    /**
-     * End the all-except hold: durable flip, one release notice with a
-     * per-channel backlog (each capped), the reader's note if any, one wake
-     * for the resident, and the reader's own look at what it had not seen.
-     */
     cancelAllExcept(source, reason, note, noteAuthor) {
         const scope = this.getAllExcept();
         if (!scope)
             return { ok: false, error: 'no all-except hold is active' };
         const { epochId, except, backlogCap } = scope.params;
         const ofEpoch = (m) => m.metadata?.tuneOut?.epochId === epochId;
-        const all = [...this.hooks.getStoredMessages().filter(ofEpoch), ...(this.hooks.getDeferredMessages?.() ?? []).filter(ofEpoch)];
+        const storedRows = this.hooks.getStoredMessages();
+        const queuedRows = this.hooks.getDeferredMessages?.() ?? [];
+        const all = [...storedRows.filter(ofEpoch), ...queuedRows.filter(ofEpoch)];
+        const earlier = releaseRecordsOf([...storedRows, ...queuedRows], epochId);
+        const releasedEarlier = keyedBy(earlier);
+        const fresh = all.filter((m) => !releasedEarlier(m));
         const byChannel = new Map();
-        for (const m of all) {
+        for (const m of fresh) {
             const meta = m.metadata;
             const serverId = meta?.serverId ?? '';
             const channelId = meta?.channelId ?? '(unknown channel)';
@@ -837,12 +915,14 @@ export class TuneOutCoordinator {
             byChannel.set(k, entry);
         }
         const historyTool = this.hooks.historyTool?.() ?? null;
+        let truncatedInAll = 0;
         const dumps = [...byChannel.values()].map(({ serverId, channelId, backlog }) => {
             // Floored here too: a hold stored before entry floored its cap can hold
             // a fraction.
             const cap = Math.floor(backlogCap);
             const shown = cap > 0 ? backlog.slice(-cap) : [];
             const truncated = backlog.length - shown.length;
+            truncatedInAll += truncated;
             const attrs = [
                 `channel=${quotedValue(this.shown(serverId, channelId))}`,
                 `messages=${backlog.length}`,
@@ -852,18 +932,15 @@ export class TuneOutCoordinator {
         });
         const summary = byChannel.size === 0
             ? 'Nothing was held.'
-            : `${all.length} message${all.length === 1 ? '' : 's'} held in ${byChannel.size} channel${byChannel.size === 1 ? '' : 's'}.`;
-        if (this.scopeReleased !== epochId) {
-            this.hooks.addMessage('user', [{ type: 'text', text: `[${NOTICE_TAG.tuneOut}: all channels except ${this.shown(except.serverId, except.channelId)} released — ${reason}, ${scope.wakeCount} wake${scope.wakeCount === 1 ? '' : 's'}]\n\n${summary}${dumps.length > 0 ? '\n\n' + dumps.join('\n\n') : ''}` }], { system: true, kind: 'tune-out-cancel', scope: 'all-except', channelId: except.channelId, epochId });
-            this.hooks.commitDeliveries?.();
-            this.scopeReleased = epochId;
-        }
-        // The release is recorded before the hold is: a failure past this point
-        // leaves the hold in force with its release already delivered. A retry in
-        // this process records only the hold's end; a restart between the two
-        // delivers the release again, where the other order lost it outright.
+            : `${fresh.length} message${fresh.length === 1 ? '' : 's'} held in ${byChannel.size} channel${byChannel.size === 1 ? '' : 's'}.`;
+        if (earlier.length === 0 || fresh.length > 0) {
+            const record = { ...keysOf(fresh), total: fresh.length, truncated: truncatedInAll, report: { ids: [], writeIds: [] } };
+            this.hooks.addMessage('user', [{ type: 'text', text: `[${NOTICE_TAG.tuneOut}: all channels except ${this.shown(except.serverId, except.channelId)} released — ${reason}, ${scope.wakeCount} wake${scope.wakeCount === 1 ? '' : 's'}]\n\n${summary}${dumps.length > 0 ? '\n\n' + dumps.join('\n\n') : ''}` }], { system: true, kind: 'tune-out-cancel', scope: 'all-except', channelId: except.channelId, epochId, released: record });
+        }
+        if (this.hooks.commitDeliveries?.() === false) {
+            throw new Error('the all-except release could not be made durable, so the hold stays in force');
+        }
         this.writeScope(null);
-        this.scopeReleased = undefined;
         this.clearScopeTimers();
         for (const [key, pending] of this.pendingWakes) {
             const [serverId, channelId] = key.split('\0');
@@ -1090,6 +1167,34 @@ export class TuneOutCoordinator {
         }, WAKE_COALESCE_MS);
         this.pendingWakes.set(key, { count: 1, timer });
     }
+    cancelPastBudget(serverId, channelId, maxWakes, from = 'wake') {
+        const key = this.key(serverId, channelId);
+        const existing = this.expiryTimers.get(key);
+        if (existing)
+            clearTimeout(existing);
+        const end = (first) => {
+            try {
+                this.expiryTimers.set(key, setTimeout(() => end(false), EXPIRY_RETRY_MS));
+                if (!this.cancel(serverId, channelId, 'tune-out-max-wakes', `wake budget exhausted (${maxWakes})`).ok) {
+                    this.clearTimers(serverId, channelId);
+                }
+            }
+            catch (err) {
+                if (first && from === 'start') {
+                    if (this.channelRegistry.getTuneOutState(serverId, channelId)) {
+                        console.error(`[tune-out] ${this.shown(serverId, channelId)} was past its wake budget at start, and its cancel failed, so it stays tuned out and its cancel is tried again in ${EXPIRY_RETRY_MS / 1000} s:`, err);
+                    }
+                    else {
+                        console.error(`[tune-out] ${this.shown(serverId, channelId)} was past its wake budget at start and its cancel ended it, but a step after the end failed:`, err);
+                    }
+                }
+                else {
+                    this.timerFailed('wake', serverId, channelId, err);
+                }
+            }
+        };
+        end(true);
+    }
     /** One coalesced wake: counted durably against the epoch that holds the channel. */
     fireWake(serverId, channelId, count) {
         if (this.channelRegistry.getTuneOutState(serverId, channelId)) {
@@ -1098,7 +1203,7 @@ export class TuneOutCoordinator {
             if (!counted)
                 return; // cancelled inside the window
             if (counted.wakeCount > counted.params.maxWakes) {
-                this.cancel(serverId, channelId, 'tune-out-max-wakes', `wake budget exhausted (${counted.params.maxWakes})`);
+                this.cancelPastBudget(serverId, channelId, counted.params.maxWakes);
                 return;
             }
             this.invokeSubconscious(serverId, channelId, 'wake', count);