@ably/ai-transport 0.2.0 → 0.3.0

package/src/core/transport/tree.ts

@@ -25,6 +25,7 @@ import type * as Ably from 'ably';
 
 import {
   HEADER_CODEC_MESSAGE_ID,
+  HEADER_EVENT_ID,
   HEADER_FORK_OF,
   HEADER_INPUT_CODEC_MESSAGE_ID,
   HEADER_INVOCATION_ID,
@@ -33,20 +34,62 @@ import {
   HEADER_ROLE,
   HEADER_RUN_CLIENT_ID,
   HEADER_RUN_ID,
+  HEADER_STREAM,
 } from '../../constants.js';
 import { EventEmitter } from '../../event-emitter.js';
 import type { Logger } from '../../logger.js';
-import type { CodecInputEvent, CodecOutputEvent, Reducer } from '../codec/types.js';
+import { getTransportHeaders } from '../../utils.js';
+import { toCodecEvents } from '../codec/codec-event.js';
+import type { CodecEvent, CodecInputEvent, CodecOutputEvent, Reducer } from '../codec/types.js';
 import type { ConversationNode, InputNode, OutputEvent, RunLifecycleEvent, RunNode, Tree } from './types.js';
+import { WireLog } from './wire-log.js';
 
 // ---------------------------------------------------------------------------
 // Internal node type
 // ---------------------------------------------------------------------------
 
-interface InternalNode<TProjection> {
+/**
+ * How long (in ms, on the Ably message-timestamp timeline) a structurally
+ * complete run's event log is retained after the node's last observed
+ * activity. Bounds cross-publisher live delivery reorder: a wire can be
+ * delivered after a higher-serial wire by at most this window. Conservative
+ * placeholder pending confirmation of the actual cross-region bound.
+ */
+export const REORDER_WINDOW_MS = 120_000;
+
+interface InternalNode<TInput extends CodecInputEvent, TOutput extends CodecOutputEvent, TProjection> {
   node: ConversationNode<TProjection>;
   /** Insertion sequence — tiebreaker for nodes with no sort serial (optimistic). */
   insertSeq: number;
+  /**
+   * The node's event log: every serial-bearing wire applied to this node, in
+   * canonical serial order. Owns its own record/refold/replay-guard/sweep
+   * mutation (see {@link WireLog}). Optimistic (serial-less) applies are not
+   * recorded.
+   */
+  log: WireLog<CodecEvent<TInput, TOutput>>;
+  /**
+   * Max Ably message timestamp (epoch ms) of everything applied to this node,
+   * including its run lifecycle events; 0 until a timestamped apply. The
+   * retention sweep measures {@link REORDER_WINDOW_MS} from here.
+   */
+  lastActivityTs: number;
+  /**
+   * Whether this run's `ai-run-start` has been observed (run nodes only —
+   * always false for input nodes). The structural half of log retention:
+   * run-start is the run's serial floor, so once it is observed no older
+   * history page can deliver further wires for this node.
+   */
+  runStartSeen: boolean;
+  /** Whether this node is already queued for sweeping (guards double-enqueue). */
+  sweepQueued: boolean;
+  /**
+   * Whether an optimistic (serial-less) seed has been folded into the
+   * projection but not into the log. The first serial-bearing wire (the echo)
+   * refolds the node from the log alone, discarding the seed, then clears
+   * this — so a codec needs no seed-replacement logic of its own.
+   */
+  optimistic: boolean;
 }
 
 /**
@@ -145,26 +188,38 @@ export interface TreeInternal<
    * @param events.outputs - Agent-published events (`ai-output` wire).
    * @param headers - Transport headers from the inbound Ably message.
    * @param serial - Ably channel serial; undefined for optimistic inserts.
+   * @param timestamp - Ably server timestamp (epoch ms) of the message —
+   *   top-level `Message.timestamp`, the message's create time on every
+   *   delivery (an append's own receive time lives in `version.timestamp`) —
+   *   or undefined for optimistic inserts. Advances the Tree's event-log
+   *   retention clock and the owning node's last-activity time.
+   * @param version - The delivery's `Message.version.serial`, or undefined
+   *   when the delivery carried none (optimistic inserts, never-mutated
+   *   deliveries from sources that omit it). Guards the node's event log
+   *   against whole-wire replays: a delivery at or below the version already
+   *   decoded into its log entry is dropped.
    */
   applyMessage(
     events: { inputs: TInput[]; outputs: TOutput[] },
     headers: Record<string, string>,
     serial?: string,
+    timestamp?: number,
+    version?: string,
   ): void;
 
   /**
    * Apply a run-lifecycle event.
    *
    * - `start`: creates the reply run (if missing) or, for an existing run,
-   *   sets RunNode.status to 'active', promotes startSerial, and backfills
+   *   sets RunNode.state to 'active', promotes startSerial, and backfills
    *   structural metadata (parent / forkOf / regenerates / invocationId).
-   * - `suspend`: sets RunNode.status to 'suspended' and records `endSerial`.
+   * - `suspend`: sets RunNode.state to 'suspended' and records `endSerial`.
    *   The run stays live so a resume under the same `runId` picks up where it
    *   left off.
-   * - `resume`: re-activates an existing suspended Run (status back to
+   * - `resume`: re-activates an existing suspended Run (state back to
    *   'active') without touching its structure or serials — a pure re-entry
    *   signal. A no-op if the Run is not yet known.
-   * - `end`: sets RunNode.status to the terminal reason and records
+   * - `end`: sets RunNode.state to the terminal reason and records
    *   `endSerial`.
    *
    * Always emits a 'run' event to subscribers.
@@ -214,7 +269,7 @@ export class DefaultTree<
   TOutput extends CodecOutputEvent,
   TProjection,
 > implements TreeInternal<TInput, TOutput, TProjection> {
-  private readonly _codec: Reducer<TInput | TOutput, TProjection>;
+  private readonly _codec: Reducer<CodecEvent<TInput, TOutput>, TProjection>;
   private readonly _logger: Logger;
   private readonly _emitter: EventEmitter<TreeEventsMap<TOutput>>;
 
@@ -222,7 +277,7 @@ export class DefaultTree<
    * All nodes indexed by their primary key ({@link nodeKey}): a reply run's
    * runId, or an input node's codec-message-id.
    */
-  private readonly _nodeIndex = new Map<string, InternalNode<TProjection>>();
+  private readonly _nodeIndex = new Map<string, InternalNode<TInput, TOutput, TProjection>>();
 
   /**
    * Maps every observed `codec-message-id` to its owning node's key
@@ -240,7 +295,7 @@ export class DefaultTree<
    * serial (optimistic) sort after all serial-bearing nodes, ordered among
    * themselves by insertion sequence.
    */
-  private readonly _sortedNodes: InternalNode<TProjection>[] = [];
+  private readonly _sortedNodes: InternalNode<TInput, TOutput, TProjection>[] = [];
 
   /**
    * Parent index: parent node key (the key its children's
@@ -273,10 +328,37 @@ export class DefaultTree<
    * the View calls `getSiblingNodes` once per visible node plus extra
    * per-message branch-anchor probes from React components.
    */
-  private _siblingCache = new Map<string, InternalNode<TProjection>[]>();
+  private _siblingCache = new Map<string, InternalNode<TInput, TOutput, TProjection>[]>();
   private _siblingCacheVersion = -1;
 
-  constructor(codec: Reducer<TInput | TOutput, TProjection>, logger: Logger) {
+  /**
+   * Index from `event-id` header to the raw Ably message that carried it.
+   * Populated incrementally as messages arrive via {@link emitAblyMessage};
+   * reads back the raw message for the agent's input-event lookup
+   * ({@link findAblyMessageByEventId}). Bounded by the Tree's lifetime — cleared
+   * when the Tree is replaced on continuity loss / session close.
+   */
+  private readonly _eventIdIndex = new Map<string, Ably.InboundMessage>();
+
+  /**
+   * Event-log retention logical clock: the max Ably message timestamp (epoch
+   * ms) observed across every apply, 0 until the first timestamped one. Only
+   * ever advances — older-page history application carries smaller timestamps
+   * and leaves it (and therefore the sweep) untouched.
+   */
+  private _clock = 0;
+
+  /**
+   * Keys of structurally complete run nodes (run-start and run-end both
+   * observed) whose event logs await the retention window, in completion
+   * order. Drained from the front whenever {@link _clock} advances; sweeping
+   * only at clock advances keeps a history page's batch atomic — applying an
+   * older page can never advance the clock, so a node cannot be swept between
+   * its run-start and the rest of its wires in the same page.
+   */
+  private readonly _sweepQueue: string[] = [];
+
+  constructor(codec: Reducer<CodecEvent<TInput, TOutput>, TProjection>, logger: Logger) {
     this._codec = codec;
     this._logger = logger;
     this._emitter = new EventEmitter<TreeEventsMap<TOutput>>(logger);
@@ -302,7 +384,10 @@ export class DefaultTree<
    * @returns Negative if a sorts before b, positive if after, zero if equal.
    */
   // Spec: AIT-CT13a
-  private _compareNodes(a: InternalNode<TProjection>, b: InternalNode<TProjection>): number {
+  private _compareNodes(
+    a: InternalNode<TInput, TOutput, TProjection>,
+    b: InternalNode<TInput, TOutput, TProjection>,
+  ): number {
     const sa = sortSerial(a.node);
     const sb = sortSerial(b.node);
     if (sa === undefined && sb === undefined) return a.insertSeq - b.insertSeq;
@@ -317,7 +402,7 @@ export class DefaultTree<
    * Insert a node into the sorted list at the correct position via binary search.
    * @param internal - The node to insert.
    */
-  private _insertSortedNode(internal: InternalNode<TProjection>): void {
+  private _insertSortedNode(internal: InternalNode<TInput, TOutput, TProjection>): void {
     const startSerial = sortSerial(internal.node);
 
     // Fast path: null-startSerial always appends to end.
@@ -345,7 +430,7 @@ export class DefaultTree<
    * Remove a node from the sorted list.
    * @param internal - The node to remove.
    */
-  private _removeSortedNode(internal: InternalNode<TProjection>): void {
+  private _removeSortedNode(internal: InternalNode<TInput, TOutput, TProjection>): void {
     const idx = this._sortedNodes.indexOf(internal);
     if (idx !== -1) this._sortedNodes.splice(idx, 1);
   }
@@ -359,7 +444,11 @@ export class DefaultTree<
    * @param entry - The internal node to insert.
    * @param parentCodecMessageId - The node's structural parent, or undefined for a root.
    */
-  private _insertNode(key: string, entry: InternalNode<TProjection>, parentCodecMessageId: string | undefined): void {
+  private _insertNode(
+    key: string,
+    entry: InternalNode<TInput, TOutput, TProjection>,
+    parentCodecMessageId: string | undefined,
+  ): void {
     this._nodeIndex.set(key, entry);
     this._addToParentIndex(parentCodecMessageId, key);
     this._insertSortedNode(entry);
@@ -373,7 +462,7 @@ export class DefaultTree<
    * optimistic-serial promotion paths when the server relay/echo arrives.
    * @param entry - The internal node whose serial was just promoted.
    */
-  private _promoteSerial(entry: InternalNode<TProjection>): void {
+  private _promoteSerial(entry: InternalNode<TInput, TOutput, TProjection>): void {
     this._removeSortedNode(entry);
     this._insertSortedNode(entry);
     this._structuralVersion++;
@@ -389,8 +478,8 @@ export class DefaultTree<
    * @param messageId - The reducer routing key (codec-message-id), or undefined.
    */
   private _foldInto(
-    entry: InternalNode<TProjection>,
-    events: (TInput | TOutput)[],
+    entry: InternalNode<TInput, TOutput, TProjection>,
+    events: CodecEvent<TInput, TOutput>[],
     serial: string | undefined,
     messageId: string | undefined,
   ): void {
@@ -403,6 +492,196 @@ export class DefaultTree<
     }
   }
 
+  /**
+   * Record a serial-bearing wire in the node's event log and fold it. Events
+   * extending the log tail (the common case — in-order live delivery) fold
+   * incrementally onto the existing projection, identical to a bare
+   * {@link _foldInto}. Events that land earlier in the log (an earlier-serial
+   * wire delivered late — cross-publisher reorder, or a history page applying
+   * an older message after a newer one) cannot be folded incrementally without
+   * corrupting serial order, so the node is refolded from the whole log via
+   * {@link _refold}.
+   *
+   * Optimistic (serial-less) applies and empty event batches are not logged;
+   * an optimistic seed folds into the projection but never into the log, and
+   * marks the node `optimistic`. The first serial-bearing wire (the echo of
+   * the optimistic input, which re-delivers the seeded content) refolds the
+   * node from the log alone — rebuilding the projection without the seed
+   * rather than folding the echo on top of it. The codec therefore never sees
+   * the seed and its echo in one projection, and needs no seed-replacement
+   * logic. The seed must be a faithful preview of the echo, since the echo's
+   * content is what survives.
+   *
+   * Whole-wire replays are dropped at the log: each entry records the highest
+   * `Message.version.serial` decoded into it (`decodedThrough`), so a
+   * version-bearing delivery the entry has already incorporated — a second
+   * hydration over a populated Tree, a remounted View's re-fetch, an agent
+   * re-walk — records nothing and folds nothing. A newer version of a
+   * discrete wire (an edited discrete) is likewise dropped; propagating edits
+   * into projections is deliberately out of scope.
+   * @param entry - The internal node whose log and projection are updated.
+   * @param events - The decoded events to fold, in wire order.
+   * @param serial - Ably channel serial; undefined for an optimistic insert.
+   * @param messageId - The reducer routing key (codec-message-id), or undefined.
+   * @param version - The delivery's `Message.version.serial`, or undefined.
+   * @param streamed - Whether the delivery is part of a streamed wire.
+   */
+  private _recordAndFold(
+    entry: InternalNode<TInput, TOutput, TProjection>,
+    events: CodecEvent<TInput, TOutput>[],
+    serial: string | undefined,
+    messageId: string | undefined,
+    version: string | undefined,
+    streamed: boolean,
+  ): void {
+    // A serial-less optimistic seed (or an empty batch) is not logged. Fold it
+    // in; a non-empty seed marks the node so its echo refolds the seed away.
+    if (serial === undefined || events.length === 0) {
+      if (serial === undefined && events.length > 0) entry.optimistic = true;
+      this._foldInto(entry, events, serial, messageId);
+      return;
+    }
+
+    const fold = entry.log.record(serial, messageId, events, version, streamed);
+    if (fold === 'dropped') {
+      // The version guard rejected a re-delivery the log already incorporated —
+      // a whole-wire replay (second hydration, remount, agent re-walk, or a
+      // `loadOlder()` re-applying a swept run's history) or an edit to a
+      // discrete. Nothing to fold.
+      this._logger.debug('Tree._recordAndFold(); version guard dropped re-delivered wire', {
+        key: nodeKey(entry.node),
+        serial,
+        version,
+        swept: entry.log.swept,
+      });
+      return;
+    }
+    if (entry.optimistic && !entry.log.swept) {
+      // First serial-bearing wire (the echo) on a node that carries an
+      // optimistic seed. The seed is in the projection but not the log, so
+      // refold from the log alone — the echo re-delivers the seeded content —
+      // rebuilding the projection without the seed instead of folding the echo
+      // on top of it.
+      entry.optimistic = false;
+      this._refold(entry);
+      return;
+    }
+    if (fold === 'refold') {
+      this._refold(entry);
+      return;
+    }
+    // 'incremental'. On a swept log this is a genuinely-new wire outside the
+    // reorder window (it should not occur) folding in arrival order — the log
+    // could not refold it.
+    if (entry.log.swept) {
+      this._logger.warn('Tree._recordAndFold(); late wire after log retention window; folding in arrival order', {
+        key: nodeKey(entry.node),
+        serial,
+      });
+    }
+    this._foldInto(entry, events, serial, messageId);
+  }
+
+  /**
+   * Rebuild a node's projection from its event log in canonical serial order:
+   * a fresh {@link Reducer.init} folded through every logged event, each with
+   * its own wire's serial and messageId. Used when a late, earlier-serial wire
+   * makes incremental folding unsound. Reducer purity (a fold is a function of
+   * its inputs alone) is what makes the rebuild faithful; the per-fold
+   * try/catch mirrors {@link _foldInto} so one throwing event can't abort the
+   * rebuild.
+   *
+   * Rebuilds the projection only; the surrounding apply emits its usual
+   * `output` event carrying just the triggering wire's events. Consumers read
+   * the rebuilt state from `node.projection` (the View recomputes its message
+   * list from it), so on the refold path the event's `events` payload is not a
+   * delta of the full projection change.
+   * @param entry - The internal node whose projection is rebuilt in place.
+   */
+  private _refold(entry: InternalNode<TInput, TOutput, TProjection>): void {
+    let projection = this._codec.init();
+    entry.log.replay((event, serial, messageId) => {
+      try {
+        projection = this._codec.fold(projection, event, { serial, messageId });
+      } catch (error) {
+        this._logger.error('Tree._refold(); fold threw', { key: nodeKey(entry.node), messageId, err: error });
+      }
+    });
+    entry.node.projection = projection;
+  }
+
+  // -------------------------------------------------------------------------
+  // Event-log retention
+  // -------------------------------------------------------------------------
+
+  /**
+   * Record activity on a node and advance the retention clock. Updates the
+   * node's `lastActivityTs` and the Tree-wide `_clock` to the given timestamp
+   * when it is newer; a clock advance drains the sweep queue. `undefined`
+   * (an optimistic local apply) advances nothing.
+   * @param entry - The node the activity belongs to.
+   * @param timestamp - Ably message timestamp (epoch ms), or undefined.
+   */
+  private _recordActivity(entry: InternalNode<TInput, TOutput, TProjection>, timestamp: number | undefined): void {
+    if (timestamp === undefined) return;
+    if (timestamp > entry.lastActivityTs) entry.lastActivityTs = timestamp;
+    if (timestamp > this._clock) {
+      this._clock = timestamp;
+      this._drainSweepQueue();
+    }
+  }
+
+  /**
+   * Queue a run node's event log for retention sweeping once the node is
+   * structurally complete: its run-start (serial floor — no older history page
+   * can add to it) and its run-end (no further agent output) have both been
+   * observed. The actual drop happens in {@link _drainSweepQueue} once the
+   * reorder window has also lapsed. No-op for input nodes (never swept — no
+   * floor marker, and their logs are bounded by one user message), for nodes
+   * already queued or swept, and while either marker is missing.
+   * @param entry - The node to consider for sweeping.
+   */
+  private _maybeQueueSweep(entry: InternalNode<TInput, TOutput, TProjection>): void {
+    const node = entry.node;
+    if (node.kind !== 'run') return;
+    if (entry.log.swept || entry.sweepQueued) return;
+    if (!entry.runStartSeen) return;
+    if (node.state.status === 'active' || node.state.status === 'suspended') return;
+    entry.sweepQueued = true;
+    this._sweepQueue.push(node.runId);
+  }
+
+  /**
+   * Drop the event logs of queued nodes whose retention window has lapsed:
+   * `lastActivityTs + REORDER_WINDOW_MS < _clock`. Drains from the front and
+   * stops at the first node still inside the window — completion order is
+   * time-ordered for live traffic, so this is amortised O(1) per apply, and
+   * stopping early only ever over-retains (memory, never correctness). Called
+   * only when the clock advances, so applying an older history page (smaller
+   * timestamps) can never sweep mid-batch. Deleted nodes are skipped.
+   */
+  private _drainSweepQueue(): void {
+    while (this._sweepQueue.length > 0) {
+      const key = this._sweepQueue[0];
+      const entry = key === undefined ? undefined : this._nodeIndex.get(key);
+      if (!entry || entry.log.swept) {
+        this._sweepQueue.shift();
+        continue;
+      }
+      if (entry.lastActivityTs + REORDER_WINDOW_MS >= this._clock) return;
+      this._sweepQueue.shift();
+      entry.sweepQueued = false;
+      // Drop the decoded payloads (the unbounded cost) but keep each entry's
+      // replay key, so a post-sweep whole-wire replay is still recognised and
+      // dropped rather than re-folded (a refold can no longer rebuild them).
+      entry.log.sweep();
+      this._logger.debug('Tree._drainSweepQueue(); dropped event-log payloads, kept replay keys', {
+        key,
+        lastActivityTs: entry.lastActivityTs,
+      });
+    }
+  }
+
   // -------------------------------------------------------------------------
   // Parent index maintenance
   // -------------------------------------------------------------------------
@@ -471,7 +750,7 @@ export class DefaultTree<
    * @returns The ordered list of sibling nodes.
    */
   // Spec: AIT-CT13b
-  private _getSiblingGroup(key: string): InternalNode<TProjection>[] {
+  private _getSiblingGroup(key: string): InternalNode<TInput, TOutput, TProjection>[] {
     if (this._siblingCacheVersion !== this._structuralVersion) {
       this._siblingCache.clear();
       this._siblingCacheVersion = this._structuralVersion;
@@ -495,7 +774,7 @@ export class DefaultTree<
     // still files/groups correctly — the parent codec-message-id is known at
     // creation, the resolved key may not be.
     const parentKey = original.parentCodecMessageId;
-    const siblings: InternalNode<TProjection>[] = [];
+    const siblings: InternalNode<TInput, TOutput, TProjection>[] = [];
     const candidateKeys = this._parentIndex.get(parentKey);
     if (candidateKeys) {
       for (const childKey of candidateKeys) {
@@ -668,6 +947,8 @@ export class DefaultTree<
     events: { inputs: TInput[]; outputs: TOutput[] },
     headers: Record<string, string>,
     serial?: string,
+    timestamp?: number,
+    version?: string,
   ): void {
     const wireRunId = headers[HEADER_RUN_ID];
     const codecMessageId = headers[HEADER_CODEC_MESSAGE_ID];
@@ -691,7 +972,7 @@ export class DefaultTree<
     }
 
     // Fold inputs first, then outputs, preserving wire order.
-    const all: (TInput | TOutput)[] = [...events.inputs, ...events.outputs];
+    const all: CodecEvent<TInput, TOutput>[] = toCodecEvents(events);
 
     // Wire-only metadata-carrier messages (e.g. `ait-regenerate`) decode to
     // zero events and don't need a node at the tree level — the eventual reply
@@ -710,9 +991,9 @@ export class DefaultTree<
     const structuralBefore = this._structuralVersion;
 
     if (inputNodeCodecMessageId !== undefined) {
-      this._applyInputMessage(inputNodeCodecMessageId, headers, serial, all);
+      this._applyInputMessage(inputNodeCodecMessageId, headers, serial, timestamp, version, all);
     } else if (wireRunId !== undefined) {
-      this._applyRunMessage(wireRunId, events, headers, serial);
+      this._applyRunMessage(wireRunId, events, headers, serial, timestamp, version);
     }
 
     if (this._structuralVersion !== structuralBefore) this._emitter.emit('update');
@@ -726,13 +1007,17 @@ export class DefaultTree<
    * @param codecMessageId - The input node's codec-message-id (its primary key).
    * @param headers - Transport headers from the inbound Ably message.
    * @param serial - Ably channel serial; undefined for an optimistic insert.
-   * @param all - The decoded input events to fold, in wire order.
+   * @param timestamp - Ably server timestamp (epoch ms); undefined for an optimistic insert.
+   * @param version - The delivery's `Message.version.serial`, or undefined.
+   * @param all - The direction-tagged input events to fold, in wire order.
    */
   private _applyInputMessage(
     codecMessageId: string,
     headers: Record<string, string>,
     serial: string | undefined,
-    all: (TInput | TOutput)[],
+    timestamp: number | undefined,
+    version: string | undefined,
+    all: CodecEvent<TInput, TOutput>[],
   ): void {
     let entry = this._nodeIndex.get(codecMessageId);
     if (!entry) {
@@ -747,7 +1032,11 @@ export class DefaultTree<
       this._promoteSerial(entry);
     }
 
-    this._foldInto(entry, all, serial, codecMessageId);
+    this._recordActivity(entry, timestamp);
+
+    // Log the wire and fold it — incrementally onto the tail in the common
+    // case, or by refolding the node if this wire arrived out of serial order.
+    this._recordAndFold(entry, all, serial, codecMessageId, version, headers[HEADER_STREAM] === 'true');
 
     // An input node owns no agent outputs; the event still fires (empty
     // outputs) so consumers observe the projection change. It has no run-id —
@@ -774,19 +1063,23 @@ export class DefaultTree<
    * @param events.outputs - Agent-published events (`ai-output` wire).
    * @param headers - Transport headers from the inbound Ably message.
    * @param serial - Ably channel serial; undefined for an optimistic insert.
+   * @param timestamp - Ably server timestamp (epoch ms); undefined for an optimistic insert.
+   * @param version - The delivery's `Message.version.serial`, or undefined.
    */
   private _applyRunMessage(
     wireRunId: string,
     events: { inputs: TInput[]; outputs: TOutput[] },
     headers: Record<string, string>,
     serial: string | undefined,
+    timestamp: number | undefined,
+    version: string | undefined,
   ): void {
     const codecMessageId = headers[HEADER_CODEC_MESSAGE_ID];
     // The triggering input's codec-message-id (the agent's echo), surfaced on
     // the `output` event as the stream's causal routing key.
     const inputCodecMessageId = headers[HEADER_INPUT_CODEC_MESSAGE_ID];
     // Fold inputs first, then outputs, preserving wire order.
-    const all: (TInput | TOutput)[] = [...events.inputs, ...events.outputs];
+    const all: CodecEvent<TInput, TOutput>[] = toCodecEvents(events);
     const outputs = events.outputs;
 
     let run = this._nodeIndex.get(wireRunId);
@@ -816,7 +1109,13 @@ export class DefaultTree<
     const ownerKey = nodeKey(run.node);
     if (codecMessageId) this._codecMessageIdToNodeKey.set(codecMessageId, ownerKey);
 
-    this._foldInto(run, all, serial, codecMessageId);
+    this._recordActivity(run, timestamp);
+
+    // Log the wire and fold it — incrementally onto the tail in the common
+    // case, or by refolding the node if this wire arrived out of serial order.
+    // `run` may be a reconciled optimistic node: record on whichever entry
+    // owns the fold.
+    this._recordAndFold(run, all, serial, codecMessageId, version, headers[HEADER_STREAM] === 'true');
 
     this._emitter.emit('output', { runId: ownerKey, inputCodecMessageId, codecMessageId, serial, events: outputs });
   }
@@ -876,8 +1175,12 @@ export class DefaultTree<
     const existing = this._nodeIndex.get(event.runId);
     if (existing?.node.kind === 'run') {
       const node = existing.node;
-      if (node.status !== 'active') {
-        node.status = 'active';
+      // Activate only a suspended run. A run-start can be observed AFTER the
+      // run's terminal event (history pages replay newest-first, so an older
+      // page delivers the start last) — like a stray resume, it must never
+      // resurrect a run that has ended.
+      if (node.state.status === 'suspended') {
+        node.state = { status: 'active' };
       }
       if (event.serial && !node.startSerial) {
         node.startSerial = event.serial;
@@ -919,10 +1222,18 @@ export class DefaultTree<
       if (node.invocationId === '' && event.invocationId !== '') {
         node.invocationId = event.invocationId;
       }
+      // The run's serial floor is now observed: no older history page can
+      // deliver further wires for this node. With a terminal status this
+      // makes the node structurally complete — eligible for log retention
+      // sweeping once the reorder window lapses.
+      existing.runStartSeen = true;
+      this._recordActivity(existing, event.timestamp);
+      this._maybeQueueSweep(existing);
     } else if (!existing) {
       const run = this._createRunFromLifecycle(event);
       this._insertNode(event.runId, run, run.node.parentCodecMessageId);
       this._indexReplyRun(run.node, event.runId);
+      this._recordActivity(run, event.timestamp);
     }
   }
 
@@ -937,8 +1248,9 @@ export class DefaultTree<
   private _applyRunSuspend(event: RunLifecycleEvent & { type: 'suspend' }): void {
     const run = this._nodeIndex.get(event.runId);
     if (run?.node.kind === 'run') {
-      run.node.status = 'suspended';
+      run.node.state = { status: 'suspended' };
       run.node.endSerial = event.serial;
+      this._recordActivity(run, event.timestamp);
     }
   }
 
@@ -955,23 +1267,33 @@ export class DefaultTree<
    */
   private _applyRunResume(event: RunLifecycleEvent & { type: 'resume' }): void {
     const run = this._nodeIndex.get(event.runId);
-    if (run?.node.kind === 'run' && run.node.status === 'suspended') {
-      run.node.status = 'active';
+    if (run?.node.kind === 'run' && run.node.state.status === 'suspended') {
+      run.node.state = { status: 'active' };
+      this._recordActivity(run, event.timestamp);
     }
   }
 
   /**
-   * Apply a run-end lifecycle event: record the terminal reason as the node's
-   * status and the serial it ended at. Status/endSerial are content, not
-   * structure, so this never mutates `_structuralVersion`; the caller owns the
-   * emits.
+   * Apply a run-end lifecycle event: record the terminal reason (and, for an
+   * error end, the error) as the node's state, plus the serial it ended at.
+   * State/endSerial are content, not structure, so this never mutates
+   * `_structuralVersion`; the caller owns the emits.
+   *
+   * A run-end for an unknown runId is a no-op: nothing else is known about the
+   * run yet, so there is no node to mark. When that happens during history
+   * replay (a page boundary falling just before the run-end, so the run's
+   * other wires arrive in later pages), the run is never marked terminal and
+   * its event log is retained for the Tree's lifetime — over-retention, never
+   * corruption.
    * @param event - The run-end lifecycle event.
    */
   private _applyRunEnd(event: RunLifecycleEvent & { type: 'end' }): void {
     const run = this._nodeIndex.get(event.runId);
     if (run?.node.kind === 'run') {
-      run.node.status = event.reason;
+      run.node.state = event.reason === 'error' ? { status: 'error', error: event.error } : { status: event.reason };
       run.node.endSerial = event.serial;
+      this._recordActivity(run, event.timestamp);
+      this._maybeQueueSweep(run);
     }
   }
 
@@ -1012,7 +1334,7 @@ export class DefaultTree<
     runId: string,
     headers: Record<string, string>,
     serial: string | undefined,
-  ): InternalNode<TProjection> {
+  ): InternalNode<TInput, TOutput, TProjection> {
     const forkOfMsgId = headers[HEADER_FORK_OF];
     return this._buildRunNode({
       runId,
@@ -1024,9 +1346,37 @@ export class DefaultTree<
       clientId: headers[HEADER_RUN_CLIENT_ID] ?? '',
       invocationId: headers[HEADER_INVOCATION_ID] ?? '',
       startSerial: serial,
+      // Created from a content wire — the run's ai-run-start has not been
+      // observed (it may still be in an unloaded older history page).
+      runStartSeen: false,
     });
   }
 
+  /**
+   * Wrap a freshly-built conversation node in its internal envelope — sort
+   * sequence, event log, and retention/promotion state. The single home for
+   * those per-node fields, so a new field is added in one place rather than at
+   * every node-construction site.
+   * @param node - The conversation node to wrap.
+   * @param runStartSeen - Whether the run's ai-run-start has been observed
+   *   (run nodes only; always false for input nodes).
+   * @returns A newly-allocated internal node ready for insertion.
+   */
+  private _wrapNode(
+    node: ConversationNode<TProjection>,
+    runStartSeen = false,
+  ): InternalNode<TInput, TOutput, TProjection> {
+    return {
+      node,
+      insertSeq: this._seqCounter++,
+      log: new WireLog<CodecEvent<TInput, TOutput>>(),
+      lastActivityTs: 0,
+      runStartSeen,
+      sweepQueued: false,
+      optimistic: false,
+    };
+  }
+
   /**
    * Allocate a RunNode from already-resolved fields. Shared by the
    * header-driven and lifecycle-driven run creators: both build the identical
@@ -1039,6 +1389,7 @@ export class DefaultTree<
    * @param params.clientId - The publishing client's id.
    * @param params.invocationId - The agent invocation id.
    * @param params.startSerial - Ably channel serial; undefined for optimistic inserts.
+   * @param params.runStartSeen - Whether the run's ai-run-start has been observed (true only for lifecycle-created runs).
    * @returns A newly-allocated internal run node ready for insertion.
    */
   private _buildRunNode(params: {
@@ -1049,7 +1400,8 @@ export class DefaultTree<
     clientId: string;
     invocationId: string;
     startSerial: string | undefined;
-  }): InternalNode<TProjection> {
+    runStartSeen: boolean;
+  }): InternalNode<TInput, TOutput, TProjection> {
     const node: RunNode<TProjection> = {
       kind: 'run',
       runId: params.runId,
@@ -1058,13 +1410,13 @@ export class DefaultTree<
       regeneratesCodecMessageId: params.regeneratesCodecMessageId,
       clientId: params.clientId,
       invocationId: params.invocationId,
-      status: 'active',
+      state: { status: 'active' },
       projection: this._codec.init(),
       startSerial: params.startSerial,
       endSerial: undefined,
     };
 
-    return { node, insertSeq: this._seqCounter++ };
+    return this._wrapNode(node, params.runStartSeen);
   }
 
   /**
@@ -1078,7 +1430,7 @@ export class DefaultTree<
     codecMessageId: string,
     headers: Record<string, string>,
     serial: string | undefined,
-  ): InternalNode<TProjection> {
+  ): InternalNode<TInput, TOutput, TProjection> {
     const forkOfMsgId = headers[HEADER_FORK_OF];
     const node: InputNode<TProjection> = {
       kind: 'input',
@@ -1090,7 +1442,7 @@ export class DefaultTree<
       projection: this._codec.init(),
       serial,
     };
-    return { node, insertSeq: this._seqCounter++ };
+    return this._wrapNode(node);
   }
 
   /**
@@ -1100,7 +1452,9 @@ export class DefaultTree<
    *   its channel serial.
    * @returns A newly-allocated internal run node ready for insertion.
    */
-  private _createRunFromLifecycle(event: RunLifecycleEvent & { type: 'start' }): InternalNode<TProjection> {
+  private _createRunFromLifecycle(
+    event: RunLifecycleEvent & { type: 'start' },
+  ): InternalNode<TInput, TOutput, TProjection> {
     const forkOfMsgId = event.forkOf;
     return this._buildRunNode({
       runId: event.runId,
@@ -1110,6 +1464,8 @@ export class DefaultTree<
       clientId: event.clientId,
       invocationId: event.invocationId,
       startSerial: event.serial,
+      // Created from the run-start itself — the serial floor is observed.
+      runStartSeen: true,
     });
   }
 
@@ -1139,13 +1495,24 @@ export class DefaultTree<
   }
 
   /**
-   * Forward a raw Ably message event to tree subscribers.
+   * Forward a raw Ably message event to tree subscribers. Also indexes the
+   * Ably message by `event-id` header (if present) for
+   * {@link findAblyMessageByEventId} lookups.
    * @param msg - The raw Ably message to emit.
    */
   emitAblyMessage(msg: Ably.InboundMessage): void {
     this._logger.trace('DefaultTree.emitAblyMessage();');
+    const headers = getTransportHeaders(msg);
+    const eventId = headers[HEADER_EVENT_ID];
+    if (eventId !== undefined && !this._eventIdIndex.has(eventId)) {
+      this._eventIdIndex.set(eventId, msg);
+    }
     this._emitter.emit('ably-message', msg);
   }
+
+  findAblyMessageByEventId(eventId: string): Ably.InboundMessage | undefined {
+    return this._eventIdIndex.get(eventId);
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -1162,6 +1529,6 @@ export class DefaultTree<
  *   emitAblyMessage). Public consumers see the narrower {@link Tree} interface.
  */
 export const createTree = <TInput extends CodecInputEvent, TOutput extends CodecOutputEvent, TProjection>(
-  codec: Reducer<TInput | TOutput, TProjection>,
+  codec: Reducer<CodecEvent<TInput, TOutput>, TProjection>,
   logger: Logger,
 ): DefaultTree<TInput, TOutput, TProjection> => new DefaultTree<TInput, TOutput, TProjection>(codec, logger);