@enbox/agent 0.5.10 → 0.5.11

package/src/sync-engine-level.ts

@@ -1,16 +1,22 @@
 import type { AbstractLevel } from 'abstract-level';
 
 import type { DwnSubscriptionHandler, ResubscribeFactory } from '@enbox/dwn-clients';
-import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDiffEntry, MessagesSyncReply, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
+import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
 
 import ms from 'ms';
 
 import { Level } from 'level';
 import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
 
+import type { ClosureEvaluationContext } from './sync-closure-types.js';
 import type { PermissionsApi } from './types/permissions.js';
 import type { EnboxAgent, EnboxPlatformAgent } from './types/agent.js';
-import type { StartSyncParams, SyncConnectivityState, SyncEngine, SyncIdentityOptions, SyncMode } from './types/sync.js';
+import type { PushResult, ReplicationLinkState, StartSyncParams, SyncConnectivityState, SyncEngine, SyncEvent, SyncEventListener, SyncIdentityOptions, SyncMode, SyncScope } from './types/sync.js';
+
+import { evaluateClosure } from './sync-closure-resolver.js';
+import { MAX_PENDING_TOKENS } from './types/sync.js';
+import { ReplicationLedger } from './sync-replication-ledger.js';
+import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
 
 import { AgentPermissionsApi } from './permissions-api.js';
 import { DwnInterface } from './types/dwn.js';
@@ -70,6 +76,80 @@ type LocalSubscription = {
   close: () => Promise<void>;
 };
 
+// ---------------------------------------------------------------------------
+// Per-link in-memory delivery-order tracking (not persisted to ledger)
+// ---------------------------------------------------------------------------
+
+/**
+ * Tracks an in-flight delivery that has been started but may not yet be
+ * durably committed. Used by the pull path to handle async completion
+ * reordering — subscription callbacks are fire-and-forget, so event B
+ * can complete before event A even though A was delivered first.
+ */
+type InFlightCommit = {
+  /** Monotonic delivery ordinal for this link. */
+  ordinal: number;
+  /** The token associated with this delivery. */
+  token: ProgressToken;
+  /** Whether processRawMessage has completed successfully. */
+  committed: boolean;
+};
+
+/**
+ * Checks whether a message's protocolPath and contextId match the link's
+ * subset scope prefixes. Returns true if the message is in scope.
+ *
+ * When the scope has no prefixes (or is kind:'full'), all messages match.
+ * When protocolPathPrefixes or contextIdPrefixes are specified, the message
+ * must match at least one prefix in each specified set.
+ *
+ * This is agent-side filtering for subset scopes. The underlying
+ * MessagesSubscribe filter only supports protocol-level scoping today —
+ * protocolPath/contextId prefix filtering at the EventLog level is a
+ * follow-up (requires dwn-sdk-js MessagesFilter extension).
+ */
+function isEventInScope(message: GenericMessage, scope: SyncScope): boolean {
+  if (scope.kind === 'full') { return true; }
+  if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) { return true; }
+
+  const desc = message.descriptor as Record<string, unknown>;
+
+  // Check protocolPath prefix.
+  if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
+    const protocolPath = desc.protocolPath as string | undefined;
+    if (!protocolPath) { return false; }
+    const matches = scope.protocolPathPrefixes.some(
+      prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/')
+    );
+    if (!matches) { return false; }
+  }
+
+  // Check contextId prefix.
+  if (scope.contextIdPrefixes && scope.contextIdPrefixes.length > 0) {
+    const contextId = (message as any).contextId as string | undefined;
+    if (!contextId) { return false; }
+    const matches = scope.contextIdPrefixes.some(
+      prefix => contextId === prefix || contextId.startsWith(prefix + '/')
+    );
+    if (!matches) { return false; }
+  }
+
+  return true;
+}
+
+/**
+ * Per-link runtime state held in memory. Not persisted — on crash,
+ * replay restarts from `contiguousAppliedToken` (idempotent apply).
+ */
+type LinkRuntimeState = {
+  /** Next ordinal to assign when a pull event is delivered. */
+  nextDeliveryOrdinal: number;
+  /** Next ordinal to check when draining committed entries. */
+  nextCommitOrdinal: number;
+  /** In-flight deliveries keyed by ordinal. */
+  inflight: Map<number, InFlightCommit>;
+};
+
 export class SyncEngineLevel implements SyncEngine {
   /**
    * Holds the instance of a `EnboxPlatformAgent` that represents the current execution context for
@@ -88,6 +168,29 @@ export class SyncEngineLevel implements SyncEngine {
   private _syncIntervalId?: ReturnType<typeof setInterval>;
   private _syncLock = false;
 
+  /**
+   * Durable replication ledger — persists per-link checkpoint state.
+   * Used by live sync to track pull/push progression independently per link.
+   * Poll-mode sync still uses the legacy `getCursor`/`setCursor` path.
+   * Lazily initialized on first use to avoid sublevel() calls on mock dbs.
+   */
+  private _ledger?: ReplicationLedger;
+
+  /**
+   * In-memory cache of active links, keyed by `{did}^{dwnUrl}^{protocol}`.
+   * Populated from the ledger on `startLiveSync`, used by subscription handlers
+   * to avoid async ledger lookups on every event.
+   */
+  private _activeLinks: Map<string, ReplicationLinkState> = new Map();
+
+  /**
+   * Per-link in-memory delivery-order tracking for the pull path. Keyed by
+   * the same link key as `_activeLinks`. Not persisted — on crash, replay
+   * restarts from `contiguousAppliedToken` and idempotent apply handles
+   * re-delivered events.
+   */
+  private _linkRuntimes: Map<string, LinkRuntimeState> = new Map();
+
   /**
    * Hex-encoded default hashes for empty subtrees at each depth, keyed by depth.
    * Lazily initialized on first use. Used by `walkTreeDiff` to detect empty subtrees
@@ -102,6 +205,14 @@ export class SyncEngineLevel implements SyncEngine {
   /** Current sync mode, set by `startSync`. */
   private _syncMode: SyncMode = 'poll';
 
+  /**
+   * Monotonic session generation counter. Incremented on every teardown.
+   * Async operations (repair, retry timers) capture the generation at start
+   * and bail if it has changed — this prevents stale work from mutating
+   * state after teardown or mode switch.
+   */
+  private _syncGeneration = 0;
+
   /** Active live pull subscriptions (remote -> local via MessagesSubscribe). */
   private _liveSubscriptions: LiveSubscription[] = [];
 
@@ -114,8 +225,35 @@ export class SyncEngineLevel implements SyncEngine {
   /** Debounce timer for batched push-on-write. */
   private _pushDebounceTimer?: ReturnType<typeof setTimeout>;
 
-  /** Pending message CIDs to push, accumulated during the debounce window. */
-  private _pendingPushCids: Map<string, { did: string; dwnUrl: string; delegateDid?: string; protocol?: string; cids: string[] }> = new Map();
+  /** Registered event listeners for observability. */
+  private _eventListeners: Set<SyncEventListener> = new Set();
+
+  /** Entry in the pending push queue — a message CID with its local EventLog token. */
+  private _pendingPushCids: Map<string, {
+    did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
+    entries: { cid: string; localToken?: ProgressToken }[];
+  }> = new Map();
+
+  /**
+   * CIDs recently received via pull subscription, keyed by `cid|dwnUrl` to
+   * scope suppression per remote endpoint. A message pulled from Provider A
+   * is only suppressed for push back to Provider A — it still fans out to
+   * Provider B and C. TTL: 60 seconds. Cap: 10,000 entries.
+   */
+  private _recentlyPulledCids: Map<string, number> = new Map();
+
+  /** TTL for echo-loop suppression entries (60 seconds). */
+  private static readonly ECHO_SUPPRESS_TTL_MS = 60_000;
+
+  /**
+   * Per-tenant closure evaluation contexts for the current live sync session.
+   * Caches ProtocolsConfigure and grant lookups across events for the same
+   * tenant. Keyed by tenantDid to prevent cross-tenant cache pollution.
+   */
+  private _closureContexts: Map<string, ClosureEvaluationContext> = new Map();
+
+  /** Maximum entries in the echo-loop suppression cache. */
+  private static readonly ECHO_SUPPRESS_MAX_ENTRIES = 10_000;
 
   /** Count of consecutive SMT sync failures (for backoff in poll mode). */
   private _consecutiveFailures = 0;
@@ -132,6 +270,14 @@ export class SyncEngineLevel implements SyncEngine {
     this._db = (db) ? db : new Level<string, string>(dataPath ?? 'DATA/AGENT/SYNC_STORE');
   }
 
+  /** Lazy accessor for the replication ledger. */
+  private get ledger(): ReplicationLedger {
+    if (!this._ledger) {
+      this._ledger = new ReplicationLedger(this._db);
+    }
+    return this._ledger;
+  }
+
   /**
    * Retrieves the `EnboxPlatformAgent` execution context.
    *
@@ -152,7 +298,39 @@ export class SyncEngineLevel implements SyncEngine {
   }
 
   get connectivityState(): SyncConnectivityState {
-    return this._connectivityState;
+    // Aggregate per-link connectivity: if any link is online, report online.
+    // If all are offline, report offline. If all unknown, report unknown.
+    // Falls back to the global _connectivityState for poll-mode (no active links).
+    if (this._activeLinks.size === 0) {
+      return this._connectivityState;
+    }
+
+    let hasOnline = false;
+    let hasOffline = false;
+    for (const link of this._activeLinks.values()) {
+      if (link.connectivity === 'online') { hasOnline = true; }
+      if (link.connectivity === 'offline') { hasOffline = true; }
+    }
+
+    if (hasOnline) { return 'online'; }
+    if (hasOffline) { return 'offline'; }
+    return 'unknown';
+  }
+
+  public on(listener: SyncEventListener): () => void {
+    this._eventListeners.add(listener);
+    return (): void => { this._eventListeners.delete(listener); };
+  }
+
+  /** Emit a sync event to all registered listeners. */
+  private emitEvent(event: SyncEvent): void {
+    for (const listener of this._eventListeners) {
+      try {
+        listener(event);
+      } catch {
+        // Don't let listener errors propagate into sync engine logic.
+      }
+    }
   }
 
   public async clear(): Promise<void> {
@@ -440,14 +618,73 @@ export class SyncEngineLevel implements SyncEngine {
       console.error('SyncEngineLevel: Error during initial live-sync catch-up', error);
     }
 
-    // Step 2: Open live subscriptions for each sync target.
+    // Step 2: Initialize replication links and open live subscriptions.
     const syncTargets = await this.getSyncTargets();
     for (const target of syncTargets) {
+      let link: ReplicationLinkState | undefined;
       try {
+        // Get or create the link in the durable ledger.
+        // Use protocol-scoped scope when a protocol is specified, otherwise full-tenant.
+        const linkScope: SyncScope = target.protocol
+          ? { kind: 'protocol', protocol: target.protocol }
+          : { kind: 'full' };
+        link = await this.ledger.getOrCreateLink({
+          tenantDid      : target.did,
+          remoteEndpoint : target.dwnUrl,
+          scope          : linkScope,
+          delegateDid    : target.delegateDid,
+          protocol       : target.protocol,
+        });
+
+        // Cache the link for fast access by subscription handlers.
+        const linkKey = this.buildCursorKey(target.did, target.dwnUrl, target.protocol);
+        this._activeLinks.set(linkKey, link);
+
+        // Open subscriptions — only transition to live if both succeed.
+        // If pull succeeds but push fails, close the pull subscription to
+        // avoid a resource leak with inconsistent state.
         await this.openLivePullSubscription(target);
-        await this.openLocalPushSubscription(target);
+        try {
+          await this.openLocalPushSubscription(target);
+        } catch (pushError) {
+          // Close the already-opened pull subscription.
+          const pullSub = this._liveSubscriptions.find(
+            s => s.did === target.did && s.dwnUrl === target.dwnUrl && s.protocol === target.protocol
+          );
+          if (pullSub) {
+            try { await pullSub.close(); } catch { /* best effort */ }
+            this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+          }
+          throw pushError;
+        }
+
+        this.emitEvent({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, from: 'initializing', to: 'live' });
+        await this.ledger.setStatus(link!, 'live');
       } catch (error: any) {
+        const linkKey = this.buildCursorKey(target.did, target.dwnUrl, target.protocol);
+
+        // Detect ProgressGap (410) — the cursor is stale, link needs SMT repair.
+        if ((error as any).isProgressGap && link) {
+          console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
+          this.emitEvent({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, reason: 'ProgressGap' });
+          const gapInfo = (error as any).gapInfo;
+          await this.transitionToRepairing(linkKey, link, {
+            resumeToken: gapInfo?.latestAvailable,
+          });
+          continue;
+        }
+
         console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
+
+        // Clean up in-memory state for the failed link so it doesn't appear
+        // active to later code. The durable link remains at 'initializing'.
+        this._activeLinks.delete(linkKey);
+        this._linkRuntimes.delete(linkKey);
+
+        // Recompute connectivity — if no live subscriptions remain, reset to unknown.
+        if (this._liveSubscriptions.length === 0) {
+          this._connectivityState = 'unknown';
+        }
       }
     }
 
@@ -467,10 +704,409 @@ export class SyncEngineLevel implements SyncEngine {
     this._syncIntervalId = setInterval(integrityCheck, intervalMilliseconds);
   }
 
+  /**
+   * Get or create the runtime state for a link.
+   */
+  private getOrCreateRuntime(linkKey: string): LinkRuntimeState {
+    let rt = this._linkRuntimes.get(linkKey);
+    if (!rt) {
+      rt = { nextDeliveryOrdinal: 0, nextCommitOrdinal: 0, inflight: new Map() };
+      this._linkRuntimes.set(linkKey, rt);
+    }
+    return rt;
+  }
+
+  /**
+   * Drain contiguously committed ordinals from the runtime state, advancing
+    * the link's pull checkpoint for each drained entry. Returns the number of
+   * entries drained (0 if the next ordinal is not yet committed).
+   */
+  private drainCommittedPull(linkKey: string): number {
+    const rt = this._linkRuntimes.get(linkKey);
+    const link = this._activeLinks.get(linkKey);
+    if (!rt || !link) { return 0; }
+
+    let drained = 0;
+    while (true) {
+      const entry = rt.inflight.get(rt.nextCommitOrdinal);
+      if (!entry || !entry.committed) { break; }
+
+      // This ordinal is committed — advance the durable checkpoint.
+      ReplicationLedger.commitContiguousToken(link.pull, entry.token);
+      ReplicationLedger.setReceivedToken(link.pull, entry.token);
+      rt.inflight.delete(rt.nextCommitOrdinal);
+      rt.nextCommitOrdinal++;
+      drained++;
+      // Note: checkpoint:pull-advance event is emitted AFTER saveLink succeeds
+      // in the caller, not here. "Advanced" means durably persisted.
+    }
+
+    return drained;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Per-link repair and degraded-poll orchestration (Phase 2)
+  // ---------------------------------------------------------------------------
+
+  /** Maximum consecutive repair attempts before falling back to degraded_poll. */
+  private static readonly MAX_REPAIR_ATTEMPTS = 3;
+
+  /** Per-link degraded-poll interval timers. */
+  private _degradedPollTimers: Map<string, ReturnType<typeof setInterval>> = new Map();
+
+  /** Per-link repair attempt counters. */
+  private _repairAttempts: Map<string, number> = new Map();
+
+  /** Per-link active repair promises — prevents concurrent repair for the same link. */
+  private _activeRepairs: Map<string, Promise<void>> = new Map();
+
+  /** Per-link retry timers for failed repairs below max attempts. */
+  private _repairRetryTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
+
+  /** Backoff schedule for repair retries (milliseconds). */
+  private static readonly REPAIR_BACKOFF_MS = [1_000, 3_000, 10_000];
+
+  /**
+   * Per-link repair context — stores ProgressGap metadata for use during
+   * repair. The `resumeToken` (from `gapInfo.latestAvailable`) is used as
+   * the post-repair checkpoint so the reopened subscription replays from
+   * a valid boundary instead of starting live-only.
+   */
+  private _repairContext: Map<string, { resumeToken?: ProgressToken }> = new Map();
+
+  /**
+   * Central helper for transitioning a link to `repairing`. Encapsulates:
+   * - status change
+   * - optional gap context storage
+   * - repair kick-off with retry scheduling on failure
+   *
+   * All code paths that set `repairing` should go through this helper to
+   * guarantee a future retry path.
+   */
+  private async transitionToRepairing(
+    linkKey: string,
+    link: ReplicationLinkState,
+    options?: { resumeToken?: ProgressToken },
+  ): Promise<void> {
+    const prevStatus = link.status;
+    const prevConnectivity = link.connectivity;
+    link.connectivity = 'offline';
+    await this.ledger.setStatus(link, 'repairing');
+
+    this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
+    if (prevConnectivity !== 'offline') {
+      this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
+    }
+
+    if (options?.resumeToken) {
+      this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
+    }
+
+    // Clear runtime ordinals immediately — stale state must not linger
+    // across repair attempts.
+    const rt = this._linkRuntimes.get(linkKey);
+    if (rt) {
+      rt.inflight.clear();
+      rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
+    }
+
+    // Kick off repair with retry scheduling on failure.
+    void this.repairLink(linkKey).catch(() => {
+      this.scheduleRepairRetry(linkKey);
+    });
+  }
+
+  /**
+   * Schedule a retry for a failed repair. Uses exponential backoff.
+   * No-op if the link is already in `degraded_poll` (timer loop owns retries)
+   * or if a retry is already scheduled.
+   */
+  private scheduleRepairRetry(linkKey: string): void {
+    // Don't schedule if already in degraded_poll or retry pending.
+    const link = this._activeLinks.get(linkKey);
+    if (!link || link.status === 'degraded_poll') { return; }
+    if (this._repairRetryTimers.has(linkKey)) { return; }
+
+    // attempts is already post-increment from doRepairLink, so subtract 1
+    // for the backoff index: first failure (attempts=1) → backoff[0]=1s.
+    const attempts = this._repairAttempts.get(linkKey) ?? 1;
+    const backoff = SyncEngineLevel.REPAIR_BACKOFF_MS;
+    const delayMs = backoff[Math.min(attempts - 1, backoff.length - 1)];
+
+    const timerGeneration = this._syncGeneration;
+    const timer = setTimeout(async (): Promise<void> => {
+      this._repairRetryTimers.delete(linkKey);
+
+      // Bail if teardown occurred since this timer was scheduled.
+      if (this._syncGeneration !== timerGeneration) { return; }
+
+      // Verify link still exists and is still repairing.
+      const currentLink = this._activeLinks.get(linkKey);
+      if (!currentLink || currentLink.status !== 'repairing') { return; }
+
+      try {
+        await this.repairLink(linkKey);
+      } catch {
+        // repairLink handles max attempts → degraded_poll internally.
+        // If still below max, schedule another retry.
+        if (currentLink.status === 'repairing') {
+          this.scheduleRepairRetry(linkKey);
+        }
+      }
+    }, delayMs);
+
+    this._repairRetryTimers.set(linkKey, timer);
+  }
+
+  /**
+   * Repair a single link. Deduplicates concurrent calls via `_activeRepairs`.
+   * If repair is already running for this link, returns the existing promise.
+   */
+  private repairLink(linkKey: string): Promise<void> {
+    const existing = this._activeRepairs.get(linkKey);
+    if (existing) { return existing; }
+
+    const promise = this.doRepairLink(linkKey).finally(() => {
+      this._activeRepairs.delete(linkKey);
+    });
+    this._activeRepairs.set(linkKey, promise);
+    return promise;
+  }
+
+  /**
+   * Internal repair implementation. Runs SMT set reconciliation for a single
+   * link, then attempts to re-establish live subscriptions. If repair succeeds,
+   * transitions to `live`. If it fails, throws so callers (degraded_poll timer,
+   * startup) can handle retry scheduling.
+   */
+  private async doRepairLink(linkKey: string): Promise<void> {
+    const link = this._activeLinks.get(linkKey);
+    if (!link) { return; }
+
+    // Capture the sync generation at repair start. If teardown occurs during
+    // any await, the generation will have incremented and we bail before
+    // mutating state — preventing the race where repair continues after teardown.
+    const generation = this._syncGeneration;
+
+    const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, protocol } = link;
+
+    this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: (this._repairAttempts.get(linkKey) ?? 0) + 1 });
+    const attempts = (this._repairAttempts.get(linkKey) ?? 0) + 1;
+    this._repairAttempts.set(linkKey, attempts);
+
+    // Step 1: Close existing subscriptions FIRST to stop old events from
+    // mutating local state while repair runs.
+    await this.closeLinkSubscriptions(link);
+    if (this._syncGeneration !== generation) { return; } // Teardown occurred.
+
+    // Step 2: Clear runtime ordinals immediately — stale state must not
+    // persist across repair attempts (successful or failed).
+    const rt = this.getOrCreateRuntime(linkKey);
+    rt.inflight.clear();
+    rt.nextDeliveryOrdinal = 0;
+    rt.nextCommitOrdinal = 0;
+
+    try {
+      // Step 3: Run SMT reconciliation for this link.
+      const localRoot = await this.getLocalRoot(did, delegateDid, protocol);
+      if (this._syncGeneration !== generation) { return; }
+      const remoteRoot = await this.getRemoteRoot(did, dwnUrl, delegateDid, protocol);
+      if (this._syncGeneration !== generation) { return; }
+
+      if (localRoot !== remoteRoot) {
+        const diff = await this.diffWithRemote({ did, dwnUrl, delegateDid, protocol });
+        if (this._syncGeneration !== generation) { return; }
+
+        if (diff.onlyRemote.length > 0) {
+          const prefetched: (MessagesSyncDiffEntry & { message: GenericMessage })[] = [];
+          const needsFetchCids: string[] = [];
+          for (const entry of diff.onlyRemote) {
+            if (!entry.message || (entry.message.descriptor.interface === 'Records' &&
+                entry.message.descriptor.method === 'Write' &&
+                (entry.message.descriptor as any).dataCid && !entry.encodedData)) {
+              needsFetchCids.push(entry.messageCid);
+            } else {
+              prefetched.push(entry as MessagesSyncDiffEntry & { message: GenericMessage });
+            }
+          }
+          await this.pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids: needsFetchCids, prefetched });
+          if (this._syncGeneration !== generation) { return; }
+        }
+
+        if (diff.onlyLocal.length > 0) {
+          await this.pushMessages({ did, dwnUrl, delegateDid, protocol, messageCids: diff.onlyLocal });
+          if (this._syncGeneration !== generation) { return; }
+        }
+      }
+
+      // Step 4: Determine the post-repair resume token.
+      // - If repair was triggered by ProgressGap, use the stored resumeToken
+      //   (from gapInfo.latestAvailable) so the reopened subscription replays
+      //   from a valid boundary, closing the race window between SMT and resubscribe.
+      // - Otherwise, use the existing contiguousAppliedToken if still valid.
+      // - Push checkpoint is NOT reset during repair: push frontier tracks what
+      //   the local EventLog has delivered to the remote. SMT repair handles
+      //   pull-side convergence; push-side convergence is handled by the diff's
+      //   onlyLocal push. The push checkpoint remains the local authority.
+      const repairCtx = this._repairContext.get(linkKey);
+      const resumeToken = repairCtx?.resumeToken ?? link.pull.contiguousAppliedToken;
+      ReplicationLedger.resetCheckpoint(link.pull, resumeToken);
+      await this.ledger.saveLink(link);
+      if (this._syncGeneration !== generation) { return; }
+
+      // Step 5: Reopen subscriptions with the repaired checkpoints.
+      const target = { did, dwnUrl, delegateDid, protocol };
+      await this.openLivePullSubscription(target);
+      if (this._syncGeneration !== generation) { return; }
+      try {
+        await this.openLocalPushSubscription({
+          ...target,
+          pushCursor: link.push.contiguousAppliedToken,
+        });
+      } catch (pushError) {
+        const pullSub = this._liveSubscriptions.find(
+          s => s.did === did && s.dwnUrl === dwnUrl && s.protocol === protocol
+        );
+        if (pullSub) {
+          try { await pullSub.close(); } catch { /* best effort */ }
+          this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+        }
+        throw pushError;
+      }
+      if (this._syncGeneration !== generation) { return; }
+
+      // Step 6: Clean up repair context and transition to live.
+      this._repairContext.delete(linkKey);
+      this._repairAttempts.delete(linkKey);
+      const retryTimer = this._repairRetryTimers.get(linkKey);
+      if (retryTimer) { clearTimeout(retryTimer); this._repairRetryTimers.delete(linkKey); }
+      const prevRepairConnectivity = link.connectivity;
+      link.connectivity = 'online';
+      await this.ledger.setStatus(link, 'live');
+      this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
+      if (prevRepairConnectivity !== 'online') {
+        this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevRepairConnectivity, to: 'online' });
+      }
+      this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: 'repairing', to: 'live' });
+
+    } catch (error: any) {
+      // If teardown occurred during repair, don't retry or enter degraded_poll.
+      if (this._syncGeneration !== generation) { return; }
+
+      console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
+      this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: attempts, error: String(error.message ?? error) });
+
+      if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
+        console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
+        await this.enterDegradedPoll(linkKey);
+        return;
+      }
+
+      // Re-throw so callers (degraded_poll timer) can handle retry scheduling.
+      throw error;
+    }
+  }
+
+  /**
+   * Close pull and push subscriptions for a specific link.
+   */
+  private async closeLinkSubscriptions(link: ReplicationLinkState): Promise<void> {
+    const { tenantDid: did, remoteEndpoint: dwnUrl, protocol } = link;
+
+    // Close pull subscription.
+    const pullSub = this._liveSubscriptions.find(
+      s => s.did === did && s.dwnUrl === dwnUrl && s.protocol === protocol
+    );
+    if (pullSub) {
+      try { await pullSub.close(); } catch { /* best effort */ }
+      this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+    }
+
+    // Close local push subscription.
+    const pushSub = this._localSubscriptions.find(
+      s => s.did === did && s.dwnUrl === dwnUrl && s.protocol === protocol
+    );
+    if (pushSub) {
+      try { await pushSub.close(); } catch { /* best effort */ }
+      this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
+    }
+  }
+
+  /**
+   * Transition a link to `degraded_poll` and start a per-link polling timer.
+   * The timer runs SMT reconciliation at a reduced frequency (30s with jitter)
+   * and attempts to re-establish live subscriptions after each successful repair.
+   */
+  private async enterDegradedPoll(linkKey: string): Promise<void> {
+    const link = this._activeLinks.get(linkKey);
+    if (!link) { return; }
+    link.connectivity = 'offline';
+
+    const prevDegradedStatus = link.status;
+    await this.ledger.setStatus(link, 'degraded_poll');
+    this._repairAttempts.delete(linkKey);
+    this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevDegradedStatus, to: 'degraded_poll' });
+    this.emitEvent({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol });
+
+    // Clear any existing timer for this link.
+    const existing = this._degradedPollTimers.get(linkKey);
+    if (existing) { clearInterval(existing); }
+
+    // Schedule per-link polling with jitter (15-30 seconds).
+    const baseInterval = 15_000;
+    const jitter = Math.floor(Math.random() * 15_000);
+    const interval = baseInterval + jitter;
+
+    const pollGeneration = this._syncGeneration;
+    const timer = setInterval(async (): Promise<void> => {
+      // Bail if teardown occurred since this timer was created.
+      if (this._syncGeneration !== pollGeneration) {
+        clearInterval(timer);
+        this._degradedPollTimers.delete(linkKey);
+        return;
+      }
+
+      // If the link was transitioned out of degraded_poll externally (e.g.,
+      // by teardown or manual intervention), stop polling.
+      if (link.status !== 'degraded_poll') {
+        clearInterval(timer);
+        this._degradedPollTimers.delete(linkKey);
+        return;
+      }
+
+      try {
+        // Attempt repair. Reset attempt counter so repairLink doesn't
+        // immediately re-enter degraded_poll on failure.
+        this._repairAttempts.set(linkKey, 0);
+        await this.ledger.setStatus(link, 'repairing');
+        await this.repairLink(linkKey);
+
+        // If repairLink succeeded, link is now 'live' — stop polling.
+        if ((link.status as string) === 'live') {
+          clearInterval(timer);
+          this._degradedPollTimers.delete(linkKey);
+        }
+      } catch {
+        // Repair failed — restore degraded_poll status so the timer continues.
+        // This is critical: repairLink sets status to 'repairing' internally,
+        // and if we don't restore degraded_poll, the next tick would see
+        // status !== 'degraded_poll' and stop the timer permanently.
+        await this.ledger.setStatus(link, 'degraded_poll');
+      }
+    }, interval);
+
+    this._degradedPollTimers.set(linkKey, timer);
+  }
+
   /**
    * Tears down all live subscriptions and push listeners.
    */
   private async teardownLiveSync(): Promise<void> {
+    // Increment generation to invalidate all in-flight async operations
+    // (repairs, retry timers, degraded-poll ticks). Any async work that
+    // captured the previous generation will bail on its next checkpoint.
+    this._syncGeneration++;
+
     // Clear the push debounce timer.
     if (this._pushDebounceTimer) {
       clearTimeout(this._pushDebounceTimer);
@@ -499,6 +1135,26 @@ export class SyncEngineLevel implements SyncEngine {
       }
     }
     this._localSubscriptions = [];
+
+    // Clear degraded-poll timers and repair state.
+    for (const timer of this._degradedPollTimers.values()) {
+      clearInterval(timer);
+    }
+    this._degradedPollTimers.clear();
+    this._repairAttempts.clear();
+    this._activeRepairs.clear();
+    for (const timer of this._repairRetryTimers.values()) {
+      clearTimeout(timer);
+    }
+    this._repairRetryTimers.clear();
+    this._repairContext.clear();
+
+    // Clear closure evaluation contexts.
+    this._closureContexts.clear();
+
+    // Clear the in-memory link and runtime state.
+    this._activeLinks.clear();
+    this._linkRuntimes.clear();
   }
 
   // ---------------------------------------------------------------------------
@@ -514,12 +1170,24 @@ export class SyncEngineLevel implements SyncEngine {
   }): Promise<void> {
     const { did, delegateDid, dwnUrl, protocol } = target;
 
-    // Resolve the cursor from the last session (if any).
+    // Resolve the cursor from the link's pull checkpoint (preferred) or legacy storage.
     const cursorKey = this.buildCursorKey(did, dwnUrl, protocol);
-    const cursor = await this.getCursor(cursorKey);
+    const link = this._activeLinks.get(cursorKey);
+    const cursor = link?.pull.contiguousAppliedToken ?? await this.getCursor(cursorKey);
 
     // Build the MessagesSubscribe filters.
-    const filters = protocol ? [{ protocol }] : [];
+    // When the link has protocolPathPrefixes, include them in the filter so the
+    // EventLog delivers only matching events (server-side filtering). This replaces
+    // the less efficient agent-side isEventInScope filtering for the pull path.
+    // Note: only the first prefix is used as the MessagesFilter field because
+    // MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
+    // would need multiple filters (OR semantics) — for now we use the first one.
+    const protocolPathPrefix = link?.scope.kind === 'protocol'
+      ? link.scope.protocolPathPrefixes?.[0]
+      : undefined;
+    const filters = protocol
+      ? [{ protocol, ...(protocolPathPrefix ? { protocolPathPrefix } : {}) }]
+      : [];
 
     // Look up permission grant for MessagesSubscribe if using a delegate.
     // The unified scope matching in AgentPermissionsApi accepts a
@@ -538,16 +1206,88 @@ export class SyncEngineLevel implements SyncEngine {
     }
 
     // Define the subscription handler that processes incoming events.
+    // NOTE: The WebSocket client fires handlers without awaiting (fire-and-forget),
+    // so multiple handlers can be in-flight concurrently. The ordinal tracker
+    // ensures the checkpoint advances only when all earlier deliveries are committed.
     const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
       if (subMessage.type === 'eose') {
-        // End-of-stored-events — catch-up complete, persist cursor.
-        await this.setCursor(cursorKey, subMessage.cursor);
-        this._connectivityState = 'online';
+        // End-of-stored-events — catch-up complete.
+        if (link) {
+          // Guard: if the link transitioned to repairing while catch-up events
+          // were being processed, skip all mutations — repair owns the state now.
+          if (link.status !== 'live' && link.status !== 'initializing') {
+            return;
+          }
+
+          if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
+            console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
+            await this.transitionToRepairing(cursorKey, link);
+            return;
+          }
+          ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+          // Drain committed entries. Do NOT unconditionally advance to the
+          // EOSE cursor — earlier stored events may still be in-flight
+          // (handlers are fire-and-forget). The checkpoint advances only as
+          // far as the contiguous drain reaches.
+          this.drainCommittedPull(cursorKey);
+          await this.ledger.saveLink(link);
+        } else {
+          await this.setCursor(cursorKey, subMessage.cursor);
+        }
+        // Transport is reachable — set connectivity to online.
+        if (link) {
+          const prevEoseConnectivity = link.connectivity;
+          link.connectivity = 'online';
+          if (prevEoseConnectivity !== 'online') {
+            this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
+          }
+        } else {
+          this._connectivityState = 'online';
+        }
         return;
       }
 
       if (subMessage.type === 'event') {
         const event: MessageEvent = subMessage.event;
+
+        // Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
+        // skip all processing. Old subscription handlers may still fire after the
+        // link transitions — these events should be ignored entirely, not just
+        // skipped at the checkpoint level.
+        if (link && link.status !== 'live' && link.status !== 'initializing') {
+          return;
+        }
+
+        // Domain validation: reject tokens from a different stream/epoch.
+        if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
+          console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
+          await this.transitionToRepairing(cursorKey, link);
+          return;
+        }
+
+        // Subset scope filtering: if the link has protocolPath/contextId prefixes,
+        // skip events that don't match. This is agent-side filtering because
+        // MessagesSubscribe only supports protocol-level filtering today.
+        //
+        // Skipped events MUST advance contiguousAppliedToken — otherwise the
+        // link would replay the same filtered-out events indefinitely after
+        // reconnect/repair. This is safe because the event is intentionally
+        // excluded from this scope and doesn't need processing.
+        if (link && !isEventInScope(event.message, link.scope)) {
+          ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+          ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
+          await this.ledger.saveLink(link);
+          return;
+        }
+
+        // Assign a delivery ordinal BEFORE async processing begins.
+        // This captures the delivery order even if processing completes out of order.
+        const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
+        const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
+        if (rt) {
+          rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
+        }
+
         try {
           // Extract inline data from the event (available for records <= 30 KB).
           let dataStream = this.extractDataStream(event);
@@ -569,12 +1309,97 @@ export class SyncEngineLevel implements SyncEngine {
 
           await this.agent.dwn.processRawMessage(did, event.message, { dataStream });
 
-          // Only advance the cursor after successful processing.
-          // If processing fails, the event will be re-delivered on
-          // reconnection (cursor-based resume from the last good point).
-          await this.setCursor(cursorKey, subMessage.cursor);
+          // Invalidate closure cache entries that may be affected by this message.
+          // Must run before closure validation so subsequent evaluations in the
+          // same session see the updated local state.
+          const closureCtxForInvalidation = this._closureContexts.get(did);
+          if (closureCtxForInvalidation) {
+            invalidateClosureCache(closureCtxForInvalidation, event.message);
+          }
+
+          // Closure validation for scoped subset sync (Phase 3).
+          // For protocol-scoped links, verify that all hard dependencies for
+          // this operation are locally present before considering it committed.
+          // Full-tenant scope bypasses this entirely (returns complete with 0 queries).
+          if (link && link.scope.kind === 'protocol') {
+            const messageStore = this.agent.dwn.node.storage.messageStore;
+            let closureCtx = this._closureContexts.get(did);
+            if (!closureCtx) {
+              closureCtx = createClosureContext(did);
+              this._closureContexts.set(did, closureCtx);
+            }
+
+            const closureResult = await evaluateClosure(
+              event.message, messageStore, link.scope, closureCtx
+            );
+
+            if (!closureResult.complete) {
+              console.warn(
+                `SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
+                `${closureResult.failure!.code} — ${closureResult.failure!.detail}`
+              );
+              await this.transitionToRepairing(cursorKey, link);
+              return;
+            }
+          }
+
+          // Squash convergence: processRawMessage triggers the DWN's built-in
+          // squash resumable task (performRecordsSquash) which runs inline and
+          // handles subset consumers correctly:
+          // - If older siblings are locally present → purges them
+          // - If squash arrives before older siblings → backstop rejects them (409)
+          // - If no older siblings are local → no-op (correct)
+          // Both sync orderings (squash-first or siblings-first) converge to
+          // the same final state. No additional sync-engine side-effect is needed.
+
+          // Track this CID for echo-loop suppression, scoped to the source endpoint.
+          const pulledCid = await Message.getCid(event.message);
+          this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
+          this.evictExpiredEchoEntries();
+
+          // Mark this ordinal as committed and drain the checkpoint.
+          // Guard: if the link transitioned to repairing while this handler was
+          // in-flight (e.g., an earlier ordinal's handler failed concurrently),
+          // skip all state mutations — the repair process owns progression now.
+          if (link && rt && link.status === 'live') {
+            const entry = rt.inflight.get(ordinal);
+            if (entry) { entry.committed = true; }
+
+            ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+            const drained = this.drainCommittedPull(cursorKey);
+            if (drained > 0) {
+              await this.ledger.saveLink(link);
+              // Emit after durable save — "advanced" means persisted.
+              if (link.pull.contiguousAppliedToken) {
+                this.emitEvent({
+                  type           : 'checkpoint:pull-advance',
+                  tenantDid      : link.tenantDid,
+                  remoteEndpoint : link.remoteEndpoint,
+                  protocol       : link.protocol,
+                  position       : link.pull.contiguousAppliedToken.position,
+                  messageCid     : link.pull.contiguousAppliedToken.messageCid,
+                });
+              }
+            }
+
+            // Overflow: too many in-flight ordinals without draining.
+            if (rt.inflight.size > MAX_PENDING_TOKENS) {
+              console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
+              await this.transitionToRepairing(cursorKey, link);
+            }
+          } else if (!link) {
+            // Legacy path: no link available, use simple cursor persistence.
+            await this.setCursor(cursorKey, subMessage.cursor);
+          }
         } catch (error: any) {
           console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
+          // A failed processRawMessage means local state is incomplete.
+          // Transition to repairing immediately — do NOT advance the checkpoint
+          // past this failure or let later ordinals commit past it. SMT
+          // reconciliation will discover and fill the gap.
+          if (link) {
+            await this.transitionToRepairing(cursorKey, link);
+          }
         }
       }
     };
@@ -599,10 +1424,12 @@ export class SyncEngineLevel implements SyncEngine {
 
     // Build a resubscribe factory so the WebSocket client can resume with
     // a fresh cursor-stamped message after reconnection.
-    const resubscribeFactory: ResubscribeFactory = async (resumeCursor?: string) => {
+    const resubscribeFactory: ResubscribeFactory = async (resumeCursor?: ProgressToken) => {
+      // On reconnect, use the latest durable checkpoint position if available.
+      const effectiveCursor = resumeCursor ?? link?.pull.contiguousAppliedToken ?? cursor;
       const resumeRequest = {
         ...subscribeRequest,
-        messageParams: { ...subscribeRequest.messageParams, cursor: resumeCursor ?? cursor },
+        messageParams: { ...subscribeRequest.messageParams, cursor: effectiveCursor },
       };
       const { message: resumeMsg } = await this.agent.dwn.processRequest(resumeRequest);
       if (!resumeMsg) {
@@ -625,9 +1452,15 @@ export class SyncEngineLevel implements SyncEngine {
         resubscribeFactory,
       },
     }) as MessagesSubscribeReply;
+    if (reply.status.code === 410) {
+      // ProgressGap — the cursor is no longer replayable. The link needs repair.
+      const gapError = new Error(`SyncEngineLevel: ProgressGap for ${did} -> ${dwnUrl}: ${reply.status.detail}`);
+      (gapError as any).isProgressGap = true;
+      (gapError as any).gapInfo = reply.error;
+      throw gapError;
+    }
     if (reply.status.code !== 200 || !reply.subscription) {
-      console.error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
-      return;
+      throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
     }
 
     this._liveSubscriptions.push({
@@ -638,7 +1471,15 @@ export class SyncEngineLevel implements SyncEngine {
       close: async (): Promise<void> => { await reply.subscription!.close(); },
     });
 
-    this._connectivityState = 'online';
+    // Set per-link connectivity to online after successful subscription setup.
+    const pullLink = this._activeLinks.get(this.buildCursorKey(did, dwnUrl, protocol));
+    if (pullLink) {
+      const prevPullConnectivity = pullLink.connectivity;
+      pullLink.connectivity = 'online';
+      if (prevPullConnectivity !== 'online') {
+        this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevPullConnectivity, to: 'online' });
+      }
+    }
   }
 
   // ---------------------------------------------------------------------------
@@ -651,6 +1492,7 @@ export class SyncEngineLevel implements SyncEngine {
    */
   private async openLocalPushSubscription(target: {
     did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
+    pushCursor?: ProgressToken;
   }): Promise<void> {
     const { did, delegateDid, dwnUrl, protocol } = target;
 
@@ -676,6 +1518,33 @@ export class SyncEngineLevel implements SyncEngine {
         return;
       }
 
+      // Subset scope filtering for push: only push events that match the
+      // link's scope prefixes. Events outside the scope are not our responsibility.
+      // Skipped events MUST advance the push checkpoint to prevent infinite
+      // replay after repair/reconnect (same reason as the pull side).
+      const pushLink = this._activeLinks.get(this.buildCursorKey(did, dwnUrl, protocol));
+      if (pushLink && !isEventInScope(subMessage.event.message, pushLink.scope)) {
+        // Guard: only mutate durable state when the link is live/initializing.
+        // During repair/degraded_poll, orchestration owns checkpoint progression.
+        if (pushLink.status !== 'live' && pushLink.status !== 'initializing') {
+          return;
+        }
+
+        // Validate token domain before committing — a stream/epoch mismatch
+        // on the local EventLog should trigger repair, not silently overwrite.
+        if (!ReplicationLedger.validateTokenDomain(pushLink.push, subMessage.cursor)) {
+          await this.transitionToRepairing(
+            this.buildCursorKey(did, dwnUrl, protocol), pushLink
+          );
+          return;
+        }
+
+        ReplicationLedger.setReceivedToken(pushLink.push, subMessage.cursor);
+        ReplicationLedger.commitContiguousToken(pushLink.push, subMessage.cursor);
+        await this.ledger.saveLink(pushLink);
+        return;
+      }
+
       // Accumulate the message CID for a debounced push.
       const targetKey = this.buildCursorKey(did, dwnUrl, protocol);
       const cid = await Message.getCid(subMessage.event.message);
@@ -683,12 +1552,19 @@ export class SyncEngineLevel implements SyncEngine {
         return;
       }
 
+      // Echo-loop suppression: skip CIDs that were recently pulled from this
+      // specific remote. A message pulled from Provider A is only suppressed
+      // for push to A — it still fans out to Provider B and C.
+      if (this.isRecentlyPulled(cid, dwnUrl)) {
+        return;
+      }
+
       let pending = this._pendingPushCids.get(targetKey);
       if (!pending) {
-        pending = { did, dwnUrl, delegateDid, protocol, cids: [] };
+        pending = { did, dwnUrl, delegateDid, protocol, entries: [] };
         this._pendingPushCids.set(targetKey, pending);
       }
-      pending.cids.push(cid);
+      pending.entries.push({ cid, localToken: subMessage.cursor });
 
       // Debounce the push.
       if (this._pushDebounceTimer) {
@@ -700,19 +1576,21 @@ export class SyncEngineLevel implements SyncEngine {
     };
 
     // Process the local subscription request.
+    // When a push cursor is provided (e.g., after repair), the local subscription
+    // replays events from that position, closing the race window where local
+    // writes during repair would otherwise be missed by push-on-write.
     const response = await this.agent.dwn.processRequest({
       author              : did,
       target              : did,
       messageType         : DwnInterface.MessagesSubscribe,
       granteeDid          : delegateDid,
-      messageParams       : { filters, permissionGrantId },
+      messageParams       : { filters, permissionGrantId, cursor: target.pushCursor },
       subscriptionHandler : subscriptionHandler as any,
     });
 
     const reply = response.reply as MessagesSubscribeReply;
     if (reply.status.code !== 200 || !reply.subscription) {
-      console.error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
-      return;
+      throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
     }
 
     this._localSubscriptions.push({
@@ -730,41 +1608,88 @@ export class SyncEngineLevel implements SyncEngine {
   private async flushPendingPushes(): Promise<void> {
     this._pushDebounceTimer = undefined;
 
-    const entries = [...this._pendingPushCids.entries()];
+    const batches = [...this._pendingPushCids.entries()];
     this._pendingPushCids.clear();
 
     // Push to all endpoints in parallel — each target is independent.
-    await Promise.all(entries.map(async ([, pending]) => {
-      const { did, dwnUrl, delegateDid, protocol, cids } = pending;
-      if (cids.length === 0) {
+    await Promise.all(batches.map(async ([targetKey, pending]) => {
+      const { did, dwnUrl, delegateDid, protocol, entries: pushEntries } = pending;
+      if (pushEntries.length === 0) {
         return;
       }
 
+      const cids = pushEntries.map(e => e.cid);
+
       try {
-        await pushMessages({
+        const result = await pushMessages({
           did, dwnUrl, delegateDid, protocol,
           messageCids    : cids,
           agent          : this.agent,
           permissionsApi : this._permissionsApi,
         });
+
+        // Advance the push checkpoint for successfully pushed entries.
+        // Push is sequential (single batch, in-order processing) so we can
+        // commit directly without ordinal tracking — there's no concurrent
+        // completion to reorder.
+        const link = this._activeLinks.get(targetKey);
+        if (link) {
+          const succeededSet = new Set(result.succeeded);
+          // Track highest contiguous success: if a CID fails, we stop advancing.
+          let hitFailure = false;
+          for (const entry of pushEntries) {
+            if (hitFailure) { break; }
+            if (succeededSet.has(entry.cid) && entry.localToken) {
+              if (!ReplicationLedger.validateTokenDomain(link.push, entry.localToken)) {
+                console.warn(`SyncEngineLevel: Push checkpoint domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
+                await this.transitionToRepairing(targetKey, link);
+                break;
+              }
+              ReplicationLedger.setReceivedToken(link.push, entry.localToken);
+              ReplicationLedger.commitContiguousToken(link.push, entry.localToken);
+            } else {
+              // This CID failed or had no token — stop advancing.
+              hitFailure = true;
+            }
+          }
+          await this.ledger.saveLink(link);
+        }
+
+        // Re-queue failed entries so they are retried on the next debounce
+        // cycle (or picked up by the SMT integrity check).
+        if (result.failed.length > 0) {
+          console.error(`SyncEngineLevel: Push-on-write failed for ${did} -> ${dwnUrl}: ${result.failed.length} of ${cids.length} messages failed`);
+          const failedSet = new Set(result.failed);
+          const failedEntries = pushEntries.filter(e => failedSet.has(e.cid));
+          let requeued = this._pendingPushCids.get(targetKey);
+          if (!requeued) {
+            requeued = { did, dwnUrl, delegateDid, protocol, entries: [] };
+            this._pendingPushCids.set(targetKey, requeued);
+          }
+          requeued.entries.push(...failedEntries);
+
+          // Schedule a retry after a short delay.
+          if (!this._pushDebounceTimer) {
+            this._pushDebounceTimer = setTimeout((): void => {
+              void this.flushPendingPushes();
+            }, PUSH_DEBOUNCE_MS * 4); // Back off: 1 second instead of 250ms.
+          }
+        }
       } catch (error: any) {
+        // Truly unexpected error (not per-message failure). Re-queue entire
+        // batch so entries aren't silently dropped from the debounce queue.
         console.error(`SyncEngineLevel: Push-on-write failed for ${did} -> ${dwnUrl}`, error);
-
-        // Re-queue the failed CIDs so they are retried on the next
-        // debounce cycle (or picked up by the SMT integrity check).
-        const targetKey = this.buildCursorKey(did, dwnUrl, protocol);
         let requeued = this._pendingPushCids.get(targetKey);
         if (!requeued) {
-          requeued = { did, dwnUrl, delegateDid, protocol, cids: [] };
+          requeued = { did, dwnUrl, delegateDid, protocol, entries: [] };
           this._pendingPushCids.set(targetKey, requeued);
         }
-        requeued.cids.push(...cids);
+        requeued.entries.push(...pushEntries);
 
-        // Schedule a retry after a short delay.
         if (!this._pushDebounceTimer) {
           this._pushDebounceTimer = setTimeout((): void => {
             void this.flushPendingPushes();
-          }, PUSH_DEBOUNCE_MS * 4); // Back off: 1 second instead of 250ms.
+          }, PUSH_DEBOUNCE_MS * 4);
         }
       }
     }));
@@ -779,10 +1704,29 @@ export class SyncEngineLevel implements SyncEngine {
     return protocol ? `${base}${CURSOR_SEPARATOR}${protocol}` : base;
   }
 
-  private async getCursor(key: string): Promise<string | undefined> {
+  /**
+   * Retrieves a stored progress token. Handles migration from old string cursors:
+   * if the stored value is a bare string (pre-ProgressToken format), it is treated
+   * as absent — the sync engine will do a full SMT reconciliation on first startup
+   * after upgrade, which is correct and safe.
+   */
+  private async getCursor(key: string): Promise<ProgressToken | undefined> {
     const cursors = this._db.sublevel('syncCursors');
     try {
-      return await cursors.get(key);
+      const raw = await cursors.get(key);
+      try {
+        const parsed = JSON.parse(raw);
+        if (parsed && typeof parsed === 'object' &&
+            typeof parsed.streamId === 'string' &&
+            typeof parsed.epoch === 'string' &&
+            typeof parsed.position === 'string' &&
+            typeof parsed.messageCid === 'string') {
+          return parsed as ProgressToken;
+        }
+      } catch {
+        // Not valid JSON (old string cursor) — treat as absent.
+      }
+      return undefined;
     } catch (error) {
       const e = error as { code: string };
       if (e.code === 'LEVEL_NOT_FOUND') {
@@ -792,9 +1736,9 @@ export class SyncEngineLevel implements SyncEngine {
     }
   }
 
-  private async setCursor(key: string, cursor: string): Promise<void> {
+  private async setCursor(key: string, cursor: ProgressToken): Promise<void> {
     const cursors = this._db.sublevel('syncCursors');
-    await cursors.put(key, cursor);
+    await cursors.put(key, JSON.stringify(cursor));
   }
 
   // ---------------------------------------------------------------------------
@@ -1164,6 +2108,52 @@ export class SyncEngineLevel implements SyncEngine {
     });
   }
 
+  // ---------------------------------------------------------------------------
+  // Echo-loop suppression
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Evicts expired entries from the echo-loop suppression cache.
+   * Also enforces the size cap by evicting oldest entries first.
+   */
+  private evictExpiredEchoEntries(): void {
+    const now = Date.now();
+
+    // Evict expired entries.
+    for (const [cid, expiry] of this._recentlyPulledCids) {
+      if (now >= expiry) {
+        this._recentlyPulledCids.delete(cid);
+      }
+    }
+
+    // Enforce size cap by evicting oldest entries.
+    if (this._recentlyPulledCids.size > SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES) {
+      const excess = this._recentlyPulledCids.size - SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES;
+      let evicted = 0;
+      for (const key of this._recentlyPulledCids.keys()) {
+        if (evicted >= excess) { break; }
+        this._recentlyPulledCids.delete(key);
+        evicted++;
+      }
+    }
+  }
+
+  /**
+   * Checks whether a CID was recently pulled from a specific remote endpoint
+   * and should not be pushed back to that same endpoint (echo-loop suppression).
+   * Does not suppress pushes to other endpoints — multi-provider fan-out works.
+   */
+  private isRecentlyPulled(cid: string, dwnUrl: string): boolean {
+    const key = `${cid}|${dwnUrl}`;
+    const expiry = this._recentlyPulledCids.get(key);
+    if (expiry === undefined) { return false; }
+    if (Date.now() >= expiry) {
+      this._recentlyPulledCids.delete(key);
+      return false;
+    }
+    return true;
+  }
+
   /**
    * Reads missing messages from the local DWN and pushes them to the remote DWN
    * in dependency order (topological sort).
@@ -1174,7 +2164,7 @@ export class SyncEngineLevel implements SyncEngine {
     delegateDid?: string;
     protocol?: string;
     messageCids: string[];
-  }): Promise<void> {
+  }): Promise<PushResult> {
     return pushMessages({
       did, dwnUrl, delegateDid, protocol, messageCids,
       agent          : this.agent,