@enbox/agent 0.5.9 → 0.5.11

package/dist/esm/sync-engine-level.js

@@ -17,6 +17,10 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
 import ms from 'ms';
 import { Level } from 'level';
 import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
+import { evaluateClosure } from './sync-closure-resolver.js';
+import { MAX_PENDING_TOKENS } from './types/sync.js';
+import { ReplicationLedger } from './sync-replication-ledger.js';
+import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
 import { AgentPermissionsApi } from './permissions-api.js';
 import { DwnInterface } from './types/dwn.js';
 import { isRecordsWrite } from './utils.js';
@@ -36,55 +40,6 @@ const MAX_DIFF_DEPTH = 16;
  */
 const BATCHED_DIFF_DEPTH = 8;
 /**
- * Maximum number of concurrent remote HTTP requests during a tree diff.
- * The binary tree walk fans out in parallel — without a limit, depth N
- * produces 2^N concurrent requests, which can exhaust server rate limits.
- */
-const REMOTE_CONCURRENCY = 4;
-/**
- * Counting semaphore for bounding concurrent async operations.
- * Used by the tree walk to limit in-flight remote HTTP requests.
- */
-class Semaphore {
-    constructor(permits) {
-        this._waiting = [];
-        this._permits = permits;
-    }
-    /** Wait until a permit is available, then consume one. */
-    acquire() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (this._permits > 0) {
-                this._permits--;
-                return;
-            }
-            return new Promise((resolve) => {
-                this._waiting.push(resolve);
-            });
-        });
-    }
-    /** Release a permit, waking the next waiter if any. */
-    release() {
-        const next = this._waiting.shift();
-        if (next) {
-            next();
-        }
-        else {
-            this._permits++;
-        }
-    }
-    /** Acquire a permit, run the task, then release regardless of outcome. */
-    run(fn) {
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.acquire();
-            try {
-                return yield fn();
-            }
-            finally {
-                this.release();
-            }
-        });
-    }
-}
 /**
  * Key for the subscription cursor sublevel. Cursors are keyed by
  * `{did}^{dwnUrl}[^{protocol}]` and store an opaque EventLog cursor string.
@@ -95,28 +50,130 @@ const CURSOR_SEPARATOR = '^';
  * we batch them and push after this delay to avoid a push per individual write.
  */
 const PUSH_DEBOUNCE_MS = 250;
+/**
+ * Checks whether a message's protocolPath and contextId match the link's
+ * subset scope prefixes. Returns true if the message is in scope.
+ *
+ * When the scope has no prefixes (or is kind:'full'), all messages match.
+ * When protocolPathPrefixes or contextIdPrefixes are specified, the message
+ * must match at least one prefix in each specified set.
+ *
+ * This is agent-side filtering for subset scopes. The underlying
+ * MessagesSubscribe filter only supports protocol-level scoping today —
+ * protocolPath/contextId prefix filtering at the EventLog level is a
+ * follow-up (requires dwn-sdk-js MessagesFilter extension).
+ */
+function isEventInScope(message, scope) {
+    if (scope.kind === 'full') {
+        return true;
+    }
+    if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) {
+        return true;
+    }
+    const desc = message.descriptor;
+    // Check protocolPath prefix.
+    if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
+        const protocolPath = desc.protocolPath;
+        if (!protocolPath) {
+            return false;
+        }
+        const matches = scope.protocolPathPrefixes.some(prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/'));
+        if (!matches) {
+            return false;
+        }
+    }
+    // Check contextId prefix.
+    if (scope.contextIdPrefixes && scope.contextIdPrefixes.length > 0) {
+        const contextId = message.contextId;
+        if (!contextId) {
+            return false;
+        }
+        const matches = scope.contextIdPrefixes.some(prefix => contextId === prefix || contextId.startsWith(prefix + '/'));
+        if (!matches) {
+            return false;
+        }
+    }
+    return true;
+}
 export class SyncEngineLevel {
     constructor({ agent, dataPath, db }) {
         this._syncLock = false;
+        /**
+         * In-memory cache of active links, keyed by `{did}^{dwnUrl}^{protocol}`.
+         * Populated from the ledger on `startLiveSync`, used by subscription handlers
+         * to avoid async ledger lookups on every event.
+         */
+        this._activeLinks = new Map();
+        /**
+         * Per-link in-memory delivery-order tracking for the pull path. Keyed by
+         * the same link key as `_activeLinks`. Not persisted — on crash, replay
+         * restarts from `contiguousAppliedToken` and idempotent apply handles
+         * re-delivered events.
+         */
+        this._linkRuntimes = new Map();
         // ---------------------------------------------------------------------------
         // Live sync state
         // ---------------------------------------------------------------------------
         /** Current sync mode, set by `startSync`. */
         this._syncMode = 'poll';
+        /**
+         * Monotonic session generation counter. Incremented on every teardown.
+         * Async operations (repair, retry timers) capture the generation at start
+         * and bail if it has changed — this prevents stale work from mutating
+         * state after teardown or mode switch.
+         */
+        this._syncGeneration = 0;
         /** Active live pull subscriptions (remote -> local via MessagesSubscribe). */
         this._liveSubscriptions = [];
         /** Active local EventLog subscriptions for push-on-write (local -> remote). */
         this._localSubscriptions = [];
         /** Connectivity state derived from subscription health. */
         this._connectivityState = 'unknown';
-        /** Pending message CIDs to push, accumulated during the debounce window. */
+        /** Registered event listeners for observability. */
+        this._eventListeners = new Set();
+        /** Entry in the pending push queue — a message CID with its local EventLog token. */
         this._pendingPushCids = new Map();
+        /**
+         * CIDs recently received via pull subscription, keyed by `cid|dwnUrl` to
+         * scope suppression per remote endpoint. A message pulled from Provider A
+         * is only suppressed for push back to Provider A — it still fans out to
+         * Provider B and C. TTL: 60 seconds. Cap: 10,000 entries.
+         */
+        this._recentlyPulledCids = new Map();
+        /**
+         * Per-tenant closure evaluation contexts for the current live sync session.
+         * Caches ProtocolsConfigure and grant lookups across events for the same
+         * tenant. Keyed by tenantDid to prevent cross-tenant cache pollution.
+         */
+        this._closureContexts = new Map();
         /** Count of consecutive SMT sync failures (for backoff in poll mode). */
         this._consecutiveFailures = 0;
+        /** Per-link degraded-poll interval timers. */
+        this._degradedPollTimers = new Map();
+        /** Per-link repair attempt counters. */
+        this._repairAttempts = new Map();
+        /** Per-link active repair promises — prevents concurrent repair for the same link. */
+        this._activeRepairs = new Map();
+        /** Per-link retry timers for failed repairs below max attempts. */
+        this._repairRetryTimers = new Map();
+        /**
+         * Per-link repair context — stores ProgressGap metadata for use during
+         * repair. The `resumeToken` (from `gapInfo.latestAvailable`) is used as
+         * the post-repair checkpoint so the reopened subscription replays from
+         * a valid boundary instead of starting live-only.
+         */
+        this._repairContext = new Map();
         this._agent = agent;
         this._permissionsApi = new AgentPermissionsApi({ agent: agent });
         this._db = (db) ? db : new Level(dataPath !== null && dataPath !== void 0 ? dataPath : 'DATA/AGENT/SYNC_STORE');
     }
+    /** Lazy accessor for the replication ledger. */
+    get ledger() {
+        if (!this._ledger) {
+            this._ledger = new ReplicationLedger(this._db);
+        }
+        return this._ledger;
+    }
     /**
      * Retrieves the `EnboxPlatformAgent` execution context.
      *
@@ -134,7 +191,44 @@ export class SyncEngineLevel {
         this._permissionsApi = new AgentPermissionsApi({ agent: agent });
     }
     get connectivityState() {
-        return this._connectivityState;
+        // Aggregate per-link connectivity: if any link is online, report online.
+        // If all are offline, report offline. If all unknown, report unknown.
+        // Falls back to the global _connectivityState for poll-mode (no active links).
+        if (this._activeLinks.size === 0) {
+            return this._connectivityState;
+        }
+        let hasOnline = false;
+        let hasOffline = false;
+        for (const link of this._activeLinks.values()) {
+            if (link.connectivity === 'online') {
+                hasOnline = true;
+            }
+            if (link.connectivity === 'offline') {
+                hasOffline = true;
+            }
+        }
+        if (hasOnline) {
+            return 'online';
+        }
+        if (hasOffline) {
+            return 'offline';
+        }
+        return 'unknown';
+    }
+    on(listener) {
+        this._eventListeners.add(listener);
+        return () => { this._eventListeners.delete(listener); };
+    }
+    /** Emit a sync event to all registered listeners. */
+    emitEvent(event) {
+        for (const listener of this._eventListeners) {
+            try {
+                listener(event);
+            }
+            catch (_a) {
+                // Don't let listener errors propagate into sync engine logic.
+            }
+        }
     }
     clear() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -404,15 +498,69 @@ export class SyncEngineLevel {
             catch (error) {
                 console.error('SyncEngineLevel: Error during initial live-sync catch-up', error);
             }
-            // Step 2: Open live subscriptions for each sync target.
+            // Step 2: Initialize replication links and open live subscriptions.
             const syncTargets = yield this.getSyncTargets();
             for (const target of syncTargets) {
+                let link;
                 try {
+                    // Get or create the link in the durable ledger.
+                    // Use protocol-scoped scope when a protocol is specified, otherwise full-tenant.
+                    const linkScope = target.protocol
+                        ? { kind: 'protocol', protocol: target.protocol }
+                        : { kind: 'full' };
+                    link = yield this.ledger.getOrCreateLink({
+                        tenantDid: target.did,
+                        remoteEndpoint: target.dwnUrl,
+                        scope: linkScope,
+                        delegateDid: target.delegateDid,
+                        protocol: target.protocol,
+                    });
+                    // Cache the link for fast access by subscription handlers.
+                    const linkKey = this.buildCursorKey(target.did, target.dwnUrl, target.protocol);
+                    this._activeLinks.set(linkKey, link);
+                    // Open subscriptions — only transition to live if both succeed.
+                    // If pull succeeds but push fails, close the pull subscription to
+                    // avoid a resource leak with inconsistent state.
                     yield this.openLivePullSubscription(target);
-                    yield this.openLocalPushSubscription(target);
+                    try {
+                        yield this.openLocalPushSubscription(target);
+                    }
+                    catch (pushError) {
+                        // Close the already-opened pull subscription.
+                        const pullSub = this._liveSubscriptions.find(s => s.did === target.did && s.dwnUrl === target.dwnUrl && s.protocol === target.protocol);
+                        if (pullSub) {
+                            try {
+                                yield pullSub.close();
+                            }
+                            catch ( /* best effort */_a) { /* best effort */ }
+                            this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+                        }
+                        throw pushError;
+                    }
+                    this.emitEvent({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, from: 'initializing', to: 'live' });
+                    yield this.ledger.setStatus(link, 'live');
                 }
                 catch (error) {
+                    const linkKey = this.buildCursorKey(target.did, target.dwnUrl, target.protocol);
+                    // Detect ProgressGap (410) — the cursor is stale, link needs SMT repair.
+                    if (error.isProgressGap && link) {
+                        console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
+                        this.emitEvent({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, reason: 'ProgressGap' });
+                        const gapInfo = error.gapInfo;
+                        yield this.transitionToRepairing(linkKey, link, {
+                            resumeToken: gapInfo === null || gapInfo === void 0 ? void 0 : gapInfo.latestAvailable,
+                        });
+                        continue;
+                    }
                     console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
+                    // Clean up in-memory state for the failed link so it doesn't appear
+                    // active to later code. The durable link remains at 'initializing'.
+                    this._activeLinks.delete(linkKey);
+                    this._linkRuntimes.delete(linkKey);
+                    // Recompute connectivity — if no live subscriptions remain, reset to unknown.
+                    if (this._liveSubscriptions.length === 0) {
+                        this._connectivityState = 'unknown';
+                    }
                 }
             }
             // Step 3: Schedule infrequent SMT integrity check.
@@ -430,11 +578,385 @@ export class SyncEngineLevel {
             this._syncIntervalId = setInterval(integrityCheck, intervalMilliseconds);
         });
     }
+    /**
+     * Get or create the runtime state for a link.
+     */
+    getOrCreateRuntime(linkKey) {
+        let rt = this._linkRuntimes.get(linkKey);
+        if (!rt) {
+            rt = { nextDeliveryOrdinal: 0, nextCommitOrdinal: 0, inflight: new Map() };
+            this._linkRuntimes.set(linkKey, rt);
+        }
+        return rt;
+    }
+    /**
+     * Drain contiguously committed ordinals from the runtime state, advancing
+      * the link's pull checkpoint for each drained entry. Returns the number of
+     * entries drained (0 if the next ordinal is not yet committed).
+     */
+    drainCommittedPull(linkKey) {
+        const rt = this._linkRuntimes.get(linkKey);
+        const link = this._activeLinks.get(linkKey);
+        if (!rt || !link) {
+            return 0;
+        }
+        let drained = 0;
+        while (true) {
+            const entry = rt.inflight.get(rt.nextCommitOrdinal);
+            if (!entry || !entry.committed) {
+                break;
+            }
+            // This ordinal is committed — advance the durable checkpoint.
+            ReplicationLedger.commitContiguousToken(link.pull, entry.token);
+            ReplicationLedger.setReceivedToken(link.pull, entry.token);
+            rt.inflight.delete(rt.nextCommitOrdinal);
+            rt.nextCommitOrdinal++;
+            drained++;
+            // Note: checkpoint:pull-advance event is emitted AFTER saveLink succeeds
+            // in the caller, not here. "Advanced" means durably persisted.
+        }
+        return drained;
+    }
+    /**
+     * Central helper for transitioning a link to `repairing`. Encapsulates:
+     * - status change
+     * - optional gap context storage
+     * - repair kick-off with retry scheduling on failure
+     *
+     * All code paths that set `repairing` should go through this helper to
+     * guarantee a future retry path.
+     */
+    transitionToRepairing(linkKey, link, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const prevStatus = link.status;
+            const prevConnectivity = link.connectivity;
+            link.connectivity = 'offline';
+            yield this.ledger.setStatus(link, 'repairing');
+            this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
+            if (prevConnectivity !== 'offline') {
+                this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
+            }
+            if (options === null || options === void 0 ? void 0 : options.resumeToken) {
+                this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
+            }
+            // Clear runtime ordinals immediately — stale state must not linger
+            // across repair attempts.
+            const rt = this._linkRuntimes.get(linkKey);
+            if (rt) {
+                rt.inflight.clear();
+                rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
+            }
+            // Kick off repair with retry scheduling on failure.
+            void this.repairLink(linkKey).catch(() => {
+                this.scheduleRepairRetry(linkKey);
+            });
+        });
+    }
+    /**
+     * Schedule a retry for a failed repair. Uses exponential backoff.
+     * No-op if the link is already in `degraded_poll` (timer loop owns retries)
+     * or if a retry is already scheduled.
+     */
+    scheduleRepairRetry(linkKey) {
+        var _a;
+        // Don't schedule if already in degraded_poll or retry pending.
+        const link = this._activeLinks.get(linkKey);
+        if (!link || link.status === 'degraded_poll') {
+            return;
+        }
+        if (this._repairRetryTimers.has(linkKey)) {
+            return;
+        }
+        // attempts is already post-increment from doRepairLink, so subtract 1
+        // for the backoff index: first failure (attempts=1) → backoff[0]=1s.
+        const attempts = (_a = this._repairAttempts.get(linkKey)) !== null && _a !== void 0 ? _a : 1;
+        const backoff = SyncEngineLevel.REPAIR_BACKOFF_MS;
+        const delayMs = backoff[Math.min(attempts - 1, backoff.length - 1)];
+        const timerGeneration = this._syncGeneration;
+        const timer = setTimeout(() => __awaiter(this, void 0, void 0, function* () {
+            this._repairRetryTimers.delete(linkKey);
+            // Bail if teardown occurred since this timer was scheduled.
+            if (this._syncGeneration !== timerGeneration) {
+                return;
+            }
+            // Verify link still exists and is still repairing.
+            const currentLink = this._activeLinks.get(linkKey);
+            if (!currentLink || currentLink.status !== 'repairing') {
+                return;
+            }
+            try {
+                yield this.repairLink(linkKey);
+            }
+            catch (_a) {
+                // repairLink handles max attempts → degraded_poll internally.
+                // If still below max, schedule another retry.
+                if (currentLink.status === 'repairing') {
+                    this.scheduleRepairRetry(linkKey);
+                }
+            }
+        }), delayMs);
+        this._repairRetryTimers.set(linkKey, timer);
+    }
+    /**
+     * Repair a single link. Deduplicates concurrent calls via `_activeRepairs`.
+     * If repair is already running for this link, returns the existing promise.
+     */
+    repairLink(linkKey) {
+        const existing = this._activeRepairs.get(linkKey);
+        if (existing) {
+            return existing;
+        }
+        const promise = this.doRepairLink(linkKey).finally(() => {
+            this._activeRepairs.delete(linkKey);
+        });
+        this._activeRepairs.set(linkKey, promise);
+        return promise;
+    }
+    /**
+     * Internal repair implementation. Runs SMT set reconciliation for a single
+     * link, then attempts to re-establish live subscriptions. If repair succeeds,
+     * transitions to `live`. If it fails, throws so callers (degraded_poll timer,
+     * startup) can handle retry scheduling.
+     */
+    doRepairLink(linkKey) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c, _d;
+            const link = this._activeLinks.get(linkKey);
+            if (!link) {
+                return;
+            }
+            // Capture the sync generation at repair start. If teardown occurs during
+            // any await, the generation will have incremented and we bail before
+            // mutating state — preventing the race where repair continues after teardown.
+            const generation = this._syncGeneration;
+            const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, protocol } = link;
+            this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: ((_a = this._repairAttempts.get(linkKey)) !== null && _a !== void 0 ? _a : 0) + 1 });
+            const attempts = ((_b = this._repairAttempts.get(linkKey)) !== null && _b !== void 0 ? _b : 0) + 1;
+            this._repairAttempts.set(linkKey, attempts);
+            // Step 1: Close existing subscriptions FIRST to stop old events from
+            // mutating local state while repair runs.
+            yield this.closeLinkSubscriptions(link);
+            if (this._syncGeneration !== generation) {
+                return;
+            } // Teardown occurred.
+            // Step 2: Clear runtime ordinals immediately — stale state must not
+            // persist across repair attempts (successful or failed).
+            const rt = this.getOrCreateRuntime(linkKey);
+            rt.inflight.clear();
+            rt.nextDeliveryOrdinal = 0;
+            rt.nextCommitOrdinal = 0;
+            try {
+                // Step 3: Run SMT reconciliation for this link.
+                const localRoot = yield this.getLocalRoot(did, delegateDid, protocol);
+                if (this._syncGeneration !== generation) {
+                    return;
+                }
+                const remoteRoot = yield this.getRemoteRoot(did, dwnUrl, delegateDid, protocol);
+                if (this._syncGeneration !== generation) {
+                    return;
+                }
+                if (localRoot !== remoteRoot) {
+                    const diff = yield this.diffWithRemote({ did, dwnUrl, delegateDid, protocol });
+                    if (this._syncGeneration !== generation) {
+                        return;
+                    }
+                    if (diff.onlyRemote.length > 0) {
+                        const prefetched = [];
+                        const needsFetchCids = [];
+                        for (const entry of diff.onlyRemote) {
+                            if (!entry.message || (entry.message.descriptor.interface === 'Records' &&
+                                entry.message.descriptor.method === 'Write' &&
+                                entry.message.descriptor.dataCid && !entry.encodedData)) {
+                                needsFetchCids.push(entry.messageCid);
+                            }
+                            else {
+                                prefetched.push(entry);
+                            }
+                        }
+                        yield this.pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids: needsFetchCids, prefetched });
+                        if (this._syncGeneration !== generation) {
+                            return;
+                        }
+                    }
+                    if (diff.onlyLocal.length > 0) {
+                        yield this.pushMessages({ did, dwnUrl, delegateDid, protocol, messageCids: diff.onlyLocal });
+                        if (this._syncGeneration !== generation) {
+                            return;
+                        }
+                    }
+                }
+                // Step 4: Determine the post-repair resume token.
+                // - If repair was triggered by ProgressGap, use the stored resumeToken
+                //   (from gapInfo.latestAvailable) so the reopened subscription replays
+                //   from a valid boundary, closing the race window between SMT and resubscribe.
+                // - Otherwise, use the existing contiguousAppliedToken if still valid.
+                // - Push checkpoint is NOT reset during repair: push frontier tracks what
+                //   the local EventLog has delivered to the remote. SMT repair handles
+                //   pull-side convergence; push-side convergence is handled by the diff's
+                //   onlyLocal push. The push checkpoint remains the local authority.
+                const repairCtx = this._repairContext.get(linkKey);
+                const resumeToken = (_c = repairCtx === null || repairCtx === void 0 ? void 0 : repairCtx.resumeToken) !== null && _c !== void 0 ? _c : link.pull.contiguousAppliedToken;
+                ReplicationLedger.resetCheckpoint(link.pull, resumeToken);
+                yield this.ledger.saveLink(link);
+                if (this._syncGeneration !== generation) {
+                    return;
+                }
+                // Step 5: Reopen subscriptions with the repaired checkpoints.
+                const target = { did, dwnUrl, delegateDid, protocol };
+                yield this.openLivePullSubscription(target);
+                if (this._syncGeneration !== generation) {
+                    return;
+                }
+                try {
+                    yield this.openLocalPushSubscription(Object.assign(Object.assign({}, target), { pushCursor: link.push.contiguousAppliedToken }));
+                }
+                catch (pushError) {
+                    const pullSub = this._liveSubscriptions.find(s => s.did === did && s.dwnUrl === dwnUrl && s.protocol === protocol);
+                    if (pullSub) {
+                        try {
+                            yield pullSub.close();
+                        }
+                        catch ( /* best effort */_e) { /* best effort */ }
+                        this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+                    }
+                    throw pushError;
+                }
+                if (this._syncGeneration !== generation) {
+                    return;
+                }
+                // Step 6: Clean up repair context and transition to live.
+                this._repairContext.delete(linkKey);
+                this._repairAttempts.delete(linkKey);
+                const retryTimer = this._repairRetryTimers.get(linkKey);
+                if (retryTimer) {
+                    clearTimeout(retryTimer);
+                    this._repairRetryTimers.delete(linkKey);
+                }
+                const prevRepairConnectivity = link.connectivity;
+                link.connectivity = 'online';
+                yield this.ledger.setStatus(link, 'live');
+                this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
+                if (prevRepairConnectivity !== 'online') {
+                    this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevRepairConnectivity, to: 'online' });
+                }
+                this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: 'repairing', to: 'live' });
+            }
+            catch (error) {
+                // If teardown occurred during repair, don't retry or enter degraded_poll.
+                if (this._syncGeneration !== generation) {
+                    return;
+                }
+                console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
+                this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: attempts, error: String((_d = error.message) !== null && _d !== void 0 ? _d : error) });
+                if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
+                    console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
+                    yield this.enterDegradedPoll(linkKey);
+                    return;
+                }
+                // Re-throw so callers (degraded_poll timer) can handle retry scheduling.
+                throw error;
+            }
+        });
+    }
+    /**
+     * Close pull and push subscriptions for a specific link.
+     */
+    closeLinkSubscriptions(link) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { tenantDid: did, remoteEndpoint: dwnUrl, protocol } = link;
+            // Close pull subscription.
+            const pullSub = this._liveSubscriptions.find(s => s.did === did && s.dwnUrl === dwnUrl && s.protocol === protocol);
+            if (pullSub) {
+                try {
+                    yield pullSub.close();
+                }
+                catch ( /* best effort */_a) { /* best effort */ }
+                this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+            }
+            // Close local push subscription.
+            const pushSub = this._localSubscriptions.find(s => s.did === did && s.dwnUrl === dwnUrl && s.protocol === protocol);
+            if (pushSub) {
+                try {
+                    yield pushSub.close();
+                }
+                catch ( /* best effort */_b) { /* best effort */ }
+                this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
+            }
+        });
+    }
+    /**
+     * Transition a link to `degraded_poll` and start a per-link polling timer.
+     * The timer runs SMT reconciliation at a reduced frequency (30s with jitter)
+     * and attempts to re-establish live subscriptions after each successful repair.
+     */
+    enterDegradedPoll(linkKey) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const link = this._activeLinks.get(linkKey);
+            if (!link) {
+                return;
+            }
+            link.connectivity = 'offline';
+            const prevDegradedStatus = link.status;
+            yield this.ledger.setStatus(link, 'degraded_poll');
+            this._repairAttempts.delete(linkKey);
+            this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevDegradedStatus, to: 'degraded_poll' });
+            this.emitEvent({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol });
+            // Clear any existing timer for this link.
+            const existing = this._degradedPollTimers.get(linkKey);
+            if (existing) {
+                clearInterval(existing);
+            }
+            // Schedule per-link polling with jitter (15-30 seconds).
+            const baseInterval = 15000;
+            const jitter = Math.floor(Math.random() * 15000);
+            const interval = baseInterval + jitter;
+            const pollGeneration = this._syncGeneration;
+            const timer = setInterval(() => __awaiter(this, void 0, void 0, function* () {
+                // Bail if teardown occurred since this timer was created.
+                if (this._syncGeneration !== pollGeneration) {
+                    clearInterval(timer);
+                    this._degradedPollTimers.delete(linkKey);
+                    return;
+                }
+                // If the link was transitioned out of degraded_poll externally (e.g.,
+                // by teardown or manual intervention), stop polling.
+                if (link.status !== 'degraded_poll') {
+                    clearInterval(timer);
+                    this._degradedPollTimers.delete(linkKey);
+                    return;
+                }
+                try {
+                    // Attempt repair. Reset attempt counter so repairLink doesn't
+                    // immediately re-enter degraded_poll on failure.
+                    this._repairAttempts.set(linkKey, 0);
+                    yield this.ledger.setStatus(link, 'repairing');
+                    yield this.repairLink(linkKey);
+                    // If repairLink succeeded, link is now 'live' — stop polling.
+                    if (link.status === 'live') {
+                        clearInterval(timer);
+                        this._degradedPollTimers.delete(linkKey);
+                    }
+                }
+                catch (_a) {
+                    // Repair failed — restore degraded_poll status so the timer continues.
+                    // This is critical: repairLink sets status to 'repairing' internally,
+                    // and if we don't restore degraded_poll, the next tick would see
+                    // status !== 'degraded_poll' and stop the timer permanently.
+                    yield this.ledger.setStatus(link, 'degraded_poll');
+                }
+            }), interval);
+            this._degradedPollTimers.set(linkKey, timer);
+        });
+    }
     /**
      * Tears down all live subscriptions and push listeners.
      */
     teardownLiveSync() {
         return __awaiter(this, void 0, void 0, function* () {
+            // Increment generation to invalidate all in-flight async operations
+            // (repairs, retry timers, degraded-poll ticks). Any async work that
+            // captured the previous generation will bail on its next checkpoint.
+            this._syncGeneration++;
             // Clear the push debounce timer.
             if (this._pushDebounceTimer) {
                 clearTimeout(this._pushDebounceTimer);
@@ -462,6 +984,23 @@ export class SyncEngineLevel {
                 }
             }
             this._localSubscriptions = [];
+            // Clear degraded-poll timers and repair state.
+            for (const timer of this._degradedPollTimers.values()) {
+                clearInterval(timer);
+            }
+            this._degradedPollTimers.clear();
+            this._repairAttempts.clear();
+            this._activeRepairs.clear();
+            for (const timer of this._repairRetryTimers.values()) {
+                clearTimeout(timer);
+            }
+            this._repairRetryTimers.clear();
+            this._repairContext.clear();
+            // Clear closure evaluation contexts.
+            this._closureContexts.clear();
+            // Clear the in-memory link and runtime state.
+            this._activeLinks.clear();
+            this._linkRuntimes.clear();
         });
     }
     // ---------------------------------------------------------------------------
@@ -473,47 +1012,118 @@ export class SyncEngineLevel {
      */
     openLivePullSubscription(target) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
             const { did, delegateDid, dwnUrl, protocol } = target;
-            // Resolve the cursor from the last session (if any).
+            // Resolve the cursor from the link's pull checkpoint (preferred) or legacy storage.
             const cursorKey = this.buildCursorKey(did, dwnUrl, protocol);
-            const cursor = yield this.getCursor(cursorKey);
+            const link = this._activeLinks.get(cursorKey);
+            const cursor = (_a = link === null || link === void 0 ? void 0 : link.pull.contiguousAppliedToken) !== null && _a !== void 0 ? _a : yield this.getCursor(cursorKey);
             // Build the MessagesSubscribe filters.
-            const filters = protocol ? [{ protocol }] : [];
+            // When the link has protocolPathPrefixes, include them in the filter so the
+            // EventLog delivers only matching events (server-side filtering). This replaces
+            // the less efficient agent-side isEventInScope filtering for the pull path.
+            // Note: only the first prefix is used as the MessagesFilter field because
+            // MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
+            // would need multiple filters (OR semantics) — for now we use the first one.
+            const protocolPathPrefix = (link === null || link === void 0 ? void 0 : link.scope.kind) === 'protocol'
+                ? (_b = link.scope.protocolPathPrefixes) === null || _b === void 0 ? void 0 : _b[0]
+                : undefined;
+            const filters = protocol
+                ? [Object.assign({ protocol }, (protocolPathPrefix ? { protocolPathPrefix } : {}))]
+                : [];
             // Look up permission grant for MessagesSubscribe if using a delegate.
+            // The unified scope matching in AgentPermissionsApi accepts a
+            // Messages.Read grant for MessagesSubscribe requests, so a single
+            // lookup is sufficient.
             let permissionGrantId;
             if (delegateDid) {
-                try {
-                    const grant = yield this._permissionsApi.getPermissionForRequest({
-                        connectedDid: did,
-                        messageType: DwnInterface.MessagesSubscribe,
-                        delegateDid,
-                        protocol,
-                        cached: true
-                    });
-                    permissionGrantId = grant.grant.id;
-                }
-                catch (_a) {
-                    // Fall back to trying MessagesRead which is a unified scope.
-                    const grant = yield this._permissionsApi.getPermissionForRequest({
-                        connectedDid: did,
-                        messageType: DwnInterface.MessagesRead,
-                        delegateDid,
-                        protocol,
-                        cached: true
-                    });
-                    permissionGrantId = grant.grant.id;
-                }
+                const grant = yield this._permissionsApi.getPermissionForRequest({
+                    connectedDid: did,
+                    messageType: DwnInterface.MessagesSubscribe,
+                    delegateDid,
+                    protocol,
+                    cached: true
+                });
+                permissionGrantId = grant.grant.id;
             }
             // Define the subscription handler that processes incoming events.
+            // NOTE: The WebSocket client fires handlers without awaiting (fire-and-forget),
+            // so multiple handlers can be in-flight concurrently. The ordinal tracker
+            // ensures the checkpoint advances only when all earlier deliveries are committed.
             const subscriptionHandler = (subMessage) => __awaiter(this, void 0, void 0, function* () {
                 if (subMessage.type === 'eose') {
-                    // End-of-stored-events — catch-up complete, persist cursor.
-                    yield this.setCursor(cursorKey, subMessage.cursor);
-                    this._connectivityState = 'online';
+                    // End-of-stored-events — catch-up complete.
+                    if (link) {
+                        // Guard: if the link transitioned to repairing while catch-up events
+                        // were being processed, skip all mutations — repair owns the state now.
+                        if (link.status !== 'live' && link.status !== 'initializing') {
+                            return;
+                        }
+                        if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
+                            console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
+                            yield this.transitionToRepairing(cursorKey, link);
+                            return;
+                        }
+                        ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+                        // Drain committed entries. Do NOT unconditionally advance to the
+                        // EOSE cursor — earlier stored events may still be in-flight
+                        // (handlers are fire-and-forget). The checkpoint advances only as
+                        // far as the contiguous drain reaches.
+                        this.drainCommittedPull(cursorKey);
+                        yield this.ledger.saveLink(link);
+                    }
+                    else {
+                        yield this.setCursor(cursorKey, subMessage.cursor);
+                    }
+                    // Transport is reachable — set connectivity to online.
+                    if (link) {
+                        const prevEoseConnectivity = link.connectivity;
+                        link.connectivity = 'online';
+                        if (prevEoseConnectivity !== 'online') {
+                            this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
+                        }
+                    }
+                    else {
+                        this._connectivityState = 'online';
+                    }
                     return;
                 }
                 if (subMessage.type === 'event') {
                     const event = subMessage.event;
+                    // Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
+                    // skip all processing. Old subscription handlers may still fire after the
+                    // link transitions — these events should be ignored entirely, not just
+                    // skipped at the checkpoint level.
+                    if (link && link.status !== 'live' && link.status !== 'initializing') {
+                        return;
+                    }
+                    // Domain validation: reject tokens from a different stream/epoch.
+                    if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
+                        console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
+                        yield this.transitionToRepairing(cursorKey, link);
+                        return;
+                    }
+                    // Subset scope filtering: if the link has protocolPath/contextId prefixes,
+                    // skip events that don't match. This is agent-side filtering because
+                    // MessagesSubscribe only supports protocol-level filtering today.
+                    //
+                    // Skipped events MUST advance contiguousAppliedToken — otherwise the
+                    // link would replay the same filtered-out events indefinitely after
+                    // reconnect/repair. This is safe because the event is intentionally
+                    // excluded from this scope and doesn't need processing.
+                    if (link && !isEventInScope(event.message, link.scope)) {
+                        ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+                        ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
+                        yield this.ledger.saveLink(link);
+                        return;
+                    }
+                    // Assign a delivery ordinal BEFORE async processing begins.
+                    // This captures the delivery order even if processing completes out of order.
+                    const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
+                    const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
+                    if (rt) {
+                        rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
+                    }
                     try {
                         // Extract inline data from the event (available for records <= 30 KB).
                         let dataStream = this.extractDataStream(event);
@@ -532,12 +1142,90 @@ export class SyncEngineLevel {
                             }
                         }
                         yield this.agent.dwn.processRawMessage(did, event.message, { dataStream });
+                        // Invalidate closure cache entries that may be affected by this message.
+                        // Must run before closure validation so subsequent evaluations in the
+                        // same session see the updated local state.
+                        const closureCtxForInvalidation = this._closureContexts.get(did);
+                        if (closureCtxForInvalidation) {
+                            invalidateClosureCache(closureCtxForInvalidation, event.message);
+                        }
+                        // Closure validation for scoped subset sync (Phase 3).
+                        // For protocol-scoped links, verify that all hard dependencies for
+                        // this operation are locally present before considering it committed.
+                        // Full-tenant scope bypasses this entirely (returns complete with 0 queries).
+                        if (link && link.scope.kind === 'protocol') {
+                            const messageStore = this.agent.dwn.node.storage.messageStore;
+                            let closureCtx = this._closureContexts.get(did);
+                            if (!closureCtx) {
+                                closureCtx = createClosureContext(did);
+                                this._closureContexts.set(did, closureCtx);
+                            }
+                            const closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
+                            if (!closureResult.complete) {
+                                console.warn(`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
+                                    `${closureResult.failure.code} — ${closureResult.failure.detail}`);
+                                yield this.transitionToRepairing(cursorKey, link);
+                                return;
+                            }
+                        }
+                        // Squash convergence: processRawMessage triggers the DWN's built-in
+                        // squash resumable task (performRecordsSquash) which runs inline and
+                        // handles subset consumers correctly:
+                        // - If older siblings are locally present → purges them
+                        // - If squash arrives before older siblings → backstop rejects them (409)
+                        // - If no older siblings are local → no-op (correct)
+                        // Both sync orderings (squash-first or siblings-first) converge to
+                        // the same final state. No additional sync-engine side-effect is needed.
+                        // Track this CID for echo-loop suppression, scoped to the source endpoint.
+                        const pulledCid = yield Message.getCid(event.message);
+                        this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
+                        this.evictExpiredEchoEntries();
+                        // Mark this ordinal as committed and drain the checkpoint.
+                        // Guard: if the link transitioned to repairing while this handler was
+                        // in-flight (e.g., an earlier ordinal's handler failed concurrently),
+                        // skip all state mutations — the repair process owns progression now.
+                        if (link && rt && link.status === 'live') {
+                            const entry = rt.inflight.get(ordinal);
+                            if (entry) {
+                                entry.committed = true;
+                            }
+                            ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+                            const drained = this.drainCommittedPull(cursorKey);
+                            if (drained > 0) {
+                                yield this.ledger.saveLink(link);
+                                // Emit after durable save — "advanced" means persisted.
+                                if (link.pull.contiguousAppliedToken) {
+                                    this.emitEvent({
+                                        type: 'checkpoint:pull-advance',
+                                        tenantDid: link.tenantDid,
+                                        remoteEndpoint: link.remoteEndpoint,
+                                        protocol: link.protocol,
+                                        position: link.pull.contiguousAppliedToken.position,
+                                        messageCid: link.pull.contiguousAppliedToken.messageCid,
+                                    });
+                                }
+                            }
+                            // Overflow: too many in-flight ordinals without draining.
+                            if (rt.inflight.size > MAX_PENDING_TOKENS) {
+                                console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
+                                yield this.transitionToRepairing(cursorKey, link);
+                            }
+                        }
+                        else if (!link) {
+                            // Legacy path: no link available, use simple cursor persistence.
+                            yield this.setCursor(cursorKey, subMessage.cursor);
+                        }
                     }
                     catch (error) {
                         console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
+                        // A failed processRawMessage means local state is incomplete.
+                        // Transition to repairing immediately — do NOT advance the checkpoint
+                        // past this failure or let later ordinals commit past it. SMT
+                        // reconciliation will discover and fill the gap.
+                        if (link) {
+                            yield this.transitionToRepairing(cursorKey, link);
+                        }
                     }
-                    // Persist cursor for resume on reconnect.
-                    yield this.setCursor(cursorKey, subMessage.cursor);
                 }
             });
             // Construct the subscribe message and send it directly to the specific
@@ -559,7 +1247,10 @@ export class SyncEngineLevel {
             // Build a resubscribe factory so the WebSocket client can resume with
             // a fresh cursor-stamped message after reconnection.
             const resubscribeFactory = (resumeCursor) => __awaiter(this, void 0, void 0, function* () {
-                const resumeRequest = Object.assign(Object.assign({}, subscribeRequest), { messageParams: Object.assign(Object.assign({}, subscribeRequest.messageParams), { cursor: resumeCursor !== null && resumeCursor !== void 0 ? resumeCursor : cursor }) });
+                var _a;
+                // On reconnect, use the latest durable checkpoint position if available.
+                const effectiveCursor = (_a = resumeCursor !== null && resumeCursor !== void 0 ? resumeCursor : link === null || link === void 0 ? void 0 : link.pull.contiguousAppliedToken) !== null && _a !== void 0 ? _a : cursor;
+                const resumeRequest = Object.assign(Object.assign({}, subscribeRequest), { messageParams: Object.assign(Object.assign({}, subscribeRequest.messageParams), { cursor: effectiveCursor }) });
                 const { message: resumeMsg } = yield this.agent.dwn.processRequest(resumeRequest);
                 if (!resumeMsg) {
                     throw new Error(`SyncEngineLevel: Failed to construct resume MessagesSubscribe for ${dwnUrl}`);
@@ -579,9 +1270,15 @@ export class SyncEngineLevel {
                     resubscribeFactory,
                 },
             });
+            if (reply.status.code === 410) {
+                // ProgressGap — the cursor is no longer replayable. The link needs repair.
+                const gapError = new Error(`SyncEngineLevel: ProgressGap for ${did} -> ${dwnUrl}: ${reply.status.detail}`);
+                gapError.isProgressGap = true;
+                gapError.gapInfo = reply.error;
+                throw gapError;
+            }
             if (reply.status.code !== 200 || !reply.subscription) {
-                console.error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
-                return;
+                throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
             }
             this._liveSubscriptions.push({
                 did,
@@ -590,7 +1287,15 @@ export class SyncEngineLevel {
                 protocol,
                 close: () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); }),
             });
-            this._connectivityState = 'online';
+            // Set per-link connectivity to online after successful subscription setup.
+            const pullLink = this._activeLinks.get(this.buildCursorKey(did, dwnUrl, protocol));
+            if (pullLink) {
+                const prevPullConnectivity = pullLink.connectivity;
+                pullLink.connectivity = 'online';
+                if (prevPullConnectivity !== 'online') {
+                    this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevPullConnectivity, to: 'online' });
+                }
+            }
         });
     }
     // ---------------------------------------------------------------------------
@@ -610,7 +1315,7 @@ export class SyncEngineLevel {
             if (delegateDid) {
                 const grant = yield this._permissionsApi.getPermissionForRequest({
                     connectedDid: did,
-                    messageType: DwnInterface.MessagesRead,
+                    messageType: DwnInterface.MessagesSubscribe,
                     delegateDid,
                     protocol,
                     cached: true,
@@ -622,18 +1327,46 @@ export class SyncEngineLevel {
                 if (subMessage.type !== 'event') {
                     return;
                 }
+                // Subset scope filtering for push: only push events that match the
+                // link's scope prefixes. Events outside the scope are not our responsibility.
+                // Skipped events MUST advance the push checkpoint to prevent infinite
+                // replay after repair/reconnect (same reason as the pull side).
+                const pushLink = this._activeLinks.get(this.buildCursorKey(did, dwnUrl, protocol));
+                if (pushLink && !isEventInScope(subMessage.event.message, pushLink.scope)) {
+                    // Guard: only mutate durable state when the link is live/initializing.
+                    // During repair/degraded_poll, orchestration owns checkpoint progression.
+                    if (pushLink.status !== 'live' && pushLink.status !== 'initializing') {
+                        return;
+                    }
+                    // Validate token domain before committing — a stream/epoch mismatch
+                    // on the local EventLog should trigger repair, not silently overwrite.
+                    if (!ReplicationLedger.validateTokenDomain(pushLink.push, subMessage.cursor)) {
+                        yield this.transitionToRepairing(this.buildCursorKey(did, dwnUrl, protocol), pushLink);
+                        return;
+                    }
+                    ReplicationLedger.setReceivedToken(pushLink.push, subMessage.cursor);
+                    ReplicationLedger.commitContiguousToken(pushLink.push, subMessage.cursor);
+                    yield this.ledger.saveLink(pushLink);
+                    return;
+                }
                 // Accumulate the message CID for a debounced push.
                 const targetKey = this.buildCursorKey(did, dwnUrl, protocol);
                 const cid = yield Message.getCid(subMessage.event.message);
                 if (cid === undefined) {
                     return;
                 }
+                // Echo-loop suppression: skip CIDs that were recently pulled from this
+                // specific remote. A message pulled from Provider A is only suppressed
+                // for push to A — it still fans out to Provider B and C.
+                if (this.isRecentlyPulled(cid, dwnUrl)) {
+                    return;
+                }
                 let pending = this._pendingPushCids.get(targetKey);
                 if (!pending) {
-                    pending = { did, dwnUrl, delegateDid, protocol, cids: [] };
+                    pending = { did, dwnUrl, delegateDid, protocol, entries: [] };
                     this._pendingPushCids.set(targetKey, pending);
                 }
-                pending.cids.push(cid);
+                pending.entries.push({ cid, localToken: subMessage.cursor });
                 // Debounce the push.
                 if (this._pushDebounceTimer) {
                     clearTimeout(this._pushDebounceTimer);
@@ -643,18 +1376,20 @@ export class SyncEngineLevel {
                 }, PUSH_DEBOUNCE_MS);
             });
             // Process the local subscription request.
+            // When a push cursor is provided (e.g., after repair), the local subscription
+            // replays events from that position, closing the race window where local
+            // writes during repair would otherwise be missed by push-on-write.
             const response = yield this.agent.dwn.processRequest({
                 author: did,
                 target: did,
                 messageType: DwnInterface.MessagesSubscribe,
                 granteeDid: delegateDid,
-                messageParams: { filters, permissionGrantId },
+                messageParams: { filters, permissionGrantId, cursor: target.pushCursor },
                 subscriptionHandler: subscriptionHandler,
             });
             const reply = response.reply;
             if (reply.status.code !== 200 || !reply.subscription) {
-                console.error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
-                return;
+                throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
             }
             this._localSubscriptions.push({
                 did,
@@ -671,25 +1406,88 @@ export class SyncEngineLevel {
     flushPendingPushes() {
         return __awaiter(this, void 0, void 0, function* () {
             this._pushDebounceTimer = undefined;
-            const entries = [...this._pendingPushCids.entries()];
+            const batches = [...this._pendingPushCids.entries()];
             this._pendingPushCids.clear();
-            for (const [, pending] of entries) {
-                const { did, dwnUrl, delegateDid, protocol, cids } = pending;
-                if (cids.length === 0) {
-                    continue;
+            // Push to all endpoints in parallel — each target is independent.
+            yield Promise.all(batches.map((_a) => __awaiter(this, [_a], void 0, function* ([targetKey, pending]) {
+                const { did, dwnUrl, delegateDid, protocol, entries: pushEntries } = pending;
+                if (pushEntries.length === 0) {
+                    return;
                 }
+                const cids = pushEntries.map(e => e.cid);
                 try {
-                    yield pushMessages({
+                    const result = yield pushMessages({
                         did, dwnUrl, delegateDid, protocol,
                         messageCids: cids,
                         agent: this.agent,
                         permissionsApi: this._permissionsApi,
                     });
+                    // Advance the push checkpoint for successfully pushed entries.
+                    // Push is sequential (single batch, in-order processing) so we can
+                    // commit directly without ordinal tracking — there's no concurrent
+                    // completion to reorder.
+                    const link = this._activeLinks.get(targetKey);
+                    if (link) {
+                        const succeededSet = new Set(result.succeeded);
+                        // Track highest contiguous success: if a CID fails, we stop advancing.
+                        let hitFailure = false;
+                        for (const entry of pushEntries) {
+                            if (hitFailure) {
+                                break;
+                            }
+                            if (succeededSet.has(entry.cid) && entry.localToken) {
+                                if (!ReplicationLedger.validateTokenDomain(link.push, entry.localToken)) {
+                                    console.warn(`SyncEngineLevel: Push checkpoint domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
+                                    yield this.transitionToRepairing(targetKey, link);
+                                    break;
+                                }
+                                ReplicationLedger.setReceivedToken(link.push, entry.localToken);
+                                ReplicationLedger.commitContiguousToken(link.push, entry.localToken);
+                            }
+                            else {
+                                // This CID failed or had no token — stop advancing.
+                                hitFailure = true;
+                            }
+                        }
+                        yield this.ledger.saveLink(link);
+                    }
+                    // Re-queue failed entries so they are retried on the next debounce
+                    // cycle (or picked up by the SMT integrity check).
+                    if (result.failed.length > 0) {
+                        console.error(`SyncEngineLevel: Push-on-write failed for ${did} -> ${dwnUrl}: ${result.failed.length} of ${cids.length} messages failed`);
+                        const failedSet = new Set(result.failed);
+                        const failedEntries = pushEntries.filter(e => failedSet.has(e.cid));
+                        let requeued = this._pendingPushCids.get(targetKey);
+                        if (!requeued) {
+                            requeued = { did, dwnUrl, delegateDid, protocol, entries: [] };
+                            this._pendingPushCids.set(targetKey, requeued);
+                        }
+                        requeued.entries.push(...failedEntries);
+                        // Schedule a retry after a short delay.
+                        if (!this._pushDebounceTimer) {
+                            this._pushDebounceTimer = setTimeout(() => {
+                                void this.flushPendingPushes();
+                            }, PUSH_DEBOUNCE_MS * 4); // Back off: 1 second instead of 250ms.
+                        }
+                    }
                 }
                 catch (error) {
+                    // Truly unexpected error (not per-message failure). Re-queue entire
+                    // batch so entries aren't silently dropped from the debounce queue.
                     console.error(`SyncEngineLevel: Push-on-write failed for ${did} -> ${dwnUrl}`, error);
+                    let requeued = this._pendingPushCids.get(targetKey);
+                    if (!requeued) {
+                        requeued = { did, dwnUrl, delegateDid, protocol, entries: [] };
+                        this._pendingPushCids.set(targetKey, requeued);
+                    }
+                    requeued.entries.push(...pushEntries);
+                    if (!this._pushDebounceTimer) {
+                        this._pushDebounceTimer = setTimeout(() => {
+                            void this.flushPendingPushes();
+                        }, PUSH_DEBOUNCE_MS * 4);
+                    }
                 }
-            }
+            })));
         });
     }
     // ---------------------------------------------------------------------------
@@ -699,11 +1497,31 @@ export class SyncEngineLevel {
         const base = `${did}${CURSOR_SEPARATOR}${dwnUrl}`;
         return protocol ? `${base}${CURSOR_SEPARATOR}${protocol}` : base;
     }
+    /**
+     * Retrieves a stored progress token. Handles migration from old string cursors:
+     * if the stored value is a bare string (pre-ProgressToken format), it is treated
+     * as absent — the sync engine will do a full SMT reconciliation on first startup
+     * after upgrade, which is correct and safe.
+     */
     getCursor(key) {
         return __awaiter(this, void 0, void 0, function* () {
             const cursors = this._db.sublevel('syncCursors');
             try {
-                return yield cursors.get(key);
+                const raw = yield cursors.get(key);
+                try {
+                    const parsed = JSON.parse(raw);
+                    if (parsed && typeof parsed === 'object' &&
+                        typeof parsed.streamId === 'string' &&
+                        typeof parsed.epoch === 'string' &&
+                        typeof parsed.position === 'string' &&
+                        typeof parsed.messageCid === 'string') {
+                        return parsed;
+                    }
+                }
+                catch (_a) {
+                    // Not valid JSON (old string cursor) — treat as absent.
+                }
+                return undefined;
             }
             catch (error) {
                 const e = error;
@@ -717,7 +1535,7 @@ export class SyncEngineLevel {
     setCursor(key, cursor) {
         return __awaiter(this, void 0, void 0, function* () {
             const cursors = this._db.sublevel('syncCursors');
-            yield cursors.put(key, cursor);
+            yield cursors.put(key, JSON.stringify(cursor));
         });
     }
     // ---------------------------------------------------------------------------
@@ -862,80 +1680,6 @@ export class SyncEngineLevel {
         });
     }
     // ---------------------------------------------------------------------------
-    // Tree Diff — walk the SMT to find divergent leaf sets
-    // ---------------------------------------------------------------------------
-    /**
-     * Walks the local and remote SMTs in parallel, recursing into subtrees whose
-     * hashes differ, until reaching `MAX_DIFF_DEPTH` where leaves are enumerated.
-     *
-     * Returns the sets of messageCids that exist only locally or only remotely.
-     */
-    walkTreeDiff(_a) {
-        return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol }) {
-            const onlyLocal = [];
-            const onlyRemote = [];
-            // Hoist permission grant lookup — resolved once and reused for all subtree/leaf requests.
-            const permissionGrantId = yield this.getSyncPermissionGrantId(did, delegateDid, protocol);
-            // Gate remote HTTP requests through a semaphore so the binary tree walk
-            // doesn't produce an exponential burst of concurrent requests.  Local
-            // DWN requests (in-process) are not gated.
-            const remoteSemaphore = new Semaphore(REMOTE_CONCURRENCY);
-            const walk = (prefix) => __awaiter(this, void 0, void 0, function* () {
-                // Get subtree hashes for this prefix from local and remote.
-                // Only the remote request is gated by the semaphore.
-                const [localHash, remoteHash] = yield Promise.all([
-                    this.getLocalSubtreeHash(did, prefix, delegateDid, protocol, permissionGrantId),
-                    remoteSemaphore.run(() => this.getRemoteSubtreeHash(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId)),
-                ]);
-                // If hashes match, this subtree is identical — skip.
-                if (localHash === remoteHash) {
-                    return;
-                }
-                // Short-circuit: if one side is the default (empty-subtree) hash, all entries
-                // on the other side are unique.  Enumerate leaves directly instead of recursing
-                // further into the tree — this avoids an exponential walk when one DWN has
-                // entries that the other lacks entirely in this subtree.
-                const emptyHash = yield this.getDefaultHashHex(prefix.length);
-                if (remoteHash === emptyHash && localHash !== emptyHash) {
-                    const localLeaves = yield this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantId);
-                    onlyLocal.push(...localLeaves);
-                    return;
-                }
-                if (localHash === emptyHash && remoteHash !== emptyHash) {
-                    const remoteLeaves = yield remoteSemaphore.run(() => this.getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId));
-                    onlyRemote.push(...remoteLeaves);
-                    return;
-                }
-                // If we've reached the maximum diff depth, enumerate leaves.
-                if (prefix.length >= MAX_DIFF_DEPTH) {
-                    const [localLeaves, remoteLeaves] = yield Promise.all([
-                        this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantId),
-                        remoteSemaphore.run(() => this.getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId)),
-                    ]);
-                    const localSet = new Set(localLeaves);
-                    const remoteSet = new Set(remoteLeaves);
-                    for (const cid of localLeaves) {
-                        if (!remoteSet.has(cid)) {
-                            onlyLocal.push(cid);
-                        }
-                    }
-                    for (const cid of remoteLeaves) {
-                        if (!localSet.has(cid)) {
-                            onlyRemote.push(cid);
-                        }
-                    }
-                    return;
-                }
-                // Recurse into left (0) and right (1) children in parallel.
-                yield Promise.all([
-                    walk(prefix + '0'),
-                    walk(prefix + '1'),
-                ]);
-            });
-            yield walk('');
-            return { onlyLocal, onlyRemote };
-        });
-    }
     // ---------------------------------------------------------------------------
     // Batched Diff — single round-trip set reconciliation
     // ---------------------------------------------------------------------------
@@ -982,7 +1726,8 @@ export class SyncEngineLevel {
                 throw new Error(`SyncEngineLevel: diff failed with ${reply.status.code}: ${reply.status.detail}`);
             }
             // Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
-            const permissionGrantIdForLeaves = yield this.getSyncPermissionGrantId(did, delegateDid, protocol);
+            // Reuse the same grant ID from step 2 (avoids redundant lookup).
+            const permissionGrantIdForLeaves = permissionGrantId;
             const onlyLocalCids = [];
             for (const prefix of (_b = reply.onlyLocal) !== null && _b !== void 0 ? _b : []) {
                 const leaves = yield this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIdForLeaves);
@@ -1073,30 +1818,6 @@ export class SyncEngineLevel {
             return (_a = reply.hash) !== null && _a !== void 0 ? _a : '';
         });
     }
-    getRemoteSubtreeHash(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            const syncMessage = yield this.agent.dwn.processRequest({
-                store: false,
-                author: did,
-                target: did,
-                messageType: DwnInterface.MessagesSync,
-                granteeDid: delegateDid,
-                messageParams: {
-                    action: 'subtree',
-                    prefix,
-                    protocol,
-                    permissionGrantId
-                }
-            });
-            const reply = yield this.agent.rpc.sendDwnRequest({
-                dwnUrl,
-                targetDid: did,
-                message: syncMessage.message,
-            });
-            return (_a = reply.hash) !== null && _a !== void 0 ? _a : '';
-        });
-    }
     /**
      * Get all leaf messageCids under a given prefix from the local DWN.
      *
@@ -1130,30 +1851,6 @@ export class SyncEngineLevel {
             return (_a = reply.entries) !== null && _a !== void 0 ? _a : [];
         });
     }
-    getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            const syncMessage = yield this.agent.dwn.processRequest({
-                store: false,
-                author: did,
-                target: did,
-                messageType: DwnInterface.MessagesSync,
-                granteeDid: delegateDid,
-                messageParams: {
-                    action: 'leaves',
-                    prefix,
-                    protocol,
-                    permissionGrantId
-                }
-            });
-            const reply = yield this.agent.rpc.sendDwnRequest({
-                dwnUrl,
-                targetDid: did,
-                message: syncMessage.message,
-            });
-            return (_a = reply.entries) !== null && _a !== void 0 ? _a : [];
-        });
-    }
     // ---------------------------------------------------------------------------
     // Pull / Push — delegates to standalone functions in sync-messages.ts
     // ---------------------------------------------------------------------------
@@ -1174,6 +1871,51 @@ export class SyncEngineLevel {
             });
         });
     }
+    // ---------------------------------------------------------------------------
+    // Echo-loop suppression
+    // ---------------------------------------------------------------------------
+    /**
+     * Evicts expired entries from the echo-loop suppression cache.
+     * Also enforces the size cap by evicting oldest entries first.
+     */
+    evictExpiredEchoEntries() {
+        const now = Date.now();
+        // Evict expired entries.
+        for (const [cid, expiry] of this._recentlyPulledCids) {
+            if (now >= expiry) {
+                this._recentlyPulledCids.delete(cid);
+            }
+        }
+        // Enforce size cap by evicting oldest entries.
+        if (this._recentlyPulledCids.size > SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES) {
+            const excess = this._recentlyPulledCids.size - SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES;
+            let evicted = 0;
+            for (const key of this._recentlyPulledCids.keys()) {
+                if (evicted >= excess) {
+                    break;
+                }
+                this._recentlyPulledCids.delete(key);
+                evicted++;
+            }
+        }
+    }
+    /**
+     * Checks whether a CID was recently pulled from a specific remote endpoint
+     * and should not be pushed back to that same endpoint (echo-loop suppression).
+     * Does not suppress pushes to other endpoints — multi-provider fan-out works.
+     */
+    isRecentlyPulled(cid, dwnUrl) {
+        const key = `${cid}|${dwnUrl}`;
+        const expiry = this._recentlyPulledCids.get(key);
+        if (expiry === undefined) {
+            return false;
+        }
+        if (Date.now() >= expiry) {
+            this._recentlyPulledCids.delete(key);
+            return false;
+        }
+        return true;
+    }
     /**
      * Reads missing messages from the local DWN and pushes them to the remote DWN
      * in dependency order (topological sort).
@@ -1265,8 +2007,19 @@ export class SyncEngineLevel {
         });
     }
 }
+/** TTL for echo-loop suppression entries (60 seconds). */
+SyncEngineLevel.ECHO_SUPPRESS_TTL_MS = 60000;
+/** Maximum entries in the echo-loop suppression cache. */
+SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES = 10000;
 /** Maximum consecutive failures before entering backoff. */
 SyncEngineLevel.MAX_CONSECUTIVE_FAILURES = 5;
 /** Backoff multiplier for consecutive failures (caps at 4x the configured interval). */
 SyncEngineLevel.MAX_BACKOFF_MULTIPLIER = 4;
+// ---------------------------------------------------------------------------
+// Per-link repair and degraded-poll orchestration (Phase 2)
+// ---------------------------------------------------------------------------
+/** Maximum consecutive repair attempts before falling back to degraded_poll. */
+SyncEngineLevel.MAX_REPAIR_ATTEMPTS = 3;
+/** Backoff schedule for repair retries (milliseconds). */
+SyncEngineLevel.REPAIR_BACKOFF_MS = [1000, 3000, 10000];
 //# sourceMappingURL=sync-engine-level.js.map