@enbox/agent 0.6.4 → 0.6.6

package/src/sync-engine-level.ts

@@ -175,7 +175,7 @@ export class SyncEngineLevel implements SyncEngine {
    */
   private _permissionsApi: PermissionsApi;
 
-  private _db: AbstractLevel<string | Buffer | Uint8Array>;
+  private readonly _db: AbstractLevel<string | Buffer | Uint8Array>;
   private _syncIntervalId?: ReturnType<typeof setInterval>;
   private _syncLock = false;
 
@@ -191,7 +191,7 @@ export class SyncEngineLevel implements SyncEngine {
    * Populated from the ledger on `startLiveSync`, used by subscription handlers
    * to avoid async ledger lookups on every event.
    */
-  private _activeLinks: Map<string, ReplicationLinkState> = new Map();
+  private readonly _activeLinks: Map<string, ReplicationLinkState> = new Map();
 
   /**
    * Per-link in-memory delivery-order tracking for the pull path. Keyed by
@@ -199,7 +199,7 @@ export class SyncEngineLevel implements SyncEngine {
    * restarts from `contiguousAppliedToken` and idempotent apply handles
    * re-delivered events.
    */
-  private _linkRuntimes: Map<string, LinkRuntimeState> = new Map();
+  private readonly _linkRuntimes: Map<string, LinkRuntimeState> = new Map();
 
   /**
    * Hex-encoded default hashes for empty subtrees at each depth, keyed by depth.
@@ -212,8 +212,8 @@ export class SyncEngineLevel implements SyncEngine {
   // Live sync state
   // ---------------------------------------------------------------------------
 
-  /** Current sync mode, set by `startSync`. */
-  private _syncMode: SyncMode = 'poll';
+  /** Current sync mode, set by `startSync`. Reset to `undefined` by `stopSync`/`clear`. */
+  private _syncMode: SyncMode | undefined = 'poll';
 
   /**
    * Monotonic session generation counter. Incremented on every teardown.
@@ -233,10 +233,10 @@ export class SyncEngineLevel implements SyncEngine {
   private _connectivityState: SyncConnectivityState = 'unknown';
 
   /** Registered event listeners for observability. */
-  private _eventListeners: Set<SyncEventListener> = new Set();
+  private readonly _eventListeners: Set<SyncEventListener> = new Set();
 
   /** Per-link push runtime: queue, debounce timer, retry state. */
-  private _pushRuntimes: Map<string, PushRuntimeState> = new Map();
+  private readonly _pushRuntimes: Map<string, PushRuntimeState> = new Map();
 
   /**
    * CIDs recently received via pull subscription, keyed by `cid|dwnUrl` to
@@ -244,7 +244,7 @@ export class SyncEngineLevel implements SyncEngine {
    * is only suppressed for push back to Provider A — it still fans out to
    * Provider B and C. TTL: 60 seconds. Cap: 10,000 entries.
    */
-  private _recentlyPulledCids: Map<string, number> = new Map();
+  private readonly _recentlyPulledCids: Map<string, number> = new Map();
 
   /** TTL for echo-loop suppression entries (60 seconds). */
   private static readonly ECHO_SUPPRESS_TTL_MS = 60_000;
@@ -254,11 +254,46 @@ export class SyncEngineLevel implements SyncEngine {
    * Caches ProtocolsConfigure and grant lookups across events for the same
    * tenant. Keyed by tenantDid to prevent cross-tenant cache pollution.
    */
-  private _closureContexts: Map<string, ClosureEvaluationContext> = new Map();
+  private readonly _closureContexts: Map<string, ClosureEvaluationContext> = new Map();
 
   /** Maximum entries in the echo-loop suppression cache. */
   private static readonly ECHO_SUPPRESS_MAX_ENTRIES = 10_000;
 
+  /** Validate `SyncIdentityOptions` for `registerIdentity` and `updateIdentityOptions`. */
+  private static validateSyncIdentityOptions(options: SyncIdentityOptions): void {
+    if (!options || !('protocols' in options)) {
+      throw new Error('SyncEngineLevel: options.protocols is required — pass \'all\' for a full replica or a non-empty protocol list.');
+    }
+    if (options.protocols !== 'all' && !Array.isArray(options.protocols)) {
+      throw new Error('SyncEngineLevel: protocols must be \'all\' or a non-empty string array.');
+    }
+    if (Array.isArray(options.protocols) && options.protocols.length === 0) {
+      throw new Error('SyncEngineLevel: protocols must be \'all\' or a non-empty array of protocol URIs. An empty array is ambiguous.');
+    }
+  }
+
+  /**
+   * Cached sync targets result from the last {@link getSyncTargets} call.
+   * Invalidated on identity registration/unregistration/update.
+   * TTL-based: cleared after 30 seconds to pick up DID document changes.
+   */
+  private _syncTargetsCache?: {
+    targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[];
+    timestamp: number;
+  };
+
+  /**
+   * Monotonic generation counter for sync target cache invalidation.
+   * Bumped on every invalidation (register/unregister/update/clear/close/stopSync).
+   * An in-flight `getSyncTargets()` captures the generation before awaiting
+   * and only writes to the cache if it hasn't changed, preventing a
+   * concurrent mutation from being masked by stale data.
+   */
+  private _syncTargetsCacheGeneration = 0;
+
+  /** TTL for the sync targets cache (30 seconds). */
+  private static readonly SYNC_TARGETS_CACHE_TTL_MS = 30_000;
+
   /** Count of consecutive SMT sync failures (for backoff in poll mode). */
   private _consecutiveFailures = 0;
 
@@ -312,6 +347,16 @@ export class SyncEngineLevel implements SyncEngine {
   set agent(agent: EnboxPlatformAgent) {
     this._agent = agent;
     this._permissionsApi = new AgentPermissionsApi({ agent: agent as EnboxAgent });
+    // Cached sync targets were resolved through the previous agent's
+    // DID resolver / endpoint lookup — invalidate so the next sync
+    // tick re-resolves through the new agent.
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
+  }
+
+  get hasActiveSubscriptions(): boolean {
+    return this._liveSubscriptions.length > 0 ||
+           this._localSubscriptions.length > 0;
   }
 
   get connectivityState(): SyncConnectivityState {
@@ -351,17 +396,24 @@ export class SyncEngineLevel implements SyncEngine {
   }
 
   public async clear(): Promise<void> {
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
     await this.teardownLiveSync();
+    this._syncMode = undefined;
     await this._permissionsApi.clear();
     await this._db.clear();
   }
 
   public async close(): Promise<void> {
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
     await this.teardownLiveSync();
     await this._db.close();
   }
 
-  public async registerIdentity({ did, options }: { did: string; options?: SyncIdentityOptions }): Promise<void> {
+  public async registerIdentity({ did, options }: { did: string; options: SyncIdentityOptions }): Promise<void> {
+    SyncEngineLevel.validateSyncIdentityOptions(options);
+
     const registeredIdentities = this._db.sublevel('registeredIdentities');
 
     const existing = await this.getIdentityOptions(did);
@@ -369,10 +421,14 @@ export class SyncEngineLevel implements SyncEngine {
       throw new Error(`SyncEngineLevel: Identity with DID ${did} is already registered.`);
     }
 
-    // if no options are provided, we default to no delegateDid and all protocols (empty array)
-    options ??= { protocols: [] };
-
     await registeredIdentities.put(did, JSON.stringify(options));
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
+
+    // If live sync is active, hot-add subscriptions for this identity.
+    if (this._syncMode === 'live') {
+      await this.addIdentityToLiveSync(did, options);
+    }
   }
 
   public async unregisterIdentity(did: string): Promise<void> {
@@ -382,7 +438,14 @@ export class SyncEngineLevel implements SyncEngine {
       throw new Error(`SyncEngineLevel: Identity with DID ${did} is not registered.`);
     }
 
+    // If live sync is active, hot-remove subscriptions for this identity.
+    if (this._syncMode === 'live') {
+      await this.removeIdentityFromLiveSync(did);
+    }
+
     await registeredIdentities.del(did);
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
   }
 
   public async getIdentityOptions(did: string): Promise<SyncIdentityOptions | undefined> {
@@ -404,6 +467,8 @@ export class SyncEngineLevel implements SyncEngine {
   }
 
   public async updateIdentityOptions({ did, options }: { did: string, options: SyncIdentityOptions }): Promise<void> {
+    SyncEngineLevel.validateSyncIdentityOptions(options);
+
     const registeredIdentities = this._db.sublevel('registeredIdentities');
     const existingOptions = await this.getIdentityOptions(did);
     if (!existingOptions) {
@@ -411,6 +476,23 @@ export class SyncEngineLevel implements SyncEngine {
     }
 
     await registeredIdentities.put(did, JSON.stringify(options));
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
+
+    // Always persist the new delegate to durable links, regardless of
+    // sync mode. If sync is stopped or polling, existing persisted links
+    // would otherwise keep the old delegateDid. When live sync starts
+    // later, initializeLinkTarget() loads the link from LevelDB without
+    // normalizing delegateDid, so repair/reconcile paths could use stale
+    // delegate data.
+    await this.ledger.updateDelegateDid(did, options.delegateDid);
+
+    // If live sync is active, tear down and rebuild subscriptions with
+    // the new options.
+    if (this._syncMode === 'live' && this.hasActiveLinksForDid(did)) {
+      await this.removeIdentityFromLiveSync(did);
+      await this.addIdentityToLiveSync(did, options);
+    }
   }
 
   // ---------------------------------------------------------------------------
@@ -534,7 +616,10 @@ export class SyncEngineLevel implements SyncEngine {
       this._syncIntervalId = undefined;
     }
 
+    this._syncTargetsCache = undefined;
+    this._syncTargetsCacheGeneration++;
     await this.teardownLiveSync();
+    this._syncMode = undefined;
   }
 
   // ---------------------------------------------------------------------------
@@ -611,95 +696,7 @@ export class SyncEngineLevel implements SyncEngine {
     // Step 2: Initialize replication links and open live subscriptions.
     // Each target's link initialization is independent — process concurrently.
     const syncTargets = await this.getSyncTargets();
-    await Promise.allSettled(syncTargets.map(async (target) => {
-      let link: ReplicationLinkState | undefined;
-      try {
-        // Get or create the link in the durable ledger.
-        // Use protocol-scoped scope when a protocol is specified, otherwise full-tenant.
-        const linkScope: SyncScope = target.protocol
-          ? { kind: 'protocol', protocol: target.protocol }
-          : { kind: 'full' };
-        link = await this.ledger.getOrCreateLink({
-          tenantDid      : target.did,
-          remoteEndpoint : target.dwnUrl,
-          scope          : linkScope,
-          delegateDid    : target.delegateDid,
-          protocol       : target.protocol,
-        });
-
-        // Cache the link for fast access by subscription handlers.
-        // Use scopeId from the link for consistent runtime identity.
-        const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
-
-        // One-time migration: if the link has no pull checkpoint, check for
-        // a legacy cursor in the old syncCursors sublevel. The legacy key
-        // used protocol, not scopeId, so we must build it the old way.
-        if (!link.pull.contiguousAppliedToken) {
-          const legacyKey = buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
-          const legacyCursor = await this.getCursor(legacyKey);
-          if (legacyCursor) {
-            ReplicationLedger.resetCheckpoint(link.pull, legacyCursor);
-            await this.ledger.saveLink(link);
-            await this.deleteLegacyCursor(legacyKey);
-          }
-        }
-
-        this._activeLinks.set(linkKey, link);
-
-        // Open subscriptions — only transition to live if both succeed.
-        // If pull succeeds but push fails, close the pull subscription to
-        // avoid a resource leak with inconsistent state.
-        const targetWithKey = { ...target, linkKey };
-        await this.openLivePullSubscription(targetWithKey);
-        try {
-          await this.openLocalPushSubscription(targetWithKey);
-        } catch (pushError) {
-          // Close the already-opened pull subscription.
-          const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
-          if (pullSub) {
-            try { await pullSub.close(); } catch { /* best effort */ }
-            this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
-          }
-          throw pushError;
-        }
-
-        this.emitEvent({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, from: 'initializing', to: 'live' });
-        await this.ledger.setStatus(link!, 'live');
-
-        // If the link was marked dirty in a previous session, schedule
-        // immediate reconciliation now that subscriptions are open.
-        if (link!.needsReconcile) {
-          this.scheduleReconcile(linkKey, 1000);
-        }
-      } catch (error: any) {
-        const linkKey = link
-          ? this.buildLinkKey(target.did, target.dwnUrl, link.scopeId)
-          : buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
-
-        // Detect ProgressGap (410) — the cursor is stale, link needs SMT repair.
-        if ((error as any).isProgressGap && link) {
-          console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
-          this.emitEvent({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, reason: 'ProgressGap' });
-          const gapInfo = (error as any).gapInfo;
-          await this.transitionToRepairing(linkKey, link, {
-            resumeToken: gapInfo?.latestAvailable,
-          });
-          return;
-        }
-
-        console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
-
-        // Clean up in-memory state for the failed link so it doesn't appear
-        // active to later code. The durable link remains at 'initializing'.
-        this._activeLinks.delete(linkKey);
-        this._linkRuntimes.delete(linkKey);
-
-        // Recompute connectivity — if no live subscriptions remain, reset to unknown.
-        if (this._liveSubscriptions.length === 0) {
-          this._connectivityState = 'unknown';
-        }
-      }
-    }));
+    await Promise.allSettled(syncTargets.map(t => this.initializeLinkTarget(t)));
 
     // Step 3: Schedule infrequent SMT integrity check.
     const integrityCheck = async (): Promise<void> => {
@@ -742,7 +739,7 @@ export class SyncEngineLevel implements SyncEngine {
     let drained = 0;
     while (true) {
       const entry = rt.inflight.get(rt.nextCommitOrdinal);
-      if (!entry || !entry.committed) { break; }
+      if (!entry?.committed) { break; }
 
       // This ordinal is committed — advance the durable checkpoint.
       ReplicationLedger.commitContiguousToken(link.pull, entry.token);
@@ -765,16 +762,16 @@ export class SyncEngineLevel implements SyncEngine {
   private static readonly MAX_REPAIR_ATTEMPTS = 3;
 
   /** Per-link degraded-poll interval timers. */
-  private _degradedPollTimers: Map<string, ReturnType<typeof setInterval>> = new Map();
+  private readonly _degradedPollTimers: Map<string, ReturnType<typeof setInterval>> = new Map();
 
   /** Per-link repair attempt counters. */
-  private _repairAttempts: Map<string, number> = new Map();
+  private readonly _repairAttempts: Map<string, number> = new Map();
 
   /** Per-link active repair promises — prevents concurrent repair for the same link. */
-  private _activeRepairs: Map<string, Promise<void>> = new Map();
+  private readonly _activeRepairs: Map<string, Promise<void>> = new Map();
 
   /** Per-link retry timers for failed repairs below max attempts. */
-  private _repairRetryTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
+  private readonly _repairRetryTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
 
   /** Backoff schedule for repair retries (milliseconds). */
   private static readonly REPAIR_BACKOFF_MS = [1_000, 3_000, 10_000];
@@ -785,7 +782,7 @@ export class SyncEngineLevel implements SyncEngine {
    * the post-repair checkpoint so the reopened subscription replays from
    * a valid boundary instead of starting live-only.
    */
-  private _repairContext: Map<string, { resumeToken?: ProgressToken }> = new Map();
+  private readonly _repairContext: Map<string, { resumeToken?: ProgressToken }> = new Map();
 
   /**
    * Central helper for transitioning a link to `repairing`. Encapsulates:
@@ -855,7 +852,7 @@ export class SyncEngineLevel implements SyncEngine {
 
       // Verify link still exists and is still repairing.
       const currentLink = this._activeLinks.get(linkKey);
-      if (!currentLink || currentLink.status !== 'repairing') { return; }
+      if (currentLink?.status !== 'repairing') { return; }
 
       try {
         await this.repairLink(linkKey);
@@ -910,6 +907,11 @@ export class SyncEngineLevel implements SyncEngine {
     // mutating state — preventing the race where repair continues after teardown.
     const generation = this._engineGeneration;
 
+    // Identity guard helper: if the DID was hot-removed and quickly re-added,
+    // `_activeLinks` may contain a *different* link object for the same key.
+    // The old repair closure must not mutate the replacement link's state.
+    const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
+
     const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, protocol } = link;
 
     this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: (this._repairAttempts.get(linkKey) ?? 0) + 1 });
@@ -919,7 +921,7 @@ export class SyncEngineLevel implements SyncEngine {
     // Step 1: Close existing subscriptions FIRST to stop old events from
     // mutating local state while repair runs.
     await this.closeLinkSubscriptions(link);
-    if (this._engineGeneration !== generation) { return; } // Teardown occurred.
+    if (this._engineGeneration !== generation || isStaleLink()) { return; }
 
     // Step 2: Clear runtime ordinals immediately — stale state must not
     // persist across repair attempts (successful or failed).
@@ -931,7 +933,7 @@ export class SyncEngineLevel implements SyncEngine {
     try {
       // Step 3: Run SMT reconciliation for this link.
       const reconcileOutcome = await this.createLinkReconciler(
-        () => this._engineGeneration === generation
+        () => this._engineGeneration === generation && !isStaleLink()
       ).reconcile({ did, dwnUrl, delegateDid, protocol });
       if (reconcileOutcome.aborted) { return; }
 
@@ -945,7 +947,7 @@ export class SyncEngineLevel implements SyncEngine {
       const resumeToken = repairCtx?.resumeToken ?? link.pull.contiguousAppliedToken;
       ReplicationLedger.resetCheckpoint(link.pull, resumeToken);
       await this.ledger.saveLink(link);
-      if (this._engineGeneration !== generation) { return; }
+      if (this._engineGeneration !== generation || isStaleLink()) { return; }
 
       // Step 5: Reopen subscriptions.
       // Mark needsReconcile BEFORE reopening — local push starts from "now",
@@ -955,7 +957,7 @@ export class SyncEngineLevel implements SyncEngine {
       // if roots already match).
       link.needsReconcile = true;
       await this.ledger.saveLink(link);
-      if (this._engineGeneration !== generation) { return; }
+      if (this._engineGeneration !== generation || isStaleLink()) { return; }
 
       const target = { did, dwnUrl, delegateDid, protocol, linkKey };
       try {
@@ -965,13 +967,13 @@ export class SyncEngineLevel implements SyncEngine {
           console.warn(`SyncEngineLevel: Stale pull resume token for ${did} -> ${dwnUrl}, resetting to start fresh`);
           ReplicationLedger.resetCheckpoint(link.pull);
           await this.ledger.saveLink(link);
-          if (this._engineGeneration !== generation) { return; }
+          if (this._engineGeneration !== generation || isStaleLink()) { return; }
           await this.openLivePullSubscription(target);
         } else {
           throw pullErr;
         }
       }
-      if (this._engineGeneration !== generation) { return; }
+      if (this._engineGeneration !== generation || isStaleLink()) { return; }
       try {
         await this.openLocalPushSubscription(target);
       } catch (pushError) {
@@ -982,7 +984,7 @@ export class SyncEngineLevel implements SyncEngine {
         }
         throw pushError;
       }
-      if (this._engineGeneration !== generation) { return; }
+      if (this._engineGeneration !== generation || isStaleLink()) { return; }
 
       // Note: post-repair reconcile to close the repair-window gap is
       // scheduled by repairLink() AFTER _activeRepairs is cleared — not
@@ -1010,8 +1012,9 @@ export class SyncEngineLevel implements SyncEngine {
       this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: 'repairing', to: 'live' });
 
     } catch (error: any) {
-      // If teardown occurred during repair, don't retry or enter degraded_poll.
-      if (this._engineGeneration !== generation) { return; }
+      // If teardown occurred during repair or the link was replaced by a
+      // hot-remove + re-add, don't retry or enter degraded_poll.
+      if (this._engineGeneration !== generation || isStaleLink()) { return; }
 
       console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
       this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: attempts, error: String(error.message ?? error) });
@@ -1070,8 +1073,14 @@ export class SyncEngineLevel implements SyncEngine {
     if (existing) { clearInterval(existing); }
 
     // Schedule per-link polling with jitter (15-30 seconds).
+    // Rejection sampling: mask to 14 bits ([0, 16383]), reject >= 15000.
     const baseInterval = 15_000;
-    const jitter = Math.floor(Math.random() * 15_000);
+    const randomBuf = new Uint32Array(1);
+    let jitter: number;
+    do {
+      crypto.getRandomValues(randomBuf);
+      jitter = randomBuf[0] & 0x3FFF;
+    } while (jitter >= baseInterval);
     const interval = baseInterval + jitter;
 
     const pollGeneration = this._engineGeneration;
@@ -1083,9 +1092,12 @@ export class SyncEngineLevel implements SyncEngine {
         return;
       }
 
-      // If the link was transitioned out of degraded_poll externally (e.g.,
-      // by teardown or manual intervention), stop polling.
-      if (link.status !== 'degraded_poll') {
+      // Resolve the *current* link from _activeLinks on each tick, not the
+      // captured closure reference. After hot-remove + re-add, the captured
+      // `link` object is stale and must not be used for status checks or
+      // ledger writes.
+      const currentLink = this._activeLinks.get(linkKey);
+      if (currentLink?.status !== 'degraded_poll') {
         clearInterval(timer);
         this._degradedPollTimers.delete(linkKey);
         return;
@@ -1095,11 +1107,11 @@ export class SyncEngineLevel implements SyncEngine {
         // Attempt repair. Reset attempt counter so repairLink doesn't
         // immediately re-enter degraded_poll on failure.
         this._repairAttempts.set(linkKey, 0);
-        await this.ledger.setStatus(link, 'repairing');
+        await this.ledger.setStatus(currentLink, 'repairing');
         await this.repairLink(linkKey);
 
         // If repairLink succeeded, link is now 'live' — stop polling.
-        if ((link.status as string) === 'live') {
+        if ((currentLink.status as string) === 'live') {
           clearInterval(timer);
           this._degradedPollTimers.delete(linkKey);
         }
@@ -1108,7 +1120,9 @@ export class SyncEngineLevel implements SyncEngine {
         // This is critical: repairLink sets status to 'repairing' internally,
         // and if we don't restore degraded_poll, the next tick would see
         // status !== 'degraded_poll' and stop the timer permanently.
-        await this.ledger.setStatus(link, 'degraded_poll');
+        if (this._activeLinks.get(linkKey) === currentLink) {
+          await this.ledger.setStatus(currentLink, 'degraded_poll');
+        }
       }
     }, interval);
 
@@ -1286,6 +1300,172 @@ export class SyncEngineLevel implements SyncEngine {
     this._linkRuntimes.clear();
   }
 
+  // ---------------------------------------------------------------------------
+  // Per-target link initialization (shared by startLiveSync + addIdentityToLiveSync)
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Initialize a single replication link target: create or resume the durable
+   * link, migrate legacy cursors, open pull + push subscriptions, and
+   * transition the link to `'live'`.
+   */
+  private async initializeLinkTarget(target: {
+    did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
+  }): Promise<void> {
+    let link: ReplicationLinkState | undefined;
+    try {
+      const linkScope: SyncScope = target.protocol
+        ? { kind: 'protocol', protocol: target.protocol }
+        : { kind: 'full' };
+      link = await this.ledger.getOrCreateLink({
+        tenantDid      : target.did,
+        remoteEndpoint : target.dwnUrl,
+        scope          : linkScope,
+        delegateDid    : target.delegateDid,
+        protocol       : target.protocol,
+      });
+
+      const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
+
+      if (!link.pull.contiguousAppliedToken) {
+        const legacyKey = buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
+        const legacyCursor = await this.getCursor(legacyKey);
+        if (legacyCursor) {
+          ReplicationLedger.resetCheckpoint(link.pull, legacyCursor);
+          await this.ledger.saveLink(link);
+          await this.deleteLegacyCursor(legacyKey);
+        }
+      }
+
+      this._activeLinks.set(linkKey, link);
+
+      const targetWithKey = { ...target, linkKey };
+      await this.openLivePullSubscription(targetWithKey);
+      try {
+        await this.openLocalPushSubscription(targetWithKey);
+      } catch (pushError) {
+        const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
+        if (pullSub) {
+          try { await pullSub.close(); } catch { /* best effort */ }
+          this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
+        }
+        throw pushError;
+      }
+
+      this.emitEvent({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, from: 'initializing', to: 'live' });
+      await this.ledger.setStatus(link, 'live');
+
+      if (link.needsReconcile) {
+        this.scheduleReconcile(linkKey, 1000);
+      }
+    } catch (error: any) {
+      const linkKey = link
+        ? this.buildLinkKey(target.did, target.dwnUrl, link.scopeId)
+        : buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
+
+      if (error.isProgressGap && link) {
+        console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
+        this.emitEvent({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, reason: 'ProgressGap' });
+        await this.transitionToRepairing(linkKey, link, {
+          resumeToken: error.gapInfo?.latestAvailable,
+        });
+        return;
+      }
+
+      console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
+
+      this._activeLinks.delete(linkKey);
+      this._linkRuntimes.delete(linkKey);
+
+      if (this._liveSubscriptions.length === 0) {
+        this._connectivityState = 'unknown';
+      }
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // Hot-add / hot-remove: per-identity live sync management
+  // ---------------------------------------------------------------------------
+
+  /** Check whether a link key belongs to a given DID. */
+  private isLinkKeyForDid(key: string, did: string): boolean {
+    return key.startsWith(did + '^') || key.startsWith(did + '_');
+  }
+
+  /** Check whether this DID has any active links. */
+  private hasActiveLinksForDid(did: string): boolean {
+    for (const key of this._activeLinks.keys()) {
+      if (this.isLinkKeyForDid(key, did)) { return true; }
+    }
+    return false;
+  }
+
+  /** Hot-add a single identity to the active live sync session. */
+  private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<void> {
+    const { protocols, delegateDid } = options;
+    const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
+    if (dwnEndpointUrls.length === 0) { return; }
+
+    const targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[] = [];
+    for (const dwnUrl of dwnEndpointUrls) {
+      if (protocols === 'all') {
+        targets.push({ did, delegateDid, dwnUrl });
+      } else {
+        for (const protocol of protocols) {
+          targets.push({ did, delegateDid, dwnUrl, protocol });
+        }
+      }
+    }
+
+    await Promise.allSettled(targets.map(t => this.initializeLinkTarget(t)));
+  }
+
+  /** Hot-remove a single identity from the active live sync session. */
+  private async removeIdentityFromLiveSync(did: string): Promise<void> {
+    for (const sub of this._liveSubscriptions.filter(s => s.did === did)) {
+      try { await sub.close(); } catch { /* best effort */ }
+    }
+    this._liveSubscriptions = this._liveSubscriptions.filter(s => s.did !== did);
+
+    for (const sub of this._localSubscriptions.filter(s => s.did === did)) {
+      try { await sub.close(); } catch { /* best effort */ }
+    }
+    this._localSubscriptions = this._localSubscriptions.filter(s => s.did !== did);
+
+    for (const [key, runtime] of this._pushRuntimes) {
+      if (runtime.did === did) {
+        if (runtime.timer) { clearTimeout(runtime.timer); }
+        this._pushRuntimes.delete(key);
+      }
+    }
+
+    for (const [key, timer] of this._degradedPollTimers) {
+      if (this.isLinkKeyForDid(key, did)) { clearInterval(timer); this._degradedPollTimers.delete(key); }
+    }
+    for (const key of this._repairAttempts.keys()) {
+      if (this.isLinkKeyForDid(key, did)) { this._repairAttempts.delete(key); }
+    }
+    for (const key of this._activeRepairs.keys()) {
+      if (this.isLinkKeyForDid(key, did)) { this._activeRepairs.delete(key); }
+    }
+    for (const key of this._repairContext.keys()) {
+      if (this.isLinkKeyForDid(key, did)) { this._repairContext.delete(key); }
+    }
+    for (const [key, timer] of this._repairRetryTimers) {
+      if (this.isLinkKeyForDid(key, did)) { clearTimeout(timer); this._repairRetryTimers.delete(key); }
+    }
+    for (const [key, timer] of this._reconcileTimers) {
+      if (this.isLinkKeyForDid(key, did)) { clearTimeout(timer); this._reconcileTimers.delete(key); }
+    }
+    for (const key of this._reconcileInFlight.keys()) {
+      if (this.isLinkKeyForDid(key, did)) { this._reconcileInFlight.delete(key); }
+    }
+    for (const key of this._activeLinks.keys()) {
+      if (this.isLinkKeyForDid(key, did)) { this._activeLinks.delete(key); this._linkRuntimes.delete(key); }
+    }
+    this._closureContexts.delete(did);
+  }
+
   // ---------------------------------------------------------------------------
   // Live pull: MessagesSubscribe to remote DWN
   // ---------------------------------------------------------------------------
@@ -1354,8 +1534,16 @@ export class SyncEngineLevel implements SyncEngine {
     // NOTE: The WebSocket client fires handlers without awaiting (fire-and-forget),
     // so multiple handlers can be in-flight concurrently. The ordinal tracker
     // ensures the checkpoint advances only when all earlier deliveries are committed.
+    // Capture the link reference at subscription-open time so we can
+    // detect remove+re-add via object identity, not just key existence.
+    const capturedLink = link;
+    const isStale = (): boolean =>
+      this._engineGeneration !== handlerGeneration ||
+      !this._activeLinks.has(cursorKey) ||
+      (capturedLink !== undefined && this._activeLinks.get(cursorKey) !== capturedLink);
+
     const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
-      if (this._engineGeneration !== handlerGeneration) {
+      if (isStale()) {
         return;
       }
 
@@ -1370,15 +1558,12 @@ export class SyncEngineLevel implements SyncEngine {
 
           if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
             console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
-            await this.transitionToRepairing(cursorKey, link);
+            if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
             return;
           }
           ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
-          // Drain committed entries. Do NOT unconditionally advance to the
-          // EOSE cursor — earlier stored events may still be in-flight
-          // (handlers are fire-and-forget). The checkpoint advances only as
-          // far as the contiguous drain reaches.
           this.drainCommittedPull(cursorKey);
+          if (isStale()) { return; }
           await this.ledger.saveLink(link);
         }
         // Transport is reachable — set connectivity to online.
@@ -1412,7 +1597,7 @@ export class SyncEngineLevel implements SyncEngine {
         // Domain validation: reject tokens from a different stream/epoch.
         if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
           console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
-          await this.transitionToRepairing(cursorKey, link);
+          if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
           return;
         }
 
@@ -1425,9 +1610,11 @@ export class SyncEngineLevel implements SyncEngine {
         // reconnect/repair. This is safe because the event is intentionally
         // excluded from this scope and doesn't need processing.
         if (link && !isEventInScope(event.message, link.scope)) {
-          ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
-          ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
-          await this.ledger.saveLink(link);
+          if (!isStale()) {
+            ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
+            ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
+            await this.ledger.saveLink(link);
+          }
           return;
         }
 
@@ -1459,6 +1646,7 @@ export class SyncEngineLevel implements SyncEngine {
           }
 
           await this.agent.dwn.processRawMessage(did, event.message, { dataStream });
+          if (isStale()) { return; }
 
           // Invalidate closure cache entries that may be affected by this message.
           // Must run before closure validation so subsequent evaluations in the
@@ -1472,7 +1660,7 @@ export class SyncEngineLevel implements SyncEngine {
           // For protocol-scoped links, verify that all hard dependencies for
           // this operation are locally present before considering it committed.
           // Full-tenant scope bypasses this entirely (returns complete with 0 queries).
-          if (link && link.scope.kind === 'protocol') {
+          if (link?.scope.kind === 'protocol') {
             const messageStore = this.agent.dwn.node.storage.messageStore;
             let closureCtx = this._closureContexts.get(did);
             if (!closureCtx) {
@@ -1486,6 +1674,8 @@ export class SyncEngineLevel implements SyncEngine {
               event.message, messageStore, link.scope, closureCtx
             );
 
+            if (isStale()) { return; }
+
             if (!closureResult.complete) {
               const failureCode = closureResult.failure!.code;
               const failureDetail = closureResult.failure!.detail;
@@ -1506,7 +1696,7 @@ export class SyncEngineLevel implements SyncEngine {
                 errorDetail    : failureDetail,
               });
 
-              await this.transitionToRepairing(cursorKey, link);
+              if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
               return;
             }
           }
@@ -1533,7 +1723,7 @@ export class SyncEngineLevel implements SyncEngine {
           // Guard: if the link transitioned to repairing while this handler was
           // in-flight (e.g., an earlier ordinal's handler failed concurrently),
           // skip all state mutations — the repair process owns progression now.
-          if (link && rt && link.status === 'live') {
+          if (link && rt && link.status === 'live' && !isStale()) {
             const entry = rt.inflight.get(ordinal);
             if (entry) { entry.committed = true; }
 
@@ -1584,7 +1774,7 @@ export class SyncEngineLevel implements SyncEngine {
           // Transition to repairing immediately — do NOT advance the checkpoint
           // past this failure or let later ordinals commit past it. SMT
           // reconciliation will discover and fill the gap.
-          if (link) {
+          if (link && !isStale()) {
             await this.transitionToRepairing(cursorKey, link);
           }
         }
@@ -1706,9 +1896,16 @@ export class SyncEngineLevel implements SyncEngine {
 
     const handlerGeneration = this._engineGeneration;
 
+    // Capture the link for identity-based staleness detection.
+    const capturedPushLink = this._activeLinks.get(target.linkKey);
+    const isPushStale = (): boolean =>
+      this._engineGeneration !== handlerGeneration ||
+      !this._activeLinks.has(target.linkKey) ||
+      (capturedPushLink !== undefined && this._activeLinks.get(target.linkKey) !== capturedPushLink);
+
     // Subscribe to the local DWN's EventLog.
     const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
-      if (this._engineGeneration !== handlerGeneration) {
+      if (isPushStale()) {
         return;
       }
 
@@ -1727,7 +1924,7 @@ export class SyncEngineLevel implements SyncEngine {
       // Accumulate the message CID for a debounced push.
       const targetKey = pushLinkKey;
       const cid = await Message.getCid(subMessage.event.message);
-      if (cid === undefined) {
+      if (cid === undefined || isPushStale()) {
         return;
       }
 
@@ -1788,11 +1985,25 @@ export class SyncEngineLevel implements SyncEngine {
   }
 
   private async flushPendingPushesForLink(linkKey: string): Promise<void> {
+    // Guard: bail if this link was hot-removed. Without this, a stale
+    // debounce timer or retry callback could send pushes after the DID
+    // was removed.
+    if (!this._activeLinks.has(linkKey)) {
+      return;
+    }
+
     const pushRuntime = this._pushRuntimes.get(linkKey);
     if (!pushRuntime) {
       return;
     }
 
+    // Capture the current active link identity so we can detect
+    // remove+re-add during the await pushMessages() call.
+    const flushLink = this._activeLinks.get(linkKey);
+    const isFlushStale = (): boolean =>
+      !this._activeLinks.has(linkKey) ||
+      (flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
+
     const { did, dwnUrl, delegateDid, protocol, entries: pushEntries, retryCount } = pushRuntime;
     pushRuntime.entries = [];
 
@@ -1814,6 +2025,10 @@ export class SyncEngineLevel implements SyncEngine {
         permissionsApi : this._permissionsApi,
       });
 
+      // If the link was replaced during pushMessages, abandon all
+      // post-push state mutations — the replacement session owns this key.
+      if (isFlushStale()) { return; }
+
       // Auto-clear dead letters for CIDs that succeeded — a previously
       // failed message may have been repaired by reconciliation.
       for (const cid of result.succeeded) {
@@ -1834,6 +2049,7 @@ export class SyncEngineLevel implements SyncEngine {
       }
 
       if (result.failed.length > 0) {
+        if (isFlushStale()) { return; }
         const failedSet = new Set(result.failed);
         const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
         this.requeueOrReconcile(linkKey, {
@@ -1850,6 +2066,7 @@ export class SyncEngineLevel implements SyncEngine {
         }
       }
     } catch (error: any) {
+      if (isFlushStale()) { return; }
       console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
       this.requeueOrReconcile(linkKey, {
         did, dwnUrl, delegateDid, protocol,
@@ -1944,10 +2161,10 @@ export class SyncEngineLevel implements SyncEngine {
   // ---------------------------------------------------------------------------
 
   /** Active reconcile timers, keyed by link key. */
-  private _reconcileTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
+  private readonly _reconcileTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
 
   /** Active reconcile operations, keyed by link key (dedup). */
-  private _reconcileInFlight: Map<string, Promise<void>> = new Map();
+  private readonly _reconcileInFlight: Map<string, Promise<void>> = new Map();
 
   /**
    * Schedule a per-link reconciliation after a short debounce. Coalesces
@@ -1962,7 +2179,14 @@ export class SyncEngineLevel implements SyncEngine {
     const timer = setTimeout((): void => {
       this._reconcileTimers.delete(linkKey);
       if (this._engineGeneration !== generation) { return; }
-      void this.reconcileLink(linkKey);
+      // Guard: bail if this link was hot-removed since the timer was
+      // scheduled. Without this, a stale timer could restart reconcile
+      // work for a DID that is no longer active.
+      if (!this._activeLinks.has(linkKey)) { return; }
+      void this.reconcileLink(linkKey).catch((): void => {
+        // Errors are already logged inside doReconcileLink; swallow here
+        // to prevent unhandled-rejection flakes in the test runner.
+      });
     }, delayMs);
     this._reconcileTimers.set(linkKey, timer);
   }
@@ -2002,13 +2226,19 @@ export class SyncEngineLevel implements SyncEngine {
     }
 
     const generation = this._engineGeneration;
+
+    // Identity guard: if the DID was hot-removed and re-added, this
+    // closure's captured `link` reference may no longer be the active
+    // link object. Bail before mutating the replacement's state.
+    const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
+
     const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, protocol } = link;
 
     try {
       const reconcileOutcome = await this.createLinkReconciler(
-        () => this._engineGeneration === generation
+        () => this._engineGeneration === generation && !isStaleLink()
       ).reconcile({ did, dwnUrl, delegateDid, protocol }, { verifyConvergence: true });
-      if (reconcileOutcome.aborted) { return; }
+      if (reconcileOutcome.aborted || isStaleLink()) { return; }
 
       if (reconcileOutcome.converged) {
         await this.ledger.clearNeedsReconcile(link);
@@ -2020,9 +2250,10 @@ export class SyncEngineLevel implements SyncEngine {
         // Roots still differ — retry after a delay. This can happen when
         // pushMessages() had permanent failures, pullMessages() partially
         // failed, or new writes arrived during reconciliation.
-        this.scheduleReconcile(linkKey, 5000);
+        if (!isStaleLink()) { this.scheduleReconcile(linkKey, 5000); }
       }
     } catch (error: any) {
+      if (isStaleLink()) { return; }
       console.error(`SyncEngineLevel: Reconciliation failed for ${did} -> ${dwnUrl}`, error);
       // Schedule retry with longer delay.
       this.scheduleReconcile(linkKey, 5000);
@@ -2218,9 +2449,9 @@ export class SyncEngineLevel implements SyncEngine {
   private async getLocalRoot(did: string, delegateDid?: string, protocol?: string): Promise<string> {
     const si = this.stateIndex;
     if (si) {
-      const rootHash = protocol !== undefined
-        ? await si.getProtocolRoot(did, protocol)
-        : await si.getRoot(did);
+      const rootHash = protocol === undefined
+        ? await si.getRoot(did)
+        : await si.getProtocolRoot(did, protocol);
       return hashToHex(rootHash);
     }
 
@@ -2363,9 +2594,9 @@ export class SyncEngineLevel implements SyncEngine {
       if (si) {
         // Fast path: direct StateIndex access (local mode).
         const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
-        const hash = protocol !== undefined
-          ? await si.getProtocolSubtreeHash(did, protocol, bitPath)
-          : await si.getSubtreeHash(did, bitPath);
+        const hash = protocol === undefined
+          ? await si.getSubtreeHash(did, bitPath)
+          : await si.getProtocolSubtreeHash(did, protocol, bitPath);
         hexHash = hashToHex(hash);
       } else {
         // Remote mode fallback.
@@ -2405,9 +2636,9 @@ export class SyncEngineLevel implements SyncEngine {
     const si = this.stateIndex;
     if (si) {
       const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
-      const hash = protocol !== undefined
-        ? await si.getProtocolSubtreeHash(did, protocol, bitPath)
-        : await si.getSubtreeHash(did, bitPath);
+      const hash = protocol === undefined
+        ? await si.getSubtreeHash(did, bitPath)
+        : await si.getProtocolSubtreeHash(did, protocol, bitPath);
       return hashToHex(hash);
     }
 
@@ -2440,9 +2671,9 @@ export class SyncEngineLevel implements SyncEngine {
     const si = this.stateIndex;
     if (si) {
       const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
-      return protocol !== undefined
-        ? await si.getProtocolLeaves(did, protocol, bitPath)
-        : await si.getLeaves(did, bitPath);
+      return protocol === undefined
+        ? await si.getLeaves(did, bitPath)
+        : await si.getProtocolLeaves(did, protocol, bitPath);
     }
 
     // Remote mode fallback.
@@ -2735,6 +2966,9 @@ export class SyncEngineLevel implements SyncEngine {
 
   /**
    * Returns the list of sync targets: (did, dwnUrl, delegateDid?, protocol?) tuples.
+   * Results are cached for up to 30 seconds to avoid redundant DID resolution
+   * on every sync tick. The cache is invalidated when identities are registered,
+   * unregistered, or updated.
    */
   private async getSyncTargets(): Promise<{
     did: string;
@@ -2742,25 +2976,41 @@ export class SyncEngineLevel implements SyncEngine {
     delegateDid?: string;
     protocol?: string;
   }[]> {
+    // Return cached targets if still valid.
+    if (this._syncTargetsCache
+        && (Date.now() - this._syncTargetsCache.timestamp) < SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS) {
+      return this._syncTargetsCache.targets;
+    }
+
+    // Capture the generation before any async work so we can detect
+    // concurrent invalidations (register/unregister/update) that would
+    // make our result stale.
+    const generationAtStart = this._syncTargetsCacheGeneration;
+
     const targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[] = [];
+    let hasRegisteredIdentities = false;
+    let anyEndpointMissing = false;
 
     for await (const [did, options] of this._db.sublevel('registeredIdentities').iterator()) {
+      hasRegisteredIdentities = true;
       let parsed: SyncIdentityOptions;
       try {
         parsed = JSON.parse(options) as SyncIdentityOptions;
       } catch (error: unknown) {
-        console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, falling back to global sync:`, error);
-        parsed = { protocols: [] };
+        console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping identity:`, error);
+        continue;
       }
+
       const { protocols, delegateDid } = parsed;
 
       const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
       if (dwnEndpointUrls.length === 0) {
+        anyEndpointMissing = true;
         continue;
       }
 
       for (const dwnUrl of dwnEndpointUrls) {
-        if (protocols.length === 0) {
+        if (protocols === 'all') {
           // Sync all protocols (global tree).
           targets.push({ did, delegateDid, dwnUrl });
         } else {
@@ -2771,6 +3021,17 @@ export class SyncEngineLevel implements SyncEngine {
       }
     }
 
+    // Only cache when:
+    // - The result is non-empty (empty = transient resolution failure).
+    // - All registered identities resolved successfully (partial =
+    //   one identity's endpoints failed transiently; caching would
+    //   suppress retries for that identity for the full TTL).
+    // - The generation hasn't changed (a concurrent register/unregister
+    //   invalidated the cache while we were awaiting).
+    const isComplete = hasRegisteredIdentities && !anyEndpointMissing;
+    if (targets.length > 0 && isComplete && this._syncTargetsCacheGeneration === generationAtStart) {
+      this._syncTargetsCache = { targets, timestamp: Date.now() };
+    }
     return targets;
   }