@enbox/agent 0.5.15 → 0.6.0

package/dist/esm/sync-engine-level.js

@@ -179,6 +179,10 @@ export class SyncEngineLevel {
         }
         return this._ledger;
     }
+    /** LevelDB sublevel for permanently failed messages (dead letters). */
+    get _deadLetters() {
+        return this._db.sublevel('deadLetters');
+    }
     /**
      * Retrieves the `EnboxPlatformAgent` execution context.
      *
@@ -475,6 +479,9 @@ export class SyncEngineLevel {
      */
     startLiveSync(intervalMilliseconds) {
         return __awaiter(this, void 0, void 0, function* () {
+            // Step 0: Register browser connectivity listeners for instant recovery
+            // on network switch, sleep/wake, or tab foregrounding. No-op in Node.
+            this.startBrowserConnectivityListeners();
             // Step 1: Initial SMT catch-up.
             try {
                 yield this.sync();
@@ -842,6 +849,11 @@ export class SyncEngineLevel {
                 const prevRepairConnectivity = link.connectivity;
                 link.connectivity = 'online';
                 yield this.ledger.setStatus(link, 'live');
+                // Auto-clear dead letters for this link — repair has verified
+                // convergence via SMT reconciliation so any previously recorded
+                // failures (closure, push-exhausted, pull-processing) for this
+                // specific link are no longer current.
+                void this.clearDeadLettersForLink(did, dwnUrl, protocol);
                 this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
                 if (prevRepairConnectivity !== 'online') {
                     this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevRepairConnectivity, to: 'online' });
@@ -959,8 +971,106 @@ export class SyncEngineLevel {
     /**
      * Tears down all live subscriptions and push listeners.
      */
+    // ---------------------------------------------------------------------------
+    // Browser connectivity: online/offline + visibilitychange
+    // ---------------------------------------------------------------------------
+    /**
+     * Registers browser `online`, `offline`, and `visibilitychange` event
+     * listeners to detect connectivity changes that WebSocket `close` events
+     * miss (NAT timeout, network switch, sleep/wake). Safe to call in Node —
+     * the guards skip registration when browser APIs are unavailable.
+     */
+    startBrowserConnectivityListeners() {
+        this.stopBrowserConnectivityListeners();
+        // Guard: only run in browser environments with the required APIs.
+        if (typeof globalThis.addEventListener !== 'function') {
+            return;
+        }
+        const generation = this._engineGeneration;
+        this._onOnline = () => {
+            if (this._engineGeneration !== generation) {
+                return;
+            }
+            console.info('SyncEngineLevel: browser online — triggering immediate integrity check');
+            // Don't set _connectivityState here — individual links will transition
+            // to online as their WebSocket connections actually recover during the
+            // sync below. The public getter uses per-link aggregation.
+            // Kick off an immediate SMT reconciliation to catch up after being offline.
+            if (!this._syncLock) {
+                this.sync().catch((err) => {
+                    console.error('SyncEngineLevel: post-online sync failed', err);
+                });
+            }
+        };
+        this._onOffline = () => {
+            if (this._engineGeneration !== generation) {
+                return;
+            }
+            console.info('SyncEngineLevel: browser offline');
+            this._connectivityState = 'offline';
+            // Transition every active link to offline so the public
+            // connectivityState getter (which aggregates per-link state)
+            // reflects the browser's network status immediately.
+            for (const link of this._activeLinks.values()) {
+                const prev = link.connectivity;
+                if (prev !== 'offline') {
+                    link.connectivity = 'offline';
+                    this.emitEvent({
+                        type: 'link:connectivity-change',
+                        tenantDid: link.tenantDid,
+                        remoteEndpoint: link.remoteEndpoint,
+                        protocol: link.protocol,
+                        from: prev,
+                        to: 'offline',
+                    });
+                }
+            }
+        };
+        this._onVisibilityChange = () => {
+            if (this._engineGeneration !== generation) {
+                return;
+            }
+            // Only act when the page becomes visible again — the user is back.
+            if (typeof document === 'undefined' || document.visibilityState !== 'visible') {
+                return;
+            }
+            console.info('SyncEngineLevel: page became visible — triggering integrity check');
+            // The device may have slept and WebSockets may be dead. An immediate
+            // sync via SMT reconciliation detects and repairs any divergence.
+            if (!this._syncLock) {
+                this.sync().catch((err) => {
+                    console.error('SyncEngineLevel: post-visibility sync failed', err);
+                });
+            }
+        };
+        globalThis.addEventListener('online', this._onOnline);
+        globalThis.addEventListener('offline', this._onOffline);
+        if (typeof document !== 'undefined') {
+            document.addEventListener('visibilitychange', this._onVisibilityChange);
+        }
+    }
+    /** Removes browser connectivity listeners if they were registered. */
+    stopBrowserConnectivityListeners() {
+        if (this._onOnline) {
+            globalThis.removeEventListener('online', this._onOnline);
+            this._onOnline = undefined;
+        }
+        if (this._onOffline) {
+            globalThis.removeEventListener('offline', this._onOffline);
+            this._onOffline = undefined;
+        }
+        if (this._onVisibilityChange && typeof document !== 'undefined') {
+            document.removeEventListener('visibilitychange', this._onVisibilityChange);
+            this._onVisibilityChange = undefined;
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Teardown
+    // ---------------------------------------------------------------------------
     teardownLiveSync() {
         return __awaiter(this, void 0, void 0, function* () {
+            // Remove browser connectivity listeners before tearing down.
+            this.stopBrowserConnectivityListeners();
             // Increment generation to invalidate all in-flight async operations
             // (repairs, retry timers, degraded-poll ticks). Any async work that
             // captured the previous generation will bail on its next checkpoint.
@@ -1079,6 +1189,7 @@ export class SyncEngineLevel {
             // so multiple handlers can be in-flight concurrently. The ordinal tracker
             // ensures the checkpoint advances only when all earlier deliveries are committed.
             const subscriptionHandler = (subMessage) => __awaiter(this, void 0, void 0, function* () {
+                var _a;
                 if (this._engineGeneration !== handlerGeneration) {
                     return;
                 }
@@ -1194,8 +1305,21 @@ export class SyncEngineLevel {
                             }
                             const closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
                             if (!closureResult.complete) {
+                                const failureCode = closureResult.failure.code;
+                                const failureDetail = closureResult.failure.detail;
                                 console.warn(`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
-                                    `${closureResult.failure.code} — ${closureResult.failure.detail}`);
+                                    `${failureCode} — ${failureDetail}`);
+                                // Record the message that triggered the closure failure.
+                                const closureCid = yield Message.getCid(event.message);
+                                void this.recordDeadLetter({
+                                    messageCid: closureCid,
+                                    tenantDid: did,
+                                    remoteEndpoint: dwnUrl,
+                                    protocol,
+                                    category: 'closure',
+                                    errorCode: failureCode,
+                                    errorDetail: failureDetail,
+                                });
                                 yield this.transitionToRepairing(cursorKey, link);
                                 return;
                             }
@@ -1212,6 +1336,9 @@ export class SyncEngineLevel {
                         const pulledCid = yield Message.getCid(event.message);
                         this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
                         this.evictExpiredEchoEntries();
+                        // Auto-clear any dead letter for this CID — it was processed
+                        // successfully, so a previous failure has been self-healed.
+                        this.clearFailedMessage(pulledCid, dwnUrl).catch(() => { });
                         // Mark this ordinal as committed and drain the checkpoint.
                         // Guard: if the link transitioned to repairing while this handler was
                         // in-flight (e.g., an earlier ordinal's handler failed concurrently),
@@ -1246,6 +1373,23 @@ export class SyncEngineLevel {
                     }
                     catch (error) {
                         console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
+                        // Record the failing message in the dead letter store before
+                        // transitioning to repair. The CID identifies which specific
+                        // message caused the transition.
+                        try {
+                            const failedCid = yield Message.getCid(event.message);
+                            void this.recordDeadLetter({
+                                messageCid: failedCid,
+                                tenantDid: did,
+                                remoteEndpoint: dwnUrl,
+                                protocol,
+                                category: 'pull-processing',
+                                errorDetail: (_a = error.message) !== null && _a !== void 0 ? _a : String(error),
+                            });
+                        }
+                        catch (_b) {
+                            // Best effort — don't let dead letter recording block repair.
+                        }
                         // A failed processRawMessage means local state is incomplete.
                         // Transition to repairing immediately — do NOT advance the checkpoint
                         // past this failure or let later ordinals commit past it. SMT
@@ -1432,6 +1576,7 @@ export class SyncEngineLevel {
     }
     flushPendingPushesForLink(linkKey) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
             const pushRuntime = this._pushRuntimes.get(linkKey);
             if (!pushRuntime) {
                 return;
@@ -1453,6 +1598,23 @@ export class SyncEngineLevel {
                     agent: this.agent,
                     permissionsApi: this._permissionsApi,
                 });
+                // Auto-clear dead letters for CIDs that succeeded — a previously
+                // failed message may have been repaired by reconciliation.
+                for (const cid of result.succeeded) {
+                    this.clearFailedMessage(cid, dwnUrl).catch(() => { });
+                }
+                // Record permanently failed messages in the dead letter store.
+                for (const entry of result.permanentlyFailed) {
+                    yield this.recordDeadLetter({
+                        messageCid: entry.cid,
+                        tenantDid: did,
+                        remoteEndpoint: dwnUrl,
+                        protocol,
+                        category: 'push-permanent',
+                        errorCode: String((_a = entry.statusCode) !== null && _a !== void 0 ? _a : ''),
+                        errorDetail: (_b = entry.detail) !== null && _b !== void 0 ? _b : 'permanent push failure',
+                    });
+                }
                 if (result.failed.length > 0) {
                     const failedSet = new Set(result.failed);
                     const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
@@ -1504,7 +1666,18 @@ export class SyncEngineLevel {
         const maxRetries = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS.length;
         const pushRuntime = this.getOrCreatePushRuntime(targetKey, pending);
         if (pending.retryCount >= maxRetries) {
-            // Retry budget exhausted — mark link dirty for reconciliation.
+            // Retry budget exhausted — record each CID as a dead letter and mark
+            // the link dirty for reconciliation.
+            for (const entry of pending.entries) {
+                void this.recordDeadLetter({
+                    messageCid: entry.cid,
+                    tenantDid: pending.did,
+                    remoteEndpoint: pending.dwnUrl,
+                    protocol: pending.protocol,
+                    category: 'push-exhausted',
+                    errorDetail: `push retries exhausted after ${maxRetries} attempts`,
+                });
+            }
             if (pushRuntime.timer) {
                 clearTimeout(pushRuntime.timer);
             }
@@ -1609,6 +1782,9 @@ export class SyncEngineLevel {
                 }
                 if (reconcileOutcome.converged) {
                     yield this.ledger.clearNeedsReconcile(link);
+                    // SMT roots match — this link is converged. Clear dead letters
+                    // scoped to this specific link (tenantDid, remoteEndpoint, protocol).
+                    void this.clearDeadLettersForLink(did, dwnUrl, protocol);
                     this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
                 }
                 else {
@@ -2035,11 +2211,22 @@ export class SyncEngineLevel {
      */
     pullMessages(_a) {
         return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched }) {
-            return pullMessages({
+            const failedCids = yield pullMessages({
                 did, dwnUrl, delegateDid, protocol, messageCids, prefetched,
                 agent: this.agent,
                 permissionsApi: this._permissionsApi,
             });
+            // Record permanently failed pull entries in the dead letter store.
+            for (const cid of failedCids) {
+                yield this.recordDeadLetter({
+                    messageCid: cid,
+                    tenantDid: did,
+                    remoteEndpoint: dwnUrl,
+                    protocol,
+                    category: 'pull-processing',
+                    errorDetail: 'pull processing failed after retry passes exhausted',
+                });
+            }
         });
     }
     // ---------------------------------------------------------------------------
@@ -2110,12 +2297,226 @@ export class SyncEngineLevel {
     static topologicalSort(messages) {
         return topologicalSort(messages);
     }
+    // ---------------------------------------------------------------------------
+    // Dead letter tracking
+    // ---------------------------------------------------------------------------
+    /**
+     * Clear dead letter entries scoped to a specific sync link. Matches on
+     * (tenantDid, remoteEndpoint, protocol) so that repairing protocol A
+     * does not erase still-valid failures for protocol B on the same remote.
+     * When `protocol` is undefined (full-tenant link), clears entries that
+     * also have no protocol.
+     */
+    clearDeadLettersForLink(tenantDid, remoteEndpoint, protocol) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const batch = [];
+            try {
+                try {
+                    for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                        _c = _f.value;
+                        _d = false;
+                        const [key, value] = _c;
+                        const entry = JSON.parse(value);
+                        if (entry.tenantDid === tenantDid &&
+                            entry.remoteEndpoint === remoteEndpoint &&
+                            entry.protocol === protocol) {
+                            batch.push({ type: 'del', key });
+                        }
+                    }
+                }
+                catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                finally {
+                    try {
+                        if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                }
+                if (batch.length > 0) {
+                    yield this._deadLetters.batch(batch);
+                }
+            }
+            catch (error) {
+                const e = error;
+                if (e.code !== 'LEVEL_DATABASE_NOT_OPEN') {
+                    throw error;
+                }
+            }
+        });
+    }
+    /**
+     * Build a compound dead letter key. Different remotes can fail the same CID
+     * for different reasons, so the key includes the remote endpoint.
+     */
+    static deadLetterKey(messageCid, remoteEndpoint) {
+        return remoteEndpoint ? `${messageCid}|${remoteEndpoint}` : messageCid;
+    }
+    recordDeadLetter(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const entry = Object.assign(Object.assign({}, params), { failedAt: new Date().toISOString() });
+            const key = SyncEngineLevel.deadLetterKey(params.messageCid, params.remoteEndpoint);
+            try {
+                yield this._deadLetters.put(key, JSON.stringify(entry));
+            }
+            catch (error) {
+                // Suppress only the expected teardown race — any other error surfaces.
+                const e = error;
+                if (e.code !== 'LEVEL_DATABASE_NOT_OPEN') {
+                    throw error;
+                }
+            }
+        });
+    }
+    getFailedMessages(tenantDid) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_2, _b, _c;
+            const entries = [];
+            try {
+                for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                    _c = _f.value;
+                    _d = false;
+                    const [, value] = _c;
+                    const entry = JSON.parse(value);
+                    if (!tenantDid || entry.tenantDid === tenantDid) {
+                        entries.push(entry);
+                    }
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            // Deterministic ordering: newest first so apps see the most recent failures.
+            entries.sort((a, b) => b.failedAt.localeCompare(a.failedAt));
+            return entries;
+        });
+    }
+    clearFailedMessage(messageCid, remoteEndpoint) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_3, _b, _c;
+            if (remoteEndpoint) {
+                // Clear a specific CID + remote pair.
+                const key = SyncEngineLevel.deadLetterKey(messageCid, remoteEndpoint);
+                try {
+                    yield this._deadLetters.get(key);
+                    yield this._deadLetters.del(key);
+                    return true;
+                }
+                catch (error) {
+                    const e = error;
+                    if (e.code === 'LEVEL_NOT_FOUND') {
+                        return false;
+                    }
+                    throw error;
+                }
+            }
+            // No remote specified — clear ALL entries for this CID (any remote).
+            let found = false;
+            const batch = [];
+            try {
+                for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                    _c = _f.value;
+                    _d = false;
+                    const [key, value] = _c;
+                    const entry = JSON.parse(value);
+                    if (entry.messageCid === messageCid) {
+                        batch.push({ type: 'del', key });
+                        found = true;
+                    }
+                }
+            }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                }
+                finally { if (e_3) throw e_3.error; }
+            }
+            if (batch.length > 0) {
+                yield this._deadLetters.batch(batch);
+            }
+            return found;
+        });
+    }
+    clearAllFailedMessages(tenantDid) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_4, _b, _c;
+            if (!tenantDid) {
+                yield this._deadLetters.clear();
+                return;
+            }
+            const batch = [];
+            try {
+                for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                    _c = _f.value;
+                    _d = false;
+                    const [key, value] = _c;
+                    const entry = JSON.parse(value);
+                    if (entry.tenantDid === tenantDid) {
+                        batch.push({ type: 'del', key });
+                    }
+                }
+            }
+            catch (e_4_1) { e_4 = { error: e_4_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                }
+                finally { if (e_4) throw e_4.error; }
+            }
+            if (batch.length > 0) {
+                yield this._deadLetters.batch(batch);
+            }
+        });
+    }
+    getSyncHealth() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_5, _b, _c;
+            let failedMessageCount = 0;
+            try {
+                for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                    _c = _f.value;
+                    _d = false;
+                    const _ = _c;
+                    failedMessageCount++;
+                }
+            }
+            catch (e_5_1) { e_5 = { error: e_5_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+                }
+                finally { if (e_5) throw e_5.error; }
+            }
+            // Count degraded links from the durable ledger, not just in-memory
+            // _activeLinks. Links persist across restarts; a repairing/degraded_poll
+            // link from a previous session must still be reported.
+            let degradedLinkCount = 0;
+            const allLinks = yield this.ledger.getAllLinks();
+            for (const link of allLinks) {
+                if (link.status === 'repairing' || link.status === 'degraded_poll') {
+                    degradedLinkCount++;
+                }
+            }
+            return {
+                connectivity: this.connectivityState,
+                failedMessageCount,
+                degradedLinkCount,
+            };
+        });
+    }
+    // ---------------------------------------------------------------------------
+    // Sync targets
+    // ---------------------------------------------------------------------------
     /**
      * Returns the list of sync targets: (did, dwnUrl, delegateDid?, protocol?) tuples.
      */
     getSyncTargets() {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
+            var _a, e_6, _b, _c;
             const targets = [];
             try {
                 for (var _d = true, _e = __asyncValues(this._db.sublevel('registeredIdentities').iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
@@ -2148,12 +2549,12 @@ export class SyncEngineLevel {
                     }
                 }
             }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            catch (e_6_1) { e_6 = { error: e_6_1 }; }
             finally {
                 try {
                     if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
                 }
-                finally { if (e_1) throw e_1.error; }
+                finally { if (e_6) throw e_6.error; }
             }
             return targets;
         });