@glance-apps/sync 1.8.0 → 1.11.0

package/src/dbEngine.js

@@ -36,6 +36,35 @@ const ts = (v) => {
   return Number.isNaN(t) ? 0 : t;
 };
 
+// ── The backoff ladder ──────────────────────────────────────────────────────
+//
+// ONE windowing mechanism for every recoverable failure, carrying a `reason`.
+// It replaces the separate quota-suppression window added in 3.3, which was
+// already built to this shape — this consolidates two parallel things rather
+// than forcing two different ones together.
+//
+// The numbers, the escalation curve, the reset rule and the in-memory
+// lifetime are the FILE TIER'S (src/engine.js): 30s doubling to a 15-minute
+// cap for writes and a 5-minute cap for reads, a flat hour on an auth
+// failure, and any success resets both the counter and the window. That file
+// was read and deliberately not refactored.
+//
+// The ONE thing that differs, and the reason this fix exists: the file tier's
+// backoff is ADVISORY — nothing reads uploadBackoffUntil except its getter,
+// so honouring it is the calling app's job. That is why "the DB engine
+// hammers" is live in every published version. Here the engine ENFORCES its
+// own window (it skips the work when called too soon, exactly as 3.3's quota
+// suppression did) AND exposes the same getters, so a scheduler can still be
+// smart without being obliged to be.
+//
+// A window never becomes terminal: it only ever delays. Whatever the
+// interval, the next ordinary cycle after it expires probes again, with no
+// user action and no restart.
+const BACKOFF_BASE_S = 30;
+const MAX_PUSH_BACKOFF_S = 15 * 60; // writes, matching the file tier's upload cap
+const MAX_PULL_BACKOFF_S = 5 * 60;  // reads, matching the file tier's download cap
+const AUTH_BACKOFF_MS = 60 * 60 * 1000; // flat hour, matching AUTH_FAILURE_BACKOFF_MS
+
 // randomUUID is missing in some non-secure-context WebViews; getRandomValues
 // is everywhere the rest of this package's crypto already runs.
 const randomUUID = () => {
@@ -197,6 +226,22 @@ export const createDbSyncEngine = (config) => {
   // stale reference the app forgot after recovery dies quietly instead of
   // hammering the server with a dead credential.
   let supersededInert = false;
+  // The two backoff windows. IN MEMORY ONLY, like the file tier's: a restart
+  // clears them, which is the right default — a relaunch is a user signalling
+  // "try now", and the failure re-opens the window immediately if it persists.
+  // Nothing here is persisted, so nothing can outlive the condition it
+  // describes or need a clearing path.
+  const newWindow = () => ({
+    until: 0,        // epoch ms; the work is skipped while this is in the future
+    strikes: 0,      // consecutive failures, driving the escalation
+    reason: null,    // 'quota' | 'auth' | 'transport'
+    code: null,      // the typed error code that opened it
+    message: null,
+    quota: null,     // the parsed descriptor, when reason === 'quota'
+    since: null,     // ISO of the most recent failure
+  });
+  const pushBackoff = newWindow();
+  const pullBackoff = newWindow();
   // Per-session cache: once the derived root key is proven against the account's
   // existing data (Part A), we never re-verify for the life of this engine.
   let verified = false;
@@ -293,6 +338,132 @@ export const createDbSyncEngine = (config) => {
     localStorage.setItem(KEY_CRED_HALT, JSON.stringify({ message, at: new Date().toISOString() }));
   };
 
+  // ── The backoff ladder: classify, open, clear ─────────────────────────────
+
+  // WHICH FAILURES OPEN A WINDOW. Returns null for the classes that must NOT
+  // be delayed:
+  //  - Readiness (PASSPHRASE_REQUIRED / ACCOUNT_ID_REQUIRED) fires BEFORE any
+  //    network call and clears the instant the app supplies the value. A
+  //    30-second delay after a user types their passphrase would read as
+  //    broken. The file tier has no equivalent class, so copying it faithfully
+  //    would have missed this.
+  //  - CREDENTIAL_INVALID is owned by the credential halt, which is terminal.
+  //    Stacking a window on top would be meaningless at best and could soften
+  //    a terminal state at worst.
+  const backoffReasonFor = (err) => {
+    const code = err && err.code;
+    if (code === 'PASSPHRASE_REQUIRED' || code === 'ACCOUNT_ID_REQUIRED') return null;
+    if (code === 'CREDENTIAL_INVALID') return null;
+    // QUOTA_EXCEEDED keeps its own reason so its descriptor keeps surfacing —
+    // but it is ONE window, not a second one stacked on a transport window.
+    if (code === 'QUOTA_EXCEEDED') return 'quota';
+    // A shared-mode 401 ("invalid device token") is a wrong/rotated token: it
+    // will not fix itself in 30 seconds and hammering an auth endpoint is the
+    // worst kind of noise, so it gets the file tier's flat hour. This is NOT
+    // the credential halt — it stays retryable and self-resuming.
+    //
+    // `cause` is consulted because the key verifier re-wraps whatever it hits
+    // as VERIFIER_UNSUPPORTED (keeping the original on err.cause), which would
+    // otherwise hide a 401 behind a fresh error carrying no status.
+    if (err && (err.status === 401 || (err.cause && err.cause.status === 401))) return 'auth';
+    return 'transport';
+  };
+
+  const openWindow = (w, err, reason, maxSeconds) => {
+    w.strikes += 1;
+    const ms = reason === 'auth'
+      ? AUTH_BACKOFF_MS
+      : Math.min(BACKOFF_BASE_S * Math.pow(2, w.strikes - 1), maxSeconds) * 1000;
+    w.until = Date.now() + ms;
+    w.reason = reason;
+    w.code = (err && err.code) || 'NETWORK_ERROR';
+    w.message = err?.message || String(err);
+    w.quota = reason === 'quota' && err && err.quota ? err.quota : null;
+    w.since = new Date().toISOString();
+  };
+
+  const clearWindow = (w) => {
+    w.until = 0; w.strikes = 0; w.reason = null;
+    w.code = null; w.message = null; w.quota = null; w.since = null;
+  };
+
+  const isOpen = (w) => w.until > Date.now();
+
+  // Record a failure against a window, honouring the do-not-delay classes.
+  // Returns the reason, or null when the failure opened no window.
+  const recordFailure = (w, err, maxSeconds) => {
+    const reason = backoffReasonFor(err);
+    if (reason) openWindow(w, err, reason, maxSeconds);
+    return reason;
+  };
+
+  // Over-quota state, preserved exactly as 3.3 exposed it: non-null from the
+  // rejection until a write actually succeeds (not merely until the window
+  // expires), so an app can keep rendering "X of Y used" while the client
+  // probes. Now derived from the push window rather than a parallel variable.
+  const getQuotaState = () => {
+    if (pushBackoff.reason !== 'quota') return null;
+    const d = pushBackoff.quota;
+    return {
+      quota: d ? d.quota : 'unknown',
+      limit: d ? d.limit : null,
+      used: d ? d.used : null,
+      requested: d ? d.requested : null,
+      message: pushBackoff.message,
+      since: pushBackoff.since,
+      retryAt: new Date(pushBackoff.until).toISOString(),
+    };
+  };
+  const isQuotaSuppressed = () => pushBackoff.reason === 'quota' && isOpen(pushBackoff);
+
+  // PER-REASON SURFACING on a cycle whose work was SKIPPED because a window is
+  // open. A quota window keeps surfacing QUOTA_EXCEEDED with its descriptor
+  // every cycle, because apps render "X of Y used" from it and a UI built
+  // after the rejection must still learn about it. A transport or auth window
+  // stays quiet after its first signal — the failure was already reported and
+  // repeating it every cycle is the noise this fix exists to remove. The
+  // status still reads 'error' in both cases: sync genuinely is not working.
+  //
+  // APP-INTEGRATION CONSEQUENCE (deliberate, do not "fix" by re-surfacing):
+  // because the cycle's onError(null) reset still fires on suppressed cycles,
+  // a quiet window means the app's LAST onError MESSAGE is cleared while
+  // onStatusChange reads 'error'. An app must therefore render standing
+  // backoff from getBackoffState() (reason + until — "retrying in Xs"), not
+  // from a remembered onError string. Surface this when integrating the
+  // three apps.
+  const surfaceStandingWindow = (w) => {
+    if (w.reason === 'quota') onError?.(w.message, 'QUOTA_EXCEEDED', false);
+  };
+
+  // The credential-rejection path, extracted UNCHANGED from the cycle's catch
+  // so a 401 surfacing from the push or the pull reaches exactly the same
+  // logic it always reached from ensureRootKey. Nothing about the halt, the
+  // identity rule, or the superseded-inertness decision is altered — this is
+  // only about where the error is allowed to arrive from. Under NO
+  // circumstances does the engine re-enroll here; recovery is user-initiated
+  // (recoverVaultSyncEngine).
+  // Returns the truthful result flag for the cycle to spread — { superseded }
+  // when this instance went inert, { halted } when the device halted — so the
+  // cycle result matches what isSuperseded()/isCredentialHalted() report,
+  // whichever path the rejection arrived by.
+  const applyCredentialRejection = (err) => {
+    const message = err?.message || String(err);
+    let outcome;
+    if (isBearerSuperseded()) {
+      // Replaced by recovery and the app kept a stale reference: go inert in
+      // memory — surfaced once, then silent — leaving the shared halt key
+      // alone so the recovered engine keeps syncing.
+      supersededInert = true;
+      outcome = { superseded: true };
+    } else {
+      setCredentialHalt(message);
+      outcome = { halted: true };
+    }
+    onError?.(message, 'CREDENTIAL_INVALID', true);
+    onStatusChange?.('error');
+    return outcome;
+  };
+
   // The halt-set identity rule (Phase 2.2). The halt key is shared by every
   // engine instance on this device, so a STALE instance — one still holding a
   // credential that recovery has since replaced — must not be able to re-set
@@ -355,8 +526,13 @@ export const createDbSyncEngine = (config) => {
   // Likewise a rejected credential (401 "invalid credential") is an auth
   // outcome, not a capability one: pass it through so the cycle's halt
   // handling sees it — and so allowUnverified can never wave a revoked
-  // device through.
-  const isPassThrough = (err) => isClientNotReady(err) || (err && err.code === 'CREDENTIAL_INVALID');
+  // device through. A quota rejection (Phase 3.3) is the same kind of
+  // mistake to make: the verifier's establishing write is a net-new entity,
+  // so an account at the row cap gets a 413 here, and relabelling it
+  // VERIFIER_UNSUPPORTED tells the user to update a server that is working
+  // correctly. Pass it through so the cycle reports the truth.
+  const isPassThrough = (err) =>
+    isClientNotReady(err) || (err && (err.code === 'CREDENTIAL_INVALID' || err.code === 'QUOTA_EXCEEDED'));
 
   const verifierUnsupported = (cause) => {
     if (isPassThrough(cause)) throw cause;
@@ -652,11 +828,22 @@ export const createDbSyncEngine = (config) => {
         if (typeof R.seq === 'number') since = Math.max(since, R.seq);
       }
 
+      // Persist the cursor PER PAGE, not once at the end. Every row in a page
+      // reaches a terminal outcome before we get here — applied, quarantined,
+      // or skipped as engine-reserved — so the cursor may safely advance past
+      // the whole page. Previously a failure on page 3 discarded the progress
+      // of pages 1 and 2, so a large backlog on a flaky connection could
+      // re-download the same pages forever and never converge; that is exactly
+      // the situation backoff is meant to help, so the two changes belong
+      // together.
+      setHighWaterMark(Math.max(getHighWaterMark(), maxSeq));
+
       hasMore = !!more;
     }
 
     // Pull is the sole writer of the pull cursor. Advance it (monotonically)
-    // from the highest seq we actually listed this run.
+    // from the highest seq we actually listed this run. Retained for the
+    // zero-page case, and harmless after the per-page advance above.
     setHighWaterMark(Math.max(getHighWaterMark(), maxSeq));
     return { maxSeq, appliedRemote, applied, skipped, skippedEntityIds };
   };
@@ -702,52 +889,135 @@ export const createDbSyncEngine = (config) => {
       return { applied: 0, skipped: 0, skippedEntityIds: [], halted: true };
     }
     if (syncing) return;
+
+    // Nothing at all can proceed while the key is unverified AND both windows
+    // are open: proving the key needs the network, and both directions are
+    // backed off. (Once verified in-session, ensureRootKey is a local check,
+    // so a single open window only skips its own half of the cycle.)
+    if (!verified && isOpen(pushBackoff) && isOpen(pullBackoff)) {
+      surfaceStandingWindow(pushBackoff);
+      onStatusChange?.('error');
+      return {
+        applied: 0, skipped: 0, skippedEntityIds: [],
+        pushSkipped: true, pullSkipped: true, quota: getQuotaState(),
+      };
+    }
+
     syncing = true;
     onStatusChange?.('uploading');
     onError?.(null, null, false);
     try {
       // Part A first: prove the key before we push anything. A globally wrong
       // key aborts here with KEY_MISMATCH (push gated, nothing uploaded) rather
-      // than quarantining thousands of rows one by one on the pull.
+      // than quarantining thousands of rows one by one on the pull. A failure
+      // here blocks BOTH directions, so it opens both windows (see the catch).
       await ensureRootKey();
 
-      await pushDirtyRows();
+      // PARTIAL FUNCTION (3.3, extended here to the pull). A failure in one
+      // direction must never cost the other: the push error is captured, the
+      // pull runs, and the pull's own error is captured too so the cursor
+      // report still happens. The dirty set is untouched by a failed push
+      // (pushDirtyRows only clears it on full ack), so backoff changes WHEN
+      // rows are retried, never WHETHER they are.
+      let pushError = null;
+      let pushSkipped = false;
+      if (isOpen(pushBackoff)) {
+        pushSkipped = true;
+      } else {
+        try {
+          await pushDirtyRows();
+          // Any success resets the window and its strike count, exactly as the
+          // file tier does. For a quota window this is also what clears the
+          // over-quota state: the operator acted, and no client action was
+          // needed or taken.
+          clearWindow(pushBackoff);
+        } catch (err) {
+          pushError = err;
+          recordFailure(pushBackoff, err, MAX_PUSH_BACKOFF_S);
+        }
+      }
+
       onStatusChange?.('downloading');
-      const pull = await pullRemoteChanges();
-      await updateDeviceCursor();
+      let pullError = null;
+      let pullSkipped = false;
+      let pull = { applied: 0, skipped: 0, skippedEntityIds: [] };
+      if (isOpen(pullBackoff)) {
+        pullSkipped = true;
+      } else {
+        try {
+          pull = await pullRemoteChanges();
+          clearWindow(pullBackoff);
+        } catch (err) {
+          pullError = err;
+          recordFailure(pullBackoff, err, MAX_PULL_BACKOFF_S);
+        }
+        // The cursor report runs whenever the pull was ATTEMPTED, success or
+        // failure — that is the 3.3 decoupling applied to the pull. It is
+        // skipped only when the pull itself was skipped, because firing a
+        // write every cycle at a server we are backing off from is the exact
+        // hammering this fix removes.
+        await updateDeviceCursor();
+      }
+
+      // A rejected credential can surface from the push or the pull once the
+      // key is verified in-session (ensureRootKey then makes no network call,
+      // so it is no longer the first thing to see a 401). Route it to the
+      // halt path from wherever it surfaced — the halt logic itself is
+      // untouched.
+      const credentialRejection =
+        (pushError && pushError.code === 'CREDENTIAL_INVALID' && pushError) ||
+        (pullError && pullError.code === 'CREDENTIAL_INVALID' && pullError) || null;
+      if (credentialRejection) {
+        const outcome = applyCredentialRejection(credentialRejection);
+        return { applied: pull.applied, skipped: pull.skipped, skippedEntityIds: pull.skippedEntityIds, ...outcome };
+      }
+
+      if (pull.skipped > 0) onRowsSkipped?.(pull.skipped, pull.skippedEntityIds);
+
+      // lastSynced means "this device is fully in step with the server", so a
+      // cycle with any failed or skipped half does not claim it.
+      if (pushError || pushSkipped || pullError || pullSkipped) {
+        // Report the fresh failure if there was one; otherwise this is a
+        // suppressed cycle, and only a quota window keeps speaking.
+        if (pushError || pullError) {
+          const err = pushError || pullError;
+          onError?.(err?.message || String(err), (err && err.code) || 'NETWORK_ERROR', false);
+        } else {
+          surfaceStandingWindow(pushSkipped ? pushBackoff : pullBackoff);
+        }
+        onStatusChange?.('error');
+        return {
+          applied: pull.applied,
+          skipped: pull.skipped,
+          skippedEntityIds: pull.skippedEntityIds,
+          pushFailed: !!pushError,
+          pushSkipped,
+          pushErrorCode: pushError ? ((pushError.code) || 'NETWORK_ERROR') : (pushSkipped ? pushBackoff.code : undefined),
+          pullFailed: !!pullError,
+          pullSkipped,
+          pullErrorCode: pullError ? ((pullError.code) || 'NETWORK_ERROR') : (pullSkipped ? pullBackoff.code : undefined),
+          quota: getQuotaState(),
+        };
+      }
 
       const now = new Date().toISOString();
       localStorage.setItem(KEY_LAST_SYNCED, now);
       onStatusChange?.('success');
 
-      if (pull.skipped > 0) onRowsSkipped?.(pull.skipped, pull.skippedEntityIds);
       return { applied: pull.applied, skipped: pull.skipped, skippedEntityIds: pull.skippedEntityIds };
     } catch (err) {
+      // Only ensureRootKey reaches here now (push and pull are captured
+      // above). It gates both directions, so its failure opens both windows.
       // eslint-disable-next-line no-console
       console.error(`[${appId}] db sync cycle error:`, err);
       const code = err && err.code ? err.code : 'NETWORK_ERROR';
       if (code === 'CREDENTIAL_INVALID') {
-        // The server rejected this engine's credential. Under NO circumstances
-        // does the engine re-enroll here: enrollment is non-idempotent, so an
-        // automatic retry would mint a server row per failure and would let a
-        // revoked device silently re-admit itself. Recovery is user-initiated
-        // only (recoverVaultSyncEngine).
-        if (isBearerSuperseded()) {
-          // This instance was replaced by recovery and the app kept a stale
-          // reference. Go inert in memory — surfaced once, then silent — and
-          // leave the shared halt key alone so the recovered engine keeps
-          // syncing.
-          supersededInert = true;
-          onError?.(err?.message || String(err), 'CREDENTIAL_INVALID', true);
-        } else {
-          // Persist the stop so no future cycle retries (a rejected
-          // credential never heals by retry) and surface it as a hard stop.
-          setCredentialHalt(err?.message || String(err));
-          onError?.(err?.message || String(err), 'CREDENTIAL_INVALID', true);
-        }
-      } else {
-        onError?.(err?.message || String(err), code, false);
+        const outcome = applyCredentialRejection(err);
+        return { applied: 0, skipped: 0, skippedEntityIds: [], ...outcome };
       }
+      const reason = recordFailure(pushBackoff, err, MAX_PUSH_BACKOFF_S);
+      if (reason) recordFailure(pullBackoff, err, MAX_PULL_BACKOFF_S);
+      onError?.(err?.message || String(err), code, false);
       onStatusChange?.('error');
       return { applied: 0, skipped: 0, skippedEntityIds: [] };
     } finally {
@@ -797,6 +1067,27 @@ export const createDbSyncEngine = (config) => {
     // True once this instance proved its bearer was superseded by recovery
     // and went inert (in-memory only; a fresh instance is unaffected).
     isSuperseded: () => supersededInert,
+    // Over-quota state (Phase 3.3): null when clear, else the parsed
+    // descriptor plus when the next probe is due. In-memory only, never a
+    // hard stop, and there is nothing to clear — it lifts by itself once the
+    // server accepts a write again. Now derived from the push backoff window.
+    getQuotaState,
+    isQuotaSuppressed,
+
+    // Backoff. The timestamps use the file tier's getter names and meaning
+    // (epoch ms before which that direction will not be attempted), so a
+    // scheduler can treat both tiers identically. Unlike the file tier the
+    // engine ALSO enforces them itself, so honouring them is optional.
+    getUploadBackoffUntil:   () => pushBackoff.until,
+    getDownloadBackoffUntil: () => pullBackoff.until,
+    // The full picture, because the timestamp alone cannot distinguish a
+    // one-hour auth backoff from a thirty-second transport one without
+    // inferring from its magnitude. `reason` is 'quota' | 'auth' |
+    // 'transport'; `code` is the typed error that opened the window.
+    getBackoffState: () => ({
+      push: { until: pushBackoff.until, reason: pushBackoff.reason, code: pushBackoff.code, strikes: pushBackoff.strikes, since: pushBackoff.since },
+      pull: { until: pullBackoff.until, reason: pullBackoff.reason, code: pullBackoff.code, strikes: pullBackoff.strikes, since: pullBackoff.since },
+    }),
     deviceId: resolvedDeviceId,
 
     // Sub-modules

package/src/index.js

@@ -47,6 +47,14 @@ export {
   // discovery and the bootstrap-secret -> per-device-credential exchange.
   fetchVaultHealth,
   enrollVaultDevice,
+  // Module-scope request diagnostics (1.11.0). The brake gates every client
+  // call after a real 429; the budget meter and the write-loop detector are
+  // visibility only. All three are per bundle realm, not per client.
+  isVaultRateLimited,
+  vaultBrakeStatus,
+  getVaultStats,
+  configureVaultDiagnostics,
+  resetVaultDiagnostics,
 } from './vaultClient.js';
 // The packaged connect flow: discover -> branch -> enroll if needed ->
 // persist credential -> build engine. Apps hand it a secret or token and get