on-zero 0.6.2 → 0.6.4

package/src/createZeroClient.tsx

@@ -29,7 +29,10 @@ import { registerClientInstance } from './instanceRegistry'
 import { createMutators } from './helpers/createMutators'
 import {
   composeRecoveryLogSink,
+  isRecoverableZeroStalePokeMessage,
   makeZeroRecovery,
+  type RecoveryGuardStorage,
+  type ScheduleReloadContext,
   type ZeroRecoveryDeps,
 } from './helpers/recoverZeroClient'
 import { getAuth } from './helpers/getAuth'
@@ -377,6 +380,20 @@ export function createZeroClientInternal<
   // with different identities would thrash this slot.
   let cachedZero: { key: string; instance: ZeroInstance } | null = null
 
+  // in-place re-mint: drop the current instance's local state then reconstruct a
+  // fresh client WITHOUT a page reload — the native-safe recovery path (a reload
+  // may never land on prod native, wedging the module latch). the mounted
+  // provider registers a bump() that changes its instanceKey; remint() drives it
+  // through the same rotate effect a real identity change uses. guarded in-memory
+  // (Hermes has no sessionStorage) so a client-not-found storm can't reconstruct
+  // in a tight loop.
+  const REMINT_GUARD_MS = 12_000
+  const REMINT_MAX_ATTEMPTS = 5
+  const REMINT_ATTEMPT_RESET_MS = 60_000
+  const remintControl: { bump: (() => void) | null } = { bump: null }
+  let lastRemintAt = 0
+  let remintAttempts = 0
+
   function invalidateZeroInstance(instanceToInvalidate: ZeroInstance | null): void {
     if (!instanceToInvalidate) return
 
@@ -406,6 +423,34 @@ export function createZeroClientInternal<
     }
   }
 
+  // supported in-place recovery: reconstruct a fresh Zero client without a page
+  // reload. by default drops the current instance's local store first (a
+  // ClientNotFound / desync means it's unusable), then bumps the provider's
+  // instanceKey so the rotate effect mints a clean client. returns false when
+  // suppressed by the rate guard or when no provider is mounted.
+  async function remint(opts: { dropLocalState?: boolean } = {}): Promise<boolean> {
+    // no mounted provider to reconstruct through — bail BEFORE the guard so an
+    // unmounted call doesn't burn an attempt or start the cooldown.
+    if (!remintControl.bump) return false
+    const now = Date.now()
+    const sinceLast = now - lastRemintAt
+    if (lastRemintAt > 0 && sinceLast < REMINT_GUARD_MS) return false
+    if (sinceLast > REMINT_ATTEMPT_RESET_MS) remintAttempts = 0
+    if (remintAttempts >= REMINT_MAX_ATTEMPTS) return false
+    lastRemintAt = now
+    remintAttempts += 1
+
+    const { dropLocalState = true } = opts
+    if (dropLocalState && latestZeroInstance) {
+      await deleteZeroInstance(latestZeroInstance).catch(() => {})
+    }
+    // re-check: the provider may have unmounted during the async delete.
+    const bump = remintControl.bump
+    if (!bump) return false
+    bump()
+    return true
+  }
+
   // when ProvideZero is rendered without a real Zero instance (SSG, disable=true,
   // or transiently while the active path is still creating its first instance),
   // we want descendants' useZero() / useConnectionState() / on-zero useQuery to
@@ -459,6 +504,23 @@ export function createZeroClientInternal<
     // awaited before a self-healing recovery reload — e.g. wait for the dev
     // origin to be reachable so the reload doesn't hit a restarting server.
     beforeReload?: () => Promise<void>
+    // take over WHEN/HOW the recovery reload happens (IDE gate + countdown,
+    // native expo-updates reload, …) while still driving the same
+    // deletes-then-reload work via ctx.performReload. default: immediate reload.
+    scheduleReload?: (ctx: ScheduleReloadContext) => void
+    // cross-reload loop-guard backing store; defaults to sessionStorage on web.
+    // inject a native KV so Hermes gets real cross-reload protection.
+    guardStorage?: RecoveryGuardStorage
+    // return true for a classified recovery log the app wants treated as benign
+    // (its own cold-boot timeout, say) so it does NOT trigger recovery.
+    benignLogFilter?: (message: string) => boolean
+    // called when the connection needs auth; return a fresh token to reconnect
+    // in place. lets an expired token auto-recover without a reload.
+    refreshAuth?: () => Promise<string | undefined>
+    // when true, mirror this instance's connection state onto
+    // document.body.dataset.zero* for e2e/diagnostics. enable on ONE instance
+    // (the control/primary) so multiple instances don't fight over the dataset.
+    connectionDataset?: boolean
   }
 
   // providezero keeps the same provider/fiber layout on ssr and client so
@@ -493,6 +555,11 @@ export function createZeroClientInternal<
     transport,
     pullIntervalMs = 30_000,
     beforeReload,
+    scheduleReload,
+    guardStorage,
+    benignLogFilter,
+    refreshAuth,
+    connectionDataset,
     disable,
     ...props
   }: ProvideZeroProps) => {
@@ -532,10 +599,23 @@ export function createZeroClientInternal<
       // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [])
 
+    // remint() reconstructs the client in place by bumping this counter, which
+    // changes instanceKey and drives the rotate effect exactly as a real
+    // identity change does. register the bump so the imperative remint() API can
+    // reach this mounted provider.
+    const [remintGeneration, setRemintGeneration] = useState(0)
+    useEffect(() => {
+      remintControl.bump = () => setRemintGeneration((generation) => generation + 1)
+      return () => {
+        remintControl.bump = null
+      }
+    }, [])
+
     // identity = every primitive option except the auth token value (token
     // changes refresh in place below; logged-in <-> logged-out still rotates
     // via hasAuth, matching zero's documented provider semantics). function
     // props (callbacks, logSink, batchViewUpdates) are bound at construction.
+    // remintGeneration is included so remint() forces a fresh instance.
     const instanceKey = JSON.stringify([
       Object.entries({ kvStore: 'mem', ...props })
         .filter(
@@ -545,6 +625,7 @@ export function createZeroClientInternal<
         .sort(([a], [b]) => (a < b ? -1 : 1)),
       hasAuth,
       transport,
+      remintGeneration,
     ])
 
     // create/rotate in an effect — commit-safe: a discarded concurrent render
@@ -589,6 +670,9 @@ export function createZeroClientInternal<
           deleteLocalState: () => deleteZeroInstance(instanceRef.current),
           zeroEvents,
           beforeReload,
+          scheduleReload,
+          guardStorage,
+          benignLogFilter,
         }
         const recovery = makeZeroRecovery(recoveryDeps)
         const createdInstance = new ZeroClient<Schema, ZeroMutators>({
@@ -653,7 +737,19 @@ export function createZeroClientInternal<
         <DisabledContext.Provider value={liveInstance ? false : 'empty'}>
           <ZeroContext.Provider value={liveInstance ?? (DISABLED_ZERO_STUB as any)}>
             {liveInstance ? <SetZeroInstance /> : null}
-            {liveInstance ? <ConnectionMonitor zeroEvents={zeroEvents} /> : null}
+            {liveInstance ? (
+              <ConnectionMonitor
+                zeroEvents={zeroEvents}
+                refreshAuth={refreshAuth}
+                exposeDataset={connectionDataset}
+                datasetCacheUrl={
+                  connectionDataset
+                    ? ((props as { cacheURL?: string; server?: string }).cacheURL ??
+                      (props as { cacheURL?: string; server?: string }).server)
+                    : undefined
+                }
+              />
+            ) : null}
             {children}
           </ZeroContext.Provider>
         </DisabledContext.Provider>
@@ -689,30 +785,79 @@ export function createZeroClientInternal<
     return null
   }
 
-  // monitors connection state and emits events (replaces onError callback removed in 0.25)
+  // monitors connection state and emits events (replaces onError callback removed
+  // in 0.25). also owns the generic-Zero connection recovery that used to live in
+  // each consumer: stale-poke reconnect, needs-auth token refresh, and optional
+  // e2e dataset bookkeeping.
   const ConnectionMonitor = memo(
     ({
       zeroEvents,
+      refreshAuth,
+      exposeDataset,
+      datasetCacheUrl,
     }: {
       zeroEvents: ReturnType<typeof createEmitter<ZeroEvent | null>>
+      refreshAuth?: () => Promise<string | undefined>
+      exposeDataset?: boolean
+      datasetCacheUrl?: string
     }) => {
+      const zeroInstance = useZero<Schema, ZeroMutators>()
       const state = useConnectionState()
       const prevState = useRef(state.name)
+      // one reconnect per distinct stale-poke reason / one refresh per needs-auth
+      // transition, so a stuck error state doesn't retry-storm.
+      const staleReconnectRef = useRef<string | null>(null)
+      const needsAuthRef = useRef(false)
 
       useEffect(() => {
-        if (state.name !== prevState.current) {
-          const reason = 'reason' in state ? state.reason : ''
-          prevState.current = state.name
-
-          if (state.name === 'error' || state.name === 'needs-auth') {
-            const message = typeof reason === 'string' ? reason : state.name
-            zeroEvents.emit({
-              type: 'error',
-              message,
-            })
+        const name = state.name
+        const reason =
+          'reason' in state && typeof state.reason === 'string' ? state.reason : ''
+
+        // mirror connection state onto the body dataset for e2e/diagnostics
+        // (enabled on one instance so instances don't clobber each other).
+        if (exposeDataset && typeof document !== 'undefined' && document.body) {
+          document.body.dataset.zeroState = name
+          if (datasetCacheUrl) document.body.dataset.zeroCacheUrl = datasetCacheUrl
+          if (reason) document.body.dataset.zeroReason = reason.slice(0, 200)
+          else delete document.body.dataset.zeroReason
+          if (name === 'connected') document.body.dataset.zeroConnected = 'true'
+          else delete document.body.dataset.zeroConnected
+        }
+
+        // stale-poke / stale-cookie: the local view is behind the server
+        // snapshot; a plain reconnect resolves it. not fatal — don't emit error.
+        if (name === 'error' && isRecoverableZeroStalePokeMessage(reason)) {
+          if (staleReconnectRef.current !== reason) {
+            staleReconnectRef.current = reason
+            void Promise.resolve(zeroInstance.connection?.connect?.()).catch(() => {})
+          }
+          return
+        }
+        if (name !== 'error') staleReconnectRef.current = null
+
+        // needs-auth: the token expired and zero won't auto-resume unless the
+        // auth string changes. refresh it and reconnect in place, once.
+        if (name === 'needs-auth') {
+          if (refreshAuth && !needsAuthRef.current) {
+            needsAuthRef.current = true
+            void refreshAuth()
+              .then((token) => {
+                if (token) zeroInstance.connection?.connect?.({ auth: token })
+              })
+              .catch(() => {})
+          }
+        } else {
+          needsAuthRef.current = false
+        }
+
+        if (name !== prevState.current) {
+          prevState.current = name
+          if (name === 'error' || name === 'needs-auth') {
+            zeroEvents.emit({ type: 'error', message: reason || name })
           }
         }
-      }, [state, zeroEvents])
+      }, [state, zeroEvents, zeroInstance, refreshAuth, exposeDataset, datasetCacheUrl])
 
       return null
     },
@@ -779,5 +924,6 @@ export function createZeroClientInternal<
     zero,
     preload,
     getQuery,
+    remint,
   }
 }

package/src/helpers/recoverZeroClient.test.ts

@@ -4,12 +4,14 @@ import { UpdateNeededReasonType } from '@rocicorp/zero'
 import { beforeEach, describe, expect, test, vi } from 'vitest'
 
 import {
+  classifyZeroRecoveryLog,
   composeRecoveryLogSink,
+  isRecoverableZeroStalePokeMessage,
   makeZeroRecovery,
   resetRecoveryStateForTests,
 } from './recoverZeroClient'
 
-import type { ZeroRecoveryDeps } from './recoverZeroClient'
+import type { ScheduleReloadContext, ZeroRecoveryDeps } from './recoverZeroClient'
 import type { ZeroEvent } from '../types'
 
 let emitterSeq = 0
@@ -131,6 +133,38 @@ describe('zero recovery', () => {
     expect(reload).toHaveBeenCalledTimes(1)
   })
 
+  test('logSink recovers on native sqlite finalized-statement local-store loss', async () => {
+    const { deps, deleteLocalState, reload } = setup()
+    const sqliteError = new Error('This statement has been finalized')
+    sqliteError.name = 'SqliteError'
+    const sink = composeRecoveryLogSink(deps)
+    sink.log('error', { bgIntervalProcess: 'Heartbeat' }, 'Error running.', sqliteError)
+    await flush()
+    expect(deleteLocalState).toHaveBeenCalledTimes(1)
+    expect(reload).toHaveBeenCalledTimes(1)
+  })
+
+  test('logSink recovers on repeated store-closed local-store loss', async () => {
+    const nowSpy = vi.spyOn(Date, 'now')
+    const { deps, deleteLocalState, reload } = setup()
+    try {
+      const sink = composeRecoveryLogSink(deps)
+      nowSpy.mockReturnValue(10_000)
+      sink.log('error', undefined, 'Failed to connect.', new Error('Store is closed'))
+      await flush()
+      expect(deleteLocalState).not.toHaveBeenCalled()
+      expect(reload).not.toHaveBeenCalled()
+
+      nowSpy.mockReturnValue(13_000)
+      sink.log('error', undefined, 'Failed to connect.', new Error('Store is closed'))
+      await flush()
+      expect(deleteLocalState).toHaveBeenCalledTimes(1)
+      expect(reload).toHaveBeenCalledTimes(1)
+    } finally {
+      nowSpy.mockRestore()
+    }
+  })
+
   test('logSink with no consumer preserves console output and still watches', async () => {
     const { deps, deleteLocalState, reload } = setup()
     const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => {})
@@ -170,6 +204,39 @@ describe('zero recovery', () => {
     expect(reload).not.toHaveBeenCalled()
   })
 
+  test('recovery log classification is narrow', () => {
+    const sqliteError = new Error('This statement has been finalized')
+    sqliteError.name = 'SqliteError'
+    expect(
+      classifyZeroRecoveryLog('error', ['Error running.', sqliteError]),
+    ).toMatchObject({
+      reason: 'sqlite-statement-finalized',
+      dropLocalState: true,
+    })
+    expect(
+      classifyZeroRecoveryLog('error', [
+        { name: 'SqliteError', message: 'database is locked' },
+      ]),
+    ).toBeUndefined()
+    expect(
+      classifyZeroRecoveryLog('warn', ['Expected IndexedDB not found']),
+    ).toBeUndefined()
+    expect(classifyZeroRecoveryLog('error', ['Store is closed'], 10_000)).toBeUndefined()
+    expect(classifyZeroRecoveryLog('error', ['Store is closed'], 11_000)).toBeUndefined()
+    expect(classifyZeroRecoveryLog('error', ['Store is closed'], 14_000)).toMatchObject({
+      reason: 'store-closed-repeat',
+      dropLocalState: true,
+    })
+    resetRecoveryStateForTests()
+    expect(
+      classifyZeroRecoveryLog(
+        'error',
+        ['Mutator "send" error on server', 'Store is closed'],
+        20_000,
+      ),
+    ).toBeUndefined()
+  })
+
   // a host with a `window` shim but no real `location` — the sootsim tenant
   // render-worker, which hides `location` for isolation. the DEFAULT reload
   // path (no injected `deps.reload`) must still drop stale IDB and not throw
@@ -208,4 +275,206 @@ describe('zero recovery', () => {
       reason: 'client state not found',
     })
   })
+
+  test('scheduleReload defers the reload AND the store delete until performReload runs', async () => {
+    const { deps, deleteLocalState, reload } = setup()
+    let captured: ScheduleReloadContext | undefined
+    deps.scheduleReload = (ctx) => {
+      captured = ctx
+    }
+    makeZeroRecovery(deps).onClientStateNotFound()
+    await flush()
+    // deferred: nothing reloaded and the store is NOT yet deleted (so the app
+    // isn't left running on a deleted store while the reload is gated).
+    expect(reload).not.toHaveBeenCalled()
+    expect(deleteLocalState).not.toHaveBeenCalled()
+    expect(captured).toMatchObject({
+      reasonKey: 'client-state-not-found',
+      dropLocalState: true,
+    })
+    await captured!.performReload()
+    await flush()
+    expect(deleteLocalState).toHaveBeenCalledTimes(1)
+    expect(reload).toHaveBeenCalledTimes(1)
+  })
+
+  test('scheduleReload that reloads directly skips the delete and beforeReload', async () => {
+    const { deps, deleteLocalState, reload } = setup()
+    const nativeReload = vi.fn()
+    const beforeReload = vi.fn(async () => {})
+    deps.beforeReload = beforeReload
+    deps.scheduleReload = () => {
+      nativeReload()
+    }
+
+    makeZeroRecovery(deps).onClientStateNotFound()
+    await flush()
+
+    expect(nativeReload).toHaveBeenCalledTimes(1)
+    expect(deleteLocalState).not.toHaveBeenCalled()
+    expect(beforeReload).not.toHaveBeenCalled()
+    expect(reload).not.toHaveBeenCalled()
+  })
+
+  test('scheduleReload performReload is idempotent (one reload even if called twice)', async () => {
+    const { deps, deleteLocalState, reload } = setup()
+    let captured: ScheduleReloadContext | undefined
+    deps.scheduleReload = (ctx) => {
+      captured = ctx
+    }
+    makeZeroRecovery(deps).onClientStateNotFound()
+    await flush()
+    await captured!.performReload()
+    await captured!.performReload()
+    await flush()
+    expect(deleteLocalState).toHaveBeenCalledTimes(1)
+    expect(reload).toHaveBeenCalledTimes(1)
+  })
+
+  test('scheduleReload deferred PAST the latch timeout still deletes and reloads exactly once', async () => {
+    vi.useFakeTimers()
+    try {
+      const { deps, deleteLocalState, reload } = setup()
+      let captured: ScheduleReloadContext | undefined
+      deps.scheduleReload = (ctx) => {
+        captured = ctx
+      }
+      makeZeroRecovery(deps).onClientStateNotFound()
+      expect(captured).toBeDefined()
+
+      // the consumer holds performReload behind a gate for longer than the latch
+      // timeout (soot's IDE gate can hold for minutes). the timeout re-opens
+      // scheduling but must NOT drop the pending delete thunk.
+      await vi.advanceTimersByTimeAsync(20_000)
+      expect(deleteLocalState).not.toHaveBeenCalled()
+      expect(reload).not.toHaveBeenCalled()
+
+      // when the consumer finally commits, the store STILL gets dropped and the
+      // reload fires exactly once — never a bare reload back onto the bad store.
+      await captured!.performReload()
+      expect(deleteLocalState).toHaveBeenCalledTimes(1)
+      expect(reload).toHaveBeenCalledTimes(1)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  test('a slow beforeReload holds the latch through the timeout window (no double reload)', async () => {
+    vi.useFakeTimers()
+    try {
+      const { deps, reload } = setup()
+      let releaseBeforeReload: () => void = () => {}
+      deps.beforeReload = () =>
+        new Promise<void>((resolve) => {
+          releaseBeforeReload = resolve
+        })
+      const recovery = makeZeroRecovery(deps)
+      // default path: performReload runs immediately, disarms the latch, then
+      // blocks on beforeReload (soot's waitForOriginReachable can exceed 15s).
+      recovery.onClientStateNotFound()
+      await vi.advanceTimersByTimeAsync(20_000)
+      // a second recovery arrives while the first reload is still in-flight.
+      recovery.onClientStateNotFound()
+      await vi.advanceTimersByTimeAsync(0)
+      expect(reload).not.toHaveBeenCalled()
+      // releasing the first beforeReload reloads exactly once, never twice.
+      releaseBeforeReload()
+      await vi.advanceTimersByTimeAsync(0)
+      expect(reload).toHaveBeenCalledTimes(1)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  test('injectable guardStorage gives cross-reload loop protection (Hermes has no sessionStorage)', async () => {
+    const store = new Map<string, string>()
+    const { deps, reload, events } = setup()
+    deps.guardStorage = {
+      getItem: (key) => store.get(key) ?? null,
+      setItem: (key, value) => {
+        store.set(key, value)
+      },
+    }
+    const recovery = makeZeroRecovery(deps)
+    recovery.onClientStateNotFound()
+    await flush()
+    expect(reload).toHaveBeenCalledTimes(1)
+    // a reload wipes in-memory state but NOT the injected store.
+    resetRecoveryStateForTests()
+    recovery.onClientStateNotFound()
+    await flush()
+    // the injected store catches the immediate re-fire → fatal, no reload storm.
+    expect(reload).toHaveBeenCalledTimes(1)
+    expect(events.some((event) => event.type === 'fatal')).toBe(true)
+  })
+
+  test('a guardStorage that throws never crashes recovery (in-memory is the floor)', async () => {
+    const { deps, reload } = setup()
+    deps.guardStorage = {
+      getItem: () => {
+        throw new Error('no storage on this platform')
+      },
+      setItem: () => {
+        throw new Error('no storage on this platform')
+      },
+    }
+    makeZeroRecovery(deps).onClientStateNotFound()
+    await flush()
+    expect(reload).toHaveBeenCalledTimes(1)
+  })
+
+  test('classifier recognizes the mutation / connection desync class', () => {
+    const cases: Array<[readonly unknown[], string]> = [
+      [['sent mutation ID 5 but expected 4'], 'mutation-desync'],
+      [['oooMutation detected'], 'mutation-desync'],
+      [['Server reported an out-of-order mutation'], 'mutation-desync'],
+      [['Ignoring mutation 3, already processed. Expected: 4'], 'mutation-desync'],
+      [['InvalidConnectionRequestLastMutationID'], 'mutation-desync'],
+      [['InvalidConnectionRequestBaseCookie'], 'connection-cookie-invalid'],
+      [['ClientNotFound: client gone'], 'client-not-found'],
+      [['connection userID mismatch'], 'connection-userid-mismatch'],
+    ]
+    for (const [args, reason] of cases) {
+      expect(classifyZeroRecoveryLog('error', args)).toMatchObject({
+        reason,
+        dropLocalState: true,
+      })
+    }
+    // still narrow: app-infra strings and non-error levels do NOT classify.
+    expect(
+      classifyZeroRecoveryLog('error', ['consecutive server-ack timeouts']),
+    ).toBeUndefined()
+    expect(
+      classifyZeroRecoveryLog('error', ['Connection attempt timed out after 10 seconds']),
+    ).toBeUndefined()
+    expect(classifyZeroRecoveryLog('warn', ['ClientNotFound'])).toBeUndefined()
+  })
+
+  test('benignLogFilter suppresses recovery for a matching classified log', async () => {
+    const { deps, deleteLocalState, reload } = setup()
+    deps.benignLogFilter = (message) => message.includes('ClientNotFound')
+    const sink = composeRecoveryLogSink(deps)
+    sink.log('error', undefined, 'ClientNotFound: gone')
+    await flush()
+    expect(deleteLocalState).not.toHaveBeenCalled()
+    expect(reload).not.toHaveBeenCalled()
+    // a non-benign desync still recovers through the same sink.
+    sink.log('error', undefined, 'sent mutation ID 5 but expected 4')
+    await flush()
+    expect(reload).toHaveBeenCalledTimes(1)
+  })
+
+  test('isRecoverableZeroStalePokeMessage matches only the stale-cookie signatures', () => {
+    expect(
+      isRecoverableZeroStalePokeMessage(
+        'Server returned unexpected base cookie during sync',
+      ),
+    ).toBe(true)
+    expect(
+      isRecoverableZeroStalePokeMessage(
+        'Received cookie 5 is < than last snapshot cookie 9, ignoring client view',
+      ),
+    ).toBe(true)
+    expect(isRecoverableZeroStalePokeMessage('client state not found')).toBe(false)
+  })
 })