on-zero 0.7.10 → 0.8.0

package/src/helpers/mutationLifecycle.test.ts

@@ -0,0 +1,180 @@
+import { beforeEach, describe, expect, test, vi } from 'vitest'
+
+import {
+  createMutationLifecycle,
+  MutationTimeoutError,
+  StaleGenerationError,
+} from './mutationLifecycle'
+
+function deferred<T>() {
+  let resolve!: (value: T | PromiseLike<T>) => void
+  let reject!: (error: unknown) => void
+  const promise = new Promise<T>((resolvePromise, rejectPromise) => {
+    resolve = resolvePromise
+    reject = rejectPromise
+  })
+  return { promise, reject, resolve }
+}
+
+function setup(threshold = 2) {
+  const recoverFromAckTimeout = vi.fn()
+  const lifecycle = createMutationLifecycle({
+    ackTimeoutRecoveryThreshold: threshold,
+    recoverFromAckTimeout,
+  })
+  lifecycle.activate()
+  return { lifecycle, recoverFromAckTimeout }
+}
+
+beforeEach(() => {
+  vi.restoreAllMocks()
+})
+
+describe('mutation lifecycle', () => {
+  test('extracts typed mutation errors and times out each settlement phase', async () => {
+    vi.useFakeTimers()
+    try {
+      const { lifecycle } = setup()
+      await expect(
+        lifecycle.awaitMutationClient(
+          { client: Promise.resolve({ type: 'error', error: { message: 'denied' } }) },
+          'save note'
+        )
+      ).rejects.toMatchObject({
+        name: 'MutationResultError',
+        label: 'save note',
+        phase: 'client',
+      })
+
+      const waiting = lifecycle.awaitMutationClient(
+        { client: new Promise(() => {}) },
+        'stuck note',
+        20
+      )
+      const timedOut = expect(waiting).rejects.toMatchObject({
+        name: 'MutationTimeoutError',
+        phase: 'client',
+        timeoutMs: 20,
+      })
+      await vi.advanceTimersByTimeAsync(20)
+      await timedOut
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  test('serializes work and skips superseded coalesced writes', async () => {
+    const { lifecycle } = setup()
+    const blocker = deferred<void>()
+    const first = lifecycle.enqueueBackgroundMutation('blocker', () => blocker.promise)
+    const superseded = vi.fn()
+    const latest = vi.fn(async () => {})
+    const oldWrite = lifecycle.enqueueBackgroundMutation('old row', superseded, {
+      coalesceKey: 'message:1',
+    })
+    const newWrite = lifecycle.enqueueBackgroundMutation('new row', latest, {
+      coalesceKey: 'message:1',
+    })
+
+    blocker.resolve()
+    await Promise.all([first, oldWrite, newWrite])
+    expect(superseded).not.toHaveBeenCalled()
+    expect(latest).toHaveBeenCalledOnce()
+  })
+
+  test('one failed background mutation does not poison later work', async () => {
+    const { lifecycle } = setup()
+    vi.spyOn(console, 'warn').mockImplementation(() => {})
+    const denied = lifecycle.enqueueBackgroundMutation('stamp', async () => {
+      throw new Error('PermissionError: stale task')
+    })
+    await expect(denied).rejects.toThrow('PermissionError')
+
+    const next = vi.fn(async () => {})
+    await lifecycle.enqueueBackgroundMutation('clear stamp', next)
+    expect(next).toHaveBeenCalledOnce()
+  })
+
+  test('recovery or close quietly drops in-flight and queued background work', async () => {
+    const { lifecycle } = setup()
+    const client = deferred<unknown>()
+    const queuedCreate = vi.fn(async () => {})
+    const inFlight = lifecycle.enqueueBackgroundMutation('in flight', () => ({
+      client: client.promise,
+    }))
+    const queued = lifecycle.enqueueBackgroundMutation('queued', queuedCreate)
+
+    lifecycle.fence()
+    await expect(inFlight).resolves.toBeUndefined()
+    await expect(queued).resolves.toBeUndefined()
+    expect(queuedCreate).not.toHaveBeenCalled()
+
+    const direct = lifecycle.awaitMutationClient(
+      { client: Promise.resolve({}) },
+      'direct after close'
+    )
+    await expect(direct).rejects.toBeInstanceOf(StaleGenerationError)
+  })
+
+  test('recovers only after consecutive server acknowledgement timeouts', async () => {
+    vi.useFakeTimers()
+    try {
+      const { lifecycle, recoverFromAckTimeout } = setup()
+      const timeout = async (label: string) => {
+        const result = lifecycle.awaitMutationServer(
+          { client: Promise.resolve({}), server: new Promise(() => {}) },
+          label,
+          10
+        )
+        const timedOut = expect(result).rejects.toMatchObject({
+          name: 'MutationTimeoutError',
+          phase: 'server',
+        })
+        await vi.advanceTimersByTimeAsync(10)
+        await timedOut
+      }
+
+      await timeout('first')
+      expect(recoverFromAckTimeout).not.toHaveBeenCalled()
+      await timeout('second')
+      expect(recoverFromAckTimeout).toHaveBeenCalledOnce()
+      expect(recoverFromAckTimeout).toHaveBeenCalledWith({
+        label: 'second',
+        timeoutMs: 10,
+        consecutiveTimeouts: 2,
+      })
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  test('a server response resets the consecutive timeout count', async () => {
+    vi.useFakeTimers()
+    try {
+      const { lifecycle, recoverFromAckTimeout } = setup()
+      const timeout = async () => {
+        const result = lifecycle.awaitMutationServer(
+          { client: Promise.resolve({}), server: new Promise(() => {}) },
+          'write',
+          10
+        )
+        const timedOut = expect(result).rejects.toBeInstanceOf(MutationTimeoutError)
+        await vi.advanceTimersByTimeAsync(10)
+        await timedOut
+      }
+
+      await timeout()
+      await expect(
+        lifecycle.awaitMutationServer(
+          { client: Promise.resolve({}), server: Promise.resolve({}) },
+          'acked write',
+          10
+        )
+      ).resolves.toEqual({})
+      await timeout()
+      expect(recoverFromAckTimeout).not.toHaveBeenCalled()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})

package/src/helpers/mutationLifecycle.ts

@@ -0,0 +1,313 @@
+export type MutationLike = {
+  client: Promise<unknown>
+  server?: Promise<unknown>
+}
+
+export type BackgroundMutationOptions = {
+  coalesceKey?: string
+  settle?: 'client' | 'server'
+  timeoutMs?: number
+}
+
+export type MutationPhase = 'client' | 'server'
+
+export class StaleGenerationError extends Error {
+  readonly label: string
+
+  constructor(label: string) {
+    super(`${label} stopped because its Zero instance was replaced or recovered`)
+    this.name = 'StaleGenerationError'
+    this.label = label
+  }
+}
+
+export class MutationTimeoutError extends Error {
+  readonly label: string
+  readonly phase: MutationPhase
+  readonly timeoutMs: number
+
+  constructor(label: string, phase: MutationPhase, timeoutMs: number) {
+    super(
+      phase === 'server'
+        ? `${label} server acknowledgement did not complete within ${timeoutMs}ms`
+        : `${label} did not complete within ${timeoutMs}ms`
+    )
+    this.name = 'MutationTimeoutError'
+    this.label = label
+    this.phase = phase
+    this.timeoutMs = timeoutMs
+  }
+}
+
+export class MutationResultError extends Error {
+  readonly label: string
+  readonly phase: MutationPhase
+  readonly result: unknown
+
+  constructor(label: string, phase: MutationPhase, result: unknown, message: string) {
+    super(`${label} failed${phase === 'server' ? ' on server' : ''}: ${message}`)
+    this.name = 'MutationResultError'
+    this.label = label
+    this.phase = phase
+    this.result = result
+  }
+}
+
+export function isStaleGenerationError(error: unknown): error is StaleGenerationError {
+  return error instanceof StaleGenerationError
+}
+
+export function mutationErrorMessage(result: unknown): string | null {
+  if (!result || typeof result !== 'object') return null
+  const typed = result as { type?: unknown; error?: unknown }
+  if (typed.type !== 'error') return null
+  return describeMutationError(typed.error)
+}
+
+function describeMutationError(error: unknown): string {
+  if (error instanceof Error) return error.message
+  if (error && typeof error === 'object') {
+    const typed = error as { type?: unknown; message?: unknown }
+    const parts = [typed.type, typed.message].filter(
+      (part): part is string => typeof part === 'string' && part.length > 0
+    )
+    if (parts.length) return parts.join(': ')
+  }
+  return String(error)
+}
+
+type GenerationWaiter = {
+  reject: () => void
+}
+
+type BackgroundMutationQueue = {
+  tail: Promise<void>
+  lastErrorMessage: string | null
+  coalesceSequence: Map<string, number>
+}
+
+export type MutationLifecycle = ReturnType<typeof createMutationLifecycle>
+
+export function createMutationLifecycle(options: {
+  ackTimeoutRecoveryThreshold: number
+  recoverFromAckTimeout: (input: {
+    label: string
+    timeoutMs: number
+    consecutiveTimeouts: number
+  }) => void
+}) {
+  const queue: BackgroundMutationQueue = {
+    tail: Promise.resolve(),
+    lastErrorMessage: null,
+    coalesceSequence: new Map(),
+  }
+  const waiters = new Set<GenerationWaiter>()
+  let generation = 0
+  let active = false
+  let consecutiveServerAckTimeouts = 0
+
+  function stale(label: string): StaleGenerationError {
+    return new StaleGenerationError(label)
+  }
+
+  function fence() {
+    if (!active) return
+    active = false
+    generation += 1
+    consecutiveServerAckTimeouts = 0
+    queue.coalesceSequence.clear()
+    for (const waiter of [...waiters]) {
+      waiters.delete(waiter)
+      waiter.reject()
+    }
+  }
+
+  function activate() {
+    if (active) return
+    generation += 1
+    active = true
+    consecutiveServerAckTimeouts = 0
+    queue.lastErrorMessage = null
+  }
+
+  function observeGeneration(capturedGeneration: number, label: string) {
+    let waiter: GenerationWaiter | undefined
+    const promise = new Promise<never>((_, reject) => {
+      if (!active || capturedGeneration !== generation) {
+        reject(stale(label))
+        return
+      }
+      waiter = {
+        reject: () => reject(stale(label)),
+      }
+      waiters.add(waiter)
+      if (!active || capturedGeneration !== generation) {
+        waiters.delete(waiter)
+        waiter = undefined
+        reject(stale(label))
+      }
+    })
+    return {
+      promise,
+      dispose() {
+        if (waiter) waiters.delete(waiter)
+      },
+    }
+  }
+
+  async function awaitInGeneration<T>(input: {
+    promise: Promise<T>
+    label: string
+    phase: MutationPhase
+    timeoutMs: number
+    generation: number
+  }): Promise<T> {
+    if (!active || input.generation !== generation) throw stale(input.label)
+    let timer: ReturnType<typeof setTimeout> | undefined
+    const generationChange = observeGeneration(input.generation, input.label)
+    try {
+      return await Promise.race([
+        input.promise,
+        generationChange.promise,
+        new Promise<never>((_, reject) => {
+          timer = setTimeout(
+            () =>
+              reject(new MutationTimeoutError(input.label, input.phase, input.timeoutMs)),
+            input.timeoutMs
+          )
+        }),
+      ])
+    } finally {
+      generationChange.dispose()
+      if (timer) clearTimeout(timer)
+    }
+  }
+
+  async function settleMutationClient(
+    mutation: MutationLike,
+    label: string,
+    timeoutMs: number,
+    capturedGeneration: number,
+    observeServerFailure: boolean
+  ): Promise<unknown> {
+    const result = await awaitInGeneration({
+      promise: mutation.client,
+      label,
+      phase: 'client',
+      timeoutMs,
+      generation: capturedGeneration,
+    })
+    const message = mutationErrorMessage(result)
+    if (message) throw new MutationResultError(label, 'client', result, message)
+    if (observeServerFailure) {
+      mutation.server?.catch((error: unknown) => {
+        if (!active || capturedGeneration !== generation) return
+        console.warn(`[on-zero] ${label} server replication failed:`, error)
+      })
+    }
+    return result
+  }
+
+  function awaitMutationClient(
+    mutation: MutationLike,
+    label: string,
+    timeoutMs = 30_000
+  ): Promise<unknown> {
+    return settleMutationClient(mutation, label, timeoutMs, generation, true)
+  }
+
+  async function awaitMutationServer(
+    mutation: MutationLike,
+    label: string,
+    timeoutMs = 30_000
+  ): Promise<unknown> {
+    const capturedGeneration = generation
+    const clientResult = await settleMutationClient(
+      mutation,
+      label,
+      timeoutMs,
+      capturedGeneration,
+      false
+    )
+    if (!mutation.server) return clientResult
+
+    try {
+      const serverResult = await awaitInGeneration({
+        promise: mutation.server,
+        label,
+        phase: 'server',
+        timeoutMs,
+        generation: capturedGeneration,
+      })
+      consecutiveServerAckTimeouts = 0
+      const message = mutationErrorMessage(serverResult)
+      if (message) throw new MutationResultError(label, 'server', serverResult, message)
+      return serverResult
+    } catch (error) {
+      if (error instanceof MutationTimeoutError && error.phase === 'server') {
+        consecutiveServerAckTimeouts += 1
+        if (consecutiveServerAckTimeouts >= options.ackTimeoutRecoveryThreshold) {
+          options.recoverFromAckTimeout({
+            label,
+            timeoutMs,
+            consecutiveTimeouts: consecutiveServerAckTimeouts,
+          })
+        }
+      }
+      throw error
+    }
+  }
+
+  function enqueueBackgroundMutation(
+    label: string,
+    create: () => unknown,
+    mutationOptions: BackgroundMutationOptions = {}
+  ): Promise<void> {
+    const capturedGeneration = generation
+    const { coalesceKey = '', settle = 'client', timeoutMs = 120_000 } = mutationOptions
+    const sequence = coalesceKey ? (queue.coalesceSequence.get(coalesceKey) ?? 0) + 1 : 0
+    if (coalesceKey) queue.coalesceSequence.set(coalesceKey, sequence)
+
+    const queued = queue.tail.then(async () => {
+      if (!active || capturedGeneration !== generation) throw stale(label)
+      if (coalesceKey) {
+        if (queue.coalesceSequence.get(coalesceKey) !== sequence) return
+        queue.coalesceSequence.delete(coalesceKey)
+      }
+      const result = await create()
+      if (result && typeof result === 'object' && 'client' in result) {
+        const mutation = result as MutationLike
+        if (settle === 'server' && mutation.server) {
+          await awaitMutationServer(mutation, label, timeoutMs)
+        } else {
+          await settleMutationClient(mutation, label, timeoutMs, capturedGeneration, true)
+        }
+      } else {
+        await result
+      }
+      queue.lastErrorMessage = null
+    })
+
+    const result = queued.catch((error: unknown) => {
+      if (isStaleGenerationError(error)) return
+      if (!active || capturedGeneration !== generation) return
+      const message = describeMutationError(error)
+      const alreadyLogged = queue.lastErrorMessage === message
+      queue.lastErrorMessage = message
+      if (!alreadyLogged) {
+        console.warn(`[on-zero] ${label} background mutation failed:`, error)
+      }
+      throw error
+    })
+    queue.tail = result.catch(() => {})
+    return result
+  }
+
+  return {
+    activate,
+    fence,
+    enqueueBackgroundMutation,
+    awaitMutationClient,
+    awaitMutationServer,
+  }
+}

package/src/helpers/mutatorContext.ts

@@ -1,4 +1,4 @@
-import { createAsyncContext } from '@take-out/helpers'
+import { createAsyncContext } from './asyncContext'
 
 import type { AuthData, MutatorContext } from '../types'
 
@@ -33,7 +33,7 @@ export function isInZeroMutation() {
 
 export function runWithContext<T>(
   context: MutatorContext,
-  fn: () => T | Promise<T>,
+  fn: () => T | Promise<T>
 ): Promise<T> {
   return asyncContext.run(context, fn)
 }
@@ -48,7 +48,7 @@ export function getScopedAuthData(): AuthData | null | undefined {
 
 export function runWithAuthScope<T>(
   authData: AuthData | null,
-  fn: () => T | Promise<T>,
+  fn: () => T | Promise<T>
 ): Promise<T> {
   return authScopeContext.run(authData, fn)
 }

package/src/helpers/platform.native.ts

@@ -0,0 +1,2 @@
+export const IS_SERVER = false
+export const IS_SERVER_RUNTIME = false

package/src/helpers/platform.ts

@@ -0,0 +1,16 @@
+export const IS_SERVER_RUNTIME: boolean =
+  process.env.VITE_ENVIRONMENT === 'ssr' ||
+  (process.env.VITE_ENVIRONMENT !== 'client' &&
+    typeof process !== 'undefined' &&
+    !!process.versions &&
+    !!(process.versions.node || (process.versions as any).bun))
+
+const isBrowser =
+  process.env.VITE_ENVIRONMENT === 'client' ||
+  (process.env.VITE_ENVIRONMENT !== 'ssr' &&
+    typeof navigator !== 'undefined' &&
+    typeof location !== 'undefined')
+
+export const IS_SERVER: boolean =
+  process.env.VITE_ENVIRONMENT === 'ssr' ||
+  (process.env.VITE_ENVIRONMENT !== 'client' && !isBrowser)

package/src/helpers/prettyFormatZeroQuery.ts

@@ -1,10 +1,10 @@
-import { ellipsis } from '@take-out/helpers'
+import { ellipsis } from './ellipsis'
 
 import type { Query } from '@rocicorp/zero'
 
 export const prettyFormatZeroQuery = (
   query: Query<any, any, any>,
-  mode: 'full' | 'minimal' = 'full',
+  mode: 'full' | 'minimal' = 'full'
 ): string => {
   const astObject = query['_completeAst']?.()
 

package/src/helpers/queryContext.ts

@@ -1,4 +1,4 @@
-import { createAsyncContext } from '@take-out/helpers'
+import { createAsyncContext } from './asyncContext'
 
 import type { AuthData } from '../types'
 
@@ -14,7 +14,7 @@ export function isInQueryContext() {
 
 export function runWithQueryContext<T>(
   context: { authData: AuthData | null },
-  fn: () => T | Promise<T>,
+  fn: () => T | Promise<T>
 ): Promise<T> {
   return asyncContext.run(context, fn)
 }

package/src/helpers/recoverZeroClient.test.ts

@@ -1,8 +1,8 @@
-// @vitest-environment jsdom
-import { createEmitter } from '@take-out/helpers'
 import { UpdateNeededReasonType } from '@rocicorp/zero'
 import { beforeEach, describe, expect, test, vi } from 'vitest'
 
+// @vitest-environment jsdom
+import { createEmitter } from './emitter'
 import {
   classifyZeroRecoveryLog,
   composeRecoveryLogSink,
@@ -11,8 +11,8 @@ import {
   resetRecoveryStateForTests,
 } from './recoverZeroClient'
 
-import type { ScheduleReloadContext, ZeroRecoveryDeps } from './recoverZeroClient'
 import type { ZeroEvent } from '../types'
+import type { ScheduleReloadContext, ZeroRecoveryDeps } from './recoverZeroClient'
 
 let emitterSeq = 0
 
@@ -48,6 +48,7 @@ describe('zero recovery', () => {
     expect(events).toEqual([
       {
         type: 'recovering',
+        reasonKey: 'SchemaVersionNotSupported',
         reason: expect.stringContaining('SchemaVersionNotSupported'),
       },
     ])
@@ -208,23 +209,23 @@ describe('zero recovery', () => {
     const sqliteError = new Error('This statement has been finalized')
     sqliteError.name = 'SqliteError'
     expect(
-      classifyZeroRecoveryLog('error', ['Error running.', sqliteError]),
+      classifyZeroRecoveryLog('error', ['Error running.', sqliteError])
     ).toMatchObject({
-      reason: 'sqlite-statement-finalized',
+      reasonKey: 'sqlite-statement-finalized',
       dropLocalState: true,
     })
     expect(
       classifyZeroRecoveryLog('error', [
         { name: 'SqliteError', message: 'database is locked' },
-      ]),
+      ])
     ).toBeUndefined()
     expect(
-      classifyZeroRecoveryLog('warn', ['Expected IndexedDB not found']),
+      classifyZeroRecoveryLog('warn', ['Expected IndexedDB not found'])
     ).toBeUndefined()
     expect(classifyZeroRecoveryLog('error', ['Store is closed'], 10_000)).toBeUndefined()
     expect(classifyZeroRecoveryLog('error', ['Store is closed'], 11_000)).toBeUndefined()
     expect(classifyZeroRecoveryLog('error', ['Store is closed'], 14_000)).toMatchObject({
-      reason: 'store-closed-repeat',
+      reasonKey: 'store-closed-repeat',
       dropLocalState: true,
     })
     resetRecoveryStateForTests()
@@ -232,8 +233,8 @@ describe('zero recovery', () => {
       classifyZeroRecoveryLog(
         'error',
         ['Mutator "send" error on server', 'Store is closed'],
-        20_000,
-      ),
+        20_000
+      )
     ).toBeUndefined()
   })
 
@@ -246,7 +247,7 @@ describe('zero recovery', () => {
     const events: ZeroEvent[] = []
     const zeroEvents = createEmitter<ZeroEvent | null>(
       `test-recover-${emitterSeq++}`,
-      null,
+      null
     )
     zeroEvents.listen((event) => {
       if (event) events.push(event)
@@ -272,6 +273,7 @@ describe('zero recovery', () => {
     expect(deleteLocalState).toHaveBeenCalledTimes(1)
     expect(events).toContainEqual({
       type: 'recovering',
+      reasonKey: 'client-state-not-found',
       reason: 'client state not found',
     })
   })
@@ -436,23 +438,23 @@ describe('zero recovery', () => {
     ]
     for (const [args, reason] of cases) {
       expect(classifyZeroRecoveryLog('error', args)).toMatchObject({
-        reason,
+        reasonKey: reason,
         dropLocalState: true,
       })
     }
-    // still narrow: app-infra strings and non-error levels do NOT classify.
+    // ack timeouts use the typed mutation path; synthetic strings do not classify.
     expect(
-      classifyZeroRecoveryLog('error', ['consecutive server-ack timeouts']),
+      classifyZeroRecoveryLog('error', ['consecutive server-ack timeouts'])
     ).toBeUndefined()
     expect(
-      classifyZeroRecoveryLog('error', ['Connection attempt timed out after 10 seconds']),
+      classifyZeroRecoveryLog('error', ['Connection attempt timed out after 10 seconds'])
     ).toBeUndefined()
     expect(classifyZeroRecoveryLog('warn', ['ClientNotFound'])).toBeUndefined()
   })
 
-  test('benignLogFilter suppresses recovery for a matching classified log', async () => {
+  test('benignLogPatterns suppress recovery for matching classified logs', async () => {
     const { deps, deleteLocalState, reload } = setup()
-    deps.benignLogFilter = (message) => message.includes('ClientNotFound')
+    deps.benignLogPatterns = ['ClientNotFound']
     const sink = composeRecoveryLogSink(deps)
     sink.log('error', undefined, 'ClientNotFound: gone')
     await flush()
@@ -467,13 +469,13 @@ describe('zero recovery', () => {
   test('isRecoverableZeroStalePokeMessage matches only the stale-cookie signatures', () => {
     expect(
       isRecoverableZeroStalePokeMessage(
-        'Server returned unexpected base cookie during sync',
-      ),
+        'Server returned unexpected base cookie during sync'
+      )
     ).toBe(true)
     expect(
       isRecoverableZeroStalePokeMessage(
-        'Received cookie 5 is < than last snapshot cookie 9, ignoring client view',
-      ),
+        'Received cookie 5 is < than last snapshot cookie 9, ignoring client view'
+      )
     ).toBe(true)
     expect(isRecoverableZeroStalePokeMessage('client state not found')).toBe(false)
   })