on-zero 0.7.2-1784530701923 → 0.7.2

package/src/createZeroServerCore.ts

@@ -0,0 +1,296 @@
+import {
+  PushProcessor,
+  handleQueryRequest as zeroHandleQueryRequest,
+} from '@rocicorp/zero/server'
+
+import { runAsyncTaskEntries } from './asyncTasks'
+import {
+  createAsyncTaskCollector,
+  createZeroDirectServerRuntime,
+  resolveServerQuery,
+  type TransactionAttemptLifecycle,
+  type ZeroServerCommonOptions,
+  type ZeroServerDatabase,
+} from './createZeroDirectServer'
+import { createMutators } from './helpers/createMutators'
+import { runWithAuthScope } from './helpers/mutatorContext'
+import { runWithQueryContext } from './helpers/queryContext'
+
+import type { AuthData, GenericModels } from './types'
+import type { Schema as ZeroSchema } from '@rocicorp/zero'
+
+type AlreadyProcessedMutation = {
+  id: {
+    clientID: string
+    id: number
+  }
+  result: {
+    error: 'alreadyProcessed'
+    details?: unknown
+  }
+}
+
+type PushRequestBody = {
+  clientGroupID?: unknown
+}
+
+export type BackgroundTaskScheduler = (promise: Promise<void>) => void
+
+export type HandleMutationRequestOptions = {
+  authData: AuthData | null
+  request: Request
+  scheduleBackgroundTask?: BackgroundTaskScheduler
+  skipAsyncTasks?: boolean
+}
+
+export type PersistAlreadyProcessedArgs = {
+  clientGroupID: string
+  mutations: AlreadyProcessedMutation[]
+  upstreamSchema: string
+}
+
+type ZeroServerCoreOptions<
+  Schema extends ZeroSchema,
+  Models extends GenericModels,
+  ServerActions extends Record<string, unknown>,
+> = ZeroServerCommonOptions<Schema, Models, ServerActions> & {
+  createDatabase: () => ZeroServerDatabase
+  persistAlreadyProcessed?: (args: PersistAlreadyProcessedArgs) => Promise<void>
+}
+
+export function createZeroServerCore<
+  Schema extends ZeroSchema,
+  Models extends GenericModels,
+  ServerActions extends Record<string, unknown>,
+>({
+  createDatabase,
+  createServerActions,
+  defaultAllowAdminRole = 'all',
+  defaultMutateAuthData,
+  models,
+  mutations: mutationValidators,
+  persistAlreadyProcessed,
+  queries,
+  schema,
+  validateMutation,
+  validateQuery,
+}: ZeroServerCoreOptions<Schema, Models, ServerActions>) {
+  const { permissions, server } = createZeroDirectServerRuntime({
+    createDatabase,
+    createServerActions,
+    defaultAllowAdminRole,
+    defaultMutateAuthData,
+    models,
+    mutations: mutationValidators,
+    queries,
+    schema,
+    validateMutation,
+    validateQuery,
+  })
+
+  const handleMutationRequest = async ({
+    authData,
+    request,
+    scheduleBackgroundTask,
+    skipAsyncTasks,
+  }: HandleMutationRequestOptions) => {
+    const tasks = createAsyncTaskCollector()
+    const recoveryRequest = request.clone()
+    const mutators = createMutators({
+      enqueueTask: tasks.enqueueTask,
+      can: permissions.can,
+      createServerActions,
+      environment: 'server',
+      models,
+      authData,
+      validateMutation,
+      mutationValidators,
+    })
+    const processor = new PushProcessor(
+      withTransactionLifecycle(createDatabase(), tasks.lifecycle),
+    )
+    const response = await processor.process(mutators, request)
+
+    if (persistAlreadyProcessed) {
+      const parsed = await parseAlreadyProcessed(recoveryRequest, response)
+      if (parsed) await persistAlreadyProcessed(parsed)
+    }
+
+    if (!skipAsyncTasks && tasks.committed.length) {
+      const entries = tasks.committed.map((entry) => ({
+        ...entry,
+        task: () => runWithAuthScope(authData, entry.task),
+      }))
+      await runAsyncTaskEntries(entries, { background: scheduleBackgroundTask })
+    }
+
+    return {
+      response,
+      asyncTasks: tasks.committed,
+    }
+  }
+
+  const handleQueryRequest = async ({
+    authData,
+    request,
+  }: {
+    authData: AuthData | null
+    request: Request
+  }) => {
+    if (!queries) {
+      throw new Error(
+        'No queries registered with createZeroServer. ' +
+          'Pass the syncedQueries registry to createZeroServer via the queries option.',
+      )
+    }
+
+    const handler = (name: string, args: unknown) =>
+      resolveServerQuery({
+        authData,
+        name,
+        args,
+        queries,
+        permissions,
+        validateQuery,
+      })
+
+    const userID = typeof authData?.id === 'string' ? authData.id : undefined
+    const response = await runWithQueryContext(
+      { authData: authData || ({} as AuthData) },
+      () =>
+        // the legacy overload omits userID when auth does not identify a user.
+        userID === undefined
+          ? zeroHandleQueryRequest(handler, schema, request)
+          : zeroHandleQueryRequest({ handler, schema, request, userID }),
+    )
+
+    return { response }
+  }
+
+  return {
+    handleMutationRequest,
+    handleQueryRequest,
+    ...server,
+  }
+}
+
+export type ZeroServer<
+  Schema extends ZeroSchema,
+  Models extends GenericModels,
+  ServerActions extends Record<string, unknown>,
+> = ReturnType<typeof createZeroServerCore<Schema, Models, ServerActions>>
+
+function withTransactionLifecycle(
+  database: ZeroServerDatabase,
+  lifecycle: TransactionAttemptLifecycle,
+): ZeroServerDatabase {
+  return {
+    async transaction(callback, input) {
+      try {
+        const value = await database.transaction(async (tx, hooks) => {
+          lifecycle.beginAttempt()
+          try {
+            const result = await callback(tx, {
+              ...hooks,
+              async updateClientMutationID() {
+                const result = await hooks.updateClientMutationID()
+                if (input && input.mutationID < result.lastMutationID) {
+                  throw new ExpectedDuplicateMutation(
+                    input.clientID,
+                    input.mutationID,
+                    result.lastMutationID,
+                  )
+                }
+                return result
+              },
+            })
+            lifecycle.completeAttempt()
+            return result
+          } catch (error) {
+            lifecycle.rollbackAttempt()
+            throw error
+          }
+        }, input)
+        lifecycle.commitTransaction()
+        return value
+      } catch (error) {
+        lifecycle.rollbackTransaction()
+        if (error instanceof ExpectedDuplicateMutation) {
+          return {
+            id: {
+              clientID: error.clientID,
+              id: error.mutationID,
+            },
+            result: {
+              error: 'alreadyProcessed',
+              details: error.message,
+            },
+          } as never
+        }
+        throw error
+      }
+    },
+  }
+}
+
+class ExpectedDuplicateMutation extends Error {
+  constructor(
+    readonly clientID: string,
+    readonly mutationID: number,
+    expectedMutationID: number | bigint,
+  ) {
+    super(
+      `Ignoring mutation from ${clientID} with ID ${mutationID} as it was already processed. Expected: ${expectedMutationID}`,
+    )
+  }
+}
+
+async function parseAlreadyProcessed(
+  request: Request,
+  response: unknown,
+): Promise<PersistAlreadyProcessedArgs | undefined> {
+  const mutations = alreadyProcessedMutations(response)
+  if (mutations.length === 0) return
+
+  const upstreamSchema = new URL(request.url).searchParams.get('schema')
+  if (!upstreamSchema) return
+
+  const body = (await request.json()) as PushRequestBody
+  if (typeof body.clientGroupID !== 'string') return
+
+  return {
+    clientGroupID: body.clientGroupID,
+    mutations,
+    upstreamSchema,
+  }
+}
+
+function alreadyProcessedMutations(response: unknown): AlreadyProcessedMutation[] {
+  if (!response || typeof response !== 'object') return []
+  const mutations = (response as { mutations?: unknown }).mutations
+  if (!Array.isArray(mutations)) return []
+
+  return mutations.filter((mutation): mutation is AlreadyProcessedMutation => {
+    if (!mutation || typeof mutation !== 'object') return false
+    const id = (mutation as { id?: unknown }).id
+    const result = (mutation as { result?: unknown }).result
+    return (
+      !!id &&
+      typeof id === 'object' &&
+      typeof (id as { clientID?: unknown }).clientID === 'string' &&
+      typeof (id as { id?: unknown }).id === 'number' &&
+      !!result &&
+      typeof result === 'object' &&
+      (result as { error?: unknown }).error === 'alreadyProcessed'
+    )
+  })
+}
+
+export type {
+  ValidateMutationArgs,
+  ValidateMutationFn,
+  ValidateQueryArgs,
+  ValidateQueryFn,
+  ZeroServerCommonOptions,
+  ZeroServerDatabase,
+} from './createZeroDirectServer'

package/src/helpers/createMutators.ts

@@ -30,7 +30,6 @@ export function createMutators<Models extends GenericModels>({
   models,
   validateMutation,
   mutationValidators,
-  resolveAuthData,
 }: {
   environment: 'server' | 'client'
   authData: AuthData | null
@@ -41,7 +40,6 @@ export function createMutators<Models extends GenericModels>({
   validateMutation?: ValidateMutationFn
   /** valibot schemas keyed by model.mutationName, auto-validates args before running */
   mutationValidators?: Record<string, Record<string, any>>
-  resolveAuthData?: () => AuthData | null
 }): GetZeroMutators<Models> {
   const serverActions = createServerActions?.()
 
@@ -56,8 +54,7 @@ export function createMutators<Models extends GenericModels>({
         tx,
         // on client, read authData dynamically to avoid stale closure during auth transitions
         // (ZeroProvider recreates Zero instance in useEffect, but mutations can run before that)
-        authData:
-          environment === 'client' ? getAuthData() : (resolveAuthData?.() ?? authData),
+        authData: environment === 'client' ? getAuthData() : authData,
         environment,
         can,
         server:
@@ -171,8 +168,7 @@ export function createMutators<Models extends GenericModels>({
       // then run user-provided validation hook as escape hatch
       if (validateMutation) {
         await validateMutation({
-          authData:
-            environment === 'client' ? getAuthData() : (resolveAuthData?.() ?? authData),
+          authData: environment === 'client' ? getAuthData() : authData,
           tableName,
           mutatorName,
           args: args[1],

package/src/httpPull/auth.test.ts

@@ -0,0 +1,208 @@
+import { afterEach, describe, expect, test } from 'vitest'
+
+import {
+  eventually,
+  sleep,
+  startZeroHttpHarness,
+  waitForComplete,
+  type ZeroHttpHarness,
+} from './testHarness'
+
+let harness: ZeroHttpHarness | undefined
+
+type UserRow = { id: string; name: string }
+type MemberRow = { id: string; projectId: string; userId: string }
+type ProjectWithMembers = {
+  id: string
+  ownerId: string
+  name: string
+  members: MemberRow[]
+}
+
+afterEach(async () => {
+  await harness?.close()
+  harness = undefined
+})
+
+describe('zero-http auth parity', () => {
+  test('real clients only emit rows visible to their auth token', async () => {
+    harness = await startZeroHttpHarness({
+      seed: {
+        user: [
+          { id: 'u1', name: 'ada' },
+          { id: 'u2', name: 'ben' },
+        ],
+        project: [
+          { id: 'p-u1', ownerId: 'u1', name: 'u1 private' },
+          { id: 'p-u2', ownerId: 'u2', name: 'u2 private' },
+          { id: 'p-shared', ownerId: 'u2', name: 'shared' },
+        ],
+        member: [
+          { id: 'm-u1-owner', projectId: 'p-u1', userId: 'u1' },
+          { id: 'm-u2-owner', projectId: 'p-u2', userId: 'u2' },
+          { id: 'm-shared-u1', projectId: 'p-shared', userId: 'u1' },
+          { id: 'm-shared-u2', projectId: 'p-shared', userId: 'u2' },
+        ],
+      },
+    })
+    const u1 = harness.createZero('u1')
+    const u2 = harness.createZero('u2')
+
+    const u1Users = u1.query.user.materialize()
+    const u1Projects = u1.query.project.related('members').materialize()
+    const u2Users = u2.query.user.materialize()
+    const u2Projects = u2.query.project.related('members').materialize()
+
+    const u1UserEmissions: UserRow[][] = []
+    const u1ProjectEmissions: ProjectWithMembers[][] = []
+    const u2UserEmissions: UserRow[][] = []
+    const u2ProjectEmissions: ProjectWithMembers[][] = []
+
+    const stops = [
+      captureRows(u1Users, u1UserEmissions, normalizeUsers),
+      captureRows(u1Projects, u1ProjectEmissions, normalizeProjects),
+      captureRows(u2Users, u2UserEmissions, normalizeUsers),
+      captureRows(u2Projects, u2ProjectEmissions, normalizeProjects),
+    ]
+
+    try {
+      const [u1UsersComplete, u1ProjectsComplete, u2UsersComplete, u2ProjectsComplete] =
+        await Promise.all([
+          waitForComplete<UserRow[]>(u1Users),
+          waitForComplete<ProjectWithMembers[]>(u1Projects),
+          waitForComplete<UserRow[]>(u2Users),
+          waitForComplete<ProjectWithMembers[]>(u2Projects),
+        ])
+
+      expect(normalizeUsers(u1UsersComplete)).toEqual([{ id: 'u1', name: 'ada' }])
+      expect(normalizeProjects(u1ProjectsComplete)).toEqual([
+        {
+          id: 'p-shared',
+          ownerId: 'u2',
+          name: 'shared',
+          members: [
+            { id: 'm-shared-u1', projectId: 'p-shared', userId: 'u1' },
+            { id: 'm-shared-u2', projectId: 'p-shared', userId: 'u2' },
+          ],
+        },
+        {
+          id: 'p-u1',
+          ownerId: 'u1',
+          name: 'u1 private',
+          members: [{ id: 'm-u1-owner', projectId: 'p-u1', userId: 'u1' }],
+        },
+      ])
+
+      expect(normalizeUsers(u2UsersComplete)).toEqual([{ id: 'u2', name: 'ben' }])
+      expect(normalizeProjects(u2ProjectsComplete)).toEqual([
+        {
+          id: 'p-shared',
+          ownerId: 'u2',
+          name: 'shared',
+          members: [
+            { id: 'm-shared-u1', projectId: 'p-shared', userId: 'u1' },
+            { id: 'm-shared-u2', projectId: 'p-shared', userId: 'u2' },
+          ],
+        },
+        {
+          id: 'p-u2',
+          ownerId: 'u2',
+          name: 'u2 private',
+          members: [{ id: 'm-u2-owner', projectId: 'p-u2', userId: 'u2' }],
+        },
+      ])
+
+      assertNoUserEmission(u1UserEmissions, 'u2')
+      assertNoProjectEmission(u1ProjectEmissions, 'p-u2', 'm-u2-owner')
+      assertNoUserEmission(u2UserEmissions, 'u1')
+      assertNoProjectEmission(u2ProjectEmissions, 'p-u1', 'm-u1-owner')
+
+      await harness.transport.pull()
+      await sleep(50)
+      assertNoUserEmission(u1UserEmissions, 'u2')
+      assertNoProjectEmission(u1ProjectEmissions, 'p-u2', 'm-u2-owner')
+      assertNoUserEmission(u2UserEmissions, 'u1')
+      assertNoProjectEmission(u2ProjectEmissions, 'p-u1', 'm-u1-owner')
+    } finally {
+      for (const stop of stops) stop()
+      u1Users.destroy()
+      u1Projects.destroy()
+      u2Users.destroy()
+      u2Projects.destroy()
+    }
+  })
+
+  test('unknown token 401 reaches needs-auth without materializing data', async () => {
+    harness = await startZeroHttpHarness({
+      seed: {
+        user: [{ id: 'u1', name: 'ada' }],
+        project: [{ id: 'p-u1', ownerId: 'u1', name: 'u1 private' }],
+        member: [{ id: 'm-u1-owner', projectId: 'p-u1', userId: 'u1' }],
+      },
+    })
+    const unknown = harness.createZero('missing')
+    const projects = unknown.query.project.related('members').materialize()
+    const emissions: ProjectWithMembers[][] = []
+    const stop = captureRows(projects, emissions, normalizeProjects)
+
+    try {
+      await eventually(() =>
+        expect(unknown.connection.state.current.name).toBe('needs-auth'),
+      )
+      await sleep(50)
+
+      expect(projects.data).toEqual([])
+      expect(emissions.every((emission) => emission.length === 0)).toBe(true)
+    } finally {
+      stop()
+      projects.destroy()
+    }
+  })
+})
+
+// listener data is Immutable<T>; normalize maps to fresh mutable rows
+function captureRows<T>(
+  view: { addListener(listener: (data: any, resultType: string) => void): () => void },
+  emissions: T[],
+  normalize: (data: T) => T,
+) {
+  return view.addListener((data) => {
+    emissions.push(normalize(data))
+  })
+}
+
+function assertNoUserEmission(emissions: UserRow[][], privateUserID: string) {
+  for (const emission of emissions) {
+    expect(emission.map((row) => row.id)).not.toContain(privateUserID)
+  }
+}
+
+function assertNoProjectEmission(
+  emissions: ProjectWithMembers[][],
+  privateProjectID: string,
+  privateMemberID: string,
+) {
+  for (const emission of emissions) {
+    expect(emission.map((row) => row.id)).not.toContain(privateProjectID)
+    expect(
+      emission.flatMap((row) => row.members.map((member) => member.id)),
+    ).not.toContain(privateMemberID)
+  }
+}
+
+function normalizeUsers(users: UserRow[]) {
+  return clone(users).sort((a, b) => a.id.localeCompare(b.id))
+}
+
+function normalizeProjects(projects: ProjectWithMembers[]) {
+  return clone(projects)
+    .map((project) => ({
+      ...project,
+      members: [...project.members].sort((a, b) => a.id.localeCompare(b.id)),
+    }))
+    .sort((a, b) => a.id.localeCompare(b.id))
+}
+
+function clone<T>(value: T): T {
+  return JSON.parse(JSON.stringify(value)) as T
+}

package/src/httpPull/churn.test.ts

@@ -0,0 +1,147 @@
+import { afterEach, expect, test } from 'vitest'
+
+import {
+  eventually,
+  startZeroHttpHarness,
+  waitForComplete,
+  type ZeroHttpHarness,
+} from './testHarness'
+
+let harness: ZeroHttpHarness | undefined
+
+afterEach(async () => {
+  await harness?.close()
+  harness = undefined
+})
+
+test('mutation burst converges while explicit pulls churn', async () => {
+  let clientGroupID = ''
+  const pullErrors: unknown[] = []
+
+  harness = await startZeroHttpHarness({
+    seed: {
+      user: [{ id: 'u1', name: 'ada' }],
+      project: [],
+      member: [],
+    },
+    interceptFetch: (next) => async (input, init) => {
+      const body = init?.body ? JSON.parse(String(init.body)) : undefined
+      if (body?.clientGroupID) clientGroupID = body.clientGroupID
+      return next(input, init)
+    },
+  })
+  const zero = harness.createZero('u1')
+  const view = zero.query.project.materialize()
+  await waitForComplete<any[]>(view)
+
+  const timer = setInterval(() => {
+    void harness?.transport.pull().catch((error) => pullErrors.push(error))
+  }, 2)
+
+  try {
+    const mutations = Array.from({ length: 15 }, (_, index) =>
+      zero.mutate.project.create({
+        id: `p${index + 1}`,
+        ownerId: 'u1',
+        name: `project ${index + 1}`,
+      }),
+    )
+    await Promise.all(mutations.map((mutation) => mutation.client))
+    await Promise.all(mutations.map((mutation) => mutation.server))
+    await harness.transport.pull()
+
+    await eventually(() => {
+      expect(projectIDs(view.data)).toEqual(
+        Array.from({ length: 15 }, (_, index) => `p${index + 1}`).sort(),
+      )
+    })
+  } finally {
+    clearInterval(timer)
+  }
+
+  expect(pullErrors).toEqual([])
+  expect(zero.connection.state.current.name).toBe('connected')
+  expect(
+    harness.server
+      .rows('project')
+      .map((row) => row.id)
+      .sort(),
+  ).toEqual(Array.from({ length: 15 }, (_, index) => `p${index + 1}`).sort())
+
+  const pull = await rawPull(harness, clientGroupID)
+  expect(Object.values(pull.lastMutationIDChanges)).toContain(15)
+  view.destroy()
+})
+
+test('two clients sharing a client group converge without cookie fights', async () => {
+  const clientGroups = new Set<string>()
+  const clientIDs = new Set<string>()
+
+  harness = await startZeroHttpHarness({
+    seed: {
+      user: [{ id: 'u1', name: 'ada' }],
+      project: [{ id: 'p1', ownerId: 'u1', name: 'first' }],
+      member: [],
+    },
+    interceptFetch: (next) => async (input, init) => {
+      const body = init?.body ? JSON.parse(String(init.body)) : undefined
+      if (body?.clientGroupID) clientGroups.add(body.clientGroupID)
+      if (body?.clientID) clientIDs.add(body.clientID)
+      return next(input, init)
+    },
+  })
+
+  const sharedStorageKey = 'zero-http-shared-client-group'
+  const zeroA = harness.createZero('u1', { storageKey: sharedStorageKey })
+  const zeroB = harness.createZero('u1', { storageKey: sharedStorageKey })
+  const viewA = zeroA.query.project.materialize()
+  const viewB = zeroB.query.project.materialize()
+  await waitForComplete<any[]>(viewA)
+  await waitForComplete<any[]>(viewB)
+
+  expect(harness.transport.connections).toBe(2)
+  expect(clientGroups.size).toBe(1)
+  expect(clientIDs.size).toBe(2)
+
+  const mutation = zeroA.mutate.project.create({
+    id: 'p2',
+    ownerId: 'u1',
+    name: 'second',
+  })
+  await mutation.client
+  await mutation.server
+  await harness.transport.pull()
+
+  await eventually(() => {
+    expect(projectIDs(viewA.data)).toEqual(['p1', 'p2'])
+    expect(projectIDs(viewB.data)).toEqual(['p1', 'p2'])
+  })
+  expect(zeroA.connection.state.current.name).toBe('connected')
+  expect(zeroB.connection.state.current.name).toBe('connected')
+
+  viewA.destroy()
+  viewB.destroy()
+})
+
+async function rawPull(harness: ZeroHttpHarness, clientGroupID: string) {
+  const response = await fetch(`${harness.server.url}/pull`, {
+    method: 'POST',
+    headers: {
+      authorization: 'Bearer token-u1',
+      'content-type': 'application/json',
+    },
+    body: JSON.stringify({
+      clientID: 'raw-pull',
+      clientGroupID,
+      cookie: null,
+    }),
+  })
+  expect(response.status).toBe(200)
+  return response.json() as Promise<{
+    lastMutationIDChanges: Record<string, number>
+  }>
+}
+
+function projectIDs(rows: any[]) {
+  return rows.map((row) => row.id).sort()
+}