on-zero 0.6.12 → 0.6.13

package/src/generate.ts

@@ -18,6 +18,12 @@ import type { ExtractedMutation, ModelMutations, SchemaColumn } from './generate
 
 const hash = (s: string) => createHash('sha256').update(s).digest('hex')
 
+const isGeneratorSourceFile = (name: string) =>
+  name.endsWith('.ts') &&
+  !name.endsWith('.d.ts') &&
+  !name.endsWith('.test.ts') &&
+  !name.endsWith('.spec.ts')
+
 // hash every .ts input the generator reads (all of baseDir except the generated
 // output dir + node_modules) so a dev-boot regen can be skipped when nothing
 // changed. the expensive part of generate() — loading the typescript compiler
@@ -36,11 +42,7 @@ function hashInputTree(baseDir: string, generatedDir: string): string {
       if (entry.isDirectory()) {
         if (entry.name === 'node_modules' || full === generatedDir) continue
         walk(full)
-      } else if (
-        entry.isFile() &&
-        entry.name.endsWith('.ts') &&
-        !entry.name.endsWith('.d.ts')
-      ) {
+      } else if (entry.isFile() && isGeneratorSourceFile(entry.name)) {
         parts.push(`${full}\0${readFileSync(full, 'utf-8')}`)
       }
     }
@@ -614,8 +616,9 @@ function tsTypeToValibot(
 
 type SchemaTable = {
   name: string
+  serverName?: string
   columns: Record<string, SchemaColumn>
-  primaryKey: string[]
+  primaryKey: readonly string[]
 }
 
 type SchemaRelationHop = {
@@ -685,7 +688,10 @@ export function generateDrizzleSchemaFile(schema: DrizzleZeroSchema): string {
       .join('\n')
     const pkArgs = t.primaryKey.map((k) => `'${k}'`).join(', ')
 
-    lines.push(`const ${tableName}Table = table('${t.name}')`)
+    const tableSource = t.serverName
+      ? `table(${JSON.stringify(t.name)}).from(${JSON.stringify(t.serverName)})`
+      : `table(${JSON.stringify(t.name)})`
+    lines.push(`const ${tableName}Table = ${tableSource}`)
     lines.push(`  .columns({`)
     lines.push(colEntries)
     lines.push(`  })`)
@@ -711,15 +717,24 @@ export function generateDrizzleSchemaFile(schema: DrizzleZeroSchema): string {
           return `    ${relName}: ${fn}({\n      sourceField: [${sf}],\n      destSchema: ${hop.destSchema}Table,\n      destField: [${df}],\n    })`
         }
         // many-to-many (2 hops)
+        if (hops.length !== 2) {
+          throw new Error(
+            `Relationship ${tableName}.${relName} must have one or two hops`,
+          )
+        }
+        const cardinality = hops[0]!.cardinality
+        if (hops.some((hop) => hop.cardinality !== cardinality)) {
+          throw new Error(`Relationship ${tableName}.${relName} must use one cardinality`)
+        }
+        const fn = cardinality === 'one' ? 'one' : 'many'
         const hopCode = hops
           .map((hop) => {
-            const fn = hop.cardinality === 'one' ? 'one' : 'many'
             const sf = hop.sourceField.map((f) => `'${f}'`).join(', ')
             const df = hop.destField.map((f) => `'${f}'`).join(', ')
-            return `${fn}({ sourceField: [${sf}], destSchema: ${hop.destSchema}Table, destField: [${df}] })`
+            return `{ sourceField: [${sf}], destSchema: ${hop.destSchema}Table, destField: [${df}] }`
           })
           .join(', ')
-        return `    ${relName}: [${hopCode}]`
+        return `    ${relName}: ${fn}(${hopCode})`
       })
       .join(',\n')
 
@@ -813,9 +828,7 @@ export async function generate(options: GenerateOptions): Promise<GenerateResult
     }
   }
 
-  const allModelFiles = readdirSync(modelsDir)
-    .filter((f) => f.endsWith('.ts'))
-    .sort()
+  const allModelFiles = readdirSync(modelsDir).filter(isGeneratorSourceFile).sort()
 
   const filesWithSchema = allModelFiles.filter((f) =>
     readFileSync(resolve(modelsDir, f), 'utf-8').includes('export const schema = table('),
@@ -867,7 +880,7 @@ export async function generate(options: GenerateOptions): Promise<GenerateResult
 
   // generate query files if queries directory exists
   if (existsSync(queriesDir)) {
-    const queryFiles = readdirSync(queriesDir).filter((f) => f.endsWith('.ts'))
+    const queryFiles = readdirSync(queriesDir).filter(isGeneratorSourceFile)
 
     const allQueries: Array<{
       name: string
@@ -881,7 +894,7 @@ export async function generate(options: GenerateOptions): Promise<GenerateResult
     const getQueryResolver = () => {
       if (!queryResolver) {
         const allFiles = readdirSync(queriesDir)
-          .filter((f) => f.endsWith('.ts'))
+          .filter(isGeneratorSourceFile)
           .map((f) => ({
             path: resolve(queriesDir, f),
             content: readFileSync(resolve(queriesDir, f), 'utf-8'),
@@ -1037,11 +1050,7 @@ export async function generate(options: GenerateOptions): Promise<GenerateResult
         const fullPath = resolve(dir, entry.name)
         if (entry.isDirectory() && entry.name !== 'node_modules') {
           results.push(...collectTsFiles(fullPath))
-        } else if (
-          entry.isFile() &&
-          entry.name.endsWith('.ts') &&
-          !entry.name.endsWith('.d.ts')
-        ) {
+        } else if (entry.isFile() && isGeneratorSourceFile(entry.name)) {
           results.push({ path: fullPath, content: readFileSync(fullPath, 'utf-8') })
         }
       }

package/src/httpPull/transport.test.ts

@@ -134,12 +134,12 @@ describe('zero-http transport', () => {
     )
   })
 
-  test('pushOrigin routes mutations through the authoritative application server', async () => {
+  test('pullOrigin and pushOrigin route sync through the authoritative application server', async () => {
     const requests: RequestRecord[] = []
     const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
       const request = recordRequest(input, init)
       requests.push(request)
-      if (request.path === '/pull') {
+      if (request.path.endsWith('/pull')) {
         return jsonResponse({ cookie: request.body.cookie, unchanged: true })
       }
       const mutation = request.body.mutations[0]
@@ -156,6 +156,7 @@ describe('zero-http transport', () => {
     })
     const transport = installHttpPullTransport({
       origin: ORIGIN,
+      pullOrigin: 'https://app.local/zero-http',
       pushOrigin: 'https://app.local/zero-http',
       fetch,
     })
@@ -163,7 +164,7 @@ describe('zero-http transport', () => {
     const zero = createZero()
 
     await eventually(() =>
-      expect(requests.some((request) => request.path === '/pull')).toBe(true),
+      expect(requests.some((request) => request.path.endsWith('/pull'))).toBe(true),
     )
     const mutation = zero.mutate.project.create({
       id: 'p1',
@@ -175,8 +176,8 @@ describe('zero-http transport', () => {
 
     const push = requests.find((request) => request.path.endsWith('/push'))
     expect(push?.url).toBe('https://app.local/zero-http/push')
-    expect(requests.find((request) => request.path === '/pull')?.url).toBe(
-      'https://zero-http.local/pull',
+    expect(requests.find((request) => request.path.endsWith('/pull'))?.url).toBe(
+      'https://app.local/zero-http/pull',
     )
   })
 

package/src/httpPullTransport.ts

@@ -60,6 +60,7 @@ type PullResponse =
 type TransportState = {
   readonly origin: URL
   readonly originString: string
+  readonly pullOriginString: string
   readonly pushOriginString: string
   readonly fetch: typeof fetch
   readonly nativeWebSocket: WebSocketConstructor | undefined
@@ -85,9 +86,9 @@ export type HttpPullTransport = {
 
 export type HttpPullTransportOptions = {
   origin: string
-  // optional authoritative mutation endpoint base. reads and wake stay on
-  // origin, while push POSTs to <pushOrigin>/push. this supports a native read
-  // host paired with an application server that owns custom mutator execution.
+  // optional authoritative sync endpoint bases. websocket interception and
+  // wake stay on origin, while pull/push POST to their configured bases.
+  pullOrigin?: string
   pushOrigin?: string
   fetch?: typeof fetch
   // when set, every open connection also pulls on this interval so
@@ -121,6 +122,9 @@ export function installHttpPullTransport(
   const state: TransportState = {
     origin: new URL(opts.origin),
     originString: trimTrailingSlash(new URL(opts.origin).toString()),
+    pullOriginString: trimTrailingSlash(
+      new URL(opts.pullOrigin ?? opts.origin).toString(),
+    ),
     pushOriginString: trimTrailingSlash(
       new URL(opts.pushOrigin ?? opts.origin).toString(),
     ),
@@ -540,7 +544,8 @@ class ZeroHttpSocket {
   }
 
   private async postJSON(path: '/pull' | '/push', body: unknown) {
-    const base = path === '/push' ? this.state.pushOriginString : this.state.originString
+    const base =
+      path === '/push' ? this.state.pushOriginString : this.state.pullOriginString
     const response = await this.state.fetch(`${base}${path}`, {
       method: 'POST',
       headers: {

package/src/server.ts

@@ -1 +1,2 @@
 export * from './createZeroServer'
+export * from './createZeroSQLiteServer'

package/src/syncHost.test.ts

@@ -0,0 +1,216 @@
+import { DatabaseSync, type SQLInputValue } from 'node:sqlite'
+
+import { defineQueries, defineQuery } from '@rocicorp/zero'
+import * as v from 'valibot'
+import { afterEach, beforeEach, describe, expect, test } from 'vitest'
+
+import { getScopedAuthData } from './helpers/mutatorContext'
+import { zeroHttpFixtureSchema } from './httpPull/fixtureSchema'
+import { createZeroSyncHostAdapter } from './syncHost'
+import { zql } from './zql'
+
+import type { SQLiteQueryFormat } from './createZeroSQLiteServer'
+import type {
+  SyncHostApplicationTransaction,
+  SyncHostMutatorContext,
+  SyncHostTransactionExecutor,
+} from './syncHost'
+
+const queries = defineQueries({
+  project: {
+    byOwner: defineQuery(v.object({ ownerId: v.string() }), ({ args }) =>
+      zql.project.where('ownerId', args.ownerId),
+    ),
+  },
+})
+
+describe('createZeroSyncHostAdapter', () => {
+  let db: DatabaseSync
+  let executor: SyncHostTransactionExecutor
+  let transaction: SyncHostApplicationTransaction
+  let lastQueryFormat: SQLiteQueryFormat | undefined
+
+  beforeEach(() => {
+    db = new DatabaseSync(':memory:')
+    db.exec(
+      'CREATE TABLE project (id TEXT PRIMARY KEY, ownerId TEXT NOT NULL, name TEXT NOT NULL)',
+    )
+    executor = {
+      exec(sql, params = []) {
+        db.prepare(sql).run(...(params as SQLInputValue[]))
+      },
+      query<Row extends Record<string, unknown>>(sql, params = []) {
+        return db.prepare(sql).all(...(params as SQLInputValue[])) as Row[]
+      },
+      queryAst<Result>(_ast, format) {
+        lastQueryFormat = format
+        const rows = db.prepare('SELECT * FROM project ORDER BY id').all()
+        return (format.singular ? rows[0] : rows) as Result
+      },
+    }
+    transaction = async (work) => {
+      const effects: Array<() => void | Promise<void>> = []
+      db.exec('BEGIN')
+      try {
+        const value = await work(executor, {
+          defer(effect) {
+            effects.push(effect)
+          },
+        })
+        db.exec('COMMIT')
+        for (const effect of effects) await effect()
+        return value
+      } catch (error) {
+        db.exec('ROLLBACK')
+        throw error
+      }
+    }
+  })
+
+  afterEach(() => {
+    db.close()
+  })
+
+  test('adapts pipe-named mutators, auth, validation, and deferred effects', async () => {
+    let effectAuthID: string | undefined
+    let validatedAuthID: string | undefined
+    const adapter = createZeroSyncHostAdapter({
+      schema: zeroHttpFixtureSchema,
+      models: {
+        project: {
+          mutate: {
+            create: async ({ authData, server, tx }, value: Record<string, unknown>) => {
+              expect(authData?.id).toBe('u1')
+              await tx.mutate.project.insert(value as never)
+              server?.asyncTasks.push(async () => {
+                effectAuthID = getScopedAuthData()?.id as string
+              })
+            },
+          },
+        },
+      },
+      createServerActions: () => ({}),
+      queries,
+      transaction,
+      validateMutation({ authData }) {
+        validatedAuthID = authData?.id as string
+      },
+    })
+
+    expect(Object.keys(adapter.mutators)).toEqual([
+      'project|create',
+      '_zero_cleanupResults',
+    ])
+    await transaction((tx, context) =>
+      adapter.mutators['project|create']!(
+        tx,
+        { id: 'p1', ownerId: 'u1', name: 'first' },
+        mutationContext(context),
+      ),
+    )
+
+    expect(db.prepare('SELECT * FROM project').all()).toEqual([
+      { id: 'p1', ownerId: 'u1', name: 'first' },
+    ])
+    expect(validatedAuthID).toBe('u1')
+    expect(effectAuthID).toBe('u1')
+  })
+
+  test('returns a structural application error and drops rolled-back effects', async () => {
+    let effectRuns = 0
+    const adapter = createZeroSyncHostAdapter({
+      schema: zeroHttpFixtureSchema,
+      models: {
+        project: {
+          mutate: {
+            reject: async ({ server, tx }, value: Record<string, unknown>) => {
+              await tx.mutate.project.insert(value as never)
+              server?.asyncTasks.push(async () => {
+                effectRuns++
+              })
+              throw new Error('denied')
+            },
+          },
+        },
+      },
+      createServerActions: () => ({}),
+      transaction,
+    })
+
+    const error = await transaction((tx, context) =>
+      adapter.mutators['project|reject']!(
+        tx,
+        { id: 'p1', ownerId: 'u1', name: 'must roll back' },
+        mutationContext(context),
+      ),
+    ).catch((caught) => caught)
+
+    expect(error).toMatchObject({
+      name: 'MutationApplicationError',
+      message: 'denied',
+      details: 'denied',
+    })
+    expect(db.prepare('SELECT * FROM project').all()).toEqual([])
+    expect(effectRuns).toBe(0)
+  })
+
+  test('uses the host queryAst compiler surface for direct server queries', async () => {
+    db.prepare('INSERT INTO project (id, ownerId, name) VALUES (?, ?, ?)').run(
+      'p1',
+      'u1',
+      'first',
+    )
+    const adapter = createZeroSyncHostAdapter({
+      schema: zeroHttpFixtureSchema,
+      models: {},
+      createServerActions: () => ({}),
+      transaction,
+    })
+
+    await expect(adapter.server.query((q) => q.project.one())).resolves.toEqual({
+      id: 'p1',
+      ownerId: 'u1',
+      name: 'first',
+    })
+    expect(lastQueryFormat).toEqual({ relationships: {}, singular: true })
+  })
+
+  test('resolves auth-sensitive named queries and runs query validation', async () => {
+    let validation: unknown
+    const adapter = createZeroSyncHostAdapter({
+      schema: zeroHttpFixtureSchema,
+      models: {},
+      createServerActions: () => ({}),
+      queries,
+      transaction,
+      validateQuery(args) {
+        validation = args
+      },
+    })
+
+    const ast = await adapter.resolveQuery('project|byOwner', [{ ownerId: 'u1' }], {
+      userID: 'u1',
+      role: 'member',
+    })
+
+    expect(ast).toMatchObject({ table: 'project' })
+    expect(JSON.stringify(ast)).toContain('ownerId')
+    expect(JSON.stringify(ast)).toContain('u1')
+    expect(validation).toEqual({
+      authData: { id: 'u1', role: 'member', userID: 'u1' },
+      queryName: 'project|byOwner',
+      params: { ownerId: 'u1' },
+    })
+  })
+})
+
+function mutationContext(context: {
+  defer(effect: () => void | Promise<void>): void
+}): SyncHostMutatorContext {
+  return {
+    ...context,
+    claims: { userID: 'u1', role: 'member' },
+    clientID: 'client-a',
+    mutationID: '1',
+  }
+}

package/src/syncHost.ts

@@ -0,0 +1,255 @@
+import { asQueryInternals } from '@rocicorp/zero/bindings'
+
+import { createPermissions } from './createPermissions'
+import { resolveServerQuery, type ZeroServerCommonOptions } from './createZeroServerCore'
+import {
+  createZeroSQLiteServer,
+  createZeroSQLiteTransaction,
+  type SQLiteTransactionExecutor,
+  type ZeroSQLiteTransactionOptions,
+} from './createZeroSQLiteServer'
+import { createMutators } from './helpers/createMutators'
+import { runWithAuthScope } from './helpers/mutatorContext'
+import { runWithQueryContext } from './helpers/queryContext'
+import { setEnvironment, setSchema } from './state'
+
+import type { AuthData, GenericModels } from './types'
+import type { Schema as ZeroSchema } from '@rocicorp/zero'
+
+export type SyncHostJsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | SyncHostJsonValue[]
+  | { [key: string]: SyncHostJsonValue }
+
+export type SyncHostClaims = {
+  userID: string
+  [claim: string]: SyncHostJsonValue
+}
+
+export type SyncHostTransactionContext = {
+  defer(effect: () => void | Promise<void>): void
+}
+
+export type SyncHostTransactionExecutor = {
+  exec(sql: string, params?: readonly unknown[]): void | Promise<void>
+  query<Row extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    params?: readonly unknown[],
+  ): Row[] | Promise<Row[]>
+  queryAst<Result = unknown>(
+    ast: SyncHostJsonValue,
+    format: import('./createZeroSQLiteServer').SQLiteQueryFormat,
+    queryName?: string,
+  ): Result | Promise<Result>
+}
+
+export type SyncHostMutatorContext = SyncHostTransactionContext & {
+  claims: SyncHostClaims
+  clientID?: string
+  mutationID?: string | number
+}
+
+export type SyncHostRegisteredMutator = (
+  tx: SyncHostTransactionExecutor,
+  args: SyncHostJsonValue,
+  context: SyncHostMutatorContext,
+) => void | Promise<void>
+
+export type SyncHostApplicationTransaction = <Value>(
+  work: (
+    tx: SyncHostTransactionExecutor,
+    context: SyncHostTransactionContext,
+  ) => Value | Promise<Value>,
+) => Promise<Value>
+
+export type CreateZeroSyncHostAdapterOptions<
+  Schema extends ZeroSchema,
+  Models extends GenericModels,
+  ServerActions extends Record<string, unknown>,
+> = Omit<
+  ZeroServerCommonOptions<Schema, Models, ServerActions>,
+  'defaultMutateAuthData'
+> &
+  ZeroSQLiteTransactionOptions<Schema> & {
+    transaction: SyncHostApplicationTransaction
+    mapAuthData?: (claims: SyncHostClaims) => AuthData
+    defaultMutateAuthData?: Pick<AuthData, 'email' | 'id'> & Partial<AuthData>
+  }
+
+export class MutationApplicationError extends Error {
+  constructor(
+    readonly details: string,
+    message = details,
+  ) {
+    super(message)
+    this.name = 'MutationApplicationError'
+  }
+}
+
+export function createZeroSyncHostAdapter<
+  Schema extends ZeroSchema,
+  Models extends GenericModels,
+  ServerActions extends Record<string, unknown>,
+>({
+  encodeValue,
+  mapAuthData = defaultMapAuthData,
+  transaction,
+  ...options
+}: CreateZeroSyncHostAdapterOptions<Schema, Models, ServerActions>) {
+  setSchema(options.schema)
+  setEnvironment('server')
+
+  const permissions = createPermissions({
+    environment: 'server',
+    schema: options.schema,
+    adminRoleMode: options.defaultAllowAdminRole ?? 'all',
+  })
+  const transactionOptions = {
+    encodeValue,
+    schema: options.schema,
+  }
+  const mutators: Record<string, SyncHostRegisteredMutator> = {}
+
+  for (const [modelName, model] of Object.entries(options.models)) {
+    for (const mutatorName of Object.keys(model.mutate ?? {})) {
+      mutators[`${modelName}|${mutatorName}`] = async (executor, args, context) => {
+        const authData = mapAuthData(context.claims)
+        const asyncTasks: Array<() => Promise<void>> = []
+        const modelMutators = createMutators({
+          asyncTasks,
+          authData,
+          can: permissions.can,
+          createServerActions: options.createServerActions,
+          environment: 'server',
+          models: options.models,
+          mutationValidators: options.mutations,
+          validateMutation: options.validateMutation,
+        }) as Record<
+          string,
+          Record<string, (tx: unknown, args: unknown) => Promise<void>>
+        >
+        const mutation = modelMutators[modelName]?.[mutatorName]
+        if (!mutation)
+          throw new Error(`unknown on-zero mutator: ${modelName}|${mutatorName}`)
+
+        const mutationID = numericMutationID(context.mutationID)
+        const onZeroTransaction = createZeroSQLiteTransaction(
+          transactionOptions,
+          toSQLiteExecutor(executor),
+          {
+            clientGroupID: 'orez-sync-host',
+            clientID: context.clientID ?? '',
+            mutationID,
+            upstreamSchema: 'orez-sync-host',
+          },
+        )
+        try {
+          await mutation(onZeroTransaction, args)
+        } catch (error) {
+          throw asMutationApplicationError(error)
+        }
+        for (const task of asyncTasks) {
+          context.defer(() => runWithAuthScope(authData, task))
+        }
+      }
+    }
+  }
+
+  // orez owns result retention and lmids. zero clients may still emit this
+  // protocol mutation, so accepting it advances the host ledger without
+  // creating a second on-zero cleanup ledger.
+  mutators._zero_cleanupResults = async () => {}
+
+  const direct = createZeroSQLiteServer({
+    ...options,
+    encodeValue,
+    transactionProvider: (work) =>
+      transaction((executor) => work(toSQLiteExecutor(executor))),
+  })
+
+  return {
+    mutators: Object.freeze(mutators),
+    async resolveQuery(
+      name: string,
+      args: readonly SyncHostJsonValue[],
+      claims: SyncHostClaims,
+    ): Promise<SyncHostJsonValue> {
+      if (!options.queries) {
+        throw new Error(
+          'No queries registered with createZeroSyncHostAdapter. Pass the syncedQueries registry via the queries option.',
+        )
+      }
+      const authData = mapAuthData(claims)
+      const query = await runWithQueryContext({ authData }, () =>
+        resolveServerQuery({
+          authData,
+          args: args[0],
+          name,
+          permissions,
+          queries: options.queries!,
+          validateQuery: options.validateQuery,
+        }),
+      )
+      return asQueryInternals(query as never).ast as unknown as SyncHostJsonValue
+    },
+    server: {
+      mutate: direct.mutate,
+      query: direct.query,
+      transaction: direct.transaction,
+    },
+  }
+}
+
+function defaultMapAuthData(claims: SyncHostClaims): AuthData {
+  return {
+    ...claims,
+    id: typeof claims.id === 'string' ? claims.id : claims.userID,
+  } as AuthData
+}
+
+function toSQLiteExecutor(
+  executor: SyncHostTransactionExecutor,
+): SQLiteTransactionExecutor {
+  return {
+    exec(sql, params) {
+      return executor.exec(sql, params)
+    },
+    query<Row extends Record<string, unknown>>(sql: string, params?: readonly unknown[]) {
+      return executor.query<Row>(sql, params)
+    },
+    queryAst<Result>(ast, format, queryName) {
+      return executor.queryAst<Result>(
+        ast as unknown as SyncHostJsonValue,
+        format,
+        queryName,
+      )
+    },
+  }
+}
+
+function numericMutationID(value: string | number | undefined): number {
+  if (value === undefined) return 0
+  const numeric = typeof value === 'number' ? value : Number(value)
+  if (!Number.isSafeInteger(numeric) || numeric < 0) {
+    throw new TypeError(`invalid Zero mutation ID: ${value}`)
+  }
+  return numeric
+}
+
+function asMutationApplicationError(error: unknown): MutationApplicationError {
+  if (
+    typeof error === 'object' &&
+    error !== null &&
+    (error as { name?: unknown }).name === 'MutationApplicationError' &&
+    typeof (error as { details?: unknown }).details === 'string'
+  ) {
+    return error as MutationApplicationError
+  }
+  const message = error instanceof Error ? error.message : String(error)
+  const applicationError = new MutationApplicationError(message)
+  if (error instanceof Error && error.stack) applicationError.stack = error.stack
+  return applicationError
+}