on-zero 0.6.14 → 0.6.16

package/src/createZeroSQLiteServer.ts

@@ -1,60 +1,17 @@
-import { createBuilder } from '@rocicorp/zero'
-import { asQueryInternals } from '@rocicorp/zero/bindings'
-
+import { createZeroServerCore } from './createZeroServerCore'
 import {
-  createZeroServerCore,
-  type PersistAlreadyProcessedArgs,
-  type ZeroServer,
-  type ZeroServerCommonOptions,
-  type ZeroServerDatabase,
-} from './createZeroServerCore'
-
-import type { GenericModels, Transaction } from './types'
-import type { AST, Schema as ZeroSchema } from '@rocicorp/zero'
-import type {
-  TransactionProviderHooks,
-  TransactionProviderInput,
-} from '@rocicorp/zero/server'
-
-export type SQLiteTransactionExecutor = {
-  exec(sql: string, params?: readonly unknown[]): void | Promise<void>
-  query<Row extends Record<string, unknown> = Record<string, unknown>>(
-    sql: string,
-    params?: readonly unknown[],
-  ): Row[] | Promise<Row[]>
-  queryAst<Result = unknown>(
-    ast: AST,
-    format: SQLiteQueryFormat,
-    queryName?: string,
-  ): Result | Promise<Result>
-}
-
-export type SQLiteQueryFormat = {
-  relationships: Record<string, SQLiteQueryFormat>
-  singular: boolean
-}
-
-export type SQLiteTransactionProvider = <R>(
-  callback: (tx: SQLiteTransactionExecutor) => Promise<R>,
-) => Promise<R>
-
-type SQLiteColumn = ZeroSchema['tables'][string]['columns'][string]
-
-export type SQLiteValueContext = {
-  column: SQLiteColumn
-  columnName: string
-  tableName: string
-}
-
-export type ZeroSQLiteTransactionOptions<Schema extends ZeroSchema> = {
-  schema: Schema
-  encodeValue?: (value: unknown, context: SQLiteValueContext) => unknown
-}
-
-export type SQLiteLedgerTables = {
-  clients: string
-  mutations: string
-}
+  createSQLiteDatabase,
+  defaultLedgerTables,
+  encodeSQLiteValue,
+  persistAlreadyProcessedMutationResults,
+  type SQLiteLedgerTables,
+  type SQLiteTransactionProvider,
+  type SQLiteValueContext,
+} from './sqliteRuntime'
+
+import type { ZeroServer, ZeroServerCommonOptions } from './createZeroServerCore'
+import type { GenericModels } from './types'
+import type { Schema as ZeroSchema } from '@rocicorp/zero'
 
 export type CreateZeroSQLiteServerOptions<
   Schema extends ZeroSchema,
@@ -96,428 +53,4 @@ export function createZeroSQLiteServer<
   })
 }
 
-type SQLiteTransactionOptions<Schema extends ZeroSchema> = {
-  encodeValue: (value: unknown, context: SQLiteValueContext) => unknown
-  schema: Schema
-}
-
-type SQLiteDatabaseOptions<Schema extends ZeroSchema> =
-  SQLiteTransactionOptions<Schema> & {
-    initializedLedgers: Set<string>
-    ledgerTables: (upstreamSchema: string) => SQLiteLedgerTables
-    transactionProvider: SQLiteTransactionProvider
-  }
-
-function createSQLiteDatabase<Schema extends ZeroSchema>(
-  options: SQLiteDatabaseOptions<Schema>,
-): ZeroServerDatabase {
-  return {
-    async transaction<R>(
-      callback: (
-        tx: Transaction,
-        transactionHooks: TransactionProviderHooks,
-      ) => R | Promise<R>,
-      input?: TransactionProviderInput,
-    ) {
-      let committedLedgers = new Set<string>()
-      const value = await options.transactionProvider(async (executor) => {
-        const attemptLedgers = new Set<string>()
-        const ensureLedger = async (upstreamSchema: string) => {
-          const tables = options.ledgerTables(upstreamSchema)
-          const key = ledgerKey(tables)
-          if (options.initializedLedgers.has(key) || attemptLedgers.has(key)) {
-            return
-          }
-          await createLedgerTables(executor, tables)
-          attemptLedgers.add(key)
-        }
-        const transaction = createZeroSQLiteTransaction(options, executor, input)
-        const result = await callback(
-          transaction,
-          createLedgerHooks(options, executor, ensureLedger, input),
-        )
-        committedLedgers = attemptLedgers
-        return result
-      })
-      for (const key of committedLedgers) options.initializedLedgers.add(key)
-      return value
-    },
-  }
-}
-
-export function createZeroSQLiteTransaction<Schema extends ZeroSchema>(
-  options: ZeroSQLiteTransactionOptions<Schema>,
-  executor: SQLiteTransactionExecutor,
-  input?: TransactionProviderInput,
-): Transaction {
-  const transactionOptions = {
-    encodeValue: options.encodeValue ?? encodeSQLiteValue,
-    schema: options.schema,
-  }
-  const clientID = input?.clientID ?? ''
-  const mutationID = input?.mutationID ?? 0
-  const mutate = new Proxy(
-    {},
-    {
-      get(_target, tableKey) {
-        const tableName = String(tableKey)
-        if (!options.schema.tables[tableName]) {
-          throw new Error(`unknown SQLite mutation table ${tableName}`)
-        }
-        return new Proxy(
-          {},
-          {
-            get(_innerTarget, kindKey) {
-              const kind = String(kindKey)
-              if (!isCrudKind(kind)) return
-              return (value: unknown) =>
-                executeCrud(transactionOptions, executor, tableName, kind, value)
-            },
-          },
-        )
-      },
-    },
-  )
-
-  const runQuery = (
-    ast: AST,
-    format: SQLiteQueryFormat,
-    queryName?: string,
-  ): Promise<unknown> => Promise.resolve(executor.queryAst(ast, format, queryName))
-  const dbTransaction = {
-    wrappedTransaction: executor,
-    query: (sql: string, args: unknown[]) => executor.query(sql, args),
-    runQuery,
-  }
-
-  return {
-    clientID,
-    dbTransaction,
-    location: 'server',
-    mutate,
-    mutationID,
-    query: transactionOptions.schema.enableLegacyQueries
-      ? createLegacyQueryBuilder(transactionOptions.schema, (query) => {
-          const internals = asQueryInternals(query as never)
-          return runQuery(internals.ast, internals.format, internals.customQueryID?.name)
-        })
-      : undefined,
-    reason: 'authoritative',
-    async run(query: unknown) {
-      const internals = asQueryInternals(query as never)
-      return runQuery(internals.ast, internals.format, internals.customQueryID?.name)
-    },
-  } as unknown as Transaction
-}
-
-type CrudKind = 'delete' | 'insert' | 'update' | 'upsert'
-
-function isCrudKind(value: string): value is CrudKind {
-  return (
-    value === 'delete' || value === 'insert' || value === 'update' || value === 'upsert'
-  )
-}
-
-async function executeCrud<Schema extends ZeroSchema>(
-  options: SQLiteTransactionOptions<Schema>,
-  executor: SQLiteTransactionExecutor,
-  tableName: string,
-  kind: CrudKind,
-  raw: unknown,
-) {
-  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
-    throw new Error(`${tableName}.${kind} expects an object`)
-  }
-
-  const table = options.schema.tables[tableName]!
-  const values = Object.fromEntries(
-    Object.entries(raw as Record<string, unknown>).filter(
-      ([, value]) => value !== undefined,
-    ),
-  )
-  for (const columnName of Object.keys(values)) {
-    if (!table.columns[columnName]) {
-      throw new Error(`unknown SQLite column ${tableName}.${columnName}`)
-    }
-  }
-
-  const primaryValues = table.primaryKey.map((columnName) => {
-    if (!(columnName in values)) {
-      throw new Error(`${tableName}.${kind} is missing primary key ${columnName}`)
-    }
-    return encodeColumn(options, tableName, columnName, values[columnName])
-  })
-  const primaryWhere = table.primaryKey
-    .map((columnName) => `${quoteSQLiteIdentifier(serverColumn(table, columnName))} = ?`)
-    .join(' AND ')
-  const serverTableName = quoteSQLiteIdentifier(table.serverName ?? table.name)
-
-  if (kind === 'delete') {
-    await executor.exec(
-      `DELETE FROM ${serverTableName} WHERE ${primaryWhere}`,
-      primaryValues,
-    )
-    return
-  }
-
-  const columnNames = Object.keys(values)
-  if (columnNames.length === 0) {
-    throw new Error(`${tableName}.${kind} has no values`)
-  }
-
-  const serverColumns = columnNames.map((columnName) =>
-    quoteSQLiteIdentifier(serverColumn(table, columnName)),
-  )
-  const params = columnNames.map((columnName) =>
-    encodeColumn(options, tableName, columnName, values[columnName]),
-  )
-  const insert = `INSERT INTO ${serverTableName} (${serverColumns.join(
-    ', ',
-  )}) VALUES (${columnNames.map(() => '?').join(', ')})`
-
-  if (kind === 'insert') {
-    await executor.exec(insert, params)
-    return
-  }
-
-  if (kind === 'upsert') {
-    const mutableColumns = columnNames.filter(
-      (columnName) => !table.primaryKey.includes(columnName),
-    )
-    const conflictColumns = table.primaryKey
-      .map((columnName) => quoteSQLiteIdentifier(serverColumn(table, columnName)))
-      .join(', ')
-    const conflictAction =
-      mutableColumns.length === 0
-        ? 'DO NOTHING'
-        : `DO UPDATE SET ${mutableColumns
-            .map((columnName) => {
-              const column = quoteSQLiteIdentifier(serverColumn(table, columnName))
-              return `${column} = excluded.${column}`
-            })
-            .join(', ')}`
-    await executor.exec(
-      `${insert} ON CONFLICT (${conflictColumns}) ${conflictAction}`,
-      params,
-    )
-    return
-  }
-
-  const mutableColumns = columnNames.filter(
-    (columnName) => !table.primaryKey.includes(columnName),
-  )
-  if (mutableColumns.length === 0) return
-  await executor.exec(
-    `UPDATE ${serverTableName} SET ${mutableColumns
-      .map(
-        (columnName) => `${quoteSQLiteIdentifier(serverColumn(table, columnName))} = ?`,
-      )
-      .join(', ')} WHERE ${primaryWhere}`,
-    [
-      ...mutableColumns.map((columnName) =>
-        encodeColumn(options, tableName, columnName, values[columnName]),
-      ),
-      ...primaryValues,
-    ],
-  )
-}
-
-function serverColumn(table: ZeroSchema['tables'][string], columnName: string) {
-  const column = table.columns[columnName]
-  if (!column) throw new Error(`unknown SQLite column ${table.name}.${columnName}`)
-  return column.serverName ?? columnName
-}
-
-function encodeColumn<Schema extends ZeroSchema>(
-  options: SQLiteTransactionOptions<Schema>,
-  tableName: string,
-  columnName: string,
-  value: unknown,
-) {
-  return options.encodeValue(value, {
-    column: options.schema.tables[tableName]!.columns[columnName]!,
-    columnName,
-    tableName,
-  })
-}
-
-function createLegacyQueryBuilder<Schema extends ZeroSchema>(
-  schema: Schema,
-  run: (query: unknown) => Promise<unknown>,
-) {
-  const builder = createBuilder(schema)
-  return new Proxy(builder, {
-    get(target, property) {
-      return makeRunnableQuery(Reflect.get(target, property, target), run)
-    },
-  })
-}
-
-function makeRunnableQuery(
-  query: unknown,
-  run: (query: unknown) => Promise<unknown>,
-): unknown {
-  if (!isQueryBuilder(query)) return query
-  return new Proxy(query as object, {
-    get(target, property) {
-      if (property === 'run') return () => run(target)
-      if (property === 'materialize' || property === 'preload') {
-        return () => {
-          throw new Error(`${String(property)}() is not supported in server transactions`)
-        }
-      }
-      const value = Reflect.get(target, property, target)
-      if (typeof value !== 'function') return value
-      return (...args: unknown[]) =>
-        makeRunnableQuery(Reflect.apply(value, target, args), run)
-    },
-  })
-}
-
-function isQueryBuilder(value: unknown) {
-  if (!value || typeof value !== 'object') return false
-  try {
-    asQueryInternals(value as never)
-    return true
-  } catch {
-    return false
-  }
-}
-
-function createLedgerHooks<Schema extends ZeroSchema>(
-  options: SQLiteDatabaseOptions<Schema>,
-  executor: SQLiteTransactionExecutor,
-  ensureLedger: (upstreamSchema: string) => Promise<void>,
-  input?: TransactionProviderInput,
-): TransactionProviderHooks {
-  const upstreamSchema = input?.upstreamSchema ?? ''
-  const clientGroupID = input?.clientGroupID ?? ''
-  const clientID = input?.clientID ?? ''
-  const tables = options.ledgerTables(upstreamSchema)
-  const clientsTable = quoteSQLiteIdentifier(tables.clients)
-  const mutationsTable = quoteSQLiteIdentifier(tables.mutations)
-
-  return {
-    async updateClientMutationID() {
-      await ensureLedger(upstreamSchema)
-      const rows = await executor.query<{ lastMutationID: number | bigint }>(
-        `INSERT INTO ${clientsTable}
-           (clientGroupID, clientID, lastMutationID)
-         VALUES (?, ?, 1)
-         ON CONFLICT (clientGroupID, clientID)
-         DO UPDATE SET lastMutationID = lastMutationID + 1
-         RETURNING lastMutationID`,
-        [clientGroupID, clientID],
-      )
-      const lastMutationID = rows[0]?.lastMutationID
-      if (typeof lastMutationID !== 'number' && typeof lastMutationID !== 'bigint') {
-        throw new Error('SQLite mutation ledger returned no lastMutationID')
-      }
-      return { lastMutationID }
-    },
-    async writeMutationResult(result) {
-      await ensureLedger(upstreamSchema)
-      await executor.exec(
-        `INSERT INTO ${mutationsTable}
-           (clientGroupID, clientID, mutationID, result)
-         VALUES (?, ?, ?, ?)`,
-        [clientGroupID, result.id.clientID, result.id.id, JSON.stringify(result.result)],
-      )
-    },
-    async deleteMutationResults(args) {
-      await ensureLedger(upstreamSchema)
-      if ('type' in args && args.type === 'bulk') {
-        if (args.clientIDs.length === 0) return
-        await executor.exec(
-          `DELETE FROM ${mutationsTable}
-           WHERE clientGroupID = ?
-             AND clientID IN (${args.clientIDs.map(() => '?').join(', ')})`,
-          [args.clientGroupID, ...args.clientIDs],
-        )
-        return
-      }
-      await executor.exec(
-        `DELETE FROM ${mutationsTable}
-         WHERE clientGroupID = ?
-           AND clientID = ?
-           AND mutationID <= ?`,
-        [args.clientGroupID, args.clientID, args.upToMutationID],
-      )
-    },
-  }
-}
-
-async function persistAlreadyProcessedMutationResults<Schema extends ZeroSchema>(
-  options: SQLiteDatabaseOptions<Schema>,
-  { clientGroupID, mutations, upstreamSchema }: PersistAlreadyProcessedArgs,
-) {
-  const tables = options.ledgerTables(upstreamSchema)
-  const table = quoteSQLiteIdentifier(tables.mutations)
-  const key = ledgerKey(tables)
-  await options.transactionProvider(async (executor) => {
-    if (!options.initializedLedgers.has(key)) {
-      await createLedgerTables(executor, tables)
-    }
-    for (const mutation of mutations) {
-      await executor.exec(
-        `INSERT INTO ${table}
-           (clientGroupID, clientID, mutationID, result)
-         VALUES (?, ?, ?, ?)
-         ON CONFLICT (clientGroupID, clientID, mutationID) DO NOTHING`,
-        [
-          clientGroupID,
-          mutation.id.clientID,
-          mutation.id.id,
-          JSON.stringify(mutation.result),
-        ],
-      )
-    }
-  })
-  options.initializedLedgers.add(key)
-}
-
-async function createLedgerTables(
-  executor: SQLiteTransactionExecutor,
-  tables: SQLiteLedgerTables,
-) {
-  await executor.exec(
-    `CREATE TABLE IF NOT EXISTS ${quoteSQLiteIdentifier(tables.clients)} (
-       clientGroupID TEXT NOT NULL,
-       clientID TEXT NOT NULL,
-       lastMutationID INTEGER NOT NULL,
-       PRIMARY KEY (clientGroupID, clientID)
-     )`,
-  )
-  await executor.exec(
-    `CREATE TABLE IF NOT EXISTS ${quoteSQLiteIdentifier(tables.mutations)} (
-       clientGroupID TEXT NOT NULL,
-       clientID TEXT NOT NULL,
-       mutationID INTEGER NOT NULL,
-       result TEXT NOT NULL,
-       PRIMARY KEY (clientGroupID, clientID, mutationID)
-     )`,
-  )
-}
-
-function ledgerKey(tables: SQLiteLedgerTables) {
-  return `${tables.clients}\0${tables.mutations}`
-}
-
-function defaultLedgerTables(upstreamSchema: string): SQLiteLedgerTables {
-  const prefix = upstreamSchema ? `${upstreamSchema}_` : ''
-  return {
-    clients: `${prefix}clients`,
-    mutations: `${prefix}mutations`,
-  }
-}
-
-function encodeSQLiteValue(value: unknown, { column }: SQLiteValueContext) {
-  if (value === null) return null
-  if (column.type === 'boolean') return value ? 1 : 0
-  if (column.type === 'json') return JSON.stringify(value)
-  return value
-}
-
-function quoteSQLiteIdentifier(value: string) {
-  return `"${value.replaceAll('"', '""')}"`
-}
+export * from './sqliteRuntime'

package/src/createZeroServer.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from 'vitest'
+import { describe, expect, test, vi } from 'vitest'
 
 import { createZeroServer } from './createZeroServer'
 import { zeroHttpFixtureSchema } from './httpPull/fixtureSchema'
@@ -103,7 +103,7 @@ describe('createZeroServer mutation recovery', () => {
         project: {
           mutate: {
             touch: async ({ server }) => {
-              server?.asyncTasks.push(async () => {
+              server?.enqueueTask(async () => {
                 effectRuns++
               })
               throw new Error('rejected')
@@ -129,6 +129,90 @@ describe('createZeroServer mutation recovery', () => {
     expect(effectRuns).toBe(0)
   })
 
+  test('awaits barrier tasks before resolving the push response', async () => {
+    const pg = new FakePgClient()
+    let releaseBarrier!: () => void
+    let markBarrierStarted!: () => void
+    const barrierStarted = new Promise<void>((resolve) => {
+      markBarrierStarted = resolve
+    })
+    const barrier = new Promise<void>((resolve) => {
+      releaseBarrier = resolve
+    })
+    const zeroServer = createZeroServer({
+      schema: zeroHttpFixtureSchema,
+      models: {
+        project: {
+          mutate: {
+            touch: async ({ server }) => {
+              server?.enqueueTask(
+                async () => {
+                  markBarrierStarted()
+                  await barrier
+                },
+                { barrier: true },
+              )
+            },
+          },
+        },
+      },
+      createServerActions: () => ({}),
+      pool: pg as any,
+    })
+
+    let responseResolved = false
+    const resultPromise = zeroServer
+      .handleMutationRequest({
+        authData: null,
+        request: pushRequest(1),
+      })
+      .then((result) => {
+        responseResolved = true
+        return result
+      })
+
+    await barrierStarted
+    await Promise.resolve()
+    expect(responseResolved).toBe(false)
+
+    releaseBarrier()
+    const result = await resultPromise
+    expect((result.response as any).mutations[0].result).toEqual({})
+  })
+
+  test('does not fail the push when a background task rejects', async () => {
+    const pg = new FakePgClient()
+    const error = new Error('background failed')
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
+    const zeroServer = createZeroServer({
+      schema: zeroHttpFixtureSchema,
+      models: {
+        project: {
+          mutate: {
+            touch: async ({ server }) => {
+              server?.enqueueTask(async () => {
+                throw error
+              })
+            },
+          },
+        },
+      },
+      createServerActions: () => ({}),
+      pool: pg as any,
+    })
+
+    const result = await zeroServer.handleMutationRequest({
+      authData: null,
+      request: pushRequest(1),
+    })
+
+    expect((result.response as any).mutations[0].result).toEqual({})
+    await vi.waitFor(() => {
+      expect(consoleError).toHaveBeenCalledWith('[on-zero] async task failed', error)
+    })
+    consoleError.mockRestore()
+  })
+
   test('persists alreadyProcessed duplicate results for zero-cache recovery rows', async () => {
     const pg = new FakePgClient()
     const zeroServer = createZeroServer({