@o/database 0.16.18 → 0.16.19-canary.1790191946359

package/src/takeout/sqlite.ts

@@ -0,0 +1,237 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+import { drizzle, type SqliteRemoteDatabase } from 'drizzle-orm/sqlite-proxy'
+
+import type { Database, SQLQueryBindings } from 'bun:sqlite'
+import type { AnyRelations, EmptyRelations } from 'drizzle-orm'
+import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
+
+export type SqlStatementMetadata = {
+  table: string
+  publicTable: string
+  kind: 'delete' | 'insert' | 'update' | 'upsert'
+}
+
+export type SQLiteQueryFormat = {
+  relationships: Record<string, SQLiteQueryFormat>
+  singular: boolean
+}
+
+export type SQLiteExecResult = {
+  changes: number
+}
+
+export type SQLiteStatement = {
+  sql: string
+  params?: readonly unknown[]
+  metadata?: SqlStatementMetadata
+}
+
+export type SQLiteTransactionExecutor = {
+  exec(
+    sql: string,
+    params?: readonly unknown[],
+    metadata?: SqlStatementMetadata
+  ): SQLiteExecResult | Promise<SQLiteExecResult>
+  /**
+   * every statement in one call to the storage owner, in order, as one
+   * atomic step. a remote owner holds its writer across each round trip, so a
+   * list sent one statement at a time holds it for the whole exchange.
+   */
+  execMany(
+    statements: readonly SQLiteStatement[]
+  ): SQLiteExecResult[] | Promise<SQLiteExecResult[]>
+  query<Row extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    params?: readonly unknown[]
+  ): Row[] | Promise<Row[]>
+  queryAst<Result = unknown>(
+    ast: unknown,
+    format: SQLiteQueryFormat,
+    queryName?: string
+  ): Result | Promise<Result>
+}
+
+export type SQLiteTransactionProvider = <Value>(
+  work: (executor: SQLiteTransactionExecutor) => Value | Promise<Value>
+) => Promise<Value>
+
+export type SQLiteSchema = Record<string, SQLiteTable>
+
+export type CreateSQLiteDatabaseOptions<
+  TSchema extends SQLiteSchema,
+  TRelations extends AnyRelations = EmptyRelations,
+> = {
+  schema: TSchema
+  relations?: TRelations
+  transactionProvider: SQLiteTransactionProvider
+}
+
+export type SQLiteDatabaseTransaction<
+  TSchema extends SQLiteSchema,
+  TRelations extends AnyRelations,
+> = {
+  executor: SQLiteTransactionExecutor
+  drizzle: SqliteRemoteDatabase<TSchema, TRelations>
+}
+
+export type SQLiteDatabase<
+  TSchema extends SQLiteSchema,
+  TRelations extends AnyRelations,
+> = {
+  transaction<Value>(
+    work: (
+      transaction: SQLiteDatabaseTransaction<TSchema, TRelations>
+    ) => Value | Promise<Value>
+  ): Promise<Value>
+}
+
+/**
+ * Creates Drizzle only after the storage owner has started its transaction.
+ * The sqlite-proxy driver is used solely to turn its query callbacks into
+ * typed Drizzle operations; it never owns BEGIN or COMMIT.
+ */
+export function createSQLiteDatabase<
+  TSchema extends SQLiteSchema,
+  TRelations extends AnyRelations = EmptyRelations,
+>(
+  options: CreateSQLiteDatabaseOptions<TSchema, TRelations>
+): SQLiteDatabase<TSchema, TRelations> {
+  return {
+    transaction(work) {
+      return options.transactionProvider(async (executor) =>
+        work({
+          executor,
+          drizzle: createSQLiteDrizzle(executor, options),
+        })
+      )
+    },
+  }
+}
+
+export function createSQLiteDrizzle<
+  TSchema extends SQLiteSchema,
+  TRelations extends AnyRelations = EmptyRelations,
+>(
+  executor: SQLiteTransactionExecutor,
+  options: Pick<CreateSQLiteDatabaseOptions<TSchema, TRelations>, 'relations' | 'schema'>
+): SqliteRemoteDatabase<TSchema, TRelations> {
+  return drizzle<TSchema, TRelations>(
+    async (sql, params, method) => {
+      if (method === 'run') {
+        const result = await executor.exec(sql, params)
+        return { changes: result.changes, rows: [] }
+      }
+
+      const rows = await executor.query(sql, params)
+      if (method === 'get') {
+        const row = rows[0]
+        return remoteRows(row ? rowValues(row) : undefined)
+      }
+      return remoteRows(rows.map(rowValues))
+    },
+    {
+      schema: options.schema,
+      relations: options.relations,
+    }
+  )
+}
+
+export type CreateBunSQLiteExecutorOptions = {
+  database: Database
+  queryAst<Result = unknown>(
+    ast: unknown,
+    format: SQLiteQueryFormat,
+    queryName?: string
+  ): Result | Promise<Result>
+}
+
+export function createBunSQLiteExecutor({
+  database,
+  queryAst,
+}: CreateBunSQLiteExecutorOptions): SQLiteTransactionExecutor {
+  return {
+    exec(sql, params = []) {
+      const result = database.prepare(sql).run(...bunBindings(params))
+      return { changes: result.changes }
+    },
+    execMany(statements) {
+      return statements.map(({ sql, params = [] }) => ({
+        changes: database.prepare(sql).run(...bunBindings(params)).changes,
+      }))
+    },
+    query<Row extends Record<string, unknown>>(
+      sql: string,
+      params: readonly unknown[] = []
+    ): Row[] {
+      return database.prepare<Row, SQLQueryBindings[]>(sql).all(...bunBindings(params))
+    },
+    queryAst,
+  }
+}
+
+export type CreateBunSQLiteTransactionProviderOptions = CreateBunSQLiteExecutorOptions
+
+/**
+ * Bun's synchronous driver needs its own native transaction boundary. Remote
+ * and Durable Object hosts pass their protected callback directly to
+ * createSQLiteDatabase instead.
+ */
+export function createBunSQLiteTransactionProvider({
+  database,
+  queryAst,
+}: CreateBunSQLiteTransactionProviderOptions): SQLiteTransactionProvider {
+  const executor = createBunSQLiteExecutor({ database, queryAst })
+  const transactionContext = new AsyncLocalStorage<boolean>()
+  let tail = Promise.resolve()
+
+  return async (work) => {
+    if (transactionContext.getStore()) {
+      throw new Error('nested SQLite transactions are not supported')
+    }
+    // keep the native connection's transaction state exclusive across awaits.
+    const previous = tail
+    let release!: () => void
+    tail = new Promise<void>((resolve) => {
+      release = resolve
+    })
+
+    await previous
+    try {
+      return await transactionContext.run(true, async () => {
+        let began = false
+        try {
+          database.exec('BEGIN')
+          began = true
+          const value = await work(executor)
+          database.exec('COMMIT')
+          began = false
+          return value
+        } catch (error) {
+          if (began) database.exec('ROLLBACK')
+          throw error
+        }
+      })
+    } finally {
+      release()
+    }
+  }
+}
+
+function rowValues(row: Record<string, unknown>): unknown[] {
+  return Object.values(row)
+}
+
+function remoteRows(rows: unknown[] | undefined): { rows: any[] } {
+  // Drizzle's sqlite-proxy declaration requires an array even though its get()
+  // implementation recognizes an absent row. Keep that transport mismatch at
+  // this one adapter boundary rather than turning a missing row into one full
+  // of undefined column values.
+  return { rows: rows as any[] }
+}
+
+function bunBindings(params: readonly unknown[]): SQLQueryBindings[] {
+  // on-zero accepts unknown bindings so every native provider can expose one
+  // structural contract. Bun validates these at its typed boundary.
+  return params as SQLQueryBindings[]
+}

package/src/takeout/waitForDatabase.ts

@@ -0,0 +1,27 @@
+import pg from 'pg'
+
+export async function waitForDatabase(connectionString: string, maxRetries = 30) {
+  for (let i = 0; i < maxRetries; i++) {
+    try {
+      const client = new pg.Client({
+        connectionString,
+        connectionTimeoutMillis: 5_000,
+        ssl: connectionString.includes('sslmode=require')
+          ? { rejectUnauthorized: false }
+          : undefined,
+      })
+      await client.connect()
+      await client.query('SELECT 1')
+      await client.end()
+      console.info('[database] connection successful')
+      return
+    } catch {
+      const delay = Math.min(1000 * 1.5 ** i, 10000)
+      console.info(
+        `[database] waiting... attempt ${i + 1}/${maxRetries} (retry in ${delay}ms)`
+      )
+      await new Promise((resolve) => setTimeout(resolve, delay))
+    }
+  }
+  throw new Error('database connection timeout after ' + maxRetries + ' attempts')
+}

package/src/takeout/zero.ts

@@ -0,0 +1,281 @@
+import { basename } from 'node:path'
+
+import { getTableName } from 'drizzle-orm'
+import { PgTable } from 'drizzle-orm/pg-core'
+
+import { getDBClient } from './getDBClient.js'
+
+import type { PoolClient } from 'pg'
+
+// --- zero publication management ---
+
+export type ZeroPublicationOptions = {
+  connectionString: string
+  /** zero publication name, e.g. 'zero_takeout' or 'zero_chat' */
+  publicationName: string
+  /** table names excluded from publication (private/auth tables) */
+  privateTableNames: string[]
+}
+
+export async function ensureZeroPublication(options: ZeroPublicationOptions) {
+  const { connectionString, publicationName, privateTableNames } = options
+  const client = await getDBClient({ connectionString })
+
+  try {
+    const { rows: wanted } = await client.query(
+      `SELECT tablename FROM pg_tables
+       WHERE schemaname = 'public'
+         AND tablename != ALL($1)
+         AND tablename NOT LIKE '_zero_%'
+         AND tablename != 'migrations'`,
+      [privateTableNames]
+    )
+    const wantedSet = new Set(wanted.map((r: any) => r.tablename))
+
+    if (!wantedSet.size) {
+      console.info(`[zero] no public tables found for ${publicationName} publication`)
+      return
+    }
+
+    const { rows: pub } = await client.query(
+      `SELECT 1 FROM pg_publication WHERE pubname = $1`,
+      [publicationName]
+    )
+
+    if (!pub.length) {
+      try {
+        const tableList = [...wantedSet].map((t) => `"${t}"`).join(', ')
+        await client.query(`CREATE PUBLICATION ${publicationName} FOR TABLE ${tableList}`)
+        console.info(
+          `[zero] created publication ${publicationName} with ${wantedSet.size} tables`
+        )
+      } catch (e: any) {
+        if (e.code === '42710') {
+          console.info(
+            `[zero] ${publicationName} publication was created concurrently, will sync`
+          )
+        } else {
+          console.warn(
+            `[zero] could not create ${publicationName} publication: ${e.message}`
+          )
+          return
+        }
+      }
+    }
+
+    const { rows: current } = await client.query(
+      `SELECT tablename FROM pg_publication_tables
+       WHERE pubname = $1 AND schemaname = 'public'`,
+      [publicationName]
+    )
+    const currentSet = new Set(current.map((r: any) => r.tablename))
+
+    const toRemove = [...currentSet].filter((t) => !wantedSet.has(t))
+    if (toRemove.length) {
+      const dropList = toRemove.map((t) => `"${t}"`).join(', ')
+      await client.query(`ALTER PUBLICATION ${publicationName} DROP TABLE ${dropList}`)
+      console.info(`[zero] removed from ${publicationName}: ${dropList}`)
+    }
+
+    const toAdd = [...wantedSet].filter((t) => !currentSet.has(t))
+    const added: string[] = []
+    for (const table of toAdd) {
+      try {
+        await client.query(`ALTER PUBLICATION ${publicationName} ADD TABLE "${table}"`)
+        added.push(table)
+      } catch (e: any) {
+        if (e.code !== '42710') throw e
+      }
+    }
+
+    if (added.length) {
+      console.info(
+        `[zero] added to ${publicationName}: ${added.map((t) => `"${t}"`).join(', ')}`
+      )
+    }
+
+    if (!toRemove.length && !added.length) {
+      console.info(
+        `[zero] ${publicationName} publication is up to date (${currentSet.size} tables)`
+      )
+    }
+  } finally {
+    client.release()
+  }
+}
+
+// --- zero database creation ---
+
+export type ZeroDatabaseOptions = {
+  connectionString: string
+  cvrDb?: string
+  changeDb?: string
+}
+
+export async function ensureZeroDatabases(options: ZeroDatabaseOptions) {
+  const { connectionString, cvrDb, changeDb } = options
+  if (!cvrDb && !changeDb) return
+
+  const client = await getDBClient({ connectionString })
+
+  try {
+    const dbNames = [basename(cvrDb || ''), basename(changeDb || '')].filter(Boolean)
+
+    for (const name of dbNames) {
+      const { rows } = await client.query(
+        `SELECT 1 FROM pg_database WHERE datname = '${name}'`
+      )
+      if (!rows.length) {
+        await client.query(`CREATE DATABASE ${name};`)
+        console.info(`[zero] created database ${name}`)
+      }
+    }
+  } finally {
+    client.release()
+  }
+}
+
+// --- zero triggers ---
+
+export type TriggerDefinition = {
+  name: string
+  sql: string
+}
+
+export function defineTrigger(name: string, sql: string): TriggerDefinition {
+  return { name, sql }
+}
+
+/** load trigger definitions from a glob result (import.meta.glob or loadTriggersFromDir) */
+export async function loadTriggers(
+  glob: Record<string, () => Promise<unknown>>
+): Promise<TriggerDefinition[]> {
+  const triggers: TriggerDefinition[] = []
+  for (const [path, load] of Object.entries(glob).sort(([a], [b]) =>
+    a.localeCompare(b)
+  )) {
+    const mod = (await load()) as {
+      default?: TriggerDefinition
+      name?: string
+      sql?: string
+    }
+    if (mod.default) {
+      triggers.push(mod.default)
+    } else if (mod.sql) {
+      triggers.push({ name: mod.name || basename(path, '.ts'), sql: mod.sql })
+    }
+  }
+  return triggers
+}
+
+export async function ensureZeroTriggers(
+  client: PoolClient,
+  triggers: TriggerDefinition[]
+) {
+  for (const trigger of triggers) {
+    try {
+      await client.query(trigger.sql)
+    } catch (err) {
+      console.error(`[zero] failed to apply trigger ${trigger.name}:`, err)
+      throw err
+    }
+  }
+  console.info(`[zero] applied ${triggers.length} trigger definitions`)
+}
+
+// --- zero seeds ---
+
+export type SeedDefinition = {
+  name: string
+  sql: string
+}
+
+export function defineSeed(name: string, sql: string): SeedDefinition {
+  return { name, sql }
+}
+
+export async function loadSeeds(
+  glob: Record<string, () => Promise<unknown>>
+): Promise<SeedDefinition[]> {
+  const seeds: SeedDefinition[] = []
+  for (const [path, load] of Object.entries(glob).sort(([a], [b]) =>
+    a.localeCompare(b)
+  )) {
+    const mod = (await load()) as {
+      default?: SeedDefinition
+      name?: string
+      sql?: string
+    }
+    if (mod.default) {
+      seeds.push(mod.default)
+    } else if (mod.sql) {
+      seeds.push({ name: mod.name || basename(path, '.ts'), sql: mod.sql })
+    }
+  }
+  return seeds
+}
+
+export async function ensureZeroSeeds(client: PoolClient, seeds: SeedDefinition[]) {
+  for (const seed of seeds) {
+    try {
+      await client.query(seed.sql)
+    } catch (err) {
+      console.error(`[zero] failed to apply seed ${seed.name}:`, err)
+      throw err
+    }
+  }
+  console.info(`[zero] applied ${seeds.length} seed definitions`)
+}
+
+// --- init sql for pglite/browser ---
+// todo: drizzleSchemaToSQL(schema) → string for browser pglite init (currently ddl is passed as string)
+
+export type InitSqlOptions = {
+  /** DDL statements (CREATE TABLE, etc.) */
+  ddl?: string
+  triggers?: TriggerDefinition[]
+  seeds?: SeedDefinition[]
+}
+
+export function buildInitSql(options: InitSqlOptions): string {
+  const parts: string[] = []
+
+  if (options.ddl) {
+    parts.push(options.ddl)
+  }
+
+  if (options.triggers?.length) {
+    parts.push(...options.triggers.map((t) => `-- trigger: ${t.name}\n${t.sql}`))
+  }
+
+  if (options.seeds?.length) {
+    parts.push(...options.seeds.map((s) => `-- seed: ${s.name}\n${s.sql}`))
+  }
+
+  return parts.join('\n\n')
+}
+
+// --- schema helpers ---
+
+const PRIVATE = Symbol.for('take-out/database/private')
+
+/** wrap pgTable to mark tables as private (excluded from zero replication) */
+export function privateTable<T extends (...args: any[]) => any>(createTable: T): T {
+  return ((...args: any[]) => {
+    const table = createTable(...args)
+    ;(table as any)[PRIVATE] = true
+    return table
+  }) as unknown as T
+}
+
+/** check if a table is marked as private */
+export function isPrivateTable(table: unknown): boolean {
+  return !!(table && typeof table === 'object' && PRIVATE in table)
+}
+
+/** derive public table names from a schema (all tables not marked private) */
+export function derivePublicTables(schema: Record<string, unknown>): string[] {
+  return Object.values(schema)
+    .filter((v) => v instanceof PgTable && !isPrivateTable(v))
+    .map((t) => getTableName(t as PgTable))
+}