@o/database 0.16.18 → 0.16.19-canary.1790191946359

package/src/takeout/scripts/drizzle-migrations-sync.ts

@@ -0,0 +1,81 @@
+#!/usr/bin/env bun
+
+/**
+ * drizzle-kit v1 beta generates migrations as directories:
+ *   YYYYMMDDHHMMSS_name/migration.sql + snapshot.json
+ *
+ * our migration runner uses loadMigrationsFromDir which only
+ * finds flat .ts files. this script creates a flat .ts wrapper for each
+ * directory migration so the runner can pick them up.
+ *
+ * the wrapper inlines the SQL as a string constant (no ?raw import needed)
+ * which makes it work with both vite and bun.
+ */
+
+import { existsSync } from 'node:fs'
+import { readdir, readFile, writeFile, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+
+export type DrizzleSyncOptions = {
+  migrationsDir: string
+}
+
+export async function syncDrizzleMigrations(options: DrizzleSyncOptions) {
+  const { migrationsDir } = options
+  const entries = await readdir(migrationsDir, { withFileTypes: true })
+
+  // find directory-format migrations (contain migration.sql)
+  const migrationDirs = entries.filter(
+    (e) => e.isDirectory() && e.name !== 'meta' && e.name !== 'node_modules'
+  )
+
+  let created = 0
+  let updated = 0
+
+  for (const dir of migrationDirs) {
+    const sqlPath = join(migrationsDir, dir.name, 'migration.sql')
+    if (!existsSync(sqlPath)) continue
+
+    const sql = await readFile(sqlPath, 'utf-8')
+    const trimmed = sql.trim()
+
+    if (!trimmed || trimmed.startsWith('-- seed')) {
+      continue
+    }
+
+    const wrapperPath = join(migrationsDir, `${dir.name}.ts`)
+
+    // check if wrapper already exists and is up to date
+    if (existsSync(wrapperPath)) {
+      const sqlMtime = (await stat(sqlPath)).mtimeMs
+      const tsMtime = (await stat(wrapperPath)).mtimeMs
+      if (tsMtime > sqlMtime) continue
+      updated++
+    } else {
+      created++
+    }
+
+    // generate wrapper with inlined SQL
+    const escaped = trimmed
+      .replace(/\\/g, '\\\\')
+      .replace(/`/g, '\\`')
+      .replace(/\$/g, '\\$')
+    const content = `import type { PoolClient } from 'pg'
+
+const sql = \`${escaped}\`
+
+export async function up(client: PoolClient) {
+  for (const stmt of sql.split('--> statement-breakpoint')) {
+    const trimmed = stmt.trim()
+    if (trimmed) await client.query(trimmed)
+  }
+}
+`
+
+    await writeFile(wrapperPath, content)
+  }
+
+  if (created > 0) console.info(`created ${created} migration wrapper(s)`)
+  if (updated > 0) console.info(`updated ${updated} migration wrapper(s)`)
+  if (created === 0 && updated === 0) console.info('migration wrappers up to date')
+}

package/src/takeout/scripts/dump-database.ts

@@ -0,0 +1,103 @@
+import { spawn } from 'node:child_process'
+
+export type DumpDatabaseOptions = {
+  host: string
+  database: string
+  username: string
+  password?: string
+  /** output file path */
+  outputPath: string
+  /** timeout in ms (default: 5 minutes) */
+  timeout?: number
+  /** pg_dump format (default: 'custom') */
+  format?: 'custom' | 'plain' | 'directory' | 'tar'
+  /** compression level 0-9 (default: 9) */
+  compress?: number
+}
+
+export async function dumpDatabase(options: DumpDatabaseOptions): Promise<boolean> {
+  const {
+    host,
+    database,
+    username,
+    password,
+    outputPath,
+    timeout = 5 * 60 * 1000,
+    format = 'custom',
+    compress = 9,
+  } = options
+
+  return new Promise<boolean>((resolve) => {
+    const proc = spawn(
+      'pg_dump',
+      [
+        '--clean',
+        '--if-exists',
+        '--no-owner',
+        '--no-privileges',
+        '--verbose',
+        `--format=${format}`,
+        `--compress=${compress}`,
+        '--file',
+        outputPath,
+      ],
+      {
+        env: {
+          ...process.env,
+          PGPASSWORD: password,
+          PGUSER: username,
+          PGHOST: host,
+          PGDATABASE: database,
+          PGCONNECT_TIMEOUT: '10',
+        } as any,
+        stdio: 'inherit',
+      }
+    )
+
+    let timeoutId: ReturnType<typeof setTimeout>
+
+    proc.on('error', (err) => {
+      console.error('[database] pg_dump process error:', err)
+      clearTimeout(timeoutId)
+      resolve(false)
+    })
+
+    proc.on('exit', (code) => {
+      clearTimeout(timeoutId)
+      if (code === 0) {
+        resolve(true)
+      } else {
+        console.error(`[database] pg_dump exited with code ${code}`)
+        resolve(false)
+      }
+    })
+
+    timeoutId = setTimeout(() => {
+      console.error(`[database] pg_dump timed out after ${timeout}ms`)
+      proc.kill('SIGTERM')
+      setTimeout(() => {
+        if (!proc.killed) proc.kill('SIGKILL')
+      }, 5000)
+      resolve(false)
+    }, timeout)
+  })
+}
+
+export async function checkDatabaseConnection(options: {
+  host: string
+  database: string
+  username: string
+  timeout?: number
+}): Promise<boolean> {
+  const { host, database, username, timeout = 5000 } = options
+
+  return new Promise<boolean>((resolve) => {
+    const proc = spawn('pg_isready', ['-d', database, '-h', host, '-U', username], {
+      stdio: 'inherit',
+      timeout,
+    })
+
+    proc.on('error', () => resolve(false))
+    proc.on('exit', (code) => resolve(code === 0))
+  })
+}

package/src/takeout/scripts/migration-add.ts

@@ -0,0 +1,54 @@
+import { readdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export type MigrationAddOptions = {
+  migrationsDir: string
+  name?: string
+}
+
+const commonWords = ['sapphire', 'emerald', 'ruby', 'amber', 'topaz', 'onyx', 'pearl']
+
+export function addMigration(options: MigrationAddOptions): string {
+  const { migrationsDir, name } = options
+
+  // if no name is provided, pick a random one from common words
+  const migrationName =
+    name || commonWords[Math.floor(Math.random() * commonWords.length)]!
+
+  // read all files in the migrations directory
+  const files = readdirSync(migrationsDir)
+
+  // find the highest migration number from both .ts and .sql files
+  const migrationRegex = /^(\d+)[-_].*\.(ts|sql)$/
+  let maxNumber = 0
+  for (const file of files) {
+    const match = file.match(migrationRegex)
+    if (match?.[1]) {
+      const num = Number.parseInt(match[1], 10)
+      if (!Number.isNaN(num) && num > maxNumber) {
+        maxNumber = num
+      }
+    }
+  }
+
+  // calculate the next migration number and pad to 4 digits
+  const nextNumber = (maxNumber + 1).toString().padStart(4, '0')
+  const newFilename = `${nextNumber}-${migrationName}.ts`
+  const newFilePath = join(migrationsDir, newFilename)
+
+  // create a template for custom TypeScript migrations
+  const templateContent = `import type { PoolClient } from 'pg'
+
+export async function up(client: PoolClient) {
+  // implementation for applying this migration
+}
+`
+
+  // write the template to the new file
+  writeFileSync(newFilePath, templateContent)
+
+  console.info(`Created custom migration: ${newFilePath}`)
+  console.info(`For Drizzle schema migrations, run 'drizzle-kit generate' instead`)
+
+  return newFilePath
+}

package/src/takeout/scripts/pg_dump.ts

@@ -0,0 +1,46 @@
+import { spawnSync } from 'node:child_process'
+
+export type PgDumpOptions = {
+  connectionString?: string
+  host?: string
+  port?: number
+  database?: string
+  user?: string
+  password?: string
+  args?: string[]
+}
+
+export function runPgDump(options: PgDumpOptions = {}) {
+  const { connectionString, host, port, database, user, password, args = [] } = options
+
+  const env: Record<string, string> = {
+    ...process.env,
+  } as any
+
+  if (connectionString) {
+    // parse connection string and set individual env vars
+    const url = new URL(connectionString)
+    env.PGHOST = url.hostname
+    env.PGPORT = url.port || '5432'
+    env.PGDATABASE = url.pathname.slice(1)
+    env.PGUSER = url.username
+    if (url.password) {
+      env.PGPASSWORD = url.password
+    }
+  } else {
+    if (host) env.PGHOST = host
+    if (port) env.PGPORT = port.toString()
+    if (database) env.PGDATABASE = database
+    if (user) env.PGUSER = user
+    if (password) env.PGPASSWORD = password
+  }
+
+  console.info(`Running pg_dump on postgres ${env.PGHOST}/${env.PGDATABASE}`)
+
+  const result = spawnSync('pg_dump', args, {
+    stdio: ['ignore', 'inherit', 'inherit'],
+    env,
+  })
+
+  return result.status
+}

package/src/takeout/scripts/prune-backups.ts

@@ -0,0 +1,211 @@
+import { DeleteObjectsCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'
+
+interface BackupInfo {
+  key: string
+  date: Date
+  size: number
+}
+
+export async function pruneBackups(
+  bucketName: string,
+  prefix: string,
+  options?: {
+    region?: string
+    dryRun?: boolean
+  }
+): Promise<{ kept: string[]; deleted: string[] }> {
+  const region = options?.region ?? 'us-west-1'
+  const dryRun = options?.dryRun ?? false
+
+  const s3Client = new S3Client({
+    region,
+    credentials: {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    },
+  })
+
+  console.info('🧹 Pruning old backups...')
+
+  // list all backups
+  const allBackups: BackupInfo[] = []
+  let continuationToken: string | undefined
+
+  do {
+    const response = await s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: bucketName,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      })
+    )
+
+    if (response.Contents) {
+      for (const obj of response.Contents) {
+        if (obj.Key && obj.LastModified && obj.Size) {
+          // extract date from filename (format: db-backup-2025-08-21T15-50-48-103Z.sql)
+          const match = obj.Key.match(/(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z)/)
+          if (match) {
+            const dateStr = match[1]!.replace(/-(\d{2})-(\d{2})-(\d{3}Z)/, ':$1:$2.$3')
+            allBackups.push({
+              key: obj.Key,
+              date: new Date(dateStr),
+              size: obj.Size,
+            })
+          }
+        }
+      }
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  if (allBackups.length === 0) {
+    console.info('No backups found to prune')
+    return { kept: [], deleted: [] }
+  }
+
+  // sort by date (newest first)
+  allBackups.sort((a, b) => b.date.getTime() - a.date.getTime())
+
+  const now = new Date()
+  const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000)
+
+  const toKeep = new Set<string>()
+  const toDelete: string[] = []
+
+  // group backups by day and month
+  const dailyBackups = new Map<string, BackupInfo[]>()
+  const monthlyBackups = new Map<string, BackupInfo[]>()
+
+  for (const backup of allBackups) {
+    const dayKey = backup.date.toISOString().split('T')[0]! // YYYY-MM-DD
+    const monthKey = dayKey.substring(0, 7) // YYYY-MM
+
+    if (!dailyBackups.has(dayKey)) {
+      dailyBackups.set(dayKey, [])
+    }
+    const dayBackupsList = dailyBackups.get(dayKey)
+    if (dayBackupsList) {
+      dayBackupsList.push(backup)
+    }
+
+    if (!monthlyBackups.has(monthKey)) {
+      monthlyBackups.set(monthKey, [])
+    }
+    const monthBackupsList = monthlyBackups.get(monthKey)
+    if (monthBackupsList) {
+      monthBackupsList.push(backup)
+    }
+  }
+
+  // keep daily backups for last 30 days
+  for (const [dayKey, backups] of dailyBackups) {
+    const dayDate = new Date(dayKey)
+    if (dayDate >= thirtyDaysAgo) {
+      // keep the most recent backup for each day
+      const latestBackup = backups[0]
+      if (latestBackup) {
+        toKeep.add(latestBackup.key)
+      }
+    }
+  }
+
+  // keep monthly backups for older than 30 days
+  for (const [monthKey, backups] of monthlyBackups) {
+    const monthDate = new Date(monthKey + '-01')
+    if (monthDate < thirtyDaysAgo) {
+      // keep the most recent backup for each month
+      const latestBackup = backups[0]
+      if (latestBackup) {
+        toKeep.add(latestBackup.key)
+      }
+    }
+  }
+
+  // always keep the very latest backup regardless of rules
+  if (allBackups.length > 0) {
+    const latestBackup = allBackups[0]
+    if (latestBackup) {
+      toKeep.add(latestBackup.key)
+    }
+  }
+
+  // determine which backups to delete
+  for (const backup of allBackups) {
+    if (!toKeep.has(backup.key)) {
+      toDelete.push(backup.key)
+    }
+  }
+
+  console.info(`📊 Backup retention summary:`)
+  console.info(`   Total backups: ${allBackups.length}`)
+  console.info(`   Keeping: ${toKeep.size}`)
+  console.info(`   Deleting: ${toDelete.length}`)
+
+  if (toDelete.length > 0) {
+    if (dryRun) {
+      console.info('\n🔍 DRY RUN - Would delete:')
+      for (const key of toDelete.slice(0, 10)) {
+        console.info(`   - ${key}`)
+      }
+      if (toDelete.length > 10) {
+        console.info(`   ... and ${toDelete.length - 10} more`)
+      }
+    } else {
+      console.info('\n🗑️  Deleting old backups...')
+
+      // delete in batches of 1000 (S3 limit)
+      for (let i = 0; i < toDelete.length; i += 1000) {
+        const batch = toDelete.slice(i, i + 1000)
+
+        await s3Client.send(
+          new DeleteObjectsCommand({
+            Bucket: bucketName,
+            Delete: {
+              Objects: batch.map((key) => ({ Key: key })),
+              Quiet: true,
+            },
+          })
+        )
+
+        console.info(
+          `   Deleted batch ${Math.floor(i / 1000) + 1}/${Math.ceil(toDelete.length / 1000)}`
+        )
+      }
+
+      console.info(`✅ Successfully deleted ${toDelete.length} old backup(s)`)
+    }
+  } else {
+    console.info('✅ No backups need to be deleted')
+  }
+
+  // log retention details
+  console.info('\n📅 Retention details:')
+  const keptList = Array.from(toKeep).sort()
+  const recentBackups = keptList.filter((key) => {
+    const match = key.match(/(\d{4}-\d{2}-\d{2})/)
+    if (match && match[1]) {
+      const date = new Date(match[1])
+      return date >= thirtyDaysAgo
+    }
+    return false
+  })
+
+  const oldBackups = keptList.filter((key) => {
+    const match = key.match(/(\d{4}-\d{2}-\d{2})/)
+    if (match && match[1]) {
+      const date = new Date(match[1])
+      return date < thirtyDaysAgo
+    }
+    return false
+  })
+
+  console.info(`   Daily backups (last 30 days): ${recentBackups.length}`)
+  console.info(`   Monthly backups (older): ${oldBackups.length}`)
+
+  return {
+    kept: Array.from(toKeep),
+    deleted: toDelete,
+  }
+}

package/src/takeout/scripts/psql.ts

@@ -0,0 +1,51 @@
+import { spawnSync } from 'node:child_process'
+
+export type PsqlOptions = {
+  connectionString?: string
+  host?: string
+  port?: number
+  database?: string
+  user?: string
+  password?: string
+  query?: string
+}
+
+export function runPsql(options: PsqlOptions = {}) {
+  const { connectionString, host, port, database, user, password, query } = options
+
+  const env: Record<string, string> = {
+    ...process.env,
+  } as any
+
+  if (connectionString) {
+    // parse connection string and set individual env vars
+    const url = new URL(connectionString)
+    env.PGHOST = url.hostname
+    env.PGPORT = url.port || '5432'
+    env.PGDATABASE = url.pathname.slice(1)
+    env.PGUSER = url.username
+    if (url.password) {
+      env.PGPASSWORD = url.password
+    }
+  } else {
+    if (host) env.PGHOST = host
+    if (port) env.PGPORT = port.toString()
+    if (database) env.PGDATABASE = database
+    if (user) env.PGUSER = user
+    if (password) env.PGPASSWORD = password
+  }
+
+  const args: string[] = []
+  if (query) {
+    args.push('-c', query)
+  }
+
+  console.info(`Connecting to postgres ${env.PGHOST}/${env.PGDATABASE}`)
+
+  const result = spawnSync('psql', args, {
+    stdio: ['ignore', 'inherit', 'inherit'],
+    env,
+  })
+
+  return result.status
+}

package/src/takeout/seed.ts

@@ -0,0 +1,45 @@
+// standardized demo/world seed applier — the one mechanism every runtime uses
+// to apply a project's database/seed.ts rows after migrations: pg via the
+// --seed flag on database/migrate.ts, and the browser preview worker (PGlite)
+// at cold boot. same rows, same insert semantics, only the client differs.
+
+import type { InferInsertModel, Table } from 'drizzle-orm'
+
+// plain rows keyed by table name, in insert order (FK parents before children)
+export type SeedRows = Record<string, Array<Record<string, unknown>>>
+
+// typed view of SeedRows against a drizzle schema module: table exports map to
+// their insert models so seed rows typecheck against real columns, while
+// `import type` keeps the seed file drizzle-free at runtime
+export type SeedData<TSchema> = {
+  [K in keyof TSchema as TSchema[K] extends Table ? K : never]?: Array<
+    InferInsertModel<Extract<TSchema[K], Table>>
+  >
+}
+
+// the minimal client shape every runtime already has: node-pg PoolClient,
+// PGlite, and the preview worker pool all expose query(sql, params)
+export interface SeedClient {
+  query(sql: string, params?: unknown[]): Promise<unknown>
+}
+
+const quote = (id: string) => `"${id.replaceAll('"', '""')}"`
+
+// INSERT ... ON CONFLICT DO NOTHING per row, tables in key order. idempotent
+// by construction: deterministic seed ids make re-application a no-op and a
+// grown seed set back-fills only the missing rows. undefined columns are
+// omitted so they fall to the DB default.
+export async function applySeed(client: SeedClient, rows: SeedRows): Promise<void> {
+  for (const [table, tableRows] of Object.entries(rows)) {
+    for (const row of tableRows) {
+      const columns = Object.keys(row).filter((key) => row[key] !== undefined)
+      if (columns.length === 0) continue
+      const values = columns.map((key) => row[key])
+      const placeholders = columns.map((_, i) => `$${i + 1}`).join(', ')
+      await client.query(
+        `INSERT INTO ${quote(table)} (${columns.map(quote).join(', ')}) VALUES (${placeholders}) ON CONFLICT DO NOTHING`,
+        values
+      )
+    }
+  }
+}

package/src/takeout/sql.ts

@@ -0,0 +1,39 @@
+import { ellipsis } from '@o/helpers'
+
+import type { Pool, QueryResult } from 'pg'
+
+export type SqlQuery = {
+  text: string
+  values: any[]
+}
+
+export const createSql = (pool: Pool) => {
+  return (strings: TemplateStringsArray, ...values: any[]): Promise<QueryResult> => {
+    const text = strings.reduce((result, str, i) => {
+      return result + str + (i < values.length ? `$${i + 1}` : '')
+    }, '')
+
+    if (process.env.DEBUG) {
+      console.info(`sql: ${ellipsis(text, 80)}`)
+    }
+
+    return pool.query(text.trim(), values)
+  }
+}
+
+// default export for backward compatibility
+let defaultPool: Pool | null = null
+
+export const setDefaultPool = (pool: Pool) => {
+  defaultPool = pool
+}
+
+export const sql = (
+  strings: TemplateStringsArray,
+  ...values: any[]
+): Promise<QueryResult> => {
+  if (!defaultPool) {
+    throw new Error('No default pool set. Call setDefaultPool() first or use createSql()')
+  }
+  return createSql(defaultPool)(strings, ...values)
+}