orez 0.2.27 → 0.2.29

package/src/restore-stress.test.ts

@@ -1,188 +0,0 @@
-import { writeFileSync, unlinkSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-
-import { PGlite } from '@electric-sql/pglite'
-import { loadModule } from 'pgsql-parser'
-import postgres from 'postgres'
-import { describe, it, expect, beforeAll, afterEach } from 'vitest'
-
-import { execDumpFile } from './cli.js'
-import { getConfig } from './config.js'
-import { startPgProxy } from './pg-proxy.js'
-
-import type { Server } from 'node:net'
-import type { AddressInfo } from 'node:net'
-
-// generate a pg_dump-style SQL file with COPY data
-function generateDump(opts: {
-  tables: number
-  rowsPerTable: number
-  columnsPerTable: number
-}): string {
-  const lines: string[] = []
-  lines.push('-- pg_dump style output')
-  lines.push('SET statement_timeout = 0;')
-  lines.push("SET client_encoding = 'UTF8';")
-  lines.push('')
-
-  for (let t = 0; t < opts.tables; t++) {
-    const tableName = `test_table_${t}`
-    const cols = Array.from({ length: opts.columnsPerTable }, (_, i) => `col_${i} TEXT`)
-    lines.push(`CREATE TABLE ${tableName} (id SERIAL PRIMARY KEY, ${cols.join(', ')});`)
-    lines.push('')
-
-    // COPY block
-    const colNames = Array.from({ length: opts.columnsPerTable }, (_, i) => `col_${i}`)
-    lines.push(`COPY ${tableName} (id, ${colNames.join(', ')}) FROM stdin;`)
-
-    for (let r = 0; r < opts.rowsPerTable; r++) {
-      const vals = Array.from({ length: opts.columnsPerTable }, (_, i) => {
-        if (r % 17 === 0 && i === 0) return '\\N'
-        if (r % 13 === 0 && i === 1) return `value with tab\\there`
-        if (r % 11 === 0 && i === 2) return `O'Brien's "quoted" value\\nwith newline`
-        return `row_${r}_col_${i}_${'x'.repeat(20)}`
-      })
-      lines.push(`${r + 1}\t${vals.join('\t')}`)
-    }
-    lines.push('\\.')
-    lines.push('')
-  }
-
-  return lines.join('\n')
-}
-
-describe('restore stress', () => {
-  let tmpFile: string
-
-  beforeAll(async () => {
-    await loadModule()
-  })
-
-  afterEach(() => {
-    try {
-      unlinkSync(tmpFile)
-    } catch {}
-  })
-
-  it('direct: 5 tables x 500 rows', async () => {
-    const dump = generateDump({ tables: 5, rowsPerTable: 500, columnsPerTable: 5 })
-    tmpFile = join(tmpdir(), `orez-stress-${Date.now()}.sql`)
-    writeFileSync(tmpFile, dump)
-
-    const db = new PGlite({ relaxedDurability: true })
-    await db.waitReady
-
-    const { executed, skipped } = await execDumpFile(db, tmpFile)
-
-    for (let t = 0; t < 5; t++) {
-      const result = await db.query(`SELECT count(*) as cnt FROM test_table_${t}`)
-      expect(Number(result.rows[0].cnt)).toBe(500)
-    }
-
-    console.log(`direct: ${executed} executed, ${skipped} skipped`)
-    await db.close()
-  }, 60_000)
-
-  it('direct: 2 tables x 5000 rows (memory pressure)', async () => {
-    const dump = generateDump({ tables: 2, rowsPerTable: 5000, columnsPerTable: 5 })
-    tmpFile = join(tmpdir(), `orez-stress-large-${Date.now()}.sql`)
-    writeFileSync(tmpFile, dump)
-
-    const db = new PGlite({ relaxedDurability: true })
-    await db.waitReady
-
-    const { executed, skipped } = await execDumpFile(db, tmpFile)
-
-    for (let t = 0; t < 2; t++) {
-      const result = await db.query(`SELECT count(*) as cnt FROM test_table_${t}`)
-      expect(Number(result.rows[0].cnt)).toBe(5000)
-    }
-
-    console.log(`large direct: ${executed} executed, ${skipped} skipped`)
-    await db.close()
-  }, 120_000)
-
-  it('wire: 3 tables x 1000 rows through pg proxy', async () => {
-    const dump = generateDump({ tables: 3, rowsPerTable: 1000, columnsPerTable: 5 })
-    tmpFile = join(tmpdir(), `orez-stress-wire-${Date.now()}.sql`)
-    writeFileSync(tmpFile, dump)
-
-    const db = new PGlite({ relaxedDurability: true })
-    await db.waitReady
-
-    const config = { ...getConfig(), pgPort: 0 }
-    const server: Server = await startPgProxy(db, config)
-    const port = (server.address() as AddressInfo).port
-
-    try {
-      const sql = postgres({
-        host: '127.0.0.1',
-        port,
-        user: 'user',
-        password: 'password',
-        database: 'postgres',
-        max: 1,
-      })
-
-      const wireDb = { exec: (query: string) => sql.unsafe(query) as Promise<unknown> }
-      const { executed, skipped } = await execDumpFile(wireDb, tmpFile)
-
-      for (let t = 0; t < 3; t++) {
-        const result =
-          await sql`SELECT count(*) as cnt FROM test_table_${sql.unsafe(String(t))}`
-        // use direct db to verify
-      }
-
-      // verify via direct PGlite
-      for (let t = 0; t < 3; t++) {
-        const result = await db.query(`SELECT count(*) as cnt FROM test_table_${t}`)
-        expect(Number(result.rows[0].cnt)).toBe(1000)
-      }
-
-      console.log(`wire: ${executed} executed, ${skipped} skipped`)
-      await sql.end()
-    } finally {
-      server.close()
-      await db.close()
-    }
-  }, 120_000)
-
-  it('wire: 2 tables x 3000 rows (memory pressure via proxy)', async () => {
-    const dump = generateDump({ tables: 2, rowsPerTable: 3000, columnsPerTable: 8 })
-    tmpFile = join(tmpdir(), `orez-stress-wire-large-${Date.now()}.sql`)
-    writeFileSync(tmpFile, dump)
-
-    const db = new PGlite({ relaxedDurability: true })
-    await db.waitReady
-
-    const config = { ...getConfig(), pgPort: 0 }
-    const server: Server = await startPgProxy(db, config)
-    const port = (server.address() as AddressInfo).port
-
-    try {
-      const sql = postgres({
-        host: '127.0.0.1',
-        port,
-        user: 'user',
-        password: 'password',
-        database: 'postgres',
-        max: 1,
-      })
-
-      const wireDb = { exec: (query: string) => sql.unsafe(query) as Promise<unknown> }
-      const { executed, skipped } = await execDumpFile(wireDb, tmpFile)
-
-      for (let t = 0; t < 2; t++) {
-        const result = await db.query(`SELECT count(*) as cnt FROM test_table_${t}`)
-        expect(Number(result.rows[0].cnt)).toBe(3000)
-      }
-
-      console.log(`wire large: ${executed} executed, ${skipped} skipped`)
-      await sql.end()
-    } finally {
-      server.close()
-      await db.close()
-    }
-  }, 180_000)
-})

package/src/s3-local.ts

@@ -1,203 +0,0 @@
-/**
- * minimal local s3-compatible server.
- * handles GET/PUT/DELETE/HEAD for object storage.
- */
-
-import {
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  readdirSync,
-  writeFileSync,
-  unlinkSync,
-  statSync,
-} from 'node:fs'
-import {
-  createServer,
-  type Server,
-  type IncomingMessage,
-  type ServerResponse,
-} from 'node:http'
-import { join, dirname, extname } from 'node:path'
-
-import { log } from './log.js'
-
-export interface S3LocalConfig {
-  port: number
-  dataDir: string
-}
-
-const MIME_TYPES: Record<string, string> = {
-  '.jpg': 'image/jpeg',
-  '.jpeg': 'image/jpeg',
-  '.png': 'image/png',
-  '.gif': 'image/gif',
-  '.webp': 'image/webp',
-  '.svg': 'image/svg+xml',
-  '.json': 'application/json',
-  '.txt': 'text/plain',
-  '.pdf': 'application/pdf',
-  '.mp4': 'video/mp4',
-  '.mp3': 'audio/mpeg',
-}
-
-function corsHeaders(): Record<string, string> {
-  return {
-    'Access-Control-Allow-Origin': '*',
-    'Access-Control-Allow-Methods': 'GET, PUT, DELETE, HEAD, OPTIONS',
-    'Access-Control-Allow-Headers': '*',
-    'Access-Control-Expose-Headers': 'ETag, Content-Length',
-  }
-}
-
-export function startS3Local(config: S3LocalConfig): Promise<Server> {
-  const storageDir = join(config.dataDir, 's3')
-  mkdirSync(storageDir, { recursive: true })
-
-  const server = createServer((req: IncomingMessage, res: ServerResponse) => {
-    const headers = corsHeaders()
-
-    if (req.method === 'OPTIONS') {
-      res.writeHead(200, headers)
-      res.end()
-      return
-    }
-
-    const url = new URL(req.url || '/', `http://localhost:${config.port}`)
-
-    // sanitize path to prevent traversal
-    const normalized = url.pathname
-      .split('/')
-      .filter((s) => s && s !== '..' && s !== '.')
-      .join('/')
-    const filePath = join(storageDir, normalized)
-    if (!filePath.startsWith(storageDir)) {
-      res.writeHead(403, headers)
-      res.end()
-      return
-    }
-
-    try {
-      switch (req.method) {
-        case 'GET': {
-          // S3 ListObjectsV2: GET /?list-type=2&prefix=...&max-keys=...
-          if (url.searchParams.get('list-type') === '2') {
-            const prefix = url.searchParams.get('prefix') || ''
-            const maxKeys = parseInt(url.searchParams.get('max-keys') || '1000')
-            const baseDir = join(storageDir, prefix)
-            const keys: string[] = []
-
-            function walkList(dir: string) {
-              if (keys.length >= maxKeys) return
-              let entries
-              try {
-                entries = readdirSync(dir, { withFileTypes: true })
-              } catch {
-                return
-              }
-              for (const entry of entries) {
-                if (keys.length >= maxKeys) break
-                const full = join(dir, entry.name)
-                if (entry.isDirectory()) {
-                  walkList(full)
-                } else {
-                  const rel = full.slice(storageDir.length + 1)
-                  keys.push(rel)
-                }
-              }
-            }
-            walkList(baseDir)
-
-            const keysXml = keys
-              .map((k) => `<Contents><Key>${k}</Key></Contents>`)
-              .join('')
-            const xml = `<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><KeyCount>${keys.length}</KeyCount>${keysXml}</ListBucketResult>`
-            res.writeHead(200, { ...headers, 'Content-Type': 'application/xml' })
-            res.end(xml)
-            return
-          }
-
-          if (!existsSync(filePath) || statSync(filePath).isDirectory()) {
-            res.writeHead(404, {
-              ...headers,
-              'Content-Type': 'application/xml',
-            })
-            res.end('<Error><Code>NoSuchKey</Code></Error>')
-            return
-          }
-          const data = readFileSync(filePath)
-          const ext = extname(filePath)
-          const contentType = MIME_TYPES[ext] || 'application/octet-stream'
-          res.writeHead(200, {
-            ...headers,
-            'Content-Type': contentType,
-            'Content-Length': data.length.toString(),
-            ETag: `"${Buffer.from(data).length}"`,
-          })
-          res.end(data)
-          break
-        }
-
-        case 'PUT': {
-          const chunks: Buffer[] = []
-          req.on('data', (chunk: Buffer) => chunks.push(chunk))
-          req.on('end', () => {
-            mkdirSync(dirname(filePath), { recursive: true })
-            const body = Buffer.concat(chunks)
-            writeFileSync(filePath, body)
-            res.writeHead(200, {
-              ...headers,
-              ETag: `"${body.length}"`,
-            })
-            res.end()
-          })
-          break
-        }
-
-        case 'DELETE': {
-          if (existsSync(filePath)) {
-            unlinkSync(filePath)
-          }
-          res.writeHead(204, headers)
-          res.end()
-          break
-        }
-
-        case 'HEAD': {
-          if (!existsSync(filePath) || statSync(filePath).isDirectory()) {
-            res.writeHead(404, headers)
-            res.end()
-            return
-          }
-          const stat = statSync(filePath)
-          const ext2 = extname(filePath)
-          res.writeHead(200, {
-            ...headers,
-            'Content-Type': MIME_TYPES[ext2] || 'application/octet-stream',
-            'Content-Length': stat.size.toString(),
-          })
-          res.end()
-          break
-        }
-
-        default:
-          res.writeHead(405, headers)
-          res.end()
-      }
-    } catch {
-      res.writeHead(500, {
-        ...headers,
-        'Content-Type': 'application/xml',
-      })
-      res.end('<Error><Code>InternalError</Code></Error>')
-    }
-  })
-
-  return new Promise((resolve, reject) => {
-    server.listen(config.port, '127.0.0.1', () => {
-      log.s3(`listening on port ${config.port}`)
-      resolve(server)
-    })
-    server.on('error', reject)
-  })
-}

package/src/shim/hooks.mjs

@@ -1,120 +0,0 @@
-// esm loader hooks — intercept @rocicorp/zero-sqlite3 with bedrock-sqlite wasm.
-//
-// NOTE: this file is currently UNUSED. orez uses in-place CJS patching via
-// src/sqlite-mode/apply-mode.ts instead. this file is kept for potential future
-// use with Node.js ESM loader hooks (--import flag).
-//
-// __BEDROCK_PATH__ is replaced at runtime by orez before writing to tmpdir.
-
-const SHIM_URL = 'orez-sqlite-shim://shim'
-const BEDROCK_PATH = '__BEDROCK_PATH__'
-// __JOURNAL_MODE__ would be replaced at runtime (delete for wasm, wal2 for native)
-const JOURNAL_MODE = '__JOURNAL_MODE__'
-
-export function resolve(specifier, context, nextResolve) {
-  if (
-    specifier === '@rocicorp/zero-sqlite3' ||
-    specifier.startsWith('@rocicorp/zero-sqlite3/')
-  ) {
-    return { url: SHIM_URL, shortCircuit: true }
-  }
-  return nextResolve(specifier, context)
-}
-
-export function load(url, context, nextLoad) {
-  if (url === SHIM_URL) {
-    // journal mode differs between native and wasm:
-    // - native: wal2 for better concurrency
-    // - wasm: delete for compatibility (wal2 can corrupt wasm vfs)
-    const journalMode = JOURNAL_MODE === '__JOURNAL_MODE__' ? 'delete' : JOURNAL_MODE
-
-    return {
-      format: 'module',
-      shortCircuit: true,
-      source: `
-// orez sqlite shim - wraps bedrock-sqlite for zero-cache compatibility
-// journal_mode: ${journalMode}
-
-// catch uncaught exceptions from bedrock-sqlite wasm clearly
-process.on('uncaughtException', (err) => {
-  console.error('[orez-shim] UNCAUGHT EXCEPTION:', err?.message || err);
-  console.error('[orez-shim] code:', err?.code, 'name:', err?.name);
-  console.error('[orez-shim] stack:', err?.stack?.split('\\n').slice(0, 5).join('\\n'));
-  process.exit(1);
-});
-
-import { createRequire } from 'node:module';
-const require = createRequire('${BEDROCK_PATH}');
-const mod = require('${BEDROCK_PATH}');
-const OrigDatabase = mod.Database;
-const SqliteError = mod.SqliteError;
-
-function Database(...args) {
-  const db = new OrigDatabase(...args);
-  try {
-    db.pragma('journal_mode = ${journalMode}');
-    db.pragma('busy_timeout = 30000');
-    db.pragma('synchronous = normal');
-  } catch(e) {}
-  return db;
-}
-
-Database.prototype = OrigDatabase.prototype;
-Database.prototype.constructor = Database;
-Object.keys(OrigDatabase).forEach(k => { Database[k] = OrigDatabase[k]; });
-
-// api polyfills for better-sqlite3 compatibility
-Database.prototype.unsafeMode = function() { return this; };
-if (!Database.prototype.defaultSafeIntegers) {
-  Database.prototype.defaultSafeIntegers = function() { return this; };
-}
-if (!Database.prototype.serialize) {
-  Database.prototype.serialize = function() { throw new Error('not supported in wasm'); };
-}
-if (!Database.prototype.backup) {
-  Database.prototype.backup = function() { throw new Error('not supported in wasm'); };
-}
-
-// wrap pragma to skip optimize (can corrupt wasm vfs) and swallow sqlite errors
-const origPragma = OrigDatabase.prototype.pragma;
-Database.prototype.pragma = function(str, opts) {
-  if (str && str.trim().toLowerCase().startsWith('optimize')) return [];
-  try { return origPragma.call(this, str, opts); }
-  catch(e) { if (e && (e.code === 'SQLITE_CORRUPT' || e.code === 'SQLITE_IOERR')) return []; throw e; }
-};
-
-// wrap close to swallow wasm errors during shutdown
-const origClose = OrigDatabase.prototype.close;
-Database.prototype.close = function() {
-  try { return origClose.call(this); }
-  catch(e) { console.error('[orez-shim] close error (swallowed):', e?.message || e); }
-};
-
-// statement prototype polyfills
-const tmpDb = new OrigDatabase(':memory:');
-const tmpStmt = tmpDb.prepare('SELECT 1');
-const SP = Object.getPrototypeOf(tmpStmt);
-if (!SP.safeIntegers) SP.safeIntegers = function() { return this; };
-SP.scanStatus = function() { return undefined; };
-SP.scanStatusV2 = function() { return []; };
-SP.scanStatusReset = function() {};
-tmpDb.close();
-
-// scanstat constants for query planner compatibility
-Database.SQLITE_SCANSTAT_NLOOP = 0;
-Database.SQLITE_SCANSTAT_NVISIT = 1;
-Database.SQLITE_SCANSTAT_EST = 2;
-Database.SQLITE_SCANSTAT_NAME = 3;
-Database.SQLITE_SCANSTAT_EXPLAIN = 4;
-Database.SQLITE_SCANSTAT_SELECTID = 5;
-Database.SQLITE_SCANSTAT_PARENTID = 6;
-Database.SQLITE_SCANSTAT_NCYCLE = 7;
-Database.SQLITE_SCANSTAT_COMPLEX = 8;
-
-export default Database;
-export { SqliteError };
-`,
-    }
-  }
-  return nextLoad(url, context)
-}

package/src/shim/register.mjs

@@ -1,4 +0,0 @@
-// --import entrypoint that registers the esm loader hooks.
-// __HOOKS_URL__ is replaced at runtime by orez before writing to tmpdir.
-import { register } from 'node:module'
-register('__HOOKS_URL__', import.meta.url)