@factorypure/devdb 0.0.0-stage → 0.1.0

package/bin/devdb.js

@@ -0,0 +1,558 @@
+#!/usr/bin/env node
+import { execFileSync, spawn, spawnSync } from 'node:child_process'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+const HOME = process.env.DEVDB_HOME ?? path.join(os.homedir(), '.devdb')
+const GOLDENS = path.join(HOME, 'goldens')
+const CLONES = path.join(HOME, 'clones')
+const DUMPS = path.join(HOME, 'dumps')
+const RUN = path.join(HOME, 'run')
+const STATE_FILE = path.join(HOME, 'state.json')
+
+const BUCKET = 'factorypure-dev-db-snapshots'
+const LATEST_KEY = 'dev-db/latest.json'
+// apps hardcode port 3306 and only swap the host (see setupENVVars.js)
+const PORT = Number(process.env.DEVDB_PORT ?? 3306)
+const LOAD_PORT = 3399
+const BUFFER_POOL = process.env.DEVDB_BUFFER_POOL ?? '2G'
+const KEEP_GOLDENS = 2
+const MYSQL_BIN = process.env.DEVDB_MYSQL_BIN ?? '/opt/homebrew/opt/mysql@8.4/bin'
+const NAME_PATTERN = /^[a-z0-9][a-z0-9._-]{0,40}$/i
+
+const HELP = `devdb - local copies of the prod database (refreshed nightly by fpjobs dev-db-snapshot)
+
+  devdb pull [--env FILE] [--force]   download the latest snapshot and load it into a new golden copy
+                                      (--env: create the DB_USER/DB_PASSWORD app user from that .env;
+                                      defaults to ./.env when present)
+  devdb new NAME                      instant copy-on-write clone of the newest golden, started on :${PORT}
+  devdb use NAME                      stop the active clone and start NAME
+  devdb reset [NAME]                  throw away NAME (default: active) and re-clone the newest golden
+  devdb rm NAME                       delete a clone
+  devdb start | stop                  start/stop the active clone
+  devdb user [--env FILE]             create the app user from a .env on the running clone
+  devdb list                          goldens and clones
+  devdb doctor                        check prerequisites
+
+Point apps at it with DB_DEVELOPMENT_HOST = 127.0.0.1, then run the branch's pending migrations.
+Requires: brew install mysql@8.4 zstd, and AWS SSO credentials (aws sso login).`
+
+const die = (message) => {
+    console.error(`devdb: ${message}`)
+    process.exit(1)
+}
+
+const bin = (name) => path.join(MYSQL_BIN, name)
+const run = (command, args, options = {}) => execFileSync(command, args, { stdio: 'inherit', ...options })
+const capture = (command, args) => execFileSync(command, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] })
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+const listDirs = (dir) =>
+    fs.existsSync(dir)
+        ? fs
+              .readdirSync(dir, { withFileTypes: true })
+              .filter((e) => e.isDirectory())
+              .map((e) => e.name)
+              .sort()
+        : []
+
+function readState() {
+    return fs.existsSync(STATE_FILE) ? JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')) : { active: null, clones: {} }
+}
+
+function writeState(state) {
+    fs.mkdirSync(HOME, { recursive: true })
+    fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2))
+}
+
+const completedGoldens = () => listDirs(GOLDENS).filter((name) => !name.endsWith('.loading'))
+
+function latestGolden() {
+    const golden = completedGoldens().at(-1)
+    if (!golden) {
+        die('no golden copy yet, run `devdb pull` first')
+    }
+    return golden
+}
+
+function serverPaths(name, datadir) {
+    fs.mkdirSync(RUN, { recursive: true })
+    return {
+        name,
+        datadir,
+        socket: path.join(RUN, `${name}.sock`),
+        pid: path.join(RUN, `${name}.pid`),
+        log: path.join(RUN, `${name}.log`),
+    }
+}
+
+const clonePaths = (name) => serverPaths(name, path.join(CLONES, name))
+
+function validateName(name) {
+    if (!name || !NAME_PATTERN.test(name)) {
+        die('clone names may use letters, digits, ".", "_" and "-"')
+    }
+}
+
+function isRunning(paths) {
+    if (!fs.existsSync(paths.pid)) {
+        return false
+    }
+    try {
+        process.kill(Number(fs.readFileSync(paths.pid, 'utf8')), 0)
+        return true
+    } catch {
+        return false
+    }
+}
+
+function assertPortFree(port) {
+    const result = spawnSync('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-t'], { encoding: 'utf8' })
+    if (result.stdout.trim()) {
+        die(`port ${port} is already in use (pid ${result.stdout.trim()}); stop that server first (e.g. brew services stop mysql@8.4)`)
+    }
+}
+
+const ping = (paths) =>
+    spawnSync(bin('mysqladmin'), ['--no-defaults', `--socket=${paths.socket}`, '-uroot', 'ping'], { stdio: 'ignore' })
+        .status === 0
+
+async function startServer(paths, port, { loading = false } = {}) {
+    if (isRunning(paths)) {
+        return
+    }
+    assertPortFree(port)
+    const args = [
+        '--no-defaults',
+        `--datadir=${paths.datadir}`,
+        `--port=${port}`,
+        '--bind-address=127.0.0.1',
+        `--socket=${paths.socket}`,
+        `--pid-file=${paths.pid}`,
+        `--log-error=${paths.log}`,
+        '--mysqlx=OFF',
+        '--skip-log-bin',
+        // match prod
+        '--sql-mode=IGNORE_SPACE,NO_ENGINE_SUBSTITUTION',
+        '--default-time-zone=+00:00',
+        '--log-bin-trust-function-creators=1',
+        // prod has foreign keys onto non-unique keys (e.g. draft_order_line_items -> draft_orders), which 8.4 rejects by default
+        '--restrict-fk-on-non-standard-key=OFF',
+        // the apps' `mysql` npm driver can't do 8.4's default caching_sha2_password
+        '--mysql-native-password=ON',
+        '--character-set-server=utf8mb4',
+        '--collation-server=utf8mb4_0900_ai_ci',
+        '--max-connections=500',
+        `--innodb-buffer-pool-size=${BUFFER_POOL}`,
+        '--innodb-redo-log-capacity=2G',
+        // durability doesn't matter for a throwaway copy; these make loading much faster
+        `--innodb-flush-log-at-trx-commit=${loading ? 0 : 2}`,
+        ...(loading ? ['--innodb-doublewrite=OFF'] : []),
+    ]
+    spawn(bin('mysqld'), args, { detached: true, stdio: 'ignore' }).unref()
+    for (let i = 0; i < 240; i++) {
+        if (ping(paths)) {
+            return
+        }
+        await sleep(500)
+    }
+    const log = fs.existsSync(paths.log) ? fs.readFileSync(paths.log, 'utf8').split('\n').slice(-20).join('\n') : ''
+    die(`mysqld for ${paths.name} didn't start. Last log lines:\n${log}`)
+}
+
+async function stopServer(paths) {
+    if (!isRunning(paths)) {
+        return
+    }
+    spawnSync(bin('mysqladmin'), ['--no-defaults', `--socket=${paths.socket}`, '-uroot', 'shutdown'], { stdio: 'inherit' })
+    for (let i = 0; i < 600 && isRunning(paths); i++) {
+        await sleep(500)
+    }
+    if (isRunning(paths)) {
+        die(`mysqld for ${paths.name} didn't stop, see ${paths.log}`)
+    }
+}
+
+const sqlString = (value) => `'${String(value).replaceAll('\\', '\\\\').replaceAll("'", "\\'")}'`
+
+function mysqlRoot(paths, sql) {
+    const result = spawnSync(bin('mysql'), ['--no-defaults', `--socket=${paths.socket}`, '-uroot'], {
+        input: sql,
+        stdio: ['pipe', 'inherit', 'inherit'],
+    })
+    if (result.status !== 0) {
+        die('mysql command failed')
+    }
+}
+
+function parseEnvFile(file) {
+    const values = {}
+    for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
+        const match = line.match(/^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*?)\s*$/)
+        if (match) {
+            values[match[1]] = match[2].replace(/^(['"])(.*)\1$/, '$2')
+        }
+    }
+    return values
+}
+
+function appCredentials(flags) {
+    const file = flags.env ?? (fs.existsSync('.env') ? '.env' : null)
+    if (!file) {
+        return null
+    }
+    const { DB_USER, DB_PASSWORD } = parseEnvFile(file)
+    if (!DB_USER || !DB_PASSWORD) {
+        die(`${file} has no DB_USER/DB_PASSWORD`)
+    }
+    return { user: DB_USER, password: DB_PASSWORD, file }
+}
+
+function createAppUser(paths, credentials) {
+    const account = `${sqlString(credentials.user)}@'%'`
+    mysqlRoot(
+        paths,
+        `CREATE USER IF NOT EXISTS ${account} IDENTIFIED WITH mysql_native_password BY ${sqlString(credentials.password)};
+         ALTER USER ${account} IDENTIFIED WITH mysql_native_password BY ${sqlString(credentials.password)};
+         GRANT ALL ON *.* TO ${account};`
+    )
+    console.log(`App user ${credentials.user} created from ${credentials.file}`)
+}
+
+function cloneFromGolden(name, golden) {
+    fs.mkdirSync(CLONES, { recursive: true })
+    // -c uses APFS clonefile(2): instant, and blocks are only copied once either side changes
+    run('cp', ['-c', '-R', path.join(GOLDENS, golden), path.join(CLONES, name)])
+}
+
+function ensureAppUser(paths, flags) {
+    const credentials = appCredentials(flags)
+    if (credentials) {
+        createAppUser(paths, credentials)
+    } else {
+        console.log('No .env here: run `devdb user --env path/to/.env` so the apps can log in')
+    }
+}
+
+// Runs one mydumper file (zstd-compressed SQL with its own SET header) through the mysql client.
+function sqlFile(paths, file, database = 'main') {
+    return new Promise((resolve, reject) => {
+        const zstd = spawn('zstd', ['-dc', file], { stdio: ['ignore', 'pipe', 'pipe'] })
+        const mysql = spawn(
+            bin('mysql'),
+            ['--no-defaults', `--socket=${paths.socket}`, '-uroot', '--max-allowed-packet=1G', ...(database ? [database] : [])],
+            { stdio: ['pipe', 'ignore', 'pipe'] }
+        )
+        zstd.stdout.pipe(mysql.stdin)
+        let stderr = ''
+        zstd.stderr.on('data', (d) => (stderr += d))
+        mysql.stderr.on('data', (d) => (stderr += d))
+        mysql.on('close', (code) =>
+            code === 0 ? resolve() : reject(new Error(`${path.basename(file)}: ${stderr.trim()}`))
+        )
+    })
+}
+
+async function inParallel(items, concurrency, task) {
+    let next = 0
+    const worker = async () => {
+        while (next < items.length) {
+            await task(items[next++])
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker))
+}
+
+function sqlText(paths, text) {
+    const result = spawnSync(
+        bin('mysql'),
+        ['--no-defaults', `--socket=${paths.socket}`, '-uroot', 'main'],
+        { input: text, encoding: 'utf8' }
+    )
+    if (result.status !== 0) {
+        return Promise.reject(new Error(result.stderr.trim()))
+    }
+    return Promise.resolve()
+}
+
+// mydumper trigger files rely on myloader to split statements; the mysql client needs DELIMITER around bodies with `;`.
+function withTriggerDelimiters(sql) {
+    const out = []
+    let depth = 0
+    for (const line of sql.split('\n')) {
+        if (depth === 0 && /^CREATE\b.*\bTRIGGER\b.*\bBEGIN\b/i.test(line)) {
+            out.push('DELIMITER ;;')
+        }
+        const opened = depth > 0 || /^CREATE\b.*\bTRIGGER\b/i.test(line)
+        depth += opened ? (line.match(/\bBEGIN\b/gi) ?? []).length : 0
+        if (depth > 0 && /^\s*END;\s*$/i.test(line) && --depth === 0) {
+            out.push(line.replace(/;\s*$/, ';;'), 'DELIMITER ;')
+            continue
+        }
+        out.push(line)
+    }
+    return out.join('\n')
+}
+
+// myloader 1.0.5 stalls after creating a handful of tables from this dump, so the files are loaded directly.
+async function loadDump(dumpDir, paths) {
+    const files = fs.readdirSync(dumpDir)
+    const full = (f) => path.join(dumpDir, f)
+    const isView = new Set(files.filter((f) => f.endsWith('-schema-view.sql.zst')).map((f) => f.replace('-schema-view', '-schema')))
+    // view placeholder tables are skipped; the real views are created after the data
+    const tables = files.filter((f) => f.endsWith('-schema.sql.zst') && f !== 'main-schema-create.sql.zst' && !isView.has(f))
+    const views = files.filter((f) => f.endsWith('-schema-view.sql.zst'))
+    const triggers = files.filter((f) => f.endsWith('-schema-triggers.sql.zst'))
+    const data = files
+        .filter((f) => /\.\d+\.sql\.zst$/.test(f))
+        .map((f) => ({ f, size: fs.statSync(full(f)).size }))
+        .sort((a, b) => b.size - a.size)
+    const totalBytes = data.reduce((sum, d) => sum + d.size, 0)
+    const threads = os.cpus().length
+
+    await sqlFile(paths, full('main-schema-create.sql.zst'), null)
+    // one at a time: concurrent CREATE TABLEs with foreign keys deadlock on the data dictionary
+    await inParallel(tables, 1, (f) => sqlFile(paths, full(f)))
+    console.log(`Created ${tables.length} tables, loading ${data.length} data files (${(totalBytes / 1024 ** 3).toFixed(1)} GB compressed)...`)
+
+    const startedAt = Date.now()
+    let doneBytes = 0
+    let doneFiles = 0
+    const progress = setInterval(() => {
+        const minutes = ((Date.now() - startedAt) / 60000).toFixed(1)
+        console.log(`  ${doneFiles}/${data.length} files, ${Math.round((doneBytes / totalBytes) * 100)}% of bytes, ${minutes} min`)
+    }, 30000)
+    try {
+        await inParallel(data, threads, async ({ f, size }) => {
+            await sqlFile(paths, full(f))
+            doneBytes += size
+            doneFiles++
+        })
+    } finally {
+        clearInterval(progress)
+    }
+    console.log(`Data loaded in ${((Date.now() - startedAt) / 60000).toFixed(1)} min`)
+
+    // a view can select from another view, so failures get retried once the rest exist
+    let pending = views
+    for (let pass = 0; pending.length && pass < 5; pass++) {
+        const failed = []
+        for (const f of pending) {
+            await sqlFile(paths, full(f)).catch((err) => failed.push({ f, err }))
+        }
+        if (failed.length === pending.length) {
+            die(`could not create views:\n${failed.map((x) => x.err.message).join('\n')}`)
+        }
+        pending = failed.map((x) => x.f)
+    }
+    if (pending.length) {
+        die(`views still failing after retries: ${pending.join(', ')}`)
+    }
+    // created last so they don't fire for every loaded row
+    for (const f of triggers) {
+        await sqlText(paths, withTriggerDelimiters(execFileSync('zstd', ['-dc', full(f)], { encoding: 'utf8' })))
+    }
+    console.log(`Created ${views.length} views and ${triggers.length} trigger files`)
+}
+
+function parseFlags(args) {
+    const flags = {}
+    const positional = []
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--force') {
+            flags.force = true
+        } else if (args[i] === '--env') {
+            flags.env = args[++i]
+        } else {
+            positional.push(args[i])
+        }
+    }
+    return { flags, positional }
+}
+
+async function stopActive(state) {
+    if (state.active) {
+        await stopServer(clonePaths(state.active))
+    }
+}
+
+const commands = {
+    async pull(flags) {
+        const startedAt = Date.now()
+        const manifest = JSON.parse(capture('aws', ['s3', 'cp', `s3://${BUCKET}/${LATEST_KEY}`, '-']))
+        const { stamp } = manifest
+        if (completedGoldens().includes(stamp) && !flags.force) {
+            console.log(`Already have the latest snapshot (${stamp}).`)
+            return
+        }
+        const credentials = appCredentials(flags)
+        console.log(
+            `Downloading ${stamp} (${(manifest.bytes / 1024 ** 3).toFixed(1)} GB, prod snapshot from ${manifest.snapshotCreatedAt})`
+        )
+        const dumpDir = path.join(DUMPS, stamp)
+        run('aws', ['s3', 'sync', `s3://${BUCKET}/${manifest.prefix}`, dumpDir, '--only-show-errors'])
+
+        const loadingDir = path.join(GOLDENS, `${stamp}.loading`)
+        fs.rmSync(loadingDir, { recursive: true, force: true })
+        fs.mkdirSync(GOLDENS, { recursive: true })
+        run(bin('mysqld'), ['--no-defaults', '--initialize-insecure', `--datadir=${loadingDir}`])
+
+        const loader = serverPaths('loader', loadingDir)
+        await startServer(loader, LOAD_PORT, { loading: true })
+        try {
+            await loadDump(dumpDir, loader)
+            if (credentials) {
+                createAppUser(loader, credentials)
+            }
+        } finally {
+            await stopServer(loader)
+        }
+        fs.renameSync(loadingDir, path.join(GOLDENS, stamp))
+        fs.rmSync(dumpDir, { recursive: true, force: true })
+
+        for (const old of completedGoldens().slice(0, -KEEP_GOLDENS)) {
+            fs.rmSync(path.join(GOLDENS, old), { recursive: true, force: true })
+        }
+        const minutes = Math.round((Date.now() - startedAt) / 60000)
+        const { active } = readState()
+        console.log(
+            `Golden ${stamp} ready in ${minutes} min. ${active ? `Run \`devdb reset\` to move ${active} onto it` : 'Run `devdb new <name>` to create a clone'}.`
+        )
+    },
+
+    async new(flags, [name]) {
+        validateName(name)
+        const state = readState()
+        if (fs.existsSync(path.join(CLONES, name))) {
+            die(`${name} already exists (use \`devdb use ${name}\` or \`devdb reset ${name}\`)`)
+        }
+        const golden = latestGolden()
+        await stopActive(state)
+        cloneFromGolden(name, golden)
+        await startServer(clonePaths(name), PORT)
+        ensureAppUser(clonePaths(name), flags)
+        state.clones[name] = { golden, createdAt: new Date().toISOString() }
+        state.active = name
+        writeState(state)
+        console.log(`${name} (from ${golden}) is running on 127.0.0.1:${PORT}`)
+    },
+
+    async use(flags, [name]) {
+        validateName(name)
+        const state = readState()
+        if (!fs.existsSync(path.join(CLONES, name))) {
+            die(`no clone named ${name}`)
+        }
+        if (state.active !== name) {
+            await stopActive(state)
+        }
+        await startServer(clonePaths(name), PORT)
+        state.active = name
+        writeState(state)
+        console.log(`${name} is running on 127.0.0.1:${PORT}`)
+    },
+
+    async reset(flags, [name]) {
+        const state = readState()
+        name ??= state.active
+        validateName(name)
+        const golden = latestGolden()
+        await stopServer(clonePaths(name))
+        fs.rmSync(path.join(CLONES, name), { recursive: true, force: true })
+        if (state.active && state.active !== name) {
+            await stopActive(state)
+        }
+        cloneFromGolden(name, golden)
+        await startServer(clonePaths(name), PORT)
+        ensureAppUser(clonePaths(name), flags)
+        state.clones[name] = { golden, createdAt: new Date().toISOString() }
+        state.active = name
+        writeState(state)
+        console.log(`${name} reset to ${golden} and running on 127.0.0.1:${PORT}`)
+    },
+
+    async rm(flags, [name]) {
+        validateName(name)
+        const state = readState()
+        await stopServer(clonePaths(name))
+        fs.rmSync(path.join(CLONES, name), { recursive: true, force: true })
+        delete state.clones[name]
+        if (state.active === name) {
+            state.active = null
+        }
+        writeState(state)
+        console.log(`Removed ${name}`)
+    },
+
+    async start() {
+        const { active } = readState()
+        if (!active) {
+            die('no active clone, run `devdb new <name>`')
+        }
+        await startServer(clonePaths(active), PORT)
+        console.log(`${active} is running on 127.0.0.1:${PORT}`)
+    },
+
+    async stop() {
+        const { active } = readState()
+        if (active) {
+            await stopServer(clonePaths(active))
+            console.log(`Stopped ${active}`)
+        }
+    },
+
+    async user(flags) {
+        const { active } = readState()
+        const paths = active && clonePaths(active)
+        if (!paths || !isRunning(paths)) {
+            die('start a clone first')
+        }
+        const credentials = appCredentials(flags)
+        if (!credentials) {
+            die('pass --env path/to/.env')
+        }
+        createAppUser(paths, credentials)
+    },
+
+    async list() {
+        const state = readState()
+        console.log('Goldens:')
+        for (const golden of completedGoldens()) {
+            console.log(`  ${golden}`)
+        }
+        console.log('Clones:')
+        for (const name of listDirs(CLONES)) {
+            const running = isRunning(clonePaths(name)) ? `running on :${PORT}` : 'stopped'
+            const marker = state.active === name ? '*' : ' '
+            console.log(` ${marker}${name}  from ${state.clones[name]?.golden ?? '?'}  ${running}`)
+        }
+    },
+
+    async doctor() {
+        const checks = [
+            ['mysqld (brew install mysql@8.4)', () => capture(bin('mysqld'), ['--version'])],
+            ['zstd (brew install zstd)', () => capture('zstd', ['--version'])],
+            ['aws cli', () => capture('aws', ['--version'])],
+            ['AWS credentials (aws sso login)', () => capture('aws', ['s3', 'ls', `s3://${BUCKET}/dev-db/`])],
+        ]
+        let ok = true
+        for (const [label, check] of checks) {
+            try {
+                check()
+                console.log(`ok   ${label}`)
+            } catch {
+                ok = false
+                console.log(`FAIL ${label}`)
+            }
+        }
+        process.exitCode = ok ? 0 : 1
+    },
+}
+
+const [command, ...rest] = process.argv.slice(2)
+if (!command || command === 'help' || command === '--help' || !commands[command]) {
+    console.log(HELP)
+    process.exit(command && !['help', '--help'].includes(command) ? 1 : 0)
+}
+const { flags, positional } = parseFlags(rest)
+await commands[command](flags, positional)

package/package.json

@@ -1,6 +1,16 @@
 {
   "name": "@factorypure/devdb",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Local copies of the prod MySQL database for development",
+  "type": "module",
+  "bin": {
+    "devdb": "./bin/devdb.js"
+  },
+  "files": [
+    "bin"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "license": "ISC"
+}

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.