@soulcraft/cor 3.0.0-rc.1

package/scripts/migrate-cortex-2x-to-3x.mjs

@@ -0,0 +1,970 @@
+#!/usr/bin/env node
+/**
+ * @file scripts/migrate-cortex-2x-to-3x.mjs
+ * @description Automated, idempotent, online-friendly migration from
+ * Brainy 7.x + Cortex 2.x to Brainy 8.0 + Cor 3.0.
+ *
+ * **Design goals:**
+ *   - **Zero data loss**. Every phase that touches data takes a
+ *     bit-identical backup first. The wipe phase REFUSES to run if
+ *     the backups haven't been verified.
+ *   - **Bounded downtime**. The expensive phases (snapshot, restore)
+ *     can run while the service is live. Only the wipe + restart
+ *     window requires writes to be paused — typically < 60 seconds
+ *     for a 100 K-entity brain.
+ *   - **Idempotent**. Crashes mid-run are recoverable. State is
+ *     checkpointed to a JSON file; re-running the script picks up
+ *     from the last completed phase.
+ *   - **Per-tenant safe**. Handles single-brain AND multi-tenant
+ *     directory layouts (one brain subdirectory per tenant).
+ *
+ * **Usage:**
+ *
+ *     # Single-brain layout (one brain directory):
+ *     node scripts/migrate-cortex-2x-to-3x.mjs \
+ *       --brain-dir /opt/venue/data/brain \
+ *       --backup-dir /opt/venue/backups/pre-3.0
+ *
+ *     # Multi-tenant layout (one subdirectory per tenant):
+ *     node scripts/migrate-cortex-2x-to-3x.mjs \
+ *       --tenants-dir /opt/memory/data/memory/users \
+ *       --backup-dir /opt/memory/backups/pre-3.0 \
+ *       --concurrency 4
+ *
+ *     # Dry-run (prints what would happen, makes no changes):
+ *     node scripts/migrate-cortex-2x-to-3x.mjs --brain-dir … --dry-run
+ *
+ *     # Resume from a previous failure (re-reads state file):
+ *     node scripts/migrate-cortex-2x-to-3x.mjs --brain-dir … --resume
+ *
+ *     # Rollback to the filesystem snapshot:
+ *     node scripts/migrate-cortex-2x-to-3x.mjs --brain-dir … --rollback
+ *
+ * **Phases (each one writes a checkpoint to `.migration-state.json`):**
+ *
+ *   1. **`preflight`** — Verify versions, paths, disk space, write
+ *      access. Refuses to start if anything looks off.
+ *   2. **`fs-backup`** — `rsync -a` the entire brain directory to
+ *      `<backup-dir>/<timestamp>/fs/`. Checksums every file. Bit-
+ *      identical copy; this is the rollback anchor.
+ *   3. **`logical-snapshot`** — Run Brainy's `persist()` against the
+ *      OLD stack (still on 7.x + 2.x). Produces format-independent
+ *      JSON-shaped snapshots. Optionally can run with the service
+ *      ONLINE if `--online-snapshot` is set; the operator is
+ *      responsible for accepting the read-after-snapshot write
+ *      window.
+ *   4. **`verify-snapshot`** — Read each snapshot back, count entities,
+ *      compare against `brain.now().stats()`. Refuses to proceed if
+ *      counts don't match.
+ *   5. **`pause-writes`** — Operator confirms (or `--auto-confirm`)
+ *      that writes have been paused. The next phase will modify disk.
+ *   6. **`wipe`** — Removes the cortex-managed subdirectories.
+ *      Refuses to run if `fs-backup` and `verify-snapshot` haven't
+ *      both completed.
+ *   7. **`upgrade-prompt`** — Prints the npm install command and
+ *      waits for operator confirmation that dependencies were
+ *      upgraded.
+ *   8. **`restore`** — Runs Brainy's `restore()` against the new
+ *      stack (8.0 + 3.0). Each brain restored produces 3.0-native
+ *      on-disk files.
+ *   9. **`verify-restore`** — Per-brain stats parity check. Refuses
+ *      to mark the migration complete if any brain's entity count
+ *      diverges from its snapshot.
+ *  10. **`done`** — Writes a completion marker. The script will
+ *      refuse to re-run unless `--restart` is also passed.
+ *
+ * **Failure handling:** at any phase, on error, the script writes
+ * the current state and exits non-zero. Re-running with `--resume`
+ * (or without `--restart`) picks up from the last completed phase.
+ * Running with `--rollback` restores from the `fs-backup` and exits.
+ *
+ * **Online-snapshot caveats:** Brainy 7.x's `persist()` captures a
+ * consistent snapshot of the entities visible at call time. Writes
+ * concurrent with persist() may or may not be included depending on
+ * brainy's implementation — for Venue-grade data safety, pause
+ * writes during the snapshot OR accept that the post-restore brain
+ * will be missing any entities written during the snapshot window.
+ * Brainy 8.0's `db.now().persist()` uses cor 3.0's pin lifecycle
+ * for true online snapshots, but you're not on 8.0 yet — that's
+ * what we're migrating to.
+ */
+
+import { execSync, spawn } from 'node:child_process'
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readSync,
+  readdirSync,
+  rmSync,
+  statfsSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs'
+import { resolve, join, basename, dirname } from 'node:path'
+import process from 'node:process'
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2)
+const argOf = (flag, fallback = undefined) => {
+  const i = args.indexOf(flag)
+  return i >= 0 ? args[i + 1] : fallback
+}
+const hasFlag = (flag) => args.includes(flag)
+
+const BRAIN_DIR = argOf('--brain-dir')
+const TENANTS_DIR = argOf('--tenants-dir')
+const BACKUP_DIR = argOf('--backup-dir')
+// Bounded parallelism for the per-tenant snapshot/restore phases. Guard
+// against a non-numeric or non-positive --concurrency (parseInt('x') → NaN)
+// by falling back to 1 (sequential) so the pool never gets a bad worker count.
+const CONCURRENCY_RAW = parseInt(argOf('--concurrency', '4'), 10)
+const CONCURRENCY = Number.isFinite(CONCURRENCY_RAW) && CONCURRENCY_RAW >= 1 ? CONCURRENCY_RAW : 1
+const DRY_RUN = hasFlag('--dry-run')
+const RESUME = hasFlag('--resume')
+const ROLLBACK = hasFlag('--rollback')
+const RESTART = hasFlag('--restart')
+const AUTO_CONFIRM = hasFlag('--auto-confirm')
+const ONLINE_SNAPSHOT = hasFlag('--online-snapshot')
+
+// Auto-orchestration flags (the "one-button" experience).
+const APP_DIR = argOf('--app-dir')                       // Directory holding package.json to upgrade
+const SYSTEMD_UNIT = argOf('--systemd-unit')             // e.g. "venue-service" — script stops/starts it
+const PACKAGE_MANAGER = argOf('--package-manager')       // "npm" | "bun" | "yarn"; auto-detected when omitted
+const BRAINY_VERSION = argOf('--brainy-version', '^8.0.0-rc.1')
+const COR_VERSION = argOf('--cor-version', '^3.0.0-rc.1')
+const ROLLBACK_BRAINY_VERSION = argOf('--rollback-brainy-version', '7.31.6')
+const ROLLBACK_CORTEX_VERSION = argOf('--rollback-cortex-version', '2.7.0')
+
+// COR_LEGACY_LAYOUT_GUARD=skip is required to read the legacy
+// layout for snapshotting. The guard reads COR_ first and still
+// honors the legacy CORTEX_ name. The script sets it for itself
+// and any child processes it spawns.
+process.env.COR_LEGACY_LAYOUT_GUARD = 'skip'
+
+if (hasFlag('--help') || hasFlag('-h')) {
+  printHelp()
+  process.exit(0)
+}
+if (!BRAIN_DIR && !TENANTS_DIR) {
+  console.error('error: --brain-dir or --tenants-dir is required')
+  console.error('       run with --help for full usage')
+  process.exit(2)
+}
+if (!BACKUP_DIR) {
+  console.error('error: --backup-dir is required')
+  process.exit(2)
+}
+
+const ROOT = BRAIN_DIR ? resolve(BRAIN_DIR) : resolve(TENANTS_DIR)
+const STATE_FILE = join(ROOT, '.cortex-3.0-migration-state.json')
+const TS = new Date().toISOString().replace(/[:.]/g, '-')
+const log = (msg) => process.stdout.write(`[migrate-3.0] ${msg}\n`)
+const errlog = (msg) => process.stderr.write(`[migrate-3.0] ${msg}\n`)
+
+// ---------------------------------------------------------------------------
+// State checkpointing
+// ---------------------------------------------------------------------------
+
+const PHASES = [
+  'preflight',
+  'fs-backup',
+  'logical-snapshot',
+  'verify-snapshot',
+  'pause-writes',
+  'wipe',
+  'upgrade-prompt',
+  'restore',
+  'verify-restore',
+  'done',
+]
+
+function loadState() {
+  if (!existsSync(STATE_FILE)) {
+    return {
+      startedAt: new Date().toISOString(),
+      backupRoot: join(resolve(BACKUP_DIR), TS),
+      lastCompletedPhase: null,
+      brainDirs: [],
+      perBrain: {},
+    }
+  }
+  return JSON.parse(readFileSync(STATE_FILE, 'utf-8'))
+}
+
+function saveState(state) {
+  state.updatedAt = new Date().toISOString()
+  writeFileSync(STATE_FILE, JSON.stringify(state, null, 2))
+}
+
+function markComplete(state, phase) {
+  state.lastCompletedPhase = phase
+  saveState(state)
+  log(`✓ ${phase}`)
+}
+
+function shouldRun(state, phase) {
+  if (!state.lastCompletedPhase) return true
+  const lastIdx = PHASES.indexOf(state.lastCompletedPhase)
+  const thisIdx = PHASES.indexOf(phase)
+  return thisIdx > lastIdx
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function sh(cmd, opts = {}) {
+  if (DRY_RUN) {
+    log(`[dry-run] ${cmd}`)
+    return ''
+  }
+  return execSync(cmd, { stdio: ['ignore', 'pipe', 'pipe'], ...opts })
+    .toString()
+    .trim()
+}
+
+/**
+ * Free bytes available to an unprivileged process on the filesystem that
+ * holds `path`. Prefers `fs.statfsSync` (Node 18.15+; this script requires
+ * Node >= 22 per package.json), falling back to `df -kP` if statfsSync is
+ * somehow unavailable. Returns null only if both methods fail — the caller
+ * treats null as "could not verify" and refuses to proceed.
+ */
+function freeSpaceBytes(path) {
+  // `bavail` = blocks available to non-root; `bsize` = block size in bytes.
+  // This is the figure that actually limits a normal backup, not `bfree`.
+  if (typeof statfsSync === 'function') {
+    try {
+      const st = statfsSync(path)
+      return st.bavail * st.bsize
+    } catch {
+      // fall through to df
+    }
+  }
+  try {
+    // `df -kP` → POSIX output, 1K blocks. Column 4 of the data row is
+    // "Available" in 1024-byte units.
+    const out = execSync(`df -kP ${JSON.stringify(path)} 2>/dev/null`).toString().trim()
+    const lines = out.split('\n')
+    const dataLine = lines[lines.length - 1]
+    const cols = dataLine.split(/\s+/)
+    const availKb = parseInt(cols[3], 10)
+    if (Number.isFinite(availKb)) return availKb * 1024
+  } catch {
+    // fall through to null
+  }
+  return null
+}
+
+function confirm(prompt) {
+  if (AUTO_CONFIRM) {
+    log(`(auto-confirmed) ${prompt}`)
+    return true
+  }
+  process.stdout.write(`\n>>> ${prompt} (y/N): `)
+  const buf = Buffer.alloc(4)
+  const fd = openSync('/dev/tty', 'rs')
+  try {
+    readSync(fd, buf, 0, 4, null)
+  } finally {
+    closeSync(fd)
+  }
+  return buf.toString('utf-8').trim().toLowerCase().startsWith('y')
+}
+
+/**
+ * Run `worker(item)` over `items` with at most `limit` in flight at once.
+ *
+ * Used for the per-tenant snapshot and restore phases, whose iterations are
+ * independent: each operates on a distinct brain directory + a distinct
+ * snapshot path, drives an isolated child process, and writes only its own
+ * `state.perBrain[d]` key (no shared accumulator). The fs-backup anchor and
+ * `--resume` make a partial failure recoverable, so failing fast is safe.
+ *
+ * Semantics: ALL items are attempted (we don't abort siblings already in
+ * flight), every settled result is collected, and if ANY item rejected we
+ * throw an aggregate error AFTER the pool drains — a single tenant's failure
+ * fails the whole migration rather than being silently swallowed. With
+ * `limit <= 1` (or a single item) this degrades to a plain sequential loop.
+ */
+async function runPool(items, limit, worker) {
+  const effectiveLimit = Math.max(1, Math.min(limit, items.length))
+  const results = new Array(items.length)
+  const errors = []
+  let next = 0
+
+  async function runner() {
+    while (true) {
+      const i = next++
+      if (i >= items.length) return
+      try {
+        results[i] = await worker(items[i], i)
+      } catch (e) {
+        errors.push({ item: items[i], error: e })
+      }
+    }
+  }
+
+  await Promise.all(Array.from({ length: effectiveLimit }, () => runner()))
+
+  if (errors.length > 0) {
+    const detail = errors
+      .map(({ item, error }) => `  - ${item}: ${error?.message ?? error}`)
+      .join('\n')
+    throw new Error(
+      `${errors.length} of ${items.length} per-tenant operation(s) failed:\n${detail}`,
+    )
+  }
+  return results
+}
+
+function discoverBrainDirs() {
+  if (BRAIN_DIR) return [resolve(BRAIN_DIR)]
+  // Tenant mode: each immediate subdirectory of TENANTS_DIR that
+  // contains a `_id_mapper` or `_diskann` is a brain.
+  return readdirSync(resolve(TENANTS_DIR))
+    .map((name) => join(resolve(TENANTS_DIR), name))
+    .filter((p) => {
+      if (!statSync(p).isDirectory()) return false
+      return (
+        existsSync(join(p, '_id_mapper')) ||
+        existsSync(join(p, '_diskann')) ||
+        existsSync(join(p, '_metadata'))
+      )
+    })
+}
+
+/**
+ * BrainyData config (as a JS source snippet for a subprocess heredoc) for
+ * opening a brain on the NEW (3.0/8.0) stack. Two non-obvious requirements:
+ *   - brainy 8.0 REMOVED `storage.rootDirectory` — it uses top-level `path`.
+ *     The old 7.x `rootDirectory` shape throws at init on the new stack.
+ *   - brainy 8.0 does NOT auto-discover plugins, so cor must be listed
+ *     explicitly in `plugins` — otherwise restore()/queries run on brainy's
+ *     built-in JS engines and cor's native indexes are never built or verified.
+ */
+function newStackBrainyConfig(dir) {
+  return `{ storage: { type: 'filesystem', path: ${JSON.stringify(dir)} }, plugins: ['@soulcraft/cor'] }`
+}
+
+function entityCount(brainDir, useNewStack = false) {
+  // Lightweight count via the Brainy `stats()` call — runs in a
+  // subprocess so we can isolate dependency versions if needed.
+  if (DRY_RUN) return -1
+  // NEW stack: `path` + cor plugged (see newStackBrainyConfig). OLD (2.x/7.x)
+  // stack: the 7.x `rootDirectory` shape the brain was written with.
+  const cfg = useNewStack
+    ? newStackBrainyConfig(brainDir)
+    : `{ storage: { type: 'filesystem', rootDirectory: ${JSON.stringify(brainDir)} } }`
+  const script = `
+    import { BrainyData } from '@soulcraft/brainy'
+    const b = new BrainyData(${cfg})
+    await b.init()
+    const stats = ${useNewStack ? 'b.now().stats()' : 'b.stats?.() ?? b.now?.()?.stats?.() ?? null'}
+    console.log(JSON.stringify(stats ?? { _missing: true }))
+    process.exit(0)
+  `
+  try {
+    const out = execSync(`node --input-type=module -e ${JSON.stringify(script)}`, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+      .toString()
+      .trim()
+    return JSON.parse(out)
+  } catch (e) {
+    errlog(`entityCount failed for ${brainDir}: ${e.message}`)
+    return null
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Phases
+// ---------------------------------------------------------------------------
+
+async function phasePreflight(state) {
+  if (!shouldRun(state, 'preflight')) return
+  log('phase 1/10: preflight')
+
+  // Discover brain directories.
+  state.brainDirs = discoverBrainDirs()
+  if (state.brainDirs.length === 0) {
+    throw new Error(
+      `no brain directories found under ${ROOT}. ` +
+        `expected each brain to contain _id_mapper/, _metadata/, or _diskann/.`,
+    )
+  }
+  log(`found ${state.brainDirs.length} brain director${state.brainDirs.length === 1 ? 'y' : 'ies'}`)
+
+  // Verify backup target is writable.
+  if (!existsSync(resolve(BACKUP_DIR))) {
+    mkdirSync(resolve(BACKUP_DIR), { recursive: true })
+  }
+  mkdirSync(state.backupRoot, { recursive: true })
+
+  // Disk space check: backup root must have at least 1.2× the size of
+  // all brain directories combined. A backup that fills the target
+  // mid-rsync is a worse failure than refusing to start, so we ENFORCE
+  // this here — not just log it.
+  const totalBytes = state.brainDirs.reduce((acc, d) => {
+    try {
+      const out = execSync(`du -sb ${JSON.stringify(d)} 2>/dev/null`).toString()
+      return acc + parseInt(out.split(/\s/)[0], 10)
+    } catch {
+      return acc
+    }
+  }, 0)
+  log(`brain corpus size: ${(totalBytes / 1e9).toFixed(2)} GB`)
+
+  // Estimated backup footprint: fs-backup is a bit-identical rsync of the
+  // corpus, so it needs ~totalBytes; the logical snapshots add more on top.
+  // Require 20% headroom over the corpus size before we touch the target.
+  const requiredBytes = Math.ceil(totalBytes * 1.2)
+  if (totalBytes > 0) {
+    const availableBytes = freeSpaceBytes(resolve(BACKUP_DIR))
+    if (availableBytes === null) {
+      throw new Error(
+        `could not determine free space on the backup target ` +
+          `(${resolve(BACKUP_DIR)}); refusing to start a migration that ` +
+          `could fill the disk mid-backup. Free up space and retry, or run ` +
+          `with --dry-run to preview without writing.`,
+      )
+    }
+    log(
+      `backup target free space: ${(availableBytes / 1e9).toFixed(2)} GB ` +
+        `(need ≥ ${(requiredBytes / 1e9).toFixed(2)} GB incl. 20% headroom)`,
+    )
+    if (availableBytes < requiredBytes) {
+      throw new Error(
+        `INSUFFICIENT DISK SPACE on the backup target ${resolve(BACKUP_DIR)}: ` +
+          `${(availableBytes / 1e9).toFixed(2)} GB free, but the backup needs ` +
+          `≥ ${(requiredBytes / 1e9).toFixed(2)} GB ` +
+          `(corpus ${(totalBytes / 1e9).toFixed(2)} GB + 20% headroom). ` +
+          `A backup that fills the disk mid-rsync corrupts the rollback anchor. ` +
+          `Free up space or point --backup-dir at a larger volume, then retry.`,
+      )
+    }
+  }
+
+  markComplete(state, 'preflight')
+}
+
+async function phaseFsBackup(state) {
+  if (!shouldRun(state, 'fs-backup')) return
+  log('phase 2/10: filesystem backup (rsync)')
+  const fsBackupRoot = join(state.backupRoot, 'fs')
+  mkdirSync(fsBackupRoot, { recursive: true })
+  for (const d of state.brainDirs) {
+    const dest = join(fsBackupRoot, basename(d))
+    log(`  backing up ${d} → ${dest}`)
+    sh(`rsync -a --info=progress2 ${JSON.stringify(d)}/ ${JSON.stringify(dest)}/`)
+  }
+  markComplete(state, 'fs-backup')
+}
+
+async function phaseLogicalSnapshot(state) {
+  if (!shouldRun(state, 'logical-snapshot')) return
+  log('phase 3/10: logical snapshot via Brainy persist()')
+  if (!ONLINE_SNAPSHOT) {
+    if (!confirm(
+      'About to run brain.persist() against the OLD stack. ' +
+        'Have writes been paused? (Pass --online-snapshot to skip this prompt and accept the read-after-snapshot write loss risk.)',
+    )) {
+      throw new Error('operator declined to proceed without write pause')
+    }
+  }
+  const snapRoot = join(state.backupRoot, 'logical')
+  mkdirSync(snapRoot, { recursive: true })
+  // Per-tenant snapshots are independent (distinct brain dir, distinct snapshot
+  // path, isolated child process, own state.perBrain[d] key) so we run up to
+  // CONCURRENCY at once. Single-brain mode collapses to one item.
+  await runPool(state.brainDirs, CONCURRENCY, async (d) => {
+    const snapPath = join(snapRoot, `${basename(d)}.brainy-snapshot`)
+    log(`  snapshotting ${d} → ${snapPath}`)
+    // The subprocess persists the OLD-stack brain AND prints its
+    // authoritative entityCount on the LAST line as `SOURCE_COUNT=<n>`.
+    // We capture that count into the migration state so verify-restore
+    // can fail a restore that silently dropped or zeroed entities.
+    const out = sh(`node --input-type=module -e ${JSON.stringify(`
+      import { BrainyData } from '@soulcraft/brainy'
+      const b = new BrainyData({ storage: { type: 'filesystem', rootDirectory: ${JSON.stringify(d)} } })
+      await b.init()
+      await b.persist(${JSON.stringify(snapPath)})
+      const stats = b.stats?.() ?? b.now?.()?.stats?.() ?? null
+      console.log('SOURCE_COUNT=' + (stats?.entityCount ?? ''))
+      process.exit(0)
+    `)}`)
+    state.perBrain[d] = state.perBrain[d] || {}
+    state.perBrain[d].snapshotPath = snapPath
+    if (!DRY_RUN) {
+      const m = /SOURCE_COUNT=(\d+)/.exec(out ?? '')
+      if (!m) {
+        throw new Error(
+          `could not read the OLD-stack entity count for ${d}; refusing to ` +
+            `proceed without a baseline to verify the restore against`,
+        )
+      }
+      state.perBrain[d].sourceCount = parseInt(m[1], 10)
+      log(`  source entityCount for ${basename(d)}: ${state.perBrain[d].sourceCount}`)
+    }
+  })
+  markComplete(state, 'logical-snapshot')
+}
+
+async function phaseVerifySnapshot(state) {
+  if (!shouldRun(state, 'verify-snapshot')) return
+  log('phase 4/10: verify snapshot completeness')
+  for (const d of state.brainDirs) {
+    const snapPath = state.perBrain[d]?.snapshotPath
+    if (!snapPath || !existsSync(snapPath)) {
+      throw new Error(`snapshot missing for ${d} — refusing to proceed`)
+    }
+    const size = statSync(snapPath).size
+    if (size === 0) {
+      throw new Error(`snapshot for ${d} is zero bytes — refusing to proceed`)
+    }
+    log(`  ✓ ${basename(d)}: ${(size / 1e6).toFixed(2)} MB snapshot`)
+    state.perBrain[d].snapshotBytes = size
+  }
+  markComplete(state, 'verify-snapshot')
+}
+
+async function phasePauseWrites(state) {
+  if (!shouldRun(state, 'pause-writes')) return
+  log('phase 5/10: pause writes')
+  if (SYSTEMD_UNIT) {
+    log(`  stopping systemd unit: ${SYSTEMD_UNIT}`)
+    sh(`systemctl stop ${SYSTEMD_UNIT}`)
+    // Verify it's actually stopped.
+    if (!DRY_RUN) {
+      try {
+        const status = execSync(`systemctl is-active ${SYSTEMD_UNIT} 2>&1 || true`)
+          .toString()
+          .trim()
+        if (status === 'active') {
+          throw new Error(`systemctl reports ${SYSTEMD_UNIT} is still active after stop`)
+        }
+        log(`  ✓ ${SYSTEMD_UNIT} stopped (${status})`)
+        state.systemdUnit = SYSTEMD_UNIT
+      } catch (e) {
+        throw new Error(`failed to stop ${SYSTEMD_UNIT}: ${e.message}`)
+      }
+    }
+  } else {
+    log('  *** STOP YOUR SERVICE (or pause all writes) NOW ***')
+    log('      The next phase WIPES the on-disk format.')
+    log('      (Pass --systemd-unit NAME to have the script do this for you.)')
+    if (!confirm('Have writes been paused?')) {
+      throw new Error('operator did not confirm write pause')
+    }
+  }
+  markComplete(state, 'pause-writes')
+}
+
+async function phaseWipe(state) {
+  if (!shouldRun(state, 'wipe')) return
+  if (!state.lastCompletedPhase || PHASES.indexOf(state.lastCompletedPhase) < PHASES.indexOf('verify-snapshot')) {
+    throw new Error(
+      'REFUSING to wipe: fs-backup and verify-snapshot must have completed first',
+    )
+  }
+  log('phase 6/10: wipe cortex-managed subdirectories')
+  const CORTEX_SUBDIRS = ['_id_mapper', '_metadata', '_diskann', '_graph_adjacency']
+  for (const d of state.brainDirs) {
+    for (const sub of CORTEX_SUBDIRS) {
+      const path = join(d, sub)
+      if (existsSync(path)) {
+        log(`  rm -rf ${path}`)
+        if (!DRY_RUN) rmSync(path, { recursive: true, force: true })
+      }
+    }
+  }
+  markComplete(state, 'wipe')
+}
+
+async function phaseUpgradePrompt(state) {
+  if (!shouldRun(state, 'upgrade-prompt')) return
+  log('phase 7/10: upgrade dependencies')
+
+  if (APP_DIR) {
+    // Auto mode — figure out the package manager and run the upgrade.
+    const pm = PACKAGE_MANAGER ?? detectPackageManager(APP_DIR)
+    log(`  upgrading in ${APP_DIR} via ${pm}`)
+    const addCmd = pm === 'bun' ? 'bun add' : pm === 'yarn' ? 'yarn add' : 'npm install'
+    const cmd = `cd ${JSON.stringify(APP_DIR)} && ${addCmd} @soulcraft/brainy@${BRAINY_VERSION} @soulcraft/cor@${COR_VERSION}`
+    log(`  running: ${cmd}`)
+    sh(cmd)
+    // Verify the install.
+    if (!DRY_RUN) {
+      try {
+        const lsCmd = pm === 'bun' ? 'bun pm ls --json 2>/dev/null' : 'npm ls --json --depth=0 2>/dev/null'
+        const lsOut = execSync(`cd ${JSON.stringify(APP_DIR)} && ${lsCmd}`).toString()
+        const parsed = JSON.parse(lsOut)
+        const deps = parsed.dependencies ?? {}
+        const brainyResolved = deps['@soulcraft/brainy']?.version
+        const corResolved = deps['@soulcraft/cor']?.version
+        log(`  ✓ @soulcraft/brainy@${brainyResolved}`)
+        log(`  ✓ @soulcraft/cor@${corResolved}`)
+        if (!brainyResolved?.startsWith('8.') || !corResolved?.startsWith('3.')) {
+          throw new Error(
+            `upgrade did not land the expected majors. Got brainy@${brainyResolved} cor@${corResolved}. ` +
+              `Check your package.json constraints and try again.`,
+          )
+        }
+        state.brainyResolved = brainyResolved
+        state.corResolved = corResolved
+      } catch (e) {
+        throw new Error(`failed to verify dependency upgrade: ${e.message}`)
+      }
+    }
+  } else {
+    log('')
+    log('  Run the following in YOUR application directory (not this one):')
+    log('')
+    log(`    npm install @soulcraft/brainy@${BRAINY_VERSION} @soulcraft/cor@${COR_VERSION}`)
+    log('       (or `bun add` / `yarn add` equivalent)')
+    log('')
+    log('     (Pass --app-dir PATH to have the script do this for you.)')
+    log('')
+    if (!confirm('Have you upgraded @soulcraft/brainy and @soulcraft/cor to the new majors?')) {
+      throw new Error('operator did not confirm dependency upgrade')
+    }
+  }
+  markComplete(state, 'upgrade-prompt')
+}
+
+function detectPackageManager(appDir) {
+  if (existsSync(join(appDir, 'bun.lockb')) || existsSync(join(appDir, 'bun.lock'))) return 'bun'
+  if (existsSync(join(appDir, 'yarn.lock'))) return 'yarn'
+  return 'npm'  // package-lock.json or no lockfile → npm
+}
+
+async function phaseRestore(state) {
+  if (!shouldRun(state, 'restore')) return
+  log('phase 8/10: restore brains from logical snapshots')
+  // Per-tenant restores are independent (distinct brain dir + snapshot,
+  // isolated child process, no shared in-process state) so we run up to
+  // CONCURRENCY at once. Single-brain mode collapses to one item.
+  await runPool(state.brainDirs, CONCURRENCY, async (d) => {
+    const snapPath = state.perBrain[d]?.snapshotPath
+    log(`  restoring ${d} from ${snapPath}`)
+    sh(`node --input-type=module -e ${JSON.stringify(`
+      import { BrainyData } from '@soulcraft/brainy'
+      const b = new BrainyData(${newStackBrainyConfig(d)})
+      await b.init()
+      await b.restore(${JSON.stringify(snapPath)})
+      const stats = b.now().stats()
+      // Searchability smoke-check: a canonical count match does NOT prove the
+      // native indexes rebuilt — a silently-empty vector/column index passes
+      // the count check but serves nothing (the cor #44/#70 class). With cor
+      // plugged, restore() rebuilds the native indexes; confirm a query
+      // actually returns. (Rigorous per-index proof lives in cor's
+      // migrationRoundTrip.test.ts; this is the per-brain runtime gate.)
+      const probe = await b.find({ limit: 5 })
+      if ((stats.entityCount ?? 0) > 0 && probe.length === 0) {
+        throw new Error('post-restore searchability check FAILED for ' + ${JSON.stringify(d)} +
+          ': ' + stats.entityCount + ' entities restored but find() returned 0 — ' +
+          'native indexes did not rebuild. The pre-migration backup is intact; investigate before retrying.')
+      }
+      console.log(JSON.stringify(stats))
+      process.exit(0)
+    `)}`)
+  })
+  markComplete(state, 'restore')
+}
+
+async function phaseVerifyRestore(state) {
+  if (!shouldRun(state, 'verify-restore')) return
+  log('phase 9/10: verify restore parity')
+  if (DRY_RUN) {
+    log('  [dry-run] skipping restore parity check')
+    markComplete(state, 'verify-restore')
+    return
+  }
+  // Brain-by-brain: re-open the NEW stack, read entityCount, and compare
+  // against the authoritative OLD-stack count captured during phase 3.
+  // A restore that dropped or zeroed entities MUST fail here.
+  for (const d of state.brainDirs) {
+    const stats = entityCount(d, true)
+    log(`  ${basename(d)}: ${JSON.stringify(stats)}`)
+    if (stats === null) {
+      throw new Error(`failed to read stats for ${d}`)
+    }
+    const restoredCount = stats.entityCount
+    if (typeof restoredCount !== 'number') {
+      throw new Error(
+        `restored brain ${d} returned no entityCount (got ${JSON.stringify(stats)}); ` +
+          `cannot verify the restore — refusing to mark migration complete`,
+      )
+    }
+    // Hard floor: a zero-entity restore is always a failure (the OLD stack
+    // had entities — that is why we are migrating it).
+    if (restoredCount === 0) {
+      throw new Error(
+        `restored brain ${d} has ZERO entities — restore failed (DATA LOSS). ` +
+          `The pre-migration backup is intact; investigate before retrying.`,
+      )
+    }
+    const sourceCount = state.perBrain[d]?.sourceCount
+    if (typeof sourceCount === 'number') {
+      if (restoredCount !== sourceCount) {
+        throw new Error(
+          `restore parity FAILED for ${d}: OLD-stack had ${sourceCount} ` +
+            `entities, restored brain has ${restoredCount} (DATA LOSS). ` +
+            `The pre-migration backup is intact; do NOT delete it. ` +
+            `Investigate before retrying.`,
+        )
+      }
+      log(`  ✓ ${basename(d)}: parity OK (${restoredCount} entities)`)
+    } else {
+      // No baseline captured (e.g. resumed from a pre-fix state file).
+      // We still enforced the zero floor above; warn that strict parity
+      // could not be checked.
+      log(
+        `  ⚠ ${basename(d)}: ${restoredCount} entities restored, but no ` +
+          `OLD-stack baseline was captured — strict parity NOT verified`,
+      )
+    }
+    state.perBrain[d].restoredStats = stats
+    state.perBrain[d].restoredCount = restoredCount
+  }
+  markComplete(state, 'verify-restore')
+}
+
+async function phaseDone(state) {
+  if (!shouldRun(state, 'done')) return
+  log('phase 10/10: completion marker')
+
+  // Restart the service if the operator told us about it.
+  if (state.systemdUnit) {
+    log(`  starting systemd unit: ${state.systemdUnit}`)
+    sh(`systemctl start ${state.systemdUnit}`)
+    if (!DRY_RUN) {
+      const status = execSync(`systemctl is-active ${state.systemdUnit} 2>&1 || true`)
+        .toString()
+        .trim()
+      if (status !== 'active') {
+        log(`  ⚠ ${state.systemdUnit} did not become active after start (status: ${status})`)
+        log('    Check service logs immediately:')
+        log(`      journalctl -u ${state.systemdUnit} -n 100`)
+      } else {
+        log(`  ✓ ${state.systemdUnit} started`)
+      }
+    }
+  }
+
+  state.completedAt = new Date().toISOString()
+  markComplete(state, 'done')
+  log('')
+  log('Migration complete. Filesystem backup retained at:')
+  log(`  ${join(state.backupRoot, 'fs')}`)
+  log('Logical snapshots retained at:')
+  log(`  ${join(state.backupRoot, 'logical')}`)
+  log('')
+  log('You can delete those backups once the new stack has run in')
+  log('production for a few days with no rollback. State file:')
+  log(`  ${STATE_FILE}`)
+}
+
+// ---------------------------------------------------------------------------
+// Rollback
+// ---------------------------------------------------------------------------
+
+async function rollback() {
+  if (!existsSync(STATE_FILE)) {
+    errlog('no state file found; nothing to rollback')
+    process.exit(2)
+  }
+  const state = loadState()
+  const fsBackup = join(state.backupRoot, 'fs')
+  if (!existsSync(fsBackup)) {
+    errlog(`filesystem backup not found at ${fsBackup}; cannot rollback`)
+    process.exit(2)
+  }
+  log(`rolling back from ${fsBackup}`)
+  if (!confirm(
+    'This will OVERWRITE the current brain directories with the pre-migration backup. Continue?',
+  )) {
+    process.exit(1)
+  }
+
+  // Stop service if we know about it.
+  if (SYSTEMD_UNIT || state.systemdUnit) {
+    const unit = SYSTEMD_UNIT ?? state.systemdUnit
+    log(`  stopping ${unit}`)
+    sh(`systemctl stop ${unit}`)
+  }
+
+  // Restore filesystem backup.
+  for (const d of state.brainDirs) {
+    const src = join(fsBackup, basename(d))
+    log(`  rsync -a --delete ${src}/ ${d}/`)
+    sh(`rsync -a --delete ${JSON.stringify(src)}/ ${JSON.stringify(d)}/`)
+  }
+
+  // Downgrade dependencies if app-dir provided.
+  if (APP_DIR) {
+    const pm = PACKAGE_MANAGER ?? detectPackageManager(APP_DIR)
+    const addCmd = pm === 'bun' ? 'bun add' : pm === 'yarn' ? 'yarn add' : 'npm install'
+    const cmd = `cd ${JSON.stringify(APP_DIR)} && ${addCmd} @soulcraft/brainy@${ROLLBACK_BRAINY_VERSION} @soulcraft/cortex@${ROLLBACK_CORTEX_VERSION}`
+    log(`  downgrading deps: ${cmd}`)
+    sh(cmd)
+  }
+
+  // Restart service.
+  if (SYSTEMD_UNIT || state.systemdUnit) {
+    const unit = SYSTEMD_UNIT ?? state.systemdUnit
+    log(`  starting ${unit}`)
+    sh(`systemctl start ${unit}`)
+  }
+
+  log('rollback complete.')
+  if (!APP_DIR) {
+    log('Reminder: also downgrade @soulcraft/brainy + @soulcraft/cortex in your application.')
+  }
+  if (!SYSTEMD_UNIT && !state.systemdUnit) {
+    log('Reminder: also restart your service.')
+  }
+}
+
+function printHelp() {
+  console.log(`
+cortex 2.x → 3.0 migration tool
+
+USAGE:
+  node scripts/migrate-cortex-2x-to-3x.mjs --brain-dir DIR --backup-dir DIR [flags]
+  node scripts/migrate-cortex-2x-to-3x.mjs --tenants-dir DIR --backup-dir DIR [flags]
+
+REQUIRED:
+  --brain-dir PATH       Path to a single brain directory (single-tenant mode)
+  --tenants-dir PATH     Path to a directory containing per-tenant brain subdirs
+                         (multi-tenant mode)
+  --backup-dir PATH      Where to write the filesystem backup + logical snapshots
+
+OPTIONAL:
+  --concurrency N        Max per-tenant snapshots/restores run in parallel
+                         in multi-tenant (--tenants-dir) mode (default: 4).
+                         Each tenant runs in its own child process; a single
+                         tenant's failure fails the whole migration. Has no
+                         effect in single-brain (--brain-dir) mode. Values < 1
+                         or non-numeric fall back to sequential (1).
+  --online-snapshot      Skip the "have writes been paused" prompt before
+                         snapshot (only safe if your brainy version handles
+                         concurrent writes during persist correctly).
+                         UNSAFE for Venue and any production with live
+                         transactional writes (bookings, payments, etc.).
+
+AUTO-ORCHESTRATION (the "one button" mode):
+  --systemd-unit NAME    Stop NAME before wipe + start NAME after restore.
+                         (e.g. --systemd-unit venue-service)
+  --app-dir PATH         Run the dependency upgrade inside this directory.
+                         The script auto-detects npm/bun/yarn and runs the
+                         appropriate "add" command.
+  --package-manager PM   Override the auto-detection (npm | bun | yarn).
+  --brainy-version V     Target brainy version (default: ^8.0.0-rc.1).
+  --cor-version V        Target cor version (default: ^3.0.0-rc.1).
+
+OPTIONAL:
+  --auto-confirm         Skip all interactive prompts (for CI/scripted runs)
+  --dry-run              Print what would happen; make no changes
+  --resume               Pick up from the last completed phase
+                         (state at <root>/.cortex-3.0-migration-state.json)
+  --rollback             Restore from the filesystem backup and exit
+  --restart              Allow re-running after a completed migration
+                         (deletes the state file and starts over)
+  --rollback-brainy-version V    Version to downgrade to on rollback (default: 7.31.6)
+  --rollback-cortex-version V    Version to downgrade to on rollback (default: 2.7.0)
+
+EXAMPLE — Venue, fully automated one-button migration:
+  sudo node /opt/venue/node_modules/@soulcraft/cor/scripts/migrate-cortex-2x-to-3x.mjs \\
+    --brain-dir /opt/venue/data/brain \\
+    --backup-dir /opt/venue/backups/pre-3.0 \\
+    --app-dir /opt/venue \\
+    --systemd-unit venue-service
+
+  (Script stops venue-service, snapshots, upgrades npm deps, restores,
+   restarts venue-service. ~10 min total wall-clock, ~2 min downtime.)
+
+EXAMPLE — multi-tenant Memory deployment, fully automated:
+  sudo node /opt/memory/node_modules/@soulcraft/cor/scripts/migrate-cortex-2x-to-3x.mjs \\
+    --tenants-dir /opt/memory/data/memory/users \\
+    --backup-dir /opt/memory/backups/pre-3.0 \\
+    --app-dir /opt/memory \\
+    --systemd-unit memory-service \\
+    --concurrency 8
+
+EXAMPLE — manual mode (you control service stop/start and npm install):
+  node scripts/migrate-cortex-2x-to-3x.mjs \\
+    --brain-dir /opt/venue/data/brain \\
+    --backup-dir /opt/venue/backups/pre-3.0
+
+EXAMPLE — rollback after a failed migration:
+  sudo node scripts/migrate-cortex-2x-to-3x.mjs \\
+    --brain-dir /opt/venue/data/brain \\
+    --backup-dir /opt/venue/backups/pre-3.0 \\
+    --app-dir /opt/venue \\
+    --systemd-unit venue-service \\
+    --rollback
+`)
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+  if (hasFlag('--help') || hasFlag('-h')) {
+    printHelp()
+    process.exit(0)
+  }
+
+  if (ROLLBACK) {
+    await rollback()
+    return
+  }
+
+  if (RESTART && existsSync(STATE_FILE)) {
+    rmSync(STATE_FILE)
+    log('state file deleted; starting fresh')
+  }
+
+  let state = loadState()
+  if (state.lastCompletedPhase === 'done' && !RESTART) {
+    errlog('migration already complete. Pass --restart to re-run.')
+    process.exit(1)
+  }
+
+  log(`migration plan: ${PHASES.join(' → ')}`)
+  if (state.lastCompletedPhase) {
+    log(`resuming from after phase: ${state.lastCompletedPhase}`)
+  }
+  log('')
+
+  try {
+    await phasePreflight(state)
+    await phaseFsBackup(state)
+    await phaseLogicalSnapshot(state)
+    await phaseVerifySnapshot(state)
+    await phasePauseWrites(state)
+    await phaseWipe(state)
+    await phaseUpgradePrompt(state)
+    await phaseRestore(state)
+    await phaseVerifyRestore(state)
+    await phaseDone(state)
+  } catch (e) {
+    errlog(`FAILED at phase after ${state.lastCompletedPhase ?? 'startup'}: ${e.message}`)
+    errlog(`state saved to ${STATE_FILE}`)
+    errlog(`run with --resume to retry, or --rollback to revert`)
+    process.exit(1)
+  }
+}
+
+main().catch((e) => {
+  errlog(e.stack ?? e.message)
+  process.exit(1)
+})