@shawnstack/quickforge 1.7.9 → 1.7.11

package/server/index.mjs

@@ -6,9 +6,9 @@ import { randomUUID } from 'node:crypto'
 import { fileURLToPath } from 'node:url'
 import { sendJson, sendError } from './utils/response.mjs'
 import { openBrowser, openPathInFileManager } from './utils/platform.mjs'
-import { ensureStorage, dataDir, configDir, storageDir, cacheDir, logsDir, readPhysicalSessionMetadataBuckets, registerSessionMetadataCommitHook } from './storage.mjs'
-import { initializeSessionStateCutover } from './session-state-cutover.mjs'
-import { initializeSessionStateService, drainSessionJsonMirror, stopSessionStateService } from './session-state-service.mjs'
+import { ensureStorage, dataDir, configDir, storageDir, cacheDir, logsDir, createPhysicalSessionStateFsAdapter } from './storage.mjs'
+import { initializeSessionStateService, stopSessionStateService } from './session-state-service.mjs'
+import { importSessionStateFromJson } from './session-state-import.mjs'
 import { initializeShareCutover } from './share-cutover.mjs'
 import { drainShareJsonMirror, initializeShareService, stopShareService } from './share-service.mjs'
 import { initializeLanAccessCutover } from './lan-access-cutover.mjs'
@@ -57,12 +57,13 @@ import { shutdown as shutdownAgentManager, resetStaleTaskStatuses } from './agen
 import { initializeChannels, shutdownChannels } from './channels/registry.mjs'
 import { shutdownMcpConnections } from './mcp/registry.mjs'
 import { shutdownTerminalSessions } from './terminal/terminal-manager.mjs'
-import { closeSqliteStorage, getSqliteStorageSummary, initializeSqliteStorage } from './sqlite/database.mjs'
+import { closeSqliteStorage, getSqliteStorage, getSqliteStorageSummary, initializeSqliteStorage } from './sqlite/database.mjs'
 import { initializeScheduledRunsCutover } from './scheduled-runs-cutover.mjs'
 import { recoverScheduledRunsRestorePlan } from './scheduled-runs-backup.mjs'
-import { configureSessionIndex, getSessionIndexDiagnostics, initializeSessionIndex, syncSessionMetadataCommit } from './session-index-service.mjs'
+import { getSessionIndexDiagnostics, initializeSessionIndex } from './session-index-service.mjs'
 import { createNodeProcessEnv } from './utils/process-env.mjs'
 import { isAuthenticatedAppClient } from './access-policy.mjs'
+import { STARTUP_STATES, getStartupError, getStartupState, readMigrationStatus, resolveMaintenanceGate, setStartupState, withStartupRecoveryGuidance } from './startup-state.mjs'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
@@ -87,6 +88,7 @@ let shutdownPromise = null
 let shutdownStarted = false
 let qfAgentStartPromise = null
 let boundServerPort = null
+let startupInitializationPromise = null
 
 setDefaultWorkspaceRoot(process.env.QUICKFORGE_WORKSPACE_DIR || path.join(dataDir, 'workspace'))
 
@@ -368,10 +370,36 @@ async function handleApi(req, res, url, requestContext = {}) {
 
   // Health check
   if (req.method === 'GET' && pathname === '/api/health') {
+    const startupState = getStartupState()
+    if (startupState === STARTUP_STATES.FAILED) {
+      // Startup failed but the process stays alive: waitForQuickForge keeps
+      // polling (payload.ok is false) until its timeout, preserving the
+      // "startup failed" semantics for CLI/Desktop.
+      sendJson(res, 200, { ok: false, startupError: getStartupError() })
+      return
+    }
+    if (startupState === STARTUP_STATES.MIGRATING) {
+      // Maintenance window: keep the existing fields (pid, sqlite summary,
+      // ...) so clients can bind early, plus maintenance:true for the UI.
+      // Fall back to the minimal shape if the full status is not readable.
+      try {
+        sendJson(res, 200, { maintenance: true, ...(await getSystemStatus(requestContext)) })
+        return
+      } catch {
+        sendJson(res, 200, { ok: true, maintenance: true })
+        return
+      }
+    }
     sendJson(res, 200, await getSystemStatus(requestContext))
     return
   }
 
+  // Migration progress during the startup maintenance window
+  if (req.method === 'GET' && pathname === '/api/migration-status') {
+    sendJson(res, 200, readMigrationStatus())
+    return
+  }
+
   if (req.method === 'GET' && pathname === '/api/project/commands') {
     await handleProjectApi(req, res, url, requestContext)
     return
@@ -707,6 +735,20 @@ const server = createServer(async (req, res) => {
     }
 
     if (url.pathname.startsWith('/api/')) {
+      // Startup maintenance gate: while the background startup chain runs (or
+      // failed), refuse every business API except the whitelisted readiness
+      // endpoints. Non-/api paths (static assets, /share/) stay ungated so the
+      // frontend can load and show migration progress.
+      const gate = resolveMaintenanceGate(req.method, url.pathname)
+      if (gate) {
+        res.writeHead(gate.status, {
+          'content-type': 'application/json; charset=utf-8',
+          'cache-control': 'no-store',
+          'retry-after': '5',
+        })
+        res.end(JSON.stringify({ ok: false, maintenance: true, state: gate.state }))
+        return
+      }
       await handleApi(req, res, url, {
         isLocalRequest: !isRemoteRequest,
         remoteAddress,
@@ -775,51 +817,124 @@ server.on('upgrade', (req, socket, head) => {
   }
 })
 
-await ensureStorage()
-await initializeSqliteStorage({ dataDir })
-await initializeScheduledRunsCutover()
-await recoverScheduledRunsRestorePlan()
-await recoverStaleScheduledTaskRuns()
-// Session state cutover: authoritative integrity failures fail closed and
-// block startup; json_authoritative failures keep the legacy JSON path.
-await initializeSessionStateCutover()
-await initializeSessionStateService()
-await recoverSessionStateRestorePlan()
-await drainSessionJsonMirror()
-// Share storage cutover: share-store writes become SQLite-authoritative once
-// pending/authoritative; integrity failures fail closed and block startup,
-// json_authoritative failures keep the legacy JSON path. The JSON file stays
-// as the best-effort mirror drained through the share mirror queue.
-await initializeShareCutover()
-await initializeShareService()
-await recoverShareRestorePlan()
-await drainShareJsonMirror()
-// LAN access storage cutover: JSON → SQLite with the same phase machine as
-// share storage. Integrity failures while pending/authoritative fail closed
-// and block startup; json_authoritative failures keep the legacy JSON store
-// path. The JSON file stays as the best-effort mirror drained through the
-// lan-access mirror queue. Interrupted lan-access restore plans are recovered
-// (roll-forward/rollback) before the mirror drain.
-await initializeLanAccessCutover()
-await initializeLanAccessService()
-await recoverLanAccessRestorePlan()
-await drainLanAccessJsonMirror()
-await ensureDefaultGlobalSkills()
-await initializeNetworkProxy()
-installAiHttpLogger()
-initializeChannels({
-  projectRoot,
-  channelEventsUrl: `http://127.0.0.1:${port}/api/channels/events`,
-  logsDir,
-})
-await resetStaleTaskStatuses()
-configureSessionIndex({ readBuckets: readPhysicalSessionMetadataBuckets })
-registerSessionMetadataCommitHook(syncSessionMetadataCommit)
-await initializeSessionIndex()
-await initializeActiveProject()
-setActiveWorkspaceRootForFilesystem(getWorkspaceRoot())
-startScheduledTaskRunner()
-startAutoArchiveRunner()
+// --- Startup chain (P1: listen early + maintenance window) ---
+// Storage + SQLite initialization stay in front of listen (the maintenance
+// gate and /api/migration-status rely on the database). The remaining
+// migration/initialization chain runs in the background behind the gate
+// while /api/health and /api/migration-status already answer and the
+// frontend page is served, so the UI can display migration progress.
+//
+// Fail-closed semantics changed from "abort the process" to "stay alive and
+// refuse business APIs": on failure the startup state becomes 'failed',
+// /api/health answers {ok:false, startupError} (waitForQuickForge keeps
+// polling until its timeout, preserving the "startup failed" result) and
+// every non-whitelisted /api/* route keeps answering 503 maintenance until
+// the operator intervenes (fix data / restart).
+try {
+  await ensureStorage()
+  await initializeSqliteStorage({ dataDir })
+} catch (error) {
+  logger.error('QuickForge startup failed', { errorName: error?.name || 'Error', errorMessage: error?.message, stack: error?.stack })
+  flushLogger()
+  setStartupState(STARTUP_STATES.FAILED, withStartupRecoveryGuidance(error?.message))
+}
+
+// Startup-chain step timing: logs any single initialization step that takes
+// >= 500ms with its duration, so slow steps on real machines (AV scanning,
+// large libraries, network calls) surface directly in server logs instead of
+// hiding inside the listen→complete gap.
+const STARTUP_STEP_SLOW_MS = 500
+async function timedStartupStep(label, fn) {
+  const start = Date.now()
+  try {
+    return await fn()
+  } finally {
+    const durationMs = Date.now() - start
+    if (durationMs >= STARTUP_STEP_SLOW_MS) {
+      logger.info('Startup step slow', { startupStep: label, durationMs })
+    }
+  }
+}
+
+async function runStartupInitialization() {
+  await timedStartupStep('scheduled-runs-cutover', () => initializeScheduledRunsCutover())
+  await timedStartupStep('scheduled-runs-restore-plan', () => recoverScheduledRunsRestorePlan())
+  await timedStartupStep('stale-scheduled-task-runs', () => recoverStaleScheduledTaskRuns())
+  // Session state (storage v2): SQLite is authoritative by construction. The
+  // only startup work is the one-shot JSON import — when the v2 store is empty
+  // and legacy JSON session files exist, importSessionStateFromJson re-imports
+  // them (idempotent, per-session transactions, WAL checkpoint at the end);
+  // otherwise the store is ready as-is. The import runs inside the maintenance
+  // window, so /api/* stays gated (process state 'migrating') until it
+  // completes; the sessionState domain reports 'authoritative' throughout.
+  await timedStartupStep('session-state-service', () => initializeSessionStateService())
+  await timedStartupStep('session-state-import', async () => {
+    const { createSessionStateRepository } = await import('./sqlite/session-state-repository.mjs')
+    const repository = createSessionStateRepository(getSqliteStorage())
+    if (repository.count() > 0) return
+    const adapter = createPhysicalSessionStateFsAdapter()
+    let hasJsonSessions = false
+    for await (const bucket of adapter.listBuckets()) {
+      for await (const sessionId of adapter.listSessionFiles(bucket)) {
+        if (sessionId) {
+          hasJsonSessions = true
+          break
+        }
+      }
+      if (hasJsonSessions) break
+    }
+    if (!hasJsonSessions) return
+    logger.info('Session state JSON import starting (empty SQLite store, legacy files detected)', { domain: 'session-state' })
+    const { imported, skipped } = await importSessionStateFromJson({ logger })
+    logger.info('Session state JSON import finished', { domain: 'session-state', imported, skipped })
+  })
+  await timedStartupStep('session-state-restore-plan', () => recoverSessionStateRestorePlan())
+  // Share storage cutover: share-store writes become SQLite-authoritative once
+  // pending/authoritative; integrity failures fail closed and block startup,
+  // json_authoritative failures keep the legacy JSON path. The JSON file stays
+  // as the best-effort mirror drained through the share mirror queue.
+  await timedStartupStep('share-cutover', () => initializeShareCutover())
+  await timedStartupStep('share-service', () => initializeShareService())
+  await timedStartupStep('share-restore-plan', () => recoverShareRestorePlan())
+  await timedStartupStep('share-json-mirror-drain', () => drainShareJsonMirror())
+  // LAN access storage cutover: JSON → SQLite with the same phase machine as
+  // share storage. Integrity failures while pending/authoritative fail closed
+  // and block startup; json_authoritative failures keep the legacy JSON store
+  // path. The JSON file stays as the best-effort mirror drained through the
+  // lan-access mirror queue. Interrupted lan-access restore plans are recovered
+  // (roll-forward/rollback) before the mirror drain.
+  await timedStartupStep('lan-access-cutover', () => initializeLanAccessCutover())
+  await timedStartupStep('lan-access-service', () => initializeLanAccessService())
+  await timedStartupStep('lan-access-restore-plan', () => recoverLanAccessRestorePlan())
+  await timedStartupStep('lan-access-json-mirror-drain', () => drainLanAccessJsonMirror())
+  await timedStartupStep('default-global-skills', () => ensureDefaultGlobalSkills())
+  await timedStartupStep('network-proxy', () => initializeNetworkProxy())
+  installAiHttpLogger()
+  initializeChannels({
+    projectRoot,
+    channelEventsUrl: `http://127.0.0.1:${port}/api/channels/events`,
+    logsDir,
+  })
+  await timedStartupStep('reset-stale-task-statuses', () => resetStaleTaskStatuses())
+  await timedStartupStep('session-index', () => initializeSessionIndex())
+  await timedStartupStep('active-project', () => initializeActiveProject())
+  setActiveWorkspaceRootForFilesystem(getWorkspaceRoot())
+  startScheduledTaskRunner()
+  startAutoArchiveRunner()
+}
+
+if (getStartupState() !== STARTUP_STATES.FAILED) {
+  // Fire-and-forget: never blocks listen; the promise always settles and the
+  // handlers below flip the startup state for the gate and health endpoint.
+  startupInitializationPromise = runStartupInitialization().then(() => {
+    setStartupState(STARTUP_STATES.READY)
+    logger.info('QuickForge startup initialization complete.')
+  }).catch((error) => {
+    logger.error('QuickForge startup failed', { errorName: error?.name || 'Error', errorMessage: error?.message, stack: error?.stack })
+    flushLogger()
+    setStartupState(STARTUP_STATES.FAILED, withStartupRecoveryGuidance(error?.message))
+  })
+}
 
 server.on('error', (error) => {
   // Handle listen errors (most commonly EADDRINUSE). Without this, Node would
@@ -847,7 +962,13 @@ server.listen(port, host, () => {
   logger.info(`QuickForge data dir: ${dataDir}`)
   logger.info(`QuickForge project: ${getWorkspaceRoot()}`)
 
-  const pending = readCloudServiceConfig({ strict: false }).then((cloudConfig) => applyCloudServiceConfig(cloudConfig)).catch((error) => {
+  const pending = (async () => {
+    // The remote agent talks to the local business API: hold it back until the
+    // background startup chain settles so it never races the maintenance gate.
+    if (startupInitializationPromise) await startupInitializationPromise
+    if (getStartupState() === STARTUP_STATES.FAILED) return null
+    return applyCloudServiceConfig(await readCloudServiceConfig({ strict: false }))
+  })().catch((error) => {
     if (!shutdownStarted) logger.warn(`QuickForge remote agent failed to start: ${error?.message || error}`)
     return null
   }).finally(() => {

package/server/lan-access-cutover.mjs

@@ -19,6 +19,7 @@ import {
   writeLanAccessJsonFile,
 } from './lan-access-json-file.mjs'
 import { storageDir } from './storage.mjs'
+import { logger } from './utils/logger.mjs'
 
 const DEFAULT_LOCK_TTL_MS = 60_000
 const DEFAULT_WAIT_TIMEOUT_MS = 65_000
@@ -223,23 +224,29 @@ export async function initializeLanAccessCutover(options = {}) {
   const storage = options.storage || getSqliteStorage()
   const repository = options.repository || createLanAccessRepository(storage)
   configureLanAccessService({ repository, mirror: options.mirror || createDefaultLanAccessMirror() })
-  return runLanAccessMaintenance(async () => {
+  const log = options.logger || logger
+  const migrate = async () => {
     const current = readLanAccessStorageState()
     if (current.phase === LAN_ACCESS_STORAGE_PHASES.JSON_PENDING) {
       const integrity = repository.verifyIntegrity({ quickCheck: true })
       if (!integrity.ok) throw new Error('LAN access state pending integrity verification failed')
       const drained = await drainLanAccessJsonMirror()
-      if (drained.pending === 0) setLanAccessStoragePhase(LAN_ACCESS_STORAGE_PHASES.AUTHORITATIVE, { lanTokenCount: integrity.count, digest: integrity.digest, backupFile: current.backupFile })
+      if (drained.pending === 0) {
+        setLanAccessStoragePhase(LAN_ACCESS_STORAGE_PHASES.AUTHORITATIVE, { lanTokenCount: integrity.count, digest: integrity.digest, backupFile: current.backupFile })
+        log.info('LAN access cutover promoted to authoritative', { domain: 'lan-access', phase: 'authoritative', tokenCount: integrity.count })
+      }
       return readLanAccessStorageState()
     }
     if (current.phase === LAN_ACCESS_STORAGE_PHASES.AUTHORITATIVE) {
       const integrity = repository.verifyIntegrity({ quickCheck: true })
       if (!integrity.ok) throw new Error('LAN access state authoritative integrity verification failed')
       await drainLanAccessJsonMirror()
+      log.info('LAN access cutover startup check passed', { domain: 'lan-access', phase: 'authoritative' })
       return readLanAccessStorageState()
     }
 
     if (current.phase === LAN_ACCESS_STORAGE_PHASES.CUTOVER_RUNNING && current.backupFile) {
+      log.warn('LAN access cutover recovery: rerunning the JSON migration', { domain: 'lan-access', phase: 'cutover_running', backupFile: current.backupFile })
       setLanAccessStoragePhase(LAN_ACCESS_STORAGE_PHASES.JSON_AUTHORITATIVE, {
         backupFile: current.backupFile,
         diagnostic: { operation: 'cutover_recovery', recoveredFrom: LAN_ACCESS_STORAGE_PHASES.CUTOVER_RUNNING },
@@ -254,6 +261,7 @@ export async function initializeLanAccessCutover(options = {}) {
       if (first.tokenCount !== second.tokenCount || first.digest !== second.digest) {
         throw new Error('LAN access JSON source changed during cutover double read')
       }
+      log.info('LAN access cutover migration started', { domain: 'lan-access', tokenCount: first.tokenCount })
       if (!backupFile) backupFile = await writeLanAccessCutoverBackup(first, options.backupDirectory)
       const third = buildLanAccessJsonSnapshot(await readJson())
       if (first.tokenCount !== third.tokenCount || first.digest !== third.digest) {
@@ -290,7 +298,9 @@ export async function initializeLanAccessCutover(options = {}) {
           diagnostic: first.diagnostics,
         })
       }
-      return readLanAccessStorageState()
+      const result = readLanAccessStorageState()
+      log.info('LAN access cutover migration complete', { domain: 'lan-access', phase: result.phase, tokenCount: first.tokenCount })
+      return result
     } catch (error) {
       const state = readLanAccessStorageState()
       if (![LAN_ACCESS_STORAGE_PHASES.JSON_PENDING, LAN_ACCESS_STORAGE_PHASES.AUTHORITATIVE].includes(state.phase)) {
@@ -298,9 +308,16 @@ export async function initializeLanAccessCutover(options = {}) {
           backupFile,
           diagnostic: { operation: 'cutover', errorName: error?.name || 'Error', error: error?.message || String(error) },
         })
+        log.warn('LAN access cutover failed; keeping the JSON store path', { domain: 'lan-access', phase: 'json_authoritative', errorName: error?.name || 'Error', errorMessage: error?.message })
         return readLanAccessStorageState()
       }
       throw error
     }
-  }, { ...options, storage, operation: 'lan-access-cutover' })
+  }
+  try {
+    return await runLanAccessMaintenance(migrate, { ...options, storage, operation: 'lan-access-cutover' })
+  } catch (error) {
+    log.error('LAN access cutover failed and blocked startup', { domain: 'lan-access', errorName: error?.name || 'Error', errorMessage: error?.message })
+    throw error
+  }
 }

package/server/maintenance/downgrade-session-state-v1.mjs

@@ -1,126 +1,89 @@
 #!/usr/bin/env node
-import { createHash } from 'node:crypto'
+// Offline downgrade escape hatch (storage v2): materializes the authoritative
+// SQLite session store back into the legacy v1 JSON layout (per-session body
+// files + per-bucket sessions-metadata.json). The old phase-machine semantics
+// are gone — SQLite stays authoritative and this tool NEVER flips authority;
+// it only exports data so an older QuickForge build (or a human) can read it.
+//
+//   node downgrade-session-state-v1.mjs --dry-run   # report only, no writes
+//   node downgrade-session-state-v1.mjs             # write the JSON layout
 import path from 'node:path'
 import { closeSqliteStorage, initializeSqliteStorage } from '../sqlite/database.mjs'
-import { createSessionStateRepository, snapshotDigestLine } from '../sqlite/session-state-repository.mjs'
-import { buildSessionJsonSnapshot } from '../session-state-cutover.mjs'
-import {
-  SESSION_STORAGE_PHASES,
-  configureSessionStateService,
-  drainSessionJsonMirror,
-  readSessionStorageState,
-  setSessionStoragePhase,
-} from '../session-state-service.mjs'
+import { createSessionStateRepository } from '../sqlite/session-state-repository.mjs'
 import {
+  createPhysicalSessionStateFsAdapter,
   dataDir,
-  materializeSessionJsonMirrorEntry,
-  readPhysicalSessionStateBuckets,
+  removeSessionStateJsonBody,
+  writeSessionStateJsonBody,
+  writeSessionStateMetadataBucket,
 } from '../storage.mjs'
 
-const AUTHORITATIVE_PHASES = new Set(['sqlite_authoritative_json_pending', 'authoritative'])
-
 const args = new Set(process.argv.slice(2).filter((arg) => arg.startsWith('--')))
 const dryRun = args.has('--dry-run')
-const commit = args.has('--commit')
-
-function mirrorAdapter() {
-  async function materialize(entry) {
-    await materializeSessionJsonMirrorEntry(entry)
-  }
-  return { upsert: materialize, delete: materialize }
-}
 
 function exitError(message) {
   process.stderr.write(`${message}\n`)
   process.exitCode = 1
 }
 
-function isPlainObject(value) {
-  return Boolean(value) && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype
-}
-
-function canonicalize(value) {
-  if (Array.isArray(value)) return value.map(canonicalize)
-  if (!isPlainObject(value)) return value
-  return Object.fromEntries(Object.keys(value).sort().filter((key) => value[key] !== undefined).map((key) => [key, canonicalize(value[key])]))
-}
-
-function digestJson(value) {
-  return createHash('sha256').update(JSON.stringify(canonicalize(value))).digest('hex')
-}
-
-// F9 split sessions: the downgrade materializes the JSON mirror as full bodies
-// (split marker + messages inline) via drainSessionJsonMirror, so the SQLite
-// side must be digested over the ASSEMBLED representation — not the stored
-// stripped body + messages digest — to compare against buildSessionJsonSnapshot.
-function assembledSnapshotDigest(snapshot) {
-  const rows = snapshot.records.map((record) => {
-    const state = record.state?.messageStorage === 'split'
-      ? { ...record.state, messages: record.messages ?? [] }
-      : record.state
-    return snapshotDigestLine(record.scope, record.projectId || '', record.sessionId, digestJson(state), digestJson(record.metadata), '')
-  })
-  return createHash('sha256').update(rows.sort().join('\n')).digest('hex')
-}
-
 try {
   const storage = await initializeSqliteStorage({ dataDir })
   storage.health({ quickCheck: true })
   const repository = createSessionStateRepository(storage)
-  configureSessionStateService({ repository, mirror: mirrorAdapter() })
-  const state = readSessionStorageState()
-  if (state.phase === 'cutover_running') {
-    throw new Error('Session state cutover is still running; stop all QuickForge processes and retry')
-  }
-  if (state.phase === SESSION_STORAGE_PHASES.JSON_AUTHORITATIVE) {
-    throw new Error('Session state is already JSON authoritative; nothing to downgrade')
-  }
-  if (!AUTHORITATIVE_PHASES.has(state.phase)) {
-    throw new Error(`Session state is ${state.phase}; downgrade requires pending or authoritative phase`)
-  }
   const integrity = repository.verifyIntegrity({ quickCheck: true })
   if (!integrity.ok) throw new Error('Session state integrity verification failed')
   const snapshot = repository.exportSnapshot()
-  if (snapshot.count !== integrity.count || snapshot.digest !== integrity.digest) {
-    throw new Error('Session state snapshot count/digest verification failed')
+  // Lightweight integrity has no digest; the count cross-check plus the
+  // snapshot's own digest stay authoritative.
+  if (snapshot.count !== integrity.count) {
+    throw new Error('Session state snapshot count verification failed')
   }
+
+  // Reassemble every record into a full v1 JSON body (split sessions carry
+  // their messages inline again) and group metadata per bucket.
+  const bodiesByBucket = new Map()
+  const metadataByBucket = new Map()
+  for (const record of snapshot.records) {
+    const bucket = record.scope === 'project' ? { scope: 'project', projectId: record.projectId } : { scope: 'global' }
+    const key = `${bucket.scope}\0${bucket.projectId || ''}`
+    const state = record.state?.messageStorage === 'split'
+      ? { ...record.state, messages: record.messages ?? [] }
+      : record.state
+    if (!bodiesByBucket.has(key)) {
+      bodiesByBucket.set(key, { bucket, bodies: new Map() })
+      metadataByBucket.set(key, { bucket, metadata: {} })
+    }
+    bodiesByBucket.get(key).bodies.set(record.sessionId, state)
+    metadataByBucket.get(key).metadata[record.sessionId] = record.metadata
+  }
+
   const report = {
     ok: true,
-    phase: state.phase,
     count: snapshot.count,
     digest: snapshot.digest,
     dryRun,
-    commit,
     sessionsDirectory: path.join(dataDir, 'storage', 'conversations'),
+    message: 'SQLite stays authoritative; the JSON layout is an exported copy (escape hatch), not a phase switch.',
   }
 
   if (dryRun) {
-    // Read-only: report what a real run would materialize without writing
-    // anything or changing the phase.
     process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
   } else {
-    await drainSessionJsonMirror()
-    // Verify the on-disk JSON mirror exactly matches the authoritative SQLite
-    // snapshot before allowing any phase change. Split sessions are compared
-    // over their assembled representation (full body with messages), which is
-    // what the JSON mirror materializes.
-    const mirror = buildSessionJsonSnapshot(await readPhysicalSessionStateBuckets())
-    const targetDigest = assembledSnapshotDigest(snapshot)
-    if (mirror.count !== snapshot.count || mirror.digest !== targetDigest) {
-      throw new Error(`JSON mirror verification failed: SQLite ${snapshot.count}/${targetDigest} vs JSON ${mirror.count}/${mirror.digest}`)
+    const adapter = createPhysicalSessionStateFsAdapter()
+    for (const { bucket, bodies } of bodiesByBucket.values()) {
+      // Remove stale body files that no longer exist in the store before
+      // writing the fresh snapshot.
+      for await (const sessionId of adapter.listSessionFiles(bucket)) {
+        if (!bodies.has(sessionId)) await removeSessionStateJsonBody(bucket, sessionId)
+      }
+      for (const [sessionId, state] of bodies) {
+        await writeSessionStateJsonBody(bucket, sessionId, state)
+      }
     }
-    if (commit) {
-      setSessionStoragePhase(SESSION_STORAGE_PHASES.JSON_AUTHORITATIVE, {
-        stateCount: snapshot.count,
-        digest: snapshot.digest,
-        diagnostic: { operation: 'downgrade', materialized: mirror.count },
-      })
-      report.phase = SESSION_STORAGE_PHASES.JSON_AUTHORITATIVE
-      report.phaseChanged = true
-    } else {
-      report.phaseChanged = false
-      report.message = 'JSON mirror materialized; run with --commit after stopping all QuickForge processes to switch authority back to JSON'
+    for (const { bucket, metadata } of metadataByBucket.values()) {
+      await writeSessionStateMetadataBucket(bucket, metadata)
     }
+    report.materialized = snapshot.count
     process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
   }
 } catch (error) {

package/server/maintenance/export-session-state-v1.mjs

@@ -4,11 +4,8 @@ import { promises as fs } from 'node:fs'
 import path from 'node:path'
 import { closeSqliteStorage, initializeSqliteStorage } from '../sqlite/database.mjs'
 import { createSessionStateRepository } from '../sqlite/session-state-repository.mjs'
-import { readSessionStorageState } from '../session-state-service.mjs'
 import { dataDir } from '../storage.mjs'
 
-const AUTHORITATIVE_PHASES = new Set(['sqlite_authoritative_json_pending', 'authoritative'])
-
 function outputPath() {
   const argument = process.argv[2]
   if (argument) return path.resolve(argument)
@@ -23,18 +20,13 @@ try {
   const storage = await initializeSqliteStorage({ dataDir })
   storage.health({ quickCheck: true })
   const repository = createSessionStateRepository(storage)
-  const state = readSessionStorageState()
-  if (state.phase === 'cutover_running') {
-    throw new Error('Session state cutover is still running; stop all QuickForge processes and retry')
-  }
-  if (!AUTHORITATIVE_PHASES.has(state.phase)) {
-    throw new Error(`Session state is ${state.phase}; authoritative export requires the SQLite cutover to be complete`)
-  }
   const integrity = repository.verifyIntegrity({ quickCheck: true })
   if (!integrity.ok) throw new Error('Session state integrity verification failed')
   const snapshot = repository.exportSnapshot()
-  if (snapshot.count !== integrity.count || snapshot.digest !== integrity.digest) {
-    throw new Error('Session state backup count/digest verification failed')
+  // Lightweight integrity has no digest; the count cross-check plus the
+  // snapshot's own digest stay authoritative.
+  if (snapshot.count !== integrity.count) {
+    throw new Error('Session state backup count verification failed')
   }
   const backup = {
     app: 'quickforge',
@@ -42,7 +34,7 @@ try {
     exportedAt: new Date().toISOString(),
     scope: 'sessions',
     includeSecrets: false,
-    sessionState: { phase: state.phase, count: snapshot.count, digest: snapshot.digest },
+    sessionState: { phase: 'authoritative', count: snapshot.count, digest: snapshot.digest },
     data: {
       sessions: Object.fromEntries(snapshot.records.map((record) => [record.sessionId, record.state?.messageStorage === 'split'
         ? { ...record.state, messages: record.messages ?? [] }

package/server/routes/backup.mjs

@@ -5,7 +5,7 @@ import { isAuthenticatedAppClient } from '../access-policy.mjs'
 import { isScheduledRunsMaintenanceActive } from '../scheduled-runs-cutover.mjs'
 import { readScheduledTasksForBackup, restoreScheduledTasks } from '../scheduled-runs-backup.mjs'
 import { isSessionStateAuthoritative } from '../session-state-service.mjs'
-import { isSessionStateMaintenanceActive } from '../session-state-cutover.mjs'
+import { isSessionStateMaintenanceActive } from '../session-state-maintenance.mjs'
 import { exportSessionStateForBackup, restoreSessionStateSnapshot } from '../session-state-backup.mjs'
 import { isShareStorageAuthoritative } from '../share-service.mjs'
 import { isShareMaintenanceActive } from '../share-cutover.mjs'
@@ -15,6 +15,8 @@ import { isLanAccessStorageAuthoritative } from '../lan-access-service.mjs'
 import { isLanAccessMaintenanceActive } from '../lan-access-cutover.mjs'
 import { exportLanAccessStateForBackup, restoreLanAccessStateSnapshot } from '../lan-access-backup.mjs'
 import { readLanAccessJsonFile, writeLanAccessJsonFile } from '../lan-access-json-file.mjs'
+import { refreshAllSessionModels } from '../agent-manager.mjs'
+import { logger } from '../utils/logger.mjs'
 import { sendJson, readJsonBody } from '../utils/response.mjs'
 import {
   ensureStorage,
@@ -641,6 +643,11 @@ async function restoreValidatedBackup(backup, mode = 'replace') {
   if (sections.customProviders !== undefined) {
     const value = merge ? mergeRecordStore(await readStore('custom-providers'), sections.customProviders) : sections.customProviders
     await writeStore('custom-providers', value)
+    try {
+      await refreshAllSessionModels()
+    } catch (error) {
+      logger.error('Failed to refresh session models after restoring custom providers:', error)
+    }
     summary.customProviders = countKeys(value)
   }