@shawnstack/quickforge 1.7.9 → 1.7.10

package/server/session-state-cutover.mjs

@@ -1,6 +1,6 @@
 import { AsyncLocalStorage } from 'node:async_hooks'
 import { createHash, randomUUID } from 'node:crypto'
-import { promises as fs } from 'node:fs'
+import { createReadStream, createWriteStream, promises as fs } from 'node:fs'
 import path from 'node:path'
 import { getSqliteStorage } from './sqlite/database.mjs'
 import { createSessionStateRepository, snapshotDigestLine } from './sqlite/session-state-repository.mjs'
@@ -12,8 +12,8 @@ import {
   setSessionStoragePhase,
 } from './session-state-service.mjs'
 import {
+  createPhysicalSessionStateFsAdapter,
   materializeSessionJsonMirrorEntry,
-  readPhysicalSessionStateBuckets,
   storageDir,
 } from './storage.mjs'
 
@@ -68,10 +68,57 @@ function deriveMetadata(sessionId, state, bucket) {
   }
 }
 
+// Per-session normalization shared verbatim by the materialized snapshot
+// (buildSessionJsonSnapshot) and the streaming cutover source below —
+// validation, shaping and diagnostics MUST stay identical between the two
+// paths; the snapshot tests are the parity anchor. Returns null for
+// metadata-only orphans (diagnostics already recorded; orphans are excluded
+// from records, backups and the SQLite import).
+function normalizeSessionEntry(bucket, sessionId, rawState, rawMetadata, diagnostics) {
+  if (!rawState && rawMetadata) {
+    diagnostics.metadataOnly.push(sessionId)
+    diagnostics.orphanDeletes.push({ scope: bucket.scope, projectId: bucket.projectId, sessionId })
+    return null
+  }
+  if (!isPlainObject(rawState)) throw new TypeError(`Invalid session state: ${sessionId}`)
+  if (!Array.isArray(rawState.messages)) throw new TypeError(`Session messages must be an array: ${sessionId}`)
+  if (rawState.id !== undefined && rawState.id !== sessionId) throw new TypeError(`Session body id mismatch: ${sessionId}`)
+  if (rawState.scope !== undefined && rawState.scope !== bucket.scope) throw new TypeError(`Session body scope mismatch: ${sessionId}`)
+  if (bucket.scope === 'project' && rawState.projectId !== undefined && rawState.projectId !== bucket.projectId) throw new TypeError(`Session body project mismatch: ${sessionId}`)
+  if (bucket.scope === 'global' && rawState.projectId !== undefined && rawState.projectId !== null) throw new TypeError(`Global session body project mismatch: ${sessionId}`)
+  let metadata
+  if (rawMetadata === undefined) {
+    diagnostics.bodyOnly.push(sessionId)
+    metadata = deriveMetadata(sessionId, rawState, bucket)
+  } else {
+    if (!isPlainObject(rawMetadata)) throw new TypeError(`Invalid session metadata: ${sessionId}`)
+    if (rawMetadata.id !== undefined && rawMetadata.id !== sessionId) throw new TypeError(`Session metadata id mismatch: ${sessionId}`)
+    if (rawMetadata.scope !== undefined && rawMetadata.scope !== bucket.scope) throw new TypeError(`Session metadata scope mismatch: ${sessionId}`)
+    if (bucket.scope === 'project' && rawMetadata.projectId !== undefined && rawMetadata.projectId !== bucket.projectId) throw new TypeError(`Session metadata project mismatch: ${sessionId}`)
+    if (bucket.scope === 'global' && rawMetadata.projectId !== undefined && rawMetadata.projectId !== null) throw new TypeError(`Global session metadata project mismatch: ${sessionId}`)
+    metadata = structuredClone(rawMetadata)
+  }
+  const stateVersion = rawState.stateVersion ?? metadata.stateVersion ?? 0
+  if (!Number.isInteger(stateVersion) || stateVersion < 0) throw new TypeError(`Invalid session stateVersion: ${sessionId}`)
+  const state = { ...structuredClone(rawState), id: sessionId, scope: bucket.scope, stateVersion }
+  metadata = { ...metadata, id: sessionId, scope: bucket.scope, stateVersion }
+  if (bucket.scope === 'project') {
+    state.projectId = bucket.projectId
+    metadata.projectId = bucket.projectId
+  } else {
+    delete state.projectId
+    delete metadata.projectId
+  }
+  for (const field of ['pinnedAt', 'archivedAt']) {
+    if (metadata[field] !== undefined) state[field] = metadata[field]
+  }
+  return { ...bucket, sessionId, stateVersion, state, metadata, stateDigest: digestJson(state), metadataDigest: digestJson(metadata) }
+}
+
 export function buildSessionJsonSnapshot(buckets) {
   if (!Array.isArray(buckets)) throw new TypeError('Session buckets must be an array')
   const records = []
-  const diagnostics = { bodyOnly: [], metadataOnly: [], duplicateSessionIds: [] }
+  const diagnostics = { bodyOnly: [], metadataOnly: [], orphanDeletes: [], duplicateSessionIds: [] }
   const seen = new Set()
   for (const rawBucket of buckets) {
     const bucket = normalizeBucket(rawBucket)
@@ -80,49 +127,19 @@ export function buildSessionJsonSnapshot(buckets) {
     for (const sessionId of ids) {
       if (seen.has(sessionId)) diagnostics.duplicateSessionIds.push(sessionId)
       seen.add(sessionId)
-      const rawState = rawBucket.sessions[sessionId]
-      const rawMetadata = rawBucket.metadata[sessionId]
-      if (!rawState && rawMetadata) {
-        diagnostics.metadataOnly.push(sessionId)
-        continue
-      }
-      if (!isPlainObject(rawState)) throw new TypeError(`Invalid session state: ${sessionId}`)
-      if (!Array.isArray(rawState.messages)) throw new TypeError(`Session messages must be an array: ${sessionId}`)
-      if (rawState.id !== undefined && rawState.id !== sessionId) throw new TypeError(`Session body id mismatch: ${sessionId}`)
-      if (rawState.scope !== undefined && rawState.scope !== bucket.scope) throw new TypeError(`Session body scope mismatch: ${sessionId}`)
-      if (bucket.scope === 'project' && rawState.projectId !== undefined && rawState.projectId !== bucket.projectId) throw new TypeError(`Session body project mismatch: ${sessionId}`)
-      if (bucket.scope === 'global' && rawState.projectId !== undefined && rawState.projectId !== null) throw new TypeError(`Global session body project mismatch: ${sessionId}`)
-      let metadata
-      if (rawMetadata === undefined) {
-        diagnostics.bodyOnly.push(sessionId)
-        metadata = deriveMetadata(sessionId, rawState, bucket)
-      } else {
-        if (!isPlainObject(rawMetadata)) throw new TypeError(`Invalid session metadata: ${sessionId}`)
-        if (rawMetadata.id !== undefined && rawMetadata.id !== sessionId) throw new TypeError(`Session metadata id mismatch: ${sessionId}`)
-        if (rawMetadata.scope !== undefined && rawMetadata.scope !== bucket.scope) throw new TypeError(`Session metadata scope mismatch: ${sessionId}`)
-        if (bucket.scope === 'project' && rawMetadata.projectId !== undefined && rawMetadata.projectId !== bucket.projectId) throw new TypeError(`Session metadata project mismatch: ${sessionId}`)
-        if (bucket.scope === 'global' && rawMetadata.projectId !== undefined && rawMetadata.projectId !== null) throw new TypeError(`Global session metadata project mismatch: ${sessionId}`)
-        metadata = structuredClone(rawMetadata)
-      }
-      const stateVersion = rawState.stateVersion ?? metadata.stateVersion ?? 0
-      if (!Number.isInteger(stateVersion) || stateVersion < 0) throw new TypeError(`Invalid session stateVersion: ${sessionId}`)
-      const state = { ...structuredClone(rawState), id: sessionId, scope: bucket.scope, stateVersion }
-      metadata = { ...metadata, id: sessionId, scope: bucket.scope, stateVersion }
-      if (bucket.scope === 'project') {
-        state.projectId = bucket.projectId
-        metadata.projectId = bucket.projectId
-      } else {
-        delete state.projectId
-        delete metadata.projectId
-      }
-      for (const field of ['pinnedAt', 'archivedAt']) {
-        if (metadata[field] !== undefined) state[field] = metadata[field]
-      }
-      records.push({ ...bucket, sessionId, stateVersion, state, metadata, stateDigest: digestJson(state), metadataDigest: digestJson(metadata) })
+      const record = normalizeSessionEntry(bucket, sessionId, rawBucket.sessions[sessionId], rawBucket.metadata[sessionId], diagnostics)
+      if (record) records.push(record)
     }
   }
   if (diagnostics.duplicateSessionIds.length > 0) throw new TypeError(`Duplicate session ids across buckets: ${[...new Set(diagnostics.duplicateSessionIds)].join(', ')}`)
-  if (diagnostics.metadataOnly.length > 0) throw new TypeError(`Metadata-only session orphans: ${diagnostics.metadataOnly.join(', ')}`)
+  // Startup cutover holds the maintenance lock with no concurrent writes, so
+  // metadata-only orphans are stale residue of already-deleted sessions: they
+  // are dropped above (excluded from records) and recorded in diagnostics for
+  // audit instead of blocking the migration. Orphans ride the mirror delete
+  // queue (`orphanDeletes` passed to replaceAll as `mirrorDeletes`) so the
+  // drain physically clears their leftover JSON metadata; otherwise
+  // initializeSessionIndex could re-import that residue on a later boot and
+  // the orphaned index rows would fail the next startup integrity check.
   records.sort((left, right) => left.sessionId.localeCompare(right.sessionId))
   // F9 v7: the canonical snapshot digest line now includes a `messagesDigest`
   // slot. JSON imports are always non-split (messages inline in the body), so
@@ -132,48 +149,176 @@ export function buildSessionJsonSnapshot(buckets) {
   return { records, count: records.length, digest, diagnostics }
 }
 
-async function writeCutoverBackup(snapshot, directory = path.join(storageDir, 'backups')) {
+// Streaming cutover source: each pass consumes the fsAdapter lazily — one
+// bucket listing, one metadata bucket parse and one session file parse at a
+// time — so a multi-GB library never needs to be fully resident. Only the
+// small per-pass summary (digest lines + diagnostics) survives a full
+// iteration. Records are normalized by the shared normalizeSessionEntry, and
+// the summary digest is computed from digest lines sorted by sessionId, which
+// matches buildSessionJsonSnapshot's snapshot digest exactly. Bucket-local
+// iteration order is sorted session ids; duplicate/orphan detection is
+// order-independent. Returns a factory: each call produces one independent
+// { iterate, getSummary } pass over the source.
+export function createStreamingSessionSource(fsAdapter) {
+  return () => {
+    const digestEntries = []
+    const diagnostics = { bodyOnly: [], metadataOnly: [], orphanDeletes: [], duplicateSessionIds: [] }
+    const seen = new Set()
+    let count = 0
+    let completed = false
+    const iterate = async function* streamSessionRecords() {
+      for await (const rawBucket of fsAdapter.listBuckets()) {
+        const bucket = normalizeBucket(rawBucket)
+        const metadata = await fsAdapter.readMetadataBucket(rawBucket)
+        if (!isPlainObject(metadata)) throw new TypeError('Session bucket stores must be objects')
+        const fileIds = new Set()
+        for await (const sessionId of fsAdapter.listSessionFiles(rawBucket)) fileIds.add(sessionId)
+        const ids = [...new Set([...fileIds, ...Object.keys(metadata)])].sort((left, right) => left.localeCompare(right))
+        for (const sessionId of ids) {
+          if (seen.has(sessionId)) diagnostics.duplicateSessionIds.push(sessionId)
+          seen.add(sessionId)
+          const rawState = fileIds.has(sessionId) ? await fsAdapter.readSessionState(rawBucket, sessionId) : undefined
+          const record = normalizeSessionEntry(bucket, sessionId, rawState, metadata[sessionId], diagnostics)
+          if (!record) continue
+          count += 1
+          digestEntries.push({ sessionId, line: snapshotDigestLine(record.scope, record.projectId, record.sessionId, record.stateDigest, record.metadataDigest, '') })
+          yield record
+        }
+      }
+      completed = true
+    }
+    return {
+      iterate,
+      getSummary() {
+        if (!completed) throw new Error('Session source summary is not ready before the iteration completes')
+        if (diagnostics.duplicateSessionIds.length > 0) throw new TypeError(`Duplicate session ids across buckets: ${[...new Set(diagnostics.duplicateSessionIds)].join(', ')}`)
+        digestEntries.sort((left, right) => left.sessionId.localeCompare(right.sessionId))
+        const digest = createHash('sha256').update(digestEntries.map((entry) => entry.line).join('\n')).digest('hex')
+        return { count, digest, diagnostics }
+      },
+    }
+  }
+}
+
+// readBuckets keeps the historical injection path for tests and small data
+// sets: each pass calls readBuckets() exactly once, materializes one full
+// snapshot (buildSessionJsonSnapshot) and replays its records. The summary is
+// structurally identical to the streaming source, so the cutover flow below
+// is agnostic to which source factory it was given.
+function createReadBucketsSessionSource(readBuckets) {
+  return () => {
+    let snapshot = null
+    const iterate = async function* snapshotSessionRecords() {
+      snapshot = buildSessionJsonSnapshot(await readBuckets())
+      yield* snapshot.records
+    }
+    return {
+      iterate,
+      getSummary() {
+        if (!snapshot) throw new Error('Session source summary is not ready before the iteration completes')
+        return { count: snapshot.count, digest: snapshot.digest, diagnostics: snapshot.diagnostics }
+      },
+    }
+  }
+}
+
+async function summarizeSessionSource(source) {
+  let consumed = 0
+  for await (const _record of source.iterate()) consumed += 1
+  const summary = source.getSummary()
+  if (summary.count !== consumed) throw new Error('Session source summary count mismatch')
+  return summary
+}
+
+function sameSessionSourceSummary(left, right) {
+  return left.count === right.count && left.digest === right.digest
+    && JSON.stringify(left.diagnostics) === JSON.stringify(right.diagnostics)
+}
+
+// Streaming v1 backup writer: the output JSON shape matches the old
+// materialized writer exactly ({app, version, exportedAt, scope,
+// includeSecrets, sessionState, data:{sessions, sessionsMetadata}}), but
+// states are streamed record-by-record so the library never lives in memory
+// as a whole. The sessions section is written while iterating; metadata JSON
+// strings (small per-session summaries) are buffered for the second section —
+// states themselves are never buffered. Verification is byte-level: the
+// write-side sha256/byte counter is re-checked against a chunked
+// createReadStream re-read plus a first/last byte sanity check, replacing the
+// old "read whole file + re-parse + re-snapshot" verification. The streamed
+// records' digest is also re-accumulated and compared against `summary`, so a
+// source that changed since the double read fails closed here.
+async function writeCutoverBackupStream(createIteration, summary, options = {}) {
+  const directory = options.directory || path.join(storageDir, 'backups')
   await fs.mkdir(directory, { recursive: true })
   const stamp = new Date().toISOString().replace(/[:.]/g, '-')
   const finalPath = path.join(directory, `quickforge-session-state-cutover-${stamp}.json`)
   const temporaryPath = `${finalPath}.${process.pid}.${randomUUID()}.tmp`
-  const backup = {
-    app: 'quickforge',
-    version: 1,
-    exportedAt: new Date().toISOString(),
-    scope: 'sessions',
-    includeSecrets: false,
-    sessionState: { count: snapshot.count, digest: snapshot.digest },
-    data: {
-      sessions: Object.fromEntries(snapshot.records.map((record) => [record.sessionId, record.state])),
-      sessionsMetadata: Object.fromEntries(snapshot.records.map((record) => [record.sessionId, record.metadata])),
-    },
-  }
+  const openWriteStream = options.createWriteStream || createWriteStream
   try {
-    await fs.writeFile(temporaryPath, `${JSON.stringify(backup, null, 2)}\n`, 'utf8')
-    const verified = JSON.parse(await fs.readFile(temporaryPath, 'utf8'))
-    const grouped = new Map()
-    for (const [sessionId, state] of Object.entries(verified.data.sessions)) {
-      const scope = state?.scope === 'project' ? 'project' : 'global'
-      const projectId = scope === 'project' ? state?.projectId : null
-      const key = scope === 'project' ? `project:${projectId}` : 'global'
-      if (!grouped.has(key)) grouped.set(key, { scope, projectId, sessions: {}, metadata: {} })
-      grouped.get(key).sessions[sessionId] = state
-      if (Object.prototype.hasOwnProperty.call(verified.data.sessionsMetadata, sessionId)) {
-        grouped.get(key).metadata[sessionId] = verified.data.sessionsMetadata[sessionId]
+    const stream = openWriteStream(temporaryPath, { encoding: 'utf8' })
+    const failures = []
+    stream.on('error', (error) => { failures.push(error) })
+    const written = createHash('sha256')
+    let bytes = 0
+    const write = async (chunk) => {
+      written.update(chunk)
+      bytes += Buffer.byteLength(chunk, 'utf8')
+      if (stream.write(chunk, 'utf8')) {
+        if (failures.length > 0) throw failures[0]
+        return
       }
+      await new Promise((resolve, reject) => {
+        const onDrain = () => { cleanup(); resolve() }
+        const onError = (error) => { cleanup(); reject(error) }
+        const cleanup = () => { stream.off('drain', onDrain); stream.off('error', onError) }
+        stream.once('drain', onDrain)
+        stream.once('error', onError)
+      })
     }
-    for (const [sessionId, metadata] of Object.entries(verified.data.sessionsMetadata)) {
-      if (!Object.prototype.hasOwnProperty.call(verified.data.sessions, sessionId)) {
-        const scope = metadata?.scope === 'project' ? 'project' : 'global'
-        const projectId = scope === 'project' ? metadata?.projectId : null
-        const key = scope === 'project' ? `project:${projectId}` : 'global'
-        if (!grouped.has(key)) grouped.set(key, { scope, projectId, sessions: {}, metadata: {} })
-        grouped.get(key).metadata[sessionId] = metadata
-      }
+    await write(`{\n  "app": "quickforge",\n  "version": 1,\n  "exportedAt": ${JSON.stringify(new Date().toISOString())},\n  "scope": "sessions",\n  "includeSecrets": false,\n  "sessionState": {\n    "count": ${summary.count},\n    "digest": ${JSON.stringify(summary.digest)}\n  },\n  "data": {\n    "sessions": {`)
+    const digestEntries = []
+    let count = 0
+    let firstSession = true
+    const metadataChunks = []
+    for await (const record of createIteration()) {
+      digestEntries.push({ sessionId: record.sessionId, line: snapshotDigestLine(record.scope, record.projectId, record.sessionId, record.stateDigest, record.metadataDigest, '') })
+      count += 1
+      const prefix = firstSession ? '' : ','
+      await write(`${prefix}\n      ${JSON.stringify(record.sessionId)}: ${JSON.stringify(record.state)}`)
+      metadataChunks.push(`${prefix}\n      ${JSON.stringify(record.sessionId)}: ${JSON.stringify(record.metadata)}`)
+      firstSession = false
+    }
+    await write(`\n    },\n    "sessionsMetadata": {`)
+    for (const metadataChunk of metadataChunks) {
+      await write(metadataChunk)
     }
-    const verifiedSnapshot = buildSessionJsonSnapshot([...grouped.values()])
-    if (verified.sessionState?.count !== snapshot.count || verified.sessionState?.digest !== snapshot.digest || verifiedSnapshot.count !== snapshot.count || verifiedSnapshot.digest !== snapshot.digest) {
+    await write(`\n    }\n  }\n}`)
+    await new Promise((resolve, reject) => {
+      const onError = (error) => { cleanup(); reject(error) }
+      const cleanup = () => { stream.off('error', onError) }
+      stream.once('error', onError)
+      stream.end(() => { cleanup(); resolve() })
+    })
+    if (count !== summary.count) throw new Error('Session JSON source changed before cutover commit')
+    digestEntries.sort((left, right) => left.sessionId.localeCompare(right.sessionId))
+    const digest = createHash('sha256').update(digestEntries.map((entry) => entry.line).join('\n')).digest('hex')
+    if (digest !== summary.digest) throw new Error('Session JSON source changed before cutover commit')
+    const reread = createHash('sha256')
+    let rereadBytes = 0
+    let firstByte = -1
+    let lastByte = -1
+    await new Promise((resolve, reject) => {
+      const reader = createReadStream(temporaryPath)
+      reader.on('data', (chunk) => {
+        if (firstByte === -1) firstByte = chunk[0]
+        lastByte = chunk[chunk.length - 1]
+        reread.update(chunk)
+        rereadBytes += chunk.length
+      })
+      reader.once('error', reject)
+      reader.once('end', () => resolve())
+    })
+    if (reread.digest('hex') !== written.digest('hex') || rereadBytes !== bytes || firstByte !== 0x7b || lastByte !== 0x7d) {
       throw new Error('Session cutover backup verification failed')
     }
     await fs.rename(temporaryPath, finalPath)
@@ -298,6 +443,29 @@ function mirrorAdapter() {
   return { upsert: materialize, delete: materialize }
 }
 
+function integrityFailureSummary(integrity) {
+  const summary = Object.entries(integrity)
+    .filter(([key, value]) => !['ok', 'count', 'digest'].includes(key) && Number(value) > 0)
+    .map(([key, value]) => `${key}=${value}`)
+    .join(', ')
+  return summary || 'unknown'
+}
+
+// session_index is a pure projection of session_states, but
+// initializeSessionIndex rebuilds it from the JSON source, which can drift
+// from authoritative rows (e.g. orphan metadata residue re-imported after a
+// failed mirror drain). Rebuilding the projection from the authoritative
+// states is lossless self-healing: on integrity failure, rebuild the index
+// and re-verify, and only fail closed if the re-check still fails.
+function verifyIntegrityWithIndexSelfHeal(repository, phase) {
+  const first = repository.verifyIntegrity({ quickCheck: true })
+  if (first.ok) return first
+  repository.rebuildIndex()
+  const second = repository.verifyIntegrity({ quickCheck: true })
+  if (!second.ok) throw new Error(`Session state ${phase} integrity verification failed (${integrityFailureSummary(first)})`)
+  return second
+}
+
 export async function initializeSessionStateCutover(options = {}) {
   const storage = options.storage || getSqliteStorage()
   const repository = options.repository || createSessionStateRepository(storage)
@@ -305,15 +473,16 @@ export async function initializeSessionStateCutover(options = {}) {
   return runSessionStateMaintenance(async () => {
     const current = readSessionStorageState()
     if (current.phase === SESSION_STORAGE_PHASES.JSON_PENDING) {
-      const integrity = repository.verifyIntegrity({ quickCheck: true })
-      if (!integrity.ok) throw new Error('Session state pending integrity verification failed')
+      const integrity = verifyIntegrityWithIndexSelfHeal(repository, 'pending')
       const drained = await drainSessionJsonMirror()
-      if (drained.pending === 0) setSessionStoragePhase(SESSION_STORAGE_PHASES.AUTHORITATIVE, { stateCount: integrity.count, digest: integrity.digest, backupFile: current.backupFile })
+      // integrity is the lightweight (SQL-level) check — its digest is null.
+      // The verified digest persisted by replaceAll's storageState stays
+      // authoritative when promoting to authoritative.
+      if (drained.pending === 0) setSessionStoragePhase(SESSION_STORAGE_PHASES.AUTHORITATIVE, { stateCount: integrity.count, digest: current.digest, backupFile: current.backupFile })
       return readSessionStorageState()
     }
     if (current.phase === SESSION_STORAGE_PHASES.AUTHORITATIVE) {
-      const integrity = repository.verifyIntegrity({ quickCheck: true })
-      if (!integrity.ok) throw new Error('Session state authoritative integrity verification failed')
+      verifyIntegrityWithIndexSelfHeal(repository, 'authoritative')
       await drainSessionJsonMirror()
       return readSessionStorageState()
     }
@@ -327,12 +496,30 @@ export async function initializeSessionStateCutover(options = {}) {
 
     let backupFile = current.backupFile
     try {
-      const first = buildSessionJsonSnapshot(await (options.readBuckets || readPhysicalSessionStateBuckets)())
-      const second = buildSessionJsonSnapshot(await (options.readBuckets || readPhysicalSessionStateBuckets)())
-      if (first.count !== second.count || first.digest !== second.digest) throw new Error('Session JSON source changed during cutover double read')
-      if (!backupFile) backupFile = await writeCutoverBackup(first, options.backupDirectory)
-      const third = buildSessionJsonSnapshot(await (options.readBuckets || readPhysicalSessionStateBuckets)())
-      if (first.count !== third.count || first.digest !== third.digest) throw new Error('Session JSON source changed before cutover commit')
+      // Streaming cutover (step 2): the json_authoritative migration path runs
+      // FOUR independent full iterations of the JSON source — (1) summary,
+      // (2) summary re-read, (3) streaming backup write (or summary-only when
+      // a backup already exists) and (4) replaceAllStream import. Only the
+      // small summaries (digest lines + diagnostics) survive between passes,
+      // so peak memory is bounded by the largest single session instead of
+      // the whole library. options.readBuckets keeps the materialized
+      // injection path for tests; the production default streams the physical
+      // layout one file at a time via the fsAdapter.
+      const createSource = options.readBuckets
+        ? createReadBucketsSessionSource(options.readBuckets)
+        : createStreamingSessionSource(options.fsAdapter || createPhysicalSessionStateFsAdapter())
+      const first = await summarizeSessionSource(createSource())
+      const second = await summarizeSessionSource(createSource())
+      if (!sameSessionSourceSummary(first, second)) throw new Error('Session JSON source changed during cutover double read')
+      if (!backupFile) {
+        backupFile = await writeCutoverBackupStream(() => createSource().iterate(), first, {
+          directory: options.backupDirectory,
+          createWriteStream: options.createBackupWriteStream,
+        })
+      } else {
+        const third = await summarizeSessionSource(createSource())
+        if (!sameSessionSourceSummary(first, third)) throw new Error('Session JSON source changed before cutover commit')
+      }
       setSessionStoragePhase(SESSION_STORAGE_PHASES.CUTOVER_RUNNING, {
         stateCount: first.count,
         digest: first.digest,
@@ -344,16 +531,22 @@ export async function initializeSessionStateCutover(options = {}) {
         backupFile,
         diagnostic: first.diagnostics,
       }
-      repository.replaceAll(first.records, {
+      // replaceAllStream consumes the 4th source pass record-by-record inside
+      // its own immediate transaction and verifies expectedDigest there.
+      await repository.replaceAllStream(createSource().iterate(), {
         expectedCount: first.count,
         expectedDigest: first.digest,
         storageState: pendingValues,
+        mirrorDeletes: first.diagnostics.orphanDeletes,
       })
+      // replaceAll already verified expectedDigest inside its transaction, so
+      // the post-replace check re-asserts SQL-level integrity and the count
+      // only (the lightweight verification digest is null by design).
       const integrity = repository.verifyIntegrity({ quickCheck: true })
-      if (!integrity.ok || integrity.count !== first.count || integrity.digest !== first.digest) throw new Error('Session SQLite replace verification failed')
+      if (!integrity.ok || integrity.count !== first.count) throw new Error('Session SQLite replace verification failed')
       readSessionStorageState()
       const drained = await drainSessionJsonMirror()
-      if (drained.pending === 0) setSessionStoragePhase(SESSION_STORAGE_PHASES.AUTHORITATIVE, { stateCount: integrity.count, digest: integrity.digest, backupFile, diagnostic: first.diagnostics })
+      if (drained.pending === 0) setSessionStoragePhase(SESSION_STORAGE_PHASES.AUTHORITATIVE, { stateCount: integrity.count, digest: first.digest, backupFile, diagnostic: first.diagnostics })
       return readSessionStorageState()
     } catch (error) {
       const state = readSessionStorageState()

package/server/session-state-service.mjs

@@ -13,6 +13,11 @@ export const SESSION_STORAGE_PHASES = Object.freeze({
 // marker). Sessions below the threshold stay inline for backward compatibility.
 export const MESSAGES_SPLIT_THRESHOLD = 200
 
+// Mirror drain page size: the queue rows carry full state_json payloads, so
+// the drain pulls a bounded batch at a time instead of loading the whole
+// outbox (large cutover imports can enqueue thousands of entries).
+const MIRROR_DRAIN_BATCH_LIMIT = 8
+
 let repositoryInstance = null
 let cachedPhase = SESSION_STORAGE_PHASES.JSON_AUTHORITATIVE
 let jsonAdapter = null
@@ -667,25 +672,37 @@ export function replaceSessionStateSnapshot({ sessions, sessionsMetadata }, { me
 export async function drainSessionJsonMirror() {
   if (drainPromise) return drainPromise
   drainPromise = (async () => {
-    if (!mirrorAdapter) return { drained: 0, pending: repository().listMirrorQueue().length }
+    if (!mirrorAdapter) return { drained: 0, pending: repository().countMirrorQueue() }
     let drained = 0
-    for (const entry of repository().listMirrorQueue()) {
-      try {
-        if (entry.operation === 'upsert') {
-          const state = entry.state?.messageStorage === MESSAGES_SPLIT_VALUE
-            ? { ...entry.state, messages: allMessages(entry) }
-            : entry.state
-          await mirrorAdapter.upsert({ ...entry, state })
-        } else {
-          await mirrorAdapter.delete(entry)
+    // Page through the outbox in small batches. A failed entry gets its
+    // updated_at bumped by failMirror, so it sorts behind the remaining
+    // entries and the next page drains others first; entries that keep
+    // failing are retried by the scheduled drain. A batch with zero
+    // acknowledgements stops the loop (no head-of-line livelock).
+    for (;;) {
+      const batch = repository().listMirrorQueue({ limit: MIRROR_DRAIN_BATCH_LIMIT })
+      if (batch.length === 0) break
+      let acknowledged = 0
+      for (const entry of batch) {
+        try {
+          if (entry.operation === 'upsert') {
+            const state = entry.state?.messageStorage === MESSAGES_SPLIT_VALUE
+              ? { ...entry.state, messages: allMessages(entry) }
+              : entry.state
+            await mirrorAdapter.upsert({ ...entry, state })
+          } else {
+            await mirrorAdapter.delete(entry)
+          }
+          repository().acknowledgeMirror(entry)
+          drained += 1
+          acknowledged += 1
+        } catch (error) {
+          repository().failMirror(entry, error)
         }
-        repository().acknowledgeMirror(entry)
-        drained += 1
-      } catch (error) {
-        repository().failMirror(entry, error)
       }
+      if (acknowledged === 0) break
     }
-    return { drained, pending: repository().listMirrorQueue().length }
+    return { drained, pending: repository().countMirrorQueue() }
   })().finally(() => { drainPromise = null })
   const result = await drainPromise
   if (result.pending > 0) scheduleSessionJsonMirrorDrain()
@@ -700,8 +717,12 @@ export function getSessionStateDiagnostics() {
   let integrity
   let mirrorPending = null
   try {
-    integrity = repository().verifyIntegrity()
-    mirrorPending = repository().listMirrorQueue().length
+    // Lightweight (SQL-level) integrity: startup diagnostics on large stores
+    // must not re-parse every stored body. The result carries
+    // `lightweight: true`; full per-row verification stays on maintenance
+    // entry points (backup export/restore, downgrade tooling).
+    integrity = repository().verifyIntegrity({ quickCheck: true })
+    mirrorPending = repository().countMirrorQueue()
   } catch (error) {
     integrity = { ok: false, error: error?.message || String(error) }
   }