@aopslabs/aops-server 0.2.23 → 0.2.25

package/scripts/owned-set-backup.mjs

@@ -0,0 +1,437 @@
+#!/usr/bin/env node
+/**
+ * First-party backup of the relations AOPS owns, scoped by the domain providers
+ * and consistent across all of them.
+ *
+ * Two properties do the work here, and they pull in opposite directions:
+ *
+ *   The bundle is fragmented per domain, because maintenance ownership is
+ *   per domain -- a provider knows its own relations and nobody else's.
+ *
+ *   The snapshot is single and global, because the domains are not independent.
+ *   Measured: `scopeId` is a hub, and more than seven thousand projectman and
+ *   docman rows point at agentspace scopes with no foreign key to enforce it.
+ *   Five separate snapshots would each be internally consistent and mutually
+ *   wrong, and nothing in the database would object.
+ *
+ * So: one REPEATABLE READ READ ONLY transaction, N fragment files, one manifest.
+ *
+ * Values leave as ::text and return as typed casts. That hands serialization to
+ * PostgreSQL rather than to the driver -- measured to matter: the default JSON
+ * path loses jsonb integers above 2^53, and the owned set contains such columns.
+ *
+ * Nothing here writes the plaintext bundle to disk. Rows stream out of a cursor
+ * and into gzipped chunks, so the largest thing in memory is one chunk.
+ */
+import { createHash } from 'node:crypto'
+import { gzipSync } from 'node:zlib'
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync,
+         renameSync, rmSync, statSync, writeSync } from 'node:fs'
+import path from 'node:path'
+
+import { providerContractSha256 as computeProviderContractSha256 } from './domain-backup-provider.mjs'
+
+const CONTRACT = 'aops.owned-set-backup/v2'
+const RECEIPT_CONTRACT = 'aops.backup-receipt/v1'
+const CHUNK_TARGET_BYTES = 12 * 1024 * 1024
+const FETCH_ROWS = 2000
+
+const quote = (id) => `"${String(id).replace(/"/g, '""')}"`
+const sha256 = (buffer) => createHash('sha256').update(buffer).digest('hex')
+
+/** .part -> fsync -> atomic rename, 0600. A reader never sees a half-written file. */
+function writeAtomic(target, buffer) {
+  const staging = `${target}.part`
+  const handle = openSync(staging, 'w', 0o600)
+  try { writeSync(handle, buffer); fsyncSync(handle) } finally { closeSync(handle) }
+  renameSync(staging, target)
+}
+
+/**
+ * A digest over a ledger's ordered content, excluding the timestamp.
+ *
+ * Row count alone cannot distinguish two different migration histories of the
+ * same length: substitute one tag for another and the count agrees while the
+ * claim does not. `appliedAt` is left out because it records when a migration ran
+ * on this machine, not which migration ran.
+ */
+export async function readLedgerDigest(client, relation) {
+  const { rows } = await client.query(`
+    SELECT a.attname AS column, format_type(a.atttypid, a.atttypmod) AS type
+    FROM pg_attribute a
+    JOIN pg_class c ON c.oid = a.attrelid
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = 'public' AND c.relname = $1 AND a.attnum > 0 AND NOT a.attisdropped
+    ORDER BY a.attnum`, [relation])
+  if (rows.length === 0) return { rowCount: 0, contentSha256: sha256(Buffer.alloc(0)) }
+  // Timestamps are excluded BY TYPE, not by name. The column is `applied_at` in
+  // these ledgers and `appliedAt` elsewhere; excluding one spelling left the other
+  // in the digest, and every legitimately rebuilt ledger then differed from the
+  // one it was rebuilt from -- for no reason except that it was applied at a
+  // different moment, which is the one thing the digest must not care about.
+  const columns = rows
+    .filter((row) => !/^timestamp( |$)/.test(String(row.type)))
+    .map((row) => row.column)
+  const projection = columns.map((column) => `coalesce(${quote(column)}::text, '')`).join(` || '|' || `)
+  const digest = await client.query(`
+    SELECT count(*)::int AS rows,
+           coalesce(string_agg(line, E'\n' ORDER BY line), '') AS body
+    FROM (SELECT ${projection} AS line FROM public.${quote(relation)}) source`)
+  return { rowCount: digest.rows[0].rows, contentSha256: sha256(Buffer.from(digest.rows[0].body)) }
+}
+
+async function openSession(client) {
+  await client.query("SET SESSION TIME ZONE 'UTC'")
+  await client.query("SET SESSION DateStyle TO 'ISO, YMD'")
+  await client.query("SET SESSION IntervalStyle TO 'iso_8601'")
+  await client.query('SET SESSION extra_float_digits TO 3')
+  await client.query("SET SESSION bytea_output TO 'hex'")
+}
+
+async function readShape(client, relations) {
+  const columns = await client.query(`
+    SELECT c.relname AS relation, a.attname AS column, a.attnum,
+           format_type(a.atttypid, a.atttypmod) AS type, a.atttypid::int AS typeOid
+    FROM pg_attribute a
+    JOIN pg_class c ON c.oid = a.attrelid
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = 'public' AND c.relname = ANY($1)
+      AND a.attnum > 0 AND NOT a.attisdropped AND c.relkind IN ('r', 'p')
+    ORDER BY c.relname, a.attnum`, [relations])
+  const foreignKeys = await client.query(`
+    SELECT src.relname AS child, tgt.relname AS parent,
+           (SELECT string_agg(att.attname, ',' ORDER BY att.attnum)
+            FROM unnest(k.conkey) ck
+            JOIN pg_attribute att ON att.attrelid = k.conrelid AND att.attnum = ck) AS columns
+    FROM pg_constraint k
+    JOIN pg_class src ON src.oid = k.conrelid
+    JOIN pg_class tgt ON tgt.oid = k.confrelid
+    WHERE k.contype = 'f' AND src.relname = ANY($1) AND tgt.relname = ANY($1)`, [relations])
+  // A foreign key with one end outside the set means the set is not closed, and a
+  // single TRUNCATE of it would fail at restore. Fail here instead, where the
+  // message can say which relation crosses the boundary.
+  const crossing = await client.query(`
+    SELECT src.relname AS child, tgt.relname AS parent
+    FROM pg_constraint k
+    JOIN pg_class src ON src.oid = k.conrelid
+    JOIN pg_class tgt ON tgt.oid = k.confrelid
+    WHERE k.contype = 'f' AND ((src.relname = ANY($1)) <> (tgt.relname = ANY($1)))`, [relations])
+  if (crossing.rows.length > 0) {
+    const sample = crossing.rows.slice(0, 3).map((row) => `${row.child}->${row.parent}`).join(',')
+    throw new Error(`owned_set_not_fk_closed:${crossing.rows.length}:${sample}`)
+  }
+  // Only sequences OWNED BY a column of an owned relation. Reading every sequence
+  // in the schema would put a setval on something outside the boundary at restore
+  // -- the one thing this design promises not to do. The dependency is the
+  // authority here, not the name: a sequence named after one of our relations may
+  // belong to something else entirely.
+  const sequences = await client.query(`
+    SELECT s.relname AS name, seq.last_value
+    FROM pg_class s
+    JOIN pg_namespace sn ON sn.oid = s.relnamespace
+    JOIN pg_depend d ON d.objid = s.oid AND d.classid = 'pg_class'::regclass AND d.deptype = 'a'
+    JOIN pg_class t ON t.oid = d.refobjid
+    JOIN pg_namespace tn ON tn.oid = t.relnamespace
+    LEFT JOIN pg_sequences seq ON seq.schemaname = sn.nspname AND seq.sequencename = s.relname
+    WHERE s.relkind = 'S' AND sn.nspname = 'public' AND tn.nspname = 'public'
+      AND t.relname = ANY($1)`, [relations])
+  const byRelation = new Map()
+  for (const row of columns.rows) {
+    if (!byRelation.has(row.relation)) byRelation.set(row.relation, [])
+    byRelation.get(row.relation).push({ column: row.column, type: row.type, typeOid: row.typeoid })
+  }
+  return { columns: byRelation, foreignKeys: foreignKeys.rows, sequences: sequences.rows }
+}
+
+/** Parents before children. Self-references are handled at load time, not here. */
+export function topologicalOrder(relations, foreignKeys) {
+  const dependencies = new Map(relations.map((relation) => [relation, new Set()]))
+  const selfReferencing = new Map()
+  for (const edge of foreignKeys) {
+    const columns = String(edge.columns ?? '').split(',').filter(Boolean)
+    if (edge.child === edge.parent) { selfReferencing.set(edge.child, columns); continue }
+    dependencies.get(edge.child)?.add(edge.parent)
+  }
+  const order = []
+  const remaining = new Set(relations)
+  while (remaining.size > 0) {
+    const ready = [...remaining].filter((relation) =>
+      [...dependencies.get(relation)].every((parent) => !remaining.has(parent)))
+    if (ready.length === 0) throw new Error(`owned_set_foreign_key_cycle:${[...remaining].sort().join(',')}`)
+    for (const relation of ready.sort()) { order.push(relation); remaining.delete(relation) }
+  }
+  return { order, selfReferencing }
+}
+
+/**
+ * Copies the predecessor's migration closure into the bundle, verifying every
+ * byte on the way in.
+ *
+ * The physical source is the installed package, not a repository tree: the
+ * repository is what a release was built from, the installed package is what an
+ * operator actually has, and a bundle that embeds anything else embeds a claim
+ * rather than the bytes. Each file is hashed as it is copied and compared to the
+ * manifest the lineage is bound to, so a closure that shifted under us fails here
+ * instead of producing a bundle that restores a different schema than it says.
+ */
+export function embedMigrationClosure(bundleDirectory, migrationSetManifest) {
+  if (!migrationSetManifest?.roots?.length) {
+    throw new Error('owned_set_backup_closure_manifest_empty')
+  }
+  const embedded = []
+  for (const root of migrationSetManifest.roots) {
+    if (!root.sourceDir || !existsSync(root.sourceDir)) {
+      throw new Error(`owned_set_backup_closure_source_missing:${root.root}:${String(root.sourceDir)}`)
+    }
+    for (const file of root.files) {
+      // A file may arrive as bytes rather than as a path. The journal does: the
+      // bundle carries the applied prefix of the closure, and the journal that
+      // describes exactly that prefix is not a file anyone shipped. Everything
+      // else still comes off disk, and both go through the same hash and size
+      // check below -- being synthesized buys no exemption from the manifest.
+      let bytes
+      if (file.contents !== undefined) {
+        bytes = Buffer.isBuffer(file.contents) ? file.contents : Buffer.from(file.contents)
+      } else {
+        const source = path.join(root.sourceDir, file.relativePath)
+        if (!existsSync(source)) {
+          throw new Error(`owned_set_backup_closure_file_missing:${root.root}:${file.relativePath}`)
+        }
+        bytes = readFileSync(source)
+      }
+      const actual = sha256(bytes)
+      if (actual !== file.sha256) {
+        throw new Error(`owned_set_backup_closure_file_changed:${root.root}:${file.relativePath}`)
+      }
+      if (bytes.length !== file.byteLength) {
+        throw new Error(`owned_set_backup_closure_file_size_changed:${root.root}:${file.relativePath}`)
+      }
+      const target = path.join(bundleDirectory, 'closure', root.root, file.relativePath)
+      mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 })
+      writeAtomic(target, bytes)
+      embedded.push({ root: root.root, relativePath: file.relativePath, sha256: actual, byteLength: bytes.length })
+    }
+  }
+  return {
+    schemaVersion: 1,
+    rootCount: migrationSetManifest.roots.length,
+    fileCount: embedded.length,
+    byteLength: embedded.reduce((sum, file) => sum + file.byteLength, 0),
+    files: embedded,
+    migrationBundleSha256: sha256(Buffer.from(JSON.stringify(embedded))),
+  }
+}
+
+export async function backupOwnedSet({ client, providerSet, identity, bundleDirectory, receiptPath, backupSetId, migrationSetManifest }) {
+  if (!backupSetId) throw new Error('owned_set_backup_set_id_required')
+  if (!(identity?.lineageId ?? identity?.id)) throw new Error('owned_set_backup_identity_lineage_id_required')
+  if (identity?.identityMode !== 'semantic-owned-v1') {
+    // A legacy-raw lineage has no owned relation list, so there is no defensible
+    // scope to back up: the retiring predecessor relations cannot be found by
+    // intersecting with the target contract.
+    throw new Error(`owned_set_backup_not_eligible:identityMode=${String(identity?.identityMode)}`)
+  }
+  // The declaration the providers actually carry, hashed over its semantic half.
+  // A lineage that states which contract it expects is checked against it here
+  // rather than at restore time: a backup taken under the wrong contract should
+  // never be written, not merely refused later when the data is already needed.
+  // Measured: the codec holds no registry of known types. A value leaves as
+  // `::text` and returns as `$n::<the column's own format_type()>`, which
+  // round-trips enums, composites, domains, ranges and arrays a product author
+  // invents later -- proved in owned-set-codec-types.test.mjs. So there is no
+  // "unknown type" for an extension to cover, and nothing here implements one.
+  // A domain that declares an extension is therefore declaring something this
+  // build cannot honour, and refusing is the only honest answer: the alternative
+  // is a backup taken as if the declaration had been read.
+  for (const provider of providerSet.providers) {
+    if ((provider.codecExtensions ?? []).length > 0) {
+      throw new Error(`owned_set_backup_codec_extension_unsupported:${provider.domain}`)
+    }
+  }
+  const providerContractSha256 = computeProviderContractSha256(providerSet)
+  // Required, not "checked when present". The restore side carried the same
+  // shape -- an optional value feeding a truthy guard -- and there the default
+  // invocation supplied nothing, so the comparison never ran and a drifted
+  // contract passed silently. The argument that it cannot be absent here is the
+  // same argument that was wrong there: the snapshot refuses a non-semantic
+  // lineage and the validator requires the field, but this function is callable
+  // on its own, and an identity without a contract means nothing is comparing
+  // what was computed to what the lineage records.
+  if (typeof identity.providerContractSha256 !== 'string' || identity.providerContractSha256.length !== 64) {
+    throw new Error(`owned_set_backup_identity_contract_absent:${String(identity.id)}`)
+  }
+  if (identity.providerContractSha256 !== providerContractSha256) {
+    throw new Error('owned_set_backup_provider_contract_mismatch:' +
+      `${identity.providerContractSha256}:${providerContractSha256}`)
+  }
+  rmSync(bundleDirectory, { recursive: true, force: true })
+  mkdirSync(path.join(bundleDirectory, 'fragments'), { recursive: true, mode: 0o700 })
+
+  await openSession(client)
+  const allRelations = [...providerSet.payloadRelations, ...providerSet.ledgerRelations].sort()
+  const shape = await readShape(client, allRelations)
+  const { order, selfReferencing } = topologicalOrder(allRelations, shape.foreignKeys)
+  const ledgers = new Set(providerSet.ledgerRelations)
+
+  const chunks = []
+  let totalRows = 0
+  let largestChunkBytes = 0
+
+  // ONE snapshot for every domain. Fragmenting the output does not fragment the read.
+  await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
+  const ledgerEvidence = {}
+  for (const provider of providerSet.providers) {
+    const relations = order.filter((relation) =>
+      provider.payloadRelations.includes(relation) || provider.ledgerRelation === relation)
+    for (const relation of relations) {
+      const columns = shape.columns.get(relation) ?? []
+      if (columns.length === 0) continue
+      const projection = columns.map((column) => `${quote(column.column)}::text AS ${quote(column.column)}`).join(', ')
+      await client.query(`DECLARE owned_cursor NO SCROLL CURSOR FOR SELECT ${projection} FROM public.${quote(relation)}`)
+      let index = 0
+      let buffer = []
+      let bufferBytes = 0
+      const flush = () => {
+        if (buffer.length === 0) return
+        const plain = Buffer.from(`${buffer.join('\n')}\n`)
+        const packed = gzipSync(plain, { level: 6 })
+        const name = `${provider.domain}.${relation}.${String(index).padStart(4, '0')}.ndjson.gz`
+        writeAtomic(path.join(bundleDirectory, 'fragments', name), packed)
+        chunks.push({
+          domain: provider.domain,
+          relation,
+          isLedger: ledgers.has(relation),
+          index,
+          file: `fragments/${name}`,
+          rowCount: buffer.length,
+          uncompressedBytes: plain.length,
+          compressedBytes: packed.length,
+          uncompressedSha256: sha256(plain),
+          compressedSha256: sha256(packed),
+        })
+        largestChunkBytes = Math.max(largestChunkBytes, plain.length)
+        index += 1
+        buffer = []
+        bufferBytes = 0
+      }
+      let relationRows = 0
+      for (;;) {
+        const { rows } = await client.query(`FETCH ${FETCH_ROWS} FROM owned_cursor`)
+        if (rows.length === 0) break
+        for (const row of rows) {
+          const line = JSON.stringify(columns.map((column) => row[column.column]))
+          buffer.push(line)
+          bufferBytes += line.length + 1
+          relationRows += 1
+          totalRows += 1
+          if (bufferBytes >= CHUNK_TARGET_BYTES) flush()
+        }
+      }
+      flush()
+      await client.query('CLOSE owned_cursor')
+      if (ledgers.has(relation)) {
+        // The digest is computed from the same snapshot the rows were read in, so
+        // it describes the state this bundle captured rather than whatever the
+        // ledger looks like by the time anyone validates it.
+        const digest = await readLedgerDigest(client, relation)
+        ledgerEvidence[relation] = {
+          domain: provider.domain,
+          rowCount: relationRows,
+          contentSha256: digest.contentSha256,
+        }
+      }
+    }
+  }
+  await client.query('COMMIT')
+
+  const embeddedClosure = migrationSetManifest ? embedMigrationClosure(bundleDirectory, migrationSetManifest) : null
+
+  const manifest = {
+    schemaVersion: 2,
+    contract: CONTRACT,
+    backupSetId,
+    identity: {
+      // Policy records carry the id as `id`; a receipt written from `lineageId`
+      // silently records undefined, and the anchor's binding to its source
+      // lineage is exactly what must not be optional.
+      lineageId: identity.lineageId ?? identity.id,
+      identityMode: identity.identityMode,
+      postgresMajor: identity.postgresMajor,
+      ownedFingerprintSha256: identity.ownedFingerprintSha256,
+      ownedRelations: [...identity.ownedRelations].sort(),
+    },
+    // What each domain declared, and the identity of that declaration. The list
+    // alone proves nothing: an edge can be deleted from the inventory and every
+    // relation-coverage check stays green -- measured, not assumed. The hash is
+    // the part a restore can refuse on.
+    providerContractSha256,
+    providers: providerSet.providers.map((provider) => ({
+      contractSchemaVersion: provider.contractSchemaVersion,
+      domain: provider.domain,
+      payloadRelations: provider.payloadRelations,
+      ledgerRelation: provider.ledgerRelation,
+      closureRoots: provider.closureRoots,
+      outboundWeakEdges: provider.outboundWeakEdges,
+      codecExtensions: provider.codecExtensions,
+    })),
+    shape: Object.fromEntries([...shape.columns]),
+    foreignKeys: shape.foreignKeys,
+    selfReferencing: Object.fromEntries([...selfReferencing]),
+    restoreOrder: order,
+    sequences: shape.sequences.map((sequence) => ({
+      name: sequence.name,
+      lastValue: sequence.last_value === null ? null : String(sequence.last_value),
+    })),
+    // Ledgers travel as evidence, never as payload to replay. See the provider
+    // contract for why: a ledger row asserts a migration ran, and asserting that
+    // independently of the DDL is a lie nothing downstream would catch.
+    ledgerEvidence,
+    chunks,
+    totalRows,
+    // Without the closure this bundle proves data fidelity and nothing more: it
+    // cannot rebuild a schema, so it is not disaster-recovery eligible. The flag
+    // is written from what actually happened rather than from intent.
+    closure: embeddedClosure,
+    migrationSetEmbedded: embeddedClosure !== null,
+  }
+  const manifestBytes = Buffer.from(JSON.stringify(manifest, null, 2))
+  writeAtomic(path.join(bundleDirectory, 'manifest.json'), manifestBytes)
+
+  // The integrity root lives OUTSIDE the bundle. Measured reason: with the chunk
+  // hashes and the manifest in one mutable file, a coordinated edit rewrites both
+  // and the bundle validates itself. That attack was accepted by an earlier
+  // version of this design; the anchor is what refuses it.
+  const bundleSha256 = sha256(manifestBytes)
+  const receipt = {
+    schemaVersion: 1,
+    contract: RECEIPT_CONTRACT,
+    bundleSha256,
+    backupSetId,
+    sourceLineageId: identity.lineageId ?? identity.id,
+    postgresMajor: identity.postgresMajor,
+    // Also outside the bundle. The bundle digest says "these bytes did not
+    // change"; this says "they were produced under the provider contract the
+    // lineage expects", which is a different claim and the one a coordinated
+    // edit of the bundle cannot forge.
+    providerContractSha256,
+    bundlePath: path.resolve(bundleDirectory),
+  }
+  mkdirSync(path.dirname(receiptPath), { recursive: true, mode: 0o700 })
+  writeAtomic(receiptPath, Buffer.from(JSON.stringify(receipt, null, 2)))
+
+  return {
+    backupSetId,
+    bundleSha256,
+    relations: allRelations.length,
+    rows: totalRows,
+    chunks: chunks.length,
+    fragments: new Set(chunks.map((chunk) => chunk.domain)).size,
+    compressedBytes: chunks.reduce((sum, chunk) => sum + chunk.compressedBytes, 0),
+    uncompressedBytes: chunks.reduce((sum, chunk) => sum + chunk.uncompressedBytes, 0),
+    largestChunkBytes,
+    closureFiles: embeddedClosure?.fileCount ?? 0,
+    migrationSetEmbedded: embeddedClosure !== null,
+  }
+}
+
+export { CONTRACT, RECEIPT_CONTRACT }