@meith/backup 0.34.0

package/src/restore.ts

@@ -0,0 +1,332 @@
+import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, stat } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+
+import type { FileStore } from '@meith/core'
+import { ValidationError } from '@meith/core'
+import { compareVersions } from '@meith/upgrade'
+
+import { inspectArchive, type RestoreLimits } from './archive'
+import { type BackupManifest, parseManifest } from './bundle'
+import { type BackupLog, SILENT_LOG } from './create'
+import { postgresClientEnvironment, run } from './postgres-client'
+import { uploadDirectoryToStore } from './uploads'
+
+export type RestoreTargetMode = 'empty-database' | 'reset-schema'
+
+export interface RestoreTarget {
+  readonly url: string
+  readonly variable: string
+  readonly mode: RestoreTargetMode
+}
+
+export type RestoreUploadsPlan =
+  | { readonly mode: 'skip' }
+  | { readonly mode: 'directory'; readonly dir: string }
+  | {
+      readonly mode: 'store'
+      readonly store: { put: FileStore['put'] }
+      readonly description: string
+    }
+
+export interface RestoreInput {
+  readonly bundle: string
+  readonly target: RestoreTarget
+  readonly codeVersion: string
+  readonly migrate: (url: string) => Promise<number>
+  readonly uploads: RestoreUploadsPlan
+  readonly limits: RestoreLimits
+  readonly log?: BackupLog | undefined
+}
+
+export interface RestoreOutcome {
+  readonly manifest: BackupManifest
+  readonly migrationsApplied: number
+  readonly posts: number
+  readonly uploads: 'restored' | 'pushed' | 'skipped' | 'none'
+  readonly pushed: number
+}
+
+export function versionRefusal(manifestVersion: string, codeVersion: string): string | null {
+  let order: number
+  try {
+    order = compareVersions(manifestVersion, codeVersion)
+  } catch {
+    return null
+  }
+  if (order <= 0) return null
+  return (
+    `This bundle was taken by version ${manifestVersion} and this build is ${codeVersion}. ` +
+    'Migrations are forward-only, so a newer dump cannot be restored into older code: ' +
+    'deploy that version or newer first, then restore.'
+  )
+}
+
+async function validateUploadsArchive(stage: string, limits: RestoreLimits): Promise<void> {
+  const members = await inspectArchive(
+    path.join(stage, 'uploads.tar.gz'),
+    limits,
+    new Set(['-', 'd']),
+  )
+  for (const member of members) {
+    if (member.name === '.' && member.type !== 'd') {
+      throw new ValidationError('The uploads archive root is not a directory.')
+    }
+  }
+}
+
+async function refuseNonEmptyDirectory(dir: string): Promise<void> {
+  const existing = await readdir(dir).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === 'ENOENT') return undefined
+    throw error
+  })
+  if (existing !== undefined && existing.length > 0) {
+    throw new ValidationError(
+      `${dir} is not empty. Empty it first, or on the command line restore the uploads into ` +
+        'a fresh directory with --uploads-dir, the same way the database goes into a fresh ' +
+        'database.',
+    )
+  }
+}
+
+async function extractUploads(
+  stage: string,
+  dir: string,
+  limits: RestoreLimits,
+  log: BackupLog,
+): Promise<void> {
+  await validateUploadsArchive(stage, limits)
+  await refuseNonEmptyDirectory(dir)
+  await mkdir(dir, { recursive: true })
+  await run('tar', ['xzf', path.join(stage, 'uploads.tar.gz'), '-C', dir])
+  log.info(`Restored the uploads into ${dir}.`)
+}
+
+async function pushUploadsToStore(
+  stage: string,
+  limits: RestoreLimits,
+  store: { put: FileStore['put'] },
+  description: string,
+  log: BackupLog,
+): Promise<number> {
+  await validateUploadsArchive(stage, limits)
+  const dir = path.join(stage, 'uploads-extract')
+  await mkdir(dir, { recursive: true })
+  await run('tar', ['xzf', path.join(stage, 'uploads.tar.gz'), '-C', dir])
+
+  const pushed = await uploadDirectoryToStore(store, dir)
+  log.info(`Uploaded ${pushed} object(s) to ${description}.`)
+  return pushed
+}
+
+const FAIL_INHERITED_RUNS_SQL =
+  "update backup_runs set status = 'failed', finished_at = now(), heartbeat_at = now(), " +
+  "error = 'The backup was interrupted: the dump this board was restored from caught it " +
+  "mid-flight.' where status in ('queued', 'running') returning id"
+
+const RELEASE_INHERITED_LEASES_SQL =
+  'update tasks set locked_until = null where locked_until is not null returning key'
+
+async function tableExists(name: string, databaseEnvironment: NodeJS.ProcessEnv): Promise<boolean> {
+  const output = await run(
+    'psql',
+    ['-tAc', `select to_regclass('public.${name}') is not null`],
+    databaseEnvironment,
+  )
+  return output.trim() === 't'
+}
+
+function returnedRows(output: string): number {
+  return output.split('\n').filter((line) => line.trim() !== '').length
+}
+
+async function failInheritedRuns(databaseEnvironment: NodeJS.ProcessEnv): Promise<number> {
+  if (await tableExists('tasks', databaseEnvironment)) {
+    await run('psql', ['-tAc', RELEASE_INHERITED_LEASES_SQL], databaseEnvironment)
+  }
+  if (!(await tableExists('backup_runs', databaseEnvironment))) return 0
+  return returnedRows(await run('psql', ['-tAc', FAIL_INHERITED_RUNS_SQL], databaseEnvironment))
+}
+
+const RESET_SCHEMA_SQL = [
+  'drop schema if exists drizzle cascade;',
+  'drop schema if exists public cascade;',
+  'create schema public;',
+].join('\n')
+
+async function prepareTarget(
+  target: RestoreTarget,
+  databaseEnvironment: NodeJS.ProcessEnv,
+  log: BackupLog,
+): Promise<void> {
+  const tables = (
+    await run(
+      'psql',
+      ['-tAc', "select count(*) from information_schema.tables where table_schema = 'public'"],
+      databaseEnvironment,
+    )
+  ).trim()
+
+  if (target.mode === 'empty-database') {
+    if (tables !== '0') {
+      throw new ValidationError(
+        `The target database already holds ${tables} table(s). Restore into a new, ` +
+          'empty database — a restore over a live board is how a bad backup becomes ' +
+          'two lost boards.',
+      )
+    }
+    return
+  }
+
+  const members = (
+    await run(
+      'psql',
+      [
+        '-tAc',
+        "select case when to_regclass('public.users') is null then 0 " +
+          'else (select count(*) from users) end',
+      ],
+      databaseEnvironment,
+    )
+  ).trim()
+  if (members !== '0') {
+    throw new ValidationError(
+      `The target database holds ${members} member account(s). The installer only restores ` +
+        'over an empty, uninstalled board.',
+    )
+  }
+
+  if (tables !== '0') {
+    log.info(`Dropping the empty schema (${tables} table(s)) before the restore…`)
+  }
+  await run('psql', ['-v', 'ON_ERROR_STOP=1', '-q'], databaseEnvironment, RESET_SCHEMA_SQL)
+}
+
+export async function restoreBackup(input: RestoreInput): Promise<RestoreOutcome> {
+  const log = input.log ?? SILENT_LOG
+  const databaseEnvironment = postgresClientEnvironment(input.target.url, input.target.variable)
+
+  const bundleInfo = await stat(input.bundle).catch(() => undefined)
+  if (bundleInfo === undefined || !bundleInfo.isFile()) {
+    throw new ValidationError(`No such bundle: ${input.bundle}`)
+  }
+  if (bundleInfo.size > input.limits.archiveBytes) {
+    throw new ValidationError('The backup bundle exceeds MEITH_RESTORE_MAX_ARCHIVE_BYTES.')
+  }
+
+  const stage = await mkdtemp(path.join(tmpdir(), 'meith-restore-'))
+  try {
+    const stagedBundle = path.join(stage, 'bundle.tar.gz')
+    await copyFile(path.resolve(input.bundle), stagedBundle)
+    const members = await inspectArchive(stagedBundle, input.limits, new Set(['-']))
+    const possibleMembers = new Set(['manifest.json', 'db.dump', 'uploads.tar.gz'])
+    if (members.some((member) => !possibleMembers.has(member.name))) {
+      throw new ValidationError('The backup bundle contains an unexpected member.')
+    }
+    await run('tar', ['xzf', stagedBundle, '-C', stage, 'manifest.json'])
+    const manifest = parseManifest(await readFile(path.join(stage, 'manifest.json'), 'utf8'))
+    const expectedMembers = new Set(['manifest.json', 'db.dump'])
+    if (manifest.uploads === 'included') expectedMembers.add('uploads.tar.gz')
+    if (
+      members.length !== expectedMembers.size ||
+      members.some((member) => !expectedMembers.has(member.name))
+    ) {
+      throw new ValidationError('The backup bundle members do not match its manifest.')
+    }
+
+    const refusal = versionRefusal(manifest.version, input.codeVersion)
+    if (refusal !== null) throw new ValidationError(refusal)
+
+    const restoreMembers = ['db.dump']
+    if (manifest.uploads === 'included') restoreMembers.push('uploads.tar.gz')
+    await run('tar', ['xzf', stagedBundle, '-C', stage, ...restoreMembers])
+    if (manifest.uploads === 'included') {
+      await validateUploadsArchive(stage, input.limits)
+      if (input.uploads.mode === 'directory') await refuseNonEmptyDirectory(input.uploads.dir)
+    }
+
+    await prepareTarget(input.target, databaseEnvironment, log)
+
+    log.info(`Restoring the backup taken ${manifest.createdAt} (version ${manifest.version})…`)
+    await run(
+      'pg_restore',
+      [
+        '--no-owner',
+        '--no-privileges',
+        '--dbname',
+        databaseEnvironment.PGDATABASE ?? '',
+        path.join(stage, 'db.dump'),
+      ],
+      databaseEnvironment,
+    )
+
+    let step = 'applying the migrations the dump predates'
+    let migrationsApplied = 0
+    let posts = 0
+    let uploads: RestoreOutcome['uploads'] = 'none'
+    let pushed = 0
+    try {
+      migrationsApplied = await input.migrate(input.target.url)
+      log.info(
+        migrationsApplied === 0
+          ? 'Migrations: nothing to do — the dump matches this build.'
+          : `Migrations: applied ${migrationsApplied} migration(s) the dump predates.`,
+      )
+
+      const interrupted = await failInheritedRuns(databaseEnvironment)
+      if (interrupted > 0) {
+        log.info(`Marked ${interrupted} backup run(s) the dump caught mid-flight as failed.`)
+      }
+
+      posts = Number(
+        (await run('psql', ['-tAc', 'select count(*) from posts'], databaseEnvironment)).trim(),
+      )
+      log.info(`The restored board holds ${posts} post(s).`)
+
+      step = 'putting the uploads back'
+      if (manifest.uploads === 'included') {
+        if (input.uploads.mode === 'skip') {
+          uploads = 'skipped'
+        } else if (input.uploads.mode === 'store') {
+          pushed = await pushUploadsToStore(
+            stage,
+            input.limits,
+            input.uploads.store,
+            input.uploads.description,
+            log,
+          )
+          uploads = 'pushed'
+        } else {
+          await extractUploads(stage, input.uploads.dir, input.limits, log)
+          uploads = 'restored'
+        }
+      }
+    } catch (error) {
+      throw new ValidationError(
+        `The database was restored from ${path.basename(input.bundle)}, but ${step} failed: ${
+          error instanceof Error ? error.message : String(error)
+        } The board is installed and serving the restored database. ` +
+          (step === 'putting the uploads back'
+            ? 'Unpack uploads.tar.gz from the bundle into the uploads location by hand.'
+            : 'Fix the cause and run `meith migrate`.'),
+      )
+    }
+    if (manifest.uploads !== 'included') {
+      log.info(
+        manifest.filestore === 's3'
+          ? `This bundle carries no uploads — they live in the S3 bucket${
+              manifest.bucket === undefined ? '' : ` (${manifest.bucket})`
+            }.`
+          : manifest.filestore === 'blob'
+            ? 'This bundle carries no uploads, and a Vercel Blob store is not ' +
+              'something you can copy out by hand. Take another backup with ' +
+              '--uploads include while the old board still exists.'
+            : 'This bundle carries no uploads.',
+      )
+    }
+
+    return { manifest, migrationsApplied, posts, uploads, pushed }
+  } finally {
+    await rm(stage, { recursive: true, force: true })
+  }
+}

package/src/retention.ts

@@ -0,0 +1,45 @@
+import { ValidationError } from '@meith/core'
+
+import { bundleTakenAt, isBundleName } from './bundle'
+
+export const DEFAULT_KEEP = 7
+
+export interface RetentionPolicy {
+  readonly keep: number
+  readonly keepDays?: number | undefined
+}
+
+export function resolveKeep(flag: string | undefined): number {
+  if (flag === undefined) return DEFAULT_KEEP
+  if (!/^\d+$/.test(flag) || Number(flag) < 1 || !Number.isSafeInteger(Number(flag))) {
+    throw new ValidationError(`--keep must be a whole number of bundles, 1 or more, got "${flag}".`)
+  }
+  return Number(flag)
+}
+
+export function pruneCandidates(names: readonly string[], keep: number): readonly string[] {
+  return retentionCandidates(names, { keep })
+}
+
+export function retentionCandidates(
+  names: readonly string[],
+  policy: RetentionPolicy,
+  now: Date = new Date(),
+): readonly string[] {
+  const bundles = names
+    .filter((name) => isBundleName(name))
+    .sort()
+    .reverse()
+  const beyondCount = new Set(bundles.slice(Math.max(1, policy.keep)))
+
+  const keepDays = policy.keepDays ?? 0
+  if (keepDays > 0) {
+    const cutoff = now.getTime() - keepDays * 24 * 60 * 60 * 1000
+    for (const name of bundles.slice(1)) {
+      const takenAt = bundleTakenAt(name)
+      if (takenAt !== null && takenAt.getTime() < cutoff) beyondCount.add(name)
+    }
+  }
+
+  return bundles.filter((name) => beyondCount.has(name))
+}

package/src/runs.ts

@@ -0,0 +1,60 @@
+export type BackupTrigger = 'manual' | 'schedule' | 'upgrade' | 'cli'
+
+export type BackupRunStatus = 'queued' | 'running' | 'done' | 'incomplete' | 'failed'
+
+export interface BackupRunRecord {
+  readonly id: number
+  readonly trigger: BackupTrigger
+  readonly status: BackupRunStatus
+  readonly requestedByUserId: number | null
+  readonly requestedAt: Date
+  readonly startedAt: Date | null
+  readonly finishedAt: Date | null
+  readonly heartbeatAt: Date | null
+  readonly bundleName: string | null
+  readonly sizeBytes: number | null
+  readonly uploads: 'included' | 'skipped' | null
+  readonly shipped: boolean
+  readonly skippedKeys: number
+  readonly error: string | null
+}
+
+export interface BackupRunFinish {
+  readonly status: 'done' | 'incomplete' | 'failed'
+  readonly finishedAt: Date
+  readonly bundleName?: string | null | undefined
+  readonly sizeBytes?: number | null | undefined
+  readonly uploads?: 'included' | 'skipped' | null | undefined
+  readonly shipped?: boolean | undefined
+  readonly skippedKeys?: number | undefined
+  readonly error?: string | null | undefined
+}
+
+export interface BackupRunRepository {
+  enqueue(input: {
+    readonly trigger: BackupTrigger
+    readonly requestedByUserId?: number | null | undefined
+    readonly now: Date
+  }): Promise<{ readonly id: number; readonly queued: boolean }>
+
+  claimNext(now: Date): Promise<BackupRunRecord | null>
+
+  heartbeat(id: number, now: Date): Promise<void>
+
+  finish(id: number, outcome: BackupRunFinish): Promise<void>
+
+  active(now: Date, staleBefore: Date): Promise<BackupRunRecord | null>
+
+  recent(limit: number): Promise<readonly BackupRunRecord[]>
+
+  lastScheduledAt(): Promise<Date | null>
+
+  failInterrupted(now: Date, staleBefore: Date): Promise<number>
+
+  record(input: {
+    readonly trigger: BackupTrigger
+    readonly requestedByUserId?: number | null | undefined
+    readonly startedAt: Date
+    readonly outcome: BackupRunFinish
+  }): Promise<void>
+}

package/src/schedule.ts

@@ -0,0 +1,75 @@
+export type BackupFrequency = 'off' | 'daily' | 'weekly'
+
+export interface BackupSchedule {
+  readonly frequency: BackupFrequency
+  readonly hour: number
+  readonly minute: number
+  readonly weekday: number
+}
+
+export const SCHEDULE_TIME_PATTERN = /^([01]\d|2[0-3]):([0-5]\d)$/
+
+export function parseScheduleTime(value: string): { hour: number; minute: number } | null {
+  const match = SCHEDULE_TIME_PATTERN.exec(value.trim())
+  if (match === null) return null
+  return { hour: Number(match[1]), minute: Number(match[2]) }
+}
+
+export function formatScheduleTime(schedule: { hour: number; minute: number }): string {
+  return `${String(schedule.hour).padStart(2, '0')}:${String(schedule.minute).padStart(2, '0')}`
+}
+
+const DAY_MS = 24 * 60 * 60 * 1000
+
+function slotOnDay(day: Date, schedule: BackupSchedule): Date {
+  return new Date(
+    Date.UTC(
+      day.getUTCFullYear(),
+      day.getUTCMonth(),
+      day.getUTCDate(),
+      schedule.hour,
+      schedule.minute,
+    ),
+  )
+}
+
+export function latestSlotAtOrBefore(schedule: BackupSchedule, now: Date): Date | null {
+  if (schedule.frequency === 'off') return null
+
+  for (let back = 0; back < 8; back += 1) {
+    const day = new Date(now.getTime() - back * DAY_MS)
+    if (schedule.frequency === 'weekly' && day.getUTCDay() !== schedule.weekday) continue
+    const slot = slotOnDay(day, schedule)
+    if (slot.getTime() <= now.getTime()) return slot
+  }
+  return null
+}
+
+export function nextSlotAfter(schedule: BackupSchedule, from: Date): Date | null {
+  if (schedule.frequency === 'off') return null
+
+  for (let ahead = 0; ahead < 8; ahead += 1) {
+    const day = new Date(from.getTime() + ahead * DAY_MS)
+    if (schedule.frequency === 'weekly' && day.getUTCDay() !== schedule.weekday) continue
+    const slot = slotOnDay(day, schedule)
+    if (slot.getTime() > from.getTime()) return slot
+  }
+  return null
+}
+
+export function scheduledBackupDue(
+  schedule: BackupSchedule,
+  input: {
+    readonly now: Date
+    readonly lastTickAt: Date | null
+    readonly lastScheduledAt: Date | null
+  },
+): Date | null {
+  const slot = latestSlotAtOrBefore(schedule, input.now)
+  if (slot === null) return null
+  if (input.lastTickAt === null || slot.getTime() <= input.lastTickAt.getTime()) return null
+  if (input.lastScheduledAt !== null && input.lastScheduledAt.getTime() >= slot.getTime()) {
+    return null
+  }
+  return slot
+}

package/src/uploads.ts

@@ -0,0 +1,75 @@
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+
+import { type FileStore, unusableKeyReason } from '@meith/core'
+
+import { contentTypeFor } from './bundle'
+
+export interface ListableStore {
+  listKeys(): AsyncGenerator<string>
+  get(key: string): Promise<Uint8Array | undefined>
+}
+
+export interface DrainedStore {
+  readonly pulled: number
+  readonly skipped: readonly string[]
+}
+
+export async function drainStoreToDirectory(
+  store: ListableStore,
+  dir: string,
+  warn: (line: string) => void = () => undefined,
+): Promise<DrainedStore> {
+  let pulled = 0
+  const skipped: string[] = []
+
+  for await (const key of store.listKeys()) {
+    const target = path.resolve(dir, key)
+    if (target !== dir && !target.startsWith(dir + path.sep)) {
+      warn(`Skipping the object at ${JSON.stringify(key)}: its key escapes ${dir}.`)
+      skipped.push(key)
+      continue
+    }
+    const unusable = unusableKeyReason(key)
+    if (unusable !== undefined) {
+      warn(`Skipping the object at ${JSON.stringify(key)}: its key ${unusable}.`)
+      skipped.push(key)
+      continue
+    }
+    const body = await store.get(key)
+    if (body === undefined) continue
+    await mkdir(path.dirname(target), { recursive: true })
+    await writeFile(target, body)
+    pulled++
+  }
+
+  return { pulled, skipped }
+}
+
+export async function walk(dir: string): Promise<readonly string[]> {
+  const files: string[] = []
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name)
+    if (entry.isDirectory()) files.push(...(await walk(full)))
+    else if (entry.isFile()) files.push(full)
+  }
+  return files
+}
+
+export async function uploadDirectoryToStore(
+  store: { put: FileStore['put'] },
+  dir: string,
+): Promise<number> {
+  let pushed = 0
+
+  for (const file of await walk(dir)) {
+    const key = path.relative(dir, file).split(path.sep).join('/')
+    await store.put(key, await readFile(file), {
+      contentType: contentTypeFor(key),
+      visibility: 'public',
+    })
+    pushed++
+  }
+
+  return pushed
+}