@meith/backup 0.34.0

package/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jordan Harrison and the Meith contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@meith/backup",
+  "version": "0.34.0",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/meith-dev/meith.git",
+    "directory": "packages/backup"
+  },
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "files": [
+    "src",
+    "!src/**/*.test.*",
+    "!src/**/*.type-test.*"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.1111.0",
+    "@aws-sdk/s3-request-presigner": "3.1111.0",
+    "@meith/core": "0.34.0",
+    "@meith/upgrade": "0.34.0"
+  }
+}

package/src/archive.ts

@@ -0,0 +1,132 @@
+import path from 'node:path'
+
+import { ValidationError } from '@meith/core'
+
+import { run } from './postgres-client'
+
+export interface RestoreLimits {
+  readonly archiveBytes: number
+  readonly members: number
+  readonly memberBytes: number
+  readonly expandedBytes: number
+}
+
+const RESTORE_LIMIT_DEFAULTS: RestoreLimits = {
+  archiveBytes: 2 * 1024 * 1024 * 1024,
+  members: 100_000,
+  memberBytes: 1024 * 1024 * 1024,
+  expandedBytes: 8 * 1024 * 1024 * 1024,
+}
+
+function positiveInteger(value: string | undefined, variable: string, fallback: number): number {
+  if (value === undefined || value === '') return fallback
+  if (!/^\d+$/.test(value)) {
+    throw new ValidationError(`${variable} must be a positive integer number of bytes or members.`)
+  }
+  const parsed = Number(value)
+  if (!Number.isSafeInteger(parsed) || parsed < 1) {
+    throw new ValidationError(`${variable} must be a positive integer number of bytes or members.`)
+  }
+  return parsed
+}
+
+export function restoreLimits(
+  environment: Readonly<Record<string, string | undefined>>,
+): RestoreLimits {
+  return {
+    archiveBytes: positiveInteger(
+      environment.MEITH_RESTORE_MAX_ARCHIVE_BYTES,
+      'MEITH_RESTORE_MAX_ARCHIVE_BYTES',
+      RESTORE_LIMIT_DEFAULTS.archiveBytes,
+    ),
+    members: positiveInteger(
+      environment.MEITH_RESTORE_MAX_MEMBERS,
+      'MEITH_RESTORE_MAX_MEMBERS',
+      RESTORE_LIMIT_DEFAULTS.members,
+    ),
+    memberBytes: positiveInteger(
+      environment.MEITH_RESTORE_MAX_MEMBER_BYTES,
+      'MEITH_RESTORE_MAX_MEMBER_BYTES',
+      RESTORE_LIMIT_DEFAULTS.memberBytes,
+    ),
+    expandedBytes: positiveInteger(
+      environment.MEITH_RESTORE_MAX_EXPANDED_BYTES,
+      'MEITH_RESTORE_MAX_EXPANDED_BYTES',
+      RESTORE_LIMIT_DEFAULTS.expandedBytes,
+    ),
+  }
+}
+
+export interface ArchiveMember {
+  readonly name: string
+  readonly type: string
+  readonly size: number
+}
+
+function normalizedArchiveName(name: string): string | undefined {
+  if (name === '' || name.includes('\\') || name.includes('\0')) return undefined
+  const withoutDirectoryMarker = name.endsWith('/') ? name.slice(0, -1) : name
+  if (withoutDirectoryMarker === '.') return '.'
+  const normalized = withoutDirectoryMarker.replace(/^\.\//, '')
+  if (normalized === '' || path.posix.isAbsolute(normalized)) return undefined
+  const parts = normalized.split('/')
+  if (parts.some((part) => part === '' || part === '.' || part === '..')) return undefined
+  return normalized
+}
+
+function memberSize(fields: readonly string[]): number {
+  return Number(fields[1]?.includes('/') ? fields[2] : fields[4])
+}
+
+export function validateArchiveListing(
+  namesOutput: string,
+  verboseOutput: string,
+  limits: RestoreLimits,
+  allowedTypes: ReadonlySet<string>,
+): readonly ArchiveMember[] {
+  const names = namesOutput === '' ? [] : namesOutput.replace(/\n$/, '').split('\n')
+  const verbose = verboseOutput === '' ? [] : verboseOutput.replace(/\n$/, '').split('\n')
+  if (names.length !== verbose.length) {
+    throw new ValidationError('The archive has malformed member names.')
+  }
+  if (names.length > limits.members) {
+    throw new ValidationError(`The archive has more than ${limits.members} members.`)
+  }
+
+  const seen = new Set<string>()
+  let expandedBytes = 0
+  return names.map((rawName, index) => {
+    const name = normalizedArchiveName(rawName)
+    if (name === undefined || seen.has(name)) {
+      throw new ValidationError(`The archive contains an unsafe or duplicate member: ${rawName}`)
+    }
+    seen.add(name)
+
+    const fields = verbose[index]?.trim().split(/\s+/) ?? []
+    const type = fields[0]?.[0] ?? ''
+    const size = memberSize(fields)
+    if (!allowedTypes.has(type) || !Number.isSafeInteger(size) || size < 0) {
+      throw new ValidationError(`The archive contains an unsupported member: ${name}`)
+    }
+    if (size > limits.memberBytes) {
+      throw new ValidationError(`The archive member ${name} exceeds the per-member size limit.`)
+    }
+    expandedBytes += size
+    if (!Number.isSafeInteger(expandedBytes) || expandedBytes > limits.expandedBytes) {
+      throw new ValidationError('The archive exceeds the expanded-size limit.')
+    }
+    return { name, type, size }
+  })
+}
+
+export async function inspectArchive(
+  archive: string,
+  limits: RestoreLimits,
+  allowedTypes: ReadonlySet<string>,
+): Promise<readonly ArchiveMember[]> {
+  const [names, verbose] = await Promise.all([
+    run('tar', ['tzf', archive]),
+    run('tar', ['tvzf', archive]),
+  ])
+  return validateArchiveListing(names, verbose, limits, allowedTypes)
+}

package/src/bundle.ts

@@ -0,0 +1,129 @@
+import path from 'node:path'
+
+import { ValidationError } from '@meith/core'
+
+export type FilestoreDriver = 'local' | 's3' | 'blob'
+
+export type UploadsMode = 'include' | 'skip'
+
+export interface BackupManifest {
+  readonly format: 1
+  readonly createdAt: string
+  readonly version: string
+  readonly filestore: FilestoreDriver
+  readonly uploads: 'included' | 'skipped'
+  readonly bucket?: string
+  readonly skippedKeys?: readonly string[]
+}
+
+export const BUNDLE_NAME_PATTERN =
+  /^meith-backup-(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})Z\.tar\.gz$/
+
+export function isBundleName(name: string): boolean {
+  return BUNDLE_NAME_PATTERN.test(name)
+}
+
+export function bundleName(at: Date): string {
+  const stamp = at
+    .toISOString()
+    .replace(/\.\d+Z$/, 'Z')
+    .replaceAll(':', '-')
+  return `meith-backup-${stamp}.tar.gz`
+}
+
+export function bundleTakenAt(name: string): Date | null {
+  const match = BUNDLE_NAME_PATTERN.exec(name)
+  if (match === null) return null
+  const at = new Date(`${match[1]}T${match[2]}:${match[3]}:${match[4]}Z`)
+  return Number.isNaN(at.getTime()) ? null : at
+}
+
+export function resolveUploadsMode(driver: FilestoreDriver, flag: string | undefined): UploadsMode {
+  if (flag === undefined || flag === 'auto') return driver === 's3' ? 'skip' : 'include'
+  if (flag === 'include' || flag === 'skip') return flag
+  throw new ValidationError(`--uploads must be "include" or "skip", got "${flag}".`)
+}
+
+export function parseManifest(raw: string): BackupManifest {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw)
+  } catch {
+    throw new ValidationError('The bundle manifest is not valid JSON.')
+  }
+
+  const manifest = parsed as Partial<BackupManifest>
+  if (manifest.format !== 1) {
+    throw new ValidationError(
+      `This bundle declares format ${JSON.stringify(manifest.format)}; this build restores format 1.`,
+    )
+  }
+  if (manifest.uploads !== 'included' && manifest.uploads !== 'skipped') {
+    throw new ValidationError('The bundle manifest does not say whether uploads are included.')
+  }
+  if (typeof manifest.createdAt !== 'string' || typeof manifest.version !== 'string') {
+    throw new ValidationError('The bundle manifest is missing createdAt or version.')
+  }
+  if (
+    manifest.filestore !== 'local' &&
+    manifest.filestore !== 's3' &&
+    manifest.filestore !== 'blob'
+  ) {
+    throw new ValidationError('The bundle manifest does not name a known file driver.')
+  }
+
+  const skippedKeys = manifest.skippedKeys
+  if (
+    skippedKeys !== undefined &&
+    (!Array.isArray(skippedKeys) || skippedKeys.some((key) => typeof key !== 'string'))
+  ) {
+    throw new ValidationError('The bundle manifest lists skipped objects in a form it cannot read.')
+  }
+
+  return {
+    format: 1,
+    createdAt: manifest.createdAt,
+    version: manifest.version,
+    filestore: manifest.filestore,
+    uploads: manifest.uploads,
+    ...(typeof manifest.bucket === 'string' ? { bucket: manifest.bucket } : {}),
+    ...(skippedKeys === undefined || skippedKeys.length === 0 ? {} : { skippedKeys }),
+  }
+}
+
+const CONTENT_TYPES: ReadonlyMap<string, string> = new Map([
+  ['.avif', 'image/avif'],
+  ['.gif', 'image/gif'],
+  ['.jpeg', 'image/jpeg'],
+  ['.jpg', 'image/jpeg'],
+  ['.png', 'image/png'],
+  ['.svg', 'image/svg+xml'],
+  ['.webp', 'image/webp'],
+])
+
+export function contentTypeFor(key: string): string {
+  return CONTENT_TYPES.get(path.extname(key).toLowerCase()) ?? 'application/octet-stream'
+}
+
+export function formatBytes(size: number): string {
+  let value = size
+  let unit = 'B'
+  for (const next of ['KiB', 'MiB', 'GiB', 'TiB']) {
+    if (value < 1024) break
+    value /= 1024
+    unit = next
+  }
+  return unit === 'B' ? `${value} B` : `${value.toFixed(value >= 10 ? 0 : 1)} ${unit}`
+}
+
+const SKIPPED_KEYS_LISTED = 10
+
+export function skippedKeyLines(keys: readonly string[]): readonly string[] {
+  const shown = keys.slice(0, SKIPPED_KEYS_LISTED)
+  return [
+    ...shown.map((key) => `  ${JSON.stringify(key)}`),
+    ...(keys.length > shown.length
+      ? [`  …and ${keys.length - shown.length} more, listed in the bundle's manifest.json.`]
+      : []),
+  ]
+}

package/src/capability.ts

@@ -0,0 +1,10 @@
+export type BackupCapability = 'available' | 'fixture' | 'serverless'
+
+export function backupCapability(environment: {
+  readonly DATA_SOURCE?: string | undefined
+  readonly VERCEL?: string | undefined
+}): BackupCapability {
+  if (environment.DATA_SOURCE !== 'postgres') return 'fixture'
+  if (environment.VERCEL !== undefined && environment.VERCEL !== '') return 'serverless'
+  return 'available'
+}

package/src/create.ts

@@ -0,0 +1,316 @@
+import { chmod, mkdir, mkdtemp, open, readdir, rm, stat, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+
+import { ValidationError } from '@meith/core'
+
+import {
+  type BackupManifest,
+  bundleName,
+  type FilestoreDriver,
+  formatBytes,
+  isBundleName,
+  type UploadsMode,
+} from './bundle'
+import type { BackupDestination } from './destination'
+import { postgresClientEnvironment, run } from './postgres-client'
+import { type RetentionPolicy, retentionCandidates } from './retention'
+import { drainStoreToDirectory, type ListableStore } from './uploads'
+
+export interface BackupLog {
+  info(line: string): void
+  warn(line: string): void
+}
+
+export const SILENT_LOG: BackupLog = { info: () => undefined, warn: () => undefined }
+
+export interface BackupSource {
+  readonly databaseUrl: string
+  readonly databaseVariable: string
+  readonly version: string
+  readonly filestore: FilestoreDriver
+  readonly uploadsDir: string
+  readonly objectStore?:
+    | { readonly store: ListableStore; readonly origin: string; readonly bucket?: string }
+    | undefined
+}
+
+export interface BackupTarget {
+  readonly out?: string | undefined
+  readonly dir?: string | undefined
+  readonly destination?: BackupDestination | undefined
+  readonly retention: RetentionPolicy
+}
+
+export interface CreateBackupInput {
+  readonly source: BackupSource
+  readonly target: BackupTarget
+  readonly uploads: UploadsMode
+  readonly now?: Date | undefined
+  readonly log?: BackupLog | undefined
+  readonly translateWriteError?: ((error: unknown, destination: string) => never) | undefined
+}
+
+export interface BackupOutcome {
+  readonly path: string
+  readonly name: string
+  readonly size: number
+  readonly createdAt: Date
+  readonly uploads: 'included' | 'skipped'
+  readonly skippedKeys: readonly string[]
+  readonly shipped: string | null
+  readonly prunedLocal: readonly string[]
+  readonly prunedRemote: readonly string[]
+}
+
+export interface WrittenBundle {
+  readonly name: string
+  readonly size: number
+  readonly uploads: 'included' | 'skipped'
+  readonly skippedKeys: readonly string[]
+}
+
+export class BackupShippingError extends Error {
+  constructor(
+    cause: unknown,
+    readonly bundle: WrittenBundle,
+  ) {
+    super(
+      `${bundle.name} was written but not shipped: ${
+        cause instanceof Error ? cause.message : String(cause)
+      }`,
+    )
+    this.name = 'BackupShippingError'
+  }
+}
+
+export async function reserveBackupDestination(destination: string): Promise<void> {
+  const file = await open(destination, 'wx', 0o600)
+  await file.close()
+}
+
+export async function claimBackupDestination(
+  destination: string,
+  translateWriteError?: (error: unknown, destination: string) => never,
+): Promise<void> {
+  try {
+    await reserveBackupDestination(destination)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException | undefined)?.code === 'EEXIST') {
+      throw new ValidationError(
+        `backup will not write over ${destination}: something is already there. Move it aside ` +
+          'or pass a different --out. A previous run killed part-way through can leave an ' +
+          'empty or truncated bundle at the path it had claimed; that file is not a backup ' +
+          'and is safe to delete.',
+      )
+    }
+    if (translateWriteError !== undefined) translateWriteError(error, destination)
+    throw error
+  }
+}
+
+interface StagedUploads {
+  readonly uploads: 'included' | 'skipped'
+  readonly skippedKeys: readonly string[]
+}
+
+async function stageLocalUploads(
+  stage: string,
+  uploadsDir: string,
+  log: BackupLog,
+): Promise<StagedUploads> {
+  const exists = await stat(uploadsDir).then(
+    (info) => info.isDirectory(),
+    () => false,
+  )
+  if (!exists) {
+    log.info(`No uploads directory at ${uploadsDir}; the bundle carries none.`)
+    return { uploads: 'skipped', skippedKeys: [] }
+  }
+
+  await run('tar', ['czf', path.join(stage, 'uploads.tar.gz'), '-C', uploadsDir, '.'])
+  return { uploads: 'included', skippedKeys: [] }
+}
+
+async function stageObjectStoreUploads(
+  stage: string,
+  store: ListableStore,
+  origin: string,
+  log: BackupLog,
+): Promise<StagedUploads> {
+  const dir = path.join(stage, 'uploads')
+  await mkdir(dir, { recursive: true })
+
+  const { pulled, skipped } = await drainStoreToDirectory(store, dir, (line) => log.warn(line))
+
+  if (pulled === 0) {
+    log.info(`Found no objects in ${origin}; the bundle carries no uploads.`)
+    await rm(dir, { recursive: true, force: true })
+    return { uploads: 'skipped', skippedKeys: skipped }
+  }
+
+  await run('tar', ['czf', path.join(stage, 'uploads.tar.gz'), '-C', dir, '.'])
+  await rm(dir, { recursive: true, force: true })
+  log.info(`Pulled ${pulled} object(s) from ${origin}.`)
+  return { uploads: 'included', skippedKeys: skipped }
+}
+
+async function stageUploads(
+  stage: string,
+  source: BackupSource,
+  mode: UploadsMode,
+  log: BackupLog,
+): Promise<StagedUploads> {
+  if (mode === 'skip') return { uploads: 'skipped', skippedKeys: [] }
+
+  if (source.filestore === 'local') return stageLocalUploads(stage, source.uploadsDir, log)
+
+  if (source.objectStore === undefined) {
+    throw new ValidationError(
+      `The uploads live in a ${source.filestore} store and no store was given to read them from.`,
+    )
+  }
+  return stageObjectStoreUploads(stage, source.objectStore.store, source.objectStore.origin, log)
+}
+
+export async function createBackup(input: CreateBackupInput): Promise<BackupOutcome> {
+  const log = input.log ?? SILENT_LOG
+  const { source, target } = input
+  if (target.out !== undefined && target.dir !== undefined) {
+    throw new ValidationError('--out and --dir are two answers to one question; pass one.')
+  }
+
+  const now = input.now ?? new Date()
+  const name = bundleName(now)
+  if (target.dir !== undefined) await mkdir(path.resolve(target.dir), { recursive: true })
+  const out = path.resolve(
+    target.dir === undefined ? (target.out ?? name) : path.join(target.dir, name),
+  )
+
+  const stage = await mkdtemp(path.join(tmpdir(), 'meith-backup-'))
+  let destinationCreated = false
+  try {
+    await claimBackupDestination(out, input.translateWriteError)
+    destinationCreated = true
+
+    log.info(
+      source.databaseVariable === 'DIRECT_DATABASE_URL'
+        ? 'Dumping the database over DIRECT_DATABASE_URL…'
+        : 'Dumping the database…',
+    )
+    const databaseEnvironment = postgresClientEnvironment(
+      source.databaseUrl,
+      source.databaseVariable,
+    )
+    await run(
+      'pg_dump',
+      ['--format=custom', '--no-owner', '--no-privileges', '--file', path.join(stage, 'db.dump')],
+      databaseEnvironment,
+    )
+
+    const { uploads, skippedKeys } = await stageUploads(stage, source, input.uploads, log)
+
+    const bucket = source.objectStore?.bucket
+    const manifest: BackupManifest = {
+      format: 1,
+      createdAt: now.toISOString(),
+      version: source.version,
+      filestore: source.filestore,
+      uploads,
+      ...(bucket === undefined ? {} : { bucket }),
+      ...(skippedKeys.length === 0 ? {} : { skippedKeys }),
+    }
+    await writeFile(path.join(stage, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`)
+
+    const members = ['manifest.json', 'db.dump']
+    if (uploads === 'included') members.push('uploads.tar.gz')
+    await run('tar', ['czf', out, '-C', stage, ...members])
+    await chmod(out, 0o600)
+
+    const size = (await stat(out)).size
+    destinationCreated = false
+    log.info(
+      `Wrote ${out} (${formatBytes(size)}): the database dump${
+        uploads === 'included' ? ' and the uploads' : ', no uploads'
+      }.`,
+    )
+    if (uploads === 'skipped' && source.filestore === 's3' && input.uploads !== 'include') {
+      log.info(
+        'The S3 bucket was not pulled — it has its own backup story. ' +
+          'Run with --uploads include for a bundle that carries every object.',
+      )
+    }
+    if (uploads === 'skipped' && input.uploads === 'skip') {
+      log.info('Restoring this bundle gives a board whose posts have broken images.')
+    }
+
+    let shipped: string | null = null
+    let prunedRemote: readonly string[] = []
+    if (target.destination !== undefined) {
+      try {
+        await target.destination.putFile(name, out, size)
+        shipped = target.destination.description
+        log.info(`Shipped ${name} to ${shipped}.`)
+        prunedRemote = await target.destination.prune(target.retention, now)
+      } catch (error) {
+        throw new BackupShippingError(error, { name, size, uploads, skippedKeys })
+      }
+      if (prunedRemote.length > 0) {
+        log.info(
+          `Pruned ${prunedRemote.length} bundle(s) there beyond the retention policy: ` +
+            `${prunedRemote.join(', ')}.`,
+        )
+      }
+    } else {
+      log.info(
+        'Copy the bundle off this machine: a backup on the server is a backup of the ' +
+          'thing most likely to fail.',
+      )
+    }
+
+    let prunedLocal: readonly string[] = []
+    if (target.dir !== undefined) {
+      const dir = path.resolve(target.dir)
+      prunedLocal = retentionCandidates(await readdir(dir), target.retention, now)
+      for (const staleName of prunedLocal) await rm(path.join(dir, staleName), { force: true })
+      if (prunedLocal.length > 0) {
+        log.info(
+          `Pruned ${prunedLocal.length} bundle(s) in ${dir} beyond the retention policy: ` +
+            `${prunedLocal.join(', ')}.`,
+        )
+      }
+    }
+
+    return {
+      path: out,
+      name,
+      size,
+      createdAt: now,
+      uploads,
+      skippedKeys,
+      shipped,
+      prunedLocal,
+      prunedRemote,
+    }
+  } finally {
+    if (destinationCreated) await rm(out, { force: true })
+    await rm(stage, { recursive: true, force: true })
+  }
+}
+
+export interface LocalBundle {
+  readonly name: string
+  readonly size: number
+}
+
+export async function localBundles(dir: string): Promise<readonly LocalBundle[]> {
+  const names = await readdir(dir).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === 'ENOENT') return []
+    throw error
+  })
+  const bundles: LocalBundle[] = []
+  for (const name of names.filter((entry) => isBundleName(entry)).sort()) {
+    bundles.push({ name, size: (await stat(path.join(dir, name))).size })
+  }
+  return bundles
+}