@better-forge/backup 0.0.0-stage → 0.1.0

package/src/archive.ts

@@ -0,0 +1,114 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import crypto from 'node:crypto'
+
+const HASH_SIZE = 32
+
+export interface ArchiveEntry { name: string; source: string }
+
+function safeName(name: string): string {
+  const normalized = name.replaceAll('\\', '/')
+  if (!normalized || normalized.startsWith('/') || normalized.includes('../') || normalized === '..') {
+    throw new Error(`Unsafe archive path: ${name}`)
+  }
+  return normalized
+}
+
+function sha256(file: string): Buffer {
+  const hash = crypto.createHash('sha256')
+  const fd = fs.openSync(file, 'r')
+  const buffer = Buffer.allocUnsafe(1024 * 1024)
+  try {
+    let count = 0
+    while ((count = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, count))
+  } finally { fs.closeSync(fd) }
+  return hash.digest()
+}
+
+function copyBytes(input: number, output: number, size: number, hash?: crypto.Hash): void {
+  const buffer = Buffer.allocUnsafe(1024 * 1024)
+  let remaining = size
+  while (remaining > 0) {
+    const count = fs.readSync(input, buffer, 0, Math.min(buffer.length, remaining), null)
+    if (count === 0) throw new Error('Unexpected end of backup archive')
+    const chunk = buffer.subarray(0, count)
+    if (hash) hash.update(chunk)
+    fs.writeSync(output, chunk)
+    remaining -= count
+  }
+}
+
+function readExact(fd: number, size: number): Buffer {
+  const buffer = Buffer.alloc(size)
+  let offset = 0
+  while (offset < size) {
+    const count = fs.readSync(fd, buffer, offset, size - offset, null)
+    if (!count) throw new Error('Truncated backup archive')
+    offset += count
+  }
+  return buffer
+}
+
+/**
+ * The archive file format: a magic header, then for each file its name, size and SHA-256 followed by its bytes.
+ * Extraction checks every checksum and refuses names that would leave the destination. The `magic` is what identifies
+ * archives of one product, so an application keeps the one its existing archives start with.
+ */
+export function createArchiveFormat(options: { magic: string }) {
+  const MAGIC = Buffer.from(options.magic)
+
+  function createArchive(output: string, entries: ArchiveEntry[]): void {
+    const fd = fs.openSync(output, 'wx', 0o600)
+    try {
+      fs.writeSync(fd, MAGIC)
+      for (const entry of entries) {
+        const name = Buffer.from(safeName(entry.name), 'utf8')
+        const stat = fs.statSync(entry.source)
+        if (!stat.isFile()) throw new Error(`Archive source is not a file: ${entry.source}`)
+        const header = Buffer.alloc(4 + 8 + HASH_SIZE)
+        header.writeUInt32BE(name.length, 0)
+        header.writeBigUInt64BE(BigInt(stat.size), 4)
+        sha256(entry.source).copy(header, 12)
+        fs.writeSync(fd, header)
+        fs.writeSync(fd, name)
+        const input = fs.openSync(entry.source, 'r')
+        try { copyBytes(input, fd, stat.size) } finally { fs.closeSync(input) }
+      }
+      fs.writeSync(fd, Buffer.alloc(4))
+      fs.fsyncSync(fd)
+    } finally { fs.closeSync(fd) }
+  }
+
+  function extractArchive(input: string, destination: string): string[] {
+    fs.mkdirSync(destination, { recursive: true, mode: 0o700 })
+    const root = path.resolve(destination)
+    const extracted: string[] = []
+    const fd = fs.openSync(input, 'r')
+    try {
+      if (!readExact(fd, MAGIC.length).equals(MAGIC)) throw new Error('Unsupported backup format')
+      for (let index = 0; index < 10000; index++) {
+        const nameLength = readExact(fd, 4).readUInt32BE(0)
+        if (nameLength === 0) return extracted
+        if (nameLength > 4096) throw new Error('Invalid archive entry name')
+        const metadata = readExact(fd, 8 + HASH_SIZE)
+        const sizeBig = metadata.readBigUInt64BE(0)
+        if (sizeBig > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('Archive entry is too large')
+        const expectedHash = metadata.subarray(8)
+        const name = safeName(readExact(fd, nameLength).toString('utf8'))
+        const target = path.resolve(root, ...name.split('/'))
+        if (target !== root && !target.startsWith(`${root}${path.sep}`)) throw new Error('Archive path escapes destination')
+        fs.mkdirSync(path.dirname(target), { recursive: true })
+        const output = fs.openSync(target, 'wx', 0o600)
+        const hash = crypto.createHash('sha256')
+        try { copyBytes(fd, output, Number(sizeBig), hash) } finally { fs.closeSync(output) }
+        if (!crypto.timingSafeEqual(hash.digest(), expectedHash)) throw new Error(`Checksum mismatch: ${name}`)
+        extracted.push(name)
+      }
+      throw new Error('Archive contains too many entries')
+    } finally { fs.closeSync(fd) }
+  }
+
+  return { createArchive, extractArchive }
+}
+
+export type ArchiveFormat = ReturnType<typeof createArchiveFormat>

package/src/cli.ts

@@ -0,0 +1,62 @@
+import path from 'node:path'
+import type { BackupService } from './service.js'
+
+function argument(argv: string[], name: string): string | undefined {
+  const index = argv.indexOf(name)
+  return index >= 0 ? argv[index + 1] : undefined
+}
+
+function fail(error: unknown): void {
+  console.error(error instanceof Error ? error.message : error)
+  process.exitCode = 1
+}
+
+export interface BackupCliOptions {
+  service: Pick<BackupService, 'createBackup' | 'validateBackup'>
+  /** The application's database handle: opened for the backup and closed after it. */
+  database: { open: () => void; close: () => void }
+  /** A backup file name as the usage line shows it (`<name>-backup-timestamp.backup`). */
+  exampleFile: string
+}
+
+/**
+ * The command line of `backup`: makes a backup (`--output <dir>` picks where) or, with `--verify <file>` /
+ * `--verify --input <file>`, checks one. Prints JSON; failures set a non-zero exit code.
+ */
+export async function runBackupCli(argv: string[], options: BackupCliOptions): Promise<void> {
+  try {
+    const verifyMode = argv.includes('--verify')
+    const verify = argument(argv, '--input') ?? argument(argv, '--verify')
+    if (verifyMode) {
+      if (!verify || verify.startsWith('--')) throw new Error(`Usage: backup:verify -- --input <${options.exampleFile}>`)
+      const manifest = options.service.validateBackup(path.resolve(verify))
+      console.log(JSON.stringify({ valid: true, manifest }, null, 2))
+      return
+    }
+    options.database.open()
+    try {
+      const output = argument(argv, '--output')
+      console.log(JSON.stringify(await options.service.createBackup(output ? path.resolve(output) : undefined), null, 2))
+    } finally { options.database.close() }
+  } catch (error) { fail(error) }
+}
+
+export interface RestoreCliOptions {
+  restore: (input: string, extra: string) => Promise<unknown>
+  exampleFile: string
+  /** The one extra value the restore needs (for example the key the backup was made with). */
+  extra: { flag: string; placeholder: string }
+}
+
+/** The command line of `restore`: `--input <file>` and the extra flag are both required. */
+export async function runRestoreCli(argv: string[], options: RestoreCliOptions): Promise<void> {
+  const input = argument(argv, '--input')
+  const extra = argument(argv, options.extra.flag)
+  if (!input || !extra) {
+    console.error(`Usage: restore --input <${options.exampleFile}> ${options.extra.flag} ${options.extra.placeholder}`)
+    process.exitCode = 1
+    return
+  }
+  try { console.log(JSON.stringify(await options.restore(path.resolve(input), extra), null, 2)) }
+  catch (error) { fail(error) }
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export { createArchiveFormat, type ArchiveEntry, type ArchiveFormat } from './archive.js'
+export {
+  collectFiles, createBackupService, DEFAULT_BACKUP_CONFIG, DEFAULT_BACKUP_STATUS, INTERRUPTED_BACKUP_ERROR,
+  type BackupConfig, type BackupInfo, type BackupManifest, type BackupService, type BackupServiceOptions, type BackupStatus, type ProcessOwner,
+} from './service.js'
+export { createRestore, replaceFile, type RestoreExtras, type RestoreOptions, type RestoreResult } from './restore.js'
+export { createBackupRunner, spawnBackupWorker, type BackupJob, type BackupWorkerOptions } from './runner.js'
+export { createBackupScheduler, scheduleExpression, type BackupSchedulerOptions } from './scheduler.js'
+export { createBackupRoutes, type BackupRoutesOptions } from './routes.js'
+export { runBackupCli, runRestoreCli, type BackupCliOptions, type RestoreCliOptions } from './cli.js'

package/src/restore.ts

@@ -0,0 +1,111 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+import { DatabaseSync } from 'node:sqlite'
+import type { BackupManifest, BackupService } from './service.js'
+
+export interface RestoreResult { restoredAt: number; safetyBackup: string | null }
+
+/** Puts `source` at `target` by copying beside it and renaming, so a failure never leaves half a file. */
+export function replaceFile(source: string, target: string): void {
+  fs.mkdirSync(path.dirname(target), { recursive: true })
+  const incoming = `${target}.restore-incoming`
+  fs.copyFileSync(source, incoming)
+  fs.renameSync(incoming, target)
+}
+
+function removeSqliteSidecars(dbPath: string): void {
+  fs.rmSync(`${dbPath}-wal`, { force: true })
+  fs.rmSync(`${dbPath}-shm`, { force: true })
+}
+
+/** The files around the database that a restore replaces, and how to put the old ones back. */
+export interface RestoreExtras {
+  /** Copies what is there now into `rollbackDir`. */
+  save: (rollbackDir: string) => void
+  /** Replaces what is there with what the backup holds (extracted into `extractedDir`). */
+  apply: (extractedDir: string) => void
+  /** Puts back what `save` kept (and removes what the failed restore created). */
+  rollback: (rollbackDir: string) => void
+}
+
+export interface RestoreOptions<Manifest extends BackupManifest> {
+  service: BackupService<Manifest>
+  name: string
+  /** How the application is called in the messages shown to its operator. */
+  displayName: string
+  databasePath: () => string
+  /** Throws unless the application is stopped. */
+  assertStopped: () => void
+  /** The application's own database handle: opened to take the safety backup and to migrate what was restored. */
+  database: { open: () => void; close: () => void; migrate: () => void }
+  /** Called once the restore is about to replace files; the flags it captures describe what existed before. */
+  extras: () => RestoreExtras
+}
+
+/**
+ * Restores a backup over a stopped application. A safety backup of what is there now is taken first, and a failure
+ * half way puts every replaced file back.
+ */
+export function createRestore<Manifest extends BackupManifest>(options: RestoreOptions<Manifest>) {
+  const { service, database } = options
+
+  /**
+   * `prepare` runs before the backup is read (to set up the key it will need); `check` may refuse the backup by
+   * throwing, once its manifest is known.
+   */
+  return async function restoreBackup(
+    input: string,
+    hooks: { prepare?: () => void; check?: (manifest: Manifest) => void } = {},
+  ): Promise<RestoreResult> {
+    options.assertStopped()
+    hooks.prepare?.()
+    const manifest = service.validateBackup(input)
+    hooks.check?.(manifest)
+
+    const dbPath = options.databasePath()
+    if (fs.existsSync(dbPath)) {
+      const lockCheck = new DatabaseSync(dbPath)
+      try { lockCheck.exec('PRAGMA busy_timeout=0; BEGIN EXCLUSIVE; ROLLBACK') }
+      catch { lockCheck.close(); throw new Error(`Database is in use. Stop ${options.displayName} before restoring.`) }
+      lockCheck.close()
+    }
+
+    let safetyBackup: string | null = null
+    if (fs.existsSync(dbPath)) {
+      database.open()
+      try { safetyBackup = (await service.createBackup()).filename } finally { database.close() }
+    }
+
+    const temp = fs.mkdtempSync(path.join(os.tmpdir(), `${options.name}-restore-`))
+    const rollback = fs.mkdtempSync(path.join(os.tmpdir(), `${options.name}-rollback-`))
+    const hadDatabase = fs.existsSync(dbPath)
+    const extras = options.extras()
+    try {
+      service.extractArchive(input, temp)
+      if (fs.existsSync(dbPath)) fs.copyFileSync(dbPath, path.join(rollback, 'database.sqlite'))
+      extras.save(rollback)
+
+      try {
+        removeSqliteSidecars(dbPath)
+        replaceFile(path.join(temp, 'database.sqlite'), dbPath)
+        extras.apply(temp)
+        database.open()
+        database.migrate()
+        database.close()
+      } catch (error) {
+        database.close()
+        removeSqliteSidecars(dbPath)
+        const oldDb = path.join(rollback, 'database.sqlite')
+        if (fs.existsSync(oldDb)) replaceFile(oldDb, dbPath)
+        else if (!hadDatabase) fs.rmSync(dbPath, { force: true })
+        extras.rollback(rollback)
+        throw error
+      }
+      return { restoredAt: Date.now(), safetyBackup }
+    } finally {
+      fs.rmSync(temp, { recursive: true, force: true })
+      fs.rmSync(rollback, { recursive: true, force: true })
+    }
+  }
+}

package/src/routes.ts

@@ -0,0 +1,115 @@
+import type { FastifyInstance, FastifyRequest } from 'fastify'
+import fs from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import type { Audit } from '@better-forge/audit'
+import { DEFAULT_BACKUP_CONFIG, type BackupConfig, type BackupInfo, type BackupManifest, type BackupService } from './service.js'
+
+/** The part of `@fastify/multipart` the upload route uses; the application registers the plugin. */
+interface UploadedFile { file: Readable & { truncated?: boolean } }
+type FileReader = (options: { limits: { fileSize: number } }) => Promise<UploadedFile | undefined>
+
+export interface BackupRoutesOptions<Manifest extends BackupManifest> {
+  service: BackupService<Manifest>
+  audit: Audit
+  /** The signed-in caller of a request (the routes sit behind the application's own guards). */
+  identity: (req: FastifyRequest) => { userId: number; email: string }
+  /** Makes a backup (on a worker thread, in the application). */
+  createBackup: () => Promise<BackupInfo>
+  /** Verifies a backup file (on a worker thread, in the application). */
+  validateBackup: (input: string) => Promise<Manifest>
+  /** Called after the schedule changed, to apply it. */
+  restartScheduler: () => void
+  /** Fields added to the answer of a verification, next to the manifest (e.g. whether a key matches). */
+  describeManifest?: (manifest: Manifest) => Record<string, unknown>
+}
+
+/** Largest backup file the verification accepts. */
+const MAX_VALIDATION_BYTES = 2 * 1024 * 1024 * 1024
+
+function parseConfig(value: Partial<BackupConfig>): BackupConfig {
+  const config = { ...DEFAULT_BACKUP_CONFIG, ...value }
+  if (!['daily', 'weekly'].includes(config.frequency)) throw new Error('Invalid frequency')
+  if (!Number.isInteger(config.hour) || config.hour < 0 || config.hour > 23) throw new Error('Hour must be between 0 and 23')
+  if (!Number.isInteger(config.minute) || config.minute < 0 || config.minute > 59) throw new Error('Minute must be between 0 and 59')
+  if (!Number.isInteger(config.weekday) || config.weekday < 0 || config.weekday > 6) throw new Error('Weekday must be between 0 and 6')
+  if (!Number.isInteger(config.retention) || config.retention < 1 || config.retention > 365) throw new Error('Retention must be between 1 and 365')
+  return config
+}
+
+const messageOf = (error: unknown) => error instanceof Error ? error.message : String(error)
+
+/** The admin routes of the backups: list, create, schedule, download, delete and verify an uploaded file. */
+export function createBackupRoutes<Manifest extends BackupManifest>(options: BackupRoutesOptions<Manifest>) {
+  const { service, audit } = options
+  const actor = (req: FastifyRequest) => audit.actor(options.identity(req))
+
+  return async function backupRoutes(app: FastifyInstance) {
+    app.get('/', async () => ({ backups: service.listBackups(), config: service.readBackupConfig(), status: service.readBackupStatus() }))
+
+    app.post('/', async (req, reply) => {
+      try {
+        const result = await options.createBackup()
+        service.applyRetention(service.readBackupConfig().retention)
+        await audit.write(actor(req), 'create', 'backup', result.filename, result.filename)
+        return result
+      } catch (error) { return reply.code(409).send({ error: messageOf(error) }) }
+    })
+
+    app.put<{ Body: Partial<BackupConfig> }>('/config', async (req, reply) => {
+      try {
+        const config = parseConfig(req.body)
+        service.writeBackupConfig(config)
+        service.applyRetention(config.retention)
+        options.restartScheduler()
+        await audit.write(actor(req), 'update', 'backup-config', null, 'Backup schedule', {
+          enabled: config.enabled,
+          frequency: config.frequency,
+          hour: config.hour,
+          minute: config.minute,
+          weekday: config.weekday,
+          retention: config.retention,
+        })
+        return config
+      } catch (error) { return reply.code(400).send({ error: messageOf(error) }) }
+    })
+
+    app.get<{ Params: { filename: string } }>('/:filename/download', async (req, reply) => {
+      try {
+        const file = service.backupFile(req.params.filename)
+        if (!fs.existsSync(file)) return reply.code(404).send({ error: 'Backup not found' })
+        await audit.write(actor(req), 'allow', 'backup', req.params.filename, req.params.filename, { access: 'downloaded' })
+        reply.header('Content-Type', 'application/octet-stream')
+        reply.header('Content-Disposition', `attachment; filename="${path.basename(file)}"`)
+        return reply.send(fs.createReadStream(file))
+      } catch (error) { return reply.code(400).send({ error: messageOf(error) }) }
+    })
+
+    app.delete<{ Params: { filename: string }; Querystring: { confirm?: string } }>('/:filename', async (req, reply) => {
+      try {
+        if (req.query.confirm !== req.params.filename) return reply.code(400).send({ error: 'Type the exact backup filename to confirm deletion' })
+        const file = service.backupFile(req.params.filename)
+        if (!fs.existsSync(file)) return reply.code(404).send({ error: 'Backup not found' })
+        fs.rmSync(file)
+        await audit.write(actor(req), 'delete', 'backup', req.params.filename, req.params.filename)
+        return reply.code(204).send()
+      } catch (error) { return reply.code(400).send({ error: messageOf(error) }) }
+    })
+
+    app.post('/validate', async (req, reply) => {
+      const data = await (req as unknown as { file: FileReader }).file({ limits: { fileSize: MAX_VALIDATION_BYTES } })
+      if (!data) return reply.code(400).send({ error: 'No backup file' })
+      const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-upload-'))
+      const input = path.join(tempDir, 'backup.backup')
+      try {
+        await pipeline(data.file, fs.createWriteStream(input, { mode: 0o600 }))
+        if (data.file.truncated) throw new Error('Backup file exceeds the 2 GB validation limit')
+        const manifest = await options.validateBackup(input)
+        return { valid: true, manifest, ...options.describeManifest?.(manifest) }
+      } catch (error) { return reply.code(400).send({ error: messageOf(error) }) }
+      finally { fs.rmSync(tempDir, { recursive: true, force: true }) }
+    })
+  }
+}

package/src/runner.ts

@@ -0,0 +1,45 @@
+import { Worker } from 'node:worker_threads'
+import type { BackupInfo, BackupManifest, BackupService } from './service.js'
+
+export type BackupJob =
+  | { task: 'create'; outputDirectory?: string | undefined }
+  | { task: 'validate'; input: string }
+
+export interface BackupWorkerOptions {
+  /** The worker script: the application's file that imports `@better-forge/backup/worker`. */
+  workerUrl: URL
+  /** The module the worker loads; it exports `createBackup` and `validateBackup`. */
+  moduleUrl: string
+  execArgv?: string[]
+}
+
+/** Runs one backup job on a worker thread; exposed with explicit URLs so tests can drive the real worker. */
+export function spawnBackupWorker<T>(job: BackupJob, options: BackupWorkerOptions): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(options.workerUrl, { workerData: { ...job, moduleUrl: options.moduleUrl }, execArgv: options.execArgv })
+    worker.once('message', (message: { result?: T; error?: string }) => message.error ? reject(new Error(message.error)) : resolve(message.result!))
+    worker.once('error', reject)
+    worker.once('exit', (code) => { if (code !== 0) reject(new Error(`Backup worker exited with code ${code}`)) })
+  })
+}
+
+/**
+ * Backups and verification off the event loop. With `inline` (development and tests, where the worker thread cannot
+ * load TypeScript) the same core runs in the calling thread instead.
+ */
+export function createBackupRunner<Manifest extends BackupManifest>(options: BackupWorkerOptions & {
+  service: BackupService<Manifest>
+  inline: boolean
+}) {
+  function run<T>(job: BackupJob, inline: () => Promise<T>): Promise<T> {
+    return options.inline ? inline() : spawnBackupWorker(job, options)
+  }
+
+  return {
+    createBackup: (outputDirectory?: string): Promise<BackupInfo> =>
+      run({ task: 'create', outputDirectory }, () => options.service.createBackup(outputDirectory)),
+    /** Extraction and the SQLite integrity check are synchronous, so keep them off the event loop. */
+    validateBackup: (input: string): Promise<Manifest> =>
+      run({ task: 'validate', input }, async () => options.service.validateBackup(input)),
+  }
+}

package/src/scheduler.ts

@@ -0,0 +1,39 @@
+import cron, { type ScheduledTask } from 'node-cron'
+import type { BackupConfig } from './service.js'
+
+export interface BackupSchedulerOptions {
+  readConfig: () => BackupConfig
+  /** Makes one scheduled backup (and prunes old ones). A rejection is logged, never thrown. */
+  run: (config: BackupConfig) => Promise<unknown>
+  /** Prefix of the lines it logs. */
+  logPrefix?: string
+}
+
+/** The cron expression of a schedule: every day, or one weekday, at the given time. */
+export function scheduleExpression(config: BackupConfig): string {
+  return config.frequency === 'weekly' ? `${config.minute} ${config.hour} * * ${config.weekday}` : `${config.minute} ${config.hour} * * *`
+}
+
+/** Runs a backup on the schedule the configuration describes; `restart` picks up a changed configuration. */
+export function createBackupScheduler(options: BackupSchedulerOptions) {
+  const prefix = options.logPrefix ?? '[backup]'
+  let task: ScheduledTask | null = null
+
+  function stop(): void {
+    task?.destroy()
+    task = null
+  }
+
+  function restart(): void {
+    stop()
+    const config = options.readConfig()
+    if (!config.enabled) return
+    const expression = scheduleExpression(config)
+    task = cron.schedule(expression, () => {
+      options.run(config).catch((error) => console.error(`${prefix} scheduled backup failed:`, error))
+    })
+    console.log(`${prefix} Scheduler started: ${expression}`)
+  }
+
+  return { restart, stop }
+}