@meith/db 0.33.4 → 0.34.0

package/migrations/meta/_journal.json

@@ -477,6 +477,20 @@
       "when": 1788464192391,
       "tag": "0067_minimal_navigation",
       "breakpoints": true
+    },
+    {
+      "idx": 68,
+      "version": "7",
+      "when": 1788501076230,
+      "tag": "0068_backup_runs",
+      "breakpoints": true
+    },
+    {
+      "idx": 69,
+      "version": "7",
+      "when": 1788517301313,
+      "tag": "0069_backup_runs_active",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meith/db",
-  "version": "0.33.4",
+  "version": "0.34.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -22,36 +22,37 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "postgres": "^3.4.7",
-    "@meith/accounts": "0.33.4",
-    "@meith/admin": "0.33.4",
-    "@meith/antispam": "0.33.4",
-    "@meith/api": "0.33.4",
-    "@meith/attachments": "0.33.4",
-    "@meith/authorization": "0.33.4",
-    "@meith/avatars": "0.33.4",
-    "@meith/board-digest": "0.33.4",
-    "@meith/core": "0.33.4",
-    "@meith/drafts": "0.33.4",
-    "@meith/events": "0.33.4",
-    "@meith/forums": "0.33.4",
-    "@meith/groups": "0.33.4",
-    "@meith/i18n": "0.33.4",
-    "@meith/markdown": "0.33.4",
-    "@meith/marketplace": "0.33.4",
-    "@meith/messages": "0.33.4",
-    "@meith/moderation": "0.33.4",
-    "@meith/notifications": "0.33.4",
-    "@meith/plugin-kit": "0.33.4",
-    "@meith/polls": "0.33.4",
-    "@meith/profile-fields": "0.33.4",
-    "@meith/relations": "0.33.4",
-    "@meith/reputation": "0.33.4",
-    "@meith/search": "0.33.4",
-    "@meith/settings": "0.33.4",
-    "@meith/signatures": "0.33.4",
-    "@meith/subscriptions": "0.33.4",
-    "@meith/tasks": "0.33.4",
-    "@meith/threads": "0.33.4"
+    "@meith/accounts": "0.34.0",
+    "@meith/admin": "0.34.0",
+    "@meith/antispam": "0.34.0",
+    "@meith/api": "0.34.0",
+    "@meith/attachments": "0.34.0",
+    "@meith/authorization": "0.34.0",
+    "@meith/avatars": "0.34.0",
+    "@meith/backup": "0.34.0",
+    "@meith/board-digest": "0.34.0",
+    "@meith/core": "0.34.0",
+    "@meith/drafts": "0.34.0",
+    "@meith/events": "0.34.0",
+    "@meith/groups": "0.34.0",
+    "@meith/forums": "0.34.0",
+    "@meith/i18n": "0.34.0",
+    "@meith/markdown": "0.34.0",
+    "@meith/marketplace": "0.34.0",
+    "@meith/messages": "0.34.0",
+    "@meith/moderation": "0.34.0",
+    "@meith/notifications": "0.34.0",
+    "@meith/plugin-kit": "0.34.0",
+    "@meith/polls": "0.34.0",
+    "@meith/profile-fields": "0.34.0",
+    "@meith/relations": "0.34.0",
+    "@meith/reputation": "0.34.0",
+    "@meith/search": "0.34.0",
+    "@meith/settings": "0.34.0",
+    "@meith/subscriptions": "0.34.0",
+    "@meith/signatures": "0.34.0",
+    "@meith/tasks": "0.34.0",
+    "@meith/threads": "0.34.0"
   },
   "devDependencies": {
     "drizzle-kit": "^0.31.6",

package/src/backup-run-repo.ts

@@ -0,0 +1,212 @@
+import { sql } from 'drizzle-orm'
+
+import type {
+  BackupRunFinish,
+  BackupRunRecord,
+  BackupRunRepository,
+  BackupRunStatus,
+  BackupTrigger,
+} from '@meith/backup'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+import { toDate } from './row-values'
+
+const COLUMNS = sql`
+  id, trigger, status, requested_by_user_id, requested_at, started_at, finished_at,
+  heartbeat_at, bundle_name, size_bytes, uploads, shipped, skipped_keys, error
+`
+
+const UNIQUE_VIOLATION = '23505'
+
+function isUniqueViolation(error: unknown): boolean {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    ((error as { code?: unknown }).code === UNIQUE_VIOLATION ||
+      (error as { cause?: { code?: unknown } }).cause?.code === UNIQUE_VIOLATION)
+  )
+}
+
+function toRecord(row: Record<string, unknown>): BackupRunRecord {
+  const optionalDate = (value: unknown): Date | null => (value === null ? null : toDate(value))
+  return {
+    id: Number(row.id),
+    trigger: String(row.trigger) as BackupTrigger,
+    status: String(row.status) as BackupRunStatus,
+    requestedByUserId: row.requested_by_user_id === null ? null : Number(row.requested_by_user_id),
+    requestedAt: toDate(row.requested_at),
+    startedAt: optionalDate(row.started_at),
+    finishedAt: optionalDate(row.finished_at),
+    heartbeatAt: optionalDate(row.heartbeat_at),
+    bundleName: row.bundle_name === null ? null : String(row.bundle_name),
+    sizeBytes: row.size_bytes === null ? null : Number(row.size_bytes),
+    uploads:
+      row.uploads === 'included' || row.uploads === 'skipped'
+        ? (row.uploads as 'included' | 'skipped')
+        : null,
+    shipped: row.shipped === true,
+    skippedKeys: Number(row.skipped_keys),
+    error: row.error === null ? null : String(row.error),
+  }
+}
+
+export class PostgresBackupRunRepository implements BackupRunRepository {
+  constructor(private readonly db: Database) {}
+
+  async enqueue(input: {
+    readonly trigger: BackupTrigger
+    readonly requestedByUserId?: number | null | undefined
+    readonly now: Date
+  }): Promise<{ readonly id: number; readonly queued: boolean }> {
+    const pending = async (): Promise<number | undefined> => {
+      const row = resultRows<{ id: number }>(
+        await this.db.execute(sql`
+          select id from backup_runs
+           where status in ('queued', 'running')
+           order by id
+           limit 1
+        `),
+      )[0]
+      return row === undefined ? undefined : Number(row.id)
+    }
+
+    const existing = await pending()
+    if (existing !== undefined) return { id: existing, queued: false }
+
+    try {
+      const inserted = resultRows<{ id: number }>(
+        await this.db.execute(sql`
+          insert into backup_runs (trigger, status, requested_by_user_id, requested_at)
+          values (${input.trigger}, 'queued', ${input.requestedByUserId ?? null}, ${input.now})
+          returning id
+        `),
+      )[0]
+      return { id: Number(inserted?.id), queued: true }
+    } catch (error) {
+      if (!isUniqueViolation(error)) throw error
+      const raced = await pending()
+      if (raced === undefined) throw error
+      return { id: raced, queued: false }
+    }
+  }
+
+  async claimNext(now: Date): Promise<BackupRunRecord | null> {
+    let rows: Array<Record<string, unknown>>
+    try {
+      rows = resultRows(
+        await this.db.execute(sql`
+          update backup_runs
+             set status = 'running', started_at = ${now}, heartbeat_at = ${now}
+           where id = (
+             select id from backup_runs
+              where status = 'queued'
+                and not exists (select 1 from backup_runs where status = 'running')
+              order by id
+              limit 1
+                for update skip locked
+           )
+          returning ${COLUMNS}
+        `),
+      ) as Array<Record<string, unknown>>
+    } catch (error) {
+      if (isUniqueViolation(error)) return null
+      throw error
+    }
+    const row = rows[0]
+    return row === undefined ? null : toRecord(row)
+  }
+
+  async heartbeat(id: number, now: Date): Promise<void> {
+    await this.db.execute(sql`
+      update backup_runs set heartbeat_at = ${now} where id = ${id} and status = 'running'
+    `)
+  }
+
+  async finish(id: number, outcome: BackupRunFinish): Promise<void> {
+    await this.db.execute(sql`
+      update backup_runs
+         set status = ${outcome.status},
+             finished_at = ${outcome.finishedAt},
+             heartbeat_at = ${outcome.finishedAt},
+             bundle_name = ${outcome.bundleName ?? null},
+             size_bytes = ${outcome.sizeBytes ?? null},
+             uploads = ${outcome.uploads ?? null},
+             shipped = ${outcome.shipped ?? false},
+             skipped_keys = ${outcome.skippedKeys ?? 0},
+             error = ${outcome.error ?? null}
+       where id = ${id}
+    `)
+  }
+
+  async active(now: Date, staleBefore: Date): Promise<BackupRunRecord | null> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select ${COLUMNS} from backup_runs
+         where status = 'queued'
+            or (status = 'running' and coalesce(heartbeat_at, started_at, ${now}) > ${staleBefore})
+         order by id
+         limit 1
+      `),
+    ) as Array<Record<string, unknown>>
+    const row = rows[0]
+    return row === undefined ? null : toRecord(row)
+  }
+
+  async recent(limit: number): Promise<readonly BackupRunRecord[]> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select ${COLUMNS} from backup_runs order by id desc limit ${limit}
+      `),
+    ) as Array<Record<string, unknown>>
+    return rows.map(toRecord)
+  }
+
+  async lastScheduledAt(): Promise<Date | null> {
+    const rows = resultRows<{ requested_at: unknown }>(
+      await this.db.execute(sql`
+        select requested_at from backup_runs
+         where trigger = 'schedule'
+         order by id desc
+         limit 1
+      `),
+    )
+    const row = rows[0]
+    return row === undefined ? null : toDate(row.requested_at)
+  }
+
+  async failInterrupted(now: Date, staleBefore: Date): Promise<number> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        update backup_runs
+           set status = 'failed',
+               finished_at = ${now},
+               error = 'The backup was interrupted before it finished: the process running it stopped.'
+         where status = 'running'
+           and coalesce(heartbeat_at, started_at, requested_at) <= ${staleBefore}
+        returning id
+      `),
+    )
+    return rows.length
+  }
+
+  async record(input: {
+    readonly trigger: BackupTrigger
+    readonly requestedByUserId?: number | null | undefined
+    readonly startedAt: Date
+    readonly outcome: BackupRunFinish
+  }): Promise<void> {
+    const { outcome } = input
+    await this.db.execute(sql`
+      insert into backup_runs (
+        trigger, status, requested_by_user_id, requested_at, started_at, finished_at,
+        heartbeat_at, bundle_name, size_bytes, uploads, shipped, skipped_keys, error
+      ) values (
+        ${input.trigger}, ${outcome.status}, ${input.requestedByUserId ?? null},
+        ${input.startedAt}, ${input.startedAt}, ${outcome.finishedAt}, ${outcome.finishedAt},
+        ${outcome.bundleName ?? null}, ${outcome.sizeBytes ?? null}, ${outcome.uploads ?? null},
+        ${outcome.shipped ?? false}, ${outcome.skippedKeys ?? 0}, ${outcome.error ?? null}
+      )
+    `)
+  }
+}

package/src/index.ts

@@ -54,6 +54,7 @@ export {
   parseAncestorPath,
 } from './authorization-source'
 export { PostgresAvatarRepository } from './avatar-repo'
+export { PostgresBackupRunRepository } from './backup-run-repo'
 export {
   PostgresBanFilterRepository,
   PostgresBanRepository,
@@ -254,7 +255,12 @@ export {
   type StoredSearch,
 } from './search-store'
 export { SEED_GROUP_KEY, type SeedGroupKey } from './seed-groups'
-export { PostgresSettingsRepository } from './settings-repo'
+export {
+  PostgresSettingsRepository,
+  SETTING_SEAL_PURPOSE,
+  type SettingSealer,
+  settingSealer,
+} from './settings-repo'
 export { PostgresSignatureRepository } from './signature-repo'
 export {
   type BoardTotals,

package/src/schema/platform.ts

@@ -1,5 +1,6 @@
 import { sql } from 'drizzle-orm'
 import {
+  bigint,
   boolean,
   check,
   foreignKey,
@@ -148,6 +149,44 @@ export const taskLog = pgTable(
   (t) => [index('task_log_key_ran_idx').on(t.taskKey, t.ranAt.desc())],
 )
 
+export const backupRuns = pgTable(
+  'backup_runs',
+  {
+    id: integer('id').primaryKey().generatedByDefaultAsIdentity(),
+    trigger: text('trigger').notNull(),
+    status: text('status').notNull().default('queued'),
+    requestedByUserId: integer('requested_by_user_id').references(() => users.id, {
+      onDelete: 'set null',
+    }),
+    requestedAt: timestamp('requested_at', { withTimezone: true }).notNull().defaultNow(),
+    startedAt: timestamp('started_at', { withTimezone: true }),
+    finishedAt: timestamp('finished_at', { withTimezone: true }),
+    heartbeatAt: timestamp('heartbeat_at', { withTimezone: true }),
+    bundleName: text('bundle_name'),
+    sizeBytes: bigint('size_bytes', { mode: 'number' }),
+    uploads: text('uploads'),
+    shipped: boolean('shipped').notNull().default(false),
+    skippedKeys: integer('skipped_keys').notNull().default(0),
+    error: text('error'),
+  },
+  (t) => [
+    index('backup_runs_status_idx').on(t.status, t.id),
+    index('backup_runs_recent_idx').on(t.requestedAt.desc()),
+    uniqueIndex('backup_runs_active_idx')
+      .on(t.status)
+      .where(sql`${t.status} in ('queued', 'running')`),
+
+    check(
+      'backup_runs_trigger_check',
+      sql`${t.trigger} in ('manual', 'schedule', 'upgrade', 'cli')`,
+    ),
+    check(
+      'backup_runs_status_check',
+      sql`${t.status} in ('queued', 'running', 'done', 'incomplete', 'failed')`,
+    ),
+  ],
+)
+
 export const cacheVersions = pgTable('cache_versions', {
   key: text('key').primaryKey(),
   version: integer('version').notNull().default(1),

package/src/settings-repo.ts

@@ -1,24 +1,84 @@
 import { inArray, sql } from 'drizzle-orm'
 
-import type { SettingsRepository } from '@meith/settings'
+import { ConfigurationError, env, logger, openValue, sealValue } from '@meith/core'
+import { SETTING_DEFINITIONS, type SettingsRepository } from '@meith/settings'
 
 import type { Database } from './client'
 import { settings } from './schema'
 
+export const SETTING_SEAL_PURPOSE = 'meith/sealed-setting'
+
+const SEALED_KEYS: ReadonlySet<string> = new Set(
+  SETTING_DEFINITIONS.filter((definition) => definition.sealed === true).map(
+    (definition) => definition.key,
+  ),
+)
+
+export interface SettingSealer {
+  seal(key: string, value: string): Promise<string>
+  open(key: string, stored: string): Promise<string | null>
+}
+
+export function settingSealer(passphrase: string | undefined): SettingSealer {
+  return {
+    async seal(key, value) {
+      if (passphrase === undefined || passphrase.trim() === '') {
+        throw new ConfigurationError(
+          `The ${key} setting is stored sealed, which needs AUTH_SECRET set: without a key ` +
+            'to seal it with, the credential would sit in the database in the clear.',
+        )
+      }
+      return sealValue(value, passphrase, SETTING_SEAL_PURPOSE)
+    },
+    async open(_key, stored) {
+      if (passphrase === undefined || passphrase.trim() === '') return null
+      return openValue(stored, passphrase, SETTING_SEAL_PURPOSE)
+    },
+  }
+}
+
 export class PostgresSettingsRepository implements SettingsRepository {
-  constructor(private readonly db: Database) {}
+  private readonly sealer: SettingSealer
+
+  constructor(
+    private readonly db: Database,
+    sealer?: SettingSealer,
+  ) {
+    this.sealer = sealer ?? settingSealer(env.AUTH_SECRET)
+  }
 
   async loadAll(): Promise<ReadonlyMap<string, string>> {
     const rows = await this.db.select({ key: settings.key, value: settings.value }).from(settings)
-    return new Map(rows.map((r) => [r.key, r.value]))
+    const loaded = new Map<string, string>()
+    for (const row of rows) {
+      if (!SEALED_KEYS.has(row.key)) {
+        loaded.set(row.key, row.value)
+        continue
+      }
+      const opened = await this.sealer.open(row.key, row.value)
+      if (opened === null) {
+        logger({ module: 'settings' }).warn(
+          { key: row.key },
+          'a sealed setting could not be opened under AUTH_SECRET and is treated as unset',
+        )
+        continue
+      }
+      loaded.set(row.key, opened)
+    }
+    return loaded
   }
 
   async save(entries: ReadonlyMap<string, string>): Promise<void> {
     if (entries.size === 0) return
 
+    const values: { key: string; value: string }[] = []
+    for (const [key, value] of entries) {
+      values.push({ key, value: SEALED_KEYS.has(key) ? await this.sealer.seal(key, value) : value })
+    }
+
     await this.db
       .insert(settings)
-      .values([...entries].map(([key, value]) => ({ key, value })))
+      .values(values)
       .onConflictDoUpdate({
         target: settings.key,
         set: { value: sql`excluded.value`, updatedAt: new Date() },

package/src/system-health-repo.ts

@@ -41,7 +41,7 @@ export class PostgresSystemHealthRepository {
     const rows = resultRows(
       await this.db.execute(sql`
         select key, interval_seconds, enabled, last_run_at, next_run_at,
-               consecutive_failures
+               locked_until, consecutive_failures
           from tasks order by key
       `),
     ) as Array<Record<string, unknown>>
@@ -52,6 +52,7 @@ export class PostgresSystemHealthRepository {
       enabled: row.enabled === true,
       lastRunAt: row.last_run_at === null ? null : toDate(row.last_run_at),
       nextRunAt: row.next_run_at === null ? null : toDate(row.next_run_at),
+      lockedUntil: row.locked_until === null ? null : toDate(row.locked_until),
       consecutiveFailures: Number(row.consecutive_failures),
     }))
   }

package/src/task-repo.ts

@@ -49,6 +49,20 @@ export class PostgresTaskRepository implements TaskRepository {
       })
   }
 
+  async renew(input: { taskId: string; now: Date; staleBefore: Date }): Promise<boolean> {
+    const leaseMs = input.now.getTime() - input.staleBefore.getTime()
+    const lockedUntil = new Date(input.now.getTime() + leaseMs)
+    const result = await this.db.execute(sql`
+      update tasks
+         set locked_until = ${lockedUntil}
+       where key = ${input.taskId}
+         and locked_until is not null
+         and locked_until > ${input.now}
+      returning key
+    `)
+    return resultRows(result).length === 1
+  }
+
   async claim(input: {
     taskId: string
     now: Date

package/src/user-merge-map.ts

@@ -15,6 +15,7 @@ export type DiscardColumn = ReassignColumn
 export const MERGE_REASSIGN: readonly ReassignColumn[] = [
   { table: 'admin_log', column: 'user_id' },
   { table: 'admin_undo_operations', column: 'actor_user_id' },
+  { table: 'backup_runs', column: 'requested_by_user_id' },
   { table: 'attachments', column: 'uploader_user_id' },
   { table: 'ban_filters', column: 'created_by_user_id' },
   { table: 'bans', column: 'banned_by_user_id' },