@meith/db 0.33.4 → 0.35.0

package/migrations/meta/_journal.json

@@ -477,6 +477,20 @@
       "when": 1788464192391,
       "tag": "0067_minimal_navigation",
       "breakpoints": true
+    },
+    {
+      "idx": 68,
+      "version": "7",
+      "when": 1788501076230,
+      "tag": "0068_backup_runs",
+      "breakpoints": true
+    },
+    {
+      "idx": 69,
+      "version": "7",
+      "when": 1788517301313,
+      "tag": "0069_backup_runs_active",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meith/db",
-  "version": "0.33.4",
+  "version": "0.35.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -22,36 +22,37 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "postgres": "^3.4.7",
-    "@meith/accounts": "0.33.4",
-    "@meith/admin": "0.33.4",
-    "@meith/antispam": "0.33.4",
-    "@meith/api": "0.33.4",
-    "@meith/attachments": "0.33.4",
-    "@meith/authorization": "0.33.4",
-    "@meith/avatars": "0.33.4",
-    "@meith/board-digest": "0.33.4",
-    "@meith/core": "0.33.4",
-    "@meith/drafts": "0.33.4",
-    "@meith/events": "0.33.4",
-    "@meith/forums": "0.33.4",
-    "@meith/groups": "0.33.4",
-    "@meith/i18n": "0.33.4",
-    "@meith/markdown": "0.33.4",
-    "@meith/marketplace": "0.33.4",
-    "@meith/messages": "0.33.4",
-    "@meith/moderation": "0.33.4",
-    "@meith/notifications": "0.33.4",
-    "@meith/plugin-kit": "0.33.4",
-    "@meith/polls": "0.33.4",
-    "@meith/profile-fields": "0.33.4",
-    "@meith/relations": "0.33.4",
-    "@meith/reputation": "0.33.4",
-    "@meith/search": "0.33.4",
-    "@meith/settings": "0.33.4",
-    "@meith/signatures": "0.33.4",
-    "@meith/subscriptions": "0.33.4",
-    "@meith/tasks": "0.33.4",
-    "@meith/threads": "0.33.4"
+    "@meith/accounts": "0.35.0",
+    "@meith/admin": "0.35.0",
+    "@meith/antispam": "0.35.0",
+    "@meith/api": "0.35.0",
+    "@meith/attachments": "0.35.0",
+    "@meith/authorization": "0.35.0",
+    "@meith/avatars": "0.35.0",
+    "@meith/backup": "0.35.0",
+    "@meith/board-digest": "0.35.0",
+    "@meith/core": "0.35.0",
+    "@meith/drafts": "0.35.0",
+    "@meith/events": "0.35.0",
+    "@meith/forums": "0.35.0",
+    "@meith/groups": "0.35.0",
+    "@meith/i18n": "0.35.0",
+    "@meith/markdown": "0.35.0",
+    "@meith/marketplace": "0.35.0",
+    "@meith/messages": "0.35.0",
+    "@meith/moderation": "0.35.0",
+    "@meith/notifications": "0.35.0",
+    "@meith/plugin-kit": "0.35.0",
+    "@meith/polls": "0.35.0",
+    "@meith/profile-fields": "0.35.0",
+    "@meith/relations": "0.35.0",
+    "@meith/reputation": "0.35.0",
+    "@meith/search": "0.35.0",
+    "@meith/settings": "0.35.0",
+    "@meith/signatures": "0.35.0",
+    "@meith/subscriptions": "0.35.0",
+    "@meith/tasks": "0.35.0",
+    "@meith/threads": "0.35.0"
   },
   "devDependencies": {
     "drizzle-kit": "^0.31.6",

package/src/backup-run-repo.ts

@@ -0,0 +1,212 @@
+import { sql } from 'drizzle-orm'
+
+import type {
+  BackupRunFinish,
+  BackupRunRecord,
+  BackupRunRepository,
+  BackupRunStatus,
+  BackupTrigger,
+} from '@meith/backup'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+import { toDate } from './row-values'
+
+const COLUMNS = sql`
+  id, trigger, status, requested_by_user_id, requested_at, started_at, finished_at,
+  heartbeat_at, bundle_name, size_bytes, uploads, shipped, skipped_keys, error
+`
+
+const UNIQUE_VIOLATION = '23505'
+
+function isUniqueViolation(error: unknown): boolean {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    ((error as { code?: unknown }).code === UNIQUE_VIOLATION ||
+      (error as { cause?: { code?: unknown } }).cause?.code === UNIQUE_VIOLATION)
+  )
+}
+
+function toRecord(row: Record<string, unknown>): BackupRunRecord {
+  const optionalDate = (value: unknown): Date | null => (value === null ? null : toDate(value))
+  return {
+    id: Number(row.id),
+    trigger: String(row.trigger) as BackupTrigger,
+    status: String(row.status) as BackupRunStatus,
+    requestedByUserId: row.requested_by_user_id === null ? null : Number(row.requested_by_user_id),
+    requestedAt: toDate(row.requested_at),
+    startedAt: optionalDate(row.started_at),
+    finishedAt: optionalDate(row.finished_at),
+    heartbeatAt: optionalDate(row.heartbeat_at),
+    bundleName: row.bundle_name === null ? null : String(row.bundle_name),
+    sizeBytes: row.size_bytes === null ? null : Number(row.size_bytes),
+    uploads:
+      row.uploads === 'included' || row.uploads === 'skipped'
+        ? (row.uploads as 'included' | 'skipped')
+        : null,
+    shipped: row.shipped === true,
+    skippedKeys: Number(row.skipped_keys),
+    error: row.error === null ? null : String(row.error),
+  }
+}
+
+export class PostgresBackupRunRepository implements BackupRunRepository {
+  constructor(private readonly db: Database) {}
+
+  async enqueue(input: {
+    readonly trigger: BackupTrigger
+    readonly requestedByUserId?: number | null | undefined
+    readonly now: Date
+  }): Promise<{ readonly id: number; readonly queued: boolean }> {
+    const pending = async (): Promise<number | undefined> => {
+      const row = resultRows<{ id: number }>(
+        await this.db.execute(sql`
+          select id from backup_runs
+           where status in ('queued', 'running')
+           order by id
+           limit 1
+        `),
+      )[0]
+      return row === undefined ? undefined : Number(row.id)
+    }
+
+    const existing = await pending()
+    if (existing !== undefined) return { id: existing, queued: false }
+
+    try {
+      const inserted = resultRows<{ id: number }>(
+        await this.db.execute(sql`
+          insert into backup_runs (trigger, status, requested_by_user_id, requested_at)
+          values (${input.trigger}, 'queued', ${input.requestedByUserId ?? null}, ${input.now})
+          returning id
+        `),
+      )[0]
+      return { id: Number(inserted?.id), queued: true }
+    } catch (error) {
+      if (!isUniqueViolation(error)) throw error
+      const raced = await pending()
+      if (raced === undefined) throw error
+      return { id: raced, queued: false }
+    }
+  }
+
+  async claimNext(now: Date): Promise<BackupRunRecord | null> {
+    let rows: Array<Record<string, unknown>>
+    try {
+      rows = resultRows(
+        await this.db.execute(sql`
+          update backup_runs
+             set status = 'running', started_at = ${now}, heartbeat_at = ${now}
+           where id = (
+             select id from backup_runs
+              where status = 'queued'
+                and not exists (select 1 from backup_runs where status = 'running')
+              order by id
+              limit 1
+                for update skip locked
+           )
+          returning ${COLUMNS}
+        `),
+      ) as Array<Record<string, unknown>>
+    } catch (error) {
+      if (isUniqueViolation(error)) return null
+      throw error
+    }
+    const row = rows[0]
+    return row === undefined ? null : toRecord(row)
+  }
+
+  async heartbeat(id: number, now: Date): Promise<void> {
+    await this.db.execute(sql`
+      update backup_runs set heartbeat_at = ${now} where id = ${id} and status = 'running'
+    `)
+  }
+
+  async finish(id: number, outcome: BackupRunFinish): Promise<void> {
+    await this.db.execute(sql`
+      update backup_runs
+         set status = ${outcome.status},
+             finished_at = ${outcome.finishedAt},
+             heartbeat_at = ${outcome.finishedAt},
+             bundle_name = ${outcome.bundleName ?? null},
+             size_bytes = ${outcome.sizeBytes ?? null},
+             uploads = ${outcome.uploads ?? null},
+             shipped = ${outcome.shipped ?? false},
+             skipped_keys = ${outcome.skippedKeys ?? 0},
+             error = ${outcome.error ?? null}
+       where id = ${id}
+    `)
+  }
+
+  async active(now: Date, staleBefore: Date): Promise<BackupRunRecord | null> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select ${COLUMNS} from backup_runs
+         where status = 'queued'
+            or (status = 'running' and coalesce(heartbeat_at, started_at, ${now}) > ${staleBefore})
+         order by id
+         limit 1
+      `),
+    ) as Array<Record<string, unknown>>
+    const row = rows[0]
+    return row === undefined ? null : toRecord(row)
+  }
+
+  async recent(limit: number): Promise<readonly BackupRunRecord[]> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select ${COLUMNS} from backup_runs order by id desc limit ${limit}
+      `),
+    ) as Array<Record<string, unknown>>
+    return rows.map(toRecord)
+  }
+
+  async lastScheduledAt(): Promise<Date | null> {
+    const rows = resultRows<{ requested_at: unknown }>(
+      await this.db.execute(sql`
+        select requested_at from backup_runs
+         where trigger = 'schedule'
+         order by id desc
+         limit 1
+      `),
+    )
+    const row = rows[0]
+    return row === undefined ? null : toDate(row.requested_at)
+  }
+
+  async failInterrupted(now: Date, staleBefore: Date): Promise<number> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        update backup_runs
+           set status = 'failed',
+               finished_at = ${now},
+               error = 'The backup was interrupted before it finished: the process running it stopped.'
+         where status = 'running'
+           and coalesce(heartbeat_at, started_at, requested_at) <= ${staleBefore}
+        returning id
+      `),
+    )
+    return rows.length
+  }
+
+  async record(input: {
+    readonly trigger: BackupTrigger
+    readonly requestedByUserId?: number | null | undefined
+    readonly startedAt: Date
+    readonly outcome: BackupRunFinish
+  }): Promise<void> {
+    const { outcome } = input
+    await this.db.execute(sql`
+      insert into backup_runs (
+        trigger, status, requested_by_user_id, requested_at, started_at, finished_at,
+        heartbeat_at, bundle_name, size_bytes, uploads, shipped, skipped_keys, error
+      ) values (
+        ${input.trigger}, ${outcome.status}, ${input.requestedByUserId ?? null},
+        ${input.startedAt}, ${input.startedAt}, ${outcome.finishedAt}, ${outcome.finishedAt},
+        ${outcome.bundleName ?? null}, ${outcome.sizeBytes ?? null}, ${outcome.uploads ?? null},
+        ${outcome.shipped ?? false}, ${outcome.skippedKeys ?? 0}, ${outcome.error ?? null}
+      )
+    `)
+  }
+}

package/src/identity-link-repo.ts

@@ -1,4 +1,4 @@
-import { and, asc, eq } from 'drizzle-orm'
+import { and, asc, eq, gt, or, sql } from 'drizzle-orm'
 
 import type {
   LinkIdentityInput,
@@ -136,10 +136,17 @@ export class PostgresPasskeyRepository implements PasskeyRepository {
     return removed.length > 0
   }
 
-  async markUsed(passkeyId: number, signCount: number, now: Date): Promise<void> {
-    await this.db
+  async markUsed(passkeyId: number, signCount: number, now: Date): Promise<boolean> {
+    const rows = await this.db
       .update(passkeys)
       .set({ signCount, lastUsedAt: now })
-      .where(eq(passkeys.id, passkeyId))
+      .where(
+        and(
+          eq(passkeys.id, passkeyId),
+          or(eq(sql`${signCount}`, 0), gt(sql`${signCount}`, passkeys.signCount)),
+        ),
+      )
+      .returning({ id: passkeys.id })
+    return rows.length > 0
   }
 }

package/src/index.ts

@@ -54,6 +54,7 @@ export {
   parseAncestorPath,
 } from './authorization-source'
 export { PostgresAvatarRepository } from './avatar-repo'
+export { PostgresBackupRunRepository } from './backup-run-repo'
 export {
   PostgresBanFilterRepository,
   PostgresBanRepository,
@@ -254,7 +255,12 @@ export {
   type StoredSearch,
 } from './search-store'
 export { SEED_GROUP_KEY, type SeedGroupKey } from './seed-groups'
-export { PostgresSettingsRepository } from './settings-repo'
+export {
+  PostgresSettingsRepository,
+  SETTING_SEAL_PURPOSE,
+  type SettingSealer,
+  settingSealer,
+} from './settings-repo'
 export { PostgresSignatureRepository } from './signature-repo'
 export {
   type BoardTotals,

package/src/message-repo.ts

@@ -336,6 +336,29 @@ export class PostgresMessageRepository implements MessageRepository {
     return rows.length
   }
 
+  async restore(input: {
+    readonly userId: number
+    readonly copyIds: readonly number[]
+  }): Promise<number> {
+    if (input.copyIds.length === 0) return 0
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        update private_message_copies
+           set folder = case when role = 'author' then 'sent' else 'inbox' end
+         where owner_user_id = ${input.userId}
+           and folder = 'trash'
+           and id in (${sql.join(
+             input.copyIds.map((id) => sql`${id}`),
+             sql`, `,
+           )})
+        returning id
+      `),
+    ) as Array<{ id: number }>
+
+    return rows.length
+  }
+
   async remove(input: {
     readonly userId: number
     readonly copyIds: readonly number[]

package/src/notification-repo.ts

@@ -266,6 +266,15 @@ export class PostgresNotificationRepository implements NotificationRepository {
     return rows.length > 0
   }
 
+  async removeAllPushSubscriptions(userId: number): Promise<number> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        delete from push_subscriptions where user_id = ${userId} returning id
+      `),
+    ) as Array<{ id: number }>
+    return rows.length
+  }
+
   async pushSubscriptionsFor(userId: number): Promise<readonly PushSubscriptionRecord[]> {
     const rows = resultRows(
       await this.db.execute(sql`

package/src/post-repo.ts

@@ -111,7 +111,7 @@ export class PostgresPostRepository implements PostRepository {
          where r.post_id = ${postId}
            and exists (select 1 from posts p where p.id = r.post_id and p.thread_id = ${threadId})
         union all
-        select p.revision_count, p.message, p.subject, p.edited_by_user_id,
+        select p.revision_count + 1, p.message, p.subject, p.edited_by_user_id,
                coalesce(u.username, p.author_username, 'Deleted member'),
                p.edit_reason, coalesce(p.edited_at, p.created_at), true
           from posts p

package/src/post-writes.ts

@@ -6,6 +6,7 @@ import {
   CORE_RENDERING,
   type MarkdownPipeline,
   renderThrough,
+  sourceAsMarkdown,
   vocabularyOptions,
 } from '@meith/markdown'
 import type {
@@ -58,8 +59,9 @@ export class PostgresPostWriteRepository implements PostWriteRepository {
     const rows = resultRows(
       await this.db.execute(sql`
         select p.id, p.thread_id, p.forum_id, p.author_user_id, p.subject, p.message,
-               p.visibility, p.is_first_post, p.revision_count, p.created_at,
-               t.slug as thread_slug, t.title as thread_title, t.is_locked,
+               p.body_format, p.visibility, p.is_first_post, p.revision_count, p.created_at,
+               t.slug as thread_slug, t.title as thread_title,
+               t.author_user_id as thread_author_user_id, t.is_locked,
                t.visibility as thread_visibility,
                f.slug as forum_slug, f.is_open
           from posts p
@@ -74,12 +76,14 @@ export class PostgresPostWriteRepository implements PostWriteRepository {
       author_user_id: number | null
       subject: string | null
       message: string
+      body_format: number
       visibility: 'visible' | 'unapproved' | 'deleted'
       is_first_post: boolean
       revision_count: number
       created_at: Date
       thread_slug: string
       thread_title: string
+      thread_author_user_id: number | null
       is_locked: boolean
       thread_visibility: 'visible' | 'unapproved' | 'deleted'
       forum_slug: string
@@ -96,7 +100,7 @@ export class PostgresPostWriteRepository implements PostWriteRepository {
         forumId: Number(row.forum_id),
         authorUserId: row.author_user_id === null ? null : Number(row.author_user_id),
         subject: row.subject,
-        message: row.message,
+        message: sourceAsMarkdown(row.message, Number(row.body_format)),
         visibility: row.visibility,
         isFirstPost: row.is_first_post,
         revisionCount: Number(row.revision_count),
@@ -106,6 +110,7 @@ export class PostgresPostWriteRepository implements PostWriteRepository {
         id: Number(row.thread_id),
         slug: row.thread_slug,
         title: row.thread_title,
+        authorUserId: row.thread_author_user_id === null ? null : Number(row.thread_author_user_id),
         isLocked: row.is_locked,
         visibility: row.thread_visibility,
       },

package/src/schema/platform.ts

@@ -1,5 +1,6 @@
 import { sql } from 'drizzle-orm'
 import {
+  bigint,
   boolean,
   check,
   foreignKey,
@@ -148,6 +149,44 @@ export const taskLog = pgTable(
   (t) => [index('task_log_key_ran_idx').on(t.taskKey, t.ranAt.desc())],
 )
 
+export const backupRuns = pgTable(
+  'backup_runs',
+  {
+    id: integer('id').primaryKey().generatedByDefaultAsIdentity(),
+    trigger: text('trigger').notNull(),
+    status: text('status').notNull().default('queued'),
+    requestedByUserId: integer('requested_by_user_id').references(() => users.id, {
+      onDelete: 'set null',
+    }),
+    requestedAt: timestamp('requested_at', { withTimezone: true }).notNull().defaultNow(),
+    startedAt: timestamp('started_at', { withTimezone: true }),
+    finishedAt: timestamp('finished_at', { withTimezone: true }),
+    heartbeatAt: timestamp('heartbeat_at', { withTimezone: true }),
+    bundleName: text('bundle_name'),
+    sizeBytes: bigint('size_bytes', { mode: 'number' }),
+    uploads: text('uploads'),
+    shipped: boolean('shipped').notNull().default(false),
+    skippedKeys: integer('skipped_keys').notNull().default(0),
+    error: text('error'),
+  },
+  (t) => [
+    index('backup_runs_status_idx').on(t.status, t.id),
+    index('backup_runs_recent_idx').on(t.requestedAt.desc()),
+    uniqueIndex('backup_runs_active_idx')
+      .on(t.status)
+      .where(sql`${t.status} in ('queued', 'running')`),
+
+    check(
+      'backup_runs_trigger_check',
+      sql`${t.trigger} in ('manual', 'schedule', 'upgrade', 'cli')`,
+    ),
+    check(
+      'backup_runs_status_check',
+      sql`${t.status} in ('queued', 'running', 'done', 'incomplete', 'failed')`,
+    ),
+  ],
+)
+
 export const cacheVersions = pgTable('cache_versions', {
   key: text('key').primaryKey(),
   version: integer('version').notNull().default(1),

package/src/settings-repo.ts

@@ -1,24 +1,84 @@
 import { inArray, sql } from 'drizzle-orm'
 
-import type { SettingsRepository } from '@meith/settings'
+import { ConfigurationError, env, logger, openValue, sealValue } from '@meith/core'
+import { SETTING_DEFINITIONS, type SettingsRepository } from '@meith/settings'
 
 import type { Database } from './client'
 import { settings } from './schema'
 
+export const SETTING_SEAL_PURPOSE = 'meith/sealed-setting'
+
+const SEALED_KEYS: ReadonlySet<string> = new Set(
+  SETTING_DEFINITIONS.filter((definition) => definition.sealed === true).map(
+    (definition) => definition.key,
+  ),
+)
+
+export interface SettingSealer {
+  seal(key: string, value: string): Promise<string>
+  open(key: string, stored: string): Promise<string | null>
+}
+
+export function settingSealer(passphrase: string | undefined): SettingSealer {
+  return {
+    async seal(key, value) {
+      if (passphrase === undefined || passphrase.trim() === '') {
+        throw new ConfigurationError(
+          `The ${key} setting is stored sealed, which needs AUTH_SECRET set: without a key ` +
+            'to seal it with, the credential would sit in the database in the clear.',
+        )
+      }
+      return sealValue(value, passphrase, SETTING_SEAL_PURPOSE)
+    },
+    async open(_key, stored) {
+      if (passphrase === undefined || passphrase.trim() === '') return null
+      return openValue(stored, passphrase, SETTING_SEAL_PURPOSE)
+    },
+  }
+}
+
 export class PostgresSettingsRepository implements SettingsRepository {
-  constructor(private readonly db: Database) {}
+  private readonly sealer: SettingSealer
+
+  constructor(
+    private readonly db: Database,
+    sealer?: SettingSealer,
+  ) {
+    this.sealer = sealer ?? settingSealer(env.AUTH_SECRET)
+  }
 
   async loadAll(): Promise<ReadonlyMap<string, string>> {
     const rows = await this.db.select({ key: settings.key, value: settings.value }).from(settings)
-    return new Map(rows.map((r) => [r.key, r.value]))
+    const loaded = new Map<string, string>()
+    for (const row of rows) {
+      if (!SEALED_KEYS.has(row.key)) {
+        loaded.set(row.key, row.value)
+        continue
+      }
+      const opened = await this.sealer.open(row.key, row.value)
+      if (opened === null) {
+        logger({ module: 'settings' }).warn(
+          { key: row.key },
+          'a sealed setting could not be opened under AUTH_SECRET and is treated as unset',
+        )
+        continue
+      }
+      loaded.set(row.key, opened)
+    }
+    return loaded
   }
 
   async save(entries: ReadonlyMap<string, string>): Promise<void> {
     if (entries.size === 0) return
 
+    const values: { key: string; value: string }[] = []
+    for (const [key, value] of entries) {
+      values.push({ key, value: SEALED_KEYS.has(key) ? await this.sealer.seal(key, value) : value })
+    }
+
     await this.db
       .insert(settings)
-      .values([...entries].map(([key, value]) => ({ key, value })))
+      .values(values)
       .onConflictDoUpdate({
         target: settings.key,
         set: { value: sql`excluded.value`, updatedAt: new Date() },