@meith/db 0.16.0

package/src/counter-recount.ts

@@ -0,0 +1,263 @@
+import { sql } from 'drizzle-orm'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+
+export const RECOUNT_PHASES = ['threads', 'forums', 'users'] as const
+export type RecountPhase = (typeof RECOUNT_PHASES)[number]
+
+export interface RecountRun {
+  readonly phase: RecountPhase
+  readonly scanned: number
+  readonly corrected: number
+  readonly cursor: number
+  readonly nextPhase: RecountPhase
+  readonly completedPass: boolean
+}
+
+interface PhaseState {
+  phase: RecountPhase
+  cursor: number
+  passes: number
+  corrected: number
+}
+
+function nextPhase(phase: RecountPhase): RecountPhase {
+  const index = RECOUNT_PHASES.indexOf(phase)
+  return RECOUNT_PHASES[(index + 1) % RECOUNT_PHASES.length]!
+}
+
+export class PostgresCounterRecount {
+  constructor(
+    private readonly db: Database,
+    private readonly stateId = 'default',
+  ) {}
+
+  async run(batchSize = 500): Promise<RecountRun> {
+    const state = await this.loadState()
+
+    const batch = await this.scan(state.phase, state.cursor, batchSize)
+    const corrected =
+      batch.ids.length === 0 ? 0 : await this.correct(state.phase, state.cursor, batchSize)
+
+    const phaseComplete = batch.ids.length < batchSize
+    const phase = phaseComplete ? nextPhase(state.phase) : state.phase
+    const cursor = phaseComplete ? 0 : batch.maxId
+    const completedPass = phaseComplete && state.phase === RECOUNT_PHASES[RECOUNT_PHASES.length - 1]
+
+    await this.db.execute(sql`
+      update counter_recount_state
+         set phase = ${phase},
+             cursor = ${cursor},
+             passes = ${state.passes + (completedPass ? 1 : 0)},
+             corrected = ${state.corrected + corrected},
+             updated_at = now()
+       where id = ${this.stateId}
+    `)
+
+    return {
+      phase: state.phase,
+      scanned: batch.ids.length,
+      corrected,
+      cursor,
+      nextPhase: phase,
+      completedPass,
+    }
+  }
+
+  async state(): Promise<PhaseState> {
+    return this.loadState()
+  }
+
+  private async loadState(): Promise<PhaseState> {
+    await this.db.execute(sql`
+      insert into counter_recount_state (id) values (${this.stateId})
+      on conflict (id) do nothing
+    `)
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select phase, cursor, passes, corrected
+          from counter_recount_state where id = ${this.stateId}
+      `),
+    ) as Array<{ phase: string; cursor: number; passes: number; corrected: number }>
+
+    const row = rows[0]
+    const phase = RECOUNT_PHASES.find((p) => p === row?.phase) ?? 'threads'
+    return {
+      phase,
+      cursor: Number(row?.cursor ?? 0),
+      passes: Number(row?.passes ?? 0),
+      corrected: Number(row?.corrected ?? 0),
+    }
+  }
+
+  private async scan(
+    phase: RecountPhase,
+    cursor: number,
+    batchSize: number,
+  ): Promise<{ ids: number[]; maxId: number }> {
+    const table = { threads: sql`threads`, forums: sql`forums`, users: sql`users` }[phase]
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select id from ${table} where id > ${cursor} order by id limit ${batchSize}
+      `),
+    ) as Array<{ id: number }>
+
+    const ids = rows.map((r) => Number(r.id))
+    return { ids, maxId: ids.length === 0 ? cursor : Math.max(...ids) }
+  }
+
+  private async correct(phase: RecountPhase, cursor: number, batchSize: number): Promise<number> {
+    if (phase === 'threads') return this.correctThreads(cursor, batchSize)
+    if (phase === 'forums') return this.correctForums(cursor, batchSize)
+    return this.correctUsers(cursor, batchSize)
+  }
+
+  private async correctThreads(cursor: number, batchSize: number): Promise<number> {
+    const result = await this.db.execute(sql`
+      with batch as (
+        select id, created_at from threads where id > ${cursor} order by id limit ${batchSize}
+      ),
+      agg as (
+        select b.id,
+               b.created_at,
+               count(p.id)::int as post_count,
+               min(p.id) as first_post_id,
+               max(p.id) as last_post_id
+          from batch b
+          left join posts p on p.thread_id = b.id and p.visibility = 'visible'
+         group by b.id, b.created_at
+      ),
+      truth as (
+        select a.id,
+               greatest(a.post_count - 1, 0) as reply_count,
+               a.first_post_id,
+               a.last_post_id,
+               lp.author_user_id as last_post_user_id,
+               lp.author_username as last_post_username,
+               coalesce(lp.created_at, a.created_at) as last_post_at
+          from agg a
+          left join posts lp on lp.id = a.last_post_id
+      )
+      update threads t
+         set reply_count = truth.reply_count,
+             first_post_id = truth.first_post_id,
+             last_post_id = truth.last_post_id,
+             last_post_user_id = truth.last_post_user_id,
+             last_post_username = truth.last_post_username,
+             last_post_at = truth.last_post_at,
+             updated_at = now()
+        from truth
+       where t.id = truth.id
+         and (t.reply_count is distinct from truth.reply_count
+           or t.first_post_id is distinct from truth.first_post_id
+           or t.last_post_id is distinct from truth.last_post_id
+           or t.last_post_user_id is distinct from truth.last_post_user_id
+           or t.last_post_username is distinct from truth.last_post_username
+           or t.last_post_at is distinct from truth.last_post_at)
+      returning t.id
+    `)
+
+    return resultRows(result).length
+  }
+
+  private async correctForums(cursor: number, batchSize: number): Promise<number> {
+    const result = await this.db.execute(sql`
+      with batch as (
+        select id, path from forums where id > ${cursor} order by id limit ${batchSize}
+      ),
+      subtree as (
+        select b.id as root_id, d.id as forum_id
+          from batch b
+          join forums d on d.id = b.id or d.path like b.path || '.%'
+      ),
+      thread_agg as (
+        select s.root_id, count(*)::int as thread_count
+          from subtree s
+          join threads th on th.forum_id = s.forum_id and th.visibility = 'visible'
+         group by s.root_id
+      ),
+      post_agg as (
+        select s.root_id, count(*)::int as post_count
+          from subtree s
+          join posts p on p.forum_id = s.forum_id and p.visibility = 'visible'
+          join threads th on th.id = p.thread_id and th.visibility = 'visible'
+         group by s.root_id
+      ),
+      last_agg as (
+        select distinct on (s.root_id)
+               s.root_id, p.id, p.thread_id, th.title, p.author_user_id,
+               p.author_username, p.created_at
+          from subtree s
+          join posts p on p.forum_id = s.forum_id and p.visibility = 'visible'
+          join threads th on th.id = p.thread_id and th.visibility = 'visible'
+         order by s.root_id, p.created_at desc, p.id desc
+      )
+      update forums f
+         set thread_count = coalesce(ta.thread_count, 0),
+             post_count = coalesce(pa.post_count, 0),
+             last_post_id = la.id,
+             last_post_thread_id = la.thread_id,
+             last_post_thread_title = la.title,
+             last_post_user_id = la.author_user_id,
+             last_post_username = la.author_username,
+             last_post_at = la.created_at,
+             updated_at = now()
+        from batch b
+             left join thread_agg ta on ta.root_id = b.id
+             left join post_agg pa on pa.root_id = b.id
+             left join last_agg la on la.root_id = b.id
+       where f.id = b.id
+         and (f.thread_count is distinct from coalesce(ta.thread_count, 0)
+           or f.post_count is distinct from coalesce(pa.post_count, 0)
+           or f.last_post_id is distinct from la.id
+           or f.last_post_thread_id is distinct from la.thread_id
+           or f.last_post_thread_title is distinct from la.title
+           or f.last_post_user_id is distinct from la.author_user_id
+           or f.last_post_username is distinct from la.author_username
+           or f.last_post_at is distinct from la.created_at)
+      returning f.id
+    `)
+
+    return resultRows(result).length
+  }
+
+  private async correctUsers(cursor: number, batchSize: number): Promise<number> {
+    const result = await this.db.execute(sql`
+      with batch as (
+        select id from users where id > ${cursor} order by id limit ${batchSize}
+      ),
+      -- FILTER, not WHERE: a user whose every post sits in a deleted thread
+      -- must still appear in this aggregate with a count of zero. Moving the
+      -- predicate into the WHERE clause drops their row entirely, and the
+      -- update would then leave exactly the stale count this is meant to fix.
+      post_agg as (
+        select b.id,
+               (count(p.id) filter (where pt.visibility = 'visible'))::int as post_count
+          from batch b
+          left join posts p on p.author_user_id = b.id and p.visibility = 'visible'
+          left join threads pt on pt.id = p.thread_id
+         group by b.id
+      ),
+      thread_agg as (
+        select b.id, count(th.id)::int as thread_count
+          from batch b
+          left join threads th on th.author_user_id = b.id and th.visibility = 'visible'
+         group by b.id
+      )
+      update users u
+         set post_count = pa.post_count,
+             thread_count = ta.thread_count,
+             updated_at = now()
+        from post_agg pa
+             join thread_agg ta on ta.id = pa.id
+       where u.id = pa.id
+         and (u.post_count is distinct from pa.post_count
+           or u.thread_count is distinct from ta.thread_count)
+      returning u.id
+    `)
+
+    return resultRows(result).length
+  }
+}

package/src/cursor.ts

@@ -0,0 +1,16 @@
+export function encodeCursor(at: Date, id: number): string {
+  return Buffer.from(`${at.toISOString()}|${id}`, 'utf8').toString('base64url')
+}
+
+export function decodeCursor(value: string): { at: Date; id: number } | null {
+  try {
+    const [at, id] = Buffer.from(value, 'base64url').toString('utf8').split('|')
+    if (at === undefined || id === undefined) return null
+    const when = new Date(at)
+    const numeric = Number(id)
+    if (Number.isNaN(when.getTime()) || !Number.isSafeInteger(numeric)) return null
+    return { at: when, id: numeric }
+  } catch {
+    return null
+  }
+}

package/src/denormalised-username.ts

@@ -0,0 +1,34 @@
+import { sql } from 'drizzle-orm'
+
+import type { Tx } from './permission-version'
+
+export interface DenormalisedUsernameColumn {
+  readonly table: string
+  readonly column: string
+  readonly idColumn: string
+}
+
+export const DENORMALISED_USERNAME_COLUMNS: readonly DenormalisedUsernameColumn[] = [
+  { table: 'forums', column: 'last_post_username', idColumn: 'last_post_user_id' },
+  { table: 'threads', column: 'last_post_username', idColumn: 'last_post_user_id' },
+  { table: 'threads', column: 'author_username', idColumn: 'author_user_id' },
+  { table: 'posts', column: 'author_username', idColumn: 'author_user_id' },
+  { table: 'private_messages', column: 'author_username', idColumn: 'author_user_id' },
+  { table: 'announcements', column: 'author_username', idColumn: 'author_user_id' },
+  { table: 'board_stats', column: 'newest_username', idColumn: 'newest_user_id' },
+]
+
+export async function rewriteDenormalisedUsernames(
+  tx: Tx,
+  userId: number,
+  username: string,
+): Promise<void> {
+  for (const entry of DENORMALISED_USERNAME_COLUMNS) {
+    await tx.execute(sql`
+      update ${sql.raw(entry.table)}
+         set ${sql.raw(entry.column)} = ${username}
+       where ${sql.raw(entry.idColumn)} = ${userId}
+         and ${sql.raw(entry.column)} is distinct from ${username}
+    `)
+  }
+}

package/src/discovery-repo.ts

@@ -0,0 +1,140 @@
+import { type SQL, sql } from 'drizzle-orm'
+
+import type { ContentScope } from '@meith/core'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+import { audienceIsEmpty, inAudience } from './thread-audience'
+import { visibleIn } from './visibility'
+
+export interface DiscoveryScope {
+  readonly forumIds: readonly number[]
+  readonly ownThreadsOnlyForumIds: readonly number[]
+  readonly content: ContentScope
+  readonly viewerUserId: number | null
+}
+
+export interface DiscoveryRow {
+  readonly threadId: number
+  readonly forumId: number
+  readonly forumTitle: string
+  readonly forumSlug: string
+  readonly title: string
+  readonly slug: string
+  readonly authorUserId: number | null
+  readonly authorUsername: string
+  readonly replyCount: number
+  readonly lastPostAt: Date
+  readonly lastPostUsername: string | null
+}
+
+export interface DiscoveryPage {
+  readonly rows: readonly DiscoveryRow[]
+  readonly nextCursor: { readonly at: Date; readonly threadId: number } | null
+}
+
+export interface DiscoveryQuery {
+  readonly limit: number
+  readonly after: { readonly at: Date; readonly threadId: number } | null
+}
+
+export class PostgresDiscoveryRepository {
+  constructor(private readonly db: Database) {}
+
+  async activeSince(
+    since: Date,
+    query: DiscoveryQuery,
+    scope: DiscoveryScope,
+  ): Promise<DiscoveryPage> {
+    return this.page([sql`t.last_post_at >= ${since}`], query, scope)
+  }
+
+  async unanswered(query: DiscoveryQuery, scope: DiscoveryScope): Promise<DiscoveryPage> {
+    return this.page([sql`t.reply_count = 0`], query, scope)
+  }
+
+  async startedBy(
+    userId: number,
+    query: DiscoveryQuery,
+    scope: DiscoveryScope,
+  ): Promise<DiscoveryPage> {
+    return this.page([sql`t.author_user_id = ${userId}`], query, scope)
+  }
+
+  async participatedIn(
+    userId: number,
+    query: DiscoveryQuery,
+    scope: DiscoveryScope,
+  ): Promise<DiscoveryPage> {
+    return this.page(
+      [
+        sql`exists (
+          select 1 from posts p
+           where p.thread_id = t.id and p.author_user_id = ${userId}
+             and ${visibleIn(sql`p.visibility`, scope.content)}
+        )`,
+      ],
+      query,
+      scope,
+    )
+  }
+
+  private async page(
+    conditions: readonly SQL[],
+    query: DiscoveryQuery,
+    scope: DiscoveryScope,
+  ): Promise<DiscoveryPage> {
+    if (audienceIsEmpty(scope)) return { rows: [], nextCursor: null }
+
+    const where: SQL[] = [
+      inAudience(sql`t.forum_id`, sql`t.author_user_id`, scope),
+      visibleIn(sql`t.visibility`, scope.content),
+      ...conditions,
+    ]
+
+    if (query.after !== null) {
+      where.push(
+        sql`(t.last_post_at < ${query.after.at}
+             or (t.last_post_at = ${query.after.at} and t.id < ${query.after.threadId}))`,
+      )
+    }
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select t.id as thread_id, t.forum_id, f.title as forum_title,
+               f.slug as forum_slug,
+               t.title, t.slug, t.author_user_id, t.author_username,
+               t.reply_count, t.last_post_at, t.last_post_username
+          from threads t
+          join forums f on f.id = t.forum_id
+         where ${sql.join(where, sql` and `)}
+         order by t.last_post_at desc, t.id desc
+         limit ${query.limit}
+      `),
+    ) as Array<Record<string, unknown>>
+
+    const mapped: DiscoveryRow[] = rows.map((row) => ({
+      threadId: Number(row.thread_id),
+      forumId: Number(row.forum_id),
+      forumTitle: String(row.forum_title),
+      forumSlug: String(row.forum_slug),
+      title: String(row.title),
+      slug: String(row.slug),
+      authorUserId: row.author_user_id === null ? null : Number(row.author_user_id),
+      authorUsername: String(row.author_username),
+      replyCount: Number(row.reply_count),
+      lastPostAt:
+        row.last_post_at instanceof Date ? row.last_post_at : new Date(String(row.last_post_at)),
+      lastPostUsername: row.last_post_username === null ? null : String(row.last_post_username),
+    }))
+
+    const last = mapped.at(-1)
+    return {
+      rows: mapped,
+      nextCursor:
+        mapped.length < query.limit || last === undefined
+          ? null
+          : { at: last.lastPostAt, threadId: last.threadId },
+    }
+  }
+}

package/src/draft-repo.ts

@@ -0,0 +1,59 @@
+import { sql } from 'drizzle-orm'
+
+import type { Draft, DraftRepository } from '@meith/drafts'
+import { BodyFormat, sourceAsMarkdown } from '@meith/markdown'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+
+export class PostgresDraftRepository implements DraftRepository {
+  constructor(private readonly db: Database) {}
+
+  async find(userId: number, forumId: number, threadId: number | null): Promise<Draft | null> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+      select forum_id, thread_id, title, message, body_format, prefix_id, updated_at from post_drafts
+       where user_id = ${userId} and forum_id = ${forumId}
+         and thread_id is not distinct from ${threadId}
+    `),
+    ) as Array<{
+      forum_id: number
+      thread_id: number | null
+      title: string
+      message: string
+      body_format: number
+      prefix_id: number | null
+      updated_at: Date | string
+    }>
+    const row = rows[0]
+    return row === undefined
+      ? null
+      : {
+          forumId: Number(row.forum_id),
+          threadId: row.thread_id === null ? null : Number(row.thread_id),
+          title: row.title,
+          message: sourceAsMarkdown(row.message, Number(row.body_format)),
+          prefixId: row.prefix_id === null ? null : Number(row.prefix_id),
+          updatedAt: new Date(row.updated_at),
+        }
+  }
+
+  async save(userId: number, draft: Draft): Promise<void> {
+    await this.db.execute(sql`
+      insert into post_drafts (user_id, forum_id, thread_id, title, message, body_format, prefix_id)
+      values (${userId}, ${draft.forumId}, ${draft.threadId}, ${draft.title}, ${draft.message}, ${BodyFormat.Markdown}, ${draft.prefixId})
+      on conflict (${sql.raw(draft.threadId === null ? 'user_id, forum_id' : 'user_id, thread_id')})
+      where ${sql.raw(draft.threadId === null ? 'thread_id is null' : 'thread_id is not null')}
+      do update set title = excluded.title, message = excluded.message,
+                    body_format = excluded.body_format, prefix_id = excluded.prefix_id,
+                    updated_at = now()
+    `)
+  }
+
+  async remove(userId: number, forumId: number, threadId: number | null): Promise<void> {
+    await this.db.execute(sql`
+      delete from post_drafts where user_id = ${userId} and forum_id = ${forumId}
+       and thread_id is not distinct from ${threadId}
+    `)
+  }
+}