@meith/db 0.16.0

package/src/search-repo.ts

@@ -0,0 +1,359 @@
+import { type SQL, sql } from 'drizzle-orm'
+
+import type {
+  AuthorFacet,
+  ForumFacet,
+  SearchCursor,
+  SearchHit,
+  SearchQuery,
+  SearchResults,
+  SearchScope,
+  SearchSummary,
+} from '@meith/search'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+import { inAudience } from './thread-audience'
+import { visibleIn } from './visibility'
+
+const SEARCH_CONFIG = 'english'
+
+export const SEARCH_WINDOW = 20_000
+
+const TOP_AUTHORS = 24
+
+export const SEARCH_DOCUMENT_VERSION = 1
+
+const EXCERPT_HIGHLIGHT_START = String.fromCharCode(0xe000)
+const EXCERPT_HIGHLIGHT_STOP = String.fromCharCode(0xe001)
+
+const HEADLINE_OPTIONS =
+  `StartSel=${EXCERPT_HIGHLIGHT_START}, StopSel=${EXCERPT_HIGHLIGHT_STOP}, ` +
+  `MaxFragments=1, MaxWords=40, MinWords=15`
+
+const EXCERPT_HTML_ESCAPES: Readonly<Record<string, string>> = {
+  '&': '&amp;',
+  '<': '&lt;',
+  '>': '&gt;',
+  '"': '&quot;',
+  "'": '&#39;',
+}
+
+export function renderExcerptHtml(headline: string): string {
+  const escaped = headline.replace(
+    /[&<>"']/g,
+    (character) => EXCERPT_HTML_ESCAPES[character] ?? character,
+  )
+  return escaped
+    .split(EXCERPT_HIGHLIGHT_START)
+    .join('<b>')
+    .split(EXCERPT_HIGHLIGHT_STOP)
+    .join('</b>')
+}
+
+export function searchVectorSql(subject: SQL | string, message: SQL | string): SQL {
+  return sql`
+    setweight(to_tsvector(${SEARCH_CONFIG}, coalesce(${subject}, '')), 'A') ||
+    setweight(to_tsvector(${SEARCH_CONFIG}, coalesce(${message}, '')), 'B')
+  `
+}
+
+export function indexedSubjectSql(post: SQL): SQL {
+  return sql`coalesce(
+    ${post}.subject,
+    case when ${post}.is_first_post
+      then (select t.title from threads t where t.id = ${post}.thread_id)
+    end
+  )`
+}
+
+const PENDING_INDEX: SQL = sql`(search_vector is null or search_version < ${SEARCH_DOCUMENT_VERSION})`
+
+const NOTHING: SearchSummary = { total: 0, isCapped: false, forums: [], authors: [] }
+
+export interface ReindexResult {
+  readonly indexed: number
+  readonly nextCursor: number | null
+}
+
+export class PostgresSearchRepository {
+  constructor(private readonly db: Database) {}
+
+  async search(query: SearchQuery, scope: SearchScope): Promise<SearchResults> {
+    const conditions = matchConditions(query, scope)
+    if (conditions === null) return { hits: [], nextCursor: null }
+
+    const rank = rankSql(query.terms)
+    const finalOrder = orderSql(query.sort)
+
+    const selection = sql`
+      select p.id as post_id, p.thread_id, p.forum_id,
+             t.title as thread_title, t.slug as thread_slug,
+             p.author_user_id, p.author_username, p.created_at,
+             ${excerptSourceSql(query.match)} as excerpt_source,
+             ${rank} as rank
+        from posts p
+        join threads t on t.id = p.thread_id`
+
+    const candidates =
+      query.grouping === 'threads'
+        ? groupedCandidates(query, selection, conditions)
+        : flatCandidates(query, selection, conditions, rank)
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select post_id, thread_id, forum_id, thread_title, thread_slug,
+               author_user_id, author_username, created_at, rank,
+               ts_headline(${SEARCH_CONFIG}, excerpt_source,
+                           websearch_to_tsquery(${SEARCH_CONFIG}, ${query.terms}),
+                           ${HEADLINE_OPTIONS}) as excerpt
+          from ${candidates}
+         order by ${finalOrder}
+         limit ${query.limit} offset ${query.after === null ? (query.offset ?? 0) : 0}
+      `),
+    ) as Array<Record<string, unknown>>
+
+    const hits: SearchHit[] = rows.map((row) => ({
+      postId: Number(row.post_id),
+      threadId: Number(row.thread_id),
+      forumId: Number(row.forum_id),
+      threadTitle: String(row.thread_title),
+      threadSlug: String(row.thread_slug),
+      authorUserId: row.author_user_id === null ? null : Number(row.author_user_id),
+      authorUsername: String(row.author_username),
+      postedAt: row.created_at instanceof Date ? row.created_at : new Date(String(row.created_at)),
+      excerpt: renderExcerptHtml(String(row.excerpt ?? '')),
+      rank: Number(row.rank),
+    }))
+
+    const last = hits.at(-1)
+    const nextCursor: SearchCursor | null =
+      hits.length < query.limit || last === undefined
+        ? null
+        : { rank: last.rank, postId: last.postId }
+
+    return { hits, nextCursor }
+  }
+
+  async summarize(query: SearchQuery, scope: SearchScope): Promise<SearchSummary> {
+    const conditions = matchConditions(query, scope)
+    if (conditions === null) return NOTHING
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        with candidates as (
+          select p.forum_id, p.thread_id, p.author_user_id, p.author_username
+            from posts p
+            join threads t on t.id = p.thread_id
+           where ${sql.join(conditions, sql` and `)}
+           limit ${SEARCH_WINDOW + 1}
+        )
+        select 'total' as facet, null::int as id, ''::text as username,
+               count(*)::int as posts, count(distinct thread_id)::int as threads
+          from candidates
+        union all
+        select 'forum', forum_id, '', count(*)::int, count(distinct thread_id)::int
+          from candidates
+         group by forum_id
+        union all
+        select * from (
+          select 'author' as facet, author_user_id as id,
+                 coalesce(max(author_username), '') as username,
+                 count(*)::int as posts, count(distinct thread_id)::int as threads
+            from candidates
+           where author_user_id is not null
+           group by author_user_id
+           order by count(*) desc
+           limit ${TOP_AUTHORS}
+        ) as top_authors
+      `),
+    ) as Array<Record<string, unknown>>
+
+    const scanned = rows.find((row) => row.facet === 'total')
+    if (scanned === undefined) return NOTHING
+
+    const isCapped = Number(scanned.posts) > SEARCH_WINDOW
+    const count = (row: Record<string, unknown>): number =>
+      query.grouping === 'threads'
+        ? Number(row.threads)
+        : Math.min(Number(row.posts), SEARCH_WINDOW)
+
+    const forums: ForumFacet[] = []
+    const authors: AuthorFacet[] = []
+
+    for (const row of rows) {
+      if (row.facet === 'forum') {
+        forums.push({ forumId: Number(row.id), hits: count(row) })
+      } else if (row.facet === 'author') {
+        authors.push({
+          userId: Number(row.id),
+          username: String(row.username),
+          hits: count(row),
+        })
+      }
+    }
+
+    const byHits = <T extends { readonly hits: number }>(a: T, b: T): number => b.hits - a.hits
+
+    return {
+      total: count(scanned),
+      isCapped,
+      forums: forums.sort(byHits),
+      authors: authors.sort(byHits),
+    }
+  }
+
+  async reindexChunk(afterPostId: number, limit: number): Promise<ReindexResult> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        update posts p
+           set search_vector = ${searchVectorSql(indexedSubjectSql(sql`p`), sql`p.message`)},
+               search_version = ${SEARCH_DOCUMENT_VERSION}
+         where p.id in (
+           select id from posts
+            where ${PENDING_INDEX} and id > ${afterPostId}
+            order by id
+            limit ${limit}
+         )
+        returning p.id
+      `),
+    ) as Array<{ id: number }>
+
+    const ids = rows.map((row) => Number(row.id)).sort((a, b) => a - b)
+
+    return {
+      indexed: ids.length,
+      nextCursor: ids.length < limit ? null : (ids.at(-1) ?? null),
+    }
+  }
+
+  async indexProgress(): Promise<{ readonly indexed: number; readonly pending: number }> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select count(*) filter (where not ${PENDING_INDEX})::int as indexed,
+               count(*) filter (where ${PENDING_INDEX})::int as pending
+          from posts
+      `),
+    ) as Array<{ indexed: number; pending: number }>
+
+    return {
+      indexed: Number(rows[0]?.indexed ?? 0),
+      pending: Number(rows[0]?.pending ?? 0),
+    }
+  }
+
+  async invalidateIndex(): Promise<void> {
+    await this.db.execute(sql`update posts set search_vector = null, search_version = 0`)
+  }
+}
+
+function rankSql(terms: string): SQL {
+  return sql`ts_rank_cd(p.search_vector, websearch_to_tsquery(${SEARCH_CONFIG}, ${terms}))`
+}
+
+function excerptSourceSql(match: SearchQuery['match']): SQL {
+  return match === 'titles' ? sql`coalesce(p.subject, t.title)` : sql`p.message`
+}
+
+function orderSql(sort: SearchQuery['sort']): SQL {
+  return sort === 'relevance'
+    ? sql`rank desc, post_id desc`
+    : sort === 'newest'
+      ? sql`post_id desc`
+      : sql`post_id asc`
+}
+
+function matchConditions(query: SearchQuery, scope: SearchScope): SQL[] | null {
+  if (query.terms.trim() === '') return null
+
+  const allowed =
+    query.forumIds === undefined
+      ? scope.forumIds
+      : scope.forumIds.filter((id) => query.forumIds!.includes(id))
+
+  if (allowed.length === 0) return null
+  if (query.authorUserIds !== undefined && query.authorUserIds.length === 0) return null
+
+  const conditions: SQL[] = [
+    inAudience(sql`p.forum_id`, sql`t.author_user_id`, {
+      ...scope,
+      forumIds: allowed,
+    }),
+    sql`p.search_vector @@ websearch_to_tsquery(${SEARCH_CONFIG}, ${query.terms})`,
+  ]
+
+  conditions.push(visibleIn(sql`p.visibility`, scope.content))
+  conditions.push(visibleIn(sql`t.visibility`, scope.content))
+
+  if (query.match === 'titles') {
+    conditions.push(
+      sql`to_tsvector(${SEARCH_CONFIG}, coalesce(${indexedSubjectSql(sql`p`)}, ''))
+          @@ websearch_to_tsquery(${SEARCH_CONFIG}, ${query.terms})`,
+    )
+  }
+
+  if (query.authorUserIds !== undefined) {
+    conditions.push(
+      sql`p.author_user_id in (${sql.join(
+        query.authorUserIds.map((id) => sql`${id}`),
+        sql`, `,
+      )})`,
+    )
+  }
+
+  if (query.postedAfter !== undefined) conditions.push(sql`p.created_at >= ${query.postedAfter}`)
+  if (query.postedBefore !== undefined) conditions.push(sql`p.created_at < ${query.postedBefore}`)
+
+  return conditions
+}
+
+function cursorSql(query: SearchQuery, rank: SQL, postId: SQL): SQL | null {
+  if (query.after === null) return null
+
+  return query.sort === 'relevance'
+    ? sql`(${rank} < ${query.after.rank}
+           or (${rank} = ${query.after.rank} and ${postId} < ${query.after.postId}))`
+    : query.sort === 'newest'
+      ? sql`${postId} < ${query.after.postId}`
+      : sql`${postId} > ${query.after.postId}`
+}
+
+function flatCandidates(
+  query: SearchQuery,
+  selection: SQL,
+  conditions: readonly SQL[],
+  rank: SQL,
+): SQL {
+  const cursor = cursorSql(query, rank, sql`p.id`)
+  const all = cursor === null ? [...conditions] : [...conditions, cursor]
+  const scan = sql`${selection} where ${sql.join(all, sql` and `)}`
+
+  return query.sort === 'relevance'
+    ? sql`(${scan} order by p.id desc limit ${SEARCH_WINDOW}) as candidates`
+    : sql`(${scan} order by ${query.sort === 'newest' ? sql`p.id desc` : sql`p.id asc`}
+           limit ${query.limit}) as candidates`
+}
+
+function groupedCandidates(query: SearchQuery, selection: SQL, conditions: readonly SQL[]): SQL {
+  const scan = sql`${selection} where ${sql.join([...conditions], sql` and `)}
+                   order by ${query.sort === 'oldest' ? sql`p.id asc` : sql`p.id desc`}
+                   limit ${SEARCH_WINDOW}`
+
+  const representative =
+    query.sort === 'relevance'
+      ? sql`thread_id, rank desc, post_id desc`
+      : query.sort === 'newest'
+        ? sql`thread_id, post_id desc`
+        : sql`thread_id, post_id asc`
+
+  const grouped = sql`
+    select distinct on (thread_id) * from (${scan}) as scanned order by ${representative}`
+
+  const cursor = cursorSql(query, sql`rank`, sql`post_id`)
+
+  return cursor === null
+    ? sql`(select * from (${grouped}) as grouped
+            order by ${orderSql(query.sort)} limit ${query.limit}) as candidates`
+    : sql`(select * from (${grouped}) as grouped where ${cursor}
+            order by ${orderSql(query.sort)} limit ${query.limit}) as candidates`
+}

package/src/search-store.ts

@@ -0,0 +1,113 @@
+import { sql } from 'drizzle-orm'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+
+export interface StoredSearch {
+  readonly id: number
+  readonly token: string
+  readonly userId: number | null
+  readonly sessionKey: string | null
+  readonly terms: string
+  readonly filters: Readonly<Record<string, unknown>>
+  readonly hitCount: number
+  readonly createdAt: Date
+}
+
+export interface CreateSearchInput {
+  readonly token: string
+  readonly userId: number | null
+  readonly sessionKey: string | null
+  readonly terms: string
+  readonly filters: Readonly<Record<string, unknown>>
+  readonly floodSeconds: number
+}
+
+export class PostgresSearchStore {
+  constructor(private readonly db: Database) {}
+
+  async create(input: CreateSearchInput): Promise<StoredSearch | null> {
+    const owner =
+      input.userId !== null
+        ? sql`user_id = ${input.userId}`
+        : input.sessionKey !== null
+          ? sql`session_key = ${input.sessionKey}`
+          : sql`false`
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        insert into searches (token, user_id, session_key, terms, filters)
+        select ${input.token}, ${input.userId}, ${input.sessionKey}, ${input.terms},
+               ${JSON.stringify(input.filters)}::jsonb
+         where not exists (
+              select 1 from searches
+               where ${owner}
+                 and created_at > now() - make_interval(secs => ${input.floodSeconds})
+            )
+        returning id, token, user_id, session_key, terms, filters, hit_count, created_at
+      `),
+    ) as Array<Record<string, unknown>>
+
+    const row = rows[0]
+    return row === undefined ? null : toStored(row)
+  }
+
+  async recordHitCount(id: number, hitCount: number): Promise<void> {
+    await this.db.execute(sql`update searches set hit_count = ${hitCount} where id = ${id}`)
+  }
+
+  async findByToken(token: string): Promise<StoredSearch | null> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select id, token, user_id, session_key, terms, filters, hit_count, created_at
+          from searches where token = ${token}
+      `),
+    ) as Array<Record<string, unknown>>
+
+    const row = rows[0]
+    return row === undefined ? null : toStored(row)
+  }
+
+  async prune(olderThan: Date, limit = 5_000): Promise<number> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        delete from searches
+         where id in (
+           select id from searches where created_at < ${olderThan} order by id limit ${limit}
+         )
+        returning id
+      `),
+    ) as Array<{ id: number }>
+
+    return rows.length
+  }
+}
+
+export function ownsSearch(
+  search: Pick<StoredSearch, 'userId' | 'sessionKey'>,
+  viewer: { readonly userId: number | null; readonly sessionKey: string | null },
+): boolean {
+  if (search.userId !== null) return search.userId === viewer.userId
+  if (search.sessionKey !== null) {
+    return viewer.userId === null && search.sessionKey === viewer.sessionKey
+  }
+
+  return true
+}
+
+function toStored(row: Record<string, unknown>): StoredSearch {
+  const filters = row.filters
+  return {
+    id: Number(row.id),
+    token: String(row.token),
+    userId: row.user_id === null ? null : Number(row.user_id),
+    sessionKey: row.session_key === null ? null : String(row.session_key),
+    terms: String(row.terms),
+    filters:
+      typeof filters === 'object' && filters !== null && !Array.isArray(filters)
+        ? (filters as Record<string, unknown>)
+        : {},
+    hitCount: Number(row.hit_count),
+    createdAt: row.created_at instanceof Date ? row.created_at : new Date(String(row.created_at)),
+  }
+}

package/src/seed-groups.ts

@@ -0,0 +1,11 @@
+export const SEED_GROUP_KEY = {
+  guests: 'guests',
+  registered: 'registered',
+  administrators: 'administrators',
+  superModerators: 'super_moderators',
+  moderators: 'moderators',
+  awaitingActivation: 'awaiting_activation',
+  banned: 'banned',
+} as const
+
+export type SeedGroupKey = (typeof SEED_GROUP_KEY)[keyof typeof SEED_GROUP_KEY]

package/src/settings-repo.ts

@@ -0,0 +1,32 @@
+import { inArray, sql } from 'drizzle-orm'
+
+import type { SettingsRepository } from '@meith/settings'
+
+import type { Database } from './client'
+import { settings } from './schema'
+
+export class PostgresSettingsRepository implements SettingsRepository {
+  constructor(private readonly db: Database) {}
+
+  async loadAll(): Promise<ReadonlyMap<string, string>> {
+    const rows = await this.db.select({ key: settings.key, value: settings.value }).from(settings)
+    return new Map(rows.map((r) => [r.key, r.value]))
+  }
+
+  async save(entries: ReadonlyMap<string, string>): Promise<void> {
+    if (entries.size === 0) return
+
+    await this.db
+      .insert(settings)
+      .values([...entries].map(([key, value]) => ({ key, value })))
+      .onConflictDoUpdate({
+        target: settings.key,
+        set: { value: sql`excluded.value`, updatedAt: new Date() },
+      })
+  }
+
+  async delete(keys: readonly string[]): Promise<void> {
+    if (keys.length === 0) return
+    await this.db.delete(settings).where(inArray(settings.key, [...keys]))
+  }
+}

package/src/signature-repo.ts

@@ -0,0 +1,102 @@
+import { sql } from 'drizzle-orm'
+
+import { BodyFormat } from '@meith/markdown'
+import type { StoredSignature } from '@meith/signatures'
+
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+
+interface RawSignature {
+  signature: string
+  signature_html: string | null
+  signature_render_version: number
+  signature_format: number
+  signature_locked: boolean
+  signature_locked_reason: string | null
+}
+
+function toSignature(row: RawSignature): StoredSignature {
+  return {
+    signature: row.signature,
+    signatureHtml: row.signature_html,
+    signatureRenderVersion: Number(row.signature_render_version),
+    signatureFormat: Number(row.signature_format),
+    locked: row.signature_locked === true,
+    lockedReason: row.signature_locked_reason,
+  }
+}
+
+const COLUMNS = sql`
+  signature, signature_html, signature_render_version, signature_format,
+  signature_locked, signature_locked_reason
+`
+
+export class PostgresSignatureRepository {
+  constructor(private readonly db: Database) {}
+
+  async read(userId: number): Promise<StoredSignature | null> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select ${COLUMNS} from users where id = ${userId} and deleted_at is null
+      `),
+    ) as RawSignature[]
+
+    return rows[0] === undefined ? null : toSignature(rows[0])
+  }
+
+  async readMany(userIds: readonly number[]): Promise<ReadonlyMap<number, StoredSignature>> {
+    if (userIds.length === 0) return new Map()
+
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select id, ${COLUMNS}
+          from users
+         where id in (${sql.join(
+           userIds.map((id) => sql`${id}`),
+           sql`, `,
+         )})
+      `),
+    ) as Array<RawSignature & { id: number }>
+
+    return new Map(rows.map((row) => [Number(row.id), toSignature(row)]))
+  }
+
+  async save(input: {
+    readonly userId: number
+    readonly signature: string
+    readonly signatureHtml: string
+    readonly renderVersion: number
+  }): Promise<boolean> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        update users
+           set signature = ${input.signature},
+               signature_html = ${input.signatureHtml},
+               signature_render_version = ${input.renderVersion},
+               /*
+                * A save is what takes a member off the legacy format for good:
+                * what they just typed in the composer is Markdown, whatever the
+                * row said a moment ago.
+                */
+               signature_format = ${BodyFormat.Markdown}
+         where id = ${input.userId} and signature_locked = false
+        returning id
+      `),
+    ) as Array<{ id: number }>
+
+    return rows.length > 0
+  }
+
+  async setLocked(input: {
+    readonly userId: number
+    readonly locked: boolean
+    readonly reason: string | null
+  }): Promise<void> {
+    await this.db.execute(sql`
+      update users
+         set signature_locked = ${input.locked},
+             signature_locked_reason = ${input.locked ? input.reason : null}
+       where id = ${input.userId}
+    `)
+  }
+}

package/src/sql-lists.ts

@@ -0,0 +1,17 @@
+import { type SQL, sql } from 'drizzle-orm'
+
+export function idList(ids: readonly number[]): SQL {
+  if (ids.length === 0) return sql`(null)`
+  return sql`(${sql.join(
+    ids.map((id) => sql`${id}`),
+    sql`, `,
+  )})`
+}
+
+export function textList(values: readonly string[]): SQL {
+  if (values.length === 0) return sql`(null)`
+  return sql`(${sql.join(
+    values.map((v) => sql`${v}`),
+    sql`, `,
+  )})`
+}

package/src/staff-groups.ts

@@ -0,0 +1,50 @@
+import { type SQL, sql } from 'drizzle-orm'
+
+import { columnName } from './schema/permission-columns'
+
+export const PLUGIN_UNGRANTABLE_PERMISSIONS = [
+  'isAdministrator',
+  'canAccessAdminCp',
+  'isSuperModerator',
+  'canAccessModCp',
+  'canWarnUsers',
+  'canApproveContent',
+  'canEditOthersPosts',
+  'canSoftDeletePosts',
+] as const
+
+export type UngrantablePermission = (typeof PLUGIN_UNGRANTABLE_PERMISSIONS)[number]
+
+export function permissionsCarryPower(
+  permissions: Readonly<Partial<Record<UngrantablePermission, boolean | number>>>,
+): boolean {
+  return PLUGIN_UNGRANTABLE_PERMISSIONS.some((key) => permissions[key] === true)
+}
+
+export function isStaffGroupSql(groupAlias: string): SQL {
+  const columns = PLUGIN_UNGRANTABLE_PERMISSIONS.map((key) =>
+    sql.raw(`${groupAlias}.${columnName(key)}`),
+  )
+
+  return sql.join(
+    [
+      sql`${sql.raw(`${groupAlias}.is_staff_group`)} = true`,
+      ...columns.map((c) => sql`${c} = true`),
+    ],
+    sql` or `,
+  )
+}
+
+export function keptDisplayGroupSql(input: {
+  readonly column: SQL
+  readonly leavingGroupId: SQL
+  readonly arrivingGroupId: SQL
+}): SQL {
+  return sql`nullif(nullif(${input.column}, ${input.leavingGroupId}), ${input.arrivingGroupId})`
+}
+
+export function displayGroupIdSql(userAlias: string, primaryAlias: string): SQL {
+  return sql`case when (${isStaffGroupSql(primaryAlias)}) then ${sql.raw(`${userAlias}.primary_group_id`)}
+                  else coalesce(${sql.raw(`${userAlias}.display_group_id`)}, ${sql.raw(`${userAlias}.primary_group_id`)})
+             end`
+}