@meith/db 0.33.0 → 0.33.2

package/migrations/meta/_journal.json

@@ -463,6 +463,13 @@
       "when": 1788346593870,
       "tag": "0065_serious_puff_adder",
       "breakpoints": true
+    },
+    {
+      "idx": 66,
+      "version": "7",
+      "when": 1788367778886,
+      "tag": "0066_board_digest_due_index",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meith/db",
-  "version": "0.33.0",
+  "version": "0.33.2",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -22,36 +22,36 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "postgres": "^3.4.7",
-    "@meith/accounts": "0.33.0",
-    "@meith/admin": "0.33.0",
-    "@meith/antispam": "0.33.0",
-    "@meith/api": "0.33.0",
-    "@meith/attachments": "0.33.0",
-    "@meith/authorization": "0.33.0",
-    "@meith/avatars": "0.33.0",
-    "@meith/board-digest": "0.33.0",
-    "@meith/core": "0.33.0",
-    "@meith/drafts": "0.33.0",
-    "@meith/events": "0.33.0",
-    "@meith/forums": "0.33.0",
-    "@meith/groups": "0.33.0",
-    "@meith/i18n": "0.33.0",
-    "@meith/markdown": "0.33.0",
-    "@meith/marketplace": "0.33.0",
-    "@meith/messages": "0.33.0",
-    "@meith/moderation": "0.33.0",
-    "@meith/notifications": "0.33.0",
-    "@meith/plugin-kit": "0.33.0",
-    "@meith/polls": "0.33.0",
-    "@meith/profile-fields": "0.33.0",
-    "@meith/relations": "0.33.0",
-    "@meith/reputation": "0.33.0",
-    "@meith/search": "0.33.0",
-    "@meith/settings": "0.33.0",
-    "@meith/signatures": "0.33.0",
-    "@meith/subscriptions": "0.33.0",
-    "@meith/tasks": "0.33.0",
-    "@meith/threads": "0.33.0"
+    "@meith/accounts": "0.33.2",
+    "@meith/admin": "0.33.2",
+    "@meith/antispam": "0.33.2",
+    "@meith/api": "0.33.2",
+    "@meith/attachments": "0.33.2",
+    "@meith/authorization": "0.33.2",
+    "@meith/avatars": "0.33.2",
+    "@meith/board-digest": "0.33.2",
+    "@meith/core": "0.33.2",
+    "@meith/drafts": "0.33.2",
+    "@meith/events": "0.33.2",
+    "@meith/forums": "0.33.2",
+    "@meith/groups": "0.33.2",
+    "@meith/i18n": "0.33.2",
+    "@meith/marketplace": "0.33.2",
+    "@meith/messages": "0.33.2",
+    "@meith/markdown": "0.33.2",
+    "@meith/moderation": "0.33.2",
+    "@meith/notifications": "0.33.2",
+    "@meith/plugin-kit": "0.33.2",
+    "@meith/polls": "0.33.2",
+    "@meith/profile-fields": "0.33.2",
+    "@meith/relations": "0.33.2",
+    "@meith/reputation": "0.33.2",
+    "@meith/search": "0.33.2",
+    "@meith/settings": "0.33.2",
+    "@meith/signatures": "0.33.2",
+    "@meith/subscriptions": "0.33.2",
+    "@meith/tasks": "0.33.2",
+    "@meith/threads": "0.33.2"
   },
   "devDependencies": {
     "drizzle-kit": "^0.31.6",

package/src/index.ts

@@ -172,7 +172,7 @@ export {
 export { PostgresMemberProfileRepository } from './member-profile-repo'
 export { PostgresMemberSettingsRepository } from './member-settings-repo'
 export { PostgresMessageRepository } from './message-repo'
-export { migrationUrl, runMigrations } from './migrate'
+export { migrationUrl, pendingCoreMigrations, runMigrations } from './migrate'
 export { PostgresModCpRepository } from './modcp-repo'
 export { PostgresModerationQueueRepository } from './moderation-queue'
 export {

package/src/migrate.ts

@@ -1,17 +1,37 @@
-import { existsSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { existsSync, readFileSync } from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
-import { drizzle } from 'drizzle-orm/postgres-js'
-import { migrate } from 'drizzle-orm/postgres-js/migrator'
+import { sql } from 'drizzle-orm'
 import postgres from 'postgres'
 
 import { ConfigurationError, env, logger } from '@meith/core'
 
+import type { Database } from './client'
+import { resultRows } from './result-rows'
+
 const JOURNAL = path.join('meta', '_journal.json')
 
 const IMAGE_MIGRATIONS = '/app/migrations'
 
+const STATEMENT_BREAKPOINT = '--> statement-breakpoint'
+
+export const MIGRATIONS_TABLE = 'drizzle.__drizzle_migrations'
+
+export interface Migration {
+  readonly tag: string
+  readonly hash: string
+  readonly when: number
+  readonly statements: readonly string[]
+}
+
+export interface MigrationExecutor {
+  exec(statement: string): Promise<void>
+  query<T>(statement: string): Promise<readonly T[]>
+  transaction(work: (exec: (statement: string) => Promise<void>) => Promise<void>): Promise<void>
+}
+
 export function migrationFolderCandidates(input: {
   readonly explicit?: string | undefined
   readonly moduleDir?: string | undefined
@@ -31,6 +51,7 @@ export function migrationFolderCandidates(input: {
   for (;;) {
     candidates.push(path.join(dir, 'migrations'))
     candidates.push(path.join(dir, 'packages', 'db', 'migrations'))
+    candidates.push(path.join(dir, 'node_modules', '@meith', 'db', 'migrations'))
     const parent = path.dirname(dir)
     if (parent === dir) break
     dir = parent
@@ -63,6 +84,95 @@ function migrationsFolder(): string {
   )
 }
 
+export function readMigrations(folder: string): readonly Migration[] {
+  const journal = JSON.parse(readFileSync(path.join(folder, JOURNAL), 'utf8')) as {
+    readonly entries: readonly { readonly tag: string; readonly when: number }[]
+  }
+
+  return journal.entries.map((entry) => {
+    if (!Number.isSafeInteger(entry.when)) {
+      throw new ConfigurationError(`Migration ${entry.tag} has no usable timestamp in the journal.`)
+    }
+
+    const source = readFileSync(path.join(folder, `${entry.tag}.sql`), 'utf8')
+
+    return {
+      tag: entry.tag,
+      when: entry.when,
+      hash: createHash('sha256').update(source).digest('hex'),
+      statements: source.split(STATEMENT_BREAKPOINT).filter((statement) => statement.trim() !== ''),
+    }
+  })
+}
+
+async function appliedHashes(client: MigrationExecutor): Promise<ReadonlySet<string>> {
+  const [table] = await client.query<{ found: string | null }>(
+    `select to_regclass('${MIGRATIONS_TABLE}')::text as found`,
+  )
+  if (table === undefined || table.found === null) return new Set()
+
+  const rows = await client.query<{ hash: string }>(`select hash from ${MIGRATIONS_TABLE}`)
+  return new Set(rows.map((row) => row.hash))
+}
+
+export async function pendingMigrations(
+  client: MigrationExecutor,
+  migrations: readonly Migration[],
+): Promise<readonly Migration[]> {
+  const applied = await appliedHashes(client)
+  return migrations.filter((migration) => !applied.has(migration.hash))
+}
+
+export async function applyMigrations(
+  client: MigrationExecutor,
+  migrations: readonly Migration[],
+): Promise<readonly Migration[]> {
+  const pending = await pendingMigrations(client, migrations)
+  if (pending.length === 0) return pending
+
+  await client.exec('create schema if not exists drizzle')
+  await client.exec(
+    `create table if not exists ${MIGRATIONS_TABLE} (id serial primary key, ` +
+      'hash text not null, created_at bigint)',
+  )
+
+  await client.transaction(async (exec) => {
+    for (const migration of pending) {
+      for (const statement of migration.statements) await exec(statement)
+      await exec(
+        `insert into ${MIGRATIONS_TABLE} ("hash", "created_at") ` +
+          `values ('${migration.hash}', ${migration.when})`,
+      )
+    }
+  })
+
+  return pending
+}
+
+function databaseExecutor(db: Database): MigrationExecutor {
+  return {
+    async exec(statement) {
+      await db.execute(sql.raw(statement))
+    },
+    async query<T>(statement: string) {
+      return resultRows<T>(await db.execute(sql.raw(statement)))
+    },
+    async transaction(work) {
+      await db.transaction(async (tx) => {
+        await work(async (statement) => {
+          await tx.execute(sql.raw(statement))
+        })
+      })
+    },
+  }
+}
+
+export async function pendingCoreMigrations(db: Database): Promise<readonly string[]> {
+  const migrations = readMigrations(migrationsFolder())
+  const pending = await pendingMigrations(databaseExecutor(db), migrations)
+  return pending.map((migration) => migration.tag)
+}
+
 export const MIGRATION_LOCK_KEY = '-2943916371013839929'
 
 export function migrationUrl(source: {
@@ -91,39 +201,43 @@ export async function withMigrationLock<T>(
   }
 }
 
+function postgresExecutor(sql: ReturnType<typeof postgres>): MigrationExecutor {
+  return {
+    async exec(statement) {
+      await sql.unsafe(statement)
+    },
+    async query<T>(statement: string) {
+      return (await sql.unsafe(statement)) as unknown as readonly T[]
+    },
+    async transaction(work) {
+      await sql.begin(async (tx) => {
+        await work(async (statement) => {
+          await tx.unsafe(statement)
+        })
+      })
+    },
+  }
+}
+
 export async function runMigrations(
   options: { readonly folder?: string; readonly url?: string } = {},
 ): Promise<number> {
   const url = options.url ?? migrationUrl(env)
+  const migrations = readMigrations(options.folder ?? migrationsFolder())
 
   const sql = postgres(url, { max: 1, prepare: false, onnotice: () => {} })
 
   try {
     return await withMigrationLock(sql, async () => {
-      const before = await appliedCount(sql)
-      await migrate(drizzle(sql), { migrationsFolder: options.folder ?? migrationsFolder() })
-      const after = await appliedCount(sql)
+      const applied = await applyMigrations(postgresExecutor(sql), migrations)
 
-      const applied = after - before
-      logger({ component: 'migrate' }).info({ applied }, 'migrations applied')
-      return applied
+      logger({ component: 'migrate' }).info(
+        { applied: applied.length, tags: applied.map((migration) => migration.tag) },
+        'migrations applied',
+      )
+      return applied.length
     })
   } finally {
     await sql.end({ timeout: 5 })
   }
 }
-
-async function appliedCount(sql: ReturnType<typeof postgres>): Promise<number> {
-  const rows = await sql<{ count: string }[]>`
-    SELECT COUNT(*)::text AS count
-    FROM information_schema.tables
-    WHERE table_schema = 'drizzle' AND table_name = '__drizzle_migrations'
-  `
-
-  if (rows[0]?.count === '0') return 0
-
-  const applied = await sql<{ count: string }[]>`
-    SELECT COUNT(*)::text AS count FROM drizzle.__drizzle_migrations
-  `
-  return Number(applied[0]?.count ?? '0')
-}

package/src/schema/identity.ts

@@ -149,6 +149,9 @@ export const users = pgTable(
       .on(t.signatureRenderVersion, t.id)
       .where(sql`${t.signature} <> ''`),
     index('users_mass_mail_opt_in_idx').on(t.id).where(sql`${t.massMailOptInAt} is not null`),
+    index('users_board_digest_due_idx')
+      .on(t.boardDigestCadence, t.lastActiveAt)
+      .where(sql`${t.state} = 'active'`),
   ],
 )
 

package/src/system-health-repo.ts

@@ -1,5 +1,6 @@
 import { sql } from 'drizzle-orm'
 
+import { MYBB_PREFIX, PHPBB_PREFIX } from '@meith/accounts'
 import type { TaskHealthInput } from '@meith/tasks'
 
 import type { Database } from './client'
@@ -116,6 +117,20 @@ export class PostgresSystemHealthRepository {
     }
   }
 
+  async legacyPasswordHashes(): Promise<number> {
+    const rows = resultRows(
+      await this.db.execute(sql`
+        select count(*)::int as count
+          from users
+         where deleted_at is null
+           and (password_hash like ${`${MYBB_PREFIX}%`}
+                or password_hash like ${`${PHPBB_PREFIX}%`})
+      `),
+    ) as Array<Record<string, unknown>>
+
+    return Number(rows[0]?.count ?? 0)
+  }
+
   async activeConnections(): Promise<number> {
     const rows = resultRows(
       await this.db.execute(sql`