@atproto/pds 0.5.11 → 0.5.14

package/src/db/db.ts

@@ -1,140 +0,0 @@
-import assert from 'node:assert'
-import SqliteDB from 'better-sqlite3'
-import {
-  Kysely,
-  KyselyPlugin,
-  PluginTransformQueryArgs,
-  PluginTransformResultArgs,
-  QueryResult,
-  RootOperationNode,
-  SqliteDialect,
-  UnknownRow,
-  sql,
-} from 'kysely'
-import { dbLogger } from '../logger.js'
-import { retrySqlite } from './util.js'
-
-const DEFAULT_PRAGMAS = {
-  // strict: 'ON', // @TODO strictness should live on table defs instead
-}
-
-export class Database<Schema> {
-  destroyed = false
-  commitHooks: CommitHook[] = []
-
-  constructor(public db: Kysely<Schema>) {}
-
-  static sqlite<T>(
-    location: string,
-    opts?: { pragmas?: Record<string, string> },
-  ): Database<T> {
-    const sqliteDb = new SqliteDB(location, {
-      timeout: 0, // handled by application
-    })
-    const pragmas = {
-      ...DEFAULT_PRAGMAS,
-      ...(opts?.pragmas ?? {}),
-    }
-    for (const pragma of Object.keys(pragmas)) {
-      sqliteDb.pragma(`${pragma} = ${pragmas[pragma]}`)
-    }
-    const db = new Kysely<T>({
-      dialect: new SqliteDialect({
-        database: sqliteDb,
-      }),
-    })
-    return new Database(db)
-  }
-
-  async ensureWal() {
-    await sql`PRAGMA journal_mode = WAL`.execute(this.db)
-  }
-
-  async transactionNoRetry<T>(
-    fn: (db: Database<Schema>) => T | PromiseLike<T>,
-  ): Promise<T> {
-    this.assertNotTransaction()
-    const leakyTxPlugin = new LeakyTxPlugin()
-    const { hooks, txRes } = await this.db
-      .withPlugin(leakyTxPlugin)
-      .transaction()
-      .execute(async (txn) => {
-        const dbTxn = new Database(txn)
-        try {
-          const txRes = await fn(dbTxn)
-          leakyTxPlugin.endTx()
-          const hooks = dbTxn.commitHooks
-          return { hooks, txRes }
-        } catch (err) {
-          leakyTxPlugin.endTx()
-          // ensure that all in-flight queries are flushed & the connection is open
-          await txn.getExecutor().provideConnection(async () => {})
-          throw err
-        }
-      })
-    hooks.map((hook) => hook())
-    return txRes
-  }
-
-  async transaction<T>(
-    fn: (db: Database<Schema>) => T | PromiseLike<T>,
-  ): Promise<T> {
-    return retrySqlite(() => this.transactionNoRetry(fn))
-  }
-
-  async executeWithRetry<T>(query: { execute: () => Promise<T> }) {
-    if (this.isTransaction) {
-      // transaction() ensures retry on entire transaction, no need to retry individual statements.
-      return query.execute()
-    }
-    return retrySqlite(() => query.execute())
-  }
-
-  onCommit(fn: () => void) {
-    this.assertTransaction()
-    this.commitHooks.push(fn)
-  }
-
-  get isTransaction() {
-    return this.db.isTransaction
-  }
-
-  assertTransaction() {
-    assert(this.isTransaction, 'Transaction required')
-  }
-
-  assertNotTransaction() {
-    assert(!this.isTransaction, 'Cannot be in a transaction')
-  }
-
-  close(): void {
-    if (this.destroyed) return
-    this.db
-      .destroy()
-      .then(() => (this.destroyed = true))
-      .catch((err) => dbLogger.error({ err }, 'error closing db'))
-  }
-}
-
-type CommitHook = () => void
-
-class LeakyTxPlugin implements KyselyPlugin {
-  private txOver = false
-
-  endTx() {
-    this.txOver = true
-  }
-
-  transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
-    if (this.txOver) {
-      throw new Error('tx already failed')
-    }
-    return args.node
-  }
-
-  async transformResult(
-    args: PluginTransformResultArgs,
-  ): Promise<QueryResult<UnknownRow>> {
-    return args.result
-  }
-}

package/src/db/index.ts

@@ -1,4 +0,0 @@
-export * from './db.js'
-export * from './cast.js'
-export * from './migrator.js'
-export * from './util.js'

package/src/db/migrator.ts

@@ -1,40 +0,0 @@
-import { Kysely } from 'kysely'
-import { Migration, Migrator as KyselyMigrator } from 'kysely/migration'
-
-export class Migrator<T> extends KyselyMigrator {
-  constructor(
-    public db: Kysely<T>,
-    migrations: Record<string, Migration>,
-  ) {
-    super({
-      db,
-      provider: {
-        async getMigrations() {
-          return migrations
-        },
-      },
-    })
-  }
-
-  async migrateToOrThrow(migration: string) {
-    const { error, results } = await this.migrateTo(migration)
-    if (error) {
-      throw error
-    }
-    if (!results) {
-      throw new Error('An unknown failure occurred while migrating')
-    }
-    return results
-  }
-
-  async migrateToLatestOrThrow() {
-    const { error, results } = await this.migrateToLatest()
-    if (error) {
-      throw error
-    }
-    if (!results) {
-      throw new Error('An unknown failure occurred while migrating')
-    }
-    return results
-  }
-}

package/src/db/pagination.ts

@@ -1,132 +0,0 @@
-import { SqlBool, sql } from 'kysely'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { AnyQb, DbRef } from './util.js'
-
-export type Cursor = { primary: string; secondary: string }
-export type LabeledResult = {
-  primary: string | number
-  secondary: string | number
-}
-
-/**
- * The GenericKeyset is an abstract class that sets-up the interface and partial implementation
- * of a keyset-paginated cursor with two parts. There are three types involved:
- *  - Result: a raw result (i.e. a row from the db) containing data that will make-up a cursor.
- *    - E.g. { createdAt: '2022-01-01T12:00:00Z', cid: 'bafyx' }
- *  - LabeledResult: a Result processed such that the "primary" and "secondary" parts of the cursor are labeled.
- *    - E.g. { primary: '2022-01-01T12:00:00Z', secondary: 'bafyx' }
- *  - Cursor: the two string parts that make-up the packed/string cursor.
- *    - E.g. packed cursor '1641038400000::bafyx' in parts { primary: '1641038400000', secondary: 'bafyx' }
- *
- * These types relate as such. Implementers define the relations marked with a *:
- *   Result -*-> LabeledResult <-*-> Cursor <--> packed/string cursor
- *                     ↳ SQL Condition
- */
-export abstract class GenericKeyset<R, LR extends LabeledResult> {
-  constructor(
-    public primary: DbRef,
-    public secondary: DbRef,
-  ) {}
-  abstract labelResult(result: R): LR
-  abstract labeledResultToCursor(labeled: LR): Cursor
-  abstract cursorToLabeledResult(cursor: Cursor): LR
-  packFromResult(results: R | R[]): string | undefined {
-    const result = Array.isArray(results) ? results.at(-1) : results
-    if (!result) return
-    return this.pack(this.labelResult(result))
-  }
-  pack(labeled?: LR): string | undefined {
-    if (!labeled) return
-    const cursor = this.labeledResultToCursor(labeled)
-    return this.packCursor(cursor)
-  }
-  unpack(cursorStr?: string): LR | undefined {
-    const cursor = this.unpackCursor(cursorStr)
-    if (!cursor) return
-    return this.cursorToLabeledResult(cursor)
-  }
-  packCursor(cursor?: Cursor): string | undefined {
-    if (!cursor) return
-    return `${cursor.primary}::${cursor.secondary}`
-  }
-  unpackCursor(cursorStr?: string): Cursor | undefined {
-    if (!cursorStr) return
-    const result = cursorStr.split('::')
-    const [primary, secondary, ...others] = result
-    if (!primary || !secondary || others.length > 0) {
-      throw new InvalidRequestError('Malformed cursor')
-    }
-    return {
-      primary,
-      secondary,
-    }
-  }
-  getSql(labeled?: LR, direction?: 'asc' | 'desc', tryIndex?: boolean) {
-    if (labeled === undefined) return
-    if (tryIndex) {
-      // The tryIndex param will likely disappear and become the default implementation: here for now for gradual rollout query-by-query.
-      if (direction === 'asc') {
-        return sql<SqlBool>`((${this.primary}, ${this.secondary}) > (${labeled.primary}, ${labeled.secondary}))`
-      } else {
-        return sql<SqlBool>`((${this.primary}, ${this.secondary}) < (${labeled.primary}, ${labeled.secondary}))`
-      }
-    } else {
-      // @NOTE this implementation can struggle to use an index on (primary, secondary) for pagination due to the "or" usage.
-      if (direction === 'asc') {
-        return sql<SqlBool>`((${this.primary} > ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} > ${labeled.secondary}))`
-      } else {
-        return sql<SqlBool>`((${this.primary} < ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} < ${labeled.secondary}))`
-      }
-    }
-  }
-}
-
-type CreatedAtCidResult = { createdAt: string; cid: string }
-type TimeCidLabeledResult = Cursor
-
-export class TimeCidKeyset<
-  TimeCidResult = CreatedAtCidResult,
-> extends GenericKeyset<TimeCidResult, TimeCidLabeledResult> {
-  labelResult(result: TimeCidResult): TimeCidLabeledResult
-  labelResult<TimeCidResult extends CreatedAtCidResult>(result: TimeCidResult) {
-    return { primary: result.createdAt, secondary: result.cid }
-  }
-  labeledResultToCursor(labeled: TimeCidLabeledResult) {
-    return {
-      primary: new Date(labeled.primary).getTime().toString(),
-      secondary: labeled.secondary,
-    }
-  }
-  cursorToLabeledResult(cursor: Cursor) {
-    const primaryDate = new Date(parseInt(cursor.primary, 10))
-    if (isNaN(primaryDate.getTime())) {
-      throw new InvalidRequestError('Malformed cursor')
-    }
-    return {
-      primary: primaryDate.toISOString(),
-      secondary: cursor.secondary,
-    }
-  }
-}
-
-export const paginate = <
-  QB extends AnyQb,
-  K extends GenericKeyset<unknown, any>,
->(
-  qb: QB,
-  opts: {
-    limit?: number
-    cursor?: string
-    direction?: 'asc' | 'desc'
-    keyset: K
-    tryIndex?: boolean
-  },
-): QB => {
-  const { limit, cursor, keyset, direction = 'desc', tryIndex } = opts
-  const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex)
-  return qb
-    .$if(!!limit, (q) => q.limit(limit as number))
-    .orderBy(keyset.primary, direction)
-    .orderBy(keyset.secondary, direction)
-    .$if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB
-}

package/src/db/tables/moderation.ts

@@ -1,80 +0,0 @@
-import { Generated } from 'kysely'
-import { com } from '../../lexicons/index.js'
-
-export const actionTableName = 'moderation_action'
-export const actionSubjectBlobTableName = 'moderation_action_subject_blob'
-export const reportTableName = 'moderation_report'
-export const reportResolutionTableName = 'moderation_report_resolution'
-
-export interface ModerationAction {
-  id: Generated<number>
-  action:
-    | 'tools.ozone.moderation.defs#modEventTakedown'
-    | 'tools.ozone.moderation.defs#modEventAcknowledge'
-    | 'tools.ozone.moderation.defs#modEventEscalate'
-    | 'tools.ozone.moderation.defs#modEventComment'
-    | 'tools.ozone.moderation.defs#modEventLabel'
-    | 'tools.ozone.moderation.defs#modEventReport'
-    | 'tools.ozone.moderation.defs#modEventMute'
-    | 'tools.ozone.moderation.defs#modEventUnmute'
-    | 'tools.ozone.moderation.defs#modEventMuteReporter'
-    | 'tools.ozone.moderation.defs#modEventUnmuteReporter'
-    | 'tools.ozone.moderation.defs#modEventReverseTakedown'
-    | 'tools.ozone.moderation.defs#modEventEmail'
-    | 'tools.ozone.moderation.defs#modEventResolveAppeal'
-    | 'tools.ozone.moderation.defs#modEventDivert'
-    | 'tools.ozone.moderation.defs#accountEvent'
-    | 'tools.ozone.moderation.defs#identityEvent'
-    | 'tools.ozone.moderation.defs#recordEvent'
-    | 'tools.ozone.moderation.defs#modEventPriorityScore'
-  subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
-  subjectDid: string
-  subjectUri: string | null
-  subjectCid: string | null
-  createLabelVals: string | null
-  negateLabelVals: string | null
-  comment: string | null
-  createdAt: string
-  createdBy: string
-  durationInHours: number | null
-  expiresAt: string | null
-  meta: Record<string, string | boolean> | null
-}
-
-export interface ModerationActionSubjectBlob {
-  actionId: number
-  cid: string
-  recordUri: string
-}
-
-export interface ModerationReport {
-  id: Generated<number>
-  subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
-  subjectDid: string
-  subjectUri: string | null
-  subjectCid: string | null
-  reasonType:
-    | com.atproto.moderation.defs.ReasonSpam
-    | com.atproto.moderation.defs.ReasonOther
-    | com.atproto.moderation.defs.ReasonMisleading
-    | com.atproto.moderation.defs.ReasonRude
-    | com.atproto.moderation.defs.ReasonSexual
-    | com.atproto.moderation.defs.ReasonViolation
-  reason: string | null
-  reportedByDid: string
-  createdAt: string
-}
-
-export interface ModerationReportResolution {
-  reportId: number
-  actionId: number
-  createdAt: string
-  createdBy: string
-}
-
-export type PartialDB = {
-  [actionTableName]: ModerationAction
-  [actionSubjectBlobTableName]: ModerationActionSubjectBlob
-  [reportTableName]: ModerationReport
-  [reportResolutionTableName]: ModerationReportResolution
-}

package/src/db/util.ts

@@ -1,108 +0,0 @@
-import {
-  DummyDriver,
-  DynamicModule,
-  Kysely,
-  RawBuilder,
-  ReferenceExpression,
-  SelectQueryBuilder,
-  SqlBool,
-  SqliteAdapter,
-  SqliteIntrospector,
-  SqliteQueryCompiler,
-  sql,
-} from 'kysely'
-import { retry } from '@atproto/common'
-
-// Applies to repo_root or record table
-export const notSoftDeletedClause = (alias: DbRef) => {
-  return sql<SqlBool>`${alias}."takedownRef" is null`
-}
-
-export const softDeleted = (repoOrRecord: { takedownRef: string | null }) => {
-  return repoOrRecord.takedownRef !== null
-}
-
-export const countAll = sql<number>`count(*)`
-export const countDistinct = (ref: DbRef) => sql<number>`count(distinct ${ref})`
-
-// For use with doUpdateSet()
-export const excluded = <T, S>(db: Kysely<S>, col) => {
-  return sql<T>`${db.dynamic.ref(`excluded.${col}`)}`
-}
-
-// Can be useful for large where-in clauses, to get the db to use a hash lookup on the list
-export const valuesList = (vals: unknown[]) => {
-  return sql`(values (${sql.join(vals, sql`), (`)}))`
-}
-
-export const dummyDialect = {
-  createAdapter() {
-    return new SqliteAdapter()
-  },
-  createDriver() {
-    return new DummyDriver()
-  },
-  createIntrospector(db) {
-    return new SqliteIntrospector(db)
-  },
-  createQueryCompiler() {
-    return new SqliteQueryCompiler()
-  },
-}
-
-export const retrySqlite = <T>(fn: () => Promise<T>): Promise<T> => {
-  return retry(fn, {
-    retryable: retryableSqlite,
-    getWaitMs: getWaitMsSqlite,
-    maxRetries: 60, // a safety measure: getWaitMsSqlite() times out before this after 5000ms of waiting.
-  })
-}
-
-const retryableSqlite = (err: unknown) => {
-  return typeof err?.['code'] === 'string' && RETRY_ERRORS.has(err['code'])
-}
-
-// based on sqlite's backoff strategy https://github.com/sqlite/sqlite/blob/91c8e65dd4bf17d21fbf8f7073565fe1a71c8948/src/main.c#L1704-L1713
-const getWaitMsSqlite = (n: number, timeout = 5000) => {
-  if (n < 0) return null
-  let delay: number
-  let prior: number
-  if (n < DELAYS.length) {
-    delay = DELAYS[n]
-    prior = TOTALS[n]
-  } else {
-    delay = last(DELAYS)
-    prior = last(TOTALS) + delay * (n - (DELAYS.length - 1))
-  }
-  if (prior + delay > timeout) {
-    delay = timeout - prior
-    if (delay <= 0) return null
-  }
-  return delay
-}
-
-const last = <T>(arr: T[]) => arr[arr.length - 1]
-const DELAYS = [1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100]
-const TOTALS = [0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228]
-const RETRY_ERRORS = new Set([
-  'SQLITE_BUSY',
-  'SQLITE_BUSY_SNAPSHOT',
-  'SQLITE_BUSY_RECOVERY',
-  'SQLITE_BUSY_TIMEOUT',
-])
-
-export type Ref = ReferenceExpression<any, any>
-
-export type DbRef =
-  | RawBuilder<unknown>
-  | ReturnType<DynamicModule<unknown>['ref']>
-
-export type AnyQb = SelectQueryBuilder<any, any, any>
-
-export const isErrUniqueViolation = (err: unknown) => {
-  const code = err?.['code']
-  return (
-    code === '23505' || // postgres, see https://www.postgresql.org/docs/current/errcodes-appendix.html
-    code === 'SQLITE_CONSTRAINT_UNIQUE' // sqlite, see https://www.sqlite.org/rescode.html#constraint_unique
-  )
-}

package/src/did-cache/db/index.ts

@@ -1,21 +0,0 @@
-import { Database, Migrator } from '../../db/index.js'
-import migrations from './migrations.js'
-import { DidCacheSchema } from './schema.js'
-
-export * from './schema.js'
-
-export type DidCacheDb = Database<DidCacheSchema>
-
-export const getDb = (
-  location: string,
-  disableWalAutoCheckpoint = false,
-): DidCacheDb => {
-  const pragmas: Record<string, string> = disableWalAutoCheckpoint
-    ? { wal_autocheckpoint: '0', synchronous: 'NORMAL' }
-    : { synchronous: 'NORMAL' }
-  return Database.sqlite(location, { pragmas })
-}
-
-export const getMigrator = (db: DidCacheDb) => {
-  return new Migrator(db.db, migrations)
-}

package/src/did-cache/db/migrations.ts

@@ -1,17 +0,0 @@
-import { Kysely } from 'kysely'
-
-export default {
-  '001': {
-    up: async (db: Kysely<unknown>) => {
-      await db.schema
-        .createTable('did_doc')
-        .addColumn('did', 'varchar', (col) => col.primaryKey())
-        .addColumn('doc', 'text', (col) => col.notNull())
-        .addColumn('updatedAt', 'bigint', (col) => col.notNull())
-        .execute()
-    },
-    down: async (db: Kysely<unknown>) => {
-      await db.schema.dropTable('did_doc').execute()
-    },
-  },
-}

package/src/did-cache/db/schema.ts

@@ -1,9 +0,0 @@
-export interface DidDoc {
-  did: string
-  doc: string // json representation of DidDocument
-  updatedAt: number
-}
-
-export type DidCacheSchema = {
-  did_doc: DidDoc
-}

package/src/did-cache/index.ts

@@ -1,131 +0,0 @@
-import PQueue from 'p-queue'
-import { CacheResult, DidCache, DidDocument } from '@atproto/identity'
-import { excluded } from '../db/util.js'
-import { didCacheLogger } from '../logger.js'
-import { DidCacheDb, getDb, getMigrator } from './db/index.js'
-
-export class DidSqliteCache implements DidCache {
-  db: DidCacheDb
-  public pQueue: PQueue | null //null during teardown
-
-  constructor(
-    dbLocation: string,
-    public staleTTL: number,
-    public maxTTL: number,
-    disableWalAutoCheckpoint = false,
-  ) {
-    this.db = getDb(dbLocation, disableWalAutoCheckpoint)
-    this.pQueue = new PQueue()
-  }
-
-  async cacheDid(
-    did: string,
-    doc: DidDocument,
-    prevResult?: CacheResult,
-  ): Promise<void> {
-    try {
-      if (prevResult) {
-        await this.db.executeWithRetry(
-          this.db.db
-            .updateTable('did_doc')
-            .set({ doc: JSON.stringify(doc), updatedAt: Date.now() })
-            .where('did', '=', did)
-            .where('updatedAt', '=', prevResult.updatedAt),
-        )
-      } else {
-        await this.db.executeWithRetry(
-          this.db.db
-            .insertInto('did_doc')
-            .values({ did, doc: JSON.stringify(doc), updatedAt: Date.now() })
-            .onConflict((oc) =>
-              oc.column('did').doUpdateSet({
-                doc: excluded(this.db.db, 'doc'),
-                updatedAt: excluded(this.db.db, 'updatedAt'),
-              }),
-            ),
-        )
-      }
-    } catch (err) {
-      didCacheLogger.error({ did, doc, err }, 'failed to cache did')
-    }
-  }
-
-  async refreshCache(
-    did: string,
-    getDoc: () => Promise<DidDocument | null>,
-    prevResult?: CacheResult,
-  ): Promise<void> {
-    this.pQueue?.add(async () => {
-      try {
-        const doc = await getDoc()
-        if (doc) {
-          await this.cacheDid(did, doc, prevResult)
-        } else {
-          await this.clearEntry(did)
-        }
-      } catch (err) {
-        didCacheLogger.error({ did, err }, 'refreshing did cache failed')
-      }
-    })
-  }
-
-  async checkCache(did: string): Promise<CacheResult | null> {
-    try {
-      return await this.checkCacheInternal(did)
-    } catch (err) {
-      didCacheLogger.error({ did, err }, 'failed to check did cache')
-      return null
-    }
-  }
-
-  async checkCacheInternal(did: string): Promise<CacheResult | null> {
-    const res = await this.db.db
-      .selectFrom('did_doc')
-      .where('did', '=', did)
-      .selectAll()
-      .executeTakeFirst()
-    if (!res) return null
-    const now = Date.now()
-    const updatedAt = new Date(res.updatedAt).getTime()
-    const expired = now > updatedAt + this.maxTTL
-    const stale = now > updatedAt + this.staleTTL
-    return {
-      doc: JSON.parse(res.doc) as DidDocument,
-      updatedAt,
-      did,
-      stale,
-      expired,
-    }
-  }
-
-  async clearEntry(did: string): Promise<void> {
-    try {
-      await this.db.executeWithRetry(
-        this.db.db.deleteFrom('did_doc').where('did', '=', did),
-      )
-    } catch (err) {
-      didCacheLogger.error({ did, err }, 'clearing did cache entry failed')
-    }
-  }
-
-  async clear(): Promise<void> {
-    await this.db.db.deleteFrom('did_doc').execute()
-  }
-
-  async processAll() {
-    await this.pQueue?.onIdle()
-  }
-
-  async migrateOrThrow() {
-    await this.db.ensureWal()
-    await getMigrator(this.db).migrateToLatestOrThrow()
-  }
-
-  async destroy() {
-    const pQueue = this.pQueue
-    this.pQueue = null
-    pQueue?.pause()
-    pQueue?.clear()
-    await pQueue?.onIdle()
-  }
-}