@atproto/bsky 0.0.15 → 0.0.17

package/src/daemon/config.ts

@@ -0,0 +1,60 @@
+import assert from 'assert'
+
+export interface DaemonConfigValues {
+  version: string
+  dbPostgresUrl: string
+  dbPostgresSchema?: string
+  notificationsDaemonFromDid?: string
+}
+
+export class DaemonConfig {
+  constructor(private cfg: DaemonConfigValues) {}
+
+  static readEnv(overrides?: Partial<DaemonConfigValues>) {
+    const version = process.env.BSKY_VERSION || '0.0.0'
+    const dbPostgresUrl =
+      overrides?.dbPostgresUrl || process.env.DB_PRIMARY_POSTGRES_URL
+    const dbPostgresSchema =
+      overrides?.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA
+    const notificationsDaemonFromDid =
+      overrides?.notificationsDaemonFromDid ||
+      process.env.BSKY_NOTIFS_DAEMON_FROM_DID ||
+      undefined
+    assert(dbPostgresUrl)
+    return new DaemonConfig({
+      version,
+      dbPostgresUrl,
+      dbPostgresSchema,
+      notificationsDaemonFromDid,
+      ...stripUndefineds(overrides ?? {}),
+    })
+  }
+
+  get version() {
+    return this.cfg.version
+  }
+
+  get dbPostgresUrl() {
+    return this.cfg.dbPostgresUrl
+  }
+
+  get dbPostgresSchema() {
+    return this.cfg.dbPostgresSchema
+  }
+
+  get notificationsDaemonFromDid() {
+    return this.cfg.notificationsDaemonFromDid
+  }
+}
+
+function stripUndefineds(
+  obj: Record<string, unknown>,
+): Record<string, unknown> {
+  const result = {}
+  Object.entries(obj).forEach(([key, val]) => {
+    if (val !== undefined) {
+      result[key] = val
+    }
+  })
+  return result
+}

package/src/daemon/context.ts

@@ -0,0 +1,27 @@
+import { PrimaryDatabase } from '../db'
+import { DaemonConfig } from './config'
+import { Services } from './services'
+
+export class DaemonContext {
+  constructor(
+    private opts: {
+      db: PrimaryDatabase
+      cfg: DaemonConfig
+      services: Services
+    },
+  ) {}
+
+  get db(): PrimaryDatabase {
+    return this.opts.db
+  }
+
+  get cfg(): DaemonConfig {
+    return this.opts.cfg
+  }
+
+  get services(): Services {
+    return this.opts.services
+  }
+}
+
+export default DaemonContext

package/src/daemon/index.ts

@@ -0,0 +1,78 @@
+import { PrimaryDatabase } from '../db'
+import { dbLogger } from '../logger'
+import { DaemonConfig } from './config'
+import { DaemonContext } from './context'
+import { createServices } from './services'
+import { ImageUriBuilder } from '../image/uri'
+import { NotificationsDaemon } from './notifications'
+import logger from './logger'
+
+export { DaemonConfig } from './config'
+export type { DaemonConfigValues } from './config'
+
+export class BskyDaemon {
+  public ctx: DaemonContext
+  public notifications: NotificationsDaemon
+  private dbStatsInterval: NodeJS.Timer
+  private notifStatsInterval: NodeJS.Timer
+
+  constructor(opts: {
+    ctx: DaemonContext
+    notifications: NotificationsDaemon
+  }) {
+    this.ctx = opts.ctx
+    this.notifications = opts.notifications
+  }
+
+  static create(opts: { db: PrimaryDatabase; cfg: DaemonConfig }): BskyDaemon {
+    const { db, cfg } = opts
+    const imgUriBuilder = new ImageUriBuilder('https://daemon.invalid') // will not be used by daemon
+    const services = createServices({
+      imgUriBuilder,
+    })
+    const ctx = new DaemonContext({
+      db,
+      cfg,
+      services,
+    })
+    const notifications = new NotificationsDaemon(ctx)
+    return new BskyDaemon({ ctx, notifications })
+  }
+
+  async start() {
+    const { db, cfg } = this.ctx
+    const pool = db.pool
+    this.notifications.run({
+      startFromDid: cfg.notificationsDaemonFromDid,
+    })
+    this.dbStatsInterval = setInterval(() => {
+      dbLogger.info(
+        {
+          idleCount: pool.idleCount,
+          totalCount: pool.totalCount,
+          waitingCount: pool.waitingCount,
+        },
+        'db pool stats',
+      )
+    }, 10000)
+    this.notifStatsInterval = setInterval(() => {
+      logger.info(
+        {
+          count: this.notifications.count,
+          lastDid: this.notifications.lastDid,
+        },
+        'notifications daemon stats',
+      )
+    }, 10000)
+    return this
+  }
+
+  async destroy(): Promise<void> {
+    await this.notifications.destroy()
+    await this.ctx.db.close()
+    clearInterval(this.dbStatsInterval)
+    clearInterval(this.notifStatsInterval)
+  }
+}
+
+export default BskyDaemon

package/src/daemon/logger.ts

@@ -0,0 +1,6 @@
+import { subsystemLogger } from '@atproto/common'
+
+const logger: ReturnType<typeof subsystemLogger> =
+  subsystemLogger('bsky:daemon')
+
+export default logger

package/src/daemon/notifications.ts

@@ -0,0 +1,54 @@
+import { tidyNotifications } from '../services/util/notification'
+import DaemonContext from './context'
+import logger from './logger'
+
+export class NotificationsDaemon {
+  ac = new AbortController()
+  running: Promise<void> | undefined
+  count = 0
+  lastDid: string | null = null
+
+  constructor(private ctx: DaemonContext) {}
+
+  run(opts?: RunOptions) {
+    if (this.running) return
+    this.count = 0
+    this.lastDid = null
+    this.ac = new AbortController()
+    this.running = this.tidyNotifications({
+      ...opts,
+      forever: opts?.forever !== false, // run forever by default
+    })
+      .catch((err) => {
+        // allow this to cause an unhandled rejection, let deployment handle the crash.
+        logger.error({ err }, 'notifications daemon crashed')
+        throw err
+      })
+      .finally(() => (this.running = undefined))
+  }
+
+  private async tidyNotifications(opts: RunOptions) {
+    const actorService = this.ctx.services.actor(this.ctx.db)
+    for await (const { did } of actorService.all(opts)) {
+      if (this.ac.signal.aborted) return
+      try {
+        await tidyNotifications(this.ctx.db, did)
+        this.count++
+        this.lastDid = did
+      } catch (err) {
+        logger.warn({ err, did }, 'failed to tidy notifications for actor')
+      }
+    }
+  }
+
+  async destroy() {
+    this.ac.abort()
+    await this.running
+  }
+}
+
+type RunOptions = {
+  forever?: boolean
+  batchSize?: number
+  startFromDid?: string
+}

package/src/daemon/services.ts

@@ -0,0 +1,22 @@
+import { PrimaryDatabase } from '../db'
+import { ActorService } from '../services/actor'
+import { ImageUriBuilder } from '../image/uri'
+import { GraphService } from '../services/graph'
+import { LabelService } from '../services/label'
+
+export function createServices(resources: {
+  imgUriBuilder: ImageUriBuilder
+}): Services {
+  const { imgUriBuilder } = resources
+  const graph = GraphService.creator(imgUriBuilder)
+  const label = LabelService.creator(null)
+  return {
+    actor: ActorService.creator(imgUriBuilder, graph, label),
+  }
+}
+
+export type Services = {
+  actor: FromDbPrimary<ActorService>
+}
+
+type FromDbPrimary<T> = (db: PrimaryDatabase) => T

package/src/db/database-schema.ts

@@ -24,7 +24,6 @@ import * as actorSync from './tables/actor-sync'
 import * as record from './tables/record'
 import * as notification from './tables/notification'
 import * as notificationPushToken from './tables/notification-push-token'
-import * as didCache from './tables/did-cache'
 import * as moderation from './tables/moderation'
 import * as label from './tables/label'
 import * as algo from './tables/algo'
@@ -57,7 +56,6 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
   record.PartialDB &
   notification.PartialDB &
   notificationPushToken.PartialDB &
-  didCache.PartialDB &
   moderation.PartialDB &
   label.PartialDB &
   algo.PartialDB &

package/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts

@@ -0,0 +1,123 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('moderation_event')
+    .addColumn('id', 'serial', (col) => col.primaryKey())
+    .addColumn('action', 'varchar', (col) => col.notNull())
+    .addColumn('subjectType', 'varchar', (col) => col.notNull())
+    .addColumn('subjectDid', 'varchar', (col) => col.notNull())
+    .addColumn('subjectUri', 'varchar')
+    .addColumn('subjectCid', 'varchar')
+    .addColumn('comment', 'text')
+    .addColumn('meta', 'jsonb')
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('createdBy', 'varchar', (col) => col.notNull())
+    .addColumn('reversedAt', 'varchar')
+    .addColumn('reversedBy', 'varchar')
+    .addColumn('durationInHours', 'integer')
+    .addColumn('expiresAt', 'varchar')
+    .addColumn('reversedReason', 'text')
+    .addColumn('createLabelVals', 'varchar')
+    .addColumn('negateLabelVals', 'varchar')
+    .addColumn('legacyRefId', 'integer')
+    .execute()
+  await db.schema
+    .createTable('moderation_subject_status')
+    .addColumn('id', 'serial', (col) => col.primaryKey())
+
+    // Identifiers
+    .addColumn('did', 'varchar', (col) => col.notNull())
+    // Default to '' so that we can apply unique constraints on did and recordPath columns
+    .addColumn('recordPath', 'varchar', (col) => col.notNull().defaultTo(''))
+    .addColumn('blobCids', 'jsonb')
+    .addColumn('recordCid', 'varchar')
+
+    // human review team state
+    .addColumn('reviewState', 'varchar', (col) => col.notNull())
+    .addColumn('comment', 'varchar')
+    .addColumn('muteUntil', 'varchar')
+    .addColumn('lastReviewedAt', 'varchar')
+    .addColumn('lastReviewedBy', 'varchar')
+
+    // report state
+    .addColumn('lastReportedAt', 'varchar')
+
+    // visibility/intervention state
+    .addColumn('takendown', 'boolean', (col) => col.defaultTo(false).notNull())
+    .addColumn('suspendUntil', 'varchar')
+
+    // timestamps
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('updatedAt', 'varchar', (col) => col.notNull())
+    .addUniqueConstraint('moderation_status_unique_idx', ['did', 'recordPath'])
+    .execute()
+
+  await db.schema
+    .createIndex('moderation_subject_status_blob_cids_idx')
+    .on('moderation_subject_status')
+    .using('gin')
+    .column('blobCids')
+    .execute()
+
+  // Move foreign keys from moderation_action to moderation_event
+  await db.schema
+    .alterTable('record')
+    .dropConstraint('record_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .dropConstraint('actor_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .addForeignKeyConstraint(
+      'actor_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_event',
+      ['id'],
+    )
+    .execute()
+  await db.schema
+    .alterTable('record')
+    .addForeignKeyConstraint(
+      'record_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_event',
+      ['id'],
+    )
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('moderation_event').execute()
+  await db.schema.dropTable('moderation_subject_status').execute()
+
+  // Revert foreign key constraints
+  await db.schema
+    .alterTable('record')
+    .dropConstraint('record_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .dropConstraint('actor_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .addForeignKeyConstraint(
+      'actor_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_action',
+      ['id'],
+    )
+    .execute()
+  await db.schema
+    .alterTable('record')
+    .addForeignKeyConstraint(
+      'record_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_action',
+      ['id'],
+    )
+    .execute()
+}

package/src/db/migrations/20231205T000257238Z-remove-did-cache.ts

@@ -0,0 +1,14 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('did_cache').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('did_cache')
+    .addColumn('did', 'varchar', (col) => col.primaryKey())
+    .addColumn('doc', 'jsonb', (col) => col.notNull())
+    .addColumn('updatedAt', 'bigint', (col) => col.notNull())
+    .execute()
+}

package/src/db/migrations/index.ts

@@ -30,3 +30,5 @@ export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
 export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
 export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
 export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes'
+export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'
+export * as _20231205T000257238Z from './20231205T000257238Z-remove-did-cache'

package/src/db/pagination.ts

@@ -117,13 +117,36 @@ export const paginate = <
     direction?: 'asc' | 'desc'
     keyset: K
     tryIndex?: boolean
+    // By default, pg does nullsFirst
+    nullsLast?: boolean
   },
 ): QB => {
-  const { limit, cursor, keyset, direction = 'desc', tryIndex } = opts
+  const {
+    limit,
+    cursor,
+    keyset,
+    direction = 'desc',
+    tryIndex,
+    nullsLast,
+  } = opts
   const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex)
   return qb
     .if(!!limit, (q) => q.limit(limit as number))
-    .orderBy(keyset.primary, direction)
-    .orderBy(keyset.secondary, direction)
+    .if(!nullsLast, (q) =>
+      q.orderBy(keyset.primary, direction).orderBy(keyset.secondary, direction),
+    )
+    .if(!!nullsLast, (q) =>
+      q
+        .orderBy(
+          direction === 'asc'
+            ? sql`${keyset.primary} asc nulls last`
+            : sql`${keyset.primary} desc nulls last`,
+        )
+        .orderBy(
+          direction === 'asc'
+            ? sql`${keyset.secondary} asc nulls last`
+            : sql`${keyset.secondary} desc nulls last`,
+        ),
+    )
     .if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB
 }

package/src/db/{periodic-moderation-action-reversal.ts → periodic-moderation-event-reversal.ts}

@@ -2,13 +2,15 @@ import { wait } from '@atproto/common'
 import { Leader } from './leader'
 import { dbLogger } from '../logger'
 import AppContext from '../context'
+import { AtUri } from '@atproto/api'
+import { ModerationSubjectStatusRow } from '../services/moderation/types'
+import { CID } from 'multiformats/cid'
 import AtpAgent from '@atproto/api'
-import { LabelService } from '../services/label'
-import { ModerationActionRow } from '../services/moderation'
+import { retryHttp } from '../util/retry'
 
 export const MODERATION_ACTION_REVERSAL_ID = 1011
 
-export class PeriodicModerationActionReversal {
+export class PeriodicModerationEventReversal {
   leader = new Leader(
     MODERATION_ACTION_REVERSAL_ID,
     this.appContext.db.getPrimary(),
@@ -20,48 +22,50 @@ export class PeriodicModerationActionReversal {
     this.pushAgent = appContext.moderationPushAgent
   }
 
-  // invert label creation & negations
-  async reverseLabels(labelTxn: LabelService, actionRow: ModerationActionRow) {
-    let uri: string
-    let cid: string | null = null
-
-    if (actionRow.subjectUri && actionRow.subjectCid) {
-      uri = actionRow.subjectUri
-      cid = actionRow.subjectCid
-    } else {
-      uri = actionRow.subjectDid
-    }
-
-    await labelTxn.formatAndCreate(this.appContext.cfg.labelerDid, uri, cid, {
-      create: actionRow.negateLabelVals
-        ? actionRow.negateLabelVals.split(' ')
-        : undefined,
-      negate: actionRow.createLabelVals
-        ? actionRow.createLabelVals.split(' ')
-        : undefined,
-    })
-  }
-
-  async revertAction(actionRow: ModerationActionRow) {
-    const reverseAction = {
-      id: actionRow.id,
-      createdBy: actionRow.createdBy,
-      createdAt: new Date(),
-      reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`,
-    }
-
-    if (this.pushAgent) {
-      await this.pushAgent.com.atproto.admin.reverseModerationAction(
-        reverseAction,
-      )
-      return
-    }
-
+  async revertState(eventRow: ModerationSubjectStatusRow) {
     await this.appContext.db.getPrimary().transaction(async (dbTxn) => {
       const moderationTxn = this.appContext.services.moderation(dbTxn)
-      await moderationTxn.revertAction(reverseAction)
-      const labelTxn = this.appContext.services.label(dbTxn)
-      await this.reverseLabels(labelTxn, actionRow)
+      const originalEvent =
+        await moderationTxn.getLastReversibleEventForSubject(eventRow)
+      if (originalEvent) {
+        const { restored } = await moderationTxn.revertState({
+          action: originalEvent.action,
+          createdBy: originalEvent.createdBy,
+          comment:
+            '[SCHEDULED_REVERSAL] Reverting action as originally scheduled',
+          subject:
+            eventRow.recordPath && eventRow.recordCid
+              ? {
+                  uri: AtUri.make(
+                    eventRow.did,
+                    ...eventRow.recordPath.split('/'),
+                  ),
+                  cid: CID.parse(eventRow.recordCid),
+                }
+              : { did: eventRow.did },
+          createdAt: new Date(),
+        })
+
+        const { pushAgent } = this
+        if (
+          originalEvent.action === 'com.atproto.admin.defs#modEventTakedown' &&
+          restored?.subjects?.length &&
+          pushAgent
+        ) {
+          await Promise.allSettled(
+            restored.subjects.map((subject) =>
+              retryHttp(() =>
+                pushAgent.api.com.atproto.admin.updateSubjectStatus({
+                  subject,
+                  takedown: {
+                    applied: false,
+                  },
+                }),
+              ),
+            ),
+          )
+        }
+      }
     })
   }
 
@@ -69,12 +73,12 @@ export class PeriodicModerationActionReversal {
     const moderationService = this.appContext.services.moderation(
       this.appContext.db.getPrimary(),
     )
-    const actionsDueForReversal =
-      await moderationService.getActionsDueForReversal()
+    const subjectsDueForReversal =
+      await moderationService.getSubjectsDueForReversal()
 
     // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine
     // Internally, each reversal runs within its own transaction
-    await Promise.all(actionsDueForReversal.map(this.revertAction.bind(this)))
+    await Promise.all(subjectsDueForReversal.map(this.revertState.bind(this)))
   }
 
   async run() {