@atproto/bsky 0.0.15 → 0.0.17

package/src/lexicon/types/com/atproto/{admin/getModerationAction.ts → temp/importRepo.ts}

@@ -2,24 +2,28 @@
  * GENERATED CODE - DO NOT MODIFY
  */
 import express from 'express'
+import stream from 'stream'
 import { ValidationResult, BlobRef } from '@atproto/lexicon'
 import { lexicons } from '../../../../lexicons'
 import { isObj, hasProp } from '../../../../util'
 import { CID } from 'multiformats/cid'
 import { HandlerAuth } from '@atproto/xrpc-server'
-import * as ComAtprotoAdminDefs from './defs'
 
 export interface QueryParams {
-  id: number
+  /** The DID of the repo. */
+  did: string
 }
 
-export type InputSchema = undefined
-export type OutputSchema = ComAtprotoAdminDefs.ActionViewDetail
-export type HandlerInput = undefined
+export type InputSchema = string | Uint8Array
+
+export interface HandlerInput {
+  encoding: 'application/vnd.ipld.car'
+  body: stream.Readable
+}
 
 export interface HandlerSuccess {
-  encoding: 'application/json'
-  body: OutputSchema
+  encoding: 'text/plain'
+  body: Uint8Array | stream.Readable
   headers?: { [key: string]: string }
 }
 

package/src/lexicon/types/com/atproto/temp/pushBlob.ts

@@ -0,0 +1,39 @@
+/**
+ * GENERATED CODE - DO NOT MODIFY
+ */
+import express from 'express'
+import stream from 'stream'
+import { ValidationResult, BlobRef } from '@atproto/lexicon'
+import { lexicons } from '../../../../lexicons'
+import { isObj, hasProp } from '../../../../util'
+import { CID } from 'multiformats/cid'
+import { HandlerAuth } from '@atproto/xrpc-server'
+
+export interface QueryParams {
+  /** The DID of the repo. */
+  did: string
+}
+
+export type InputSchema = string | Uint8Array
+
+export interface HandlerInput {
+  encoding: '*/*'
+  body: stream.Readable
+}
+
+export interface HandlerError {
+  status: number
+  message?: string
+}
+
+export type HandlerOutput = HandlerError | void
+export type HandlerReqCtx<HA extends HandlerAuth = never> = {
+  auth: HA
+  params: QueryParams
+  input: HandlerInput
+  req: express.Request
+  res: express.Response
+}
+export type Handler<HA extends HandlerAuth = never> = (
+  ctx: HandlerReqCtx<HA>,
+) => Promise<HandlerOutput> | HandlerOutput

package/src/lexicon/types/com/atproto/{admin/reverseModerationAction.ts → temp/transferAccount.ts}

@@ -7,18 +7,23 @@ import { lexicons } from '../../../../lexicons'
 import { isObj, hasProp } from '../../../../util'
 import { CID } from 'multiformats/cid'
 import { HandlerAuth } from '@atproto/xrpc-server'
-import * as ComAtprotoAdminDefs from './defs'
 
 export interface QueryParams {}
 
 export interface InputSchema {
-  id: number
-  reason: string
-  createdBy: string
+  handle: string
+  did: string
+  plcOp: {}
   [k: string]: unknown
 }
 
-export type OutputSchema = ComAtprotoAdminDefs.ActionView
+export interface OutputSchema {
+  accessJwt: string
+  refreshJwt: string
+  handle: string
+  did: string
+  [k: string]: unknown
+}
 
 export interface HandlerInput {
   encoding: 'application/json'
@@ -34,6 +39,14 @@ export interface HandlerSuccess {
 export interface HandlerError {
   status: number
   message?: string
+  error?:
+    | 'InvalidHandle'
+    | 'InvalidPassword'
+    | 'InvalidInviteCode'
+    | 'HandleNotAvailable'
+    | 'UnsupportedDomain'
+    | 'UnresolvableDid'
+    | 'IncompatibleDidDoc'
 }
 
 export type HandlerOutput = HandlerError | HandlerSuccess

package/src/logger.ts

@@ -3,6 +3,8 @@ import { subsystemLogger } from '@atproto/common'
 
 export const dbLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:db')
+export const cacheLogger: ReturnType<typeof subsystemLogger> =
+  subsystemLogger('bsky:cache')
 export const subLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:sub')
 export const labelerLogger: ReturnType<typeof subsystemLogger> =

package/src/migrate-moderation-data.ts

@@ -0,0 +1,414 @@
+import { sql } from 'kysely'
+import { DatabaseCoordinator, PrimaryDatabase } from './index'
+import { adjustModerationSubjectStatus } from './services/moderation/status'
+import { ModerationEventRow } from './services/moderation/types'
+
+type ModerationActionRow = Omit<ModerationEventRow, 'comment' | 'meta'> & {
+  reason: string | null
+}
+
+const getEnv = () => ({
+  DB_URL:
+    process.env.MODERATION_MIGRATION_DB_URL ||
+    'postgresql://pg:password@127.0.0.1:5433/postgres',
+  DB_POOL_SIZE: Number(process.env.MODERATION_MIGRATION_DB_POOL_SIZE) || 10,
+  DB_SCHEMA: process.env.MODERATION_MIGRATION_DB_SCHEMA || 'bsky',
+})
+
+const countEntries = async (db: PrimaryDatabase) => {
+  const [allActions, allReports] = await Promise.all([
+    db.db
+      // @ts-ignore
+      .selectFrom('moderation_action')
+      // @ts-ignore
+      .select((eb) => eb.fn.count<number>('id').as('count'))
+      .executeTakeFirstOrThrow(),
+    db.db
+      // @ts-ignore
+      .selectFrom('moderation_report')
+      // @ts-ignore
+      .select((eb) => eb.fn.count<number>('id').as('count'))
+      .executeTakeFirstOrThrow(),
+  ])
+
+  return { reportsCount: allReports.count, actionsCount: allActions.count }
+}
+
+const countEvents = async (db: PrimaryDatabase) => {
+  const events = await db.db
+    .selectFrom('moderation_event')
+    .select((eb) => eb.fn.count<number>('id').as('count'))
+    .executeTakeFirstOrThrow()
+
+  return events.count
+}
+
+const getLatestReportLegacyRefId = async (db: PrimaryDatabase) => {
+  const events = await db.db
+    .selectFrom('moderation_event')
+    .select((eb) => eb.fn.max('legacyRefId').as('latestLegacyRefId'))
+    .where('action', '=', 'com.atproto.admin.defs#modEventReport')
+    .executeTakeFirstOrThrow()
+
+  return events.latestLegacyRefId
+}
+
+const countStatuses = async (db: PrimaryDatabase) => {
+  const events = await db.db
+    .selectFrom('moderation_subject_status')
+    .select((eb) => eb.fn.count<number>('id').as('count'))
+    .executeTakeFirstOrThrow()
+
+  return events.count
+}
+
+const processLegacyReports = async (
+  db: PrimaryDatabase,
+  legacyIds: number[],
+) => {
+  if (!legacyIds.length) {
+    console.log('No legacy reports to process')
+    return
+  }
+  const reports = await db.db
+    .selectFrom('moderation_event')
+    .where('action', '=', 'com.atproto.admin.defs#modEventReport')
+    .where('legacyRefId', 'in', legacyIds)
+    .orderBy('legacyRefId', 'asc')
+    .selectAll()
+    .execute()
+
+  console.log(`Processing ${reports.length} reports from ${legacyIds.length}`)
+  await db.transaction(async (tx) => {
+    // This will be slow but we need to run this in sequence
+    for (const report of reports) {
+      await adjustModerationSubjectStatus(tx, report)
+    }
+  })
+  console.log(`Completed processing ${reports.length} reports`)
+}
+
+const getReportEventsAboveLegacyId = async (
+  db: PrimaryDatabase,
+  aboveLegacyId: number,
+) => {
+  return await db.db
+    .selectFrom('moderation_event')
+    .where('action', '=', 'com.atproto.admin.defs#modEventReport')
+    .where('legacyRefId', '>', aboveLegacyId)
+    .select(sql<number>`"legacyRefId"`.as('legacyRefId'))
+    .execute()
+}
+
+const createEvents = async (
+  db: PrimaryDatabase,
+  opts?: { onlyReportsAboveId: number },
+) => {
+  const commonColumnsToSelect = [
+    'subjectDid',
+    'subjectUri',
+    'subjectType',
+    'subjectCid',
+    sql`reason`.as('comment'),
+    'createdAt',
+  ]
+  const commonColumnsToInsert = [
+    'subjectDid',
+    'subjectUri',
+    'subjectType',
+    'subjectCid',
+    'comment',
+    'createdAt',
+    'action',
+    'createdBy',
+  ] as const
+
+  let totalActions: number
+  if (!opts?.onlyReportsAboveId) {
+    await db.db
+      .insertInto('moderation_event')
+      .columns([
+        'id',
+        ...commonColumnsToInsert,
+        'createLabelVals',
+        'negateLabelVals',
+        'durationInHours',
+        'expiresAt',
+      ])
+      .expression((eb) =>
+        eb
+          // @ts-ignore
+          .selectFrom('moderation_action')
+          // @ts-ignore
+          .select([
+            'id',
+            ...commonColumnsToSelect,
+            sql`CONCAT('com.atproto.admin.defs#modEvent', UPPER(SUBSTRING(SPLIT_PART(action, '#', 2) FROM 1 FOR 1)), SUBSTRING(SPLIT_PART(action, '#', 2) FROM 2))`.as(
+              'action',
+            ),
+            'createdBy',
+            'createLabelVals',
+            'negateLabelVals',
+            'durationInHours',
+            'expiresAt',
+          ])
+          .orderBy('id', 'asc'),
+      )
+      .execute()
+
+    totalActions = await countEvents(db)
+    console.log(`Created ${totalActions} events from actions`)
+
+    await sql`SELECT setval(pg_get_serial_sequence('moderation_event', 'id'), (select max(id) from moderation_event))`.execute(
+      db.db,
+    )
+    console.log('Reset the id sequence for moderation_event')
+  } else {
+    totalActions = await countEvents(db)
+  }
+
+  await db.db
+    .insertInto('moderation_event')
+    .columns([...commonColumnsToInsert, 'meta', 'legacyRefId'])
+    .expression((eb) => {
+      const builder = eb
+        // @ts-ignore
+        .selectFrom('moderation_report')
+        // @ts-ignore
+        .select([
+          ...commonColumnsToSelect,
+          sql`'com.atproto.admin.defs#modEventReport'`.as('action'),
+          sql`"reportedByDid"`.as('createdBy'),
+          sql`json_build_object('reportType', "reasonType")`.as('meta'),
+          sql`id`.as('legacyRefId'),
+        ])
+
+      if (opts?.onlyReportsAboveId) {
+        // @ts-ignore
+        return builder.where('id', '>', opts.onlyReportsAboveId)
+      }
+
+      return builder
+    })
+    .execute()
+
+  const totalEvents = await countEvents(db)
+  console.log(`Created ${totalEvents - totalActions} events from reports`)
+
+  return
+}
+
+const setReportedAtTimestamp = async (db: PrimaryDatabase) => {
+  console.log('Initiating lastReportedAt timestamp sync')
+  const didUpdate = await sql`
+    UPDATE moderation_subject_status
+    SET "lastReportedAt" = reports."createdAt"
+    FROM (
+      select "subjectDid", "subjectUri", MAX("createdAt") as "createdAt"
+      from moderation_report
+      where "subjectUri" is null
+      group by "subjectDid", "subjectUri"
+    ) as reports
+    WHERE reports."subjectDid" = moderation_subject_status."did"
+      AND "recordPath" = ''
+      AND ("lastReportedAt" is null OR "lastReportedAt" < reports."createdAt")
+  `.execute(db.db)
+
+  console.log(
+    `Updated lastReportedAt for ${didUpdate.numUpdatedOrDeletedRows} did subject`,
+  )
+
+  const contentUpdate = await sql`
+    UPDATE moderation_subject_status
+    SET "lastReportedAt" = reports."createdAt"
+    FROM (
+      select "subjectDid", "subjectUri", MAX("createdAt") as "createdAt"
+      from moderation_report
+      where "subjectUri" is not null
+      group by "subjectDid", "subjectUri"
+    ) as reports
+    WHERE reports."subjectDid" = moderation_subject_status."did"
+      AND "recordPath" is not null
+      AND POSITION(moderation_subject_status."recordPath" IN reports."subjectUri") > 0
+      AND ("lastReportedAt" is null OR "lastReportedAt" < reports."createdAt")
+  `.execute(db.db)
+
+  console.log(
+    `Updated lastReportedAt for ${contentUpdate.numUpdatedOrDeletedRows} subject with uri`,
+  )
+}
+
+const createStatusFromActions = async (db: PrimaryDatabase) => {
+  const allEvents = await db.db
+    // @ts-ignore
+    .selectFrom('moderation_action')
+    // @ts-ignore
+    .where('reversedAt', 'is', null)
+    // @ts-ignore
+    .select((eb) => eb.fn.count<number>('id').as('count'))
+    .executeTakeFirstOrThrow()
+
+  const chunkSize = 2500
+  const totalChunks = Math.ceil(allEvents.count / chunkSize)
+
+  console.log(`Processing ${allEvents.count} actions in ${totalChunks} chunks`)
+
+  await db.transaction(async (tx) => {
+    // This is not used for pagination but only for logging purposes
+    let currentChunk = 1
+    let lastProcessedId: undefined | number = 0
+    do {
+      const eventsQuery = tx.db
+        // @ts-ignore
+        .selectFrom('moderation_action')
+        // @ts-ignore
+        .where('reversedAt', 'is', null)
+        // @ts-ignore
+        .where('id', '>', lastProcessedId)
+        .limit(chunkSize)
+        .selectAll()
+      const events = (await eventsQuery.execute()) as ModerationActionRow[]
+
+      for (const event of events) {
+        // Remap action to event data type
+        const actionParts = event.action.split('#')
+        await adjustModerationSubjectStatus(tx, {
+          ...event,
+          action: `com.atproto.admin.defs#modEvent${actionParts[1]
+            .charAt(0)
+            .toUpperCase()}${actionParts[1].slice(
+            1,
+          )}` as ModerationEventRow['action'],
+          comment: event.reason,
+          meta: null,
+        })
+      }
+
+      console.log(`Processed events chunk ${currentChunk} of ${totalChunks}`)
+      lastProcessedId = events.at(-1)?.id
+      currentChunk++
+    } while (lastProcessedId !== undefined)
+  })
+
+  console.log(`Events migration complete!`)
+
+  const totalStatuses = await countStatuses(db)
+  console.log(`Created ${totalStatuses} statuses`)
+}
+
+const remapFlagToAcknlowedge = async (db: PrimaryDatabase) => {
+  console.log('Initiating flag to ack remap')
+  const results = await sql`
+    UPDATE moderation_event
+    SET "action" = 'com.atproto.admin.defs#modEventAcknowledge'
+    WHERE action = 'com.atproto.admin.defs#modEventFlag'
+  `.execute(db.db)
+  console.log(`Remapped ${results.numUpdatedOrDeletedRows} flag actions to ack`)
+}
+
+const syncBlobCids = async (db: PrimaryDatabase) => {
+  console.log('Initiating blob cid sync')
+  const results = await sql`
+    UPDATE moderation_subject_status
+    SET "blobCids" = blob_action."cids"
+    FROM (
+        SELECT moderation_action."subjectUri", moderation_action."subjectDid", jsonb_agg(moderation_action_subject_blob."cid") as cids
+        FROM moderation_action_subject_blob
+            JOIN moderation_action
+                ON moderation_action.id = moderation_action_subject_blob."actionId"
+          WHERE moderation_action."reversedAt" is NULL
+          GROUP by moderation_action."subjectUri", moderation_action."subjectDid"
+    ) as blob_action
+    WHERE did = "subjectDid" AND position("recordPath" IN "subjectUri") > 0
+  `.execute(db.db)
+  console.log(`Updated blob cids on ${results.numUpdatedOrDeletedRows} rows`)
+}
+
+async function updateStatusFromUnresolvedReports(db: PrimaryDatabase) {
+  const { ref } = db.db.dynamic
+  const reports = await db.db
+    // @ts-ignore
+    .selectFrom('moderation_report')
+    .whereNotExists((qb) =>
+      qb
+        .selectFrom('moderation_report_resolution')
+        .selectAll()
+        // @ts-ignore
+        .whereRef('reportId', '=', ref('moderation_report.id')),
+    )
+    .select(sql<number>`moderation_report.id`.as('legacyId'))
+    .execute()
+
+  console.log('Updating statuses based on unresolved reports')
+  await processLegacyReports(
+    db,
+    reports.map((report) => report.legacyId),
+  )
+  console.log('Completed updating statuses based on unresolved reports')
+}
+
+export async function MigrateModerationData() {
+  const env = getEnv()
+  const db = new DatabaseCoordinator({
+    schema: env.DB_SCHEMA,
+    primary: {
+      url: env.DB_URL,
+      poolSize: env.DB_POOL_SIZE,
+    },
+    replicas: [],
+  })
+
+  const primaryDb = db.getPrimary()
+
+  const [counts, existingEventsCount] = await Promise.all([
+    countEntries(primaryDb),
+    countEvents(primaryDb),
+  ])
+
+  // If there are existing events in the moderation_event table, we assume that the migration has already been run
+  // so we just bring over any new reports since last run
+  if (existingEventsCount) {
+    console.log(
+      `Found ${existingEventsCount} existing events. Migrating ${counts.reportsCount} reports only, ignoring actions`,
+    )
+    const reportMigrationStartedAt = Date.now()
+    const latestReportLegacyRefId = await getLatestReportLegacyRefId(primaryDb)
+
+    if (latestReportLegacyRefId) {
+      await createEvents(primaryDb, {
+        onlyReportsAboveId: latestReportLegacyRefId,
+      })
+      const newReportEvents = await getReportEventsAboveLegacyId(
+        primaryDb,
+        latestReportLegacyRefId,
+      )
+      await processLegacyReports(
+        primaryDb,
+        newReportEvents.map((evt) => evt.legacyRefId),
+      )
+      await setReportedAtTimestamp(primaryDb)
+    } else {
+      console.log('No reports have been migrated into events yet, bailing.')
+    }
+
+    console.log(
+      `Time spent: ${(Date.now() - reportMigrationStartedAt) / 1000} seconds`,
+    )
+    console.log('Migration complete!')
+    return
+  }
+
+  const totalEntries = counts.actionsCount + counts.reportsCount
+  console.log(`Migrating ${totalEntries} rows of actions and reports`)
+  const startedAt = Date.now()
+  await createEvents(primaryDb)
+  // Important to run this before creation statuses from actions to ensure that we are not attempting to map flag actions
+  await remapFlagToAcknlowedge(primaryDb)
+  await createStatusFromActions(primaryDb)
+  await updateStatusFromUnresolvedReports(primaryDb)
+  await setReportedAtTimestamp(primaryDb)
+  await syncBlobCids(primaryDb)
+
+  console.log(`Time spent: ${(Date.now() - startedAt) / 1000 / 60} minutes`)
+  console.log('Migration complete!')
+}

package/src/redis.ts

@@ -11,12 +11,16 @@ export class Redis {
         name: opts.sentinel,
         sentinels: opts.hosts.map((h) => addressParts(h, 26379)),
         password: opts.password,
+        db: opts.db,
+        commandTimeout: opts.commandTimeout,
       })
     } else if ('host' in opts) {
       assert(opts.host)
       this.driver = new RedisDriver({
         ...addressParts(opts.host),
         password: opts.password,
+        db: opts.db,
+        commandTimeout: opts.commandTimeout,
       })
     } else {
       assert(opts.driver)
@@ -25,6 +29,10 @@ export class Redis {
     this.namespace = opts.namespace
   }
 
+  withNamespace(namespace: string): Redis {
+    return new Redis({ driver: this.driver, namespace })
+  }
+
   async readStreams(
     streams: StreamRef[],
     opts: { count: number; blockMs?: number },
@@ -97,8 +105,38 @@ export class Redis {
     return await this.driver.get(this.ns(key))
   }
 
-  async set(key: string, val: string | number) {
-    await this.driver.set(this.ns(key), val)
+  async set(key: string, val: string | number, ttlMs?: number) {
+    if (ttlMs !== undefined) {
+      await this.driver.set(this.ns(key), val, 'PX', ttlMs)
+    } else {
+      await this.driver.set(this.ns(key), val)
+    }
+  }
+
+  async getMulti(keys: string[]) {
+    const namespaced = keys.map((k) => this.ns(k))
+    const got = await this.driver.mget(...namespaced)
+    const results = {}
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]
+      results[key] = got[i]
+    }
+    return results
+  }
+
+  async setMulti(vals: Record<string, string | number>, ttlMs?: number) {
+    if (Object.keys(vals).length === 0) {
+      return
+    }
+    let builder = this.driver.multi({ pipeline: true })
+    for (const key of Object.keys(vals)) {
+      if (ttlMs !== undefined) {
+        builder = builder.set(this.ns(key), vals[key], 'PX', ttlMs)
+      } else {
+        builder = builder.set(this.ns(key), vals[key])
+      }
+    }
+    await builder.exec()
   }
 
   async del(key: string) {
@@ -152,9 +190,11 @@ export type RedisOptions = (
 ) & {
   password?: string
   namespace?: string
+  db?: number
+  commandTimeout?: number
 }
 
-function addressParts(
+export function addressParts(
   addr: string,
   defaultPort = 6379,
 ): { host: string; port: number } {