@atproto/bsky 0.0.15 → 0.0.16

package/src/migrate-moderation-data.ts

@@ -0,0 +1,414 @@
+import { sql } from 'kysely'
+import { DatabaseCoordinator, PrimaryDatabase } from './index'
+import { adjustModerationSubjectStatus } from './services/moderation/status'
+import { ModerationEventRow } from './services/moderation/types'
+
+type ModerationActionRow = Omit<ModerationEventRow, 'comment' | 'meta'> & {
+  reason: string | null
+}
+
+const getEnv = () => ({
+  DB_URL:
+    process.env.MODERATION_MIGRATION_DB_URL ||
+    'postgresql://pg:password@127.0.0.1:5433/postgres',
+  DB_POOL_SIZE: Number(process.env.MODERATION_MIGRATION_DB_POOL_SIZE) || 10,
+  DB_SCHEMA: process.env.MODERATION_MIGRATION_DB_SCHEMA || 'bsky',
+})
+
+const countEntries = async (db: PrimaryDatabase) => {
+  const [allActions, allReports] = await Promise.all([
+    db.db
+      // @ts-ignore
+      .selectFrom('moderation_action')
+      // @ts-ignore
+      .select((eb) => eb.fn.count<number>('id').as('count'))
+      .executeTakeFirstOrThrow(),
+    db.db
+      // @ts-ignore
+      .selectFrom('moderation_report')
+      // @ts-ignore
+      .select((eb) => eb.fn.count<number>('id').as('count'))
+      .executeTakeFirstOrThrow(),
+  ])
+
+  return { reportsCount: allReports.count, actionsCount: allActions.count }
+}
+
+const countEvents = async (db: PrimaryDatabase) => {
+  const events = await db.db
+    .selectFrom('moderation_event')
+    .select((eb) => eb.fn.count<number>('id').as('count'))
+    .executeTakeFirstOrThrow()
+
+  return events.count
+}
+
+const getLatestReportLegacyRefId = async (db: PrimaryDatabase) => {
+  const events = await db.db
+    .selectFrom('moderation_event')
+    .select((eb) => eb.fn.max('legacyRefId').as('latestLegacyRefId'))
+    .where('action', '=', 'com.atproto.admin.defs#modEventReport')
+    .executeTakeFirstOrThrow()
+
+  return events.latestLegacyRefId
+}
+
+const countStatuses = async (db: PrimaryDatabase) => {
+  const events = await db.db
+    .selectFrom('moderation_subject_status')
+    .select((eb) => eb.fn.count<number>('id').as('count'))
+    .executeTakeFirstOrThrow()
+
+  return events.count
+}
+
+const processLegacyReports = async (
+  db: PrimaryDatabase,
+  legacyIds: number[],
+) => {
+  if (!legacyIds.length) {
+    console.log('No legacy reports to process')
+    return
+  }
+  const reports = await db.db
+    .selectFrom('moderation_event')
+    .where('action', '=', 'com.atproto.admin.defs#modEventReport')
+    .where('legacyRefId', 'in', legacyIds)
+    .orderBy('legacyRefId', 'asc')
+    .selectAll()
+    .execute()
+
+  console.log(`Processing ${reports.length} reports from ${legacyIds.length}`)
+  await db.transaction(async (tx) => {
+    // This will be slow but we need to run this in sequence
+    for (const report of reports) {
+      await adjustModerationSubjectStatus(tx, report)
+    }
+  })
+  console.log(`Completed processing ${reports.length} reports`)
+}
+
+const getReportEventsAboveLegacyId = async (
+  db: PrimaryDatabase,
+  aboveLegacyId: number,
+) => {
+  return await db.db
+    .selectFrom('moderation_event')
+    .where('action', '=', 'com.atproto.admin.defs#modEventReport')
+    .where('legacyRefId', '>', aboveLegacyId)
+    .select(sql<number>`"legacyRefId"`.as('legacyRefId'))
+    .execute()
+}
+
+const createEvents = async (
+  db: PrimaryDatabase,
+  opts?: { onlyReportsAboveId: number },
+) => {
+  const commonColumnsToSelect = [
+    'subjectDid',
+    'subjectUri',
+    'subjectType',
+    'subjectCid',
+    sql`reason`.as('comment'),
+    'createdAt',
+  ]
+  const commonColumnsToInsert = [
+    'subjectDid',
+    'subjectUri',
+    'subjectType',
+    'subjectCid',
+    'comment',
+    'createdAt',
+    'action',
+    'createdBy',
+  ] as const
+
+  let totalActions: number
+  if (!opts?.onlyReportsAboveId) {
+    await db.db
+      .insertInto('moderation_event')
+      .columns([
+        'id',
+        ...commonColumnsToInsert,
+        'createLabelVals',
+        'negateLabelVals',
+        'durationInHours',
+        'expiresAt',
+      ])
+      .expression((eb) =>
+        eb
+          // @ts-ignore
+          .selectFrom('moderation_action')
+          // @ts-ignore
+          .select([
+            'id',
+            ...commonColumnsToSelect,
+            sql`CONCAT('com.atproto.admin.defs#modEvent', UPPER(SUBSTRING(SPLIT_PART(action, '#', 2) FROM 1 FOR 1)), SUBSTRING(SPLIT_PART(action, '#', 2) FROM 2))`.as(
+              'action',
+            ),
+            'createdBy',
+            'createLabelVals',
+            'negateLabelVals',
+            'durationInHours',
+            'expiresAt',
+          ])
+          .orderBy('id', 'asc'),
+      )
+      .execute()
+
+    totalActions = await countEvents(db)
+    console.log(`Created ${totalActions} events from actions`)
+
+    await sql`SELECT setval(pg_get_serial_sequence('moderation_event', 'id'), (select max(id) from moderation_event))`.execute(
+      db.db,
+    )
+    console.log('Reset the id sequence for moderation_event')
+  } else {
+    totalActions = await countEvents(db)
+  }
+
+  await db.db
+    .insertInto('moderation_event')
+    .columns([...commonColumnsToInsert, 'meta', 'legacyRefId'])
+    .expression((eb) => {
+      const builder = eb
+        // @ts-ignore
+        .selectFrom('moderation_report')
+        // @ts-ignore
+        .select([
+          ...commonColumnsToSelect,
+          sql`'com.atproto.admin.defs#modEventReport'`.as('action'),
+          sql`"reportedByDid"`.as('createdBy'),
+          sql`json_build_object('reportType', "reasonType")`.as('meta'),
+          sql`id`.as('legacyRefId'),
+        ])
+
+      if (opts?.onlyReportsAboveId) {
+        // @ts-ignore
+        return builder.where('id', '>', opts.onlyReportsAboveId)
+      }
+
+      return builder
+    })
+    .execute()
+
+  const totalEvents = await countEvents(db)
+  console.log(`Created ${totalEvents - totalActions} events from reports`)
+
+  return
+}
+
+const setReportedAtTimestamp = async (db: PrimaryDatabase) => {
+  console.log('Initiating lastReportedAt timestamp sync')
+  const didUpdate = await sql`
+    UPDATE moderation_subject_status
+    SET "lastReportedAt" = reports."createdAt"
+    FROM (
+      select "subjectDid", "subjectUri", MAX("createdAt") as "createdAt"
+      from moderation_report
+      where "subjectUri" is null
+      group by "subjectDid", "subjectUri"
+    ) as reports
+    WHERE reports."subjectDid" = moderation_subject_status."did"
+      AND "recordPath" = ''
+      AND ("lastReportedAt" is null OR "lastReportedAt" < reports."createdAt")
+  `.execute(db.db)
+
+  console.log(
+    `Updated lastReportedAt for ${didUpdate.numUpdatedOrDeletedRows} did subject`,
+  )
+
+  const contentUpdate = await sql`
+    UPDATE moderation_subject_status
+    SET "lastReportedAt" = reports."createdAt"
+    FROM (
+      select "subjectDid", "subjectUri", MAX("createdAt") as "createdAt"
+      from moderation_report
+      where "subjectUri" is not null
+      group by "subjectDid", "subjectUri"
+    ) as reports
+    WHERE reports."subjectDid" = moderation_subject_status."did"
+      AND "recordPath" is not null
+      AND POSITION(moderation_subject_status."recordPath" IN reports."subjectUri") > 0
+      AND ("lastReportedAt" is null OR "lastReportedAt" < reports."createdAt")
+  `.execute(db.db)
+
+  console.log(
+    `Updated lastReportedAt for ${contentUpdate.numUpdatedOrDeletedRows} subject with uri`,
+  )
+}
+
+const createStatusFromActions = async (db: PrimaryDatabase) => {
+  const allEvents = await db.db
+    // @ts-ignore
+    .selectFrom('moderation_action')
+    // @ts-ignore
+    .where('reversedAt', 'is', null)
+    // @ts-ignore
+    .select((eb) => eb.fn.count<number>('id').as('count'))
+    .executeTakeFirstOrThrow()
+
+  const chunkSize = 2500
+  const totalChunks = Math.ceil(allEvents.count / chunkSize)
+
+  console.log(`Processing ${allEvents.count} actions in ${totalChunks} chunks`)
+
+  await db.transaction(async (tx) => {
+    // This is not used for pagination but only for logging purposes
+    let currentChunk = 1
+    let lastProcessedId: undefined | number = 0
+    do {
+      const eventsQuery = tx.db
+        // @ts-ignore
+        .selectFrom('moderation_action')
+        // @ts-ignore
+        .where('reversedAt', 'is', null)
+        // @ts-ignore
+        .where('id', '>', lastProcessedId)
+        .limit(chunkSize)
+        .selectAll()
+      const events = (await eventsQuery.execute()) as ModerationActionRow[]
+
+      for (const event of events) {
+        // Remap action to event data type
+        const actionParts = event.action.split('#')
+        await adjustModerationSubjectStatus(tx, {
+          ...event,
+          action: `com.atproto.admin.defs#modEvent${actionParts[1]
+            .charAt(0)
+            .toUpperCase()}${actionParts[1].slice(
+            1,
+          )}` as ModerationEventRow['action'],
+          comment: event.reason,
+          meta: null,
+        })
+      }
+
+      console.log(`Processed events chunk ${currentChunk} of ${totalChunks}`)
+      lastProcessedId = events.at(-1)?.id
+      currentChunk++
+    } while (lastProcessedId !== undefined)
+  })
+
+  console.log(`Events migration complete!`)
+
+  const totalStatuses = await countStatuses(db)
+  console.log(`Created ${totalStatuses} statuses`)
+}
+
+const remapFlagToAcknlowedge = async (db: PrimaryDatabase) => {
+  console.log('Initiating flag to ack remap')
+  const results = await sql`
+    UPDATE moderation_event
+    SET "action" = 'com.atproto.admin.defs#modEventAcknowledge'
+    WHERE action = 'com.atproto.admin.defs#modEventFlag'
+  `.execute(db.db)
+  console.log(`Remapped ${results.numUpdatedOrDeletedRows} flag actions to ack`)
+}
+
+const syncBlobCids = async (db: PrimaryDatabase) => {
+  console.log('Initiating blob cid sync')
+  const results = await sql`
+    UPDATE moderation_subject_status
+    SET "blobCids" = blob_action."cids"
+    FROM (
+        SELECT moderation_action."subjectUri", moderation_action."subjectDid", jsonb_agg(moderation_action_subject_blob."cid") as cids
+        FROM moderation_action_subject_blob
+            JOIN moderation_action
+                ON moderation_action.id = moderation_action_subject_blob."actionId"
+          WHERE moderation_action."reversedAt" is NULL
+          GROUP by moderation_action."subjectUri", moderation_action."subjectDid"
+    ) as blob_action
+    WHERE did = "subjectDid" AND position("recordPath" IN "subjectUri") > 0
+  `.execute(db.db)
+  console.log(`Updated blob cids on ${results.numUpdatedOrDeletedRows} rows`)
+}
+
+async function updateStatusFromUnresolvedReports(db: PrimaryDatabase) {
+  const { ref } = db.db.dynamic
+  const reports = await db.db
+    // @ts-ignore
+    .selectFrom('moderation_report')
+    .whereNotExists((qb) =>
+      qb
+        .selectFrom('moderation_report_resolution')
+        .selectAll()
+        // @ts-ignore
+        .whereRef('reportId', '=', ref('moderation_report.id')),
+    )
+    .select(sql<number>`moderation_report.id`.as('legacyId'))
+    .execute()
+
+  console.log('Updating statuses based on unresolved reports')
+  await processLegacyReports(
+    db,
+    reports.map((report) => report.legacyId),
+  )
+  console.log('Completed updating statuses based on unresolved reports')
+}
+
+export async function MigrateModerationData() {
+  const env = getEnv()
+  const db = new DatabaseCoordinator({
+    schema: env.DB_SCHEMA,
+    primary: {
+      url: env.DB_URL,
+      poolSize: env.DB_POOL_SIZE,
+    },
+    replicas: [],
+  })
+
+  const primaryDb = db.getPrimary()
+
+  const [counts, existingEventsCount] = await Promise.all([
+    countEntries(primaryDb),
+    countEvents(primaryDb),
+  ])
+
+  // If there are existing events in the moderation_event table, we assume that the migration has already been run
+  // so we just bring over any new reports since last run
+  if (existingEventsCount) {
+    console.log(
+      `Found ${existingEventsCount} existing events. Migrating ${counts.reportsCount} reports only, ignoring actions`,
+    )
+    const reportMigrationStartedAt = Date.now()
+    const latestReportLegacyRefId = await getLatestReportLegacyRefId(primaryDb)
+
+    if (latestReportLegacyRefId) {
+      await createEvents(primaryDb, {
+        onlyReportsAboveId: latestReportLegacyRefId,
+      })
+      const newReportEvents = await getReportEventsAboveLegacyId(
+        primaryDb,
+        latestReportLegacyRefId,
+      )
+      await processLegacyReports(
+        primaryDb,
+        newReportEvents.map((evt) => evt.legacyRefId),
+      )
+      await setReportedAtTimestamp(primaryDb)
+    } else {
+      console.log('No reports have been migrated into events yet, bailing.')
+    }
+
+    console.log(
+      `Time spent: ${(Date.now() - reportMigrationStartedAt) / 1000} seconds`,
+    )
+    console.log('Migration complete!')
+    return
+  }
+
+  const totalEntries = counts.actionsCount + counts.reportsCount
+  console.log(`Migrating ${totalEntries} rows of actions and reports`)
+  const startedAt = Date.now()
+  await createEvents(primaryDb)
+  // Important to run this before creation statuses from actions to ensure that we are not attempting to map flag actions
+  await remapFlagToAcknlowedge(primaryDb)
+  await createStatusFromActions(primaryDb)
+  await updateStatusFromUnresolvedReports(primaryDb)
+  await setReportedAtTimestamp(primaryDb)
+  await syncBlobCids(primaryDb)
+
+  console.log(`Time spent: ${(Date.now() - startedAt) / 1000 / 60} minutes`)
+  console.log('Migration complete!')
+}

package/src/services/actor/views.ts

@@ -45,10 +45,7 @@ export class ActorViews {
       viewer,
       ...opts,
     })
-    return this.profilePresentation(dids, hydrated, {
-      viewer,
-      ...opts,
-    })
+    return this.profilePresentation(dids, hydrated, viewer)
   }
 
   async profilesBasic(
@@ -62,10 +59,7 @@ export class ActorViews {
       viewer,
       includeSoftDeleted: opts?.includeSoftDeleted,
     })
-    return this.profileBasicPresentation(dids, hydrated, {
-      viewer,
-      omitLabels: opts?.omitLabels,
-    })
+    return this.profileBasicPresentation(dids, hydrated, viewer, opts)
   }
 
   async profilesList(
@@ -293,11 +287,8 @@ export class ActorViews {
       labels: Labels
       bam: BlockAndMuteState
     },
-    opts?: {
-      viewer?: string | null
-    },
+    viewer: string | null,
   ): ProfileViewMap {
-    const { viewer } = opts ?? {}
     const { profiles, lists, labels, bam } = state
     return dids.reduce((acc, did) => {
       const prof = profiles[did]
@@ -357,12 +348,12 @@ export class ActorViews {
   profileBasicPresentation(
     dids: string[],
     state: ProfileHydrationState,
+    viewer: string | null,
     opts?: {
-      viewer?: string | null
       omitLabels?: boolean
     },
   ): ProfileViewMap {
-    const result = this.profilePresentation(dids, state, opts)
+    const result = this.profilePresentation(dids, state, viewer)
     return Object.values(result).reduce((acc, prof) => {
       const profileBasic = {
         did: prof.did,

package/src/services/feed/index.ts

@@ -44,6 +44,7 @@ import {
 import { FeedViews } from './views'
 import { LabelCache } from '../../label-cache'
 import { threadgateToPostUri, postToThreadgateUri } from './util'
+import { mapDefined } from '@atproto/common'
 
 export * from './types'
 
@@ -205,6 +206,11 @@ export class FeedService {
     }, {} as Record<string, FeedRow>)
   }
 
+  async postUrisToFeedItems(uris: string[]): Promise<FeedRow[]> {
+    const feedItems = await this.getFeedItems(uris)
+    return mapDefined(uris, (uri) => feedItems[uri])
+  }
+
   feedItemRefs(items: FeedRow[]) {
     const actorDids = new Set<string>()
     const postUris = new Set<string>()
@@ -399,20 +405,32 @@ export class FeedService {
     const actorInfos = this.services.actor.views.profileBasicPresentation(
       [...nestedDids],
       feedState,
-      { viewer },
+      viewer,
     )
     const recordEmbedViews: RecordEmbedViewRecordMap = {}
     for (const uri of nestedUris) {
       const collection = new AtUri(uri).collection
       if (collection === ids.AppBskyFeedGenerator && feedGenInfos[uri]) {
-        recordEmbedViews[uri] = {
-          $type: 'app.bsky.feed.defs#generatorView',
-          ...this.views.formatFeedGeneratorView(feedGenInfos[uri], actorInfos),
+        const genView = this.views.formatFeedGeneratorView(
+          feedGenInfos[uri],
+          actorInfos,
+        )
+        if (genView) {
+          recordEmbedViews[uri] = {
+            $type: 'app.bsky.feed.defs#generatorView',
+            ...genView,
+          }
         }
       } else if (collection === ids.AppBskyGraphList && listViews[uri]) {
-        recordEmbedViews[uri] = {
-          $type: 'app.bsky.graph.defs#listView',
-          ...this.services.graph.formatListView(listViews[uri], actorInfos),
+        const listView = this.services.graph.formatListView(
+          listViews[uri],
+          actorInfos,
+        )
+        if (listView) {
+          recordEmbedViews[uri] = {
+            $type: 'app.bsky.graph.defs#listView',
+            ...listView,
+          }
         }
       } else if (collection === ids.AppBskyFeedPost && feedState.posts[uri]) {
         const formatted = this.views.formatPostView(
@@ -423,6 +441,7 @@ export class FeedService {
           feedState.embeds,
           feedState.labels,
           feedState.lists,
+          viewer,
         )
         recordEmbedViews[uri] = this.views.getRecordEmbedView(
           uri,

package/src/services/feed/util.ts

@@ -36,43 +36,72 @@ export const invalidReplyRoot = (
   return parent.record.reply?.root.uri !== replyRoot
 }
 
-export const violatesThreadGate = async (
-  db: DatabaseSchema,
-  did: string,
-  owner: string,
-  root: PostRecord | null,
+type ParsedThreadGate = {
+  canReply?: boolean
+  allowMentions?: boolean
+  allowFollowing?: boolean
+  allowListUris?: string[]
+}
+
+export const parseThreadGate = (
+  replierDid: string,
+  ownerDid: string,
+  rootPost: PostRecord | null,
   gate: GateRecord | null,
-) => {
-  if (did === owner) return false
-  if (!gate?.allow) return false
+): ParsedThreadGate => {
+  if (replierDid === ownerDid) {
+    return { canReply: true }
+  }
+  // if gate.allow is unset then *any* reply is allowed, if it is an empty array then *no* reply is allowed
+  if (!gate || !gate.allow) {
+    return { canReply: true }
+  }
 
-  const allowMentions = gate.allow.find(isMentionRule)
-  const allowFollowing = gate.allow.find(isFollowingRule)
+  const allowMentions = !!gate.allow.find(isMentionRule)
+  const allowFollowing = !!gate.allow.find(isFollowingRule)
   const allowListUris = gate.allow?.filter(isListRule).map((item) => item.list)
 
   // check mentions first since it's quick and synchronous
   if (allowMentions) {
-    const isMentioned = root?.facets?.some((facet) => {
-      return facet.features.some((item) => isMention(item) && item.did === did)
+    const isMentioned = rootPost?.facets?.some((facet) => {
+      return facet.features.some(
+        (item) => isMention(item) && item.did === replierDid,
+      )
     })
     if (isMentioned) {
-      return false
+      return { canReply: true, allowMentions, allowFollowing, allowListUris }
     }
   }
+  return { allowMentions, allowFollowing, allowListUris }
+}
 
-  // check follows and list containment
-  if (!allowFollowing && !allowListUris.length) {
+export const violatesThreadGate = async (
+  db: DatabaseSchema,
+  replierDid: string,
+  ownerDid: string,
+  rootPost: PostRecord | null,
+  gate: GateRecord | null,
+) => {
+  const {
+    canReply,
+    allowFollowing,
+    allowListUris = [],
+  } = parseThreadGate(replierDid, ownerDid, rootPost, gate)
+  if (canReply) {
+    return false
+  }
+  if (!allowFollowing && !allowListUris?.length) {
     return true
   }
   const { ref } = db.dynamic
   const nullResult = sql<null>`${null}`
   const check = await db
-    .selectFrom(valuesList([did]).as(sql`subject (did)`))
+    .selectFrom(valuesList([replierDid]).as(sql`subject (did)`))
     .select([
       allowFollowing
         ? db
             .selectFrom('follow')
-            .where('creator', '=', owner)
+            .where('creator', '=', ownerDid)
             .whereRef('subjectDid', '=', ref('subject.did'))
             .select('creator')
             .as('isFollowed')
@@ -91,8 +120,7 @@ export const violatesThreadGate = async (
 
   if (allowFollowing && check?.isFollowed) {
     return false
-  }
-  if (allowListUris.length && check?.isInList) {
+  } else if (allowListUris.length && check?.isInList) {
     return false
   }