@atproto/bsky 0.0.28 → 0.0.30

package/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts

@@ -12,11 +12,17 @@ export default function (server: Server, ctx: AppContext) {
     handler: async ({ auth, params }) => {
       const { limit, cursor, query } = params
       const requester = auth.credentials.iss
+      if (LikeCountKeyset.clearlyBad(cursor)) {
+        return {
+          encoding: 'application/json',
+          body: { feeds: [] },
+        }
+      }
+
       const db = ctx.db.getReplica()
       const { ref } = db.db.dynamic
       const feedService = ctx.services.feed(db)
       const actorService = ctx.services.actor(db)
-
       let inner = db.db
         .selectFrom('feed_generator')
         .select([

package/src/api/index.ts

@@ -1,6 +1,5 @@
 import { Server } from '../lexicon'
 import AppContext from '../context'
-import describeFeedGenerator from './app/bsky/feed/describeFeedGenerator'
 import getTimeline from './app/bsky/feed/getTimeline'
 import getActorFeeds from './app/bsky/feed/getActorFeeds'
 import getSuggestedFeeds from './app/bsky/feed/getSuggestedFeeds'
@@ -8,7 +7,6 @@ import getAuthorFeed from './app/bsky/feed/getAuthorFeed'
 import getFeed from './app/bsky/feed/getFeed'
 import getFeedGenerator from './app/bsky/feed/getFeedGenerator'
 import getFeedGenerators from './app/bsky/feed/getFeedGenerators'
-import getFeedSkeleton from './app/bsky/feed/getFeedSkeleton'
 import getLikes from './app/bsky/feed/getLikes'
 import getListFeed from './app/bsky/feed/getListFeed'
 import getPostThread from './app/bsky/feed/getPostThread'
@@ -40,7 +38,6 @@ import listNotifications from './app/bsky/notification/listNotifications'
 import updateSeen from './app/bsky/notification/updateSeen'
 import registerPush from './app/bsky/notification/registerPush'
 import getPopularFeedGenerators from './app/bsky/unspecced/getPopularFeedGenerators'
-import getTimelineSkeleton from './app/bsky/unspecced/getTimelineSkeleton'
 import getTaggedSuggestions from './app/bsky/unspecced/getTaggedSuggestions'
 import getSubjectStatus from './com/atproto/admin/getSubjectStatus'
 import updateSubjectStatus from './com/atproto/admin/updateSubjectStatus'
@@ -57,7 +54,6 @@ export * as blobResolver from './blob-resolver'
 
 export default function (server: Server, ctx: AppContext) {
   // app.bsky
-  describeFeedGenerator(server, ctx)
   getTimeline(server, ctx)
   getActorFeeds(server, ctx)
   getSuggestedFeeds(server, ctx)
@@ -65,7 +61,6 @@ export default function (server: Server, ctx: AppContext) {
   getFeed(server, ctx)
   getFeedGenerator(server, ctx)
   getFeedGenerators(server, ctx)
-  getFeedSkeleton(server, ctx)
   getLikes(server, ctx)
   getListFeed(server, ctx)
   getPostThread(server, ctx)
@@ -97,7 +92,6 @@ export default function (server: Server, ctx: AppContext) {
   updateSeen(server, ctx)
   registerPush(server, ctx)
   getPopularFeedGenerators(server, ctx)
-  getTimelineSkeleton(server, ctx)
   getTaggedSuggestions(server, ctx)
   // com.atproto
   getSubjectStatus(server, ctx)

package/src/auto-moderator/index.ts

@@ -8,21 +8,12 @@ import { BackgroundQueue } from '../background'
 import { IndexerConfig } from '../indexer/config'
 import { buildBasicAuth } from '../auth-verifier'
 import { CID } from 'multiformats/cid'
-import { ImageFlagger } from './abyss'
 import { HiveLabeler, ImgLabeler } from './hive'
 import { KeywordLabeler, TextLabeler } from './keyword'
 import { ids } from '../lexicon/lexicons'
-import { Abyss } from './abyss'
-import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher'
-import {
-  REASONOTHER,
-  REASONVIOLATION,
-} from '../lexicon/types/com/atproto/moderation/defs'
 
 export class AutoModerator {
   public pushAgent: AtpAgent
-  public imageFlagger?: ImageFlagger
-  public textFlagger?: TextFlagger
   public imgLabeler?: ImgLabeler
   public textLabeler?: TextLabeler
 
@@ -34,24 +25,9 @@ export class AutoModerator {
       backgroundQueue: BackgroundQueue
     },
   ) {
-    const { hiveApiKey, abyssEndpoint, abyssPassword } = ctx.cfg
+    const { hiveApiKey } = ctx.cfg
     this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined
     this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords)
-    if (abyssEndpoint && abyssPassword) {
-      this.imageFlagger = new Abyss(abyssEndpoint, abyssPassword, ctx)
-    } else {
-      log.error(
-        { abyssEndpoint, abyssPassword },
-        'abyss not properly configured',
-      )
-    }
-
-    if (ctx.cfg.fuzzyMatchB64) {
-      this.textFlagger = FuzzyMatcher.fromB64(
-        ctx.cfg.fuzzyMatchB64,
-        ctx.cfg.fuzzyFalsePositiveB64,
-      )
-    }
 
     const url = new URL(ctx.cfg.moderationPushUrl)
     this.pushAgent = new AtpAgent({ service: url.origin })
@@ -64,35 +40,17 @@ export class AutoModerator {
   processRecord(uri: AtUri, cid: CID, obj: unknown) {
     this.ctx.backgroundQueue.add(async () => {
       const { text, imgs } = getFieldsFromRecord(obj, uri)
-      await Promise.all([
-        this.labelRecord(uri, cid, text, imgs).catch((err) => {
-          log.error(
-            { err, uri: uri.toString(), record: obj },
-            'failed to label record',
-          )
-        }),
-        this.flagRecordText(uri, cid, text).catch((err) => {
-          log.error(
-            { err, uri: uri.toString(), record: obj },
-            'failed to check record for text flagging',
-          )
-        }),
-        this.checkImgForTakedown(uri, cid, imgs).catch((err) => {
-          log.error(
-            { err, uri: uri.toString(), record: obj },
-            'failed to check img for takedown',
-          )
-        }),
-      ])
+      await this.labelRecord(uri, cid, text, imgs).catch((err) => {
+        log.error(
+          { err, uri: uri.toString(), record: obj },
+          'failed to label record',
+        )
+      })
     })
   }
 
-  processHandle(handle: string, did: string) {
-    this.ctx.backgroundQueue.add(async () => {
-      await this.flagSubjectText(handle, { did }).catch((err) => {
-        log.error({ err, handle, did }, 'failed to label handle')
-      })
-    })
+  processHandle(_handle: string, _did: string) {
+    // no-op since this functionality moved to auto-mod service
   }
 
   async labelRecord(uri: AtUri, recordCid: CID, text: string[], imgs: CID[]) {
@@ -108,131 +66,6 @@ export class AutoModerator {
     await this.pushLabels(uri, recordCid, labels)
   }
 
-  async flagRecordText(uri: AtUri, cid: CID, text: string[]) {
-    if (
-      ![
-        ids.AppBskyActorProfile,
-        ids.AppBskyGraphList,
-        ids.AppBskyFeedGenerator,
-      ].includes(uri.collection)
-    ) {
-      return
-    }
-    return this.flagSubjectText(text.join(' '), { uri, cid })
-  }
-
-  async flagSubjectText(
-    text: string,
-    subject: { did: string } | { uri: AtUri; cid: CID },
-  ) {
-    if (!this.textFlagger) return
-    const matches = this.textFlagger.getMatches(text)
-    if (matches.length < 1) return
-    const formattedSubject =
-      'did' in subject
-        ? {
-            $type: 'com.atproto.admin.defs#repoRef',
-            did: subject.did,
-          }
-        : {
-            $type: 'com.atproto.repo.strongRef',
-            uri: subject.uri.toString(),
-            cid: subject.cid.toString(),
-          }
-
-    await this.pushAgent.api.com.atproto.admin.emitModerationEvent({
-      event: {
-        $type: 'com.atproto.admin.defs#modEventReport',
-        comment: `Automatically flagged for possible slurs: ${matches.join(
-          ', ',
-        )}`,
-        reportType: REASONOTHER,
-      },
-      subject: formattedSubject,
-      createdBy: this.ctx.cfg.serverDid,
-    })
-  }
-
-  async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) {
-    if (imgCids.length < 0) return
-    const results = await Promise.all(
-      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)),
-    )
-    const takedownCids: CID[] = []
-    for (let i = 0; i < results.length; i++) {
-      if (results.at(i)?.length) {
-        takedownCids.push(imgCids[i])
-      }
-    }
-    if (takedownCids.length === 0) return
-    try {
-      await this.persistTakedown(
-        uri,
-        recordCid,
-        takedownCids,
-        dedupe(results.flat()),
-      )
-    } catch (err) {
-      log.error(
-        {
-          err,
-          uri: uri.toString(),
-          imgCids: imgCids.map((c) => c.toString()),
-          results,
-        },
-        'failed to persist takedown',
-      )
-    }
-  }
-
-  async persistTakedown(
-    uri: AtUri,
-    recordCid: CID,
-    takedownCids: CID[],
-    labels: string[],
-  ) {
-    const reportReason = `automated takedown (${labels.join(
-      ', ',
-    )}). account needs review and possibly additional action`
-    const takedownReason = `automated takedown for labels: ${labels.join(', ')}`
-    log.warn(
-      {
-        uri: uri.toString(),
-        blobCids: takedownCids,
-        labels,
-      },
-      'hard takedown of record (and blobs) based on auto-matching',
-    )
-
-    await this.pushAgent.api.com.atproto.admin.emitModerationEvent({
-      event: {
-        $type: 'com.atproto.admin.defs#modEventReport',
-        comment: reportReason,
-        reportType: REASONVIOLATION,
-      },
-      subject: {
-        $type: 'com.atproto.repo.strongRef',
-        uri: uri.toString(),
-        cid: recordCid.toString(),
-      },
-      createdBy: this.ctx.cfg.serverDid,
-    })
-
-    await this.pushAgent.com.atproto.admin.emitModerationEvent({
-      event: {
-        $type: 'com.atproto.admin.defs#modEventTakedown',
-        comment: takedownReason,
-      },
-      subject: {
-        $type: 'com.atproto.repo.strongRef',
-        uri: uri.toString(),
-        cid: recordCid.toString(),
-      },
-      subjectBlobCids: takedownCids.map((c) => c.toString()),
-      createdBy: this.ctx.cfg.serverDid,
-    })
-  }
-
   async pushLabels(uri: AtUri, cid: CID, labels: string[]): Promise<void> {
     if (labels.length < 1) return
 

package/src/context.ts

@@ -9,7 +9,6 @@ import { ImageUriBuilder } from './image/uri'
 import { Services } from './services'
 import DidRedisCache from './did-cache'
 import { BackgroundQueue } from './background'
-import { MountedAlgos } from './feed-gen/types'
 import { Redis } from './redis'
 import { AuthVerifier } from './auth-verifier'
 import { BsyncClient } from './bsync'
@@ -30,7 +29,6 @@ export class AppContext {
       searchAgent?: AtpAgent
       bsyncClient?: BsyncClient
       courierClient?: CourierClient
-      algos: MountedAlgos
       authVerifier: AuthVerifier
     },
   ) {}
@@ -99,10 +97,6 @@ export class AppContext {
   get backgroundQueue(): BackgroundQueue {
     return this.opts.backgroundQueue
   }
-
-  get algos(): MountedAlgos {
-    return this.opts.algos
-  }
 }
 
 export default AppContext

package/src/db/pagination.ts

@@ -27,6 +27,9 @@ export abstract class GenericKeyset<R, LR extends LabeledResult> {
   abstract labelResult(result: R): LR
   abstract labeledResultToCursor(labeled: LR): Cursor
   abstract cursorToLabeledResult(cursor: Cursor): LR
+  static clearlyBad(cursor?: string) {
+    return cursor !== undefined && !cursor.includes('::')
+  }
   packFromResult(results: R | R[]): string | undefined {
     const result = Array.isArray(results) ? results.at(-1) : results
     if (!result) return

package/src/index.ts

@@ -28,7 +28,6 @@ import {
   ImageProcessingServerInvalidator,
 } from './image/invalidator'
 import { BackgroundQueue } from './background'
-import { MountedAlgos } from './feed-gen/types'
 import { AtpAgent } from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
 import { Redis } from './redis'
@@ -37,14 +36,12 @@ import { authWithApiKey as bsyncAuth, createBsyncClient } from './bsync'
 import { authWithApiKey as courierAuth, createCourierClient } from './courier'
 
 export type { ServerConfigValues } from './config'
-export type { MountedAlgos } from './feed-gen/types'
 export { ServerConfig } from './config'
 export { Database, PrimaryDatabase, DatabaseCoordinator } from './db'
 export { Redis } from './redis'
 export { ViewMaintainer } from './db/views'
 export { AppContext } from './context'
 export type { ImageInvalidator } from './image/invalidator'
-export { makeAlgos } from './feed-gen'
 export * from './daemon'
 export * from './indexer'
 export * from './ingester'
@@ -67,9 +64,8 @@ export class BskyAppView {
     config: ServerConfig
     signingKey: Keypair
     imgInvalidator?: ImageInvalidator
-    algos?: MountedAlgos
   }): BskyAppView {
-    const { db, redis, config, signingKey, algos = {} } = opts
+    const { db, redis, config, signingKey } = opts
     let maybeImgInvalidator = opts.imgInvalidator
     const app = express()
     app.set('trust proxy', true)
@@ -170,7 +166,6 @@ export class BskyAppView {
       searchAgent,
       bsyncClient,
       courierClient,
-      algos,
       authVerifier,
     })
 

package/src/indexer/config.ts

@@ -15,11 +15,7 @@ export interface IndexerConfigValues {
   didCacheMaxTTL: number
   handleResolveNameservers?: string[]
   hiveApiKey?: string
-  abyssEndpoint?: string
-  abyssPassword?: string
   imgUriEndpoint?: string
-  fuzzyMatchB64?: string
-  fuzzyFalsePositiveB64?: string
   labelerKeywords: Record<string, string>
   moderationPushUrl: string
   courierUrl?: string
@@ -89,8 +85,6 @@ export class IndexerConfig {
       process.env.BSKY_COURIER_IGNORE_BAD_TLS === 'true'
     assert(courierHttpVersion === '1.1' || courierHttpVersion === '2')
     const hiveApiKey = process.env.HIVE_API_KEY || undefined
-    const abyssEndpoint = process.env.ABYSS_ENDPOINT
-    const abyssPassword = process.env.ABYSS_PASSWORD
     const imgUriEndpoint = process.env.IMG_URI_ENDPOINT
     const indexerPartitionIds =
       overrides?.indexerPartitionIds ||
@@ -109,9 +103,6 @@ export class IndexerConfig {
     const ingesterPartitionCount =
       maybeParseInt(process.env.INGESTER_PARTITION_COUNT) ?? 64
     const labelerKeywords = {}
-    const fuzzyMatchB64 = process.env.FUZZY_MATCH_B64 || undefined
-    const fuzzyFalsePositiveB64 =
-      process.env.FUZZY_FALSE_POSITIVE_B64 || undefined
     const pushNotificationEndpoint = process.env.PUSH_NOTIFICATION_ENDPOINT
     assert(dbPostgresUrl)
     assert(redisHost || (redisSentinelName && redisSentinelHosts?.length))
@@ -135,8 +126,6 @@ export class IndexerConfig {
       courierHttpVersion,
       courierIgnoreBadTls,
       hiveApiKey,
-      abyssEndpoint,
-      abyssPassword,
       imgUriEndpoint,
       indexerPartitionIds,
       indexerConcurrency,
@@ -146,8 +135,6 @@ export class IndexerConfig {
       indexerPort,
       ingesterPartitionCount,
       labelerKeywords,
-      fuzzyMatchB64,
-      fuzzyFalsePositiveB64,
       pushNotificationEndpoint,
       ...stripUndefineds(overrides ?? {}),
     })
@@ -225,14 +212,6 @@ export class IndexerConfig {
     return this.cfg.hiveApiKey
   }
 
-  get abyssEndpoint() {
-    return this.cfg.abyssEndpoint
-  }
-
-  get abyssPassword() {
-    return this.cfg.abyssPassword
-  }
-
   get imgUriEndpoint() {
     return this.cfg.imgUriEndpoint
   }
@@ -269,14 +248,6 @@ export class IndexerConfig {
     return this.cfg.labelerKeywords
   }
 
-  get fuzzyMatchB64() {
-    return this.cfg.fuzzyMatchB64
-  }
-
-  get fuzzyFalsePositiveB64() {
-    return this.cfg.fuzzyFalsePositiveB64
-  }
-
   get pushNotificationEndpoint() {
     return this.cfg.pushNotificationEndpoint
   }

package/src/lexicon/index.ts

@@ -127,7 +127,6 @@ import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/
 import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
 import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
 import * as AppBskyUnspeccedGetTaggedSuggestions from './types/app/bsky/unspecced/getTaggedSuggestions'
-import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton'
 import * as AppBskyUnspeccedSearchActorsSkeleton from './types/app/bsky/unspecced/searchActorsSkeleton'
 import * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced/searchPostsSkeleton'
 
@@ -1649,17 +1648,6 @@ export class AppBskyUnspeccedNS {
     return this._server.xrpc.method(nsid, cfg)
   }
 
-  getTimelineSkeleton<AV extends AuthVerifier>(
-    cfg: ConfigOf<
-      AV,
-      AppBskyUnspeccedGetTimelineSkeleton.Handler<ExtractAuth<AV>>,
-      AppBskyUnspeccedGetTimelineSkeleton.HandlerReqCtx<ExtractAuth<AV>>
-    >,
-  ) {
-    const nsid = 'app.bsky.unspecced.getTimelineSkeleton' // @ts-ignore
-    return this._server.xrpc.method(nsid, cfg)
-  }
-
   searchActorsSkeleton<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,