@atproto/bsky 0.0.28 → 0.0.29

package/src/auto-moderator/index.ts

@@ -8,21 +8,12 @@ import { BackgroundQueue } from '../background'
 import { IndexerConfig } from '../indexer/config'
 import { buildBasicAuth } from '../auth-verifier'
 import { CID } from 'multiformats/cid'
-import { ImageFlagger } from './abyss'
 import { HiveLabeler, ImgLabeler } from './hive'
 import { KeywordLabeler, TextLabeler } from './keyword'
 import { ids } from '../lexicon/lexicons'
-import { Abyss } from './abyss'
-import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher'
-import {
-  REASONOTHER,
-  REASONVIOLATION,
-} from '../lexicon/types/com/atproto/moderation/defs'
 
 export class AutoModerator {
   public pushAgent: AtpAgent
-  public imageFlagger?: ImageFlagger
-  public textFlagger?: TextFlagger
   public imgLabeler?: ImgLabeler
   public textLabeler?: TextLabeler
 
@@ -34,24 +25,9 @@ export class AutoModerator {
       backgroundQueue: BackgroundQueue
     },
   ) {
-    const { hiveApiKey, abyssEndpoint, abyssPassword } = ctx.cfg
+    const { hiveApiKey } = ctx.cfg
     this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined
     this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords)
-    if (abyssEndpoint && abyssPassword) {
-      this.imageFlagger = new Abyss(abyssEndpoint, abyssPassword, ctx)
-    } else {
-      log.error(
-        { abyssEndpoint, abyssPassword },
-        'abyss not properly configured',
-      )
-    }
-
-    if (ctx.cfg.fuzzyMatchB64) {
-      this.textFlagger = FuzzyMatcher.fromB64(
-        ctx.cfg.fuzzyMatchB64,
-        ctx.cfg.fuzzyFalsePositiveB64,
-      )
-    }
 
     const url = new URL(ctx.cfg.moderationPushUrl)
     this.pushAgent = new AtpAgent({ service: url.origin })
@@ -64,35 +40,17 @@ export class AutoModerator {
   processRecord(uri: AtUri, cid: CID, obj: unknown) {
     this.ctx.backgroundQueue.add(async () => {
       const { text, imgs } = getFieldsFromRecord(obj, uri)
-      await Promise.all([
-        this.labelRecord(uri, cid, text, imgs).catch((err) => {
-          log.error(
-            { err, uri: uri.toString(), record: obj },
-            'failed to label record',
-          )
-        }),
-        this.flagRecordText(uri, cid, text).catch((err) => {
-          log.error(
-            { err, uri: uri.toString(), record: obj },
-            'failed to check record for text flagging',
-          )
-        }),
-        this.checkImgForTakedown(uri, cid, imgs).catch((err) => {
-          log.error(
-            { err, uri: uri.toString(), record: obj },
-            'failed to check img for takedown',
-          )
-        }),
-      ])
+      await this.labelRecord(uri, cid, text, imgs).catch((err) => {
+        log.error(
+          { err, uri: uri.toString(), record: obj },
+          'failed to label record',
+        )
+      })
     })
   }
 
-  processHandle(handle: string, did: string) {
-    this.ctx.backgroundQueue.add(async () => {
-      await this.flagSubjectText(handle, { did }).catch((err) => {
-        log.error({ err, handle, did }, 'failed to label handle')
-      })
-    })
+  processHandle(_handle: string, _did: string) {
+    // no-op since this functionality moved to auto-mod service
   }
 
   async labelRecord(uri: AtUri, recordCid: CID, text: string[], imgs: CID[]) {
@@ -108,131 +66,6 @@ export class AutoModerator {
     await this.pushLabels(uri, recordCid, labels)
   }
 
-  async flagRecordText(uri: AtUri, cid: CID, text: string[]) {
-    if (
-      ![
-        ids.AppBskyActorProfile,
-        ids.AppBskyGraphList,
-        ids.AppBskyFeedGenerator,
-      ].includes(uri.collection)
-    ) {
-      return
-    }
-    return this.flagSubjectText(text.join(' '), { uri, cid })
-  }
-
-  async flagSubjectText(
-    text: string,
-    subject: { did: string } | { uri: AtUri; cid: CID },
-  ) {
-    if (!this.textFlagger) return
-    const matches = this.textFlagger.getMatches(text)
-    if (matches.length < 1) return
-    const formattedSubject =
-      'did' in subject
-        ? {
-            $type: 'com.atproto.admin.defs#repoRef',
-            did: subject.did,
-          }
-        : {
-            $type: 'com.atproto.repo.strongRef',
-            uri: subject.uri.toString(),
-            cid: subject.cid.toString(),
-          }
-
-    await this.pushAgent.api.com.atproto.admin.emitModerationEvent({
-      event: {
-        $type: 'com.atproto.admin.defs#modEventReport',
-        comment: `Automatically flagged for possible slurs: ${matches.join(
-          ', ',
-        )}`,
-        reportType: REASONOTHER,
-      },
-      subject: formattedSubject,
-      createdBy: this.ctx.cfg.serverDid,
-    })
-  }
-
-  async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) {
-    if (imgCids.length < 0) return
-    const results = await Promise.all(
-      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)),
-    )
-    const takedownCids: CID[] = []
-    for (let i = 0; i < results.length; i++) {
-      if (results.at(i)?.length) {
-        takedownCids.push(imgCids[i])
-      }
-    }
-    if (takedownCids.length === 0) return
-    try {
-      await this.persistTakedown(
-        uri,
-        recordCid,
-        takedownCids,
-        dedupe(results.flat()),
-      )
-    } catch (err) {
-      log.error(
-        {
-          err,
-          uri: uri.toString(),
-          imgCids: imgCids.map((c) => c.toString()),
-          results,
-        },
-        'failed to persist takedown',
-      )
-    }
-  }
-
-  async persistTakedown(
-    uri: AtUri,
-    recordCid: CID,
-    takedownCids: CID[],
-    labels: string[],
-  ) {
-    const reportReason = `automated takedown (${labels.join(
-      ', ',
-    )}). account needs review and possibly additional action`
-    const takedownReason = `automated takedown for labels: ${labels.join(', ')}`
-    log.warn(
-      {
-        uri: uri.toString(),
-        blobCids: takedownCids,
-        labels,
-      },
-      'hard takedown of record (and blobs) based on auto-matching',
-    )
-
-    await this.pushAgent.api.com.atproto.admin.emitModerationEvent({
-      event: {
-        $type: 'com.atproto.admin.defs#modEventReport',
-        comment: reportReason,
-        reportType: REASONVIOLATION,
-      },
-      subject: {
-        $type: 'com.atproto.repo.strongRef',
-        uri: uri.toString(),
-        cid: recordCid.toString(),
-      },
-      createdBy: this.ctx.cfg.serverDid,
-    })
-
-    await this.pushAgent.com.atproto.admin.emitModerationEvent({
-      event: {
-        $type: 'com.atproto.admin.defs#modEventTakedown',
-        comment: takedownReason,
-      },
-      subject: {
-        $type: 'com.atproto.repo.strongRef',
-        uri: uri.toString(),
-        cid: recordCid.toString(),
-      },
-      subjectBlobCids: takedownCids.map((c) => c.toString()),
-      createdBy: this.ctx.cfg.serverDid,
-    })
-  }
-
   async pushLabels(uri: AtUri, cid: CID, labels: string[]): Promise<void> {
     if (labels.length < 1) return
 

package/src/context.ts

@@ -9,7 +9,6 @@ import { ImageUriBuilder } from './image/uri'
 import { Services } from './services'
 import DidRedisCache from './did-cache'
 import { BackgroundQueue } from './background'
-import { MountedAlgos } from './feed-gen/types'
 import { Redis } from './redis'
 import { AuthVerifier } from './auth-verifier'
 import { BsyncClient } from './bsync'
@@ -30,7 +29,6 @@ export class AppContext {
       searchAgent?: AtpAgent
       bsyncClient?: BsyncClient
       courierClient?: CourierClient
-      algos: MountedAlgos
       authVerifier: AuthVerifier
     },
   ) {}
@@ -99,10 +97,6 @@ export class AppContext {
   get backgroundQueue(): BackgroundQueue {
     return this.opts.backgroundQueue
   }
-
-  get algos(): MountedAlgos {
-    return this.opts.algos
-  }
 }
 
 export default AppContext

package/src/db/pagination.ts

@@ -27,6 +27,9 @@ export abstract class GenericKeyset<R, LR extends LabeledResult> {
   abstract labelResult(result: R): LR
   abstract labeledResultToCursor(labeled: LR): Cursor
   abstract cursorToLabeledResult(cursor: Cursor): LR
+  static clearlyBad(cursor?: string) {
+    return cursor !== undefined && !cursor.includes('::')
+  }
   packFromResult(results: R | R[]): string | undefined {
     const result = Array.isArray(results) ? results.at(-1) : results
     if (!result) return

package/src/index.ts

@@ -28,7 +28,6 @@ import {
   ImageProcessingServerInvalidator,
 } from './image/invalidator'
 import { BackgroundQueue } from './background'
-import { MountedAlgos } from './feed-gen/types'
 import { AtpAgent } from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
 import { Redis } from './redis'
@@ -37,14 +36,12 @@ import { authWithApiKey as bsyncAuth, createBsyncClient } from './bsync'
 import { authWithApiKey as courierAuth, createCourierClient } from './courier'
 
 export type { ServerConfigValues } from './config'
-export type { MountedAlgos } from './feed-gen/types'
 export { ServerConfig } from './config'
 export { Database, PrimaryDatabase, DatabaseCoordinator } from './db'
 export { Redis } from './redis'
 export { ViewMaintainer } from './db/views'
 export { AppContext } from './context'
 export type { ImageInvalidator } from './image/invalidator'
-export { makeAlgos } from './feed-gen'
 export * from './daemon'
 export * from './indexer'
 export * from './ingester'
@@ -67,9 +64,8 @@ export class BskyAppView {
     config: ServerConfig
     signingKey: Keypair
     imgInvalidator?: ImageInvalidator
-    algos?: MountedAlgos
   }): BskyAppView {
-    const { db, redis, config, signingKey, algos = {} } = opts
+    const { db, redis, config, signingKey } = opts
     let maybeImgInvalidator = opts.imgInvalidator
     const app = express()
     app.set('trust proxy', true)
@@ -170,7 +166,6 @@ export class BskyAppView {
       searchAgent,
       bsyncClient,
       courierClient,
-      algos,
       authVerifier,
     })
 

package/src/indexer/config.ts

@@ -15,11 +15,7 @@ export interface IndexerConfigValues {
   didCacheMaxTTL: number
   handleResolveNameservers?: string[]
   hiveApiKey?: string
-  abyssEndpoint?: string
-  abyssPassword?: string
   imgUriEndpoint?: string
-  fuzzyMatchB64?: string
-  fuzzyFalsePositiveB64?: string
   labelerKeywords: Record<string, string>
   moderationPushUrl: string
   courierUrl?: string
@@ -89,8 +85,6 @@ export class IndexerConfig {
       process.env.BSKY_COURIER_IGNORE_BAD_TLS === 'true'
     assert(courierHttpVersion === '1.1' || courierHttpVersion === '2')
     const hiveApiKey = process.env.HIVE_API_KEY || undefined
-    const abyssEndpoint = process.env.ABYSS_ENDPOINT
-    const abyssPassword = process.env.ABYSS_PASSWORD
     const imgUriEndpoint = process.env.IMG_URI_ENDPOINT
     const indexerPartitionIds =
       overrides?.indexerPartitionIds ||
@@ -109,9 +103,6 @@ export class IndexerConfig {
     const ingesterPartitionCount =
       maybeParseInt(process.env.INGESTER_PARTITION_COUNT) ?? 64
     const labelerKeywords = {}
-    const fuzzyMatchB64 = process.env.FUZZY_MATCH_B64 || undefined
-    const fuzzyFalsePositiveB64 =
-      process.env.FUZZY_FALSE_POSITIVE_B64 || undefined
     const pushNotificationEndpoint = process.env.PUSH_NOTIFICATION_ENDPOINT
     assert(dbPostgresUrl)
     assert(redisHost || (redisSentinelName && redisSentinelHosts?.length))
@@ -135,8 +126,6 @@ export class IndexerConfig {
       courierHttpVersion,
       courierIgnoreBadTls,
       hiveApiKey,
-      abyssEndpoint,
-      abyssPassword,
       imgUriEndpoint,
       indexerPartitionIds,
       indexerConcurrency,
@@ -146,8 +135,6 @@ export class IndexerConfig {
       indexerPort,
       ingesterPartitionCount,
       labelerKeywords,
-      fuzzyMatchB64,
-      fuzzyFalsePositiveB64,
       pushNotificationEndpoint,
       ...stripUndefineds(overrides ?? {}),
     })
@@ -225,14 +212,6 @@ export class IndexerConfig {
     return this.cfg.hiveApiKey
   }
 
-  get abyssEndpoint() {
-    return this.cfg.abyssEndpoint
-  }
-
-  get abyssPassword() {
-    return this.cfg.abyssPassword
-  }
-
   get imgUriEndpoint() {
     return this.cfg.imgUriEndpoint
   }
@@ -269,14 +248,6 @@ export class IndexerConfig {
     return this.cfg.labelerKeywords
   }
 
-  get fuzzyMatchB64() {
-    return this.cfg.fuzzyMatchB64
-  }
-
-  get fuzzyFalsePositiveB64() {
-    return this.cfg.fuzzyFalsePositiveB64
-  }
-
   get pushNotificationEndpoint() {
     return this.cfg.pushNotificationEndpoint
   }

package/src/lexicon/index.ts

@@ -127,7 +127,6 @@ import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/
 import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
 import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
 import * as AppBskyUnspeccedGetTaggedSuggestions from './types/app/bsky/unspecced/getTaggedSuggestions'
-import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton'
 import * as AppBskyUnspeccedSearchActorsSkeleton from './types/app/bsky/unspecced/searchActorsSkeleton'
 import * as AppBskyUnspeccedSearchPostsSkeleton from './types/app/bsky/unspecced/searchPostsSkeleton'
 
@@ -1649,17 +1648,6 @@ export class AppBskyUnspeccedNS {
     return this._server.xrpc.method(nsid, cfg)
   }
 
-  getTimelineSkeleton<AV extends AuthVerifier>(
-    cfg: ConfigOf<
-      AV,
-      AppBskyUnspeccedGetTimelineSkeleton.Handler<ExtractAuth<AV>>,
-      AppBskyUnspeccedGetTimelineSkeleton.HandlerReqCtx<ExtractAuth<AV>>
-    >,
-  ) {
-    const nsid = 'app.bsky.unspecced.getTimelineSkeleton' // @ts-ignore
-    return this._server.xrpc.method(nsid, cfg)
-  }
-
   searchActorsSkeleton<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,

package/src/lexicon/lexicons.ts

@@ -91,6 +91,7 @@ export const schemaDict = {
               'lex:com.atproto.admin.defs#modEventEscalate',
               'lex:com.atproto.admin.defs#modEventMute',
               'lex:com.atproto.admin.defs#modEventEmail',
+              'lex:com.atproto.admin.defs#modEventResolveAppeal',
             ],
           },
           subject: {
@@ -147,6 +148,7 @@ export const schemaDict = {
               'lex:com.atproto.admin.defs#modEventAcknowledge',
               'lex:com.atproto.admin.defs#modEventEscalate',
               'lex:com.atproto.admin.defs#modEventMute',
+              'lex:com.atproto.admin.defs#modEventEmail',
               'lex:com.atproto.admin.defs#modEventResolveAppeal',
             ],
           },
@@ -1450,6 +1452,16 @@ export const schemaDict = {
               description:
                 'Sort direction for the events. Defaults to descending order of created at timestamp.',
             },
+            createdAfter: {
+              type: 'string',
+              format: 'datetime',
+              description: 'Retrieve events created after a given timestamp',
+            },
+            createdBefore: {
+              type: 'string',
+              format: 'datetime',
+              description: 'Retrieve events created before a given timestamp',
+            },
             subject: {
               type: 'string',
               format: 'uri',
@@ -1466,6 +1478,37 @@ export const schemaDict = {
               maximum: 100,
               default: 50,
             },
+            hasComment: {
+              type: 'boolean',
+              description: 'If true, only events with comments are returned',
+            },
+            comment: {
+              type: 'string',
+              description:
+                'If specified, only events with comments containing the keyword are returned',
+            },
+            addedLabels: {
+              type: 'array',
+              items: {
+                type: 'string',
+              },
+              description:
+                'If specified, only events where all of these labels were added are returned',
+            },
+            removedLabels: {
+              type: 'array',
+              items: {
+                type: 'string',
+              },
+              description:
+                'If specified, only events where all of these labels were removed are returned',
+            },
+            reportTypes: {
+              type: 'array',
+              items: {
+                type: 'string',
+              },
+            },
             cursor: {
               type: 'string',
             },
@@ -8100,55 +8143,6 @@ export const schemaDict = {
       },
     },
   },
-  AppBskyUnspeccedGetTimelineSkeleton: {
-    lexicon: 1,
-    id: 'app.bsky.unspecced.getTimelineSkeleton',
-    defs: {
-      main: {
-        type: 'query',
-        description:
-          'DEPRECATED: a skeleton of a timeline. Unspecced and will be unavailable soon.',
-        parameters: {
-          type: 'params',
-          properties: {
-            limit: {
-              type: 'integer',
-              minimum: 1,
-              maximum: 100,
-              default: 50,
-            },
-            cursor: {
-              type: 'string',
-            },
-          },
-        },
-        output: {
-          encoding: 'application/json',
-          schema: {
-            type: 'object',
-            required: ['feed'],
-            properties: {
-              cursor: {
-                type: 'string',
-              },
-              feed: {
-                type: 'array',
-                items: {
-                  type: 'ref',
-                  ref: 'lex:app.bsky.feed.defs#skeletonFeedPost',
-                },
-              },
-            },
-          },
-        },
-        errors: [
-          {
-            name: 'UnknownFeed',
-          },
-        ],
-      },
-    },
-  },
   AppBskyUnspeccedSearchActorsSkeleton: {
     lexicon: 1,
     id: 'app.bsky.unspecced.searchActorsSkeleton',
@@ -8438,7 +8432,6 @@ export const ids = {
     'app.bsky.unspecced.getPopularFeedGenerators',
   AppBskyUnspeccedGetTaggedSuggestions:
     'app.bsky.unspecced.getTaggedSuggestions',
-  AppBskyUnspeccedGetTimelineSkeleton: 'app.bsky.unspecced.getTimelineSkeleton',
   AppBskyUnspeccedSearchActorsSkeleton:
     'app.bsky.unspecced.searchActorsSkeleton',
   AppBskyUnspeccedSearchPostsSkeleton: 'app.bsky.unspecced.searchPostsSkeleton',

package/src/lexicon/types/com/atproto/admin/defs.ts

@@ -40,6 +40,7 @@ export interface ModEventView {
     | ModEventEscalate
     | ModEventMute
     | ModEventEmail
+    | ModEventResolveAppeal
     | { $type: string; [k: string]: unknown }
   subject:
     | RepoRef
@@ -76,6 +77,7 @@ export interface ModEventViewDetail {
     | ModEventAcknowledge
     | ModEventEscalate
     | ModEventMute
+    | ModEventEmail
     | ModEventResolveAppeal
     | { $type: string; [k: string]: unknown }
   subject:

package/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts

@@ -15,10 +15,23 @@ export interface QueryParams {
   createdBy?: string
   /** Sort direction for the events. Defaults to descending order of created at timestamp. */
   sortDirection: 'asc' | 'desc'
+  /** Retrieve events created after a given timestamp */
+  createdAfter?: string
+  /** Retrieve events created before a given timestamp */
+  createdBefore?: string
   subject?: string
   /** If true, events on all record types (posts, lists, profile etc.) owned by the did are returned */
   includeAllUserRecords: boolean
   limit: number
+  /** If true, only events with comments are returned */
+  hasComment?: boolean
+  /** If specified, only events with comments containing the keyword are returned */
+  comment?: string
+  /** If specified, only events where all of these labels were added are returned */
+  addedLabels?: string[]
+  /** If specified, only events where all of these labels were removed are returned */
+  removedLabels?: string[]
+  reportTypes?: string[]
   cursor?: string
 }
 

package/src/logger.ts

@@ -1,5 +1,8 @@
+import pino from 'pino'
 import pinoHttp from 'pino-http'
+import * as jose from 'jose'
 import { subsystemLogger } from '@atproto/common'
+import { parseBasicAuth } from './auth-verifier'
 
 export const dbLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:db')
@@ -21,5 +24,34 @@ export const loggerMiddleware = pinoHttp({
         message: err?.message,
       }
     },
+    req: (req) => {
+      const serialized = pino.stdSerializers.req(req)
+      const authHeader = serialized.headers.authorization || ''
+      let auth: string | undefined = undefined
+      if (authHeader.startsWith('Bearer ')) {
+        const token = authHeader.slice('Bearer '.length)
+        const { iss } = jose.decodeJwt(token)
+        if (iss) {
+          auth = 'Bearer ' + iss
+        } else {
+          auth = 'Bearer Invalid'
+        }
+      }
+      if (authHeader.startsWith('Basic ')) {
+        const parsed = parseBasicAuth(authHeader)
+        if (!parsed) {
+          auth = 'Basic Invalid'
+        } else {
+          auth = 'Basic ' + parsed.username
+        }
+      }
+      return {
+        ...serialized,
+        headers: {
+          ...serialized.headers,
+          authorization: auth,
+        },
+      }
+    },
   },
 })

package/tests/auto-moderator/labeler.test.ts

@@ -103,6 +103,8 @@ describe('labeler', () => {
       subject: uri.toString(),
       limit: 10,
       types: [],
+      addedLabels: [],
+      removedLabels: [],
     })
     expect(events.length).toBe(1)
     expect(events[0]).toMatchObject({

package/tests/feed-generation.test.ts

@@ -86,12 +86,6 @@ describe('feed generation', () => {
     await network.close()
   })
 
-  // @TODO enable once getFeed is implemented
-  it('describes the feed generator', async () => {
-    const res = await agent.api.app.bsky.feed.describeFeedGenerator()
-    expect(res.data.did).toBe(network.bsky.ctx.cfg.feedGenDid)
-  })
-
   it('feed gen records can be created.', async () => {
     const all = await pdsAgent.api.app.bsky.feed.generator.create(
       { repo: alice, rkey: 'all' },

package/tests/views/notifications.test.ts

@@ -296,4 +296,13 @@ describe('notification views', () => {
       ),
     )
   })
+
+  it('fails open on clearly bad cursor.', async () => {
+    const { data: notifs } =
+      await agent.api.app.bsky.notification.listNotifications(
+        { cursor: 'bad' },
+        { headers: await network.serviceHeaders(alice) },
+      )
+    expect(notifs).toEqual({ notifications: [] })
+  })
 })

package/tests/views/timeline.test.ts

@@ -298,4 +298,12 @@ describe('timeline views', () => {
       ),
     )
   })
+
+  it('fails open on clearly bad cursor.', async () => {
+    const { data: timeline } = await agent.api.app.bsky.feed.getTimeline(
+      { cursor: 'bad' },
+      { headers: await network.serviceHeaders(alice) },
+    )
+    expect(timeline).toEqual({ feed: [] })
+  })
 })

package/dist/api/app/bsky/feed/describeFeedGenerator.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/api/app/bsky/feed/getFeedSkeleton.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;