@atproto/bsky 0.0.28 → 0.0.30

package/src/auto-moderator/fuzzy-matcher.ts

@@ -1,126 +0,0 @@
-import { dedupeStrs } from '@atproto/common'
-import * as ui8 from 'uint8arrays'
-
-export interface TextFlagger {
-  getMatches(string: string): string[]
-}
-
-export class FuzzyMatcher implements TextFlagger {
-  private bannedWords: Set<string>
-  private falsePositives: Set<string>
-
-  constructor(bannedWords: string[], falsePositives: string[] = []) {
-    this.bannedWords = new Set(bannedWords.map((word) => word.toLowerCase()))
-    this.falsePositives = new Set(
-      falsePositives.map((word) => word.toLowerCase()),
-    )
-  }
-
-  static fromB64(bannedB64: string, falsePositivesB64?: string) {
-    return new FuzzyMatcher(
-      decode(bannedB64),
-      falsePositivesB64 ? decode(falsePositivesB64) : undefined,
-    )
-  }
-
-  private normalize(domain: string): string[] {
-    const withoutSymbols = domain.replace(/[\W_]+/g, '') // Remove non-alphanumeric characters
-    const lowercase = withoutSymbols.toLowerCase()
-
-    // Replace common leetspeak characters
-    const leetSpeakReplacements: { [key: string]: string[] } = {
-      '0': ['o'],
-      '8': ['b'],
-      '3': ['e'],
-      '4': ['a'],
-      '6': ['g'],
-      '1': ['i', 'l'],
-      '5': ['s'],
-      '7': ['t'],
-    }
-
-    return this.generatePermutations(lowercase, leetSpeakReplacements)
-  }
-
-  private generatePermutations(
-    domain: string,
-    leetSpeakReplacements: { [key: string]: string[] },
-  ): string[] {
-    const results: string[] = []
-
-    const leetChars = Object.keys(leetSpeakReplacements)
-    const firstLeetCharIndex = [...domain].findIndex((char) =>
-      leetChars.includes(char),
-    )
-
-    if (firstLeetCharIndex === -1) {
-      // No leetspeak characters left in the string
-      results.push(domain)
-    } else {
-      const char = domain[firstLeetCharIndex]
-      const beforeChar = domain.slice(0, firstLeetCharIndex)
-      const afterChar = domain.slice(firstLeetCharIndex + 1)
-
-      // For each replacement, generate all possible combinations
-      for (const replacement of leetSpeakReplacements[char]) {
-        const replaced = beforeChar + replacement + afterChar
-
-        // Recursively generate all permutations for the rest of the string
-        const otherPermutations = this.generatePermutations(
-          replaced,
-          leetSpeakReplacements,
-        )
-
-        // Add these permutations to the results
-        results.push(...otherPermutations)
-      }
-    }
-
-    return dedupeStrs(results)
-  }
-
-  public getMatches(domain: string): string[] {
-    const normalizedDomains = this.normalize(domain)
-
-    const foundUnacceptableWords: string[] = []
-
-    for (const normalizedDomain of normalizedDomains) {
-      for (const word of this.bannedWords) {
-        const match = normalizedDomain.indexOf(word)
-        if (match > -1) {
-          let isFalsePositive = false
-          for (const falsePositive of this.falsePositives) {
-            const s_fp = falsePositive.indexOf(word)
-            const s_nd = match - s_fp
-            const wordToMatch = normalizedDomain.slice(
-              s_nd,
-              s_nd + falsePositive.length,
-            )
-            if (wordToMatch === falsePositive) {
-              isFalsePositive = true
-              break
-            }
-          }
-
-          if (!isFalsePositive) {
-            foundUnacceptableWords.push(word)
-          }
-        }
-      }
-    }
-
-    if (foundUnacceptableWords.length > 0) {
-      return foundUnacceptableWords
-    }
-
-    return []
-  }
-}
-
-export const decode = (encoded: string): string[] => {
-  return ui8.toString(ui8.fromString(encoded, 'base64'), 'utf8').split(',')
-}
-
-export const encode = (words: string[]): string => {
-  return ui8.toString(ui8.fromString(words.join(','), 'utf8'), 'base64')
-}

package/src/feed-gen/bsky-team.ts

@@ -1,42 +0,0 @@
-import { NotEmptyArray } from '@atproto/common'
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
-import AppContext from '../context'
-import { paginate } from '../db/pagination'
-import { AlgoHandler, AlgoResponse } from './types'
-import { FeedKeyset } from '../api/app/bsky/util/feed'
-
-const BSKY_TEAM: NotEmptyArray<string> = [
-  'did:plc:z72i7hdynmk6r22z27h6tvur', // @bsky.app
-  'did:plc:ewvi7nxzyoun6zhxrhs64oiz', // @atproto.com
-  'did:plc:eon2iu7v3x2ukgxkqaf7e5np', // @safety.bsky.app
-]
-
-const handler: AlgoHandler = async (
-  ctx: AppContext,
-  params: SkeletonParams,
-  _viewer: string | null,
-): Promise<AlgoResponse> => {
-  const { limit = 50, cursor } = params
-  const db = ctx.db.getReplica('feed')
-  const feedService = ctx.services.feed(db)
-
-  const { ref } = db.db.dynamic
-
-  const postsQb = feedService
-    .selectPostQb()
-    .where('post.creator', 'in', BSKY_TEAM)
-
-  const keyset = new FeedKeyset(ref('sortAt'), ref('cid'))
-
-  let feedQb = db.db.selectFrom(postsQb.as('feed_items')).selectAll()
-  feedQb = paginate(feedQb, { limit, cursor, keyset })
-
-  const feedItems = await feedQb.execute()
-
-  return {
-    feedItems,
-    cursor: keyset.packFromResult(feedItems),
-  }
-}
-
-export default handler

package/src/feed-gen/hot-classic.ts

@@ -1,55 +0,0 @@
-import AppContext from '../context'
-import { NotEmptyArray } from '@atproto/common'
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
-import { paginate } from '../db/pagination'
-import { AlgoHandler, AlgoResponse } from './types'
-import { FeedKeyset } from '../api/app/bsky/util/feed'
-import { valuesList } from '../db/util'
-
-const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = ['!no-promote']
-
-const handler: AlgoHandler = async (
-  ctx: AppContext,
-  params: SkeletonParams,
-  _viewer: string | null,
-): Promise<AlgoResponse> => {
-  const { limit = 50, cursor } = params
-  const db = ctx.db.getReplica('feed')
-  const feedService = ctx.services.feed(db)
-
-  const { ref } = db.db.dynamic
-
-  const postsQb = feedService
-    .selectPostQb()
-    .leftJoin('post_agg', 'post_agg.uri', 'post.uri')
-    .leftJoin('post_embed_record', 'post_embed_record.postUri', 'post.uri')
-    .where('post_agg.likeCount', '>=', 12)
-    .where('post.replyParent', 'is', null)
-    .whereNotExists((qb) =>
-      qb
-        .selectFrom('label')
-        .selectAll()
-        .whereRef('val', 'in', valuesList(NO_WHATS_HOT_LABELS))
-        .where('neg', '=', false)
-        .where((clause) =>
-          clause
-            .whereRef('label.uri', '=', ref('post.creator'))
-            .orWhereRef('label.uri', '=', ref('post.uri'))
-            .orWhereRef('label.uri', '=', ref('post_embed_record.embedUri')),
-        ),
-    )
-
-  const keyset = new FeedKeyset(ref('sortAt'), ref('cid'))
-
-  let feedQb = db.db.selectFrom(postsQb.as('feed_items')).selectAll()
-  feedQb = paginate(feedQb, { limit, cursor, keyset })
-
-  const feedItems = await feedQb.execute()
-
-  return {
-    feedItems,
-    cursor: keyset.packFromResult(feedItems),
-  }
-}
-
-export default handler

package/src/feed-gen/index.ts

@@ -1,17 +0,0 @@
-import { AtUri } from '@atproto/syntax'
-import { ids } from '../lexicon/lexicons'
-import bskyTeam from './bsky-team'
-import hotClassic from './hot-classic'
-import mutuals from './mutuals'
-import { MountedAlgos } from './types'
-
-const feedgenUri = (did, name) =>
-  AtUri.make(did, ids.AppBskyFeedGenerator, name).toString()
-
-// These are custom algorithms that will be mounted directly onto an AppView
-// Feel free to remove, update to your own, or serve the following logic at a record that you control
-export const makeAlgos = (did: string): MountedAlgos => ({
-  [feedgenUri(did, 'bsky-team')]: bskyTeam,
-  [feedgenUri(did, 'hot-classic')]: hotClassic,
-  [feedgenUri(did, 'mutuals')]: mutuals,
-})

package/src/feed-gen/mutuals.ts

@@ -1,57 +0,0 @@
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
-import AppContext from '../context'
-import { paginate } from '../db/pagination'
-import { AlgoHandler, AlgoResponse } from './types'
-import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed'
-import { AuthRequiredError } from '@atproto/xrpc-server'
-
-const handler: AlgoHandler = async (
-  ctx: AppContext,
-  params: SkeletonParams,
-  viewer: string | null,
-): Promise<AlgoResponse> => {
-  if (!viewer) {
-    throw new AuthRequiredError('This feed requires being logged-in')
-  }
-
-  const { limit = 50, cursor } = params
-  const db = ctx.db.getReplica('feed')
-  const feedService = ctx.services.feed(db)
-  const { ref } = db.db.dynamic
-
-  const mutualsSubquery = db.db
-    .selectFrom('follow')
-    .where('follow.creator', '=', viewer)
-    .whereExists((qb) =>
-      qb
-        .selectFrom('follow as follow_inner')
-        .whereRef('follow_inner.creator', '=', 'follow.subjectDid')
-        .where('follow_inner.subjectDid', '=', viewer)
-        .selectAll(),
-    )
-    .select('follow.subjectDid')
-
-  const keyset = new FeedKeyset(ref('feed_item.sortAt'), ref('feed_item.cid'))
-  const sortFrom = keyset.unpack(cursor)?.primary
-
-  let feedQb = feedService
-    .selectFeedItemQb()
-    .where('feed_item.type', '=', 'post') // ensures originatorDid is post.creator
-    .where((qb) =>
-      qb
-        .where('originatorDid', '=', viewer)
-        .orWhere('originatorDid', 'in', mutualsSubquery),
-    )
-    .where('feed_item.sortAt', '>', getFeedDateThreshold(sortFrom))
-
-  feedQb = paginate(feedQb, { limit, cursor, keyset })
-
-  const feedItems = await feedQb.execute()
-
-  return {
-    feedItems,
-    cursor: keyset.packFromResult(feedItems),
-  }
-}
-
-export default handler

package/src/feed-gen/types.ts

@@ -1,32 +0,0 @@
-import AppContext from '../context'
-import { SkeletonFeedPost } from '../lexicon/types/app/bsky/feed/defs'
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
-import { FeedRow } from '../services/feed'
-
-export type AlgoResponse = {
-  feedItems: FeedRow[]
-  resHeaders?: Record<string, string>
-  cursor?: string
-}
-
-export type AlgoHandler = (
-  ctx: AppContext,
-  params: SkeletonParams,
-  viewer: string | null,
-) => Promise<AlgoResponse>
-
-export type MountedAlgos = Record<string, AlgoHandler>
-
-export const toSkeletonItem = (feedItem: {
-  uri: string
-  postUri: string
-}): SkeletonFeedPost => ({
-  post: feedItem.postUri,
-  reason:
-    feedItem.uri === feedItem.postUri
-      ? undefined
-      : {
-          $type: 'app.bsky.feed.defs#skeletonReasonRepost',
-          repost: feedItem.uri,
-        },
-})

package/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts

@@ -1,49 +0,0 @@
-/**
- * GENERATED CODE - DO NOT MODIFY
- */
-import express from 'express'
-import { ValidationResult, BlobRef } from '@atproto/lexicon'
-import { lexicons } from '../../../../lexicons'
-import { isObj, hasProp } from '../../../../util'
-import { CID } from 'multiformats/cid'
-import { HandlerAuth } from '@atproto/xrpc-server'
-import * as AppBskyFeedDefs from '../feed/defs'
-
-export interface QueryParams {
-  limit: number
-  cursor?: string
-}
-
-export type InputSchema = undefined
-
-export interface OutputSchema {
-  cursor?: string
-  feed: AppBskyFeedDefs.SkeletonFeedPost[]
-  [k: string]: unknown
-}
-
-export type HandlerInput = undefined
-
-export interface HandlerSuccess {
-  encoding: 'application/json'
-  body: OutputSchema
-  headers?: { [key: string]: string }
-}
-
-export interface HandlerError {
-  status: number
-  message?: string
-  error?: 'UnknownFeed'
-}
-
-export type HandlerOutput = HandlerError | HandlerSuccess
-export type HandlerReqCtx<HA extends HandlerAuth = never> = {
-  auth: HA
-  params: QueryParams
-  input: HandlerInput
-  req: express.Request
-  res: express.Response
-}
-export type Handler<HA extends HandlerAuth = never> = (
-  ctx: HandlerReqCtx<HA>,
-) => Promise<HandlerOutput> | HandlerOutput

package/tests/algos/hot-classic.test.ts

@@ -1,87 +0,0 @@
-import AtpAgent, { AtUri } from '@atproto/api'
-import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env'
-import { makeAlgos } from '../../src'
-
-describe('algo hot-classic', () => {
-  let network: TestNetwork
-  let agent: AtpAgent
-  let sc: SeedClient
-
-  // account dids, for convenience
-  let alice: string
-  let bob: string
-
-  const feedPublisherDid = 'did:example:feed-publisher'
-  const feedUri = AtUri.make(
-    feedPublisherDid,
-    'app.bsky.feed.generator',
-    'hot-classic',
-  ).toString()
-
-  beforeAll(async () => {
-    network = await TestNetwork.create({
-      dbPostgresSchema: 'bsky_algo_hot_classic',
-      bsky: { algos: makeAlgos(feedPublisherDid) },
-    })
-    agent = new AtpAgent({ service: network.bsky.url })
-    sc = network.getSeedClient()
-    await basicSeed(sc)
-
-    alice = sc.dids.alice
-    bob = sc.dids.bob
-    await network.processAll()
-  })
-
-  afterAll(async () => {
-    await network.close()
-  })
-
-  it('returns well liked posts', async () => {
-    const img = await sc.uploadFile(
-      alice,
-      '../dev-env/src/seed/img/key-landscape-small.jpg',
-      'image/jpeg',
-    )
-    const one = await sc.post(alice, 'first post', undefined, [img])
-    const two = await sc.post(bob, 'bobby boi')
-    const three = await sc.reply(bob, one.ref, one.ref, 'reply')
-
-    for (let i = 0; i < 12; i++) {
-      const name = `user${i}`
-      await sc.createAccount(name, {
-        handle: `user${i}.test`,
-        email: `user${i}@test.com`,
-        password: 'password',
-      })
-      await sc.like(sc.dids[name], one.ref)
-      await sc.like(sc.dids[name], two.ref)
-      await sc.like(sc.dids[name], three.ref)
-    }
-    await network.processAll()
-
-    const res = await agent.api.app.bsky.feed.getFeed(
-      { feed: feedUri },
-      { headers: await network.serviceHeaders(alice) },
-    )
-    const feedUris = res.data.feed.map((i) => i.post.uri).sort()
-    const expected = [one.ref.uriStr, two.ref.uriStr].sort()
-    expect(feedUris).toEqual(expected)
-  })
-
-  it('paginates', async () => {
-    const res = await agent.api.app.bsky.feed.getFeed(
-      { feed: feedUri },
-      { headers: await network.serviceHeaders(alice) },
-    )
-    const first = await agent.api.app.bsky.feed.getFeed(
-      { feed: feedUri, limit: 1 },
-      { headers: await network.serviceHeaders(alice) },
-    )
-    const second = await agent.api.app.bsky.feed.getFeed(
-      { feed: feedUri, cursor: first.data.cursor },
-      { headers: await network.serviceHeaders(alice) },
-    )
-
-    expect([...first.data.feed, ...second.data.feed]).toEqual(res.data.feed)
-  })
-})

package/tests/auto-moderator/fuzzy-matcher.test.ts

@@ -1,163 +0,0 @@
-import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env'
-import { FuzzyMatcher, encode } from '../../src/auto-moderator/fuzzy-matcher'
-import { AtpAgent } from '@atproto/api'
-import { ImageInvalidator } from '../../src/image/invalidator'
-
-describe('fuzzy matcher', () => {
-  let network: TestNetwork
-  let agent: AtpAgent
-  let sc: SeedClient
-  let fuzzyMatcher: FuzzyMatcher
-
-  let alice: string
-
-  beforeAll(async () => {
-    network = await TestNetwork.create({
-      dbPostgresSchema: 'fuzzy_matcher',
-      bsky: {
-        imgInvalidator: new NoopInvalidator(),
-        indexer: {
-          fuzzyMatchB64: encode(['evil']),
-        },
-      },
-    })
-    fuzzyMatcher = new FuzzyMatcher(['evil', 'mean', 'bad'], ['baddie'])
-    agent = network.pds.getClient()
-    sc = network.getSeedClient()
-    await basicSeed(sc)
-    await network.processAll()
-    alice = sc.dids.alice
-  })
-
-  afterAll(async () => {
-    await network.close()
-  })
-
-  const getAllReports = () => {
-    return network.ozone.ctx.db.db
-      .selectFrom('moderation_event')
-      .where('action', '=', 'com.atproto.admin.defs#modEventReport')
-      .selectAll()
-      .orderBy('id', 'asc')
-      .execute()
-  }
-
-  it('identifies fuzzy matches', () => {
-    expect(fuzzyMatcher.getMatches('evil.john.test')).toMatchObject(['evil'])
-    expect(fuzzyMatcher.getMatches('john.evil.test')).toMatchObject(['evil'])
-    expect(fuzzyMatcher.getMatches('john.test.evil')).toMatchObject(['evil'])
-    expect(fuzzyMatcher.getMatches('ev1l.test.john')).toMatchObject(['evil'])
-    expect(fuzzyMatcher.getMatches('ev-1l.test.john')).toMatchObject(['evil'])
-    expect(fuzzyMatcher.getMatches('ev-11.test.john')).toMatchObject(['evil'])
-    expect(fuzzyMatcher.getMatches('ev.-1.l-test.john')).toMatchObject(['evil'])
-  })
-
-  it('identifies fuzzy false positivies', () => {
-    expect(fuzzyMatcher.getMatches('john.test')).toHaveLength(0)
-    expect(fuzzyMatcher.getMatches('good.john.test')).toHaveLength(0)
-    expect(fuzzyMatcher.getMatches('john.baddie.test')).toHaveLength(0)
-  })
-
-  it('doesnt label any of the content in the seed', async () => {
-    const reports = await getAllReports()
-    expect(reports.length).toBe(0)
-  })
-
-  it('flags a handle with an unacceptable word', async () => {
-    await sc.updateHandle(alice, 'evil.test')
-    await network.processAll()
-    const reports = await getAllReports()
-    expect(reports.length).toBe(1)
-    expect(reports.at(-1)?.subjectDid).toEqual(alice)
-  })
-
-  it('flags a profile with an unacceptable displayName', async () => {
-    const res = await agent.api.com.atproto.repo.putRecord(
-      {
-        repo: alice,
-        collection: 'app.bsky.actor.profile',
-        rkey: 'self',
-        record: {
-          displayName: 'evil alice',
-        },
-      },
-      { headers: sc.getHeaders(alice), encoding: 'application/json' },
-    )
-    await network.processAll()
-
-    const reports = await getAllReports()
-    expect(reports.length).toBe(2)
-    expect(reports.at(-1)?.subjectUri).toEqual(res.data.uri)
-    expect(reports.at(-1)?.subjectCid).toEqual(res.data.cid)
-  })
-
-  it('flags a list with an unacceptable name', async () => {
-    const res = await agent.api.com.atproto.repo.createRecord(
-      {
-        repo: alice,
-        collection: 'app.bsky.graph.list',
-        rkey: 'list',
-        record: {
-          name: 'myevillist',
-          purpose: 'app.bsky.graph.defs#modList',
-          createdAt: new Date().toISOString(),
-        },
-      },
-      { headers: sc.getHeaders(alice), encoding: 'application/json' },
-    )
-    await network.processAll()
-
-    const reports = await getAllReports()
-    expect(reports.length).toBe(3)
-    expect(reports.at(-1)?.subjectUri).toEqual(res.data.uri)
-    expect(reports.at(-1)?.subjectCid).toEqual(res.data.cid)
-  })
-
-  it('flags a feed generator with an unacceptable displayName', async () => {
-    const res = await agent.api.com.atproto.repo.createRecord(
-      {
-        repo: alice,
-        collection: 'app.bsky.feed.generator',
-        rkey: 'generator',
-        record: {
-          did: alice,
-          displayName: 'myevilfeed',
-          createdAt: new Date().toISOString(),
-        },
-      },
-      { headers: sc.getHeaders(alice), encoding: 'application/json' },
-    )
-    await network.processAll()
-
-    const reports = await getAllReports()
-    expect(reports.length).toBe(4)
-    expect(reports.at(-1)?.subjectUri).toEqual(res.data.uri)
-    expect(reports.at(-1)?.subjectCid).toEqual(res.data.cid)
-  })
-
-  it('flags a record with an unacceptable rkey', async () => {
-    const res = await agent.api.com.atproto.repo.createRecord(
-      {
-        repo: alice,
-        collection: 'app.bsky.feed.generator',
-        rkey: 'evilrkey',
-        record: {
-          did: alice,
-          displayName: 'totally fine feed',
-          createdAt: new Date().toISOString(),
-        },
-      },
-      { headers: sc.getHeaders(alice), encoding: 'application/json' },
-    )
-    await network.processAll()
-
-    const reports = await getAllReports()
-    expect(reports.length).toBe(5)
-    expect(reports.at(-1)?.subjectUri).toEqual(res.data.uri)
-    expect(reports.at(-1)?.subjectCid).toEqual(res.data.cid)
-  })
-})
-
-class NoopInvalidator implements ImageInvalidator {
-  async invalidate() {}
-}