@atproto/bsky 0.0.27 → 0.0.29

package/src/lexicon/types/{app/bsky/unspecced/getTimelineSkeleton.ts → com/atproto/temp/checkSignupQueue.ts}

@@ -7,18 +7,15 @@ import { lexicons } from '../../../../lexicons'
 import { isObj, hasProp } from '../../../../util'
 import { CID } from 'multiformats/cid'
 import { HandlerAuth } from '@atproto/xrpc-server'
-import * as AppBskyFeedDefs from '../feed/defs'
 
-export interface QueryParams {
-  limit: number
-  cursor?: string
-}
+export interface QueryParams {}
 
 export type InputSchema = undefined
 
 export interface OutputSchema {
-  cursor?: string
-  feed: AppBskyFeedDefs.SkeletonFeedPost[]
+  activated: boolean
+  placeInQueue?: number
+  estimatedTimeMs?: number
   [k: string]: unknown
 }
 
@@ -33,7 +30,6 @@ export interface HandlerSuccess {
 export interface HandlerError {
   status: number
   message?: string
-  error?: 'UnknownFeed'
 }
 
 export type HandlerOutput = HandlerError | HandlerSuccess

package/src/logger.ts

@@ -1,5 +1,8 @@
+import pino from 'pino'
 import pinoHttp from 'pino-http'
+import * as jose from 'jose'
 import { subsystemLogger } from '@atproto/common'
+import { parseBasicAuth } from './auth-verifier'
 
 export const dbLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:db')
@@ -21,5 +24,34 @@ export const loggerMiddleware = pinoHttp({
         message: err?.message,
       }
     },
+    req: (req) => {
+      const serialized = pino.stdSerializers.req(req)
+      const authHeader = serialized.headers.authorization || ''
+      let auth: string | undefined = undefined
+      if (authHeader.startsWith('Bearer ')) {
+        const token = authHeader.slice('Bearer '.length)
+        const { iss } = jose.decodeJwt(token)
+        if (iss) {
+          auth = 'Bearer ' + iss
+        } else {
+          auth = 'Bearer Invalid'
+        }
+      }
+      if (authHeader.startsWith('Basic ')) {
+        const parsed = parseBasicAuth(authHeader)
+        if (!parsed) {
+          auth = 'Basic Invalid'
+        } else {
+          auth = 'Basic ' + parsed.username
+        }
+      }
+      return {
+        ...serialized,
+        headers: {
+          ...serialized.headers,
+          authorization: auth,
+        },
+      }
+    },
   },
 })

package/tests/auto-moderator/labeler.test.ts

@@ -103,6 +103,8 @@ describe('labeler', () => {
       subject: uri.toString(),
       limit: 10,
       types: [],
+      addedLabels: [],
+      removedLabels: [],
     })
     expect(events.length).toBe(1)
     expect(events[0]).toMatchObject({

package/tests/feed-generation.test.ts

@@ -86,12 +86,6 @@ describe('feed generation', () => {
     await network.close()
   })
 
-  // @TODO enable once getFeed is implemented
-  it('describes the feed generator', async () => {
-    const res = await agent.api.app.bsky.feed.describeFeedGenerator()
-    expect(res.data.did).toBe(network.bsky.ctx.cfg.feedGenDid)
-  })
-
   it('feed gen records can be created.', async () => {
     const all = await pdsAgent.api.app.bsky.feed.generator.create(
       { repo: alice, rkey: 'all' },

package/tests/views/notifications.test.ts

@@ -296,4 +296,13 @@ describe('notification views', () => {
       ),
     )
   })
+
+  it('fails open on clearly bad cursor.', async () => {
+    const { data: notifs } =
+      await agent.api.app.bsky.notification.listNotifications(
+        { cursor: 'bad' },
+        { headers: await network.serviceHeaders(alice) },
+      )
+    expect(notifs).toEqual({ notifications: [] })
+  })
 })

package/tests/views/timeline.test.ts

@@ -298,4 +298,12 @@ describe('timeline views', () => {
       ),
     )
   })
+
+  it('fails open on clearly bad cursor.', async () => {
+    const { data: timeline } = await agent.api.app.bsky.feed.getTimeline(
+      { cursor: 'bad' },
+      { headers: await network.serviceHeaders(alice) },
+    )
+    expect(timeline).toEqual({ feed: [] })
+  })
 })

package/dist/api/app/bsky/feed/describeFeedGenerator.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/api/app/bsky/feed/getFeedSkeleton.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/api/app/bsky/unspecced/getTimelineSkeleton.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/auto-moderator/abyss.d.ts

@@ -1,48 +0,0 @@
-import { CID } from 'multiformats/cid';
-import { AtUri } from '@atproto/syntax';
-import { PrimaryDatabase } from '../db';
-import { IdResolver } from '@atproto/identity';
-export interface ImageFlagger {
-    scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>;
-}
-export declare class Abyss implements ImageFlagger {
-    endpoint: string;
-    protected password: string;
-    ctx: {
-        db: PrimaryDatabase;
-        idResolver: IdResolver;
-    };
-    protected auth: string;
-    constructor(endpoint: string, password: string, ctx: {
-        db: PrimaryDatabase;
-        idResolver: IdResolver;
-    });
-    scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>;
-    makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp>;
-    parseRes(res: ScannerResp): string[];
-    getReqUrl(params: {
-        did: string;
-        uri: string;
-    }): string;
-}
-type ScannerResp = {
-    blob: unknown;
-    match?: {
-        status: string;
-        hits: ScannerHit[];
-    };
-    classify?: {
-        hits?: unknown[];
-    };
-    review?: {
-        state?: string;
-        ticketId?: string;
-    };
-};
-type ScannerHit = {
-    hashType: string;
-    hashValue: string;
-    label: string;
-    corpus: string;
-};
-export {};

package/dist/auto-moderator/fuzzy-matcher.d.ts

@@ -1,14 +0,0 @@
-export interface TextFlagger {
-    getMatches(string: string): string[];
-}
-export declare class FuzzyMatcher implements TextFlagger {
-    private bannedWords;
-    private falsePositives;
-    constructor(bannedWords: string[], falsePositives?: string[]);
-    static fromB64(bannedB64: string, falsePositivesB64?: string): FuzzyMatcher;
-    private normalize;
-    private generatePermutations;
-    getMatches(domain: string): string[];
-}
-export declare const decode: (encoded: string) => string[];
-export declare const encode: (words: string[]) => string;

package/dist/feed-gen/bsky-team.d.ts

@@ -1,3 +0,0 @@
-import { AlgoHandler } from './types';
-declare const handler: AlgoHandler;
-export default handler;

package/dist/feed-gen/hot-classic.d.ts

@@ -1,3 +0,0 @@
-import { AlgoHandler } from './types';
-declare const handler: AlgoHandler;
-export default handler;

package/dist/feed-gen/index.d.ts

@@ -1,2 +0,0 @@
-import { MountedAlgos } from './types';
-export declare const makeAlgos: (did: string) => MountedAlgos;

package/dist/feed-gen/mutuals.d.ts

@@ -1,3 +0,0 @@
-import { AlgoHandler } from './types';
-declare const handler: AlgoHandler;
-export default handler;

package/dist/feed-gen/types.d.ts

@@ -1,15 +0,0 @@
-import AppContext from '../context';
-import { SkeletonFeedPost } from '../lexicon/types/app/bsky/feed/defs';
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton';
-import { FeedRow } from '../services/feed';
-export type AlgoResponse = {
-    feedItems: FeedRow[];
-    resHeaders?: Record<string, string>;
-    cursor?: string;
-};
-export type AlgoHandler = (ctx: AppContext, params: SkeletonParams, viewer: string | null) => Promise<AlgoResponse>;
-export type MountedAlgos = Record<string, AlgoHandler>;
-export declare const toSkeletonItem: (feedItem: {
-    uri: string;
-    postUri: string;
-}) => SkeletonFeedPost;

package/src/api/app/bsky/feed/describeFeedGenerator.ts

@@ -1,21 +0,0 @@
-import { MethodNotImplementedError } from '@atproto/xrpc-server'
-import { Server } from '../../../../lexicon'
-import AppContext from '../../../../context'
-
-export default function (server: Server, ctx: AppContext) {
-  server.app.bsky.feed.describeFeedGenerator(async () => {
-    if (!ctx.cfg.feedGenDid) {
-      throw new MethodNotImplementedError()
-    }
-
-    const feeds = Object.keys(ctx.algos).map((uri) => ({ uri }))
-
-    return {
-      encoding: 'application/json',
-      body: {
-        did: ctx.cfg.feedGenDid,
-        feeds,
-      },
-    }
-  })
-}

package/src/api/app/bsky/feed/getFeedSkeleton.ts

@@ -1,30 +0,0 @@
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { Server } from '../../../../lexicon'
-import AppContext from '../../../../context'
-import { toSkeletonItem } from '../../../../feed-gen/types'
-
-export default function (server: Server, ctx: AppContext) {
-  server.app.bsky.feed.getFeedSkeleton({
-    auth: ctx.authVerifier.standardOptional,
-    handler: async ({ params, auth }) => {
-      const { feed } = params
-      const viewer = auth.credentials.iss
-      const localAlgo = ctx.algos[feed]
-
-      if (!localAlgo) {
-        throw new InvalidRequestError('Unknown feed', 'UnknownFeed')
-      }
-
-      const result = await localAlgo(ctx, params, viewer)
-
-      return {
-        encoding: 'application/json',
-        body: {
-          // @TODO should we proactively filter blocks/mutes from the skeleton, or treat this similar to other custom feeds?
-          feed: result.feedItems.map(toSkeletonItem),
-          cursor: result.cursor,
-        },
-      }
-    },
-  })
-}

package/src/api/app/bsky/unspecced/getTimelineSkeleton.ts

@@ -1,26 +0,0 @@
-import { Server } from '../../../../lexicon'
-import AppContext from '../../../../context'
-import { skeleton } from '../feed/getTimeline'
-import { toSkeletonItem } from '../../../../feed-gen/types'
-
-// THIS IS A TEMPORARY UNSPECCED ROUTE
-export default function (server: Server, ctx: AppContext) {
-  server.app.bsky.unspecced.getTimelineSkeleton({
-    auth: ctx.authVerifier.standard,
-    handler: async ({ auth, params }) => {
-      const db = ctx.db.getReplica('timeline')
-      const feedService = ctx.services.feed(db)
-      const viewer = auth.credentials.iss
-
-      const result = await skeleton({ ...params, viewer }, { db, feedService })
-
-      return {
-        encoding: 'application/json',
-        body: {
-          feed: result.feedItems.map(toSkeletonItem),
-          cursor: result.cursor,
-        },
-      }
-    },
-  })
-}

package/src/auto-moderator/abyss.ts

@@ -1,114 +0,0 @@
-import axios from 'axios'
-import { CID } from 'multiformats/cid'
-import { AtUri } from '@atproto/syntax'
-import * as ui8 from 'uint8arrays'
-import { resolveBlob } from '../api/blob-resolver'
-import { retryHttp } from '../util/retry'
-import { PrimaryDatabase } from '../db'
-import { IdResolver } from '@atproto/identity'
-import { labelerLogger as log } from '../logger'
-
-export interface ImageFlagger {
-  scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>
-}
-
-export class Abyss implements ImageFlagger {
-  protected auth: string
-
-  constructor(
-    public endpoint: string,
-    protected password: string,
-    public ctx: { db: PrimaryDatabase; idResolver: IdResolver },
-  ) {
-    this.auth = basicAuth(this.password)
-  }
-
-  async scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]> {
-    const start = Date.now()
-    const res = await retryHttp(async () => {
-      try {
-        return await this.makeReq(did, cid, uri)
-      } catch (err) {
-        log.warn({ err, did, cid: cid.toString() }, 'abyss request failed')
-        throw err
-      }
-    })
-    log.info(
-      { res, did, cid: cid.toString(), duration: Date.now() - start },
-      'abyss response',
-    )
-    return this.parseRes(res)
-  }
-
-  async makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp> {
-    const { stream, contentType } = await resolveBlob(
-      did,
-      cid,
-      this.ctx.db,
-      this.ctx.idResolver,
-    )
-    const { data } = await axios.post(
-      this.getReqUrl({ did, uri: uri.toString() }),
-      stream,
-      {
-        headers: {
-          'Content-Type': contentType,
-          authorization: this.auth,
-        },
-        timeout: 10000,
-      },
-    )
-    return data
-  }
-
-  parseRes(res: ScannerResp): string[] {
-    if (!res.match || res.match.status !== 'success') {
-      return []
-    }
-    const labels: string[] = []
-    for (const hit of res.match.hits) {
-      if (TAKEDOWN_LABELS.includes(hit.label)) {
-        labels.push(hit.label)
-      }
-    }
-    return labels
-  }
-
-  getReqUrl(params: { did: string; uri: string }) {
-    const search = new URLSearchParams(params)
-    return `${
-      this.endpoint
-    }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}`
-  }
-}
-
-const TAKEDOWN_LABELS = ['csam', 'csem']
-
-type ScannerResp = {
-  blob: unknown
-  match?: {
-    status: string
-    hits: ScannerHit[]
-  }
-  classify?: {
-    hits?: unknown[]
-  }
-  review?: {
-    state?: string
-    ticketId?: string
-  }
-}
-
-type ScannerHit = {
-  hashType: string
-  hashValue: string
-  label: string
-  corpus: string
-}
-
-const basicAuth = (password: string) => {
-  return (
-    'Basic ' +
-    ui8.toString(ui8.fromString(`admin:${password}`, 'utf8'), 'base64pad')
-  )
-}

package/src/auto-moderator/fuzzy-matcher.ts

@@ -1,126 +0,0 @@
-import { dedupeStrs } from '@atproto/common'
-import * as ui8 from 'uint8arrays'
-
-export interface TextFlagger {
-  getMatches(string: string): string[]
-}
-
-export class FuzzyMatcher implements TextFlagger {
-  private bannedWords: Set<string>
-  private falsePositives: Set<string>
-
-  constructor(bannedWords: string[], falsePositives: string[] = []) {
-    this.bannedWords = new Set(bannedWords.map((word) => word.toLowerCase()))
-    this.falsePositives = new Set(
-      falsePositives.map((word) => word.toLowerCase()),
-    )
-  }
-
-  static fromB64(bannedB64: string, falsePositivesB64?: string) {
-    return new FuzzyMatcher(
-      decode(bannedB64),
-      falsePositivesB64 ? decode(falsePositivesB64) : undefined,
-    )
-  }
-
-  private normalize(domain: string): string[] {
-    const withoutSymbols = domain.replace(/[\W_]+/g, '') // Remove non-alphanumeric characters
-    const lowercase = withoutSymbols.toLowerCase()
-
-    // Replace common leetspeak characters
-    const leetSpeakReplacements: { [key: string]: string[] } = {
-      '0': ['o'],
-      '8': ['b'],
-      '3': ['e'],
-      '4': ['a'],
-      '6': ['g'],
-      '1': ['i', 'l'],
-      '5': ['s'],
-      '7': ['t'],
-    }
-
-    return this.generatePermutations(lowercase, leetSpeakReplacements)
-  }
-
-  private generatePermutations(
-    domain: string,
-    leetSpeakReplacements: { [key: string]: string[] },
-  ): string[] {
-    const results: string[] = []
-
-    const leetChars = Object.keys(leetSpeakReplacements)
-    const firstLeetCharIndex = [...domain].findIndex((char) =>
-      leetChars.includes(char),
-    )
-
-    if (firstLeetCharIndex === -1) {
-      // No leetspeak characters left in the string
-      results.push(domain)
-    } else {
-      const char = domain[firstLeetCharIndex]
-      const beforeChar = domain.slice(0, firstLeetCharIndex)
-      const afterChar = domain.slice(firstLeetCharIndex + 1)
-
-      // For each replacement, generate all possible combinations
-      for (const replacement of leetSpeakReplacements[char]) {
-        const replaced = beforeChar + replacement + afterChar
-
-        // Recursively generate all permutations for the rest of the string
-        const otherPermutations = this.generatePermutations(
-          replaced,
-          leetSpeakReplacements,
-        )
-
-        // Add these permutations to the results
-        results.push(...otherPermutations)
-      }
-    }
-
-    return dedupeStrs(results)
-  }
-
-  public getMatches(domain: string): string[] {
-    const normalizedDomains = this.normalize(domain)
-
-    const foundUnacceptableWords: string[] = []
-
-    for (const normalizedDomain of normalizedDomains) {
-      for (const word of this.bannedWords) {
-        const match = normalizedDomain.indexOf(word)
-        if (match > -1) {
-          let isFalsePositive = false
-          for (const falsePositive of this.falsePositives) {
-            const s_fp = falsePositive.indexOf(word)
-            const s_nd = match - s_fp
-            const wordToMatch = normalizedDomain.slice(
-              s_nd,
-              s_nd + falsePositive.length,
-            )
-            if (wordToMatch === falsePositive) {
-              isFalsePositive = true
-              break
-            }
-          }
-
-          if (!isFalsePositive) {
-            foundUnacceptableWords.push(word)
-          }
-        }
-      }
-    }
-
-    if (foundUnacceptableWords.length > 0) {
-      return foundUnacceptableWords
-    }
-
-    return []
-  }
-}
-
-export const decode = (encoded: string): string[] => {
-  return ui8.toString(ui8.fromString(encoded, 'base64'), 'utf8').split(',')
-}
-
-export const encode = (words: string[]): string => {
-  return ui8.toString(ui8.fromString(words.join(','), 'utf8'), 'base64')
-}

package/src/feed-gen/bsky-team.ts

@@ -1,42 +0,0 @@
-import { NotEmptyArray } from '@atproto/common'
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
-import AppContext from '../context'
-import { paginate } from '../db/pagination'
-import { AlgoHandler, AlgoResponse } from './types'
-import { FeedKeyset } from '../api/app/bsky/util/feed'
-
-const BSKY_TEAM: NotEmptyArray<string> = [
-  'did:plc:z72i7hdynmk6r22z27h6tvur', // @bsky.app
-  'did:plc:ewvi7nxzyoun6zhxrhs64oiz', // @atproto.com
-  'did:plc:eon2iu7v3x2ukgxkqaf7e5np', // @safety.bsky.app
-]
-
-const handler: AlgoHandler = async (
-  ctx: AppContext,
-  params: SkeletonParams,
-  _viewer: string | null,
-): Promise<AlgoResponse> => {
-  const { limit = 50, cursor } = params
-  const db = ctx.db.getReplica('feed')
-  const feedService = ctx.services.feed(db)
-
-  const { ref } = db.db.dynamic
-
-  const postsQb = feedService
-    .selectPostQb()
-    .where('post.creator', 'in', BSKY_TEAM)
-
-  const keyset = new FeedKeyset(ref('sortAt'), ref('cid'))
-
-  let feedQb = db.db.selectFrom(postsQb.as('feed_items')).selectAll()
-  feedQb = paginate(feedQb, { limit, cursor, keyset })
-
-  const feedItems = await feedQb.execute()
-
-  return {
-    feedItems,
-    cursor: keyset.packFromResult(feedItems),
-  }
-}
-
-export default handler

package/src/feed-gen/hot-classic.ts

@@ -1,55 +0,0 @@
-import AppContext from '../context'
-import { NotEmptyArray } from '@atproto/common'
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
-import { paginate } from '../db/pagination'
-import { AlgoHandler, AlgoResponse } from './types'
-import { FeedKeyset } from '../api/app/bsky/util/feed'
-import { valuesList } from '../db/util'
-
-const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = ['!no-promote']
-
-const handler: AlgoHandler = async (
-  ctx: AppContext,
-  params: SkeletonParams,
-  _viewer: string | null,
-): Promise<AlgoResponse> => {
-  const { limit = 50, cursor } = params
-  const db = ctx.db.getReplica('feed')
-  const feedService = ctx.services.feed(db)
-
-  const { ref } = db.db.dynamic
-
-  const postsQb = feedService
-    .selectPostQb()
-    .leftJoin('post_agg', 'post_agg.uri', 'post.uri')
-    .leftJoin('post_embed_record', 'post_embed_record.postUri', 'post.uri')
-    .where('post_agg.likeCount', '>=', 12)
-    .where('post.replyParent', 'is', null)
-    .whereNotExists((qb) =>
-      qb
-        .selectFrom('label')
-        .selectAll()
-        .whereRef('val', 'in', valuesList(NO_WHATS_HOT_LABELS))
-        .where('neg', '=', false)
-        .where((clause) =>
-          clause
-            .whereRef('label.uri', '=', ref('post.creator'))
-            .orWhereRef('label.uri', '=', ref('post.uri'))
-            .orWhereRef('label.uri', '=', ref('post_embed_record.embedUri')),
-        ),
-    )
-
-  const keyset = new FeedKeyset(ref('sortAt'), ref('cid'))
-
-  let feedQb = db.db.selectFrom(postsQb.as('feed_items')).selectAll()
-  feedQb = paginate(feedQb, { limit, cursor, keyset })
-
-  const feedItems = await feedQb.execute()
-
-  return {
-    feedItems,
-    cursor: keyset.packFromResult(feedItems),
-  }
-}
-
-export default handler

package/src/feed-gen/index.ts

@@ -1,17 +0,0 @@
-import { AtUri } from '@atproto/syntax'
-import { ids } from '../lexicon/lexicons'
-import bskyTeam from './bsky-team'
-import hotClassic from './hot-classic'
-import mutuals from './mutuals'
-import { MountedAlgos } from './types'
-
-const feedgenUri = (did, name) =>
-  AtUri.make(did, ids.AppBskyFeedGenerator, name).toString()
-
-// These are custom algorithms that will be mounted directly onto an AppView
-// Feel free to remove, update to your own, or serve the following logic at a record that you control
-export const makeAlgos = (did: string): MountedAlgos => ({
-  [feedgenUri(did, 'bsky-team')]: bskyTeam,
-  [feedgenUri(did, 'hot-classic')]: hotClassic,
-  [feedgenUri(did, 'mutuals')]: mutuals,
-})