@atproto/bsky 0.0.28 → 0.0.30

package/src/lexicon/types/com/atproto/temp/transferAccount.ts

@@ -6,7 +6,7 @@ import { ValidationResult, BlobRef } from '@atproto/lexicon'
 import { lexicons } from '../../../../lexicons'
 import { isObj, hasProp } from '../../../../util'
 import { CID } from 'multiformats/cid'
-import { HandlerAuth } from '@atproto/xrpc-server'
+import { HandlerAuth, HandlerPipeThrough } from '@atproto/xrpc-server'
 
 export interface QueryParams {}
 
@@ -49,7 +49,7 @@ export interface HandlerError {
     | 'IncompatibleDidDoc'
 }
 
-export type HandlerOutput = HandlerError | HandlerSuccess
+export type HandlerOutput = HandlerError | HandlerSuccess | HandlerPipeThrough
 export type HandlerReqCtx<HA extends HandlerAuth = never> = {
   auth: HA
   params: QueryParams

package/src/logger.ts

@@ -1,5 +1,8 @@
+import pino from 'pino'
 import pinoHttp from 'pino-http'
+import * as jose from 'jose'
 import { subsystemLogger } from '@atproto/common'
+import { parseBasicAuth } from './auth-verifier'
 
 export const dbLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:db')
@@ -21,5 +24,34 @@ export const loggerMiddleware = pinoHttp({
         message: err?.message,
       }
     },
+    req: (req) => {
+      const serialized = pino.stdSerializers.req(req)
+      const authHeader = serialized.headers.authorization || ''
+      let auth: string | undefined = undefined
+      if (authHeader.startsWith('Bearer ')) {
+        const token = authHeader.slice('Bearer '.length)
+        const { iss } = jose.decodeJwt(token)
+        if (iss) {
+          auth = 'Bearer ' + iss
+        } else {
+          auth = 'Bearer Invalid'
+        }
+      }
+      if (authHeader.startsWith('Basic ')) {
+        const parsed = parseBasicAuth(authHeader)
+        if (!parsed) {
+          auth = 'Basic Invalid'
+        } else {
+          auth = 'Basic ' + parsed.username
+        }
+      }
+      return {
+        ...serialized,
+        headers: {
+          ...serialized.headers,
+          authorization: auth,
+        },
+      }
+    },
   },
 })

package/src/services/feed/index.ts

@@ -139,6 +139,7 @@ export class FeedService {
   async getPostInfos(
     postUris: string[],
     viewer: string | null,
+    includeSoftDeleted?: boolean,
   ): Promise<PostInfoMap> {
     if (postUris.length < 1) return {}
     const db = this.db.db
@@ -149,8 +150,12 @@ export class FeedService {
       .innerJoin('actor', 'actor.did', 'post.creator')
       .innerJoin('record', 'record.uri', 'post.uri')
       .leftJoin('post_agg', 'post_agg.uri', 'post.uri')
-      .where(notSoftDeletedClause(ref('actor'))) // Ensures post reply parent/roots get omitted from views when taken down
-      .where(notSoftDeletedClause(ref('record')))
+      .if(!includeSoftDeleted, (qb) =>
+        qb.where(notSoftDeletedClause(ref('actor'))),
+      ) // Ensures post reply parent/roots get omitted from views when taken down
+      .if(!includeSoftDeleted, (qb) =>
+        qb.where(notSoftDeletedClause(ref('record'))),
+      )
       .select([
         'post.uri as uri',
         'post.cid as cid',
@@ -249,12 +254,13 @@ export class FeedService {
       dids: Set<string>
       uris: Set<string>
       viewer: string | null
+      includeSoftDeleted?: boolean
     },
     depth = 0,
   ): Promise<FeedHydrationState> {
     const { viewer, dids, uris } = refs
     const [posts, threadgates, labels, bam] = await Promise.all([
-      this.getPostInfos(Array.from(uris), viewer),
+      this.getPostInfos(Array.from(uris), viewer, refs.includeSoftDeleted),
       this.threadgatesByPostUri(Array.from(uris)),
       this.services.label.getLabelsForSubjects([...uris, ...dids]),
       this.services.graph.getBlockAndMuteState(
@@ -266,7 +272,7 @@ export class FeedService {
     const [profileState, blocks, lists] = await Promise.all([
       this.services.actor.views.profileHydration(
         Array.from(dids),
-        { viewer },
+        { viewer, includeSoftDeleted: refs.includeSoftDeleted },
         { bam, labels },
       ),
       this.blocksForPosts(posts, bam),

package/src/services/feed/views.ts

@@ -208,7 +208,9 @@ export class FeedViews {
     const post = posts[uri]
     const gate = threadgates[uri]
     const author = actors[post?.creator]
-    if (!post || !author) return undefined
+    if (!post || !author) {
+      return undefined
+    }
     const postLabels = labels[uri] ?? []
     const postSelfLabels = getSelfLabels({
       uri: post.uri,

package/tests/auto-moderator/labeler.test.ts

@@ -103,6 +103,8 @@ describe('labeler', () => {
       subject: uri.toString(),
       limit: 10,
       types: [],
+      addedLabels: [],
+      removedLabels: [],
     })
     expect(events.length).toBe(1)
     expect(events[0]).toMatchObject({

package/tests/feed-generation.test.ts

@@ -86,12 +86,6 @@ describe('feed generation', () => {
     await network.close()
   })
 
-  // @TODO enable once getFeed is implemented
-  it('describes the feed generator', async () => {
-    const res = await agent.api.app.bsky.feed.describeFeedGenerator()
-    expect(res.data.did).toBe(network.bsky.ctx.cfg.feedGenDid)
-  })
-
   it('feed gen records can be created.', async () => {
     const all = await pdsAgent.api.app.bsky.feed.generator.create(
       { repo: alice, rkey: 'all' },

package/tests/views/author-feed.test.ts

@@ -138,7 +138,7 @@ describe('pds author feed views', () => {
     )
   })
 
-  it('blocked by actor takedown.', async () => {
+  it('non-admins blocked by actor takedown.', async () => {
     const { data: preBlock } = await agent.api.app.bsky.feed.getAuthorFeed(
       { actor: alice },
       { headers: await network.serviceHeaders(carol) },
@@ -163,11 +163,18 @@ describe('pds author feed views', () => {
       },
     )
 
-    const attempt = agent.api.app.bsky.feed.getAuthorFeed(
+    const attemptAsUser = agent.api.app.bsky.feed.getAuthorFeed(
       { actor: alice },
       { headers: await network.serviceHeaders(carol) },
     )
-    await expect(attempt).rejects.toThrow('Profile not found')
+    await expect(attemptAsUser).rejects.toThrow('Profile not found')
+
+    const attemptAsAdmin = await agent.api.app.bsky.feed.getAuthorFeed(
+      { actor: alice },
+      { headers: network.bsky.adminAuthHeaders() },
+    )
+    expect(attemptAsAdmin.data.feed.length).toBeGreaterThan(0)
+    expect(attemptAsAdmin.data.feed.length).toEqual(preBlock.feed.length)
 
     // Cleanup
     await agent.api.com.atproto.admin.updateSubjectStatus(
@@ -215,13 +222,26 @@ describe('pds author feed views', () => {
       },
     )
 
-    const { data: postBlock } = await agent.api.app.bsky.feed.getAuthorFeed(
-      { actor: alice },
-      { headers: await network.serviceHeaders(carol) },
+    const [{ data: postBlockAsUser }, { data: postBlockAsAdmin }] =
+      await Promise.all([
+        agent.api.app.bsky.feed.getAuthorFeed(
+          { actor: alice },
+          { headers: await network.serviceHeaders(carol) },
+        ),
+        agent.api.app.bsky.feed.getAuthorFeed(
+          { actor: alice },
+          { headers: network.bsky.adminAuthHeaders() },
+        ),
+      ])
+
+    expect(postBlockAsUser.feed.length).toEqual(preBlock.feed.length - 1)
+    expect(postBlockAsUser.feed.map((item) => item.post.uri)).not.toContain(
+      post.uri,
+    )
+    expect(postBlockAsAdmin.feed.length).toEqual(preBlock.feed.length)
+    expect(postBlockAsAdmin.feed.map((item) => item.post.uri)).toContain(
+      post.uri,
     )
-
-    expect(postBlock.feed.length).toEqual(preBlock.feed.length - 1)
-    expect(postBlock.feed.map((item) => item.post.uri)).not.toContain(post.uri)
 
     // Cleanup
     await agent.api.com.atproto.admin.updateSubjectStatus(

package/tests/views/notifications.test.ts

@@ -296,4 +296,13 @@ describe('notification views', () => {
       ),
     )
   })
+
+  it('fails open on clearly bad cursor.', async () => {
+    const { data: notifs } =
+      await agent.api.app.bsky.notification.listNotifications(
+        { cursor: 'bad' },
+        { headers: await network.serviceHeaders(alice) },
+      )
+    expect(notifs).toEqual({ notifications: [] })
+  })
 })

package/tests/views/timeline.test.ts

@@ -298,4 +298,12 @@ describe('timeline views', () => {
       ),
     )
   })
+
+  it('fails open on clearly bad cursor.', async () => {
+    const { data: timeline } = await agent.api.app.bsky.feed.getTimeline(
+      { cursor: 'bad' },
+      { headers: await network.serviceHeaders(alice) },
+    )
+    expect(timeline).toEqual({ feed: [] })
+  })
 })

package/dist/api/app/bsky/feed/describeFeedGenerator.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/api/app/bsky/feed/getFeedSkeleton.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/api/app/bsky/unspecced/getTimelineSkeleton.d.ts

@@ -1,3 +0,0 @@
-import { Server } from '../../../../lexicon';
-import AppContext from '../../../../context';
-export default function (server: Server, ctx: AppContext): void;

package/dist/auto-moderator/abyss.d.ts

@@ -1,48 +0,0 @@
-import { CID } from 'multiformats/cid';
-import { AtUri } from '@atproto/syntax';
-import { PrimaryDatabase } from '../db';
-import { IdResolver } from '@atproto/identity';
-export interface ImageFlagger {
-    scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>;
-}
-export declare class Abyss implements ImageFlagger {
-    endpoint: string;
-    protected password: string;
-    ctx: {
-        db: PrimaryDatabase;
-        idResolver: IdResolver;
-    };
-    protected auth: string;
-    constructor(endpoint: string, password: string, ctx: {
-        db: PrimaryDatabase;
-        idResolver: IdResolver;
-    });
-    scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>;
-    makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp>;
-    parseRes(res: ScannerResp): string[];
-    getReqUrl(params: {
-        did: string;
-        uri: string;
-    }): string;
-}
-type ScannerResp = {
-    blob: unknown;
-    match?: {
-        status: string;
-        hits: ScannerHit[];
-    };
-    classify?: {
-        hits?: unknown[];
-    };
-    review?: {
-        state?: string;
-        ticketId?: string;
-    };
-};
-type ScannerHit = {
-    hashType: string;
-    hashValue: string;
-    label: string;
-    corpus: string;
-};
-export {};

package/dist/auto-moderator/fuzzy-matcher.d.ts

@@ -1,14 +0,0 @@
-export interface TextFlagger {
-    getMatches(string: string): string[];
-}
-export declare class FuzzyMatcher implements TextFlagger {
-    private bannedWords;
-    private falsePositives;
-    constructor(bannedWords: string[], falsePositives?: string[]);
-    static fromB64(bannedB64: string, falsePositivesB64?: string): FuzzyMatcher;
-    private normalize;
-    private generatePermutations;
-    getMatches(domain: string): string[];
-}
-export declare const decode: (encoded: string) => string[];
-export declare const encode: (words: string[]) => string;

package/dist/feed-gen/bsky-team.d.ts

@@ -1,3 +0,0 @@
-import { AlgoHandler } from './types';
-declare const handler: AlgoHandler;
-export default handler;

package/dist/feed-gen/hot-classic.d.ts

@@ -1,3 +0,0 @@
-import { AlgoHandler } from './types';
-declare const handler: AlgoHandler;
-export default handler;

package/dist/feed-gen/index.d.ts

@@ -1,2 +0,0 @@
-import { MountedAlgos } from './types';
-export declare const makeAlgos: (did: string) => MountedAlgos;

package/dist/feed-gen/mutuals.d.ts

@@ -1,3 +0,0 @@
-import { AlgoHandler } from './types';
-declare const handler: AlgoHandler;
-export default handler;

package/dist/feed-gen/types.d.ts

@@ -1,15 +0,0 @@
-import AppContext from '../context';
-import { SkeletonFeedPost } from '../lexicon/types/app/bsky/feed/defs';
-import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton';
-import { FeedRow } from '../services/feed';
-export type AlgoResponse = {
-    feedItems: FeedRow[];
-    resHeaders?: Record<string, string>;
-    cursor?: string;
-};
-export type AlgoHandler = (ctx: AppContext, params: SkeletonParams, viewer: string | null) => Promise<AlgoResponse>;
-export type MountedAlgos = Record<string, AlgoHandler>;
-export declare const toSkeletonItem: (feedItem: {
-    uri: string;
-    postUri: string;
-}) => SkeletonFeedPost;

package/dist/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.d.ts

@@ -1,35 +0,0 @@
-import express from 'express';
-import { HandlerAuth } from '@atproto/xrpc-server';
-import * as AppBskyFeedDefs from '../feed/defs';
-export interface QueryParams {
-    limit: number;
-    cursor?: string;
-}
-export type InputSchema = undefined;
-export interface OutputSchema {
-    cursor?: string;
-    feed: AppBskyFeedDefs.SkeletonFeedPost[];
-    [k: string]: unknown;
-}
-export type HandlerInput = undefined;
-export interface HandlerSuccess {
-    encoding: 'application/json';
-    body: OutputSchema;
-    headers?: {
-        [key: string]: string;
-    };
-}
-export interface HandlerError {
-    status: number;
-    message?: string;
-    error?: 'UnknownFeed';
-}
-export type HandlerOutput = HandlerError | HandlerSuccess;
-export type HandlerReqCtx<HA extends HandlerAuth = never> = {
-    auth: HA;
-    params: QueryParams;
-    input: HandlerInput;
-    req: express.Request;
-    res: express.Response;
-};
-export type Handler<HA extends HandlerAuth = never> = (ctx: HandlerReqCtx<HA>) => Promise<HandlerOutput> | HandlerOutput;

package/src/api/app/bsky/feed/describeFeedGenerator.ts

@@ -1,21 +0,0 @@
-import { MethodNotImplementedError } from '@atproto/xrpc-server'
-import { Server } from '../../../../lexicon'
-import AppContext from '../../../../context'
-
-export default function (server: Server, ctx: AppContext) {
-  server.app.bsky.feed.describeFeedGenerator(async () => {
-    if (!ctx.cfg.feedGenDid) {
-      throw new MethodNotImplementedError()
-    }
-
-    const feeds = Object.keys(ctx.algos).map((uri) => ({ uri }))
-
-    return {
-      encoding: 'application/json',
-      body: {
-        did: ctx.cfg.feedGenDid,
-        feeds,
-      },
-    }
-  })
-}

package/src/api/app/bsky/feed/getFeedSkeleton.ts

@@ -1,30 +0,0 @@
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { Server } from '../../../../lexicon'
-import AppContext from '../../../../context'
-import { toSkeletonItem } from '../../../../feed-gen/types'
-
-export default function (server: Server, ctx: AppContext) {
-  server.app.bsky.feed.getFeedSkeleton({
-    auth: ctx.authVerifier.standardOptional,
-    handler: async ({ params, auth }) => {
-      const { feed } = params
-      const viewer = auth.credentials.iss
-      const localAlgo = ctx.algos[feed]
-
-      if (!localAlgo) {
-        throw new InvalidRequestError('Unknown feed', 'UnknownFeed')
-      }
-
-      const result = await localAlgo(ctx, params, viewer)
-
-      return {
-        encoding: 'application/json',
-        body: {
-          // @TODO should we proactively filter blocks/mutes from the skeleton, or treat this similar to other custom feeds?
-          feed: result.feedItems.map(toSkeletonItem),
-          cursor: result.cursor,
-        },
-      }
-    },
-  })
-}

package/src/api/app/bsky/unspecced/getTimelineSkeleton.ts

@@ -1,26 +0,0 @@
-import { Server } from '../../../../lexicon'
-import AppContext from '../../../../context'
-import { skeleton } from '../feed/getTimeline'
-import { toSkeletonItem } from '../../../../feed-gen/types'
-
-// THIS IS A TEMPORARY UNSPECCED ROUTE
-export default function (server: Server, ctx: AppContext) {
-  server.app.bsky.unspecced.getTimelineSkeleton({
-    auth: ctx.authVerifier.standard,
-    handler: async ({ auth, params }) => {
-      const db = ctx.db.getReplica('timeline')
-      const feedService = ctx.services.feed(db)
-      const viewer = auth.credentials.iss
-
-      const result = await skeleton({ ...params, viewer }, { db, feedService })
-
-      return {
-        encoding: 'application/json',
-        body: {
-          feed: result.feedItems.map(toSkeletonItem),
-          cursor: result.cursor,
-        },
-      }
-    },
-  })
-}

package/src/auto-moderator/abyss.ts

@@ -1,114 +0,0 @@
-import axios from 'axios'
-import { CID } from 'multiformats/cid'
-import { AtUri } from '@atproto/syntax'
-import * as ui8 from 'uint8arrays'
-import { resolveBlob } from '../api/blob-resolver'
-import { retryHttp } from '../util/retry'
-import { PrimaryDatabase } from '../db'
-import { IdResolver } from '@atproto/identity'
-import { labelerLogger as log } from '../logger'
-
-export interface ImageFlagger {
-  scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>
-}
-
-export class Abyss implements ImageFlagger {
-  protected auth: string
-
-  constructor(
-    public endpoint: string,
-    protected password: string,
-    public ctx: { db: PrimaryDatabase; idResolver: IdResolver },
-  ) {
-    this.auth = basicAuth(this.password)
-  }
-
-  async scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]> {
-    const start = Date.now()
-    const res = await retryHttp(async () => {
-      try {
-        return await this.makeReq(did, cid, uri)
-      } catch (err) {
-        log.warn({ err, did, cid: cid.toString() }, 'abyss request failed')
-        throw err
-      }
-    })
-    log.info(
-      { res, did, cid: cid.toString(), duration: Date.now() - start },
-      'abyss response',
-    )
-    return this.parseRes(res)
-  }
-
-  async makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp> {
-    const { stream, contentType } = await resolveBlob(
-      did,
-      cid,
-      this.ctx.db,
-      this.ctx.idResolver,
-    )
-    const { data } = await axios.post(
-      this.getReqUrl({ did, uri: uri.toString() }),
-      stream,
-      {
-        headers: {
-          'Content-Type': contentType,
-          authorization: this.auth,
-        },
-        timeout: 10000,
-      },
-    )
-    return data
-  }
-
-  parseRes(res: ScannerResp): string[] {
-    if (!res.match || res.match.status !== 'success') {
-      return []
-    }
-    const labels: string[] = []
-    for (const hit of res.match.hits) {
-      if (TAKEDOWN_LABELS.includes(hit.label)) {
-        labels.push(hit.label)
-      }
-    }
-    return labels
-  }
-
-  getReqUrl(params: { did: string; uri: string }) {
-    const search = new URLSearchParams(params)
-    return `${
-      this.endpoint
-    }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}`
-  }
-}
-
-const TAKEDOWN_LABELS = ['csam', 'csem']
-
-type ScannerResp = {
-  blob: unknown
-  match?: {
-    status: string
-    hits: ScannerHit[]
-  }
-  classify?: {
-    hits?: unknown[]
-  }
-  review?: {
-    state?: string
-    ticketId?: string
-  }
-}
-
-type ScannerHit = {
-  hashType: string
-  hashValue: string
-  label: string
-  corpus: string
-}
-
-const basicAuth = (password: string) => {
-  return (
-    'Basic ' +
-    ui8.toString(ui8.fromString(`admin:${password}`, 'utf8'), 'base64pad')
-  )
-}