@atproto/bsky 0.0.11 → 0.0.13

package/src/api/com/atproto/admin/takeModerationAction.ts

@@ -1,6 +1,10 @@
 import { CID } from 'multiformats/cid'
 import { AtUri } from '@atproto/syntax'
-import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
+import {
+  AuthRequiredError,
+  InvalidRequestError,
+  UpstreamFailureError,
+} from '@atproto/xrpc-server'
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
 import {
@@ -9,6 +13,8 @@ import {
   TAKEDOWN,
 } from '../../../../lexicon/types/com/atproto/admin/defs'
 import { getSubject, getAction } from '../moderation/util'
+import { TakedownSubjects } from '../../../../services/moderation'
+import { retryHttp } from '../../../../util/retry'
 
 export default function (server: Server, ctx: AppContext) {
   server.com.atproto.admin.takeModerationAction({
@@ -52,7 +58,7 @@ export default function (server: Server, ctx: AppContext) {
 
       validateLabels([...(createLabelVals ?? []), ...(negateLabelVals ?? [])])
 
-      const moderationAction = await db.transaction(async (dbTxn) => {
+      const { result, takenDown } = await db.transaction(async (dbTxn) => {
         const moderationTxn = ctx.services.moderation(dbTxn)
         const labelTxn = ctx.services.label(dbTxn)
 
@@ -67,13 +73,15 @@ export default function (server: Server, ctx: AppContext) {
           durationInHours,
         })
 
+        let takenDown: TakedownSubjects | undefined
+
         if (
           result.action === TAKEDOWN &&
           result.subjectType === 'com.atproto.admin.defs#repoRef' &&
           result.subjectDid
         ) {
           // No credentials to revoke on appview
-          await moderationTxn.takedownRepo({
+          takenDown = await moderationTxn.takedownRepo({
             takedownId: result.id,
             did: result.subjectDid,
           })
@@ -82,11 +90,13 @@ export default function (server: Server, ctx: AppContext) {
         if (
           result.action === TAKEDOWN &&
           result.subjectType === 'com.atproto.repo.strongRef' &&
-          result.subjectUri
+          result.subjectUri &&
+          result.subjectCid
         ) {
-          await moderationTxn.takedownRecord({
+          takenDown = await moderationTxn.takedownRecord({
             takedownId: result.id,
             uri: new AtUri(result.subjectUri),
+            cid: CID.parse(result.subjectCid),
             blobCids: subjectBlobCids?.map((cid) => CID.parse(cid)) ?? [],
           })
         }
@@ -98,12 +108,36 @@ export default function (server: Server, ctx: AppContext) {
           { create: createLabelVals, negate: negateLabelVals },
         )
 
-        return result
+        return { result, takenDown }
       })
 
+      if (takenDown) {
+        const { did, subjects } = takenDown
+        if (did && subjects.length > 0) {
+          const agent = await ctx.pdsAdminAgent(did)
+          const results = await Promise.allSettled(
+            subjects.map((subject) =>
+              retryHttp(() =>
+                agent.api.com.atproto.admin.updateSubjectStatus({
+                  subject,
+                  takedown: {
+                    applied: true,
+                    ref: result.id.toString(),
+                  },
+                }),
+              ),
+            ),
+          )
+          const hadFailure = results.some((r) => r.status === 'rejected')
+          if (hadFailure) {
+            throw new UpstreamFailureError('failed to apply action on PDS')
+          }
+        }
+      }
+
       return {
         encoding: 'application/json',
-        body: await moderationService.views.action(moderationAction),
+        body: await moderationService.views.action(result),
       }
     },
   })

package/src/api/com/atproto/admin/util.ts

@@ -0,0 +1,50 @@
+import AppContext from '../../../../context'
+import {
+  RepoView,
+  RepoViewDetail,
+  AccountView,
+} from '../../../../lexicon/types/com/atproto/admin/defs'
+
+export const getPdsAccountInfo = async (
+  ctx: AppContext,
+  did: string,
+): Promise<AccountView | null> => {
+  try {
+    const agent = await ctx.pdsAdminAgent(did)
+    const res = await agent.api.com.atproto.admin.getAccountInfo({ did })
+    return res.data
+  } catch (err) {
+    return null
+  }
+}
+
+export const addAccountInfoToRepoViewDetail = (
+  repoView: RepoViewDetail,
+  accountInfo: AccountView | null,
+  includeEmail = false,
+): RepoViewDetail => {
+  if (!accountInfo) return repoView
+  return {
+    ...repoView,
+    email: includeEmail ? accountInfo.email : undefined,
+    invitedBy: accountInfo.invitedBy,
+    invitesDisabled: accountInfo.invitesDisabled,
+    inviteNote: accountInfo.inviteNote,
+    invites: accountInfo.invites,
+  }
+}
+
+export const addAccountInfoToRepoView = (
+  repoView: RepoView,
+  accountInfo: AccountView | null,
+  includeEmail = false,
+): RepoView => {
+  if (!accountInfo) return repoView
+  return {
+    ...repoView,
+    email: includeEmail ? accountInfo.email : undefined,
+    invitedBy: accountInfo.invitedBy,
+    invitesDisabled: accountInfo.invitesDisabled,
+    inviteNote: accountInfo.inviteNote,
+  }
+}

package/src/api/well-known.ts

@@ -12,6 +12,14 @@ export const createRouter = (ctx: AppContext): express.Router => {
     res.json({
       '@context': ['https://www.w3.org/ns/did/v1'],
       id: ctx.cfg.serverDid,
+      verificationMethod: [
+        {
+          id: `${ctx.cfg.serverDid}#atproto`,
+          type: 'Multikey',
+          controller: ctx.cfg.serverDid,
+          publicKeyMultibase: ctx.signingKey.did().replace('did:key:', ''),
+        },
+      ],
       service: [
         {
           id: '#bsky_notif',

package/src/auth.ts

@@ -14,11 +14,18 @@ export const authVerifier =
     if (!jwtStr) {
       throw new AuthRequiredError('missing jwt', 'MissingJwt')
     }
-    const did = await verifyJwt(jwtStr, opts.aud, async (did: string) => {
-      const atprotoData = await idResolver.did.resolveAtprotoData(did)
-      return atprotoData.signingKey
-    })
-    return { credentials: { did }, artifacts: { aud: opts.aud } }
+    const payload = await verifyJwt(
+      jwtStr,
+      opts.aud,
+      async (did, forceRefresh) => {
+        const atprotoData = await idResolver.did.resolveAtprotoData(
+          did,
+          forceRefresh,
+        )
+        return atprotoData.signingKey
+      },
+    )
+    return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } }
   }
 
 export const authOptionalVerifier =

package/src/auto-moderator/index.ts

@@ -271,6 +271,7 @@ export class AutoModerator {
         await modSrvc.takedownRecord({
           takedownId: action.id,
           uri: uri,
+          cid: recordCid,
           blobCids: takedownCids,
         })
       })

package/src/context.ts

@@ -1,5 +1,8 @@
 import * as plc from '@did-plc/lib'
 import { IdResolver } from '@atproto/identity'
+import { AtpAgent } from '@atproto/api'
+import { Keypair } from '@atproto/crypto'
+import { createServiceJwt } from '@atproto/xrpc-server'
 import { DatabaseCoordinator } from './db'
 import { ServerConfig } from './config'
 import { ImageUriBuilder } from './image/uri'
@@ -10,7 +13,6 @@ import { BackgroundQueue } from './background'
 import { MountedAlgos } from './feed-gen/types'
 import { LabelCache } from './label-cache'
 import { NotificationServer } from './notifications'
-import { AtpAgent } from '@atproto/api'
 
 export class AppContext {
   constructor(
@@ -19,6 +21,7 @@ export class AppContext {
       imgUriBuilder: ImageUriBuilder
       cfg: ServerConfig
       services: Services
+      signingKey: Keypair
       idResolver: IdResolver
       didCache: DidSqlCache
       labelCache: LabelCache
@@ -45,6 +48,10 @@ export class AppContext {
     return this.opts.services
   }
 
+  get signingKey(): Keypair {
+    return this.opts.signingKey
+  }
+
   get plcClient(): plc.Client {
     return new plc.Client(this.cfg.didPlcUrl)
   }
@@ -91,6 +98,23 @@ export class AppContext {
     return auth.roleVerifier(this.cfg)
   }
 
+  async serviceAuthJwt(aud: string) {
+    const iss = this.cfg.serverDid
+    return createServiceJwt({
+      iss,
+      aud,
+      keypair: this.signingKey,
+    })
+  }
+
+  async pdsAdminAgent(did: string): Promise<AtpAgent> {
+    const data = await this.idResolver.did.resolveAtprotoData(did)
+    const agent = new AtpAgent({ service: data.pds })
+    const jwt = await this.serviceAuthJwt(did)
+    agent.api.setHeader('authorization', `Bearer ${jwt}`)
+    return agent
+  }
+
   get backgroundQueue(): BackgroundQueue {
     return this.opts.backgroundQueue
   }

package/src/db/migrations/20230929T192920807Z-record-cursor-indexes.ts

@@ -0,0 +1,40 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createIndex('like_creator_cursor_idx')
+    .on('like')
+    .columns(['creator', 'sortAt', 'cid'])
+    .execute()
+  await db.schema
+    .createIndex('follow_creator_cursor_idx')
+    .on('follow')
+    .columns(['creator', 'sortAt', 'cid'])
+    .execute()
+  await db.schema
+    .createIndex('follow_subject_cursor_idx')
+    .on('follow')
+    .columns(['subjectDid', 'sortAt', 'cid'])
+    .execute()
+
+  // drop old indices that are superceded by these
+  await db.schema.dropIndex('like_creator_idx').execute()
+  await db.schema.dropIndex('follow_subjectdid_idx').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createIndex('like_creator_idx')
+    .on('like')
+    .column('creator')
+    .execute()
+  await db.schema
+    .createIndex('follow_subjectdid_idx')
+    .on('follow')
+    .column('subjectDid')
+    .execute()
+
+  await db.schema.dropIndex('like_creator_cursor_idx').execute()
+  await db.schema.dropIndex('follow_creator_cursor_idx').execute()
+  await db.schema.dropIndex('follow_subject_cursor_idx').execute()
+}

package/src/db/migrations/index.ts

@@ -29,3 +29,4 @@ export * as _20230830T205507322Z from './20230830T205507322Z-suggested-feeds'
 export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
 export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
 export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
+export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes'

package/src/did-cache.ts

@@ -1,6 +1,7 @@
 import PQueue from 'p-queue'
 import { CacheResult, DidCache, DidDocument } from '@atproto/identity'
 import { PrimaryDatabase } from './db'
+import { excluded } from './db/util'
 import { dbLogger } from './logger'
 
 export class DidSqlCache implements DidCache {
@@ -16,25 +17,42 @@ export class DidSqlCache implements DidCache {
     this.pQueue = new PQueue()
   }
 
-  async cacheDid(did: string, doc: DidDocument): Promise<void> {
-    await this.db.db
-      .insertInto('did_cache')
-      .values({ did, doc, updatedAt: Date.now() })
-      .onConflict((oc) =>
-        oc.column('did').doUpdateSet({ doc, updatedAt: Date.now() }),
-      )
-      .executeTakeFirst()
+  async cacheDid(
+    did: string,
+    doc: DidDocument,
+    prevResult?: CacheResult,
+  ): Promise<void> {
+    if (prevResult) {
+      await this.db.db
+        .updateTable('did_cache')
+        .set({ doc, updatedAt: Date.now() })
+        .where('did', '=', did)
+        .where('updatedAt', '=', prevResult.updatedAt)
+        .execute()
+    } else {
+      await this.db.db
+        .insertInto('did_cache')
+        .values({ did, doc, updatedAt: Date.now() })
+        .onConflict((oc) =>
+          oc.column('did').doUpdateSet({
+            doc: excluded(this.db.db, 'doc'),
+            updatedAt: excluded(this.db.db, 'updatedAt'),
+          }),
+        )
+        .executeTakeFirst()
+    }
   }
 
   async refreshCache(
     did: string,
     getDoc: () => Promise<DidDocument | null>,
+    prevResult?: CacheResult,
   ): Promise<void> {
     this.pQueue?.add(async () => {
       try {
         const doc = await getDoc()
         if (doc) {
-          await this.cacheDid(did, doc)
+          await this.cacheDid(did, doc, prevResult)
         } else {
           await this.clearEntry(did)
         }
@@ -51,20 +69,17 @@ export class DidSqlCache implements DidCache {
       .selectAll()
       .executeTakeFirst()
     if (!res) return null
+
     const now = Date.now()
     const updatedAt = new Date(res.updatedAt).getTime()
-
     const expired = now > updatedAt + this.maxTTL
-    if (expired) {
-      return null
-    }
-
     const stale = now > updatedAt + this.staleTTL
     return {
       doc: res.doc,
       updatedAt,
       did,
       stale,
+      expired,
     }
   }
 

package/src/feed-gen/with-friends.ts

@@ -15,7 +15,7 @@ const handler: AlgoHandler = async (
 
   const { ref } = db.db.dynamic
 
-  const keyset = new FeedKeyset(ref('post.indexedAt'), ref('post.cid'))
+  const keyset = new FeedKeyset(ref('post.sortAt'), ref('post.cid'))
   const sortFrom = keyset.unpack(cursor)?.primary
 
   let postsQb = feedService
@@ -24,7 +24,7 @@ const handler: AlgoHandler = async (
     .innerJoin('post_agg', 'post_agg.uri', 'post.uri')
     .where('post_agg.likeCount', '>=', 5)
     .where('follow.creator', '=', requester)
-    .where('post.indexedAt', '>', getFeedDateThreshold(sortFrom))
+    .where('post.sortAt', '>', getFeedDateThreshold(sortFrom))
 
   postsQb = paginate(postsQb, { limit, cursor, keyset, tryIndex: true })
 

package/src/index.ts

@@ -26,6 +26,7 @@ import { MountedAlgos } from './feed-gen/types'
 import { LabelCache } from './label-cache'
 import { NotificationServer } from './notifications'
 import { AtpAgent } from '@atproto/api'
+import { Keypair } from '@atproto/crypto'
 
 export type { ServerConfigValues } from './config'
 export type { MountedAlgos } from './feed-gen/types'
@@ -54,10 +55,11 @@ export class BskyAppView {
   static create(opts: {
     db: DatabaseCoordinator
     config: ServerConfig
+    signingKey: Keypair
     imgInvalidator?: ImageInvalidator
     algos?: MountedAlgos
   }): BskyAppView {
-    const { db, config, algos = {} } = opts
+    const { db, config, signingKey, algos = {} } = opts
     let maybeImgInvalidator = opts.imgInvalidator
     const app = express()
     app.use(cors())
@@ -116,6 +118,7 @@ export class BskyAppView {
       cfg: config,
       services,
       imgUriBuilder,
+      signingKey,
       idResolver,
       didCache,
       labelCache,

package/src/indexer/subscription.ts

@@ -1,7 +1,7 @@
 import assert from 'node:assert'
 import { CID } from 'multiformats/cid'
 import { AtUri } from '@atproto/syntax'
-import { cborDecode, wait } from '@atproto/common'
+import { cborDecode, wait, handleAllSettledErrors } from '@atproto/common'
 import { DisconnectError } from '@atproto/xrpc-server'
 import {
   WriteOpAction,
@@ -343,23 +343,3 @@ type PreparedDelete = {
 }
 
 type PreparedWrite = PreparedCreate | PreparedUpdate | PreparedDelete
-
-function handleAllSettledErrors(results: PromiseSettledResult<unknown>[]) {
-  const errors = results.filter(isRejected).map((res) => res.reason)
-  if (errors.length === 0) {
-    return
-  }
-  if (errors.length === 1) {
-    throw errors[0]
-  }
-  throw new AggregateError(
-    errors,
-    'Multiple errors: ' + errors.map((err) => err?.message).join('\n'),
-  )
-}
-
-function isRejected(
-  result: PromiseSettledResult<unknown>,
-): result is PromiseRejectedResult {
-  return result.status === 'rejected'
-}

package/src/lexicon/index.ts

@@ -12,6 +12,7 @@ import { schemas } from './lexicons'
 import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites'
 import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes'
 import * as ComAtprotoAdminEnableAccountInvites from './types/com/atproto/admin/enableAccountInvites'
+import * as ComAtprotoAdminGetAccountInfo from './types/com/atproto/admin/getAccountInfo'
 import * as ComAtprotoAdminGetInviteCodes from './types/com/atproto/admin/getInviteCodes'
 import * as ComAtprotoAdminGetModerationAction from './types/com/atproto/admin/getModerationAction'
 import * as ComAtprotoAdminGetModerationActions from './types/com/atproto/admin/getModerationActions'
@@ -19,6 +20,7 @@ import * as ComAtprotoAdminGetModerationReport from './types/com/atproto/admin/g
 import * as ComAtprotoAdminGetModerationReports from './types/com/atproto/admin/getModerationReports'
 import * as ComAtprotoAdminGetRecord from './types/com/atproto/admin/getRecord'
 import * as ComAtprotoAdminGetRepo from './types/com/atproto/admin/getRepo'
+import * as ComAtprotoAdminGetSubjectStatus from './types/com/atproto/admin/getSubjectStatus'
 import * as ComAtprotoAdminResolveModerationReports from './types/com/atproto/admin/resolveModerationReports'
 import * as ComAtprotoAdminReverseModerationAction from './types/com/atproto/admin/reverseModerationAction'
 import * as ComAtprotoAdminSearchRepos from './types/com/atproto/admin/searchRepos'
@@ -26,6 +28,7 @@ import * as ComAtprotoAdminSendEmail from './types/com/atproto/admin/sendEmail'
 import * as ComAtprotoAdminTakeModerationAction from './types/com/atproto/admin/takeModerationAction'
 import * as ComAtprotoAdminUpdateAccountEmail from './types/com/atproto/admin/updateAccountEmail'
 import * as ComAtprotoAdminUpdateAccountHandle from './types/com/atproto/admin/updateAccountHandle'
+import * as ComAtprotoAdminUpdateSubjectStatus from './types/com/atproto/admin/updateSubjectStatus'
 import * as ComAtprotoIdentityResolveHandle from './types/com/atproto/identity/resolveHandle'
 import * as ComAtprotoIdentityUpdateHandle from './types/com/atproto/identity/updateHandle'
 import * as ComAtprotoLabelQueryLabels from './types/com/atproto/label/queryLabels'
@@ -56,6 +59,7 @@ import * as ComAtprotoServerRequestAccountDelete from './types/com/atproto/serve
 import * as ComAtprotoServerRequestEmailConfirmation from './types/com/atproto/server/requestEmailConfirmation'
 import * as ComAtprotoServerRequestEmailUpdate from './types/com/atproto/server/requestEmailUpdate'
 import * as ComAtprotoServerRequestPasswordReset from './types/com/atproto/server/requestPasswordReset'
+import * as ComAtprotoServerReserveSigningKey from './types/com/atproto/server/reserveSigningKey'
 import * as ComAtprotoServerResetPassword from './types/com/atproto/server/resetPassword'
 import * as ComAtprotoServerRevokeAppPassword from './types/com/atproto/server/revokeAppPassword'
 import * as ComAtprotoServerUpdateEmail from './types/com/atproto/server/updateEmail'
@@ -224,6 +228,17 @@ export class AdminNS {
     return this._server.xrpc.method(nsid, cfg)
   }
 
+  getAccountInfo<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoAdminGetAccountInfo.Handler<ExtractAuth<AV>>,
+      ComAtprotoAdminGetAccountInfo.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.admin.getAccountInfo' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
   getInviteCodes<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,
@@ -301,6 +316,17 @@ export class AdminNS {
     return this._server.xrpc.method(nsid, cfg)
   }
 
+  getSubjectStatus<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoAdminGetSubjectStatus.Handler<ExtractAuth<AV>>,
+      ComAtprotoAdminGetSubjectStatus.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.admin.getSubjectStatus' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
   resolveModerationReports<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,
@@ -377,6 +403,17 @@ export class AdminNS {
     const nsid = 'com.atproto.admin.updateAccountHandle' // @ts-ignore
     return this._server.xrpc.method(nsid, cfg)
   }
+
+  updateSubjectStatus<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoAdminUpdateSubjectStatus.Handler<ExtractAuth<AV>>,
+      ComAtprotoAdminUpdateSubjectStatus.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.admin.updateSubjectStatus' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
 }
 
 export class IdentityNS {
@@ -748,6 +785,17 @@ export class ServerNS {
     return this._server.xrpc.method(nsid, cfg)
   }
 
+  reserveSigningKey<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoServerReserveSigningKey.Handler<ExtractAuth<AV>>,
+      ComAtprotoServerReserveSigningKey.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.server.reserveSigningKey' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
   resetPassword<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,