@atproto/pds 0.5.2 → 0.5.4

package/src/api/com/atproto/server/createAccount.ts

@@ -1,7 +1,7 @@
 import * as plc from '@did-plc/lib'
 import { isEmailValid } from '@hapi/address'
 import { isDisposableEmail } from 'disposable-email-domains-js'
-import { DidDocument, MINUTE, check } from '@atproto/common'
+import { MINUTE, check } from '@atproto/common'
 import { ExportableKeypair, Keypair, Secp256k1Keypair } from '@atproto/crypto'
 import { AtprotoData, ensureAtpDocument } from '@atproto/identity'
 import { DidString } from '@atproto/syntax'
@@ -10,12 +10,10 @@ import {
   InvalidRequestError,
   Server,
 } from '@atproto/xrpc-server'
-import { AccountStatus } from '../../../../account-manager/account-manager.js'
 import { NEW_PASSWORD_MAX_LENGTH } from '../../../../account-manager/helpers/scrypt.js'
 import { AppContext } from '../../../../context.js'
 import { baseNormalizeAndValidate } from '../../../../handle/index.js'
 import { com } from '../../../../lexicons/index.js'
-import { syncEvtDataFromCommit } from '../../../../sequencer/index.js'
 import { safeResolveDidDoc } from './util.js'
 
 export default function (server: Server, ctx: AppContext) {
@@ -47,68 +45,85 @@ export default function (server: Server, ctx: AppContext) {
         ? await validateInputsForEntrywayPds(ctx, input.body)
         : await validateInputsForLocalPds(ctx, input.body, requester)
 
-      let didDoc: DidDocument | undefined
-      let creds: { accessJwt: string; refreshJwt: string }
       await ctx.actorStore.create(did, signingKey)
+
       try {
-        const commit = await ctx.actorStore.transact(did, (actorTxn) =>
-          actorTxn.repo.createRepo([]),
-        )
+        const commit = await ctx.actorStore.transact(did, (actorTxn) => {
+          return actorTxn.repo.createRepo([])
+        })
+
+        const canTombstone =
+          // @NOTE IMPORTANT Because the user may be bringing their own did, we
+          // must make sure not to tombstone their did on failure if we didn't
+          // create it here.
+          !ctx.entrywayClient && !input.body.did && !!plcOp
 
         // Generate a real did with PLC
         if (plcOp) {
-          try {
-            await ctx.plcClient.sendOperation(did, plcOp)
-          } catch (err) {
-            req.log.error(
-              { didKey: ctx.plcRotationKey.did(), handle },
-              'failed to create did:plc',
-            )
-            throw err
-          }
+          await ctx.plcClient.sendOperation(did, plcOp)
         }
 
-        didDoc = await safeResolveDidDoc(ctx, did, true)
+        try {
+          const didDoc = await safeResolveDidDoc(ctx, did, true)
 
-        creds = await ctx.accountManager.createAccountAndSession({
-          did,
-          handle,
-          email,
-          password,
-          repoCid: commit.cid,
-          repoRev: commit.rev,
-          inviteCode,
-          deactivated,
-        })
-
-        if (!deactivated) {
-          await ctx.sequencer.sequenceIdentityEvt(did, handle)
-          await ctx.sequencer.sequenceAccountEvt(did, AccountStatus.Active)
-          await ctx.sequencer.sequenceCommit(did, commit)
-          await ctx.sequencer.sequenceSyncEvt(
+          const creds = await ctx.accountManager.createAccountAndSession({
             did,
-            syncEvtDataFromCommit(commit),
-          )
+            handle,
+            email,
+            password,
+            repoCid: commit.cid,
+            repoRev: commit.rev,
+            inviteCode,
+            deactivated,
+          })
+
+          try {
+            const sequenceEvt = !deactivated
+            if (sequenceEvt) {
+              await ctx.sequencer.sequenceAccountCreation(did, handle, commit)
+            }
+
+            try {
+              await ctx.actorStore
+                .clearReservedKeypair(signingKey.did(), did)
+                .catch((err) => {
+                  // @NOTE This is a cleanup operation so we won't fail the whole
+                  // flow if it fails, but we log it just in case
+                  req.log.error(
+                    { did, signingKeyDid: signingKey.did(), err },
+                    'Failed to clear reserved keypair',
+                  )
+                })
+
+              return {
+                encoding: 'application/json' as const,
+                body: {
+                  handle,
+                  did: did,
+                  // @ts-expect-error https://github.com/bluesky-social/atproto/pull/4406
+                  didDoc,
+                  accessJwt: creds.accessJwt,
+                  refreshJwt: creds.refreshJwt,
+                },
+              }
+            } catch (err) {
+              if (sequenceEvt) await ctx.sequencer.sequenceAccountDeletion(did)
+              throw err
+            }
+          } catch (err) {
+            await ctx.accountManager.deleteAccount(did)
+            throw err
+          }
+        } catch (err) {
+          if (canTombstone) {
+            await ctx.plcClient.tombstone(did, ctx.plcRotationKey)
+          }
+          throw err
         }
-        await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev)
-        await ctx.actorStore.clearReservedKeypair(signingKey.did(), did)
       } catch (err) {
-        // this will only be reached if the actor store _did not_ exist before
         await ctx.actorStore.destroy(did)
         throw err
       }
-
-      return {
-        encoding: 'application/json' as const,
-        body: {
-          handle,
-          did: did,
-          // @ts-expect-error https://github.com/bluesky-social/atproto/pull/4406
-          didDoc,
-          accessJwt: creds.accessJwt,
-          refreshJwt: creds.refreshJwt,
-        },
-      }
     },
   })
 }
@@ -117,9 +132,10 @@ const validateInputsForEntrywayPds = async (
   ctx: AppContext,
   input: com.atproto.server.createAccount.$InputBody,
 ) => {
-  const { did, plcOp } = input
   const handle = baseNormalizeAndValidate(input.handle)
-  if (!did || !input.plcOp) {
+
+  const { did, plcOp } = input
+  if (!did || !plcOp) {
     throw new InvalidRequestError(
       'non-entryway pds requires bringing a DID and plcOp',
     )
@@ -244,7 +260,7 @@ const validateInputsForLocalPds = async (
   } else {
     const formatted = await formatDidAndPlcOp(ctx, handle, input, signingKey)
     did = formatted.did as DidString
-    plcOp = formatted.plcOp
+    plcOp = formatted.op
   }
 
   return {
@@ -264,10 +280,7 @@ const formatDidAndPlcOp = async (
   handle: string,
   input: com.atproto.server.createAccount.$InputBody,
   signingKey: Keypair,
-): Promise<{
-  did: string
-  plcOp: plc.Operation | null
-}> => {
+) => {
   // if the user is not bringing a DID, then we format a create op for PLC
   const rotationKeys = [ctx.plcRotationKey.did()]
   if (ctx.cfg.identity.recoveryDidKey) {
@@ -276,17 +289,13 @@ const formatDidAndPlcOp = async (
   if (input.recoveryKey) {
     rotationKeys.unshift(input.recoveryKey)
   }
-  const plcCreate = await plc.createOp({
+  return plc.createOp({
     signingKey: signingKey.did(),
     rotationKeys,
     handle,
     pds: ctx.cfg.service.publicUrl,
     signer: ctx.plcRotationKey,
   })
-  return {
-    did: plcCreate.did,
-    plcOp: plcCreate.op,
-  }
 }
 const validateAtprotoData = (
   data: AtprotoData,

package/src/api/com/atproto/server/deactivateAccount.ts

@@ -38,7 +38,7 @@ export default function (server: Server, ctx: AppContext) {
           body.deleteAfter ?? null,
         )
         const status = await ctx.accountManager.getAccountStatus(requester)
-        await ctx.sequencer.sequenceAccountEvt(requester, status)
+        await ctx.sequencer.sequenceAccount(requester, status)
       },
     })
   }

package/src/api/com/atproto/server/deleteAccount.ts

@@ -4,7 +4,6 @@ import {
   InvalidRequestError,
   Server,
 } from '@atproto/xrpc-server'
-import { AccountStatus } from '../../../../account-manager/account-manager.js'
 import { OLD_PASSWORD_MAX_LENGTH } from '../../../../account-manager/helpers/scrypt.js'
 import { AppContext } from '../../../../context.js'
 import { com } from '../../../../lexicons/index.js'
@@ -54,13 +53,16 @@ export default function (server: Server, ctx: AppContext) {
         'delete_account',
         token,
       )
-      await ctx.actorStore.destroy(did)
+
+      // @NOTE Order matters here: first "unlink" the account by removing it
+      // from the account manager database ("source of truth"), then notify the
+      // sequencer, and finally cleanup files from the file system.
       await ctx.accountManager.deleteAccount(did)
-      const accountSeq = await ctx.sequencer.sequenceAccountEvt(
-        did,
-        AccountStatus.Deleted,
-      )
-      await ctx.sequencer.deleteAllForUser(did, [accountSeq])
+      try {
+        await ctx.sequencer.sequenceAccountDeletion(did)
+      } finally {
+        await ctx.actorStore.destroy(did)
+      }
     },
   })
 }

package/src/api/com/atproto/sync/getRepo.ts

@@ -1,5 +1,9 @@
 import stream from 'node:stream'
-import { byteIterableToStream } from '@atproto/common'
+import {
+  MINUTE,
+  byteIterableToStream,
+  coalesceByteStream,
+} from '@atproto/common'
 import { InvalidRequestError, Server } from '@atproto/xrpc-server'
 import {
   RepoRootNotFoundError,
@@ -11,6 +15,8 @@ import { AppContext } from '../../../../context.js'
 import { com } from '../../../../lexicons/index.js'
 import { assertRepoAvailability } from './util.js'
 
+const CAR_STREAM_CHUNK_SIZE = 64 * 1024
+
 export default function (server: Server, ctx: AppContext) {
   server.add(com.atproto.sync.getRepo, {
     auth: ctx.authVerifier.authorizationOrAdminTokenOptional({
@@ -19,7 +25,11 @@ export default function (server: Server, ctx: AppContext) {
         // always allow
       },
     }),
-    handler: async ({ params, auth }) => {
+    rateLimit: {
+      durationMs: 5 * MINUTE,
+      points: 6000,
+    },
+    handler: async ({ req, params, auth }) => {
       const { did, since } = params
       await assertRepoAvailability(ctx, did, isUserOrAdmin(auth, did))
 
@@ -27,7 +37,15 @@ export default function (server: Server, ctx: AppContext) {
 
       return {
         encoding: 'application/vnd.ipld.car' as const,
-        body: carStream,
+        // @NOTE If the client asked for compression (via "accept-encoding"), we
+        // coalesce the CAR stream into larger chunks to improve compression
+        // efficiency. See https://github.com/bluesky-social/atproto/pull/5078
+        //
+        // @TODO This would be better handled by xrpc-server and/or the
+        // compression middleware instead of manually coalescing the stream.
+        body: req.headers['accept-encoding']
+          ? coalesceByteStream(carStream, CAR_STREAM_CHUNK_SIZE)
+          : carStream,
       }
     },
   })
@@ -39,11 +57,14 @@ export const getCarStream = async (
   since?: string,
 ): Promise<stream.Readable> => {
   const actorDb = await ctx.actorStore.openDb(did)
-  let carStream: stream.Readable
   try {
     const storage = new SqlRepoReader(actorDb)
     const carIter = await storage.getCarStream(since)
-    carStream = byteIterableToStream(carIter)
+    const carStream = byteIterableToStream(carIter)
+    const closeDb = () => actorDb.close()
+    carStream.on('error', closeDb)
+    carStream.on('close', closeDb)
+    return carStream
   } catch (err) {
     await actorDb.close()
     if (err instanceof RepoRootNotFoundError) {
@@ -51,8 +72,4 @@ export const getCarStream = async (
     }
     throw err
   }
-  const closeDb = () => actorDb.close()
-  carStream.on('error', closeDb)
-  carStream.on('close', closeDb)
-  return carStream
 }

package/src/index.ts

@@ -14,11 +14,9 @@ import httpTerminator from 'http-terminator'
 // eslint-disable-next-line import/no-named-as-default-member
 const { createHttpTerminator } = httpTerminator
 type HttpTerminator = ReturnType<typeof createHttpTerminator>
-import { DAY, HOUR, MINUTE, SECOND } from '@atproto/common'
+import { DAY, SECOND } from '@atproto/common'
 import {
-  MemoryRateLimiter,
   MethodHandler,
-  RedisRateLimiter,
   ResponseType,
   XRPCError,
   createServer,
@@ -32,6 +30,7 @@ import * as error from './error.js'
 import { app } from './lexicons.js'
 import { loggerMiddleware } from './logger.js'
 import { proxyHandler } from './pipethrough.js'
+import { buildRateLimitsConfig } from './rate-limits.js'
 import compression from './util/compression.js'
 import * as wellKnown from './well-known.js'
 
@@ -113,45 +112,7 @@ export class PDS {
 
         return XRPCError.fromError(err)
       },
-      rateLimits: rateLimits.enabled
-        ? {
-            creator: ctx.redisScratch
-              ? (opts) => new RedisRateLimiter(ctx.redisScratch, opts)
-              : (opts) => new MemoryRateLimiter(opts),
-            bypass: ({ req }) => {
-              const { bypassKey, bypassIps } = rateLimits
-              if (
-                bypassKey &&
-                bypassKey === req.headers['x-ratelimit-bypass']
-              ) {
-                return true
-              }
-              if (bypassIps && bypassIps.includes(req.ip)) {
-                return true
-              }
-              return false
-            },
-            global: [
-              {
-                name: 'global-ip',
-                durationMs: 5 * MINUTE,
-                points: 3000,
-              },
-            ],
-            shared: [
-              {
-                name: 'repo-write-hour',
-                durationMs: HOUR,
-                points: 5000, // creates=3, puts=2, deletes=1
-              },
-              {
-                name: 'repo-write-day',
-                durationMs: DAY,
-                points: 35000, // creates=3, puts=2, deletes=1
-              },
-            ],
-          }
-        : undefined,
+      rateLimits: buildRateLimitsConfig(rateLimits, ctx.redisScratch),
     })
 
     apiRoutes(server, ctx)

package/src/rate-limits.ts

@@ -0,0 +1,59 @@
+import type { Redis } from 'ioredis'
+import { DAY, HOUR, MINUTE } from '@atproto/common'
+import { MemoryRateLimiter, RedisRateLimiter } from '@atproto/xrpc-server'
+import type { Options } from '@atproto/xrpc-server'
+import type { RateLimitsConfig } from './config/index.js'
+
+type RateLimitDescriptions = NonNullable<Options['rateLimits']>
+
+const SYNC_GET_REPO_PATH = '/xrpc/com.atproto.sync.getRepo'
+
+export const buildRateLimitsConfig = (
+  rateLimits: RateLimitsConfig,
+  redisScratch?: Redis,
+): RateLimitDescriptions | undefined => {
+  if (!rateLimits.enabled) return undefined
+
+  return {
+    creator: redisScratch
+      ? (opts) => new RedisRateLimiter(redisScratch, opts)
+      : (opts) => new MemoryRateLimiter(opts),
+    bypass: ({ req }) => {
+      const { bypassKey, bypassIps } = rateLimits
+      if (bypassKey && bypassKey === req.headers['x-ratelimit-bypass']) {
+        return true
+      }
+      if (bypassIps && bypassIps.includes(req.ip)) {
+        return true
+      }
+      return false
+    },
+    global: [
+      {
+        name: 'global-ip',
+        durationMs: 5 * MINUTE,
+        points: 3000,
+        // getRepo can be a high-volume sync path, so it has its own endpoint
+        // limit and should not consume the shared global read bucket.
+        calcKey: ({ req }) => {
+          if (req.path === SYNC_GET_REPO_PATH) {
+            return null
+          }
+          return req.ip
+        },
+      },
+    ],
+    shared: [
+      {
+        name: 'repo-write-hour',
+        durationMs: HOUR,
+        points: 5000, // creates=3, puts=2, deletes=1
+      },
+      {
+        name: 'repo-write-day',
+        durationMs: DAY,
+        points: 35000, // creates=3, puts=2, deletes=1
+      },
+    ],
+  }
+}

package/src/scripts/publish-identity.ts

@@ -48,7 +48,7 @@ export const publishIdentityEvtForDids = async (
 ) => {
   for (const did of dids) {
     try {
-      await ctx.sequencer.sequenceIdentityEvt(did)
+      await ctx.sequencer.sequenceIdentity(did)
       console.log(`published identity evt for ${did}`)
     } catch (err) {
       console.error(`failed to sequence new identity evt for ${did}: ${err}`)

package/src/scripts/rebuild-repo.ts

@@ -106,7 +106,7 @@ export const rebuildRepo = async (
   const syncData = await ctx.actorStore.read(did, (store) =>
     store.repo.getSyncEventData(),
   )
-  await ctx.sequencer.sequenceSyncEvt(did, syncData)
+  await ctx.sequencer.sequenceSync(did, syncData)
 }
 
 const promptContinue = async (): Promise<boolean> => {

package/src/scripts/rotate-keys.ts

@@ -100,13 +100,13 @@ const rotateKeysForRepos = async (
         return
       }
       try {
-        await ctx.sequencer.sequenceIdentityEvt(did)
+        await ctx.sequencer.sequenceIdentity(did)
       } catch (err) {
         console.error(`failed to sequence new identity evt for ${did}: ${err}`)
         return
       }
       try {
-        await ctx.sequencer.sequenceSyncEvt(did, syncData)
+        await ctx.sequencer.sequenceSync(did, syncData)
       } catch (err) {
         console.error(`failed to sequence for ${did}: ${err}`)
         return

package/src/scripts/sequencer-recovery/recoverer.ts

@@ -179,12 +179,16 @@ const processRepoCreation = async (
 
 const processAccountEvt = async (ctx: RecovererContext, evt: AccountEvt) => {
   // do not need to process deactivation/takedowns because we backup account DB as well
-  if (evt.status !== AccountStatus.Deleted) {
-    return
+
+  if (evt.status === AccountStatus.Deleted) {
+    // In case an account deletion was sequenced, let's make sure to (first)
+    // delete the accounts database, and (then) unlink the actor store from the
+    // file system. Order matters here.
+    await ctx.accountManager.deleteAccount(evt.did)
+
+    const { directory } = await ctx.actorStore.getLocation(evt.did)
+    await rmIfExists(directory, true)
   }
-  const { directory } = await ctx.actorStore.getLocation(evt.did)
-  await rmIfExists(directory, true)
-  await ctx.accountManager.deleteAccount(evt.did)
 }
 
 const trackBlobs = async (

package/src/sequencer/sequencer.ts

@@ -24,6 +24,7 @@ import {
   formatSeqCommit,
   formatSeqIdentityEvt,
   formatSeqSyncEvt,
+  syncEvtDataFromCommit,
 } from './events.js'
 
 export * from './events.js'
@@ -161,51 +162,79 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
     await wait(waitTime)
   }
 
-  async sequenceEvt(evt: RepoSeqInsert): Promise<number> {
-    const [{ seq }] = await this.db.executeWithRetry(
-      this.db.db.insertInto('repo_seq').values(evt).returning('seq'),
+  protected async sequenceEvts(
+    events: readonly RepoSeqInsert[],
+  ): Promise<number[]> {
+    if (!events.length) return []
+    const rows = await this.db.executeWithRetry(
+      this.db.db.insertInto('repo_seq').values(events).returning('seq'),
     )
     this.crawlers.notifyOfUpdate()
-    return seq
+    return rows.map((row) => row.seq)
   }
 
-  async sequenceCommit(
+  public async sequenceCommit(
     did: DidString,
     commitData: CommitDataWithOps,
-  ): Promise<number> {
-    const evt = await formatSeqCommit(did, commitData)
-    return this.sequenceEvt(evt)
+  ): Promise<void> {
+    await this.sequenceEvts([await formatSeqCommit(did, commitData)])
   }
 
-  async sequenceSyncEvt(did: DidString, data: SyncEvtData) {
-    const evt = await formatSeqSyncEvt(did, data)
-    return this.sequenceEvt(evt)
+  public async sequenceSync(did: DidString, data: SyncEvtData): Promise<void> {
+    await this.sequenceEvts([await formatSeqSyncEvt(did, data)])
   }
 
-  async sequenceIdentityEvt(
+  public async sequenceIdentity(
     did: DidString,
     handle?: HandleString,
-  ): Promise<number> {
-    const evt = await formatSeqIdentityEvt(did, handle)
-    return this.sequenceEvt(evt)
+  ): Promise<void> {
+    await this.sequenceEvts([await formatSeqIdentityEvt(did, handle)])
   }
 
-  async sequenceAccountEvt(
+  public async sequenceAccount(
     did: DidString,
     status: AccountStatus,
-  ): Promise<number> {
-    const evt = await formatSeqAccountEvt(did, status)
-    return this.sequenceEvt(evt)
+  ): Promise<void> {
+    await this.sequenceEvts([await formatSeqAccountEvt(did, status)])
   }
 
-  async deleteAllForUser(did: string, excludingSeqs: number[] = []) {
+  public async sequenceAccountCreation(
+    did: DidString,
+    handle: HandleString,
+    commit: CommitDataWithOps,
+  ): Promise<void> {
+    // Atomically sequence all events
+    await this.sequenceEvts([
+      await formatSeqIdentityEvt(did, handle),
+      await formatSeqAccountEvt(did, AccountStatus.Active),
+      await formatSeqCommit(did, commit),
+      await formatSeqSyncEvt(did, syncEvtDataFromCommit(commit)),
+    ])
+  }
+
+  public async sequenceAccountActivation(
+    did: DidString,
+    handle: HandleString,
+    status: AccountStatus,
+    syncData: SyncEvtData,
+  ): Promise<void> {
+    // Atomically sequence all events
+    await this.sequenceEvts([
+      await formatSeqAccountEvt(did, status),
+      await formatSeqIdentityEvt(did, handle),
+      await formatSeqSyncEvt(did, syncData),
+    ])
+  }
+
+  public async sequenceAccountDeletion(did: DidString) {
+    const [seq] = await this.sequenceEvts([
+      await formatSeqAccountEvt(did, AccountStatus.Deleted),
+    ])
     await this.db.executeWithRetry(
       this.db.db
         .deleteFrom('repo_seq')
         .where('did', '=', did)
-        .if(excludingSeqs.length > 0, (qb) =>
-          qb.where('seq', 'not in', excludingSeqs),
-        ),
+        .where('seq', '!=', seq),
     )
   }
 }