@atproto/pds 0.4.176 → 0.4.177

package/src/account-manager/db/migrations/007-lexicon-failures-index.ts

@@ -0,0 +1,14 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createIndex('lexicon_failures_idx')
+    .on('lexicon')
+    // https://github.com/kysely-org/kysely/issues/302
+    .expression(sql`"updatedAt" DESC) WHERE ("lexicon" is NULL`)
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropIndex('lexicon_failures_idx').execute()
+}

package/src/account-manager/db/migrations/index.ts

@@ -4,6 +4,7 @@ import * as mig003 from './003-privileged-app-passwords'
 import * as mig004 from './004-oauth'
 import * as mig005 from './005-oauth-account-management'
 import * as mig006 from './006-oauth-permission-sets'
+import * as mig007 from './007-lexicon-failures-index'
 
 export default {
   '001': mig001,
@@ -12,4 +13,5 @@ export default {
   '004': mig004,
   '005': mig005,
   '006': mig006,
+  '007': mig007,
 }

package/src/account-manager/helpers/lexicon.ts

@@ -1,5 +1,5 @@
 import { Insertable } from 'kysely'
-import { LexiconData } from '@atproto/oauth-provider'
+import { LEXICON_REFRESH_FREQUENCY, LexiconData } from '@atproto/oauth-provider'
 import { fromDateISO, fromJson, toDateISO, toJson } from '../../db'
 import { AccountDb, Lexicon } from '../db'
 
@@ -20,6 +20,19 @@ export async function upsert(db: AccountDb, nsid: string, data: LexiconData) {
       .values({ ...updates, nsid })
       .onConflict((oc) => oc.column('nsid').doUpdateSet(updates)),
   )
+
+  // Garbage collection: remove old, never resolved, lexicons.
+  // Uses "lexicon_failures_idx"
+  await db.executeWithRetry(
+    db.db
+      .deleteFrom('lexicon')
+      .where('lexicon', 'is', null)
+      .where(
+        'updatedAt',
+        '<',
+        toDateISO(new Date(Date.now() - LEXICON_REFRESH_FREQUENCY)),
+      ),
+  )
 }
 
 export async function find(

package/src/account-manager/scope-reference-getter.ts

@@ -0,0 +1,92 @@
+import Redis from 'ioredis'
+import { Agent, ComAtprotoTempDereferenceScope } from '@atproto/api'
+import { DAY, backoffMs, retry } from '@atproto/common'
+import { InvalidTokenError, OAuthScope } from '@atproto/oauth-provider'
+import { UpstreamFailureError } from '@atproto/xrpc-server'
+import { CachedGetter, GetterOptions } from '@atproto-labs/simple-store'
+import { SimpleStoreMemory } from '@atproto-labs/simple-store-memory'
+import { SimpleStoreRedis } from '@atproto-labs/simple-store-redis'
+
+const { InvalidScopeReferenceError } = ComAtprotoTempDereferenceScope
+const PREFIX = 'ref:'
+
+type ScopeReference = `${typeof PREFIX}${string}`
+const isScopeReference = (scope?: OAuthScope): scope is ScopeReference =>
+  scope != null && scope.startsWith(PREFIX) && !scope.includes(' ')
+
+const identity = <T>(value: T): T => value
+
+export class ScopeReferenceGetter extends CachedGetter<
+  ScopeReference,
+  OAuthScope
+> {
+  constructor(
+    protected readonly entryway: Agent,
+    redis?: Redis,
+  ) {
+    super(
+      async (scope, options) => {
+        return retry(async () => this.fetchDereferencedScope(scope, options), {
+          maxRetries: 3,
+          getWaitMs: (n) => backoffMs(n, 250, 2000),
+          retryable: (err) =>
+            !options?.signal?.aborted &&
+            !(err instanceof InvalidScopeReferenceError),
+        })
+      },
+      redis
+        ? new SimpleStoreRedis(redis, {
+            // tradeoff between wasted memory usage (by no longer used scopes)
+            // and amount of requests to entryway:
+            ttl: 1 * DAY,
+
+            keyPrefix: `auth-scope-${PREFIX}`,
+            encode: identity,
+            decode: identity,
+          })
+        : new SimpleStoreMemory({ max: 1000 }),
+    )
+  }
+
+  protected async fetchDereferencedScope(
+    scope: ScopeReference,
+    options?: GetterOptions,
+  ): Promise<OAuthScope> {
+    const response = await this.entryway.com.atproto.temp.dereferenceScope(
+      { scope },
+      {
+        signal: options?.signal,
+        headers: options?.noCache ? { 'Cache-Control': 'no-cache' } : undefined,
+      },
+    )
+
+    // @NOTE the part after `PREFIX` (in the input scope) is the CID of the
+    // scope string returned by entryway. Since there is a trust
+    // relationship with the entryway, we don't need to verify or enforce
+    // that here.
+
+    return response.data.scope
+  }
+
+  async dereference(scope?: OAuthScope): Promise<undefined | OAuthScope> {
+    if (!isScopeReference(scope)) return scope
+
+    return this.get(scope).catch(handleDereferenceError)
+  }
+}
+
+function handleDereferenceError(cause: unknown): never {
+  if (cause instanceof InvalidScopeReferenceError) {
+    // The scope reference cannot be found on the server.
+    // Consider the session as invalid, allowing entryway to
+    // re-build the scope as the user re-authenticates. This
+    // should never happen though.
+    throw InvalidTokenError.from(cause, 'DPoP')
+  }
+
+  throw new UpstreamFailureError(
+    'Failed to fetch token permissions',
+    undefined,
+    { cause },
+  )
+}

package/src/actor-store/actor-store.ts

@@ -1,18 +1,14 @@
 import assert from 'node:assert'
 import fs, { mkdir } from 'node:fs/promises'
 import path from 'node:path'
-import {
-  chunkArray,
-  fileExists,
-  readIfExists,
-  rmIfExists,
-} from '@atproto/common'
+import { fileExists, readIfExists, rmIfExists } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import { ExportableKeypair, Keypair } from '@atproto/crypto'
 import { InvalidRequestError } from '@atproto/xrpc-server'
 import { ActorStoreConfig } from '../config'
 import { retrySqlite } from '../db'
 import { DiskBlobStore } from '../disk-blobstore'
+import { blobStoreLogger } from '../logger'
 import { ActorStoreReader } from './actor-store-reader'
 import { ActorStoreResources } from './actor-store-resources'
 import { ActorStoreTransactor } from './actor-store-transactor'
@@ -137,9 +133,9 @@ export class ActorStore {
       const cids = await this.read(did, async (store) =>
         store.repo.blob.getBlobCids(),
       )
-      await Promise.allSettled(
-        chunkArray(cids, 500).map((chunk) => blobstore.deleteMany(chunk)),
-      )
+      await blobstore.deleteMany(cids).catch((err) => {
+        blobStoreLogger.error('Failed to delete blobs', { did, cids, err })
+      })
     }
 
     const { directory } = await this.getLocation(did)

package/src/actor-store/blob/transactor.ts

@@ -3,7 +3,13 @@ import stream from 'node:stream'
 import bytes from 'bytes'
 import { fromStream as fileTypeFromStream } from 'file-type'
 import { CID } from 'multiformats/cid'
-import { cloneStream, sha256RawToCid, streamSize } from '@atproto/common'
+import PQueue from 'p-queue'
+import {
+  SECOND,
+  cloneStream,
+  sha256RawToCid,
+  streamSize,
+} from '@atproto/common'
 import { BlobRef } from '@atproto/lexicon'
 import { BlobNotFoundError, BlobStore, WriteOpAction } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
@@ -11,12 +17,8 @@ import { InvalidRequestError } from '@atproto/xrpc-server'
 import { BackgroundQueue } from '../../background'
 import * as img from '../../image'
 import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
-import {
-  PreparedBlobRef,
-  PreparedDelete,
-  PreparedUpdate,
-  PreparedWrite,
-} from '../../repo/types'
+import { blobStoreLogger as log } from '../../logger'
+import { PreparedBlobRef, PreparedWrite } from '../../repo/types'
 import { ActorDb, Blob as BlobTable } from '../db'
 import { BlobReader } from './reader'
 
@@ -113,38 +115,68 @@ export class BlobTransactor extends BlobReader {
   async processWriteBlobs(rev: string, writes: PreparedWrite[]) {
     await this.deleteDereferencedBlobs(writes)
 
-    const blobPromises: Promise<void>[] = []
+    const ac = new AbortController()
+
+    // Limit the number of parallel requests made to the BlobStore by using a
+    // a queue with concurrency management.
+    type Task = () => Promise<void>
+    const tasks: Task[] = []
+
     for (const write of writes) {
-      if (
-        write.action === WriteOpAction.Create ||
-        write.action === WriteOpAction.Update
-      ) {
+      if (isCreate(write) || isUpdate(write)) {
         for (const blob of write.blobs) {
-          blobPromises.push(this.verifyBlobAndMakePermanent(blob))
-          blobPromises.push(this.associateBlob(blob, write.uri))
+          tasks.push(async () => {
+            if (ac.signal.aborted) return
+            await this.associateBlob(blob, write.uri)
+            await this.verifyBlobAndMakePermanent(blob, ac.signal)
+          })
         }
       }
     }
-    await Promise.all(blobPromises)
+
+    try {
+      const queue = new PQueue({
+        concurrency: 20,
+        // The blob store should already limit the time of every operation. We
+        // add a timeout here as an extra precaution.
+        timeout: 60 * SECOND,
+        throwOnTimeout: true,
+      })
+
+      // Will reject as soon as any task fails, causing the "finally" block
+      // below to run, aborting every other pending tasks.
+      await queue.addAll(tasks)
+    } finally {
+      ac.abort()
+    }
   }
 
-  async updateBlobTakedownStatus(blob: CID, takedown: StatusAttr) {
+  async updateBlobTakedownStatus(cid: CID, takedown: StatusAttr) {
     const takedownRef = takedown.applied
       ? takedown.ref ?? new Date().toISOString()
       : null
     await this.db.db
       .updateTable('blob')
       .set({ takedownRef })
-      .where('cid', '=', blob.toString())
+      .where('cid', '=', cid.toString())
       .executeTakeFirst()
+
     try {
+      // @NOTE find a way to not perform i/o operations during the transaction
+      // (typically by using a state in the "blob" table, and another process to
+      // handle the actual i/o)
       if (takedown.applied) {
-        await this.blobstore.quarantine(blob)
+        await this.blobstore.quarantine(cid)
       } else {
-        await this.blobstore.unquarantine(blob)
+        await this.blobstore.unquarantine(cid)
       }
     } catch (err) {
       if (!(err instanceof BlobNotFoundError)) {
+        log.error(
+          { err, cid: cid.toString() },
+          'could not update blob takedown status',
+        )
+
         throw err
       }
     }
@@ -154,21 +186,17 @@ export class BlobTransactor extends BlobReader {
     writes: PreparedWrite[],
     skipBlobStore?: boolean,
   ) {
-    const deletes = writes.filter(
-      (w) => w.action === WriteOpAction.Delete,
-    ) as PreparedDelete[]
-    const updates = writes.filter(
-      (w) => w.action === WriteOpAction.Update,
-    ) as PreparedUpdate[]
+    const deletes = writes.filter(isDelete)
+    const updates = writes.filter(isUpdate)
     const uris = [...deletes, ...updates].map((w) => w.uri.toString())
     if (uris.length === 0) return
 
     const deletedRepoBlobs = await this.db.db
       .deleteFrom('record_blob')
       .where('recordUri', 'in', uris)
-      .returningAll()
+      .returning('blobCid')
       .execute()
-    if (deletedRepoBlobs.length < 1) return
+    if (deletedRepoBlobs.length === 0) return
 
     const deletedRepoBlobCids = deletedRepoBlobs.map((row) => row.blobCid)
     const duplicateCids = await this.db.db
@@ -178,53 +206,85 @@ export class BlobTransactor extends BlobReader {
       .execute()
 
     const newBlobCids = writes
-      .map((w) =>
-        w.action === WriteOpAction.Create || w.action === WriteOpAction.Update
-          ? w.blobs
-          : [],
-      )
-      .flat()
-      .map((b) => b.cid.toString())
+      .filter((w) => isUpdate(w) || isCreate(w))
+      .flatMap((w) => w.blobs.map((b) => b.cid.toString()))
+
     const cidsToKeep = [
       ...newBlobCids,
       ...duplicateCids.map((row) => row.blobCid),
     ]
+
     const cidsToDelete = deletedRepoBlobCids.filter(
       (cid) => !cidsToKeep.includes(cid),
     )
-    if (cidsToDelete.length < 1) return
+    if (cidsToDelete.length === 0) return
 
     await this.db.db
       .deleteFrom('blob')
       .where('cid', 'in', cidsToDelete)
       .execute()
+
     if (!skipBlobStore) {
       this.db.onCommit(() => {
         this.backgroundQueue.add(async () => {
-          await Promise.allSettled(
-            cidsToDelete.map((cid) => this.blobstore.delete(CID.parse(cid))),
-          )
+          try {
+            const cids = cidsToDelete.map((cid) => CID.parse(cid))
+            await this.blobstore.deleteMany(cids)
+          } catch (err) {
+            log.error(
+              { err, cids: cidsToDelete },
+              'could not delete blobs from blobstore',
+            )
+          }
         })
       })
     }
   }
 
-  async verifyBlobAndMakePermanent(blob: PreparedBlobRef): Promise<void> {
+  async verifyBlobAndMakePermanent(
+    blob: PreparedBlobRef,
+    signal?: AbortSignal,
+  ): Promise<void> {
     const found = await this.db.db
       .selectFrom('blob')
-      .selectAll()
+      .select(['tempKey', 'size', 'mimeType'])
       .where('cid', '=', blob.cid.toString())
       .where('takedownRef', 'is', null)
       .executeTakeFirst()
+
+    signal?.throwIfAborted()
+
     if (!found) {
       throw new InvalidRequestError(
         `Could not find blob: ${blob.cid.toString()}`,
         'BlobNotFound',
       )
     }
+
     if (found.tempKey) {
       verifyBlob(blob, found)
-      await this.blobstore.makePermanent(found.tempKey, blob.cid)
+
+      // @NOTE it is less than ideal to perform async (i/o) operations during a
+      // transaction. Especially since there have been instances of the actor-db
+      // being locked, requiring to kick the processes.
+
+      // The better solution would be to update the blob state in the database
+      // (e.g. "makeItPermanent") and to process those updates outside of the
+      // transaction.
+
+      await this.blobstore
+        .makePermanent(found.tempKey, blob.cid)
+        .catch((err) => {
+          log.error(
+            { err, cid: blob.cid.toString() },
+            'could not make blob permanent',
+          )
+
+          throw err
+        })
+
+      signal?.throwIfAborted()
+
       await this.db.db
         .updateTable('blob')
         .set({ tempKey: null })
@@ -300,7 +360,10 @@ function acceptedMime(mime: string, accepted: string[]): boolean {
   return accepted.includes(mime)
 }
 
-function verifyBlob(blob: PreparedBlobRef, found: BlobTable) {
+function verifyBlob(
+  blob: PreparedBlobRef,
+  found: Pick<BlobTable, 'size' | 'mimeType'>,
+) {
   const throwInvalid = (msg: string, errName = 'InvalidBlob') => {
     throw new InvalidRequestError(msg, errName)
   }
@@ -328,3 +391,13 @@ function verifyBlob(blob: PreparedBlobRef, found: BlobTable) {
     )
   }
 }
+
+function isCreate(write: PreparedWrite) {
+  return write.action === WriteOpAction.Create
+}
+function isUpdate(write: PreparedWrite) {
+  return write.action === WriteOpAction.Update
+}
+function isDelete(write: PreparedWrite) {
+  return write.action === WriteOpAction.Delete
+}

package/src/actor-store/record/reader.ts

@@ -213,19 +213,21 @@ export class RecordReader {
   // Ensures that we don't end-up with duplicate likes, reposts, and follows from race conditions.
 
   async getBacklinkConflicts(uri: AtUri, record: RepoRecord): Promise<AtUri[]> {
-    const recordBacklinks = getBacklinks(uri, record)
-    const conflicts = await Promise.all(
-      recordBacklinks.map((backlink) =>
-        this.getRecordBacklinks({
-          collection: uri.collection,
-          path: backlink.path,
-          linkTo: backlink.linkTo,
-        }),
-      ),
-    )
+    const conflicts: AtUri[] = []
+
+    for (const backlink of getBacklinks(uri, record)) {
+      const backlinks = await this.getRecordBacklinks({
+        collection: uri.collection,
+        path: backlink.path,
+        linkTo: backlink.linkTo,
+      })
+
+      for (const { rkey } of backlinks) {
+        conflicts.push(AtUri.make(uri.hostname, uri.collection, rkey))
+      }
+    }
+
     return conflicts
-      .flat()
-      .map(({ rkey }) => AtUri.make(uri.hostname, uri.collection, rkey))
   }
 
   async listExistingBlocks(): Promise<CidSet> {

package/src/actor-store/repo/sql-repo-reader.ts

@@ -59,20 +59,18 @@ export class SqlRepoReader extends ReadableBlockstore {
     const missing = new CidSet(cached.missing)
     const missingStr = cached.missing.map((c) => c.toString())
     const blocks = new BlockMap()
-    await Promise.all(
-      chunkArray(missingStr, 500).map(async (batch) => {
-        const res = await this.db.db
-          .selectFrom('repo_block')
-          .where('repo_block.cid', 'in', batch)
-          .select(['repo_block.cid as cid', 'repo_block.content as content'])
-          .execute()
-        for (const row of res) {
-          const cid = CID.parse(row.cid)
-          blocks.set(cid, row.content)
-          missing.delete(cid)
-        }
-      }),
-    )
+    for (const batch of chunkArray(missingStr, 500)) {
+      const res = await this.db.db
+        .selectFrom('repo_block')
+        .where('repo_block.cid', 'in', batch)
+        .select(['repo_block.cid as cid', 'repo_block.content as content'])
+        .execute()
+      for (const row of res) {
+        const cid = CID.parse(row.cid)
+        blocks.set(cid, row.content)
+        missing.delete(cid)
+      }
+    }
     this.cache.addMap(blocks)
     blocks.addMap(cached.blocks)
     return { blocks, missing: missing.toList() }

package/src/actor-store/repo/sql-repo-transactor.ts

@@ -45,24 +45,20 @@ export class SqlRepoTransactor extends SqlRepoReader implements RepoStorage {
   }
 
   async putMany(toPut: BlockMap, rev: string): Promise<void> {
-    const blocks: RepoBlock[] = []
-    toPut.forEach((bytes, cid) => {
-      blocks.push({
-        cid: cid.toString(),
-        repoRev: rev,
-        size: bytes.length,
-        content: bytes,
-      })
-    })
-    await Promise.all(
-      chunkArray(blocks, 50).map((batch) =>
-        this.db.db
-          .insertInto('repo_block')
-          .values(batch)
-          .onConflict((oc) => oc.doNothing())
-          .execute(),
-      ),
-    )
+    const blocks: RepoBlock[] = Array.from(toPut, ([cid, bytes]) => ({
+      cid: cid.toString(),
+      repoRev: rev,
+      size: bytes.length,
+      content: bytes,
+    }))
+
+    for (const batch of chunkArray(blocks, 50)) {
+      await this.db.db
+        .insertInto('repo_block')
+        .values(batch)
+        .onConflict((oc) => oc.doNothing())
+        .execute()
+    }
   }
 
   async deleteMany(cids: CID[]) {
@@ -75,11 +71,9 @@ export class SqlRepoTransactor extends SqlRepoReader implements RepoStorage {
   }
 
   async applyCommit(commit: CommitData, isCreate?: boolean) {
-    await Promise.all([
-      this.updateRoot(commit.cid, commit.rev, isCreate),
-      this.putMany(commit.newBlocks, commit.rev),
-      this.deleteMany(commit.removedCids.toList()),
-    ])
+    await this.updateRoot(commit.cid, commit.rev, isCreate)
+    await this.putMany(commit.newBlocks, commit.rev)
+    await this.deleteMany(commit.removedCids.toList())
   }
 
   async updateRoot(cid: CID, rev: string, isCreate = false): Promise<void> {

package/src/actor-store/repo/transactor.ts

@@ -55,11 +55,10 @@ export class RepoTransactor extends RepoReader {
       this.signingKey,
       writes.map(createWriteToOp),
     )
-    await Promise.all([
-      this.storage.applyCommit(commit, true),
-      this.indexWrites(writes, commit.rev),
-      this.blob.processWriteBlobs(commit.rev, writes),
-    ])
+    await this.storage.applyCommit(commit, true)
+    await this.indexWrites(writes, commit.rev)
+    await this.blob.processWriteBlobs(commit.rev, writes)
+
     const ops = writes.map((w) => ({
       action: 'create' as const,
       path: formatDataKey(w.uri.collection, w.uri.rkey),
@@ -87,14 +86,13 @@ export class RepoTransactor extends RepoReader {
       throw new InvalidRequestError('Too many writes. Max event size: 2MB')
     }
 
-    await Promise.all([
-      // persist the commit to repo storage
-      this.storage.applyCommit(commit),
-      // & send to indexing
-      this.indexWrites(writes, commit.rev),
-      // process blobs
-      this.blob.processWriteBlobs(commit.rev, writes),
-    ])
+    // persist the commit to repo storage
+    await this.storage.applyCommit(commit)
+    // & send to indexing
+    await this.indexWrites(writes, commit.rev)
+    // process blobs
+    await this.blob.processWriteBlobs(commit.rev, writes)
+
     return commit
   }
 
@@ -184,25 +182,24 @@ export class RepoTransactor extends RepoReader {
 
   async indexWrites(writes: PreparedWrite[], rev: string) {
     this.db.assertTransaction()
-    await Promise.all(
-      writes.map(async (write) => {
-        if (
-          write.action === WriteOpAction.Create ||
-          write.action === WriteOpAction.Update
-        ) {
-          await this.record.indexRecord(
-            write.uri,
-            write.cid,
-            write.record,
-            write.action,
-            rev,
-            this.now,
-          )
-        } else if (write.action === WriteOpAction.Delete) {
-          await this.record.deleteRecord(write.uri)
-        }
-      }),
-    )
+
+    for (const write of writes) {
+      if (
+        write.action === WriteOpAction.Create ||
+        write.action === WriteOpAction.Update
+      ) {
+        await this.record.indexRecord(
+          write.uri,
+          write.cid,
+          write.record,
+          write.action,
+          rev,
+          this.now,
+        )
+      } else if (write.action === WriteOpAction.Delete) {
+        await this.record.deleteRecord(write.uri)
+      }
+    }
   }
 
   async getDuplicateRecordCids(

package/src/api/com/atproto/admin/updateSubjectStatus.ts

@@ -19,16 +19,16 @@ export default function (server: Server, ctx: AppContext) {
           await ctx.accountManager.takedownAccount(subject.did, takedown)
         } else if (isStrongRef(subject)) {
           const uri = new AtUri(subject.uri)
-          await ctx.actorStore.transact(uri.hostname, (store) =>
-            store.record.updateRecordTakedownStatus(uri, takedown),
-          )
+          await ctx.actorStore.transact(uri.hostname, async (store) => {
+            await store.record.updateRecordTakedownStatus(uri, takedown)
+          })
         } else if (isRepoBlobRef(subject)) {
-          await ctx.actorStore.transact(subject.did, (store) =>
-            store.repo.blob.updateBlobTakedownStatus(
+          await ctx.actorStore.transact(subject.did, async (store) => {
+            await store.repo.blob.updateBlobTakedownStatus(
               CID.parse(subject.cid),
               takedown,
-            ),
-          )
+            )
+          })
         } else {
           throw new InvalidRequestError('Invalid subject')
         }