@atproto/pds 0.4.176 → 0.4.178

package/src/actor-store/blob/transactor.ts

@@ -3,7 +3,13 @@ import stream from 'node:stream'
 import bytes from 'bytes'
 import { fromStream as fileTypeFromStream } from 'file-type'
 import { CID } from 'multiformats/cid'
-import { cloneStream, sha256RawToCid, streamSize } from '@atproto/common'
+import PQueue from 'p-queue'
+import {
+  SECOND,
+  cloneStream,
+  sha256RawToCid,
+  streamSize,
+} from '@atproto/common'
 import { BlobRef } from '@atproto/lexicon'
 import { BlobNotFoundError, BlobStore, WriteOpAction } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
@@ -11,12 +17,8 @@ import { InvalidRequestError } from '@atproto/xrpc-server'
 import { BackgroundQueue } from '../../background'
 import * as img from '../../image'
 import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
-import {
-  PreparedBlobRef,
-  PreparedDelete,
-  PreparedUpdate,
-  PreparedWrite,
-} from '../../repo/types'
+import { blobStoreLogger as log } from '../../logger'
+import { PreparedBlobRef, PreparedWrite } from '../../repo/types'
 import { ActorDb, Blob as BlobTable } from '../db'
 import { BlobReader } from './reader'
 
@@ -113,38 +115,68 @@ export class BlobTransactor extends BlobReader {
   async processWriteBlobs(rev: string, writes: PreparedWrite[]) {
     await this.deleteDereferencedBlobs(writes)
 
-    const blobPromises: Promise<void>[] = []
+    const ac = new AbortController()
+
+    // Limit the number of parallel requests made to the BlobStore by using a
+    // a queue with concurrency management.
+    type Task = () => Promise<void>
+    const tasks: Task[] = []
+
     for (const write of writes) {
-      if (
-        write.action === WriteOpAction.Create ||
-        write.action === WriteOpAction.Update
-      ) {
+      if (isCreate(write) || isUpdate(write)) {
         for (const blob of write.blobs) {
-          blobPromises.push(this.verifyBlobAndMakePermanent(blob))
-          blobPromises.push(this.associateBlob(blob, write.uri))
+          tasks.push(async () => {
+            if (ac.signal.aborted) return
+            await this.associateBlob(blob, write.uri)
+            await this.verifyBlobAndMakePermanent(blob, ac.signal)
+          })
         }
       }
     }
-    await Promise.all(blobPromises)
+
+    try {
+      const queue = new PQueue({
+        concurrency: 20,
+        // The blob store should already limit the time of every operation. We
+        // add a timeout here as an extra precaution.
+        timeout: 60 * SECOND,
+        throwOnTimeout: true,
+      })
+
+      // Will reject as soon as any task fails, causing the "finally" block
+      // below to run, aborting every other pending tasks.
+      await queue.addAll(tasks)
+    } finally {
+      ac.abort()
+    }
   }
 
-  async updateBlobTakedownStatus(blob: CID, takedown: StatusAttr) {
+  async updateBlobTakedownStatus(cid: CID, takedown: StatusAttr) {
     const takedownRef = takedown.applied
       ? takedown.ref ?? new Date().toISOString()
       : null
     await this.db.db
       .updateTable('blob')
       .set({ takedownRef })
-      .where('cid', '=', blob.toString())
+      .where('cid', '=', cid.toString())
       .executeTakeFirst()
+
     try {
+      // @NOTE find a way to not perform i/o operations during the transaction
+      // (typically by using a state in the "blob" table, and another process to
+      // handle the actual i/o)
       if (takedown.applied) {
-        await this.blobstore.quarantine(blob)
+        await this.blobstore.quarantine(cid)
       } else {
-        await this.blobstore.unquarantine(blob)
+        await this.blobstore.unquarantine(cid)
       }
     } catch (err) {
       if (!(err instanceof BlobNotFoundError)) {
+        log.error(
+          { err, cid: cid.toString() },
+          'could not update blob takedown status',
+        )
+
         throw err
       }
     }
@@ -154,21 +186,17 @@ export class BlobTransactor extends BlobReader {
     writes: PreparedWrite[],
     skipBlobStore?: boolean,
   ) {
-    const deletes = writes.filter(
-      (w) => w.action === WriteOpAction.Delete,
-    ) as PreparedDelete[]
-    const updates = writes.filter(
-      (w) => w.action === WriteOpAction.Update,
-    ) as PreparedUpdate[]
+    const deletes = writes.filter(isDelete)
+    const updates = writes.filter(isUpdate)
     const uris = [...deletes, ...updates].map((w) => w.uri.toString())
     if (uris.length === 0) return
 
     const deletedRepoBlobs = await this.db.db
       .deleteFrom('record_blob')
       .where('recordUri', 'in', uris)
-      .returningAll()
+      .returning('blobCid')
       .execute()
-    if (deletedRepoBlobs.length < 1) return
+    if (deletedRepoBlobs.length === 0) return
 
     const deletedRepoBlobCids = deletedRepoBlobs.map((row) => row.blobCid)
     const duplicateCids = await this.db.db
@@ -178,53 +206,85 @@ export class BlobTransactor extends BlobReader {
       .execute()
 
     const newBlobCids = writes
-      .map((w) =>
-        w.action === WriteOpAction.Create || w.action === WriteOpAction.Update
-          ? w.blobs
-          : [],
-      )
-      .flat()
-      .map((b) => b.cid.toString())
+      .filter((w) => isUpdate(w) || isCreate(w))
+      .flatMap((w) => w.blobs.map((b) => b.cid.toString()))
+
     const cidsToKeep = [
       ...newBlobCids,
       ...duplicateCids.map((row) => row.blobCid),
     ]
+
     const cidsToDelete = deletedRepoBlobCids.filter(
       (cid) => !cidsToKeep.includes(cid),
     )
-    if (cidsToDelete.length < 1) return
+    if (cidsToDelete.length === 0) return
 
     await this.db.db
       .deleteFrom('blob')
       .where('cid', 'in', cidsToDelete)
       .execute()
+
     if (!skipBlobStore) {
       this.db.onCommit(() => {
         this.backgroundQueue.add(async () => {
-          await Promise.allSettled(
-            cidsToDelete.map((cid) => this.blobstore.delete(CID.parse(cid))),
-          )
+          try {
+            const cids = cidsToDelete.map((cid) => CID.parse(cid))
+            await this.blobstore.deleteMany(cids)
+          } catch (err) {
+            log.error(
+              { err, cids: cidsToDelete },
+              'could not delete blobs from blobstore',
+            )
+          }
         })
       })
     }
   }
 
-  async verifyBlobAndMakePermanent(blob: PreparedBlobRef): Promise<void> {
+  async verifyBlobAndMakePermanent(
+    blob: PreparedBlobRef,
+    signal?: AbortSignal,
+  ): Promise<void> {
     const found = await this.db.db
       .selectFrom('blob')
-      .selectAll()
+      .select(['tempKey', 'size', 'mimeType'])
       .where('cid', '=', blob.cid.toString())
       .where('takedownRef', 'is', null)
       .executeTakeFirst()
+
+    signal?.throwIfAborted()
+
     if (!found) {
       throw new InvalidRequestError(
         `Could not find blob: ${blob.cid.toString()}`,
         'BlobNotFound',
       )
     }
+
     if (found.tempKey) {
       verifyBlob(blob, found)
-      await this.blobstore.makePermanent(found.tempKey, blob.cid)
+
+      // @NOTE it is less than ideal to perform async (i/o) operations during a
+      // transaction. Especially since there have been instances of the actor-db
+      // being locked, requiring to kick the processes.
+
+      // The better solution would be to update the blob state in the database
+      // (e.g. "makeItPermanent") and to process those updates outside of the
+      // transaction.
+
+      await this.blobstore
+        .makePermanent(found.tempKey, blob.cid)
+        .catch((err) => {
+          log.error(
+            { err, cid: blob.cid.toString() },
+            'could not make blob permanent',
+          )
+
+          throw err
+        })
+
+      signal?.throwIfAborted()
+
       await this.db.db
         .updateTable('blob')
         .set({ tempKey: null })
@@ -300,7 +360,10 @@ function acceptedMime(mime: string, accepted: string[]): boolean {
   return accepted.includes(mime)
 }
 
-function verifyBlob(blob: PreparedBlobRef, found: BlobTable) {
+function verifyBlob(
+  blob: PreparedBlobRef,
+  found: Pick<BlobTable, 'size' | 'mimeType'>,
+) {
   const throwInvalid = (msg: string, errName = 'InvalidBlob') => {
     throw new InvalidRequestError(msg, errName)
   }
@@ -328,3 +391,13 @@ function verifyBlob(blob: PreparedBlobRef, found: BlobTable) {
     )
   }
 }
+
+function isCreate(write: PreparedWrite) {
+  return write.action === WriteOpAction.Create
+}
+function isUpdate(write: PreparedWrite) {
+  return write.action === WriteOpAction.Update
+}
+function isDelete(write: PreparedWrite) {
+  return write.action === WriteOpAction.Delete
+}

package/src/actor-store/record/reader.ts

@@ -213,19 +213,21 @@ export class RecordReader {
   // Ensures that we don't end-up with duplicate likes, reposts, and follows from race conditions.
 
   async getBacklinkConflicts(uri: AtUri, record: RepoRecord): Promise<AtUri[]> {
-    const recordBacklinks = getBacklinks(uri, record)
-    const conflicts = await Promise.all(
-      recordBacklinks.map((backlink) =>
-        this.getRecordBacklinks({
-          collection: uri.collection,
-          path: backlink.path,
-          linkTo: backlink.linkTo,
-        }),
-      ),
-    )
+    const conflicts: AtUri[] = []
+
+    for (const backlink of getBacklinks(uri, record)) {
+      const backlinks = await this.getRecordBacklinks({
+        collection: uri.collection,
+        path: backlink.path,
+        linkTo: backlink.linkTo,
+      })
+
+      for (const { rkey } of backlinks) {
+        conflicts.push(AtUri.make(uri.hostname, uri.collection, rkey))
+      }
+    }
+
     return conflicts
-      .flat()
-      .map(({ rkey }) => AtUri.make(uri.hostname, uri.collection, rkey))
   }
 
   async listExistingBlocks(): Promise<CidSet> {

package/src/actor-store/repo/sql-repo-reader.ts

@@ -59,20 +59,18 @@ export class SqlRepoReader extends ReadableBlockstore {
     const missing = new CidSet(cached.missing)
     const missingStr = cached.missing.map((c) => c.toString())
     const blocks = new BlockMap()
-    await Promise.all(
-      chunkArray(missingStr, 500).map(async (batch) => {
-        const res = await this.db.db
-          .selectFrom('repo_block')
-          .where('repo_block.cid', 'in', batch)
-          .select(['repo_block.cid as cid', 'repo_block.content as content'])
-          .execute()
-        for (const row of res) {
-          const cid = CID.parse(row.cid)
-          blocks.set(cid, row.content)
-          missing.delete(cid)
-        }
-      }),
-    )
+    for (const batch of chunkArray(missingStr, 500)) {
+      const res = await this.db.db
+        .selectFrom('repo_block')
+        .where('repo_block.cid', 'in', batch)
+        .select(['repo_block.cid as cid', 'repo_block.content as content'])
+        .execute()
+      for (const row of res) {
+        const cid = CID.parse(row.cid)
+        blocks.set(cid, row.content)
+        missing.delete(cid)
+      }
+    }
     this.cache.addMap(blocks)
     blocks.addMap(cached.blocks)
     return { blocks, missing: missing.toList() }

package/src/actor-store/repo/sql-repo-transactor.ts

@@ -45,24 +45,20 @@ export class SqlRepoTransactor extends SqlRepoReader implements RepoStorage {
   }
 
   async putMany(toPut: BlockMap, rev: string): Promise<void> {
-    const blocks: RepoBlock[] = []
-    toPut.forEach((bytes, cid) => {
-      blocks.push({
-        cid: cid.toString(),
-        repoRev: rev,
-        size: bytes.length,
-        content: bytes,
-      })
-    })
-    await Promise.all(
-      chunkArray(blocks, 50).map((batch) =>
-        this.db.db
-          .insertInto('repo_block')
-          .values(batch)
-          .onConflict((oc) => oc.doNothing())
-          .execute(),
-      ),
-    )
+    const blocks: RepoBlock[] = Array.from(toPut, ([cid, bytes]) => ({
+      cid: cid.toString(),
+      repoRev: rev,
+      size: bytes.length,
+      content: bytes,
+    }))
+
+    for (const batch of chunkArray(blocks, 50)) {
+      await this.db.db
+        .insertInto('repo_block')
+        .values(batch)
+        .onConflict((oc) => oc.doNothing())
+        .execute()
+    }
   }
 
   async deleteMany(cids: CID[]) {
@@ -75,11 +71,9 @@ export class SqlRepoTransactor extends SqlRepoReader implements RepoStorage {
   }
 
   async applyCommit(commit: CommitData, isCreate?: boolean) {
-    await Promise.all([
-      this.updateRoot(commit.cid, commit.rev, isCreate),
-      this.putMany(commit.newBlocks, commit.rev),
-      this.deleteMany(commit.removedCids.toList()),
-    ])
+    await this.updateRoot(commit.cid, commit.rev, isCreate)
+    await this.putMany(commit.newBlocks, commit.rev)
+    await this.deleteMany(commit.removedCids.toList())
   }
 
   async updateRoot(cid: CID, rev: string, isCreate = false): Promise<void> {

package/src/actor-store/repo/transactor.ts

@@ -55,11 +55,10 @@ export class RepoTransactor extends RepoReader {
       this.signingKey,
       writes.map(createWriteToOp),
     )
-    await Promise.all([
-      this.storage.applyCommit(commit, true),
-      this.indexWrites(writes, commit.rev),
-      this.blob.processWriteBlobs(commit.rev, writes),
-    ])
+    await this.storage.applyCommit(commit, true)
+    await this.indexWrites(writes, commit.rev)
+    await this.blob.processWriteBlobs(commit.rev, writes)
+
     const ops = writes.map((w) => ({
       action: 'create' as const,
       path: formatDataKey(w.uri.collection, w.uri.rkey),
@@ -87,14 +86,13 @@ export class RepoTransactor extends RepoReader {
       throw new InvalidRequestError('Too many writes. Max event size: 2MB')
     }
 
-    await Promise.all([
-      // persist the commit to repo storage
-      this.storage.applyCommit(commit),
-      // & send to indexing
-      this.indexWrites(writes, commit.rev),
-      // process blobs
-      this.blob.processWriteBlobs(commit.rev, writes),
-    ])
+    // persist the commit to repo storage
+    await this.storage.applyCommit(commit)
+    // & send to indexing
+    await this.indexWrites(writes, commit.rev)
+    // process blobs
+    await this.blob.processWriteBlobs(commit.rev, writes)
+
     return commit
   }
 
@@ -184,25 +182,24 @@ export class RepoTransactor extends RepoReader {
 
   async indexWrites(writes: PreparedWrite[], rev: string) {
     this.db.assertTransaction()
-    await Promise.all(
-      writes.map(async (write) => {
-        if (
-          write.action === WriteOpAction.Create ||
-          write.action === WriteOpAction.Update
-        ) {
-          await this.record.indexRecord(
-            write.uri,
-            write.cid,
-            write.record,
-            write.action,
-            rev,
-            this.now,
-          )
-        } else if (write.action === WriteOpAction.Delete) {
-          await this.record.deleteRecord(write.uri)
-        }
-      }),
-    )
+
+    for (const write of writes) {
+      if (
+        write.action === WriteOpAction.Create ||
+        write.action === WriteOpAction.Update
+      ) {
+        await this.record.indexRecord(
+          write.uri,
+          write.cid,
+          write.record,
+          write.action,
+          rev,
+          this.now,
+        )
+      } else if (write.action === WriteOpAction.Delete) {
+        await this.record.deleteRecord(write.uri)
+      }
+    }
   }
 
   async getDuplicateRecordCids(

package/src/api/com/atproto/admin/updateSubjectStatus.ts

@@ -19,16 +19,16 @@ export default function (server: Server, ctx: AppContext) {
           await ctx.accountManager.takedownAccount(subject.did, takedown)
         } else if (isStrongRef(subject)) {
           const uri = new AtUri(subject.uri)
-          await ctx.actorStore.transact(uri.hostname, (store) =>
-            store.record.updateRecordTakedownStatus(uri, takedown),
-          )
+          await ctx.actorStore.transact(uri.hostname, async (store) => {
+            await store.record.updateRecordTakedownStatus(uri, takedown)
+          })
         } else if (isRepoBlobRef(subject)) {
-          await ctx.actorStore.transact(subject.did, (store) =>
-            store.repo.blob.updateBlobTakedownStatus(
+          await ctx.actorStore.transact(subject.did, async (store) => {
+            await store.repo.blob.updateBlobTakedownStatus(
               CID.parse(subject.cid),
               takedown,
-            ),
-          )
+            )
+          })
         } else {
           throw new InvalidRequestError('Invalid subject')
         }

package/src/api/com/atproto/repo/importRepo.ts

@@ -1,5 +1,4 @@
 import { CID } from 'multiformats/cid'
-import PQueue from 'p-queue'
 import { TID } from '@atproto/common'
 import { BlobRef, LexValue, RepoRecord } from '@atproto/lexicon'
 import {
@@ -11,7 +10,6 @@ import {
 } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
 import { InvalidRequestError } from '@atproto/xrpc-server'
-import { ActorStoreTransactor } from '../../../../actor-store/actor-store-transactor'
 import { ACCESS_FULL } from '../../../../auth-scope'
 import { AppContext } from '../../../../context'
 import { Server } from '../../../../lexicon'
@@ -32,53 +30,47 @@ export default function (server: Server, ctx: AppContext) {
 
       const { did } = auth.credentials
 
-      await ctx.actorStore.transact(did, (store) =>
-        importRepo(store, input.body),
-      )
-    },
-  })
-}
+      // @NOTE process as much as we can before the transaction, in particular
+      // the reading of the body stream.
+      const { roots, blocks } = await readCarStream(input.body)
+      if (roots.length !== 1) {
+        await blocks.dump()
+        throw new InvalidRequestError('expected one root')
+      }
 
-const importRepo = async (
-  actorStore: ActorStoreTransactor,
-  incomingCar: AsyncIterable<Uint8Array>,
-) => {
-  const now = new Date().toISOString()
-  const rev = TID.nextStr()
-  const did = actorStore.repo.did
+      const blockMap = new BlockMap()
+      for await (const block of blocks) {
+        blockMap.set(block.cid, block.bytes)
+      }
 
-  const { roots, blocks } = await readCarStream(incomingCar)
-  if (roots.length !== 1) {
-    await blocks.dump()
-    throw new InvalidRequestError('expected one root')
-  }
-  const blockMap = new BlockMap()
-  for await (const block of blocks) {
-    blockMap.set(block.cid, block.bytes)
-  }
-  const currRepo = await actorStore.repo.maybeLoadRepo()
-  const diff = await verifyDiff(
-    currRepo,
-    blockMap,
-    roots[0],
-    undefined,
-    undefined,
-    { ensureLeaves: false },
-  )
-  diff.commit.rev = rev
-  await actorStore.repo.storage.applyCommit(diff.commit, currRepo === null)
-  const recordQueue = new PQueue({ concurrency: 50 })
-  const controller = new AbortController()
-  for (const write of diff.writes) {
-    recordQueue
-      .add(
-        async () => {
+      await ctx.actorStore.transact(did, async (store) => {
+        const now = new Date().toISOString()
+        const rev = TID.nextStr()
+        const did = store.repo.did
+
+        const currRepo = await store.repo.maybeLoadRepo()
+        const diff = await verifyDiff(
+          currRepo,
+          blockMap,
+          roots[0],
+          undefined,
+          undefined,
+          { ensureLeaves: false },
+        )
+        diff.commit.rev = rev
+        await store.repo.storage.applyCommit(diff.commit, currRepo === null)
+
+        // @NOTE There is no point in performing the following concurrently
+        // since better-sqlite3 is synchronous.
+        for (const write of diff.writes) {
           const uri = AtUri.make(did, write.collection, write.rkey)
           if (write.action === WriteOpAction.Delete) {
-            await actorStore.record.deleteRecord(uri)
+            await store.record.deleteRecord(uri)
           } else {
             let parsedRecord: RepoRecord
             try {
+              // @NOTE getAndParseRecord returns a promise for historical
+              // reasons but it's internal processing is actually synchronous.
               const parsed = await getAndParseRecord(blockMap, write.cid)
               parsedRecord = parsed.record
             } catch {
@@ -86,7 +78,8 @@ const importRepo = async (
                 `Could not parse record at '${write.collection}/${write.rkey}'`,
               )
             }
-            const indexRecord = actorStore.record.indexRecord(
+
+            await store.record.indexRecord(
               uri,
               write.cid,
               parsedRecord,
@@ -95,19 +88,12 @@ const importRepo = async (
               now,
             )
             const recordBlobs = findBlobRefs(parsedRecord)
-            const indexRecordBlobs = actorStore.repo.blob.insertBlobs(
-              uri.toString(),
-              recordBlobs,
-            )
-            await Promise.all([indexRecord, indexRecordBlobs])
+            await store.repo.blob.insertBlobs(uri.toString(), recordBlobs)
           }
-        },
-        { signal: controller.signal },
-      )
-      .catch((err) => controller.abort(err))
-  }
-  await recordQueue.onIdle()
-  controller.signal.throwIfAborted()
+        }
+      })
+    },
+  })
 }
 
 export const findBlobRefs = (val: LexValue, layer = 0): BlobRef[] => {