@atproto/pds 0.4.91 → 0.4.92

package/src/actor-store/actor-store-transactor.ts

@@ -0,0 +1,31 @@
+import { Keypair } from '@atproto/crypto'
+import { ActorStoreResources } from './actor-store-resources'
+import { ActorDb } from './db'
+import { PreferenceTransactor } from './preference/transactor'
+import { RecordTransactor } from './record/transactor'
+import { RepoTransactor } from './repo/transactor'
+
+export class ActorStoreTransactor {
+  public readonly record: RecordTransactor
+  public readonly repo: RepoTransactor
+  public readonly pref: PreferenceTransactor
+
+  constructor(
+    public readonly did: string,
+    protected readonly db: ActorDb,
+    protected readonly keypair: Keypair,
+    protected readonly resources: ActorStoreResources,
+  ) {
+    const blobstore = resources.blobstore(did)
+
+    this.record = new RecordTransactor(db, blobstore)
+    this.pref = new PreferenceTransactor(db)
+    this.repo = new RepoTransactor(
+      db,
+      blobstore,
+      did,
+      keypair,
+      resources.backgroundQueue,
+    )
+  }
+}

package/src/actor-store/actor-store-writer.ts

@@ -0,0 +1,17 @@
+import { ActorStoreTransactor } from './actor-store-transactor'
+
+export class ActorStoreWriter extends ActorStoreTransactor {
+  async transact<T>(
+    fn: (fn: ActorStoreTransactor) => T | PromiseLike<T>,
+  ): Promise<T> {
+    return this.db.transaction((dbTxn) => {
+      const transactor = new ActorStoreTransactor(
+        this.did,
+        dbTxn,
+        this.keypair,
+        this.resources,
+      )
+      return fn(transactor)
+    })
+  }
+}

package/src/actor-store/{index.ts → actor-store.ts}

@@ -1,35 +1,23 @@
-import path from 'path'
-import assert from 'assert'
-import fs from 'fs/promises'
-import * as crypto from '@atproto/crypto'
-import { Keypair, ExportableKeypair } from '@atproto/crypto'
-import { BlobStore } from '@atproto/repo'
+import assert from 'node:assert'
+import fs, { mkdir } from 'node:fs/promises'
+import path from 'node:path'
 import {
   chunkArray,
   fileExists,
   readIfExists,
   rmIfExists,
 } from '@atproto/common'
-import { ActorDb, getDb, getMigrator } from './db'
-import { BackgroundQueue } from '../background'
-import { RecordReader } from './record/reader'
-import { PreferenceReader } from './preference/reader'
-import { RepoReader } from './repo/reader'
-import { RepoTransactor } from './repo/transactor'
-import { PreferenceTransactor } from './preference/transactor'
+import * as crypto from '@atproto/crypto'
+import { ExportableKeypair, Keypair } from '@atproto/crypto'
 import { InvalidRequestError } from '@atproto/xrpc-server'
-import { RecordTransactor } from './record/transactor'
-import { CID } from 'multiformats/cid'
-import DiskBlobStore from '../disk-blobstore'
-import { mkdir } from 'fs/promises'
 import { ActorStoreConfig } from '../config'
 import { retrySqlite } from '../db'
-
-type ActorStoreResources = {
-  blobstore: (did: string) => BlobStore
-  backgroundQueue: BackgroundQueue
-  reservedKeyDir?: string
-}
+import { DiskBlobStore } from '../disk-blobstore'
+import { ActorStoreReader } from './actor-store-reader'
+import { ActorStoreResources } from './actor-store-resources'
+import { ActorStoreTransactor } from './actor-store-transactor'
+import { ActorStoreWriter } from './actor-store-writer'
+import { ActorDb, getDb, getMigrator } from './db'
 
 export class ActorStore {
   reservedKeyDir: string
@@ -82,50 +70,39 @@ export class ActorStore {
     return db
   }
 
-  async read<T>(did: string, fn: ActorStoreReadFn<T>) {
+  async read<T>(did: string, fn: (fn: ActorStoreReader) => T | PromiseLike<T>) {
     const db = await this.openDb(did)
     try {
-      const reader = createActorReader(did, db, this.resources, () =>
-        this.keypair(did),
-      )
-      return await fn(reader)
+      const getKeypair = () => this.keypair(did)
+      return await fn(new ActorStoreReader(did, db, this.resources, getKeypair))
     } finally {
       db.close()
     }
   }
 
-  async transact<T>(did: string, fn: ActorStoreTransactFn<T>) {
+  async transact<T>(
+    did: string,
+    fn: (fn: ActorStoreTransactor) => T | PromiseLike<T>,
+  ) {
     const keypair = await this.keypair(did)
     const db = await this.openDb(did)
     try {
       return await db.transaction((dbTxn) => {
-        const store = createActorTransactor(did, dbTxn, keypair, this.resources)
-        return fn(store)
+        return fn(new ActorStoreTransactor(did, dbTxn, keypair, this.resources))
       })
     } finally {
       db.close()
     }
   }
 
-  async writeNoTransaction<T>(did: string, fn: ActorStoreWriterFn<T>) {
+  async writeNoTransaction<T>(
+    did: string,
+    fn: (fn: ActorStoreWriter) => T | PromiseLike<T>,
+  ) {
     const keypair = await this.keypair(did)
     const db = await this.openDb(did)
     try {
-      const writer = createActorTransactor(did, db, keypair, this.resources)
-      return await fn({
-        ...writer,
-        transact: async <T>(fn: ActorStoreTransactFn<T>): Promise<T> => {
-          return db.transaction((dbTxn) => {
-            const transactor = createActorTransactor(
-              did,
-              dbTxn,
-              keypair,
-              this.resources,
-            )
-            return fn(transactor)
-          })
-        },
-      })
+      return await fn(new ActorStoreWriter(did, db, keypair, this.resources))
     } finally {
       db.close()
     }
@@ -157,10 +134,9 @@ export class ActorStore {
     if (blobstore instanceof DiskBlobStore) {
       await blobstore.deleteAll()
     } else {
-      const blobRows = await this.read(did, (store) =>
-        store.db.db.selectFrom('blob').select('cid').execute(),
+      const cids = await this.read(did, async (store) =>
+        store.repo.blob.getBlobCids(),
       )
-      const cids = blobRows.map((row) => CID.parse(row.cid))
       await Promise.allSettled(
         chunkArray(cids, 500).map((chunk) => blobstore.deleteMany(chunk)),
       )
@@ -226,73 +202,6 @@ const loadKey = async (loc: string): Promise<ExportableKeypair | undefined> => {
   return crypto.Secp256k1Keypair.import(privKey, { exportable: true })
 }
 
-const createActorTransactor = (
-  did: string,
-  db: ActorDb,
-  keypair: Keypair,
-  resources: ActorStoreResources,
-): ActorStoreTransactor => {
-  const { blobstore, backgroundQueue } = resources
-  const userBlobstore = blobstore(did)
-  return {
-    did,
-    db,
-    repo: new RepoTransactor(db, did, keypair, userBlobstore, backgroundQueue),
-    record: new RecordTransactor(db, userBlobstore),
-    pref: new PreferenceTransactor(db),
-  }
-}
-
-const createActorReader = (
-  did: string,
-  db: ActorDb,
-  resources: ActorStoreResources,
-  getKeypair: () => Promise<Keypair>,
-): ActorStoreReader => {
-  const { blobstore } = resources
-  return {
-    did,
-    db,
-    repo: new RepoReader(db, blobstore(did)),
-    record: new RecordReader(db),
-    pref: new PreferenceReader(db),
-    keypair: getKeypair,
-    transact: async <T>(fn: ActorStoreTransactFn<T>): Promise<T> => {
-      const keypair = await getKeypair()
-      return db.transaction((dbTxn) => {
-        const store = createActorTransactor(did, dbTxn, keypair, resources)
-        return fn(store)
-      })
-    },
-  }
-}
-
-export type ActorStoreReadFn<T> = (fn: ActorStoreReader) => Promise<T>
-export type ActorStoreTransactFn<T> = (fn: ActorStoreTransactor) => Promise<T>
-export type ActorStoreWriterFn<T> = (fn: ActorStoreWriter) => Promise<T>
-
-export type ActorStoreReader = {
-  did: string
-  db: ActorDb
-  repo: RepoReader
-  record: RecordReader
-  pref: PreferenceReader
-  keypair: () => Promise<Keypair>
-  transact: <T>(fn: ActorStoreTransactFn<T>) => Promise<T>
-}
-
-export type ActorStoreTransactor = {
-  did: string
-  db: ActorDb
-  repo: RepoTransactor
-  record: RecordTransactor
-  pref: PreferenceTransactor
-}
-
-export type ActorStoreWriter = ActorStoreTransactor & {
-  transact: <T>(fn: ActorStoreTransactFn<T>) => Promise<T>
-}
-
 function assertSafePathPart(part: string) {
   const normalized = path.normalize(part)
   assert(

package/src/actor-store/blob/reader.ts

@@ -1,10 +1,10 @@
-import stream from 'stream'
+import stream from 'node:stream'
 import { CID } from 'multiformats/cid'
 import { BlobNotFoundError, BlobStore } from '@atproto/repo'
 import { InvalidRequestError } from '@atproto/xrpc-server'
-import { ActorDb } from '../db'
 import { countAll, countDistinct, notSoftDeletedClause } from '../../db/util'
 import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
+import { ActorDb } from '../db'
 
 export class BlobReader {
   constructor(
@@ -95,6 +95,16 @@ export class BlobReader {
     return res.map((row) => row.recordUri)
   }
 
+  async getBlobsForRecord(recordUri: string): Promise<string[]> {
+    const res = await this.db.db
+      .selectFrom('blob')
+      .innerJoin('record_blob', 'record_blob.blobCid', 'blob.cid')
+      .where('recordUri', '=', recordUri)
+      .select('blob.cid')
+      .execute()
+    return res.map((row) => row.cid)
+  }
+
   async blobCount(): Promise<number> {
     const res = await this.db.db
       .selectFrom('blob')
@@ -138,4 +148,9 @@ export class BlobReader {
       recordUri: row.recordUri,
     }))
   }
+
+  async getBlobCids() {
+    const blobRows = await this.db.db.selectFrom('blob').select('cid').execute()
+    return blobRows.map((row) => CID.parse(row.cid))
+  }
 }

package/src/actor-store/blob/transactor.ts

@@ -1,24 +1,24 @@
-import stream from 'stream'
-import crypto from 'crypto'
-import { CID } from 'multiformats/cid'
+import crypto from 'node:crypto'
+import stream from 'node:stream'
 import bytes from 'bytes'
 import { fromStream as fileTypeFromStream } from 'file-type'
+import { CID } from 'multiformats/cid'
+import { cloneStream, sha256RawToCid, streamSize } from '@atproto/common'
+import { BlobRef } from '@atproto/lexicon'
 import { BlobNotFoundError, BlobStore, WriteOpAction } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
-import { cloneStream, sha256RawToCid, streamSize } from '@atproto/common'
 import { InvalidRequestError } from '@atproto/xrpc-server'
-import { BlobRef } from '@atproto/lexicon'
-import { ActorDb, Blob as BlobTable } from '../db'
+import { BackgroundQueue } from '../../background'
+import * as img from '../../image'
+import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
 import {
   PreparedBlobRef,
-  PreparedWrite,
   PreparedDelete,
   PreparedUpdate,
+  PreparedWrite,
 } from '../../repo/types'
-import * as img from '../../image'
-import { BackgroundQueue } from '../../background'
+import { ActorDb, Blob as BlobTable } from '../db'
 import { BlobReader } from './reader'
-import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
 
 export type BlobMetadata = {
   tempKey: string
@@ -38,6 +38,21 @@ export class BlobTransactor extends BlobReader {
     super(db, blobstore)
   }
 
+  async insertBlobs(recordUri: string, blobs: Iterable<BlobRef>) {
+    const values = Array.from(blobs, (cid) => ({
+      recordUri,
+      blobCid: cid.ref.toString(),
+    }))
+
+    if (values.length) {
+      await this.db.db
+        .insertInto('record_blob')
+        .values(values)
+        .onConflict((oc) => oc.doNothing())
+        .execute()
+    }
+  }
+
   async uploadBlobAndGetMetadata(
     userSuggestedMime: string,
     blobStream: stream.Readable,

package/src/actor-store/db/index.ts

@@ -1,6 +1,6 @@
-import { DatabaseSchema } from './schema'
 import { Database, Migrator } from '../../db'
 import migrations from './migrations'
+import { DatabaseSchema } from './schema'
 export * from './schema'
 
 export type ActorDb = Database<DatabaseSchema>

package/src/actor-store/db/schema/index.ts

@@ -1,10 +1,10 @@
 import * as accountPref from './account-pref'
-import * as repoRoot from './repo-root'
-import * as record from './record'
 import * as backlink from './backlink'
-import * as repoBlock from './repo-block'
 import * as blob from './blob'
+import * as record from './record'
 import * as recordBlob from './record-blob'
+import * as repoBlock from './repo-block'
+import * as repoRoot from './repo-root'
 
 export type DatabaseSchema = accountPref.PartialDB &
   repoRoot.PartialDB &

package/src/actor-store/migrate.ts

@@ -1,6 +1,6 @@
 import { sql } from 'kysely'
-import AppContext from '../context'
 import PQueue from 'p-queue'
+import { AppContext } from '../context'
 
 export const forEachActorStore = async (
   ctx: AppContext,

package/src/actor-store/preference/transactor.ts

@@ -1,10 +1,10 @@
 import { InvalidRequestError } from '@atproto/xrpc-server'
+import { AuthScope } from '../../auth-verifier'
 import {
-  PreferenceReader,
   AccountPreference,
+  PreferenceReader,
   prefMatchNamespace,
 } from './reader'
-import { AuthScope } from '../../auth-verifier'
 import { prefInScope } from './util'
 
 export class PreferenceTransactor extends PreferenceReader {

package/src/actor-store/record/reader.ts

@@ -1,12 +1,21 @@
+import { CID } from 'multiformats/cid'
+import { RepoRecord } from '@atproto/lexicon'
+import { CidSet, cborToLexRecord, formatDataKey } from '@atproto/repo'
 import * as syntax from '@atproto/syntax'
 import { AtUri, ensureValidAtUri } from '@atproto/syntax'
-import { cborToLexRecord } from '@atproto/repo'
-import { CID } from 'multiformats/cid'
 import { countAll, notSoftDeletedClause } from '../../db/util'
 import { ids } from '../../lexicon/lexicons'
-import { ActorDb, Backlink } from '../db'
+import { Record as ProfileRecord } from '../../lexicon/types/app/bsky/actor/profile'
+import { Record as PostRecord } from '../../lexicon/types/app/bsky/feed/post'
 import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
-import { RepoRecord } from '@atproto/lexicon'
+import { LocalRecords } from '../../read-after-write/types'
+import { ActorDb, Backlink } from '../db'
+
+export type RecordDescript = {
+  uri: string
+  path: string
+  cid: CID
+}
 
 export class RecordReader {
   constructor(public db: ActorDb) {}
@@ -19,6 +28,30 @@ export class RecordReader {
     return res?.count ?? 0
   }
 
+  async listAll(): Promise<RecordDescript[]> {
+    const records: RecordDescript[] = []
+    let cursor: string | undefined = ''
+    while (cursor !== undefined) {
+      const res = await this.db.db
+        .selectFrom('record')
+        .select(['uri', 'cid'])
+        .where('uri', '>', cursor)
+        .orderBy('uri', 'asc')
+        .limit(1000)
+        .execute()
+      for (const row of res) {
+        const parsed = new AtUri(row.uri)
+        records.push({
+          uri: row.uri,
+          path: formatDataKey(parsed.collection, parsed.rkey),
+          cid: CID.parse(row.cid),
+        })
+      }
+      cursor = res.at(-1)?.uri
+    }
+    return records
+  }
+
   async listCollections(): Promise<string[]> {
     const collections = await this.db.db
       .selectFrom('record')
@@ -194,6 +227,94 @@ export class RecordReader {
       .flat()
       .map(({ rkey }) => AtUri.make(uri.hostname, uri.collection, rkey))
   }
+
+  async listExistingBlocks(): Promise<CidSet> {
+    const cids = new CidSet()
+    let cursor: string | undefined = ''
+    while (cursor !== undefined) {
+      const res = await this.db.db
+        .selectFrom('repo_block')
+        .select('cid')
+        .where('cid', '>', cursor)
+        .orderBy('cid', 'asc')
+        .limit(1000)
+        .execute()
+      for (const row of res) {
+        cids.add(CID.parse(row.cid))
+      }
+      cursor = res.at(-1)?.cid
+    }
+    return cids
+  }
+
+  async getProfileRecord() {
+    const row = await this.db.db
+      .selectFrom('record')
+      .leftJoin('repo_block', 'repo_block.cid', 'record.cid')
+      .where('record.collection', '=', ids.AppBskyActorProfile)
+      .where('record.rkey', '=', 'self')
+      .selectAll()
+      .executeTakeFirst()
+
+    if (!row?.content) return null
+
+    return cborToLexRecord(row.content) as ProfileRecord
+  }
+
+  async getRecordsSinceRev(rev: string): Promise<LocalRecords> {
+    const result: LocalRecords = { count: 0, profile: null, posts: [] }
+
+    const res = await this.db.db
+      .selectFrom('record')
+      .innerJoin('repo_block', 'repo_block.cid', 'record.cid')
+      .select([
+        'repo_block.content',
+        'uri',
+        'repo_block.cid',
+        'record.indexedAt',
+      ])
+      .where('record.repoRev', '>', rev)
+      .limit(10)
+      .orderBy('record.repoRev', 'asc')
+      .execute()
+
+    // sanity check to ensure that the clock received is not before _all_ local records (for instance in case of account migration)
+    if (res.length > 0) {
+      const sanityCheckRes = await this.db.db
+        .selectFrom('record')
+        .selectAll()
+        .where('record.repoRev', '<=', rev)
+        .limit(1)
+        .executeTakeFirst()
+
+      if (!sanityCheckRes) {
+        return result
+      }
+    }
+
+    for (const cur of res) {
+      result.count++
+
+      const uri = new AtUri(cur.uri)
+      if (uri.collection === ids.AppBskyActorProfile && uri.rkey === 'self') {
+        result.profile = {
+          uri,
+          cid: CID.parse(cur.cid),
+          indexedAt: cur.indexedAt,
+          record: cborToLexRecord(cur.content) as ProfileRecord,
+        }
+      } else if (uri.collection === ids.AppBskyFeedPost) {
+        result.posts.push({
+          uri,
+          cid: CID.parse(cur.cid),
+          indexedAt: cur.indexedAt,
+          record: cborToLexRecord(cur.content) as PostRecord,
+        })
+      }
+    }
+
+    return result
+  }
 }
 
 // @NOTE in the future this can be replaced with a more generic routine that pulls backlinks based on lex docs.

package/src/actor-store/record/transactor.ts

@@ -1,11 +1,11 @@
 import { CID } from 'multiformats/cid'
-import { AtUri } from '@atproto/syntax'
+import { RepoRecord } from '@atproto/lexicon'
 import { BlobStore, WriteOpAction } from '@atproto/repo'
+import { AtUri } from '@atproto/syntax'
+import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
 import { dbLogger as log } from '../../logger'
 import { ActorDb, Backlink } from '../db'
 import { RecordReader, getBacklinks } from './reader'
-import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
-import { RepoRecord } from '@atproto/lexicon'
 
 export class RecordTransactor extends RecordReader {
   constructor(

package/src/actor-store/repo/reader.ts

@@ -1,8 +1,8 @@
 import { BlobStore } from '@atproto/repo'
-import { SqlRepoReader } from './sql-repo-reader'
 import { BlobReader } from '../blob/reader'
 import { ActorDb } from '../db'
 import { RecordReader } from '../record/reader'
+import { SqlRepoReader } from './sql-repo-reader'
 
 export class RepoReader {
   blob: BlobReader

package/src/actor-store/repo/sql-repo-reader.ts

@@ -1,14 +1,14 @@
+import { sql } from 'kysely'
+import { CID } from 'multiformats/cid'
+import { chunkArray, wait } from '@atproto/common'
 import {
   BlockMap,
   CidSet,
   ReadableBlockstore,
   writeCarStream,
 } from '@atproto/repo'
-import { chunkArray, wait } from '@atproto/common'
-import { CID } from 'multiformats/cid'
-import { ActorDb } from '../db'
-import { sql } from 'kysely'
 import { countAll } from '../../db'
+import { ActorDb } from '../db'
 
 export class SqlRepoReader extends ReadableBlockstore {
   cache: BlockMap = new BlockMap()
@@ -19,13 +19,14 @@ export class SqlRepoReader extends ReadableBlockstore {
 
   async getRoot(): Promise<CID> {
     const root = await this.getRootDetailed()
-    return root?.cid ?? null
+    return root.cid
   }
 
   async getRootDetailed(): Promise<{ cid: CID; rev: string }> {
     const res = await this.db.db
       .selectFrom('repo_root')
-      .selectAll()
+      .select(['cid', 'rev'])
+      .limit(1)
       .executeTakeFirstOrThrow()
     return {
       cid: CID.parse(res.cid),

package/src/actor-store/repo/sql-repo-transactor.ts

@@ -1,6 +1,6 @@
-import { CommitData, RepoStorage, BlockMap } from '@atproto/repo'
-import { chunkArray } from '@atproto/common'
 import { CID } from 'multiformats/cid'
+import { chunkArray } from '@atproto/common'
+import { BlockMap, CommitData, RepoStorage } from '@atproto/repo'
 import { ActorDb, RepoBlock } from '../db'
 import { SqlRepoReader } from './sql-repo-reader'
 

package/src/actor-store/repo/transactor.ts

@@ -1,9 +1,10 @@
 import { CID } from 'multiformats/cid'
 import * as crypto from '@atproto/crypto'
 import { BlobStore, CommitData, Repo, WriteOpAction } from '@atproto/repo'
-import { InvalidRequestError } from '@atproto/xrpc-server'
 import { AtUri } from '@atproto/syntax'
-import { SqlRepoTransactor } from './sql-repo-transactor'
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import { BackgroundQueue } from '../../background'
+import { createWriteToOp, writeToOp } from '../../repo'
 import {
   BadCommitSwapError,
   BadRecordSwapError,
@@ -11,31 +12,37 @@ import {
   PreparedWrite,
 } from '../../repo/types'
 import { BlobTransactor } from '../blob/transactor'
-import { createWriteToOp, writeToOp } from '../../repo'
-import { BackgroundQueue } from '../../background'
 import { ActorDb } from '../db'
 import { RecordTransactor } from '../record/transactor'
 import { RepoReader } from './reader'
+import { SqlRepoTransactor } from './sql-repo-transactor'
 
 export class RepoTransactor extends RepoReader {
   blob: BlobTransactor
   record: RecordTransactor
   storage: SqlRepoTransactor
-  now: string
 
   constructor(
     public db: ActorDb,
+    public blobstore: BlobStore,
     public did: string,
     public signingKey: crypto.Keypair,
-    public blobstore: BlobStore,
     public backgroundQueue: BackgroundQueue,
-    now?: string,
+    public now: string = new Date().toISOString(),
   ) {
     super(db, blobstore)
     this.blob = new BlobTransactor(db, blobstore, backgroundQueue)
     this.record = new RecordTransactor(db, blobstore)
-    this.now = now ?? new Date().toISOString()
-    this.storage = new SqlRepoTransactor(db, this.did, this.now)
+    this.storage = new SqlRepoTransactor(db, did, now)
+  }
+
+  async maybeLoadRepo(): Promise<Repo | null> {
+    const res = await this.db.db
+      .selectFrom('repo_root')
+      .select('cid')
+      .limit(1)
+      .executeTakeFirst()
+    return res ? Repo.load(this.storage, CID.parse(res.cid)) : null
   }
 
   async createRepo(writes: PreparedCreate[]): Promise<CommitData> {