@atproto/pds 0.5.12 → 0.5.14

package/src/actor-store/repo/sql-repo-reader.ts

@@ -1,150 +0,0 @@
-import { sql } from 'kysely'
-import { chunkArray } from '@atproto/common'
-import { Cid, parseCid } from '@atproto/lex-data'
-import {
-  BlockMap,
-  CarBlock,
-  CidSet,
-  ReadableBlockstore,
-  writeCarStream,
-} from '@atproto/repo'
-import { countAll } from '../../db/index.js'
-import { ActorDb } from '../db/index.js'
-
-export class SqlRepoReader extends ReadableBlockstore {
-  cache: BlockMap = new BlockMap()
-
-  constructor(public db: ActorDb) {
-    super()
-  }
-
-  async getRoot(): Promise<Cid> {
-    const root = await this.getRootDetailed()
-    return root.cid
-  }
-
-  async getRootDetailed(): Promise<{ cid: Cid; rev: string }> {
-    const res = await this.db.db
-      .selectFrom('repo_root')
-      .select(['cid', 'rev'])
-      .limit(1)
-      .executeTakeFirstOrThrow()
-    return {
-      cid: parseCid(res.cid),
-      rev: res.rev,
-    }
-  }
-
-  async getBytes(cid: Cid): Promise<Uint8Array | null> {
-    const cached = this.cache.get(cid)
-    if (cached) return cached
-    const found = await this.db.db
-      .selectFrom('repo_block')
-      .where('repo_block.cid', '=', cid.toString())
-      .select('content')
-      .executeTakeFirst()
-    if (!found) return null
-    this.cache.set(cid, found.content)
-    return found.content
-  }
-
-  async has(cid: Cid): Promise<boolean> {
-    const got = await this.getBytes(cid)
-    return !!got
-  }
-
-  async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
-    const cached = this.cache.getMany(cids)
-    if (cached.missing.length < 1) return cached
-    const missing = new CidSet(cached.missing)
-    const missingStr = cached.missing.map((c) => c.toString())
-    const blocks = new BlockMap()
-    for (const batch of chunkArray(missingStr, 500)) {
-      const res = await this.db.db
-        .selectFrom('repo_block')
-        .where('repo_block.cid', 'in', batch)
-        .select(['repo_block.cid as cid', 'repo_block.content as content'])
-        .execute()
-      for (const row of res) {
-        const cid = parseCid(row.cid)
-        blocks.set(cid, row.content)
-        missing.delete(cid)
-      }
-    }
-    this.cache.addMap(blocks)
-    blocks.addMap(cached.blocks)
-    return { blocks, missing: missing.toList() }
-  }
-
-  async getCarStream(since?: string) {
-    const root = await this.getRoot()
-    if (!root) {
-      throw new RepoRootNotFoundError()
-    }
-    return writeCarStream(root, this.iterateCarBlocks(since))
-  }
-
-  async *iterateCarBlocks(since?: string): AsyncIterable<CarBlock> {
-    let cursor: RevCursor | undefined = undefined
-    // allow us to write to car while fetching the next page
-    do {
-      const res = await this.getBlockRange(since, cursor)
-      for (const row of res) {
-        yield {
-          cid: parseCid(row.cid),
-          bytes: row.content,
-        }
-      }
-      const lastRow = res.at(-1)
-      if (lastRow && lastRow.repoRev) {
-        cursor = {
-          cid: parseCid(lastRow.cid),
-          rev: lastRow.repoRev,
-        }
-      } else {
-        cursor = undefined
-      }
-    } while (cursor)
-  }
-
-  async getBlockRange(since?: string, cursor?: RevCursor) {
-    const { ref } = this.db.db.dynamic
-    let builder = this.db.db
-      .selectFrom('repo_block')
-      .select(['cid', 'repoRev', 'content'])
-      .orderBy('repoRev', 'desc')
-      .orderBy('cid', 'desc')
-      .limit(500)
-    if (cursor) {
-      // use this syntax to ensure we hit the index
-      builder = builder.where(
-        sql<boolean>`((${ref('repoRev')}, ${ref('cid')}) < (${
-          cursor.rev
-        }, ${cursor.cid.toString()}))`,
-      )
-    }
-    if (since) {
-      builder = builder.where('repoRev', '>', since)
-    }
-    return builder.execute()
-  }
-
-  async countBlocks(): Promise<number> {
-    const res = await this.db.db
-      .selectFrom('repo_block')
-      .select(countAll.as('count'))
-      .executeTakeFirst()
-    return res?.count ?? 0
-  }
-
-  async destroy(): Promise<void> {
-    throw new Error('Destruction of SQL repo storage not allowed at runtime')
-  }
-}
-
-type RevCursor = {
-  cid: Cid
-  rev: string
-}
-
-export class RepoRootNotFoundError extends Error {}

package/src/actor-store/repo/sql-repo-transactor.ts

@@ -1,105 +0,0 @@
-import { chunkArray } from '@atproto/common'
-import { Cid, parseCid } from '@atproto/lex-data'
-import { BlockMap, CommitData, RepoStorage } from '@atproto/repo'
-import { ActorDb, RepoBlock } from '../db/index.js'
-import { SqlRepoReader } from './sql-repo-reader.js'
-
-export class SqlRepoTransactor extends SqlRepoReader implements RepoStorage {
-  cache = new BlockMap()
-  now: string
-
-  constructor(
-    public db: ActorDb,
-    public did: string,
-    now?: string,
-  ) {
-    super(db)
-    this.now = now ?? new Date().toISOString()
-  }
-
-  // proactively cache all blocks from a particular commit (to prevent multiple roundtrips)
-  async cacheRev(rev: string): Promise<void> {
-    const res = await this.db.db
-      .selectFrom('repo_block')
-      .where('repoRev', '=', rev)
-      .select(['repo_block.cid', 'repo_block.content'])
-      .limit(15)
-      .execute()
-    for (const row of res) {
-      this.cache.set(parseCid(row.cid), row.content)
-    }
-  }
-
-  async putBlock(cid: Cid, block: Uint8Array, rev: string): Promise<void> {
-    await this.db.db
-      .insertInto('repo_block')
-      .values({
-        cid: cid.toString(),
-        repoRev: rev,
-        size: block.length,
-        content: block,
-      })
-      .onConflict((oc) => oc.doNothing())
-      .execute()
-    this.cache.set(cid, block)
-  }
-
-  async putMany(toPut: BlockMap, rev: string): Promise<void> {
-    const blocks: RepoBlock[] = Array.from(toPut, ([cid, bytes]) => ({
-      cid: cid.toString(),
-      repoRev: rev,
-      size: bytes.length,
-      content: bytes,
-    }))
-
-    for (const batch of chunkArray(blocks, 50)) {
-      await this.db.db
-        .insertInto('repo_block')
-        .values(batch)
-        .onConflict((oc) => oc.doNothing())
-        .execute()
-    }
-  }
-
-  async deleteMany(cids: Cid[]) {
-    if (cids.length < 1) return
-    const cidStrs = cids.map((c) => c.toString())
-    await this.db.db
-      .deleteFrom('repo_block')
-      .where('cid', 'in', cidStrs)
-      .execute()
-  }
-
-  async applyCommit(commit: CommitData, isCreate?: boolean) {
-    await this.updateRoot(commit.cid, commit.rev, isCreate)
-    await this.putMany(commit.newBlocks, commit.rev)
-    await this.deleteMany(commit.removedCids.toList())
-  }
-
-  async updateRoot(cid: Cid, rev: string, isCreate = false): Promise<void> {
-    if (isCreate) {
-      await this.db.db
-        .insertInto('repo_root')
-        .values({
-          did: this.did,
-          cid: cid.toString(),
-          rev: rev,
-          indexedAt: this.now,
-        })
-        .execute()
-    } else {
-      await this.db.db
-        .updateTable('repo_root')
-        .set({
-          cid: cid.toString(),
-          rev: rev,
-          indexedAt: this.now,
-        })
-        .execute()
-    }
-  }
-
-  async destroy(): Promise<void> {
-    throw new Error('Destruction of SQL repo storage not allowed at runtime')
-  }
-}

package/src/actor-store/repo/transactor.ts

@@ -1,227 +0,0 @@
-import * as crypto from '@atproto/crypto'
-import { Cid, parseCid } from '@atproto/lex-data'
-import { BlobStore, Repo, WriteOpAction, formatDataKey } from '@atproto/repo'
-import { AtUri } from '@atproto/syntax'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { BackgroundQueue } from '../../background.js'
-import { createWriteToOp, writeToOp } from '../../repo/index.js'
-import {
-  BadCommitSwapError,
-  BadRecordSwapError,
-  CommitDataWithOps,
-  CommitOp,
-  PreparedCreate,
-  PreparedWrite,
-} from '../../repo/types.js'
-import { BlobTransactor } from '../blob/transactor.js'
-import { ActorDb } from '../db/index.js'
-import { RecordTransactor } from '../record/transactor.js'
-import { RepoReader } from './reader.js'
-import { SqlRepoTransactor } from './sql-repo-transactor.js'
-
-export class RepoTransactor extends RepoReader {
-  blob: BlobTransactor
-  record: RecordTransactor
-  storage: SqlRepoTransactor
-
-  constructor(
-    public db: ActorDb,
-    public blobstore: BlobStore,
-    public did: string,
-    public signingKey: crypto.Keypair,
-    public backgroundQueue: BackgroundQueue,
-    public now: string = new Date().toISOString(),
-  ) {
-    super(db, blobstore)
-    this.blob = new BlobTransactor(db, blobstore, backgroundQueue)
-    this.record = new RecordTransactor(db, blobstore)
-    this.storage = new SqlRepoTransactor(db, did, now)
-  }
-
-  async maybeLoadRepo(): Promise<Repo | null> {
-    const res = await this.db.db
-      .selectFrom('repo_root')
-      .select('cid')
-      .limit(1)
-      .executeTakeFirst()
-    return res ? Repo.load(this.storage, parseCid(res.cid)) : null
-  }
-
-  async createRepo(writes: PreparedCreate[]): Promise<CommitDataWithOps> {
-    this.db.assertTransaction()
-    const commit = await Repo.formatInitCommit(
-      this.storage,
-      this.did,
-      this.signingKey,
-      writes.map(createWriteToOp),
-    )
-    await this.storage.applyCommit(commit, true)
-    await this.indexWrites(writes, commit.rev)
-    await this.blob.processWriteBlobs(commit.rev, writes)
-
-    const ops = writes.map((w) => ({
-      action: 'create' as const,
-      path: formatDataKey(w.uri.collection, w.uri.rkey),
-      cid: w.cid,
-    }))
-    return {
-      ...commit,
-      ops,
-      prevData: null,
-    }
-  }
-
-  async processWrites(
-    writes: PreparedWrite[],
-    swapCommitCid?: Cid,
-  ): Promise<CommitDataWithOps> {
-    this.db.assertTransaction()
-    if (writes.length > 200) {
-      throw new InvalidRequestError('Too many writes. Max: 200')
-    }
-
-    const commit = await this.formatCommit(writes, swapCommitCid)
-    // Do not allow commits > 2MB
-    if (commit.relevantBlocks.byteSize > 2000000) {
-      throw new InvalidRequestError('Too many writes. Max event size: 2MB')
-    }
-
-    // persist the commit to repo storage
-    await this.storage.applyCommit(commit)
-    // & send to indexing
-    await this.indexWrites(writes, commit.rev)
-    // process blobs
-    await this.blob.processWriteBlobs(commit.rev, writes)
-
-    return commit
-  }
-
-  async formatCommit(
-    writes: PreparedWrite[],
-    swapCommit?: Cid,
-  ): Promise<CommitDataWithOps> {
-    // this is not in a txn, so this won't actually hold the lock,
-    // we just check if it is currently held by another txn
-    const currRoot = await this.storage.getRootDetailed()
-    if (!currRoot) {
-      throw new InvalidRequestError(`No repo root found for ${this.did}`)
-    }
-    if (swapCommit && !currRoot.cid.equals(swapCommit)) {
-      throw new BadCommitSwapError(currRoot.cid)
-    }
-    // cache last commit since there's likely overlap
-    await this.storage.cacheRev(currRoot.rev)
-    const newRecordCids: Cid[] = []
-    const delAndUpdateUris: AtUri[] = []
-    const commitOps: CommitOp[] = []
-    for (const write of writes) {
-      const { action, uri, swapCid } = write
-      if (action !== WriteOpAction.Delete) {
-        newRecordCids.push(write.cid)
-      }
-      if (action !== WriteOpAction.Create) {
-        delAndUpdateUris.push(uri)
-      }
-      const record = await this.record.getRecord(uri, null, true)
-      const currRecord = record ? parseCid(record.cid) : null
-
-      const op: CommitOp = {
-        action,
-        path: formatDataKey(uri.collection, uri.rkey),
-        cid: write.action === WriteOpAction.Delete ? null : write.cid,
-      }
-      if (currRecord) {
-        op.prev = currRecord
-      }
-      commitOps.push(op)
-      if (swapCid !== undefined) {
-        if (action === WriteOpAction.Create && swapCid !== null) {
-          throw new BadRecordSwapError(currRecord) // There should be no current record for a create
-        }
-        if (action === WriteOpAction.Update && swapCid === null) {
-          throw new BadRecordSwapError(currRecord) // There should be a current record for an update
-        }
-        if (action === WriteOpAction.Delete && swapCid === null) {
-          throw new BadRecordSwapError(currRecord) // There should be a current record for a delete
-        }
-        if (
-          // If we have either a current CID or a swap CID
-          (currRecord || swapCid) &&
-          // We make sure that both are provided and equal
-          !(swapCid && currRecord && currRecord.equals(swapCid))
-        ) {
-          throw new BadRecordSwapError(currRecord)
-        }
-      }
-    }
-
-    const repo = await Repo.load(this.storage, currRoot.cid)
-    const prevData = repo.commit.data
-    const writeOps = writes.map(writeToOp)
-    const commit = await repo.formatCommit(writeOps, this.signingKey)
-
-    // find blocks that would be deleted but are referenced by another record
-    const dupeRecordCids = await this.getDuplicateRecordCids(
-      commit.removedCids.toList(),
-      delAndUpdateUris,
-    )
-    for (const cid of dupeRecordCids) {
-      commit.removedCids.delete(cid)
-    }
-
-    // find blocks that are relevant to ops but not included in diff
-    // (for instance a record that was moved but cid stayed the same)
-    const newRecordBlocks = commit.relevantBlocks.getMany(newRecordCids)
-    if (newRecordBlocks.missing.length > 0) {
-      const missingBlocks = await this.storage.getBlocks(
-        newRecordBlocks.missing,
-      )
-      commit.relevantBlocks.addMap(missingBlocks.blocks)
-    }
-    return {
-      ...commit,
-      ops: commitOps,
-      prevData,
-    }
-  }
-
-  async indexWrites(writes: PreparedWrite[], rev: string) {
-    this.db.assertTransaction()
-
-    for (const write of writes) {
-      if (
-        write.action === WriteOpAction.Create ||
-        write.action === WriteOpAction.Update
-      ) {
-        await this.record.indexRecord(
-          write.uri,
-          write.cid,
-          write.record,
-          write.action,
-          rev,
-          this.now,
-        )
-      } else if (write.action === WriteOpAction.Delete) {
-        await this.record.deleteRecord(write.uri)
-      }
-    }
-  }
-
-  async getDuplicateRecordCids(
-    cids: Cid[],
-    touchedUris: AtUri[],
-  ): Promise<Cid[]> {
-    if (touchedUris.length === 0 || cids.length === 0) {
-      return []
-    }
-    const cidStrs = cids.map((c) => c.toString())
-    const uriStrs = touchedUris.map((u) => u.toString())
-    const res = await this.db.db
-      .selectFrom('record')
-      .where('cid', 'in', cidStrs)
-      .where('uri', 'not in', uriStrs)
-      .select('cid')
-      .execute()
-    return res.map((row) => parseCid(row.cid))
-  }
-}

package/src/api/app/bsky/actor/getPreferences.ts

@@ -1,57 +0,0 @@
-import { Server } from '@atproto/xrpc-server'
-import { AuthScope, isAccessFull } from '../../../../auth-scope.js'
-import { AppContext } from '../../../../context.js'
-import { app } from '../../../../lexicons/index.js'
-import {
-  bareDidFromProxyTo,
-  computeProxyTo,
-  pipethrough,
-} from '../../../../pipethrough.js'
-
-export default function (server: Server, ctx: AppContext) {
-  const { bskyAppView } = ctx
-  if (!bskyAppView) return
-
-  server.add(app.bsky.actor.getPreferences, {
-    auth: ctx.authVerifier.authorization({
-      additional: [AuthScope.Takendown],
-      authorize: (permissions, { req }) => {
-        const lxm = app.bsky.actor.getPreferences.$lxm
-        const aud = computeProxyTo(ctx, req, lxm)
-        permissions.assertRpc({ aud, lxm })
-      },
-    }),
-    handler: async ({ auth, req }) => {
-      const { did } = auth.credentials
-
-      // If the request has a proxy header different from the bsky app view,
-      // we need to proxy the request to the requested app view.
-      // @TODO This behavior should not be implemented as part of the XRPC framework
-      const lxm = app.bsky.actor.getPreferences.$lxm
-      const aud = computeProxyTo(ctx, req, lxm)
-      if (aud !== `${bskyAppView.did}#bsky_appview`) {
-        // Phase 1 of service auth updates: outbound JWT keeps bare-DID aud.
-        return pipethrough(ctx, req, {
-          iss: did,
-          aud: bareDidFromProxyTo(aud),
-          lxm,
-        })
-      }
-
-      const hasAccessFull =
-        auth.credentials.type === 'access' &&
-        isAccessFull(auth.credentials.scope)
-
-      const preferences = await ctx.actorStore.read(did, (store) => {
-        return store.pref.getPreferences('app.bsky', {
-          hasAccessFull,
-        })
-      })
-
-      return {
-        encoding: 'application/json' as const,
-        body: { preferences },
-      }
-    },
-  })
-}

package/src/api/app/bsky/actor/getProfile.ts

@@ -1,41 +0,0 @@
-import { Server } from '@atproto/xrpc-server'
-import { AppContext } from '../../../../context.js'
-import { app } from '../../../../lexicons/index.js'
-import { computeProxyTo } from '../../../../pipethrough.js'
-import {
-  MungeFn,
-  pipethroughReadAfterWrite,
-} from '../../../../read-after-write/index.js'
-
-export default function (server: Server, ctx: AppContext) {
-  if (!ctx.bskyAppView) return
-
-  server.add(app.bsky.actor.getProfile, {
-    auth: ctx.authVerifier.authorization({
-      authorize: (permissions, { req }) => {
-        const lxm = app.bsky.actor.getProfile.$lxm
-        const aud = computeProxyTo(ctx, req, lxm)
-        permissions.assertRpc({ aud, lxm })
-      },
-    }),
-    handler: async (reqCtx) => {
-      return pipethroughReadAfterWrite(
-        ctx,
-        reqCtx,
-        app.bsky.actor.getProfile,
-        getProfileMunge,
-      )
-    },
-  })
-}
-
-const getProfileMunge: MungeFn<app.bsky.actor.getProfile.$OutputBody> = async (
-  localViewer,
-  original,
-  local,
-  requester,
-) => {
-  if (!local.profile) return original
-  if (original.did !== requester) return original
-  return localViewer.updateProfileDetailed(original, local.profile.record)
-}

package/src/api/app/bsky/actor/getProfiles.ts

@@ -1,51 +0,0 @@
-import { Server } from '@atproto/xrpc-server'
-import { AppContext } from '../../../../context.js'
-import { app } from '../../../../lexicons/index.js'
-import { computeProxyTo } from '../../../../pipethrough.js'
-import {
-  MungeFn,
-  pipethroughReadAfterWrite,
-} from '../../../../read-after-write/index.js'
-
-export default function (server: Server, ctx: AppContext) {
-  if (!ctx.bskyAppView) return
-
-  server.add(app.bsky.actor.getProfiles, {
-    auth: ctx.authVerifier.authorization({
-      authorize: (permissions, { req }) => {
-        const lxm = app.bsky.actor.getProfiles.$lxm
-        const aud = computeProxyTo(ctx, req, lxm)
-        permissions.assertRpc({ aud, lxm })
-      },
-    }),
-    opts: {
-      // @TODO remove after grace period has passed, behavior is non-standard.
-      // temporarily added for compat w/ previous version of xrpc-server to avoid breakage of a few specified parties.
-      paramsParseLoose: true,
-    },
-    handler: async (reqCtx) => {
-      return pipethroughReadAfterWrite(
-        ctx,
-        reqCtx,
-        app.bsky.actor.getProfiles,
-        getProfilesMunge,
-      )
-    },
-  })
-}
-
-const getProfilesMunge: MungeFn<
-  app.bsky.actor.getProfiles.$OutputBody
-> = async (localViewer, original, local, requester) => {
-  const localProf = local.profile
-  if (!localProf) return original
-
-  const profiles = original.profiles.map((prof) => {
-    if (prof.did !== requester) return prof
-    return localViewer.updateProfileDetailed(prof, localProf.record)
-  })
-  return {
-    ...original,
-    profiles,
-  }
-}

package/src/api/app/bsky/actor/index.ts

@@ -1,13 +0,0 @@
-import { Server } from '@atproto/xrpc-server'
-import { AppContext } from '../../../../context.js'
-import getPreferences from './getPreferences.js'
-import getProfile from './getProfile.js'
-import getProfiles from './getProfiles.js'
-import putPreferences from './putPreferences.js'
-
-export default function (server: Server, ctx: AppContext) {
-  getPreferences(server, ctx)
-  getProfile(server, ctx)
-  getProfiles(server, ctx)
-  putPreferences(server, ctx)
-}