@atproto/pds 0.5.11 → 0.5.14

package/src/account-manager/scope-reference-getter.ts

@@ -1,106 +0,0 @@
-import { Redis } from 'ioredis'
-import { DAY, backoffMs, retry } from '@atproto/common'
-import { Client, XrpcError } from '@atproto/lex'
-import { InvalidTokenError, OAuthScope } from '@atproto/oauth-provider'
-import { UpstreamFailureError } from '@atproto/xrpc-server'
-import { CachedGetter, GetterOptions } from '@atproto-labs/simple-store'
-import { SimpleStoreMemory } from '@atproto-labs/simple-store-memory'
-import { SimpleStoreRedis } from '@atproto-labs/simple-store-redis'
-import { com } from '../lexicons.js'
-import { oauthLogger } from '../logger.js'
-
-const PREFIX = 'ref:'
-
-type ScopeReference = `${typeof PREFIX}${string}`
-const isScopeReference = (scope?: OAuthScope): scope is ScopeReference =>
-  scope != null && scope.startsWith(PREFIX) && !scope.includes(' ')
-
-const identity = <T>(value: T): T => value
-
-export class ScopeReferenceGetter extends CachedGetter<
-  ScopeReference,
-  OAuthScope
-> {
-  constructor(
-    protected readonly entryway: Client,
-    redis?: Redis,
-  ) {
-    super(
-      async (ref, options) => {
-        return retry(async () => this.fetchDereferencedScope(ref, options), {
-          maxRetries: 3,
-          getWaitMs: (n) => backoffMs(n, 250, 2000),
-          retryable: (err) =>
-            !options?.signal?.aborted &&
-            err instanceof XrpcError &&
-            err.shouldRetry(),
-        })
-      },
-      redis
-        ? new SimpleStoreRedis(redis, {
-            // tradeoff between wasted memory usage (by no longer used scopes)
-            // and amount of requests to entryway:
-            ttl: 1 * DAY,
-
-            keyPrefix: `auth-scope-${PREFIX}`,
-            encode: identity,
-            decode: identity,
-          })
-        : new SimpleStoreMemory({ max: 1000 }),
-    )
-  }
-
-  protected async fetchDereferencedScope(
-    ref: ScopeReference,
-    opts?: GetterOptions,
-  ): Promise<OAuthScope> {
-    oauthLogger.info({ ref }, 'Fetching scope reference')
-
-    try {
-      const { scope } = await this.entryway.call(
-        com.atproto.temp.dereferenceScope,
-        { scope: ref },
-        {
-          signal: opts?.signal,
-          headers: opts?.noCache ? { 'Cache-Control': 'no-cache' } : undefined,
-        },
-      )
-
-      oauthLogger.info({ ref, scope }, 'Successfully fetched scope reference')
-
-      // @NOTE the part after `PREFIX` (in the input scope) is the CID of the
-      // scope string returned by entryway. Since there is a trust
-      // relationship with the entryway, we don't need to verify or enforce
-      // that here.
-
-      return scope
-    } catch (err) {
-      oauthLogger.error({ err, ref }, 'Failed to fetch scope reference')
-
-      throw err
-    }
-  }
-
-  async dereference(scope?: OAuthScope): Promise<undefined | OAuthScope> {
-    oauthLogger.debug({ scope }, 'Dereferencing scope')
-
-    if (!isScopeReference(scope)) return scope
-    return this.get(scope).catch(handleDereferenceError)
-  }
-}
-
-function handleDereferenceError(cause: unknown): never {
-  if (cause instanceof XrpcError && cause.error === 'InvalidScopeReference') {
-    // The scope reference cannot be found on the server.
-    // Consider the session as invalid, allowing entryway to
-    // re-build the scope as the user re-authenticates. This
-    // should never happen though.
-    throw InvalidTokenError.from(cause, 'DPoP')
-  }
-
-  throw new UpstreamFailureError(
-    'Failed to fetch token permissions',
-    undefined,
-    { cause },
-  )
-}

package/src/actor-store/actor-store-reader.ts

@@ -1,45 +0,0 @@
-import { Keypair } from '@atproto/crypto'
-import { ActorStoreResources } from './actor-store-resources.js'
-import { ActorStoreTransactor } from './actor-store-transactor.js'
-import { ActorDb } from './db/index.js'
-import { PreferenceReader } from './preference/reader.js'
-import { RecordReader } from './record/reader.js'
-import { RepoReader } from './repo/reader.js'
-
-export class ActorStoreReader {
-  public readonly repo: RepoReader
-  public readonly record: RecordReader
-  public readonly pref: PreferenceReader
-
-  constructor(
-    public readonly did: string,
-    protected readonly db: ActorDb,
-    protected readonly resources: ActorStoreResources,
-    public readonly keypair: () => Promise<Keypair>,
-  ) {
-    const blobstore = resources.blobstore(did)
-
-    this.repo = new RepoReader(db, blobstore)
-    this.record = new RecordReader(db)
-    this.pref = new PreferenceReader(db)
-
-    // Invoke "keypair" once. Also avoids leaking "this" as keypair context.
-    let keypairPromise: Promise<Keypair>
-    this.keypair = () => (keypairPromise ??= Promise.resolve().then(keypair))
-  }
-
-  async transact<T>(
-    fn: (fn: ActorStoreTransactor) => T | PromiseLike<T>,
-  ): Promise<T> {
-    const keypair = await this.keypair()
-    return this.db.transaction((dbTxn) => {
-      const store = new ActorStoreTransactor(
-        this.did,
-        dbTxn,
-        keypair,
-        this.resources,
-      )
-      return fn(store)
-    })
-  }
-}

package/src/actor-store/actor-store-resources.ts

@@ -1,8 +0,0 @@
-import { BlobStore } from '@atproto/repo'
-import { BackgroundQueue } from '../background.js'
-
-export type ActorStoreResources = {
-  blobstore: (did: string) => BlobStore
-  backgroundQueue: BackgroundQueue
-  reservedKeyDir?: string
-}

package/src/actor-store/actor-store-transactor.ts

@@ -1,31 +0,0 @@
-import { Keypair } from '@atproto/crypto'
-import { ActorStoreResources } from './actor-store-resources.js'
-import { ActorDb } from './db/index.js'
-import { PreferenceTransactor } from './preference/transactor.js'
-import { RecordTransactor } from './record/transactor.js'
-import { RepoTransactor } from './repo/transactor.js'
-
-export class ActorStoreTransactor {
-  public readonly record: RecordTransactor
-  public readonly repo: RepoTransactor
-  public readonly pref: PreferenceTransactor
-
-  constructor(
-    public readonly did: string,
-    protected readonly db: ActorDb,
-    protected readonly keypair: Keypair,
-    protected readonly resources: ActorStoreResources,
-  ) {
-    const blobstore = resources.blobstore(did)
-
-    this.record = new RecordTransactor(db, blobstore)
-    this.pref = new PreferenceTransactor(db)
-    this.repo = new RepoTransactor(
-      db,
-      blobstore,
-      did,
-      keypair,
-      resources.backgroundQueue,
-    )
-  }
-}

package/src/actor-store/actor-store-writer.ts

@@ -1,17 +0,0 @@
-import { ActorStoreTransactor } from './actor-store-transactor.js'
-
-export class ActorStoreWriter extends ActorStoreTransactor {
-  async transact<T>(
-    fn: (fn: ActorStoreTransactor) => T | PromiseLike<T>,
-  ): Promise<T> {
-    return this.db.transaction((dbTxn) => {
-      const transactor = new ActorStoreTransactor(
-        this.did,
-        dbTxn,
-        this.keypair,
-        this.resources,
-      )
-      return fn(transactor)
-    })
-  }
-}

package/src/actor-store/actor-store.ts

@@ -1,210 +0,0 @@
-import assert from 'node:assert'
-import fs, { mkdir } from 'node:fs/promises'
-import path from 'node:path'
-import { fileExists, readIfExists, rmIfExists } from '@atproto/common'
-import * as crypto from '@atproto/crypto'
-import { ExportableKeypair, Keypair } from '@atproto/crypto'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { ActorStoreConfig } from '../config/index.js'
-import { retrySqlite } from '../db/index.js'
-import { DiskBlobStore } from '../disk-blobstore.js'
-import { blobStoreLogger } from '../logger.js'
-import { ActorStoreReader } from './actor-store-reader.js'
-import { ActorStoreResources } from './actor-store-resources.js'
-import { ActorStoreTransactor } from './actor-store-transactor.js'
-import { ActorStoreWriter } from './actor-store-writer.js'
-import { ActorDb, getDb, getMigrator } from './db/index.js'
-
-export class ActorStore {
-  reservedKeyDir: string
-
-  constructor(
-    public cfg: ActorStoreConfig,
-    public resources: ActorStoreResources,
-  ) {
-    this.reservedKeyDir = path.join(cfg.directory, 'reserved_keys')
-  }
-
-  async getLocation(did: string) {
-    const didHash = await crypto.sha256Hex(did)
-    const directory = path.join(this.cfg.directory, didHash.slice(0, 2), did)
-    const dbLocation = path.join(directory, `store.sqlite`)
-    const keyLocation = path.join(directory, `key`)
-    return { directory, dbLocation, keyLocation }
-  }
-
-  async exists(did: string): Promise<boolean> {
-    const location = await this.getLocation(did)
-    return await fileExists(location.dbLocation)
-  }
-
-  async keypair(did: string): Promise<Keypair> {
-    const { keyLocation } = await this.getLocation(did)
-    const privKey = await fs.readFile(keyLocation)
-    return crypto.Secp256k1Keypair.import(privKey)
-  }
-
-  async openDb(did: string): Promise<ActorDb> {
-    const { dbLocation } = await this.getLocation(did)
-    const exists = await fileExists(dbLocation)
-    if (!exists) {
-      throw new InvalidRequestError('Repo not found', 'NotFound')
-    }
-
-    const db = getDb(dbLocation, this.cfg.disableWalAutoCheckpoint)
-
-    // run a simple select with retry logic to ensure the db is ready (not in wal recovery mode)
-    try {
-      await retrySqlite(() =>
-        db.db.selectFrom('repo_root').selectAll().execute(),
-      )
-    } catch (err) {
-      db.close()
-      throw err
-    }
-
-    return db
-  }
-
-  async read<T>(did: string, fn: (fn: ActorStoreReader) => T | PromiseLike<T>) {
-    const db = await this.openDb(did)
-    try {
-      const getKeypair = () => this.keypair(did)
-      return await fn(new ActorStoreReader(did, db, this.resources, getKeypair))
-    } finally {
-      db.close()
-    }
-  }
-
-  async transact<T>(
-    did: string,
-    fn: (fn: ActorStoreTransactor) => T | PromiseLike<T>,
-  ) {
-    const keypair = await this.keypair(did)
-    const db = await this.openDb(did)
-    try {
-      return await db.transaction((dbTxn) => {
-        return fn(new ActorStoreTransactor(did, dbTxn, keypair, this.resources))
-      })
-    } finally {
-      db.close()
-    }
-  }
-
-  async writeNoTransaction<T>(
-    did: string,
-    fn: (fn: ActorStoreWriter) => T | PromiseLike<T>,
-  ) {
-    const keypair = await this.keypair(did)
-    const db = await this.openDb(did)
-    try {
-      return await fn(new ActorStoreWriter(did, db, keypair, this.resources))
-    } finally {
-      db.close()
-    }
-  }
-
-  async create(did: string, keypair: ExportableKeypair) {
-    const { directory, dbLocation, keyLocation } = await this.getLocation(did)
-    // ensure subdir exists
-    await mkdir(directory, { recursive: true })
-    const exists = await fileExists(dbLocation)
-    if (exists) {
-      throw new InvalidRequestError('Repo already exists', 'AlreadyExists')
-    }
-    const privKey = await keypair.export()
-    await fs.writeFile(keyLocation, privKey)
-
-    const db: ActorDb = getDb(dbLocation, this.cfg.disableWalAutoCheckpoint)
-    try {
-      await db.ensureWal()
-      const migrator = getMigrator(db)
-      await migrator.migrateToLatestOrThrow()
-    } finally {
-      db.close()
-    }
-  }
-
-  async destroy(did: string) {
-    const blobstore = this.resources.blobstore(did)
-    if (blobstore instanceof DiskBlobStore) {
-      await blobstore.deleteAll()
-    } else {
-      const cids = await this.read(did, async (store) =>
-        store.repo.blob.getBlobCids(),
-      )
-      await blobstore.deleteMany(cids).catch((err) => {
-        blobStoreLogger.error({ did, cids, err }, 'Failed to delete blobs')
-      })
-    }
-
-    const { directory } = await this.getLocation(did)
-    await rmIfExists(directory, true)
-  }
-
-  async reserveKeypair(did?: string): Promise<string> {
-    let keyLoc: string | undefined
-    if (did) {
-      assertSafePathPart(did)
-      keyLoc = path.join(this.reservedKeyDir, did)
-      const maybeKey = await loadKey(keyLoc)
-      if (maybeKey) {
-        return maybeKey.did()
-      }
-    }
-    const keypair = await crypto.Secp256k1Keypair.create({ exportable: true })
-    const keyDid = keypair.did()
-    keyLoc = keyLoc ?? path.join(this.reservedKeyDir, keyDid)
-    await mkdir(this.reservedKeyDir, { recursive: true })
-    await fs.writeFile(keyLoc, await keypair.export())
-    return keyDid
-  }
-
-  async getReservedKeypair(
-    signingKeyOrDid: string,
-  ): Promise<ExportableKeypair | undefined> {
-    return loadKey(path.join(this.reservedKeyDir, signingKeyOrDid))
-  }
-
-  async clearReservedKeypair(keyDid: string, did?: string) {
-    await rmIfExists(path.join(this.reservedKeyDir, keyDid))
-    if (did) {
-      await rmIfExists(path.join(this.reservedKeyDir, did))
-    }
-  }
-
-  async storePlcOp(did: string, op: Uint8Array) {
-    const { directory } = await this.getLocation(did)
-    const opLoc = path.join(directory, `did-op`)
-    await fs.writeFile(opLoc, op)
-  }
-
-  async getPlcOp(did: string): Promise<Uint8Array> {
-    const { directory } = await this.getLocation(did)
-    const opLoc = path.join(directory, `did-op`)
-    return await fs.readFile(opLoc)
-  }
-
-  async clearPlcOp(did: string) {
-    const { directory } = await this.getLocation(did)
-    const opLoc = path.join(directory, `did-op`)
-    await rmIfExists(opLoc)
-  }
-}
-
-const loadKey = async (loc: string): Promise<ExportableKeypair | undefined> => {
-  const privKey = await readIfExists(loc)
-  if (!privKey) return undefined
-  return crypto.Secp256k1Keypair.import(privKey, { exportable: true })
-}
-
-function assertSafePathPart(part: string) {
-  const normalized = path.normalize(part)
-  assert(
-    part === normalized &&
-      !part.startsWith('.') &&
-      !part.includes('/') &&
-      !part.includes('\\'),
-    `unsafe path part: ${part}`,
-  )
-}

package/src/actor-store/blob/reader.ts

@@ -1,160 +0,0 @@
-import stream from 'node:stream'
-import { Cid, parseCid } from '@atproto/lex-data'
-import { BlobNotFoundError, BlobStore } from '@atproto/repo'
-import { AtUriString } from '@atproto/syntax'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { countAll, countDistinct, notSoftDeletedClause } from '../../db/util.js'
-import { com } from '../../lexicons/index.js'
-import { ActorDb } from '../db/index.js'
-
-export class BlobReader {
-  constructor(
-    public db: ActorDb,
-    public blobstore: BlobStore,
-  ) {}
-
-  async getBlobMetadata(
-    cid: Cid,
-  ): Promise<{ size: number; mimeType?: `${string}/${string}` }> {
-    const { ref } = this.db.db.dynamic
-    const found = await this.db.db
-      .selectFrom('blob')
-      .selectAll()
-      .where('blob.cid', '=', cid.toString())
-      .where(notSoftDeletedClause(ref('blob')))
-      .executeTakeFirst()
-    if (!found) {
-      throw new InvalidRequestError('Blob not found')
-    }
-    return {
-      size: found.size,
-      mimeType: found.mimeType as `${string}/${string}` | undefined,
-    }
-  }
-
-  async getBlob(cid: Cid): Promise<{
-    size: number
-    mimeType?: `${string}/${string}`
-    stream: stream.Readable
-  }> {
-    const metadata = await this.getBlobMetadata(cid)
-    const stream = await this.blobstore.getStream(cid).catch((err) => {
-      if (err instanceof BlobNotFoundError) {
-        throw new InvalidRequestError('Blob not found')
-      }
-      throw err
-    })
-
-    return { ...metadata, stream }
-  }
-
-  async listBlobs(opts: {
-    since?: string
-    cursor?: string
-    limit: number
-  }): Promise<string[]> {
-    const { since, cursor, limit } = opts
-    let builder = this.db.db
-      .selectFrom('record_blob')
-      .select('blobCid')
-      .orderBy('blobCid', 'asc')
-      .groupBy('blobCid')
-      .limit(limit)
-    if (since) {
-      builder = builder
-        .innerJoin('record', 'record.uri', 'record_blob.recordUri')
-        .where('record.repoRev', '>', since)
-    }
-    if (cursor) {
-      builder = builder.where('blobCid', '>', cursor)
-    }
-    const res = await builder.execute()
-    return res.map((row) => row.blobCid)
-  }
-
-  async getBlobTakedownStatus(
-    cid: Cid,
-  ): Promise<com.atproto.admin.defs.StatusAttr | null> {
-    const res = await this.db.db
-      .selectFrom('blob')
-      .select('takedownRef')
-      .where('cid', '=', cid.toString())
-      .executeTakeFirst()
-    if (!res) return null
-    return res.takedownRef
-      ? { applied: true, ref: res.takedownRef }
-      : { applied: false }
-  }
-
-  async hasRecordsForBlob(cid: Cid): Promise<boolean> {
-    const res = await this.db.db
-      .selectFrom('record_blob')
-      .where('blobCid', '=', cid.toString())
-      .select('blobCid')
-      .limit(1)
-      .executeTakeFirst()
-    return res != null
-  }
-
-  async getBlobsForRecord(recordUri: string): Promise<string[]> {
-    const res = await this.db.db
-      .selectFrom('blob')
-      .innerJoin('record_blob', 'record_blob.blobCid', 'blob.cid')
-      .where('recordUri', '=', recordUri)
-      .select('blob.cid')
-      .execute()
-    return res.map((row) => row.cid)
-  }
-
-  async blobCount(): Promise<number> {
-    const res = await this.db.db
-      .selectFrom('blob')
-      .select(countAll.as('count'))
-      .executeTakeFirst()
-    return res?.count ?? 0
-  }
-
-  async recordBlobCount(): Promise<number> {
-    const { ref } = this.db.db.dynamic
-    const res = await this.db.db
-      .selectFrom('record_blob')
-      .select(countDistinct(ref('blobCid')).as('count'))
-      .executeTakeFirst()
-    return res?.count ?? 0
-  }
-
-  async listMissingBlobs(opts: {
-    cursor?: string
-    limit: number
-  }): Promise<{ cid: string; recordUri: AtUriString }[]> {
-    const { cursor, limit } = opts
-    let builder = this.db.db
-      .selectFrom('record_blob')
-      .where(({ not, exists, selectFrom }) =>
-        not(
-          exists(
-            selectFrom('blob')
-              .selectAll()
-              .whereRef('blob.cid', '=', 'record_blob.blobCid'),
-          ),
-        ),
-      )
-      .selectAll()
-      .orderBy('blobCid', 'asc')
-      .groupBy('blobCid')
-      .limit(limit)
-    if (cursor) {
-      builder = builder.where('blobCid', '>', cursor)
-    }
-    const res = await builder.execute()
-    return res.map((row) => ({
-      cid: row.blobCid,
-      recordUri: row.recordUri as AtUriString,
-    }))
-  }
-
-  async getBlobCids() {
-    const blobRows = await this.db.db.selectFrom('blob').select('cid').execute()
-    return blobRows.map((row) => parseCid(row.cid))
-  }
-}