@atproto/pds 0.5.11 → 0.5.14

package/src/actor-store/preference/reader.ts

@@ -1,48 +0,0 @@
-import { app } from '../../lexicons/index.js'
-import { ActorDb } from '../db/index.js'
-import {
-  PrefAllowedOptions,
-  getAgeFromDatestring,
-  prefAllowed,
-} from './util.js'
-
-export class PreferenceReader {
-  constructor(public db: ActorDb) {}
-
-  async getPreferences(
-    namespace: string,
-    opts: PrefAllowedOptions,
-  ): Promise<AccountPreference[]> {
-    const prefsRes = await this.db.db
-      .selectFrom('account_pref')
-      .orderBy('id')
-      .selectAll()
-      .execute()
-
-    const prefs = prefsRes
-      .filter((pref) => !namespace || prefMatchNamespace(namespace, pref.name))
-      .map((pref) => JSON.parse(pref.valueJson) as AccountPreference)
-    const personalDetailsPref = prefs.find(
-      app.bsky.actor.defs.personalDetailsPref.$isTypeOf,
-    )
-
-    if (personalDetailsPref?.birthDate) {
-      const age = getAgeFromDatestring(personalDetailsPref.birthDate)
-      prefs.push(
-        app.bsky.actor.defs.declaredAgePref.$build({
-          isOverAge13: age >= 13,
-          isOverAge16: age >= 16,
-          isOverAge18: age >= 18,
-        }),
-      )
-    }
-
-    return prefs.filter((pref) => prefAllowed(pref.$type, opts))
-  }
-}
-
-export type AccountPreference = app.bsky.actor.defs.Preferences[number]
-
-export const prefMatchNamespace = (namespace: string, fullname: string) => {
-  return fullname === namespace || fullname.startsWith(`${namespace}.`)
-}

package/src/actor-store/preference/transactor.ts

@@ -1,55 +0,0 @@
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import {
-  AccountPreference,
-  PreferenceReader,
-  prefMatchNamespace,
-} from './reader.js'
-import { PrefAllowedOptions, isReadOnlyPref, prefAllowed } from './util.js'
-
-export class PreferenceTransactor extends PreferenceReader {
-  async putPreferences(
-    values: AccountPreference[],
-    namespace: string,
-    opts: PrefAllowedOptions,
-  ): Promise<void> {
-    this.db.assertTransaction()
-    if (!values.every((value) => prefMatchNamespace(namespace, value.$type))) {
-      throw new InvalidRequestError(
-        `Some preferences are not in the ${namespace} namespace`,
-      )
-    }
-    const forbiddenPrefs = values.filter((val) => !prefAllowed(val.$type, opts))
-    if (forbiddenPrefs.length > 0) {
-      throw new InvalidRequestError(
-        `Do not have authorization to set preferences: ${forbiddenPrefs.map((p) => p.$type).join(', ')}`,
-      )
-    }
-    // get all current prefs for user and prep new pref rows
-    const allPrefs = await this.db.db
-      .selectFrom('account_pref')
-      .select(['id', 'name'])
-      .execute()
-    const putPrefs = values
-      .filter((value) => !isReadOnlyPref(value.$type))
-      .map((value) => {
-        return {
-          name: value.$type,
-          valueJson: JSON.stringify(value),
-        }
-      })
-    const allPrefIdsInNamespace = allPrefs
-      .filter((pref) => prefMatchNamespace(namespace, pref.name))
-      .filter((pref) => prefAllowed(pref.name, opts))
-      .map((pref) => pref.id)
-    // replace all prefs in given namespace
-    if (allPrefIdsInNamespace.length) {
-      await this.db.db
-        .deleteFrom('account_pref')
-        .where('id', 'in', allPrefIdsInNamespace)
-        .execute()
-    }
-    if (putPrefs.length) {
-      await this.db.db.insertInto('account_pref').values(putPrefs).execute()
-    }
-  }
-}

package/src/actor-store/preference/util.ts

@@ -1,39 +0,0 @@
-import { app } from '../../lexicons/index.js'
-
-export type PrefAllowedOptions = {
-  hasAccessFull?: boolean
-}
-
-export function prefAllowed(
-  prefType: string,
-  options?: PrefAllowedOptions,
-): boolean {
-  if (options?.hasAccessFull === true) {
-    return true
-  }
-
-  return !isFullAccessOnlyPref(prefType)
-}
-
-export function isFullAccessOnlyPref(
-  type: string,
-): type is typeof app.bsky.actor.defs.personalDetailsPref.$type {
-  return type === app.bsky.actor.defs.personalDetailsPref.$type
-}
-
-export function isReadOnlyPref(
-  type: string,
-): type is typeof app.bsky.actor.defs.declaredAgePref.$type {
-  return type === app.bsky.actor.defs.declaredAgePref.$type
-}
-
-export function getAgeFromDatestring(birthDate: string): number {
-  const bday = new Date(birthDate)
-  const today = new Date()
-  let age = today.getFullYear() - bday.getFullYear()
-  const m = today.getMonth() - bday.getMonth()
-  if (m < 0 || (m === 0 && today.getDate() < bday.getDate())) {
-    age--
-  }
-  return age
-}

package/src/actor-store/record/reader.ts

@@ -1,374 +0,0 @@
-import { Cid, LexMap, parseCid } from '@atproto/lex-data'
-import { CidSet, cborToLexRecord, formatDataKey } from '@atproto/repo'
-import {
-  AtUri,
-  AtUriString,
-  DatetimeString,
-  NsidString,
-  ensureValidAtUri,
-  ensureValidDid,
-} from '@atproto/syntax'
-import { countAll, notSoftDeletedClause } from '../../db/util.js'
-import { app, com } from '../../lexicons/index.js'
-import { LocalRecords } from '../../read-after-write/types.js'
-import { ActorDb, Backlink } from '../db/index.js'
-
-export type RecordDescript = {
-  uri: string
-  path: string
-  cid: Cid
-}
-
-export class RecordReader {
-  constructor(public db: ActorDb) {}
-
-  async recordCount(): Promise<number> {
-    const res = await this.db.db
-      .selectFrom('record')
-      .select(countAll.as('count'))
-      .executeTakeFirst()
-    return res?.count ?? 0
-  }
-
-  async listAll(): Promise<RecordDescript[]> {
-    const records: RecordDescript[] = []
-    let cursor: string | undefined = ''
-    while (cursor !== undefined) {
-      const res = await this.db.db
-        .selectFrom('record')
-        .select(['uri', 'cid'])
-        .where('uri', '>', cursor)
-        .orderBy('uri', 'asc')
-        .limit(1000)
-        .execute()
-      for (const row of res) {
-        const parsed = new AtUri(row.uri)
-        records.push({
-          uri: row.uri,
-          path: formatDataKey(parsed.collection, parsed.rkey),
-          cid: parseCid(row.cid),
-        })
-      }
-      cursor = res.at(-1)?.uri
-    }
-    return records
-  }
-
-  async listCollections(): Promise<NsidString[]> {
-    const collections = await this.db.db
-      .selectFrom('record')
-      .select('collection')
-      .groupBy('collection')
-      .execute()
-
-    return collections.map((row) => row.collection as NsidString)
-  }
-
-  async listRecordsForCollection(opts: {
-    collection: string
-    limit: number
-    reverse: boolean
-    cursor?: string
-    rkeyStart?: string
-    rkeyEnd?: string
-    includeSoftDeleted?: boolean
-  }): Promise<{ uri: AtUriString; cid: string; value: LexMap }[]> {
-    const {
-      collection,
-      limit,
-      reverse,
-      cursor,
-      rkeyStart,
-      rkeyEnd,
-      includeSoftDeleted = false,
-    } = opts
-
-    const { ref } = this.db.db.dynamic
-    let builder = this.db.db
-      .selectFrom('record')
-      .innerJoin('repo_block', 'repo_block.cid', 'record.cid')
-      .where('record.collection', '=', collection)
-      .$if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('record'))),
-      )
-      .orderBy('record.rkey', reverse ? 'asc' : 'desc')
-      .limit(limit)
-      .selectAll()
-
-    // prioritize cursor but fall back to soon-to-be-depcreated rkey start/end
-    if (cursor !== undefined) {
-      if (reverse) {
-        builder = builder.where('record.rkey', '>', cursor)
-      } else {
-        builder = builder.where('record.rkey', '<', cursor)
-      }
-    } else {
-      if (rkeyStart !== undefined) {
-        builder = builder.where('record.rkey', '>', rkeyStart)
-      }
-      if (rkeyEnd !== undefined) {
-        builder = builder.where('record.rkey', '<', rkeyEnd)
-      }
-    }
-    const res = await builder.execute()
-    return res.map((row) => ({
-      uri: row.uri as AtUriString,
-      cid: row.cid,
-      value: cborToLexRecord(row.content),
-    }))
-  }
-
-  async getRecord(
-    uri: AtUri | AtUriString,
-    cid: string | null,
-    includeSoftDeleted = false,
-  ): Promise<{
-    uri: string
-    cid: string
-    value: LexMap
-    indexedAt: string
-    takedownRef: string | null
-  } | null> {
-    const { ref } = this.db.db.dynamic
-    let builder = this.db.db
-      .selectFrom('record')
-      .innerJoin('repo_block', 'repo_block.cid', 'record.cid')
-      .where('record.uri', '=', uri.toString())
-      .selectAll()
-      .$if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('record'))),
-      )
-    if (cid) {
-      builder = builder.where('record.cid', '=', cid)
-    }
-    const record = await builder.executeTakeFirst()
-    if (!record) return null
-    return {
-      uri: record.uri,
-      cid: record.cid,
-      value: cborToLexRecord(record.content),
-      indexedAt: record.indexedAt,
-      takedownRef: record.takedownRef ? record.takedownRef.toString() : null,
-    }
-  }
-
-  async hasRecord(
-    uri: AtUri,
-    cid: string | null,
-    includeSoftDeleted = false,
-  ): Promise<boolean> {
-    const { ref } = this.db.db.dynamic
-    let builder = this.db.db
-      .selectFrom('record')
-      .select('uri')
-      .where('record.uri', '=', uri.toString())
-      .$if(!includeSoftDeleted, (qb) =>
-        qb.where(notSoftDeletedClause(ref('record'))),
-      )
-    if (cid) {
-      builder = builder.where('record.cid', '=', cid)
-    }
-    const record = await builder.executeTakeFirst()
-    return !!record
-  }
-
-  async getRecordTakedownStatus(
-    uri: AtUri,
-  ): Promise<com.atproto.admin.defs.StatusAttr | null> {
-    const res = await this.db.db
-      .selectFrom('record')
-      .select('takedownRef')
-      .where('uri', '=', uri.toString())
-      .executeTakeFirst()
-    if (!res) return null
-    return res.takedownRef
-      ? { applied: true, ref: res.takedownRef }
-      : { applied: false }
-  }
-
-  async getCurrentRecordCid(uri: AtUri): Promise<Cid | null> {
-    const res = await this.db.db
-      .selectFrom('record')
-      .select('cid')
-      .where('uri', '=', uri.toString())
-      .executeTakeFirst()
-    return res ? parseCid(res.cid) : null
-  }
-
-  async getRecordBacklinks(opts: {
-    collection: string
-    path: string
-    linkTo: string
-  }) {
-    const { collection, path, linkTo } = opts
-    return await this.db.db
-      .selectFrom('record')
-      .innerJoin('backlink', 'backlink.uri', 'record.uri')
-      .where('backlink.path', '=', path)
-      .where('backlink.linkTo', '=', linkTo)
-      .where('record.collection', '=', collection)
-      .selectAll('record')
-      .execute()
-  }
-
-  // @NOTE this logic is a placeholder until we allow users to specify these constraints themselves.
-  // Ensures that we don't end-up with duplicate likes, reposts, and follows from race conditions.
-
-  async getBacklinkConflicts(uri: AtUri, record: LexMap): Promise<AtUri[]> {
-    const conflicts: AtUri[] = []
-
-    for (const backlink of getBacklinks(uri, record)) {
-      const backlinks = await this.getRecordBacklinks({
-        collection: uri.collection,
-        path: backlink.path,
-        linkTo: backlink.linkTo,
-      })
-
-      for (const { rkey } of backlinks) {
-        conflicts.push(AtUri.make(uri.hostname, uri.collection, rkey))
-      }
-    }
-
-    return conflicts
-  }
-
-  async listExistingBlocks(): Promise<CidSet> {
-    const cids = new CidSet()
-    let cursor: string | undefined = ''
-    while (cursor !== undefined) {
-      const res = await this.db.db
-        .selectFrom('repo_block')
-        .select('cid')
-        .where('cid', '>', cursor)
-        .orderBy('cid', 'asc')
-        .limit(1000)
-        .execute()
-      for (const row of res) {
-        cids.add(parseCid(row.cid))
-      }
-      cursor = res.at(-1)?.cid
-    }
-    return cids
-  }
-
-  async getProfileRecord() {
-    const row = await this.db.db
-      .selectFrom('record')
-      .leftJoin('repo_block', 'repo_block.cid', 'record.cid')
-      .where('record.collection', '=', app.bsky.actor.profile.$type)
-      .where('record.rkey', '=', 'self')
-      .selectAll()
-      .executeTakeFirst()
-
-    if (!row?.content) return null
-
-    return cborToLexRecord(row.content) as app.bsky.actor.profile.Main
-  }
-
-  async getRecordsSinceRev(rev: string): Promise<LocalRecords> {
-    const result: LocalRecords = { count: 0, profile: null, posts: [] }
-
-    const res = await this.db.db
-      .selectFrom('record')
-      .innerJoin('repo_block', 'repo_block.cid', 'record.cid')
-      .select([
-        'repo_block.content',
-        'uri',
-        'repo_block.cid',
-        'record.indexedAt',
-      ])
-      .where('record.repoRev', '>', rev)
-      .limit(10)
-      .orderBy('record.repoRev', 'asc')
-      .execute()
-
-    // sanity check to ensure that the clock received is not before _all_ local records (for instance in case of account migration)
-    if (res.length > 0) {
-      const sanityCheckRes = await this.db.db
-        .selectFrom('record')
-        .selectAll()
-        .where('record.repoRev', '<=', rev)
-        .limit(1)
-        .executeTakeFirst()
-
-      if (!sanityCheckRes) {
-        return result
-      }
-    }
-
-    for (const cur of res) {
-      result.count++
-
-      const uri = new AtUri(cur.uri)
-      if (
-        uri.collection === app.bsky.actor.profile.$type &&
-        uri.rkey === 'self'
-      ) {
-        result.profile = {
-          uri,
-          cid: parseCid(cur.cid),
-          indexedAt: cur.indexedAt as DatetimeString,
-          record: cborToLexRecord(cur.content) as app.bsky.actor.profile.Main,
-        }
-      } else if (uri.collection === app.bsky.feed.post.$type) {
-        result.posts.push({
-          uri,
-          cid: parseCid(cur.cid),
-          indexedAt: cur.indexedAt as DatetimeString,
-          record: cborToLexRecord(cur.content) as app.bsky.feed.post.Main,
-        })
-      }
-    }
-
-    return result
-  }
-}
-
-// @NOTE in the future this can be replaced with a more generic routine that pulls backlinks based on lex docs.
-// For now we just want to ensure we're tracking links from follows, blocks, likes, and reposts.
-
-export const getBacklinks = (uri: AtUri, record: LexMap): Backlink[] => {
-  if (
-    record?.['$type'] === app.bsky.graph.follow.$type ||
-    record?.['$type'] === app.bsky.graph.block.$type
-  ) {
-    const subject = record['subject']
-    if (typeof subject !== 'string') {
-      return []
-    }
-    try {
-      ensureValidDid(subject)
-    } catch {
-      return []
-    }
-    return [
-      {
-        uri: uri.toString(),
-        path: 'subject',
-        linkTo: subject,
-      },
-    ]
-  }
-  if (
-    record?.['$type'] === app.bsky.feed.like.$type ||
-    record?.['$type'] === app.bsky.feed.repost.$type
-  ) {
-    const subject = record['subject']
-    if (typeof subject?.['uri'] !== 'string') {
-      return []
-    }
-    try {
-      ensureValidAtUri(subject['uri'])
-    } catch {
-      return []
-    }
-    return [
-      {
-        uri: uri.toString(),
-        path: 'subject.uri',
-        linkTo: subject['uri'],
-      },
-    ]
-  }
-  return []
-}

package/src/actor-store/record/transactor.ts

@@ -1,111 +0,0 @@
-import { Cid, LexMap } from '@atproto/lex-data'
-import { BlobStore, WriteOpAction } from '@atproto/repo'
-import { AtUri } from '@atproto/syntax'
-import { com } from '../../lexicons/index.js'
-import { dbLogger as log } from '../../logger.js'
-import { ActorDb, Backlink } from '../db/index.js'
-import { RecordReader, getBacklinks } from './reader.js'
-
-export class RecordTransactor extends RecordReader {
-  constructor(
-    public db: ActorDb,
-    public blobstore: BlobStore,
-  ) {
-    super(db)
-  }
-
-  async indexRecord(
-    uri: AtUri,
-    cid: Cid,
-    record: LexMap | null,
-    action: WriteOpAction.Create | WriteOpAction.Update = WriteOpAction.Create,
-    repoRev: string,
-    timestamp?: string,
-  ) {
-    log.debug({ uri }, 'indexing record')
-    const row = {
-      uri: uri.toString(),
-      cid: cid.toString(),
-      collection: uri.collection,
-      rkey: uri.rkey,
-      repoRev: repoRev,
-      indexedAt: timestamp || new Date().toISOString(),
-    }
-    if (!uri.hostname.startsWith('did:')) {
-      throw new Error('Expected indexed URI to contain DID')
-    } else if (row.collection.length < 1) {
-      throw new Error('Expected indexed URI to contain a collection')
-    } else if (row.rkey.length < 1) {
-      throw new Error('Expected indexed URI to contain a record key')
-    }
-
-    // Track current version of record
-    await this.db.db
-      .insertInto('record')
-      .values(row)
-      .onConflict((oc) =>
-        oc.column('uri').doUpdateSet({
-          cid: row.cid,
-          repoRev: repoRev,
-          indexedAt: row.indexedAt,
-        }),
-      )
-      .execute()
-
-    if (record !== null) {
-      // Maintain backlinks
-      const backlinks = getBacklinks(uri, record)
-      if (action === WriteOpAction.Update) {
-        // On update just recreate backlinks from scratch for the record, so we can clear out
-        // the old ones. E.g. for weird cases like updating a follow to be for a different did.
-        await this.removeBacklinksByUri(uri)
-      }
-      await this.addBacklinks(backlinks)
-    }
-
-    log.info({ uri }, 'indexed record')
-  }
-
-  async deleteRecord(uri: AtUri) {
-    log.debug({ uri }, 'deleting indexed record')
-    const deleteQuery = this.db.db
-      .deleteFrom('record')
-      .where('uri', '=', uri.toString())
-    const backlinkQuery = this.db.db
-      .deleteFrom('backlink')
-      .where('uri', '=', uri.toString())
-    await Promise.all([deleteQuery.execute(), backlinkQuery.execute()])
-
-    log.info({ uri }, 'deleted indexed record')
-  }
-
-  async removeBacklinksByUri(uri: AtUri) {
-    await this.db.db
-      .deleteFrom('backlink')
-      .where('uri', '=', uri.toString())
-      .execute()
-  }
-
-  async addBacklinks(backlinks: Backlink[]) {
-    if (backlinks.length === 0) return
-    await this.db.db
-      .insertInto('backlink')
-      .values(backlinks)
-      .onConflict((oc) => oc.doNothing())
-      .execute()
-  }
-
-  async updateRecordTakedownStatus(
-    uri: AtUri,
-    takedown: com.atproto.admin.defs.StatusAttr,
-  ) {
-    const takedownRef = takedown.applied
-      ? takedown.ref ?? new Date().toISOString()
-      : null
-    await this.db.db
-      .updateTable('record')
-      .set({ takedownRef })
-      .where('uri', '=', uri.toString())
-      .executeTakeFirst()
-  }
-}

package/src/actor-store/repo/reader.ts

@@ -1,31 +0,0 @@
-import { BlobStore } from '@atproto/repo'
-import { SyncEvtData } from '../../repo/index.js'
-import { BlobReader } from '../blob/reader.js'
-import { ActorDb } from '../db/index.js'
-import { RecordReader } from '../record/reader.js'
-import { SqlRepoReader } from './sql-repo-reader.js'
-
-export class RepoReader {
-  blob: BlobReader
-  record: RecordReader
-  storage: SqlRepoReader
-
-  constructor(
-    public db: ActorDb,
-    public blobstore: BlobStore,
-  ) {
-    this.blob = new BlobReader(db, blobstore)
-    this.record = new RecordReader(db)
-    this.storage = new SqlRepoReader(db)
-  }
-
-  async getSyncEventData(): Promise<SyncEvtData> {
-    const root = await this.storage.getRootDetailed()
-    const { blocks } = await this.storage.getBlocks([root.cid])
-    return {
-      cid: root.cid,
-      rev: root.rev,
-      blocks,
-    }
-  }
-}