@atproto/pds 0.5.11 → 0.5.14

package/src/scripts/rotate-keys.ts

@@ -1,146 +0,0 @@
-import assert from 'node:assert'
-import fs from 'node:fs/promises'
-import * as plc from '@did-plc/lib'
-import PQueue from 'p-queue'
-import AtpAgent from '@atproto/api'
-import { Keypair } from '@atproto/crypto'
-import { IdResolver } from '@atproto/identity'
-import { DidString, isDidString } from '@atproto/lex'
-import { ActorStore } from '../actor-store/actor-store.js'
-import { SyncEvtData } from '../repo/index.js'
-import { Sequencer } from '../sequencer/index.js'
-import { getRecoveryDbFromSequencerLoc } from './sequencer-recovery/recovery-db.js'
-import { parseIntArg } from './util.js'
-
-export type RotateKeysContext = {
-  sequencer: Sequencer
-  actorStore: ActorStore
-  idResolver: IdResolver
-  plcClient: plc.Client
-  plcRotationKey: Keypair
-  entrywayAdminAgent?: AtpAgent
-}
-
-export const rotateKeys = async (ctx: RotateKeysContext, args: string[]) => {
-  const dids = args
-  assert(dids.every(isDidString), 'All arguments must be DIDs')
-  await rotateKeysForRepos(ctx, dids, 10)
-}
-
-export const rotateKeysFromFile = async (
-  ctx: RotateKeysContext,
-  args: string[],
-) => {
-  const filepath = args[0]
-  if (!filepath) {
-    throw new Error('Expected filepath as argument')
-  }
-  const concurrency = args[1] ? parseIntArg(args[1]) : 25
-  const file = await fs.readFile(filepath)
-  const dids = file
-    .toString()
-    .split('\n')
-    .map((did) => did.trim())
-    .filter((did) => did.startsWith('did:plc'))
-
-  assert(dids.every(isDidString), 'File contains invalid DIDs')
-
-  await rotateKeysForRepos(ctx, dids, concurrency)
-}
-
-export const rotateKeysRecovery = async (
-  ctx: RotateKeysContext,
-  args: string[],
-) => {
-  const concurrency = args[1] ? parseIntArg(args[0]) : 10
-
-  const recoveryDb = await getRecoveryDbFromSequencerLoc(
-    ctx.sequencer.dbLocation,
-  )
-  const rows = await recoveryDb.db
-    .selectFrom('new_account')
-    .select('did')
-    .where('new_account.published', '=', 0)
-    .execute()
-  const dids = rows.map((r) => r.did as DidString)
-
-  await rotateKeysForRepos(ctx, dids, concurrency, async (did) => {
-    await recoveryDb.db
-      .updateTable('new_account')
-      .set({ published: 1 })
-      .where('did', '=', did)
-      .execute()
-  })
-}
-
-const rotateKeysForRepos = async (
-  ctx: RotateKeysContext,
-  dids: DidString[],
-  concurrency: number,
-  onSuccess?: (did: DidString) => Promise<void>,
-) => {
-  const queue = new PQueue({ concurrency })
-  let completed = 0
-  for (const did of dids) {
-    queue.add(async () => {
-      try {
-        await updatePlcSigningKey(ctx, did)
-      } catch (err) {
-        console.error(`failed to update key for ${did}: ${err}`)
-        return
-      }
-      let syncData: SyncEvtData
-      try {
-        syncData = await ctx.actorStore.transact(did, async (actorTxn) => {
-          await actorTxn.repo.processWrites([])
-          return actorTxn.repo.getSyncEventData()
-        })
-      } catch (err) {
-        console.error(`failed to write new commit for ${did}: ${err}`)
-        return
-      }
-      try {
-        await ctx.sequencer.sequenceIdentity(did)
-      } catch (err) {
-        console.error(`failed to sequence new identity evt for ${did}: ${err}`)
-        return
-      }
-      try {
-        await ctx.sequencer.sequenceSync(did, syncData)
-      } catch (err) {
-        console.error(`failed to sequence for ${did}: ${err}`)
-        return
-      }
-      if (onSuccess) {
-        await onSuccess(did)
-      }
-      completed++
-      if (completed % 10 === 0) {
-        console.log(`${completed}/${dids.length}`)
-      }
-    })
-  }
-  await queue.onIdle()
-  console.log('DONE')
-}
-
-const updatePlcSigningKey = async (ctx: RotateKeysContext, did: DidString) => {
-  const updateTo = await ctx.actorStore.keypair(did)
-  const currSigningKey = await ctx.idResolver.did.resolveAtprotoKey(did, true)
-  if (updateTo.did() === currSigningKey) {
-    // already up to date
-    return
-  }
-  if (ctx.entrywayAdminAgent) {
-    await ctx.entrywayAdminAgent.api.com.atproto.admin.updateAccountSigningKey({
-      did,
-      signingKey: updateTo.did(),
-    })
-  } else {
-    await ctx.plcClient.updateAtprotoKey(
-      did,
-      ctx.plcRotationKey,
-      updateTo.did(),
-    )
-  }
-}

package/src/scripts/sequencer-recovery/index.ts

@@ -1,23 +0,0 @@
-import { parseIntArg } from '../util.js'
-import { Recoverer, RecovererContextNoDb } from './recoverer.js'
-import { getRecoveryDbFromSequencerLoc } from './recovery-db.js'
-
-export const sequencerRecovery = async (
-  ctx: RecovererContextNoDb,
-  args: string[],
-) => {
-  const cursor = args[0] ? parseIntArg(args[0]) : 0
-  const concurrency = args[1] ? parseIntArg(args[1]) : 10
-
-  const recoveryDb = await getRecoveryDbFromSequencerLoc(
-    ctx.sequencer.dbLocation,
-  )
-
-  const recover = new Recoverer(
-    { ...ctx, recoveryDb },
-    {
-      concurrency,
-    },
-  )
-  await recover.run(cursor)
-}

package/src/scripts/sequencer-recovery/recoverer.ts

@@ -1,297 +0,0 @@
-import { rmIfExists } from '@atproto/common'
-import { Secp256k1Keypair } from '@atproto/crypto'
-import { DidString, isNsidString, isRecordKeyString } from '@atproto/lex'
-import {
-  BlockMap,
-  CidSet,
-  CommitData,
-  WriteOpAction,
-  cborToLexRecord,
-  parseDataKey,
-  readCar,
-} from '@atproto/repo'
-import {
-  AccountManager,
-  AccountStatus,
-} from '../../account-manager/account-manager.js'
-import { ActorStoreTransactor } from '../../actor-store/actor-store-transactor.js'
-import { ActorStore } from '../../actor-store/actor-store.js'
-import { countAll } from '../../db/index.js'
-import {
-  PreparedWrite,
-  prepareCreate,
-  prepareDelete,
-  prepareUpdate,
-} from '../../repo/index.js'
-import {
-  AccountEvt,
-  CommitEvt,
-  SeqEvt,
-  Sequencer,
-} from '../../sequencer/index.js'
-import { RecoveryDb } from './recovery-db.js'
-import { UserQueues } from './user-queues.js'
-
-export type RecovererContextNoDb = {
-  sequencer: Sequencer
-  accountManager: AccountManager
-  actorStore: ActorStore
-}
-
-export type RecovererContext = RecovererContextNoDb & {
-  recoveryDb: RecoveryDb
-}
-
-const PAGE_SIZE = 5000
-
-export class Recoverer {
-  queues: UserQueues
-  failed: Set<string>
-
-  constructor(
-    public ctx: RecovererContext,
-    opts: { concurrency: number },
-  ) {
-    this.queues = new UserQueues(opts.concurrency)
-    this.failed = new Set()
-  }
-
-  async run(startCursor = 0) {
-    const failed = await this.ctx.recoveryDb.db
-      .selectFrom('failed')
-      .select('did')
-      .execute()
-    for (const row of failed) {
-      this.failed.add(row.did)
-    }
-
-    const totalRes = await this.ctx.sequencer.db.db
-      .selectFrom('repo_seq')
-      .select(countAll.as('count'))
-      .executeTakeFirstOrThrow()
-    const totalEvts = totalRes.count
-    let completed = 0
-
-    let cursor: number | undefined = startCursor
-    while (cursor !== undefined) {
-      const page = await this.ctx.sequencer.requestSeqRange({
-        earliestSeq: cursor,
-        limit: PAGE_SIZE,
-      })
-      page.forEach((evt) => this.processEvent(evt))
-      cursor = page.at(-1)?.seq
-
-      await this.queues.onEmpty()
-
-      completed += PAGE_SIZE
-      const percentComplete = (completed / totalEvts) * 100
-      console.log(`${percentComplete.toFixed(2)}% - ${cursor}`)
-    }
-
-    await this.queues.processAll()
-  }
-
-  async processAll() {
-    await this.queues.processAll()
-  }
-
-  async destroy() {
-    await this.queues.destroy()
-  }
-
-  processEvent(evt: SeqEvt) {
-    const did = didFromEvt(evt)
-    if (!did) {
-      return
-    }
-    this.queues.addToUser(did, async () => {
-      if (this.failed.has(did)) {
-        return
-      }
-      await processSeqEvt(this.ctx, evt).catch(async (err) => {
-        this.failed.add(did)
-        await trackFailure(this.ctx.recoveryDb, did, err)
-      })
-    })
-  }
-}
-
-export const processSeqEvt = async (ctx: RecovererContext, evt: SeqEvt) => {
-  // only need to process commits & tombstones
-  if (evt.type === 'account') {
-    await processAccountEvt(ctx, evt.evt)
-  }
-  if (evt.type === 'commit') {
-    await processCommit(ctx, evt.evt).catch()
-  }
-}
-
-const processCommit = async (ctx: RecovererContext, evt: CommitEvt) => {
-  const did = evt.repo
-  const { writes, blocks } = await parseCommitEvt(evt)
-  if (evt.since === null) {
-    const actorExists = await ctx.actorStore.exists(did)
-    if (!actorExists) {
-      await processRepoCreation(ctx, evt, writes, blocks)
-      return
-    }
-  }
-  await ctx.actorStore.transact(did, async (actorTxn) => {
-    const root = await actorTxn.repo.storage.getRootDetailed()
-    if (root.rev >= evt.rev) {
-      return
-    }
-    const commit = await actorTxn.repo.formatCommit(writes)
-    commit.newBlocks = blocks
-    commit.cid = evt.commit
-    commit.rev = evt.rev
-    await actorTxn.repo.storage.applyCommit(commit)
-    await actorTxn.repo.indexWrites(writes, commit.rev)
-    await trackBlobs(actorTxn, writes)
-  })
-}
-
-const processRepoCreation = async (
-  ctx: RecovererContext,
-  evt: CommitEvt,
-  writes: PreparedWrite[],
-  blocks: BlockMap,
-) => {
-  const did = evt.repo
-  const keypair = await Secp256k1Keypair.create({ exportable: true })
-  await ctx.actorStore.create(did, keypair)
-  const commit: CommitData = {
-    cid: evt.commit,
-    rev: evt.rev,
-    since: evt.since,
-    prev: null,
-    newBlocks: blocks,
-    relevantBlocks: new BlockMap(),
-    removedCids: new CidSet(),
-  }
-  await ctx.actorStore.transact(did, async (actorTxn) => {
-    await actorTxn.repo.storage.applyCommit(commit, true)
-    await actorTxn.repo.indexWrites(writes, commit.rev)
-    await actorTxn.repo.blob.processWriteBlobs(commit.rev, writes)
-  })
-  await trackNewAccount(ctx.recoveryDb, did)
-}
-
-const processAccountEvt = async (ctx: RecovererContext, evt: AccountEvt) => {
-  // do not need to process deactivation/takedowns because we backup account DB as well
-
-  if (evt.status === AccountStatus.Deleted) {
-    // In case an account deletion was sequenced, let's make sure to (first)
-    // delete the accounts database, and (then) unlink the actor store from the
-    // file system. Order matters here.
-    await ctx.accountManager.deleteAccount(evt.did)
-
-    const { directory } = await ctx.actorStore.getLocation(evt.did)
-    await rmIfExists(directory, true)
-  }
-}
-
-const trackBlobs = async (
-  store: ActorStoreTransactor,
-  writes: PreparedWrite[],
-) => {
-  await store.repo.blob.deleteDereferencedBlobs(writes)
-
-  for (const write of writes) {
-    if (
-      write.action === WriteOpAction.Create ||
-      write.action === WriteOpAction.Update
-    ) {
-      for (const blob of write.blobs) {
-        await store.repo.blob.insertBlobMetadata(blob)
-        await store.repo.blob.associateBlob(blob, write.uri)
-      }
-    }
-  }
-}
-
-const trackFailure = async (
-  recoveryDb: RecoveryDb,
-  did: DidString,
-  err: unknown,
-) => {
-  await recoveryDb.db
-    .insertInto('failed')
-    .values({
-      did,
-      error: err?.toString(),
-      fixed: 0,
-    })
-    .onConflict((oc) => oc.doNothing())
-    .execute()
-}
-
-const trackNewAccount = async (recoveryDb: RecoveryDb, did: DidString) => {
-  await recoveryDb.db
-    .insertInto('new_account')
-    .values({
-      did,
-      published: 0,
-    })
-    .onConflict((oc) => oc.doNothing())
-    .execute()
-}
-
-const parseCommitEvt = async (
-  evt: CommitEvt,
-): Promise<{
-  writes: PreparedWrite[]
-  blocks: BlockMap
-}> => {
-  const did = evt.repo
-  const evtCar = await readCar(evt.blocks, { skipCidVerification: true })
-  const writesUnfiltered = await Promise.all(
-    evt.ops.map(async (op) => {
-      const { collection, rkey } = parseDataKey(op.path)
-      if (!isNsidString(collection)) return undefined
-      if (!isRecordKeyString(rkey)) return undefined
-
-      if (op.action === 'delete') {
-        return prepareDelete({ did, collection, rkey })
-      }
-      if (!op.cid) return undefined
-
-      const recordBytes = evtCar.blocks.get(op.cid)
-      if (!recordBytes) return undefined
-      const record = cborToLexRecord(recordBytes)
-
-      if (op.action === 'create') {
-        return prepareCreate({
-          did,
-          collection,
-          rkey,
-          record,
-          validate: false,
-        })
-      } else {
-        return prepareUpdate({
-          did,
-          collection,
-          rkey,
-          record,
-          validate: false,
-        })
-      }
-    }),
-  )
-
-  return {
-    writes: writesUnfiltered.filter((w) => w != null),
-    blocks: evtCar.blocks,
-  }
-}
-
-const didFromEvt = (evt: SeqEvt): DidString | null => {
-  if (evt.type === 'account') {
-    return evt.evt.did
-  } else if (evt.type === 'commit') {
-    return evt.evt.repo
-  } else {
-    return null
-  }
-}

package/src/scripts/sequencer-recovery/recovery-db.ts

@@ -1,65 +0,0 @@
-import path from 'node:path'
-import { Kysely } from 'kysely'
-import { DidString } from '@atproto/syntax'
-import { Database, Migrator } from '../../db/index.js'
-
-export interface NewAccount {
-  did: DidString
-  published: 0 | 1
-}
-
-export interface Failed {
-  did: DidString
-  error: string | null
-  fixed: 0 | 1
-}
-
-export type RecoveryDbSchema = {
-  new_account: NewAccount
-  failed: Failed
-}
-
-export type RecoveryDb = Database<RecoveryDbSchema>
-
-export const getRecoveryDbFromSequencerLoc = (
-  sequencerLoc: string,
-): Promise<RecoveryDb> => {
-  const recoveryDbLoc = path.join(path.dirname(sequencerLoc), 'recovery.sqlite')
-  return getAndMigrateRecoveryDb(recoveryDbLoc)
-}
-
-export const getAndMigrateRecoveryDb = async (
-  location: string,
-  disableWalAutoCheckpoint = false,
-): Promise<RecoveryDb> => {
-  const pragmas: Record<string, string> = disableWalAutoCheckpoint
-    ? { wal_autocheckpoint: '0' }
-    : {}
-  const db = Database.sqlite<RecoveryDbSchema>(location, pragmas)
-  const migrator = new Migrator(db.db, migrations)
-  await migrator.migrateToLatestOrThrow()
-  return db
-}
-
-const migrations = {
-  '001': {
-    up: async (db: Kysely<unknown>) => {
-      await db.schema
-        .createTable('new_account')
-        .addColumn('did', 'varchar', (col) => col.primaryKey())
-        .addColumn('published', 'int2', (col) => col.notNull())
-        .execute()
-
-      await db.schema
-        .createTable('failed')
-        .addColumn('did', 'varchar', (col) => col.primaryKey())
-        .addColumn('error', 'varchar')
-        .addColumn('fixed', 'int2', (col) => col.notNull())
-        .execute()
-    },
-    down: async (db: Kysely<unknown>) => {
-      await db.schema.dropTable('new_account').execute()
-      await db.schema.dropTable('failed').execute()
-    },
-  },
-}

package/src/scripts/sequencer-recovery/repair-repos.ts

@@ -1,49 +0,0 @@
-import { DidString } from '@atproto/syntax'
-import { parseRepoSeqRows } from '../../sequencer/index.js'
-import { rebuildRepo } from '../rebuild-repo.js'
-import {
-  RecovererContext,
-  RecovererContextNoDb,
-  processSeqEvt,
-} from './recoverer.js'
-import { getRecoveryDbFromSequencerLoc } from './recovery-db.js'
-
-export const repairRepos = async (ctx: RecovererContextNoDb) => {
-  const recoveryDb = await getRecoveryDbFromSequencerLoc(
-    ctx.sequencer.dbLocation,
-  )
-  const repairRes = await recoveryDb.db
-    .selectFrom('failed')
-    .select('did')
-    .where('failed.fixed', '=', 0)
-    .execute()
-  const dids = repairRes.map((row) => row.did)
-  let fixed = 0
-  for (const did of dids) {
-    await rebuildRepo(ctx, did, false)
-    await recoverFromSequencer({ ...ctx, recoveryDb }, did)
-    fixed++
-    console.log(`${fixed}/${dids.length}`)
-  }
-}
-
-const recoverFromSequencer = async (ctx: RecovererContext, did: DidString) => {
-  const didEvts = await ctx.sequencer.db.db
-    .selectFrom('repo_seq')
-    .selectAll()
-    .where('did', '=', did)
-    .orderBy('seq', 'asc')
-    .execute()
-  const seqEvts = parseRepoSeqRows(didEvts)
-  for (const evt of seqEvts) {
-    await processSeqEvt(ctx, evt)
-  }
-  await ctx.recoveryDb.db
-    .updateTable('failed')
-    .set({
-      fixed: 1,
-      error: null,
-    })
-    .where('did', '=', did)
-    .execute()
-}

package/src/scripts/sequencer-recovery/user-queues.ts

@@ -1,44 +0,0 @@
-import PQueue from 'p-queue'
-
-type Queue = InstanceType<typeof PQueue>
-
-export class UserQueues {
-  main: Queue
-  queues: Map<string, Queue> = new Map()
-
-  constructor(concurrency: number) {
-    this.main = new PQueue({ concurrency })
-  }
-
-  async addToUser(did: string, task: () => Promise<void>) {
-    if (this.main.isPaused) return
-    return this.main.add(() => {
-      return this.getQueue(did).add(task)
-    })
-  }
-
-  private getQueue(did: string) {
-    let queue = this.queues.get(did)
-    if (!queue) {
-      queue = new PQueue({ concurrency: 1 })
-      queue.once('idle', () => this.queues.delete(did))
-      this.queues.set(did, queue)
-    }
-    return queue
-  }
-
-  async onEmpty() {
-    await this.main.onEmpty()
-  }
-
-  async processAll() {
-    await this.main.onIdle()
-  }
-
-  async destroy() {
-    this.main.pause()
-    this.main.clear()
-    this.queues.forEach((q) => q.clear())
-    await this.processAll()
-  }
-}

package/src/scripts/util.ts

@@ -1,7 +0,0 @@
-export const parseIntArg = (arg: string): number => {
-  const parsed = parseInt(arg, 10)
-  if (isNaN(parsed)) {
-    throw new Error(`Invalid arg, expected number: ${arg}`)
-  }
-  return parsed
-}

package/src/sequencer/db/index.ts

@@ -1,21 +0,0 @@
-import { Database, Migrator } from '../../db/index.js'
-import migrations from './migrations/index.js'
-import { SequencerDbSchema } from './schema.js'
-
-export * from './schema.js'
-
-export type SequencerDb = Database<SequencerDbSchema>
-
-export const getDb = (
-  location: string,
-  disableWalAutoCheckpoint = false,
-): SequencerDb => {
-  const pragmas: Record<string, string> = disableWalAutoCheckpoint
-    ? { wal_autocheckpoint: '0' }
-    : {}
-  return Database.sqlite(location, pragmas)
-}
-
-export const getMigrator = (db: Database<SequencerDbSchema>) => {
-  return new Migrator(db.db, migrations)
-}

package/src/sequencer/db/migrations/001-init.ts

@@ -1,35 +0,0 @@
-import { Kysely } from 'kysely'
-
-export async function up(db: Kysely<unknown>): Promise<void> {
-  await db.schema
-    .createTable('repo_seq')
-    .addColumn('seq', 'integer', (col) => col.autoIncrement().primaryKey())
-    .addColumn('did', 'varchar', (col) => col.notNull())
-    .addColumn('eventType', 'varchar', (col) => col.notNull())
-    .addColumn('event', 'blob', (col) => col.notNull())
-    .addColumn('invalidated', 'int2', (col) => col.notNull().defaultTo(0))
-    .addColumn('sequencedAt', 'varchar', (col) => col.notNull())
-    .execute()
-  // for filtering seqs based on did
-  await db.schema
-    .createIndex('repo_seq_did_idx')
-    .on('repo_seq')
-    .column('did')
-    .execute()
-  // for filtering seqs based on event type
-  await db.schema
-    .createIndex('repo_seq_event_type_idx')
-    .on('repo_seq')
-    .column('eventType')
-    .execute()
-  // for entering into the seq stream at a particular time
-  await db.schema
-    .createIndex('repo_seq_sequenced_at_index')
-    .on('repo_seq')
-    .column('sequencedAt')
-    .execute()
-}
-
-export async function down(db: Kysely<unknown>): Promise<void> {
-  await db.schema.dropTable('repo_seq').execute()
-}

package/src/sequencer/db/migrations/index.ts

@@ -1,5 +0,0 @@
-import * as init from './001-init.js'
-
-export default {
-  '001': init,
-}