@atproto/pds 0.5.12 → 0.5.14

package/src/repo/prepare.ts

@@ -1,266 +0,0 @@
-import { TID } from '@atproto/common'
-import { RecordSchema } from '@atproto/lex'
-import { encode } from '@atproto/lex-cbor'
-import {
-  Cid,
-  LexMap,
-  TypedBlobRef,
-  TypedLexMap,
-  cidForCbor,
-  enumBlobRefs,
-  isLegacyBlobRef,
-} from '@atproto/lex-data'
-import {
-  RecordCreateOp,
-  RecordDeleteOp,
-  RecordUpdateOp,
-  RecordWriteOp,
-  WriteOpAction,
-} from '@atproto/repo'
-import {
-  AtUri,
-  DidString,
-  NsidString,
-  RecordKeyString,
-  isValidRecordKey,
-} from '@atproto/syntax'
-import { hasExplicitSlur } from '../handle/explicit-slurs.js'
-import { app, chat, com } from '../lexicons/index.js'
-import {
-  InvalidRecordError,
-  PreparedCreate,
-  PreparedDelete,
-  PreparedUpdate,
-  PreparedWrite,
-  ValidationStatus,
-} from './types.js'
-
-// @TODO replace this with automatically fetched (& built) schemas
-const knownSchemas = new Map<string, RecordSchema>(
-  [
-    app.bsky.actor.profile.main,
-    app.bsky.actor.status.main,
-    app.bsky.feed.generator.main,
-    app.bsky.feed.like.main,
-    app.bsky.feed.post.main,
-    app.bsky.feed.postgate.main,
-    app.bsky.feed.repost.main,
-    app.bsky.feed.threadgate.main,
-    app.bsky.graph.block.main,
-    app.bsky.graph.follow.main,
-    app.bsky.graph.list.main,
-    app.bsky.graph.listblock.main,
-    app.bsky.graph.listitem.main,
-    app.bsky.graph.starterpack.main,
-    app.bsky.graph.verification.main,
-    app.bsky.labeler.service.main,
-    app.bsky.notification.declaration.main,
-    chat.bsky.actor.declaration.main,
-    com.atproto.lexicon.schema.main,
-    com.germnetwork.declaration.main,
-  ].map((schema: RecordSchema) => [schema.$type, schema]),
-)
-
-const validateRecord = (
-  record: TypedLexMap,
-  rkey: RecordKeyString,
-  opts: {
-    validate?: boolean
-    validationPath?: (string | number)[]
-  },
-): undefined | ValidationStatus => {
-  // If validation is explicitly disabled, skip it
-  if (opts.validate === false) {
-    return undefined
-  }
-
-  // @TODO add support for lexicon resolution to fetch the schema dynamically
-  const schema = knownSchemas.get(record.$type)
-  if (!schema) {
-    // If validation is explicitly requested, throw if unable to validate
-    if (opts.validate === true) {
-      throw new InvalidRecordError(`Unknown lexicon type: ${record.$type}`)
-    } else {
-      return 'unknown'
-    }
-  }
-
-  const rkeyResult = schema.keySchema.safeValidate(rkey)
-  if (!rkeyResult.success) {
-    throw new InvalidRecordError(
-      `Invalid record key for ${record.$type}: ${rkeyResult.reason.message}`,
-      { cause: rkeyResult.reason },
-    )
-  }
-
-  const recordResult = schema.safeValidate(record, {
-    path: opts.validationPath ?? ['record'],
-  })
-  if (!recordResult.success) {
-    throw new InvalidRecordError(
-      `Invalid ${record.$type} record: ${recordResult.reason.message}`,
-      { cause: recordResult.reason },
-    )
-  }
-
-  return 'valid'
-}
-
-export const prepareCreate = async (opts: {
-  did: DidString
-  collection: NsidString
-  rkey?: RecordKeyString
-  swapCid?: Cid | null
-  record: LexMap
-  validate?: boolean
-  validationPath?: (string | number)[]
-}): Promise<PreparedCreate> => {
-  const { cid, uri, record, blobs, validationStatus } = await prepareWrite(opts)
-
-  return {
-    action: WriteOpAction.Create,
-    uri,
-    cid,
-    swapCid: opts.swapCid,
-    record,
-    blobs,
-    validationStatus,
-  }
-}
-
-export const prepareUpdate = async (opts: {
-  did: DidString
-  collection: NsidString
-  rkey: RecordKeyString
-  swapCid?: Cid | null
-  record: LexMap
-  validate?: boolean
-  validationPath?: (string | number)[]
-}): Promise<PreparedUpdate> => {
-  const { cid, uri, record, blobs, validationStatus } = await prepareWrite(opts)
-
-  return {
-    action: WriteOpAction.Update,
-    uri,
-    cid,
-    swapCid: opts.swapCid,
-    record,
-    blobs,
-    validationStatus,
-  }
-}
-
-async function prepareWrite(opts: {
-  did: string
-  collection: NsidString
-  rkey?: RecordKeyString
-  record: LexMap
-  validate?: boolean
-  validationPath?: (string | number)[]
-}): Promise<{
-  record: TypedLexMap
-  blobs: TypedBlobRef[]
-  validationStatus?: ValidationStatus
-  uri: AtUri
-  cid: Cid
-}> {
-  const record: null | TypedLexMap =
-    opts.record.$type === undefined
-      ? { ...opts.record, $type: opts.collection }
-      : opts.record.$type === opts.collection
-        ? (opts.record as TypedLexMap)
-        : null
-
-  if (!record) {
-    throw new InvalidRecordError(
-      `Invalid $type: expected ${opts.collection}, got ${opts.record.$type}`,
-    )
-  }
-
-  // @NOTE the rkey will be validated against the schema later
-  if (opts.rkey != null) {
-    if (!isValidRecordKey(opts.rkey)) {
-      throw new InvalidRecordError(`Invalid record key: ${opts.rkey}`)
-    }
-    if (hasExplicitSlur(opts.rkey)) {
-      throw new InvalidRecordError('Unacceptable slur in record key')
-    }
-  }
-
-  const nextRkey = TID.next()
-  const rkey = opts.rkey || nextRkey.toString()
-
-  return {
-    record,
-    // @NOTE we validate before enumerating blobs, so that we can provide more
-    // accurate validations error (esp. in case of legacy blobs).
-    validationStatus: validateRecord(record, rkey, opts),
-    blobs: Array.from(
-      enumBlobRefs(record, { strict: false, allowLegacy: true }),
-      (blob) => {
-        // @NOTE as we migrated from legacy blobs to non legacy blobs, we wanted
-        // to prevent the creation of legacy blobs. Note that this prevents the
-        // creation of (legitimate) records that have the same shape as legacy
-        // blob refs ({ cid: "<cid>", mimeType: "<mime-type>" }), but this was
-        // deemed an acceptable tradeoff to prevent the creation of new legacy
-        // blobs. Since that migration happened a while ago, we can probably
-        // remove this check in the future, by removing the "allowLegacy" option.
-        if (isLegacyBlobRef(blob)) {
-          throw new InvalidRecordError(
-            `Legacy blobs are not allowed (${blob.cid})`,
-          )
-        }
-        return blob
-      },
-    ),
-    uri: AtUri.make(opts.did, opts.collection, rkey),
-    cid: await cidForCbor(encode(record)),
-  }
-}
-
-export const prepareDelete = (opts: {
-  did: DidString
-  collection: NsidString
-  rkey: RecordKeyString
-  swapCid?: Cid | null
-}): PreparedDelete => {
-  const { did, collection, rkey, swapCid } = opts
-  return {
-    action: WriteOpAction.Delete,
-    uri: AtUri.make(did, collection, rkey),
-    swapCid,
-  }
-}
-
-export const createWriteToOp = (write: PreparedCreate): RecordCreateOp => ({
-  action: WriteOpAction.Create,
-  collection: write.uri.collectionSafe,
-  rkey: write.uri.rkeySafe,
-  record: write.record,
-})
-
-export const updateWriteToOp = (write: PreparedUpdate): RecordUpdateOp => ({
-  action: WriteOpAction.Update,
-  collection: write.uri.collectionSafe,
-  rkey: write.uri.rkeySafe,
-  record: write.record,
-})
-
-export const deleteWriteToOp = (write: PreparedDelete): RecordDeleteOp => ({
-  action: WriteOpAction.Delete,
-  collection: write.uri.collectionSafe,
-  rkey: write.uri.rkeySafe,
-})
-
-export const writeToOp = (write: PreparedWrite): RecordWriteOp => {
-  switch (write.action) {
-    case WriteOpAction.Create:
-      return createWriteToOp(write)
-    case WriteOpAction.Update:
-      return updateWriteToOp(write)
-    case WriteOpAction.Delete:
-      return deleteWriteToOp(write)
-    default:
-      throw new Error(`Unrecognized action: ${write}`)
-  }
-}

package/src/repo/types.ts

@@ -1,65 +0,0 @@
-import { Cid, LexMap, TypedBlobRef } from '@atproto/lex-data'
-import { BlockMap, CommitData, WriteOpAction } from '@atproto/repo'
-import { AtUri } from '@atproto/syntax'
-
-export type ValidationStatus = 'valid' | 'unknown'
-
-export type PreparedCreate = {
-  action: WriteOpAction.Create
-  uri: AtUri
-  cid: Cid
-  swapCid?: Cid | null
-  record: LexMap
-  blobs: TypedBlobRef[]
-  validationStatus?: ValidationStatus
-}
-
-export type PreparedUpdate = {
-  action: WriteOpAction.Update
-  uri: AtUri
-  cid: Cid
-  swapCid?: Cid | null
-  record: LexMap
-  blobs: TypedBlobRef[]
-  validationStatus?: ValidationStatus
-}
-
-export type PreparedDelete = {
-  action: WriteOpAction.Delete
-  uri: AtUri
-  swapCid?: Cid | null
-}
-
-export type CommitOp = {
-  action: 'create' | 'update' | 'delete'
-  path: string
-  cid: Cid | null
-  prev?: Cid
-}
-
-export type CommitDataWithOps = CommitData & {
-  ops: CommitOp[]
-  prevData: Cid | null
-}
-
-export type PreparedWrite = PreparedCreate | PreparedUpdate | PreparedDelete
-
-export type SyncEvtData = {
-  cid: Cid
-  rev: string
-  blocks: BlockMap
-}
-
-export class InvalidRecordError extends Error {}
-
-export class BadCommitSwapError extends Error {
-  constructor(public cid: Cid) {
-    super(`Commit was at ${cid.toString()}`)
-  }
-}
-
-export class BadRecordSwapError extends Error {
-  constructor(public cid: Cid | null) {
-    super(`Record was at ${cid?.toString() ?? 'null'}`)
-  }
-}

package/src/scripts/README.md

@@ -1,40 +0,0 @@
-# PDS Scripts
-
-This directory includes some low-level administrative scripts primarily meant to help recover from situations of data loss or repository corruption.
-
-These scripts are included in the Docker image. The recommended way to run them is to shell into the PDS container (`docker exec -it pds /bin/sh`) and run the relevant script using `node run-script.js SCRIPT_NAME`.
-
-### rebuild-repo
-
-Rebuild a repo's MST and sign a new commit based on data stored in the actor's record table. Intended to be used if a repository is corrupted and there are missing MST blocks.
-
-`node run-script.js rebuild-repo DID`
-
-### publish-identity
-
-Publishes an identity event on the PDS's outgoing firehose for the relevant DID. Intended to be used if a user's identity is out of date and a refresh of their identity needs to be pushed through the system.
-
-`node run-script.js publish-identity DID`
-`node run-script.js publish-identity DID1 DID2 DID3 ...`
-`node run-script.js publish-identity-file dids.txt` (where `dids.txt` is a `\n` delimited text file of dids)
-
-### rotate-keys
-
-Ensures that an account's signing key in their PLC DID document matches the signing key that the PDS is holidng for them locally. If not, then update their PLC document. Does not work for `did:web`s. Intended to be used in recovery situations where an accounts' signing key is lost and it needs to be re-generated. This script _does not_ regenerate the key.
-
-`node run-script.js rotate-keys DID`
-`node run-script.js rotate-keys DID1 DID2 DID3 ...`
-`node run-script.js rotate-keys-file dids.txt` (where `dids.txt` is a `\n` delimited text file of dids)
-`node run-script.js rotate-keys-recovery` (to be used after `sequencer-recovery` script)
-
-### sequencer-recovery
-
-Replays the sequencer file on top of actor stores. Creates new actor stores & keys for actors that do not exist but are in the sequencer. Deletes actor stores & entries in the accounts DB when processing an account deletion on the stream. This script is meant to be re-runnable. Though because it processes events in parallel, it is important to be discerning about the cursor you pick up from if you stop & start the script.
-
-Does _not_ rotate signing keys even if it generates them. Signing keys that need to be rotated are stored in a recovery DB and can be actually rotated with the `rotate-keys-recovery` script.
-
-Intended to be used for recovery from data loss.
-
-`node run-script.js sequencer-recovery START_CURSOR CONCURRENCY` (both params are optional & default to `0` and `10` respectively)
-
-Failures are also tracked in the recovery DB and can be recovered from with `node run-script.js recovery-repair-repos`. Which will rebuild repos (a la `rebuild-repo`) and then play back the events from the sequencer only pertaining to the recovered DIDs.

package/src/scripts/index.ts

@@ -1,20 +0,0 @@
-import { publishIdentity, publishIdentityFromFile } from './publish-identity.js'
-import { rebuildRepo } from './rebuild-repo.js'
-import {
-  rotateKeys,
-  rotateKeysFromFile,
-  rotateKeysRecovery,
-} from './rotate-keys.js'
-import { sequencerRecovery } from './sequencer-recovery/index.js'
-import { repairRepos } from './sequencer-recovery/repair-repos.js'
-
-export const scripts = {
-  'rebuild-repo': rebuildRepo,
-  'sequencer-recovery': sequencerRecovery,
-  'recovery-repair-repos': repairRepos,
-  'rotate-keys': rotateKeys,
-  'rotate-keys-file': rotateKeysFromFile,
-  'rotate-keys-recovery': rotateKeysRecovery,
-  'publish-identity': publishIdentity,
-  'publish-identity-file': publishIdentityFromFile,
-}

package/src/scripts/publish-identity.ts

@@ -1,60 +0,0 @@
-import assert from 'node:assert'
-import fs from 'node:fs/promises'
-import { wait } from '@atproto/common'
-import { isDidString } from '@atproto/lex'
-import { DidString } from '@atproto/syntax'
-import { Sequencer } from '../sequencer/index.js'
-import { parseIntArg } from './util.js'
-
-export type PublishIdentityContext = {
-  sequencer: Sequencer
-}
-
-export const publishIdentity = async (
-  ctx: PublishIdentityContext,
-  args: string[],
-) => {
-  const dids = args
-  assert(dids.every(isDidString), 'All arguments must be DIDs')
-  await publishIdentityEvtForDids(ctx, dids)
-  console.log('DONE')
-}
-
-export const publishIdentityFromFile = async (
-  ctx: PublishIdentityContext,
-  args: string[],
-) => {
-  const filepath = args[0]
-  if (!filepath) {
-    throw new Error('Expected filepath as argument')
-  }
-  const timeBetween = args[1] ? parseIntArg(args[1]) : 5
-  const file = await fs.readFile(filepath)
-  const dids = file
-    .toString()
-    .split('\n')
-    .map((did) => did.trim())
-
-  assert(dids.every(isDidString), 'File contains invalid DIDs')
-
-  await publishIdentityEvtForDids(ctx, dids, timeBetween)
-  console.log('DONE')
-}
-
-export const publishIdentityEvtForDids = async (
-  ctx: PublishIdentityContext,
-  dids: DidString[],
-  timeBetween = 0,
-) => {
-  for (const did of dids) {
-    try {
-      await ctx.sequencer.sequenceIdentity(did)
-      console.log(`published identity evt for ${did}`)
-    } catch (err) {
-      console.error(`failed to sequence new identity evt for ${did}: ${err}`)
-    }
-    if (timeBetween > 0) {
-      await wait(timeBetween)
-    }
-  }
-}

package/src/scripts/rebuild-repo.ts

@@ -1,119 +0,0 @@
-import assert from 'node:assert'
-import readline from 'node:readline/promises'
-import { TID } from '@atproto/common'
-import { DidString, isDidString } from '@atproto/lex'
-import {
-  BlockMap,
-  CidSet,
-  MST,
-  MemoryBlockstore,
-  signCommit,
-} from '@atproto/repo'
-import { AccountManager } from '../account-manager/account-manager.js'
-import { ActorStore } from '../actor-store/actor-store.js'
-import { Sequencer } from '../sequencer/index.js'
-
-export interface RebuildContext {
-  sequencer: Sequencer
-  accountManager: AccountManager
-  actorStore: ActorStore
-}
-
-export const rebuildRepoScript = async (
-  ctx: RebuildContext,
-  args: string[],
-) => {
-  const did = args[0]
-  assert(isDidString(did), 'Expected DID as argument')
-  return rebuildRepo(ctx, did, true)
-}
-
-export const rebuildRepo = async (
-  ctx: RebuildContext,
-  did: DidString,
-  promptUser: boolean,
-) => {
-  const memoryStore = new MemoryBlockstore()
-  const commit = await ctx.actorStore.transact(did, async (store) => {
-    const rootDetails = await store.repo.storage.getRootDetailed()
-    const records = await store.record.listAll()
-    const existingCids = await store.record.listExistingBlocks()
-
-    // increment existing rev by 1 ms
-    const revTid = TID.fromStr(rootDetails.rev)
-    const rev = TID.fromTime(
-      revTid.timestamp() + 1,
-      revTid.clockid(),
-    ).toString()
-
-    let mst = await MST.create(memoryStore)
-    for (const record of records) {
-      mst = await mst.add(record.path, record.cid)
-    }
-    const newBlocks = new BlockMap()
-    for await (const node of mst.walk()) {
-      if (node.isTree()) {
-        const pointer = await node.getPointer()
-        if (!existingCids.has(pointer)) {
-          const serialized = await node.serialize()
-          newBlocks.set(serialized.cid, serialized.bytes)
-        }
-      }
-    }
-    const mstCids = await mst.allCids()
-    const toDelete = new CidSet(existingCids.toList()).subtractSet(mstCids)
-    const newCommit = await signCommit(
-      {
-        did,
-        version: 3,
-        rev,
-        prev: null,
-        data: await mst.getPointer(),
-      },
-      store.repo.signingKey,
-    )
-    const commitCid = await newBlocks.add(newCommit)
-
-    if (promptUser) {
-      console.log('Record count: ', records.length)
-      console.log('Existing blocks: ', existingCids.toList().length)
-      console.log('Deleting blocks:', toDelete.toList().length)
-      console.log('Adding blocks: ', newBlocks.size)
-
-      const shouldContinue = await promptContinue()
-      if (!shouldContinue) {
-        throw new Error('Aborted')
-      }
-    }
-
-    await store.repo.storage.deleteMany(toDelete.toList())
-    await store.repo.storage.putMany(newBlocks, rev)
-    await store.repo.storage.updateRoot(commitCid, rev)
-    return {
-      cid: commitCid,
-      rev,
-      since: null,
-      prev: null,
-      newBlocks,
-      relevantBlocks: newBlocks,
-      removedCids: toDelete,
-      ops: [],
-      blobs: new CidSet(),
-      prevData: null,
-    }
-  })
-  await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev)
-  const syncData = await ctx.actorStore.read(did, (store) =>
-    store.repo.getSyncEventData(),
-  )
-  await ctx.sequencer.sequenceSync(did, syncData)
-}
-
-const promptContinue = async (): Promise<boolean> => {
-  const rl = readline.createInterface({
-    input: process.stdin,
-    output: process.stdout,
-  })
-  const answer = await rl.question('Continue? y/n ')
-  return answer === ''
-}