@atproto/pds 0.4.216 → 0.4.217

package/src/scripts/rotate-keys.ts

@@ -1,9 +1,11 @@
+import assert from 'node:assert'
 import fs from 'node:fs/promises'
 import * as plc from '@did-plc/lib'
 import PQueue from 'p-queue'
 import AtpAgent from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
 import { IdResolver } from '@atproto/identity'
+import { DidString, isDidString } from '@atproto/lex'
 import { ActorStore } from '../actor-store/actor-store'
 import { SyncEvtData } from '../repo'
 import { Sequencer } from '../sequencer'
@@ -21,6 +23,7 @@ export type RotateKeysContext = {
 
 export const rotateKeys = async (ctx: RotateKeysContext, args: string[]) => {
   const dids = args
+  assert(dids.every(isDidString), 'All arguments must be DIDs')
   await rotateKeysForRepos(ctx, dids, 10)
 }
 
@@ -40,6 +43,8 @@ export const rotateKeysFromFile = async (
     .map((did) => did.trim())
     .filter((did) => did.startsWith('did:plc'))
 
+  assert(dids.every(isDidString), 'File contains invalid DIDs')
+
   await rotateKeysForRepos(ctx, dids, concurrency)
 }
 
@@ -57,7 +62,7 @@ export const rotateKeysRecovery = async (
     .select('did')
     .where('new_account.published', '=', 0)
     .execute()
-  const dids = rows.map((r) => r.did)
+  const dids = rows.map((r) => r.did as DidString)
 
   await rotateKeysForRepos(ctx, dids, concurrency, async (did) => {
     await recoveryDb.db
@@ -70,9 +75,9 @@ export const rotateKeysRecovery = async (
 
 const rotateKeysForRepos = async (
   ctx: RotateKeysContext,
-  dids: string[],
+  dids: DidString[],
   concurrency: number,
-  onSuccess?: (did: string) => Promise<void>,
+  onSuccess?: (did: DidString) => Promise<void>,
 ) => {
   const queue = new PQueue({ concurrency })
   let completed = 0
@@ -119,7 +124,7 @@ const rotateKeysForRepos = async (
   console.log('DONE')
 }
 
-const updatePlcSigningKey = async (ctx: RotateKeysContext, did: string) => {
+const updatePlcSigningKey = async (ctx: RotateKeysContext, did: DidString) => {
   const updateTo = await ctx.actorStore.keypair(did)
   const currSigningKey = await ctx.idResolver.did.resolveAtprotoKey(did, true)
   if (updateTo.did() === currSigningKey) {

package/src/scripts/sequencer-recovery/recoverer.ts

@@ -1,5 +1,6 @@
 import { rmIfExists } from '@atproto/common'
 import { Secp256k1Keypair } from '@atproto/crypto'
+import { DidString, isNsidString, isRecordKeyString } from '@atproto/lex'
 import {
   BlockMap,
   CidSet,
@@ -202,7 +203,7 @@ const trackBlobs = async (
 
 const trackFailure = async (
   recoveryDb: RecoveryDb,
-  did: string,
+  did: DidString,
   err: unknown,
 ) => {
   await recoveryDb.db
@@ -216,7 +217,7 @@ const trackFailure = async (
     .execute()
 }
 
-const trackNewAccount = async (recoveryDb: RecoveryDb, did: string) => {
+const trackNewAccount = async (recoveryDb: RecoveryDb, did: DidString) => {
   await recoveryDb.db
     .insertInto('new_account')
     .values({
@@ -238,10 +239,14 @@ const parseCommitEvt = async (
   const writesUnfiltered = await Promise.all(
     evt.ops.map(async (op) => {
       const { collection, rkey } = parseDataKey(op.path)
+      if (!isNsidString(collection)) return undefined
+      if (!isRecordKeyString(rkey)) return undefined
+
       if (op.action === 'delete') {
         return prepareDelete({ did, collection, rkey })
       }
       if (!op.cid) return undefined
+
       const recordBytes = evtCar.blocks.get(op.cid)
       if (!recordBytes) return undefined
       const record = cborToLexRecord(recordBytes)
@@ -265,16 +270,14 @@ const parseCommitEvt = async (
       }
     }),
   )
-  const writes = writesUnfiltered.filter(
-    (w) => w !== undefined,
-  ) as PreparedWrite[]
+
   return {
-    writes,
+    writes: writesUnfiltered.filter((w) => w != null),
     blocks: evtCar.blocks,
   }
 }
 
-const didFromEvt = (evt: SeqEvt): string | null => {
+const didFromEvt = (evt: SeqEvt): DidString | null => {
   if (evt.type === 'account') {
     return evt.evt.did
   } else if (evt.type === 'commit') {

package/src/scripts/sequencer-recovery/recovery-db.ts

@@ -1,14 +1,15 @@
 import path from 'node:path'
 import { Kysely } from 'kysely'
+import { DidString } from '@atproto/syntax'
 import { Database, Migrator } from '../../db'
 
 export interface NewAccount {
-  did: string
+  did: DidString
   published: 0 | 1
 }
 
 export interface Failed {
-  did: string
+  did: DidString
   error: string | null
   fixed: 0 | 1
 }

package/src/scripts/sequencer-recovery/repair-repos.ts

@@ -1,3 +1,4 @@
+import { DidString } from '@atproto/syntax'
 import { parseRepoSeqRows } from '../../sequencer'
 import { rebuildRepo } from '../rebuild-repo'
 import {
@@ -26,7 +27,7 @@ export const repairRepos = async (ctx: RecovererContextNoDb) => {
   }
 }
 
-const recoverFromSequencer = async (ctx: RecovererContext, did: string) => {
+const recoverFromSequencer = async (ctx: RecovererContext, did: DidString) => {
   const didEvts = await ctx.sequencer.db.db
     .selectFrom('repo_seq')
     .selectAll()

package/src/sequencer/events.ts

@@ -1,6 +1,14 @@
 import assert from 'node:assert'
 import { z } from 'zod'
-import { cborEncode, schema } from '@atproto/common'
+import { schema } from '@atproto/common'
+import {
+  DatetimeString,
+  DidString,
+  HandleString,
+  isDidString,
+  isHandleString,
+} from '@atproto/lex'
+import { encode as cborEncode } from '@atproto/lex-cbor'
 import { BlockMap, blocksToCarFile } from '@atproto/repo'
 import { AccountStatus } from '../account-manager/account-manager'
 import { CommitDataWithOps, SyncEvtData } from '../repo'
@@ -37,7 +45,7 @@ export const formatSeqCommit = async (
 }
 
 export const formatSeqSyncEvt = async (
-  did: string,
+  did: DidString,
   data: SyncEvtData,
 ): Promise<RepoSeqInsert> => {
   const blocks = await blocksToCarFile(data.cid, data.blocks)
@@ -72,8 +80,8 @@ export const syncEvtDataFromCommit = (
 }
 
 export const formatSeqIdentityEvt = async (
-  did: string,
-  handle?: string,
+  did: DidString,
+  handle?: HandleString,
 ): Promise<RepoSeqInsert> => {
   const evt: IdentityEvt = {
     did,
@@ -90,7 +98,7 @@ export const formatSeqIdentityEvt = async (
 }
 
 export const formatSeqAccountEvt = async (
-  did: string,
+  did: DidString,
   status: AccountStatus,
 ): Promise<RepoSeqInsert> => {
   const evt: AccountEvt = {
@@ -115,16 +123,19 @@ export const commitEvtOp = z.object({
     z.literal('update'),
     z.literal('delete'),
   ]),
+  // @TODO should we validate that the "path" is a valid "<nsid>/<rkey>" ?
   path: z.string(),
   cid: schema.cid.nullable(),
   prev: schema.cid.optional(),
 })
 export type CommitEvtOp = z.infer<typeof commitEvtOp>
 
+// @TODO This runtime code is only used to generate "types". We should either
+// make use of it or only use types.
 export const commitEvt = z.object({
   rebase: z.boolean(),
   tooBig: z.boolean(),
-  repo: z.string(),
+  repo: z.string().refine(isDidString),
   commit: schema.cid,
   rev: z.string(),
   since: z.string().nullable(),
@@ -136,20 +147,20 @@ export const commitEvt = z.object({
 export type CommitEvt = z.infer<typeof commitEvt>
 
 export const syncEvt = z.object({
-  did: z.string(),
+  did: z.string().refine(isDidString),
   blocks: schema.bytes,
   rev: z.string(),
 })
 export type SyncEvt = z.infer<typeof syncEvt>
 
 export const identityEvt = z.object({
-  did: z.string(),
-  handle: z.string().optional(),
+  did: z.string().refine(isDidString),
+  handle: z.string().refine(isHandleString).optional(),
 })
 export type IdentityEvt = z.infer<typeof identityEvt>
 
 export const accountEvt = z.object({
-  did: z.string(),
+  did: z.string().refine(isDidString),
   active: z.boolean(),
   status: z
     .enum([
@@ -165,25 +176,25 @@ export type AccountEvt = z.infer<typeof accountEvt>
 type TypedCommitEvt = {
   type: 'commit'
   seq: number
-  time: string
+  time: DatetimeString
   evt: CommitEvt
 }
 type TypedSyncEvt = {
   type: 'sync'
   seq: number
-  time: string
+  time: DatetimeString
   evt: SyncEvt
 }
 type TypedIdentityEvt = {
   type: 'identity'
   seq: number
-  time: string
+  time: DatetimeString
   evt: IdentityEvt
 }
 type TypedAccountEvt = {
   type: 'account'
   seq: number
-  time: string
+  time: DatetimeString
   evt: AccountEvt
 }
 export type SeqEvt =

package/src/sequencer/sequencer.ts

@@ -1,6 +1,8 @@
 import EventEmitter from 'node:events'
 import TypedEmitter from 'typed-emitter'
-import { SECOND, cborDecode, wait } from '@atproto/common'
+import { SECOND, wait } from '@atproto/common'
+import { decode as cborDecode } from '@atproto/lex-cbor'
+import { DatetimeString, DidString, HandleString } from '@atproto/syntax'
 import { AccountStatus } from '../account-manager/helpers/account'
 import { Crawlers } from '../crawlers'
 import { seqLogger as log } from '../logger'
@@ -168,29 +170,32 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
   }
 
   async sequenceCommit(
-    did: string,
+    did: DidString,
     commitData: CommitDataWithOps,
   ): Promise<number> {
     const evt = await formatSeqCommit(did, commitData)
-    return await this.sequenceEvt(evt)
+    return this.sequenceEvt(evt)
   }
 
-  async sequenceSyncEvt(did: string, data: SyncEvtData) {
+  async sequenceSyncEvt(did: DidString, data: SyncEvtData) {
     const evt = await formatSeqSyncEvt(did, data)
-    return await this.sequenceEvt(evt)
+    return this.sequenceEvt(evt)
   }
 
-  async sequenceIdentityEvt(did: string, handle?: string): Promise<number> {
+  async sequenceIdentityEvt(
+    did: DidString,
+    handle?: HandleString,
+  ): Promise<number> {
     const evt = await formatSeqIdentityEvt(did, handle)
-    return await this.sequenceEvt(evt)
+    return this.sequenceEvt(evt)
   }
 
   async sequenceAccountEvt(
-    did: string,
+    did: DidString,
     status: AccountStatus,
   ): Promise<number> {
     const evt = await formatSeqAccountEvt(did, status)
-    return await this.sequenceEvt(evt)
+    return this.sequenceEvt(evt)
   }
 
   async deleteAllForUser(did: string, excludingSeqs: number[] = []) {
@@ -217,28 +222,28 @@ export const parseRepoSeqRows = (rows: RepoSeqEntry[]): SeqEvt[] => {
       seqEvts.push({
         type: 'commit',
         seq: row.seq,
-        time: row.sequencedAt,
+        time: row.sequencedAt as DatetimeString,
         evt: evt as CommitEvt,
       })
     } else if (row.eventType === 'sync') {
       seqEvts.push({
         type: 'sync',
         seq: row.seq,
-        time: row.sequencedAt,
+        time: row.sequencedAt as DatetimeString,
         evt: evt as SyncEvt,
       })
     } else if (row.eventType === 'identity') {
       seqEvts.push({
         type: 'identity',
         seq: row.seq,
-        time: row.sequencedAt,
+        time: row.sequencedAt as DatetimeString,
         evt: evt as IdentityEvt,
       })
     } else if (row.eventType === 'account') {
       seqEvts.push({
         type: 'account',
         seq: row.seq,
-        time: row.sequencedAt,
+        time: row.sequencedAt as DatetimeString,
         evt: evt as AccountEvt,
       })
     }

package/src/well-known.ts

@@ -1,11 +1,12 @@
 import { Router } from 'express'
+import { HandleString } from '@atproto/syntax'
 import { AppContext } from './context'
 
 export const createRouter = (ctx: AppContext): Router => {
   const router = Router()
 
   router.get('/.well-known/atproto-did', async function (req, res) {
-    const handle = req.hostname
+    const handle = req.hostname as HandleString
     const supportedHandle = ctx.cfg.identity.serviceHandleDomains.some(
       (host) => handle.endsWith(host) || handle === host.slice(1),
     )

package/tests/_util.ts

@@ -1,14 +1,10 @@
 import { Server } from 'node:http'
 import { AddressInfo } from 'node:net'
 import { type Express } from 'express'
-import { CID } from 'multiformats/cid'
-import { ToolsOzoneModerationDefs } from '@atproto/api'
-import { lexToJson } from '@atproto/lexicon'
+import { ToolsOzoneModerationDefs, lexToJson } from '@atproto/api'
+import { isCidString } from '@atproto/lex'
+import { isCid, isPlainObject } from '@atproto/lex-data'
 import { AtUri } from '@atproto/syntax'
-import {
-  FeedViewPost,
-  isReasonRepost,
-} from '../src/lexicon/types/app/bsky/feed/defs'
 
 // Swap out identifiers and dates with stable
 // values for the purpose of snapshot testing
@@ -20,9 +16,8 @@ export const forSnapshot = (obj: unknown) => {
   const unknown = { [kTake]: 'unknown' }
   const toWalk = lexToJson(obj as any) // remove any blobrefs/cids
   return mapLeafValues(toWalk, (item) => {
-    const asCid = CID.asCID(item)
-    if (asCid !== null) {
-      return take(cids, asCid.toString())
+    if (isCid(item)) {
+      return take(cids, item.toString())
     }
     if (typeof item !== 'string') {
       return item
@@ -86,14 +81,7 @@ export const forSnapshot = (obj: unknown) => {
     if (str.match(/^\d+::pds-public-url-/)) {
       return '0000000000000::invite-code'
     }
-    let isCid: boolean
-    try {
-      CID.parse(str)
-      isCid = true
-    } catch (_err) {
-      isCid = false
-    }
-    if (isCid) {
+    if (isCidString(str)) {
       return take(cids, str)
     }
     return item
@@ -102,14 +90,6 @@ export const forSnapshot = (obj: unknown) => {
 
 // Feed testing utils
 
-export const getOriginator = (item: FeedViewPost) => {
-  if (isReasonRepost(item.reason)) {
-    return item.reason.by.did
-  } else {
-    return item.post.author.did
-  }
-}
-
 // Useful for remapping ids in snapshot testing, to make snapshots deterministic.
 // E.g. you may use this to map this:
 //   [{ uri: 'did://rad'}, { uri: 'did://bad' }, { uri: 'did://rad'}]
@@ -142,7 +122,7 @@ const mapLeafValues = (obj: unknown, fn: (val: unknown) => unknown) => {
   if (Array.isArray(obj)) {
     return obj.map((item) => mapLeafValues(item, fn))
   }
-  if (obj && typeof obj === 'object') {
+  if (isPlainObject(obj)) {
     return Object.entries(obj).reduce(
       (collect, [name, value]) =>
         Object.assign(collect, { [name]: mapLeafValues(value, fn) }),

package/tests/account-deactivation.test.ts

@@ -21,7 +21,7 @@ describe('account deactivation', () => {
     })
 
     sc = network.getSeedClient()
-    agent = network.pds.getClient()
+    agent = network.pds.getAgent()
 
     await basicSeed(sc)
     alice = sc.dids.alice

package/tests/account-deletion.test.ts

@@ -39,7 +39,7 @@ describe('account deletion', () => {
     // @ts-expect-error Error due to circular dependency with the dev-env package
     ctx = network.pds.ctx
     mailer = ctx.mailer
-    agent = network.pds.getClient()
+    agent = network.pds.getAgent()
     sc = network.getSeedClient()
     await basicSeed(sc)
     carol = sc.accounts[sc.dids.carol]

package/tests/account-migration.test.ts

@@ -7,7 +7,7 @@ import {
   mockNetworkUtilities,
 } from '@atproto/dev-env'
 import { readCar } from '@atproto/repo'
-import { ids } from '../src/lexicon/lexicons'
+import { DidString } from '@atproto/syntax'
 
 describe('account migration', () => {
   let network: TestNetworkNoAppView
@@ -17,7 +17,7 @@ describe('account migration', () => {
   let oldAgent: AtpAgent
   let newAgent: AtpAgent
 
-  let alice: string
+  let alice: DidString
 
   beforeAll(async () => {
     network = await TestNetworkNoAppView.create({
@@ -29,8 +29,8 @@ describe('account migration', () => {
     mockNetworkUtilities(newPds)
 
     sc = network.getSeedClient()
-    oldAgent = network.pds.getClient()
-    newAgent = newPds.getClient()
+    oldAgent = network.pds.getAgent()
+    newAgent = newPds.getAgent()
 
     await sc.createAccount('alice', {
       email: 'alice@test.com',
@@ -81,7 +81,7 @@ describe('account migration', () => {
 
     const serviceJwtRes = await oldAgent.com.atproto.server.getServiceAuth({
       aud: newServerDid,
-      lxm: ids.ComAtprotoServerCreateAccount,
+      lxm: 'com.atproto.server.createAccount',
     })
     const serviceJwt = serviceJwtRes.data.token
 

package/tests/account.test.ts

@@ -1,9 +1,12 @@
+import assert from 'node:assert'
 import { EventEmitter, once } from 'node:events'
 import Mail from 'nodemailer/lib/mailer'
 import { AtpAgent, ComAtprotoServerResetPassword } from '@atproto/api'
 import * as crypto from '@atproto/crypto'
 import { TestNetworkNoAppView } from '@atproto/dev-env'
 import { IdResolver } from '@atproto/identity'
+import { isDidString } from '@atproto/lex'
+import { DidString } from '@atproto/syntax'
 import { AppContext } from '../src'
 import { ServerMailer } from '../src/mailer'
 
@@ -22,6 +25,14 @@ describe('account', () => {
   const mailCatcher = new EventEmitter()
   let _origSendMail
 
+  const tryHandle = async (handle: string) => {
+    await agent.api.com.atproto.server.createAccount({
+      email: 'john@test.com',
+      handle,
+      password: 'test123',
+    })
+  }
+
   beforeAll(async () => {
     network = await TestNetworkNoAppView.create({
       dbPostgresSchema: 'account',
@@ -36,7 +47,7 @@ describe('account', () => {
     // @ts-expect-error Error due to circular dependency with the dev-env package
     ctx = network.pds.ctx
     idResolver = network.pds.ctx.idResolver
-    agent = network.pds.getClient()
+    agent = network.pds.getAgent()
 
     // Catch emails for use in tests
     _origSendMail = mailer.transporter.sendMail
@@ -70,7 +81,11 @@ describe('account', () => {
       handle: 'did:bad-handle.test',
       password: 'asdf',
     })
-    await expect(promise).rejects.toThrow('Input/handle must be a valid handle')
+    await expect(promise).rejects.toMatchObject({
+      error: 'InvalidRequest',
+      message: expect.stringContaining('handle'),
+      // - Input/handle must be a valid handle
+    })
   })
 
   describe('email validation', () => {
@@ -89,13 +104,15 @@ describe('account', () => {
         handle: 'bad-email.test',
         password: 'asdf',
       })
-      await expect(promise).rejects.toThrow(
-        'This email address is not supported, please use a different email.',
-      )
+      await expect(promise).rejects.toMatchObject({
+        error: 'InvalidRequest',
+        message: expect.stringContaining('email'),
+        // - This email address is not supported, please use a different email.
+      })
     })
   })
 
-  let did: string
+  let did: DidString
   let jwt: string
 
   it('creates an account', async () => {
@@ -104,12 +121,13 @@ describe('account', () => {
       handle,
       password,
     })
-    did = res.data.did
-    jwt = res.data.accessJwt
 
-    expect(typeof jwt).toBe('string')
-    expect(did.startsWith('did:plc:')).toBeTruthy()
+    expect(typeof res.data.accessJwt).toBe('string')
+    assert(isDidString(res.data.did), 'Did is not a valid DidString')
     expect(res.data.handle).toEqual(handle)
+
+    did = res.data.did
+    jwt = res.data.accessJwt
   })
 
   it('generates a properly formatted PLC DID', async () => {
@@ -292,50 +310,54 @@ describe('account', () => {
     ).rejects.toThrow('Handle already taken: bob.test')
   })
 
-  it('disallows improperly formatted handles', async () => {
-    const tryHandle = async (handle: string) => {
-      await agent.api.com.atproto.server.createAccount({
-        email: 'john@test.com',
-        handle,
-        password: 'test123',
+  it('validates input through lexicon schema', async () => {
+    for (const invalidHandle of [
+      'did:john',
+      'jo_hn.test',
+      'jo!hn.test',
+      'jo%hn.test',
+      'jo&hn.test',
+      'jo*hn.test',
+      'jo|hn.test',
+      'jo:hn.test',
+      'jo/hn.test',
+    ]) {
+      await expect(tryHandle(invalidHandle)).rejects.toMatchObject({
+        error: 'InvalidRequest',
+        message: expect.stringContaining('handle'),
       })
     }
-    await expect(tryHandle('did:john')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('john.bsky.io')).rejects.toThrow(
-      'Not a supported handle domain',
-    )
-    await expect(tryHandle('j.test')).rejects.toThrow('Handle too short')
-    await expect(tryHandle('jayromy-johnber12345678910.test')).rejects.toThrow(
-      'Handle too long',
-    )
-    await expect(tryHandle('jo_hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo!hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo%hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo&hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo*hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo|hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo:hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('jo/hn.test')).rejects.toThrow(
-      'Input/handle must be a valid handle',
-    )
-    await expect(tryHandle('about.test')).rejects.toThrow('Reserved handle')
-    await expect(tryHandle('atp.test')).rejects.toThrow('Reserved handle')
+  })
+
+  it('disallows improperly formatted handles', async () => {
+    await expect(tryHandle('j.test')).rejects.toMatchObject({
+      error: 'InvalidHandle',
+      message: 'Handle too short',
+    })
+    await expect(
+      tryHandle('jayromy-johnber12345678910.test'),
+    ).rejects.toMatchObject({
+      error: 'InvalidHandle',
+      message: 'Handle too long',
+    })
+  })
+
+  it('disallows reserved handles', async () => {
+    await expect(tryHandle('john.bsky.io')).rejects.toMatchObject({
+      error: 'UnsupportedDomain',
+      message: 'Not a supported handle domain',
+    })
+  })
+
+  it('disallows reserved handles', async () => {
+    await expect(tryHandle('about.test')).rejects.toMatchObject({
+      error: 'HandleNotAvailable',
+      message: 'Reserved handle',
+    })
+    await expect(tryHandle('atp.test')).rejects.toMatchObject({
+      error: 'HandleNotAvailable',
+      message: 'Reserved handle',
+    })
   })
 
   it('handles racing signups for same handle', async () => {