@atproto/pds 0.4.28 → 0.4.29

package/src/lexicon/types/com/atproto/sync/getRepoStatus.ts

@@ -0,0 +1,52 @@
+/**
+ * GENERATED CODE - DO NOT MODIFY
+ */
+import express from 'express'
+import { ValidationResult, BlobRef } from '@atproto/lexicon'
+import { lexicons } from '../../../../lexicons'
+import { isObj, hasProp } from '../../../../util'
+import { CID } from 'multiformats/cid'
+import { HandlerAuth, HandlerPipeThrough } from '@atproto/xrpc-server'
+
+export interface QueryParams {
+  /** The DID of the repo. */
+  did: string
+}
+
+export type InputSchema = undefined
+
+export interface OutputSchema {
+  did: string
+  active: boolean
+  /** If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. */
+  status?: 'takendown' | 'suspended' | 'deactivated' | (string & {})
+  /** Optional field, the current rev of the repo, if active=true */
+  rev?: string
+  [k: string]: unknown
+}
+
+export type HandlerInput = undefined
+
+export interface HandlerSuccess {
+  encoding: 'application/json'
+  body: OutputSchema
+  headers?: { [key: string]: string }
+}
+
+export interface HandlerError {
+  status: number
+  message?: string
+  error?: 'RepoNotFound'
+}
+
+export type HandlerOutput = HandlerError | HandlerSuccess | HandlerPipeThrough
+export type HandlerReqCtx<HA extends HandlerAuth = never> = {
+  auth: HA
+  params: QueryParams
+  input: HandlerInput
+  req: express.Request
+  res: express.Response
+}
+export type Handler<HA extends HandlerAuth = never> = (
+  ctx: HandlerReqCtx<HA>,
+) => Promise<HandlerOutput> | HandlerOutput

package/src/lexicon/types/com/atproto/sync/listBlobs.ts

@@ -36,6 +36,7 @@ export interface HandlerSuccess {
 export interface HandlerError {
   status: number
   message?: string
+  error?: 'RepoNotFound' | 'RepoTakendown' | 'RepoSuspended' | 'RepoDeactivated'
 }
 
 export type HandlerOutput = HandlerError | HandlerSuccess | HandlerPipeThrough

package/src/lexicon/types/com/atproto/sync/listRepos.ts

@@ -51,6 +51,9 @@ export interface Repo {
   /** Current repo commit CID */
   head: string
   rev: string
+  active?: boolean
+  /** If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. */
+  status?: 'takendown' | 'suspended' | 'deactivated' | (string & {})
   [k: string]: unknown
 }
 

package/src/lexicon/types/com/atproto/sync/subscribeRepos.ts

@@ -16,6 +16,7 @@ export interface QueryParams {
 export type OutputSchema =
   | Commit
   | Identity
+  | Account
   | Handle
   | Migrate
   | Tombstone
@@ -77,6 +78,8 @@ export interface Identity {
   seq: number
   did: string
   time: string
+  /** The current handle for the account, or 'handle.invalid' if validation fails. This field is optional, might have been validated or passed-through from an upstream source. Semantics and behaviors for PDS vs Relay may evolve in the future; see atproto specs for more details. */
+  handle?: string
   [k: string]: unknown
 }
 
@@ -92,7 +95,31 @@ export function validateIdentity(v: unknown): ValidationResult {
   return lexicons.validate('com.atproto.sync.subscribeRepos#identity', v)
 }
 
-/** Represents an update of the account's handle, or transition to/from invalid state. NOTE: Will be deprecated in favor of #identity. */
+/** Represents a change to an account's status on a host (eg, PDS or Relay). The semantics of this event are that the status is at the host which emitted the event, not necessarily that at the currently active PDS. Eg, a Relay takedown would emit a takedown with active=false, even if the PDS is still active. */
+export interface Account {
+  seq: number
+  did: string
+  time: string
+  /** Indicates that the account has a repository which can be fetched from the host that emitted this event. */
+  active: boolean
+  /** If active=false, this optional field indicates a reason for why the account is not active. */
+  status?: 'takendown' | 'suspended' | 'deleted' | 'deactivated' | (string & {})
+  [k: string]: unknown
+}
+
+export function isAccount(v: unknown): v is Account {
+  return (
+    isObj(v) &&
+    hasProp(v, '$type') &&
+    v.$type === 'com.atproto.sync.subscribeRepos#account'
+  )
+}
+
+export function validateAccount(v: unknown): ValidationResult {
+  return lexicons.validate('com.atproto.sync.subscribeRepos#account', v)
+}
+
+/** DEPRECATED -- Use #identity event instead */
 export interface Handle {
   seq: number
   did: string
@@ -113,7 +140,7 @@ export function validateHandle(v: unknown): ValidationResult {
   return lexicons.validate('com.atproto.sync.subscribeRepos#handle', v)
 }
 
-/** Represents an account moving from one PDS instance to another. NOTE: not implemented; account migration uses #identity instead */
+/** DEPRECATED -- Use #account event instead */
 export interface Migrate {
   seq: number
   did: string
@@ -134,7 +161,7 @@ export function validateMigrate(v: unknown): ValidationResult {
   return lexicons.validate('com.atproto.sync.subscribeRepos#migrate', v)
 }
 
-/** Indicates that an account has been deleted. NOTE: may be deprecated in favor of #identity or a future #account event */
+/** DEPRECATED -- Use #account event instead */
 export interface Tombstone {
   seq: number
   did: string

package/src/scripts/index.ts

@@ -0,0 +1,5 @@
+import { rebuildRepo } from './rebuild-repo'
+
+export const scripts = {
+  'rebuild-repo': rebuildRepo,
+}

package/src/scripts/rebuild-repo.ts

@@ -0,0 +1,143 @@
+import readline from 'node:readline/promises'
+import { CID } from 'multiformats/cid'
+import {
+  BlockMap,
+  CidSet,
+  MST,
+  MemoryBlockstore,
+  formatDataKey,
+  signCommit,
+} from '@atproto/repo'
+import { AtUri } from '@atproto/syntax'
+import { TID } from '@atproto/common'
+import { ActorStoreTransactor } from '../actor-store'
+import AppContext from '../context'
+
+export const rebuildRepo = async (ctx: AppContext, args: string[]) => {
+  const did = args[0]
+  if (!did || !did.startsWith('did:')) {
+    throw new Error('Expected DID as argument')
+  }
+
+  const memoryStore = new MemoryBlockstore()
+  const rev = TID.nextStr()
+  const commit = await ctx.actorStore.transact(did, async (store) => {
+    const [records, existingCids] = await Promise.all([
+      listAllRecords(store),
+      listExistingBlocks(store),
+    ])
+    let mst = await MST.create(memoryStore)
+    for (const record of records) {
+      mst = await mst.add(record.path, record.cid)
+    }
+    const newBlocks = new BlockMap()
+    for await (const node of mst.walk()) {
+      if (node.isTree()) {
+        const pointer = await node.getPointer()
+        if (!existingCids.has(pointer)) {
+          const serialized = await node.serialize()
+          newBlocks.set(serialized.cid, serialized.bytes)
+        }
+      }
+    }
+    const mstCids = await mst.allCids()
+    const toDelete = new CidSet(existingCids.toList()).subtractSet(mstCids)
+    const newCommit = await signCommit(
+      {
+        did,
+        version: 3,
+        rev,
+        prev: null,
+        data: await mst.getPointer(),
+      },
+      store.repo.signingKey,
+    )
+    const commitCid = await newBlocks.add(newCommit)
+
+    console.log('Record count: ', records.length)
+    console.log('Existing blocks: ', existingCids.toList().length)
+    console.log('Deleting blocks:', toDelete.toList().length)
+    console.log('Adding blocks: ', newBlocks.size)
+
+    const shouldContinue = await promptContinue()
+    if (!shouldContinue) {
+      throw new Error('Aborted')
+    }
+
+    await store.repo.storage.deleteMany(toDelete.toList())
+    await store.repo.storage.putMany(newBlocks, rev)
+    await store.repo.storage.updateRoot(commitCid, rev)
+    return {
+      cid: commitCid,
+      rev,
+      since: null,
+      prev: null,
+      newBlocks,
+      removedCids: toDelete,
+    }
+  })
+  await ctx.accountManager.updateRepoRoot(did, commit.cid, rev)
+  await ctx.sequencer.sequenceCommit(did, commit, [])
+}
+
+const listExistingBlocks = async (
+  store: ActorStoreTransactor,
+): Promise<CidSet> => {
+  const cids = new CidSet()
+  let cursor: string | undefined = ''
+  while (cursor !== undefined) {
+    const res = await store.db.db
+      .selectFrom('repo_block')
+      .select('cid')
+      .where('cid', '>', cursor)
+      .orderBy('cid', 'asc')
+      .limit(1000)
+      .execute()
+    for (const row of res) {
+      cids.add(CID.parse(row.cid))
+    }
+    cursor = res.at(-1)?.cid
+  }
+  return cids
+}
+
+const listAllRecords = async (
+  store: ActorStoreTransactor,
+): Promise<RecordDescript[]> => {
+  const records: RecordDescript[] = []
+  let cursor: string | undefined = ''
+  while (cursor !== undefined) {
+    const res = await store.db.db
+      .selectFrom('record')
+      .select(['uri', 'cid'])
+      .where('uri', '>', cursor)
+      .orderBy('uri', 'asc')
+      .limit(1000)
+      .execute()
+    for (const row of res) {
+      const parsed = new AtUri(row.uri)
+      records.push({
+        uri: row.uri,
+        path: formatDataKey(parsed.collection, parsed.rkey),
+        cid: CID.parse(row.cid),
+      })
+    }
+    cursor = res.at(-1)?.uri
+  }
+  return records
+}
+
+const promptContinue = async (): Promise<boolean> => {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  })
+  const answer = await rl.question('Continue? y/n ')
+  return answer === ''
+}
+
+type RecordDescript = {
+  uri: string
+  path: string
+  cid: CID
+}

package/src/sequencer/db/schema.ts

@@ -6,6 +6,7 @@ export type RepoSeqEventType =
   | 'handle'
   | 'migrate'
   | 'identity'
+  | 'account'
   | 'tombstone'
 
 export interface RepoSeq {

package/src/sequencer/events.ts

@@ -10,6 +10,7 @@ import {
 import { PreparedWrite } from '../repo'
 import { CID } from 'multiformats/cid'
 import { RepoSeqInsert } from './db'
+import { AccountStatus } from '../account-manager'
 
 export const formatSeqCommit = async (
   did: string,
@@ -83,10 +84,14 @@ export const formatSeqHandleUpdate = async (
 
 export const formatSeqIdentityEvt = async (
   did: string,
+  handle?: string,
 ): Promise<RepoSeqInsert> => {
   const evt: IdentityEvt = {
     did,
   }
+  if (handle) {
+    evt.handle = handle
+  }
   return {
     did,
     eventType: 'identity',
@@ -95,6 +100,26 @@ export const formatSeqIdentityEvt = async (
   }
 }
 
+export const formatSeqAccountEvt = async (
+  did: string,
+  status: AccountStatus,
+): Promise<RepoSeqInsert> => {
+  const evt: AccountEvt = {
+    did,
+    active: status === 'active',
+  }
+  if (status !== AccountStatus.Active) {
+    evt.status = status
+  }
+
+  return {
+    did,
+    eventType: 'account',
+    event: cborEncode(evt),
+    sequencedAt: new Date().toISOString(),
+  }
+}
+
 export const formatSeqTombstone = async (
   did: string,
 ): Promise<RepoSeqInsert> => {
@@ -142,9 +167,24 @@ export type HandleEvt = z.infer<typeof handleEvt>
 
 export const identityEvt = z.object({
   did: z.string(),
+  handle: z.string().optional(),
 })
 export type IdentityEvt = z.infer<typeof identityEvt>
 
+export const accountEvt = z.object({
+  did: z.string(),
+  active: z.boolean(),
+  status: z
+    .enum([
+      AccountStatus.Takendown,
+      AccountStatus.Suspended,
+      AccountStatus.Deleted,
+      AccountStatus.Deactivated,
+    ])
+    .optional(),
+})
+export type AccountEvt = z.infer<typeof accountEvt>
+
 export const tombstoneEvt = z.object({
   did: z.string(),
 })
@@ -168,6 +208,12 @@ type TypedIdentityEvt = {
   time: string
   evt: IdentityEvt
 }
+type TypedAccountEvt = {
+  type: 'account'
+  seq: number
+  time: string
+  evt: AccountEvt
+}
 type TypedTombstoneEvt = {
   type: 'tombstone'
   seq: number
@@ -178,4 +224,5 @@ export type SeqEvt =
   | TypedCommitEvt
   | TypedHandleEvt
   | TypedIdentityEvt
+  | TypedAccountEvt
   | TypedTombstoneEvt

package/src/sequencer/sequencer.ts

@@ -4,11 +4,13 @@ import { seqLogger as log } from '../logger'
 import { SECOND, cborDecode, wait } from '@atproto/common'
 import { CommitData } from '@atproto/repo'
 import {
+  AccountEvt,
   CommitEvt,
   HandleEvt,
   IdentityEvt,
   SeqEvt,
   TombstoneEvt,
+  formatSeqAccountEvt,
   formatSeqCommit,
   formatSeqHandleUpdate,
   formatSeqIdentityEvt,
@@ -23,6 +25,7 @@ import {
 } from './db'
 import { PreparedWrite } from '../repo'
 import { Crawlers } from '../crawlers'
+import { AccountStatus } from '../account-manager/helpers/account'
 
 export * from './events'
 
@@ -154,6 +157,13 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
           time: row.sequencedAt,
           evt: evt as IdentityEvt,
         })
+      } else if (row.eventType === 'account') {
+        seqEvts.push({
+          type: 'account',
+          seq: row.seq,
+          time: row.sequencedAt,
+          evt: evt as AccountEvt,
+        })
       } else if (row.eventType === 'tombstone') {
         seqEvts.push({
           type: 'tombstone',
@@ -197,43 +207,54 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
     await wait(waitTime)
   }
 
-  async sequenceEvt(evt: RepoSeqInsert) {
-    await this.db.executeWithRetry(
-      this.db.db.insertInto('repo_seq').values(evt),
+  async sequenceEvt(evt: RepoSeqInsert): Promise<number> {
+    const res = await this.db.executeWithRetry(
+      this.db.db.insertInto('repo_seq').values(evt).returningAll(),
     )
     this.crawlers.notifyOfUpdate()
+    return res[0].seq
   }
 
   async sequenceCommit(
     did: string,
     commitData: CommitData,
     writes: PreparedWrite[],
-  ) {
+  ): Promise<number> {
     const evt = await formatSeqCommit(did, commitData, writes)
-    await this.sequenceEvt(evt)
+    return await this.sequenceEvt(evt)
   }
 
-  async sequenceHandleUpdate(did: string, handle: string) {
+  async sequenceHandleUpdate(did: string, handle: string): Promise<number> {
     const evt = await formatSeqHandleUpdate(did, handle)
-    await this.sequenceEvt(evt)
+    return await this.sequenceEvt(evt)
+  }
+
+  async sequenceIdentityEvt(did: string, handle?: string): Promise<number> {
+    const evt = await formatSeqIdentityEvt(did, handle)
+    return await this.sequenceEvt(evt)
   }
 
-  async sequenceIdentityEvt(did: string) {
-    const evt = await formatSeqIdentityEvt(did)
-    await this.sequenceEvt(evt)
+  async sequenceAccountEvt(
+    did: string,
+    status: AccountStatus,
+  ): Promise<number> {
+    const evt = await formatSeqAccountEvt(did, status)
+    return await this.sequenceEvt(evt)
   }
 
-  async sequenceTombstone(did: string) {
+  async sequenceTombstone(did: string): Promise<number> {
     const evt = await formatSeqTombstone(did)
-    await this.sequenceEvt(evt)
+    return await this.sequenceEvt(evt)
   }
 
-  async deleteAllForUser(did: string) {
+  async deleteAllForUser(did: string, excludingSeqs: number[] = []) {
     await this.db.executeWithRetry(
       this.db.db
         .deleteFrom('repo_seq')
         .where('did', '=', did)
-        .where('eventType', '!=', 'tombstone'),
+        .if(excludingSeqs.length > 0, (qb) =>
+          qb.where('seq', 'not in', excludingSeqs),
+        ),
     )
   }
 }

package/tests/account-deactivation.test.ts

@@ -1,5 +1,10 @@
 import AtpAgent from '@atproto/api'
-import { SeedClient, TestNetworkNoAppView, basicSeed } from '@atproto/dev-env'
+import {
+  ImageRef,
+  SeedClient,
+  TestNetworkNoAppView,
+  basicSeed,
+} from '@atproto/dev-env'
 
 describe('account deactivation', () => {
   let network: TestNetworkNoAppView
@@ -8,6 +13,7 @@ describe('account deactivation', () => {
   let agent: AtpAgent
 
   let alice: string
+  let aliceAvatar: ImageRef
 
   beforeAll(async () => {
     network = await TestNetworkNoAppView.create({
@@ -19,6 +25,16 @@ describe('account deactivation', () => {
 
     await basicSeed(sc)
     alice = sc.dids.alice
+
+    aliceAvatar = await sc.uploadFile(
+      alice,
+      '../dev-env/src/seed/img/key-portrait-small.jpg',
+      'image/jpeg',
+    )
+    await sc.updateProfile(alice, {
+      avatar: aliceAvatar.image,
+    })
+
     await network.processAll()
   })
 
@@ -33,16 +49,25 @@ describe('account deactivation', () => {
     )
   })
 
+  it('returns deactivated status', async () => {
+    const res = await agent.com.atproto.sync.getRepoStatus({ did: alice })
+    expect(res.data).toEqual({
+      did: alice,
+      active: false,
+      status: 'deactivated',
+    })
+  })
+
   it('no longer serves repo data', async () => {
     await expect(
       agent.com.atproto.sync.getRepo({ did: alice }),
-    ).rejects.toThrow()
+    ).rejects.toThrow(/Repo has been deactivated/)
     await expect(
       agent.com.atproto.sync.getLatestCommit({ did: alice }),
-    ).rejects.toThrow()
+    ).rejects.toThrow(/Repo has been deactivated/)
     await expect(
       agent.com.atproto.sync.listBlobs({ did: alice }),
-    ).rejects.toThrow()
+    ).rejects.toThrow(/Repo has been deactivated/)
     const recordUri = sc.posts[alice][0].ref.uri
     await expect(
       agent.com.atproto.sync.getRecord({
@@ -50,7 +75,7 @@ describe('account deactivation', () => {
         collection: recordUri.collection,
         rkey: recordUri.rkey,
       }),
-    ).rejects.toThrow()
+    ).rejects.toThrow(/Repo has been deactivated/)
     await expect(
       agent.com.atproto.repo.getRecord({
         repo: alice,
@@ -62,17 +87,18 @@ describe('account deactivation', () => {
       agent.com.atproto.repo.describeRepo({
         repo: alice,
       }),
-    ).rejects.toThrow()
+    ).rejects.toThrow(/Repo has been deactivated/)
 
-    const blobCid = sc.profiles[alice].avatar.cid
     await expect(
       agent.com.atproto.sync.getBlob({
         did: alice,
-        cid: blobCid,
+        cid: aliceAvatar.image.ref.toString(),
       }),
-    ).rejects.toThrow()
+    ).rejects.toThrow(/Repo has been deactivated/)
     const listedRepos = await agent.com.atproto.sync.listRepos()
-    expect(listedRepos.data.repos.find((r) => r.did === alice)).toBeUndefined()
+    const listedAlice = listedRepos.data.repos.find((r) => r.did === alice)
+    expect(listedAlice?.active).toBe(false)
+    expect(listedAlice?.status).toBe('deactivated')
   })
 
   it('no longer resolves handle', async () => {

package/tests/account-deletion.test.ts

@@ -143,10 +143,18 @@ describe('account deletion', () => {
     expect(updatedDbContents.userAccounts).toEqual(
       initialDbContents.userAccounts.filter((row) => row.did !== carol.did),
     )
-    // check all seqs for this did are gone, except for the tombstone
+    // check we didn't touch other user seqs
     expect(
-      updatedDbContents.repoSeqs.filter((row) => row.eventType !== 'tombstone'),
+      updatedDbContents.repoSeqs.filter((row) => row.did !== carol.did),
     ).toEqual(initialDbContents.repoSeqs.filter((row) => row.did !== carol.did))
+    // check all seqs for this did are gone, except for the tombstone & account events
+    expect(
+      updatedDbContents.repoSeqs
+        .filter((row) => row.did === carol.did)
+        .every(
+          (row) => row.eventType === 'tombstone' || row.eventType === 'account',
+        ),
+    ).toBe(true)
     // check we do have a tombstone for this did
     expect(
       updatedDbContents.repoSeqs.filter(

package/tests/moderation.test.ts

@@ -238,12 +238,12 @@ describe('moderation', () => {
       }
       // public, disallow
       const attempt1 = agent.api.com.atproto.sync.getBlob(blobParams)
-      await expect(attempt1).rejects.toThrow('Blob not found')
+      await expect(attempt1).rejects.toThrow(/Repo has been takendown/)
       // logged-in, disallow
       const attempt2 = agent.api.com.atproto.sync.getBlob(blobParams, {
         headers: sc.getHeaders(sc.dids.bob),
       })
-      await expect(attempt2).rejects.toThrow('Blob not found')
+      await expect(attempt2).rejects.toThrow(/Repo has been takendown/)
       // logged-in as account, allow
       const res1 = await agent.api.com.atproto.sync.getBlob(blobParams, {
         headers: sc.getHeaders(sc.dids.carol),

package/tests/proxied/notif.test.ts

@@ -37,6 +37,7 @@ describe('notif service proxy', () => {
       }
       return x
     })
+    await network.pds.ctx.idResolver.did.resolve(notifDid, true)
   })
 
   afterAll(async () => {

package/tests/sequencer.test.ts

@@ -25,8 +25,8 @@ describe('sequencer', () => {
     await userSeed(sc)
     alice = sc.dids.alice
     bob = sc.dids.bob
-    // 10 events in userSeed
-    totalEvts = 10
+    // 14 events in userSeed
+    totalEvts = 14
   })
 
   afterAll(async () => {

package/tests/sync/list.test.ts

@@ -29,6 +29,7 @@ describe('sync listing', () => {
       sc.dids.carol,
       sc.dids.dan,
     ])
+    expect(res.data.repos.every((r) => r.active === true)).toBe(true)
   })
 
   it('paginates listed hosted repos', async () => {