@atproto/pds 0.4.104 → 0.4.106

package/src/scripts/rebuild-repo.ts

@@ -77,7 +77,10 @@ export const rebuildRepo = async (ctx: AppContext, args: string[]) => {
     }
   })
   await ctx.accountManager.updateRepoRoot(did, commit.cid, rev)
-  await ctx.sequencer.sequenceCommit(did, commit)
+  const syncData = await ctx.actorStore.read(did, (store) =>
+    store.repo.getSyncEventData(),
+  )
+  await ctx.sequencer.sequenceSyncEvt(did, syncData)
 }
 
 const promptContinue = async (): Promise<boolean> => {

package/src/sequencer/db/schema.ts

@@ -1,13 +1,6 @@
 import { Generated, GeneratedAlways, Insertable, Selectable } from 'kysely'
 
-export type RepoSeqEventType =
-  | 'append'
-  | 'rebase'
-  | 'handle'
-  | 'migrate'
-  | 'identity'
-  | 'account'
-  | 'tombstone'
+export type RepoSeqEventType = 'append' | 'sync' | 'identity' | 'account'
 
 export interface RepoSeq {
   seq: GeneratedAlways<number>

package/src/sequencer/events.ts

@@ -1,8 +1,9 @@
+import assert from 'node:assert'
 import { z } from 'zod'
 import { cborEncode, noUndefinedVals, schema } from '@atproto/common'
 import { BlockMap, blocksToCarFile } from '@atproto/repo'
-import { AccountStatus } from '../account-manager'
-import { CommitDataWithOps } from '../repo'
+import { AccountStatus } from '../account-manager/account-manager'
+import { CommitDataWithOps, SyncEvtData } from '../repo'
 import { RepoSeqInsert } from './db'
 
 export const formatSeqCommit = async (
@@ -13,41 +14,18 @@ export const formatSeqCommit = async (
   blocksToSend.addMap(commitData.newBlocks)
   blocksToSend.addMap(commitData.relevantBlocks)
 
-  let evt: CommitEvt
-
-  // If event is too big (max 200 ops or 1MB of data)
-  if (commitData.ops.length > 200 || blocksToSend.byteSize > 1000000) {
-    const justRoot = new BlockMap()
-    const rootBlock = blocksToSend.get(commitData.cid)
-    if (rootBlock) {
-      justRoot.set(commitData.cid, rootBlock)
-    }
-
-    evt = {
-      rebase: false,
-      tooBig: true,
-      repo: did,
-      commit: commitData.cid,
-      rev: commitData.rev,
-      since: commitData.since,
-      blocks: await blocksToCarFile(commitData.cid, justRoot),
-      ops: [],
-      blobs: [],
-      prevData: commitData.prevData ?? undefined,
-    }
-  } else {
-    evt = {
-      rebase: false,
-      tooBig: false,
-      repo: did,
-      commit: commitData.cid,
-      rev: commitData.rev,
-      since: commitData.since,
-      blocks: await blocksToCarFile(commitData.cid, blocksToSend),
-      ops: commitData.ops,
-      blobs: commitData.blobs.toList(),
-      prevData: commitData.prevData ?? undefined,
-    }
+  const evt = {
+    repo: did,
+    commit: commitData.cid,
+    rev: commitData.rev,
+    since: commitData.since,
+    blocks: await blocksToCarFile(commitData.cid, blocksToSend),
+    ops: commitData.ops,
+    prevData: commitData.prevData ?? undefined,
+    // deprecated (but still required) fields
+    rebase: false,
+    tooBig: false,
+    blobs: [],
   }
 
   return {
@@ -58,22 +36,41 @@ export const formatSeqCommit = async (
   }
 }
 
-export const formatSeqHandleUpdate = async (
+export const formatSeqSyncEvt = async (
   did: string,
-  handle: string,
+  data: SyncEvtData,
 ): Promise<RepoSeqInsert> => {
-  const evt: HandleEvt = {
+  const blocks = await blocksToCarFile(data.cid, data.blocks)
+  const evt: SyncEvt = {
     did,
-    handle,
+    rev: data.rev,
+    blocks,
   }
   return {
     did,
-    eventType: 'handle',
+    eventType: 'sync',
     event: cborEncode(evt),
     sequencedAt: new Date().toISOString(),
   }
 }
 
+export const syncEvtDataFromCommit = (
+  commitData: CommitDataWithOps,
+): SyncEvtData => {
+  const { blocks, missing } = commitData.relevantBlocks.getMany([
+    commitData.cid,
+  ])
+  assert(
+    !missing.length,
+    'commit block was not found, could not build sync event',
+  )
+  return {
+    rev: commitData.rev,
+    cid: commitData.cid,
+    blocks,
+  }
+}
+
 export const formatSeqIdentityEvt = async (
   did: string,
   handle?: string,
@@ -112,20 +109,6 @@ export const formatSeqAccountEvt = async (
   }
 }
 
-export const formatSeqTombstone = async (
-  did: string,
-): Promise<RepoSeqInsert> => {
-  const evt: TombstoneEvt = {
-    did,
-  }
-  return {
-    did,
-    eventType: 'tombstone',
-    event: cborEncode(evt),
-    sequencedAt: new Date().toISOString(),
-  }
-}
-
 export const commitEvtOp = z.object({
   action: z.union([
     z.literal('create'),
@@ -152,11 +135,12 @@ export const commitEvt = z.object({
 })
 export type CommitEvt = z.infer<typeof commitEvt>
 
-export const handleEvt = z.object({
+export const syncEvt = z.object({
   did: z.string(),
-  handle: z.string(),
+  blocks: schema.bytes,
+  rev: z.string(),
 })
-export type HandleEvt = z.infer<typeof handleEvt>
+export type SyncEvt = z.infer<typeof syncEvt>
 
 export const identityEvt = z.object({
   did: z.string(),
@@ -178,22 +162,17 @@ export const accountEvt = z.object({
 })
 export type AccountEvt = z.infer<typeof accountEvt>
 
-export const tombstoneEvt = z.object({
-  did: z.string(),
-})
-export type TombstoneEvt = z.infer<typeof tombstoneEvt>
-
 type TypedCommitEvt = {
   type: 'commit'
   seq: number
   time: string
   evt: CommitEvt
 }
-type TypedHandleEvt = {
-  type: 'handle'
+type TypedSyncEvt = {
+  type: 'sync'
   seq: number
   time: string
-  evt: HandleEvt
+  evt: SyncEvt
 }
 type TypedIdentityEvt = {
   type: 'identity'
@@ -207,15 +186,8 @@ type TypedAccountEvt = {
   time: string
   evt: AccountEvt
 }
-type TypedTombstoneEvt = {
-  type: 'tombstone'
-  seq: number
-  time: string
-  evt: TombstoneEvt
-}
 export type SeqEvt =
   | TypedCommitEvt
-  | TypedHandleEvt
+  | TypedSyncEvt
   | TypedIdentityEvt
   | TypedAccountEvt
-  | TypedTombstoneEvt

package/src/sequencer/sequencer.ts

@@ -4,7 +4,7 @@ import { SECOND, cborDecode, wait } from '@atproto/common'
 import { AccountStatus } from '../account-manager/helpers/account'
 import { Crawlers } from '../crawlers'
 import { seqLogger as log } from '../logger'
-import { CommitDataWithOps } from '../repo'
+import { CommitDataWithOps, SyncEvtData } from '../repo'
 import {
   RepoSeqEntry,
   RepoSeqInsert,
@@ -15,15 +15,13 @@ import {
 import {
   AccountEvt,
   CommitEvt,
-  HandleEvt,
   IdentityEvt,
   SeqEvt,
-  TombstoneEvt,
+  SyncEvt,
   formatSeqAccountEvt,
   formatSeqCommit,
-  formatSeqHandleUpdate,
   formatSeqIdentityEvt,
-  formatSeqTombstone,
+  formatSeqSyncEvt,
 } from './events'
 
 export * from './events'
@@ -135,19 +133,19 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
         continue
       }
       const evt = cborDecode(row.event)
-      if (row.eventType === 'append' || row.eventType === 'rebase') {
+      if (row.eventType === 'append') {
         seqEvts.push({
           type: 'commit',
           seq: row.seq,
           time: row.sequencedAt,
           evt: evt as CommitEvt,
         })
-      } else if (row.eventType === 'handle') {
+      } else if (row.eventType === 'sync') {
         seqEvts.push({
-          type: 'handle',
+          type: 'sync',
           seq: row.seq,
           time: row.sequencedAt,
-          evt: evt as HandleEvt,
+          evt: evt as SyncEvt,
         })
       } else if (row.eventType === 'identity') {
         seqEvts.push({
@@ -163,13 +161,6 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
           time: row.sequencedAt,
           evt: evt as AccountEvt,
         })
-      } else if (row.eventType === 'tombstone') {
-        seqEvts.push({
-          type: 'tombstone',
-          seq: row.seq,
-          time: row.sequencedAt,
-          evt: evt as TombstoneEvt,
-        })
       }
     }
 
@@ -222,8 +213,8 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
     return await this.sequenceEvt(evt)
   }
 
-  async sequenceHandleUpdate(did: string, handle: string): Promise<number> {
-    const evt = await formatSeqHandleUpdate(did, handle)
+  async sequenceSyncEvt(did: string, data: SyncEvtData) {
+    const evt = await formatSeqSyncEvt(did, data)
     return await this.sequenceEvt(evt)
   }
 
@@ -240,11 +231,6 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
     return await this.sequenceEvt(evt)
   }
 
-  async sequenceTombstone(did: string): Promise<number> {
-    const evt = await formatSeqTombstone(did)
-    return await this.sequenceEvt(evt)
-  }
-
   async deleteAllForUser(did: string, excludingSeqs: number[] = []) {
     await this.db.executeWithRetry(
       this.db.db

package/tests/account-deletion.test.ts

@@ -151,14 +151,12 @@ describe('account deletion', () => {
     expect(
       updatedDbContents.repoSeqs
         .filter((row) => row.did === carol.did)
-        .every(
-          (row) => row.eventType === 'tombstone' || row.eventType === 'account',
-        ),
+        .every((row) => row.eventType === 'account'),
     ).toBe(true)
-    // check we do have a tombstone for this did
+    // check we do have a account (deletion) event for this did
     expect(
       updatedDbContents.repoSeqs.filter(
-        (row) => row.did === carol.did && row.eventType === 'tombstone',
+        (row) => row.did === carol.did && row.eventType === 'account',
       ).length,
     ).toEqual(1)
     expect(updatedDbContents.appPasswords).toEqual(