@atproto/pds 0.4.100 → 0.4.102

package/src/lexicon/lexicons.ts

@@ -3517,7 +3517,14 @@ export const schemaDict = {
                 type: 'string',
                 description:
                   'If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted.',
-                knownValues: ['takendown', 'suspended', 'deactivated'],
+                knownValues: [
+                  'takendown',
+                  'suspended',
+                  'deleted',
+                  'deactivated',
+                  'desynchronized',
+                  'throttled',
+                ],
               },
               rev: {
                 type: 'string',
@@ -3671,7 +3678,14 @@ export const schemaDict = {
             type: 'string',
             description:
               'If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted.',
-            knownValues: ['takendown', 'suspended', 'deactivated'],
+            knownValues: [
+              'takendown',
+              'suspended',
+              'deleted',
+              'deactivated',
+              'desynchronized',
+              'throttled',
+            ],
           },
         },
       },
@@ -3810,6 +3824,7 @@ export const schemaDict = {
             type: 'union',
             refs: [
               'lex:com.atproto.sync.subscribeRepos#commit',
+              'lex:com.atproto.sync.subscribeRepos#sync',
               'lex:com.atproto.sync.subscribeRepos#identity',
               'lex:com.atproto.sync.subscribeRepos#account',
               'lex:com.atproto.sync.subscribeRepos#handle',
@@ -3847,7 +3862,7 @@ export const schemaDict = {
           'blobs',
           'time',
         ],
-        nullable: ['prev', 'since'],
+        nullable: ['since'],
         properties: {
           seq: {
             type: 'integer',
@@ -3860,22 +3875,18 @@ export const schemaDict = {
           tooBig: {
             type: 'boolean',
             description:
-              'Indicates that this commit contained too many ops, or data size was too large. Consumers will need to make a separate request to get missing data.',
+              'DEPRECATED -- replaced by #sync event and data limits. Indicates that this commit contained too many ops, or data size was too large. Consumers will need to make a separate request to get missing data.',
           },
           repo: {
             type: 'string',
             format: 'did',
-            description: 'The repo this event comes from.',
+            description:
+              "The repo this event comes from. Note that all other message types name this field 'did'.",
           },
           commit: {
             type: 'cid-link',
             description: 'Repo commit object CID.',
           },
-          prev: {
-            type: 'cid-link',
-            description:
-              'DEPRECATED -- unused. WARNING -- nullable and optional; stick with optional to ensure golang interoperability.',
-          },
           rev: {
             type: 'string',
             format: 'tid',
@@ -3891,8 +3902,8 @@ export const schemaDict = {
           blocks: {
             type: 'bytes',
             description:
-              'CAR file containing relevant blocks, as a diff since the previous repo state.',
-            maxLength: 1000000,
+              "CAR file containing relevant blocks, as a diff since the previous repo state. The commit must be included as a block, and the commit block CID must be the first entry in the CAR header 'roots' list.",
+            maxLength: 2000000,
           },
           ops: {
             type: 'array',
@@ -3909,9 +3920,49 @@ export const schemaDict = {
             items: {
               type: 'cid-link',
               description:
-                'List of new blobs (by CID) referenced by records in this commit.',
+                'DEPRECATED -- will soon always be empty. List of new blobs (by CID) referenced by records in this commit.',
             },
           },
+          prevData: {
+            type: 'cid-link',
+            description:
+              "The root CID of the MST tree for the previous commit from this repo (indicated by the 'since' revision field in this message). Corresponds to the 'data' field in the repo commit object. NOTE: this field is effectively required for the 'inductive' version of firehose.",
+          },
+          time: {
+            type: 'string',
+            format: 'datetime',
+            description:
+              'Timestamp of when this message was originally broadcast.',
+          },
+        },
+      },
+      sync: {
+        type: 'object',
+        description:
+          'Updates the repo to a new state, without necessarily including that state on the firehose. Used to recover from broken commit streams, data loss incidents, or in situations where upstream host does not know recent state of the repository.',
+        required: ['seq', 'did', 'blocks', 'rev', 'time'],
+        properties: {
+          seq: {
+            type: 'integer',
+            description: 'The stream sequence number of this message.',
+          },
+          did: {
+            type: 'string',
+            format: 'did',
+            description:
+              'The account this repo event corresponds to. Must match that in the commit object.',
+          },
+          blocks: {
+            type: 'bytes',
+            description:
+              "CAR file containing the commit, as a block. The CAR header must include the commit block CID as the first 'root'.",
+            maxLength: 10000,
+          },
+          rev: {
+            type: 'string',
+            description:
+              'The rev of the commit. This value must match that in the commit object.',
+          },
           time: {
             type: 'string',
             format: 'datetime',
@@ -3971,7 +4022,14 @@ export const schemaDict = {
             type: 'string',
             description:
               'If active=false, this optional field indicates a reason for why the account is not active.',
-            knownValues: ['takendown', 'suspended', 'deleted', 'deactivated'],
+            knownValues: [
+              'takendown',
+              'suspended',
+              'deleted',
+              'deactivated',
+              'desynchronized',
+              'throttled',
+            ],
           },
         },
       },
@@ -4068,6 +4126,11 @@ export const schemaDict = {
             description:
               'For creates and updates, the new record CID. For deletions, null.',
           },
+          prev: {
+            type: 'cid-link',
+            description:
+              'For updates and deletes, the previous record CID (required for inductive firehose). For creations, field should not be defined.',
+          },
         },
       },
     },
@@ -10979,6 +11042,40 @@ export const schemaDict = {
       },
     },
   },
+  ChatBskyConvoUpdateAllRead: {
+    lexicon: 1,
+    id: 'chat.bsky.convo.updateAllRead',
+    defs: {
+      main: {
+        type: 'procedure',
+        input: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            properties: {
+              status: {
+                type: 'string',
+                knownValues: ['request', 'accepted'],
+              },
+            },
+          },
+        },
+        output: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            required: ['updatedCount'],
+            properties: {
+              updatedCount: {
+                description: 'The count of updated convos.',
+                type: 'integer',
+              },
+            },
+          },
+        },
+      },
+    },
+  },
   ChatBskyConvoUpdateRead: {
     lexicon: 1,
     id: 'chat.bsky.convo.updateRead',
@@ -11777,8 +11874,8 @@ export const schemaDict = {
       },
       modEventComment: {
         type: 'object',
-        description: 'Add a comment to a subject',
-        required: ['comment'],
+        description:
+          'Add a comment to a subject. An empty comment will clear any previously set sticky comment.',
         properties: {
           comment: {
             type: 'string',
@@ -14445,6 +14542,7 @@ export const ids = {
   ChatBskyConvoSendMessage: 'chat.bsky.convo.sendMessage',
   ChatBskyConvoSendMessageBatch: 'chat.bsky.convo.sendMessageBatch',
   ChatBskyConvoUnmuteConvo: 'chat.bsky.convo.unmuteConvo',
+  ChatBskyConvoUpdateAllRead: 'chat.bsky.convo.updateAllRead',
   ChatBskyConvoUpdateRead: 'chat.bsky.convo.updateRead',
   ChatBskyModerationGetActorMetadata: 'chat.bsky.moderation.getActorMetadata',
   ChatBskyModerationGetMessageContext: 'chat.bsky.moderation.getMessageContext',

package/src/lexicon/types/chat/bsky/convo/updateAllRead.ts

@@ -0,0 +1,53 @@
+/**
+ * GENERATED CODE - DO NOT MODIFY
+ */
+import express from 'express'
+import { ValidationResult, BlobRef } from '@atproto/lexicon'
+import { CID } from 'multiformats/cid'
+import { validate as _validate } from '../../../../lexicons'
+import { $Typed, is$typed as _is$typed, OmitKey } from '../../../../util'
+import { HandlerAuth, HandlerPipeThrough } from '@atproto/xrpc-server'
+
+const is$typed = _is$typed,
+  validate = _validate
+const id = 'chat.bsky.convo.updateAllRead'
+
+export interface QueryParams {}
+
+export interface InputSchema {
+  status?: 'request' | 'accepted' | (string & {})
+}
+
+export interface OutputSchema {
+  /** The count of updated convos. */
+  updatedCount: number
+}
+
+export interface HandlerInput {
+  encoding: 'application/json'
+  body: InputSchema
+}
+
+export interface HandlerSuccess {
+  encoding: 'application/json'
+  body: OutputSchema
+  headers?: { [key: string]: string }
+}
+
+export interface HandlerError {
+  status: number
+  message?: string
+}
+
+export type HandlerOutput = HandlerError | HandlerSuccess | HandlerPipeThrough
+export type HandlerReqCtx<HA extends HandlerAuth = never> = {
+  auth: HA
+  params: QueryParams
+  input: HandlerInput
+  req: express.Request
+  res: express.Response
+  resetRouteRateLimits: () => Promise<void>
+}
+export type Handler<HA extends HandlerAuth = never> = (
+  ctx: HandlerReqCtx<HA>,
+) => Promise<HandlerOutput> | HandlerOutput

package/src/lexicon/types/com/atproto/sync/getRepoStatus.ts

@@ -23,7 +23,14 @@ export interface OutputSchema {
   did: string
   active: boolean
   /** If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. */
-  status?: 'takendown' | 'suspended' | 'deactivated' | (string & {})
+  status?:
+    | 'takendown'
+    | 'suspended'
+    | 'deleted'
+    | 'deactivated'
+    | 'desynchronized'
+    | 'throttled'
+    | (string & {})
   /** Optional field, the current rev of the repo, if active=true */
   rev?: string
 }

package/src/lexicon/types/com/atproto/sync/listRepos.ts

@@ -58,7 +58,14 @@ export interface Repo {
   rev: string
   active?: boolean
   /** If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. */
-  status?: 'takendown' | 'suspended' | 'deactivated' | (string & {})
+  status?:
+    | 'takendown'
+    | 'suspended'
+    | 'deleted'
+    | 'deactivated'
+    | 'desynchronized'
+    | 'throttled'
+    | (string & {})
 }
 
 const hashRepo = 'repo'

package/src/lexicon/types/com/atproto/sync/subscribeRepos.ts

@@ -19,6 +19,7 @@ export interface QueryParams {
 
 export type OutputSchema =
   | $Typed<Commit>
+  | $Typed<Sync>
   | $Typed<Identity>
   | $Typed<Account>
   | $Typed<Handle>
@@ -45,22 +46,22 @@ export interface Commit {
   seq: number
   /** DEPRECATED -- unused */
   rebase: boolean
-  /** Indicates that this commit contained too many ops, or data size was too large. Consumers will need to make a separate request to get missing data. */
+  /** DEPRECATED -- replaced by #sync event and data limits. Indicates that this commit contained too many ops, or data size was too large. Consumers will need to make a separate request to get missing data. */
   tooBig: boolean
-  /** The repo this event comes from. */
+  /** The repo this event comes from. Note that all other message types name this field 'did'. */
   repo: string
   /** Repo commit object CID. */
   commit: CID
-  /** DEPRECATED -- unused. WARNING -- nullable and optional; stick with optional to ensure golang interoperability. */
-  prev?: CID | null
   /** The rev of the emitted commit. Note that this information is also in the commit object included in blocks, unless this is a tooBig event. */
   rev: string
   /** The rev of the last emitted commit from this repo (if any). */
   since: string | null
-  /** CAR file containing relevant blocks, as a diff since the previous repo state. */
+  /** CAR file containing relevant blocks, as a diff since the previous repo state. The commit must be included as a block, and the commit block CID must be the first entry in the CAR header 'roots' list. */
   blocks: Uint8Array
   ops: RepoOp[]
   blobs: CID[]
+  /** The root CID of the MST tree for the previous commit from this repo (indicated by the 'since' revision field in this message). Corresponds to the 'data' field in the repo commit object. NOTE: this field is effectively required for the 'inductive' version of firehose. */
+  prevData?: CID
   /** Timestamp of when this message was originally broadcast. */
   time: string
 }
@@ -75,6 +76,31 @@ export function validateCommit<V>(v: V) {
   return validate<Commit & V>(v, id, hashCommit)
 }
 
+/** Updates the repo to a new state, without necessarily including that state on the firehose. Used to recover from broken commit streams, data loss incidents, or in situations where upstream host does not know recent state of the repository. */
+export interface Sync {
+  $type?: 'com.atproto.sync.subscribeRepos#sync'
+  /** The stream sequence number of this message. */
+  seq: number
+  /** The account this repo event corresponds to. Must match that in the commit object. */
+  did: string
+  /** CAR file containing the commit, as a block. The CAR header must include the commit block CID as the first 'root'. */
+  blocks: Uint8Array
+  /** The rev of the commit. This value must match that in the commit object. */
+  rev: string
+  /** Timestamp of when this message was originally broadcast. */
+  time: string
+}
+
+const hashSync = 'sync'
+
+export function isSync<V>(v: V) {
+  return is$typed(v, id, hashSync)
+}
+
+export function validateSync<V>(v: V) {
+  return validate<Sync & V>(v, id, hashSync)
+}
+
 /** Represents a change to an account's identity. Could be an updated handle, signing key, or pds hosting endpoint. Serves as a prod to all downstream services to refresh their identity cache. */
 export interface Identity {
   $type?: 'com.atproto.sync.subscribeRepos#identity'
@@ -104,7 +130,14 @@ export interface Account {
   /** Indicates that the account has a repository which can be fetched from the host that emitted this event. */
   active: boolean
   /** If active=false, this optional field indicates a reason for why the account is not active. */
-  status?: 'takendown' | 'suspended' | 'deleted' | 'deactivated' | (string & {})
+  status?:
+    | 'takendown'
+    | 'suspended'
+    | 'deleted'
+    | 'deactivated'
+    | 'desynchronized'
+    | 'throttled'
+    | (string & {})
 }
 
 const hashAccount = 'account'
@@ -196,6 +229,8 @@ export interface RepoOp {
   path: string
   /** For creates and updates, the new record CID. For deletions, null. */
   cid: CID | null
+  /** For updates and deletes, the previous record CID (required for inductive firehose). For creations, field should not be defined. */
+  prev?: CID
 }
 
 const hashRepoOp = 'repoOp'

package/src/lexicon/types/tools/ozone/moderation/defs.ts

@@ -284,10 +284,10 @@ export function validateModEventResolveAppeal<V>(v: V) {
   return validate<ModEventResolveAppeal & V>(v, id, hashModEventResolveAppeal)
 }
 
-/** Add a comment to a subject */
+/** Add a comment to a subject. An empty comment will clear any previously set sticky comment. */
 export interface ModEventComment {
   $type?: 'tools.ozone.moderation.defs#modEventComment'
-  comment: string
+  comment?: string
   /** Make the comment persistent on the subject */
   sticky?: boolean
 }

package/src/repo/types.ts

@@ -1,6 +1,6 @@
 import { CID } from 'multiformats/cid'
 import { RepoRecord } from '@atproto/lexicon'
-import { WriteOpAction } from '@atproto/repo'
+import { CidSet, CommitData, WriteOpAction } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
 
 export type ValidationStatus = 'valid' | 'unknown' | undefined
@@ -42,6 +42,19 @@ export type PreparedDelete = {
   swapCid?: CID | null
 }
 
+export type CommitOp = {
+  action: 'create' | 'update' | 'delete'
+  path: string
+  cid: CID | null
+  prev?: CID
+}
+
+export type CommitDataWithOps = CommitData & {
+  ops: CommitOp[]
+  blobs: CidSet
+  prevData: CID | null
+}
+
 export type PreparedWrite = PreparedCreate | PreparedUpdate | PreparedDelete
 
 export class InvalidRecordError extends Error {}

package/src/scripts/rebuild-repo.ts

@@ -71,10 +71,13 @@ export const rebuildRepo = async (ctx: AppContext, args: string[]) => {
       newBlocks,
       relevantBlocks: newBlocks,
       removedCids: toDelete,
+      ops: [],
+      blobs: new CidSet(),
+      prevData: null,
     }
   })
   await ctx.accountManager.updateRepoRoot(did, commit.cid, rev)
-  await ctx.sequencer.sequenceCommit(did, commit, [])
+  await ctx.sequencer.sequenceCommit(did, commit)
 }
 
 const promptContinue = async (): Promise<boolean> => {

package/src/sequencer/events.ts

@@ -1,74 +1,59 @@
-import { CID } from 'multiformats/cid'
 import { z } from 'zod'
-import { cborEncode, schema } from '@atproto/common'
-import {
-  BlockMap,
-  CidSet,
-  CommitData,
-  WriteOpAction,
-  blocksToCarFile,
-} from '@atproto/repo'
+import { cborEncode, noUndefinedVals, schema } from '@atproto/common'
+import { BlockMap, blocksToCarFile } from '@atproto/repo'
 import { AccountStatus } from '../account-manager'
-import { PreparedWrite } from '../repo'
+import { CommitDataWithOps } from '../repo'
 import { RepoSeqInsert } from './db'
 
 export const formatSeqCommit = async (
   did: string,
-  commitData: CommitData,
-  writes: PreparedWrite[],
+  commitData: CommitDataWithOps,
 ): Promise<RepoSeqInsert> => {
-  let tooBig: boolean
-  const ops: CommitEvtOp[] = []
-  const blobs = new CidSet()
-  let carSlice: Uint8Array
-
   const blocksToSend = new BlockMap()
   blocksToSend.addMap(commitData.newBlocks)
   blocksToSend.addMap(commitData.relevantBlocks)
 
-  // max 200 ops or 1MB of data
-  if (writes.length > 200 || blocksToSend.byteSize > 1000000) {
-    tooBig = true
+  let evt: CommitEvt
+
+  // If event is too big (max 200 ops or 1MB of data)
+  if (commitData.ops.length > 200 || blocksToSend.byteSize > 1000000) {
     const justRoot = new BlockMap()
     const rootBlock = blocksToSend.get(commitData.cid)
     if (rootBlock) {
       justRoot.set(commitData.cid, rootBlock)
     }
-    carSlice = await blocksToCarFile(commitData.cid, justRoot)
+
+    evt = {
+      rebase: false,
+      tooBig: true,
+      repo: did,
+      commit: commitData.cid,
+      rev: commitData.rev,
+      since: commitData.since,
+      blocks: await blocksToCarFile(commitData.cid, justRoot),
+      ops: [],
+      blobs: [],
+      prevData: commitData.prevData ?? undefined,
+    }
   } else {
-    tooBig = false
-    for (const w of writes) {
-      const path = w.uri.collection + '/' + w.uri.rkey
-      let cid: CID | null
-      if (w.action === WriteOpAction.Delete) {
-        cid = null
-      } else {
-        cid = w.cid
-        w.blobs.forEach((blob) => {
-          blobs.add(blob.cid)
-        })
-      }
-      ops.push({ action: w.action, path, cid })
+    evt = {
+      rebase: false,
+      tooBig: false,
+      repo: did,
+      commit: commitData.cid,
+      rev: commitData.rev,
+      since: commitData.since,
+      blocks: await blocksToCarFile(commitData.cid, blocksToSend),
+      ops: commitData.ops,
+      blobs: commitData.blobs.toList(),
+      prevData: commitData.prevData ?? undefined,
     }
-    carSlice = await blocksToCarFile(commitData.cid, blocksToSend)
   }
 
-  const evt: CommitEvt = {
-    rebase: false,
-    tooBig,
-    repo: did,
-    commit: commitData.cid,
-    prev: commitData.prev,
-    rev: commitData.rev,
-    since: commitData.since,
-    ops,
-    blocks: carSlice,
-    blobs: blobs.toList(),
-  }
   return {
     did,
     eventType: 'append' as const,
-    event: cborEncode(evt),
+    event: cborEncode(noUndefinedVals(evt)),
     sequencedAt: new Date().toISOString(),
   }
 }
@@ -149,6 +134,7 @@ export const commitEvtOp = z.object({
   ]),
   path: z.string(),
   cid: schema.cid.nullable(),
+  prev: schema.cid.optional(),
 })
 export type CommitEvtOp = z.infer<typeof commitEvtOp>
 
@@ -157,12 +143,12 @@ export const commitEvt = z.object({
   tooBig: z.boolean(),
   repo: z.string(),
   commit: schema.cid,
-  prev: schema.cid.nullable(),
   rev: z.string(),
   since: z.string().nullable(),
   blocks: schema.bytes,
   ops: z.array(commitEvtOp),
   blobs: z.array(schema.cid),
+  prevData: schema.cid.optional(),
 })
 export type CommitEvt = z.infer<typeof commitEvt>
 

package/src/sequencer/sequencer.ts

@@ -1,11 +1,10 @@
 import EventEmitter from 'node:events'
 import TypedEmitter from 'typed-emitter'
 import { SECOND, cborDecode, wait } from '@atproto/common'
-import { CommitData } from '@atproto/repo'
 import { AccountStatus } from '../account-manager/helpers/account'
 import { Crawlers } from '../crawlers'
 import { seqLogger as log } from '../logger'
-import { PreparedWrite } from '../repo'
+import { CommitDataWithOps } from '../repo'
 import {
   RepoSeqEntry,
   RepoSeqInsert,
@@ -217,10 +216,9 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
 
   async sequenceCommit(
     did: string,
-    commitData: CommitData,
-    writes: PreparedWrite[],
+    commitData: CommitDataWithOps,
   ): Promise<number> {
-    const evt = await formatSeqCommit(did, commitData, writes)
+    const evt = await formatSeqCommit(did, commitData)
     return await this.sequenceEvt(evt)
   }
 

package/tests/crud.test.ts

@@ -1191,7 +1191,7 @@ describe('crud operations', () => {
           record: {
             text: 'x',
             createdAt: new Date().toISOString(),
-            deepObject: createDeepObject(3000),
+            deepObject: createDeepObject(4000),
           },
         }),
         {

package/tests/sequencer.test.ts

@@ -241,11 +241,7 @@ describe('sequencer', () => {
       (store) => store.repo.formatCommit(writes),
     )
 
-    const repoSeqInsert = await formatSeqCommit(
-      sc.dids.alice,
-      writeCommit,
-      writes,
-    )
+    const repoSeqInsert = await formatSeqCommit(sc.dids.alice, writeCommit)
 
     const evt = cborDecode<sequencer.CommitEvt>(repoSeqInsert.event)
     expect(evt.tooBig).toBe(true)