@atproto/pds 0.5.5 → 0.5.6

package/dist/sequencer/sequencer.js.map

@@ -1 +1 @@
-{"version":3,"file":"sequencer.js","sourceRoot":"","sources":["../../src/sequencer/sequencer.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAA;AAErE,OAAO,EAAE,SAAS,IAAI,GAAG,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAIL,KAAK,EACL,WAAW,GACZ,MAAM,eAAe,CAAA;AACtB,OAAO,EAML,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,aAAa,CAAA;AAEpB,cAAc,aAAa,CAAA;AAE3B,MAAM,OAAO,SAAU,SAAS,YAA2C;IAMzE,YACS,UAAkB,EAClB,QAAkB,EAClB,QAAQ,GAAG,CAAC,EACnB,wBAAwB,GAAG,KAAK;QAEhC,KAAK,EAAE,CAAA;QALA,eAAU,GAAV,UAAU,CAAQ;QAClB,aAAQ,GAAR,QAAQ,CAAU;QAClB,aAAQ,GAAR,QAAQ,CAAI;QAPrB,cAAS,GAAG,KAAK,CAAA;QACjB,gBAAW,GAAyB,IAAI,CAAA;QACxC,uBAAkB,GAAG,CAAC,CAAA;QASpB,0EAA0E;QAC1E,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;QACzB,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;IACvD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAA;QACzB,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACrC,MAAM,QAAQ,CAAC,sBAAsB,EAAE,CAAA;QACvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAClC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;QACxB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;aACtB,KAAK,CAAC,CAAC,CAAC;aACR,gBAAgB,EAAE,CAAA;QACrB,OAAO,GAAG,EAAE,GAAG,IAAI,IAAI,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC;aACzB,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,gBAAgB,EAAE,CAAA;QACrB,OAAO,GAAG,IAAI,IAAI,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAClC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC;aAC7B,KAAK,CAAC,CAAC,CAAC;aACR,gBAAgB,EAAE,CAAA;QACrB,OAAO,GAAG,IAAI,IAAI,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAKrB;QACC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,IAAI,CAAA;QAE5D,IAAI,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,EAAE;aACnB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAC/B,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,CAAA;QAC9C,CAAC;QACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;QAC7C,CAAC;QACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,YAAY,CAAC,CAAA;QACxD,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;QAClC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC;IAEO,KAAK,CAAC,MAAM;QAClB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAM;QAC1B,gDAAgD;QAChD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC;gBACtC,WAAW,EAAE,IAAI,CAAC,QAAQ;gBAC1B,KAAK,EAAE,IAAI;aACZ,CAAC,CAAA;YACF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;gBAC3B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;gBACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAA;YACnD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACjC,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,6BAA6B,CAAC,CAAA;YAC1E,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;YAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAClC,CAAC;IACH,CAAC;IAED,+EAA+E;IACvE,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC,kBAAkB,EAAE,CAAA;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC,CAAA;QACvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;IACtB,CAAC;IAES,KAAK,CAAC,YAAY,CAC1B,MAAgC;QAEhC,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,EAAE,CAAA;QAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CACzC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAClE,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAA;QAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnC,CAAC;IAEM,KAAK,CAAC,cAAc,CACzB,GAAc,EACd,UAA6B;QAE7B,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,eAAe,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IACnE,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAc,EAAE,IAAiB;QACzD,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9D,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAC3B,GAAc,EACd,MAAqB;QAErB,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IACpE,CAAC;IAEM,KAAK,CAAC,eAAe,CAC1B,GAAc,EACd,MAAqB;QAErB,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IACnE,CAAC;IAEM,KAAK,CAAC,uBAAuB,CAClC,GAAc,EACd,MAAoB,EACpB,MAAyB;QAEzB,iCAAiC;QACjC,MAAM,IAAI,CAAC,YAAY,CAAC;YACtB,MAAM,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC;YACvC,MAAM,mBAAmB,CAAC,GAAG,EAAE,aAAa,CAAC,MAAM,CAAC;YACpD,MAAM,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC;YAClC,MAAM,gBAAgB,CAAC,GAAG,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;SAC3D,CAAC,CAAA;IACJ,CAAC;IAEM,KAAK,CAAC,yBAAyB,CACpC,GAAc,EACd,MAAoB,EACpB,MAAqB,EACrB,QAAqB;QAErB,iCAAiC;QACjC,MAAM,IAAI,CAAC,YAAY,CAAC;YACtB,MAAM,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC;YACtC,MAAM,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC;YACvC,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC;SACtC,CAAC,CAAA;IACJ,CAAC;IAEM,KAAK,CAAC,uBAAuB,CAAC,GAAc;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YACpC,MAAM,mBAAmB,CAAC,GAAG,EAAE,aAAa,CAAC,OAAO,CAAC;SACtD,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAC5B,IAAI,CAAC,EAAE,CAAC,EAAE;aACP,UAAU,CAAC,UAAU,CAAC;aACtB,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC;aACtB,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAC3B,CAAA;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAoB,EAAY,EAAE;IACjE,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,gDAAgD;QAChD,IAAI,GAAG,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACrB,SAAQ;QACV,CAAC;QACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACjC,IAAI,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAgB;aACtB,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAc;aACpB,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,UAAU;gBAChB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAkB;aACxB,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,SAAS;gBACf,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAiB;aACvB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAWD,eAAe,SAAS,CAAA","sourcesContent":["import EventEmitter from 'node:events'\nimport type TypedEmitter from 'typed-emitter'\nimport { SECOND, wait } from '@atproto/common'\nimport { decode as cborDecode } from '@atproto/lex-cbor'\nimport { DatetimeString, DidString, HandleString } from '@atproto/syntax'\nimport { AccountStatus } from '../account-manager/helpers/account.js'\nimport { Crawlers } from '../crawlers.js'\nimport { seqLogger as log } from '../logger.js'\nimport { CommitDataWithOps, SyncEvtData } from '../repo/index.js'\nimport {\n  RepoSeqEntry,\n  RepoSeqInsert,\n  SequencerDb,\n  getDb,\n  getMigrator,\n} from './db/index.js'\nimport {\n  AccountEvt,\n  CommitEvt,\n  IdentityEvt,\n  SeqEvt,\n  SyncEvt,\n  formatSeqAccountEvt,\n  formatSeqCommit,\n  formatSeqIdentityEvt,\n  formatSeqSyncEvt,\n  syncEvtDataFromCommit,\n} from './events.js'\n\nexport * from './events.js'\n\nexport class Sequencer extends (EventEmitter as new () => SequencerEmitter) {\n  db: SequencerDb\n  destroyed = false\n  pollPromise: Promise<void> | null = null\n  triesWithNoResults = 0\n\n  constructor(\n    public dbLocation: string,\n    public crawlers: Crawlers,\n    public lastSeen = 0,\n    disableWalAutoCheckpoint = false,\n  ) {\n    super()\n    // note: this does not err when surpassed, just prints a warning to stderr\n    this.setMaxListeners(100)\n    this.db = getDb(dbLocation, disableWalAutoCheckpoint)\n  }\n\n  async start() {\n    await this.db.ensureWal()\n    const migrator = getMigrator(this.db)\n    await migrator.migrateToLatestOrThrow()\n    const curr = await this.curr()\n    this.lastSeen = curr ?? 0\n    if (this.pollPromise === null) {\n      this.pollPromise = this.pollDb()\n    }\n  }\n\n  async destroy() {\n    this.destroyed = true\n    if (this.pollPromise) {\n      await this.pollPromise\n    }\n    this.emit('close')\n  }\n\n  async curr(): Promise<number | null> {\n    const got = await this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .orderBy('seq', 'desc')\n      .limit(1)\n      .executeTakeFirst()\n    return got?.seq ?? null\n  }\n\n  async next(cursor: number): Promise<SeqRow | null> {\n    const got = await this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .where('seq', '>', cursor)\n      .limit(1)\n      .orderBy('seq', 'asc')\n      .executeTakeFirst()\n    return got || null\n  }\n\n  async earliestAfterTime(time: string): Promise<SeqRow | null> {\n    const got = await this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .where('sequencedAt', '>=', time)\n      .orderBy('sequencedAt', 'asc')\n      .limit(1)\n      .executeTakeFirst()\n    return got || null\n  }\n\n  async requestSeqRange(opts: {\n    earliestSeq?: number\n    latestSeq?: number\n    earliestTime?: string\n    limit?: number\n  }): Promise<SeqEvt[]> {\n    const { earliestSeq, latestSeq, earliestTime, limit } = opts\n\n    let seqQb = this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .orderBy('seq', 'asc')\n      .where('invalidated', '=', 0)\n    if (earliestSeq !== undefined) {\n      seqQb = seqQb.where('seq', '>', earliestSeq)\n    }\n    if (latestSeq !== undefined) {\n      seqQb = seqQb.where('seq', '<=', latestSeq)\n    }\n    if (earliestTime !== undefined) {\n      seqQb = seqQb.where('sequencedAt', '>=', earliestTime)\n    }\n    if (limit !== undefined) {\n      seqQb = seqQb.limit(limit)\n    }\n\n    const rows = await seqQb.execute()\n    if (rows.length < 1) {\n      return []\n    }\n\n    return parseRepoSeqRows(rows)\n  }\n\n  private async pollDb(): Promise<void> {\n    if (this.destroyed) return\n    // if already polling, do not start another poll\n    try {\n      const evts = await this.requestSeqRange({\n        earliestSeq: this.lastSeen,\n        limit: 1000,\n      })\n      if (evts.length > 0) {\n        this.triesWithNoResults = 0\n        this.emit('events', evts)\n        this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen\n      } else {\n        await this.exponentialBackoff()\n      }\n      this.pollPromise = this.pollDb()\n    } catch (err) {\n      log.error({ err, lastSeen: this.lastSeen }, 'sequencer failed to poll db')\n      await this.exponentialBackoff()\n      this.pollPromise = this.pollDb()\n    }\n  }\n\n  // when no results, exponential backoff on pulling, with a max of a second wait\n  private async exponentialBackoff(): Promise<void> {\n    this.triesWithNoResults++\n    const waitTime = Math.min(Math.pow(2, this.triesWithNoResults), SECOND)\n    await wait(waitTime)\n  }\n\n  protected async sequenceEvts(\n    events: readonly RepoSeqInsert[],\n  ): Promise<number[]> {\n    if (!events.length) return []\n    const rows = await this.db.executeWithRetry(\n      this.db.db.insertInto('repo_seq').values(events).returning('seq'),\n    )\n    this.crawlers.notifyOfUpdate()\n    return rows.map((row) => row.seq)\n  }\n\n  public async sequenceCommit(\n    did: DidString,\n    commitData: CommitDataWithOps,\n  ): Promise<void> {\n    await this.sequenceEvts([await formatSeqCommit(did, commitData)])\n  }\n\n  public async sequenceSync(did: DidString, data: SyncEvtData): Promise<void> {\n    await this.sequenceEvts([await formatSeqSyncEvt(did, data)])\n  }\n\n  public async sequenceIdentity(\n    did: DidString,\n    handle?: HandleString,\n  ): Promise<void> {\n    await this.sequenceEvts([await formatSeqIdentityEvt(did, handle)])\n  }\n\n  public async sequenceAccount(\n    did: DidString,\n    status: AccountStatus,\n  ): Promise<void> {\n    await this.sequenceEvts([await formatSeqAccountEvt(did, status)])\n  }\n\n  public async sequenceAccountCreation(\n    did: DidString,\n    handle: HandleString,\n    commit: CommitDataWithOps,\n  ): Promise<void> {\n    // Atomically sequence all events\n    await this.sequenceEvts([\n      await formatSeqIdentityEvt(did, handle),\n      await formatSeqAccountEvt(did, AccountStatus.Active),\n      await formatSeqCommit(did, commit),\n      await formatSeqSyncEvt(did, syncEvtDataFromCommit(commit)),\n    ])\n  }\n\n  public async sequenceAccountActivation(\n    did: DidString,\n    handle: HandleString,\n    status: AccountStatus,\n    syncData: SyncEvtData,\n  ): Promise<void> {\n    // Atomically sequence all events\n    await this.sequenceEvts([\n      await formatSeqAccountEvt(did, status),\n      await formatSeqIdentityEvt(did, handle),\n      await formatSeqSyncEvt(did, syncData),\n    ])\n  }\n\n  public async sequenceAccountDeletion(did: DidString) {\n    const [seq] = await this.sequenceEvts([\n      await formatSeqAccountEvt(did, AccountStatus.Deleted),\n    ])\n    await this.db.executeWithRetry(\n      this.db.db\n        .deleteFrom('repo_seq')\n        .where('did', '=', did)\n        .where('seq', '!=', seq),\n    )\n  }\n}\n\nexport const parseRepoSeqRows = (rows: RepoSeqEntry[]): SeqEvt[] => {\n  const seqEvts: SeqEvt[] = []\n  for (const row of rows) {\n    // should never hit this because of WHERE clause\n    if (row.seq === null) {\n      continue\n    }\n    const evt = cborDecode(row.event)\n    if (row.eventType === 'append') {\n      seqEvts.push({\n        type: 'commit',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as CommitEvt,\n      })\n    } else if (row.eventType === 'sync') {\n      seqEvts.push({\n        type: 'sync',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as SyncEvt,\n      })\n    } else if (row.eventType === 'identity') {\n      seqEvts.push({\n        type: 'identity',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as IdentityEvt,\n      })\n    } else if (row.eventType === 'account') {\n      seqEvts.push({\n        type: 'account',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as AccountEvt,\n      })\n    }\n  }\n  return seqEvts\n}\n\ntype SeqRow = RepoSeqEntry\n\ntype SequencerEvents = {\n  events: (evts: SeqEvt[]) => void\n  close: () => void\n}\n\nexport type SequencerEmitter = TypedEmitter.default<SequencerEvents>\n\nexport default Sequencer\n"]}
+{"version":3,"file":"sequencer.js","sourceRoot":"","sources":["../../src/sequencer/sequencer.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAA;AAErE,OAAO,EAAE,SAAS,IAAI,GAAG,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAIL,KAAK,EACL,WAAW,GACZ,MAAM,eAAe,CAAA;AACtB,OAAO,EAML,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,aAAa,CAAA;AAEpB,cAAc,aAAa,CAAA;AAE3B,MAAM,OAAO,SAAU,SAAS,YAA2C;IAMzE,YACS,UAAkB,EAClB,QAAkB,EAClB,QAAQ,GAAG,CAAC,EACnB,wBAAwB,GAAG,KAAK;QAEhC,KAAK,EAAE,CAAA;QALA,eAAU,GAAV,UAAU,CAAQ;QAClB,aAAQ,GAAR,QAAQ,CAAU;QAClB,aAAQ,GAAR,QAAQ,CAAI;QAPrB,cAAS,GAAG,KAAK,CAAA;QACjB,gBAAW,GAAyB,IAAI,CAAA;QACxC,uBAAkB,GAAG,CAAC,CAAA;QASpB,0EAA0E;QAC1E,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;QACzB,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAA;IACvD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAA;QACzB,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACrC,MAAM,QAAQ,CAAC,sBAAsB,EAAE,CAAA;QACvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAClC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;QACxB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;aACtB,KAAK,CAAC,CAAC,CAAC;aACR,gBAAgB,EAAE,CAAA;QACrB,OAAO,GAAG,EAAE,GAAG,IAAI,IAAI,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC;aACzB,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,gBAAgB,EAAE,CAAA;QACrB,OAAO,GAAG,IAAI,IAAI,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAClC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE;aACzB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC;aAC7B,KAAK,CAAC,CAAC,CAAC;aACR,gBAAgB,EAAE,CAAA;QACrB,OAAO,GAAG,IAAI,IAAI,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAKrB;QACC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,IAAI,CAAA;QAE5D,IAAI,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,EAAE;aACnB,UAAU,CAAC,UAAU,CAAC;aACtB,SAAS,EAAE;aACX,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAC/B,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,CAAA;QAC9C,CAAC;QACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;QAC7C,CAAC;QACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,YAAY,CAAC,CAAA;QACxD,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;QAClC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC;IAEO,KAAK,CAAC,MAAM;QAClB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAM;QAC1B,gDAAgD;QAChD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC;gBACtC,WAAW,EAAE,IAAI,CAAC,QAAQ;gBAC1B,KAAK,EAAE,IAAI;aACZ,CAAC,CAAA;YACF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;gBAC3B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;gBACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAA;YACnD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACjC,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,6BAA6B,CAAC,CAAA;YAC1E,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;YAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAClC,CAAC;IACH,CAAC;IAED,+EAA+E;IACvE,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC,kBAAkB,EAAE,CAAA;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC,CAAA;QACvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAgC;QACjD,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,EAAE,CAAA;QAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CACzC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAClE,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAA;QAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnC,CAAC;IAEM,KAAK,CAAC,cAAc,CACzB,GAAc,EACd,UAA6B;QAE7B,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,eAAe,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IACnE,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAc,EAAE,IAAiB;QACzD,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9D,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAC3B,GAAc,EACd,MAAqB;QAErB,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IACpE,CAAC;IAEM,KAAK,CAAC,eAAe,CAC1B,GAAc,EACd,MAAqB;QAErB,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IACnE,CAAC;IAEM,KAAK,CAAC,uBAAuB,CAClC,GAAc,EACd,MAAoB,EACpB,MAAyB;QAEzB,iCAAiC;QACjC,MAAM,IAAI,CAAC,YAAY,CAAC;YACtB,MAAM,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC;YACvC,MAAM,mBAAmB,CAAC,GAAG,EAAE,aAAa,CAAC,MAAM,CAAC;YACpD,MAAM,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC;YAClC,MAAM,gBAAgB,CAAC,GAAG,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;SAC3D,CAAC,CAAA;IACJ,CAAC;IAEM,KAAK,CAAC,yBAAyB,CACpC,GAAc,EACd,MAAoB,EACpB,MAAqB,EACrB,QAAqB;QAErB,iCAAiC;QACjC,MAAM,IAAI,CAAC,YAAY,CAAC;YACtB,MAAM,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC;YACtC,MAAM,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC;YACvC,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC;SACtC,CAAC,CAAA;IACJ,CAAC;IAEM,KAAK,CAAC,uBAAuB,CAAC,GAAc;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YACpC,MAAM,mBAAmB,CAAC,GAAG,EAAE,aAAa,CAAC,OAAO,CAAC;SACtD,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAC5B,IAAI,CAAC,EAAE,CAAC,EAAE;aACP,UAAU,CAAC,UAAU,CAAC;aACtB,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC;aACtB,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAC3B,CAAA;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAoB,EAAY,EAAE;IACjE,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,gDAAgD;QAChD,IAAI,GAAG,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACrB,SAAQ;QACV,CAAC;QACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACjC,IAAI,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAgB;aACtB,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAc;aACpB,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,UAAU;gBAChB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAkB;aACxB,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,SAAS;gBACf,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,WAA6B;gBACvC,GAAG,EAAE,GAAiB;aACvB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAWD,eAAe,SAAS,CAAA","sourcesContent":["import EventEmitter from 'node:events'\nimport type TypedEmitter from 'typed-emitter'\nimport { SECOND, wait } from '@atproto/common'\nimport { decode as cborDecode } from '@atproto/lex-cbor'\nimport { DatetimeString, DidString, HandleString } from '@atproto/syntax'\nimport { AccountStatus } from '../account-manager/helpers/account.js'\nimport { Crawlers } from '../crawlers.js'\nimport { seqLogger as log } from '../logger.js'\nimport { CommitDataWithOps, SyncEvtData } from '../repo/index.js'\nimport {\n  RepoSeqEntry,\n  RepoSeqInsert,\n  SequencerDb,\n  getDb,\n  getMigrator,\n} from './db/index.js'\nimport {\n  AccountEvt,\n  CommitEvt,\n  IdentityEvt,\n  SeqEvt,\n  SyncEvt,\n  formatSeqAccountEvt,\n  formatSeqCommit,\n  formatSeqIdentityEvt,\n  formatSeqSyncEvt,\n  syncEvtDataFromCommit,\n} from './events.js'\n\nexport * from './events.js'\n\nexport class Sequencer extends (EventEmitter as new () => SequencerEmitter) {\n  db: SequencerDb\n  destroyed = false\n  pollPromise: Promise<void> | null = null\n  triesWithNoResults = 0\n\n  constructor(\n    public dbLocation: string,\n    public crawlers: Crawlers,\n    public lastSeen = 0,\n    disableWalAutoCheckpoint = false,\n  ) {\n    super()\n    // note: this does not err when surpassed, just prints a warning to stderr\n    this.setMaxListeners(100)\n    this.db = getDb(dbLocation, disableWalAutoCheckpoint)\n  }\n\n  async start() {\n    await this.db.ensureWal()\n    const migrator = getMigrator(this.db)\n    await migrator.migrateToLatestOrThrow()\n    const curr = await this.curr()\n    this.lastSeen = curr ?? 0\n    if (this.pollPromise === null) {\n      this.pollPromise = this.pollDb()\n    }\n  }\n\n  async destroy() {\n    this.destroyed = true\n    if (this.pollPromise) {\n      await this.pollPromise\n    }\n    this.emit('close')\n  }\n\n  async curr(): Promise<number | null> {\n    const got = await this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .orderBy('seq', 'desc')\n      .limit(1)\n      .executeTakeFirst()\n    return got?.seq ?? null\n  }\n\n  async next(cursor: number): Promise<SeqRow | null> {\n    const got = await this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .where('seq', '>', cursor)\n      .limit(1)\n      .orderBy('seq', 'asc')\n      .executeTakeFirst()\n    return got || null\n  }\n\n  async earliestAfterTime(time: string): Promise<SeqRow | null> {\n    const got = await this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .where('sequencedAt', '>=', time)\n      .orderBy('sequencedAt', 'asc')\n      .limit(1)\n      .executeTakeFirst()\n    return got || null\n  }\n\n  async requestSeqRange(opts: {\n    earliestSeq?: number\n    latestSeq?: number\n    earliestTime?: string\n    limit?: number\n  }): Promise<SeqEvt[]> {\n    const { earliestSeq, latestSeq, earliestTime, limit } = opts\n\n    let seqQb = this.db.db\n      .selectFrom('repo_seq')\n      .selectAll()\n      .orderBy('seq', 'asc')\n      .where('invalidated', '=', 0)\n    if (earliestSeq !== undefined) {\n      seqQb = seqQb.where('seq', '>', earliestSeq)\n    }\n    if (latestSeq !== undefined) {\n      seqQb = seqQb.where('seq', '<=', latestSeq)\n    }\n    if (earliestTime !== undefined) {\n      seqQb = seqQb.where('sequencedAt', '>=', earliestTime)\n    }\n    if (limit !== undefined) {\n      seqQb = seqQb.limit(limit)\n    }\n\n    const rows = await seqQb.execute()\n    if (rows.length < 1) {\n      return []\n    }\n\n    return parseRepoSeqRows(rows)\n  }\n\n  private async pollDb(): Promise<void> {\n    if (this.destroyed) return\n    // if already polling, do not start another poll\n    try {\n      const evts = await this.requestSeqRange({\n        earliestSeq: this.lastSeen,\n        limit: 1000,\n      })\n      if (evts.length > 0) {\n        this.triesWithNoResults = 0\n        this.emit('events', evts)\n        this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen\n      } else {\n        await this.exponentialBackoff()\n      }\n      this.pollPromise = this.pollDb()\n    } catch (err) {\n      log.error({ err, lastSeen: this.lastSeen }, 'sequencer failed to poll db')\n      await this.exponentialBackoff()\n      this.pollPromise = this.pollDb()\n    }\n  }\n\n  // when no results, exponential backoff on pulling, with a max of a second wait\n  private async exponentialBackoff(): Promise<void> {\n    this.triesWithNoResults++\n    const waitTime = Math.min(Math.pow(2, this.triesWithNoResults), SECOND)\n    await wait(waitTime)\n  }\n\n  async sequenceEvts(events: readonly RepoSeqInsert[]): Promise<number[]> {\n    if (!events.length) return []\n    const rows = await this.db.executeWithRetry(\n      this.db.db.insertInto('repo_seq').values(events).returning('seq'),\n    )\n    this.crawlers.notifyOfUpdate()\n    return rows.map((row) => row.seq)\n  }\n\n  public async sequenceCommit(\n    did: DidString,\n    commitData: CommitDataWithOps,\n  ): Promise<void> {\n    await this.sequenceEvts([await formatSeqCommit(did, commitData)])\n  }\n\n  public async sequenceSync(did: DidString, data: SyncEvtData): Promise<void> {\n    await this.sequenceEvts([await formatSeqSyncEvt(did, data)])\n  }\n\n  public async sequenceIdentity(\n    did: DidString,\n    handle?: HandleString,\n  ): Promise<void> {\n    await this.sequenceEvts([await formatSeqIdentityEvt(did, handle)])\n  }\n\n  public async sequenceAccount(\n    did: DidString,\n    status: AccountStatus,\n  ): Promise<void> {\n    await this.sequenceEvts([await formatSeqAccountEvt(did, status)])\n  }\n\n  public async sequenceAccountCreation(\n    did: DidString,\n    handle: HandleString,\n    commit: CommitDataWithOps,\n  ): Promise<void> {\n    // Atomically sequence all events\n    await this.sequenceEvts([\n      await formatSeqIdentityEvt(did, handle),\n      await formatSeqAccountEvt(did, AccountStatus.Active),\n      await formatSeqCommit(did, commit),\n      await formatSeqSyncEvt(did, syncEvtDataFromCommit(commit)),\n    ])\n  }\n\n  public async sequenceAccountActivation(\n    did: DidString,\n    handle: HandleString,\n    status: AccountStatus,\n    syncData: SyncEvtData,\n  ): Promise<void> {\n    // Atomically sequence all events\n    await this.sequenceEvts([\n      await formatSeqAccountEvt(did, status),\n      await formatSeqIdentityEvt(did, handle),\n      await formatSeqSyncEvt(did, syncData),\n    ])\n  }\n\n  public async sequenceAccountDeletion(did: DidString) {\n    const [seq] = await this.sequenceEvts([\n      await formatSeqAccountEvt(did, AccountStatus.Deleted),\n    ])\n    await this.db.executeWithRetry(\n      this.db.db\n        .deleteFrom('repo_seq')\n        .where('did', '=', did)\n        .where('seq', '!=', seq),\n    )\n  }\n}\n\nexport const parseRepoSeqRows = (rows: RepoSeqEntry[]): SeqEvt[] => {\n  const seqEvts: SeqEvt[] = []\n  for (const row of rows) {\n    // should never hit this because of WHERE clause\n    if (row.seq === null) {\n      continue\n    }\n    const evt = cborDecode(row.event)\n    if (row.eventType === 'append') {\n      seqEvts.push({\n        type: 'commit',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as CommitEvt,\n      })\n    } else if (row.eventType === 'sync') {\n      seqEvts.push({\n        type: 'sync',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as SyncEvt,\n      })\n    } else if (row.eventType === 'identity') {\n      seqEvts.push({\n        type: 'identity',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as IdentityEvt,\n      })\n    } else if (row.eventType === 'account') {\n      seqEvts.push({\n        type: 'account',\n        seq: row.seq,\n        time: row.sequencedAt as DatetimeString,\n        evt: evt as AccountEvt,\n      })\n    }\n  }\n  return seqEvts\n}\n\ntype SeqRow = RepoSeqEntry\n\ntype SequencerEvents = {\n  events: (evts: SeqEvt[]) => void\n  close: () => void\n}\n\nexport type SequencerEmitter = TypedEmitter.default<SequencerEvents>\n\nexport default Sequencer\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/pds",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "license": "MIT",
   "description": "Reference implementation of atproto Personal Data Server (PDS)",
   "keywords": [
@@ -32,7 +32,7 @@
     "ioredis": "^5.3.2",
     "jose": "^5.0.1",
     "key-encoder": "^2.0.3",
-    "kysely": "^0.22.0",
+    "kysely": "^0.29.2",
     "nodemailer": "^6.8.0",
     "nodemailer-html-to-text": "^3.2.0",
     "p-queue": "^8.0.0",
@@ -44,24 +44,24 @@
     "zod": "^3.23.8",
     "@atproto-labs/fetch-node": "^0.3.1",
     "@atproto-labs/simple-store": "^0.4.1",
-    "@atproto-labs/simple-store-redis": "^0.1.1",
+    "@atproto-labs/simple-store-memory": "^0.2.1",
     "@atproto-labs/xrpc-utils": "^0.1.1",
+    "@atproto-labs/simple-store-redis": "^0.1.1",
     "@atproto/common": "^0.6.3",
+    "@atproto/aws": "^0.3.1",
     "@atproto/crypto": "^0.5.1",
-    "@atproto-labs/simple-store-memory": "^0.2.1",
     "@atproto/did": "^0.5.1",
     "@atproto/identity": "^0.5.1",
-    "@atproto/aws": "^0.3.1",
+    "@atproto/lex": "^0.1.4",
     "@atproto/lex-cbor": "^0.1.1",
     "@atproto/lex-data": "^0.1.2",
-    "@atproto/lex": "^0.1.4",
-    "@atproto/lex-json": "^0.1.1",
-    "@atproto/oauth-provider": "^0.18.4",
     "@atproto/oauth-scopes": "^0.5.1",
     "@atproto/repo": "^0.10.1",
+    "@atproto/lex-json": "^0.1.1",
+    "@atproto/syntax": "^0.6.2",
     "@atproto/xrpc": "^0.8.1",
-    "@atproto/xrpc-server": "^0.11.2",
-    "@atproto/syntax": "^0.6.2"
+    "@atproto/oauth-provider": "^0.19.0",
+    "@atproto/xrpc-server": "^0.11.2"
   },
   "devDependencies": {
     "@did-plc/server": "^0.0.1",
@@ -78,10 +78,10 @@
     "puppeteer": "^23.11.1",
     "ts-node": "^10.8.2",
     "ws": "^8.12.0",
-    "@atproto/api": "^0.20.16",
-    "@atproto/bsky": "^0.0.242",
+    "@atproto/bsky": "^0.0.243",
     "@atproto/lex-document": "^0.1.1",
-    "@atproto/oauth-client-browser-example": "^0.1.1"
+    "@atproto/oauth-client-browser-example": "^0.1.1",
+    "@atproto/api": "^0.20.16"
   },
   "type": "module",
   "exports": {

package/src/account-manager/account-manager.ts

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import { KeyObject } from 'node:crypto'
 import { Client as PlcClient } from '@did-plc/lib'
 import { isEmailValid } from '@hapi/address'
@@ -12,9 +13,16 @@ import {
   isAtIdentifierString,
 } from '@atproto/lex'
 import { Cid } from '@atproto/lex-data'
-import { currentDatetimeString, isValidTld } from '@atproto/syntax'
+import {
+  INVALID_HANDLE,
+  currentDatetimeString,
+  isValidTld,
+} from '@atproto/syntax'
 import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
+import { ActorStore } from '../actor-store/actor-store.js'
+import { assertValidDidDocumentForService } from '../api/com/atproto/server/util.js'
 import { AuthScope } from '../auth-scope.js'
+import { ServerConfig } from '../config/config.js'
 import { softDeleted } from '../db/index.js'
 import { hasExplicitSlur } from '../handle/explicit-slurs.js'
 import {
@@ -27,9 +35,10 @@ import { httpLogger } from '../logger.js'
 import { ServerMailer } from '../mailer/index.js'
 import { Sequencer } from '../sequencer/index.js'
 import { AccountDb, EmailTokenPurpose, getDb, getMigrator } from './db/index.js'
-import * as account from './helpers/account.js'
+import * as accountHelpers from './helpers/account.js'
 import { AccountStatus, ActorAccount } from './helpers/account.js'
 import * as auth from './helpers/auth.js'
+import * as authorizedClientHelper from './helpers/authorized-client.js'
 import * as emailToken from './helpers/email-token.js'
 import * as invite from './helpers/invite.js'
 import * as password from './helpers/password.js'
@@ -53,7 +62,7 @@ export { AccountStatus, formatAccountStatus } from './helpers/account.js'
  */
 export class InvalidPasswordError extends AuthRequiredError {
   constructor(
-    public readonly did: string,
+    public readonly did: DidString,
     errorMessage = 'Invalid identifier or password',
   ) {
     super(errorMessage)
@@ -69,17 +78,24 @@ export class AccountManager {
   readonly db: AccountDb
 
   constructor(
+    readonly cfg: ServerConfig,
+    readonly actorStore: ActorStore,
     readonly idResolver: IdResolver,
     readonly jwtKey: KeyObject,
     readonly mailer: ServerMailer,
     readonly sequencer: Sequencer,
     readonly plcClient: PlcClient,
     readonly plcRotationKey: Keypair,
-    readonly serviceDid: string,
-    readonly serviceHandleDomains: string[],
-    db: AccountManagerDbConfig,
   ) {
-    this.db = getDb(db.accountDbLoc, db.disableWalAutoCheckpoint)
+    this.db = getDb(cfg.db.accountDbLoc, cfg.db.disableWalAutoCheckpoint)
+  }
+
+  get serviceDid(): DidString {
+    return this.cfg.service.did
+  }
+
+  get serviceHandleDomains(): string[] {
+    return this.cfg.identity.serviceHandleDomains
   }
 
   async migrateOrThrow() {
@@ -96,23 +112,23 @@ export class AccountManager {
 
   async getAccount(
     handleOrDid: AtIdentifierString,
-    flags?: account.AvailabilityFlags,
+    flags?: accountHelpers.AvailabilityFlags,
   ): Promise<ActorAccount | null> {
-    return account.getAccount(this.db, handleOrDid, flags)
+    return accountHelpers.getAccount(this.db, handleOrDid, flags)
   }
 
   async getAccounts(
     dids: DidString[],
-    flags?: account.AvailabilityFlags,
+    flags?: accountHelpers.AvailabilityFlags,
   ): Promise<Map<string, ActorAccount>> {
-    return account.getAccounts(this.db, dids, flags)
+    return accountHelpers.getAccounts(this.db, dids, flags)
   }
 
   async getAccountByEmail(
     email: string,
-    flags?: account.AvailabilityFlags,
+    flags?: accountHelpers.AvailabilityFlags,
   ): Promise<ActorAccount | null> {
-    return account.getAccountByEmail(this.db, email, flags)
+    return accountHelpers.getAccountByEmail(this.db, email, flags)
   }
 
   async isAccountActivated(did: DidString): Promise<boolean> {
@@ -123,22 +139,26 @@ export class AccountManager {
 
   async getDidForActor(
     handleOrDid: AtIdentifierString,
-    flags?: account.AvailabilityFlags,
+    flags?: accountHelpers.AvailabilityFlags,
   ): Promise<DidString | null> {
     const got = await this.getAccount(handleOrDid, flags)
     return got?.did ?? null
   }
 
-  async getAccountStatus(
-    handleOrDid: AtIdentifierString,
-  ): Promise<AccountStatus> {
+  async getAccountStatus(handleOrDid: AtIdentifierString) {
     const got = await this.getAccount(handleOrDid, {
       includeDeactivated: true,
       includeTakenDown: true,
     })
 
-    const res = account.formatAccountStatus(got)
-    return res.active ? AccountStatus.Active : res.status
+    const { active, status = active ? AccountStatus.Active : undefined } =
+      accountHelpers.formatAccountStatus(got)
+    assert(status != null)
+    return { status, account: got } as
+      | { status: AccountStatus.Deleted; account: null }
+      | { status: AccountStatus.Takendown; account: ActorAccount }
+      | { status: AccountStatus.Deactivated; account: ActorAccount }
+      | { status: AccountStatus.Active; account: ActorAccount }
   }
 
   async normalizeAndValidateHandle(
@@ -230,10 +250,14 @@ export class AccountManager {
         await invite.ensureInviteIsAvailable(dbTxn, inviteCode)
       }
 
-      await account.registerActor(dbTxn, { did, handle, deactivated })
+      await accountHelpers.registerActor(dbTxn, { did, handle, deactivated })
 
       if (email && passwordScrypt) {
-        await account.registerAccount(dbTxn, { did, email, passwordScrypt })
+        await accountHelpers.registerAccount(dbTxn, {
+          did,
+          email,
+          passwordScrypt,
+        })
       }
 
       await invite.recordInviteUse(dbTxn, {
@@ -359,7 +383,7 @@ export class AccountManager {
     did: DidString,
     handle: HandleString,
   ): Promise<void> {
-    await account.updateHandle(this.db, did, handle)
+    await accountHelpers.updateHandle(this.db, did, handle)
 
     try {
       await this.sequencer.sequenceIdentity(did, handle)
@@ -369,36 +393,105 @@ export class AccountManager {
   }
 
   async deleteAccount(did: DidString) {
-    return account.deleteAccount(this.db, did)
+    return accountHelpers.deleteAccount(this.db, did)
   }
 
   async takedownAccount(
     did: DidString,
     takedown: com.atproto.admin.defs.StatusAttr,
   ) {
-    await this.db.transaction(async (dbTxn) =>
-      Promise.all([
-        account.updateAccountTakedownStatus(dbTxn, did, takedown),
-        auth.revokeRefreshTokensByDid(dbTxn, did),
-        token.removeByDidQB(dbTxn, did).execute(),
-      ]),
-    )
+    await this.db.transaction(async (dbTxn) => {
+      await accountHelpers.updateAccountTakedownStatus(dbTxn, did, takedown)
+      await auth.revokeRefreshTokensByDid(dbTxn, did)
+      await token.removeByDid(dbTxn, did)
+    })
+
+    await this.sequenceAccountStatus(did)
   }
 
   async getAccountAdminStatus(did: DidString) {
-    return account.getAccountAdminStatus(this.db, did)
+    return accountHelpers.getAccountAdminStatus(this.db, did)
   }
 
   async updateRepoRoot(did: DidString, cid: Cid, rev: string) {
     return repo.updateRoot(this.db, did, cid, rev)
   }
 
-  async deactivateAccount(did: DidString, deleteAfter: string | null) {
-    return account.deactivateAccount(this.db, did, deleteAfter)
+  async deactivateAccount(
+    did: DidString,
+    options?: {
+      deleteCredentials?: boolean
+      deleteAfter?: string | null
+    },
+  ) {
+    const wasUpdated = await this.db.transaction(async (dbTxn) => {
+      if (options?.deleteCredentials) {
+        await token.removeByDid(dbTxn, did)
+        await authorizedClientHelper.deleteAllAuthorizedClients(dbTxn, did)
+        await password.deleteAllAppPasswords(dbTxn, did)
+      }
+
+      return accountHelpers.deactivateAccount(
+        dbTxn,
+        did,
+        options?.deleteAfter ?? null,
+      )
+    })
+
+    if (!wasUpdated) {
+      throw new InvalidRequestError('Account not found')
+    }
+
+    const accountStatus = await this.getAccountStatus(did)
+
+    // Account is likely soft-deleted (takendown)
+    if (accountStatus.status === AccountStatus.Deleted) {
+      throw new InvalidRequestError('Account not found')
+    }
+
+    await this.sequencer.sequenceAccount(did, accountStatus.status)
+
+    return accountStatus
   }
 
   async activateAccount(did: DidString) {
-    return account.activateAccount(this.db, did)
+    await assertValidDidDocumentForService(this, did)
+
+    const found = await accountHelpers.activateAccount(this.db, did, {
+      // We cannot activate a takendown account
+      includeTakenDown: false,
+      includeDeactivated: true,
+    })
+    if (!found) {
+      throw new InvalidRequestError('user not found', 'AccountNotFound')
+    }
+
+    const accountStatus = await this.getAccountStatus(did)
+
+    const { account, status } = accountStatus
+
+    if (status === AccountStatus.Deleted) {
+      // A concurrent operation deleted the account
+      throw new InvalidRequestError('user not found', 'AccountNotFound')
+    }
+
+    const syncData = await this.actorStore.read(did, (store) => {
+      return store.repo.getSyncEventData()
+    })
+
+    await this.sequencer.sequenceAccountActivation(
+      did,
+      account.handle ?? INVALID_HANDLE,
+      status,
+      syncData,
+    )
+
+    return accountStatus
+  }
+
+  async sequenceAccountStatus(did: DidString) {
+    const { status } = await this.getAccountStatus(did)
+    await this.sequencer.sequenceAccount(did, status)
   }
 
   // Auth
@@ -556,6 +649,12 @@ export class AccountManager {
     did: DidString,
     passwordStr: string,
   ): Promise<boolean> {
+    if (passwordStr.length > scrypt.OLD_PASSWORD_MAX_LENGTH) {
+      // @NOTE Avoid throwing from here to avoid leaking account email validity
+      // through error messages.
+      return false
+    }
+
     return password.verifyAccountPassword(this.db, did, passwordStr)
   }
 
@@ -691,7 +790,7 @@ export class AccountManager {
     const now = currentDatetimeString()
     await this.db.transaction(async (dbTxn) => {
       await emailToken.deleteEmailToken(dbTxn, did, 'confirm_email')
-      await account.setEmailConfirmedAt(dbTxn, did, now)
+      await accountHelpers.setEmailConfirmedAt(dbTxn, did, now)
     })
 
     user.emailConfirmedAt = now
@@ -787,7 +886,7 @@ export class AccountManager {
   async updateAccountEmail(opts: { did: DidString; email: string }) {
     const { did, email } = opts
     await this.db.transaction(async (dbTxn) => {
-      await account.updateEmail(dbTxn, did, email)
+      await accountHelpers.updateEmail(dbTxn, did, email)
       await emailToken.deleteAllEmailTokens(dbTxn, did)
     })
   }

package/src/account-manager/db/migrations/005-oauth-account-management.ts

@@ -85,13 +85,14 @@ export async function up(
         .select('authenticatedAt as createdAt') // Best we can do
         .select('authenticatedAt as updatedAt')
         .where('remember', '=', 1)
-        .whereExists((qb) =>
+        .where(({ exists, selectFrom }) =>
           // device_account does not have fkey on account.did,
           // so we satisfy account_device_did_fk with this condition.
-          qb
-            .selectFrom('account')
-            .selectAll()
-            .whereRef('account.did', '=', 'device_account.did'),
+          exists(
+            selectFrom('account')
+              .selectAll()
+              .whereRef('account.did', '=', 'device_account.did'),
+          ),
         ),
     )
     .onConflict((oc) => oc.doNothing())

package/src/account-manager/db/schema/authorization-request.ts

@@ -4,6 +4,7 @@ import {
   ClientAuthLegacy,
   Code,
   DeviceId,
+  Did,
   OAuthAuthorizationRequestParameters,
   OAuthClientId,
   RequestId,
@@ -12,7 +13,7 @@ import { DateISO, JsonEncoded } from '../../../db/index.js'
 
 export interface AuthorizationRequest {
   id: RequestId
-  did: string | null
+  did: Did | null
   deviceId: DeviceId | null
 
   clientId: OAuthClientId

package/src/account-manager/db/schema/authorized-client.ts

@@ -1,9 +1,13 @@
 import { Selectable } from 'kysely'
-import { AuthorizedClientData, OAuthClientId } from '@atproto/oauth-provider'
+import {
+  AuthorizedClientData,
+  Did,
+  OAuthClientId,
+} from '@atproto/oauth-provider'
 import { DateISO, JsonEncoded } from '../../../db/index.js'
 
 export interface AuthorizedClient {
-  did: string
+  did: Did
   clientId: OAuthClientId
 
   createdAt: DateISO

package/src/account-manager/db/schema/token.ts

@@ -4,18 +4,18 @@ import {
   ClientAuthLegacy,
   Code,
   DeviceId,
+  Did,
   OAuthAuthorizationDetails,
   OAuthAuthorizationRequestParameters,
   OAuthClientId,
   RefreshToken,
-  Sub,
   TokenId,
 } from '@atproto/oauth-provider'
 import { DateISO, JsonEncoded } from '../../../db/cast.js'
 
 export interface Token {
   id: Generated<number>
-  did: Sub
+  did: Did
 
   tokenId: TokenId
   createdAt: DateISO

package/src/account-manager/helpers/account-device.ts

@@ -1,6 +1,5 @@
 import assert from 'node:assert'
-import { DeviceId } from '@atproto/oauth-provider'
-import { DidString } from '@atproto/syntax'
+import { DeviceId, Did } from '@atproto/oauth-provider'
 import { toDateISO } from '../../db/index.js'
 import { AccountDb } from '../db/index.js'
 import { selectAccountQB } from './account.js'
@@ -26,13 +25,10 @@ export function upsertQB(db: AccountDb, deviceId: DeviceId, did: string) {
 
 export function selectQB(
   db: AccountDb,
-  filter: {
-    sub?: string
-    deviceId?: DeviceId
-  },
+  filter: { did?: Did; deviceId?: DeviceId },
 ) {
   assert(
-    filter.sub != null || filter.deviceId != null,
+    filter.did != null || filter.deviceId != null,
     'Either sub or deviceId must be provided',
   )
 
@@ -52,10 +48,8 @@ export function selectQB(
         'device.ipAddress',
         'device.lastSeenAt',
       ])
-      .if(filter.sub != null, (qb) =>
-        qb.where('actor.did', '=', filter.sub! as DidString),
-      )
-      .if(filter.deviceId != null, (qb) =>
+      .$if(filter.did != null, (qb) => qb.where('actor.did', '=', filter.did!))
+      .$if(filter.deviceId != null, (qb) =>
         qb.where('account_device.deviceId', '=', filter.deviceId!),
       )
   )

package/src/account-manager/helpers/account.ts

@@ -46,8 +46,10 @@ export const selectAccountQB = (db: AccountDb, flags?: AvailabilityFlags) => {
   return db.db
     .selectFrom('actor')
     .leftJoin('account', 'actor.did', 'account.did')
-    .if(!includeTakenDown, (qb) => qb.where(notSoftDeletedClause(ref('actor'))))
-    .if(!includeDeactivated, (qb) =>
+    .$if(!includeTakenDown, (qb) =>
+      qb.where(notSoftDeletedClause(ref('actor'))),
+    )
+    .$if(!includeDeactivated, (qb) =>
       qb.where('actor.deactivatedAt', 'is', null),
     )
     .select([
@@ -69,11 +71,11 @@ export const getAccount = async (
   flags?: AvailabilityFlags,
 ): Promise<ActorAccount | null> => {
   const found = await selectAccountQB(db, flags)
-    .where((qb) => {
+    .where((eb) => {
       if (isDidIdentifier(handleOrDid)) {
-        return qb.where('actor.did', '=', handleOrDid)
+        return eb('actor.did', '=', handleOrDid)
       } else {
-        return qb.where('actor.handle', '=', handleOrDid)
+        return eb('actor.handle', '=', handleOrDid)
       }
     })
     .executeTakeFirst()
@@ -202,12 +204,16 @@ export const updateHandle = async (
       .updateTable('actor')
       .set({ handle })
       .where('did', '=', did)
-      .whereNotExists(
-        db.db
-          .selectFrom('actor')
-          .where('handle', '=', handle)
-          .where('did', '!=', did)
-          .selectAll(),
+      .where(({ not, exists }) =>
+        not(
+          exists(
+            db.db
+              .selectFrom('actor')
+              .where('handle', '=', handle)
+              .where('did', '!=', did)
+              .selectAll(),
+          ),
+        ),
       ),
   )
   if (res.numUpdatedRows < 1) {
@@ -290,8 +296,8 @@ export const deactivateAccount = async (
   db: AccountDb,
   did: DidString,
   deleteAfter: string | null,
-) => {
-  await db.executeWithRetry(
+): Promise<boolean> => {
+  const [res] = await db.executeWithRetry(
     db.db
       .updateTable('actor')
       .set({
@@ -300,18 +306,35 @@ export const deactivateAccount = async (
       })
       .where('did', '=', did),
   )
+
+  return res.numUpdatedRows > 0
 }
 
-export const activateAccount = async (db: AccountDb, did: DidString) => {
-  await db.executeWithRetry(
+export const activateAccount = async (
+  db: AccountDb,
+  did: DidString,
+  flags?: AvailabilityFlags,
+): Promise<boolean> => {
+  const { includeTakenDown = false, includeDeactivated = true } = flags ?? {}
+  const { ref } = db.db.dynamic
+
+  const [res] = await db.executeWithRetry(
     db.db
       .updateTable('actor')
       .set({
         deactivatedAt: null,
         deleteAfter: null,
       })
-      .where('did', '=', did),
+      .where('did', '=', did)
+      .$if(!includeTakenDown, (q) =>
+        q.where(notSoftDeletedClause(ref('actor'))),
+      )
+      .$if(!includeDeactivated, (q) =>
+        q.where('actor.deactivatedAt', 'is not', null),
+      ),
   )
+
+  return res.numUpdatedRows > 0
 }
 
 export const formatAccountStatus = (