@atproto/pds 0.5.51 → 0.5.55

package/dist/scripts/rebuild-repo.js.map

@@ -1 +1 @@
-{"version":3,"file":"rebuild-repo.js","sourceRoot":"","sources":["../../src/scripts/rebuild-repo.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,QAAQ,MAAM,wBAAwB,CAAA;AAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AACrC,OAAO,EAAkB,WAAW,EAAE,MAAM,cAAc,CAAA;AAC1D,OAAO,EACL,QAAQ,EACR,MAAM,EACN,GAAG,EACH,gBAAgB,EAChB,UAAU,GACX,MAAM,eAAe,CAAA;AAWtB,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EACpC,GAAmB,EACnB,IAAc,EACd,EAAE;IACF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACpD,OAAO,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;AACpC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAmB,EACnB,GAAc,EACd,UAAmB,EACnB,EAAE;IACF,MAAM,WAAW,GAAG,IAAI,gBAAgB,EAAE,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAChE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAA;QAC9D,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;QAC5C,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAA;QAE5D,iCAAiC;QACjC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CACtB,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,EACtB,MAAM,CAAC,OAAO,EAAE,CACjB,CAAC,QAAQ,EAAE,CAAA;QAEZ,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,GAAG,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QAC9C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,QAAQ,EAAE,CAAA;QAChC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;gBACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;oBACzC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,CAAA;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;QACnC,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QACvE,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC;YACE,GAAG;YACH,OAAO,EAAE,CAAC;YACV,GAAG;YACH,IAAI,EAAE,IAAI;YACV,IAAI,EAAE,MAAM,GAAG,CAAC,UAAU,EAAE;SAC7B,EACD,KAAK,CAAC,IAAI,CAAC,UAAU,CACtB,CAAA;QACD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEhD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YAC7C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAA;YAC9D,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAA;YACzD,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;YAE9C,MAAM,cAAc,GAAG,MAAM,cAAc,EAAE,CAAA;YAC7C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAA;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QACtD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAChD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QACnD,OAAO;YACL,GAAG,EAAE,SAAS;YACd,GAAG;YACH,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,SAAS;YACT,cAAc,EAAE,SAAS;YACzB,WAAW,EAAE,QAAQ;YACrB,GAAG,EAAE,EAAE;YACP,KAAK,EAAE,IAAI,MAAM,EAAE;YACnB,QAAQ,EAAE,IAAI;SACf,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,GAAG,CAAC,cAAc,CAAC,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;IACpE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CACxD,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAC9B,CAAA;IACD,MAAM,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;AACjD,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,KAAK,IAAsB,EAAE;IAClD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAA;IAClD,OAAO,MAAM,KAAK,EAAE,CAAA;AACtB,CAAC,CAAA","sourcesContent":["import assert from 'node:assert'\nimport readline from 'node:readline/promises'\nimport { TID } from '@atproto/common'\nimport { type DidString, isDidString } from '@atproto/lex'\nimport {\n  BlockMap,\n  CidSet,\n  MST,\n  MemoryBlockstore,\n  signCommit,\n} from '@atproto/repo'\nimport type { AccountManager } from '../account-manager/account-manager.js'\nimport type { ActorStore } from '../actor-store/actor-store.js'\nimport type { Sequencer } from '../sequencer/index.js'\n\nexport interface RebuildContext {\n  sequencer: Sequencer\n  accountManager: AccountManager\n  actorStore: ActorStore\n}\n\nexport const rebuildRepoScript = async (\n  ctx: RebuildContext,\n  args: string[],\n) => {\n  const did = args[0]\n  assert(isDidString(did), 'Expected DID as argument')\n  return rebuildRepo(ctx, did, true)\n}\n\nexport const rebuildRepo = async (\n  ctx: RebuildContext,\n  did: DidString,\n  promptUser: boolean,\n) => {\n  const memoryStore = new MemoryBlockstore()\n  const commit = await ctx.actorStore.transact(did, async (store) => {\n    const rootDetails = await store.repo.storage.getRootDetailed()\n    const records = await store.record.listAll()\n    const existingCids = await store.record.listExistingBlocks()\n\n    // increment existing rev by 1 ms\n    const revTid = TID.fromStr(rootDetails.rev)\n    const rev = TID.fromTime(\n      revTid.timestamp() + 1,\n      revTid.clockid(),\n    ).toString()\n\n    let mst = await MST.create(memoryStore)\n    for (const record of records) {\n      mst = await mst.add(record.path, record.cid)\n    }\n    const newBlocks = new BlockMap()\n    for await (const node of mst.walk()) {\n      if (node.isTree()) {\n        const pointer = await node.getPointer()\n        if (!existingCids.has(pointer)) {\n          const serialized = await node.serialize()\n          newBlocks.set(serialized.cid, serialized.bytes)\n        }\n      }\n    }\n    const mstCids = await mst.allCids()\n    const toDelete = new CidSet(existingCids.toList()).subtractSet(mstCids)\n    const newCommit = await signCommit(\n      {\n        did,\n        version: 3,\n        rev,\n        prev: null,\n        data: await mst.getPointer(),\n      },\n      store.repo.signingKey,\n    )\n    const commitCid = await newBlocks.add(newCommit)\n\n    if (promptUser) {\n      console.log('Record count: ', records.length)\n      console.log('Existing blocks: ', existingCids.toList().length)\n      console.log('Deleting blocks:', toDelete.toList().length)\n      console.log('Adding blocks: ', newBlocks.size)\n\n      const shouldContinue = await promptContinue()\n      if (!shouldContinue) {\n        throw new Error('Aborted')\n      }\n    }\n\n    await store.repo.storage.deleteMany(toDelete.toList())\n    await store.repo.storage.putMany(newBlocks, rev)\n    await store.repo.storage.updateRoot(commitCid, rev)\n    return {\n      cid: commitCid,\n      rev,\n      since: null,\n      prev: null,\n      newBlocks,\n      relevantBlocks: newBlocks,\n      removedCids: toDelete,\n      ops: [],\n      blobs: new CidSet(),\n      prevData: null,\n    }\n  })\n  await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev)\n  const syncData = await ctx.actorStore.read(did, (store) =>\n    store.repo.getSyncEventData(),\n  )\n  await ctx.sequencer.sequenceSync(did, syncData)\n}\n\nconst promptContinue = async (): Promise<boolean> => {\n  const rl = readline.createInterface({\n    input: process.stdin,\n    output: process.stdout,\n  })\n  const answer = await rl.question('Continue? y/n ')\n  return answer === ''\n}\n"]}
+{"version":3,"file":"rebuild-repo.js","sourceRoot":"","sources":["../../src/scripts/rebuild-repo.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,QAAQ,MAAM,wBAAwB,CAAA;AAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AACrC,OAAO,EAAkB,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjE,OAAO,EACL,QAAQ,EACR,MAAM,EACN,GAAG,EACH,gBAAgB,EAChB,UAAU,GACX,MAAM,eAAe,CAAA;AAWtB,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EACpC,GAAmB,EACnB,IAAc,EACd,EAAE;IACF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACpD,OAAO,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;AACpC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAmB,EACnB,GAAc,EACd,UAAmB,EACnB,EAAE;IACF,MAAM,WAAW,GAAG,IAAI,gBAAgB,EAAE,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAChE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAA;QAC9D,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;QAC5C,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAA;QAE5D,iCAAiC;QACjC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CACtB,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,EACtB,MAAM,CAAC,OAAO,EAAE,CACjB,CAAC,QAAQ,EAAE,CAAA;QAEZ,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,GAAG,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QAC9C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,QAAQ,EAAE,CAAA;QAChC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;gBACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;oBACzC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,CAAA;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;QACnC,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QACvE,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC;YACE,GAAG;YACH,OAAO,EAAE,CAAC;YACV,GAAG;YACH,IAAI,EAAE,IAAI;YACV,IAAI,EAAE,MAAM,GAAG,CAAC,UAAU,EAAE;SAC7B,EACD,KAAK,CAAC,IAAI,CAAC,UAAU,CACtB,CAAA;QACD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEhD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YAC7C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAA;YAC9D,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAA;YACzD,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;YAE9C,MAAM,cAAc,GAAG,MAAM,cAAc,EAAE,CAAA;YAC7C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAA;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QACtD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAChD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QACnD,OAAO;YACL,GAAG,EAAE,SAAS;YACd,GAAG;YACH,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,SAAS;YACT,cAAc,EAAE,SAAS;YACzB,WAAW,EAAE,QAAQ;YACrB,GAAG,EAAE,EAAE;YACP,KAAK,EAAE,IAAI,MAAM,EAAE;YACnB,QAAQ,EAAE,IAAI;SACf,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,GAAG,CAAC,cAAc,CAAC,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;IACpE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CACxD,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAC9B,CAAA;IACD,MAAM,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;AACjD,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,KAAK,IAAsB,EAAE;IAClD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAA;IAClD,OAAO,MAAM,KAAK,EAAE,CAAA;AACtB,CAAC,CAAA","sourcesContent":["import assert from 'node:assert'\nimport readline from 'node:readline/promises'\nimport { TID } from '@atproto/common'\nimport { type DidString, isDidString } from '@atproto/lex-schema'\nimport {\n  BlockMap,\n  CidSet,\n  MST,\n  MemoryBlockstore,\n  signCommit,\n} from '@atproto/repo'\nimport type { AccountManager } from '../account-manager/account-manager.js'\nimport type { ActorStore } from '../actor-store/actor-store.js'\nimport type { Sequencer } from '../sequencer/index.js'\n\nexport interface RebuildContext {\n  sequencer: Sequencer\n  accountManager: AccountManager\n  actorStore: ActorStore\n}\n\nexport const rebuildRepoScript = async (\n  ctx: RebuildContext,\n  args: string[],\n) => {\n  const did = args[0]\n  assert(isDidString(did), 'Expected DID as argument')\n  return rebuildRepo(ctx, did, true)\n}\n\nexport const rebuildRepo = async (\n  ctx: RebuildContext,\n  did: DidString,\n  promptUser: boolean,\n) => {\n  const memoryStore = new MemoryBlockstore()\n  const commit = await ctx.actorStore.transact(did, async (store) => {\n    const rootDetails = await store.repo.storage.getRootDetailed()\n    const records = await store.record.listAll()\n    const existingCids = await store.record.listExistingBlocks()\n\n    // increment existing rev by 1 ms\n    const revTid = TID.fromStr(rootDetails.rev)\n    const rev = TID.fromTime(\n      revTid.timestamp() + 1,\n      revTid.clockid(),\n    ).toString()\n\n    let mst = await MST.create(memoryStore)\n    for (const record of records) {\n      mst = await mst.add(record.path, record.cid)\n    }\n    const newBlocks = new BlockMap()\n    for await (const node of mst.walk()) {\n      if (node.isTree()) {\n        const pointer = await node.getPointer()\n        if (!existingCids.has(pointer)) {\n          const serialized = await node.serialize()\n          newBlocks.set(serialized.cid, serialized.bytes)\n        }\n      }\n    }\n    const mstCids = await mst.allCids()\n    const toDelete = new CidSet(existingCids.toList()).subtractSet(mstCids)\n    const newCommit = await signCommit(\n      {\n        did,\n        version: 3,\n        rev,\n        prev: null,\n        data: await mst.getPointer(),\n      },\n      store.repo.signingKey,\n    )\n    const commitCid = await newBlocks.add(newCommit)\n\n    if (promptUser) {\n      console.log('Record count: ', records.length)\n      console.log('Existing blocks: ', existingCids.toList().length)\n      console.log('Deleting blocks:', toDelete.toList().length)\n      console.log('Adding blocks: ', newBlocks.size)\n\n      const shouldContinue = await promptContinue()\n      if (!shouldContinue) {\n        throw new Error('Aborted')\n      }\n    }\n\n    await store.repo.storage.deleteMany(toDelete.toList())\n    await store.repo.storage.putMany(newBlocks, rev)\n    await store.repo.storage.updateRoot(commitCid, rev)\n    return {\n      cid: commitCid,\n      rev,\n      since: null,\n      prev: null,\n      newBlocks,\n      relevantBlocks: newBlocks,\n      removedCids: toDelete,\n      ops: [],\n      blobs: new CidSet(),\n      prevData: null,\n    }\n  })\n  await ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev)\n  const syncData = await ctx.actorStore.read(did, (store) =>\n    store.repo.getSyncEventData(),\n  )\n  await ctx.sequencer.sequenceSync(did, syncData)\n}\n\nconst promptContinue = async (): Promise<boolean> => {\n  const rl = readline.createInterface({\n    input: process.stdin,\n    output: process.stdout,\n  })\n  const answer = await rl.question('Continue? y/n ')\n  return answer === ''\n}\n"]}

package/dist/scripts/rotate-keys.js

@@ -1,7 +1,7 @@
 import assert from 'node:assert';
 import fs from 'node:fs/promises';
 import PQueue from 'p-queue';
-import { isDidString } from '@atproto/lex';
+import { isDidString } from '@atproto/lex-schema';
 import { getRecoveryDbFromSequencerLoc } from './sequencer-recovery/recovery-db.js';
 import { parseIntArg } from './util.js';
 export const rotateKeys = async (ctx, args) => {

package/dist/scripts/rotate-keys.js.map

@@ -1 +1 @@
-{"version":3,"file":"rotate-keys.js","sourceRoot":"","sources":["../../src/scripts/rotate-keys.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,MAAM,kBAAkB,CAAA;AAEjC,OAAO,MAAM,MAAM,SAAS,CAAA;AAG5B,OAAO,EAAkB,WAAW,EAAE,MAAM,cAAc,CAAA;AAI1D,OAAO,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAA;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAUvC,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAsB,EAAE,IAAc,EAAE,EAAE;IACzE,MAAM,IAAI,GAAG,IAAI,CAAA;IACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,4BAA4B,CAAC,CAAA;IAC7D,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;AACzC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,GAAsB,EACtB,IAAc,EACd,EAAE;IACF,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAClD,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACvD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,IAAI;SACd,QAAQ,EAAE;SACV,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;SACxB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAA;IAE7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,4BAA4B,CAAC,CAAA;IAE7D,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,CAAC,CAAA;AAClD,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,GAAsB,EACtB,IAAc,EACd,EAAE;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEvD,MAAM,UAAU,GAAG,MAAM,6BAA6B,CACpD,GAAG,CAAC,SAAS,CAAC,UAAU,CACzB,CAAA;IACD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,EAAE;SAC7B,UAAU,CAAC,aAAa,CAAC;SACzB,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;SACtC,OAAO,EAAE,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAgB,CAAC,CAAA;IAEhD,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC7D,MAAM,UAAU,CAAC,EAAE;aAChB,WAAW,CAAC,aAAa,CAAC;aAC1B,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;aACrB,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC;aACtB,OAAO,EAAE,CAAA;IACd,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,kBAAkB,GAAG,KAAK,EAC9B,GAAsB,EACtB,IAAiB,EACjB,WAAmB,EACnB,SAA6C,EAC7C,EAAE;IACF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IACzC,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YACrC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBACxD,OAAM;YACR,CAAC;YACD,IAAI,QAAqB,CAAA;YACzB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;oBAC/D,MAAM,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;oBACrC,OAAO,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAA;gBACzC,CAAC,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,kCAAkC,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBAC9D,OAAM;YACR,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAA;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBACvE,OAAM;YACR,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,0BAA0B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBACtD,OAAM;YACR,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,SAAS,CAAC,GAAG,CAAC,CAAA;YACtB,CAAC;YACD,SAAS,EAAE,CAAA;YACX,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IACpB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;AACrB,CAAC,CAAA;AAED,MAAM,mBAAmB,GAAG,KAAK,EAAE,GAAsB,EAAE,GAAc,EAAE,EAAE;IAC3E,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAClD,MAAM,cAAc,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAC5E,IAAI,QAAQ,CAAC,GAAG,EAAE,KAAK,cAAc,EAAE,CAAC;QACtC,qBAAqB;QACrB,OAAM;IACR,CAAC;IACD,MAAM,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAA;AAC/E,CAAC,CAAA","sourcesContent":["import assert from 'node:assert'\nimport fs from 'node:fs/promises'\nimport type * as plc from '@did-plc/lib'\nimport PQueue from 'p-queue'\nimport type { Keypair } from '@atproto/crypto'\nimport type { IdResolver } from '@atproto/identity'\nimport { type DidString, isDidString } from '@atproto/lex'\nimport type { ActorStore } from '../actor-store/actor-store.js'\nimport type { SyncEvtData } from '../repo/index.js'\nimport type { Sequencer } from '../sequencer/index.js'\nimport { getRecoveryDbFromSequencerLoc } from './sequencer-recovery/recovery-db.js'\nimport { parseIntArg } from './util.js'\n\nexport type RotateKeysContext = {\n  sequencer: Sequencer\n  actorStore: ActorStore\n  idResolver: IdResolver\n  plcClient: plc.Client\n  plcRotationKey: Keypair\n}\n\nexport const rotateKeys = async (ctx: RotateKeysContext, args: string[]) => {\n  const dids = args\n  assert(dids.every(isDidString), 'All arguments must be DIDs')\n  await rotateKeysForRepos(ctx, dids, 10)\n}\n\nexport const rotateKeysFromFile = async (\n  ctx: RotateKeysContext,\n  args: string[],\n) => {\n  const filepath = args[0]\n  if (!filepath) {\n    throw new Error('Expected filepath as argument')\n  }\n  const concurrency = args[1] ? parseIntArg(args[1]) : 25\n  const file = await fs.readFile(filepath)\n  const dids = file\n    .toString()\n    .split('\\n')\n    .map((did) => did.trim())\n    .filter((did) => did.startsWith('did:plc'))\n\n  assert(dids.every(isDidString), 'File contains invalid DIDs')\n\n  await rotateKeysForRepos(ctx, dids, concurrency)\n}\n\nexport const rotateKeysRecovery = async (\n  ctx: RotateKeysContext,\n  args: string[],\n) => {\n  const concurrency = args[1] ? parseIntArg(args[0]) : 10\n\n  const recoveryDb = await getRecoveryDbFromSequencerLoc(\n    ctx.sequencer.dbLocation,\n  )\n  const rows = await recoveryDb.db\n    .selectFrom('new_account')\n    .select('did')\n    .where('new_account.published', '=', 0)\n    .execute()\n  const dids = rows.map((r) => r.did as DidString)\n\n  await rotateKeysForRepos(ctx, dids, concurrency, async (did) => {\n    await recoveryDb.db\n      .updateTable('new_account')\n      .set({ published: 1 })\n      .where('did', '=', did)\n      .execute()\n  })\n}\n\nconst rotateKeysForRepos = async (\n  ctx: RotateKeysContext,\n  dids: DidString[],\n  concurrency: number,\n  onSuccess?: (did: DidString) => Promise<void>,\n) => {\n  const queue = new PQueue({ concurrency })\n  let completed = 0\n  for (const did of dids) {\n    queue.add(async () => {\n      try {\n        await updatePlcSigningKey(ctx, did)\n      } catch (err) {\n        console.error(`failed to update key for ${did}: ${err}`)\n        return\n      }\n      let syncData: SyncEvtData\n      try {\n        syncData = await ctx.actorStore.transact(did, async (actorTxn) => {\n          await actorTxn.repo.processWrites([])\n          return actorTxn.repo.getSyncEventData()\n        })\n      } catch (err) {\n        console.error(`failed to write new commit for ${did}: ${err}`)\n        return\n      }\n      try {\n        await ctx.sequencer.sequenceIdentity(did)\n      } catch (err) {\n        console.error(`failed to sequence new identity evt for ${did}: ${err}`)\n        return\n      }\n      try {\n        await ctx.sequencer.sequenceSync(did, syncData)\n      } catch (err) {\n        console.error(`failed to sequence for ${did}: ${err}`)\n        return\n      }\n      if (onSuccess) {\n        await onSuccess(did)\n      }\n      completed++\n      if (completed % 10 === 0) {\n        console.log(`${completed}/${dids.length}`)\n      }\n    })\n  }\n  await queue.onIdle()\n  console.log('DONE')\n}\n\nconst updatePlcSigningKey = async (ctx: RotateKeysContext, did: DidString) => {\n  const updateTo = await ctx.actorStore.keypair(did)\n  const currSigningKey = await ctx.idResolver.did.resolveAtprotoKey(did, true)\n  if (updateTo.did() === currSigningKey) {\n    // already up to date\n    return\n  }\n  await ctx.plcClient.updateAtprotoKey(did, ctx.plcRotationKey, updateTo.did())\n}\n"]}
+{"version":3,"file":"rotate-keys.js","sourceRoot":"","sources":["../../src/scripts/rotate-keys.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,MAAM,kBAAkB,CAAA;AAEjC,OAAO,MAAM,MAAM,SAAS,CAAA;AAG5B,OAAO,EAAkB,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAIjE,OAAO,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAA;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAUvC,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAsB,EAAE,IAAc,EAAE,EAAE;IACzE,MAAM,IAAI,GAAG,IAAI,CAAA;IACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,4BAA4B,CAAC,CAAA;IAC7D,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;AACzC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,GAAsB,EACtB,IAAc,EACd,EAAE;IACF,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAClD,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACvD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,IAAI;SACd,QAAQ,EAAE;SACV,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;SACxB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAA;IAE7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,4BAA4B,CAAC,CAAA;IAE7D,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,CAAC,CAAA;AAClD,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,GAAsB,EACtB,IAAc,EACd,EAAE;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEvD,MAAM,UAAU,GAAG,MAAM,6BAA6B,CACpD,GAAG,CAAC,SAAS,CAAC,UAAU,CACzB,CAAA;IACD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,EAAE;SAC7B,UAAU,CAAC,aAAa,CAAC;SACzB,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;SACtC,OAAO,EAAE,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAgB,CAAC,CAAA;IAEhD,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC7D,MAAM,UAAU,CAAC,EAAE;aAChB,WAAW,CAAC,aAAa,CAAC;aAC1B,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;aACrB,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC;aACtB,OAAO,EAAE,CAAA;IACd,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,kBAAkB,GAAG,KAAK,EAC9B,GAAsB,EACtB,IAAiB,EACjB,WAAmB,EACnB,SAA6C,EAC7C,EAAE;IACF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IACzC,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YACrC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBACxD,OAAM;YACR,CAAC;YACD,IAAI,QAAqB,CAAA;YACzB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;oBAC/D,MAAM,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;oBACrC,OAAO,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAA;gBACzC,CAAC,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,kCAAkC,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBAC9D,OAAM;YACR,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAA;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBACvE,OAAM;YACR,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,0BAA0B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;gBACtD,OAAM;YACR,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,SAAS,CAAC,GAAG,CAAC,CAAA;YACtB,CAAC;YACD,SAAS,EAAE,CAAA;YACX,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IACpB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;AACrB,CAAC,CAAA;AAED,MAAM,mBAAmB,GAAG,KAAK,EAAE,GAAsB,EAAE,GAAc,EAAE,EAAE;IAC3E,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAClD,MAAM,cAAc,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAC5E,IAAI,QAAQ,CAAC,GAAG,EAAE,KAAK,cAAc,EAAE,CAAC;QACtC,qBAAqB;QACrB,OAAM;IACR,CAAC;IACD,MAAM,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAA;AAC/E,CAAC,CAAA","sourcesContent":["import assert from 'node:assert'\nimport fs from 'node:fs/promises'\nimport type * as plc from '@did-plc/lib'\nimport PQueue from 'p-queue'\nimport type { Keypair } from '@atproto/crypto'\nimport type { IdResolver } from '@atproto/identity'\nimport { type DidString, isDidString } from '@atproto/lex-schema'\nimport type { ActorStore } from '../actor-store/actor-store.js'\nimport type { SyncEvtData } from '../repo/index.js'\nimport type { Sequencer } from '../sequencer/index.js'\nimport { getRecoveryDbFromSequencerLoc } from './sequencer-recovery/recovery-db.js'\nimport { parseIntArg } from './util.js'\n\nexport type RotateKeysContext = {\n  sequencer: Sequencer\n  actorStore: ActorStore\n  idResolver: IdResolver\n  plcClient: plc.Client\n  plcRotationKey: Keypair\n}\n\nexport const rotateKeys = async (ctx: RotateKeysContext, args: string[]) => {\n  const dids = args\n  assert(dids.every(isDidString), 'All arguments must be DIDs')\n  await rotateKeysForRepos(ctx, dids, 10)\n}\n\nexport const rotateKeysFromFile = async (\n  ctx: RotateKeysContext,\n  args: string[],\n) => {\n  const filepath = args[0]\n  if (!filepath) {\n    throw new Error('Expected filepath as argument')\n  }\n  const concurrency = args[1] ? parseIntArg(args[1]) : 25\n  const file = await fs.readFile(filepath)\n  const dids = file\n    .toString()\n    .split('\\n')\n    .map((did) => did.trim())\n    .filter((did) => did.startsWith('did:plc'))\n\n  assert(dids.every(isDidString), 'File contains invalid DIDs')\n\n  await rotateKeysForRepos(ctx, dids, concurrency)\n}\n\nexport const rotateKeysRecovery = async (\n  ctx: RotateKeysContext,\n  args: string[],\n) => {\n  const concurrency = args[1] ? parseIntArg(args[0]) : 10\n\n  const recoveryDb = await getRecoveryDbFromSequencerLoc(\n    ctx.sequencer.dbLocation,\n  )\n  const rows = await recoveryDb.db\n    .selectFrom('new_account')\n    .select('did')\n    .where('new_account.published', '=', 0)\n    .execute()\n  const dids = rows.map((r) => r.did as DidString)\n\n  await rotateKeysForRepos(ctx, dids, concurrency, async (did) => {\n    await recoveryDb.db\n      .updateTable('new_account')\n      .set({ published: 1 })\n      .where('did', '=', did)\n      .execute()\n  })\n}\n\nconst rotateKeysForRepos = async (\n  ctx: RotateKeysContext,\n  dids: DidString[],\n  concurrency: number,\n  onSuccess?: (did: DidString) => Promise<void>,\n) => {\n  const queue = new PQueue({ concurrency })\n  let completed = 0\n  for (const did of dids) {\n    queue.add(async () => {\n      try {\n        await updatePlcSigningKey(ctx, did)\n      } catch (err) {\n        console.error(`failed to update key for ${did}: ${err}`)\n        return\n      }\n      let syncData: SyncEvtData\n      try {\n        syncData = await ctx.actorStore.transact(did, async (actorTxn) => {\n          await actorTxn.repo.processWrites([])\n          return actorTxn.repo.getSyncEventData()\n        })\n      } catch (err) {\n        console.error(`failed to write new commit for ${did}: ${err}`)\n        return\n      }\n      try {\n        await ctx.sequencer.sequenceIdentity(did)\n      } catch (err) {\n        console.error(`failed to sequence new identity evt for ${did}: ${err}`)\n        return\n      }\n      try {\n        await ctx.sequencer.sequenceSync(did, syncData)\n      } catch (err) {\n        console.error(`failed to sequence for ${did}: ${err}`)\n        return\n      }\n      if (onSuccess) {\n        await onSuccess(did)\n      }\n      completed++\n      if (completed % 10 === 0) {\n        console.log(`${completed}/${dids.length}`)\n      }\n    })\n  }\n  await queue.onIdle()\n  console.log('DONE')\n}\n\nconst updatePlcSigningKey = async (ctx: RotateKeysContext, did: DidString) => {\n  const updateTo = await ctx.actorStore.keypair(did)\n  const currSigningKey = await ctx.idResolver.did.resolveAtprotoKey(did, true)\n  if (updateTo.did() === currSigningKey) {\n    // already up to date\n    return\n  }\n  await ctx.plcClient.updateAtprotoKey(did, ctx.plcRotationKey, updateTo.did())\n}\n"]}

package/dist/scripts/sequencer-recovery/recoverer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recoverer.d.ts","sourceRoot":"","sources":["../../../src/scripts/sequencer-recovery/recoverer.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,KAAK,cAAc,EAEpB,MAAM,0CAA0C,CAAA;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAA;AAQlE,OAAO,KAAK,EAGV,MAAM,EACN,SAAS,EACV,MAAM,0BAA0B,CAAA;AACjC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAE7C,MAAM,MAAM,oBAAoB,GAAG;IACjC,SAAS,EAAE,SAAS,CAAA;IACpB,cAAc,EAAE,cAAc,CAAA;IAC9B,UAAU,EAAE,UAAU,CAAA;CACvB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG,oBAAoB,GAAG;IACpD,UAAU,EAAE,UAAU,CAAA;CACvB,CAAA;AAID,qBAAa,SAAS;IAKX,GAAG,EAAE,gBAAgB;IAJ9B,MAAM,EAAE,UAAU,CAAA;IAClB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAEnB,YACS,GAAG,EAAE,gBAAgB,EAC5B,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,EAI9B;IAEK,GAAG,CAAC,WAAW,SAAI,iBAiCxB;IAEK,UAAU,kBAEf;IAEK,OAAO,kBAEZ;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,QAcvB;CACF;AAED,eAAO,MAAM,aAAa,QAAe,gBAAgB,OAAO,MAAM,kBAQrE,CAAA"}
+{"version":3,"file":"recoverer.d.ts","sourceRoot":"","sources":["../../../src/scripts/sequencer-recovery/recoverer.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,KAAK,cAAc,EAEpB,MAAM,0CAA0C,CAAA;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAA;AAQlE,OAAO,KAAK,EAGV,MAAM,EACN,SAAS,EACV,MAAM,0BAA0B,CAAA;AACjC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAE7C,MAAM,MAAM,oBAAoB,GAAG;IACjC,SAAS,EAAE,SAAS,CAAA;IACpB,cAAc,EAAE,cAAc,CAAA;IAC9B,UAAU,EAAE,UAAU,CAAA;CACvB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG,oBAAoB,GAAG;IACpD,UAAU,EAAE,UAAU,CAAA;CACvB,CAAA;AAID,qBAAa,SAAS;IAKX,GAAG,EAAE,gBAAgB;IAJ9B,MAAM,EAAE,UAAU,CAAA;IAClB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAEnB,YACS,GAAG,EAAE,gBAAgB,EAC5B,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,EAI9B;IAEK,GAAG,CAAC,WAAW,SAAI,iBAiCxB;IAEK,UAAU,kBAEf;IAEK,OAAO,kBAEZ;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,QAcvB;CACF;AAED,eAAO,MAAM,aAAa,QAAe,gBAAgB,OAAO,MAAM,kBAQrE,CAAA"}

package/dist/scripts/sequencer-recovery/recoverer.js

@@ -1,6 +1,6 @@
 import { rmIfExists } from '@atproto/common';
 import { Secp256k1Keypair } from '@atproto/crypto';
-import { isNsidString, isRecordKeyString } from '@atproto/lex';
+import { isNsidString, isRecordKeyString, } from '@atproto/lex-schema';
 import { BlockMap, CidSet, WriteOpAction, cborToLexRecord, parseDataKey, readCar, } from '@atproto/repo';
 import { AccountStatus, } from '../../account-manager/account-manager.js';
 import { countAll } from '../../db/index.js';

package/dist/scripts/sequencer-recovery/recoverer.js.map

@@ -1 +1 @@
-{"version":3,"file":"recoverer.js","sourceRoot":"","sources":["../../../src/scripts/sequencer-recovery/recoverer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAkB,YAAY,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAC9E,OAAO,EACL,QAAQ,EACR,MAAM,EAEN,aAAa,EACb,eAAe,EACf,YAAY,EACZ,OAAO,GACR,MAAM,eAAe,CAAA;AACtB,OAAO,EAEL,aAAa,GACd,MAAM,0CAA0C,CAAA;AAGjD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAC5C,OAAO,EAEL,aAAa,EACb,aAAa,EACb,aAAa,GACd,MAAM,qBAAqB,CAAA;AAQ5B,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAY7C,MAAM,SAAS,GAAG,IAAI,CAAA;AAEtB,MAAM,OAAO,SAAS;IAIpB,YACS,GAAqB,EAC5B,IAA6B;mBADtB,GAAG;QAGV,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;aACxC,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,KAAK,CAAC;aACb,OAAO,EAAE,CAAA;QACZ,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC1B,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE;aAC5C,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;aAC5B,uBAAuB,EAAE,CAAA;QAC5B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAA;QAChC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,IAAI,MAAM,GAAuB,WAAW,CAAA;QAC5C,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC;gBACpD,WAAW,EAAE,MAAM;gBACnB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAA;YACF,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7C,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAA;YAEzB,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;YAE3B,SAAS,IAAI,SAAS,CAAA;YACtB,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,GAAG,CAAA;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,MAAM,EAAE,CAAC,CAAA;QAC3D,CAAC;QAED,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;IAC7B,CAAC;IAED,YAAY,CAAC,GAAW;QACtB,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAM;QACR,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAM;YACR,CAAC;YACD,MAAM,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACrD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACpB,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;YACnD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,GAAqB,EAAE,GAAW,EAAE,EAAE;IACxE,4CAA4C;IAC5C,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAA;IACvC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAA;IAC3C,CAAC;AACH,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,KAAK,EAAE,GAAqB,EAAE,GAAc,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,CAAA;IACpD,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;YACnD,OAAM;QACR,CAAC;IACH,CAAC;IACD,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACpD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAA;QAC1D,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;YACxB,OAAM;QACR,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;QACvD,MAAM,CAAC,SAAS,GAAG,MAAM,CAAA;QACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAA;QACvB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAA;QACpB,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QAC/C,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QACnD,MAAM,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,mBAAmB,GAAG,KAAK,EAC/B,GAAqB,EACrB,GAAc,EACd,MAAuB,EACvB,MAAgB,EAChB,EAAE;IACF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IACnE,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,MAAM,GAAe;QACzB,GAAG,EAAE,GAAG,CAAC,MAAM;QACf,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,MAAM;QACjB,cAAc,EAAE,IAAI,QAAQ,EAAE;QAC9B,WAAW,EAAE,IAAI,MAAM,EAAE;KAC1B,CAAA;IACD,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACpD,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QACrD,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QACnD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAChE,CAAC,CAAC,CAAA;IACF,MAAM,eAAe,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;AAC5C,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG,KAAK,EAAE,GAAqB,EAAE,GAAe,EAAE,EAAE;IACzE,qFAAqF;IAErF,IAAI,GAAG,CAAC,MAAM,KAAK,aAAa,CAAC,OAAO,EAAE,CAAC;QACzC,wEAAwE;QACxE,2EAA2E;QAC3E,mCAAmC;QACnC,MAAM,GAAG,CAAC,cAAc,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAE/C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/D,MAAM,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;AACH,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,KAAK,EACtB,KAA2B,EAC3B,MAAuB,EACvB,EAAE;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAA;IAErD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IACE,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM;YACrC,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM,EACrC,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC/B,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAA;gBAC9C,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,KAAK,EACxB,UAAsB,EACtB,GAAc,EACd,GAAY,EACZ,EAAE;IACF,MAAM,UAAU,CAAC,EAAE;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC;QACN,GAAG;QACH,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE;QACtB,KAAK,EAAE,CAAC;KACT,CAAC;SACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;SAClC,OAAO,EAAE,CAAA;AACd,CAAC,CAAA;AAED,MAAM,eAAe,GAAG,KAAK,EAAE,UAAsB,EAAE,GAAc,EAAE,EAAE;IACvE,MAAM,UAAU,CAAC,EAAE;SAChB,UAAU,CAAC,aAAa,CAAC;SACzB,MAAM,CAAC;QACN,GAAG;QACH,SAAS,EAAE,CAAC;KACb,CAAC;SACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;SAClC,OAAO,EAAE,CAAA;AACd,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,KAAK,EAC1B,GAAc,EAIb,EAAE;IACH,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAA;IACvE,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACvB,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAClD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YAAE,OAAO,SAAS,CAAA;QAC/C,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAA;QAE9C,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,aAAa,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QACjD,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,GAAG;YAAE,OAAO,SAAS,CAAA;QAE7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,WAAW;YAAE,OAAO,SAAS,CAAA;QAClC,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,CAAC,CAAA;QAE3C,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,aAAa,CAAC;gBACnB,GAAG;gBACH,UAAU;gBACV,IAAI;gBACJ,MAAM;gBACN,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,aAAa,CAAC;gBACnB,GAAG;gBACH,UAAU;gBACV,IAAI;gBACJ,MAAM;gBACN,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAC,CACH,CAAA;IAED,OAAO;QACL,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC;QACjD,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAA;AACH,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,GAAW,EAAoB,EAAE;IACnD,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAA;IACpB,CAAC;SAAM,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAA;IACrB,CAAC;SAAM,CAAC;QACN,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC,CAAA","sourcesContent":["import { rmIfExists } from '@atproto/common'\nimport { Secp256k1Keypair } from '@atproto/crypto'\nimport { type DidString, isNsidString, isRecordKeyString } from '@atproto/lex'\nimport {\n  BlockMap,\n  CidSet,\n  type CommitData,\n  WriteOpAction,\n  cborToLexRecord,\n  parseDataKey,\n  readCar,\n} from '@atproto/repo'\nimport {\n  type AccountManager,\n  AccountStatus,\n} from '../../account-manager/account-manager.js'\nimport type { ActorStoreTransactor } from '../../actor-store/actor-store-transactor.js'\nimport type { ActorStore } from '../../actor-store/actor-store.js'\nimport { countAll } from '../../db/index.js'\nimport {\n  type PreparedWrite,\n  prepareCreate,\n  prepareDelete,\n  prepareUpdate,\n} from '../../repo/index.js'\nimport type {\n  AccountEvt,\n  CommitEvt,\n  SeqEvt,\n  Sequencer,\n} from '../../sequencer/index.js'\nimport type { RecoveryDb } from './recovery-db.js'\nimport { UserQueues } from './user-queues.js'\n\nexport type RecovererContextNoDb = {\n  sequencer: Sequencer\n  accountManager: AccountManager\n  actorStore: ActorStore\n}\n\nexport type RecovererContext = RecovererContextNoDb & {\n  recoveryDb: RecoveryDb\n}\n\nconst PAGE_SIZE = 5000\n\nexport class Recoverer {\n  queues: UserQueues\n  failed: Set<string>\n\n  constructor(\n    public ctx: RecovererContext,\n    opts: { concurrency: number },\n  ) {\n    this.queues = new UserQueues(opts.concurrency)\n    this.failed = new Set()\n  }\n\n  async run(startCursor = 0) {\n    const failed = await this.ctx.recoveryDb.db\n      .selectFrom('failed')\n      .select('did')\n      .execute()\n    for (const row of failed) {\n      this.failed.add(row.did)\n    }\n\n    const totalRes = await this.ctx.sequencer.db.db\n      .selectFrom('repo_seq')\n      .select(countAll.as('count'))\n      .executeTakeFirstOrThrow()\n    const totalEvts = totalRes.count\n    let completed = 0\n\n    let cursor: number | undefined = startCursor\n    while (cursor !== undefined) {\n      const page = await this.ctx.sequencer.requestSeqRange({\n        earliestSeq: cursor,\n        limit: PAGE_SIZE,\n      })\n      page.forEach((evt) => this.processEvent(evt))\n      cursor = page.at(-1)?.seq\n\n      await this.queues.onEmpty()\n\n      completed += PAGE_SIZE\n      const percentComplete = (completed / totalEvts) * 100\n      console.log(`${percentComplete.toFixed(2)}% - ${cursor}`)\n    }\n\n    await this.queues.processAll()\n  }\n\n  async processAll() {\n    await this.queues.processAll()\n  }\n\n  async destroy() {\n    await this.queues.destroy()\n  }\n\n  processEvent(evt: SeqEvt) {\n    const did = didFromEvt(evt)\n    if (!did) {\n      return\n    }\n    this.queues.addToUser(did, async () => {\n      if (this.failed.has(did)) {\n        return\n      }\n      await processSeqEvt(this.ctx, evt).catch(async (err) => {\n        this.failed.add(did)\n        await trackFailure(this.ctx.recoveryDb, did, err)\n      })\n    })\n  }\n}\n\nexport const processSeqEvt = async (ctx: RecovererContext, evt: SeqEvt) => {\n  // only need to process commits & tombstones\n  if (evt.type === 'account') {\n    await processAccountEvt(ctx, evt.evt)\n  }\n  if (evt.type === 'commit') {\n    await processCommit(ctx, evt.evt).catch()\n  }\n}\n\nconst processCommit = async (ctx: RecovererContext, evt: CommitEvt) => {\n  const did = evt.repo\n  const { writes, blocks } = await parseCommitEvt(evt)\n  if (evt.since === null) {\n    const actorExists = await ctx.actorStore.exists(did)\n    if (!actorExists) {\n      await processRepoCreation(ctx, evt, writes, blocks)\n      return\n    }\n  }\n  await ctx.actorStore.transact(did, async (actorTxn) => {\n    const root = await actorTxn.repo.storage.getRootDetailed()\n    if (root.rev >= evt.rev) {\n      return\n    }\n    const commit = await actorTxn.repo.formatCommit(writes)\n    commit.newBlocks = blocks\n    commit.cid = evt.commit\n    commit.rev = evt.rev\n    await actorTxn.repo.storage.applyCommit(commit)\n    await actorTxn.repo.indexWrites(writes, commit.rev)\n    await trackBlobs(actorTxn, writes)\n  })\n}\n\nconst processRepoCreation = async (\n  ctx: RecovererContext,\n  evt: CommitEvt,\n  writes: PreparedWrite[],\n  blocks: BlockMap,\n) => {\n  const did = evt.repo\n  const keypair = await Secp256k1Keypair.create({ exportable: true })\n  await ctx.actorStore.create(did, keypair)\n  const commit: CommitData = {\n    cid: evt.commit,\n    rev: evt.rev,\n    since: evt.since,\n    prev: null,\n    newBlocks: blocks,\n    relevantBlocks: new BlockMap(),\n    removedCids: new CidSet(),\n  }\n  await ctx.actorStore.transact(did, async (actorTxn) => {\n    await actorTxn.repo.storage.applyCommit(commit, true)\n    await actorTxn.repo.indexWrites(writes, commit.rev)\n    await actorTxn.repo.blob.processWriteBlobs(commit.rev, writes)\n  })\n  await trackNewAccount(ctx.recoveryDb, did)\n}\n\nconst processAccountEvt = async (ctx: RecovererContext, evt: AccountEvt) => {\n  // do not need to process deactivation/takedowns because we backup account DB as well\n\n  if (evt.status === AccountStatus.Deleted) {\n    // In case an account deletion was sequenced, let's make sure to (first)\n    // delete the accounts database, and (then) unlink the actor store from the\n    // file system. Order matters here.\n    await ctx.accountManager.deleteAccount(evt.did)\n\n    const { directory } = await ctx.actorStore.getLocation(evt.did)\n    await rmIfExists(directory, true)\n  }\n}\n\nconst trackBlobs = async (\n  store: ActorStoreTransactor,\n  writes: PreparedWrite[],\n) => {\n  await store.repo.blob.deleteDereferencedBlobs(writes)\n\n  for (const write of writes) {\n    if (\n      write.action === WriteOpAction.Create ||\n      write.action === WriteOpAction.Update\n    ) {\n      for (const blob of write.blobs) {\n        await store.repo.blob.insertBlobMetadata(blob)\n        await store.repo.blob.associateBlob(blob, write.uri)\n      }\n    }\n  }\n}\n\nconst trackFailure = async (\n  recoveryDb: RecoveryDb,\n  did: DidString,\n  err: unknown,\n) => {\n  await recoveryDb.db\n    .insertInto('failed')\n    .values({\n      did,\n      error: err?.toString(),\n      fixed: 0,\n    })\n    .onConflict((oc) => oc.doNothing())\n    .execute()\n}\n\nconst trackNewAccount = async (recoveryDb: RecoveryDb, did: DidString) => {\n  await recoveryDb.db\n    .insertInto('new_account')\n    .values({\n      did,\n      published: 0,\n    })\n    .onConflict((oc) => oc.doNothing())\n    .execute()\n}\n\nconst parseCommitEvt = async (\n  evt: CommitEvt,\n): Promise<{\n  writes: PreparedWrite[]\n  blocks: BlockMap\n}> => {\n  const did = evt.repo\n  const evtCar = await readCar(evt.blocks, { skipCidVerification: true })\n  const writesUnfiltered = await Promise.all(\n    evt.ops.map(async (op) => {\n      const { collection, rkey } = parseDataKey(op.path)\n      if (!isNsidString(collection)) return undefined\n      if (!isRecordKeyString(rkey)) return undefined\n\n      if (op.action === 'delete') {\n        return prepareDelete({ did, collection, rkey })\n      }\n      if (!op.cid) return undefined\n\n      const recordBytes = evtCar.blocks.get(op.cid)\n      if (!recordBytes) return undefined\n      const record = cborToLexRecord(recordBytes)\n\n      if (op.action === 'create') {\n        return prepareCreate({\n          did,\n          collection,\n          rkey,\n          record,\n          validate: false,\n        })\n      } else {\n        return prepareUpdate({\n          did,\n          collection,\n          rkey,\n          record,\n          validate: false,\n        })\n      }\n    }),\n  )\n\n  return {\n    writes: writesUnfiltered.filter((w) => w != null),\n    blocks: evtCar.blocks,\n  }\n}\n\nconst didFromEvt = (evt: SeqEvt): DidString | null => {\n  if (evt.type === 'account') {\n    return evt.evt.did\n  } else if (evt.type === 'commit') {\n    return evt.evt.repo\n  } else {\n    return null\n  }\n}\n"]}
+{"version":3,"file":"recoverer.js","sourceRoot":"","sources":["../../../src/scripts/sequencer-recovery/recoverer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAEL,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,QAAQ,EACR,MAAM,EAEN,aAAa,EACb,eAAe,EACf,YAAY,EACZ,OAAO,GACR,MAAM,eAAe,CAAA;AACtB,OAAO,EAEL,aAAa,GACd,MAAM,0CAA0C,CAAA;AAGjD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAC5C,OAAO,EAEL,aAAa,EACb,aAAa,EACb,aAAa,GACd,MAAM,qBAAqB,CAAA;AAQ5B,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAY7C,MAAM,SAAS,GAAG,IAAI,CAAA;AAEtB,MAAM,OAAO,SAAS;IAIpB,YACS,GAAqB,EAC5B,IAA6B;mBADtB,GAAG;QAGV,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;aACxC,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,KAAK,CAAC;aACb,OAAO,EAAE,CAAA;QACZ,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC1B,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE;aAC5C,UAAU,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;aAC5B,uBAAuB,EAAE,CAAA;QAC5B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAA;QAChC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,IAAI,MAAM,GAAuB,WAAW,CAAA;QAC5C,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC;gBACpD,WAAW,EAAE,MAAM;gBACnB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAA;YACF,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7C,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAA;YAEzB,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;YAE3B,SAAS,IAAI,SAAS,CAAA;YACtB,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,GAAG,CAAA;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,MAAM,EAAE,CAAC,CAAA;QAC3D,CAAC;QAED,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;IAC7B,CAAC;IAED,YAAY,CAAC,GAAW;QACtB,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAM;QACR,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAM;YACR,CAAC;YACD,MAAM,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACrD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACpB,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;YACnD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,GAAqB,EAAE,GAAW,EAAE,EAAE;IACxE,4CAA4C;IAC5C,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAA;IACvC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAA;IAC3C,CAAC;AACH,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,KAAK,EAAE,GAAqB,EAAE,GAAc,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,CAAA;IACpD,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;YACnD,OAAM;QACR,CAAC;IACH,CAAC;IACD,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACpD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAA;QAC1D,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;YACxB,OAAM;QACR,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;QACvD,MAAM,CAAC,SAAS,GAAG,MAAM,CAAA;QACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAA;QACvB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAA;QACpB,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QAC/C,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QACnD,MAAM,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,mBAAmB,GAAG,KAAK,EAC/B,GAAqB,EACrB,GAAc,EACd,MAAuB,EACvB,MAAgB,EAChB,EAAE;IACF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IACnE,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,MAAM,GAAe;QACzB,GAAG,EAAE,GAAG,CAAC,MAAM;QACf,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,MAAM;QACjB,cAAc,EAAE,IAAI,QAAQ,EAAE;QAC9B,WAAW,EAAE,IAAI,MAAM,EAAE;KAC1B,CAAA;IACD,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACpD,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QACrD,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QACnD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAChE,CAAC,CAAC,CAAA;IACF,MAAM,eAAe,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;AAC5C,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG,KAAK,EAAE,GAAqB,EAAE,GAAe,EAAE,EAAE;IACzE,qFAAqF;IAErF,IAAI,GAAG,CAAC,MAAM,KAAK,aAAa,CAAC,OAAO,EAAE,CAAC;QACzC,wEAAwE;QACxE,2EAA2E;QAC3E,mCAAmC;QACnC,MAAM,GAAG,CAAC,cAAc,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAE/C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/D,MAAM,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;AACH,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,KAAK,EACtB,KAA2B,EAC3B,MAAuB,EACvB,EAAE;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAA;IAErD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IACE,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM;YACrC,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM,EACrC,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC/B,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAA;gBAC9C,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,KAAK,EACxB,UAAsB,EACtB,GAAc,EACd,GAAY,EACZ,EAAE;IACF,MAAM,UAAU,CAAC,EAAE;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC;QACN,GAAG;QACH,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE;QACtB,KAAK,EAAE,CAAC;KACT,CAAC;SACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;SAClC,OAAO,EAAE,CAAA;AACd,CAAC,CAAA;AAED,MAAM,eAAe,GAAG,KAAK,EAAE,UAAsB,EAAE,GAAc,EAAE,EAAE;IACvE,MAAM,UAAU,CAAC,EAAE;SAChB,UAAU,CAAC,aAAa,CAAC;SACzB,MAAM,CAAC;QACN,GAAG;QACH,SAAS,EAAE,CAAC;KACb,CAAC;SACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;SAClC,OAAO,EAAE,CAAA;AACd,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,KAAK,EAC1B,GAAc,EAIb,EAAE;IACH,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAA;IACvE,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACvB,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAClD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YAAE,OAAO,SAAS,CAAA;QAC/C,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAA;QAE9C,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,aAAa,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QACjD,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,GAAG;YAAE,OAAO,SAAS,CAAA;QAE7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,WAAW;YAAE,OAAO,SAAS,CAAA;QAClC,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,CAAC,CAAA;QAE3C,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,aAAa,CAAC;gBACnB,GAAG;gBACH,UAAU;gBACV,IAAI;gBACJ,MAAM;gBACN,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,aAAa,CAAC;gBACnB,GAAG;gBACH,UAAU;gBACV,IAAI;gBACJ,MAAM;gBACN,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAC,CACH,CAAA;IAED,OAAO;QACL,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC;QACjD,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAA;AACH,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,GAAW,EAAoB,EAAE;IACnD,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAA;IACpB,CAAC;SAAM,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAA;IACrB,CAAC;SAAM,CAAC;QACN,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC,CAAA","sourcesContent":["import { rmIfExists } from '@atproto/common'\nimport { Secp256k1Keypair } from '@atproto/crypto'\nimport {\n  type DidString,\n  isNsidString,\n  isRecordKeyString,\n} from '@atproto/lex-schema'\nimport {\n  BlockMap,\n  CidSet,\n  type CommitData,\n  WriteOpAction,\n  cborToLexRecord,\n  parseDataKey,\n  readCar,\n} from '@atproto/repo'\nimport {\n  type AccountManager,\n  AccountStatus,\n} from '../../account-manager/account-manager.js'\nimport type { ActorStoreTransactor } from '../../actor-store/actor-store-transactor.js'\nimport type { ActorStore } from '../../actor-store/actor-store.js'\nimport { countAll } from '../../db/index.js'\nimport {\n  type PreparedWrite,\n  prepareCreate,\n  prepareDelete,\n  prepareUpdate,\n} from '../../repo/index.js'\nimport type {\n  AccountEvt,\n  CommitEvt,\n  SeqEvt,\n  Sequencer,\n} from '../../sequencer/index.js'\nimport type { RecoveryDb } from './recovery-db.js'\nimport { UserQueues } from './user-queues.js'\n\nexport type RecovererContextNoDb = {\n  sequencer: Sequencer\n  accountManager: AccountManager\n  actorStore: ActorStore\n}\n\nexport type RecovererContext = RecovererContextNoDb & {\n  recoveryDb: RecoveryDb\n}\n\nconst PAGE_SIZE = 5000\n\nexport class Recoverer {\n  queues: UserQueues\n  failed: Set<string>\n\n  constructor(\n    public ctx: RecovererContext,\n    opts: { concurrency: number },\n  ) {\n    this.queues = new UserQueues(opts.concurrency)\n    this.failed = new Set()\n  }\n\n  async run(startCursor = 0) {\n    const failed = await this.ctx.recoveryDb.db\n      .selectFrom('failed')\n      .select('did')\n      .execute()\n    for (const row of failed) {\n      this.failed.add(row.did)\n    }\n\n    const totalRes = await this.ctx.sequencer.db.db\n      .selectFrom('repo_seq')\n      .select(countAll.as('count'))\n      .executeTakeFirstOrThrow()\n    const totalEvts = totalRes.count\n    let completed = 0\n\n    let cursor: number | undefined = startCursor\n    while (cursor !== undefined) {\n      const page = await this.ctx.sequencer.requestSeqRange({\n        earliestSeq: cursor,\n        limit: PAGE_SIZE,\n      })\n      page.forEach((evt) => this.processEvent(evt))\n      cursor = page.at(-1)?.seq\n\n      await this.queues.onEmpty()\n\n      completed += PAGE_SIZE\n      const percentComplete = (completed / totalEvts) * 100\n      console.log(`${percentComplete.toFixed(2)}% - ${cursor}`)\n    }\n\n    await this.queues.processAll()\n  }\n\n  async processAll() {\n    await this.queues.processAll()\n  }\n\n  async destroy() {\n    await this.queues.destroy()\n  }\n\n  processEvent(evt: SeqEvt) {\n    const did = didFromEvt(evt)\n    if (!did) {\n      return\n    }\n    this.queues.addToUser(did, async () => {\n      if (this.failed.has(did)) {\n        return\n      }\n      await processSeqEvt(this.ctx, evt).catch(async (err) => {\n        this.failed.add(did)\n        await trackFailure(this.ctx.recoveryDb, did, err)\n      })\n    })\n  }\n}\n\nexport const processSeqEvt = async (ctx: RecovererContext, evt: SeqEvt) => {\n  // only need to process commits & tombstones\n  if (evt.type === 'account') {\n    await processAccountEvt(ctx, evt.evt)\n  }\n  if (evt.type === 'commit') {\n    await processCommit(ctx, evt.evt).catch()\n  }\n}\n\nconst processCommit = async (ctx: RecovererContext, evt: CommitEvt) => {\n  const did = evt.repo\n  const { writes, blocks } = await parseCommitEvt(evt)\n  if (evt.since === null) {\n    const actorExists = await ctx.actorStore.exists(did)\n    if (!actorExists) {\n      await processRepoCreation(ctx, evt, writes, blocks)\n      return\n    }\n  }\n  await ctx.actorStore.transact(did, async (actorTxn) => {\n    const root = await actorTxn.repo.storage.getRootDetailed()\n    if (root.rev >= evt.rev) {\n      return\n    }\n    const commit = await actorTxn.repo.formatCommit(writes)\n    commit.newBlocks = blocks\n    commit.cid = evt.commit\n    commit.rev = evt.rev\n    await actorTxn.repo.storage.applyCommit(commit)\n    await actorTxn.repo.indexWrites(writes, commit.rev)\n    await trackBlobs(actorTxn, writes)\n  })\n}\n\nconst processRepoCreation = async (\n  ctx: RecovererContext,\n  evt: CommitEvt,\n  writes: PreparedWrite[],\n  blocks: BlockMap,\n) => {\n  const did = evt.repo\n  const keypair = await Secp256k1Keypair.create({ exportable: true })\n  await ctx.actorStore.create(did, keypair)\n  const commit: CommitData = {\n    cid: evt.commit,\n    rev: evt.rev,\n    since: evt.since,\n    prev: null,\n    newBlocks: blocks,\n    relevantBlocks: new BlockMap(),\n    removedCids: new CidSet(),\n  }\n  await ctx.actorStore.transact(did, async (actorTxn) => {\n    await actorTxn.repo.storage.applyCommit(commit, true)\n    await actorTxn.repo.indexWrites(writes, commit.rev)\n    await actorTxn.repo.blob.processWriteBlobs(commit.rev, writes)\n  })\n  await trackNewAccount(ctx.recoveryDb, did)\n}\n\nconst processAccountEvt = async (ctx: RecovererContext, evt: AccountEvt) => {\n  // do not need to process deactivation/takedowns because we backup account DB as well\n\n  if (evt.status === AccountStatus.Deleted) {\n    // In case an account deletion was sequenced, let's make sure to (first)\n    // delete the accounts database, and (then) unlink the actor store from the\n    // file system. Order matters here.\n    await ctx.accountManager.deleteAccount(evt.did)\n\n    const { directory } = await ctx.actorStore.getLocation(evt.did)\n    await rmIfExists(directory, true)\n  }\n}\n\nconst trackBlobs = async (\n  store: ActorStoreTransactor,\n  writes: PreparedWrite[],\n) => {\n  await store.repo.blob.deleteDereferencedBlobs(writes)\n\n  for (const write of writes) {\n    if (\n      write.action === WriteOpAction.Create ||\n      write.action === WriteOpAction.Update\n    ) {\n      for (const blob of write.blobs) {\n        await store.repo.blob.insertBlobMetadata(blob)\n        await store.repo.blob.associateBlob(blob, write.uri)\n      }\n    }\n  }\n}\n\nconst trackFailure = async (\n  recoveryDb: RecoveryDb,\n  did: DidString,\n  err: unknown,\n) => {\n  await recoveryDb.db\n    .insertInto('failed')\n    .values({\n      did,\n      error: err?.toString(),\n      fixed: 0,\n    })\n    .onConflict((oc) => oc.doNothing())\n    .execute()\n}\n\nconst trackNewAccount = async (recoveryDb: RecoveryDb, did: DidString) => {\n  await recoveryDb.db\n    .insertInto('new_account')\n    .values({\n      did,\n      published: 0,\n    })\n    .onConflict((oc) => oc.doNothing())\n    .execute()\n}\n\nconst parseCommitEvt = async (\n  evt: CommitEvt,\n): Promise<{\n  writes: PreparedWrite[]\n  blocks: BlockMap\n}> => {\n  const did = evt.repo\n  const evtCar = await readCar(evt.blocks, { skipCidVerification: true })\n  const writesUnfiltered = await Promise.all(\n    evt.ops.map(async (op) => {\n      const { collection, rkey } = parseDataKey(op.path)\n      if (!isNsidString(collection)) return undefined\n      if (!isRecordKeyString(rkey)) return undefined\n\n      if (op.action === 'delete') {\n        return prepareDelete({ did, collection, rkey })\n      }\n      if (!op.cid) return undefined\n\n      const recordBytes = evtCar.blocks.get(op.cid)\n      if (!recordBytes) return undefined\n      const record = cborToLexRecord(recordBytes)\n\n      if (op.action === 'create') {\n        return prepareCreate({\n          did,\n          collection,\n          rkey,\n          record,\n          validate: false,\n        })\n      } else {\n        return prepareUpdate({\n          did,\n          collection,\n          rkey,\n          record,\n          validate: false,\n        })\n      }\n    }),\n  )\n\n  return {\n    writes: writesUnfiltered.filter((w) => w != null),\n    blocks: evtCar.blocks,\n  }\n}\n\nconst didFromEvt = (evt: SeqEvt): DidString | null => {\n  if (evt.type === 'account') {\n    return evt.evt.did\n  } else if (evt.type === 'commit') {\n    return evt.evt.repo\n  } else {\n    return null\n  }\n}\n"]}

package/dist/sequencer/events.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { type DatetimeString, type DidString, type HandleString } from '@atproto/lex';
+import { type DatetimeString, type DidString, type HandleString } from '@atproto/lex-schema';
 import { AccountStatus } from '../account-manager/account-manager.js';
 import type { CommitDataWithOps, SyncEvtData } from '../repo/index.js';
 import type { RepoSeqInsert } from './db/index.js';
@@ -11,13 +11,13 @@ export declare const formatSeqAccountEvt: (did: DidString, status: AccountStatus
 export declare const commitEvtOp: z.ZodObject<{
     action: z.ZodUnion<[z.ZodLiteral<"create">, z.ZodLiteral<"update">, z.ZodLiteral<"delete">]>;
     path: z.ZodString;
-    cid: z.ZodNullable<z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>>;
-    prev: z.ZodOptional<z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>>;
+    cid: z.ZodNullable<z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>>;
+    prev: z.ZodOptional<z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>>;
 }, "strip", z.ZodTypeAny, {
     action: "create" | "delete" | "update";
     path: string;
-    cid: import("multiformats/cid").CID<unknown, number, number, any> | null;
-    prev?: import("multiformats/cid").CID<unknown, number, number, any> | undefined;
+    cid: import("multiformats").CID<unknown, number, number, any> | null;
+    prev?: import("multiformats").CID<unknown, number, number, any> | undefined;
 }, {
     action: "create" | "delete" | "update";
     path: string;
@@ -29,44 +29,44 @@ export declare const commitEvt: z.ZodObject<{
     rebase: z.ZodBoolean;
     tooBig: z.ZodBoolean;
     repo: z.ZodEffects<z.ZodString, `did:${string}:${string}`, string>;
-    commit: z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>;
+    commit: z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>;
     rev: z.ZodString;
     since: z.ZodNullable<z.ZodString>;
     blocks: z.ZodType<Uint8Array<ArrayBufferLike>, z.ZodTypeDef, Uint8Array<ArrayBufferLike>>;
     ops: z.ZodArray<z.ZodObject<{
         action: z.ZodUnion<[z.ZodLiteral<"create">, z.ZodLiteral<"update">, z.ZodLiteral<"delete">]>;
         path: z.ZodString;
-        cid: z.ZodNullable<z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>>;
-        prev: z.ZodOptional<z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>>;
+        cid: z.ZodNullable<z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>>;
+        prev: z.ZodOptional<z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>>;
     }, "strip", z.ZodTypeAny, {
         action: "create" | "delete" | "update";
         path: string;
-        cid: import("multiformats/cid").CID<unknown, number, number, any> | null;
-        prev?: import("multiformats/cid").CID<unknown, number, number, any> | undefined;
+        cid: import("multiformats").CID<unknown, number, number, any> | null;
+        prev?: import("multiformats").CID<unknown, number, number, any> | undefined;
     }, {
         action: "create" | "delete" | "update";
         path: string;
         cid?: unknown;
         prev?: unknown;
     }>, "many">;
-    blobs: z.ZodArray<z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>, "many">;
-    prevData: z.ZodOptional<z.ZodEffects<z.ZodUnknown, import("multiformats/cid").CID<unknown, number, number, any>, unknown>>;
+    blobs: z.ZodArray<z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>, "many">;
+    prevData: z.ZodOptional<z.ZodEffects<z.ZodUnknown, import("multiformats").CID<unknown, number, number, any>, unknown>>;
 }, "strip", z.ZodTypeAny, {
     rebase: boolean;
     tooBig: boolean;
     repo: `did:${string}:${string}`;
-    commit: import("multiformats/cid").CID<unknown, number, number, any>;
+    commit: import("multiformats").CID<unknown, number, number, any>;
     rev: string;
     since: string | null;
     blocks: Uint8Array<ArrayBufferLike>;
     ops: {
         action: "create" | "delete" | "update";
         path: string;
-        cid: import("multiformats/cid").CID<unknown, number, number, any> | null;
-        prev?: import("multiformats/cid").CID<unknown, number, number, any> | undefined;
+        cid: import("multiformats").CID<unknown, number, number, any> | null;
+        prev?: import("multiformats").CID<unknown, number, number, any> | undefined;
     }[];
-    blobs: import("multiformats/cid").CID<unknown, number, number, any>[];
-    prevData?: import("multiformats/cid").CID<unknown, number, number, any> | undefined;
+    blobs: import("multiformats").CID<unknown, number, number, any>[];
+    prevData?: import("multiformats").CID<unknown, number, number, any> | undefined;
 }, {
     rebase: boolean;
     tooBig: boolean;

package/dist/sequencer/events.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/sequencer/events.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,YAAY,EAGlB,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAA;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAElD,eAAO,MAAM,eAAe,QACrB,MAAM,cACC,iBAAiB,KAC5B,OAAO,CAAC,aAAa,CAyBvB,CAAA;AAED,eAAO,MAAM,gBAAgB,QACtB,SAAS,QACR,WAAW,KAChB,OAAO,CAAC,aAAa,CAavB,CAAA;AAED,eAAO,MAAM,qBAAqB,eACpB,iBAAiB,KAC5B,WAaF,CAAA;AAED,eAAO,MAAM,oBAAoB,QAC1B,SAAS,WACL,YAAY,KACpB,OAAO,CAAC,aAAa,CAWvB,CAAA;AAED,eAAO,MAAM,mBAAmB,QACzB,SAAS,UACN,aAAa,KACpB,OAAO,CAAC,aAAa,CAYvB,CAAA;AAED,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;EAUtB,CAAA;AACF,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAA;AAIrD,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWpB,CAAA;AACF,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAA;AAEjD,eAAO,MAAM,OAAO;;;;;;;;;;;;EAIlB,CAAA;AACF,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,OAAO,CAAC,CAAA;AAE7C,eAAO,MAAM,WAAW;;;;;;;;;EAGtB,CAAA;AACF,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAA;AAErD,eAAO,MAAM,UAAU;;;;;;;;;;;;EAWrB,CAAA;AACF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA;AAEnD,KAAK,cAAc,GAAG;IACpB,IAAI,EAAE,QAAQ,CAAA;IACd,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,SAAS,CAAA;CACf,CAAA;AACD,KAAK,YAAY,GAAG;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,OAAO,CAAA;CACb,CAAA;AACD,KAAK,gBAAgB,GAAG;IACtB,IAAI,EAAE,UAAU,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,WAAW,CAAA;CACjB,CAAA;AACD,KAAK,eAAe,GAAG;IACrB,IAAI,EAAE,SAAS,CAAA;IACf,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,UAAU,CAAA;CAChB,CAAA;AACD,MAAM,MAAM,MAAM,GAChB,cAAc,GAAG,YAAY,GAAG,gBAAgB,GAAG,eAAe,CAAA"}
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/sequencer/events.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,YAAY,EAGlB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAA;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAElD,eAAO,MAAM,eAAe,QACrB,MAAM,cACC,iBAAiB,KAC5B,OAAO,CAAC,aAAa,CAyBvB,CAAA;AAED,eAAO,MAAM,gBAAgB,QACtB,SAAS,QACR,WAAW,KAChB,OAAO,CAAC,aAAa,CAavB,CAAA;AAED,eAAO,MAAM,qBAAqB,eACpB,iBAAiB,KAC5B,WAaF,CAAA;AAED,eAAO,MAAM,oBAAoB,QAC1B,SAAS,WACL,YAAY,KACpB,OAAO,CAAC,aAAa,CAWvB,CAAA;AAED,eAAO,MAAM,mBAAmB,QACzB,SAAS,UACN,aAAa,KACpB,OAAO,CAAC,aAAa,CAYvB,CAAA;AAED,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;EAUtB,CAAA;AACF,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAA;AAIrD,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWpB,CAAA;AACF,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAA;AAEjD,eAAO,MAAM,OAAO;;;;;;;;;;;;EAIlB,CAAA;AACF,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,OAAO,CAAC,CAAA;AAE7C,eAAO,MAAM,WAAW;;;;;;;;;EAGtB,CAAA;AACF,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAA;AAErD,eAAO,MAAM,UAAU;;;;;;;;;;;;EAWrB,CAAA;AACF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA;AAEnD,KAAK,cAAc,GAAG;IACpB,IAAI,EAAE,QAAQ,CAAA;IACd,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,SAAS,CAAA;CACf,CAAA;AACD,KAAK,YAAY,GAAG;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,OAAO,CAAA;CACb,CAAA;AACD,KAAK,gBAAgB,GAAG;IACtB,IAAI,EAAE,UAAU,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,WAAW,CAAA;CACjB,CAAA;AACD,KAAK,eAAe,GAAG;IACrB,IAAI,EAAE,SAAS,CAAA;IACf,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,cAAc,CAAA;IACpB,GAAG,EAAE,UAAU,CAAA;CAChB,CAAA;AACD,MAAM,MAAM,MAAM,GAChB,cAAc,GAAG,YAAY,GAAG,gBAAgB,GAAG,eAAe,CAAA"}

package/dist/sequencer/events.js

@@ -1,8 +1,8 @@
 import assert from 'node:assert';
 import { z } from 'zod';
 import { schema } from '@atproto/common';
-import { isDidString, isHandleString, } from '@atproto/lex';
 import { encode as cborEncode } from '@atproto/lex-cbor';
+import { isDidString, isHandleString, } from '@atproto/lex-schema';
 import { BlockMap, blocksToCarFile } from '@atproto/repo';
 import { AccountStatus } from '../account-manager/account-manager.js';
 export const formatSeqCommit = async (did, commitData) => {

package/dist/sequencer/events.js.map

@@ -1 +1 @@
-{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/sequencer/events.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAIL,WAAW,EACX,cAAc,GACf,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,mBAAmB,CAAA;AACxD,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAA;AAIrE,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,GAAW,EACX,UAA6B,EACL,EAAE;IAC1B,MAAM,YAAY,GAAG,IAAI,QAAQ,EAAE,CAAA;IACnC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;IACzC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;IAE9C,MAAM,GAAG,GAAG;QACV,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,UAAU,CAAC,GAAG;QACtB,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,MAAM,EAAE,MAAM,eAAe,CAAC,UAAU,CAAC,GAAG,EAAE,YAAY,CAAC;QAC3D,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,SAAS;QAC1C,yCAAyC;QACzC,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,EAAE;KACV,CAAA;IAED,OAAO;QACL,GAAG;QACH,SAAS,EAAE,QAAiB;QAC5B,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,GAAc,EACd,IAAiB,EACO,EAAE;IAC1B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3D,MAAM,GAAG,GAAY;QACnB,GAAG;QACH,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM;KACP,CAAA;IACD,OAAO;QACL,GAAG;QACH,SAAS,EAAE,MAAM;QACjB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,UAA6B,EAChB,EAAE;IACf,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,cAAc,CAAC,OAAO,CAAC;QAC5D,UAAU,CAAC,GAAG;KACf,CAAC,CAAA;IACF,MAAM,CACJ,CAAC,OAAO,CAAC,MAAM,EACf,wDAAwD,CACzD,CAAA;IACD,OAAO;QACL,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,GAAc,EACd,MAAqB,EACG,EAAE;IAC1B,MAAM,GAAG,GAAgB,MAAM,CAAC,EAAE;QAChC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE;QACjB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAA;IAEX,OAAO;QACL,GAAG;QACH,SAAS,EAAE,UAAU;QACrB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,GAAc,EACd,MAAqB,EACG,EAAE;IAC1B,MAAM,GAAG,GACP,MAAM,KAAK,aAAa,CAAC,MAAM;QAC7B,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACvB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;IAEpC,OAAO;QACL,GAAG;QACH,SAAS,EAAE,SAAS;QACpB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC;QACd,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QACnB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QACnB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;KACpB,CAAC;IACF,wEAAwE;IACxE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAA;AAGF,6EAA6E;AAC7E,oCAAoC;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC,GAAG;IAClB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,MAAM,EAAE,MAAM,CAAC,KAAK;IACpB,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;IACzB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAA;AAGF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC,KAAK;IACpB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;CAChB,CAAC,CAAA;AAGF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAA;AAGF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,MAAM,EAAE,CAAC;SACN,IAAI,CAAC;QACJ,aAAa,CAAC,SAAS;QACvB,aAAa,CAAC,SAAS;QACvB,aAAa,CAAC,OAAO;QACrB,aAAa,CAAC,WAAW;KAC1B,CAAC;SACD,QAAQ,EAAE;CACd,CAAC,CAAA","sourcesContent":["import assert from 'node:assert'\nimport { z } from 'zod'\nimport { schema } from '@atproto/common'\nimport {\n  type DatetimeString,\n  type DidString,\n  type HandleString,\n  isDidString,\n  isHandleString,\n} from '@atproto/lex'\nimport { encode as cborEncode } from '@atproto/lex-cbor'\nimport { BlockMap, blocksToCarFile } from '@atproto/repo'\nimport { AccountStatus } from '../account-manager/account-manager.js'\nimport type { CommitDataWithOps, SyncEvtData } from '../repo/index.js'\nimport type { RepoSeqInsert } from './db/index.js'\n\nexport const formatSeqCommit = async (\n  did: string,\n  commitData: CommitDataWithOps,\n): Promise<RepoSeqInsert> => {\n  const blocksToSend = new BlockMap()\n  blocksToSend.addMap(commitData.newBlocks)\n  blocksToSend.addMap(commitData.relevantBlocks)\n\n  const evt = {\n    repo: did,\n    commit: commitData.cid,\n    rev: commitData.rev,\n    since: commitData.since,\n    blocks: await blocksToCarFile(commitData.cid, blocksToSend),\n    ops: commitData.ops,\n    prevData: commitData.prevData ?? undefined,\n    // deprecated (but still required) fields\n    rebase: false,\n    tooBig: false,\n    blobs: [],\n  }\n\n  return {\n    did,\n    eventType: 'append' as const,\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const formatSeqSyncEvt = async (\n  did: DidString,\n  data: SyncEvtData,\n): Promise<RepoSeqInsert> => {\n  const blocks = await blocksToCarFile(data.cid, data.blocks)\n  const evt: SyncEvt = {\n    did,\n    rev: data.rev,\n    blocks,\n  }\n  return {\n    did,\n    eventType: 'sync',\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const syncEvtDataFromCommit = (\n  commitData: CommitDataWithOps,\n): SyncEvtData => {\n  const { blocks, missing } = commitData.relevantBlocks.getMany([\n    commitData.cid,\n  ])\n  assert(\n    !missing.length,\n    'commit block was not found, could not build sync event',\n  )\n  return {\n    rev: commitData.rev,\n    cid: commitData.cid,\n    blocks,\n  }\n}\n\nexport const formatSeqIdentityEvt = async (\n  did: DidString,\n  handle?: HandleString,\n): Promise<RepoSeqInsert> => {\n  const evt: IdentityEvt = handle //\n    ? { did, handle }\n    : { did }\n\n  return {\n    did,\n    eventType: 'identity',\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const formatSeqAccountEvt = async (\n  did: DidString,\n  status: AccountStatus,\n): Promise<RepoSeqInsert> => {\n  const evt: AccountEvt =\n    status === AccountStatus.Active\n      ? { did, active: true }\n      : { did, active: false, status }\n\n  return {\n    did,\n    eventType: 'account',\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const commitEvtOp = z.object({\n  action: z.union([\n    z.literal('create'),\n    z.literal('update'),\n    z.literal('delete'),\n  ]),\n  // @TODO should we validate that the \"path\" is a valid \"<nsid>/<rkey>\" ?\n  path: z.string(),\n  cid: schema.cid.nullable(),\n  prev: schema.cid.optional(),\n})\nexport type CommitEvtOp = z.infer<typeof commitEvtOp>\n\n// @TODO This runtime code is only used to generate \"types\". We should either\n// make use of it or only use types.\nexport const commitEvt = z.object({\n  rebase: z.boolean(),\n  tooBig: z.boolean(),\n  repo: z.string().refine(isDidString),\n  commit: schema.cid,\n  rev: z.string(),\n  since: z.string().nullable(),\n  blocks: schema.bytes,\n  ops: z.array(commitEvtOp),\n  blobs: z.array(schema.cid),\n  prevData: schema.cid.optional(),\n})\nexport type CommitEvt = z.infer<typeof commitEvt>\n\nexport const syncEvt = z.object({\n  did: z.string().refine(isDidString),\n  blocks: schema.bytes,\n  rev: z.string(),\n})\nexport type SyncEvt = z.infer<typeof syncEvt>\n\nexport const identityEvt = z.object({\n  did: z.string().refine(isDidString),\n  handle: z.string().refine(isHandleString).optional(),\n})\nexport type IdentityEvt = z.infer<typeof identityEvt>\n\nexport const accountEvt = z.object({\n  did: z.string().refine(isDidString),\n  active: z.boolean(),\n  status: z\n    .enum([\n      AccountStatus.Takendown,\n      AccountStatus.Suspended,\n      AccountStatus.Deleted,\n      AccountStatus.Deactivated,\n    ])\n    .optional(),\n})\nexport type AccountEvt = z.infer<typeof accountEvt>\n\ntype TypedCommitEvt = {\n  type: 'commit'\n  seq: number\n  time: DatetimeString\n  evt: CommitEvt\n}\ntype TypedSyncEvt = {\n  type: 'sync'\n  seq: number\n  time: DatetimeString\n  evt: SyncEvt\n}\ntype TypedIdentityEvt = {\n  type: 'identity'\n  seq: number\n  time: DatetimeString\n  evt: IdentityEvt\n}\ntype TypedAccountEvt = {\n  type: 'account'\n  seq: number\n  time: DatetimeString\n  evt: AccountEvt\n}\nexport type SeqEvt =\n  TypedCommitEvt | TypedSyncEvt | TypedIdentityEvt | TypedAccountEvt\n"]}
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/sequencer/events.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,mBAAmB,CAAA;AACxD,OAAO,EAIL,WAAW,EACX,cAAc,GACf,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAA;AAIrE,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,GAAW,EACX,UAA6B,EACL,EAAE;IAC1B,MAAM,YAAY,GAAG,IAAI,QAAQ,EAAE,CAAA;IACnC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;IACzC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;IAE9C,MAAM,GAAG,GAAG;QACV,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,UAAU,CAAC,GAAG;QACtB,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,MAAM,EAAE,MAAM,eAAe,CAAC,UAAU,CAAC,GAAG,EAAE,YAAY,CAAC;QAC3D,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,SAAS;QAC1C,yCAAyC;QACzC,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,EAAE;KACV,CAAA;IAED,OAAO;QACL,GAAG;QACH,SAAS,EAAE,QAAiB;QAC5B,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,GAAc,EACd,IAAiB,EACO,EAAE;IAC1B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3D,MAAM,GAAG,GAAY;QACnB,GAAG;QACH,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM;KACP,CAAA;IACD,OAAO;QACL,GAAG;QACH,SAAS,EAAE,MAAM;QACjB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,UAA6B,EAChB,EAAE;IACf,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,cAAc,CAAC,OAAO,CAAC;QAC5D,UAAU,CAAC,GAAG;KACf,CAAC,CAAA;IACF,MAAM,CACJ,CAAC,OAAO,CAAC,MAAM,EACf,wDAAwD,CACzD,CAAA;IACD,OAAO;QACL,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,GAAG,EAAE,UAAU,CAAC,GAAG;QACnB,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,GAAc,EACd,MAAqB,EACG,EAAE;IAC1B,MAAM,GAAG,GAAgB,MAAM,CAAC,EAAE;QAChC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE;QACjB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAA;IAEX,OAAO;QACL,GAAG;QACH,SAAS,EAAE,UAAU;QACrB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,GAAc,EACd,MAAqB,EACG,EAAE;IAC1B,MAAM,GAAG,GACP,MAAM,KAAK,aAAa,CAAC,MAAM;QAC7B,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACvB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;IAEpC,OAAO;QACL,GAAG;QACH,SAAS,EAAE,SAAS;QACpB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC;QACd,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QACnB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QACnB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;KACpB,CAAC;IACF,wEAAwE;IACxE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAA;AAGF,6EAA6E;AAC7E,oCAAoC;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC,GAAG;IAClB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,MAAM,EAAE,MAAM,CAAC,KAAK;IACpB,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;IACzB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAA;AAGF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC,KAAK;IACpB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;CAChB,CAAC,CAAA;AAGF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAA;AAGF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,MAAM,EAAE,CAAC;SACN,IAAI,CAAC;QACJ,aAAa,CAAC,SAAS;QACvB,aAAa,CAAC,SAAS;QACvB,aAAa,CAAC,OAAO;QACrB,aAAa,CAAC,WAAW;KAC1B,CAAC;SACD,QAAQ,EAAE;CACd,CAAC,CAAA","sourcesContent":["import assert from 'node:assert'\nimport { z } from 'zod'\nimport { schema } from '@atproto/common'\nimport { encode as cborEncode } from '@atproto/lex-cbor'\nimport {\n  type DatetimeString,\n  type DidString,\n  type HandleString,\n  isDidString,\n  isHandleString,\n} from '@atproto/lex-schema'\nimport { BlockMap, blocksToCarFile } from '@atproto/repo'\nimport { AccountStatus } from '../account-manager/account-manager.js'\nimport type { CommitDataWithOps, SyncEvtData } from '../repo/index.js'\nimport type { RepoSeqInsert } from './db/index.js'\n\nexport const formatSeqCommit = async (\n  did: string,\n  commitData: CommitDataWithOps,\n): Promise<RepoSeqInsert> => {\n  const blocksToSend = new BlockMap()\n  blocksToSend.addMap(commitData.newBlocks)\n  blocksToSend.addMap(commitData.relevantBlocks)\n\n  const evt = {\n    repo: did,\n    commit: commitData.cid,\n    rev: commitData.rev,\n    since: commitData.since,\n    blocks: await blocksToCarFile(commitData.cid, blocksToSend),\n    ops: commitData.ops,\n    prevData: commitData.prevData ?? undefined,\n    // deprecated (but still required) fields\n    rebase: false,\n    tooBig: false,\n    blobs: [],\n  }\n\n  return {\n    did,\n    eventType: 'append' as const,\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const formatSeqSyncEvt = async (\n  did: DidString,\n  data: SyncEvtData,\n): Promise<RepoSeqInsert> => {\n  const blocks = await blocksToCarFile(data.cid, data.blocks)\n  const evt: SyncEvt = {\n    did,\n    rev: data.rev,\n    blocks,\n  }\n  return {\n    did,\n    eventType: 'sync',\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const syncEvtDataFromCommit = (\n  commitData: CommitDataWithOps,\n): SyncEvtData => {\n  const { blocks, missing } = commitData.relevantBlocks.getMany([\n    commitData.cid,\n  ])\n  assert(\n    !missing.length,\n    'commit block was not found, could not build sync event',\n  )\n  return {\n    rev: commitData.rev,\n    cid: commitData.cid,\n    blocks,\n  }\n}\n\nexport const formatSeqIdentityEvt = async (\n  did: DidString,\n  handle?: HandleString,\n): Promise<RepoSeqInsert> => {\n  const evt: IdentityEvt = handle //\n    ? { did, handle }\n    : { did }\n\n  return {\n    did,\n    eventType: 'identity',\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const formatSeqAccountEvt = async (\n  did: DidString,\n  status: AccountStatus,\n): Promise<RepoSeqInsert> => {\n  const evt: AccountEvt =\n    status === AccountStatus.Active\n      ? { did, active: true }\n      : { did, active: false, status }\n\n  return {\n    did,\n    eventType: 'account',\n    event: cborEncode(evt),\n    sequencedAt: new Date().toISOString(),\n  }\n}\n\nexport const commitEvtOp = z.object({\n  action: z.union([\n    z.literal('create'),\n    z.literal('update'),\n    z.literal('delete'),\n  ]),\n  // @TODO should we validate that the \"path\" is a valid \"<nsid>/<rkey>\" ?\n  path: z.string(),\n  cid: schema.cid.nullable(),\n  prev: schema.cid.optional(),\n})\nexport type CommitEvtOp = z.infer<typeof commitEvtOp>\n\n// @TODO This runtime code is only used to generate \"types\". We should either\n// make use of it or only use types.\nexport const commitEvt = z.object({\n  rebase: z.boolean(),\n  tooBig: z.boolean(),\n  repo: z.string().refine(isDidString),\n  commit: schema.cid,\n  rev: z.string(),\n  since: z.string().nullable(),\n  blocks: schema.bytes,\n  ops: z.array(commitEvtOp),\n  blobs: z.array(schema.cid),\n  prevData: schema.cid.optional(),\n})\nexport type CommitEvt = z.infer<typeof commitEvt>\n\nexport const syncEvt = z.object({\n  did: z.string().refine(isDidString),\n  blocks: schema.bytes,\n  rev: z.string(),\n})\nexport type SyncEvt = z.infer<typeof syncEvt>\n\nexport const identityEvt = z.object({\n  did: z.string().refine(isDidString),\n  handle: z.string().refine(isHandleString).optional(),\n})\nexport type IdentityEvt = z.infer<typeof identityEvt>\n\nexport const accountEvt = z.object({\n  did: z.string().refine(isDidString),\n  active: z.boolean(),\n  status: z\n    .enum([\n      AccountStatus.Takendown,\n      AccountStatus.Suspended,\n      AccountStatus.Deleted,\n      AccountStatus.Deactivated,\n    ])\n    .optional(),\n})\nexport type AccountEvt = z.infer<typeof accountEvt>\n\ntype TypedCommitEvt = {\n  type: 'commit'\n  seq: number\n  time: DatetimeString\n  evt: CommitEvt\n}\ntype TypedSyncEvt = {\n  type: 'sync'\n  seq: number\n  time: DatetimeString\n  evt: SyncEvt\n}\ntype TypedIdentityEvt = {\n  type: 'identity'\n  seq: number\n  time: DatetimeString\n  evt: IdentityEvt\n}\ntype TypedAccountEvt = {\n  type: 'account'\n  seq: number\n  time: DatetimeString\n  evt: AccountEvt\n}\nexport type SeqEvt =\n  TypedCommitEvt | TypedSyncEvt | TypedIdentityEvt | TypedAccountEvt\n"]}

package/dist/telemetry.js

@@ -1,8 +1,8 @@
 import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk';
 import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
-import { BetterSqlite3Instrumentation } from 'opentelemetry-plugin-better-sqlite3';
 import pkg from '@atproto/pds/package.json' with { type: 'json' };
 import { setup } from '@atproto-labs/opentelemetry-node';
+import { BetterSqlite3Instrumentation } from '@atproto-labs/opentelemetry-node/better-sqlite3-plugin';
 await setup(() => ({
     name: pkg.name,
     version: pkg.version,

package/dist/telemetry.js.map

@@ -1 +1 @@
-{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAA;AAC/E,OAAO,EAAE,4BAA4B,EAAE,MAAM,qCAAqC,CAAA;AAClF,OAAO,GAAG,MAAM,2BAA2B,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,kCAAkC,CAAA;AAExD,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,IAAI;IACd,OAAO,EAAE,GAAG,CAAC,OAAO;IACpB,gBAAgB,EAAE;QAChB,IAAI,kBAAkB,EAAE;QACxB,IAAI,sBAAsB,EAAE;QAC5B,IAAI,4BAA4B,EAAE;KACnC;CACF,CAAC,CAAC,CAAA","sourcesContent":["import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk'\nimport { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'\nimport { BetterSqlite3Instrumentation } from 'opentelemetry-plugin-better-sqlite3'\nimport pkg from '@atproto/pds/package.json' with { type: 'json' }\nimport { setup } from '@atproto-labs/opentelemetry-node'\n\nawait setup(() => ({\n  name: pkg.name,\n  version: pkg.version,\n  instrumentations: [\n    new AwsInstrumentation(),\n    new IORedisInstrumentation(),\n    new BetterSqlite3Instrumentation(),\n  ],\n}))\n"]}
+{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAA;AAC/E,OAAO,GAAG,MAAM,2BAA2B,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,kCAAkC,CAAA;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,wDAAwD,CAAA;AAErG,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,IAAI;IACd,OAAO,EAAE,GAAG,CAAC,OAAO;IACpB,gBAAgB,EAAE;QAChB,IAAI,kBAAkB,EAAE;QACxB,IAAI,sBAAsB,EAAE;QAC5B,IAAI,4BAA4B,EAAE;KACnC;CACF,CAAC,CAAC,CAAA","sourcesContent":["import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk'\nimport { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'\nimport pkg from '@atproto/pds/package.json' with { type: 'json' }\nimport { setup } from '@atproto-labs/opentelemetry-node'\nimport { BetterSqlite3Instrumentation } from '@atproto-labs/opentelemetry-node/better-sqlite3-plugin'\n\nawait setup(() => ({\n  name: pkg.name,\n  version: pkg.version,\n  instrumentations: [\n    new AwsInstrumentation(),\n    new IORedisInstrumentation(),\n    new BetterSqlite3Instrumentation(),\n  ],\n}))\n"]}

package/example.env

@@ -52,4 +52,3 @@ PDS_DISABLE_SSRF_PROTECTION=0
 #OTEL_NODE_RESOURCE_DETECTORS="container,env,host"
 #OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector.example.com"
 #OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf"
-#OTEL_SEMCONV_STABILITY_OPT_IN="http"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/pds",
-  "version": "0.5.51",
+  "version": "0.5.55",
   "license": "MIT",
   "description": "Reference implementation of atproto Personal Data Server (PDS)",
   "keywords": [
@@ -35,13 +35,34 @@
     "node": ">=22"
   },
   "dependencies": {
+    "@atproto-labs/fetch-node": "^0.4.5",
+    "@atproto-labs/opentelemetry-node": "^0.5.0",
+    "@atproto-labs/simple-store": "^0.5.1",
+    "@atproto-labs/simple-store-memory": "^0.2.6",
+    "@atproto-labs/simple-store-redis": "^0.1.6",
+    "@atproto-labs/xrpc-utils": "^0.1.30",
+    "@atproto/aws": "^0.3.23",
+    "@atproto/common": "^0.8.9",
+    "@atproto/crypto": "^0.5.7",
+    "@atproto/did": "^0.5.6",
+    "@atproto/identity": "^0.5.23",
+    "@atproto/lex-cbor": "^0.1.9",
+    "@atproto/lex-client": "^0.4.2",
+    "@atproto/lex-data": "^0.1.10",
+    "@atproto/lex-document": "^0.1.16",
+    "@atproto/lex-json": "^0.1.9",
+    "@atproto/lex-schema": "^0.2.11",
+    "@atproto/oauth-provider": "^0.25.5",
+    "@atproto/oauth-scopes": "^0.5.14",
+    "@atproto/repo": "^0.11.4",
+    "@atproto/syntax": "^0.7.7",
+    "@atproto/xrpc-server": "^0.13.9",
     "@did-plc/lib": "^0.0.4",
     "@hapi/address": "^5.1.1",
     "@noble/curves": "^1.7.0",
-    "@opentelemetry/api": "^1.9.1",
-    "@opentelemetry/api-logs": "^0.219.0",
-    "@opentelemetry/instrumentation-aws-sdk": "^0.74.0",
-    "@opentelemetry/instrumentation-ioredis": "^0.67.0",
+    "@opentelemetry/api": "^1.10.0",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.79.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.72.0",
     "better-sqlite3": "^12.0.0",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
@@ -56,37 +77,19 @@
     "kysely": "^0.29.2",
     "nodemailer": "^6.8.0",
     "nodemailer-html-to-text": "^3.2.0",
-    "opentelemetry-plugin-better-sqlite3": "^1.13.0",
     "p-queue": "^8.0.0",
     "pino": "^10.3.1",
     "pino-http": "^11.0.0",
     "strtok3": "^10.3.5",
     "uint8arrays": "^5.0.0",
     "undici": "^8.12.2",
-    "zod": "^3.23.8",
-    "@atproto-labs/fetch-node": "^0.4.3",
-    "@atproto-labs/opentelemetry-node": "^0.4.0",
-    "@atproto-labs/simple-store": "^0.5.1",
-    "@atproto-labs/simple-store-memory": "^0.2.6",
-    "@atproto-labs/simple-store-redis": "^0.1.6",
-    "@atproto-labs/xrpc-utils": "^0.1.27",
-    "@atproto/aws": "^0.3.20",
-    "@atproto/common": "^0.8.6",
-    "@atproto/crypto": "^0.5.7",
-    "@atproto/did": "^0.5.6",
-    "@atproto/identity": "^0.5.20",
-    "@atproto/lex": "^0.4.3",
-    "@atproto/lex-cbor": "^0.1.7",
-    "@atproto/lex-data": "^0.1.8",
-    "@atproto/lex-document": "^0.1.14",
-    "@atproto/lex-json": "^0.1.7",
-    "@atproto/oauth-provider": "^0.25.1",
-    "@atproto/oauth-scopes": "^0.5.14",
-    "@atproto/repo": "^0.11.1",
-    "@atproto/syntax": "^0.7.7",
-    "@atproto/xrpc-server": "^0.13.6"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
+    "@atproto/api": "^0.30.3",
+    "@atproto/lex": "^0.4.7",
+    "@atproto/oauth-client-browser-example": "^0.2.1",
+    "@atproto/oauth-client-node": "^0.5.16",
     "@did-plc/server": "^0.0.1",
     "@jest/globals": "^30.0.0",
     "@types/cors": "^2.8.12",
@@ -97,13 +100,10 @@
     "get-port": "^6.1.2",
     "jest": "^30.5.2",
     "puppeteer": "^25.16.0",
-    "ws": "^8.12.0",
-    "@atproto/api": "^0.29.5",
-    "@atproto/oauth-client-browser-example": "^0.2.1",
-    "@atproto/oauth-client-node": "^0.5.13"
+    "ws": "^8.12.0"
   },
   "scripts": {
-    "lex:build": "lex build --clear --indexFile",
+    "lex:build": "lex build --clear --indexFile --lib @atproto/lex-schema",
     "lex:install": "lex install",
     "postlex:install": "pnpm run lex:build",
     "lex:update": "lex update",