@atproto/repo 0.8.12 → 0.9.0

package/src/data-diff.ts

@@ -1,4 +1,4 @@
-import { CID } from 'multiformats'
+import { Cid } from '@atproto/lex-data'
 import { BlockMap } from './block-map'
 import { CidSet } from './cid-set'
 import { MST, NodeEntry, mstDiff } from './mst'
@@ -37,7 +37,7 @@ export class DataDiff {
     }
   }
 
-  leafAdd(key: string, cid: CID) {
+  leafAdd(key: string, cid: Cid) {
     this.adds[key] = { key, cid }
     if (this.removedCids.has(cid)) {
       this.removedCids.delete(cid)
@@ -46,14 +46,14 @@ export class DataDiff {
     }
   }
 
-  leafUpdate(key: string, prev: CID, cid: CID) {
+  leafUpdate(key: string, prev: Cid, cid: Cid) {
     if (prev.equals(cid)) return
     this.updates[key] = { key, prev, cid }
     this.removedCids.add(prev)
     this.newLeafCids.add(cid)
   }
 
-  leafDelete(key: string, cid: CID) {
+  leafDelete(key: string, cid: Cid) {
     this.deletes[key] = { key, cid }
     if (this.newLeafCids.has(cid)) {
       this.newLeafCids.delete(cid)
@@ -62,7 +62,7 @@ export class DataDiff {
     }
   }
 
-  treeAdd(cid: CID, bytes: Uint8Array) {
+  treeAdd(cid: Cid, bytes: Uint8Array) {
     if (this.removedCids.has(cid)) {
       this.removedCids.delete(cid)
     } else {
@@ -70,7 +70,7 @@ export class DataDiff {
     }
   }
 
-  treeDelete(cid: CID) {
+  treeDelete(cid: Cid) {
     if (this.newMstBlocks.has(cid)) {
       this.newMstBlocks.delete(cid)
     } else {
@@ -102,16 +102,16 @@ export class DataDiff {
 
 export type DataAdd = {
   key: string
-  cid: CID
+  cid: Cid
 }
 
 export type DataUpdate = {
   key: string
-  prev: CID
-  cid: CID
+  prev: Cid
+  cid: Cid
 }
 
 export type DataDelete = {
   key: string
-  cid: CID
+  cid: Cid
 }

package/src/error.ts

@@ -1,8 +1,8 @@
-import { CID } from 'multiformats/cid'
+import { Cid } from '@atproto/lex-data'
 
 export class MissingBlockError extends Error {
   constructor(
-    public cid: CID,
+    public cid: Cid,
     def?: string,
   ) {
     let msg = `block not found: ${cid.toString()}`
@@ -16,7 +16,7 @@ export class MissingBlockError extends Error {
 export class MissingBlocksError extends Error {
   constructor(
     public context: string,
-    public cids: CID[],
+    public cids: Cid[],
   ) {
     const cidStr = cids.map((c) => c.toString())
     super(`missing ${context} blocks: ${cidStr}`)
@@ -25,8 +25,8 @@ export class MissingBlocksError extends Error {
 
 export class MissingCommitBlocksError extends Error {
   constructor(
-    public commit: CID,
-    public cids: CID[],
+    public commit: Cid,
+    public cids: Cid[],
   ) {
     const cidStr = cids.map((c) => c.toString())
     super(`missing blocks for commit ${commit.toString()}: ${cidStr}`)
@@ -35,7 +35,7 @@ export class MissingCommitBlocksError extends Error {
 
 export class UnexpectedObjectError extends Error {
   constructor(
-    public cid: CID,
+    public cid: Cid,
     public def: string,
   ) {
     super(`unexpected object at ${cid.toString()}, expected: ${def}`)

package/src/logger.ts

@@ -1,3 +1,4 @@
+// @TODO remove dependency on @atproto/common and subsystemLogger
 import { subsystemLogger } from '@atproto/common'
 
 export const logger: ReturnType<typeof subsystemLogger> =

package/src/mst/mst.ts

@@ -1,14 +1,12 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats'
 import { z } from 'zod'
-import { cidForCbor, dataToCborBlock, schema as common } from '@atproto/common'
+import { cidForLex, encode } from '@atproto/lex-cbor'
+import { Cid, cidForCbor } from '@atproto/lex-data'
 import { BlockMap } from '../block-map'
 import { CidSet } from '../cid-set'
 import { MissingBlockError, MissingBlocksError } from '../error'
 import * as parse from '../parse'
 import { ReadableBlockstore } from '../storage'
-import { CarBlock } from '../types'
+import { CarBlock, schema } from '../types'
 import * as util from './util'
 
 /**
@@ -18,7 +16,7 @@ import * as util from './util'
  * Keys are laid out in alphabetic order.
  * The key insight of an MST is that each key is hashed and starting 0s are counted
  * to determine which layer it falls on (5 zeros for ~32 fanout).
- * This is a merkle tree, so each subtree is referred to by it's hash (CID).
+ * This is a merkle tree, so each subtree is referred to by it's hash (Cid).
  * When a leaf is changed, ever tree on the path to that leaf is changed as well,
  * thereby updating the root hash.
  *
@@ -44,11 +42,11 @@ import * as util from './util'
  * Then the first will be described as `prefix: 0, key: 'bsky/posts/abcdefg'`,
  * and the second will be described as `prefix: 16, key: 'hi'.`
  */
-const subTreePointer = z.nullable(common.cid)
+const subTreePointer = z.nullable(schema.cid)
 const treeEntry = z.object({
   p: z.number(), // prefix count of ascii chars that this key shares with the prev key
-  k: common.bytes, // the rest of the key outside the shared prefix
-  v: common.cid, // value
+  k: schema.bytes, // the rest of the key outside the shared prefix
+  v: schema.cid, // value
   t: subTreePointer, // next subtree (to the right of leaf)
 })
 const nodeData = z.object({
@@ -72,12 +70,12 @@ export class MST {
   storage: ReadableBlockstore
   entries: NodeEntry[] | null
   layer: number | null
-  pointer: CID
+  pointer: Cid
   outdatedPointer = false
 
   constructor(
     storage: ReadableBlockstore,
-    pointer: CID,
+    pointer: Cid,
     entries: NodeEntry[] | null,
     layer: number | null,
   ) {
@@ -104,14 +102,14 @@ export class MST {
   ): Promise<MST> {
     const { layer = null } = opts || {}
     const entries = await util.deserializeNodeData(storage, data, opts)
-    const pointer = await cidForCbor(data)
+    const pointer = await cidForLex(data)
     return new MST(storage, pointer, entries, layer)
   }
 
   // this is really a *lazy* load, doesn't actually touch storage
   static load(
     storage: ReadableBlockstore,
-    cid: CID,
+    cid: Cid,
     opts?: Partial<MstOpts>,
   ): MST {
     const { layer = null } = opts || {}
@@ -147,12 +145,12 @@ export class MST {
 
       return this.entries
     }
-    throw new Error('No entries or CID provided')
+    throw new Error('No entries or Cid provided')
   }
 
   // We don't hash the node on every mutation for performance reasons
   // Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed
-  async getPointer(): Promise<CID> {
+  async getPointer(): Promise<Cid> {
     if (!this.outdatedPointer) return this.pointer
     const { cid } = await this.serialize()
     this.pointer = cid
@@ -160,7 +158,7 @@ export class MST {
     return this.pointer
   }
 
-  async serialize(): Promise<{ cid: CID; bytes: Uint8Array }> {
+  async serialize(): Promise<{ cid: Cid; bytes: Uint8Array }> {
     let entries = await this.getEntries()
     const outdated = entries.filter(
       (e) => e.isTree() && e.outdatedPointer,
@@ -170,11 +168,9 @@ export class MST {
       entries = await this.getEntries()
     }
     const data = util.serializeNodeData(entries)
-    const block = await dataToCborBlock(data)
-    return {
-      cid: block.cid,
-      bytes: block.bytes,
-    }
+    const bytes = encode(data)
+    const cid = await cidForCbor(bytes)
+    return { cid, bytes }
   }
 
   // In most cases, we get the layer of a node from a hint on creation
@@ -210,7 +206,7 @@ export class MST {
   // -------------------
 
   // Return the necessary blocks to persist the MST to repo storage
-  async getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }> {
+  async getUnstoredBlocks(): Promise<{ root: Cid; blocks: BlockMap }> {
     const blocks = new BlockMap()
     const pointer = await this.getPointer()
     const alreadyHas = await this.storage.has(pointer)
@@ -229,7 +225,7 @@ export class MST {
 
   // Adds a new leaf for the given key/value pair
   // Throws if a leaf with that key already exists
-  async add(key: string, value: CID, knownZeros?: number): Promise<MST> {
+  async add(key: string, value: Cid, knownZeros?: number): Promise<MST> {
     util.ensureValidMstKey(key)
     const keyZeros = knownZeros ?? (await util.leadingZerosOnHash(key))
     const layer = await this.getLayer()
@@ -299,7 +295,7 @@ export class MST {
   }
 
   // Gets the value at the given key
-  async get(key: string): Promise<CID | null> {
+  async get(key: string): Promise<Cid | null> {
     const index = await this.findGtOrEqualLeafIndex(key)
     const found = await this.atIndex(index)
     if (found && found.isLeaf() && found.key === key) {
@@ -314,7 +310,7 @@ export class MST {
 
   // Edits the value at the given key
   // Throws if the given key does not exist
-  async update(key: string, value: CID): Promise<MST> {
+  async update(key: string, value: Cid): Promise<MST> {
     util.ensureValidMstKey(key)
     const index = await this.findGtOrEqualLeafIndex(key)
     const found = await this.atIndex(index)
@@ -767,8 +763,8 @@ export class MST {
     }
   }
 
-  async cidsForPath(key: string): Promise<CID[]> {
-    const cids: CID[] = [await this.getPointer()]
+  async cidsForPath(key: string): Promise<Cid[]> {
+    const cids: Cid[] = [await this.getPointer()]
     const index = await this.findGtOrEqualLeafIndex(key)
     const found = await this.atIndex(index)
     if (found && found.isLeaf() && found.key === key) {
@@ -875,7 +871,7 @@ export class MST {
 export class Leaf {
   constructor(
     public key: string,
-    public value: CID,
+    public value: Cid,
   ) {}
 
   isTree(): this is MST {

package/src/mst/util.ts

@@ -1,12 +1,25 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats'
-import * as uint8arrays from 'uint8arrays'
-import { cidForCbor } from '@atproto/common'
 import { sha256 } from '@atproto/crypto'
+import { cidForLex } from '@atproto/lex-cbor'
+import { Cid } from '@atproto/lex-data'
 import { ReadableBlockstore } from '../storage'
 import { Leaf, MST, MstOpts, NodeData, NodeEntry } from './mst'
 
+function toAscii(bytes: Uint8Array): string {
+  let string = ''
+  for (let i = 0; i < bytes.length; i++) {
+    string += String.fromCharCode(bytes[i])
+  }
+  return string
+}
+
+function fromAscii(str: string): Uint8Array {
+  const bytes = new Uint8Array(str.length)
+  for (let i = 0; i < str.length; i++) {
+    bytes[i] = str.charCodeAt(i)
+  }
+  return bytes
+}
+
 export const leadingZerosOnHash = async (key: string | Uint8Array) => {
   const hash = await sha256(key)
   let leadingZeros = 0
@@ -48,8 +61,8 @@ export const deserializeNodeData = async (
   }
   let lastKey = ''
   for (const entry of data.e) {
-    const keyStr = uint8arrays.toString(entry.k, 'ascii')
-    const key = lastKey.slice(0, entry.p) + keyStr
+    const keyStr = toAscii(entry.k)
+    const key = `${lastKey.slice(0, entry.p)}${keyStr}`
     ensureValidMstKey(key)
     entries.push(new Leaf(key, entry.v))
     lastKey = key
@@ -82,7 +95,7 @@ export const serializeNodeData = (entries: NodeEntry[]): NodeData => {
       throw new Error('Not a valid node: two subtrees next to each other')
     }
     i++
-    let subtree: CID | null = null
+    let subtree: Cid | null = null
     if (next?.isTree()) {
       subtree = next.pointer
       i++
@@ -91,7 +104,7 @@ export const serializeNodeData = (entries: NodeEntry[]): NodeData => {
     const prefixLen = countPrefixLen(lastKey, leaf.key)
     data.e.push({
       p: prefixLen,
-      k: uint8arrays.fromString(leaf.key.slice(prefixLen), 'ascii'),
+      k: fromAscii(leaf.key.slice(prefixLen)),
       v: leaf.value,
       t: subtree,
     })
@@ -111,9 +124,9 @@ export const countPrefixLen = (a: string, b: string): number => {
   return i
 }
 
-export const cidForEntries = async (entries: NodeEntry[]): Promise<CID> => {
+export const cidForEntries = async (entries: NodeEntry[]): Promise<Cid> => {
   const data = serializeNodeData(entries)
-  return cidForCbor(data)
+  return cidForLex(data)
 }
 
 export const isValidMstKey = (str: string): boolean => {

package/src/parse.ts

@@ -1,16 +1,14 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats/cid'
-import { cborDecode, check } from '@atproto/common'
-import { RepoRecord } from '@atproto/lexicon'
+import { check } from '@atproto/common-web'
+import { decode } from '@atproto/lex-cbor'
+import { Cid, LexMap } from '@atproto/lex-data'
 import { BlockMap } from './block-map'
 import { MissingBlockError, UnexpectedObjectError } from './error'
 import { cborToLexRecord } from './util'
 
 export const getAndParseRecord = async (
   blocks: BlockMap,
-  cid: CID,
-): Promise<{ record: RepoRecord; bytes: Uint8Array }> => {
+  cid: Cid,
+): Promise<{ record: LexMap; bytes: Uint8Array }> => {
   const bytes = blocks.get(cid)
   if (!bytes) {
     throw new MissingBlockError(cid, 'record')
@@ -21,7 +19,7 @@ export const getAndParseRecord = async (
 
 export const getAndParseByDef = async <T>(
   blocks: BlockMap,
-  cid: CID,
+  cid: Cid,
   def: check.Def<T>,
 ): Promise<{ obj: T; bytes: Uint8Array }> => {
   const bytes = blocks.get(cid)
@@ -33,10 +31,10 @@ export const getAndParseByDef = async <T>(
 
 export const parseObjByDef = <T>(
   bytes: Uint8Array,
-  cid: CID,
+  cid: Cid,
   def: check.Def<T>,
 ): { obj: T; bytes: Uint8Array } => {
-  const obj = cborDecode(bytes)
+  const obj = decode(bytes)
   const res = def.schema.safeParse(obj)
   if (res.success) {
     return { obj: res.data, bytes }

package/src/readable-repo.ts

@@ -1,7 +1,4 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats/cid'
-import { RepoRecord } from '@atproto/lexicon'
+import { Cid, LexMap } from '@atproto/lex-data'
 import { MissingBlocksError } from './error'
 import log from './logger'
 import { MST } from './mst'
@@ -14,14 +11,14 @@ type Params = {
   storage: ReadableBlockstore
   data: MST
   commit: Commit
-  cid: CID
+  cid: Cid
 }
 
 export class ReadableRepo {
   storage: ReadableBlockstore
   data: MST
   commit: Commit
-  cid: CID
+  cid: Cid
 
   constructor(params: Params) {
     this.storage = params.storage
@@ -30,7 +27,7 @@ export class ReadableRepo {
     this.cid = params.cid
   }
 
-  static async load(storage: ReadableBlockstore, commitCid: CID) {
+  static async load(storage: ReadableBlockstore, commitCid: Cid) {
     const commit = await storage.readObj(commitCid, def.versionedCommit)
     const data = await MST.load(storage, commit.data)
     log.info({ did: commit.did }, 'loaded repo for')
@@ -53,8 +50,8 @@ export class ReadableRepo {
   async *walkRecords(from?: string): AsyncIterable<{
     collection: string
     rkey: string
-    cid: CID
-    record: RepoRecord
+    cid: Cid
+    record: LexMap
   }> {
     for await (const leaf of this.data.walkLeavesFrom(from ?? '')) {
       const { collection, rkey } = util.parseDataKey(leaf.key)

package/src/repo.ts

@@ -1,9 +1,7 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats/cid'
-import { TID, dataToCborBlock } from '@atproto/common'
+import { TID } from '@atproto/common-web'
 import * as crypto from '@atproto/crypto'
-import { lexToIpld } from '@atproto/lexicon'
+import { encode } from '@atproto/lex-cbor'
+import { Cid, cidForCbor } from '@atproto/lex-data'
 import { BlockMap } from './block-map'
 import { CidSet } from './cid-set'
 import { DataDiff } from './data-diff'
@@ -25,7 +23,7 @@ type Params = {
   storage: RepoStorage
   data: MST
   commit: Commit
-  cid: CID
+  cid: Cid
 }
 
 export class Repo extends ReadableRepo {
@@ -101,7 +99,7 @@ export class Repo extends ReadableRepo {
     return Repo.createFromCommit(storage, commit)
   }
 
-  static async load(storage: RepoStorage, cid?: CID) {
+  static async load(storage: RepoStorage, cid?: Cid) {
     const commitCid = cid || (await storage.getRoot())
     if (!commitCid) {
       throw new Error('No cid provided and none in storage')
@@ -171,15 +169,18 @@ export class Repo extends ReadableRepo {
       },
       keypair,
     )
-    const commitBlock = await dataToCborBlock(lexToIpld(commit))
-    if (!commitBlock.cid.equals(this.cid)) {
-      newBlocks.set(commitBlock.cid, commitBlock.bytes)
-      relevantBlocks.set(commitBlock.cid, commitBlock.bytes)
+
+    const commitBytes = encode(commit)
+    const commitCid = await cidForCbor(commitBytes)
+
+    if (!commitCid.equals(this.cid)) {
+      newBlocks.set(commitCid, commitBytes)
+      relevantBlocks.set(commitCid, commitBytes)
       removedCids.add(this.cid)
     }
 
     return {
-      cid: commitBlock.cid,
+      cid: commitCid,
       rev,
       since: this.commit.rev,
       prev: this.cid,

package/src/storage/memory-blockstore.ts

@@ -1,4 +1,4 @@
-import { CID } from 'multiformats/cid'
+import { Cid } from '@atproto/lex-data'
 import { BlockMap } from '../block-map'
 import { CommitData } from '../types'
 import { ReadableBlockstore } from './readable-blockstore'
@@ -9,7 +9,7 @@ export class MemoryBlockstore
   implements RepoStorage
 {
   blocks: BlockMap
-  root: CID | null = null
+  root: Cid | null = null
   rev: string | null = null
 
   constructor(blocks?: BlockMap) {
@@ -20,23 +20,23 @@ export class MemoryBlockstore
     }
   }
 
-  async getRoot(): Promise<CID | null> {
+  async getRoot(): Promise<Cid | null> {
     return this.root
   }
 
-  async getBytes(cid: CID): Promise<Uint8Array | null> {
+  async getBytes(cid: Cid): Promise<Uint8Array | null> {
     return this.blocks.get(cid) || null
   }
 
-  async has(cid: CID): Promise<boolean> {
+  async has(cid: Cid): Promise<boolean> {
     return this.blocks.has(cid)
   }
 
-  async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> {
+  async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
     return this.blocks.getMany(cids)
   }
 
-  async putBlock(cid: CID, block: Uint8Array): Promise<void> {
+  async putBlock(cid: Cid, block: Uint8Array): Promise<void> {
     this.blocks.set(cid, block)
   }
 
@@ -44,7 +44,7 @@ export class MemoryBlockstore
     this.blocks.addMap(blocks)
   }
 
-  async updateRoot(cid: CID, rev: string): Promise<void> {
+  async updateRoot(cid: Cid, rev: string): Promise<void> {
     this.root = cid
     this.rev = rev
   }

package/src/storage/readable-blockstore.ts

@@ -1,29 +1,26 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats/cid'
-import { check } from '@atproto/common'
-import { RepoRecord } from '@atproto/lexicon'
+import { check } from '@atproto/common-web'
+import { Cid, LexMap } from '@atproto/lex-data'
 import { BlockMap } from '../block-map'
 import { MissingBlockError } from '../error'
-import * as parse from '../parse'
+import { parseObjByDef } from '../parse'
 import { cborToLexRecord } from '../util'
 
 export abstract class ReadableBlockstore {
-  abstract getBytes(cid: CID): Promise<Uint8Array | null>
-  abstract has(cid: CID): Promise<boolean>
-  abstract getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+  abstract getBytes(cid: Cid): Promise<Uint8Array | null>
+  abstract has(cid: Cid): Promise<boolean>
+  abstract getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }>
 
   async attemptRead<T>(
-    cid: CID,
+    cid: Cid,
     def: check.Def<T>,
   ): Promise<{ obj: T; bytes: Uint8Array } | null> {
     const bytes = await this.getBytes(cid)
     if (!bytes) return null
-    return parse.parseObjByDef(bytes, cid, def)
+    return parseObjByDef(bytes, cid, def)
   }
 
   async readObjAndBytes<T>(
-    cid: CID,
+    cid: Cid,
     def: check.Def<T>,
   ): Promise<{ obj: T; bytes: Uint8Array }> {
     const read = await this.attemptRead(cid, def)
@@ -33,12 +30,12 @@ export abstract class ReadableBlockstore {
     return read
   }
 
-  async readObj<T>(cid: CID, def: check.Def<T>): Promise<T> {
+  async readObj<T>(cid: Cid, def: check.Def<T>): Promise<T> {
     const obj = await this.readObjAndBytes(cid, def)
     return obj.obj
   }
 
-  async attemptReadRecord(cid: CID): Promise<RepoRecord | null> {
+  async attemptReadRecord(cid: Cid): Promise<LexMap | null> {
     try {
       return await this.readRecord(cid)
     } catch {
@@ -46,7 +43,7 @@ export abstract class ReadableBlockstore {
     }
   }
 
-  async readRecord(cid: CID): Promise<RepoRecord> {
+  async readRecord(cid: Cid): Promise<LexMap> {
     const bytes = await this.getBytes(cid)
     if (!bytes) {
       throw new MissingBlockError(cid)

package/src/storage/sync-storage.ts

@@ -1,4 +1,4 @@
-import { CID } from 'multiformats/cid'
+import { Cid } from '@atproto/lex-data'
 import { BlockMap } from '../block-map'
 import { ReadableBlockstore } from './readable-blockstore'
 
@@ -10,13 +10,13 @@ export class SyncStorage extends ReadableBlockstore {
     super()
   }
 
-  async getBytes(cid: CID): Promise<Uint8Array | null> {
+  async getBytes(cid: Cid): Promise<Uint8Array | null> {
     const got = await this.staged.getBytes(cid)
     if (got) return got
     return this.saved.getBytes(cid)
   }
 
-  async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> {
+  async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
     const fromStaged = await this.staged.getBlocks(cids)
     const fromSaved = await this.saved.getBlocks(fromStaged.missing)
     const blocks = fromStaged.blocks
@@ -27,7 +27,7 @@ export class SyncStorage extends ReadableBlockstore {
     }
   }
 
-  async has(cid: CID): Promise<boolean> {
+  async has(cid: Cid): Promise<boolean> {
     return (await this.staged.has(cid)) || (await this.saved.has(cid))
   }
 }