@atproto/repo 0.10.2 → 0.10.3

package/src/mst/util.ts

@@ -1,160 +0,0 @@
-import { sha256 } from '@atproto/crypto'
-import { cidForLex } from '@atproto/lex-cbor'
-import { Cid } from '@atproto/lex-data'
-import { ReadableBlockstore } from '../storage/index.js'
-import { Leaf, MST, MstOpts, NodeData, NodeEntry } from './mst.js'
-
-function toAscii(bytes: Uint8Array): string {
-  let string = ''
-  for (let i = 0; i < bytes.length; i++) {
-    string += String.fromCharCode(bytes[i])
-  }
-  return string
-}
-
-function fromAscii(str: string): Uint8Array<ArrayBuffer> {
-  const bytes = new Uint8Array(str.length)
-  for (let i = 0; i < str.length; i++) {
-    bytes[i] = str.charCodeAt(i)
-  }
-  return bytes
-}
-
-export const leadingZerosOnHash = async (key: string | Uint8Array) => {
-  const hash = await sha256(key)
-  let leadingZeros = 0
-  for (let i = 0; i < hash.length; i++) {
-    const byte = hash[i]
-    if (byte < 64) leadingZeros++
-    if (byte < 16) leadingZeros++
-    if (byte < 4) leadingZeros++
-    if (byte === 0) {
-      leadingZeros++
-    } else {
-      break
-    }
-  }
-  return leadingZeros
-}
-
-export const layerForEntries = async (
-  entries: NodeEntry[],
-): Promise<number | null> => {
-  const firstLeaf = entries.find((entry) => entry.isLeaf())
-  if (!firstLeaf || firstLeaf.isTree()) return null
-  return await leadingZerosOnHash(firstLeaf.key)
-}
-
-export const deserializeNodeData = async (
-  storage: ReadableBlockstore,
-  data: NodeData,
-  opts?: Partial<MstOpts>,
-): Promise<NodeEntry[]> => {
-  const { layer } = opts || {}
-  const entries: NodeEntry[] = []
-  if (data.l !== null) {
-    entries.push(
-      await MST.load(storage, data.l, {
-        layer: layer ? layer - 1 : undefined,
-      }),
-    )
-  }
-  let lastKey = ''
-  for (const entry of data.e) {
-    const keyStr = toAscii(entry.k)
-    const key = `${lastKey.slice(0, entry.p)}${keyStr}`
-    ensureValidMstKey(key)
-    entries.push(new Leaf(key, entry.v))
-    lastKey = key
-    if (entry.t !== null) {
-      entries.push(
-        await MST.load(storage, entry.t, {
-          layer: layer ? layer - 1 : undefined,
-        }),
-      )
-    }
-  }
-  return entries
-}
-
-export const serializeNodeData = (entries: NodeEntry[]): NodeData => {
-  const data: NodeData = {
-    l: null,
-    e: [],
-  }
-  let i = 0
-  if (entries[0]?.isTree()) {
-    i++
-    data.l = entries[0].pointer
-  }
-  let lastKey = ''
-  while (i < entries.length) {
-    const leaf = entries[i]
-    const next = entries[i + 1]
-    if (!leaf.isLeaf()) {
-      throw new Error('Not a valid node: two subtrees next to each other')
-    }
-    i++
-    let subtree: Cid | null = null
-    if (next?.isTree()) {
-      subtree = next.pointer
-      i++
-    }
-    ensureValidMstKey(leaf.key)
-    const prefixLen = countPrefixLen(lastKey, leaf.key)
-    data.e.push({
-      p: prefixLen,
-      k: fromAscii(leaf.key.slice(prefixLen)),
-      v: leaf.value,
-      t: subtree,
-    })
-
-    lastKey = leaf.key
-  }
-  return data
-}
-
-export const countPrefixLen = (a: string, b: string): number => {
-  let i
-  for (i = 0; i < a.length; i++) {
-    if (a[i] !== b[i]) {
-      break
-    }
-  }
-  return i
-}
-
-export const cidForEntries = async (entries: NodeEntry[]): Promise<Cid> => {
-  const data = serializeNodeData(entries)
-  return cidForLex(data)
-}
-
-export const isValidMstKey = (str: string): boolean => {
-  const split = str.split('/')
-  return (
-    str.length <= 1024 &&
-    split.length === 2 &&
-    split[0].length > 0 &&
-    split[1].length > 0 &&
-    isValidChars(split[0]) &&
-    isValidChars(split[1])
-  )
-}
-
-export const validCharsRegex = /^[a-zA-Z0-9_~\-:.]*$/
-
-export const isValidChars = (str: string): boolean => {
-  return str.match(validCharsRegex) !== null
-}
-
-export const ensureValidMstKey = (str: string) => {
-  if (!isValidMstKey(str)) {
-    throw new InvalidMstKeyError(str)
-  }
-}
-
-export class InvalidMstKeyError extends Error {
-  constructor(public key: string) {
-    super(`Not a valid MST key: ${key}`)
-  }
-}

package/src/mst/walker.ts

@@ -1,118 +0,0 @@
-import { MST, NodeEntry } from './mst.js'
-
-type WalkerStatusDone = {
-  done: true
-}
-
-type WalkerStatusProgress = {
-  done: false
-  curr: NodeEntry
-  walking: MST | null // walking set to null if `curr` is the root of the tree
-  index: number
-}
-
-type WalkerStatus = WalkerStatusDone | WalkerStatusProgress
-
-export class MstWalker {
-  stack: WalkerStatus[] = []
-  status: WalkerStatus
-
-  constructor(public root: MST) {
-    this.status = {
-      done: false,
-      curr: root,
-      walking: null,
-      index: 0,
-    }
-  }
-
-  // return the current layer of the node you are walking
-  layer(): number {
-    if (this.status.done) {
-      throw new Error('Walk is done')
-    }
-    if (this.status.walking) {
-      return this.status.walking.layer ?? 0
-    }
-    // if curr is the root of the tree, add 1
-    if (this.status.curr.isTree()) {
-      return (this.status.curr.layer ?? 0) + 1
-    }
-    throw new Error('Could not identify layer of walk')
-  }
-
-  // move to the next node in the subtree, skipping over the subtree
-  async stepOver(): Promise<void> {
-    if (this.status.done) return
-    // if stepping over the root of the node, we're done
-    if (this.status.walking === null) {
-      this.status = { done: true }
-      return
-    }
-    const entries = await this.status.walking.getEntries()
-    this.status.index++
-    const next = entries[this.status.index]
-    if (!next) {
-      const popped = this.stack.pop()
-      if (!popped) {
-        this.status = { done: true }
-        return
-      } else {
-        this.status = popped
-        await this.stepOver()
-        return
-      }
-    } else {
-      this.status.curr = next
-    }
-  }
-
-  // step into a subtree, throws if currently pointed at a leaf
-  async stepInto(): Promise<void> {
-    if (this.status.done) return
-    // edge case for very start of walk
-    if (this.status.walking === null) {
-      if (!this.status.curr.isTree()) {
-        throw new Error('The root of the tree cannot be a leaf')
-      }
-      const next = await this.status.curr.atIndex(0)
-      if (!next) {
-        this.status = { done: true }
-      } else {
-        this.status = {
-          done: false,
-          walking: this.status.curr,
-          curr: next,
-          index: 0,
-        }
-      }
-      return
-    }
-    if (!this.status.curr.isTree()) {
-      throw new Error('No tree at pointer, cannot step into')
-    }
-
-    const next = await this.status.curr.atIndex(0)
-    if (!next) {
-      throw new Error(
-        'Tried to step into a node with 0 entries which is invalid',
-      )
-    }
-
-    this.stack.push({ ...this.status })
-    this.status.walking = this.status.curr
-    this.status.curr = next
-    this.status.index = 0
-  }
-
-  // advance the pointer to the next node in the tree,
-  // stepping into the current node if necessary
-  async advance(): Promise<void> {
-    if (this.status.done) return
-    if (this.status.curr.isLeaf()) {
-      await this.stepOver()
-    } else {
-      await this.stepInto()
-    }
-  }
-}

package/src/parse.ts

@@ -1,44 +0,0 @@
-import { check } from '@atproto/common-web'
-import { decode } from '@atproto/lex-cbor'
-import { Cid, LexMap } from '@atproto/lex-data'
-import { BlockMap } from './block-map.js'
-import { MissingBlockError, UnexpectedObjectError } from './error.js'
-import { cborToLexRecord } from './util.js'
-
-export const getAndParseRecord = async (
-  blocks: BlockMap,
-  cid: Cid,
-): Promise<{ record: LexMap; bytes: Uint8Array }> => {
-  const bytes = blocks.get(cid)
-  if (!bytes) {
-    throw new MissingBlockError(cid, 'record')
-  }
-  const record = cborToLexRecord(bytes)
-  return { record, bytes }
-}
-
-export const getAndParseByDef = async <T>(
-  blocks: BlockMap,
-  cid: Cid,
-  def: check.Def<T>,
-): Promise<{ obj: T; bytes: Uint8Array }> => {
-  const bytes = blocks.get(cid)
-  if (!bytes) {
-    throw new MissingBlockError(cid, def.name)
-  }
-  return parseObjByDef(bytes, cid, def)
-}
-
-export const parseObjByDef = <T>(
-  bytes: Uint8Array,
-  cid: Cid,
-  def: check.Def<T>,
-): { obj: T; bytes: Uint8Array } => {
-  const obj = decode(bytes)
-  const res = def.schema.safeParse(obj)
-  if (res.success) {
-    return { obj: res.data, bytes }
-  } else {
-    throw new UnexpectedObjectError(cid, def.name)
-  }
-}

package/src/readable-repo.ts

@@ -1,86 +0,0 @@
-import { Cid, LexMap } from '@atproto/lex-data'
-import { MissingBlocksError } from './error.js'
-import log from './logger.js'
-import { MST } from './mst/index.js'
-import * as parse from './parse.js'
-import { ReadableBlockstore } from './storage/index.js'
-import { Commit, RepoContents, def } from './types.js'
-import * as util from './util.js'
-
-type Params = {
-  storage: ReadableBlockstore
-  data: MST
-  commit: Commit
-  cid: Cid
-}
-
-export class ReadableRepo {
-  storage: ReadableBlockstore
-  data: MST
-  commit: Commit
-  cid: Cid
-
-  constructor(params: Params) {
-    this.storage = params.storage
-    this.data = params.data
-    this.commit = params.commit
-    this.cid = params.cid
-  }
-
-  static async load(storage: ReadableBlockstore, commitCid: Cid) {
-    const commit = await storage.readObj(commitCid, def.versionedCommit)
-    const data = await MST.load(storage, commit.data)
-    log.info({ did: commit.did }, 'loaded repo for')
-    return new ReadableRepo({
-      storage,
-      data,
-      commit: util.ensureV3Commit(commit),
-      cid: commitCid,
-    })
-  }
-
-  get did(): string {
-    return this.commit.did
-  }
-
-  get version(): number {
-    return this.commit.version
-  }
-
-  async *walkRecords(from?: string): AsyncIterable<{
-    collection: string
-    rkey: string
-    cid: Cid
-    record: LexMap
-  }> {
-    for await (const leaf of this.data.walkLeavesFrom(from ?? '')) {
-      const { collection, rkey } = util.parseDataKey(leaf.key)
-      const record = await this.storage.readRecord(leaf.value)
-      yield { collection, rkey, cid: leaf.value, record }
-    }
-  }
-
-  async getRecord(collection: string, rkey: string): Promise<unknown | null> {
-    const dataKey = collection + '/' + rkey
-    const cid = await this.data.get(dataKey)
-    if (!cid) return null
-    return this.storage.readObj(cid, def.unknown)
-  }
-
-  async getContents(): Promise<RepoContents> {
-    const entries = await this.data.list()
-    const cids = entries.map((e) => e.value)
-    const { blocks, missing } = await this.storage.getBlocks(cids)
-    if (missing.length > 0) {
-      throw new MissingBlocksError('getContents record', missing)
-    }
-    const contents: RepoContents = {}
-    for (const entry of entries) {
-      const { collection, rkey } = util.parseDataKey(entry.key)
-      contents[collection] ??= {}
-      const parsed = await parse.getAndParseRecord(blocks, entry.value)
-      contents[collection][rkey] = parsed.record
-    }
-    return contents
-  }
-}

package/src/repo.ts

@@ -1,236 +0,0 @@
-import { TID } from '@atproto/common-web'
-import * as crypto from '@atproto/crypto'
-import { encode } from '@atproto/lex-cbor'
-import { Cid, cidForCbor } from '@atproto/lex-data'
-import { BlockMap } from './block-map.js'
-import { CidSet } from './cid-set.js'
-import { DataDiff } from './data-diff.js'
-import log from './logger.js'
-import { MST } from './mst/index.js'
-import { ReadableRepo } from './readable-repo.js'
-import { RepoStorage } from './storage/index.js'
-import {
-  Commit,
-  CommitData,
-  RecordCreateOp,
-  RecordWriteOp,
-  WriteOpAction,
-  def,
-} from './types.js'
-import * as util from './util.js'
-
-type Params = {
-  storage: RepoStorage
-  data: MST
-  commit: Commit
-  cid: Cid
-}
-
-export class Repo extends ReadableRepo {
-  storage: RepoStorage
-
-  constructor(params: Params) {
-    super(params)
-    this.storage = params.storage
-  }
-
-  static async formatInitCommit(
-    storage: RepoStorage,
-    did: string,
-    keypair: crypto.Keypair,
-    initialWrites: RecordCreateOp[] = [],
-    revOverride?: string,
-  ): Promise<CommitData> {
-    const newBlocks = new BlockMap()
-
-    let data = await MST.create(storage)
-    for (const record of initialWrites) {
-      const cid = await newBlocks.add(record.record)
-      const dataKey = util.formatDataKey(record.collection, record.rkey)
-      data = await data.add(dataKey, cid)
-    }
-    const dataCid = await data.getPointer()
-    const diff = await DataDiff.of(data, null)
-    newBlocks.addMap(diff.newMstBlocks)
-
-    const rev = revOverride ?? TID.nextStr()
-    const commit = await util.signCommit(
-      {
-        did,
-        version: 3,
-        rev,
-        prev: null, // added for backwards compatibility with v2
-        data: dataCid,
-      },
-      keypair,
-    )
-    const commitCid = await newBlocks.add(commit)
-    return {
-      cid: commitCid,
-      rev,
-      since: null,
-      prev: null,
-      newBlocks,
-      relevantBlocks: newBlocks,
-      removedCids: diff.removedCids,
-    }
-  }
-
-  static async createFromCommit(
-    storage: RepoStorage,
-    commit: CommitData,
-  ): Promise<Repo> {
-    await storage.applyCommit(commit)
-    return Repo.load(storage, commit.cid)
-  }
-
-  static async create(
-    storage: RepoStorage,
-    did: string,
-    keypair: crypto.Keypair,
-    initialWrites: RecordCreateOp[] = [],
-  ): Promise<Repo> {
-    const commit = await Repo.formatInitCommit(
-      storage,
-      did,
-      keypair,
-      initialWrites,
-    )
-    return Repo.createFromCommit(storage, commit)
-  }
-
-  static async load(storage: RepoStorage, cid?: Cid) {
-    const commitCid = cid || (await storage.getRoot())
-    if (!commitCid) {
-      throw new Error('No cid provided and none in storage')
-    }
-    const commit = await storage.readObj(commitCid, def.versionedCommit)
-    const data = await MST.load(storage, commit.data)
-    log.info({ did: commit.did }, 'loaded repo for')
-    return new Repo({
-      storage,
-      data,
-      commit: util.ensureV3Commit(commit),
-      cid: commitCid,
-    })
-  }
-
-  async formatCommit(
-    toWrite: RecordWriteOp | RecordWriteOp[],
-    keypair: crypto.Keypair,
-  ): Promise<CommitData> {
-    const writes = Array.isArray(toWrite) ? toWrite : [toWrite]
-    const leaves = new BlockMap()
-
-    let data = this.data
-    for (const write of writes) {
-      if (write.action === WriteOpAction.Create) {
-        const cid = await leaves.add(write.record)
-        const dataKey = write.collection + '/' + write.rkey
-        data = await data.add(dataKey, cid)
-      } else if (write.action === WriteOpAction.Update) {
-        const cid = await leaves.add(write.record)
-        const dataKey = write.collection + '/' + write.rkey
-        data = await data.update(dataKey, cid)
-      } else if (write.action === WriteOpAction.Delete) {
-        const dataKey = write.collection + '/' + write.rkey
-        data = await data.delete(dataKey)
-      }
-    }
-
-    const dataCid = await data.getPointer()
-    const diff = await DataDiff.of(data, this.data)
-    const newBlocks = diff.newMstBlocks
-    const removedCids = diff.removedCids
-
-    const proofs = await Promise.all(
-      writes.map((op) =>
-        data.getCoveringProof(util.formatDataKey(op.collection, op.rkey)),
-      ),
-    )
-    const relevantBlocks = new BlockMap()
-    for (const proof of proofs) relevantBlocks.addMap(proof)
-
-    const addedLeaves = leaves.getMany(diff.newLeafCids.toList())
-    if (addedLeaves.missing.length > 0) {
-      throw new Error(`Missing leaf blocks: ${addedLeaves.missing}`)
-    }
-    newBlocks.addMap(addedLeaves.blocks)
-    relevantBlocks.addMap(addedLeaves.blocks)
-
-    const rev = TID.nextStr(this.commit.rev)
-    const commit = await util.signCommit(
-      {
-        did: this.did,
-        version: 3,
-        rev,
-        prev: null, // added for backwards compatibility with v2
-        data: dataCid,
-      },
-      keypair,
-    )
-
-    const commitBytes = encode(commit)
-    const commitCid = await cidForCbor(commitBytes)
-
-    if (!commitCid.equals(this.cid)) {
-      newBlocks.set(commitCid, commitBytes)
-      relevantBlocks.set(commitCid, commitBytes)
-      removedCids.add(this.cid)
-    }
-
-    return {
-      cid: commitCid,
-      rev,
-      since: this.commit.rev,
-      prev: this.cid,
-      newBlocks,
-      relevantBlocks,
-      removedCids,
-    }
-  }
-
-  async applyCommit(commitData: CommitData): Promise<Repo> {
-    await this.storage.applyCommit(commitData)
-    return Repo.load(this.storage, commitData.cid)
-  }
-
-  async applyWrites(
-    toWrite: RecordWriteOp | RecordWriteOp[],
-    keypair: crypto.Keypair,
-  ): Promise<Repo> {
-    const commit = await this.formatCommit(toWrite, keypair)
-    return this.applyCommit(commit)
-  }
-
-  async formatResignCommit(rev: string, keypair: crypto.Keypair) {
-    const commit = await util.signCommit(
-      {
-        did: this.did,
-        version: 3,
-        rev,
-        prev: null, // added for backwards compatibility with v2
-        data: this.commit.data,
-      },
-      keypair,
-    )
-    const newBlocks = new BlockMap()
-    const commitCid = await newBlocks.add(commit)
-    return {
-      cid: commitCid,
-      rev,
-      since: null,
-      prev: null,
-      newBlocks,
-      relevantBlocks: newBlocks,
-      removedCids: new CidSet([this.cid]),
-    }
-  }
-
-  async resignCommit(rev: string, keypair: crypto.Keypair) {
-    const formatted = await this.formatResignCommit(rev, keypair)
-    return this.applyCommit(formatted)
-  }
-}
-
-export default Repo

package/src/storage/index.ts

@@ -1,4 +0,0 @@
-export * from './readable-blockstore.js'
-export * from './memory-blockstore.js'
-export * from './sync-storage.js'
-export * from './types.js'