@atproto/repo 0.0.1 → 0.1.0

package/src/parse.ts

@@ -0,0 +1,44 @@
+import { check, cborDecode } from '@atproto/common'
+import { RepoRecord } from '@atproto/lexicon'
+import { CID } from 'multiformats/cid'
+import BlockMap from './block-map'
+import { MissingBlockError, UnexpectedObjectError } from './error'
+import { cborToLexRecord } from './util'
+
+export const getAndParseRecord = async (
+  blocks: BlockMap,
+  cid: CID,
+): Promise<{ record: RepoRecord; bytes: Uint8Array }> => {
+  const bytes = blocks.get(cid)
+  if (!bytes) {
+    throw new MissingBlockError(cid, 'record')
+  }
+  const record = await cborToLexRecord(bytes)
+  return { record, bytes }
+}
+
+export const getAndParseByDef = async <T>(
+  blocks: BlockMap,
+  cid: CID,
+  def: check.Def<T>,
+): Promise<{ obj: T; bytes: Uint8Array }> => {
+  const bytes = blocks.get(cid)
+  if (!bytes) {
+    throw new MissingBlockError(cid, def.name)
+  }
+  return parseObjByDef(bytes, cid, def)
+}
+
+export const parseObjByDef = <T>(
+  bytes: Uint8Array,
+  cid: CID,
+  def: check.Def<T>,
+): { obj: T; bytes: Uint8Array } => {
+  const obj = cborDecode(bytes)
+  const res = def.schema.safeParse(obj)
+  if (res.success) {
+    return { obj: res.data, bytes }
+  } else {
+    throw new UnexpectedObjectError(cid, def.name)
+  }
+}

package/src/readable-repo.ts

@@ -0,0 +1,75 @@
+import { CID } from 'multiformats/cid'
+import { Commit, def, DataStore, RepoContents } from './types'
+import { ReadableBlockstore } from './storage'
+import { MST } from './mst'
+import log from './logger'
+import * as util from './util'
+import * as parse from './parse'
+import { MissingBlocksError } from './error'
+
+type Params = {
+  storage: ReadableBlockstore
+  data: DataStore
+  commit: Commit
+  cid: CID
+}
+
+export class ReadableRepo {
+  storage: ReadableBlockstore
+  data: DataStore
+  commit: Commit
+  cid: CID
+
+  constructor(params: Params) {
+    this.storage = params.storage
+    this.data = params.data
+    this.commit = params.commit
+    this.cid = params.cid
+  }
+
+  static async load(storage: ReadableBlockstore, commitCid: CID) {
+    const commit = await storage.readObj(commitCid, def.commit)
+    const data = await MST.load(storage, commit.data)
+    log.info({ did: commit.did }, 'loaded repo for')
+    return new ReadableRepo({
+      storage,
+      data,
+      commit,
+      cid: commitCid,
+    })
+  }
+
+  get did(): string {
+    return this.commit.did
+  }
+
+  get version(): number {
+    return this.commit.version
+  }
+
+  async getRecord(collection: string, rkey: string): Promise<unknown | null> {
+    const dataKey = collection + '/' + rkey
+    const cid = await this.data.get(dataKey)
+    if (!cid) return null
+    return this.storage.readObj(cid, def.unknown)
+  }
+
+  async getContents(): Promise<RepoContents> {
+    const entries = await this.data.list()
+    const cids = entries.map((e) => e.value)
+    const { blocks, missing } = await this.storage.getBlocks(cids)
+    if (missing.length > 0) {
+      throw new MissingBlocksError('getContents record', missing)
+    }
+    const contents: RepoContents = {}
+    for (const entry of entries) {
+      const { collection, rkey } = util.parseDataKey(entry.key)
+      contents[collection] ??= {}
+      const parsed = await parse.getAndParseRecord(blocks, entry.value)
+      contents[collection][rkey] = parsed.record
+    }
+    return contents
+  }
+}
+
+export default ReadableRepo

package/src/repo.ts

@@ -1,311 +1,181 @@
 import { CID } from 'multiformats/cid'
-import { CarWriter } from '@ipld/car'
-import { BlockWriter } from '@ipld/car/writer'
+import * as crypto from '@atproto/crypto'
 import {
-  RepoRoot,
   Commit,
   def,
   DataStore,
-  RepoMeta,
   RecordCreateOp,
   RecordWriteOp,
+  CommitData,
+  WriteOpAction,
 } from './types'
-import { streamToArray } from '@atproto/common'
-import IpldStore from './blockstore/ipld-store'
-import * as auth from '@atproto/auth'
+import { RepoStorage } from './storage'
 import { MST } from './mst'
+import DataDiff from './data-diff'
 import log from './logger'
+import BlockMap from './block-map'
+import { ReadableRepo } from './readable-repo'
 import * as util from './util'
 
 type Params = {
-  blockstore: IpldStore
+  storage: RepoStorage
   data: DataStore
   commit: Commit
-  root: RepoRoot
-  meta: RepoMeta
   cid: CID
-  stagedWrites: RecordWriteOp[]
 }
 
-export class Repo {
-  blockstore: IpldStore
-  data: DataStore
-  commit: Commit
-  root: RepoRoot
-  meta: RepoMeta
-  cid: CID
-  stagedWrites: RecordWriteOp[]
+export class Repo extends ReadableRepo {
+  storage: RepoStorage
 
   constructor(params: Params) {
-    this.blockstore = params.blockstore
-    this.data = params.data
-    this.commit = params.commit
-    this.root = params.root
-    this.meta = params.meta
-    this.cid = params.cid
-    this.stagedWrites = params.stagedWrites
+    super(params)
+    this.storage = params.storage
   }
 
-  static async create(
-    blockstore: IpldStore,
+  static async formatInitCommit(
+    storage: RepoStorage,
     did: string,
-    authStore: auth.AuthStore,
-    initialRecords: RecordCreateOp[] = [],
-  ): Promise<Repo> {
-    let tokenCid: CID | null = null
-    if (!(await authStore.canSignForDid(did))) {
-      const foundUcan = await authStore.findUcan(auth.maintenanceCap(did))
-      if (foundUcan === null) {
-        throw new Error(`No valid Ucan for creating repo`)
-      }
-      tokenCid = await blockstore.stage(auth.encodeUcan(foundUcan))
-    }
-
-    let data = await MST.create(blockstore)
-    for (const write of initialRecords) {
-      const cid = await blockstore.stage(write.value)
-      const dataKey = write.collection + '/' + write.rkey
+    keypair: crypto.Keypair,
+    initialWrites: RecordCreateOp[] = [],
+  ): Promise<CommitData> {
+    const newBlocks = new BlockMap()
+
+    let data = await MST.create(storage)
+    for (const record of initialWrites) {
+      const cid = await newBlocks.add(record.record)
+      const dataKey = util.formatDataKey(record.collection, record.rkey)
       data = await data.add(dataKey, cid)
     }
-    const dataCid = await data.stage()
 
-    const meta: RepoMeta = {
-      did,
-      version: 1,
-      datastore: 'mst',
-    }
-    const metaCid = await blockstore.stage(meta)
+    const unstoredData = await data.getUnstoredBlocks()
+    newBlocks.addMap(unstoredData.blocks)
+
+    const commit = await util.signCommit(
+      {
+        did,
+        version: 2,
+        prev: null,
+        data: unstoredData.root,
+      },
+      keypair,
+    )
+    const commitCid = await newBlocks.add(commit)
 
-    const root: RepoRoot = {
-      meta: metaCid,
+    return {
+      commit: commitCid,
       prev: null,
-      auth_token: tokenCid,
-      data: dataCid,
+      blocks: newBlocks,
     }
+  }
 
-    const rootCid = await blockstore.stage(root)
-    const commit: Commit = {
-      root: rootCid,
-      sig: await authStore.sign(rootCid.bytes),
-    }
-
-    const cid = await blockstore.stage(commit)
-
-    await blockstore.saveStaged()
+  static async createFromCommit(
+    storage: RepoStorage,
+    commit: CommitData,
+  ): Promise<Repo> {
+    await storage.applyCommit(commit)
+    return Repo.load(storage, commit.commit)
+  }
 
-    log.info({ did }, `created repo`)
-    return new Repo({
-      blockstore,
-      data,
-      commit,
-      root,
-      meta,
-      cid,
-      stagedWrites: [],
-    })
+  static async create(
+    storage: RepoStorage,
+    did: string,
+    keypair: crypto.Keypair,
+    initialWrites: RecordCreateOp[] = [],
+  ): Promise<Repo> {
+    const commit = await Repo.formatInitCommit(
+      storage,
+      did,
+      keypair,
+      initialWrites,
+    )
+    return Repo.createFromCommit(storage, commit)
   }
 
-  static async load(blockstore: IpldStore, cid: CID) {
-    const commit = await blockstore.get(cid, def.commit)
-    const root = await blockstore.get(commit.root, def.repoRoot)
-    const meta = await blockstore.get(root.meta, def.repoMeta)
-    const data = await MST.load(blockstore, root.data)
-    log.info({ did: meta.did }, 'loaded repo for')
+  static async load(storage: RepoStorage, cid?: CID) {
+    const commitCid = cid || (await storage.getHead())
+    if (!commitCid) {
+      throw new Error('No cid provided and none in storage')
+    }
+    const commit = await storage.readObj(commitCid, def.commit)
+    const data = await MST.load(storage, commit.data)
+    log.info({ did: commit.did }, 'loaded repo for')
     return new Repo({
-      blockstore,
+      storage,
       data,
       commit,
-      root,
-      meta,
-      cid,
-      stagedWrites: [],
-    })
-  }
-
-  private updateRepo(params: Partial<Params>): Repo {
-    return new Repo({
-      blockstore: params.blockstore || this.blockstore,
-      data: params.data || this.data,
-      commit: params.commit || this.commit,
-      root: params.root || this.root,
-      meta: params.meta || this.meta,
-      cid: params.cid || this.cid,
-      stagedWrites: params.stagedWrites || this.stagedWrites,
+      cid: commitCid,
     })
   }
 
-  get did(): string {
-    return this.meta.did
-  }
-
-  async getRecord(collection: string, rkey: string): Promise<unknown | null> {
-    const dataKey = collection + '/' + rkey
-    const cid = await this.data.get(dataKey)
-    if (!cid) return null
-    return this.blockstore.getUnchecked(cid)
-  }
-
-  stageUpdate(write: RecordWriteOp | RecordWriteOp[]): Repo {
-    const writeArr = Array.isArray(write) ? write : [write]
-    return this.updateRepo({
-      stagedWrites: [...this.stagedWrites, ...writeArr],
-    })
-  }
+  async formatCommit(
+    toWrite: RecordWriteOp | RecordWriteOp[],
+    keypair: crypto.Keypair,
+  ): Promise<CommitData> {
+    const writes = Array.isArray(toWrite) ? toWrite : [toWrite]
+    const commitBlocks = new BlockMap()
 
-  async createCommit(
-    authStore: auth.AuthStore,
-    performUpdate?: (prev: CID, curr: CID) => Promise<CID | null>,
-  ): Promise<Repo> {
     let data = this.data
-    for (const write of this.stagedWrites) {
-      if (write.action === 'create') {
-        const cid = await this.blockstore.stage(write.value)
+    for (const write of writes) {
+      if (write.action === WriteOpAction.Create) {
+        const cid = await commitBlocks.add(write.record)
         const dataKey = write.collection + '/' + write.rkey
         data = await data.add(dataKey, cid)
-      } else if (write.action === 'update') {
-        const cid = await this.blockstore.stage(write.value)
+      } else if (write.action === WriteOpAction.Update) {
+        const cid = await commitBlocks.add(write.record)
         const dataKey = write.collection + '/' + write.rkey
         data = await data.update(dataKey, cid)
-      } else if (write.action === 'delete') {
+      } else if (write.action === WriteOpAction.Delete) {
         const dataKey = write.collection + '/' + write.rkey
         data = await data.delete(dataKey)
       }
     }
-    const token = (await authStore.canSignForDid(this.did))
-      ? null
-      : await util.ucanForOperation(this.data, data, this.did, authStore)
-    const tokenCid = token ? await this.blockstore.stage(token) : null
-
-    const dataCid = await data.stage()
-    const root: RepoRoot = {
-      meta: this.root.meta,
-      prev: this.cid,
-      auth_token: tokenCid,
-      data: dataCid,
-    }
-    const rootCid = await this.blockstore.stage(root)
-    const commit: Commit = {
-      root: rootCid,
-      sig: await authStore.sign(rootCid.bytes),
-    }
-    const commitCid = await this.blockstore.stage(commit)
-
-    if (performUpdate) {
-      const rebaseOn = await performUpdate(this.cid, commitCid)
-      if (rebaseOn) {
-        await this.blockstore.clearStaged()
-        const rebaseRepo = await Repo.load(this.blockstore, rebaseOn)
-        return rebaseRepo.createCommit(authStore, performUpdate)
-      } else {
-        await this.blockstore.saveStaged()
-      }
-    } else {
-      await this.blockstore.saveStaged()
-    }
 
-    return this.updateRepo({
-      cid: commitCid,
-      root,
-      commit,
-      data,
-      stagedWrites: [],
-    })
-  }
-
-  async revert(count: number): Promise<Repo> {
-    let revertTo = this.cid
-    for (let i = 0; i < count; i++) {
-      const commit = await this.blockstore.get(revertTo, def.commit)
-      const root = await this.blockstore.get(commit.root, def.repoRoot)
-      if (root.prev === null) {
-        throw new Error(`Could not revert ${count} commits`)
+    const unstoredData = await data.getUnstoredBlocks()
+    commitBlocks.addMap(unstoredData.blocks)
+
+    // ensure we're not missing any blocks that were removed and then readded in this commit
+    const diff = await DataDiff.of(data, this.data)
+    const found = commitBlocks.getMany(diff.newCidList())
+    if (found.missing.length > 0) {
+      const fromStorage = await this.storage.getBlocks(found.missing)
+      if (fromStorage.missing.length > 0) {
+        // this shouldn't ever happen
+        throw new Error(
+          'Could not find block for commit in Datastore or storage',
+        )
       }
-      revertTo = root.prev
+      commitBlocks.addMap(fromStorage.blocks)
     }
-    return Repo.load(this.blockstore, revertTo)
-  }
 
-  // CAR FILES
-  // -----------
-
-  async getCarNoHistory(): Promise<Uint8Array> {
-    return this.openCar((car: BlockWriter) => {
-      return this.writeCheckoutToCarStream(car)
-    })
-  }
-
-  async getDiffCar(to: CID | null): Promise<Uint8Array> {
-    return this.openCar((car: BlockWriter) => {
-      return this.writeCommitsToCarStream(car, to, this.cid)
-    })
-  }
-
-  async getFullHistory(): Promise<Uint8Array> {
-    return this.getDiffCar(null)
-  }
+    const commit = await util.signCommit(
+      {
+        did: this.did,
+        version: 2,
+        prev: this.cid,
+        data: unstoredData.root,
+      },
+      keypair,
+    )
+    const commitCid = await commitBlocks.add(commit)
 
-  private async openCar(
-    fn: (car: BlockWriter) => Promise<void>,
-  ): Promise<Uint8Array> {
-    const { writer, out } = CarWriter.create([this.cid])
-    try {
-      await fn(writer)
-    } finally {
-      writer.close()
+    return {
+      commit: commitCid,
+      prev: this.cid,
+      blocks: commitBlocks,
     }
-    return streamToArray(out)
   }
 
-  async writeCheckoutToCarStream(car: BlockWriter): Promise<void> {
-    const commit = await this.blockstore.get(this.cid, def.commit)
-    const root = await this.blockstore.get(commit.root, def.repoRoot)
-    await this.blockstore.addToCar(car, this.cid)
-    await this.blockstore.addToCar(car, commit.root)
-    await this.blockstore.addToCar(car, root.meta)
-    if (root.auth_token) {
-      await this.blockstore.addToCar(car, root.auth_token)
-    }
-    await this.data.writeToCarStream(car)
+  async applyCommit(commitData: CommitData): Promise<Repo> {
+    await this.storage.applyCommit(commitData)
+    return Repo.load(this.storage, commitData.commit)
   }
 
-  async writeCommitsToCarStream(
-    car: BlockWriter,
-    oldestCommit: CID | null,
-    recentCommit: CID,
-  ): Promise<void> {
-    const commitPath = await util.getCommitPath(
-      this.blockstore,
-      oldestCommit,
-      recentCommit,
-    )
-    if (commitPath === null) {
-      throw new Error('Could not find shared history')
-    }
-    if (commitPath.length === 0) return
-    const firstHeadInPath = await Repo.load(this.blockstore, commitPath[0])
-    // handle the first commit
-    let prevHead: Repo | null =
-      firstHeadInPath.root.prev !== null
-        ? await Repo.load(this.blockstore, firstHeadInPath.root.prev)
-        : null
-    for (const commit of commitPath) {
-      const nextHead = await Repo.load(this.blockstore, commit)
-      await this.blockstore.addToCar(car, nextHead.cid)
-      await this.blockstore.addToCar(car, nextHead.commit.root)
-      await this.blockstore.addToCar(car, nextHead.root.meta)
-      if (nextHead.root.auth_token) {
-        await this.blockstore.addToCar(car, nextHead.root.auth_token)
-      }
-      if (prevHead === null) {
-        await nextHead.data.writeToCarStream(car)
-      } else {
-        const diff = await prevHead.data.diff(nextHead.data)
-        await Promise.all(
-          diff.newCidList().map((cid) => this.blockstore.addToCar(car, cid)),
-        )
-      }
-      prevHead = nextHead
-    }
+  async applyWrites(
+    toWrite: RecordWriteOp | RecordWriteOp[],
+    keypair: crypto.Keypair,
+  ): Promise<Repo> {
+    const commit = await this.formatCommit(toWrite, keypair)
+    return this.applyCommit(commit)
   }
 }
 

package/src/storage/index.ts

@@ -1 +1,5 @@
+export * from './readable-blockstore'
+export * from './repo-storage'
+export * from './memory-blockstore'
+export * from './sync-storage'
 export * from './types'

package/src/storage/memory-blockstore.ts

@@ -0,0 +1,114 @@
+import { CID } from 'multiformats/cid'
+import { CommitData, def } from '../types'
+import BlockMap from '../block-map'
+import { MST } from '../mst'
+import DataDiff from '../data-diff'
+import { MissingCommitBlocksError } from '../error'
+import RepoStorage from './repo-storage'
+
+export class MemoryBlockstore extends RepoStorage {
+  blocks: BlockMap
+  head: CID | null = null
+
+  constructor(blocks?: BlockMap) {
+    super()
+    this.blocks = new BlockMap()
+    if (blocks) {
+      this.blocks.addMap(blocks)
+    }
+  }
+
+  async getHead(): Promise<CID | null> {
+    return this.head
+  }
+
+  async getBytes(cid: CID): Promise<Uint8Array | null> {
+    return this.blocks.get(cid) || null
+  }
+
+  async has(cid: CID): Promise<boolean> {
+    return this.blocks.has(cid)
+  }
+
+  async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> {
+    return this.blocks.getMany(cids)
+  }
+
+  async putBlock(cid: CID, block: Uint8Array): Promise<void> {
+    this.blocks.set(cid, block)
+  }
+
+  async putMany(blocks: BlockMap): Promise<void> {
+    this.blocks.addMap(blocks)
+  }
+
+  async indexCommits(commits: CommitData[]): Promise<void> {
+    commits.forEach((commit) => {
+      this.blocks.addMap(commit.blocks)
+    })
+  }
+
+  async updateHead(cid: CID, _prev: CID | null): Promise<void> {
+    this.head = cid
+  }
+
+  async applyCommit(commit: CommitData): Promise<void> {
+    this.blocks.addMap(commit.blocks)
+    this.head = commit.commit
+  }
+
+  async getCommitPath(
+    latest: CID,
+    earliest: CID | null,
+  ): Promise<CID[] | null> {
+    let curr: CID | null = latest
+    const path: CID[] = []
+    while (curr !== null) {
+      path.push(curr)
+      const commit = await this.readObj(curr, def.commit)
+      if (!earliest && commit.prev === null) {
+        return path.reverse()
+      } else if (earliest && commit.prev.equals(earliest)) {
+        return path.reverse()
+      }
+      curr = commit.prev
+    }
+    return null
+  }
+
+  async getBlocksForCommits(
+    commits: CID[],
+  ): Promise<{ [commit: string]: BlockMap }> {
+    const commitData: { [commit: string]: BlockMap } = {}
+    let prevData: MST | null = null
+    for (const commitCid of commits) {
+      const commit = await this.readObj(commitCid, def.commit)
+      const data = await MST.load(this, commit.data)
+      const diff = await DataDiff.of(data, prevData)
+      const { blocks, missing } = await this.getBlocks([
+        commitCid,
+        ...diff.newCidList(),
+      ])
+      if (missing.length > 0) {
+        throw new MissingCommitBlocksError(commitCid, missing)
+      }
+      commitData[commitCid.toString()] = blocks
+      prevData = data
+    }
+    return commitData
+  }
+
+  async sizeInBytes(): Promise<number> {
+    let total = 0
+    this.blocks.forEach((bytes) => {
+      total += bytes.byteLength
+    })
+    return total
+  }
+
+  async destroy(): Promise<void> {
+    this.blocks.clear()
+  }
+}
+
+export default MemoryBlockstore