@atproto/repo 0.0.1 → 0.2.0

package/src/storage/memory-blockstore.ts

@@ -0,0 +1,133 @@
+import { CID } from 'multiformats/cid'
+import { CommitData, def, RebaseData } from '../types'
+import BlockMap from '../block-map'
+import { MST } from '../mst'
+import DataDiff from '../data-diff'
+import { MissingCommitBlocksError } from '../error'
+import RepoStorage from './repo-storage'
+import CidSet from '../cid-set'
+
+export class MemoryBlockstore extends RepoStorage {
+  blocks: BlockMap
+  head: CID | null = null
+
+  constructor(blocks?: BlockMap) {
+    super()
+    this.blocks = new BlockMap()
+    if (blocks) {
+      this.blocks.addMap(blocks)
+    }
+  }
+
+  async getHead(): Promise<CID | null> {
+    return this.head
+  }
+
+  async getBytes(cid: CID): Promise<Uint8Array | null> {
+    return this.blocks.get(cid) || null
+  }
+
+  async has(cid: CID): Promise<boolean> {
+    return this.blocks.has(cid)
+  }
+
+  async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> {
+    return this.blocks.getMany(cids)
+  }
+
+  async putBlock(cid: CID, block: Uint8Array): Promise<void> {
+    this.blocks.set(cid, block)
+  }
+
+  async putMany(blocks: BlockMap): Promise<void> {
+    this.blocks.addMap(blocks)
+  }
+
+  async indexCommits(commits: CommitData[]): Promise<void> {
+    commits.forEach((commit) => {
+      this.blocks.addMap(commit.blocks)
+    })
+  }
+
+  async updateHead(cid: CID, _prev: CID | null): Promise<void> {
+    this.head = cid
+  }
+
+  async applyCommit(commit: CommitData): Promise<void> {
+    this.blocks.addMap(commit.blocks)
+    this.head = commit.commit
+  }
+
+  async getCommitPath(
+    latest: CID,
+    earliest: CID | null,
+  ): Promise<CID[] | null> {
+    let curr: CID | null = latest
+    const path: CID[] = []
+    while (curr !== null) {
+      path.push(curr)
+      const commit = await this.readObj(curr, def.commit)
+      if (!earliest && commit.prev === null) {
+        return path.reverse()
+      } else if (earliest && commit.prev.equals(earliest)) {
+        return path.reverse()
+      }
+      curr = commit.prev
+    }
+    return null
+  }
+
+  async getBlocksForCommits(
+    commits: CID[],
+  ): Promise<{ [commit: string]: BlockMap }> {
+    const commitData: { [commit: string]: BlockMap } = {}
+    let prevData: MST | null = null
+    for (const commitCid of commits) {
+      const commit = await this.readObj(commitCid, def.commit)
+      const data = await MST.load(this, commit.data)
+      const diff = await DataDiff.of(data, prevData)
+      const { blocks, missing } = await this.getBlocks([
+        commitCid,
+        ...diff.newCidList(),
+      ])
+      if (missing.length > 0) {
+        throw new MissingCommitBlocksError(commitCid, missing)
+      }
+      commitData[commitCid.toString()] = blocks
+      prevData = data
+    }
+    return commitData
+  }
+
+  async applyRebase(rebase: RebaseData) {
+    this.putMany(rebase.blocks)
+    const allCids = new CidSet([
+      ...rebase.preservedCids,
+      ...rebase.blocks.cids(),
+    ])
+    const toDelete: CID[] = []
+    this.blocks.forEach((_bytes, cid) => {
+      if (!allCids.has(cid)) {
+        toDelete.push(cid)
+      }
+    })
+    for (const cid of toDelete) {
+      this.blocks.delete(cid)
+    }
+    await this.updateHead(rebase.commit, null)
+  }
+
+  async sizeInBytes(): Promise<number> {
+    let total = 0
+    this.blocks.forEach((bytes) => {
+      total += bytes.byteLength
+    })
+    return total
+  }
+
+  async destroy(): Promise<void> {
+    this.blocks.clear()
+  }
+}
+
+export default MemoryBlockstore

package/src/storage/readable-blockstore.ts

@@ -0,0 +1,56 @@
+import { check } from '@atproto/common'
+import { RepoRecord } from '@atproto/lexicon'
+import { CID } from 'multiformats/cid'
+import BlockMap from '../block-map'
+import { MissingBlockError } from '../error'
+import * as parse from '../parse'
+import { cborToLexRecord } from '../util'
+
+export abstract class ReadableBlockstore {
+  abstract getBytes(cid: CID): Promise<Uint8Array | null>
+  abstract has(cid: CID): Promise<boolean>
+  abstract getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+
+  async attemptRead<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array } | null> {
+    const bytes = await this.getBytes(cid)
+    if (!bytes) return null
+    return parse.parseObjByDef(bytes, cid, def)
+  }
+
+  async readObjAndBytes<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array }> {
+    const read = await this.attemptRead(cid, def)
+    if (!read) {
+      throw new MissingBlockError(cid, def.name)
+    }
+    return read
+  }
+
+  async readObj<T>(cid: CID, def: check.Def<T>): Promise<T> {
+    const obj = await this.readObjAndBytes(cid, def)
+    return obj.obj
+  }
+
+  async attemptReadRecord(cid: CID): Promise<RepoRecord | null> {
+    try {
+      return await this.readRecord(cid)
+    } catch {
+      return null
+    }
+  }
+
+  async readRecord(cid: CID): Promise<RepoRecord> {
+    const bytes = await this.getBytes(cid)
+    if (!bytes) {
+      throw new MissingBlockError(cid)
+    }
+    return cborToLexRecord(bytes)
+  }
+}
+
+export default ReadableBlockstore

package/src/storage/repo-storage.ts

@@ -0,0 +1,43 @@
+import { CID } from 'multiformats/cid'
+import BlockMap from '../block-map'
+import { CommitBlockData, CommitData, RebaseData } from '../types'
+import ReadableBlockstore from './readable-blockstore'
+
+export abstract class RepoStorage extends ReadableBlockstore {
+  abstract getHead(forUpdate?: boolean): Promise<CID | null>
+  abstract getCommitPath(
+    latest: CID,
+    earliest: CID | null,
+  ): Promise<CID[] | null>
+  abstract getBlocksForCommits(
+    commits: CID[],
+  ): Promise<{ [commit: string]: BlockMap }>
+
+  abstract putBlock(cid: CID, block: Uint8Array): Promise<void>
+  abstract putMany(blocks: BlockMap): Promise<void>
+  abstract updateHead(cid: CID, prev: CID | null): Promise<void>
+  abstract indexCommits(commit: CommitData[]): Promise<void>
+  abstract applyRebase(rebase: RebaseData): Promise<void>
+
+  async applyCommit(commit: CommitData): Promise<void> {
+    await Promise.all([
+      this.indexCommits([commit]),
+      this.updateHead(commit.commit, commit.prev),
+    ])
+  }
+
+  async getCommits(
+    latest: CID,
+    earliest: CID | null,
+  ): Promise<CommitBlockData[] | null> {
+    const commitPath = await this.getCommitPath(latest, earliest)
+    if (!commitPath) return null
+    const blocksByCommit = await this.getBlocksForCommits(commitPath)
+    return commitPath.map((commit) => ({
+      commit,
+      blocks: blocksByCommit[commit.toString()] || new BlockMap(),
+    }))
+  }
+}
+
+export default RepoStorage

package/src/storage/sync-storage.ts

@@ -0,0 +1,35 @@
+import { CID } from 'multiformats/cid'
+import BlockMap from '../block-map'
+import ReadableBlockstore from './readable-blockstore'
+
+export class SyncStorage extends ReadableBlockstore {
+  constructor(
+    public staged: ReadableBlockstore,
+    public saved: ReadableBlockstore,
+  ) {
+    super()
+  }
+
+  async getBytes(cid: CID): Promise<Uint8Array | null> {
+    const got = await this.staged.getBytes(cid)
+    if (got) return got
+    return this.saved.getBytes(cid)
+  }
+
+  async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> {
+    const fromStaged = await this.staged.getBlocks(cids)
+    const fromSaved = await this.saved.getBlocks(fromStaged.missing)
+    const blocks = fromStaged.blocks
+    blocks.addMap(fromSaved.blocks)
+    return {
+      blocks,
+      missing: fromSaved.missing,
+    }
+  }
+
+  async has(cid: CID): Promise<boolean> {
+    return (await this.staged.has(cid)) || (await this.saved.has(cid))
+  }
+}
+
+export default SyncStorage

package/src/storage/types.ts

@@ -5,8 +5,12 @@ export interface BlobStore {
   putTemp(bytes: Uint8Array | stream.Readable): Promise<string>
   makePermanent(key: string, cid: CID): Promise<void>
   putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>
+  quarantine(cid: CID): Promise<void>
+  unquarantine(cid: CID): Promise<void>
   getBytes(cid: CID): Promise<Uint8Array>
   getStream(cid: CID): Promise<stream.Readable>
+  hasStored(cid: CID): Promise<boolean>
+  delete(cid: CID): Promise<void>
 }
 
 export class BlobNotFoundError extends Error {}

package/src/sync/consumer.ts

@@ -0,0 +1,140 @@
+import { CID } from 'multiformats/cid'
+import { MemoryBlockstore, RepoStorage } from '../storage'
+import Repo from '../repo'
+import * as verify from '../verify'
+import * as util from '../util'
+import { CommitData, RepoContents, WriteLog } from '../types'
+import CidSet from '../cid-set'
+import { MissingBlocksError } from '../error'
+
+// Checkouts
+// -------------
+
+export const loadCheckout = async (
+  storage: RepoStorage,
+  repoCar: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<{ root: CID; contents: RepoContents }> => {
+  const { root, blocks } = await util.readCarWithRoot(repoCar)
+  const updateStorage = new MemoryBlockstore(blocks)
+  const checkout = await verify.verifyCheckout(
+    updateStorage,
+    root,
+    did,
+    signingKey,
+  )
+
+  const checkoutBlocks = await updateStorage.getBlocks(
+    checkout.newCids.toList(),
+  )
+  if (checkoutBlocks.missing.length > 0) {
+    throw new MissingBlocksError('sync', checkoutBlocks.missing)
+  }
+  await Promise.all([
+    storage.putMany(checkoutBlocks.blocks),
+    storage.updateHead(root, null),
+  ])
+
+  return {
+    root,
+    contents: checkout.contents,
+  }
+}
+
+// Diffs
+// -------------
+
+export const loadFullRepo = async (
+  storage: RepoStorage,
+  repoCar: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<{ root: CID; writeLog: WriteLog; repo: Repo }> => {
+  const { root, blocks } = await util.readCarWithRoot(repoCar)
+  const updateStorage = new MemoryBlockstore(blocks)
+  const updates = await verify.verifyFullHistory(
+    updateStorage,
+    root,
+    did,
+    signingKey,
+  )
+
+  const [writeLog] = await Promise.all([
+    persistUpdates(storage, updateStorage, updates),
+    storage.updateHead(root, null),
+  ])
+
+  const repo = await Repo.load(storage, root)
+
+  return {
+    root,
+    writeLog,
+    repo,
+  }
+}
+
+export const loadDiff = async (
+  repo: Repo,
+  diffCar: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<{ root: CID; writeLog: WriteLog }> => {
+  const { root, blocks } = await util.readCarWithRoot(diffCar)
+  const updateStorage = new MemoryBlockstore(blocks)
+  const updates = await verify.verifyUpdates(
+    repo,
+    updateStorage,
+    root,
+    did,
+    signingKey,
+  )
+
+  const [writeLog] = await Promise.all([
+    persistUpdates(repo.storage, updateStorage, updates),
+    repo.storage.updateHead(root, repo.cid),
+  ])
+
+  return {
+    root,
+    writeLog,
+  }
+}
+
+// Helpers
+// -------------
+
+export const persistUpdates = async (
+  storage: RepoStorage,
+  updateStorage: RepoStorage,
+  updates: verify.VerifiedUpdate[],
+): Promise<WriteLog> => {
+  const newCids = new CidSet()
+  for (const update of updates) {
+    newCids.addSet(update.newCids)
+  }
+
+  const diffBlocks = await updateStorage.getBlocks(newCids.toList())
+  if (diffBlocks.missing.length > 0) {
+    throw new MissingBlocksError('sync', diffBlocks.missing)
+  }
+  const commits: CommitData[] = updates.map((update) => {
+    const forCommit = diffBlocks.blocks.getMany(update.newCids.toList())
+    if (forCommit.missing.length > 0) {
+      throw new MissingBlocksError('sync', forCommit.missing)
+    }
+    return {
+      commit: update.commit,
+      prev: update.prev,
+      blocks: forCommit.blocks,
+    }
+  })
+
+  await storage.indexCommits(commits)
+
+  return Promise.all(
+    updates.map((upd) =>
+      util.diffToWriteDescripts(upd.diff, diffBlocks.blocks),
+    ),
+  )
+}

package/src/sync/index.ts

@@ -0,0 +1,2 @@
+export * from './consumer'
+export * from './provider'

package/src/sync/provider.ts

@@ -0,0 +1,91 @@
+import { def, RecordPath } from '../types'
+import { BlockWriter } from '@ipld/car/writer'
+import { CID } from 'multiformats/cid'
+import CidSet from '../cid-set'
+import { MissingBlocksError } from '../error'
+import { RepoStorage } from '../storage'
+import * as util from '../util'
+import { MST } from '../mst'
+
+// Checkouts
+// -------------
+
+export const getCheckout = (
+  storage: RepoStorage,
+  commitCid: CID,
+): AsyncIterable<Uint8Array> => {
+  return util.writeCar(commitCid, async (car: BlockWriter) => {
+    const commit = await storage.readObjAndBytes(commitCid, def.commit)
+    await car.put({ cid: commitCid, bytes: commit.bytes })
+    const mst = MST.load(storage, commit.obj.data)
+    await mst.writeToCarStream(car)
+  })
+}
+
+// Commits
+// -------------
+
+export const getCommits = (
+  storage: RepoStorage,
+  latest: CID,
+  earliest: CID | null,
+): AsyncIterable<Uint8Array> => {
+  return util.writeCar(latest, (car: BlockWriter) => {
+    return writeCommitsToCarStream(storage, car, latest, earliest)
+  })
+}
+
+export const getFullRepo = (
+  storage: RepoStorage,
+  cid: CID,
+): AsyncIterable<Uint8Array> => {
+  return getCommits(storage, cid, null)
+}
+
+export const writeCommitsToCarStream = async (
+  storage: RepoStorage,
+  car: BlockWriter,
+  latest: CID,
+  earliest: CID | null,
+): Promise<void> => {
+  const commits = await storage.getCommits(latest, earliest)
+  if (commits === null) {
+    throw new Error('Could not find shared history')
+  }
+  if (commits.length === 0) return
+  for (const commit of commits) {
+    for (const entry of commit.blocks.entries()) {
+      await car.put(entry)
+    }
+  }
+}
+
+// Narrow slices
+// -------------
+
+export const getRecords = (
+  storage: RepoStorage,
+  commitCid: CID,
+  paths: RecordPath[],
+): AsyncIterable<Uint8Array> => {
+  return util.writeCar(commitCid, async (car: BlockWriter) => {
+    const commit = await storage.readObjAndBytes(commitCid, def.commit)
+    await car.put({ cid: commitCid, bytes: commit.bytes })
+    const mst = MST.load(storage, commit.obj.data)
+    const cidsForPaths = await Promise.all(
+      paths.map((p) =>
+        mst.cidsForPath(util.formatDataKey(p.collection, p.rkey)),
+      ),
+    )
+    const allCids = cidsForPaths.reduce((acc, cur) => {
+      return acc.addSet(new CidSet(cur))
+    }, new CidSet())
+    const found = await storage.getBlocks(allCids.toList())
+    if (found.missing.length > 0) {
+      throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
+    }
+    for (const block of found.blocks.entries()) {
+      await car.put(block)
+    }
+  })
+}