@atproto/repo 0.10.2 → 0.10.3

package/src/storage/memory-blockstore.ts

@@ -1,76 +0,0 @@
-import { Cid } from '@atproto/lex-data'
-import { BlockMap } from '../block-map.js'
-import { CommitData } from '../types.js'
-import { ReadableBlockstore } from './readable-blockstore.js'
-import { RepoStorage } from './types.js'
-
-export class MemoryBlockstore
-  extends ReadableBlockstore
-  implements RepoStorage
-{
-  blocks: BlockMap
-  root: Cid | null = null
-  rev: string | null = null
-
-  constructor(blocks?: BlockMap) {
-    super()
-    this.blocks = new BlockMap()
-    if (blocks) {
-      this.blocks.addMap(blocks)
-    }
-  }
-
-  async getRoot(): Promise<Cid | null> {
-    return this.root
-  }
-
-  async getBytes(cid: Cid): Promise<Uint8Array | null> {
-    return this.blocks.get(cid) || null
-  }
-
-  async has(cid: Cid): Promise<boolean> {
-    return this.blocks.has(cid)
-  }
-
-  async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
-    return this.blocks.getMany(cids)
-  }
-
-  async putBlock(cid: Cid, block: Uint8Array): Promise<void> {
-    this.blocks.set(cid, block)
-  }
-
-  async putMany(blocks: BlockMap): Promise<void> {
-    this.blocks.addMap(blocks)
-  }
-
-  async updateRoot(cid: Cid, rev: string): Promise<void> {
-    this.root = cid
-    this.rev = rev
-  }
-
-  async applyCommit(commit: CommitData): Promise<void> {
-    this.root = commit.cid
-    const rmCids = commit.removedCids.toList()
-    for (const cid of rmCids) {
-      this.blocks.delete(cid)
-    }
-    commit.newBlocks.forEach((bytes, cid) => {
-      this.blocks.set(cid, bytes)
-    })
-  }
-
-  async sizeInBytes(): Promise<number> {
-    let total = 0
-    this.blocks.forEach((bytes) => {
-      total += bytes.byteLength
-    })
-    return total
-  }
-
-  async destroy(): Promise<void> {
-    this.blocks.clear()
-  }
-}
-
-export default MemoryBlockstore

package/src/storage/readable-blockstore.ts

@@ -1,55 +0,0 @@
-import { check } from '@atproto/common-web'
-import { Cid, LexMap } from '@atproto/lex-data'
-import { BlockMap } from '../block-map.js'
-import { MissingBlockError } from '../error.js'
-import { parseObjByDef } from '../parse.js'
-import { cborToLexRecord } from '../util.js'
-
-export abstract class ReadableBlockstore {
-  abstract getBytes(cid: Cid): Promise<Uint8Array | null>
-  abstract has(cid: Cid): Promise<boolean>
-  abstract getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }>
-
-  async attemptRead<T>(
-    cid: Cid,
-    def: check.Def<T>,
-  ): Promise<{ obj: T; bytes: Uint8Array } | null> {
-    const bytes = await this.getBytes(cid)
-    if (!bytes) return null
-    return parseObjByDef(bytes, cid, def)
-  }
-
-  async readObjAndBytes<T>(
-    cid: Cid,
-    def: check.Def<T>,
-  ): Promise<{ obj: T; bytes: Uint8Array }> {
-    const read = await this.attemptRead(cid, def)
-    if (!read) {
-      throw new MissingBlockError(cid, def.name)
-    }
-    return read
-  }
-
-  async readObj<T>(cid: Cid, def: check.Def<T>): Promise<T> {
-    const obj = await this.readObjAndBytes(cid, def)
-    return obj.obj
-  }
-
-  async attemptReadRecord(cid: Cid): Promise<LexMap | null> {
-    try {
-      return await this.readRecord(cid)
-    } catch {
-      return null
-    }
-  }
-
-  async readRecord(cid: Cid): Promise<LexMap> {
-    const bytes = await this.getBytes(cid)
-    if (!bytes) {
-      throw new MissingBlockError(cid)
-    }
-    return cborToLexRecord(bytes)
-  }
-}
-
-export default ReadableBlockstore

package/src/storage/sync-storage.ts

@@ -1,35 +0,0 @@
-import { Cid } from '@atproto/lex-data'
-import { BlockMap } from '../block-map.js'
-import { ReadableBlockstore } from './readable-blockstore.js'
-
-export class SyncStorage extends ReadableBlockstore {
-  constructor(
-    public staged: ReadableBlockstore,
-    public saved: ReadableBlockstore,
-  ) {
-    super()
-  }
-
-  async getBytes(cid: Cid): Promise<Uint8Array | null> {
-    const got = await this.staged.getBytes(cid)
-    if (got) return got
-    return this.saved.getBytes(cid)
-  }
-
-  async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
-    const fromStaged = await this.staged.getBlocks(cids)
-    const fromSaved = await this.saved.getBlocks(fromStaged.missing)
-    const blocks = fromStaged.blocks
-    blocks.addMap(fromSaved.blocks)
-    return {
-      blocks,
-      missing: fromSaved.missing,
-    }
-  }
-
-  async has(cid: Cid): Promise<boolean> {
-    return (await this.staged.has(cid)) || (await this.saved.has(cid))
-  }
-}
-
-export default SyncStorage

package/src/storage/types.ts

@@ -1,47 +0,0 @@
-import type { Readable } from 'node:stream'
-import { check } from '@atproto/common-web'
-import { Cid, LexMap } from '@atproto/lex-data'
-import { BlockMap } from '../block-map.js'
-import { CommitData } from '../types.js'
-
-export interface RepoStorage {
-  // Writable
-  getRoot(): Promise<Cid | null>
-  putBlock(cid: Cid, block: Uint8Array, rev: string): Promise<void>
-  putMany(blocks: BlockMap, rev: string): Promise<void>
-  updateRoot(cid: Cid, rev: string): Promise<void>
-  applyCommit(commit: CommitData)
-
-  // Readable
-  getBytes(cid: Cid): Promise<Uint8Array | null>
-  has(cid: Cid): Promise<boolean>
-  getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }>
-  attemptRead<T>(
-    cid: Cid,
-    def: check.Def<T>,
-  ): Promise<{ obj: T; bytes: Uint8Array } | null>
-  readObjAndBytes<T>(
-    cid: Cid,
-    def: check.Def<T>,
-  ): Promise<{ obj: T; bytes: Uint8Array }>
-  readObj<T>(cid: Cid, def: check.Def<T>): Promise<T>
-  attemptReadRecord(cid: Cid): Promise<LexMap | null>
-  readRecord(cid: Cid): Promise<LexMap>
-}
-
-// @TODO make this less node-js specific by using AsyncIterable<Uint8Array> instead of Readable
-export interface BlobStore {
-  putTemp(bytes: Uint8Array | Readable): Promise<string>
-  makePermanent(key: string, cid: Cid): Promise<void>
-  putPermanent(cid: Cid, bytes: Uint8Array | Readable): Promise<void>
-  quarantine(cid: Cid): Promise<void>
-  unquarantine(cid: Cid): Promise<void>
-  getBytes(cid: Cid): Promise<Uint8Array>
-  getStream(cid: Cid): Promise<Readable>
-  hasTemp(key: string): Promise<boolean>
-  hasStored(cid: Cid): Promise<boolean>
-  delete(cid: Cid): Promise<void>
-  deleteMany(cid: Cid[]): Promise<void>
-}
-
-export class BlobNotFoundError extends Error {}

package/src/sync/consumer.ts

@@ -1,207 +0,0 @@
-import { Cid } from '@atproto/lex-data'
-import { BlockMap } from '../block-map.js'
-import { readCarWithRoot } from '../car.js'
-import { DataDiff } from '../data-diff.js'
-import { MST } from '../mst/index.js'
-import { ReadableRepo } from '../readable-repo.js'
-import {
-  MemoryBlockstore,
-  ReadableBlockstore,
-  SyncStorage,
-} from '../storage/index.js'
-import {
-  RecordCidClaim,
-  RecordClaim,
-  VerifiedDiff,
-  VerifiedRepo,
-  def,
-} from '../types.js'
-import * as util from '../util.js'
-
-export const verifyRepoCar = async (
-  carBytes: Uint8Array,
-  did?: string,
-  signingKey?: string,
-): Promise<VerifiedRepo> => {
-  const car = await readCarWithRoot(carBytes)
-  return verifyRepo(car.blocks, car.root, did, signingKey)
-}
-
-export const verifyRepo = async (
-  blocks: BlockMap,
-  head: Cid,
-  did?: string,
-  signingKey?: string,
-  opts?: { ensureLeaves?: boolean },
-): Promise<VerifiedRepo> => {
-  const diff = await verifyDiff(null, blocks, head, did, signingKey, opts)
-  const creates = util.ensureCreates(diff.writes)
-  return {
-    creates,
-    commit: diff.commit,
-  }
-}
-
-export const verifyDiffCar = async (
-  repo: ReadableRepo | null,
-  carBytes: Uint8Array,
-  did?: string,
-  signingKey?: string,
-  opts?: { ensureLeaves?: boolean },
-): Promise<VerifiedDiff> => {
-  const car = await readCarWithRoot(carBytes)
-  return verifyDiff(repo, car.blocks, car.root, did, signingKey, opts)
-}
-
-export const verifyDiff = async (
-  repo: ReadableRepo | null,
-  updateBlocks: BlockMap,
-  updateRoot: Cid,
-  did?: string,
-  signingKey?: string,
-  opts?: { ensureLeaves?: boolean },
-): Promise<VerifiedDiff> => {
-  const { ensureLeaves = true } = opts ?? {}
-  const stagedStorage = new MemoryBlockstore(updateBlocks)
-  const updateStorage = repo
-    ? new SyncStorage(stagedStorage, repo.storage)
-    : stagedStorage
-  const updated = await verifyRepoRoot(
-    updateStorage,
-    updateRoot,
-    did,
-    signingKey,
-  )
-  const diff = await DataDiff.of(updated.data, repo?.data ?? null)
-  const writes = await util.diffToWriteDescripts(diff)
-  const newBlocks = diff.newMstBlocks
-  const leaves = updateBlocks.getMany(diff.newLeafCids.toList())
-  if (leaves.missing.length > 0 && ensureLeaves) {
-    throw new Error(`missing leaf blocks: ${leaves.missing}`)
-  }
-  newBlocks.addMap(leaves.blocks)
-  const removedCids = diff.removedCids
-  const commitCid = await newBlocks.add(updated.commit)
-  // ensure the commit cid actually changed
-  if (repo) {
-    if (commitCid.equals(repo.cid)) {
-      newBlocks.delete(commitCid)
-    } else {
-      removedCids.add(repo.cid)
-    }
-  }
-  return {
-    writes,
-    commit: {
-      cid: updated.cid,
-      rev: updated.commit.rev,
-      prev: repo?.cid ?? null,
-      since: repo?.commit.rev ?? null,
-      newBlocks,
-      relevantBlocks: newBlocks,
-      removedCids,
-    },
-  }
-}
-
-// @NOTE only verifies the root, not the repo contents
-const verifyRepoRoot = async (
-  storage: ReadableBlockstore,
-  head: Cid,
-  did?: string,
-  signingKey?: string,
-): Promise<ReadableRepo> => {
-  const repo = await ReadableRepo.load(storage, head)
-  if (did !== undefined && repo.did !== did) {
-    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
-  }
-  if (signingKey !== undefined) {
-    const validSig = await util.verifyCommitSig(repo.commit, signingKey)
-    if (!validSig) {
-      throw new RepoVerificationError(
-        `Invalid signature on commit: ${repo.cid.toString()}`,
-      )
-    }
-  }
-  return repo
-}
-
-export const verifyProofs = async (
-  proofs: Uint8Array,
-  claims: RecordCidClaim[],
-  did: string,
-  didKey: string,
-): Promise<{ verified: RecordCidClaim[]; unverified: RecordCidClaim[] }> => {
-  const car = await readCarWithRoot(proofs)
-  const blockstore = new MemoryBlockstore(car.blocks)
-  const commit = await blockstore.readObj(car.root, def.commit)
-  if (commit.did !== did) {
-    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
-  }
-  const validSig = await util.verifyCommitSig(commit, didKey)
-  if (!validSig) {
-    throw new RepoVerificationError(
-      `Invalid signature on commit: ${car.root.toString()}`,
-    )
-  }
-  const mst = MST.load(blockstore, commit.data)
-  const verified: RecordCidClaim[] = []
-  const unverified: RecordCidClaim[] = []
-  for (const claim of claims) {
-    const found = await mst.get(
-      util.formatDataKey(claim.collection, claim.rkey),
-    )
-    const record = found ? await blockstore.readObj(found, def.map) : null
-    if (claim.cid === null) {
-      if (record === null) {
-        verified.push(claim)
-      } else {
-        unverified.push(claim)
-      }
-    } else {
-      if (found?.equals(claim.cid)) {
-        verified.push(claim)
-      } else {
-        unverified.push(claim)
-      }
-    }
-  }
-  return { verified, unverified }
-}
-
-export const verifyRecords = async (
-  proofs: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<RecordClaim[]> => {
-  const car = await readCarWithRoot(proofs)
-  const blockstore = new MemoryBlockstore(car.blocks)
-  const commit = await blockstore.readObj(car.root, def.commit)
-  if (commit.did !== did) {
-    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
-  }
-  const validSig = await util.verifyCommitSig(commit, signingKey)
-  if (!validSig) {
-    throw new RepoVerificationError(
-      `Invalid signature on commit: ${car.root.toString()}`,
-    )
-  }
-  const mst = MST.load(blockstore, commit.data)
-
-  const records: RecordClaim[] = []
-  const leaves = await mst.reachableLeaves()
-  for (const leaf of leaves) {
-    const { collection, rkey } = util.parseDataKey(leaf.key)
-    const record = await blockstore.attemptReadRecord(leaf.value)
-    if (record) {
-      records.push({
-        collection,
-        rkey,
-        record,
-      })
-    }
-  }
-  return records
-}
-
-export class RepoVerificationError extends Error {}

package/src/sync/index.ts

@@ -1,2 +0,0 @@
-export * from './consumer.js'
-export * from './provider.js'

package/src/sync/provider.ts

@@ -1,67 +0,0 @@
-import { Cid } from '@atproto/lex-data'
-import { writeCarStream } from '../car.js'
-import { CidSet } from '../cid-set.js'
-import { MissingBlocksError } from '../error.js'
-import { MST } from '../mst/index.js'
-import { ReadableBlockstore, RepoStorage } from '../storage/index.js'
-import { CarBlock, RecordPath, def } from '../types.js'
-import * as util from '../util.js'
-
-// Full Repo
-// -------------
-
-export const getFullRepo = (
-  storage: RepoStorage,
-  commitCid: Cid,
-): AsyncIterable<Uint8Array> => {
-  return writeCarStream(commitCid, iterateFullRepo(storage, commitCid))
-}
-
-async function* iterateFullRepo(
-  storage: RepoStorage,
-  commitCid: Cid,
-): AsyncGenerator<CarBlock> {
-  const commit = await storage.readObjAndBytes(commitCid, def.commit)
-  yield { cid: commitCid, bytes: commit.bytes }
-  const mst = MST.load(storage, commit.obj.data)
-  for await (const block of mst.carBlockStream()) {
-    yield block
-  }
-}
-
-// Narrow slices
-// -------------
-
-export const getRecords = (
-  storage: ReadableBlockstore,
-  commitCid: Cid,
-  paths: RecordPath[],
-): AsyncIterable<Uint8Array> => {
-  return writeCarStream(
-    commitCid,
-    iterateRecordBlocks(storage, commitCid, paths),
-  )
-}
-
-async function* iterateRecordBlocks(
-  storage: ReadableBlockstore,
-  commitCid: Cid,
-  paths: RecordPath[],
-): AsyncGenerator<CarBlock> {
-  const commit = await storage.readObjAndBytes(commitCid, def.commit)
-  yield { cid: commitCid, bytes: commit.bytes }
-  const mst = MST.load(storage, commit.obj.data)
-  const cidsForPaths = await Promise.all(
-    paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))),
-  )
-  const allCids = cidsForPaths.reduce((acc, cur) => {
-    return acc.addSet(new CidSet(cur))
-  }, new CidSet())
-  const found = await storage.getBlocks(allCids.toList())
-  if (found.missing.length > 0) {
-    throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
-  }
-  for (const block of found.blocks.entries()) {
-    yield block
-  }
-}