@atproto/repo 0.0.1 → 0.1.0

package/src/storage/readable-blockstore.ts

@@ -0,0 +1,56 @@
+import { check } from '@atproto/common'
+import { RepoRecord } from '@atproto/lexicon'
+import { CID } from 'multiformats/cid'
+import BlockMap from '../block-map'
+import { MissingBlockError } from '../error'
+import * as parse from '../parse'
+import { cborToLexRecord } from '../util'
+
+export abstract class ReadableBlockstore {
+  abstract getBytes(cid: CID): Promise<Uint8Array | null>
+  abstract has(cid: CID): Promise<boolean>
+  abstract getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+
+  async attemptRead<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array } | null> {
+    const bytes = await this.getBytes(cid)
+    if (!bytes) return null
+    return parse.parseObjByDef(bytes, cid, def)
+  }
+
+  async readObjAndBytes<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array }> {
+    const read = await this.attemptRead(cid, def)
+    if (!read) {
+      throw new MissingBlockError(cid, def.name)
+    }
+    return read
+  }
+
+  async readObj<T>(cid: CID, def: check.Def<T>): Promise<T> {
+    const obj = await this.readObjAndBytes(cid, def)
+    return obj.obj
+  }
+
+  async attemptReadRecord(cid: CID): Promise<RepoRecord | null> {
+    try {
+      return await this.readRecord(cid)
+    } catch {
+      return null
+    }
+  }
+
+  async readRecord(cid: CID): Promise<RepoRecord> {
+    const bytes = await this.getBytes(cid)
+    if (!bytes) {
+      throw new MissingBlockError(cid)
+    }
+    return cborToLexRecord(bytes)
+  }
+}
+
+export default ReadableBlockstore

package/src/storage/repo-storage.ts

@@ -0,0 +1,42 @@
+import { CID } from 'multiformats/cid'
+import BlockMap from '../block-map'
+import { CommitBlockData, CommitData } from '../types'
+import ReadableBlockstore from './readable-blockstore'
+
+export abstract class RepoStorage extends ReadableBlockstore {
+  abstract getHead(forUpdate?: boolean): Promise<CID | null>
+  abstract getCommitPath(
+    latest: CID,
+    earliest: CID | null,
+  ): Promise<CID[] | null>
+  abstract getBlocksForCommits(
+    commits: CID[],
+  ): Promise<{ [commit: string]: BlockMap }>
+
+  abstract putBlock(cid: CID, block: Uint8Array): Promise<void>
+  abstract putMany(blocks: BlockMap): Promise<void>
+  abstract updateHead(cid: CID, prev: CID | null): Promise<void>
+  abstract indexCommits(commit: CommitData[]): Promise<void>
+
+  async applyCommit(commit: CommitData): Promise<void> {
+    await Promise.all([
+      this.indexCommits([commit]),
+      this.updateHead(commit.commit, commit.prev),
+    ])
+  }
+
+  async getCommits(
+    latest: CID,
+    earliest: CID | null,
+  ): Promise<CommitBlockData[] | null> {
+    const commitPath = await this.getCommitPath(latest, earliest)
+    if (!commitPath) return null
+    const blocksByCommit = await this.getBlocksForCommits(commitPath)
+    return commitPath.map((commit) => ({
+      commit,
+      blocks: blocksByCommit[commit.toString()] || new BlockMap(),
+    }))
+  }
+}
+
+export default RepoStorage

package/src/storage/sync-storage.ts

@@ -0,0 +1,35 @@
+import { CID } from 'multiformats/cid'
+import BlockMap from '../block-map'
+import ReadableBlockstore from './readable-blockstore'
+
+export class SyncStorage extends ReadableBlockstore {
+  constructor(
+    public staged: ReadableBlockstore,
+    public saved: ReadableBlockstore,
+  ) {
+    super()
+  }
+
+  async getBytes(cid: CID): Promise<Uint8Array | null> {
+    const got = await this.staged.getBytes(cid)
+    if (got) return got
+    return this.saved.getBytes(cid)
+  }
+
+  async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> {
+    const fromStaged = await this.staged.getBlocks(cids)
+    const fromSaved = await this.saved.getBlocks(fromStaged.missing)
+    const blocks = fromStaged.blocks
+    blocks.addMap(fromSaved.blocks)
+    return {
+      blocks,
+      missing: fromSaved.missing,
+    }
+  }
+
+  async has(cid: CID): Promise<boolean> {
+    return (await this.staged.has(cid)) || (await this.saved.has(cid))
+  }
+}
+
+export default SyncStorage

package/src/storage/types.ts

@@ -5,8 +5,11 @@ export interface BlobStore {
   putTemp(bytes: Uint8Array | stream.Readable): Promise<string>
   makePermanent(key: string, cid: CID): Promise<void>
   putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>
+  quarantine(cid: CID): Promise<void>
+  unquarantine(cid: CID): Promise<void>
   getBytes(cid: CID): Promise<Uint8Array>
   getStream(cid: CID): Promise<stream.Readable>
+  delete(cid: CID): Promise<void>
 }
 
 export class BlobNotFoundError extends Error {}

package/src/sync/consumer.ts

@@ -0,0 +1,137 @@
+import { CID } from 'multiformats/cid'
+import { MemoryBlockstore, RepoStorage } from '../storage'
+import Repo from '../repo'
+import * as verify from '../verify'
+import * as util from '../util'
+import { CommitData, RepoContents, WriteLog } from '../types'
+import CidSet from '../cid-set'
+import { MissingBlocksError } from '../error'
+
+// Checkouts
+// -------------
+
+export const loadCheckout = async (
+  storage: RepoStorage,
+  repoCar: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<{ root: CID; contents: RepoContents }> => {
+  const { root, blocks } = await util.readCarWithRoot(repoCar)
+  const updateStorage = new MemoryBlockstore(blocks)
+  const checkout = await verify.verifyCheckout(
+    updateStorage,
+    root,
+    did,
+    signingKey,
+  )
+
+  const checkoutBlocks = await updateStorage.getBlocks(
+    checkout.newCids.toList(),
+  )
+  if (checkoutBlocks.missing.length > 0) {
+    throw new MissingBlocksError('sync', checkoutBlocks.missing)
+  }
+  await Promise.all([
+    storage.putMany(checkoutBlocks.blocks),
+    storage.updateHead(root, null),
+  ])
+
+  return {
+    root,
+    contents: checkout.contents,
+  }
+}
+
+// Diffs
+// -------------
+
+export const loadFullRepo = async (
+  storage: RepoStorage,
+  repoCar: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<{ root: CID; writeLog: WriteLog }> => {
+  const { root, blocks } = await util.readCarWithRoot(repoCar)
+  const updateStorage = new MemoryBlockstore(blocks)
+  const updates = await verify.verifyFullHistory(
+    updateStorage,
+    root,
+    did,
+    signingKey,
+  )
+
+  const [writeLog] = await Promise.all([
+    persistUpdates(storage, updateStorage, updates),
+    storage.updateHead(root, null),
+  ])
+
+  return {
+    root,
+    writeLog,
+  }
+}
+
+export const loadDiff = async (
+  repo: Repo,
+  diffCar: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<{ root: CID; writeLog: WriteLog }> => {
+  const { root, blocks } = await util.readCarWithRoot(diffCar)
+  const updateStorage = new MemoryBlockstore(blocks)
+  const updates = await verify.verifyUpdates(
+    repo,
+    updateStorage,
+    root,
+    did,
+    signingKey,
+  )
+
+  const [writeLog] = await Promise.all([
+    persistUpdates(repo.storage, updateStorage, updates),
+    repo.storage.updateHead(root, repo.cid),
+  ])
+
+  return {
+    root,
+    writeLog,
+  }
+}
+
+// Helpers
+// -------------
+
+export const persistUpdates = async (
+  storage: RepoStorage,
+  updateStorage: RepoStorage,
+  updates: verify.VerifiedUpdate[],
+): Promise<WriteLog> => {
+  const newCids = new CidSet()
+  for (const update of updates) {
+    newCids.addSet(update.newCids)
+  }
+
+  const diffBlocks = await updateStorage.getBlocks(newCids.toList())
+  if (diffBlocks.missing.length > 0) {
+    throw new MissingBlocksError('sync', diffBlocks.missing)
+  }
+  const commits: CommitData[] = updates.map((update) => {
+    const forCommit = diffBlocks.blocks.getMany(update.newCids.toList())
+    if (forCommit.missing.length > 0) {
+      throw new MissingBlocksError('sync', forCommit.missing)
+    }
+    return {
+      commit: update.commit,
+      prev: update.prev,
+      blocks: forCommit.blocks,
+    }
+  })
+
+  await storage.indexCommits(commits)
+
+  return Promise.all(
+    updates.map((upd) =>
+      util.diffToWriteDescripts(upd.diff, diffBlocks.blocks),
+    ),
+  )
+}

package/src/sync/index.ts

@@ -0,0 +1,2 @@
+export * from './consumer'
+export * from './provider'

package/src/sync/provider.ts

@@ -0,0 +1,91 @@
+import { Commit, def, RecordPath } from '../types'
+import { BlockWriter } from '@ipld/car/writer'
+import { CID } from 'multiformats/cid'
+import CidSet from '../cid-set'
+import { MissingBlocksError } from '../error'
+import { RepoStorage } from '../storage'
+import * as util from '../util'
+import { MST } from '../mst'
+
+// Checkouts
+// -------------
+
+export const getCheckout = async (
+  storage: RepoStorage,
+  commitCid: CID,
+): Promise<Uint8Array> => {
+  return util.writeCar(commitCid, async (car: BlockWriter) => {
+    const commit = await storage.readObjAndBytes(commitCid, def.commit)
+    await car.put({ cid: commitCid, bytes: commit.bytes })
+    const mst = MST.load(storage, commit.obj.data)
+    await mst.writeToCarStream(car)
+  })
+}
+
+// Diffs
+// -------------
+
+export const getDiff = async (
+  storage: RepoStorage,
+  latest: CID,
+  earliest: CID | null,
+): Promise<Uint8Array> => {
+  return util.writeCar(latest, (car: BlockWriter) => {
+    return writeCommitsToCarStream(storage, car, latest, earliest)
+  })
+}
+
+export const getFullRepo = async (
+  storage: RepoStorage,
+  cid: CID,
+): Promise<Uint8Array> => {
+  return getDiff(storage, cid, null)
+}
+
+export const writeCommitsToCarStream = async (
+  storage: RepoStorage,
+  car: BlockWriter,
+  latest: CID,
+  earliest: CID | null,
+): Promise<void> => {
+  const commits = await storage.getCommits(latest, earliest)
+  if (commits === null) {
+    throw new Error('Could not find shared history')
+  }
+  if (commits.length === 0) return
+  for (const commit of commits) {
+    for (const entry of commit.blocks.entries()) {
+      await car.put(entry)
+    }
+  }
+}
+
+// Narrow slices
+// -------------
+
+export const getRecords = async (
+  storage: RepoStorage,
+  commitCid: CID,
+  paths: RecordPath[],
+): Promise<Uint8Array> => {
+  return util.writeCar(commitCid, async (car: BlockWriter) => {
+    const commit = await storage.readObjAndBytes(commitCid, def.commit)
+    await car.put({ cid: commitCid, bytes: commit.bytes })
+    const mst = MST.load(storage, commit.obj.data)
+    const cidsForPaths = await Promise.all(
+      paths.map((p) =>
+        mst.cidsForPath(util.formatDataKey(p.collection, p.rkey)),
+      ),
+    )
+    const allCids = cidsForPaths.reduce((acc, cur) => {
+      return acc.addSet(new CidSet(cur))
+    }, new CidSet())
+    const found = await storage.getBlocks(allCids.toList())
+    if (found.missing.length > 0) {
+      throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
+    }
+    for (const block of found.blocks.entries()) {
+      await car.put(block)
+    }
+  })
+}

package/src/types.ts

@@ -1,88 +1,127 @@
 import { z } from 'zod'
 import { BlockWriter } from '@ipld/car/writer'
-import { def as common } from '@atproto/common'
+import { schema as common, def as commonDef } from '@atproto/common'
 import { CID } from 'multiformats'
-import { DataDiff } from './mst'
+import BlockMap from './block-map'
+import { RepoRecord } from '@atproto/lexicon'
 
-const repoMeta = z.object({
+// Repo nodes
+// ---------------
+
+const unsignedCommit = z.object({
   did: z.string(),
   version: z.number(),
-  datastore: z.string(),
-})
-export type RepoMeta = z.infer<typeof repoMeta>
-
-const repoRoot = z.object({
-  meta: common.cid,
   prev: common.cid.nullable(),
-  auth_token: common.cid.nullable(),
   data: common.cid,
 })
-export type RepoRoot = z.infer<typeof repoRoot>
+export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
 const commit = z.object({
-  root: common.cid,
+  did: z.string(),
+  version: z.number(),
+  prev: common.cid.nullable(),
+  data: common.cid,
   sig: common.bytes,
 })
 export type Commit = z.infer<typeof commit>
 
-export const cidCreateOp = z.object({
-  action: z.literal('create'),
-  collection: z.string(),
-  rkey: z.string(),
-  cid: common.cid,
-})
-export type CidCreateOp = z.infer<typeof cidCreateOp>
+export const schema = {
+  ...common,
+  commit,
+}
 
-export const cidUpdateOp = z.object({
-  action: z.literal('update'),
-  collection: z.string(),
-  rkey: z.string(),
-  cid: common.cid,
-})
-export type CidUpdateOp = z.infer<typeof cidUpdateOp>
+export const def = {
+  ...commonDef,
+  commit: {
+    name: 'commit',
+    schema: schema.commit,
+  },
+}
 
-export const deleteOp = z.object({
-  action: z.literal('delete'),
-  collection: z.string(),
-  rkey: z.string(),
-})
-export type DeleteOp = z.infer<typeof deleteOp>
+// Repo Operations
+// ---------------
 
-export const cidWriteOp = z.union([cidCreateOp, cidUpdateOp, deleteOp])
-export type CidWriteOp = z.infer<typeof cidWriteOp>
+export enum WriteOpAction {
+  Create = 'create',
+  Update = 'update',
+  Delete = 'delete',
+}
 
-export const recordCreateOp = z.object({
-  action: z.literal('create'),
-  collection: z.string(),
-  rkey: z.string(),
-  value: z.any(),
-})
-export type RecordCreateOp = z.infer<typeof recordCreateOp>
+export type RecordCreateOp = {
+  action: WriteOpAction.Create
+  collection: string
+  rkey: string
+  record: RepoRecord
+}
 
-export const recordUpdateOp = z.object({
-  action: z.literal('update'),
-  collection: z.string(),
-  rkey: z.string(),
-  value: z.any(),
-})
-export type RecordUpdateOp = z.infer<typeof recordUpdateOp>
+export type RecordUpdateOp = {
+  action: WriteOpAction.Update
+  collection: string
+  rkey: string
+  record: RepoRecord
+}
 
-export const recordWriteOp = z.union([recordCreateOp, recordUpdateOp, deleteOp])
-export type RecordWriteOp = z.infer<typeof recordWriteOp>
+export type RecordDeleteOp = {
+  action: WriteOpAction.Delete
+  collection: string
+  rkey: string
+}
 
-export const def = {
-  ...common,
-  repoMeta,
-  repoRoot,
-  commit,
-  cidWriteOp,
-  recordWriteOp,
+export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp
+
+export type RecordCreateDescript = RecordCreateOp & {
+  cid: CID
 }
 
-export interface CarStreamable {
-  writeToCarStream(car: BlockWriter): Promise<void>
+export type RecordUpdateDescript = RecordUpdateOp & {
+  prev: CID
+  cid: CID
+}
+
+export type RecordDeleteDescript = RecordDeleteOp & {
+  cid: CID
+}
+
+export type RecordWriteDescript =
+  | RecordCreateDescript
+  | RecordUpdateDescript
+  | RecordDeleteDescript
+
+export type WriteLog = RecordWriteDescript[][]
+
+// Updates/Commits
+// ---------------
+
+export type CommitBlockData = {
+  commit: CID
+  blocks: BlockMap
 }
 
+export type CommitData = CommitBlockData & {
+  prev: CID | null
+}
+
+export type RepoUpdate = CommitData & {
+  ops: RecordWriteOp[]
+}
+
+export type CollectionContents = Record<string, RepoRecord>
+export type RepoContents = Record<string, CollectionContents>
+
+export type RecordPath = {
+  collection: string
+  rkey: string
+}
+
+export type RecordClaim = {
+  collection: string
+  rkey: string
+  record: RepoRecord | null
+}
+
+// DataStores
+// ---------------
+
 export type DataValue = {
   key: string
   value: CID
@@ -93,9 +132,9 @@ export interface DataStore {
   update(key: string, value: CID): Promise<DataStore>
   delete(key: string): Promise<DataStore>
   get(key: string): Promise<CID | null>
-  list(count: number, after?: string, before?: string): Promise<DataValue[]>
+  list(count?: number, after?: string, before?: string): Promise<DataValue[]>
   listWithPrefix(prefix: string, count?: number): Promise<DataValue[]>
-  diff(other: DataStore): Promise<DataDiff>
-  stage(): Promise<CID>
+  getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }>
   writeToCarStream(car: BlockWriter): Promise<void>
+  cidsForPath(key: string): Promise<CID[]>
 }