@atproto/repo 0.0.1 → 0.2.0

package/tests/sync/checkout.test.ts

@@ -0,0 +1,75 @@
+import * as crypto from '@atproto/crypto'
+import { CidSet, Repo, RepoContents, RepoVerificationError } from '../../src'
+import { MemoryBlockstore } from '../../src/storage'
+import * as sync from '../../src/sync'
+
+import * as util from '../_util'
+import { streamToBuffer } from '@atproto/common'
+import { CarReader } from '@ipld/car/reader'
+
+describe('Checkout Sync', () => {
+  let storage: MemoryBlockstore
+  let syncStorage: MemoryBlockstore
+  let repo: Repo
+  let keypair: crypto.Keypair
+  let repoData: RepoContents
+
+  const repoDid = 'did:example:test'
+
+  beforeAll(async () => {
+    storage = new MemoryBlockstore()
+    keypair = await crypto.Secp256k1Keypair.create()
+    repo = await Repo.create(storage, repoDid, keypair)
+    syncStorage = new MemoryBlockstore()
+    const filled = await util.fillRepo(repo, keypair, 20)
+    repo = filled.repo
+    repoData = filled.data
+  })
+
+  it('sync a non-historical repo checkout', async () => {
+    const checkoutCar = await streamToBuffer(
+      sync.getCheckout(storage, repo.cid),
+    )
+    const checkout = await sync.loadCheckout(
+      syncStorage,
+      checkoutCar,
+      repoDid,
+      keypair.did(),
+    )
+    const checkoutRepo = await Repo.load(syncStorage, checkout.root)
+    const contents = await checkoutRepo.getContents()
+    expect(contents).toEqual(repoData)
+    expect(checkout.contents).toEqual(repoData)
+  })
+
+  it('does not sync unneeded blocks during checkout', async () => {
+    const commitPath = await storage.getCommitPath(repo.cid, null)
+    if (!commitPath) {
+      throw new Error('Could not get commitPath')
+    }
+    const hasGenesisCommit = await syncStorage.has(commitPath[0])
+    expect(hasGenesisCommit).toBeFalsy()
+  })
+
+  it('does not sync duplicate blocks', async () => {
+    const carBytes = await streamToBuffer(sync.getCheckout(storage, repo.cid))
+    const car = await CarReader.fromBytes(carBytes)
+    const cids = new CidSet()
+    for await (const block of car.blocks()) {
+      if (cids.has(block.cid)) {
+        throw new Error(`duplicate block: :${block.cid.toString()}`)
+      }
+      cids.add(block.cid)
+    }
+  })
+
+  it('throws on a bad signature', async () => {
+    const badRepo = await util.addBadCommit(repo, keypair)
+    const checkoutCar = await streamToBuffer(
+      sync.getCheckout(storage, badRepo.cid),
+    )
+    await expect(
+      sync.loadCheckout(syncStorage, checkoutCar, repoDid, keypair.did()),
+    ).rejects.toThrow(RepoVerificationError)
+  })
+})

package/tests/sync/diff.test.ts

@@ -0,0 +1,92 @@
+import * as crypto from '@atproto/crypto'
+import { Repo, RepoContents } from '../../src'
+import { MemoryBlockstore } from '../../src/storage'
+import * as sync from '../../src/sync'
+
+import * as util from '../_util'
+import { streamToBuffer } from '@atproto/common'
+
+describe('Diff Sync', () => {
+  let storage: MemoryBlockstore
+  let syncStorage: MemoryBlockstore
+  let repo: Repo
+  let keypair: crypto.Keypair
+  let repoData: RepoContents
+
+  const repoDid = 'did:example:test'
+
+  beforeAll(async () => {
+    storage = new MemoryBlockstore()
+    keypair = await crypto.Secp256k1Keypair.create()
+    repo = await Repo.create(storage, repoDid, keypair)
+    syncStorage = new MemoryBlockstore()
+  })
+
+  let syncRepo: Repo
+
+  it('syncs an empty repo', async () => {
+    const car = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const loaded = await sync.loadFullRepo(
+      syncStorage,
+      car,
+      repoDid,
+      keypair.did(),
+    )
+    syncRepo = await Repo.load(syncStorage, loaded.root)
+    const data = await syncRepo.data.list(10)
+    expect(data.length).toBe(0)
+  })
+
+  it('syncs a repo that is starting from scratch', async () => {
+    const filled = await util.fillRepo(repo, keypair, 100)
+    repo = filled.repo
+    repoData = filled.data
+
+    const car = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const loaded = await sync.loadFullRepo(
+      syncStorage,
+      car,
+      repoDid,
+      keypair.did(),
+    )
+    syncRepo = await Repo.load(syncStorage, loaded.root)
+    const contents = await syncRepo.getContents()
+    expect(contents).toEqual(repoData)
+    await util.verifyRepoDiff(loaded.writeLog, {}, repoData)
+  })
+
+  it('syncs a repo that is behind', async () => {
+    // add more to providers's repo & have consumer catch up
+    const beforeData = structuredClone(repoData)
+    const edited = await util.editRepo(repo, repoData, keypair, {
+      adds: 20,
+      updates: 20,
+      deletes: 20,
+    })
+    repo = edited.repo
+    repoData = edited.data
+    const diffCar = await streamToBuffer(
+      sync.getCommits(storage, repo.cid, syncRepo.cid),
+    )
+    const loaded = await sync.loadDiff(
+      syncRepo,
+      diffCar,
+      repoDid,
+      keypair.did(),
+    )
+    syncRepo = await Repo.load(syncStorage, loaded.root)
+    const contents = await syncRepo.getContents()
+    expect(contents).toEqual(repoData)
+    await util.verifyRepoDiff(loaded.writeLog, beforeData, repoData)
+  })
+
+  it('throws on a bad signature', async () => {
+    const badRepo = await util.addBadCommit(repo, keypair)
+    const diffCar = await streamToBuffer(
+      sync.getCommits(storage, badRepo.cid, syncRepo.cid),
+    )
+    await expect(
+      sync.loadDiff(syncRepo, diffCar, repoDid, keypair.did()),
+    ).rejects.toThrow('Invalid signature on commit')
+  })
+})

package/tests/sync/narrow.test.ts

@@ -0,0 +1,149 @@
+import { TID, streamToBuffer } from '@atproto/common'
+import * as crypto from '@atproto/crypto'
+import { RecordClaim, Repo, RepoContents } from '../../src'
+import { MemoryBlockstore } from '../../src/storage'
+import * as verify from '../../src/verify'
+import * as sync from '../../src/sync'
+
+import * as util from '../_util'
+
+describe('Narrow Sync', () => {
+  let storage: MemoryBlockstore
+  let repo: Repo
+  let keypair: crypto.Keypair
+  let repoData: RepoContents
+
+  const repoDid = 'did:example:test'
+
+  beforeAll(async () => {
+    storage = new MemoryBlockstore()
+    keypair = await crypto.Secp256k1Keypair.create()
+    repo = await Repo.create(storage, repoDid, keypair)
+    const filled = await util.fillRepo(repo, keypair, 5)
+    repo = filled.repo
+    repoData = filled.data
+  })
+
+  const getProofs = async (claims: RecordClaim[]) => {
+    return streamToBuffer(sync.getRecords(storage, repo.cid, claims))
+  }
+
+  const doVerify = (proofs: Uint8Array, claims: RecordClaim[]) => {
+    return verify.verifyProofs(proofs, claims, repoDid, keypair.did())
+  }
+
+  it('verifies valid records', async () => {
+    const claims = util.contentsToClaims(repoData)
+    const proofs = await getProofs(claims)
+    const results = await doVerify(proofs, claims)
+    expect(results.verified.length).toBeGreaterThan(0)
+    expect(results.verified).toEqual(claims)
+    expect(results.unverified.length).toBe(0)
+  })
+
+  it('verifies record nonexistence', async () => {
+    const claims: RecordClaim[] = [
+      {
+        collection: util.testCollections[0],
+        rkey: TID.nextStr(), // does not exist
+        record: null,
+      },
+    ]
+    const proofs = await getProofs(claims)
+    const results = await doVerify(proofs, claims)
+    expect(results.verified.length).toBeGreaterThan(0)
+    expect(results.verified).toEqual(claims)
+    expect(results.unverified.length).toBe(0)
+  })
+
+  it('does not verify a record that doesnt exist', async () => {
+    const realClaims = util.contentsToClaims(repoData)
+    const claims: RecordClaim[] = [
+      {
+        ...realClaims[0],
+        rkey: TID.nextStr(),
+      },
+    ]
+    const proofs = await getProofs(claims)
+    const results = await doVerify(proofs, claims)
+    expect(results.verified.length).toBe(0)
+    expect(results.unverified.length).toBeGreaterThan(0)
+    expect(results.unverified).toEqual(claims)
+  })
+
+  it('does not verify an invalid record at a real path', async () => {
+    const realClaims = util.contentsToClaims(repoData)
+    const claims: RecordClaim[] = [
+      {
+        ...realClaims[0],
+        record: util.generateObject(),
+      },
+    ]
+    const proofs = await getProofs(claims)
+    const results = await doVerify(proofs, claims)
+    expect(results.verified.length).toBe(0)
+    expect(results.unverified.length).toBeGreaterThan(0)
+    expect(results.unverified).toEqual(claims)
+  })
+
+  it('does not verify a delete where the record does exist', async () => {
+    const realClaims = util.contentsToClaims(repoData)
+    const claims: RecordClaim[] = [
+      {
+        collection: realClaims[0].collection,
+        rkey: realClaims[0].rkey,
+        record: null,
+      },
+    ]
+    const proofs = await getProofs(claims)
+    const results = await doVerify(proofs, claims)
+    expect(results.verified.length).toBe(0)
+    expect(results.unverified.length).toBeGreaterThan(0)
+    expect(results.unverified).toEqual(claims)
+  })
+
+  it('can determine record proofs from car file', async () => {
+    const possible = util.contentsToClaims(repoData)
+    const claims = [
+      //random sampling of records
+      possible[0],
+      possible[4],
+      possible[5],
+      possible[8],
+    ]
+    const proofs = await getProofs(claims)
+    const records = await verify.verifyRecords(proofs, repoDid, keypair.did())
+    for (const record of records) {
+      const foundClaim = claims.find(
+        (claim) =>
+          claim.collection === record.collection && claim.rkey === record.rkey,
+      )
+      if (!foundClaim) {
+        throw new Error('Could not find record for claim')
+      }
+      expect(foundClaim.record).toEqual(
+        repoData[record.collection][record.rkey],
+      )
+    }
+  })
+
+  it('verifyRecords throws on a bad signature', async () => {
+    const badRepo = await util.addBadCommit(repo, keypair)
+    const claims = util.contentsToClaims(repoData)
+    const proofs = await streamToBuffer(
+      sync.getRecords(storage, badRepo.cid, claims),
+    )
+    const fn = verify.verifyRecords(proofs, repoDid, keypair.did())
+    await expect(fn).rejects.toThrow(verify.RepoVerificationError)
+  })
+
+  it('verifyProofs throws on a bad signature', async () => {
+    const badRepo = await util.addBadCommit(repo, keypair)
+    const claims = util.contentsToClaims(repoData)
+    const proofs = await streamToBuffer(
+      sync.getRecords(storage, badRepo.cid, claims),
+    )
+    const fn = verify.verifyProofs(proofs, claims, repoDid, keypair.did())
+    await expect(fn).rejects.toThrow(verify.RepoVerificationError)
+  })
+})

package/tests/util.test.ts

@@ -0,0 +1,21 @@
+import { dataToCborBlock, wait } from '@atproto/common'
+import { writeCar } from '../src'
+
+describe('Utils', () => {
+  describe('writeCar()', () => {
+    it('propagates errors', async () => {
+      const iterate = async () => {
+        const iter = writeCar(null, async (car) => {
+          await wait(1)
+          const block = await dataToCborBlock({ test: 1 })
+          await car.put(block)
+          throw new Error('Oops!')
+        })
+        for await (const bytes of iter) {
+          // no-op
+        }
+      }
+      await expect(iterate).rejects.toThrow('Oops!')
+    })
+  })
+})