@atproto/repo 0.8.13 → 0.9.1

package/src/sync/provider.ts

@@ -1,10 +1,10 @@
-import { CID } from 'multiformats/cid'
+import { Cid } from '@atproto/lex-data'
 import { writeCarStream } from '../car'
 import { CidSet } from '../cid-set'
 import { MissingBlocksError } from '../error'
 import { MST } from '../mst'
 import { ReadableBlockstore, RepoStorage } from '../storage'
-import { RecordPath, def } from '../types'
+import { CarBlock, RecordPath, def } from '../types'
 import * as util from '../util'
 
 // Full Repo
@@ -12,12 +12,15 @@ import * as util from '../util'
 
 export const getFullRepo = (
   storage: RepoStorage,
-  commitCid: CID,
+  commitCid: Cid,
 ): AsyncIterable<Uint8Array> => {
   return writeCarStream(commitCid, iterateFullRepo(storage, commitCid))
 }
 
-async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
+async function* iterateFullRepo(
+  storage: RepoStorage,
+  commitCid: Cid,
+): AsyncGenerator<CarBlock> {
   const commit = await storage.readObjAndBytes(commitCid, def.commit)
   yield { cid: commitCid, bytes: commit.bytes }
   const mst = MST.load(storage, commit.obj.data)
@@ -31,7 +34,7 @@ async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
 
 export const getRecords = (
   storage: ReadableBlockstore,
-  commitCid: CID,
+  commitCid: Cid,
   paths: RecordPath[],
 ): AsyncIterable<Uint8Array> => {
   return writeCarStream(
@@ -42,9 +45,9 @@ export const getRecords = (
 
 async function* iterateRecordBlocks(
   storage: ReadableBlockstore,
-  commitCid: CID,
+  commitCid: Cid,
   paths: RecordPath[],
-) {
+): AsyncGenerator<CarBlock> {
   const commit = await storage.readObjAndBytes(commitCid, def.commit)
   yield { cid: commitCid, bytes: commit.bytes }
   const mst = MST.load(storage, commit.obj.data)

package/src/types.ts

@@ -1,41 +1,50 @@
-import { CID } from 'multiformats'
 import { z } from 'zod'
-import { schema as common } from '@atproto/common'
-import { def as commonDef } from '@atproto/common-web'
-import { RepoRecord } from '@atproto/lexicon'
+import { Cid, LexMap, ifCid } from '@atproto/lex-data'
+import { NsidString, RecordKeyString } from '@atproto/syntax'
 import { BlockMap } from './block-map'
 import { CidSet } from './cid-set'
 
 // Repo nodes
 // ---------------
 
+const cidSchema = z.unknown().transform((input, ctx): Cid => {
+  const cid = ifCid(input)
+  if (cid) return cid
+
+  ctx.addIssue({
+    code: z.ZodIssueCode.custom,
+    message: 'Not a valid CID',
+  })
+  return z.NEVER
+})
+
 const unsignedCommit = z.object({
   did: z.string(),
   version: z.literal(3),
-  data: common.cid,
+  data: cidSchema,
   rev: z.string(),
   // `prev` added for backwards compatibility with v2, no requirement of keeping around history
-  prev: common.cid.nullable(),
+  prev: cidSchema.nullable(),
 })
 export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
 const commit = z.object({
   did: z.string(),
   version: z.literal(3),
-  data: common.cid,
+  data: cidSchema,
   rev: z.string(),
-  prev: common.cid.nullable(),
-  sig: common.bytes,
+  prev: cidSchema.nullable(),
+  sig: z.instanceof(Uint8Array),
 })
 export type Commit = z.infer<typeof commit>
 
 const legacyV2Commit = z.object({
   did: z.string(),
   version: z.literal(2),
-  data: common.cid,
+  data: cidSchema,
   rev: z.string().optional(),
-  prev: common.cid.nullable(),
-  sig: common.bytes,
+  prev: cidSchema.nullable(),
+  sig: z.instanceof(Uint8Array),
 })
 export type LegacyV2Commit = z.infer<typeof legacyV2Commit>
 
@@ -46,14 +55,46 @@ const versionedCommit = z.discriminatedUnion('version', [
 export type VersionedCommit = z.infer<typeof versionedCommit>
 
 export const schema = {
-  ...common,
+  cid: cidSchema,
+  carHeader: z.object({
+    version: z.literal(1),
+    roots: z.array(cidSchema),
+  }),
+  bytes: z.instanceof(Uint8Array),
+  string: z.string(),
+  array: z.array(z.unknown()),
+  map: z.record(z.string(), z.unknown()),
+  unknown: z.unknown(),
   commit,
   legacyV2Commit,
   versionedCommit,
 }
 
 export const def = {
-  ...commonDef,
+  cid: {
+    name: 'cid',
+    schema: schema.cid,
+  },
+  carHeader: {
+    name: 'CAR header',
+    schema: schema.carHeader,
+  },
+  bytes: {
+    name: 'bytes',
+    schema: schema.bytes,
+  },
+  string: {
+    name: 'string',
+    schema: schema.string,
+  },
+  map: {
+    name: 'map',
+    schema: schema.map,
+  },
+  unknown: {
+    name: 'unknown',
+    schema: schema.unknown,
+  },
   commit: {
     name: 'commit',
     schema: schema.commit,
@@ -75,46 +116,46 @@ export enum WriteOpAction {
 
 export type RecordCreateOp = {
   action: WriteOpAction.Create
-  collection: string
-  rkey: string
-  record: RepoRecord
+  collection: NsidString
+  rkey: RecordKeyString
+  record: LexMap
 }
 
 export type RecordUpdateOp = {
   action: WriteOpAction.Update
-  collection: string
-  rkey: string
-  record: RepoRecord
+  collection: NsidString
+  rkey: RecordKeyString
+  record: LexMap
 }
 
 export type RecordDeleteOp = {
   action: WriteOpAction.Delete
-  collection: string
-  rkey: string
+  collection: NsidString
+  rkey: RecordKeyString
 }
 
 export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp
 
 export type RecordCreateDescript = {
   action: WriteOpAction.Create
-  collection: string
-  rkey: string
-  cid: CID
+  collection: NsidString
+  rkey: RecordKeyString
+  cid: Cid
 }
 
 export type RecordUpdateDescript = {
   action: WriteOpAction.Update
-  collection: string
-  rkey: string
-  prev: CID
-  cid: CID
+  collection: NsidString
+  rkey: RecordKeyString
+  prev: Cid
+  cid: Cid
 }
 
 export type RecordDeleteDescript = {
   action: WriteOpAction.Delete
-  collection: string
-  rkey: string
-  cid: CID
+  collection: NsidString
+  rkey: RecordKeyString
+  cid: Cid
 }
 
 export type RecordWriteDescript =
@@ -128,10 +169,10 @@ export type WriteLog = RecordWriteDescript[][]
 // ---------------
 
 export type CommitData = {
-  cid: CID
+  cid: Cid
   rev: string
   since: string | null
-  prev: CID | null
+  prev: Cid | null
   newBlocks: BlockMap
   relevantBlocks: BlockMap
   removedCids: CidSet
@@ -141,14 +182,14 @@ export type RepoUpdate = CommitData & {
   ops: RecordWriteOp[]
 }
 
-export type CollectionContents = Record<string, RepoRecord>
-export type RepoContents = Record<string, CollectionContents>
+export type CollectionContents = Record<string, LexMap>
+export type RepoContents = Record<NsidString, CollectionContents>
 
-export type RepoRecordWithCid = { cid: CID; value: RepoRecord }
+export type RepoRecordWithCid = { cid: Cid; value: LexMap }
 export type CollectionContentsWithCids = Record<string, RepoRecordWithCid>
 export type RepoContentsWithCids = Record<string, CollectionContentsWithCids>
 
-export type DatastoreContents = Record<string, CID>
+export type DatastoreContents = Record<string, Cid>
 
 export type RecordPath = {
   collection: string
@@ -158,13 +199,13 @@ export type RecordPath = {
 export type RecordCidClaim = {
   collection: string
   rkey: string
-  cid: CID | null
+  cid: Cid | null
 }
 
 export type RecordClaim = {
   collection: string
   rkey: string
-  record: RepoRecord | null
+  record: LexMap | null
 }
 
 // Sync
@@ -181,6 +222,6 @@ export type VerifiedRepo = {
 }
 
 export type CarBlock = {
-  cid: CID
+  cid: Cid
   bytes: Uint8Array
 }

package/src/util.ts

@@ -1,8 +1,8 @@
-import * as cbor from '@ipld/dag-cbor'
-import { TID, cborDecode, check, cidForCbor, schema } from '@atproto/common'
+import { TID } from '@atproto/common-web'
 import * as crypto from '@atproto/crypto'
 import { Keypair } from '@atproto/crypto'
-import { LexValue, RepoRecord, ipldToLex, lexToIpld } from '@atproto/lexicon'
+import * as cbor from '@atproto/lex-cbor'
+import { Cid, LexMap, LexValue, isPlainObject } from '@atproto/lex-data'
 import { DataDiff } from './data-diff'
 import {
   Commit,
@@ -66,9 +66,9 @@ export const ensureCreates = (
 }
 
 export const parseDataKey = (key: string): RecordPath => {
-  const parts = key.split('/')
-  if (parts.length !== 2) throw new Error(`Invalid record key: ${key}`)
-  return { collection: parts[0], rkey: parts[1] }
+  const { length, 0: collection, 1: rkey } = key.split('/')
+  if (length !== 2) throw new Error(`Invalid record key: ${key}`)
+  return { collection, rkey }
 }
 
 export const formatDataKey = (collection: string, rkey: string): string => {
@@ -100,21 +100,17 @@ export const verifyCommitSig = async (
   return crypto.verifySignature(didKey, encoded, sig)
 }
 
-export const cborToLex = (val: Uint8Array): LexValue => {
-  return ipldToLex(cborDecode(val))
-}
+export const cborToLex: (val: Uint8Array) => LexValue = cbor.decode
 
-export const cborToLexRecord = (val: Uint8Array): RepoRecord => {
+export const cborToLexRecord = (val: Uint8Array): LexMap => {
   const parsed = cborToLex(val)
-  if (!check.is(parsed, schema.map)) {
+  if (!isPlainObject(parsed)) {
     throw new Error('lexicon records be a json object')
   }
   return parsed
 }
 
-export const cidForRecord = async (val: LexValue) => {
-  return cidForCbor(lexToIpld(val))
-}
+export const cidForRecord: (val: LexValue) => Promise<Cid> = cbor.cidForLex
 
 export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
   if (commit.version === 3) {
@@ -127,3 +123,24 @@ export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
     }
   }
 }
+
+export async function concatBytesAsync(iterable: AsyncIterable<Uint8Array>) {
+  const chunks: Uint8Array[] = []
+  for await (const chunk of iterable) chunks.push(chunk)
+  return concatBytes(chunks)
+}
+
+/**
+ * This is the same as {@link Buffer.concat}, without the `totalLength` argument.
+ */
+export function concatBytes(chunks: readonly Uint8Array[]): Uint8Array {
+  let totalLength = 0
+  for (const chunk of chunks) totalLength += chunk.byteLength
+  const result = new Uint8Array(totalLength)
+  let offset = 0
+  for (const chunk of chunks) {
+    result.set(chunk, offset)
+    offset += chunk.byteLength
+  }
+  return result
+}

package/tests/_util.ts

@@ -1,8 +1,10 @@
 import fs from 'node:fs'
-import { CID } from 'multiformats'
-import { TID, dataToCborBlock } from '@atproto/common'
+import { TID } from '@atproto/common-web'
 import * as crypto from '@atproto/crypto'
 import { Keypair, randomBytes } from '@atproto/crypto'
+import * as cbor from '@atproto/lex-cbor'
+import { Cid, cidForCbor, parseCid } from '@atproto/lex-data'
+import { NsidString } from '@atproto/syntax'
 import {
   BlockMap,
   CollectionContents,
@@ -18,15 +20,16 @@ import { MST } from '../src/mst'
 import { Repo } from '../src/repo'
 import { RepoStorage } from '../src/storage'
 
-type IdMapping = Record<string, CID>
+type IdMapping = Record<string, Cid>
 
-export const randomCid = async (storage?: RepoStorage): Promise<CID> => {
-  const block = await dataToCborBlock({ test: randomStr(50) })
+export const randomCid = async (storage?: RepoStorage): Promise<Cid> => {
+  const bytes = cbor.encode({ test: randomStr(50) })
+  const cid = await cidForCbor(bytes)
   if (storage) {
     // @ts-expect-error FIXME remove this comment (and fix the TS error)
-    await storage.putBlock(block.cid, block.bytes)
+    await storage.putBlock(cid, bytes)
   }
-  return block.cid
+  return cid
 }
 
 export const generateBulkDataKeys = async (
@@ -78,7 +81,10 @@ export const generateObject = (): Record<string, string> => {
 // Mass repo mutations & checking
 // -------------------------------
 
-export const testCollections = ['com.example.posts', 'com.example.likes']
+export const testCollections: NsidString[] = [
+  'com.example.posts',
+  'com.example.likes',
+]
 
 export const fillRepo = async (
   repo: Repo,
@@ -173,7 +179,7 @@ export const formatEdit = async (
 export const pathsForOps = (ops: RecordWriteOp[]): RecordPath[] =>
   ops.map((op) => ({ collection: op.collection, rkey: op.rkey }))
 
-export const saveMst = async (storage: RepoStorage, mst: MST): Promise<CID> => {
+export const saveMst = async (storage: RepoStorage, mst: MST): Promise<Cid> => {
   const diff = await mst.getUnstoredBlocks()
   // @ts-expect-error FIXME remove this comment (and fix the TS error)
   await storage.putMany(diff.blocks)
@@ -237,13 +243,23 @@ export const writeMstLog = async (filename: string, tree: MST) => {
   fs.writeFileSync(filename, log)
 }
 
-export const saveMstEntries = (filename: string, entries: [string, CID][]) => {
+export const saveMstEntries = (filename: string, entries: [string, Cid][]) => {
   const writable = entries.map(([key, val]) => [key, val.toString()])
   fs.writeFileSync(filename, JSON.stringify(writable))
 }
 
-export const loadMstEntries = (filename: string): [string, CID][] => {
+export const loadMstEntries = (filename: string): [string, Cid][] => {
   const contents = fs.readFileSync(filename)
   const parsed = JSON.parse(contents.toString())
-  return parsed.map(([key, value]) => [key, CID.parse(value)])
+  return parsed.map(([key, value]) => [key, parseCid(value)])
+}
+
+export async function toBuffer(
+  stream: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+): Promise<Buffer> {
+  const chunks: Uint8Array[] = []
+  for await (const chunk of stream) {
+    chunks.push(chunk)
+  }
+  return Buffer.concat(chunks)
 }

package/tests/car.test.ts

@@ -1,28 +1,48 @@
-import { CID } from 'multiformats/cid'
-import * as ui8 from 'uint8arrays'
-import { dataToCborBlock, streamToBytes, wait } from '@atproto/common'
+import { wait } from '@atproto/common-web'
+import { encode } from '@atproto/lex-cbor'
+import {
+  Cid,
+  LexValue,
+  cidForCbor,
+  fromBase64,
+  parseCid,
+  toBase64,
+} from '@atproto/lex-data'
 import { CarBlock, readCarStream, writeCarStream } from '../src'
 import fixtures from './car-file-fixtures.json'
 
+async function dataToCborBlock(data: LexValue): Promise<{
+  cid: Cid
+  bytes: Uint8Array
+}> {
+  const bytes = encode(data)
+  const cid = await cidForCbor(bytes)
+  return { cid, bytes }
+}
+
 describe('car', () => {
   for (const fixture of fixtures) {
     it('correctly writes car files', async () => {
-      const root = CID.parse(fixture.root)
+      const root = parseCid(fixture.root)
       async function* blockIter() {
         for (const block of fixture.blocks) {
-          const cid = CID.parse(block.cid)
-          const bytes = ui8.fromString(block.bytes, 'base64')
+          const cid = parseCid(block.cid)
+          const bytes = fromBase64(block.bytes, 'base64')
           yield { cid, bytes }
         }
       }
       const carStream = writeCarStream(root, blockIter())
-      const car = await streamToBytes(carStream)
-      const carB64 = ui8.toString(car, 'base64')
-      expect(carB64).toEqual(fixture.car)
+      const chunks: Uint8Array[] = []
+      for await (const chunk of carStream) {
+        chunks.push(chunk)
+      }
+      const car = Buffer.concat(chunks)
+      // @NOTE Not using car.toString('base64') because of padding differences
+      expect(toBase64(car)).toEqual(fixture.car)
     })
 
     it('correctly reads carfiles', async () => {
-      const carStream = [ui8.fromString(fixture.car, 'base64')]
+      const carStream = [fromBase64(fixture.car, 'base64')]
       const { roots, blocks } = await readCarStream(carStream)
       expect(roots.length).toBe(1)
       expect(roots[0].toString()).toEqual(fixture.root)
@@ -33,7 +53,7 @@ describe('car', () => {
       expect(carBlocks.length).toEqual(fixture.blocks.length)
       for (let i = 0; i < carBlocks.length; i++) {
         expect(carBlocks[i].cid.toString()).toEqual(fixture.blocks[i].cid)
-        expect(ui8.toString(carBlocks[i].bytes, 'base64')).toEqual(
+        expect(toBase64(carBlocks[i].bytes, 'base64')).toEqual(
           fixture.blocks[i].bytes,
         )
       }

package/tests/commit-proofs.test.ts

@@ -1,4 +1,4 @@
-import { CID } from 'multiformats'
+import { parseCid } from '@atproto/lex-data'
 import { BlockMap } from '../src'
 import { MST } from '../src/mst'
 import { MemoryBlockstore } from '../src/storage'
@@ -8,7 +8,7 @@ describe('commit proofs', () => {
   for (const fixture of fixtures) {
     it(fixture.comment, async () => {
       const { leafValue, keys, adds, dels } = fixture
-      const leaf = CID.parse(leafValue)
+      const leaf = parseCid(leafValue)
 
       const storage = new MemoryBlockstore()
       let mst = await MST.create(storage)
@@ -31,7 +31,7 @@ describe('commit proofs', () => {
         [...adds, ...dels].map((key) => mst.getCoveringProof(key)),
       )
       const proof = proofs.reduce((acc, cur) => acc.addMap(cur), new BlockMap())
-      const blocksInProof = fixture.blocksInProof.map((cid) => CID.parse(cid))
+      const blocksInProof = fixture.blocksInProof.map(parseCid)
       for (const cid of blocksInProof) {
         expect(proof.has(cid)).toBe(true)
       }

package/tests/covering-proofs.test.ts

@@ -1,4 +1,4 @@
-import { CID } from 'multiformats'
+import { parseCid } from '@atproto/lex-data'
 import { BlockMap } from '../src'
 import { MST } from '../src/mst'
 import { MemoryBlockstore } from '../src/storage'
@@ -25,7 +25,7 @@ describe('covering proofs', () => {
    */
   it('two deep split ', async () => {
     const storage = new MemoryBlockstore()
-    const cid = CID.parse(
+    const cid = parseCid(
       'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
     )
 
@@ -65,7 +65,7 @@ describe('covering proofs', () => {
    */
   it('two deep leafless splits ', async () => {
     const storage = new MemoryBlockstore()
-    const cid = CID.parse(
+    const cid = parseCid(
       'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
     )
 
@@ -103,7 +103,7 @@ describe('covering proofs', () => {
    */
   it('add on edge with neighbor two layers down', async () => {
     const storage = new MemoryBlockstore()
-    const cid = CID.parse(
+    const cid = parseCid(
       'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
     )
 
@@ -140,7 +140,7 @@ describe('covering proofs', () => {
    */
   it('merge and split in multi op commit', async () => {
     const storage = new MemoryBlockstore()
-    const cid = CID.parse(
+    const cid = parseCid(
       'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
     )
 
@@ -206,7 +206,7 @@ describe('covering proofs', () => {
    */
   it('complex multi-op commit', async () => {
     const storage = new MemoryBlockstore()
-    const cid = CID.parse(
+    const cid = parseCid(
       'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
     )