@atproto/repo 0.8.12 → 0.9.0

package/src/storage/types.ts

@@ -1,49 +1,47 @@
-/* eslint-disable import/no-deprecated */
-
-import stream from 'node:stream'
-import { CID } from 'multiformats/cid'
-import { check } from '@atproto/common'
-import { RepoRecord } from '@atproto/lexicon'
+import type { Readable } from 'node:stream'
+import { check } from '@atproto/common-web'
+import { Cid, LexMap } from '@atproto/lex-data'
 import { BlockMap } from '../block-map'
 import { CommitData } from '../types'
 
 export interface RepoStorage {
   // Writable
-  getRoot(): Promise<CID | null>
-  putBlock(cid: CID, block: Uint8Array, rev: string): Promise<void>
+  getRoot(): Promise<Cid | null>
+  putBlock(cid: Cid, block: Uint8Array, rev: string): Promise<void>
   putMany(blocks: BlockMap, rev: string): Promise<void>
-  updateRoot(cid: CID, rev: string): Promise<void>
+  updateRoot(cid: Cid, rev: string): Promise<void>
   applyCommit(commit: CommitData)
 
   // Readable
-  getBytes(cid: CID): Promise<Uint8Array | null>
-  has(cid: CID): Promise<boolean>
-  getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+  getBytes(cid: Cid): Promise<Uint8Array | null>
+  has(cid: Cid): Promise<boolean>
+  getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }>
   attemptRead<T>(
-    cid: CID,
+    cid: Cid,
     def: check.Def<T>,
   ): Promise<{ obj: T; bytes: Uint8Array } | null>
   readObjAndBytes<T>(
-    cid: CID,
+    cid: Cid,
     def: check.Def<T>,
   ): Promise<{ obj: T; bytes: Uint8Array }>
-  readObj<T>(cid: CID, def: check.Def<T>): Promise<T>
-  attemptReadRecord(cid: CID): Promise<RepoRecord | null>
-  readRecord(cid: CID): Promise<RepoRecord>
+  readObj<T>(cid: Cid, def: check.Def<T>): Promise<T>
+  attemptReadRecord(cid: Cid): Promise<LexMap | null>
+  readRecord(cid: Cid): Promise<LexMap>
 }
 
+// @TODO make this less node-js specific by using AsyncIterable<Uint8Array> instead of Readable
 export interface BlobStore {
-  putTemp(bytes: Uint8Array | stream.Readable): Promise<string>
-  makePermanent(key: string, cid: CID): Promise<void>
-  putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>
-  quarantine(cid: CID): Promise<void>
-  unquarantine(cid: CID): Promise<void>
-  getBytes(cid: CID): Promise<Uint8Array>
-  getStream(cid: CID): Promise<stream.Readable>
+  putTemp(bytes: Uint8Array | Readable): Promise<string>
+  makePermanent(key: string, cid: Cid): Promise<void>
+  putPermanent(cid: Cid, bytes: Uint8Array | Readable): Promise<void>
+  quarantine(cid: Cid): Promise<void>
+  unquarantine(cid: Cid): Promise<void>
+  getBytes(cid: Cid): Promise<Uint8Array>
+  getStream(cid: Cid): Promise<Readable>
   hasTemp(key: string): Promise<boolean>
-  hasStored(cid: CID): Promise<boolean>
-  delete(cid: CID): Promise<void>
-  deleteMany(cid: CID[]): Promise<void>
+  hasStored(cid: Cid): Promise<boolean>
+  delete(cid: Cid): Promise<void>
+  deleteMany(cid: Cid[]): Promise<void>
 }
 
 export class BlobNotFoundError extends Error {}

package/src/sync/consumer.ts

@@ -1,4 +1,4 @@
-import { CID } from 'multiformats/cid'
+import { Cid } from '@atproto/lex-data'
 import { BlockMap } from '../block-map'
 import { readCarWithRoot } from '../car'
 import { DataDiff } from '../data-diff'
@@ -25,7 +25,7 @@ export const verifyRepoCar = async (
 
 export const verifyRepo = async (
   blocks: BlockMap,
-  head: CID,
+  head: Cid,
   did?: string,
   signingKey?: string,
   opts?: { ensureLeaves?: boolean },
@@ -52,7 +52,7 @@ export const verifyDiffCar = async (
 export const verifyDiff = async (
   repo: ReadableRepo | null,
   updateBlocks: BlockMap,
-  updateRoot: CID,
+  updateRoot: Cid,
   did?: string,
   signingKey?: string,
   opts?: { ensureLeaves?: boolean },
@@ -103,7 +103,7 @@ export const verifyDiff = async (
 // @NOTE only verifies the root, not the repo contents
 const verifyRepoRoot = async (
   storage: ReadableBlockstore,
-  head: CID,
+  head: Cid,
   did?: string,
   signingKey?: string,
 ): Promise<ReadableRepo> => {
@@ -155,7 +155,7 @@ export const verifyProofs = async (
         unverified.push(claim)
       }
     } else {
-      if (claim.cid.equals(found)) {
+      if (found?.equals(claim.cid)) {
         verified.push(claim)
       } else {
         unverified.push(claim)

package/src/sync/provider.ts

@@ -1,10 +1,10 @@
-import { CID } from 'multiformats/cid'
+import { Cid } from '@atproto/lex-data'
 import { writeCarStream } from '../car'
 import { CidSet } from '../cid-set'
 import { MissingBlocksError } from '../error'
 import { MST } from '../mst'
 import { ReadableBlockstore, RepoStorage } from '../storage'
-import { RecordPath, def } from '../types'
+import { CarBlock, RecordPath, def } from '../types'
 import * as util from '../util'
 
 // Full Repo
@@ -12,12 +12,15 @@ import * as util from '../util'
 
 export const getFullRepo = (
   storage: RepoStorage,
-  commitCid: CID,
+  commitCid: Cid,
 ): AsyncIterable<Uint8Array> => {
   return writeCarStream(commitCid, iterateFullRepo(storage, commitCid))
 }
 
-async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
+async function* iterateFullRepo(
+  storage: RepoStorage,
+  commitCid: Cid,
+): AsyncGenerator<CarBlock> {
   const commit = await storage.readObjAndBytes(commitCid, def.commit)
   yield { cid: commitCid, bytes: commit.bytes }
   const mst = MST.load(storage, commit.obj.data)
@@ -31,7 +34,7 @@ async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
 
 export const getRecords = (
   storage: ReadableBlockstore,
-  commitCid: CID,
+  commitCid: Cid,
   paths: RecordPath[],
 ): AsyncIterable<Uint8Array> => {
   return writeCarStream(
@@ -42,9 +45,9 @@ export const getRecords = (
 
 async function* iterateRecordBlocks(
   storage: ReadableBlockstore,
-  commitCid: CID,
+  commitCid: Cid,
   paths: RecordPath[],
-) {
+): AsyncGenerator<CarBlock> {
   const commit = await storage.readObjAndBytes(commitCid, def.commit)
   yield { cid: commitCid, bytes: commit.bytes }
   const mst = MST.load(storage, commit.obj.data)

package/src/types.ts

@@ -1,43 +1,50 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats'
 import { z } from 'zod'
-import { schema as common } from '@atproto/common'
-import { def as commonDef } from '@atproto/common-web'
-import { RepoRecord } from '@atproto/lexicon'
+import { Cid, LexMap, ifCid } from '@atproto/lex-data'
+import { NsidString, RecordKeyString } from '@atproto/syntax'
 import { BlockMap } from './block-map'
 import { CidSet } from './cid-set'
 
 // Repo nodes
 // ---------------
 
+const cidSchema = z.unknown().transform((input, ctx): Cid => {
+  const cid = ifCid(input)
+  if (cid) return cid
+
+  ctx.addIssue({
+    code: z.ZodIssueCode.custom,
+    message: 'Not a valid CID',
+  })
+  return z.NEVER
+})
+
 const unsignedCommit = z.object({
   did: z.string(),
   version: z.literal(3),
-  data: common.cid,
+  data: cidSchema,
   rev: z.string(),
   // `prev` added for backwards compatibility with v2, no requirement of keeping around history
-  prev: common.cid.nullable(),
+  prev: cidSchema.nullable(),
 })
 export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
 const commit = z.object({
   did: z.string(),
   version: z.literal(3),
-  data: common.cid,
+  data: cidSchema,
   rev: z.string(),
-  prev: common.cid.nullable(),
-  sig: common.bytes,
+  prev: cidSchema.nullable(),
+  sig: z.instanceof(Uint8Array),
 })
 export type Commit = z.infer<typeof commit>
 
 const legacyV2Commit = z.object({
   did: z.string(),
   version: z.literal(2),
-  data: common.cid,
+  data: cidSchema,
   rev: z.string().optional(),
-  prev: common.cid.nullable(),
-  sig: common.bytes,
+  prev: cidSchema.nullable(),
+  sig: z.instanceof(Uint8Array),
 })
 export type LegacyV2Commit = z.infer<typeof legacyV2Commit>
 
@@ -48,14 +55,46 @@ const versionedCommit = z.discriminatedUnion('version', [
 export type VersionedCommit = z.infer<typeof versionedCommit>
 
 export const schema = {
-  ...common,
+  cid: cidSchema,
+  carHeader: z.object({
+    version: z.literal(1),
+    roots: z.array(cidSchema),
+  }),
+  bytes: z.instanceof(Uint8Array),
+  string: z.string(),
+  array: z.array(z.unknown()),
+  map: z.record(z.string(), z.unknown()),
+  unknown: z.unknown(),
   commit,
   legacyV2Commit,
   versionedCommit,
 }
 
 export const def = {
-  ...commonDef,
+  cid: {
+    name: 'cid',
+    schema: schema.cid,
+  },
+  carHeader: {
+    name: 'CAR header',
+    schema: schema.carHeader,
+  },
+  bytes: {
+    name: 'bytes',
+    schema: schema.bytes,
+  },
+  string: {
+    name: 'string',
+    schema: schema.string,
+  },
+  map: {
+    name: 'map',
+    schema: schema.map,
+  },
+  unknown: {
+    name: 'unknown',
+    schema: schema.unknown,
+  },
   commit: {
     name: 'commit',
     schema: schema.commit,
@@ -77,46 +116,46 @@ export enum WriteOpAction {
 
 export type RecordCreateOp = {
   action: WriteOpAction.Create
-  collection: string
-  rkey: string
-  record: RepoRecord
+  collection: NsidString
+  rkey: RecordKeyString
+  record: LexMap
 }
 
 export type RecordUpdateOp = {
   action: WriteOpAction.Update
-  collection: string
-  rkey: string
-  record: RepoRecord
+  collection: NsidString
+  rkey: RecordKeyString
+  record: LexMap
 }
 
 export type RecordDeleteOp = {
   action: WriteOpAction.Delete
-  collection: string
-  rkey: string
+  collection: NsidString
+  rkey: RecordKeyString
 }
 
 export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp
 
 export type RecordCreateDescript = {
   action: WriteOpAction.Create
-  collection: string
-  rkey: string
-  cid: CID
+  collection: NsidString
+  rkey: RecordKeyString
+  cid: Cid
 }
 
 export type RecordUpdateDescript = {
   action: WriteOpAction.Update
-  collection: string
-  rkey: string
-  prev: CID
-  cid: CID
+  collection: NsidString
+  rkey: RecordKeyString
+  prev: Cid
+  cid: Cid
 }
 
 export type RecordDeleteDescript = {
   action: WriteOpAction.Delete
-  collection: string
-  rkey: string
-  cid: CID
+  collection: NsidString
+  rkey: RecordKeyString
+  cid: Cid
 }
 
 export type RecordWriteDescript =
@@ -130,10 +169,10 @@ export type WriteLog = RecordWriteDescript[][]
 // ---------------
 
 export type CommitData = {
-  cid: CID
+  cid: Cid
   rev: string
   since: string | null
-  prev: CID | null
+  prev: Cid | null
   newBlocks: BlockMap
   relevantBlocks: BlockMap
   removedCids: CidSet
@@ -143,14 +182,14 @@ export type RepoUpdate = CommitData & {
   ops: RecordWriteOp[]
 }
 
-export type CollectionContents = Record<string, RepoRecord>
-export type RepoContents = Record<string, CollectionContents>
+export type CollectionContents = Record<string, LexMap>
+export type RepoContents = Record<NsidString, CollectionContents>
 
-export type RepoRecordWithCid = { cid: CID; value: RepoRecord }
+export type RepoRecordWithCid = { cid: Cid; value: LexMap }
 export type CollectionContentsWithCids = Record<string, RepoRecordWithCid>
 export type RepoContentsWithCids = Record<string, CollectionContentsWithCids>
 
-export type DatastoreContents = Record<string, CID>
+export type DatastoreContents = Record<string, Cid>
 
 export type RecordPath = {
   collection: string
@@ -160,13 +199,13 @@ export type RecordPath = {
 export type RecordCidClaim = {
   collection: string
   rkey: string
-  cid: CID | null
+  cid: Cid | null
 }
 
 export type RecordClaim = {
   collection: string
   rkey: string
-  record: RepoRecord | null
+  record: LexMap | null
 }
 
 // Sync
@@ -183,6 +222,6 @@ export type VerifiedRepo = {
 }
 
 export type CarBlock = {
-  cid: CID
+  cid: Cid
   bytes: Uint8Array
 }

package/src/util.ts

@@ -1,10 +1,8 @@
-/* eslint-disable import/no-deprecated */
-
-import * as cbor from '@ipld/dag-cbor'
-import { TID, cborDecode, check, cidForCbor, schema } from '@atproto/common'
+import { TID } from '@atproto/common-web'
 import * as crypto from '@atproto/crypto'
 import { Keypair } from '@atproto/crypto'
-import { LexValue, RepoRecord, ipldToLex, lexToIpld } from '@atproto/lexicon'
+import * as cbor from '@atproto/lex-cbor'
+import { Cid, LexMap, LexValue, isPlainObject } from '@atproto/lex-data'
 import { DataDiff } from './data-diff'
 import {
   Commit,
@@ -68,9 +66,9 @@ export const ensureCreates = (
 }
 
 export const parseDataKey = (key: string): RecordPath => {
-  const parts = key.split('/')
-  if (parts.length !== 2) throw new Error(`Invalid record key: ${key}`)
-  return { collection: parts[0], rkey: parts[1] }
+  const { length, 0: collection, 1: rkey } = key.split('/')
+  if (length !== 2) throw new Error(`Invalid record key: ${key}`)
+  return { collection, rkey }
 }
 
 export const formatDataKey = (collection: string, rkey: string): string => {
@@ -102,21 +100,17 @@ export const verifyCommitSig = async (
   return crypto.verifySignature(didKey, encoded, sig)
 }
 
-export const cborToLex = (val: Uint8Array): LexValue => {
-  return ipldToLex(cborDecode(val))
-}
+export const cborToLex: (val: Uint8Array) => LexValue = cbor.decode
 
-export const cborToLexRecord = (val: Uint8Array): RepoRecord => {
+export const cborToLexRecord = (val: Uint8Array): LexMap => {
   const parsed = cborToLex(val)
-  if (!check.is(parsed, schema.map)) {
+  if (!isPlainObject(parsed)) {
     throw new Error('lexicon records be a json object')
   }
   return parsed
 }
 
-export const cidForRecord = async (val: LexValue) => {
-  return cidForCbor(lexToIpld(val))
-}
+export const cidForRecord: (val: LexValue) => Promise<Cid> = cbor.cidForLex
 
 export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
   if (commit.version === 3) {
@@ -129,3 +123,24 @@ export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
     }
   }
 }
+
+export async function concatBytesAsync(iterable: AsyncIterable<Uint8Array>) {
+  const chunks: Uint8Array[] = []
+  for await (const chunk of iterable) chunks.push(chunk)
+  return concatBytes(chunks)
+}
+
+/**
+ * This is the same as {@link Buffer.concat}, without the `totalLength` argument.
+ */
+export function concatBytes(chunks: readonly Uint8Array[]): Uint8Array {
+  let totalLength = 0
+  for (const chunk of chunks) totalLength += chunk.byteLength
+  const result = new Uint8Array(totalLength)
+  let offset = 0
+  for (const chunk of chunks) {
+    result.set(chunk, offset)
+    offset += chunk.byteLength
+  }
+  return result
+}