@atproto/space 0.0.0-spaces-alpha-20260818022935

package/src/repo-commit.ts

@@ -0,0 +1,171 @@
+import {
+  type Keypair,
+  hkdfSha256,
+  hmacSha256,
+  randomBytes,
+  verifySignature,
+} from '@atproto/crypto'
+import { type Cid, ui8Equals } from '@atproto/lex-data'
+import { LtHash } from './lthash.js'
+import {
+  COMMIT_VERSION,
+  type CommitCtx,
+  type RecordPath,
+  type RepoIndex,
+  type RepoOp,
+  type SignedCommit,
+} from './types.js'
+import { formatSetHashElement } from './util.js'
+
+export class RepoCommit {
+  constructor(public setHash: LtHash = new LtHash()) {}
+
+  // nullish state yields an empty repo
+  static fromState(state: Uint8Array | null | undefined): RepoCommit {
+    return new RepoCommit(new LtHash(state))
+  }
+
+  static fromRecords(records: Iterable<RecordPath & { cid: Cid }>): RepoCommit {
+    const commit = new RepoCommit()
+    for (const { collection, rkey, cid } of records) {
+      commit.add(collection, rkey, cid)
+    }
+    return commit
+  }
+
+  // Fold in every record an index describes, to compare against a commit.
+  static fromIndex(index: RepoIndex): RepoCommit {
+    const commit = new RepoCommit()
+    for (const [path, cid] of Object.entries(index)) {
+      commit.setHash.add(`${path}/${cid.toString()}`)
+    }
+    return commit
+  }
+
+  add(collection: string, rkey: string, cid: Cid): this {
+    this.setHash.add(formatSetHashElement(collection, rkey, cid))
+    return this
+  }
+
+  remove(collection: string, rkey: string, cid: Cid): this {
+    this.setHash.remove(formatSetHashElement(collection, rkey, cid))
+    return this
+  }
+
+  applyOp(op: RepoOp): this {
+    if (op.prev) this.remove(op.collection, op.rkey, op.prev)
+    if (op.cid) this.add(op.collection, op.rkey, op.cid)
+    return this
+  }
+
+  applyOps(ops: Iterable<RepoOp>): this {
+    for (const op of ops) {
+      this.applyOp(op)
+    }
+    return this
+  }
+
+  /**
+   * Whether this repo's contents match a signed commit. Verify the commit first —
+   * on its own this says nothing about authenticity.
+   */
+  matches(commit: SignedCommit): boolean {
+    return ui8Equals(this.setHash.digest(), commit.hash)
+  }
+
+  /**
+   * Sign a commit over the current contents.
+   *
+   * The signature covers only the ctx, never the digest, so a leaked commit proves
+   * nothing about what the author wrote. The digest is bound to the ctx by a
+   * symmetric MAC instead: readers get integrity, third parties get nothing. A
+   * fresh `ikm` per commit means each reader receives a distinct one.
+   */
+  async sign(ctx: CommitCtx, keypair: Keypair): Promise<SignedCommit> {
+    const hash = this.setHash.digest()
+    const ikm = randomBytes(32)
+    const ctxBytes = encodeCommitCtx(ctx, ikm)
+    return {
+      ver: COMMIT_VERSION,
+      hash,
+      ikm,
+      mac: computeMac(ikm, ctxBytes, hash),
+      sig: await keypair.sign(ctxBytes),
+      rev: ctx.rev,
+    }
+  }
+}
+
+/**
+ * Verify a commit's signature (authenticity) and MAC (integrity). Once this passes,
+ * `hash` is trusted as the author's claim about their repo, which is what makes
+ * {@link RepoCommit#matches} meaningful.
+ */
+export const verifyCommit = async (
+  commit: SignedCommit,
+  ctx: CommitCtx,
+  didKey: string,
+): Promise<boolean> => {
+  if (commit.ver !== COMMIT_VERSION) return false
+  if (commit.rev !== ctx.rev) return false
+
+  const ctxBytes = encodeCommitCtx(ctx, commit.ikm)
+  const mac = computeMac(commit.ikm, ctxBytes, commit.hash)
+  if (!ui8Equals(mac, commit.mac)) return false
+
+  return verifySignature(didKey, ctxBytes, commit.sig)
+}
+
+const computeMac = (
+  ikm: Uint8Array,
+  ctxBytes: Uint8Array,
+  hash: Uint8Array,
+): Uint8Array => {
+  return hmacSha256(hkdfSha256(ikm, ctxBytes), hash)
+}
+
+const DOMAIN_PREFIX = new TextEncoder().encode('atproto-space-v1')
+
+/**
+ * ```
+ * ctx = "atproto-space-v1"
+ *    || uint16be(len(space))  || space
+ *    || uint16be(len(author)) || author
+ *    || uint16be(len(rev))    || rev
+ *    || uint16be(len(ikm))    || ikm
+ * ```
+ *
+ * Length prefixes are big-endian per the TLS variable-length vector convention —
+ * deliberately the opposite byte order from the set hash's lanes.
+ */
+export const encodeCommitCtx = (
+  ctx: CommitCtx,
+  ikm: Uint8Array,
+): Uint8Array => {
+  const encoder = new TextEncoder()
+  const fields = [
+    encoder.encode(ctx.space),
+    encoder.encode(ctx.author),
+    encoder.encode(ctx.rev),
+    ikm,
+  ]
+
+  let size = DOMAIN_PREFIX.length
+  for (const field of fields) {
+    if (field.length > 0xffff) {
+      throw new Error('commit ctx field exceeds uint16 length prefix')
+    }
+    size += 2 + field.length
+  }
+
+  const out = new Uint8Array(size)
+  out.set(DOMAIN_PREFIX)
+  let offset = DOMAIN_PREFIX.length
+  for (const field of fields) {
+    out[offset++] = (field.length >>> 8) & 0xff
+    out[offset++] = field.length & 0xff
+    out.set(field, offset)
+    offset += field.length
+  }
+  return out
+}

package/src/sync/consumer.ts

@@ -0,0 +1,149 @@
+import { type check, readCarStream } from '@atproto/common'
+import { decode } from '@atproto/lex-cbor'
+import type { Cid, LexMap } from '@atproto/lex-data'
+import { RepoVerificationError } from '../error.js'
+import { RepoCommit, verifyCommit } from '../repo-commit.js'
+import {
+  type CommitCtx,
+  type RepoIndex,
+  type SignedCommit,
+  type SpaceRecord,
+  def,
+} from '../types.js'
+import { parseRecordPath } from '../util.js'
+
+export type VerifyRepoParams = {
+  space: string
+  author: string
+  didKey: string
+  // False for an index-only car, which carries no record blocks. Defaults to true.
+  expectValues?: boolean
+}
+
+export type VerifiedRecord = {
+  collection: string
+  rkey: string
+  cid: Cid
+  record: SpaceRecord
+}
+
+export type VerifiedRepo = {
+  commit: SignedCommit
+  index: RepoIndex
+  repo: RepoCommit
+  records: AsyncGenerator<VerifiedRecord>
+}
+
+/**
+ * Verify a serialized repo, streaming out its records.
+ *
+ * The three stages follow the CAR's layout, so nothing needs buffering: the
+ * commit, then the index against the commit's hash (which authenticates every
+ * path/cid pair without reading a record), then each block against its index
+ * entry. That last stage runs as `records` is drained, so it must be consumed to
+ * know the repo was complete — {@link verifyRepoCarFull} does that.
+ */
+export const verifyRepoCar = async (
+  car: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+  params: VerifyRepoParams,
+): Promise<VerifiedRepo> => {
+  // The reader hashes every block against its cid as it streams.
+  const { roots, blocks } = await readCarStream(car)
+  if (roots.length !== 2) {
+    throw new RepoVerificationError(
+      `expected 2 car roots (commit, index), got ${roots.length}`,
+    )
+  }
+  const [commitRoot, indexRoot] = roots
+
+  const commitBlock = await blocks.next()
+  if (commitBlock.done || !commitBlock.value.cid.equals(commitRoot)) {
+    throw new RepoVerificationError('expected the commit block to lead the car')
+  }
+  const commit = parseBlock(commitBlock.value.bytes, def.signedCommit)
+
+  const ctx: CommitCtx = { ...params, rev: commit.rev }
+  if (!(await verifyCommit(commit, ctx, params.didKey))) {
+    throw new RepoVerificationError('commit failed verification')
+  }
+
+  const indexBlock = await blocks.next()
+  if (indexBlock.done || !indexBlock.value.cid.equals(indexRoot)) {
+    throw new RepoVerificationError(
+      'expected the index block to follow the commit',
+    )
+  }
+  const index = parseBlock(indexBlock.value.bytes, def.repoIndex)
+
+  const repo = RepoCommit.fromIndex(index)
+  if (!repo.matches(commit)) {
+    throw new RepoVerificationError('index does not match the commit hash')
+  }
+
+  return {
+    commit,
+    index,
+    repo,
+    records: verifyRecords(blocks, index, params.expectValues !== false),
+  }
+}
+
+/**
+ * Verify a serialized repo and collect its records. Prefer {@link verifyRepoCar}
+ * for large repos, where streaming avoids holding every record at once.
+ */
+export const verifyRepoCarFull = async (
+  car: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+  params: VerifyRepoParams,
+): Promise<Omit<VerifiedRepo, 'records'> & { records: VerifiedRecord[] }> => {
+  const { records, ...rest } = await verifyRepoCar(car, params)
+  const collected: VerifiedRecord[] = []
+  for await (const record of records) {
+    collected.push(record)
+  }
+  return { ...rest, records: collected }
+}
+
+// The car holds one record block per index entry, in the index's order.
+async function* verifyRecords(
+  blocks: AsyncIterable<{ cid: Cid; bytes: Uint8Array }>,
+  index: RepoIndex,
+  expectValues: boolean,
+): AsyncGenerator<VerifiedRecord> {
+  const paths = Object.keys(index)
+  let i = 0
+
+  for await (const block of blocks) {
+    if (i >= paths.length) {
+      throw new RepoVerificationError('car has more blocks than index entries')
+    }
+    const path = paths[i]
+    const cid = index[path]
+    i++
+
+    if (!block.cid.equals(cid)) {
+      throw new RepoVerificationError(
+        `expected block ${cid} at ${path}, got ${block.cid}`,
+      )
+    }
+    const { collection, rkey } = parseRecordPath(path)
+    yield { collection, rkey, cid, record: decode(block.bytes) as LexMap }
+  }
+
+  const isIndexOnly = !expectValues && i === 0
+  if (i < paths.length && !isIndexOnly) {
+    throw new RepoVerificationError(
+      `car is missing ${paths.length - i} record(s) named in the index`,
+    )
+  }
+}
+
+const parseBlock = <T>(bytes: Uint8Array, def: check.Def<T>): T => {
+  const parsed = def.schema.safeParse(decode(bytes))
+  if (!parsed.success) {
+    throw new RepoVerificationError(
+      `invalid ${def.name}: ${parsed.error.message}`,
+    )
+  }
+  return parsed.data
+}

package/src/sync/index.ts

@@ -0,0 +1,2 @@
+export * from './consumer.js'
+export * from './provider.js'

package/src/sync/provider.ts

@@ -0,0 +1,82 @@
+import { type CarBlock, writeCarStream } from '@atproto/common'
+import { cidForLex, encode } from '@atproto/lex-cbor'
+import { type Cid, cidForCbor } from '@atproto/lex-data'
+import type { RepoIndex, SignedCommit, SpaceRecord } from '../types.js'
+import { formatRecordPath } from '../util.js'
+
+export type SerializedRecord = {
+  collection: string
+  rkey: string
+  cid: Cid
+  bytes: Uint8Array
+}
+
+/**
+ * Serialize a repo as a car: two roots in order — the signed commit, then the
+ * index — followed by one block per index entry, in the index's order. Blobs are
+ * excluded.
+ *
+ * Records are collected up front because the index has to precede the blocks it
+ * describes.
+ *
+ * With `excludeValues` only the two roots are written. The index still
+ * authenticates against the commit, since the set hash is folded from the index's
+ * entries rather than from the blocks.
+ */
+export async function* serializeRepo(
+  commit: SignedCommit,
+  records: AsyncIterable<SerializedRecord> | Iterable<SerializedRecord>,
+  opts: { excludeValues?: boolean } = {},
+): AsyncIterable<Uint8Array> {
+  const byPath = new Map<string, SerializedRecord>()
+  for await (const record of records) {
+    byPath.set(formatRecordPath(record.collection, record.rkey), record)
+  }
+  // A consumer walks the index as the cbor encoder ordered its keys, so blocks
+  // have to follow that order too.
+  const paths = [...byPath.keys()].sort(byCanonicalKey)
+
+  const index: RepoIndex = {}
+  for (const path of paths) {
+    index[path] = byPath.get(path)!.cid
+  }
+
+  const commitBytes = encode(commit)
+  const indexBytes = encode(index)
+  const [commitRoot, indexRoot] = await Promise.all([
+    cidForCbor(commitBytes),
+    cidForCbor(indexBytes),
+  ])
+
+  yield* writeCarStream(
+    [commitRoot, indexRoot],
+    (function* (): Generator<CarBlock> {
+      yield { cid: commitRoot, bytes: commitBytes }
+      yield { cid: indexRoot, bytes: indexBytes }
+      if (opts.excludeValues) return
+      for (const path of paths) {
+        const record = byPath.get(path)!
+        yield { cid: record.cid, bytes: record.bytes }
+      }
+    })(),
+  )
+}
+
+// Canonical dag-cbor map key order: shortest first, then bytewise.
+const byCanonicalKey = (a: string, b: string): number => {
+  if (a.length !== b.length) return a.length - b.length
+  return a < b ? -1 : a > b ? 1 : 0
+}
+
+export const serializeRecord = async (
+  collection: string,
+  rkey: string,
+  record: SpaceRecord,
+): Promise<SerializedRecord> => {
+  return {
+    collection,
+    rkey,
+    cid: await cidForLex(record),
+    bytes: encode(record),
+  }
+}

package/src/types.ts

@@ -0,0 +1,62 @@
+import { z } from 'zod'
+import type { check } from '@atproto/common'
+import { type Cid, type LexMap, ifCid } from '@atproto/lex-data'
+
+export type SpaceRecord = LexMap
+
+export type RecordPath = {
+  collection: string
+  rkey: string
+}
+
+// A create has no `prev`, a delete no `cid`, an update both.
+export type RepoOp = RecordPath & {
+  cid: Cid | null
+  prev: Cid | null
+}
+
+export const COMMIT_VERSION = 1
+
+export type CommitCtx = {
+  space: string
+  author: string
+  rev: string
+}
+
+const cidSchema = z.unknown().transform((input, ctx): Cid => {
+  const cid = ifCid(input)
+  if (cid) return cid
+
+  ctx.addIssue({
+    code: z.ZodIssueCode.custom,
+    message: 'Not a valid CID',
+  })
+  return z.NEVER
+})
+
+const bytes = z.instanceof(Uint8Array<ArrayBufferLike>)
+
+const signedCommit = z.object({
+  ver: z.literal(COMMIT_VERSION),
+  hash: bytes,
+  ikm: bytes,
+  sig: bytes,
+  mac: bytes,
+  rev: z.string(),
+})
+export type SignedCommit = z.infer<typeof signedCommit>
+
+// `{collection}/{rkey}` to the record's cid.
+const repoIndex = z.record(z.string(), cidSchema)
+export type RepoIndex = z.infer<typeof repoIndex>
+
+export const def = {
+  signedCommit: {
+    name: 'signed commit',
+    schema: signedCommit,
+  } as check.Def<SignedCommit>,
+  repoIndex: {
+    name: 'repo index',
+    schema: repoIndex,
+  } as check.Def<RepoIndex>,
+}

package/src/util.ts

@@ -0,0 +1,21 @@
+import type { Cid } from '@atproto/lex-data'
+import type { RecordPath } from './types.js'
+
+export const formatRecordPath = (collection: string, rkey: string): string => {
+  return collection + '/' + rkey
+}
+
+export const parseRecordPath = (path: string): RecordPath => {
+  const { length, 0: collection, 1: rkey } = path.split('/')
+  if (length !== 2) throw new Error(`invalid record path: ${path}`)
+  return { collection, rkey }
+}
+
+// The element a record contributes to a repo's set hash.
+export const formatSetHashElement = (
+  collection: string,
+  rkey: string,
+  cid: Cid,
+): string => {
+  return `${collection}/${rkey}/${cid.toString()}`
+}