@atproto/repo 0.10.1 → 0.10.3

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @atproto/repo
 
+## 0.10.3
+
+### Patch Changes
+
+- [#5099](https://github.com/bluesky-social/atproto/pull/5099) [`b43ec31`](https://github.com/bluesky-social/atproto/commit/b43ec31f247f4461725b01226885f88bd430ca07) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Update TypeScript build to rely on references to composite internal projects
+
+- [#5099](https://github.com/bluesky-social/atproto/pull/5099) [`b43ec31`](https://github.com/bluesky-social/atproto/commit/b43ec31f247f4461725b01226885f88bd430ca07) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Bundle only necessary files in the NPM tarball, including the `CHANGELOG.md` and `README.md` files (if present).
+
+- Updated dependencies [[`28a0b58`](https://github.com/bluesky-social/atproto/commit/28a0b588147863eaef948cd2bb8fc0f19d08cda9), [`b43ec31`](https://github.com/bluesky-social/atproto/commit/b43ec31f247f4461725b01226885f88bd430ca07), [`b43ec31`](https://github.com/bluesky-social/atproto/commit/b43ec31f247f4461725b01226885f88bd430ca07), [`b43ec31`](https://github.com/bluesky-social/atproto/commit/b43ec31f247f4461725b01226885f88bd430ca07), [`b43ec31`](https://github.com/bluesky-social/atproto/commit/b43ec31f247f4461725b01226885f88bd430ca07)]:
+  - @atproto/syntax@0.6.4
+  - @atproto/lex-cbor@0.1.3
+  - @atproto/lex-data@0.1.4
+  - @atproto/common-web@0.5.3
+  - @atproto/common@0.6.5
+  - @atproto/crypto@0.5.3
+
+## 0.10.2
+
+### Patch Changes
+
+- [#5151](https://github.com/bluesky-social/atproto/pull/5151) [`a51c45d`](https://github.com/bluesky-social/atproto/commit/a51c45d38f6bd7b8765f640e564cf921d52162e7) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Update dependencies
+
+- Updated dependencies [[`f2cf8f7`](https://github.com/bluesky-social/atproto/commit/f2cf8f7fc5f3a10847f2e6d785e5fa2244ee8cfb), [`a51c45d`](https://github.com/bluesky-social/atproto/commit/a51c45d38f6bd7b8765f640e564cf921d52162e7), [`f2cf8f7`](https://github.com/bluesky-social/atproto/commit/f2cf8f7fc5f3a10847f2e6d785e5fa2244ee8cfb)]:
+  - @atproto/common@0.6.4
+  - @atproto/common-web@0.5.2
+  - @atproto/crypto@0.5.2
+  - @atproto/lex-cbor@0.1.2
+  - @atproto/lex-data@0.1.3
+  - @atproto/syntax@0.6.3
+
 ## 0.10.1
 
 ### Patch Changes

package/package.json

@@ -1,34 +1,26 @@
 {
   "name": "@atproto/repo",
-  "version": "0.10.1",
+  "version": "0.10.3",
+  "engines": {
+    "node": ">=22"
+  },
   "license": "MIT",
   "description": "atproto repo and MST implementation",
   "keywords": [
     "atproto",
     "mst"
   ],
-  "engines": {
-    "node": ">=22"
-  },
   "homepage": "https://atproto.com",
   "repository": {
     "type": "git",
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/repo"
   },
-  "dependencies": {
-    "varint": "^6.0.0",
-    "zod": "^3.23.8",
-    "@atproto/common": "^0.6.3",
-    "@atproto/common-web": "^0.5.1",
-    "@atproto/crypto": "^0.5.1",
-    "@atproto/lex-cbor": "^0.1.1",
-    "@atproto/lex-data": "^0.1.2",
-    "@atproto/syntax": "^0.6.2"
-  },
-  "devDependencies": {
-    "jest": "^30.0.0"
-  },
+  "files": [
+    "./dist",
+    "./README.md",
+    "./CHANGELOG.md"
+  ],
   "type": "module",
   "exports": {
     ".": {
@@ -36,6 +28,19 @@
       "default": "./dist/index.js"
     }
   },
+  "dependencies": {
+    "varint": "^6.0.0",
+    "zod": "^3.23.8",
+    "@atproto/common-web": "^0.5.3",
+    "@atproto/crypto": "^0.5.3",
+    "@atproto/lex-cbor": "^0.1.3",
+    "@atproto/syntax": "^0.6.4",
+    "@atproto/lex-data": "^0.1.4",
+    "@atproto/common": "^0.6.5"
+  },
+  "devDependencies": {
+    "jest": "^30.0.0"
+  },
   "scripts": {
     "test": "NODE_OPTIONS=--experimental-vm-modules jest",
     "test:profile": "node --experimental-vm-modules --inspect ../../node_modules/.bin/jest",

package/jest.config.cjs

@@ -1,24 +0,0 @@
-/** @type {import('jest').Config} */
-module.exports = {
-  displayName: 'Repo',
-  transform: {
-    '^.+\\.(t|j)s$': [
-      '@swc/jest',
-      {
-        jsc: {
-          parser: { syntax: 'typescript', importAttributes: true },
-          experimental: { keepImportAttributes: true },
-          transform: {},
-        },
-        module: { type: 'es6' },
-      },
-    ],
-  },
-  extensionsToTreatAsEsm: ['.ts'],
-  transformIgnorePatterns: [],
-  setupFiles: ['<rootDir>/../../test.setup.ts'],
-  moduleNameMapper: {
-    '^varint$': '<rootDir>/../../jest.varint-shim.cjs',
-    '^(\\.\\.?\\/.+)\\.js$': ['$1.ts', '$1.js'],
-  },
-}

package/src/block-map.ts

@@ -1,131 +0,0 @@
-import { encode } from '@atproto/lex-cbor'
-import {
-  Cid,
-  LexValue,
-  cidForCbor,
-  parseCid,
-  ui8Equals,
-} from '@atproto/lex-data'
-
-export class BlockMap implements Iterable<[cid: Cid, bytes: Uint8Array]> {
-  private map: Map<string, Uint8Array> = new Map()
-
-  constructor(entries?: Iterable<readonly [cid: Cid, bytes: Uint8Array]>) {
-    if (entries) {
-      for (const [cid, bytes] of entries) {
-        this.set(cid, bytes)
-      }
-    }
-  }
-
-  async add(value: LexValue): Promise<Cid> {
-    const bytes = encode(value)
-    const cid = await cidForCbor(bytes)
-    this.set(cid, bytes)
-    return cid
-  }
-
-  set(cid: Cid, bytes: Uint8Array): BlockMap {
-    this.map.set(cid.toString(), bytes)
-    return this
-  }
-
-  get(cid: Cid): Uint8Array | undefined {
-    return this.map.get(cid.toString())
-  }
-
-  delete(cid: Cid): BlockMap {
-    this.map.delete(cid.toString())
-    return this
-  }
-
-  getMany(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } {
-    const missing: Cid[] = []
-    const blocks = new BlockMap()
-    for (const cid of cids) {
-      const got = this.map.get(cid.toString())
-      if (got) {
-        blocks.set(cid, got)
-      } else {
-        missing.push(cid)
-      }
-    }
-    return { blocks, missing }
-  }
-
-  has(cid: Cid): boolean {
-    return this.map.has(cid.toString())
-  }
-
-  clear(): void {
-    this.map.clear()
-  }
-
-  forEach(cb: (bytes: Uint8Array, cid: Cid) => void): void {
-    for (const [cid, bytes] of this) cb(bytes, cid)
-  }
-
-  entries(): Entry[] {
-    return Array.from(this, toEntry)
-  }
-
-  cids(): Cid[] {
-    return Array.from(this.keys())
-  }
-
-  addMap(toAdd: BlockMap): BlockMap {
-    for (const [cid, bytes] of toAdd) this.set(cid, bytes)
-    return this
-  }
-
-  get size(): number {
-    return this.map.size
-  }
-
-  get byteSize(): number {
-    let size = 0
-    for (const bytes of this.values()) size += bytes.length
-    return size
-  }
-
-  equals(other: BlockMap): boolean {
-    if (this.size !== other.size) {
-      return false
-    }
-    for (const [cid, bytes] of this) {
-      const otherBytes = other.get(cid)
-      if (!otherBytes) return false
-      if (!ui8Equals(bytes, otherBytes)) {
-        return false
-      }
-    }
-    return true
-  }
-
-  *keys(): Generator<Cid, void, unknown> {
-    for (const key of this.map.keys()) {
-      yield parseCid(key)
-    }
-  }
-
-  *values(): Generator<Uint8Array, void, unknown> {
-    yield* this.map.values()
-  }
-
-  *[Symbol.iterator](): Generator<[Cid, Uint8Array], void, unknown> {
-    for (const [key, value] of this.map) {
-      yield [parseCid(key), value]
-    }
-  }
-}
-
-function toEntry([cid, bytes]: readonly [Cid, Uint8Array]): Entry {
-  return { cid, bytes }
-}
-
-type Entry = {
-  cid: Cid
-  bytes: Uint8Array
-}
-
-export default BlockMap

package/src/car.ts

@@ -1,357 +0,0 @@
-import { setImmediate } from 'node:timers/promises'
-// eslint-disable-next-line import/default, import/no-named-as-default-member
-import varint from 'varint'
-import * as cbor from '@atproto/lex-cbor'
-import { Cid, decodeCid, isCidForBytes } from '@atproto/lex-data'
-import { BlockMap } from './block-map.js'
-import { CarBlock, schema } from './types.js'
-import { concatBytesAsync } from './util.js'
-
-export async function* writeCarStream(
-  root: Cid | null,
-  blocks: AsyncIterable<CarBlock>,
-): AsyncIterable<Uint8Array> {
-  const header = new Uint8Array(
-    cbor.encode({
-      version: 1,
-      roots: root ? [root] : [],
-    }),
-  )
-  yield new Uint8Array(varint.encode(header.byteLength))
-  yield header
-  for await (const block of blocks) {
-    yield new Uint8Array(
-      varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength),
-    )
-    yield block.cid.bytes
-    yield block.bytes
-  }
-}
-
-export async function blocksToCarFile(
-  root: Cid | null,
-  blocks: BlockMap,
-): Promise<Uint8Array> {
-  return concatBytesAsync(blocksToCarStream(root, blocks))
-}
-
-export const blocksToCarStream = (
-  root: Cid | null,
-  blocks: BlockMap,
-): AsyncIterable<Uint8Array> => {
-  return writeCarStream(root, iterateBlocks(blocks))
-}
-
-async function* iterateBlocks(blocks: BlockMap) {
-  for (const entry of blocks.entries()) {
-    yield { cid: entry.cid, bytes: entry.bytes }
-  }
-}
-
-export type ReadCarOptions = {
-  /**
-   * When true, does not verify CID-to-content mapping within CAR.
-   */
-  skipCidVerification?: boolean
-}
-
-export const readCar = async (
-  bytes: Uint8Array,
-  opts?: ReadCarOptions,
-): Promise<{ roots: Cid[]; blocks: BlockMap }> => {
-  const { roots, blocks } = await readCarReader(new Ui8Reader(bytes), opts)
-  const blockMap = new BlockMap()
-  for await (const block of blocks) {
-    blockMap.set(block.cid, block.bytes)
-  }
-  return { roots, blocks: blockMap }
-}
-
-export const readCarWithRoot = async (
-  bytes: Uint8Array,
-  opts?: ReadCarOptions,
-): Promise<{ root: Cid; blocks: BlockMap }> => {
-  const { roots, blocks } = await readCar(bytes, opts)
-  if (roots.length !== 1) {
-    throw new Error(`Expected one root, got ${roots.length}`)
-  }
-  const root = roots[0]
-  return {
-    root,
-    blocks,
-  }
-}
-export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
-  dump: () => Promise<void>
-}
-
-export const readCarStream = async (
-  car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
-  opts?: ReadCarOptions,
-): Promise<{
-  roots: Cid[]
-  blocks: CarBlockIterable
-}> => {
-  return readCarReader(new BufferedReader(car), opts)
-}
-
-export const readCarReader = async (
-  reader: BytesReader,
-  opts?: ReadCarOptions,
-): Promise<{
-  roots: Cid[]
-  blocks: CarBlockIterable
-}> => {
-  try {
-    const headerSize = await readVarint(reader)
-    if (headerSize === null) {
-      throw new Error('Could not parse CAR header')
-    }
-    const headerBytes = await reader.read(headerSize)
-    const header = cbor.decode(headerBytes)
-    const result = schema.carHeader.safeParse(header)
-    if (!result.success) {
-      throw new Error('Could not parse CAR header', { cause: result.error })
-    }
-    return {
-      roots: result.data.roots,
-      blocks: readCarBlocksIter(reader, opts),
-    }
-  } catch (err) {
-    await reader.close()
-    throw err
-  }
-}
-
-const readCarBlocksIter = (
-  reader: BytesReader,
-  opts?: ReadCarOptions,
-): CarBlockIterable => {
-  let generator = readCarBlocksIterGenerator(reader)
-  if (!opts?.skipCidVerification) {
-    generator = verifyIncomingCarBlocks(generator)
-  }
-  return Object.assign(generator, {
-    async dump() {
-      // try/finally to ensure that reader.close is called even if blocks.return throws.
-      try {
-        // Prevent the iterator from being started after this method is called.
-        await generator.return()
-      } finally {
-        // @NOTE the "finally" block of the async generator won't be called
-        // if the iteration was never started so we need to manually close here.
-        await reader.close()
-      }
-    },
-  })
-}
-
-async function* readCarBlocksIterGenerator(
-  reader: BytesReader,
-): AsyncGenerator<CarBlock, void, unknown> {
-  let blocks = 0
-  try {
-    while (!reader.isDone) {
-      const blockSize = await readVarint(reader)
-      if (blockSize === null) {
-        break
-      }
-      const blockBytes = await reader.read(blockSize)
-      const cid = decodeCid(blockBytes.subarray(0, 36))
-      const bytes = blockBytes.subarray(36)
-      yield { cid, bytes }
-
-      // yield to the event loop every 25 blocks
-      // in the case the incoming CAR is synchronous, this can end up jamming up the thread
-      blocks++
-      if (blocks % 25 === 0) {
-        await setImmediate()
-      }
-    }
-  } finally {
-    await reader.close()
-  }
-}
-
-export async function* verifyIncomingCarBlocks(
-  car: AsyncIterable<CarBlock>,
-): AsyncGenerator<CarBlock, void, unknown> {
-  for await (const block of car) {
-    if (!(await isCidForBytes(block.cid, block.bytes))) {
-      throw new Error(`Not a valid CID for bytes (${block.cid.toString()})`)
-    }
-    yield block
-  }
-}
-
-const readVarint = async (reader: BytesReader): Promise<number | null> => {
-  let done = false
-  const bytes: Uint8Array[] = []
-  while (!done) {
-    const byte = await reader.read(1)
-    if (byte.byteLength === 0) {
-      if (bytes.length > 0) {
-        throw new Error('could not parse varint')
-      } else {
-        return null
-      }
-    }
-    bytes.push(byte)
-    if (byte[0] < 128) {
-      done = true
-    }
-  }
-  const concatted = Buffer.concat(bytes)
-  return varint.decode(concatted)
-}
-
-interface BytesReader {
-  isDone: boolean
-  read(bytesToRead: number): Promise<Uint8Array>
-  close(): Promise<void>
-}
-
-class Ui8Reader implements BytesReader {
-  idx = 0
-  isDone = false
-
-  constructor(public bytes: Uint8Array) {}
-
-  async read(bytesToRead: number): Promise<Uint8Array> {
-    const value = this.bytes.subarray(this.idx, this.idx + bytesToRead)
-    this.idx += bytesToRead
-    if (this.idx >= this.bytes.length) {
-      this.isDone = true
-    }
-    return value
-  }
-
-  async close(): Promise<void> {}
-}
-
-/**
- * This code was optimized for performance. See
- * {@link https://github.com/bluesky-social/atproto/pull/4729 #4729} for more details
- * and benchmarks.
- */
-class BufferedReader implements BytesReader {
-  iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>
-  isDone = false
-
-  /** fifo list of chunks to consume */
-  private chunks: Uint8Array[] = []
-
-  constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {
-    this.iterator =
-      Symbol.asyncIterator in stream
-        ? stream[Symbol.asyncIterator]()
-        : stream[Symbol.iterator]()
-  }
-
-  /** Number of bytes currently buffered and available for reading */
-  get bufferedByteLength() {
-    let total = 0
-    for (let i = 0; i < this.chunks.length; i++) {
-      total += this.chunks[i].byteLength
-    }
-    return total
-  }
-
-  /**
-   * @note concurrent reads are **NOT** supported by the current implementation
-   * and would require call to readUntilBuffered to be using a fifo lock for
-   * read()s to be processed in fifo order.
-   */
-  async read(bytesToRead: number): Promise<Uint8Array> {
-    const bytesNeeded = bytesToRead - this.bufferedByteLength
-    if (bytesNeeded > 0 && !this.isDone) {
-      await this.readUntilBuffered(bytesNeeded)
-    }
-
-    const resultLength = Math.min(bytesToRead, this.bufferedByteLength)
-    if (resultLength <= 0) return new Uint8Array()
-
-    const firstChunk = this.consumeChunk(resultLength)
-    if (firstChunk.byteLength === resultLength) {
-      // If the data consumed from the first chunk contains all we need, return
-      // it as-is. This allows to avoid any copy operation.
-      return firstChunk
-    }
-
-    // The first chunk does not have all the data we need. We have to copy
-    // multiple chunks into a larger buffer
-    const result = new Uint8Array(resultLength)
-    let resultWriteIndex = 0
-
-    // Copy the first chunk into the result buffer
-    result.set(firstChunk, resultWriteIndex)
-    resultWriteIndex += firstChunk.byteLength
-
-    // Copy more chunks as needed (we use do-while because we *know* we need
-    // more than one chunk)
-    do {
-      const missingLength = resultLength - resultWriteIndex
-      const currentChunk = this.consumeChunk(missingLength)
-
-      result.set(currentChunk, resultWriteIndex)
-      resultWriteIndex += currentChunk.byteLength
-    } while (resultWriteIndex < resultLength)
-
-    return result
-  }
-
-  private async readUntilBuffered(bytesNeeded: number) {
-    let bytesRead = 0
-    while (bytesRead < bytesNeeded) {
-      const next = await this.iterator.next()
-      if (next.done) {
-        this.isDone = true
-        break
-      } else {
-        this.chunks.push(next.value)
-        bytesRead += next.value.byteLength
-      }
-    }
-    return bytesRead
-  }
-
-  private consumeChunk(bytesToConsume: number) {
-    const firstChunk = this.chunks[0]!
-    if (bytesToConsume < firstChunk.byteLength) {
-      // return a sub-view of the data being read and replace the first chunk
-      // with a sub-view that does not contain that data.
-
-      // @NOTE for some reason, subarray() revealed to be 7-8% slower in NodeJS
-      // benchmarks.
-
-      // this.chunks[0] = firstChunk.subarray(bytesToConsume)
-      // return firstChunk.subarray(0, bytesToConsume)
-
-      this.chunks[0] = new Uint8Array(
-        firstChunk.buffer,
-        firstChunk.byteOffset + bytesToConsume,
-        firstChunk.byteLength - bytesToConsume,
-      )
-      return new Uint8Array(
-        firstChunk.buffer,
-        firstChunk.byteOffset,
-        bytesToConsume,
-      )
-    } else {
-      // First chunk is being read in full, discard it
-      this.chunks.shift()
-      return firstChunk
-    }
-  }
-
-  async close(): Promise<void> {
-    try {
-      if (!this.isDone && this.iterator.return) {
-        await this.iterator.return()
-      }
-    } finally {
-      this.isDone = true
-      this.chunks.length = 0
-    }
-  }
-}

package/src/cid-set.ts

@@ -1,55 +0,0 @@
-import { Cid, parseCid } from '@atproto/lex-data'
-
-export class CidSet implements Iterable<Cid> {
-  private set: Set<string>
-
-  constructor(arr: Cid[] = []) {
-    const strArr = arr.map((c) => c.toString())
-    this.set = new Set(strArr)
-  }
-
-  add(cid: Cid): CidSet {
-    this.set.add(cid.toString())
-    return this
-  }
-
-  addSet(toMerge: CidSet): CidSet {
-    for (const c of toMerge.set) this.set.add(c)
-    return this
-  }
-
-  subtractSet(toSubtract: CidSet): CidSet {
-    for (const c of toSubtract.set) this.set.delete(c)
-    return this
-  }
-
-  delete(cid: Cid) {
-    this.set.delete(cid.toString())
-    return this
-  }
-
-  has(cid: Cid): boolean {
-    return this.set.has(cid.toString())
-  }
-
-  size(): number {
-    return this.set.size
-  }
-
-  clear(): CidSet {
-    this.set.clear()
-    return this
-  }
-
-  toList(): Cid[] {
-    return Array.from(this)
-  }
-
-  *[Symbol.iterator](): Generator<Cid, void, unknown> {
-    for (const c of this.set) {
-      yield parseCid(c)
-    }
-  }
-}
-
-export default CidSet