@fireproof/vendor 0.0.0-smoke

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts

@@ -0,0 +1,39 @@
+import filter from 'it-filter'
+import map from 'it-map'
+import parallel from 'it-parallel'
+import { pipe } from 'it-pipe'
+import { CustomProgressEvent } from 'progress-events'
+import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js'
+
+const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
+  async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
+    const offset = options.offset ?? 0
+    const length = options.length ?? node.Links.length
+    const links = node.Links.slice(offset, length)
+
+    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:directory', {
+      cid
+    }))
+
+    yield * pipe(
+      links,
+      source => map(source, link => {
+        return async () => {
+          const linkName = link.Name ?? ''
+          const linkPath = `${path}/${linkName}`
+          const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options)
+          return result.entry
+        }
+      }),
+      source => parallel(source, {
+        ordered: true,
+        concurrency: options.blockReadConcurrency
+      }),
+      source => filter(source, entry => entry != null)
+    )
+  }
+
+  return yieldDirectoryContent
+}
+
+export default directoryContent

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts

@@ -0,0 +1,198 @@
+import * as dagPb from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import map from 'it-map'
+import parallel from 'it-parallel'
+import { pipe } from 'it-pipe'
+import { type Pushable, pushable } from 'it-pushable'
+import * as raw from 'multiformats/codecs/raw'
+import PQueue from 'p-queue'
+import { CustomProgressEvent } from 'progress-events'
+import { NotUnixFSError, OverReadError, UnderReadError } from '../../../errors.js'
+import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
+import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
+import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage, ExportProgress, ExportWalk } from '../../../index.js'
+
+async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8Array, queue: Pushable<Uint8Array>, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise<void> {
+  // a `raw` node
+  if (node instanceof Uint8Array) {
+    const buf = extractDataFromBlock(node, streamPosition, start, end)
+
+    queue.push(buf)
+
+    return
+  }
+
+  if (node.Data == null) {
+    throw new NotUnixFSError('no data in PBNode')
+  }
+
+  let file: UnixFS
+
+  try {
+    file = UnixFS.unmarshal(node.Data)
+  } catch (err: any) {
+    throw new NotUnixFSError(err.message)
+  }
+
+  // might be a unixfs `raw` node or have data on intermediate nodes
+  if (file.data != null) {
+    const data = file.data
+    const buf = extractDataFromBlock(data, streamPosition, start, end)
+
+    queue.push(buf)
+
+    streamPosition += BigInt(buf.byteLength)
+  }
+
+  const childOps: Array<{ link: dagPb.PBLink, blockStart: bigint }> = []
+
+  if (node.Links.length !== file.blockSizes.length) {
+    throw new NotUnixFSError('Inconsistent block sizes and dag links')
+  }
+
+  for (let i = 0; i < node.Links.length; i++) {
+    const childLink = node.Links[i]
+    const childStart = streamPosition // inclusive
+    const childEnd = childStart + file.blockSizes[i] // exclusive
+
+    if ((start >= childStart && start < childEnd) || // child has offset byte
+        (end >= childStart && end <= childEnd) || // child has end byte
+        (start < childStart && end > childEnd)) { // child is between offset and end bytes
+      childOps.push({
+        link: childLink,
+        blockStart: streamPosition
+      })
+    }
+
+    streamPosition = childEnd
+
+    if (streamPosition > end) {
+      break
+    }
+  }
+
+  await pipe(
+    childOps,
+    (source) => map(source, (op) => {
+      return async () => {
+        const block = await blockstore.get(op.link.Hash, options)
+
+        return {
+          ...op,
+          block
+        }
+      }
+    }),
+    (source) => parallel(source, {
+      ordered: true,
+      concurrency: options.blockReadConcurrency
+    }),
+    async (source) => {
+      for await (const { link, block, blockStart } of source) {
+        let child: dagPb.PBNode | Uint8Array
+        switch (link.Hash.code) {
+          case dagPb.code:
+            child = dagPb.decode(block)
+            break
+          case raw.code:
+            child = block
+            break
+          default:
+            queue.end(new NotUnixFSError(`Unsupported codec: ${link.Hash.code}`))
+            return
+        }
+
+        // create a queue for this child - we use a queue instead of recursion
+        // to avoid overflowing the stack
+        const childQueue = new PQueue({
+          concurrency: 1
+        })
+        // if any of the child jobs error, end the read queue with the error
+        childQueue.on('error', error => {
+          queue.end(error)
+        })
+
+        // if the job rejects the 'error' event will be emitted on the child queue
+        void childQueue.add(async () => {
+          options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:file', {
+            cid: link.Hash
+          }))
+
+          await walkDAG(blockstore, child, queue, blockStart, start, end, options)
+        })
+
+        // wait for this child to complete before moving on to the next
+        await childQueue.onIdle()
+      }
+    }
+  )
+
+  if (streamPosition >= end) {
+    queue.end()
+  }
+}
+
+const fileContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
+  async function * yieldFileContent (options: ExporterOptions = {}): UnixfsV1FileContent {
+    const fileSize = unixfs.fileSize()
+
+    if (fileSize === undefined) {
+      throw new Error('File was a directory')
+    }
+
+    const {
+      start,
+      end
+    } = validateOffsetAndLength(fileSize, options.offset, options.length)
+
+    if (end === 0n) {
+      return
+    }
+
+    let read = 0n
+    const wanted = end - start
+    const queue = pushable()
+
+    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:file', {
+      cid
+    }))
+
+    void walkDAG(blockstore, node, queue, 0n, start, end, options)
+      .catch(err => {
+        queue.end(err)
+      })
+
+    for await (const buf of queue) {
+      if (buf == null) {
+        continue
+      }
+
+      read += BigInt(buf.byteLength)
+
+      if (read > wanted) {
+        queue.end()
+        throw new OverReadError('Read too many bytes - the file size reported by the UnixFS data in the root node may be incorrect')
+      }
+
+      if (read === wanted) {
+        queue.end()
+      }
+
+      options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:unixfs:file', {
+        bytesRead: read,
+        totalBytes: wanted,
+        fileSize
+      }))
+
+      yield buf
+    }
+
+    if (read < wanted) {
+      throw new UnderReadError('Traversed entire DAG but did not read enough bytes')
+    }
+  }
+
+  return yieldFileContent
+}
+
+export default fileContent

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts

@@ -0,0 +1,76 @@
+import { decode, type PBNode } from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import map from 'it-map'
+import parallel from 'it-parallel'
+import { pipe } from 'it-pipe'
+import { CustomProgressEvent } from 'progress-events'
+import { NotUnixFSError } from '../../../errors.js'
+import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js'
+
+const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
+  function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
+    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
+      cid
+    }))
+
+    return listDirectory(node, path, resolve, depth, blockstore, options)
+  }
+
+  return yieldHamtDirectoryContent
+}
+
+async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent {
+  const links = node.Links
+
+  if (node.Data == null) {
+    throw new NotUnixFSError('no data in PBNode')
+  }
+
+  let dir: UnixFS
+  try {
+    dir = UnixFS.unmarshal(node.Data)
+  } catch (err: any) {
+    throw new NotUnixFSError(err.message)
+  }
+
+  if (dir.fanout == null) {
+    throw new NotUnixFSError('missing fanout')
+  }
+
+  const padLength = (dir.fanout - 1n).toString(16).length
+
+  const results = pipe(
+    links,
+    source => map(source, link => {
+      return async () => {
+        const name = link.Name != null ? link.Name.substring(padLength) : null
+
+        if (name != null && name !== '') {
+          const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options)
+
+          return { entries: result.entry == null ? [] : [result.entry] }
+        } else {
+          // descend into subshard
+          const block = await blockstore.get(link.Hash, options)
+          node = decode(block)
+
+          options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
+            cid: link.Hash
+          }))
+
+          return { entries: listDirectory(node, path, resolve, depth, blockstore, options) }
+        }
+      }
+    }),
+    source => parallel(source, {
+      ordered: true,
+      concurrency: options.blockReadConcurrency
+    })
+  )
+
+  for await (const { entries } of results) {
+    yield * entries
+  }
+}
+
+export default hamtShardedDirectoryContent

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.ts

@@ -0,0 +1,37 @@
+import { CustomProgressEvent } from 'progress-events'
+import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
+import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
+import type { ExporterOptions, ExportProgress, ExportWalk, UnixfsV1Resolver } from '../../../index.js'
+
+const rawContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
+  function * yieldRawContent (options: ExporterOptions = {}): Generator<Uint8Array, void, undefined> {
+    if (unixfs.data == null) {
+      throw new Error('Raw block had no data')
+    }
+
+    const size = unixfs.data.length
+
+    const {
+      start,
+      end
+    } = validateOffsetAndLength(size, options.offset, options.length)
+
+    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:raw', {
+      cid
+    }))
+
+    const buf = extractDataFromBlock(unixfs.data, 0n, start, end)
+
+    options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:unixfs:raw', {
+      bytesRead: BigInt(buf.byteLength),
+      totalBytes: end - start,
+      fileSize: BigInt(unixfs.data.byteLength)
+    }))
+
+    yield buf
+  }
+
+  return yieldRawContent
+}
+
+export default rawContent

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts

@@ -0,0 +1,121 @@
+import { decode, type PBNode } from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import { NotFoundError, NotUnixFSError } from '../../errors.js'
+import findShardCid from '../../utils/find-cid-in-shard.js'
+import contentDirectory from './content/directory.js'
+import contentFile from './content/file.js'
+import contentHamtShardedDirectory from './content/hamt-sharded-directory.js'
+import type { Resolver, UnixfsV1Resolver } from '../../index.js'
+import type { CID } from 'multiformats/cid'
+
+const findLinkCid = (node: PBNode, name: string): CID | undefined => {
+  const link = node.Links.find(link => link.Name === name)
+
+  return link?.Hash
+}
+
+const contentExporters: Record<string, UnixfsV1Resolver> = {
+  raw: contentFile,
+  file: contentFile,
+  directory: contentDirectory,
+  'hamt-sharded-directory': contentHamtShardedDirectory,
+  metadata: (cid, node, unixfs, path, resolve, depth, blockstore) => {
+    return () => []
+  },
+  symlink: (cid, node, unixfs, path, resolve, depth, blockstore) => {
+    return () => []
+  }
+}
+
+// @ts-expect-error types are wrong
+const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
+  const block = await blockstore.get(cid, options)
+  const node = decode(block)
+  let unixfs
+  let next
+
+  if (name == null) {
+    name = cid.toString()
+  }
+
+  if (node.Data == null) {
+    throw new NotUnixFSError('no data in PBNode')
+  }
+
+  try {
+    unixfs = UnixFS.unmarshal(node.Data)
+  } catch (err: any) {
+    // non-UnixFS dag-pb node? It could happen.
+    throw new NotUnixFSError(err.message)
+  }
+
+  if (path == null) {
+    path = name
+  }
+
+  if (toResolve.length > 0) {
+    let linkCid
+
+    if (unixfs?.type === 'hamt-sharded-directory') {
+      // special case - unixfs v1 hamt shards
+      linkCid = await findShardCid(node, toResolve[0], blockstore)
+    } else {
+      linkCid = findLinkCid(node, toResolve[0])
+    }
+
+    if (linkCid == null) {
+      throw new NotFoundError('file does not exist')
+    }
+
+    // remove the path component we have resolved
+    const nextName = toResolve.shift()
+    const nextPath = `${path}/${nextName}`
+
+    next = {
+      cid: linkCid,
+      toResolve,
+      name: nextName ?? '',
+      path: nextPath
+    }
+  }
+
+  const content = contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, blockstore)
+
+  if (content == null) {
+    throw new NotFoundError('could not find content exporter')
+  }
+
+  if (unixfs.isDirectory()) {
+    return {
+      entry: {
+        type: 'directory',
+        name,
+        path,
+        cid,
+        content,
+        unixfs,
+        depth,
+        node,
+        size: unixfs.fileSize()
+      },
+      next
+    }
+  }
+
+  return {
+    entry: {
+      type: 'file',
+      name,
+      path,
+      cid,
+      content,
+      unixfs,
+      depth,
+      node,
+      size: unixfs.fileSize()
+    },
+    next
+  }
+}
+
+export default unixFsResolver

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/utils/extract-data-from-block.ts

@@ -0,0 +1,24 @@
+function extractDataFromBlock (block: Uint8Array, blockStart: bigint, requestedStart: bigint, requestedEnd: bigint): Uint8Array {
+  const blockLength = BigInt(block.length)
+  const blockEnd = BigInt(blockStart + blockLength)
+
+  if (requestedStart >= blockEnd || requestedEnd < blockStart) {
+    // If we are looking for a byte range that is starts after the start of the block,
+    // return an empty block.  This can happen when internal nodes contain data
+    return new Uint8Array(0)
+  }
+
+  if (requestedEnd >= blockStart && requestedEnd < blockEnd) {
+    // If the end byte is in the current block, truncate the block to the end byte
+    block = block.subarray(0, Number(requestedEnd - blockStart))
+  }
+
+  if (requestedStart >= blockStart && requestedStart < blockEnd) {
+    // If the start byte is in the current block, skip to the start byte
+    block = block.subarray(Number(requestedStart - blockStart))
+  }
+
+  return block
+}
+
+export default extractDataFromBlock

package/src/ipfs-unixfs-exporter/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts

@@ -0,0 +1,149 @@
+import { decode, type PBLink, type PBNode } from '@ipld/dag-pb'
+import { murmur3128 } from '@multiformats/murmur3'
+import { Bucket, type BucketPosition, createHAMT } from 'hamt-sharding'
+import { UnixFS } from 'ipfs-unixfs'
+import { NotUnixFSError } from '../errors.js'
+import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js'
+import type { CID } from 'multiformats/cid'
+
+// FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js
+const hashFn = async function (buf: Uint8Array): Promise<Uint8Array> {
+  return (await murmur3128.encode(buf))
+    // Murmur3 outputs 128 bit but, accidentally, IPFS Go's
+    // implementation only uses the first 64, so we must do the same
+    // for parity..
+    .slice(0, 8)
+    // Invert buffer because that's how Go impl does it
+    .reverse()
+}
+
+const addLinksToHamtBucket = async (links: PBLink[], bucket: Bucket<boolean>, rootBucket: Bucket<boolean>): Promise<void> => {
+  const padLength = (bucket.tableSize() - 1).toString(16).length
+  await Promise.all(
+    links.map(async link => {
+      if (link.Name == null) {
+        // TODO(@rvagg): what do? this is technically possible
+        throw new Error('Unexpected Link without a Name')
+      }
+      if (link.Name.length === padLength) {
+        const pos = parseInt(link.Name, 16)
+
+        bucket._putObjectAt(pos, new Bucket({
+          hash: rootBucket._options.hash,
+          bits: rootBucket._options.bits
+        }, bucket, pos))
+        return
+      }
+
+      await rootBucket.put(link.Name.substring(2), true)
+    })
+  )
+}
+
+const toPrefix = (position: number, padLength: number): string => {
+  return position
+    .toString(16)
+    .toUpperCase()
+    .padStart(padLength, '0')
+    .substring(0, padLength)
+}
+
+const toBucketPath = (position: BucketPosition<boolean>): Array<Bucket<boolean>> => {
+  let bucket = position.bucket
+  const path = []
+
+  while (bucket._parent != null) {
+    path.push(bucket)
+
+    bucket = bucket._parent
+  }
+
+  path.push(bucket)
+
+  return path.reverse()
+}
+
+const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions): Promise<CID | undefined> => {
+  if (context == null) {
+    if (node.Data == null) {
+      throw new NotUnixFSError('no data in PBNode')
+    }
+
+    let dir: UnixFS
+    try {
+      dir = UnixFS.unmarshal(node.Data)
+    } catch (err: any) {
+      throw new NotUnixFSError(err.message)
+    }
+
+    if (dir.type !== 'hamt-sharded-directory') {
+      throw new NotUnixFSError('not a HAMT')
+    }
+    if (dir.fanout == null) {
+      throw new NotUnixFSError('missing fanout')
+    }
+
+    const rootBucket = createHAMT<boolean>({
+      hashFn,
+      bits: Math.log2(Number(dir.fanout))
+    })
+
+    context = {
+      rootBucket,
+      hamtDepth: 1,
+      lastBucket: rootBucket
+    }
+  }
+
+  const padLength = (context.lastBucket.tableSize() - 1).toString(16).length
+
+  await addLinksToHamtBucket(node.Links, context.lastBucket, context.rootBucket)
+
+  const position = await context.rootBucket._findNewBucketAndPos(name)
+  let prefix = toPrefix(position.pos, padLength)
+  const bucketPath = toBucketPath(position)
+
+  if (bucketPath.length > context.hamtDepth) {
+    context.lastBucket = bucketPath[context.hamtDepth]
+
+    prefix = toPrefix(context.lastBucket._posAtParent, padLength)
+  }
+
+  const link = node.Links.find(link => {
+    if (link.Name == null) {
+      return false
+    }
+
+    const entryPrefix = link.Name.substring(0, padLength)
+    const entryName = link.Name.substring(padLength)
+
+    if (entryPrefix !== prefix) {
+      // not the entry or subshard we're looking for
+      return false
+    }
+
+    if (entryName !== '' && entryName !== name) {
+      // not the entry we're looking for
+      return false
+    }
+
+    return true
+  })
+
+  if (link == null) {
+    return
+  }
+
+  if (link.Name != null && link.Name.substring(padLength) === name) {
+    return link.Hash
+  }
+
+  context.hamtDepth++
+
+  const block = await blockstore.get(link.Hash, options)
+  node = decode(block)
+
+  return findShardCid(node, name, blockstore, context, options)
+}
+
+export default findShardCid