@fireproof/vendor 1.0.4 → 1.1.0-dev

package/src/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts

@@ -1,39 +0,0 @@
-import filter from 'it-filter'
-import map from 'it-map'
-import parallel from 'it-parallel'
-import { pipe } from 'it-pipe'
-import { CustomProgressEvent } from 'progress-events'
-import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js'
-
-const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
-  async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
-    const offset = options.offset ?? 0
-    const length = options.length ?? node.Links.length
-    const links = node.Links.slice(offset, length)
-
-    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:directory', {
-      cid
-    }))
-
-    yield * pipe(
-      links,
-      source => map(source, link => {
-        return async () => {
-          const linkName = link.Name ?? ''
-          const linkPath = `${path}/${linkName}`
-          const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options)
-          return result.entry
-        }
-      }),
-      source => parallel(source, {
-        ordered: true,
-        concurrency: options.blockReadConcurrency
-      }),
-      source => filter(source, entry => entry != null)
-    )
-  }
-
-  return yieldDirectoryContent
-}
-
-export default directoryContent

package/src/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts

@@ -1,198 +0,0 @@
-import * as dagPb from '@ipld/dag-pb'
-import { UnixFS } from 'ipfs-unixfs'
-import map from 'it-map'
-import parallel from 'it-parallel'
-import { pipe } from 'it-pipe'
-import { type Pushable, pushable } from 'it-pushable'
-import * as raw from 'multiformats/codecs/raw'
-import PQueue from 'p-queue'
-import { CustomProgressEvent } from 'progress-events'
-import { NotUnixFSError, OverReadError, UnderReadError } from '../../../errors.js'
-import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
-import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
-import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage, ExportProgress, ExportWalk } from '../../../index.js'
-
-async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8Array, queue: Pushable<Uint8Array>, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise<void> {
-  // a `raw` node
-  if (node instanceof Uint8Array) {
-    const buf = extractDataFromBlock(node, streamPosition, start, end)
-
-    queue.push(buf)
-
-    return
-  }
-
-  if (node.Data == null) {
-    throw new NotUnixFSError('no data in PBNode')
-  }
-
-  let file: UnixFS
-
-  try {
-    file = UnixFS.unmarshal(node.Data)
-  } catch (err: any) {
-    throw new NotUnixFSError(err.message)
-  }
-
-  // might be a unixfs `raw` node or have data on intermediate nodes
-  if (file.data != null) {
-    const data = file.data
-    const buf = extractDataFromBlock(data, streamPosition, start, end)
-
-    queue.push(buf)
-
-    streamPosition += BigInt(buf.byteLength)
-  }
-
-  const childOps: Array<{ link: dagPb.PBLink, blockStart: bigint }> = []
-
-  if (node.Links.length !== file.blockSizes.length) {
-    throw new NotUnixFSError('Inconsistent block sizes and dag links')
-  }
-
-  for (let i = 0; i < node.Links.length; i++) {
-    const childLink = node.Links[i]
-    const childStart = streamPosition // inclusive
-    const childEnd = childStart + file.blockSizes[i] // exclusive
-
-    if ((start >= childStart && start < childEnd) || // child has offset byte
-        (end >= childStart && end <= childEnd) || // child has end byte
-        (start < childStart && end > childEnd)) { // child is between offset and end bytes
-      childOps.push({
-        link: childLink,
-        blockStart: streamPosition
-      })
-    }
-
-    streamPosition = childEnd
-
-    if (streamPosition > end) {
-      break
-    }
-  }
-
-  await pipe(
-    childOps,
-    (source) => map(source, (op) => {
-      return async () => {
-        const block = await blockstore.get(op.link.Hash, options)
-
-        return {
-          ...op,
-          block
-        }
-      }
-    }),
-    (source) => parallel(source, {
-      ordered: true,
-      concurrency: options.blockReadConcurrency
-    }),
-    async (source) => {
-      for await (const { link, block, blockStart } of source) {
-        let child: dagPb.PBNode | Uint8Array
-        switch (link.Hash.code) {
-          case dagPb.code:
-            child = dagPb.decode(block)
-            break
-          case raw.code:
-            child = block
-            break
-          default:
-            queue.end(new NotUnixFSError(`Unsupported codec: ${link.Hash.code}`))
-            return
-        }
-
-        // create a queue for this child - we use a queue instead of recursion
-        // to avoid overflowing the stack
-        const childQueue = new PQueue({
-          concurrency: 1
-        })
-        // if any of the child jobs error, end the read queue with the error
-        childQueue.on('error', error => {
-          queue.end(error)
-        })
-
-        // if the job rejects the 'error' event will be emitted on the child queue
-        void childQueue.add(async () => {
-          options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:file', {
-            cid: link.Hash
-          }))
-
-          await walkDAG(blockstore, child, queue, blockStart, start, end, options)
-        })
-
-        // wait for this child to complete before moving on to the next
-        await childQueue.onIdle()
-      }
-    }
-  )
-
-  if (streamPosition >= end) {
-    queue.end()
-  }
-}
-
-const fileContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
-  async function * yieldFileContent (options: ExporterOptions = {}): UnixfsV1FileContent {
-    const fileSize = unixfs.fileSize()
-
-    if (fileSize === undefined) {
-      throw new Error('File was a directory')
-    }
-
-    const {
-      start,
-      end
-    } = validateOffsetAndLength(fileSize, options.offset, options.length)
-
-    if (end === 0n) {
-      return
-    }
-
-    let read = 0n
-    const wanted = end - start
-    const queue = pushable()
-
-    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:file', {
-      cid
-    }))
-
-    void walkDAG(blockstore, node, queue, 0n, start, end, options)
-      .catch(err => {
-        queue.end(err)
-      })
-
-    for await (const buf of queue) {
-      if (buf == null) {
-        continue
-      }
-
-      read += BigInt(buf.byteLength)
-
-      if (read > wanted) {
-        queue.end()
-        throw new OverReadError('Read too many bytes - the file size reported by the UnixFS data in the root node may be incorrect')
-      }
-
-      if (read === wanted) {
-        queue.end()
-      }
-
-      options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:unixfs:file', {
-        bytesRead: read,
-        totalBytes: wanted,
-        fileSize
-      }))
-
-      yield buf
-    }
-
-    if (read < wanted) {
-      throw new UnderReadError('Traversed entire DAG but did not read enough bytes')
-    }
-  }
-
-  return yieldFileContent
-}
-
-export default fileContent

package/src/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts

@@ -1,76 +0,0 @@
-import { decode, type PBNode } from '@ipld/dag-pb'
-import { UnixFS } from 'ipfs-unixfs'
-import map from 'it-map'
-import parallel from 'it-parallel'
-import { pipe } from 'it-pipe'
-import { CustomProgressEvent } from 'progress-events'
-import { NotUnixFSError } from '../../../errors.js'
-import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js'
-
-const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
-  function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
-    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
-      cid
-    }))
-
-    return listDirectory(node, path, resolve, depth, blockstore, options)
-  }
-
-  return yieldHamtDirectoryContent
-}
-
-async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent {
-  const links = node.Links
-
-  if (node.Data == null) {
-    throw new NotUnixFSError('no data in PBNode')
-  }
-
-  let dir: UnixFS
-  try {
-    dir = UnixFS.unmarshal(node.Data)
-  } catch (err: any) {
-    throw new NotUnixFSError(err.message)
-  }
-
-  if (dir.fanout == null) {
-    throw new NotUnixFSError('missing fanout')
-  }
-
-  const padLength = (dir.fanout - 1n).toString(16).length
-
-  const results = pipe(
-    links,
-    source => map(source, link => {
-      return async () => {
-        const name = link.Name != null ? link.Name.substring(padLength) : null
-
-        if (name != null && name !== '') {
-          const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options)
-
-          return { entries: result.entry == null ? [] : [result.entry] }
-        } else {
-          // descend into subshard
-          const block = await blockstore.get(link.Hash, options)
-          node = decode(block)
-
-          options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
-            cid: link.Hash
-          }))
-
-          return { entries: listDirectory(node, path, resolve, depth, blockstore, options) }
-        }
-      }
-    }),
-    source => parallel(source, {
-      ordered: true,
-      concurrency: options.blockReadConcurrency
-    })
-  )
-
-  for await (const { entries } of results) {
-    yield * entries
-  }
-}
-
-export default hamtShardedDirectoryContent

package/src/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.ts

@@ -1,37 +0,0 @@
-import { CustomProgressEvent } from 'progress-events'
-import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
-import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
-import type { ExporterOptions, ExportProgress, ExportWalk, UnixfsV1Resolver } from '../../../index.js'
-
-const rawContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
-  function * yieldRawContent (options: ExporterOptions = {}): Generator<Uint8Array, void, undefined> {
-    if (unixfs.data == null) {
-      throw new Error('Raw block had no data')
-    }
-
-    const size = unixfs.data.length
-
-    const {
-      start,
-      end
-    } = validateOffsetAndLength(size, options.offset, options.length)
-
-    options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:raw', {
-      cid
-    }))
-
-    const buf = extractDataFromBlock(unixfs.data, 0n, start, end)
-
-    options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:unixfs:raw', {
-      bytesRead: BigInt(buf.byteLength),
-      totalBytes: end - start,
-      fileSize: BigInt(unixfs.data.byteLength)
-    }))
-
-    yield buf
-  }
-
-  return yieldRawContent
-}
-
-export default rawContent

package/src/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts

@@ -1,121 +0,0 @@
-import { decode, type PBNode } from '@ipld/dag-pb'
-import { UnixFS } from 'ipfs-unixfs'
-import { NotFoundError, NotUnixFSError } from '../../errors.js'
-import findShardCid from '../../utils/find-cid-in-shard.js'
-import contentDirectory from './content/directory.js'
-import contentFile from './content/file.js'
-import contentHamtShardedDirectory from './content/hamt-sharded-directory.js'
-import type { Resolver, UnixfsV1Resolver } from '../../index.js'
-import type { CID } from 'multiformats/cid'
-
-const findLinkCid = (node: PBNode, name: string): CID | undefined => {
-  const link = node.Links.find(link => link.Name === name)
-
-  return link?.Hash
-}
-
-const contentExporters: Record<string, UnixfsV1Resolver> = {
-  raw: contentFile,
-  file: contentFile,
-  directory: contentDirectory,
-  'hamt-sharded-directory': contentHamtShardedDirectory,
-  metadata: (cid, node, unixfs, path, resolve, depth, blockstore) => {
-    return () => []
-  },
-  symlink: (cid, node, unixfs, path, resolve, depth, blockstore) => {
-    return () => []
-  }
-}
-
-// @ts-expect-error types are wrong
-const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
-  const block = await blockstore.get(cid, options)
-  const node = decode(block)
-  let unixfs
-  let next
-
-  if (name == null) {
-    name = cid.toString()
-  }
-
-  if (node.Data == null) {
-    throw new NotUnixFSError('no data in PBNode')
-  }
-
-  try {
-    unixfs = UnixFS.unmarshal(node.Data)
-  } catch (err: any) {
-    // non-UnixFS dag-pb node? It could happen.
-    throw new NotUnixFSError(err.message)
-  }
-
-  if (path == null) {
-    path = name
-  }
-
-  if (toResolve.length > 0) {
-    let linkCid
-
-    if (unixfs?.type === 'hamt-sharded-directory') {
-      // special case - unixfs v1 hamt shards
-      linkCid = await findShardCid(node, toResolve[0], blockstore)
-    } else {
-      linkCid = findLinkCid(node, toResolve[0])
-    }
-
-    if (linkCid == null) {
-      throw new NotFoundError('file does not exist')
-    }
-
-    // remove the path component we have resolved
-    const nextName = toResolve.shift()
-    const nextPath = `${path}/${nextName}`
-
-    next = {
-      cid: linkCid,
-      toResolve,
-      name: nextName ?? '',
-      path: nextPath
-    }
-  }
-
-  const content = contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, blockstore)
-
-  if (content == null) {
-    throw new NotFoundError('could not find content exporter')
-  }
-
-  if (unixfs.isDirectory()) {
-    return {
-      entry: {
-        type: 'directory',
-        name,
-        path,
-        cid,
-        content,
-        unixfs,
-        depth,
-        node,
-        size: unixfs.fileSize()
-      },
-      next
-    }
-  }
-
-  return {
-    entry: {
-      type: 'file',
-      name,
-      path,
-      cid,
-      content,
-      unixfs,
-      depth,
-      node,
-      size: unixfs.fileSize()
-    },
-    next
-  }
-}
-
-export default unixFsResolver

package/src/ipfs-unixfs-exporter/src/utils/extract-data-from-block.ts

@@ -1,24 +0,0 @@
-function extractDataFromBlock (block: Uint8Array, blockStart: bigint, requestedStart: bigint, requestedEnd: bigint): Uint8Array {
-  const blockLength = BigInt(block.length)
-  const blockEnd = BigInt(blockStart + blockLength)
-
-  if (requestedStart >= blockEnd || requestedEnd < blockStart) {
-    // If we are looking for a byte range that is starts after the start of the block,
-    // return an empty block.  This can happen when internal nodes contain data
-    return new Uint8Array(0)
-  }
-
-  if (requestedEnd >= blockStart && requestedEnd < blockEnd) {
-    // If the end byte is in the current block, truncate the block to the end byte
-    block = block.subarray(0, Number(requestedEnd - blockStart))
-  }
-
-  if (requestedStart >= blockStart && requestedStart < blockEnd) {
-    // If the start byte is in the current block, skip to the start byte
-    block = block.subarray(Number(requestedStart - blockStart))
-  }
-
-  return block
-}
-
-export default extractDataFromBlock

package/src/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts

@@ -1,149 +0,0 @@
-import { decode, type PBLink, type PBNode } from '@ipld/dag-pb'
-import { murmur3128 } from '@multiformats/murmur3'
-import { Bucket, type BucketPosition, createHAMT } from 'hamt-sharding'
-import { UnixFS } from 'ipfs-unixfs'
-import { NotUnixFSError } from '../errors.js'
-import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js'
-import type { CID } from 'multiformats/cid'
-
-// FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js
-const hashFn = async function (buf: Uint8Array): Promise<Uint8Array> {
-  return (await murmur3128.encode(buf))
-    // Murmur3 outputs 128 bit but, accidentally, IPFS Go's
-    // implementation only uses the first 64, so we must do the same
-    // for parity..
-    .slice(0, 8)
-    // Invert buffer because that's how Go impl does it
-    .reverse()
-}
-
-const addLinksToHamtBucket = async (links: PBLink[], bucket: Bucket<boolean>, rootBucket: Bucket<boolean>): Promise<void> => {
-  const padLength = (bucket.tableSize() - 1).toString(16).length
-  await Promise.all(
-    links.map(async link => {
-      if (link.Name == null) {
-        // TODO(@rvagg): what do? this is technically possible
-        throw new Error('Unexpected Link without a Name')
-      }
-      if (link.Name.length === padLength) {
-        const pos = parseInt(link.Name, 16)
-
-        bucket._putObjectAt(pos, new Bucket({
-          hash: rootBucket._options.hash,
-          bits: rootBucket._options.bits
-        }, bucket, pos))
-        return
-      }
-
-      await rootBucket.put(link.Name.substring(2), true)
-    })
-  )
-}
-
-const toPrefix = (position: number, padLength: number): string => {
-  return position
-    .toString(16)
-    .toUpperCase()
-    .padStart(padLength, '0')
-    .substring(0, padLength)
-}
-
-const toBucketPath = (position: BucketPosition<boolean>): Array<Bucket<boolean>> => {
-  let bucket = position.bucket
-  const path = []
-
-  while (bucket._parent != null) {
-    path.push(bucket)
-
-    bucket = bucket._parent
-  }
-
-  path.push(bucket)
-
-  return path.reverse()
-}
-
-const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions): Promise<CID | undefined> => {
-  if (context == null) {
-    if (node.Data == null) {
-      throw new NotUnixFSError('no data in PBNode')
-    }
-
-    let dir: UnixFS
-    try {
-      dir = UnixFS.unmarshal(node.Data)
-    } catch (err: any) {
-      throw new NotUnixFSError(err.message)
-    }
-
-    if (dir.type !== 'hamt-sharded-directory') {
-      throw new NotUnixFSError('not a HAMT')
-    }
-    if (dir.fanout == null) {
-      throw new NotUnixFSError('missing fanout')
-    }
-
-    const rootBucket = createHAMT<boolean>({
-      hashFn,
-      bits: Math.log2(Number(dir.fanout))
-    })
-
-    context = {
-      rootBucket,
-      hamtDepth: 1,
-      lastBucket: rootBucket
-    }
-  }
-
-  const padLength = (context.lastBucket.tableSize() - 1).toString(16).length
-
-  await addLinksToHamtBucket(node.Links, context.lastBucket, context.rootBucket)
-
-  const position = await context.rootBucket._findNewBucketAndPos(name)
-  let prefix = toPrefix(position.pos, padLength)
-  const bucketPath = toBucketPath(position)
-
-  if (bucketPath.length > context.hamtDepth) {
-    context.lastBucket = bucketPath[context.hamtDepth]
-
-    prefix = toPrefix(context.lastBucket._posAtParent, padLength)
-  }
-
-  const link = node.Links.find(link => {
-    if (link.Name == null) {
-      return false
-    }
-
-    const entryPrefix = link.Name.substring(0, padLength)
-    const entryName = link.Name.substring(padLength)
-
-    if (entryPrefix !== prefix) {
-      // not the entry or subshard we're looking for
-      return false
-    }
-
-    if (entryName !== '' && entryName !== name) {
-      // not the entry we're looking for
-      return false
-    }
-
-    return true
-  })
-
-  if (link == null) {
-    return
-  }
-
-  if (link.Name != null && link.Name.substring(padLength) === name) {
-    return link.Hash
-  }
-
-  context.hamtDepth++
-
-  const block = await blockstore.get(link.Hash, options)
-  node = decode(block)
-
-  return findShardCid(node, name, blockstore, context, options)
-}
-
-export default findShardCid