@helia/unixfs 1.0.3 → 1.0.4

package/src/commands/utils/hamt-utils.ts

@@ -1,8 +1,4 @@
 import * as dagPB from '@ipld/dag-pb'
-import {
-  Bucket,
-  createHAMT
-} from 'hamt-sharding'
 import { DirSharded } from './dir-sharded.js'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
@@ -13,131 +9,23 @@ import {
   hamtHashFn,
   hamtBucketBits
 } from './hamt-constants.js'
-import type { PBLink, PBNode } from '@ipld/dag-pb/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { Mtime } from 'ipfs-unixfs'
-import type { Directory } from './cid-to-directory.js'
 import type { AbortOptions } from '@libp2p/interfaces'
 import type { ImportResult } from 'ipfs-unixfs-importer'
 import { persist } from './persist.js'
+import { InfiniteHash, wrapHash } from './consumable-hash.js'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+// @ts-expect-error no types
+import SparseArray from 'sparse-array'
+import type { PersistOptions } from './persist.js'
 
 const log = logger('helia:unixfs:commands:utils:hamt-utils')
 
-export interface UpdateHamtResult {
-  node: PBNode
-  cid: CID
-  size: number
-}
-
 export interface UpdateHamtDirectoryOptions extends AbortOptions {
   cidVersion: Version
 }
 
-export const updateHamtDirectory = async (pbNode: PBNode, blockstore: Blockstore, bucket: Bucket<any>, options: UpdateHamtDirectoryOptions): Promise<UpdateHamtResult> => {
-  if (pbNode.Data == null) {
-    throw new Error('Could not update HAMT directory because parent had no data')
-  }
-
-  // update parent with new bit field
-  const node = UnixFS.unmarshal(pbNode.Data)
-  const dir = new UnixFS({
-    type: 'hamt-sharded-directory',
-    data: Uint8Array.from(bucket._children.bitField().reverse()),
-    fanout: BigInt(bucket.tableSize()),
-    hashType: hamtHashCode,
-    mode: node.mode,
-    mtime: node.mtime
-  })
-
-  const updatedPbNode = {
-    Data: dir.marshal(),
-    Links: pbNode.Links
-  }
-
-  const buf = dagPB.encode(dagPB.prepare(updatedPbNode))
-  const cid = await persist(buf, blockstore, options)
-
-  return {
-    node: updatedPbNode,
-    cid,
-    size: pbNode.Links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.byteLength)
-  }
-}
-
-export const recreateHamtLevel = async (blockstore: Blockstore, links: PBLink[], rootBucket: Bucket<any>, parentBucket: Bucket<any>, positionAtParent: number, options: AbortOptions): Promise<Bucket<any>> => {
-  // recreate this level of the HAMT
-  const bucket = new Bucket({
-    hash: rootBucket._options.hash,
-    bits: rootBucket._options.bits
-  }, parentBucket, positionAtParent)
-  parentBucket._putObjectAt(positionAtParent, bucket)
-
-  await addLinksToHamtBucket(blockstore, links, bucket, rootBucket, options)
-
-  return bucket
-}
-
-export const recreateInitialHamtLevel = async (links: PBLink[]): Promise<Bucket<any>> => {
-  const bucket = createHAMT<any>({
-    hashFn: hamtHashFn,
-    bits: hamtBucketBits
-  })
-
-  // populate sub bucket but do not recurse as we do not want to load the whole shard
-  await Promise.all(
-    links.map(async link => {
-      const linkName = (link.Name ?? '')
-
-      if (linkName.length === 2) {
-        const pos = parseInt(linkName, 16)
-        const subBucket = new Bucket({
-          hash: bucket._options.hash,
-          bits: bucket._options.bits
-        }, bucket, pos)
-
-        bucket._putObjectAt(pos, subBucket)
-        return
-      }
-
-      await bucket.put(linkName.substring(2), {
-        size: link.Tsize,
-        cid: link.Hash
-      })
-    })
-  )
-
-  return bucket
-}
-
-export const addLinksToHamtBucket = async (blockstore: Blockstore, links: PBLink[], bucket: Bucket<any>, rootBucket: Bucket<any>, options: AbortOptions): Promise<void> => {
-  await Promise.all(
-    links.map(async link => {
-      const linkName = (link.Name ?? '')
-
-      if (linkName.length === 2) {
-        log('Populating sub bucket', linkName)
-        const pos = parseInt(linkName, 16)
-        const block = await blockstore.get(link.Hash, options)
-        const node = dagPB.decode(block)
-
-        const subBucket = new Bucket({
-          hash: rootBucket._options.hash,
-          bits: rootBucket._options.bits
-        }, bucket, pos)
-        bucket._putObjectAt(pos, subBucket)
-
-        await addLinksToHamtBucket(blockstore, node.Links, subBucket, rootBucket, options)
-        return
-      }
-
-      await rootBucket.put(linkName.substring(2), {
-        size: link.Tsize,
-        cid: link.Hash
-      })
-    })
-  )
-}
-
 export const toPrefix = (position: number): string => {
   return position
     .toString(16)
@@ -146,149 +34,175 @@ export const toPrefix = (position: number): string => {
     .substring(0, 2)
 }
 
-export interface HamtPathSegment {
-  bucket?: Bucket<any>
-  prefix?: string
-  node?: PBNode
-  cid?: CID
-  size?: number
+export interface CreateShardOptions {
+  mtime?: Mtime
+  mode?: number
+  cidVersion: Version
 }
 
-export const generatePath = async (root: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<HamtPathSegment[]> => {
-  // start at the root bucket and descend, loading nodes as we go
-  const rootBucket = await recreateInitialHamtLevel(root.node.Links)
-  const position = await rootBucket._findNewBucketAndPos(name)
-  const path: HamtPathSegment[] = [{
-    bucket: position.bucket,
-    prefix: toPrefix(position.pos)
-  }]
-  let currentBucket = position.bucket
-
-  while (currentBucket !== rootBucket) {
-    path.push({
-      bucket: currentBucket,
-      prefix: toPrefix(currentBucket._posAtParent)
+export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise<ImportResult> => {
+  const shard = new DirSharded({
+    root: true,
+    dir: true,
+    parent: undefined,
+    parentKey: undefined,
+    path: '',
+    dirty: true,
+    flat: false,
+    mtime: options.mtime,
+    mode: options.mode
+  }, options)
+
+  for (let i = 0; i < contents.length; i++) {
+    await shard._bucket.put(contents[i].name, {
+      size: contents[i].size,
+      cid: contents[i].cid
     })
+  }
 
-    if (currentBucket._parent == null) {
-      break
-    }
+  const res = await last(shard.flush(blockstore))
 
-    currentBucket = currentBucket._parent
+  if (res == null) {
+    throw new Error('Flushing shard yielded no result')
   }
 
-  // add the root bucket to the path
-  path.push({
-    bucket: rootBucket,
-    node: root.node
-  })
+  return res
+}
+
+export interface HAMTPath {
+  prefix: string
+  children: SparseArray
+  node: dagPB.PBNode
+}
+
+export const updateShardedDirectory = async (path: HAMTPath[], blockstore: Blockstore, options: PersistOptions): Promise<{ cid: CID, node: dagPB.PBNode }> => {
+  // persist any metadata on the shard root
+  const shardRoot = UnixFS.unmarshal(path[0].node.Data ?? new Uint8Array(0))
 
+  // this is always the same
+  const fanout = BigInt(Math.pow(2, hamtBucketBits))
+
+  // start from the leaf and ascend to the root
   path.reverse()
 
-  // load PbNode for each path segment
-  for (let i = 1; i < path.length; i++) {
-    const segment = path[i]
-    const previousSegment = path[i - 1]
+  let cid: CID | undefined
+  let node: dagPB.PBNode | undefined
 
-    if (previousSegment.node == null) {
-      throw new Error('Could not generate HAMT path')
-    }
+  for (let i = 0; i < path.length; i++) {
+    const isRoot = i === path.length - 1
+    const segment = path[i]
 
-    // find prefix in links
-    const link = previousSegment.node.Links
-      .filter(link => (link.Name ?? '').substring(0, 2) === segment.prefix)
-      .pop()
+    // go-ipfs uses little endian, that's why we have to
+    // reverse the bit field before storing it
+    const data = Uint8Array.from(segment.children.bitField().reverse())
+    const dir = new UnixFS({
+      type: 'hamt-sharded-directory',
+      data,
+      fanout,
+      hashType: hamtHashCode
+    })
 
-    // entry was not in shard
-    if (link == null) {
-      // reached bottom of tree, file will be added to the current bucket
-      log(`Link ${segment.prefix}${name} will be added`)
-      // return path
-      continue
+    if (isRoot) {
+      dir.mtime = shardRoot.mtime
+      dir.mode = shardRoot.mode
     }
 
-    const linkName = link.Name ?? ''
-
-    // found entry
-    if (linkName === `${segment.prefix}${name}`) {
-      log(`Link ${segment.prefix}${name} will be replaced`)
-      // file already existed, file will be added to the current bucket
-      // return path
-      continue
+    node = {
+      Data: dir.marshal(),
+      Links: segment.node.Links
     }
 
-    // found subshard
-    log(`Found subshard ${segment.prefix}`)
-    const block = await blockstore.get(link.Hash)
-    const node = segment.node = dagPB.decode(block)
+    const block = dagPB.encode(dagPB.prepare(node))
 
-    // subshard hasn't been loaded, descend to the next level of the HAMT
-    if (path[i + 1] == null) {
-      log(`Loaded new subshard ${segment.prefix}`)
+    cid = await persist(block, blockstore, options)
 
-      if (segment.bucket == null || segment.prefix == null) {
-        throw new Error('Shard was invalid')
+    if (!isRoot) {
+      // update link in parent sub-shard
+      const nextSegment = path[i + 1]
+
+      if (nextSegment == null) {
+        throw new Error('Was not operating on shard root but also had no parent?')
       }
 
-      await recreateHamtLevel(blockstore, node.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options)
-      const position = await rootBucket._findNewBucketAndPos(name)
+      log('updating link in parent sub-shard with prefix %s', nextSegment.prefix)
 
-      // i--
-      path.push({
-        bucket: position.bucket,
-        prefix: toPrefix(position.pos),
-        node
+      nextSegment.node.Links = nextSegment.node.Links.filter(l => l.Name !== nextSegment.prefix)
+      nextSegment.node.Links.push({
+        Name: nextSegment.prefix,
+        Hash: cid,
+        Tsize: segment.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), block.byteLength)
       })
-
-      continue
-    }
-
-    if (segment.bucket == null) {
-      throw new Error('Shard was invalid')
     }
-
-    // add intermediate links to bucket
-    await addLinksToHamtBucket(blockstore, node.Links, segment.bucket, rootBucket, options)
   }
 
-  await rootBucket.put(name, true)
-
-  path.reverse()
+  if (cid == null || node == null) {
+    throw new Error('Noting persisted')
+  }
 
-  return path
+  return { cid, node }
 }
 
-export interface CreateShardOptions {
-  mtime?: Mtime
-  mode?: number
-  cidVersion: Version
-}
+export const recreateShardedDirectory = async (cid: CID, fileName: string, blockstore: Blockstore, options: AbortOptions): Promise<{ path: HAMTPath[], hash: InfiniteHash }> => {
+  const wrapped = wrapHash(hamtHashFn)
+  const hash = wrapped(uint8ArrayFromString(fileName))
+  const path: HAMTPath[] = []
 
-export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise<ImportResult> => {
-  const shard = new DirSharded({
-    root: true,
-    dir: true,
-    parent: undefined,
-    parentKey: undefined,
-    path: '',
-    dirty: true,
-    flat: false,
-    mtime: options.mtime,
-    mode: options.mode
-  }, options)
+  // descend the HAMT, loading each layer as we head towards the target child
+  while (true) {
+    const block = await blockstore.get(cid, options)
+    const node = dagPB.decode(block)
+    const children = new SparseArray()
+    const index = await hash.take(hamtBucketBits)
+    const prefix = toPrefix(index)
 
-  for (let i = 0; i < contents.length; i++) {
-    await shard._bucket.put(contents[i].name, {
-      size: contents[i].size,
-      cid: contents[i].cid
+    path.push({
+      prefix,
+      children,
+      node
     })
-  }
 
-  const res = await last(shard.flush(blockstore))
+    let childLink: dagPB.PBLink | undefined
 
-  if (res == null) {
-    throw new Error('Flushing shard yielded no result')
+    // update sparsearray child layout - the bitfield is used as the data field for the
+    // intermediate DAG node so this is required to generate consistent hashes
+    for (const link of node.Links) {
+      const linkName = link.Name ?? ''
+
+      if (linkName.length < 2) {
+        throw new Error('Invalid HAMT - link name was too short')
+      }
+
+      const position = parseInt(linkName.substring(0, 2), 16)
+      children.set(position, true)
+
+      // we found the child we are looking for
+      if (linkName.startsWith(prefix)) {
+        childLink = link
+      }
+    }
+
+    if (childLink == null) {
+      log('no link found with prefix %s for %s', prefix, fileName)
+      // hash.untake(hamtBucketBits)
+      break
+    }
+
+    const linkName = childLink.Name ?? ''
+
+    if (linkName.length < 2) {
+      throw new Error('Invalid HAMT - link name was too short')
+    }
+
+    if (linkName.length === 2) {
+      // found sub-shard
+      cid = childLink.Hash
+      log('descend into sub-shard with prefix %s', linkName)
+
+      continue
+    }
+
+    break
   }
 
-  return res
+  return { path, hash }
 }

package/src/commands/utils/remove-link.ts

@@ -4,10 +4,9 @@ import type { CID, Version } from 'multiformats/cid'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
 import {
-  generatePath,
-  HamtPathSegment,
-  updateHamtDirectory,
-  UpdateHamtDirectoryOptions
+  recreateShardedDirectory,
+  UpdateHamtDirectoryOptions,
+  updateShardedDirectory
 } from './hamt-utils.js'
 import type { PBNode } from '@ipld/dag-pb'
 import type { Blockstore } from 'interface-blockstore'
@@ -77,131 +76,60 @@ const removeFromDirectory = async (parent: Directory, name: string, blockstore:
 }
 
 const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ cid: CID, node: PBNode }> => {
-  const path = await generatePath(parent, name, blockstore, options)
+  const { path } = await recreateShardedDirectory(parent.cid, name, blockstore, options)
+  const finalSegment = path[path.length - 1]
 
-  // remove file from root bucket
-  const rootBucket = path[path.length - 1].bucket
-
-  if (rootBucket == null) {
-    throw new Error('Could not generate HAMT path')
+  if (finalSegment == null) {
+    throw new Error('Invalid HAMT, could not generate path')
   }
 
-  await rootBucket.del(name)
-
-  // update all nodes in the shard path
-  return await updateShard(path, name, blockstore, options)
-}
-
-/**
- * The `path` param is a list of HAMT path segments starting with th
- */
-const updateShard = async (path: HamtPathSegment[], name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ node: PBNode, cid: CID }> => {
-  const fileName = `${path[0].prefix}${name}`
-
-  // skip first path segment as it is the file to remove
-  for (let i = 1; i < path.length; i++) {
-    const lastPrefix = path[i - 1].prefix
-    const segment = path[i]
-
-    if (segment.node == null) {
-      throw new InvalidParametersError('Path segment had no associated PBNode')
-    }
-
-    const link = segment.node.Links
-      .find(link => (link.Name ?? '').substring(0, 2) === lastPrefix)
+  const linkName = finalSegment.node.Links.filter(l => (l.Name ?? '').substring(2) === name).map(l => l.Name).pop()
 
-    if (link == null) {
-      throw new InvalidParametersError(`No link found with prefix ${lastPrefix} for file ${name}`)
-    }
+  if (linkName == null) {
+    throw new Error('File not found')
+  }
 
-    if (link.Name == null) {
-      throw new InvalidParametersError(`${lastPrefix} link had no name`)
-    }
+  const prefix = linkName.substring(0, 2)
+  const index = parseInt(prefix, 16)
 
-    if (link.Name === fileName) {
-      log(`removing existing link ${link.Name}`)
+  // remove the file from the shard
+  finalSegment.node.Links = finalSegment.node.Links.filter(link => link.Name !== linkName)
+  finalSegment.children.unset(index)
 
-      const links = segment.node.Links.filter((nodeLink) => {
-        return nodeLink.Name !== link.Name
-      })
-
-      if (segment.bucket == null) {
-        throw new Error('Segment bucket was missing')
+  if (finalSegment.node.Links.length === 1) {
+    // replace the subshard with the last remaining file in the parent
+    while (true) {
+      if (path.length === 1) {
+        break
       }
 
-      await segment.bucket.del(name)
-
-      const result = await updateHamtDirectory({
-        Data: segment.node.Data,
-        Links: links
-      }, blockstore, segment.bucket, options)
+      const segment = path[path.length - 1]
 
-      segment.node = result.node
-      segment.cid = result.cid
-      segment.size = result.size
-    }
-
-    if (link.Name === lastPrefix) {
-      log(`updating subshard with prefix ${lastPrefix}`)
-
-      const lastSegment = path[i - 1]
-
-      if (lastSegment.node?.Links.length === 1) {
-        log(`removing subshard for ${lastPrefix}`)
-
-        // convert subshard back to normal file entry
-        const link = lastSegment.node.Links[0]
-        link.Name = `${lastPrefix}${(link.Name ?? '').substring(2)}`
-
-        // remove existing prefix
-        segment.node.Links = segment.node.Links.filter((link) => {
-          return link.Name !== lastPrefix
-        })
-
-        // add new child
-        segment.node.Links.push(link)
-      } else {
-        // replace subshard entry
-        log(`replacing subshard for ${lastPrefix}`)
+      if (segment == null || segment.node.Links.length > 1) {
+        break
+      }
 
-        // remove existing prefix
-        segment.node.Links = segment.node.Links.filter((link) => {
-          return link.Name !== lastPrefix
-        })
+      // remove final segment
+      path.pop()
 
-        if (lastSegment.cid == null) {
-          throw new Error('Did not persist previous segment')
-        }
+      const nextSegment = path[path.length - 1]
 
-        // add new child
-        segment.node.Links.push({
-          Name: lastPrefix,
-          Hash: lastSegment.cid,
-          Tsize: lastSegment.size
-        })
+      if (nextSegment == null) {
+        break
       }
 
-      if (segment.bucket == null) {
-        throw new Error('Segment bucket was missing')
-      }
+      const link = segment.node.Links[0]
 
-      const result = await updateHamtDirectory(segment.node, blockstore, segment.bucket, options)
-      segment.node = result.node
-      segment.cid = result.cid
-      segment.size = result.size
+      nextSegment.node.Links = nextSegment.node.Links.filter(l => !(l.Name ?? '').startsWith(nextSegment.prefix))
+      nextSegment.node.Links.push({
+        Hash: link.Hash,
+        Name: `${nextSegment.prefix}${(link.Name ?? '').substring(2)}`,
+        Tsize: link.Tsize
+      })
     }
   }
 
-  const rootSegment = path[path.length - 1]
-
-  if (rootSegment == null || rootSegment.cid == null || rootSegment.node == null) {
-    throw new InvalidParametersError('Failed to update shard')
-  }
-
-  return {
-    cid: rootSegment.cid,
-    node: rootSegment.node
-  }
+  return await updateShardedDirectory(path, blockstore, options)
 }
 
 const convertToFlatDirectory = async (parent: Directory, blockstore: Blockstore, options: RmLinkOptions): Promise<RemoveLinkResult> => {

package/src/commands/utils/resolve.ts

@@ -8,7 +8,7 @@ import { cidToDirectory } from './cid-to-directory.js'
 import { cidToPBLink } from './cid-to-pblink.js'
 import type { Blockstore } from 'interface-blockstore'
 
-const log = logger('helia:unixfs:components:utils:add-link')
+const log = logger('helia:unixfs:components:utils:resolve')
 
 export interface Segment {
   name: string
@@ -33,12 +33,12 @@ export interface ResolveResult {
 }
 
 export async function resolve (cid: CID, path: string | undefined, blockstore: Blockstore, options: AbortOptions): Promise<ResolveResult> {
-  log('resolve "%s" under %c', path, cid)
-
   if (path == null || path === '') {
     return { cid }
   }
 
+  log('resolve "%s" under %c', path, cid)
+
   const parts = path.split('/').filter(Boolean)
   const segments: Segment[] = [{
     name: '',
@@ -50,6 +50,8 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B
     const part = parts[i]
     const result = await exporter(cid, blockstore, options)
 
+    log('resolving "%s"', part, result)
+
     if (result.type === 'file') {
       if (i < parts.length - 1) {
         throw new InvalidParametersError('Path was invalid')
@@ -62,6 +64,7 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B
       for await (const entry of result.content()) {
         if (entry.name === part) {
           dirCid = entry.cid
+          break
         }
       }
 
@@ -81,6 +84,8 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B
     }
   }
 
+  log('resolved %s to %c', path, cid)
+
   return {
     cid,
     path,