@helia/unixfs 0.0.0

package/src/commands/utils/dir-sharded.ts

@@ -0,0 +1,219 @@
+import { encode, prepare } from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import { persist } from './persist.js'
+import { createHAMT, Bucket, BucketChild } from 'hamt-sharding'
+import {
+  hamtHashCode,
+  hamtHashFn,
+  hamtBucketBits
+} from './hamt-constants.js'
+import type { CID, Version } from 'multiformats/cid'
+import type { PBNode } from '@ipld/dag-pb/interface'
+import type { Mtime } from 'ipfs-unixfs'
+import type { BlockCodec } from 'multiformats/codecs/interface'
+import type { Blockstore } from 'ipfs-unixfs-importer'
+
+export interface ImportResult {
+  cid: CID
+  node: PBNode
+  size: number
+}
+
+export interface DirContents {
+  cid?: CID
+  size?: number
+}
+
+export interface DirOptions {
+  mtime?: Mtime
+  mode?: number
+  codec?: BlockCodec<any, any>
+  cidVersion?: Version
+  onlyHash?: boolean
+  signal?: AbortSignal
+}
+
+export interface DirProps {
+  root: boolean
+  dir: boolean
+  path: string
+  dirty: boolean
+  flat: boolean
+  parent?: Dir
+  parentKey?: string
+  unixfs?: UnixFS
+  mode?: number
+  mtime?: Mtime
+}
+
+export abstract class Dir {
+  protected options: DirOptions
+  protected root: boolean
+  protected dir: boolean
+  protected path: string
+  protected dirty: boolean
+  protected flat: boolean
+  protected parent?: Dir
+  protected parentKey?: string
+  protected unixfs?: UnixFS
+  protected mode?: number
+  public mtime?: Mtime
+  protected cid?: CID
+  protected size?: number
+
+  constructor (props: DirProps, options: DirOptions) {
+    this.options = options ?? {}
+    this.root = props.root
+    this.dir = props.dir
+    this.path = props.path
+    this.dirty = props.dirty
+    this.flat = props.flat
+    this.parent = props.parent
+    this.parentKey = props.parentKey
+    this.unixfs = props.unixfs
+    this.mode = props.mode
+    this.mtime = props.mtime
+  }
+}
+
+export class DirSharded extends Dir {
+  public _bucket: Bucket<DirContents>
+
+  constructor (props: DirProps, options: DirOptions) {
+    super(props, options)
+
+    /** @type {Bucket<DirContents>} */
+    this._bucket = createHAMT({
+      hashFn: hamtHashFn,
+      bits: hamtBucketBits
+    })
+  }
+
+  async put (name: string, value: DirContents): Promise<void> {
+    await this._bucket.put(name, value)
+  }
+
+  async get (name: string): Promise<DirContents | undefined> {
+    return await this._bucket.get(name)
+  }
+
+  childCount (): number {
+    return this._bucket.leafCount()
+  }
+
+  directChildrenCount (): number {
+    return this._bucket.childrenCount()
+  }
+
+  onlyChild (): Bucket<DirContents> | BucketChild<DirContents> {
+    return this._bucket.onlyChild()
+  }
+
+  async * eachChildSeries (): AsyncGenerator<{ key: string, child: DirContents }> {
+    for await (const { key, value } of this._bucket.eachLeafSeries()) {
+      yield {
+        key,
+        child: value
+      }
+    }
+  }
+
+  async * flush (blockstore: Blockstore): AsyncIterable<ImportResult> {
+    yield * flush(this._bucket, blockstore, this, this.options)
+  }
+}
+
+async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot: any, options: DirOptions): AsyncIterable<ImportResult> {
+  const children = bucket._children
+  const links = []
+  let childrenSize = 0
+
+  for (let i = 0; i < children.length; i++) {
+    const child = children.get(i)
+
+    if (child == null) {
+      continue
+    }
+
+    const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
+
+    if (child instanceof Bucket) {
+      let shard: ImportResult | undefined
+
+      for await (const subShard of flush(child, blockstore, null, options)) {
+        shard = subShard
+      }
+
+      if (shard == null) {
+        throw new Error('Could not flush sharded directory, no subshard found')
+      }
+
+      links.push({
+        Name: labelPrefix,
+        Tsize: shard.size,
+        Hash: shard.cid
+      })
+      childrenSize += shard.size
+    } else if (typeof child.value.flush === 'function') {
+      const dir = child.value
+      let flushedDir
+
+      for await (const entry of dir.flush(blockstore)) {
+        flushedDir = entry
+
+        yield flushedDir
+      }
+
+      const label = labelPrefix + child.key
+      links.push({
+        Name: label,
+        Tsize: flushedDir.size,
+        Hash: flushedDir.cid
+      })
+
+      childrenSize += flushedDir.size // eslint-disable-line @typescript-eslint/restrict-plus-operands
+    } else {
+      const value = child.value
+
+      if (value.cid == null) {
+        continue
+      }
+
+      const label = labelPrefix + child.key
+      const size = value.size
+
+      links.push({
+        Name: label,
+        Tsize: size,
+        Hash: value.cid
+      })
+      childrenSize += size ?? 0 // eslint-disable-line @typescript-eslint/restrict-plus-operands
+    }
+  }
+
+  // go-ipfs uses little endian, that's why we have to
+  // reverse the bit field before storing it
+  const data = Uint8Array.from(children.bitField().reverse())
+  const dir = new UnixFS({
+    type: 'hamt-sharded-directory',
+    data,
+    fanout: bucket.tableSize(),
+    hashType: hamtHashCode,
+    mtime: shardRoot?.mtime,
+    mode: shardRoot?.mode
+  })
+
+  const node = {
+    Data: dir.marshal(),
+    Links: links
+  }
+  const buffer = encode(prepare(node))
+  const cid = await persist(buffer, blockstore, options)
+  const size = buffer.length + childrenSize
+
+  yield {
+    cid,
+    node,
+    size
+  }
+}

package/src/commands/utils/errors.ts

@@ -0,0 +1,31 @@
+import { HeliaError } from '@helia/interface/errors'
+
+export class NotUnixFSError extends HeliaError {
+  constructor (message = 'not a Unixfs node') {
+    super(message, 'NotUnixFSError', 'ERR_NOT_UNIXFS')
+  }
+}
+
+export class InvalidPBNodeError extends HeliaError {
+  constructor (message = 'invalid PBNode') {
+    super(message, 'InvalidPBNodeError', 'ERR_INVALID_PBNODE')
+  }
+}
+
+export class UnknownError extends HeliaError {
+  constructor (message = 'unknown error') {
+    super(message, 'InvalidPBNodeError', 'ERR_UNKNOWN_ERROR')
+  }
+}
+
+export class AlreadyExistsError extends HeliaError {
+  constructor (message = 'path already exists') {
+    super(message, 'NotUnixFSError', 'ERR_ALREADY_EXISTS')
+  }
+}
+
+export class DoesNotExistError extends HeliaError {
+  constructor (message = 'path does not exist') {
+    super(message, 'NotUnixFSError', 'ERR_DOES_NOT_EXIST')
+  }
+}

package/src/commands/utils/hamt-constants.ts

@@ -0,0 +1,14 @@
+import { murmur3128 } from '@multiformats/murmur3'
+
+export const hamtHashCode = murmur3128.code
+export const hamtBucketBits = 8
+
+export async function hamtHashFn (buf: Uint8Array): Promise<Uint8Array> {
+  return (await murmur3128.encode(buf))
+    // Murmur3 outputs 128 bit but, accidentally, IPFS Go's
+    // implementation only uses the first 64, so we must do the same
+    // for parity..
+    .subarray(0, 8)
+    // Invert buffer because that's how Go impl does it
+    .reverse()
+}

package/src/commands/utils/hamt-utils.ts

@@ -0,0 +1,285 @@
+import * as dagPB from '@ipld/dag-pb'
+import {
+  Bucket,
+  createHAMT
+} from 'hamt-sharding'
+import { DirSharded } from './dir-sharded.js'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import last from 'it-last'
+import { CID } from 'multiformats/cid'
+import {
+  hamtHashCode,
+  hamtHashFn,
+  hamtBucketBits
+} from './hamt-constants.js'
+import type { PBLink, PBNode } from '@ipld/dag-pb/interface'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { Blockstore } from 'interface-blockstore'
+import type { Mtime } from 'ipfs-unixfs'
+import type { Directory } from './cid-to-directory.js'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { ImportResult } from 'ipfs-unixfs-importer'
+
+const log = logger('helia:unixfs:commands:utils:hamt-utils')
+
+export interface UpdateHamtResult {
+  node: PBNode
+  cid: CID
+  size: number
+}
+
+export const updateHamtDirectory = async (parent: Directory, blockstore: Blockstore, links: PBLink[], bucket: Bucket<any>, options: AbortOptions): Promise<UpdateHamtResult> => {
+  if (parent.node.Data == null) {
+    throw new Error('Could not update HAMT directory because parent had no data')
+  }
+
+  // update parent with new bit field
+  const data = Uint8Array.from(bucket._children.bitField().reverse())
+  const node = UnixFS.unmarshal(parent.node.Data)
+  const dir = new UnixFS({
+    type: 'hamt-sharded-directory',
+    data,
+    fanout: bucket.tableSize(),
+    hashType: hamtHashCode,
+    mode: node.mode,
+    mtime: node.mtime
+  })
+
+  parent.node = {
+    Data: dir.marshal(),
+    Links: links.sort((a, b) => (a.Name ?? '').localeCompare(b.Name ?? ''))
+  }
+  const buf = dagPB.encode(parent.node)
+  const hash = await sha256.digest(buf)
+  const cid = CID.create(parent.cid.version, dagPB.code, hash)
+
+  await blockstore.put(cid, buf, options)
+
+  return {
+    node: parent.node,
+    cid,
+    size: links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.length)
+  }
+}
+
+export const recreateHamtLevel = async (blockstore: Blockstore, links: PBLink[], rootBucket: Bucket<any>, parentBucket: Bucket<any>, positionAtParent: number, options: AbortOptions): Promise<Bucket<any>> => {
+  // recreate this level of the HAMT
+  const bucket = new Bucket({
+    hash: rootBucket._options.hash,
+    bits: rootBucket._options.bits
+  }, parentBucket, positionAtParent)
+  parentBucket._putObjectAt(positionAtParent, bucket)
+
+  await addLinksToHamtBucket(blockstore, links, bucket, rootBucket, options)
+
+  return bucket
+}
+
+export const recreateInitialHamtLevel = async (links: PBLink[]): Promise<Bucket<any>> => {
+  const bucket = createHAMT({
+    hashFn: hamtHashFn,
+    bits: hamtBucketBits
+  })
+
+  // populate sub bucket but do not recurse as we do not want to pull whole shard in
+  await Promise.all(
+    links.map(async link => {
+      const linkName = (link.Name ?? '')
+
+      if (linkName.length === 2) {
+        const pos = parseInt(linkName, 16)
+
+        const subBucket = new Bucket({
+          hash: bucket._options.hash,
+          bits: bucket._options.bits
+        }, bucket, pos)
+        bucket._putObjectAt(pos, subBucket)
+
+        await Promise.resolve(); return
+      }
+
+      await bucket.put(linkName.substring(2), {
+        size: link.Tsize,
+        cid: link.Hash
+      })
+    })
+  )
+
+  return bucket
+}
+
+export const addLinksToHamtBucket = async (blockstore: Blockstore, links: PBLink[], bucket: Bucket<any>, rootBucket: Bucket<any>, options: AbortOptions): Promise<void> => {
+  await Promise.all(
+    links.map(async link => {
+      const linkName = (link.Name ?? '')
+
+      if (linkName.length === 2) {
+        log('Populating sub bucket', linkName)
+        const pos = parseInt(linkName, 16)
+        const block = await blockstore.get(link.Hash, options)
+        const node = dagPB.decode(block)
+
+        const subBucket = new Bucket({
+          hash: rootBucket._options.hash,
+          bits: rootBucket._options.bits
+        }, bucket, pos)
+        bucket._putObjectAt(pos, subBucket)
+
+        await addLinksToHamtBucket(blockstore, node.Links, subBucket, rootBucket, options)
+
+        await Promise.resolve(); return
+      }
+
+      await rootBucket.put(linkName.substring(2), {
+        size: link.Tsize,
+        cid: link.Hash
+      })
+    })
+  )
+}
+
+export const toPrefix = (position: number): string => {
+  return position
+    .toString(16)
+    .toUpperCase()
+    .padStart(2, '0')
+    .substring(0, 2)
+}
+
+export interface HamtPath {
+  rootBucket: Bucket<any>
+  path: Array<{
+    bucket: Bucket<any>
+    prefix: string
+    node?: PBNode
+  }>
+}
+
+export const generatePath = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<HamtPath> => {
+  // start at the root bucket and descend, loading nodes as we go
+  const rootBucket = await recreateInitialHamtLevel(parent.node.Links)
+  const position = await rootBucket._findNewBucketAndPos(name)
+
+  // the path to the root bucket
+  const path: Array<{ bucket: Bucket<any>, prefix: string, node?: PBNode }> = [{
+    bucket: position.bucket,
+    prefix: toPrefix(position.pos)
+  }]
+  let currentBucket = position.bucket
+
+  while (currentBucket !== rootBucket) {
+    path.push({
+      bucket: currentBucket,
+      prefix: toPrefix(currentBucket._posAtParent)
+    })
+
+    // @ts-expect-error - only the root bucket's parent will be undefined
+    currentBucket = currentBucket._parent
+  }
+
+  path.reverse()
+  path[0].node = parent.node
+
+  // load PbNode for each path segment
+  for (let i = 0; i < path.length; i++) {
+    const segment = path[i]
+
+    if (segment.node == null) {
+      throw new Error('Could not generate HAMT path')
+    }
+
+    // find prefix in links
+    const link = segment.node.Links
+      .filter(link => (link.Name ?? '').substring(0, 2) === segment.prefix)
+      .pop()
+
+    // entry was not in shard
+    if (link == null) {
+      // reached bottom of tree, file will be added to the current bucket
+      log(`Link ${segment.prefix}${name} will be added`)
+      // return path
+      continue
+    }
+
+    // found entry
+    if (link.Name === `${segment.prefix}${name}`) {
+      log(`Link ${segment.prefix}${name} will be replaced`)
+      // file already existed, file will be added to the current bucket
+      // return path
+      continue
+    }
+
+    // found subshard
+    log(`Found subshard ${segment.prefix}`)
+    const block = await blockstore.get(link.Hash, options)
+    const node = dagPB.decode(block)
+
+    // subshard hasn't been loaded, descend to the next level of the HAMT
+    if (path[i + 1] == null) {
+      log(`Loaded new subshard ${segment.prefix}`)
+
+      await recreateHamtLevel(blockstore, node.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options)
+      const position = await rootBucket._findNewBucketAndPos(name)
+
+      // i--
+      path.push({
+        bucket: position.bucket,
+        prefix: toPrefix(position.pos),
+        node
+      })
+
+      continue
+    }
+
+    const nextSegment = path[i + 1]
+
+    // add intermediate links to bucket
+    await addLinksToHamtBucket(blockstore, node.Links, nextSegment.bucket, rootBucket, options)
+
+    nextSegment.node = node
+  }
+
+  await rootBucket.put(name, true)
+
+  path.reverse()
+
+  return {
+    rootBucket,
+    path
+  }
+}
+
+export interface CreateShardOptions {
+  mtime?: Mtime
+  mode?: number
+}
+
+export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: number, cid: CID }>, options: CreateShardOptions = {}): Promise<ImportResult> => {
+  const shard = new DirSharded({
+    root: true,
+    dir: true,
+    parent: undefined,
+    parentKey: undefined,
+    path: '',
+    dirty: true,
+    flat: false,
+    mtime: options.mtime,
+    mode: options.mode
+  }, options)
+
+  for (let i = 0; i < contents.length; i++) {
+    await shard._bucket.put(contents[i].name, {
+      size: contents[i].size,
+      cid: contents[i].cid
+    })
+  }
+
+  const res = await last(shard.flush(blockstore))
+
+  if (res == null) {
+    throw new Error('Flushing shard yielded no result')
+  }
+
+  return res
+}

package/src/commands/utils/persist.ts

@@ -0,0 +1,22 @@
+import { CID, Version } from 'multiformats/cid'
+import * as dagPB from '@ipld/dag-pb'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { BlockCodec } from 'multiformats/codecs/interface'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Blockstore } from 'interface-blockstore'
+
+export interface PersistOptions extends AbortOptions {
+  codec?: BlockCodec<any, any>
+  cidVersion?: Version
+}
+
+export const persist = async (buffer: Uint8Array, blockstore: Blockstore, options: PersistOptions = {}): Promise<CID> => {
+  const multihash = await sha256.digest(buffer)
+  const cid = CID.create(options.cidVersion ?? 1, dagPB.code, multihash)
+
+  await blockstore.put(cid, buffer, {
+    signal: options.signal
+  })
+
+  return cid
+}