@helia/unixfs 0.0.0 → 1.0.0

package/src/commands/utils/dir-sharded.ts

@@ -1,39 +1,28 @@
-import { encode, prepare } from '@ipld/dag-pb'
+import { encode, PBLink, prepare } from '@ipld/dag-pb'
 import { UnixFS } from 'ipfs-unixfs'
-import { persist } from './persist.js'
+import { persist, PersistOptions } from './persist.js'
 import { createHAMT, Bucket, BucketChild } from 'hamt-sharding'
 import {
   hamtHashCode,
-  hamtHashFn,
-  hamtBucketBits
+  hamtHashFn
 } from './hamt-constants.js'
-import type { CID, Version } from 'multiformats/cid'
-import type { PBNode } from '@ipld/dag-pb/interface'
+import { CID } from 'multiformats/cid'
 import type { Mtime } from 'ipfs-unixfs'
-import type { BlockCodec } from 'multiformats/codecs/interface'
-import type { Blockstore } from 'ipfs-unixfs-importer'
+import type { Blockstore } from 'interface-blockstore'
 
-export interface ImportResult {
-  cid: CID
-  node: PBNode
-  size: number
-}
-
-export interface DirContents {
-  cid?: CID
-  size?: number
+interface InProgressImportResult extends ImportResult {
+  single?: boolean
+  originalPath?: string
 }
 
-export interface DirOptions {
-  mtime?: Mtime
-  mode?: number
-  codec?: BlockCodec<any, any>
-  cidVersion?: Version
-  onlyHash?: boolean
-  signal?: AbortSignal
+interface ImportResult {
+  cid: CID
+  size: bigint
+  path?: string
+  unixfs?: UnixFS
 }
 
-export interface DirProps {
+interface DirProps {
   root: boolean
   dir: boolean
   path: string
@@ -46,23 +35,25 @@ export interface DirProps {
   mtime?: Mtime
 }
 
-export abstract class Dir {
-  protected options: DirOptions
-  protected root: boolean
-  protected dir: boolean
-  protected path: string
-  protected dirty: boolean
-  protected flat: boolean
-  protected parent?: Dir
-  protected parentKey?: string
-  protected unixfs?: UnixFS
-  protected mode?: number
+abstract class Dir {
+  public options: PersistOptions
+  public root: boolean
+  public dir: boolean
+  public path: string
+  public dirty: boolean
+  public flat: boolean
+  public parent?: Dir
+  public parentKey?: string
+  public unixfs?: UnixFS
+  public mode?: number
   public mtime?: Mtime
-  protected cid?: CID
-  protected size?: number
+  public cid?: CID
+  public size?: number
+  public nodeSize?: number
 
-  constructor (props: DirProps, options: DirOptions) {
+  constructor (props: DirProps, options: PersistOptions) {
     this.options = options ?? {}
+
     this.root = props.root
     this.dir = props.dir
     this.path = props.path
@@ -74,26 +65,36 @@ export abstract class Dir {
     this.mode = props.mode
     this.mtime = props.mtime
   }
+
+  abstract put (name: string, value: InProgressImportResult | Dir): Promise<void>
+  abstract get (name: string): Promise<InProgressImportResult | Dir | undefined>
+  abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }>
+  abstract flush (blockstore: Blockstore): AsyncGenerator<ImportResult>
+  abstract estimateNodeSize (): number
+  abstract childCount (): number
 }
 
 export class DirSharded extends Dir {
-  public _bucket: Bucket<DirContents>
+  public _bucket: Bucket<InProgressImportResult | Dir>
 
-  constructor (props: DirProps, options: DirOptions) {
+  constructor (props: DirProps, options: PersistOptions) {
     super(props, options)
 
-    /** @type {Bucket<DirContents>} */
     this._bucket = createHAMT({
       hashFn: hamtHashFn,
-      bits: hamtBucketBits
+      bits: 8
     })
   }
 
-  async put (name: string, value: DirContents): Promise<void> {
+  async put (name: string, value: InProgressImportResult | Dir): Promise<void> {
+    this.cid = undefined
+    this.size = undefined
+    this.nodeSize = undefined
+
     await this._bucket.put(name, value)
   }
 
-  async get (name: string): Promise<DirContents | undefined> {
+  async get (name: string): Promise<InProgressImportResult | Dir | undefined> {
     return await this._bucket.get(name)
   }
 
@@ -105,11 +106,11 @@ export class DirSharded extends Dir {
     return this._bucket.childrenCount()
   }
 
-  onlyChild (): Bucket<DirContents> | BucketChild<DirContents> {
+  onlyChild (): Bucket<InProgressImportResult | Dir> | BucketChild<InProgressImportResult | Dir> {
     return this._bucket.onlyChild()
   }
 
-  async * eachChildSeries (): AsyncGenerator<{ key: string, child: DirContents }> {
+  async * eachChildSeries (): AsyncGenerator<{ key: string, child: InProgressImportResult | Dir }> {
     for await (const { key, value } of this._bucket.eachLeafSeries()) {
       yield {
         key,
@@ -118,15 +119,30 @@ export class DirSharded extends Dir {
     }
   }
 
-  async * flush (blockstore: Blockstore): AsyncIterable<ImportResult> {
-    yield * flush(this._bucket, blockstore, this, this.options)
+  estimateNodeSize (): number {
+    if (this.nodeSize !== undefined) {
+      return this.nodeSize
+    }
+
+    this.nodeSize = calculateSize(this._bucket, this, this.options)
+
+    return this.nodeSize
+  }
+
+  async * flush (blockstore: Blockstore): AsyncGenerator<ImportResult> {
+    for await (const entry of flush(this._bucket, blockstore, this, this.options)) {
+      yield {
+        ...entry,
+        path: this.path
+      }
+    }
   }
 }
 
-async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot: any, options: DirOptions): AsyncIterable<ImportResult> {
+async function * flush (bucket: Bucket<Dir | InProgressImportResult>, blockstore: Blockstore, shardRoot: DirSharded | null, options: PersistOptions): AsyncIterable<ImportResult> {
   const children = bucket._children
-  const links = []
-  let childrenSize = 0
+  const links: PBLink[] = []
+  let childrenSize = 0n
 
   for (let i = 0; i < children.length; i++) {
     const child = children.get(i)
@@ -138,7 +154,7 @@ async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot:
     const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
 
     if (child instanceof Bucket) {
-      let shard: ImportResult | undefined
+      let shard
 
       for await (const subShard of flush(child, blockstore, null, options)) {
         shard = subShard
@@ -150,13 +166,13 @@ async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot:
 
       links.push({
         Name: labelPrefix,
-        Tsize: shard.size,
+        Tsize: Number(shard.size),
         Hash: shard.cid
       })
       childrenSize += shard.size
-    } else if (typeof child.value.flush === 'function') {
+    } else if (isDir(child.value)) {
       const dir = child.value
-      let flushedDir
+      let flushedDir: ImportResult | undefined
 
       for await (const entry of dir.flush(blockstore)) {
         flushedDir = entry
@@ -164,14 +180,18 @@ async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot:
         yield flushedDir
       }
 
+      if (flushedDir == null) {
+        throw new Error('Did not flush dir')
+      }
+
       const label = labelPrefix + child.key
       links.push({
         Name: label,
-        Tsize: flushedDir.size,
+        Tsize: Number(flushedDir.size),
         Hash: flushedDir.cid
       })
 
-      childrenSize += flushedDir.size // eslint-disable-line @typescript-eslint/restrict-plus-operands
+      childrenSize += flushedDir.size
     } else {
       const value = child.value
 
@@ -184,10 +204,10 @@ async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot:
 
       links.push({
         Name: label,
-        Tsize: size,
+        Tsize: Number(size),
         Hash: value.cid
       })
-      childrenSize += size ?? 0 // eslint-disable-line @typescript-eslint/restrict-plus-operands
+      childrenSize += BigInt(size ?? 0)
     }
   }
 
@@ -197,7 +217,7 @@ async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot:
   const dir = new UnixFS({
     type: 'hamt-sharded-directory',
     data,
-    fanout: bucket.tableSize(),
+    fanout: BigInt(bucket.tableSize()),
     hashType: hamtHashCode,
     mtime: shardRoot?.mtime,
     mode: shardRoot?.mode
@@ -209,11 +229,90 @@ async function * flush (bucket: Bucket<any>, blockstore: Blockstore, shardRoot:
   }
   const buffer = encode(prepare(node))
   const cid = await persist(buffer, blockstore, options)
-  const size = buffer.length + childrenSize
+  const size = BigInt(buffer.byteLength) + childrenSize
 
   yield {
     cid,
-    node,
+    unixfs: dir,
     size
   }
 }
+
+function isDir (obj: any): obj is Dir {
+  return typeof obj.flush === 'function'
+}
+
+function calculateSize (bucket: Bucket<any>, shardRoot: DirSharded | null, options: PersistOptions): number {
+  const children = bucket._children
+  const links: PBLink[] = []
+
+  for (let i = 0; i < children.length; i++) {
+    const child = children.get(i)
+
+    if (child == null) {
+      continue
+    }
+
+    const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
+
+    if (child instanceof Bucket) {
+      const size = calculateSize(child, null, options)
+
+      links.push({
+        Name: labelPrefix,
+        Tsize: Number(size),
+        Hash: options.cidVersion === 0 ? CID_V0 : CID_V1
+      })
+    } else if (typeof child.value.flush === 'function') {
+      const dir = child.value
+      const size = dir.nodeSize()
+
+      links.push({
+        Name: labelPrefix + child.key,
+        Tsize: Number(size),
+        Hash: options.cidVersion === 0 ? CID_V0 : CID_V1
+      })
+    } else {
+      const value = child.value
+
+      if (value.cid == null) {
+        continue
+      }
+
+      const label = labelPrefix + child.key
+      const size = value.size
+
+      links.push({
+        Name: label,
+        Tsize: Number(size),
+        Hash: value.cid
+      })
+    }
+  }
+
+  // go-ipfs uses little endian, that's why we have to
+  // reverse the bit field before storing it
+  const data = Uint8Array.from(children.bitField().reverse())
+  const dir = new UnixFS({
+    type: 'hamt-sharded-directory',
+    data,
+    fanout: BigInt(bucket.tableSize()),
+    hashType: hamtHashCode,
+    mtime: shardRoot?.mtime,
+    mode: shardRoot?.mode
+  })
+
+  const buffer = encode(prepare({
+    Data: dir.marshal(),
+    Links: links
+  }))
+
+  return buffer.length
+}
+
+// we use these to calculate the node size to use as a check for whether a directory
+// should be sharded or not. Since CIDs have a constant length and We're only
+// interested in the data length and not the actual content identifier we can use
+// any old CID instead of having to hash the data which is expensive.
+export const CID_V0 = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')
+export const CID_V1 = CID.parse('zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi')

package/src/commands/utils/errors.ts

@@ -1,31 +1,65 @@
-import { HeliaError } from '@helia/interface/errors'
+export abstract class UnixFSError extends Error {
+  public readonly name: string
+  public readonly code: string
 
-export class NotUnixFSError extends HeliaError {
+  constructor (message: string, name: string, code: string) {
+    super(message)
+
+    this.name = name
+    this.code = code
+  }
+}
+
+export class NotUnixFSError extends UnixFSError {
   constructor (message = 'not a Unixfs node') {
     super(message, 'NotUnixFSError', 'ERR_NOT_UNIXFS')
   }
 }
 
-export class InvalidPBNodeError extends HeliaError {
+export class InvalidPBNodeError extends UnixFSError {
   constructor (message = 'invalid PBNode') {
     super(message, 'InvalidPBNodeError', 'ERR_INVALID_PBNODE')
   }
 }
 
-export class UnknownError extends HeliaError {
+export class UnknownError extends UnixFSError {
   constructor (message = 'unknown error') {
     super(message, 'InvalidPBNodeError', 'ERR_UNKNOWN_ERROR')
   }
 }
 
-export class AlreadyExistsError extends HeliaError {
+export class AlreadyExistsError extends UnixFSError {
   constructor (message = 'path already exists') {
-    super(message, 'NotUnixFSError', 'ERR_ALREADY_EXISTS')
+    super(message, 'AlreadyExistsError', 'ERR_ALREADY_EXISTS')
   }
 }
 
-export class DoesNotExistError extends HeliaError {
+export class DoesNotExistError extends UnixFSError {
   constructor (message = 'path does not exist') {
-    super(message, 'NotUnixFSError', 'ERR_DOES_NOT_EXIST')
+    super(message, 'DoesNotExistError', 'ERR_DOES_NOT_EXIST')
+  }
+}
+
+export class NoContentError extends UnixFSError {
+  constructor (message = 'no content') {
+    super(message, 'NoContentError', 'ERR_NO_CONTENT')
+  }
+}
+
+export class NotAFileError extends UnixFSError {
+  constructor (message = 'not a file') {
+    super(message, 'NotAFileError', 'ERR_NOT_A_FILE')
+  }
+}
+
+export class NotADirectoryError extends UnixFSError {
+  constructor (message = 'not a directory') {
+    super(message, 'NotADirectoryError', 'ERR_NOT_A_DIRECTORY')
+  }
+}
+
+export class InvalidParametersError extends UnixFSError {
+  constructor (message = 'invalid parameters') {
+    super(message, 'InvalidParametersError', 'ERR_INVALID_PARAMETERS')
   }
 }

package/src/commands/utils/hamt-constants.ts

@@ -1,6 +1,6 @@
 import { murmur3128 } from '@multiformats/murmur3'
 
-export const hamtHashCode = murmur3128.code
+export const hamtHashCode = BigInt(murmur3128.code)
 export const hamtBucketBits = 8
 
 export async function hamtHashFn (buf: Uint8Array): Promise<Uint8Array> {

package/src/commands/utils/hamt-utils.ts

@@ -7,19 +7,19 @@ import { DirSharded } from './dir-sharded.js'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
 import last from 'it-last'
-import { CID } from 'multiformats/cid'
+import type { CID, Version } from 'multiformats/cid'
 import {
   hamtHashCode,
   hamtHashFn,
   hamtBucketBits
 } from './hamt-constants.js'
 import type { PBLink, PBNode } from '@ipld/dag-pb/interface'
-import { sha256 } from 'multiformats/hashes/sha2'
 import type { Blockstore } from 'interface-blockstore'
 import type { Mtime } from 'ipfs-unixfs'
 import type { Directory } from './cid-to-directory.js'
 import type { AbortOptions } from '@libp2p/interfaces'
 import type { ImportResult } from 'ipfs-unixfs-importer'
+import { persist } from './persist.js'
 
 const log = logger('helia:unixfs:commands:utils:hamt-utils')
 
@@ -29,37 +29,38 @@ export interface UpdateHamtResult {
   size: number
 }
 
-export const updateHamtDirectory = async (parent: Directory, blockstore: Blockstore, links: PBLink[], bucket: Bucket<any>, options: AbortOptions): Promise<UpdateHamtResult> => {
-  if (parent.node.Data == null) {
+export interface UpdateHamtDirectoryOptions extends AbortOptions {
+  cidVersion: Version
+}
+
+export const updateHamtDirectory = async (pbNode: PBNode, blockstore: Blockstore, bucket: Bucket<any>, options: UpdateHamtDirectoryOptions): Promise<UpdateHamtResult> => {
+  if (pbNode.Data == null) {
     throw new Error('Could not update HAMT directory because parent had no data')
   }
 
   // update parent with new bit field
-  const data = Uint8Array.from(bucket._children.bitField().reverse())
-  const node = UnixFS.unmarshal(parent.node.Data)
+  const node = UnixFS.unmarshal(pbNode.Data)
   const dir = new UnixFS({
     type: 'hamt-sharded-directory',
-    data,
-    fanout: bucket.tableSize(),
+    data: Uint8Array.from(bucket._children.bitField().reverse()),
+    fanout: BigInt(bucket.tableSize()),
     hashType: hamtHashCode,
     mode: node.mode,
     mtime: node.mtime
   })
 
-  parent.node = {
+  const updatedPbNode = {
     Data: dir.marshal(),
-    Links: links.sort((a, b) => (a.Name ?? '').localeCompare(b.Name ?? ''))
+    Links: pbNode.Links
   }
-  const buf = dagPB.encode(parent.node)
-  const hash = await sha256.digest(buf)
-  const cid = CID.create(parent.cid.version, dagPB.code, hash)
 
-  await blockstore.put(cid, buf, options)
+  const buf = dagPB.encode(dagPB.prepare(updatedPbNode))
+  const cid = await persist(buf, blockstore, options)
 
   return {
-    node: parent.node,
+    node: updatedPbNode,
     cid,
-    size: links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.length)
+    size: pbNode.Links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.byteLength)
   }
 }
 
@@ -77,26 +78,24 @@ export const recreateHamtLevel = async (blockstore: Blockstore, links: PBLink[],
 }
 
 export const recreateInitialHamtLevel = async (links: PBLink[]): Promise<Bucket<any>> => {
-  const bucket = createHAMT({
+  const bucket = createHAMT<any>({
     hashFn: hamtHashFn,
     bits: hamtBucketBits
   })
 
-  // populate sub bucket but do not recurse as we do not want to pull whole shard in
+  // populate sub bucket but do not recurse as we do not want to load the whole shard
   await Promise.all(
     links.map(async link => {
       const linkName = (link.Name ?? '')
 
       if (linkName.length === 2) {
         const pos = parseInt(linkName, 16)
-
         const subBucket = new Bucket({
           hash: bucket._options.hash,
           bits: bucket._options.bits
         }, bucket, pos)
-        bucket._putObjectAt(pos, subBucket)
 
-        await Promise.resolve(); return
+        bucket._putObjectAt(pos, subBucket)
       }
 
       await bucket.put(linkName.substring(2), {
@@ -127,8 +126,6 @@ export const addLinksToHamtBucket = async (blockstore: Blockstore, links: PBLink
         bucket._putObjectAt(pos, subBucket)
 
         await addLinksToHamtBucket(blockstore, node.Links, subBucket, rootBucket, options)
-
-        await Promise.resolve(); return
       }
 
       await rootBucket.put(linkName.substring(2), {
@@ -147,22 +144,19 @@ export const toPrefix = (position: number): string => {
     .substring(0, 2)
 }
 
-export interface HamtPath {
-  rootBucket: Bucket<any>
-  path: Array<{
-    bucket: Bucket<any>
-    prefix: string
-    node?: PBNode
-  }>
+export interface HamtPathSegment {
+  bucket?: Bucket<any>
+  prefix?: string
+  node?: PBNode
+  cid?: CID
+  size?: number
 }
 
-export const generatePath = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<HamtPath> => {
+export const generatePath = async (root: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<HamtPathSegment[]> => {
   // start at the root bucket and descend, loading nodes as we go
-  const rootBucket = await recreateInitialHamtLevel(parent.node.Links)
+  const rootBucket = await recreateInitialHamtLevel(root.node.Links)
   const position = await rootBucket._findNewBucketAndPos(name)
-
-  // the path to the root bucket
-  const path: Array<{ bucket: Bucket<any>, prefix: string, node?: PBNode }> = [{
+  const path: HamtPathSegment[] = [{
     bucket: position.bucket,
     prefix: toPrefix(position.pos)
   }]
@@ -174,23 +168,32 @@ export const generatePath = async (parent: Directory, name: string, blockstore:
       prefix: toPrefix(currentBucket._posAtParent)
     })
 
-    // @ts-expect-error - only the root bucket's parent will be undefined
+    if (currentBucket._parent == null) {
+      break
+    }
+
     currentBucket = currentBucket._parent
   }
 
+  // add the root bucket to the path
+  path.push({
+    bucket: rootBucket,
+    node: root.node
+  })
+
   path.reverse()
-  path[0].node = parent.node
 
   // load PbNode for each path segment
-  for (let i = 0; i < path.length; i++) {
+  for (let i = 1; i < path.length; i++) {
     const segment = path[i]
+    const previousSegment = path[i - 1]
 
-    if (segment.node == null) {
+    if (previousSegment.node == null) {
       throw new Error('Could not generate HAMT path')
     }
 
     // find prefix in links
-    const link = segment.node.Links
+    const link = previousSegment.node.Links
       .filter(link => (link.Name ?? '').substring(0, 2) === segment.prefix)
       .pop()
 
@@ -202,8 +205,10 @@ export const generatePath = async (parent: Directory, name: string, blockstore:
       continue
     }
 
+    const linkName = link.Name ?? ''
+
     // found entry
-    if (link.Name === `${segment.prefix}${name}`) {
+    if (linkName === `${segment.prefix}${name}`) {
       log(`Link ${segment.prefix}${name} will be replaced`)
       // file already existed, file will be added to the current bucket
       // return path
@@ -212,13 +217,17 @@ export const generatePath = async (parent: Directory, name: string, blockstore:
 
     // found subshard
     log(`Found subshard ${segment.prefix}`)
-    const block = await blockstore.get(link.Hash, options)
+    const block = await blockstore.get(link.Hash)
     const node = dagPB.decode(block)
 
     // subshard hasn't been loaded, descend to the next level of the HAMT
     if (path[i + 1] == null) {
       log(`Loaded new subshard ${segment.prefix}`)
 
+      if (segment.bucket == null || segment.prefix == null) {
+        throw new Error('Shard was invalid')
+      }
+
       await recreateHamtLevel(blockstore, node.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options)
       const position = await rootBucket._findNewBucketAndPos(name)
 
@@ -232,30 +241,30 @@ export const generatePath = async (parent: Directory, name: string, blockstore:
       continue
     }
 
-    const nextSegment = path[i + 1]
+    if (segment.bucket == null) {
+      throw new Error('Shard was invalid')
+    }
 
     // add intermediate links to bucket
-    await addLinksToHamtBucket(blockstore, node.Links, nextSegment.bucket, rootBucket, options)
+    await addLinksToHamtBucket(blockstore, node.Links, segment.bucket, rootBucket, options)
 
-    nextSegment.node = node
+    segment.node = node
   }
 
   await rootBucket.put(name, true)
 
   path.reverse()
 
-  return {
-    rootBucket,
-    path
-  }
+  return path
 }
 
 export interface CreateShardOptions {
   mtime?: Mtime
   mode?: number
+  cidVersion: Version
 }
 
-export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: number, cid: CID }>, options: CreateShardOptions = {}): Promise<ImportResult> => {
+export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise<ImportResult> => {
   const shard = new DirSharded({
     root: true,
     dir: true,