@helia/unixfs 4.0.2 → 4.0.3-325b36f

package/src/commands/add.ts

@@ -1,31 +1,36 @@
-import { type ByteStream, type DirectoryCandidate, type FileCandidate, importBytes, importByteStream, type ImportCandidateStream, importDirectory, importer, type ImporterOptions, importFile, type ImportResult } from 'ipfs-unixfs-importer'
+import { importBytes, importByteStream, importer } from 'ipfs-unixfs-importer'
 import { fixedSize } from 'ipfs-unixfs-importer/chunker'
 import { balanced } from 'ipfs-unixfs-importer/layout'
+import first from 'it-first'
+import last from 'it-last'
+import { InvalidParametersError } from '../errors.js'
+import type { FileCandidate, AddOptions, AddFileOptions } from '../index.js'
 import type { PutStore } from '../unixfs.js'
+import type { ByteStream, DirectoryCandidate, ImportCandidateStream, ImportResult } from 'ipfs-unixfs-importer'
 import type { CID } from 'multiformats/cid'
 
 /**
  * Default importer settings match Filecoin
  */
-const defaultImporterSettings: ImporterOptions = {
+const defaultImporterSettings: AddOptions = {
   cidVersion: 1,
   rawLeaves: true,
   layout: balanced({
     maxChildrenPerNode: 1024
   }),
   chunker: fixedSize({
-    chunkSize: 1048576
+    chunkSize: 1_048_576
   })
 }
 
-export async function * addAll (source: ImportCandidateStream, blockstore: PutStore, options: Partial<ImporterOptions> = {}): AsyncGenerator<ImportResult, void, unknown> {
+export async function * addAll (source: ImportCandidateStream, blockstore: PutStore, options: Partial<AddOptions> = {}): AsyncGenerator<ImportResult, void, unknown> {
   yield * importer(source, blockstore, {
     ...defaultImporterSettings,
     ...options
   })
 }
 
-export async function addBytes (bytes: Uint8Array, blockstore: PutStore, options: Partial<ImporterOptions> = {}): Promise<CID> {
+export async function addBytes (bytes: Uint8Array, blockstore: PutStore, options: Partial<AddFileOptions> = {}): Promise<CID> {
   const { cid } = await importBytes(bytes, blockstore, {
     ...defaultImporterSettings,
     ...options
@@ -34,7 +39,7 @@ export async function addBytes (bytes: Uint8Array, blockstore: PutStore, options
   return cid
 }
 
-export async function addByteStream (bytes: ByteStream, blockstore: PutStore, options: Partial<ImporterOptions> = {}): Promise<CID> {
+export async function addByteStream (bytes: ByteStream, blockstore: PutStore, options: Partial<AddFileOptions> = {}): Promise<CID> {
   const { cid } = await importByteStream(bytes, blockstore, {
     ...defaultImporterSettings,
     ...options
@@ -43,23 +48,48 @@ export async function addByteStream (bytes: ByteStream, blockstore: PutStore, op
   return cid
 }
 
-export async function addFile (file: FileCandidate, blockstore: PutStore, options: Partial<ImporterOptions> = {}): Promise<CID> {
-  const { cid } = await importFile(file, blockstore, {
+export async function addFile (file: FileCandidate, blockstore: PutStore, options: Partial<AddFileOptions> = {}): Promise<CID> {
+  if (file.path == null) {
+    throw new InvalidParametersError('path is required')
+  }
+
+  if (file.content == null) {
+    throw new InvalidParametersError('content is required')
+  }
+
+  const result = await last(addAll([file], blockstore, {
     ...defaultImporterSettings,
-    ...options
-  })
+    ...options,
+    wrapWithDirectory: true
+  }))
 
-  return cid
+  if (result == null) {
+    throw new InvalidParametersError('Nothing imported')
+  }
+
+  return result.cid
 }
 
-export async function addDirectory (dir: Partial<DirectoryCandidate>, blockstore: PutStore, options: Partial<ImporterOptions> = {}): Promise<CID> {
-  const { cid } = await importDirectory({
+export async function addDirectory (dir: Partial<DirectoryCandidate>, blockstore: PutStore, options: Partial<AddFileOptions> = {}): Promise<CID> {
+  // @ts-expect-error field is not in the types
+  if (dir.content != null) {
+    throw new InvalidParametersError('Directories cannot have content, use addFile instead')
+  }
+
+  const ord = dir.path == null ? first : last
+
+  const result = await ord(addAll([{
     ...dir,
     path: dir.path ?? '-'
-  }, blockstore, {
+  }], blockstore, {
     ...defaultImporterSettings,
-    ...options
-  })
+    ...options,
+    wrapWithDirectory: dir.path != null
+  }))
 
-  return cid
+  if (result == null) {
+    throw new InvalidParametersError('Nothing imported')
+  }
+
+  return result.cid
 }

package/src/commands/stat.ts

@@ -1,17 +1,21 @@
 import * as dagPb from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
+import { ScalableCuckooFilter } from '@libp2p/utils/filters'
 import { UnixFS } from 'ipfs-unixfs'
-import { exporter } from 'ipfs-unixfs-exporter'
+import { exporter, type RawNode, type UnixFSDirectory, type UnixFSFile } from 'ipfs-unixfs-exporter'
 import mergeOpts from 'merge-options'
 import * as raw from 'multiformats/codecs/raw'
 import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js'
 import { resolve } from './utils/resolve.js'
-import type { StatOptions, UnixFSStats } from '../index.js'
+import type { ExtendedStatOptions, ExtendedDirectoryStats, ExtendedFileStats, StatOptions, DirectoryStats, FileStats, RawStats, ExtendedRawStats } from '../index.js'
 import type { GetStore, HasStore } from '../unixfs.js'
-import type { AbortOptions } from '@libp2p/interface'
-import type { Mtime } from 'ipfs-unixfs'
+import type { Filter } from '@libp2p/utils/filters'
 import type { CID } from 'multiformats/cid'
 
+// https://github.com/ipfs/specs/blob/main/UNIXFS.md#metadata
+const DEFAULT_DIR_MODE = 0x755
+const DEFAULT_FILE_MODE = 0x644
+
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:stat')
 
@@ -19,7 +23,9 @@ const defaultOptions: StatOptions = {
 
 }
 
-export async function stat (cid: CID, blockstore: GetStore & HasStore, options: Partial<StatOptions> = {}): Promise<UnixFSStats> {
+export async function stat (cid: CID, blockstore: GetStore & HasStore, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+export async function stat (cid: CID, blockstore: GetStore & HasStore, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
+export async function stat (cid: CID, blockstore: GetStore & HasStore, options: Partial<ExtendedStatOptions> = {}): Promise<any> {
   const opts: StatOptions = mergeOptions(defaultOptions, options)
   const resolved = await resolve(cid, options.path, blockstore, opts)
 
@@ -27,116 +33,186 @@ export async function stat (cid: CID, blockstore: GetStore & HasStore, options:
 
   const result = await exporter(resolved.cid, blockstore, opts)
 
-  if (result.type !== 'file' && result.type !== 'directory' && result.type !== 'raw') {
-    throw new NotUnixFSError()
+  if (result.type === 'raw') {
+    if (options.extended === true) {
+      return createExtendedRawStats(result)
+    }
+
+    return createRawStats(result)
+  } else if (result.type === 'file' || result.type === 'directory') {
+    if (options.extended === true) {
+      return createExtendedStats(result, blockstore, options.filter ?? new ScalableCuckooFilter({ filterSize: 1024 }), options)
+    }
+
+    return createStats(result)
   }
 
-  let fileSize: bigint = 0n
-  let dagSize: bigint = 0n
-  let localFileSize: bigint = 0n
-  let localDagSize: bigint = 0n
-  let blocks: number = 0
-  let mode: number | undefined
-  let mtime: Mtime | undefined
-  const type = result.type
-  let unixfs: UnixFS | undefined
+  throw new NotUnixFSError()
+}
 
-  if (result.type === 'raw') {
-    fileSize = BigInt(result.node.byteLength)
-    dagSize = BigInt(result.node.byteLength)
-    localFileSize = BigInt(result.node.byteLength)
-    localDagSize = BigInt(result.node.byteLength)
-    blocks = 1
+function createStats (entry: UnixFSFile | UnixFSDirectory): FileStats | DirectoryStats {
+  return {
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: entry.unixfs,
+    mode: entry.unixfs.mode ?? (entry.unixfs.isDirectory() ? DEFAULT_DIR_MODE : DEFAULT_FILE_MODE),
+    mtime: entry.unixfs.mtime,
+    size: entry.unixfs.fileSize()
   }
+}
 
-  if (result.type === 'directory') {
-    fileSize = 0n
-    dagSize = BigInt(result.unixfs.marshal().byteLength)
-    localFileSize = 0n
-    localDagSize = dagSize
-    blocks = 1
-    mode = result.unixfs.mode
-    mtime = result.unixfs.mtime
-    unixfs = result.unixfs
+async function createExtendedStats (entry: UnixFSFile | UnixFSDirectory, blockstore: GetStore & HasStore, filter: Filter, options: StatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats> {
+  const stats = await inspectDag(entry.cid, blockstore, false, filter, options)
+
+  return {
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: entry.unixfs,
+    size: entry.unixfs.isDirectory() ? stats.dirSize : entry.unixfs.fileSize(),
+    mode: entry.unixfs.mode ?? (entry.unixfs.isDirectory() ? DEFAULT_DIR_MODE : DEFAULT_FILE_MODE),
+    mtime: entry.unixfs.mtime,
+    localSize: stats.localSize,
+    dagSize: stats.dagSize,
+    deduplicatedDagSize: stats.deduplicatedDagSize,
+    blocks: stats.blocks,
+    uniqueBlocks: stats.uniqueBlocks
   }
+}
 
-  if (result.type === 'file') {
-    const results = await inspectDag(resolved.cid, blockstore, opts)
-
-    fileSize = result.unixfs.fileSize()
-    dagSize = BigInt((result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0))
-    localFileSize = BigInt(results.localFileSize)
-    localDagSize = BigInt(results.localDagSize)
-    blocks = results.blocks
-    mode = result.unixfs.mode
-    mtime = result.unixfs.mtime
-    unixfs = result.unixfs
+function createRawStats (entry: RawNode): RawStats {
+  return {
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: undefined,
+    mode: DEFAULT_FILE_MODE,
+    mtime: undefined,
+    size: BigInt(entry.node.byteLength)
   }
+}
 
+function createExtendedRawStats (entry: RawNode): ExtendedRawStats {
   return {
-    cid: resolved.cid,
-    mode,
-    mtime,
-    fileSize,
-    dagSize,
-    localFileSize,
-    localDagSize,
-    blocks,
-    type,
-    unixfs
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: undefined,
+    mode: DEFAULT_FILE_MODE,
+    mtime: undefined,
+    size: BigInt(entry.node.byteLength),
+    localSize: BigInt(entry.node.byteLength),
+    dagSize: BigInt(entry.node.byteLength),
+    deduplicatedDagSize: BigInt(entry.node.byteLength),
+    blocks: 1n,
+    uniqueBlocks: 1n
   }
 }
 
 interface InspectDagResults {
-  localFileSize: number
-  localDagSize: number
-  blocks: number
+  dirSize: bigint
+  localSize: bigint
+  dagSize: bigint
+  deduplicatedDagSize: bigint
+  blocks: bigint
+  uniqueBlocks: bigint
 }
 
-async function inspectDag (cid: CID, blockstore: GetStore & HasStore, options: AbortOptions): Promise<InspectDagResults> {
-  const results = {
-    localFileSize: 0,
-    localDagSize: 0,
-    blocks: 0
+async function inspectDag (cid: CID, blockstore: GetStore & HasStore, isFile: boolean, filter: Filter, options: StatOptions): Promise<InspectDagResults> {
+  const results: InspectDagResults = {
+    dirSize: 0n,
+    localSize: 0n,
+    dagSize: 0n,
+    deduplicatedDagSize: 0n,
+    blocks: 0n,
+    uniqueBlocks: 0n
   }
 
-  if (await blockstore.has(cid, options)) {
+  try {
+    const alreadyTraversed = filter.has(cid.bytes)
+    filter.add(cid.bytes)
+
     const block = await blockstore.get(cid, options)
     results.blocks++
-    results.localDagSize += block.byteLength
+    results.dagSize += BigInt(block.byteLength)
+
+    if (!alreadyTraversed) {
+      results.uniqueBlocks++
+      results.deduplicatedDagSize += BigInt(block.byteLength)
+    }
 
     if (cid.code === raw.code) {
-      results.localFileSize += block.byteLength
+      results.localSize += BigInt(block.byteLength)
+
+      if (isFile) {
+        results.dirSize += BigInt(block.byteLength)
+      }
     } else if (cid.code === dagPb.code) {
       const pbNode = dagPb.decode(block)
 
+      let unixfs: UnixFS | undefined
+
+      if (pbNode.Data != null) {
+        unixfs = UnixFS.unmarshal(pbNode.Data)
+      }
+
       if (pbNode.Links.length > 0) {
         // intermediate node
         for (const link of pbNode.Links) {
-          const linkResult = await inspectDag(link.Hash, blockstore, options)
+          const linkResult = await inspectDag(link.Hash, blockstore, linkIsFile(link, unixfs), filter, options)
 
-          results.localFileSize += linkResult.localFileSize
-          results.localDagSize += linkResult.localDagSize
+          results.localSize += linkResult.localSize
+          results.dagSize += linkResult.dagSize
+          results.deduplicatedDagSize += linkResult.deduplicatedDagSize
           results.blocks += linkResult.blocks
+          results.uniqueBlocks += linkResult.uniqueBlocks
+          results.dirSize += linkResult.dirSize
+        }
+
+        // multi-block file node
+        if (isFile && unixfs != null) {
+          results.dirSize += unixfs.fileSize()
         }
       } else {
-        // leaf node
-        if (pbNode.Data == null) {
+        if (unixfs == null) {
           throw new InvalidPBNodeError(`PBNode ${cid.toString()} had no data`)
         }
 
-        const unixfs = UnixFS.unmarshal(pbNode.Data)
-
-        if (unixfs.data == null) {
-          throw new InvalidPBNodeError(`UnixFS node ${cid.toString()} had no data`)
+        // multi-block file leaf node
+        if (unixfs.data != null) {
+          results.localSize += BigInt(unixfs.data.byteLength ?? 0)
         }
 
-        results.localFileSize += unixfs.data.byteLength ?? 0
+        // single-block file node
+        if (isFile) {
+          results.dirSize += unixfs.fileSize()
+        }
       }
     } else {
       throw new UnknownError(`${cid.toString()} was neither DAG_PB nor RAW`)
     }
+  } catch (err: any) {
+    if (err.name !== 'NotFoundError' || options.offline !== true) {
+      throw err
+    }
   }
 
   return results
 }
+
+function linkIsFile (link: dagPb.PBLink, parent?: UnixFS): boolean {
+  if (parent == null) {
+    return false
+  }
+
+  const name = link.Name
+
+  if (name == null) {
+    return false
+  }
+
+  if (parent.type === 'directory') {
+    return true
+  } else if (parent.type === 'hamt-sharded-directory' && name.length > 2) {
+    return true
+  }
+
+  return false
+}

package/src/commands/utils/add-link.ts

@@ -165,7 +165,7 @@ const addToShardedDirectory = async (parent: Directory, child: Required<PBLink>,
         throw new AlreadyExistsError()
       }
 
-      log('overwriting %s in subshard', child.Name)
+      log('overwriting %s in sub-shard', child.Name)
       finalSegment.node.Links = finalSegment.node.Links.filter(l => l.Name !== linkName)
       finalSegment.node.Links.push({
         Name: linkName,
@@ -173,10 +173,10 @@ const addToShardedDirectory = async (parent: Directory, child: Required<PBLink>,
         Tsize: child.Tsize
       })
     } else if (existingLink.Name?.length === 2) {
-      throw new Error('Existing link was subshard?!')
+      throw new Error('Existing link was sub-shard?!')
     } else {
       // conflict, add a new HAMT segment
-      log('prefix %s already exists, creating new subshard', prefix)
+      log('prefix %s already exists, creating new sub-shard', prefix)
       // find the sibling we are going to replace
       const index = finalSegment.node.Links.findIndex(l => l.Name?.startsWith(prefix))
       const sibling = finalSegment.node.Links.splice(index, 1)[0]
@@ -186,7 +186,7 @@ const addToShardedDirectory = async (parent: Directory, child: Required<PBLink>,
       const wrapped = wrapHash(hamtHashFn)
       const siblingHash = wrapped(uint8ArrayFromString(siblingName))
 
-      // discard hash bits until we reach the subshard depth
+      // discard hash bits until we reach the sub-shard depth
       for (let i = 0; i < path.length; i++) {
         await siblingHash.take(hamtBucketBits)
       }

package/src/commands/utils/dir-sharded.ts

@@ -161,7 +161,7 @@ async function * flush (bucket: Bucket<Dir | InProgressImportResult>, blockstore
       }
 
       if (shard == null) {
-        throw new Error('Could not flush sharded directory, no subshard found')
+        throw new Error('Could not flush sharded directory, no sub-shard found')
       }
 
       links.push({

package/src/commands/utils/remove-link.ts

@@ -96,7 +96,7 @@ const removeFromShardedDirectory = async (parent: Directory, name: string, block
   finalSegment.children.unset(index)
 
   if (finalSegment.node.Links.length === 1) {
-    // replace the subshard with the last remaining file in the parent
+    // replace the sub-shard with the last remaining file in the parent
     while (true) {
       if (path.length === 1) {
         break

package/src/errors.ts

@@ -18,7 +18,7 @@ export class NotUnixFSError extends UnixFSError {
 
 export class InvalidPBNodeError extends UnixFSError {
   constructor (message = 'invalid PBNode') {
-    super(message, 'InvalidPBNodeError', 'ERR_INVALID_PBNODE')
+    super(message, 'InvalidPBNodeError', 'ERR_INVALID_PB_NODE')
   }
 }