@helia/unixfs 4.0.3-c0bf36e → 5.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/unixfs",
-  "version": "4.0.3-c0bf36e",
+  "version": "5.0.0",
   "description": "A Helia-compatible wrapper for UnixFS",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/unixfs#readme",
@@ -74,10 +74,11 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/interface": "5.2.1-c0bf36e",
+    "@helia/interface": "^5.2.1",
     "@ipld/dag-pb": "^4.1.3",
     "@libp2p/interface": "^2.2.1",
     "@libp2p/logger": "^5.1.4",
+    "@libp2p/utils": "^6.6.0",
     "@multiformats/murmur3": "^2.1.8",
     "hamt-sharding": "^3.0.6",
     "interface-blockstore": "^5.3.1",

package/src/commands/stat.ts

@@ -1,17 +1,21 @@
 import * as dagPb from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
+import { ScalableCuckooFilter } from '@libp2p/utils/filters'
 import { UnixFS } from 'ipfs-unixfs'
-import { exporter } from 'ipfs-unixfs-exporter'
+import { exporter, type RawNode, type UnixFSDirectory, type UnixFSFile } from 'ipfs-unixfs-exporter'
 import mergeOpts from 'merge-options'
 import * as raw from 'multiformats/codecs/raw'
 import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js'
 import { resolve } from './utils/resolve.js'
-import type { StatOptions, UnixFSStats } from '../index.js'
+import type { ExtendedStatOptions, ExtendedDirectoryStats, ExtendedFileStats, StatOptions, DirectoryStats, FileStats, RawStats, ExtendedRawStats } from '../index.js'
 import type { GetStore, HasStore } from '../unixfs.js'
-import type { AbortOptions } from '@libp2p/interface'
-import type { Mtime } from 'ipfs-unixfs'
+import type { Filter } from '@libp2p/utils/filters'
 import type { CID } from 'multiformats/cid'
 
+// https://github.com/ipfs/specs/blob/main/UNIXFS.md#metadata
+const DEFAULT_DIR_MODE = 0x755
+const DEFAULT_FILE_MODE = 0x644
+
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:stat')
 
@@ -19,7 +23,9 @@ const defaultOptions: StatOptions = {
 
 }
 
-export async function stat (cid: CID, blockstore: GetStore & HasStore, options: Partial<StatOptions> = {}): Promise<UnixFSStats> {
+export async function stat (cid: CID, blockstore: GetStore & HasStore, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+export async function stat (cid: CID, blockstore: GetStore & HasStore, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
+export async function stat (cid: CID, blockstore: GetStore & HasStore, options: Partial<ExtendedStatOptions> = {}): Promise<any> {
   const opts: StatOptions = mergeOptions(defaultOptions, options)
   const resolved = await resolve(cid, options.path, blockstore, opts)
 
@@ -27,116 +33,186 @@ export async function stat (cid: CID, blockstore: GetStore & HasStore, options:
 
   const result = await exporter(resolved.cid, blockstore, opts)
 
-  if (result.type !== 'file' && result.type !== 'directory' && result.type !== 'raw') {
-    throw new NotUnixFSError()
+  if (result.type === 'raw') {
+    if (options.extended === true) {
+      return createExtendedRawStats(result)
+    }
+
+    return createRawStats(result)
+  } else if (result.type === 'file' || result.type === 'directory') {
+    if (options.extended === true) {
+      return createExtendedStats(result, blockstore, options.filter ?? new ScalableCuckooFilter({ filterSize: 1024 }), options)
+    }
+
+    return createStats(result)
   }
 
-  let fileSize: bigint = 0n
-  let dagSize: bigint = 0n
-  let localFileSize: bigint = 0n
-  let localDagSize: bigint = 0n
-  let blocks: number = 0
-  let mode: number | undefined
-  let mtime: Mtime | undefined
-  const type = result.type
-  let unixfs: UnixFS | undefined
+  throw new NotUnixFSError()
+}
 
-  if (result.type === 'raw') {
-    fileSize = BigInt(result.node.byteLength)
-    dagSize = BigInt(result.node.byteLength)
-    localFileSize = BigInt(result.node.byteLength)
-    localDagSize = BigInt(result.node.byteLength)
-    blocks = 1
+function createStats (entry: UnixFSFile | UnixFSDirectory): FileStats | DirectoryStats {
+  return {
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: entry.unixfs,
+    mode: entry.unixfs.mode ?? (entry.unixfs.isDirectory() ? DEFAULT_DIR_MODE : DEFAULT_FILE_MODE),
+    mtime: entry.unixfs.mtime,
+    size: entry.unixfs.fileSize()
   }
+}
 
-  if (result.type === 'directory') {
-    fileSize = 0n
-    dagSize = BigInt(result.unixfs.marshal().byteLength)
-    localFileSize = 0n
-    localDagSize = dagSize
-    blocks = 1
-    mode = result.unixfs.mode
-    mtime = result.unixfs.mtime
-    unixfs = result.unixfs
+async function createExtendedStats (entry: UnixFSFile | UnixFSDirectory, blockstore: GetStore & HasStore, filter: Filter, options: StatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats> {
+  const stats = await inspectDag(entry.cid, blockstore, false, filter, options)
+
+  return {
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: entry.unixfs,
+    size: entry.unixfs.isDirectory() ? stats.dirSize : entry.unixfs.fileSize(),
+    mode: entry.unixfs.mode ?? (entry.unixfs.isDirectory() ? DEFAULT_DIR_MODE : DEFAULT_FILE_MODE),
+    mtime: entry.unixfs.mtime,
+    localSize: stats.localSize,
+    dagSize: stats.dagSize,
+    deduplicatedDagSize: stats.deduplicatedDagSize,
+    blocks: stats.blocks,
+    uniqueBlocks: stats.uniqueBlocks
   }
+}
 
-  if (result.type === 'file') {
-    const results = await inspectDag(resolved.cid, blockstore, opts)
-
-    fileSize = result.unixfs.fileSize()
-    dagSize = BigInt((result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0))
-    localFileSize = BigInt(results.localFileSize)
-    localDagSize = BigInt(results.localDagSize)
-    blocks = results.blocks
-    mode = result.unixfs.mode
-    mtime = result.unixfs.mtime
-    unixfs = result.unixfs
+function createRawStats (entry: RawNode): RawStats {
+  return {
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: undefined,
+    mode: DEFAULT_FILE_MODE,
+    mtime: undefined,
+    size: BigInt(entry.node.byteLength)
   }
+}
 
+function createExtendedRawStats (entry: RawNode): ExtendedRawStats {
   return {
-    cid: resolved.cid,
-    mode,
-    mtime,
-    fileSize,
-    dagSize,
-    localFileSize,
-    localDagSize,
-    blocks,
-    type,
-    unixfs
+    type: entry.type,
+    cid: entry.cid,
+    unixfs: undefined,
+    mode: DEFAULT_FILE_MODE,
+    mtime: undefined,
+    size: BigInt(entry.node.byteLength),
+    localSize: BigInt(entry.node.byteLength),
+    dagSize: BigInt(entry.node.byteLength),
+    deduplicatedDagSize: BigInt(entry.node.byteLength),
+    blocks: 1n,
+    uniqueBlocks: 1n
   }
 }
 
 interface InspectDagResults {
-  localFileSize: number
-  localDagSize: number
-  blocks: number
+  dirSize: bigint
+  localSize: bigint
+  dagSize: bigint
+  deduplicatedDagSize: bigint
+  blocks: bigint
+  uniqueBlocks: bigint
 }
 
-async function inspectDag (cid: CID, blockstore: GetStore & HasStore, options: AbortOptions): Promise<InspectDagResults> {
-  const results = {
-    localFileSize: 0,
-    localDagSize: 0,
-    blocks: 0
+async function inspectDag (cid: CID, blockstore: GetStore & HasStore, isFile: boolean, filter: Filter, options: StatOptions): Promise<InspectDagResults> {
+  const results: InspectDagResults = {
+    dirSize: 0n,
+    localSize: 0n,
+    dagSize: 0n,
+    deduplicatedDagSize: 0n,
+    blocks: 0n,
+    uniqueBlocks: 0n
   }
 
-  if (await blockstore.has(cid, options)) {
+  try {
+    const alreadyTraversed = filter.has(cid.bytes)
+    filter.add(cid.bytes)
+
     const block = await blockstore.get(cid, options)
     results.blocks++
-    results.localDagSize += block.byteLength
+    results.dagSize += BigInt(block.byteLength)
+
+    if (!alreadyTraversed) {
+      results.uniqueBlocks++
+      results.deduplicatedDagSize += BigInt(block.byteLength)
+    }
 
     if (cid.code === raw.code) {
-      results.localFileSize += block.byteLength
+      results.localSize += BigInt(block.byteLength)
+
+      if (isFile) {
+        results.dirSize += BigInt(block.byteLength)
+      }
     } else if (cid.code === dagPb.code) {
       const pbNode = dagPb.decode(block)
 
+      let unixfs: UnixFS | undefined
+
+      if (pbNode.Data != null) {
+        unixfs = UnixFS.unmarshal(pbNode.Data)
+      }
+
       if (pbNode.Links.length > 0) {
         // intermediate node
         for (const link of pbNode.Links) {
-          const linkResult = await inspectDag(link.Hash, blockstore, options)
+          const linkResult = await inspectDag(link.Hash, blockstore, linkIsFile(link, unixfs), filter, options)
 
-          results.localFileSize += linkResult.localFileSize
-          results.localDagSize += linkResult.localDagSize
+          results.localSize += linkResult.localSize
+          results.dagSize += linkResult.dagSize
+          results.deduplicatedDagSize += linkResult.deduplicatedDagSize
           results.blocks += linkResult.blocks
+          results.uniqueBlocks += linkResult.uniqueBlocks
+          results.dirSize += linkResult.dirSize
+        }
+
+        // multi-block file node
+        if (isFile && unixfs != null) {
+          results.dirSize += unixfs.fileSize()
         }
       } else {
-        // leaf node
-        if (pbNode.Data == null) {
+        if (unixfs == null) {
           throw new InvalidPBNodeError(`PBNode ${cid.toString()} had no data`)
         }
 
-        const unixfs = UnixFS.unmarshal(pbNode.Data)
-
-        if (unixfs.data == null) {
-          throw new InvalidPBNodeError(`UnixFS node ${cid.toString()} had no data`)
+        // multi-block file leaf node
+        if (unixfs.data != null) {
+          results.localSize += BigInt(unixfs.data.byteLength ?? 0)
         }
 
-        results.localFileSize += unixfs.data.byteLength ?? 0
+        // single-block file node
+        if (isFile) {
+          results.dirSize += unixfs.fileSize()
+        }
       }
     } else {
       throw new UnknownError(`${cid.toString()} was neither DAG_PB nor RAW`)
     }
+  } catch (err: any) {
+    if (err.name !== 'NotFoundError' || options.offline !== true) {
+      throw err
+    }
   }
 
   return results
 }
+
+function linkIsFile (link: dagPb.PBLink, parent?: UnixFS): boolean {
+  if (parent == null) {
+    return false
+  }
+
+  const name = link.Name
+
+  if (name == null) {
+    return false
+  }
+
+  if (parent.type === 'directory') {
+    return true
+  } else if (parent.type === 'hamt-sharded-directory' && name.length > 2) {
+    return true
+  }
+
+  return false
+}

package/src/index.ts

@@ -49,6 +49,7 @@
 import { UnixFS as UnixFSClass } from './unixfs.js'
 import type { GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks'
 import type { AbortOptions } from '@libp2p/interface'
+import type { Filter } from '@libp2p/utils/filters'
 import type { Blockstore } from 'interface-blockstore'
 import type { Mtime, UnixFS as IPFSUnixFS } from 'ipfs-unixfs'
 import type { ExporterProgressEvents, UnixFSEntry } from 'ipfs-unixfs-exporter'
@@ -248,15 +249,32 @@ export interface StatOptions extends AbortOptions, ProgressOptions<GetEvents> {
 
   /**
    * If true, do not perform any network operations and throw if blocks are
-   * missing from the local store. (default: false)
+   * missing from the local store.
+   *
+   * @default false
    */
   offline?: boolean
 }
 
+export interface ExtendedStatOptions extends StatOptions {
+  /**
+   * If true, traverse the whole DAG to return additional stats. If all data is
+   * not in the local blockstore, this may involve fetching them from the
+   * network.
+   */
+  extended: true
+
+  /**
+   * By default CIDs are deduplicated using a `ScalableCuckooFilter` - if you
+   * wish to use a different filter, pass it here.
+   */
+  filter?: Filter
+}
+
 /**
  * Statistics relating to a UnixFS DAG
  */
-export interface UnixFSStats {
+export interface Stats {
   /**
    * The file or directory CID
    */
@@ -265,7 +283,7 @@ export interface UnixFSStats {
   /**
    * The file or directory mode
    */
-  mode?: number
+  mode: number
 
   /**
    * The file or directory mtime
@@ -273,41 +291,112 @@ export interface UnixFSStats {
   mtime?: Mtime
 
   /**
-   * The size of the file in bytes
+   * The type of UnixFS node - 'file' or 'directory'
    */
-  fileSize: bigint
+  type: 'file' | 'directory' | 'raw'
 
   /**
-   * The size of the DAG that holds the file in bytes
+   * UnixFS metadata about this file or directory
    */
-  dagSize: bigint
+  unixfs?: IPFSUnixFS
+
+  /**
+   * The size in bytes of the file as reported by the UnixFS metadata stored in
+   * the root DAG node, or if the CID resolves to a raw node, the size of the
+   * block that holds it.
+   *
+   * For directories this will return `0` as no size information is available in
+   * the root block - instead please stat with the `extended` option to traverse
+   * the DAG and calculate the size.
+   */
+  size: bigint
+}
+
+export interface FileStats extends Stats {
+  type: 'file'
+  unixfs: IPFSUnixFS
+}
+
+export interface DirectoryStats extends Stats {
+  type: 'directory'
+  unixfs: IPFSUnixFS
+}
+
+export interface RawStats extends Stats {
+  type: 'raw'
+  unixfs: undefined
+}
 
+/**
+ * More detailed statistics relating to a UnixFS DAG. These can involve
+ * traversing the DAG behind the CID so can involve network operations and/or
+ * more disk activity.
+ */
+export interface ExtendedStats extends Stats {
   /**
-   * How much of the file is in the local block store
+   * How many blocks make up the DAG.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  localFileSize: bigint
+  blocks: bigint
 
   /**
-   * How much of the DAG that holds the file is in the local blockstore
+   * How many unique blocks make up the DAG - this count does not include any
+   * blocks that appear in the DAG more than once.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  localDagSize: bigint
+  uniqueBlocks: bigint
 
   /**
-   * How many blocks make up the DAG - nb. this will only be accurate
-   * if all blocks are present in the local blockstore
+   * The size of the DAG that holds the file or directory in bytes - this is
+   * the sum of all block sizes so includes any protobuf overhead, etc.
+   *
+   * Duplicate blocks are included in this measurement.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  blocks: number
+  dagSize: bigint
 
   /**
-   * The type of file
+   * Similar to `dagSize` except duplicate blocks are not included in the
+   * reported amount.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  type: 'file' | 'directory' | 'raw'
+  deduplicatedDagSize: bigint
 
   /**
-   * UnixFS metadata about this file or directory. Will not be present
-   * if the node is a `raw` type.
+   * How much of the file or directory is in the local block store. If this is a
+   * directory it will include the `localSize` of all child files and
+   * directories.
+   *
+   * It does not include protobuf overhead, for that see `dagSize`.
+   *
+   * nb. if the `offline` option is `true`, and not all blocks for the
+   * file/directory are in the blockstore, this number may be smaller than
+   * `size`.
    */
-  unixfs?: IPFSUnixFS
+  localSize: bigint
+}
+
+export interface ExtendedFileStats extends ExtendedStats {
+  type: 'file'
+  unixfs: IPFSUnixFS
+}
+
+export interface ExtendedDirectoryStats extends ExtendedStats {
+  type: 'directory'
+  unixfs: IPFSUnixFS
+}
+
+export interface ExtendedRawStats extends ExtendedStats {
+  type: 'raw'
+  unixfs: undefined
 }
 
 /**
@@ -571,7 +660,8 @@ export interface UnixFS {
    * console.info(stats)
    * ```
    */
-  stat(cid: CID, options?: Partial<StatOptions>): Promise<UnixFSStats>
+  stat(cid: CID, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+  stat(cid: CID, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
 
   /**
    * Update the mtime of a UnixFS DAG

package/src/unixfs.ts

@@ -7,7 +7,7 @@ import { mkdir } from './commands/mkdir.js'
 import { rm } from './commands/rm.js'
 import { stat } from './commands/stat.js'
 import { touch } from './commands/touch.js'
-import type { AddOptions, CatOptions, ChmodOptions, CpOptions, FileCandidate, LsOptions, MkdirOptions, RmOptions, StatOptions, TouchOptions, UnixFSComponents, UnixFS as UnixFSInterface, UnixFSStats } from './index.js'
+import type { AddOptions, CatOptions, ChmodOptions, CpOptions, ExtendedStatOptions, ExtendedDirectoryStats, ExtendedFileStats, FileCandidate, LsOptions, MkdirOptions, RmOptions, StatOptions, TouchOptions, UnixFSComponents, DirectoryStats, FileStats, UnixFS as UnixFSInterface, RawStats, ExtendedRawStats } from './index.js'
 import type { Blockstore } from 'interface-blockstore'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
 import type { ByteStream, DirectoryCandidate, ImportCandidateStream, ImportResult } from 'ipfs-unixfs-importer'
@@ -68,7 +68,9 @@ export class UnixFS implements UnixFSInterface {
     return rm(cid, path, this.components.blockstore, options)
   }
 
-  async stat (cid: CID, options: Partial<StatOptions> = {}): Promise<UnixFSStats> {
+  async stat (cid: CID, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+  async stat (cid: CID, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
+  async stat (cid: CID, options: Partial<StatOptions> = {}): Promise<FileStats | DirectoryStats | RawStats> {
     return stat(cid, this.components.blockstore, options)
   }