@helia/unixfs 4.0.2 → 4.0.3-325b36f

package/src/index.ts

@@ -49,10 +49,11 @@
 import { UnixFS as UnixFSClass } from './unixfs.js'
 import type { GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks'
 import type { AbortOptions } from '@libp2p/interface'
+import type { Filter } from '@libp2p/utils/filters'
 import type { Blockstore } from 'interface-blockstore'
 import type { Mtime, UnixFS as IPFSUnixFS } from 'ipfs-unixfs'
 import type { ExporterProgressEvents, UnixFSEntry } from 'ipfs-unixfs-exporter'
-import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImporterProgressEvents, ImportResult } from 'ipfs-unixfs-importer'
+import type { ByteStream, DirectoryCandidate, ImportCandidateStream, ImporterOptions, ImporterProgressEvents, ImportResult, ImportContent } from 'ipfs-unixfs-importer'
 import type { CID, Version } from 'multiformats/cid'
 import type { ProgressOptions } from 'progress-events'
 
@@ -60,6 +61,13 @@ export interface UnixFSComponents {
   blockstore: Pick<Blockstore, 'get' | 'put' | 'has'>
 }
 
+export interface FileCandidate<T extends ImportContent = ImportContent> {
+  path: string
+  content: T
+  mtime?: Mtime
+  mode?: number
+}
+
 export type AddEvents = PutBlockProgressEvents
 | ImporterProgressEvents
 
@@ -67,6 +75,8 @@ export interface AddOptions extends AbortOptions, Omit<ImporterOptions, 'onProgr
 
 }
 
+export type AddFileOptions = Omit<AddOptions, 'wrapWithDirectory'>
+
 export type GetEvents = GetBlockProgressEvents
 | ExporterProgressEvents
 
@@ -233,21 +243,38 @@ export interface RmOptions extends AbortOptions, ProgressOptions<GetEvents | Put
  */
 export interface StatOptions extends AbortOptions, ProgressOptions<GetEvents> {
   /**
-   * An optional path to allow statting paths inside directories
+   * An optional path to allow getting stats of paths inside directories
    */
   path?: string
 
   /**
    * If true, do not perform any network operations and throw if blocks are
-   * missing from the local store. (default: false)
+   * missing from the local store.
+   *
+   * @default false
    */
   offline?: boolean
 }
 
+export interface ExtendedStatOptions extends StatOptions {
+  /**
+   * If true, traverse the whole DAG to return additional stats. If all data is
+   * not in the local blockstore, this may involve fetching them from the
+   * network.
+   */
+  extended: true
+
+  /**
+   * By default CIDs are deduplicated using a `ScalableCuckooFilter` - if you
+   * wish to use a different filter, pass it here.
+   */
+  filter?: Filter
+}
+
 /**
  * Statistics relating to a UnixFS DAG
  */
-export interface UnixFSStats {
+export interface Stats {
   /**
    * The file or directory CID
    */
@@ -256,7 +283,7 @@ export interface UnixFSStats {
   /**
    * The file or directory mode
    */
-  mode?: number
+  mode: number
 
   /**
    * The file or directory mtime
@@ -264,41 +291,112 @@ export interface UnixFSStats {
   mtime?: Mtime
 
   /**
-   * The size of the file in bytes
+   * The type of UnixFS node - 'file' or 'directory'
    */
-  fileSize: bigint
+  type: 'file' | 'directory' | 'raw'
 
   /**
-   * The size of the DAG that holds the file in bytes
+   * UnixFS metadata about this file or directory
    */
-  dagSize: bigint
+  unixfs?: IPFSUnixFS
 
   /**
-   * How much of the file is in the local block store
+   * The size in bytes of the file as reported by the UnixFS metadata stored in
+   * the root DAG node, or if the CID resolves to a raw node, the size of the
+   * block that holds it.
+   *
+   * For directories this will return `0` as no size information is available in
+   * the root block - instead please stat with the `extended` option to traverse
+   * the DAG and calculate the size.
    */
-  localFileSize: bigint
+  size: bigint
+}
+
+export interface FileStats extends Stats {
+  type: 'file'
+  unixfs: IPFSUnixFS
+}
+
+export interface DirectoryStats extends Stats {
+  type: 'directory'
+  unixfs: IPFSUnixFS
+}
 
+export interface RawStats extends Stats {
+  type: 'raw'
+  unixfs: undefined
+}
+
+/**
+ * More detailed statistics relating to a UnixFS DAG. These can involve
+ * traversing the DAG behind the CID so can involve network operations and/or
+ * more disk activity.
+ */
+export interface ExtendedStats extends Stats {
   /**
-   * How much of the DAG that holds the file is in the local blockstore
+   * How many blocks make up the DAG.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  localDagSize: bigint
+  blocks: bigint
 
   /**
-   * How many blocks make up the DAG - nb. this will only be accurate
-   * if all blocks are present in the local blockstore
+   * How many unique blocks make up the DAG - this count does not include any
+   * blocks that appear in the DAG more than once.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  blocks: number
+  uniqueBlocks: bigint
 
   /**
-   * The type of file
+   * The size of the DAG that holds the file or directory in bytes - this is
+   * the sum of all block sizes so includes any protobuf overhead, etc.
+   *
+   * Duplicate blocks are included in this measurement.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  type: 'file' | 'directory' | 'raw'
+  dagSize: bigint
 
   /**
-   * UnixFS metadata about this file or directory. Will not be present
-   * if the node is a `raw` type.
+   * Similar to `dagSize` except duplicate blocks are not included in the
+   * reported amount.
+   *
+   * nb. this will only be accurate if either all blocks are present in the
+   * local blockstore or the `offline` option was not `true`
    */
-  unixfs?: IPFSUnixFS
+  deduplicatedDagSize: bigint
+
+  /**
+   * How much of the file or directory is in the local block store. If this is a
+   * directory it will include the `localSize` of all child files and
+   * directories.
+   *
+   * It does not include protobuf overhead, for that see `dagSize`.
+   *
+   * nb. if the `offline` option is `true`, and not all blocks for the
+   * file/directory are in the blockstore, this number may be smaller than
+   * `size`.
+   */
+  localSize: bigint
+}
+
+export interface ExtendedFileStats extends ExtendedStats {
+  type: 'file'
+  unixfs: IPFSUnixFS
+}
+
+export interface ExtendedDirectoryStats extends ExtendedStats {
+  type: 'directory'
+  unixfs: IPFSUnixFS
+}
+
+export interface ExtendedRawStats extends ExtendedStats {
+  type: 'raw'
+  unixfs: undefined
 }
 
 /**
@@ -362,7 +460,11 @@ export interface UnixFS {
   addAll(source: ImportCandidateStream, options?: Partial<AddOptions>): AsyncIterable<ImportResult>
 
   /**
-   * Add a single `Uint8Array` to your Helia node as a file.
+   * Add a single `Uint8Array` to your Helia node and receive a CID that will
+   * resolve to it.
+   *
+   * If you want to preserve a file name or other metadata such as modification
+   * time or mode, use `addFile` instead.
    *
    * @example
    *
@@ -372,10 +474,14 @@ export interface UnixFS {
    * console.info(cid)
    * ```
    */
-  addBytes(bytes: Uint8Array, options?: Partial<AddOptions>): Promise<CID>
+  addBytes(bytes: Uint8Array, options?: Partial<AddFileOptions>): Promise<CID>
 
   /**
-   * Add a stream of `Uint8Array` to your Helia node as a file.
+   * Add a stream of `Uint8Array`s to your Helia node and receive a CID that
+   * will resolve to them.
+   *
+   * If you want to preserve a file name or other metadata such as modification
+   * time or mode, use `addFile` instead.
    *
    * @example
    *
@@ -388,10 +494,14 @@ export interface UnixFS {
    * console.info(cid)
    * ```
    */
-  addByteStream(bytes: ByteStream, options?: Partial<AddOptions>): Promise<CID>
+  addByteStream(bytes: ByteStream, options?: Partial<AddFileOptions>): Promise<CID>
 
   /**
-   * Add a file to your Helia node with optional metadata.
+   * Add a file to your Helia node with metadata. The returned CID will resolve
+   * to a directory with one file entry.
+   *
+   * If you don't care about file names and just want a CID that will resolve to
+   * the contents of the file, use `addBytes` or `addByeStream` instead.
    *
    * @example
    *
@@ -409,20 +519,42 @@ export interface UnixFS {
    * console.info(cid)
    * ```
    */
-  addFile(file: FileCandidate, options?: Partial<AddOptions>): Promise<CID>
+  addFile(file: FileCandidate, options?: Partial<AddFileOptions>): Promise<CID>
 
   /**
    * Add a directory to your Helia node.
    *
    * @example
    *
+   * If no path is specified, the returned CID will resolve to an empty
+   * directory.
+   *
    * ```typescript
    * const cid = await fs.addDirectory()
    *
-   * console.info(cid)
+   * console.info(cid) // empty directory CID
+   * ```
+   *
+   * @example
+   *
+   * If a path is specified, the CID will resolve to a directory that contains
+   * an empty directory with the specified name.
+   *
+   * ```typescript
+   * const cid = await fs.addDirectory({
+   *   path: 'my-dir'
+   * })
+   *
+   * console.info(cid) // containing directory CID
+   *
+   * const stat = await fs.stat(cid, {
+   *   path: 'my-dir'
+   * })
+   *
+   * console.info(stat.cid) // empty directory CID
    * ```
    */
-  addDirectory(dir?: Partial<DirectoryCandidate>, options?: Partial<AddOptions>): Promise<CID>
+  addDirectory(dir?: Partial<DirectoryCandidate>, options?: Partial<AddFileOptions>): Promise<CID>
 
   /**
    * Retrieve the contents of a file from your Helia node.
@@ -528,7 +660,8 @@ export interface UnixFS {
    * console.info(stats)
    * ```
    */
-  stat(cid: CID, options?: Partial<StatOptions>): Promise<UnixFSStats>
+  stat(cid: CID, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+  stat(cid: CID, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
 
   /**
    * Update the mtime of a UnixFS DAG

package/src/unixfs.ts

@@ -7,10 +7,10 @@ import { mkdir } from './commands/mkdir.js'
 import { rm } from './commands/rm.js'
 import { stat } from './commands/stat.js'
 import { touch } from './commands/touch.js'
-import type { AddOptions, CatOptions, ChmodOptions, CpOptions, LsOptions, MkdirOptions, RmOptions, StatOptions, TouchOptions, UnixFSComponents, UnixFS as UnixFSInterface, UnixFSStats } from './index.js'
+import type { AddOptions, CatOptions, ChmodOptions, CpOptions, ExtendedStatOptions, ExtendedDirectoryStats, ExtendedFileStats, FileCandidate, LsOptions, MkdirOptions, RmOptions, StatOptions, TouchOptions, UnixFSComponents, DirectoryStats, FileStats, UnixFS as UnixFSInterface, RawStats, ExtendedRawStats } from './index.js'
 import type { Blockstore } from 'interface-blockstore'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
-import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImportResult } from 'ipfs-unixfs-importer'
+import type { ByteStream, DirectoryCandidate, ImportCandidateStream, ImportResult } from 'ipfs-unixfs-importer'
 import type { CID } from 'multiformats/cid'
 
 export type PutStore = Pick<Blockstore, 'put'>
@@ -68,7 +68,9 @@ export class UnixFS implements UnixFSInterface {
     return rm(cid, path, this.components.blockstore, options)
   }
 
-  async stat (cid: CID, options: Partial<StatOptions> = {}): Promise<UnixFSStats> {
+  async stat (cid: CID, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+  async stat (cid: CID, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
+  async stat (cid: CID, options: Partial<StatOptions> = {}): Promise<FileStats | DirectoryStats | RawStats> {
     return stat(cid, this.components.blockstore, options)
   }
 

package/src/utils/glob-source.ts

@@ -50,6 +50,25 @@ export interface GlobSourceResult {
 
 /**
  * Create an async iterator that yields paths that match requested glob pattern
+ *
+ * @example
+ *
+ * ```ts
+ * import { unixfs, globSource } from '@helia/unixfs'
+ * import { createHelia } from 'helia'
+ *
+ * const helia = await createHelia()
+ * const fs = unixfs(helia)
+ *
+ * for await (const entry of fs.addAll(globSource(
+ *  '/path/to/dir',
+ *  '**\/*'
+ * ), {
+ *   wrapWithDirectory: true
+ * })) {
+ *   console.info(entry)
+ * }
+ * ```
  */
 export async function * globSource (cwd: string, pattern: string, options: GlobSourceOptions = {}): AsyncGenerator<ImportCandidate & GlobSourceResult> {
   if (typeof pattern !== 'string') {

package/src/utils/url-source.ts

@@ -1,13 +1,66 @@
 import { UnknownError } from '../errors.js'
-import type { FileCandidate } from 'ipfs-unixfs-importer'
+import type { FileCandidate } from '../index.js'
+
+/**
+ * Import a file directly from a URL. The path of the file will be the path
+ * section of the URL.
+ *
+ * @example
+ *
+ * ```ts
+ * import { unixfs, urlSource } from '@helia/unixfs'
+ * import { createHelia } from 'helia'
+ *
+ * const helia = await createHelia()
+ * const fs = unixfs(helia)
+ *
+ * const cid = await fs.addFile(urlSource('http://example.com/path/to/file.html))
+ * const stat = await fs.stat(cid)
+ *
+ * console.info(stat)
+ * // { cid: CID(...), type: 'directory', ... }
+ *
+ * for await (const entry of fs.ls(cid)) {
+ *   console.info(entry)
+ *   // { type: 'file', name: 'file.html', cid: CID(...), ... }
+ * }
+ * ```
+ */
+export function urlSource (url: URL | string, options?: RequestInit): FileCandidate<AsyncGenerator<Uint8Array, void, unknown>> {
+  url = new URL(url)
 
-export function urlSource (url: URL, options?: RequestInit): FileCandidate<AsyncGenerator<Uint8Array, void, unknown>> {
   return {
     path: decodeURIComponent(new URL(url).pathname.split('/').pop() ?? ''),
     content: readURLContent(url, options)
   }
 }
 
+/**
+ * Import a file directly from a URL ignoring the file name or any containing
+ * directory.
+ *
+ * @example
+ *
+ * ```ts
+ * import { unixfs, urlByteSource } from '@helia/unixfs'
+ * import { createHelia } from 'helia'
+ *
+ * const helia = await createHelia()
+ * const fs = unixfs(helia)
+ *
+ * const cid = await fs.addByteSource(urlByteSource('http://example.com/path/to/file.html))
+ * const stat = await fs.stat(cid)
+ *
+ * console.info(stat)
+ * // { type: 'file', cid: CID(...), ... }
+ * ```
+ */
+export function urlByteSource (url: URL | string, options?: RequestInit): AsyncGenerator<Uint8Array, void, unknown> {
+  url = new URL(url)
+
+  return readURLContent(url, options)
+}
+
 async function * readURLContent (url: URL, options?: RequestInit): AsyncGenerator<Uint8Array, void, unknown> {
   const response = await globalThis.fetch(url, options)
 

package/dist/typedoc-urls.json

@@ -1,56 +0,0 @@
-{
-  "AlreadyExistsError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.AlreadyExistsError.html",
-  "./errors:AlreadyExistsError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.AlreadyExistsError.html",
-  "DoesNotExistError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.DoesNotExistError.html",
-  "./errors:DoesNotExistError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.DoesNotExistError.html",
-  "InvalidPBNodeError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.InvalidPBNodeError.html",
-  "./errors:InvalidPBNodeError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.InvalidPBNodeError.html",
-  "InvalidParametersError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.InvalidParametersError.html",
-  "./errors:InvalidParametersError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.InvalidParametersError.html",
-  "NoContentError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NoContentError.html",
-  "./errors:NoContentError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NoContentError.html",
-  "NotADirectoryError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NotADirectoryError.html",
-  "./errors:NotADirectoryError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NotADirectoryError.html",
-  "NotAFileError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NotAFileError.html",
-  "./errors:NotAFileError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NotAFileError.html",
-  "NotUnixFSError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NotUnixFSError.html",
-  "./errors:NotUnixFSError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.NotUnixFSError.html",
-  "UnixFSError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.UnixFSError.html",
-  "./errors:UnixFSError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.UnixFSError.html",
-  "UnknownError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.UnknownError.html",
-  "./errors:UnknownError": "https://ipfs.github.io/helia/classes/_helia_unixfs.errors.UnknownError.html",
-  "AddOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.AddOptions.html",
-  ".:AddOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.AddOptions.html",
-  "CatOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.CatOptions.html",
-  ".:CatOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.CatOptions.html",
-  "ChmodOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.ChmodOptions.html",
-  ".:ChmodOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.ChmodOptions.html",
-  "CpOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.CpOptions.html",
-  ".:CpOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.CpOptions.html",
-  "GlobSourceOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.GlobSourceOptions.html",
-  "GlobSourceResult": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.GlobSourceResult.html",
-  "LsOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.LsOptions.html",
-  ".:LsOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.LsOptions.html",
-  "MkdirOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.MkdirOptions.html",
-  ".:MkdirOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.MkdirOptions.html",
-  "RmOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.RmOptions.html",
-  ".:RmOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.RmOptions.html",
-  "StatOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.StatOptions.html",
-  ".:StatOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.StatOptions.html",
-  "TouchOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.TouchOptions.html",
-  ".:TouchOptions": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.TouchOptions.html",
-  "UnixFS": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.UnixFS.html",
-  ".:UnixFS": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.UnixFS.html",
-  "UnixFSComponents": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.UnixFSComponents.html",
-  ".:UnixFSComponents": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.UnixFSComponents.html",
-  "UnixFSStats": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.UnixFSStats.html",
-  ".:UnixFSStats": "https://ipfs.github.io/helia/interfaces/_helia_unixfs.index.UnixFSStats.html",
-  "AddEvents": "https://ipfs.github.io/helia/types/_helia_unixfs.index.AddEvents.html",
-  ".:AddEvents": "https://ipfs.github.io/helia/types/_helia_unixfs.index.AddEvents.html",
-  "GetEvents": "https://ipfs.github.io/helia/types/_helia_unixfs.index.GetEvents.html",
-  ".:GetEvents": "https://ipfs.github.io/helia/types/_helia_unixfs.index.GetEvents.html",
-  "globSource": "https://ipfs.github.io/helia/functions/_helia_unixfs.index.globSource.html",
-  "unixfs": "https://ipfs.github.io/helia/functions/_helia_unixfs.index.unixfs-1.html",
-  ".:unixfs": "https://ipfs.github.io/helia/functions/_helia_unixfs.index.unixfs-1.html",
-  "urlSource": "https://ipfs.github.io/helia/functions/_helia_unixfs.index.urlSource.html"
-}