@helia/unixfs 1.2.4 → 1.4.0

package/src/commands/touch.ts

@@ -108,7 +108,7 @@ export async function touch (cid: CID, blockstore: Blocks, options: Partial<Touc
     return updatePathCids(root.cid, resolved, blockstore, opts)
   }
 
-  const block = await blockstore.get(resolved.cid)
+  const block = await blockstore.get(resolved.cid, options)
   let metadata: UnixFS
   let links: PBLink[] = []
 

package/src/commands/utils/add-link.ts

@@ -52,12 +52,12 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
 
   const result = await addToDirectory(parent, child, blockstore, options)
 
-  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) {
+  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options)) {
     log('converting directory to sharded directory')
 
     const converted = await convertToShardedDirectory(result, blockstore)
     result.cid = converted.cid
-    result.node = dagPB.decode(await blockstore.get(converted.cid))
+    result.node = dagPB.decode(await blockstore.get(converted.cid, options))
   }
 
   return result

package/src/commands/utils/is-over-shard-threshold.ts

@@ -3,6 +3,7 @@ import { UnixFS } from 'ipfs-unixfs'
 import { CID_V0, CID_V1 } from './dir-sharded.js'
 import type { Blocks } from '@helia/interface/blocks'
 import type { PBNode } from '@ipld/dag-pb'
+import type { AbortOptions } from '@libp2p/interfaces'
 
 /**
  * Estimate node size only based on DAGLink name and CID byte lengths
@@ -10,7 +11,7 @@ import type { PBNode } from '@ipld/dag-pb'
  *
  * If the node is a hamt sharded directory the calculation is based on if it was a regular directory.
  */
-export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number): Promise<boolean> {
+export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number, options: AbortOptions): Promise<boolean> {
   if (node.Data == null) {
     throw new Error('DagPB node had no data')
   }
@@ -21,7 +22,7 @@ export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, th
   if (unixfs.type === 'directory') {
     size = estimateNodeSize(node)
   } else if (unixfs.type === 'hamt-sharded-directory') {
-    size = await estimateShardSize(node, 0, threshold, blockstore)
+    size = await estimateShardSize(node, 0, threshold, blockstore, options)
   } else {
     throw new Error('Can only estimate the size of directories or shards')
   }
@@ -42,7 +43,7 @@ function estimateNodeSize (node: PBNode): number {
   return size
 }
 
-async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks): Promise<number> {
+async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks, options: AbortOptions): Promise<number> {
   if (current > max) {
     return max
   }
@@ -67,10 +68,10 @@ async function estimateShardSize (node: PBNode, current: number, max: number, bl
     current += link.Hash.bytes.byteLength
 
     if (link.Hash.code === dagPb.code) {
-      const block = await blockstore.get(link.Hash)
+      const block = await blockstore.get(link.Hash, options)
       const node = dagPb.decode(block)
 
-      current += await estimateShardSize(node, current, max, blockstore)
+      current += await estimateShardSize(node, current, max, blockstore, options)
     }
   }
 

package/src/commands/utils/remove-link.ts

@@ -41,7 +41,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B
 
     const result = await removeFromShardedDirectory(parent, name, blockstore, options)
 
-    if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) {
+    if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options))) {
       log('converting shard to flat directory %c', parent.cid)
 
       return convertToFlatDirectory(result, blockstore, options)

package/src/index.ts

@@ -29,6 +29,18 @@
  *   console.info(entry)
  * }
  * ```
+ *
+ * @example
+ *
+ * Recursively adding a directory (Node.js-compatibly environments only):
+ *
+ * ```typescript
+ * import { globSource } from '@helia/unixfs'
+ *
+ * for await (const entry of fs.addAll(globSource('path/to/containing/dir', 'glob-pattern'))) {
+ *   console.info(entry)
+ * }
+ * ```
  */
 
 import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js'
@@ -80,6 +92,12 @@ export interface CatOptions extends AbortOptions, ProgressOptions<GetEvents> {
    * An optional path to allow reading files inside directories
    */
   path?: string
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -102,6 +120,12 @@ export interface ChmodOptions extends AbortOptions, ProgressOptions<GetEvents |
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -118,6 +142,12 @@ export interface CpOptions extends AbortOptions, ProgressOptions<GetEvents | Put
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -139,6 +169,12 @@ export interface LsOptions extends AbortOptions, ProgressOptions<GetEvents> {
    * Stop reading the directory contents after this many directory entries
    */
   length?: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -171,6 +207,12 @@ export interface MkdirOptions extends AbortOptions, ProgressOptions<GetEvents |
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -182,6 +224,12 @@ export interface RmOptions extends AbortOptions, ProgressOptions<GetEvents | Put
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -192,6 +240,12 @@ export interface StatOptions extends AbortOptions, ProgressOptions<GetEvents> {
    * An optional path to allow statting paths inside directories
    */
   path?: string
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -275,6 +329,12 @@ export interface TouchOptions extends AbortOptions, ProgressOptions<GetEvents |
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -559,3 +619,6 @@ class DefaultUnixFS implements UnixFS {
 export function unixfs (helia: { blockstore: Blocks }): UnixFS {
   return new DefaultUnixFS(helia)
 }
+
+export { globSource } from './utils/glob-source.js'
+export { urlSource } from './utils/url-source.js'

package/src/utils/glob-source.ts

@@ -0,0 +1,92 @@
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
+import Path from 'node:path'
+import glob from 'it-glob'
+import { InvalidParametersError } from '../errors.js'
+import type { MtimeLike } from 'ipfs-unixfs'
+import type { ImportCandidateStream } from 'ipfs-unixfs-importer'
+
+export interface GlobSourceOptions {
+  /**
+   * Include .dot files in matched paths
+   */
+  hidden?: boolean
+
+  /**
+   * follow symlinks
+   */
+  followSymlinks?: boolean
+
+  /**
+   * Preserve mode
+   */
+  preserveMode?: boolean
+
+  /**
+   * Preserve mtime
+   */
+  preserveMtime?: boolean
+
+  /**
+   * mode to use - if preserveMode is true this will be ignored
+   */
+  mode?: number
+
+  /**
+   * mtime to use - if preserveMtime is true this will be ignored
+   */
+  mtime?: MtimeLike
+}
+
+export interface GlobSourceResult {
+  path: string
+  content: AsyncIterable<Uint8Array> | undefined
+  mode: number | undefined
+  mtime: MtimeLike | undefined
+}
+
+/**
+ * Create an async iterator that yields paths that match requested glob pattern
+ */
+export async function * globSource (cwd: string, pattern: string, options: GlobSourceOptions = {}): ImportCandidateStream {
+  if (typeof pattern !== 'string') {
+    throw new InvalidParametersError('Pattern must be a string')
+  }
+
+  if (!Path.isAbsolute(cwd)) {
+    cwd = Path.resolve(process.cwd(), cwd)
+  }
+
+  const globOptions = Object.assign({}, {
+    nodir: false,
+    realpath: false,
+    absolute: true,
+    dot: Boolean(options.hidden),
+    follow: options.followSymlinks != null ? options.followSymlinks : true
+  })
+
+  for await (const p of glob(cwd, pattern, globOptions)) {
+    const stat = await fsp.stat(p)
+
+    let mode = options.mode
+
+    if (options.preserveMode === true) {
+      mode = stat.mode
+    }
+
+    let mtime = options.mtime
+
+    if (options.preserveMtime === true) {
+      mtime = stat.mtime
+    }
+
+    yield {
+      path: toPosix(p.replace(cwd, '')),
+      content: stat.isFile() ? fs.createReadStream(p) : undefined,
+      mode,
+      mtime
+    }
+  }
+}
+
+const toPosix = (path: string): string => path.replace(/\\/g, '/')

package/src/utils/url-source.ts

@@ -0,0 +1,35 @@
+import { UnknownError } from '../errors.js'
+import type { FileCandidate } from 'ipfs-unixfs-importer'
+
+export function urlSource (url: URL, options?: RequestInit): FileCandidate<AsyncGenerator<Uint8Array, void, unknown>> {
+  return {
+    path: decodeURIComponent(new URL(url).pathname.split('/').pop() ?? ''),
+    content: readURLContent(url, options)
+  }
+}
+
+async function * readURLContent (url: URL, options?: RequestInit): AsyncGenerator<Uint8Array, void, unknown> {
+  const response = await globalThis.fetch(url, options)
+
+  if (response.body == null) {
+    throw new UnknownError('HTTP response did not have a body')
+  }
+
+  const reader = response.body.getReader()
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+
+      if (done) {
+        return
+      }
+
+      if (value != null) {
+        yield value
+      }
+    }
+  } finally {
+    reader.releaseLock()
+  }
+}