@helia/unixfs 5.1.0 → 6.0.0

package/src/commands/mkdir.ts

@@ -1,30 +1,20 @@
 import * as dagPB from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
-import { mergeOptions as mergeOpts } from '@libp2p/utils/merge-options'
 import { UnixFS } from 'ipfs-unixfs'
 import { exporter } from 'ipfs-unixfs-exporter'
 import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
+import { DEFAULT_CID_VERSION } from '../constants.ts'
 import { InvalidParametersError, NotADirectoryError } from '../errors.js'
 import { addLink } from './utils/add-link.js'
 import { cidToDirectory } from './utils/cid-to-directory.js'
 import { cidToPBLink } from './utils/cid-to-pblink.js'
-import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
 import type { MkdirOptions } from '../index.js'
 import type { GetStore, PutStore } from '../unixfs.js'
 
-const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:mkdir')
 
-const defaultOptions: MkdirOptions = {
-  cidVersion: 1,
-  force: false,
-  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
-}
-
 export async function mkdir (parentCid: CID, dirname: string, blockstore: GetStore & PutStore, options: Partial<MkdirOptions> = {}): Promise<CID> {
-  const opts: MkdirOptions = mergeOptions(defaultOptions, options)
-
   if (dirname.includes('/')) {
     throw new InvalidParametersError('Path must not have slashes')
   }
@@ -39,8 +29,8 @@ export async function mkdir (parentCid: CID, dirname: string, blockstore: GetSto
 
   const metadata = new UnixFS({
     type: 'directory',
-    mode: opts.mode,
-    mtime: opts.mtime
+    mode: options.mode,
+    mtime: options.mtime
   })
 
   // Persist the new parent PBNode
@@ -50,7 +40,7 @@ export async function mkdir (parentCid: CID, dirname: string, blockstore: GetSto
   }
   const buf = dagPB.encode(node)
   const hash = await sha256.digest(buf)
-  const emptyDirCid = CID.create(opts.cidVersion, dagPB.code, hash)
+  const emptyDirCid = CID.create(options.cidVersion ?? DEFAULT_CID_VERSION, dagPB.code, hash)
 
   await blockstore.put(emptyDirCid, buf)
 
@@ -58,15 +48,15 @@ export async function mkdir (parentCid: CID, dirname: string, blockstore: GetSto
     directory,
     pblink
   ] = await Promise.all([
-    cidToDirectory(parentCid, blockstore, opts),
-    cidToPBLink(emptyDirCid, dirname, blockstore, opts)
+    cidToDirectory(parentCid, blockstore, options),
+    cidToPBLink(emptyDirCid, dirname, blockstore, options)
   ])
 
   log('adding empty dir called %s to %c', dirname, parentCid)
 
   const result = await addLink(directory, pblink, blockstore, {
-    ...opts,
-    allowOverwriting: opts.force
+    ...options,
+    allowOverwriting: options.force
   })
 
   return result.cid

package/src/commands/rm.ts

@@ -1,33 +1,24 @@
 import { logger } from '@libp2p/logger'
-import { mergeOptions as mergeOpts } from '@libp2p/utils/merge-options'
 import { InvalidParametersError } from '../errors.js'
 import { cidToDirectory } from './utils/cid-to-directory.js'
-import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
 import { removeLink } from './utils/remove-link.js'
 import type { RmOptions } from '../index.js'
 import type { GetStore, PutStore } from '../unixfs.js'
 import type { CID } from 'multiformats/cid'
 
-const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:rm')
 
-const defaultOptions: RmOptions = {
-  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
-}
-
 export async function rm (target: CID, name: string, blockstore: GetStore & PutStore, options: Partial<RmOptions> = {}): Promise<CID> {
-  const opts: RmOptions = mergeOptions(defaultOptions, options)
-
   if (name.includes('/')) {
     throw new InvalidParametersError('Name must not have slashes')
   }
 
-  const directory = await cidToDirectory(target, blockstore, opts)
+  const directory = await cidToDirectory(target, blockstore, options)
 
   log('Removing %s from %c', name, target)
 
   const result = await removeLink(directory, name, blockstore, {
-    ...opts,
+    ...options,
     cidVersion: target.version
   })
 

package/src/commands/stat.ts

@@ -1,15 +1,15 @@
 import * as dagPb from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
-import { ScalableCuckooFilter } from '@libp2p/utils/filters'
-import { mergeOptions as mergeOpts } from '@libp2p/utils/merge-options'
+import { ScalableCuckooFilter } from '@libp2p/utils'
 import { UnixFS } from 'ipfs-unixfs'
 import { exporter } from 'ipfs-unixfs-exporter'
+import toBuffer from 'it-to-buffer'
 import * as raw from 'multiformats/codecs/raw'
 import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js'
 import { resolve } from './utils/resolve.js'
 import type { ExtendedStatOptions, ExtendedDirectoryStats, ExtendedFileStats, StatOptions, DirectoryStats, FileStats, RawStats, ExtendedRawStats } from '../index.js'
 import type { GetStore, HasStore } from '../unixfs.js'
-import type { Filter } from '@libp2p/utils/filters'
+import type { Filter } from '@libp2p/utils'
 import type { RawNode, UnixFSDirectory, UnixFSFile } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 
@@ -17,22 +17,16 @@ import type { CID } from 'multiformats/cid'
 const DEFAULT_DIR_MODE = 0x755
 const DEFAULT_FILE_MODE = 0x644
 
-const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:stat')
 
-const defaultOptions: StatOptions = {
-
-}
-
 export async function stat (cid: CID, blockstore: GetStore & HasStore, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
 export async function stat (cid: CID, blockstore: GetStore & HasStore, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
 export async function stat (cid: CID, blockstore: GetStore & HasStore, options: Partial<ExtendedStatOptions> = {}): Promise<any> {
-  const opts: StatOptions = mergeOptions(defaultOptions, options)
-  const resolved = await resolve(cid, options.path, blockstore, opts)
+  const resolved = await resolve(cid, options.path, blockstore, options)
 
   log('stat %c', resolved.cid)
 
-  const result = await exporter(resolved.cid, blockstore, opts)
+  const result = await exporter(resolved.cid, blockstore, options)
 
   if (result.type === 'raw') {
     if (options.extended === true) {
@@ -130,7 +124,7 @@ async function inspectDag (cid: CID, blockstore: GetStore & HasStore, isFile: bo
     const alreadyTraversed = filter.has(cid.bytes)
     filter.add(cid.bytes)
 
-    const block = await blockstore.get(cid, options)
+    const block = await toBuffer(blockstore.get(cid, options))
     results.blocks++
     results.dagSize += BigInt(block.byteLength)
 

package/src/commands/touch.ts

@@ -1,41 +1,33 @@
 import * as dagPB from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
-import { mergeOptions as mergeOpts } from '@libp2p/utils/merge-options'
 import { UnixFS } from 'ipfs-unixfs'
 import { recursive } from 'ipfs-unixfs-exporter'
 import { importer } from 'ipfs-unixfs-importer'
 import last from 'it-last'
 import { pipe } from 'it-pipe'
+import toBuffer from 'it-to-buffer'
 import { CID } from 'multiformats/cid'
 import * as raw from 'multiformats/codecs/raw'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js'
-import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
 import { persist } from './utils/persist.js'
 import { resolve, updatePathCids } from './utils/resolve.js'
 import type { TouchOptions } from '../index.js'
 import type { GetStore, PutStore } from '../unixfs.js'
 import type { PBNode, PBLink } from '@ipld/dag-pb'
 
-const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:touch')
 
-const defaultOptions: TouchOptions = {
-  recursive: false,
-  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
-}
-
 export async function touch (cid: CID, blockstore: GetStore & PutStore, options: Partial<TouchOptions> = {}): Promise<CID> {
-  const opts: TouchOptions = mergeOptions(defaultOptions, options)
-  const resolved = await resolve(cid, opts.path, blockstore, opts)
-  const mtime = opts.mtime ?? {
+  const resolved = await resolve(cid, options.path, blockstore, options)
+  const mtime = options.mtime ?? {
     secs: BigInt(Math.round(Date.now() / 1000)),
     nsecs: 0
   }
 
   log('touch %c %o', resolved.cid, mtime)
 
-  if (opts.recursive) {
+  if (options.recursive) {
     // recursively export from root CID, change perms of each entry then reimport
     // but do not reimport files, only manipulate dag-pb nodes
     const root = await pipe(
@@ -69,7 +61,7 @@ export async function touch (cid: CID, blockstore: GetStore & PutStore, options:
       },
       // @ts-expect-error blockstore types are incompatible
       (source) => importer(source, blockstore, {
-        ...opts,
+        ...options,
         dagBuilder: async function * (source, block) {
           for await (const entry of source) {
             yield async function () {
@@ -78,7 +70,7 @@ export async function touch (cid: CID, blockstore: GetStore & PutStore, options:
 
               const buf = dagPB.encode(node)
               const updatedCid = await persist(buf, block, {
-                ...opts,
+                ...options,
                 cidVersion: cid.version
               })
 
@@ -105,10 +97,10 @@ export async function touch (cid: CID, blockstore: GetStore & PutStore, options:
       throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`)
     }
 
-    return updatePathCids(root.cid, resolved, blockstore, opts)
+    return updatePathCids(root.cid, resolved, blockstore, options)
   }
 
-  const block = await blockstore.get(resolved.cid, options)
+  const block = await toBuffer(blockstore.get(resolved.cid, options))
   let metadata: UnixFS
   let links: PBLink[] = []
 
@@ -136,5 +128,5 @@ export async function touch (cid: CID, blockstore: GetStore & PutStore, options:
 
   await blockstore.put(updatedCid, updatedBlock)
 
-  return updatePathCids(updatedCid, resolved, blockstore, opts)
+  return updatePathCids(updatedCid, resolved, blockstore, options)
 }

package/src/commands/utils/add-link.ts

@@ -1,11 +1,13 @@
 import * as dagPB from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
+import toBuffer from 'it-to-buffer'
 import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
 // @ts-expect-error no types
 import SparseArray from 'sparse-array'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES } from '../../constants.ts'
 import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from '../../errors.js'
 import { wrapHash } from './consumable-hash.js'
 import { hamtBucketBits, hamtHashFn } from './hamt-constants.js'
@@ -31,9 +33,9 @@ export interface AddLinkResult {
 }
 
 export interface AddLinkOptions extends AbortOptions {
-  allowOverwriting: boolean
-  shardSplitThresholdBytes: number
-  cidVersion: Version
+  allowOverwriting?: boolean
+  shardSplitThresholdBytes?: number
+  cidVersion?: Version
 }
 
 export async function addLink (parent: Directory, child: Required<PBLink>, blockstore: GetStore & PutStore, options: AddLinkOptions): Promise<AddLinkResult> {
@@ -53,12 +55,12 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
 
   const result = await addToDirectory(parent, child, blockstore, options)
 
-  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options)) {
+  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes ?? DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES, options)) {
     log('converting directory to sharded directory')
 
     const converted = await convertToShardedDirectory(result, blockstore)
     result.cid = converted.cid
-    result.node = dagPB.decode(await blockstore.get(converted.cid, options))
+    result.node = dagPB.decode(await toBuffer(blockstore.get(converted.cid, options)))
   }
 
   return result

package/src/commands/utils/hamt-utils.ts

@@ -2,6 +2,7 @@ import * as dagPB from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
 import last from 'it-last'
+import toBuffer from 'it-to-buffer'
 // @ts-expect-error no types
 import SparseArray from 'sparse-array'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
@@ -25,7 +26,7 @@ import type { CID, Version } from 'multiformats/cid'
 const log = logger('helia:unixfs:commands:utils:hamt-utils')
 
 export interface UpdateHamtDirectoryOptions extends AbortOptions {
-  cidVersion: Version
+  cidVersion?: Version
 }
 
 export const toPrefix = (position: number): string => {
@@ -39,7 +40,7 @@ export const toPrefix = (position: number): string => {
 export interface CreateShardOptions {
   mtime?: Mtime
   mode?: number
-  cidVersion: Version
+  cidVersion?: Version
 }
 
 export const createShard = async (blockstore: PutStore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise<ImportResult> => {
@@ -151,7 +152,7 @@ export const recreateShardedDirectory = async (cid: CID, fileName: string, block
 
   // descend the HAMT, loading each layer as we head towards the target child
   while (true) {
-    const block = await blockstore.get(cid, options)
+    const block = await toBuffer(blockstore.get(cid, options))
     const node = dagPB.decode(block)
     const children = new SparseArray()
     const index = await hash.take(hamtBucketBits)

package/src/commands/utils/is-over-shard-threshold.ts

@@ -1,5 +1,6 @@
 import * as dagPb from '@ipld/dag-pb'
 import { UnixFS } from 'ipfs-unixfs'
+import toBuffer from 'it-to-buffer'
 import { CID_V0, CID_V1 } from './dir-sharded.js'
 import type { GetStore } from '../../unixfs.js'
 import type { PBNode } from '@ipld/dag-pb'
@@ -68,7 +69,7 @@ async function estimateShardSize (node: PBNode, current: number, max: number, bl
     current += link.Hash.bytes.byteLength
 
     if (link.Hash.code === dagPb.code) {
-      const block = await blockstore.get(link.Hash, options)
+      const block = await toBuffer(blockstore.get(link.Hash, options))
       const node = dagPb.decode(block)
 
       current += await estimateShardSize(node, current, max, blockstore, options)

package/src/commands/utils/persist.ts

@@ -1,13 +1,14 @@
 import * as dagPb from '@ipld/dag-pb'
 import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
+import { DEFAULT_CID_VERSION } from '../../constants.ts'
 import type { PutStore } from '../../unixfs.js'
 import type { Version as CIDVersion } from 'multiformats/cid'
 import type { BlockCodec } from 'multiformats/codecs/interface'
 
 export interface PersistOptions {
   codec?: BlockCodec<any, any>
-  cidVersion: CIDVersion
+  cidVersion?: CIDVersion
   signal?: AbortSignal
 }
 
@@ -17,7 +18,7 @@ export const persist = async (buffer: Uint8Array, blockstore: PutStore, options:
   }
 
   const multihash = await sha256.digest(buffer)
-  const cid = CID.create(options.cidVersion, options.codec.code, multihash)
+  const cid = CID.create(options.cidVersion ?? DEFAULT_CID_VERSION, options.codec.code, multihash)
 
   await blockstore.put(cid, buffer, {
     ...options,

package/src/commands/utils/remove-link.ts

@@ -2,6 +2,7 @@ import * as dagPB from '@ipld/dag-pb'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
 import { exporter } from 'ipfs-unixfs-exporter'
+import { DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES } from '../../constants.ts'
 import { InvalidParametersError, InvalidPBNodeError } from '../../errors.js'
 import {
   recreateShardedDirectory,
@@ -20,8 +21,8 @@ import type { CID, Version } from 'multiformats/cid'
 const log = logger('helia:unixfs:utils:remove-link')
 
 export interface RmLinkOptions extends AbortOptions {
-  shardSplitThresholdBytes: number
-  cidVersion: Version
+  shardSplitThresholdBytes?: number
+  cidVersion?: Version
 }
 
 export interface RemoveLinkResult {
@@ -41,7 +42,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: P
 
     const result = await removeFromShardedDirectory(parent, name, blockstore, options)
 
-    if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options))) {
+    if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes ?? DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES, options))) {
       log('converting shard to flat directory %c', parent.cid)
 
       return convertToFlatDirectory(result, blockstore, options)

package/src/commands/utils/resolve.ts

@@ -55,7 +55,7 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: G
 }
 
 export interface UpdatePathCidsOptions extends AbortOptions {
-  shardSplitThresholdBytes: number
+  shardSplitThresholdBytes?: number
 }
 
 /**

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const DEFAULT_CID_VERSION = 1
+export const DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES = 262144

package/src/index.ts

@@ -78,7 +78,7 @@
 import { UnixFS as UnixFSClass } from './unixfs.js'
 import type { GetBlockProgressEvents, ProviderOptions, PutBlockProgressEvents } from '@helia/interface/blocks'
 import type { AbortOptions } from '@libp2p/interface'
-import type { Filter } from '@libp2p/utils/filters'
+import type { Filter } from '@libp2p/utils'
 import type { Blockstore } from 'interface-blockstore'
 import type { Mtime, UnixFS as IPFSUnixFS } from 'ipfs-unixfs'
 import type { ExporterProgressEvents, UnixFSEntry, UnixFSBasicEntry } from 'ipfs-unixfs-exporter'

package/dist/src/commands/utils/constants.d.ts

@@ -1,2 +0,0 @@
-export declare const SHARD_SPLIT_THRESHOLD_BYTES = 262144;
-//# sourceMappingURL=constants.d.ts.map

package/dist/src/commands/utils/constants.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../../src/commands/utils/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,2BAA2B,SAAS,CAAA"}

package/dist/src/commands/utils/constants.js

@@ -1,2 +0,0 @@
-export const SHARD_SPLIT_THRESHOLD_BYTES = 262144;
-//# sourceMappingURL=constants.js.map

package/dist/src/commands/utils/constants.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../src/commands/utils/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,2BAA2B,GAAG,MAAM,CAAA"}

package/src/commands/utils/constants.ts

@@ -1 +0,0 @@
-export const SHARD_SPLIT_THRESHOLD_BYTES = 262144