@helia/unixfs 0.0.0 → 1.0.1

package/src/commands/cp.ts

@@ -1,5 +1,4 @@
-import { InvalidParametersError } from '@helia/interface/errors'
-import type { Blockstore } from 'ipfs-unixfs-exporter'
+import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 import type { CpOptions } from '../index.js'
 import mergeOpts from 'merge-options'
@@ -7,12 +6,15 @@ import { logger } from '@libp2p/logger'
 import { addLink } from './utils/add-link.js'
 import { cidToPBLink } from './utils/cid-to-pblink.js'
 import { cidToDirectory } from './utils/cid-to-directory.js'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+import { InvalidParametersError } from './utils/errors.js'
 
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:cp')
 
-const defaultOptions = {
-  force: false
+const defaultOptions: CpOptions = {
+  force: false,
+  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
 }
 
 export async function cp (source: CID, target: CID, name: string, blockstore: Blockstore, options: Partial<CpOptions> = {}): Promise<CID> {
@@ -34,6 +36,7 @@ export async function cp (source: CID, target: CID, name: string, blockstore: Bl
 
   const result = await addLink(directory, pblink, blockstore, {
     allowOverwriting: opts.force,
+    cidVersion: target.version,
     ...opts
   })
 

package/src/commands/ls.ts

@@ -1,13 +1,14 @@
-import { NoContentError, NotADirectoryError } from '@helia/interface/errors'
-import { Blockstore, exporter, UnixFSEntry } from 'ipfs-unixfs-exporter'
+import { exporter, UnixFSEntry } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 import type { LsOptions } from '../index.js'
 import { resolve } from './utils/resolve.js'
 import mergeOpts from 'merge-options'
+import type { Blockstore } from 'interface-blockstore'
+import { NoContentError, NotADirectoryError } from './utils/errors.js'
 
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 
-const defaultOptions = {
+const defaultOptions: LsOptions = {
 
 }
 

package/src/commands/mkdir.ts

@@ -1,4 +1,3 @@
-import { InvalidParametersError, NotADirectoryError } from '@helia/interface/errors'
 import { CID } from 'multiformats/cid'
 import mergeOpts from 'merge-options'
 import { logger } from '@libp2p/logger'
@@ -11,13 +10,16 @@ import { sha256 } from 'multiformats/hashes/sha2'
 import { exporter } from 'ipfs-unixfs-exporter'
 import { cidToDirectory } from './utils/cid-to-directory.js'
 import { cidToPBLink } from './utils/cid-to-pblink.js'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+import { InvalidParametersError, NotADirectoryError } from './utils/errors.js'
 
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:mkdir')
 
-const defaultOptions = {
+const defaultOptions: MkdirOptions = {
   cidVersion: 1,
-  force: false
+  force: false,
+  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
 }
 
 export async function mkdir (parentCid: CID, dirname: string, blockstore: Blockstore, options: Partial<MkdirOptions> = {}): Promise<CID> {

package/src/commands/rm.ts

@@ -1,17 +1,18 @@
-import { InvalidParametersError } from '@helia/interface/errors'
-import type { Blockstore } from 'ipfs-unixfs-exporter'
+import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 import type { RmOptions } from '../index.js'
 import mergeOpts from 'merge-options'
 import { logger } from '@libp2p/logger'
 import { removeLink } from './utils/remove-link.js'
 import { cidToDirectory } from './utils/cid-to-directory.js'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
+import { InvalidParametersError } from './utils/errors.js'
 
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:rm')
 
-const defaultOptions = {
-
+const defaultOptions: RmOptions = {
+  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
 }
 
 export async function rm (target: CID, name: string, blockstore: Blockstore, options: Partial<RmOptions> = {}): Promise<CID> {
@@ -25,7 +26,10 @@ export async function rm (target: CID, name: string, blockstore: Blockstore, opt
 
   log('Removing %s from %c', name, target)
 
-  const result = await removeLink(directory, name, blockstore, opts)
+  const result = await removeLink(directory, name, blockstore, {
+    ...opts,
+    cidVersion: target.version
+  })
 
   return result.cid
 }

package/src/commands/stat.ts

@@ -1,4 +1,4 @@
-import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import { exporter } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 import type { StatOptions, UnixFSStats } from '../index.js'
 import mergeOpts from 'merge-options'
@@ -10,11 +10,12 @@ import type { AbortOptions } from '@libp2p/interfaces'
 import type { Mtime } from 'ipfs-unixfs'
 import { resolve } from './utils/resolve.js'
 import * as raw from 'multiformats/codecs/raw'
+import type { Blockstore } from 'interface-blockstore'
 
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:stat')
 
-const defaultOptions = {
+const defaultOptions: StatOptions = {
 
 }
 
@@ -30,43 +31,46 @@ export async function stat (cid: CID, blockstore: Blockstore, options: Partial<S
     throw new NotUnixFSError()
   }
 
-  let fileSize: number = 0
-  let dagSize: number = 0
-  let localFileSize: number = 0
-  let localDagSize: number = 0
+  let fileSize: bigint = 0n
+  let dagSize: bigint = 0n
+  let localFileSize: bigint = 0n
+  let localDagSize: bigint = 0n
   let blocks: number = 0
   let mode: number | undefined
   let mtime: Mtime | undefined
   const type = result.type
+  let unixfs: UnixFS | undefined
 
   if (result.type === 'raw') {
-    fileSize = result.node.byteLength
-    dagSize = result.node.byteLength
-    localFileSize = result.node.byteLength
-    localDagSize = result.node.byteLength
+    fileSize = BigInt(result.node.byteLength)
+    dagSize = BigInt(result.node.byteLength)
+    localFileSize = BigInt(result.node.byteLength)
+    localDagSize = BigInt(result.node.byteLength)
     blocks = 1
   }
 
   if (result.type === 'directory') {
-    fileSize = 0
-    dagSize = result.unixfs.marshal().byteLength
-    localFileSize = 0
+    fileSize = 0n
+    dagSize = BigInt(result.unixfs.marshal().byteLength)
+    localFileSize = 0n
     localDagSize = dagSize
     blocks = 1
     mode = result.unixfs.mode
     mtime = result.unixfs.mtime
+    unixfs = result.unixfs
   }
 
   if (result.type === 'file') {
     const results = await inspectDag(resolved.cid, blockstore, opts)
 
     fileSize = result.unixfs.fileSize()
-    dagSize = (result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0)
-    localFileSize = results.localFileSize
-    localDagSize = results.localDagSize
+    dagSize = BigInt((result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0))
+    localFileSize = BigInt(results.localFileSize)
+    localDagSize = BigInt(results.localDagSize)
     blocks = results.blocks
     mode = result.unixfs.mode
     mtime = result.unixfs.mtime
+    unixfs = result.unixfs
   }
 
   return {
@@ -78,7 +82,8 @@ export async function stat (cid: CID, blockstore: Blockstore, options: Partial<S
     localFileSize,
     localDagSize,
     blocks,
-    type
+    type,
+    unixfs
   }
 }
 

package/src/commands/touch.ts

@@ -15,19 +15,21 @@ import last from 'it-last'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { resolve, updatePathCids } from './utils/resolve.js'
 import * as raw from 'multiformats/codecs/raw'
+import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js'
 
 const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
 const log = logger('helia:unixfs:touch')
 
-const defaultOptions = {
-  recursive: false
+const defaultOptions: TouchOptions = {
+  recursive: false,
+  shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES
 }
 
 export async function touch (cid: CID, blockstore: Blockstore, options: Partial<TouchOptions> = {}): Promise<CID> {
   const opts: TouchOptions = mergeOptions(defaultOptions, options)
   const resolved = await resolve(cid, opts.path, blockstore, opts)
   const mtime = opts.mtime ?? {
-    secs: Date.now() / 1000,
+    secs: BigInt(Math.round(Date.now() / 1000)),
     nsecs: 0
   }
 
@@ -65,28 +67,29 @@ export async function touch (cid: CID, blockstore: Blockstore, options: Partial<
           }
         }
       },
-      // @ts-expect-error we account for the incompatible source type with our custom dag builder below
       (source) => importer(source, blockstore, {
         ...opts,
-        pin: false,
-        dagBuilder: async function * (source, block, opts) {
+        dagBuilder: async function * (source, block) {
           for await (const entry of source) {
             yield async function () {
               // @ts-expect-error cannot derive type
               const node: PBNode = entry.content
 
               const buf = dagPB.encode(node)
-              const cid = await persist(buf, block, opts)
+              const updatedCid = await persist(buf, block, {
+                ...opts,
+                cidVersion: cid.version
+              })
 
               if (node.Data == null) {
-                throw new InvalidPBNodeError(`${cid} had no data`)
+                throw new InvalidPBNodeError(`${updatedCid} had no data`)
               }
 
               const unixfs = UnixFS.unmarshal(node.Data)
 
               return {
-                cid,
-                size: buf.length,
+                cid: updatedCid,
+                size: BigInt(buf.length),
                 path: entry.path,
                 unixfs
               }
@@ -101,7 +104,7 @@ export async function touch (cid: CID, blockstore: Blockstore, options: Partial<
       throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`)
     }
 
-    return await updatePathCids(root.cid, resolved, blockstore, options)
+    return await updatePathCids(root.cid, resolved, blockstore, opts)
   }
 
   const block = await blockstore.get(resolved.cid)
@@ -132,5 +135,5 @@ export async function touch (cid: CID, blockstore: Blockstore, options: Partial<
 
   await blockstore.put(updatedCid, updatedBlock)
 
-  return await updatePathCids(updatedCid, resolved, blockstore, options)
+  return await updatePathCids(updatedCid, resolved, blockstore, opts)
 }

package/src/commands/utils/add-link.ts

@@ -1,5 +1,5 @@
 import * as dagPB from '@ipld/dag-pb'
-import { CID } from 'multiformats/cid'
+import { CID, Version } from 'multiformats/cid'
 import { logger } from '@libp2p/logger'
 import { UnixFS } from 'ipfs-unixfs'
 import { DirSharded } from './dir-sharded.js'
@@ -12,15 +12,15 @@ import {
   addLinksToHamtBucket
 } from './hamt-utils.js'
 import last from 'it-last'
-import type { Blockstore } from 'ipfs-unixfs-exporter'
 import type { PBNode, PBLink } from '@ipld/dag-pb/interface'
 import { sha256 } from 'multiformats/hashes/sha2'
 import type { Bucket } from 'hamt-sharding'
-import { AlreadyExistsError, InvalidPBNodeError } from './errors.js'
-import { InvalidParametersError } from '@helia/interface/errors'
+import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from './errors.js'
 import type { ImportResult } from 'ipfs-unixfs-importer'
 import type { AbortOptions } from '@libp2p/interfaces'
 import type { Directory } from './cid-to-directory.js'
+import type { Blockstore } from 'interface-blockstore'
+import { isOverShardThreshold } from './is-over-shard-threshold.js'
 
 const log = logger('helia:unixfs:components:utils:add-link')
 
@@ -32,6 +32,8 @@ export interface AddLinkResult {
 
 export interface AddLinkOptions extends AbortOptions {
   allowOverwriting: boolean
+  shardSplitThresholdBytes: number
+  cidVersion: Version
 }
 
 export async function addLink (parent: Directory, child: Required<PBLink>, blockstore: Blockstore, options: AddLinkOptions): Promise<AddLinkResult> {
@@ -39,19 +41,6 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
     throw new InvalidParametersError('Invalid parent passed to addLink')
   }
 
-  // FIXME: this should work on block size not number of links
-  if (parent.node.Links.length >= 1000) {
-    log('converting directory to sharded directory')
-
-    const result = await convertToShardedDirectory(parent, blockstore)
-    parent.cid = result.cid
-    parent.node = dagPB.decode(await blockstore.get(result.cid))
-  }
-
-  if (parent.node.Data == null) {
-    throw new InvalidParametersError('Invalid parent passed to addLink')
-  }
-
   const meta = UnixFS.unmarshal(parent.node.Data)
 
   if (meta.type === 'hamt-sharded-directory') {
@@ -62,7 +51,17 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
 
   log(`adding ${child.Name} (${child.Hash}) to regular directory`)
 
-  return await addToDirectory(parent, child, blockstore, options)
+  const result = await addToDirectory(parent, child, blockstore, options)
+
+  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) {
+    log('converting directory to sharded directory')
+
+    const converted = await convertToShardedDirectory(result, blockstore)
+    result.cid = converted.cid
+    result.node = dagPB.decode(await blockstore.get(converted.cid))
+  }
+
+  return result
 }
 
 const convertToShardedDirectory = async (parent: Directory, blockstore: Blockstore): Promise<ImportResult> => {
@@ -74,11 +73,12 @@ const convertToShardedDirectory = async (parent: Directory, blockstore: Blocksto
 
   const result = await createShard(blockstore, parent.node.Links.map(link => ({
     name: (link.Name ?? ''),
-    size: link.Tsize ?? 0,
+    size: BigInt(link.Tsize ?? 0),
     cid: link.Hash
   })), {
     mode: unixfs.mode,
-    mtime: unixfs.mtime
+    mtime: unixfs.mtime,
+    cidVersion: parent.cid.version
   })
 
   log(`Converted directory to sharded directory ${result.cid}`)
@@ -112,7 +112,7 @@ const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Bloc
     const secs = Math.floor(ms / 1000)
 
     node.mtime = {
-      secs,
+      secs: BigInt(secs),
       nsecs: (ms - (secs * 1000)) * 1000
     }
 
@@ -154,13 +154,7 @@ const addToShardedDirectory = async (parent: Directory, child: Required<PBLink>,
 
   // we have written out the shard, but only one sub-shard will have been written so replace it in the original shard
   const parentLinks = parent.node.Links.filter((link) => {
-    const matches = (link.Name ?? '').substring(0, 2) === path[0].prefix
-
-    if (matches && !options.allowOverwriting) {
-      throw new AlreadyExistsError()
-    }
-
-    return !matches
+    return (link.Name ?? '').substring(0, 2) !== path[0].prefix
   })
 
   const newLink = node.Links
@@ -172,7 +166,10 @@ const addToShardedDirectory = async (parent: Directory, child: Required<PBLink>,
 
   parentLinks.push(newLink)
 
-  return await updateHamtDirectory(parent, blockstore, parentLinks, path[0].bucket, options)
+  return await updateHamtDirectory({
+    Data: parent.node.Data,
+    Links: parentLinks
+  }, blockstore, path[0].bucket, options)
 }
 
 const addFileToShardedDirectory = async (parent: Directory, child: Required<PBLink>, blockstore: Blockstore, options: AddLinkOptions): Promise<{ shard: DirSharded, path: BucketPath[] }> => {
@@ -202,7 +199,7 @@ const addFileToShardedDirectory = async (parent: Directory, child: Required<PBLi
   if (node.mtime != null) {
     // update mtime if previously set
     shard.mtime = {
-      secs: Math.round(Date.now() / 1000)
+      secs: BigInt(Math.round(Date.now() / 1000))
     }
   }
 
@@ -233,6 +230,10 @@ const addFileToShardedDirectory = async (parent: Directory, child: Required<PBLi
     }
 
     if (link.Name === `${segment.prefix}${child.Name}`) {
+      if (!options.allowOverwriting) {
+        throw new AlreadyExistsError()
+      }
+
       // file already existed, file will be added to the current bucket
       log(`Link ${segment.prefix}${child.Name} will be replaced`)
       index = path.length
@@ -279,7 +280,7 @@ const addFileToShardedDirectory = async (parent: Directory, child: Required<PBLi
 
   // finally add the new file into the shard
   await shard._bucket.put(child.Name, {
-    size: child.Tsize,
+    size: BigInt(child.Tsize),
     cid: child.Hash
   })
 

package/src/commands/utils/cid-to-directory.ts

@@ -1,15 +1,15 @@
-import { NotADirectoryError } from '@helia/interface/errors'
-import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 import type { PBNode } from '@ipld/dag-pb'
-import type { AbortOptions } from '@libp2p/interfaces'
+import type { Blockstore } from 'interface-blockstore'
+import { NotADirectoryError } from './errors.js'
 
 export interface Directory {
   cid: CID
   node: PBNode
 }
 
-export async function cidToDirectory (cid: CID, blockstore: Blockstore, options: AbortOptions = {}): Promise<Directory> {
+export async function cidToDirectory (cid: CID, blockstore: Blockstore, options: ExporterOptions = {}): Promise<Directory> {
   const entry = await exporter(cid, blockstore, options)
 
   if (entry.type !== 'directory') {

package/src/commands/utils/cid-to-pblink.ts

@@ -1,11 +1,11 @@
-import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 import { NotUnixFSError } from './errors.js'
 import * as dagPb from '@ipld/dag-pb'
 import type { PBNode, PBLink } from '@ipld/dag-pb'
-import type { AbortOptions } from '@libp2p/interfaces'
+import type { Blockstore } from 'interface-blockstore'
 
-export async function cidToPBLink (cid: CID, name: string, blockstore: Blockstore, options?: AbortOptions): Promise<Required<PBLink>> {
+export async function cidToPBLink (cid: CID, name: string, blockstore: Blockstore, options?: ExporterOptions): Promise<Required<PBLink>> {
   const sourceEntry = await exporter(cid, blockstore, options)
 
   if (sourceEntry.type !== 'directory' && sourceEntry.type !== 'file' && sourceEntry.type !== 'raw') {

package/src/commands/utils/constants.ts

@@ -0,0 +1,2 @@
+
+export const SHARD_SPLIT_THRESHOLD_BYTES = 262144