@helia/unixfs 0.0.0

package/src/commands/stat.ts

@@ -0,0 +1,137 @@
+import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import type { StatOptions, UnixFSStats } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js'
+import * as dagPb from '@ipld/dag-pb'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Mtime } from 'ipfs-unixfs'
+import { resolve } from './utils/resolve.js'
+import * as raw from 'multiformats/codecs/raw'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:stat')
+
+const defaultOptions = {
+
+}
+
+export async function stat (cid: CID, blockstore: Blockstore, options: Partial<StatOptions> = {}): Promise<UnixFSStats> {
+  const opts: StatOptions = mergeOptions(defaultOptions, options)
+  const resolved = await resolve(cid, options.path, blockstore, opts)
+
+  log('stat %c', resolved.cid)
+
+  const result = await exporter(resolved.cid, blockstore, opts)
+
+  if (result.type !== 'file' && result.type !== 'directory' && result.type !== 'raw') {
+    throw new NotUnixFSError()
+  }
+
+  let fileSize: number = 0
+  let dagSize: number = 0
+  let localFileSize: number = 0
+  let localDagSize: number = 0
+  let blocks: number = 0
+  let mode: number | undefined
+  let mtime: Mtime | undefined
+  const type = result.type
+
+  if (result.type === 'raw') {
+    fileSize = result.node.byteLength
+    dagSize = result.node.byteLength
+    localFileSize = result.node.byteLength
+    localDagSize = result.node.byteLength
+    blocks = 1
+  }
+
+  if (result.type === 'directory') {
+    fileSize = 0
+    dagSize = result.unixfs.marshal().byteLength
+    localFileSize = 0
+    localDagSize = dagSize
+    blocks = 1
+    mode = result.unixfs.mode
+    mtime = result.unixfs.mtime
+  }
+
+  if (result.type === 'file') {
+    const results = await inspectDag(resolved.cid, blockstore, opts)
+
+    fileSize = result.unixfs.fileSize()
+    dagSize = (result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0)
+    localFileSize = results.localFileSize
+    localDagSize = results.localDagSize
+    blocks = results.blocks
+    mode = result.unixfs.mode
+    mtime = result.unixfs.mtime
+  }
+
+  return {
+    cid: resolved.cid,
+    mode,
+    mtime,
+    fileSize,
+    dagSize,
+    localFileSize,
+    localDagSize,
+    blocks,
+    type
+  }
+}
+
+interface InspectDagResults {
+  localFileSize: number
+  localDagSize: number
+  blocks: number
+}
+
+async function inspectDag (cid: CID, blockstore: Blockstore, options: AbortOptions): Promise<InspectDagResults> {
+  const results = {
+    localFileSize: 0,
+    localDagSize: 0,
+    blocks: 0
+  }
+
+  if (await blockstore.has(cid, options)) {
+    const block = await blockstore.get(cid, options)
+    results.blocks++
+    results.localDagSize += block.byteLength
+
+    if (cid.code === raw.code) {
+      results.localFileSize += block.byteLength
+    } else if (cid.code === dagPb.code) {
+      const pbNode = dagPb.decode(block)
+
+      if (pbNode.Links.length > 0) {
+        // intermediate node
+        for (const link of pbNode.Links) {
+          const linkResult = await inspectDag(link.Hash, blockstore, options)
+
+          results.localFileSize += linkResult.localFileSize
+          results.localDagSize += linkResult.localDagSize
+          results.blocks += linkResult.blocks
+        }
+      } else {
+        // leaf node
+        if (pbNode.Data == null) {
+          throw new InvalidPBNodeError(`PBNode ${cid.toString()} had no data`)
+        }
+
+        const unixfs = UnixFS.unmarshal(pbNode.Data)
+
+        if (unixfs.data == null) {
+          throw new InvalidPBNodeError(`UnixFS node ${cid.toString()} had no data`)
+        }
+
+        results.localFileSize += unixfs.data.byteLength ?? 0
+      }
+    } else {
+      throw new UnknownError(`${cid.toString()} was neither DAG_PB nor RAW`)
+    }
+  }
+
+  return results
+}

package/src/commands/touch.ts

@@ -0,0 +1,136 @@
+import { recursive } from 'ipfs-unixfs-exporter'
+import { CID } from 'multiformats/cid'
+import type { TouchOptions } from '../index.js'
+import mergeOpts from 'merge-options'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { pipe } from 'it-pipe'
+import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js'
+import * as dagPB from '@ipld/dag-pb'
+import type { PBNode, PBLink } from '@ipld/dag-pb'
+import { importer } from 'ipfs-unixfs-importer'
+import { persist } from './utils/persist.js'
+import type { Blockstore } from 'interface-blockstore'
+import last from 'it-last'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { resolve, updatePathCids } from './utils/resolve.js'
+import * as raw from 'multiformats/codecs/raw'
+
+const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
+const log = logger('helia:unixfs:touch')
+
+const defaultOptions = {
+  recursive: false
+}
+
+export async function touch (cid: CID, blockstore: Blockstore, options: Partial<TouchOptions> = {}): Promise<CID> {
+  const opts: TouchOptions = mergeOptions(defaultOptions, options)
+  const resolved = await resolve(cid, opts.path, blockstore, opts)
+  const mtime = opts.mtime ?? {
+    secs: Date.now() / 1000,
+    nsecs: 0
+  }
+
+  log('touch %c %o', resolved.cid, mtime)
+
+  if (opts.recursive) {
+    // recursively export from root CID, change perms of each entry then reimport
+    // but do not reimport files, only manipulate dag-pb nodes
+    const root = await pipe(
+      async function * () {
+        for await (const entry of recursive(resolved.cid, blockstore)) {
+          let metadata: UnixFS
+          let links: PBLink[]
+
+          if (entry.type === 'raw') {
+            metadata = new UnixFS({ data: entry.node })
+            links = []
+          } else if (entry.type === 'file' || entry.type === 'directory') {
+            metadata = entry.unixfs
+            links = entry.node.Links
+          } else {
+            throw new NotUnixFSError()
+          }
+
+          metadata.mtime = mtime
+
+          const node = {
+            Data: metadata.marshal(),
+            Links: links
+          }
+
+          yield {
+            path: entry.path,
+            content: node
+          }
+        }
+      },
+      // @ts-expect-error we account for the incompatible source type with our custom dag builder below
+      (source) => importer(source, blockstore, {
+        ...opts,
+        pin: false,
+        dagBuilder: async function * (source, block, opts) {
+          for await (const entry of source) {
+            yield async function () {
+              // @ts-expect-error cannot derive type
+              const node: PBNode = entry.content
+
+              const buf = dagPB.encode(node)
+              const cid = await persist(buf, block, opts)
+
+              if (node.Data == null) {
+                throw new InvalidPBNodeError(`${cid} had no data`)
+              }
+
+              const unixfs = UnixFS.unmarshal(node.Data)
+
+              return {
+                cid,
+                size: buf.length,
+                path: entry.path,
+                unixfs
+              }
+            }
+          }
+        }
+      }),
+      async (nodes) => await last(nodes)
+    )
+
+    if (root == null) {
+      throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`)
+    }
+
+    return await updatePathCids(root.cid, resolved, blockstore, options)
+  }
+
+  const block = await blockstore.get(resolved.cid)
+  let metadata: UnixFS
+  let links: PBLink[] = []
+
+  if (resolved.cid.code === raw.code) {
+    metadata = new UnixFS({ data: block })
+  } else {
+    const node = dagPB.decode(block)
+    links = node.Links
+
+    if (node.Data == null) {
+      throw new InvalidPBNodeError(`${resolved.cid.toString()} had no data`)
+    }
+
+    metadata = UnixFS.unmarshal(node.Data)
+  }
+
+  metadata.mtime = mtime
+  const updatedBlock = dagPB.encode({
+    Data: metadata.marshal(),
+    Links: links
+  })
+
+  const hash = await sha256.digest(updatedBlock)
+  const updatedCid = CID.create(resolved.cid.version, dagPB.code, hash)
+
+  await blockstore.put(updatedCid, updatedBlock)
+
+  return await updatePathCids(updatedCid, resolved, blockstore, options)
+}

package/src/commands/utils/add-link.ts

@@ -0,0 +1,319 @@
+import * as dagPB from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import { DirSharded } from './dir-sharded.js'
+import {
+  updateHamtDirectory,
+  recreateHamtLevel,
+  recreateInitialHamtLevel,
+  createShard,
+  toPrefix,
+  addLinksToHamtBucket
+} from './hamt-utils.js'
+import last from 'it-last'
+import type { Blockstore } from 'ipfs-unixfs-exporter'
+import type { PBNode, PBLink } from '@ipld/dag-pb/interface'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { Bucket } from 'hamt-sharding'
+import { AlreadyExistsError, InvalidPBNodeError } from './errors.js'
+import { InvalidParametersError } from '@helia/interface/errors'
+import type { ImportResult } from 'ipfs-unixfs-importer'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Directory } from './cid-to-directory.js'
+
+const log = logger('helia:unixfs:components:utils:add-link')
+
+export interface AddLinkResult {
+  node: PBNode
+  cid: CID
+  size: number
+}
+
+export interface AddLinkOptions extends AbortOptions {
+  allowOverwriting: boolean
+}
+
+export async function addLink (parent: Directory, child: Required<PBLink>, blockstore: Blockstore, options: AddLinkOptions): Promise<AddLinkResult> {
+  if (parent.node.Data == null) {
+    throw new InvalidParametersError('Invalid parent passed to addLink')
+  }
+
+  // FIXME: this should work on block size not number of links
+  if (parent.node.Links.length >= 1000) {
+    log('converting directory to sharded directory')
+
+    const result = await convertToShardedDirectory(parent, blockstore)
+    parent.cid = result.cid
+    parent.node = dagPB.decode(await blockstore.get(result.cid))
+  }
+
+  if (parent.node.Data == null) {
+    throw new InvalidParametersError('Invalid parent passed to addLink')
+  }
+
+  const meta = UnixFS.unmarshal(parent.node.Data)
+
+  if (meta.type === 'hamt-sharded-directory') {
+    log('adding link to sharded directory')
+
+    return await addToShardedDirectory(parent, child, blockstore, options)
+  }
+
+  log(`adding ${child.Name} (${child.Hash}) to regular directory`)
+
+  return await addToDirectory(parent, child, blockstore, options)
+}
+
+const convertToShardedDirectory = async (parent: Directory, blockstore: Blockstore): Promise<ImportResult> => {
+  if (parent.node.Data == null) {
+    throw new InvalidParametersError('Invalid parent passed to convertToShardedDirectory')
+  }
+
+  const unixfs = UnixFS.unmarshal(parent.node.Data)
+
+  const result = await createShard(blockstore, parent.node.Links.map(link => ({
+    name: (link.Name ?? ''),
+    size: link.Tsize ?? 0,
+    cid: link.Hash
+  })), {
+    mode: unixfs.mode,
+    mtime: unixfs.mtime
+  })
+
+  log(`Converted directory to sharded directory ${result.cid}`)
+
+  return result
+}
+
+const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Blockstore, options: AddLinkOptions): Promise<AddLinkResult> => {
+  // Remove existing link if it exists
+  const parentLinks = parent.node.Links.filter((link) => {
+    const matches = link.Name === child.Name
+
+    if (matches && !options.allowOverwriting) {
+      throw new AlreadyExistsError()
+    }
+
+    return !matches
+  })
+  parentLinks.push(child)
+
+  if (parent.node.Data == null) {
+    throw new InvalidPBNodeError('Parent node with no data passed to addToDirectory')
+  }
+
+  const node = UnixFS.unmarshal(parent.node.Data)
+
+  let data
+  if (node.mtime != null) {
+    // Update mtime if previously set
+    const ms = Date.now()
+    const secs = Math.floor(ms / 1000)
+
+    node.mtime = {
+      secs,
+      nsecs: (ms - (secs * 1000)) * 1000
+    }
+
+    data = node.marshal()
+  } else {
+    data = parent.node.Data
+  }
+  parent.node = dagPB.prepare({
+    Data: data,
+    Links: parentLinks
+  })
+
+  // Persist the new parent PbNode
+  const buf = dagPB.encode(parent.node)
+  const hash = await sha256.digest(buf)
+  const cid = CID.create(parent.cid.version, dagPB.code, hash)
+
+  await blockstore.put(cid, buf)
+
+  return {
+    node: parent.node,
+    cid,
+    size: buf.length
+  }
+}
+
+const addToShardedDirectory = async (parent: Directory, child: Required<PBLink>, blockstore: Blockstore, options: AddLinkOptions): Promise<AddLinkResult> => {
+  const {
+    shard, path
+  } = await addFileToShardedDirectory(parent, child, blockstore, options)
+  const result = await last(shard.flush(blockstore))
+
+  if (result == null) {
+    throw new Error('No result from flushing shard')
+  }
+
+  const block = await blockstore.get(result.cid)
+  const node = dagPB.decode(block)
+
+  // we have written out the shard, but only one sub-shard will have been written so replace it in the original shard
+  const parentLinks = parent.node.Links.filter((link) => {
+    const matches = (link.Name ?? '').substring(0, 2) === path[0].prefix
+
+    if (matches && !options.allowOverwriting) {
+      throw new AlreadyExistsError()
+    }
+
+    return !matches
+  })
+
+  const newLink = node.Links
+    .find(link => (link.Name ?? '').substring(0, 2) === path[0].prefix)
+
+  if (newLink == null) {
+    throw new Error(`No link found with prefix ${path[0].prefix}`)
+  }
+
+  parentLinks.push(newLink)
+
+  return await updateHamtDirectory(parent, blockstore, parentLinks, path[0].bucket, options)
+}
+
+const addFileToShardedDirectory = async (parent: Directory, child: Required<PBLink>, blockstore: Blockstore, options: AddLinkOptions): Promise<{ shard: DirSharded, path: BucketPath[] }> => {
+  if (parent.node.Data == null) {
+    throw new InvalidPBNodeError('Parent node with no data passed to addFileToShardedDirectory')
+  }
+
+  // start at the root bucket and descend, loading nodes as we go
+  const rootBucket = await recreateInitialHamtLevel(parent.node.Links)
+  const node = UnixFS.unmarshal(parent.node.Data)
+
+  const shard = new DirSharded({
+    root: true,
+    dir: true,
+    parent: undefined,
+    parentKey: undefined,
+    path: '',
+    dirty: true,
+    flat: false,
+    mode: node.mode
+  }, {
+    ...options,
+    cidVersion: parent.cid.version
+  })
+  shard._bucket = rootBucket
+
+  if (node.mtime != null) {
+    // update mtime if previously set
+    shard.mtime = {
+      secs: Math.round(Date.now() / 1000)
+    }
+  }
+
+  // load subshards until the bucket & position no longer changes
+  const position = await rootBucket._findNewBucketAndPos(child.Name)
+  const path = toBucketPath(position)
+  path[0].node = parent.node
+  let index = 0
+
+  while (index < path.length) {
+    const segment = path[index]
+    index++
+    const node = segment.node
+
+    if (node == null) {
+      throw new Error('Segment had no node')
+    }
+
+    const link = node.Links
+      .find(link => (link.Name ?? '').substring(0, 2) === segment.prefix)
+
+    if (link == null) {
+      // prefix is new, file will be added to the current bucket
+      log(`Link ${segment.prefix}${child.Name} will be added`)
+      index = path.length
+
+      break
+    }
+
+    if (link.Name === `${segment.prefix}${child.Name}`) {
+      // file already existed, file will be added to the current bucket
+      log(`Link ${segment.prefix}${child.Name} will be replaced`)
+      index = path.length
+
+      break
+    }
+
+    if ((link.Name ?? '').length > 2) {
+      // another file had the same prefix, will be replaced with a subshard
+      log(`Link ${link.Name} ${link.Hash} will be replaced with a subshard`)
+      index = path.length
+
+      break
+    }
+
+    // load sub-shard
+    log(`Found subshard ${segment.prefix}`)
+    const block = await blockstore.get(link.Hash)
+    const subShard = dagPB.decode(block)
+
+    // subshard hasn't been loaded, descend to the next level of the HAMT
+    if (path[index] == null) {
+      log(`Loaded new subshard ${segment.prefix}`)
+      await recreateHamtLevel(blockstore, subShard.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options)
+
+      const position = await rootBucket._findNewBucketAndPos(child.Name)
+
+      path.push({
+        bucket: position.bucket,
+        prefix: toPrefix(position.pos),
+        node: subShard
+      })
+
+      break
+    }
+
+    const nextSegment = path[index]
+
+    // add next levels worth of links to bucket
+    await addLinksToHamtBucket(blockstore, subShard.Links, nextSegment.bucket, rootBucket, options)
+
+    nextSegment.node = subShard
+  }
+
+  // finally add the new file into the shard
+  await shard._bucket.put(child.Name, {
+    size: child.Tsize,
+    cid: child.Hash
+  })
+
+  return {
+    shard, path
+  }
+}
+
+export interface BucketPath {
+  bucket: Bucket<any>
+  prefix: string
+  node?: PBNode
+}
+
+const toBucketPath = (position: { pos: number, bucket: Bucket<any> }): BucketPath[] => {
+  const path = [{
+    bucket: position.bucket,
+    prefix: toPrefix(position.pos)
+  }]
+
+  let bucket = position.bucket._parent
+  let positionInBucket = position.bucket._posAtParent
+
+  while (bucket != null) {
+    path.push({
+      bucket,
+      prefix: toPrefix(positionInBucket)
+    })
+
+    positionInBucket = bucket._posAtParent
+    bucket = bucket._parent
+  }
+
+  path.reverse()
+
+  return path
+}

package/src/commands/utils/cid-to-directory.ts

@@ -0,0 +1,23 @@
+import { NotADirectoryError } from '@helia/interface/errors'
+import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import type { PBNode } from '@ipld/dag-pb'
+import type { AbortOptions } from '@libp2p/interfaces'
+
+export interface Directory {
+  cid: CID
+  node: PBNode
+}
+
+export async function cidToDirectory (cid: CID, blockstore: Blockstore, options: AbortOptions = {}): Promise<Directory> {
+  const entry = await exporter(cid, blockstore, options)
+
+  if (entry.type !== 'directory') {
+    throw new NotADirectoryError(`${cid.toString()} was not a UnixFS directory`)
+  }
+
+  return {
+    cid,
+    node: entry.node
+  }
+}

package/src/commands/utils/cid-to-pblink.ts

@@ -0,0 +1,26 @@
+import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import type { CID } from 'multiformats/cid'
+import { NotUnixFSError } from './errors.js'
+import * as dagPb from '@ipld/dag-pb'
+import type { PBNode, PBLink } from '@ipld/dag-pb'
+import type { AbortOptions } from '@libp2p/interfaces'
+
+export async function cidToPBLink (cid: CID, name: string, blockstore: Blockstore, options?: AbortOptions): Promise<Required<PBLink>> {
+  const sourceEntry = await exporter(cid, blockstore, options)
+
+  if (sourceEntry.type !== 'directory' && sourceEntry.type !== 'file' && sourceEntry.type !== 'raw') {
+    throw new NotUnixFSError(`${cid.toString()} was not a UnixFS node`)
+  }
+
+  return {
+    Name: name,
+    Tsize: sourceEntry.node instanceof Uint8Array ? sourceEntry.node.byteLength : dagNodeTsize(sourceEntry.node),
+    Hash: cid
+  }
+}
+
+function dagNodeTsize (node: PBNode): number {
+  const linkSizes = node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0)
+
+  return dagPb.encode(node).byteLength + linkSizes
+}