@helia/unixfs 0.0.0

package/src/commands/utils/remove-link.ts

@@ -0,0 +1,151 @@
+
+import * as dagPB from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import { logger } from '@libp2p/logger'
+import { UnixFS } from 'ipfs-unixfs'
+import {
+  generatePath,
+  updateHamtDirectory,
+  UpdateHamtResult
+} from './hamt-utils.js'
+import type { PBNode, PBLink } from '@ipld/dag-pb'
+import type { Blockstore } from 'interface-blockstore'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { Bucket } from 'hamt-sharding'
+import type { Directory } from './cid-to-directory.js'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { InvalidPBNodeError } from './errors.js'
+import { InvalidParametersError } from '@helia/interface/errors'
+
+const log = logger('helia:unixfs:utils:remove-link')
+
+export interface RemoveLinkResult {
+  node: PBNode
+  cid: CID
+}
+
+export async function removeLink (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<RemoveLinkResult> {
+  if (parent.node.Data == null) {
+    throw new InvalidPBNodeError('Parent node had no data')
+  }
+
+  const meta = UnixFS.unmarshal(parent.node.Data)
+
+  if (meta.type === 'hamt-sharded-directory') {
+    log(`Removing ${name} from sharded directory`)
+
+    return await removeFromShardedDirectory(parent, name, blockstore, options)
+  }
+
+  log(`Removing link ${name} regular directory`)
+
+  return await removeFromDirectory(parent, name, blockstore, options)
+}
+
+const removeFromDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<RemoveLinkResult> => {
+  // Remove existing link if it exists
+  parent.node.Links = parent.node.Links.filter((link) => {
+    return link.Name !== name
+  })
+
+  const parentBlock = dagPB.encode(parent.node)
+  const hash = await sha256.digest(parentBlock)
+  const parentCid = CID.create(parent.cid.version, dagPB.code, hash)
+
+  await blockstore.put(parentCid, parentBlock, options)
+
+  log(`Updated regular directory ${parentCid}`)
+
+  return {
+    node: parent.node,
+    cid: parentCid
+  }
+}
+
+const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise<UpdateHamtResult> => {
+  const {
+    rootBucket, path
+  } = await generatePath(parent, name, blockstore, options)
+
+  await rootBucket.del(name)
+
+  const {
+    node
+  } = await updateShard(parent, blockstore, path, name, options)
+
+  return await updateHamtDirectory(parent, blockstore, node.Links, rootBucket, options)
+}
+
+const updateShard = async (parent: Directory, blockstore: Blockstore, positions: Array<{ bucket: Bucket<any>, prefix: string, node?: PBNode }>, name: string, options: AbortOptions): Promise<{ node: PBNode, cid: CID, size: number }> => {
+  const last = positions.pop()
+
+  if (last == null) {
+    throw new InvalidParametersError('Could not find parent')
+  }
+
+  const {
+    bucket,
+    prefix,
+    node
+  } = last
+
+  if (node == null) {
+    throw new InvalidParametersError('Could not find parent')
+  }
+
+  const link = node.Links
+    .find(link => (link.Name ?? '').substring(0, 2) === prefix)
+
+  if (link == null) {
+    throw new InvalidParametersError(`No link found with prefix ${prefix} for file ${name}`)
+  }
+
+  if (link.Name === `${prefix}${name}`) {
+    log(`Removing existing link ${link.Name}`)
+
+    const links = node.Links.filter((nodeLink) => {
+      return nodeLink.Name !== link.Name
+    })
+
+    await bucket.del(name)
+
+    parent.node = node
+
+    return await updateHamtDirectory(parent, blockstore, links, bucket, options)
+  }
+
+  log(`Descending into sub-shard ${link.Name} for ${prefix}${name}`)
+
+  const result = await updateShard(parent, blockstore, positions, name, options)
+
+  const child: Required<PBLink> = {
+    Hash: result.cid,
+    Tsize: result.size,
+    Name: prefix
+  }
+
+  if (result.node.Links.length === 1) {
+    log(`Removing subshard for ${prefix}`)
+
+    // convert shard back to normal dir
+    const link = result.node.Links[0]
+
+    child.Name = `${prefix}${(link.Name ?? '').substring(2)}`
+    child.Hash = link.Hash
+    child.Tsize = link.Tsize ?? 0
+  }
+
+  log(`Updating shard ${prefix} with name ${child.Name}`)
+
+  return await updateShardParent(parent, child, prefix, blockstore, bucket, options)
+}
+
+const updateShardParent = async (parent: Directory, child: Required<PBLink>, oldName: string, blockstore: Blockstore, bucket: Bucket<any>, options: AbortOptions): Promise<UpdateHamtResult> => {
+  // Remove existing link if it exists
+  const parentLinks = parent.node.Links.filter((link) => {
+    return link.Name !== oldName
+  })
+  parentLinks.push(child)
+
+  return await updateHamtDirectory(parent, blockstore, parentLinks, bucket, options)
+}

package/src/commands/utils/resolve.ts

@@ -0,0 +1,130 @@
+import type { CID } from 'multiformats/cid'
+import { Blockstore, exporter } from 'ipfs-unixfs-exporter'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { InvalidParametersError } from '@helia/interface/errors'
+import { logger } from '@libp2p/logger'
+import { DoesNotExistError } from './errors.js'
+import { addLink } from './add-link.js'
+import { cidToDirectory } from './cid-to-directory.js'
+import { cidToPBLink } from './cid-to-pblink.js'
+
+const log = logger('helia:unixfs:components:utils:add-link')
+
+export interface Segment {
+  name: string
+  cid: CID
+  size: number
+}
+
+export interface ResolveResult {
+  /**
+   * The CID at the end of the path
+   */
+  cid: CID
+
+  path?: string
+
+  /**
+   * If present, these are the CIDs and path segments that were traversed through to reach the final CID
+   *
+   * If not present, there was no path passed or the path was an empty string
+   */
+  segments?: Segment[]
+}
+
+export async function resolve (cid: CID, path: string | undefined, blockstore: Blockstore, options: AbortOptions): Promise<ResolveResult> {
+  log('resolve "%s" under %c', path, cid)
+
+  if (path == null || path === '') {
+    return { cid }
+  }
+
+  const parts = path.split('/').filter(Boolean)
+  const segments: Segment[] = [{
+    name: '',
+    cid,
+    size: 0
+  }]
+
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i]
+    const result = await exporter(cid, blockstore, options)
+
+    if (result.type === 'file') {
+      if (i < parts.length - 1) {
+        throw new InvalidParametersError('Path was invalid')
+      }
+
+      cid = result.cid
+    } else if (result.type === 'directory') {
+      let dirCid: CID | undefined
+
+      for await (const entry of result.content()) {
+        if (entry.name === part) {
+          dirCid = entry.cid
+        }
+      }
+
+      if (dirCid == null) {
+        throw new DoesNotExistError('Could not find path in directory')
+      }
+
+      cid = dirCid
+
+      segments.push({
+        name: part,
+        cid,
+        size: result.size
+      })
+    } else {
+      throw new InvalidParametersError('Could not resolve path')
+    }
+  }
+
+  return {
+    cid,
+    path,
+    segments
+  }
+}
+
+/**
+ * Where we have descended into a DAG to update a child node, ascend up the DAG creating
+ * new hashes and blocks for the changed content
+ */
+export async function updatePathCids (cid: CID, result: ResolveResult, blockstore: Blockstore, options: AbortOptions): Promise<CID> {
+  if (result.segments == null || result.segments.length === 0) {
+    return cid
+  }
+
+  let child = result.segments.pop()
+
+  if (child == null) {
+    throw new Error('Insufficient segments')
+  }
+
+  child.cid = cid
+
+  result.segments.reverse()
+
+  for (const parent of result.segments) {
+    const [
+      directory,
+      pblink
+    ] = await Promise.all([
+      cidToDirectory(parent.cid, blockstore, options),
+      cidToPBLink(child.cid, child.name, blockstore, options)
+    ])
+
+    const result = await addLink(directory, pblink, blockstore, {
+      ...options,
+      allowOverwriting: true
+    })
+
+    cid = result.cid
+    parent.cid = cid
+    child = parent
+  }
+
+  return cid
+}

package/src/index.ts

@@ -0,0 +1,174 @@
+import type { CID, Version } from 'multiformats/cid'
+import type { Blockstore } from 'interface-blockstore'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { ImportCandidate, ImportResult, UserImporterOptions } from 'ipfs-unixfs-importer'
+import { add, addStream } from './commands/add.js'
+import { cat } from './commands/cat.js'
+import { mkdir } from './commands/mkdir.js'
+import type { Mtime } from 'ipfs-unixfs'
+import { cp } from './commands/cp.js'
+import { rm } from './commands/rm.js'
+import { stat } from './commands/stat.js'
+import { touch } from './commands/touch.js'
+import { chmod } from './commands/chmod.js'
+import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
+import { ls } from './commands/ls.js'
+
+export interface UnixFSComponents {
+  blockstore: Blockstore
+}
+
+export interface CatOptions extends AbortOptions {
+  offset?: number
+  length?: number
+  path?: string
+}
+
+export interface ChmodOptions extends AbortOptions {
+  recursive: boolean
+  path?: string
+}
+
+export interface CpOptions extends AbortOptions {
+  force: boolean
+}
+
+export interface LsOptions extends AbortOptions {
+  path?: string
+  offset?: number
+  length?: number
+}
+
+export interface MkdirOptions extends AbortOptions {
+  cidVersion: Version
+  force: boolean
+  mode?: number
+  mtime?: Mtime
+}
+
+export interface RmOptions extends AbortOptions {
+
+}
+
+export interface StatOptions extends AbortOptions {
+  path?: string
+}
+
+export interface UnixFSStats {
+  /**
+   * The file or directory CID
+   */
+  cid: CID
+
+  /**
+   * The file or directory mode
+   */
+  mode?: number
+
+  /**
+   * The file or directory mtime
+   */
+  mtime?: Mtime
+
+  /**
+   * The size of the file in bytes
+   */
+  fileSize: number
+
+  /**
+   * The size of the DAG that holds the file in bytes
+   */
+  dagSize: number
+
+  /**
+   * How much of the file is in the local block store
+   */
+  localFileSize: number
+
+  /**
+   * How much of the DAG that holds the file is in the local blockstore
+   */
+  localDagSize: number
+
+  /**
+   * How many blocks make up the DAG - nb. this will only be accurate
+   * if all blocks are present in the local blockstore
+   */
+  blocks: number
+
+  /**
+   * The type of file
+   */
+  type: 'file' | 'directory' | 'raw'
+}
+
+export interface TouchOptions extends AbortOptions {
+  mtime?: Mtime
+  path?: string
+  recursive: boolean
+}
+
+export interface UnixFS {
+  add: (source: Uint8Array | Iterator<Uint8Array> | AsyncIterator<Uint8Array> | ImportCandidate, options?: Partial<UserImporterOptions>) => Promise<CID>
+  addStream: (source: Iterable<ImportCandidate> | AsyncIterable<ImportCandidate>, options?: Partial<UserImporterOptions>) => AsyncGenerator<ImportResult>
+  cat: (cid: CID, options?: Partial<CatOptions>) => AsyncIterable<Uint8Array>
+  chmod: (source: CID, mode: number, options?: Partial<ChmodOptions>) => Promise<CID>
+  cp: (source: CID, target: CID, name: string, options?: Partial<CpOptions>) => Promise<CID>
+  ls: (cid: CID, options?: Partial<LsOptions>) => AsyncIterable<UnixFSEntry>
+  mkdir: (cid: CID, dirname: string, options?: Partial<MkdirOptions>) => Promise<CID>
+  rm: (cid: CID, path: string, options?: Partial<RmOptions>) => Promise<CID>
+  stat: (cid: CID, options?: Partial<StatOptions>) => Promise<UnixFSStats>
+  touch: (cid: CID, options?: Partial<TouchOptions>) => Promise<CID>
+}
+
+class DefaultUnixFS implements UnixFS {
+  private readonly components: UnixFSComponents
+
+  constructor (components: UnixFSComponents) {
+    this.components = components
+  }
+
+  async add (source: Uint8Array | Iterator<Uint8Array> | AsyncIterator<Uint8Array> | ImportCandidate, options: Partial<UserImporterOptions> = {}): Promise<CID> {
+    return await add(source, this.components.blockstore, options)
+  }
+
+  async * addStream (source: Iterable<ImportCandidate> | AsyncIterable<ImportCandidate>, options: Partial<UserImporterOptions> = {}): AsyncGenerator<ImportResult> {
+    yield * addStream(source, this.components.blockstore, options)
+  }
+
+  async * cat (cid: CID, options: Partial<CatOptions> = {}): AsyncIterable<Uint8Array> {
+    yield * cat(cid, this.components.blockstore, options)
+  }
+
+  async chmod (source: CID, mode: number, options: Partial<ChmodOptions> = {}): Promise<CID> {
+    return await chmod(source, mode, this.components.blockstore, options)
+  }
+
+  async cp (source: CID, target: CID, name: string, options: Partial<CpOptions> = {}): Promise<CID> {
+    return await cp(source, target, name, this.components.blockstore, options)
+  }
+
+  async * ls (cid: CID, options: Partial<LsOptions> = {}): AsyncIterable<UnixFSEntry> {
+    yield * ls(cid, this.components.blockstore, options)
+  }
+
+  async mkdir (cid: CID, dirname: string, options: Partial<MkdirOptions> = {}): Promise<CID> {
+    return await mkdir(cid, dirname, this.components.blockstore, options)
+  }
+
+  async rm (cid: CID, path: string, options: Partial<RmOptions> = {}): Promise<CID> {
+    return await rm(cid, path, this.components.blockstore, options)
+  }
+
+  async stat (cid: CID, options: Partial<StatOptions> = {}): Promise<UnixFSStats> {
+    return await stat(cid, this.components.blockstore, options)
+  }
+
+  async touch (cid: CID, options: Partial<TouchOptions> = {}): Promise<CID> {
+    return await touch(cid, this.components.blockstore, options)
+  }
+}
+
+export function unixfs (helia: { blockstore: Blockstore }): UnixFS {
+  return new DefaultUnixFS(helia)
+}