@helia/verified-fetch 6.0.0 → 6.1.0

package/src/url-resolver.ts

@@ -1,15 +1,9 @@
 import { DoesNotExistError } from '@helia/unixfs/errors'
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagJson from '@ipld/dag-json'
-import * as dagPb from '@ipld/dag-pb'
 import { peerIdFromString } from '@libp2p/peer-id'
 import { InvalidParametersError, walkPath } from 'ipfs-unixfs-exporter'
-import toBuffer from 'it-to-buffer'
 import { CID } from 'multiformats/cid'
-import * as json from 'multiformats/codecs/json'
-import * as raw from 'multiformats/codecs/raw'
 import QuickLRU from 'quick-lru'
-import { SESSION_CACHE_MAX_SIZE, SESSION_CACHE_TTL_MS, CODEC_CBOR, CODEC_IDENTITY } from './constants.ts'
+import { SESSION_CACHE_MAX_SIZE, SESSION_CACHE_TTL_MS } from './constants.ts'
 import { ServerTiming } from './utils/server-timing.ts'
 import type { ResolveURLResult, URLResolver as URLResolverInterface } from './index.ts'
 import type { DNSLink } from '@helia/dnslink'
@@ -17,49 +11,30 @@ import type { IPNSResolver } from '@helia/ipns'
 import type { AbortOptions } from '@libp2p/interface'
 import type { Helia, SessionBlockstore } from 'helia'
 import type { Blockstore } from 'interface-blockstore'
-import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
+import type { PathEntry } from 'ipfs-unixfs-exporter'
 
 // 1 year in seconds for ipfs content
 const IPFS_CONTENT_TTL = 29030400
 
-const ENTITY_CODECS = [
-  CODEC_CBOR,
-  json.code,
-  raw.code
-]
-
-/**
- * These are supported by the UnixFS exporter
- */
-const EXPORTABLE_CODECS = [
-  dagPb.code,
-  dagCbor.code,
-  dagJson.code,
-  raw.code
-]
-
 interface GetBlockstoreOptions extends AbortOptions {
   session?: boolean
 }
 
 export interface WalkPathResult {
   ipfsRoots: CID[]
-  terminalElement: UnixFSEntry
+  terminalElement: PathEntry
   blockstore: Blockstore
 }
 
-function basicEntry (type: 'raw' | 'object', cid: CID, bytes: Uint8Array): UnixFSEntry {
+function basicEntry (cid: CID): PathEntry {
   return {
+    cid,
     name: cid.toString(),
     path: cid.toString(),
-    depth: 0,
-    type,
-    node: bytes,
-    cid,
-    size: BigInt(bytes.byteLength),
-    content: async function * () {
-      yield bytes
-    }
+    roots: [
+      cid
+    ],
+    remainder: []
   }
 }
 
@@ -77,7 +52,6 @@ export interface URLResolverInit {
 export interface ResolveURLOptions extends AbortOptions {
   session?: boolean
   isRawBlockRequest?: boolean
-  onlyIfCached?: boolean
 }
 
 export class URLResolver implements URLResolverInterface {
@@ -195,16 +169,14 @@ export class URLResolver implements URLResolverInterface {
     const cid = CID.parse(url.hostname)
     const blockstore = this.getBlockstore(cid, options)
 
-    if (EXPORTABLE_CODECS.includes(cid.code)) {
+    try {
       const ipfsRoots: CID[] = []
-      let terminalElement: UnixFSEntry | undefined
+      let terminalElement: PathEntry | undefined
       const ipfsPath = toIPFSPath(url)
 
-      // @ts-expect-error offline is a helia option
       for await (const entry of walkPath(ipfsPath, blockstore, {
         ...options,
-        offline: options.onlyIfCached === true,
-        extended: options.isRawBlockRequest !== true
+        yieldSubShards: true
       })) {
         ipfsRoots.push(entry.cid)
         terminalElement = entry
@@ -219,31 +191,17 @@ export class URLResolver implements URLResolverInterface {
         terminalElement,
         blockstore
       }
-    }
-
-    let bytes: Uint8Array
-
-    if (cid.multihash.code === CODEC_IDENTITY) {
-      bytes = cid.multihash.digest
-    } else {
-      bytes = await toBuffer(blockstore.get(cid, options))
-    }
-
-    // entity codecs contain all the bytes for an entity in one block and no
-    // path walking outside of that block is possible
-    if (ENTITY_CODECS.includes(cid.code)) {
-      return {
-        ipfsRoots: [cid],
-        terminalElement: basicEntry('object', cid, bytes),
-        blockstore
+    } catch (err: any) {
+      if (err.name === 'NoResolverError') {
+        // may be an unknown codec
+        return {
+          ipfsRoots: [cid],
+          terminalElement: basicEntry(cid),
+          blockstore
+        }
       }
-    }
 
-    // may be an unknown codec
-    return {
-      ipfsRoots: [cid],
-      terminalElement: basicEntry('raw', cid, bytes),
-      blockstore
+      throw err
     }
   }
 }

package/src/utils/error-to-response.ts

@@ -26,7 +26,7 @@ export function errorToResponse (resource: Resource | string, err: any, init?: R
   }
 
   // path was not under DAG root
-  if (['ERR_NO_PROP', 'ERR_NO_TERMINAL_ELEMENT', 'ERR_NOT_FOUND'].includes(err.code)) {
+  if (['ERR_BAD_PATH', 'ERR_NO_TERMINAL_ELEMENT', 'ERR_NOT_FOUND'].includes(err.code)) {
     return notFoundResponse(resource)
   }
 

package/src/utils/get-tar-stream.ts

@@ -1,5 +1,7 @@
 import { NotUnixFSError } from '@helia/unixfs/errors'
-import { exporter, recursive } from 'ipfs-unixfs-exporter'
+import { InvalidParametersError } from '@libp2p/interface'
+import { exporter, recursive, walkPath } from 'ipfs-unixfs-exporter'
+import last from 'it-last'
 import map from 'it-map'
 import { pipe } from 'it-pipe'
 import { pack } from 'it-tar'
@@ -10,9 +12,14 @@ import type { TarEntryHeader, TarImportCandidate } from 'it-tar'
 
 const EXPORTABLE = ['file', 'raw', 'directory']
 
-function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
+function toHeader (file: UnixFSEntry, path: string): Partial<TarEntryHeader> & { name: string } {
   let mode: number | undefined
   let mtime: Date | undefined
+  let size = 0n
+
+  if (file.type === 'file' || file.type === 'raw' || file.type === 'identity') {
+    size = file.size
+  }
 
   if (file.type === 'file' || file.type === 'directory') {
     mode = file.unixfs.mode
@@ -20,21 +27,21 @@ function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string
   }
 
   return {
-    name: file.path,
+    name: path,
     mode,
     mtime,
-    size: Number(file.size),
+    size: Number(size),
     type: file.type === 'directory' ? 'directory' : 'file'
   }
 }
 
-function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
+function toTarImportCandidate (entry: UnixFSEntry, path: string): TarImportCandidate {
   if (!EXPORTABLE.includes(entry.type)) {
     throw new NotUnixFSError(`${entry.type} is not a UnixFS node`)
   }
 
   const candidate: TarImportCandidate = {
-    header: toHeader(entry)
+    header: toHeader(entry, path)
   }
 
   if (entry.type === 'file' || entry.type === 'raw') {
@@ -45,11 +52,17 @@ function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
 }
 
 export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
-  const file = await exporter(ipfsPath, blockstore, options)
+  const entry = await last(walkPath(ipfsPath, blockstore, options))
+
+  if (entry == null) {
+    throw new InvalidParametersError(`Could not walk path "${ipfsPath}"`)
+  }
+
+  const file = await exporter(entry.cid, blockstore, options)
 
   if (file.type === 'file' || file.type === 'raw') {
     yield * pipe(
-      [toTarImportCandidate(file)],
+      [toTarImportCandidate(file, entry.path)],
       pack()
     )
 
@@ -58,8 +71,11 @@ export async function * tarStream (ipfsPath: string, blockstore: Blockstore, opt
 
   if (file.type === 'directory') {
     yield * pipe(
-      recursive(ipfsPath, blockstore, options),
-      (source) => map(source, (entry) => toTarImportCandidate(entry)),
+      recursive(file.cid, blockstore, options),
+      (source) => map(source, async (entry) => {
+        const file = await exporter(entry.cid, blockstore, options)
+        return toTarImportCandidate(file, entry.path)
+      }),
       pack()
     )
 

package/src/verified-fetch.ts

@@ -1,6 +1,6 @@
 import { dnsLink } from '@helia/dnslink'
 import { ipnsResolver } from '@helia/ipns'
-import { AbortError, isPeerId, isPublicKey } from '@libp2p/interface'
+import { isPeerId, isPublicKey } from '@libp2p/interface'
 import { CID } from 'multiformats/cid'
 import { CustomProgressEvent } from 'progress-events'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
@@ -8,6 +8,7 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { CarPlugin } from './plugins/plugin-handle-car.js'
 import { IpldPlugin } from './plugins/plugin-handle-ipld.js'
 import { IpnsRecordPlugin } from './plugins/plugin-handle-ipns-record.js'
+import { RawPlugin } from './plugins/plugin-handle-raw.ts'
 import { TarPlugin } from './plugins/plugin-handle-tar.js'
 import { UnixFSPlugin } from './plugins/plugin-handle-unixfs.js'
 import { URLResolver } from './url-resolver.ts'
@@ -19,7 +20,7 @@ import { getETag, ifNoneMatches } from './utils/get-e-tag.js'
 import { getRangeHeader } from './utils/get-range-header.ts'
 import { stringToIpfsUrl } from './utils/parse-resource.ts'
 import { setCacheControlHeader } from './utils/response-headers.js'
-import { internalServerErrorResponse, notAcceptableResponse, notImplementedResponse, notModifiedResponse } from './utils/responses.js'
+import { notAcceptableResponse, notImplementedResponse, notModifiedResponse } from './utils/responses.js'
 import { ServerTiming } from './utils/server-timing.js'
 import type { AcceptHeader, CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, ResourceDetail, VerifiedFetchInit as VerifiedFetchOptions, VerifiedFetchPlugin, PluginContext, PluginOptions } from './index.js'
 import type { DNSLink } from '@helia/dnslink'
@@ -108,7 +109,8 @@ export class VerifiedFetch {
       new IpldPlugin(pluginOptions),
       new CarPlugin(pluginOptions),
       new TarPlugin(pluginOptions),
-      new IpnsRecordPlugin(pluginOptions)
+      new IpnsRecordPlugin(pluginOptions),
+      new RawPlugin(pluginOptions)
     ]
 
     const customPlugins = init.plugins?.map((pluginFactory) => pluginFactory(pluginOptions)) ?? []
@@ -149,6 +151,10 @@ export class VerifiedFetch {
       const headers = new Headers(options?.headers)
       const serverTiming = new ServerTiming()
 
+      if (options != null) {
+        options.offline ??= headers.get('cache-control') === 'only-if-cached'
+      }
+
       options?.onProgress?.(new CustomProgressEvent<ResourceDetail>('verified-fetch:request:start', { resource }))
 
       const range = getRangeHeader(resource, headers)
@@ -201,8 +207,7 @@ export class VerifiedFetch {
 
       const resolveResult = await this.urlResolver.resolve(url, serverTiming, {
         ...options,
-        isRawBlockRequest: isRawBlockRequest(headers),
-        onlyIfCached: headers.get('cache-control') === 'only-if-cached'
+        isRawBlockRequest: isRawBlockRequest(headers)
       })
 
       options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:resolve', {
@@ -408,21 +413,21 @@ export class VerifiedFetch {
     let pluginHandled = false
 
     for (const plugin of plugins) {
-      try {
-        this.log('invoking plugin: %s', plugin.id)
-        pluginsUsed.add(plugin.id)
+      // try {
+      this.log('invoking plugin: %s', plugin.id)
+      pluginsUsed.add(plugin.id)
 
-        const maybeResponse = await plugin.handle(context)
+      const maybeResponse = await plugin.handle(context)
 
-        this.log('plugin response %s %o', plugin.id, maybeResponse)
+      this.log('plugin response %s %o', plugin.id, maybeResponse)
 
-        if (maybeResponse != null) {
-          // if a plugin returns a final Response, short-circuit
-          finalResponse = maybeResponse
-          pluginHandled = true
-          break
-        }
-      } catch (err: any) {
+      if (maybeResponse != null) {
+        // if a plugin returns a final Response, short-circuit
+        finalResponse = maybeResponse
+        pluginHandled = true
+        break
+      }
+      /* } catch (err: any) {
         if (context.options?.signal?.aborted) {
           throw new AbortError(context.options?.signal?.reason)
         }
@@ -430,7 +435,7 @@ export class VerifiedFetch {
         this.log.error('error in plugin %s - %e', plugin.id, err)
 
         return internalServerErrorResponse(context.resource, err)
-      }
+      } */
 
       if (finalResponse != null) {
         this.log.trace('plugin %s produced final response', plugin.id)