@helia/verified-fetch 7.0.4 → 7.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/verified-fetch",
-  "version": "7.0.4",
+  "version": "7.2.0",
   "description": "A fetch-like API for obtaining verified & trustless IPFS content on the web",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/verified-fetch#readme",
@@ -164,14 +164,14 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "@helia/block-brokers": "^5.1.0",
-    "@helia/car": "^5.3.7",
+    "@helia/block-brokers": "^5.2.1",
+    "@helia/car": "^5.4.0",
     "@helia/delegated-routing-v1-http-api-client": "^6.0.1",
-    "@helia/dnslink": "^1.1.4",
-    "@helia/interface": "^6.1.0",
-    "@helia/ipns": "^9.1.8",
-    "@helia/routers": "^5.0.2",
-    "@helia/unixfs": "^7.0.3",
+    "@helia/dnslink": "^1.2.0",
+    "@helia/interface": "^6.2.0",
+    "@helia/ipns": "^9.2.0",
+    "@helia/routers": "^5.1.0",
+    "@helia/unixfs": "^7.2.0",
     "@ipld/dag-cbor": "^9.2.3",
     "@ipld/dag-json": "^10.2.4",
     "@ipld/dag-pb": "^4.1.5",
@@ -182,8 +182,10 @@
     "@libp2p/webrtc": "^6.0.8",
     "@libp2p/websockets": "^10.1.0",
     "@multiformats/dns": "^1.0.6",
+    "@multiformats/multiaddr": "^13.0.1",
+    "@multiformats/multiaddr-matcher": "^3.0.2",
     "file-type": "^21.1.1",
-    "helia": "^6.0.18",
+    "helia": "^6.1.1",
     "interface-blockstore": "^6.0.1",
     "ipfs-unixfs-exporter": "^15.0.2",
     "ipns": "^10.1.3",
@@ -202,10 +204,10 @@
     "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
-    "@helia/dag-cbor": "^5.0.0",
-    "@helia/dag-json": "^5.0.0",
-    "@helia/http": "^3.0.5",
-    "@helia/json": "^5.0.0",
+    "@helia/dag-cbor": "^5.1.0",
+    "@helia/dag-json": "^5.1.0",
+    "@helia/http": "^3.1.1",
+    "@helia/json": "^5.1.0",
     "@ipld/car": "^5.4.2",
     "@libp2p/crypto": "^5.1.13",
     "@libp2p/logger": "^6.2.0",
@@ -214,7 +216,7 @@
     "aegir": "^47.0.24",
     "blockstore-core": "^6.1.1",
     "browser-readablestream-to-it": "^2.0.9",
-    "cborg": "^4.2.11",
+    "cborg": "^5.1.0",
     "ipfs-unixfs-importer": "^16.0.1",
     "it-all": "^3.0.8",
     "magic-bytes.js": "^1.12.1",

package/src/index.ts

@@ -772,6 +772,77 @@
  *   - Any thrown error immediately stops the pipeline and returns the error response.
  *
  * For a detailed explanation of the pipeline, please refer to the discussion in [Issue #167](https://github.com/ipfs/helia-verified-fetch/issues/167).
+ *
+ * ### Server-Timing
+ *
+ * Detailed timing is found in the [Server-Timing](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Server-Timing)
+ * HTTP header that is returned with every response when a resource is requested
+ * with the `withServerTiming` init option set to `true`.
+ *
+ * To prevent the header value growing too large, PeerIDs/CIDs are truncated to
+ * their first 10 characters and common strings are abbreviated.
+ *
+ * The values you may expect to see are described in the following table. Note
+ * that not all of them may be present in a given response.
+ *
+ * Router, block broker and transport abbreviations used in the `desc` fields
+ * follow.
+ *
+ * | Timing metric      | Elaboration     | Detail                          | Example |
+ * | ------------------ | --------------- | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+ * | d                  | DNSLink.resolve | Resolving a DNSLink to an IPFS path or IPNS name                                                                                                                                                      | `d;dur=0.200`                                  |
+ * | i                  | IPFS.resolve    | Resolving a CID + path to a CID                                                                                                                                                                       | `i;dur=0.200`                                  |
+ * | n                  | IPNS.resolve    | Resolving an IPNS name to an IPFS path                                                                                                                                                                | `n;dur=0.200`                                  |
+ * | p                  | Provider        | A provider was found. The `desc` field contains the routing system that found the provider and the first 10 characters of the PeerId                                                                  | `p;dur=0.000;desc="h,bagqbeaawn"`              |
+ * | f                  | Find Providers  | The total duration of the routing systems Find Providers operation. The `desc` field contains the routing system and how many providers were found                                                    | `f;dur=2.000;desc="h,4"`                       |
+ * | c                  | Connect         | How long it took to connect to a provider. The `desc` field contains the type of provider, the first 10 characters of their PeerId and the transport used                                             | `b;dur=0.000;desc="t,bagqbeaa7n,bafybeigoc,t"` |
+ * | b                  | Block           | How long it took to retrieve a block from the provider once connected. The `desc` field contains the type of provider, the first 10 characters of their PeerId and the first 10 characters of the CID | `b;dur=0.000;desc="t,bagqbeaa7n,bafybeigoc"`   |
+ *
+ * A full header might look like:
+ *
+ * ```
+ * i;dur=0.000,p;dur=0.000;desc="h,bagqbeaawn",p;dur=0.000;desc="h,bagqbeaawn",p;dur=1.000;desc="h,bagqbeaa7n",p;dur=1.000;desc="h,bagqbeaa7n",f;dur=1.000;desc="h,4",f;dur=1.000;desc="h,4",f;dur=144.000;desc="l,0",f;dur=144.000;desc="l,0",c;dur=206.000;desc="t,bagqbeaa7n,h",b;dur=1.000;desc="t,bagqbeaa7n,bafybeigoc"
+ * ```
+ *
+ * Here resolving a CID to a CID+path took less than a millisecond (e.g. a bare
+ * CID was requested).
+ *
+ * Two HTTP Gateway providers were found in the routing (`bagqbeaawn` and
+ * `bagqbeaa7n`). They are found twice because two block brokers are configured
+ * (bitswap and trustless-gateway) which both make a routing request (results
+ * are cached internally so the duration to find them the second time differs).
+ *
+ * It took 206ms to connect to `bagqbeaa7n` over HTTP, and 1s to retrieve the
+ * block for the CID `bafybeigo` from the Trustless Gateway `bagqbeaa7n`.
+ *
+ * All PeerIDs and CIDs above are truncated to 10 characters.
+ *
+ * #### Router abbreviations
+ *
+ * | Router | Elaboration           |
+ * | ------ | --------------------- |
+ * | h      | HTTP Gateway          |
+ * | l      | Libp2p (e.g. Kad-DHT) |
+ *
+ * #### Block broker abbreviations
+ *
+ * | Block Broker | Elaboration       |
+ * | ------------ | ----------------- |
+ * | t            | Trustless Gateway |
+ * | b            | Bitswap           |
+ *
+ * #### Transport abbreviations
+ *
+ * | Transport | Elaboration   |
+ * | --------- | ------------- |
+ * | t         | TCP           |
+ * | h         | HTTP          |
+ * | w         | WebSockets    |
+ * | r         | WebRTC        |
+ * | d         | WebRTC-Direct |
+ * | q         | QUIC          |
+ * | b         | WebTransport  |
+ * | u         | Unknown       |
  */
 
 import { bitswap, trustlessGateway } from '@helia/block-brokers'
@@ -903,7 +974,7 @@ export interface PluginContext extends ResolveURLResult, Omit<VerifiedFetchInit,
   /**
    * A callback that receives progress events
    */
-  onProgress?(evt: ProgressEvent): void
+  onProgress?(evt: VerifiedFetchProgressEvents): void
 
   /**
    * Any async operations should be invoked using server timings to allow
@@ -1218,6 +1289,17 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<Verified
    */
   supportWebRedirects?: boolean
 
+  /**
+   * If a HAMT-sharded directory is encountered, paths will be translated
+   * automatically, e.g. a request for `QmHamt/bar.txt` will be translated to
+   * `QmHamt/F0/A1bar.txt` internally, pass `false` here to not perform this
+   * translation which will then treat HAMT shard structures as
+   * regular directories.
+   *
+   * @default true
+   */
+  translateHAMTPath?: boolean
+
   /**
    * If true, only operate on the local blockstore, do not perform any network
    * operations.

package/src/url-resolver.ts

@@ -6,6 +6,7 @@ import { CID } from 'multiformats/cid'
 import QuickLRU from 'quick-lru'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { CODEC_LIBP2P_KEY, SESSION_CACHE_MAX_SIZE, SESSION_CACHE_TTL_MS } from './constants.ts'
+import { abbreviate } from './utils/abbreviate.ts'
 import { applyRedirects } from './utils/apply-redirect.ts'
 import { ServerTiming } from './utils/server-timing.ts'
 import type { ResolveURLOptions, ResolveURLResult, URLResolver as URLResolverInterface } from './index.ts'
@@ -124,7 +125,7 @@ export class URLResolver implements URLResolverInterface {
   }
 
   private async resolveDNSLink (url: URL, serverTiming: ServerTiming, options?: ResolveURLOptions): Promise<ResolveURLResult | Response> {
-    const results = await serverTiming.time('dnsLink.resolve', `Resolve DNSLink ${url.hostname}`, this.components.dnsLink.resolve(url.hostname, options))
+    const results = await serverTiming.time(abbreviate('dnsLink.resolve'), '', this.components.dnsLink.resolve(url.hostname, options))
     const result = results?.[0]
 
     if (result == null) {
@@ -161,7 +162,7 @@ export class URLResolver implements URLResolverInterface {
 
   private async resolveIPNSName (url: URL, serverTiming: ServerTiming, options?: ResolveURLOptions): Promise<ResolveURLResult | Response> {
     const peerId = peerIdFromString(url.hostname)
-    const result = await serverTiming.time('ipns.resolve', `Resolve IPNS name ${peerId}`, this.components.ipnsResolver.resolve(peerId, options))
+    const result = await serverTiming.time(abbreviate('ipns.resolve'), '', this.components.ipnsResolver.resolve(peerId, options))
     const path = normalizePath(`${result.path ?? ''}/${url.pathname}`)
 
     const ipfsUrl = new URL(`ipfs://${result.cid}${path}`)
@@ -180,7 +181,7 @@ export class URLResolver implements URLResolverInterface {
   }
 
   private async resolveIPFSPath (url: URL, serverTiming: ServerTiming, options?: ResolveURLOptions): Promise<ResolveURLResult | Response> {
-    const walkPathResult = await serverTiming.time('ipfs.resolve', '', this.walkPath(url, options))
+    const walkPathResult = await serverTiming.time(abbreviate('ipfs.resolve'), '', this.walkPath(url, options))
 
     if (walkPathResult instanceof Response) {
       return walkPathResult

package/src/utils/abbreviate.ts

@@ -0,0 +1,57 @@
+import { HTTP, HTTPS, QUIC, QUIC_V1, TCP, WebRTC, WebRTCDirect, WebSockets, WebSocketsSecure, WebTransport } from '@multiformats/multiaddr-matcher'
+import type { Multiaddr } from '@multiformats/multiaddr'
+
+const ABBREVIATIONS: Record<string, string> = {
+  // operations
+  'ipfs.resolve': 'i',
+  'dnsLink.resolve': 'd',
+  'ipns.resolve': 'n',
+  'found-provider': 'p',
+  'find-providers': 'f',
+  connect: 'c',
+  block: 'b',
+
+  // routers
+  'http-gateway-router': 'h',
+  'libp2p-router': 'l',
+
+  // block brokers
+  'trustless-gateway': 't',
+  bitswap: 'b'
+}
+
+export function abbreviate (str: string): string {
+  return ABBREVIATIONS[str] ?? str
+}
+
+export function abbreviateAddress (ma: Multiaddr): string {
+  if (TCP.exactMatch(ma)) {
+    return 't'
+  }
+
+  if (HTTP.exactMatch(ma) || HTTPS.exactMatch(ma)) {
+    return 'h'
+  }
+
+  if (WebSockets.exactMatch(ma) || WebSocketsSecure.exactMatch(ma)) {
+    return 'w'
+  }
+
+  if (WebRTC.exactMatch(ma)) {
+    return 'r'
+  }
+
+  if (WebRTCDirect.exactMatch(ma)) {
+    return 'd'
+  }
+
+  if (QUIC.exactMatch(ma) || QUIC_V1.exactMatch(ma)) {
+    return 'q'
+  }
+
+  if (WebTransport.exactMatch(ma)) {
+    return 'b'
+  }
+
+  return 'u'
+}

package/src/utils/server-timing.ts

@@ -25,6 +25,6 @@ export class ServerTiming {
   }
 
   add (name: string, description: string, duration: number | string): void {
-    this.headers.push(`${name};dur=${Number(duration).toFixed(this.precision)};desc="${description}"`)
+    this.headers.push(`${name};dur=${Number(duration).toFixed(this.precision)}${description === '' ? '' : `;desc="${description}"`}`)
   }
 }

package/src/verified-fetch.ts

@@ -12,6 +12,7 @@ import { RawPlugin } from './plugins/plugin-handle-raw.ts'
 import { TarPlugin } from './plugins/plugin-handle-tar.js'
 import { UnixFSPlugin } from './plugins/plugin-handle-unixfs.js'
 import { URLResolver } from './url-resolver.ts'
+import { abbreviate, abbreviateAddress } from './utils/abbreviate.ts'
 import { contentTypeParser } from './utils/content-type-parser.js'
 import { getContentType, getSupportedContentTypes, CONTENT_TYPE_OCTET_STREAM, MEDIA_TYPE_IPNS_RECORD, CONTENT_TYPE_IPNS } from './utils/content-types.ts'
 import { errorToObject } from './utils/error-to-object.ts'
@@ -58,6 +59,10 @@ function isIPNSRecordRequest (headers: Headers): boolean {
   return mediaType === MEDIA_TYPE_IPNS_RECORD
 }
 
+function truncate (obj?: any): string {
+  return `${obj}`.substring(0, 10)
+}
+
 export class VerifiedFetch {
   private readonly helia: Helia
   private readonly ipnsResolver: IPNSResolver
@@ -211,6 +216,10 @@ export class VerifiedFetch {
         return this.handleFinalResponse(accept)
       }
 
+      const routingTimers: Record<string, { start: number, found: number }> = {}
+      const connectTimers: Record<string, { start: number, transport: string }> = {}
+      const blockTimers: Record<string, number> = {}
+
       const context: PluginContext = {
         ...options,
         ...resolveResult,
@@ -219,7 +228,56 @@ export class VerifiedFetch {
         range,
         serverTiming,
         headers,
-        requestedMimeTypes
+        requestedMimeTypes,
+        onProgress: (evt) => {
+          options?.onProgress?.(evt)
+
+          if (evt.type === 'helia:routing:find-providers:start') {
+            routingTimers[evt.detail.routing] = {
+              start: Date.now(),
+              found: 0
+            }
+          } else if (evt.type === 'helia:routing:find-providers:provider') {
+            if (routingTimers[evt.detail.routing] == null) {
+              return
+            }
+
+            routingTimers[evt.detail.routing].found++
+
+            serverTiming.add(abbreviate('found-provider'), `${abbreviate(evt.detail.routing)},${truncate(evt.detail.provider.id)}`, Date.now() - routingTimers[evt.detail.routing].start)
+          } else if (evt.type === 'helia:routing:find-providers:end') {
+            const routing = routingTimers[evt.detail.routing]
+
+            if (routing == null) {
+              return
+            }
+
+            serverTiming.add(abbreviate('find-providers'), `${abbreviate(evt.detail.routing)},${routing.found}`, Date.now() - routing.start)
+          } else if (evt.type === 'helia:block-broker:connect') {
+            connectTimers[`connect-${evt.detail.broker}-${evt.detail.provider}`] = {
+              start: Date.now(),
+              transport: ''
+            }
+          } else if (evt.type === 'helia:block-broker:connected') {
+            const start = connectTimers[`connect-${evt.detail.broker}-${evt.detail.provider}`]
+
+            if (start == null) {
+              return
+            }
+
+            serverTiming.add(abbreviate('connect'), `${abbreviate(evt.detail.broker)},${truncate(evt.detail.provider)},${abbreviateAddress(evt.detail.address)}`, Date.now() - start.start)
+          } else if (evt.type === 'helia:block-broker:request-block') {
+            blockTimers[`block-${evt.detail.broker}-${evt.detail.cid}-${evt.detail.provider}`] = Date.now()
+          } else if (evt.type === 'helia:block-broker:receive-block') {
+            const start = blockTimers[`block-${evt.detail.broker}-${evt.detail.cid}-${evt.detail.provider}`]
+
+            if (start == null) {
+              return
+            }
+
+            serverTiming.add(abbreviate('block'), `${abbreviate(evt.detail.broker)},${truncate(evt.detail.provider)},${truncate(evt.detail.cid)}`, Date.now() - start)
+          }
+        }
       }
 
       this.log.trace('finding handler for cid code "0x%s" and response content types %s', resolveResult.terminalElement.cid.code.toString(16), accept.map(header => header.contentType.mediaType).join(', '))