@helia/verified-fetch 3.2.3 → 4.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +10 -52
  2. package/dist/index.min.js +86 -71
  3. package/dist/index.min.js.map +4 -4
  4. package/dist/src/constants.d.ts +2 -0
  5. package/dist/src/constants.d.ts.map +1 -0
  6. package/dist/src/constants.js +2 -0
  7. package/dist/src/constants.js.map +1 -0
  8. package/dist/src/index.d.ts +63 -61
  9. package/dist/src/index.d.ts.map +1 -1
  10. package/dist/src/index.js +12 -54
  11. package/dist/src/index.js.map +1 -1
  12. package/dist/src/plugins/index.d.ts +0 -1
  13. package/dist/src/plugins/index.d.ts.map +1 -1
  14. package/dist/src/plugins/index.js +0 -1
  15. package/dist/src/plugins/index.js.map +1 -1
  16. package/dist/src/plugins/plugin-base.d.ts.map +1 -1
  17. package/dist/src/plugins/plugin-base.js +3 -2
  18. package/dist/src/plugins/plugin-base.js.map +1 -1
  19. package/dist/src/plugins/plugin-handle-car.d.ts.map +1 -1
  20. package/dist/src/plugins/plugin-handle-car.js +37 -28
  21. package/dist/src/plugins/plugin-handle-car.js.map +1 -1
  22. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.d.ts +1 -1
  23. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.d.ts.map +1 -1
  24. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.js +1 -2
  25. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.js.map +1 -1
  26. package/dist/src/plugins/plugin-handle-dag-cbor.d.ts.map +1 -1
  27. package/dist/src/plugins/plugin-handle-dag-cbor.js +5 -6
  28. package/dist/src/plugins/plugin-handle-dag-cbor.js.map +1 -1
  29. package/dist/src/plugins/plugin-handle-dag-pb.d.ts.map +1 -1
  30. package/dist/src/plugins/plugin-handle-dag-pb.js +24 -27
  31. package/dist/src/plugins/plugin-handle-dag-pb.js.map +1 -1
  32. package/dist/src/plugins/plugin-handle-dag-walk.d.ts +8 -4
  33. package/dist/src/plugins/plugin-handle-dag-walk.d.ts.map +1 -1
  34. package/dist/src/plugins/plugin-handle-dag-walk.js +13 -9
  35. package/dist/src/plugins/plugin-handle-dag-walk.js.map +1 -1
  36. package/dist/src/plugins/plugin-handle-ipns-record.d.ts +1 -1
  37. package/dist/src/plugins/plugin-handle-ipns-record.d.ts.map +1 -1
  38. package/dist/src/plugins/plugin-handle-ipns-record.js +16 -24
  39. package/dist/src/plugins/plugin-handle-ipns-record.js.map +1 -1
  40. package/dist/src/plugins/plugin-handle-json.d.ts.map +1 -1
  41. package/dist/src/plugins/plugin-handle-json.js +5 -5
  42. package/dist/src/plugins/plugin-handle-json.js.map +1 -1
  43. package/dist/src/plugins/plugin-handle-raw.d.ts.map +1 -1
  44. package/dist/src/plugins/plugin-handle-raw.js +21 -12
  45. package/dist/src/plugins/plugin-handle-raw.js.map +1 -1
  46. package/dist/src/plugins/plugin-handle-tar.d.ts.map +1 -1
  47. package/dist/src/plugins/plugin-handle-tar.js +1 -2
  48. package/dist/src/plugins/plugin-handle-tar.js.map +1 -1
  49. package/dist/src/plugins/types.d.ts +15 -15
  50. package/dist/src/plugins/types.d.ts.map +1 -1
  51. package/dist/src/url-resolver.d.ts +21 -0
  52. package/dist/src/url-resolver.d.ts.map +1 -0
  53. package/dist/src/url-resolver.js +118 -0
  54. package/dist/src/url-resolver.js.map +1 -0
  55. package/dist/src/utils/byte-range-context.d.ts +3 -3
  56. package/dist/src/utils/byte-range-context.d.ts.map +1 -1
  57. package/dist/src/utils/byte-range-context.js +1 -1
  58. package/dist/src/utils/byte-range-context.js.map +1 -1
  59. package/dist/src/utils/content-type-parser.d.ts.map +1 -1
  60. package/dist/src/utils/content-type-parser.js +0 -10
  61. package/dist/src/utils/content-type-parser.js.map +1 -1
  62. package/dist/src/utils/error-to-object.d.ts +6 -0
  63. package/dist/src/utils/error-to-object.d.ts.map +1 -0
  64. package/dist/src/utils/error-to-object.js +20 -0
  65. package/dist/src/utils/error-to-object.js.map +1 -0
  66. package/dist/src/utils/get-content-type.d.ts +3 -3
  67. package/dist/src/utils/get-content-type.d.ts.map +1 -1
  68. package/dist/src/utils/get-content-type.js +1 -1
  69. package/dist/src/utils/get-content-type.js.map +1 -1
  70. package/dist/src/utils/get-e-tag.d.ts +1 -1
  71. package/dist/src/utils/get-offset-and-length.d.ts +6 -0
  72. package/dist/src/utils/get-offset-and-length.d.ts.map +1 -0
  73. package/dist/src/utils/get-offset-and-length.js +46 -0
  74. package/dist/src/utils/get-offset-and-length.js.map +1 -0
  75. package/dist/src/utils/get-resolved-accept-header.d.ts +2 -2
  76. package/dist/src/utils/get-resolved-accept-header.d.ts.map +1 -1
  77. package/dist/src/utils/get-stream-from-async-iterable.d.ts +2 -2
  78. package/dist/src/utils/get-stream-from-async-iterable.d.ts.map +1 -1
  79. package/dist/src/utils/get-stream-from-async-iterable.js +2 -2
  80. package/dist/src/utils/get-stream-from-async-iterable.js.map +1 -1
  81. package/dist/src/utils/handle-redirects.d.ts.map +1 -1
  82. package/dist/src/utils/handle-redirects.js +3 -3
  83. package/dist/src/utils/handle-redirects.js.map +1 -1
  84. package/dist/src/utils/ipfs-path-to-string.d.ts +6 -0
  85. package/dist/src/utils/ipfs-path-to-string.d.ts.map +1 -0
  86. package/dist/src/utils/ipfs-path-to-string.js +10 -0
  87. package/dist/src/utils/ipfs-path-to-string.js.map +1 -0
  88. package/dist/src/utils/is-accept-explicit.d.ts +6 -4
  89. package/dist/src/utils/is-accept-explicit.d.ts.map +1 -1
  90. package/dist/src/utils/is-accept-explicit.js +7 -4
  91. package/dist/src/utils/is-accept-explicit.js.map +1 -1
  92. package/dist/src/utils/parse-url-string.d.ts +1 -55
  93. package/dist/src/utils/parse-url-string.d.ts.map +1 -1
  94. package/dist/src/utils/parse-url-string.js +16 -217
  95. package/dist/src/utils/parse-url-string.js.map +1 -1
  96. package/dist/src/utils/response-headers.d.ts +1 -1
  97. package/dist/src/utils/response-headers.d.ts.map +1 -1
  98. package/dist/src/utils/responses.d.ts +3 -2
  99. package/dist/src/utils/responses.d.ts.map +1 -1
  100. package/dist/src/utils/responses.js +12 -1
  101. package/dist/src/utils/responses.js.map +1 -1
  102. package/dist/src/utils/select-output-type.d.ts +6 -2
  103. package/dist/src/utils/select-output-type.d.ts.map +1 -1
  104. package/dist/src/utils/select-output-type.js +28 -37
  105. package/dist/src/utils/select-output-type.js.map +1 -1
  106. package/dist/src/utils/server-timing.d.ts +5 -11
  107. package/dist/src/utils/server-timing.d.ts.map +1 -1
  108. package/dist/src/utils/server-timing.js +17 -15
  109. package/dist/src/utils/server-timing.js.map +1 -1
  110. package/dist/src/utils/walk-path.js +2 -2
  111. package/dist/src/utils/walk-path.js.map +1 -1
  112. package/dist/src/verified-fetch.d.ts +3 -10
  113. package/dist/src/verified-fetch.d.ts.map +1 -1
  114. package/dist/src/verified-fetch.js +99 -80
  115. package/dist/src/verified-fetch.js.map +1 -1
  116. package/dist/typedoc-urls.json +13 -4
  117. package/package.json +35 -36
  118. package/src/constants.ts +1 -0
  119. package/src/index.ts +79 -70
  120. package/src/plugins/index.ts +0 -1
  121. package/src/plugins/plugin-base.ts +3 -2
  122. package/src/plugins/plugin-handle-car.ts +53 -31
  123. package/src/plugins/plugin-handle-dag-cbor-html-preview.ts +4 -3
  124. package/src/plugins/plugin-handle-dag-cbor.ts +8 -6
  125. package/src/plugins/plugin-handle-dag-pb.ts +34 -26
  126. package/src/plugins/plugin-handle-dag-walk.ts +15 -9
  127. package/src/plugins/plugin-handle-ipns-record.ts +21 -24
  128. package/src/plugins/plugin-handle-json.ts +6 -5
  129. package/src/plugins/plugin-handle-raw.ts +27 -13
  130. package/src/plugins/plugin-handle-tar.ts +3 -2
  131. package/src/plugins/types.ts +18 -16
  132. package/src/url-resolver.ts +159 -0
  133. package/src/utils/byte-range-context.ts +4 -4
  134. package/src/utils/content-type-parser.ts +5 -11
  135. package/src/utils/error-to-object.ts +22 -0
  136. package/src/utils/get-content-type.ts +5 -4
  137. package/src/utils/get-e-tag.ts +1 -1
  138. package/src/utils/get-offset-and-length.ts +54 -0
  139. package/src/utils/get-resolved-accept-header.ts +2 -2
  140. package/src/utils/get-stream-from-async-iterable.ts +4 -4
  141. package/src/utils/handle-redirects.ts +10 -3
  142. package/src/utils/ipfs-path-to-string.ts +9 -0
  143. package/src/utils/is-accept-explicit.ts +14 -7
  144. package/src/utils/parse-url-string.ts +20 -286
  145. package/src/utils/response-headers.ts +1 -1
  146. package/src/utils/responses.ts +16 -2
  147. package/src/utils/select-output-type.ts +38 -44
  148. package/src/utils/server-timing.ts +17 -30
  149. package/src/utils/walk-path.ts +2 -2
  150. package/src/verified-fetch.ts +119 -92
  151. package/dist/src/plugins/errors.d.ts +0 -25
  152. package/dist/src/plugins/errors.d.ts.map +0 -1
  153. package/dist/src/plugins/errors.js +0 -33
  154. package/dist/src/plugins/errors.js.map +0 -1
  155. package/dist/src/types.d.ts +0 -16
  156. package/dist/src/types.d.ts.map +0 -1
  157. package/dist/src/types.js +0 -2
  158. package/dist/src/types.js.map +0 -1
  159. package/dist/src/utils/parse-resource.d.ts +0 -18
  160. package/dist/src/utils/parse-resource.d.ts.map +0 -1
  161. package/dist/src/utils/parse-resource.js +0 -27
  162. package/dist/src/utils/parse-resource.js.map +0 -1
  163. package/src/plugins/errors.ts +0 -37
  164. package/src/types.ts +0 -17
  165. package/src/utils/parse-resource.ts +0 -42
package/src/index.ts CHANGED
@@ -641,13 +641,13 @@
641
641
  * Inspects the current `PluginContext` (which includes the CID, path, query, accept header, etc.)
642
642
  * and returns `true` if the plugin can operate on the current state of the request.
643
643
  *
644
- * - **`handle(context: PluginContext): Promise<Response | null>`**
645
- * Performs the plugin’s work. It may:
646
- * - **Return a final `Response`**: This stops the pipeline immediately.
647
- * - **Return `null`**: This indicates that the plugin has only partially processed the request
648
- * (for example, by performing path walking or decoding) and the pipeline should continue.
649
- * - **Throw a `PluginError`**: This logs a non-fatal error and continues the pipeline.
650
- * - **Throw a `PluginFatalError`**: This logs a fatal error and stops the pipeline immediately.
644
+ * - **`handle(context: PluginContext): Promise<Response | undefined>`**
645
+ * Performs the plugin’s work. It will only be executed if `canHandle` previously returned `true`.
646
+ * It may:
647
+ * - **Return a `Response`**: This stops the pipeline immediately and returns the response.
648
+ * - **Return `undefined`**: This indicates that the plugin has only partially processed the request
649
+ * (for example, by performing path walking or decoding) and the pipeline should continue.
650
+ * - **Throw an `Error`**: An Internal Server Error will be returned
651
651
  *
652
652
  * ### Plugin Pipeline
653
653
  *
@@ -745,12 +745,12 @@
745
745
  * canHandle(context: PluginContext): boolean {
746
746
  * // Only handle requests if the Accept header matches your custom type
747
747
  * // Or check context for pathDetails, custom values, etc...
748
- * return context.accept === 'application/vnd.my-custom-type'
748
+ * return context.accept?.mimeType === 'application/vnd.my-custom-type'
749
749
  * }
750
750
  *
751
751
  * async handle(context: PluginContext): Promise<Response | null> {
752
752
  * // Perform any partial processing here, e.g., modify the context:
753
- * context.customProcessed = true;
753
+ * context.customProcessed = true
754
754
  *
755
755
  * // If you are ready to finalize the response:
756
756
  * return new Response('Hello, world!', {
@@ -758,7 +758,7 @@
758
758
  * headers: {
759
759
  * 'Content-Type': 'text/plain'
760
760
  * }
761
- * });
761
+ * })
762
762
  *
763
763
  * // Or, if further processing is needed by another plugin, simply return null.
764
764
  * }
@@ -784,47 +784,6 @@
784
784
  * const fetch = await createVerifiedFetch(helia, { plugins })
785
785
  * ```
786
786
  *
787
- * ---
788
- *
789
- * ### Error Handling in the Plugin Pipeline
790
- *
791
- * Verified‑Fetch distinguishes between two types of errors thrown by plugins:
792
- *
793
- * - **PluginError (Non‑Fatal):**
794
- * - Use this when your plugin encounters an issue that should be logged but does not prevent the pipeline
795
- * from continuing.
796
- * - When a plugin throws a `PluginError`, the error is logged and the pipeline continues with the next plugin.
797
- *
798
- * - **PluginFatalError (Fatal):**
799
- * - Use this when a critical error occurs that should immediately abort the request.
800
- * - When a plugin throws a `PluginFatalError`, the pipeline immediately terminates and the provided error
801
- * response is returned.
802
- *
803
- * @example Plugin error Handling
804
- *
805
- * ```typescript
806
- * import { PluginError, PluginFatalError } from '@helia/verified-fetch'
807
- *
808
- * // async handle(context: PluginContext): Promise<Response | null> {
809
- * const recoverable = Math.random() > 0.5 // Use more sophisticated logic here ;)
810
- * if (recoverable === true) {
811
- * throw new PluginError('MY_CUSTOM_WARNING', 'A non‑fatal issue occurred', {
812
- * details: {
813
- * someKey: 'Additional details here'
814
- * }
815
- * });
816
- * }
817
- *
818
- * if (recoverable === false) {
819
- * throw new PluginFatalError('MY_CUSTOM_FATAL', 'A critical error occurred', {
820
- * response: new Response('Something happened', { status: 500 }) // Required: supply your own error response
821
- * });
822
- * }
823
- *
824
- * // Otherwise, continue processing...
825
- * // }
826
- * ```
827
- *
828
787
  * ### How the Plugin Pipeline Works
829
788
  *
830
789
  * - **Shared Context:**
@@ -842,8 +801,7 @@
842
801
  * This means you do not have to specify a rigid order, each plugin simply checks the context and acts if appropriate.
843
802
  *
844
803
  * - **Error Handling:**
845
- * - A thrown `PluginError` is considered non‑fatal and is logged, allowing the pipeline to continue.
846
- * - A thrown `PluginFatalError` immediately stops the pipeline and returns the error response.
804
+ * - Any thrown error immediately stops the pipeline and returns the error response.
847
805
  *
848
806
  * For a detailed explanation of the pipeline, please refer to the discussion in [Issue #167](https://github.com/ipfs/helia-verified-fetch/issues/167).
849
807
  */
@@ -857,10 +815,10 @@ import { createLibp2p } from 'libp2p'
857
815
  import { getLibp2pConfig } from './utils/libp2p-defaults.js'
858
816
  import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
859
817
  import type { VerifiedFetchPluginFactory } from './plugins/types.js'
860
- import type { ContentTypeParser } from './types.js'
818
+ import type { DNSLink, ResolveProgressEvents as ResolveDNSLinkProgressEvents } from '@helia/dnslink'
861
819
  import type { GetBlockProgressEvents, Helia, Routing } from '@helia/interface'
862
- import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
863
- import type { Libp2p, ServiceMap } from '@libp2p/interface'
820
+ import type { IPNSResolver } from '@helia/ipns'
821
+ import type { AbortOptions, Libp2p, ServiceMap } from '@libp2p/interface'
864
822
  import type { DNSResolvers, DNS } from '@multiformats/dns'
865
823
  import type { DNSResolver } from '@multiformats/dns/resolvers'
866
824
  import type { HeliaInit } from 'helia'
@@ -868,6 +826,25 @@ import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter'
868
826
  import type { Libp2pOptions } from 'libp2p'
869
827
  import type { CID } from 'multiformats/cid'
870
828
  import type { ProgressEvent, ProgressOptions } from 'progress-events'
829
+
830
+ export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor'
831
+
832
+ export type SupportedBodyTypes = string | Uint8Array | ArrayBuffer | Blob | ReadableStream<Uint8Array> | null
833
+
834
+ /**
835
+ * A ContentTypeParser attempts to return the mime type of a given file. It
836
+ * receives the first chunk of the file data and the file name, if it is
837
+ * available. The function can be sync or async and if it returns/resolves to
838
+ * `undefined`, `application/octet-stream` will be used.
839
+ */
840
+ export interface ContentTypeParser {
841
+ /**
842
+ * Attempt to determine a mime type, either via of the passed bytes or the
843
+ * filename if it is available.
844
+ */
845
+ (bytes: Uint8Array, fileName?: string): Promise<string | undefined> | string | undefined
846
+ }
847
+
871
848
  /**
872
849
  * The types for the first argument of the `verifiedFetch` function.
873
850
  */
@@ -879,7 +856,7 @@ export interface ResourceDetail {
879
856
 
880
857
  export interface CIDDetail {
881
858
  cid: CID
882
- path: string
859
+ path?: string
883
860
  }
884
861
 
885
862
  export interface CIDDetailError extends CIDDetail {
@@ -995,24 +972,32 @@ export interface CreateVerifiedFetchOptions {
995
972
  * If you want to replace one of the default plugins, you can do so by passing a plugin with the same name.
996
973
  */
997
974
  plugins?: VerifiedFetchPluginFactory[]
998
- }
999
975
 
1000
- export type { ContentTypeParser } from './types.js'
976
+ /**
977
+ * Used to resolve IPNS names
978
+ */
979
+ ipnsResolver?: IPNSResolver
1001
980
 
1002
- export type BubbledProgressEvents =
1003
- // unixfs-exporter
1004
- ExporterProgressEvents |
1005
- // helia blockstore
1006
- GetBlockProgressEvents |
1007
- // ipns
1008
- ResolveDNSLinkProgressEvents
981
+ /**
982
+ * Used to resolve DNSLink entries to IPNS names or CIDs
983
+ */
984
+ dnsLink?: DNSLink
985
+
986
+ /**
987
+ * Used to turn URLs into CIDs/paths
988
+ */
989
+ urlResolver?: URLResolver
990
+ }
1009
991
 
1010
992
  export type VerifiedFetchProgressEvents =
1011
993
  ProgressEvent<'verified-fetch:request:start', CIDDetail> |
1012
994
  ProgressEvent<'verified-fetch:request:info', string> |
1013
- ProgressEvent<'verified-fetch:request:progress:chunk', CIDDetail> |
995
+ ProgressEvent<'verified-fetch:request:progress:chunk'> |
1014
996
  ProgressEvent<'verified-fetch:request:end', CIDDetail> |
1015
- ProgressEvent<'verified-fetch:request:error', CIDDetailError>
997
+ ProgressEvent<'verified-fetch:request:error', CIDDetailError> |
998
+ ExporterProgressEvents |
999
+ GetBlockProgressEvents |
1000
+ ResolveDNSLinkProgressEvents
1016
1001
 
1017
1002
  /**
1018
1003
  * Options for the `fetch` function returned by `createVerifiedFetch`.
@@ -1021,7 +1006,7 @@ export type VerifiedFetchProgressEvents =
1021
1006
  * passed to `fetch` in browsers, plus an `onProgress` option to listen for
1022
1007
  * progress events.
1023
1008
  */
1024
- export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledProgressEvents | VerifiedFetchProgressEvents> {
1009
+ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<VerifiedFetchProgressEvents> {
1025
1010
  /**
1026
1011
  * If true, try to create a blockstore session - this can reduce overall
1027
1012
  * network traffic by first querying for a set of peers that have the data we
@@ -1072,6 +1057,30 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledP
1072
1057
  withServerTiming?: boolean
1073
1058
  }
1074
1059
 
1060
+ export interface ResolveURLOptions extends ProgressOptions<VerifiedFetchProgressEvents>, AbortOptions {
1061
+
1062
+ }
1063
+
1064
+ export interface UrlQuery extends Record<string, string | unknown> {
1065
+ format?: RequestFormatShorthand
1066
+ download?: boolean
1067
+ filename?: string
1068
+ 'dag-scope'?: string
1069
+ }
1070
+
1071
+ export interface ResolveURLResult {
1072
+ cid: CID
1073
+ protocol: string
1074
+ ttl: number
1075
+ path: string
1076
+ query: UrlQuery
1077
+ ipfsPath: string
1078
+ }
1079
+
1080
+ export interface URLResolver {
1081
+ resolve (resource: Resource, options?: ResolveURLOptions): Promise<ResolveURLResult>
1082
+ }
1083
+
1075
1084
  /**
1076
1085
  * Create and return a Helia node
1077
1086
  */
@@ -1116,7 +1125,7 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni
1116
1125
  init.logger.forComponent('helia:verified-fetch').trace('created verified-fetch with libp2p config: %j', libp2pConfig)
1117
1126
  }
1118
1127
 
1119
- const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options)
1128
+ const verifiedFetchInstance = new VerifiedFetchClass(init, options)
1120
1129
  async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise<Response> {
1121
1130
  return verifiedFetchInstance.fetch(resource, options)
1122
1131
  }
@@ -2,7 +2,6 @@
2
2
  * This file is the entry into all things we export from the `src/plugins` directory.
3
3
  */
4
4
 
5
- export { PluginError, PluginFatalError } from './errors.js'
6
5
  export { BasePlugin } from './plugin-base.js'
7
6
  export type { PluginOptions, PluginContext, VerifiedFetchPluginFactory } from './types.js'
8
7
  export * from './plugins.js'
@@ -17,9 +17,10 @@ export abstract class BasePlugin implements VerifiedFetchPlugin {
17
17
  protected _log?: Logger
18
18
 
19
19
  get log (): Logger {
20
- // instantiate the logger lazily because it depends on the id, which is not set until after the constructor is called
20
+ // instantiate the logger lazily because it depends on the id, which is not
21
+ // set until after the constructor is called
21
22
  if (this._log == null) {
22
- this._log = this.pluginOptions.logger.forComponent(this.id)
23
+ this._log = this.pluginOptions.logger.newScope(this.id)
23
24
  }
24
25
  return this._log
25
26
  }
@@ -1,11 +1,12 @@
1
1
  import { BlockExporter, car, CIDPath, SubgraphExporter, UnixFSExporter } from '@helia/car'
2
- import { CarWriter } from '@ipld/car'
3
2
  import { code as dagPbCode } from '@ipld/dag-pb'
3
+ import { createScalableCuckooFilter } from '@libp2p/utils'
4
4
  import toBrowserReadableStream from 'it-to-browser-readablestream'
5
+ import { getOffsetAndLength } from '../utils/get-offset-and-length.ts'
5
6
  import { okRangeResponse } from '../utils/responses.js'
6
7
  import { BasePlugin } from './plugin-base.js'
7
8
  import type { PluginContext } from './types.js'
8
- import type { ExportCarOptions } from '@helia/car'
9
+ import type { ExportCarOptions, UnixFSExporterOptions } from '@helia/car'
9
10
 
10
11
  function getFilename ({ cid, ipfsPath, query }: Pick<PluginContext, 'query' | 'cid' | 'ipfsPath'>): string {
11
12
  if (query.filename != null) {
@@ -13,7 +14,10 @@ function getFilename ({ cid, ipfsPath, query }: Pick<PluginContext, 'query' | 'c
13
14
  }
14
15
 
15
16
  // convert context.ipfsPath to a filename. replace all / with _, replace prefix protocol with empty string
16
- const filename = ipfsPath.replace(/\/ipfs\//, '').replace(/\/ipns\//, '').replace(/\//g, '_')
17
+ const filename = ipfsPath
18
+ .replace(/\/ipfs\//, '')
19
+ .replace(/\/ipns\//, '')
20
+ .replace(/\//g, '_')
17
21
 
18
22
  return `${filename}.car`
19
23
  }
@@ -22,9 +26,16 @@ function getFilename ({ cid, ipfsPath, query }: Pick<PluginContext, 'query' | 'c
22
26
  type DagScope = 'all' | 'entity' | 'block'
23
27
  function getDagScope ({ query }: Pick<PluginContext, 'query'>): DagScope | null {
24
28
  const dagScope = query['dag-scope']
29
+
25
30
  if (dagScope === 'all' || dagScope === 'entity' || dagScope === 'block') {
26
31
  return dagScope
27
32
  }
33
+
34
+ // entity-bytes implies entity scope
35
+ if (query['entity-bytes']) {
36
+ return 'entity'
37
+ }
38
+
28
39
  return 'all'
29
40
  }
30
41
 
@@ -36,22 +47,27 @@ export class CarPlugin extends BasePlugin {
36
47
  readonly id = 'car-plugin'
37
48
 
38
49
  canHandle (context: PluginContext): boolean {
39
- this.log('checking if we can handle %c with accept %s', context.cid, context.accept)
40
50
  if (context.byteRangeContext == null) {
41
51
  return false
42
52
  }
53
+
43
54
  if (context.pathDetails == null) {
44
55
  return false
45
56
  }
46
57
 
47
- return context.accept?.startsWith('application/vnd.ipld.car') === true || context.query.format === 'car' // application/vnd.ipld.car
58
+ return context.accept?.mimeType.startsWith('application/vnd.ipld.car') === true || context.query.format === 'car' // application/vnd.ipld.car
48
59
  }
49
60
 
50
61
  async handle (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext'>>): Promise<Response> {
51
- const { options, pathDetails, cid } = context
62
+ const { options, pathDetails, cid, query, accept } = context
63
+
64
+ const order = accept?.options.order === 'dfs' ? 'dfs' : 'unk'
65
+ const duplicates = accept?.options.dups !== 'n'
66
+
52
67
  if (pathDetails == null) {
53
68
  throw new Error('attempted to handle request for car with no path details')
54
69
  }
70
+
55
71
  const { getBlockstore, helia } = this.pluginOptions
56
72
  context.reqFormat = 'car'
57
73
  context.query.download = true
@@ -63,46 +79,52 @@ export class CarPlugin extends BasePlugin {
63
79
  getCodec: helia.getCodec,
64
80
  logger: helia.logger
65
81
  })
66
- const ipfsRootsWithoutDagRoot = pathDetails.ipfsRoots.filter(pathCid => !pathCid.equals(cid))
82
+
67
83
  const carExportOptions: ExportCarOptions = {
68
- ...options
84
+ ...options,
85
+ includeTraversalBlocks: true
86
+ }
87
+
88
+ if (!duplicates) {
89
+ carExportOptions.blockFilter = createScalableCuckooFilter(1024)
69
90
  }
70
- if (ipfsRootsWithoutDagRoot.length > 0) {
71
- carExportOptions.traversal = new CIDPath(ipfsRootsWithoutDagRoot)
91
+
92
+ if (pathDetails.ipfsRoots.length > 1) {
93
+ carExportOptions.traversal = new CIDPath(pathDetails.ipfsRoots)
72
94
  }
95
+
73
96
  const dagScope = getDagScope(context)
74
- // root should be the terminal element if it exists, otherwise the root cid.. because of this, we can't use the @helia/car stream() method.
75
- const root = pathDetails.terminalElement.cid ?? cid
97
+ const target = pathDetails.terminalElement.cid ?? cid
98
+
76
99
  if (dagScope === 'block') {
77
100
  carExportOptions.exporter = new BlockExporter()
78
101
  } else if (dagScope === 'entity') {
79
- // if its unixFS, we need to enumerate a directory, or get all blocks for the entity, otherwise, use blockExporter
80
- if (root.code === dagPbCode) {
81
- carExportOptions.exporter = new UnixFSExporter()
102
+ // if its unixFS, we need to enumerate a directory, or get all/some blocks
103
+ // for the entity, otherwise, use blockExporter
104
+ if (target.code === dagPbCode) {
105
+ const options: UnixFSExporterOptions = {
106
+ listingOnly: true
107
+ }
108
+
109
+ const slice = getOffsetAndLength(pathDetails.terminalElement, query['entity-bytes']?.toString())
110
+ options.offset = slice.offset
111
+ options.length = slice.length
112
+
113
+ carExportOptions.exporter = new UnixFSExporter(options)
82
114
  } else {
83
115
  carExportOptions.exporter = new BlockExporter()
84
116
  }
85
117
  } else {
86
118
  carExportOptions.exporter = new SubgraphExporter()
87
119
  }
88
- const { writer, out } = CarWriter.create(root)
89
- const iter = async function * (): AsyncIterable<Uint8Array> {
90
- for await (const buf of out) {
91
- yield buf
92
- }
93
- }
94
120
 
95
- // the root passed to export should be the root CID of the DAG, not the terminal element.
96
- c.export(cid, writer, carExportOptions)
97
- .catch((err) => {
98
- this.log.error('error exporting car - %e', err)
99
- })
100
- // export will close the writer when it's done, no finally needed.
121
+ context.byteRangeContext.setBody(toBrowserReadableStream(c.export(target, carExportOptions)))
101
122
 
102
- context.byteRangeContext.setBody(toBrowserReadableStream(iter()))
103
-
104
- const response = okRangeResponse(context.resource, context.byteRangeContext.getBody('application/vnd.ipld.car; version=1'), { byteRangeContext: context.byteRangeContext, log: this.log })
105
- response.headers.set('content-type', context.byteRangeContext.getContentType() ?? 'application/vnd.ipld.car; version=1')
123
+ const response = okRangeResponse(context.resource, context.byteRangeContext.getBody('application/vnd.ipld.car; version=1'), {
124
+ byteRangeContext: context.byteRangeContext,
125
+ log: this.log
126
+ })
127
+ response.headers.set('content-type', context.byteRangeContext.getContentType() ?? `application/vnd.ipld.car; version=1; order=${order}; dups=${duplicates ? 'y' : 'n'}`)
106
128
 
107
129
  return response
108
130
  }
@@ -51,18 +51,19 @@ export class DagCborHtmlPreviewPlugin extends BasePlugin {
51
51
  readonly codes = [ipldDagCbor.code]
52
52
 
53
53
  canHandle ({ cid, accept, pathDetails }: PluginContext): boolean {
54
- this.log('checking if we can handle %c with accept %s', cid, accept)
55
54
  if (pathDetails == null) {
56
55
  return false
57
56
  }
57
+
58
58
  if (!isObjectNode(pathDetails.terminalElement)) {
59
59
  return false
60
60
  }
61
+
61
62
  if (cid.code !== ipldDagCbor.code) {
62
63
  return false
63
64
  }
64
65
 
65
- if (accept == null || !accept.includes('text/html')) {
66
+ if (accept == null || !accept?.mimeType.includes('text/html')) {
66
67
  return false
67
68
  }
68
69
 
@@ -97,7 +98,7 @@ export class DagCborHtmlPreviewPlugin extends BasePlugin {
97
98
  })
98
99
  }
99
100
 
100
- getHtml ({ path, obj, cid }: { path: string, obj: Record<string, any>, cid: CID }): string {
101
+ getHtml ({ path, obj, cid }: { path?: string, obj: Record<string, any>, cid: CID }): string {
101
102
  const style = `
102
103
  :root {
103
104
  --sans-serif: "Plex", system-ui, -apple-system, Segoe UI, Roboto, Ubuntu, Cantarell, Noto Sans, sans-serif;
@@ -16,21 +16,23 @@ export class DagCborPlugin extends BasePlugin {
16
16
  readonly codes = [ipldDagCbor.code]
17
17
 
18
18
  canHandle ({ cid, accept, pathDetails, byteRangeContext, plugins }: PluginContext): boolean {
19
- this.log('checking if we can handle %c with accept %s', cid, accept)
20
19
  if (pathDetails == null) {
21
20
  return false
22
21
  }
22
+
23
23
  if (!isObjectNode(pathDetails.terminalElement)) {
24
24
  return false
25
25
  }
26
+
26
27
  if (cid.code !== ipldDagCbor.code) {
27
28
  return false
28
29
  }
30
+
29
31
  if (byteRangeContext == null) {
30
32
  return false
31
33
  }
32
34
 
33
- if (accept != null && accept.includes('text/html') && plugins.includes('dag-cbor-plugin-html-preview')) {
35
+ if (accept != null && accept.mimeType === 'text/html' && plugins.includes('dag-cbor-plugin-html-preview')) {
34
36
  // let the dag-cbor-html-preview plugin handle it
35
37
  return false
36
38
  }
@@ -47,10 +49,10 @@ export class DagCborPlugin extends BasePlugin {
47
49
 
48
50
  let body: string | Uint8Array
49
51
 
50
- if (accept === 'application/octet-stream' || accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') {
52
+ if (accept?.mimeType === 'application/octet-stream' || accept?.mimeType === 'application/vnd.ipld.dag-cbor' || accept?.mimeType === 'application/cbor') {
51
53
  // skip decoding
52
54
  body = block
53
- } else if (accept === 'application/vnd.ipld.dag-json') {
55
+ } else if (accept?.mimeType === 'application/vnd.ipld.dag-json') {
54
56
  try {
55
57
  // if vnd.ipld.dag-json has been specified, convert to the format - note
56
58
  // that this supports more data types than regular JSON, the content-type
@@ -65,7 +67,7 @@ export class DagCborPlugin extends BasePlugin {
65
67
  try {
66
68
  body = dagCborToSafeJSON(block)
67
69
  } catch (err) {
68
- if (accept === 'application/json') {
70
+ if (accept?.mimeType === 'application/json') {
69
71
  this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err)
70
72
 
71
73
  return notAcceptableResponse(resource)
@@ -78,7 +80,7 @@ export class DagCborPlugin extends BasePlugin {
78
80
 
79
81
  context.byteRangeContext.setBody(body)
80
82
 
81
- const responseContentType = accept ?? (body instanceof Uint8Array ? 'application/octet-stream' : 'application/json')
83
+ const responseContentType = accept?.mimeType ?? (body instanceof Uint8Array ? 'application/octet-stream' : 'application/json')
82
84
  const response = okRangeResponse(resource, context.byteRangeContext.getBody(responseContentType), { byteRangeContext: context.byteRangeContext, log: this.log })
83
85
 
84
86
  response.headers.set('content-type', context.byteRangeContext.getContentType() ?? responseContentType)
@@ -6,7 +6,7 @@ import { CustomProgressEvent } from 'progress-events'
6
6
  import { getContentType } from '../utils/get-content-type.js'
7
7
  import { getStreamFromAsyncIterable } from '../utils/get-stream-from-async-iterable.js'
8
8
  import { setIpfsRoots } from '../utils/response-headers.js'
9
- import { badGatewayResponse, badRangeResponse, movedPermanentlyResponse, notSupportedResponse, okRangeResponse } from '../utils/responses.js'
9
+ import { badGatewayResponse, badRangeResponse, movedPermanentlyResponse, okRangeResponse } from '../utils/responses.js'
10
10
  import { BasePlugin } from './plugin-base.js'
11
11
  import type { PluginContext } from './types.js'
12
12
  import type { CIDDetail } from '../index.js'
@@ -18,15 +18,21 @@ import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
18
18
  export class DagPbPlugin extends BasePlugin {
19
19
  readonly id = 'dag-pb-plugin'
20
20
  readonly codes = [dagPbCode]
21
+
21
22
  canHandle ({ cid, accept, pathDetails, byteRangeContext }: PluginContext): boolean {
22
- this.log('checking if we can handle %c with accept %s', cid, accept)
23
23
  if (pathDetails == null) {
24
24
  return false
25
25
  }
26
+
26
27
  if (byteRangeContext == null) {
27
28
  return false
28
29
  }
29
30
 
31
+ // TODO: this may be too restrictive?
32
+ if (accept != null && accept.mimeType !== 'application/octet-stream') {
33
+ return false
34
+ }
35
+
30
36
  return cid.code === dagPbCode
31
37
  }
32
38
 
@@ -55,8 +61,8 @@ export class DagPbPlugin extends BasePlugin {
55
61
  }
56
62
 
57
63
  async handle (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext' | 'pathDetails'>>): Promise<Response | null> {
58
- const { cid, options, withServerTiming = false, pathDetails, query } = context
59
- const { handleServerTiming, contentTypeParser, helia, getBlockstore } = this.pluginOptions
64
+ const { cid, options, pathDetails, query } = context
65
+ const { contentTypeParser, helia, getBlockstore } = this.pluginOptions
60
66
  const log = this.log
61
67
  let resource = context.resource
62
68
  let path = context.path
@@ -93,10 +99,10 @@ export class DagPbPlugin extends BasePlugin {
93
99
  try {
94
100
  log.trace('found directory at %c/%s, looking for index.html', cid, path)
95
101
 
96
- const entry = await handleServerTiming('exporter-dir', '', async () => exporter(`/ipfs/${dirCid}/${rootFilePath}`, helia.blockstore, {
102
+ const entry = await context.serverTiming.time('exporter-dir', '', exporter(`/ipfs/${dirCid}/${rootFilePath}`, helia.blockstore, {
97
103
  signal: options?.signal,
98
104
  onProgress: options?.onProgress
99
- }), withServerTiming)
105
+ }))
100
106
 
101
107
  log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
102
108
  path = rootFilePath
@@ -105,21 +111,20 @@ export class DagPbPlugin extends BasePlugin {
105
111
  if (options?.signal?.aborted) {
106
112
  throw new AbortError(options?.signal?.reason)
107
113
  }
108
- this.log.error('error loading path %c/%s', dirCid, rootFilePath, err)
114
+
115
+ this.log.error('error loading path %c/%s - %e', dirCid, rootFilePath, err)
116
+
109
117
  context.isDirectory = true
110
118
  context.directoryEntries = []
111
119
  context.modified++
120
+
112
121
  this.log.trace('attempting to get directory entries because index.html was not found')
113
- try {
114
- for await (const dirItem of fs.ls(dirCid, { signal: options?.signal, onProgress: options?.onProgress, extended: false })) {
115
- context.directoryEntries.push(dirItem)
116
- }
117
- // dir-index-html plugin or dir-index-json (future idea?) plugin should handle this
118
- return null
119
- } catch (e) {
120
- log.error('error listing directory %c', dirCid, e)
121
- return notSupportedResponse('Unable to get directory contents')
122
+ for await (const dirItem of fs.ls(dirCid, { signal: options?.signal, onProgress: options?.onProgress, extended: false })) {
123
+ context.directoryEntries.push(dirItem)
122
124
  }
125
+
126
+ // dir-index-html plugin or dir-index-json (future idea?) plugin should handle this
127
+ return null
123
128
  } finally {
124
129
  options?.onProgress?.(new CustomProgressEvent<CIDDetail>('verified-fetch:request:end', { cid: dirCid, path: rootFilePath }))
125
130
  }
@@ -136,10 +141,10 @@ export class DagPbPlugin extends BasePlugin {
136
141
  }
137
142
 
138
143
  try {
139
- const entry = await handleServerTiming('exporter-file', '', async () => exporter(resolvedCID, helia.blockstore, {
144
+ const entry = await context.serverTiming.time('exporter-file', '', exporter(resolvedCID, helia.blockstore, {
140
145
  signal: options?.signal,
141
146
  onProgress: options?.onProgress
142
- }), withServerTiming)
147
+ }))
143
148
 
144
149
  let firstChunk: Uint8Array
145
150
  let contentType: string
@@ -152,13 +157,13 @@ export class DagPbPlugin extends BasePlugin {
152
157
  })
153
158
  log('got async iterator for %c/%s', cid, path)
154
159
 
155
- const streamAndFirstChunk = await handleServerTiming('stream-and-chunk', '', async () => getStreamFromAsyncIterable(asyncIter, path ?? '', this.pluginOptions.logger, {
160
+ const streamAndFirstChunk = await context.serverTiming.time('stream-and-chunk', '', getStreamFromAsyncIterable(asyncIter, path, this.pluginOptions.logger, {
156
161
  onProgress: options?.onProgress,
157
162
  signal: options?.signal
158
- }), withServerTiming)
163
+ }))
159
164
  const stream = streamAndFirstChunk.stream
160
165
  firstChunk = streamAndFirstChunk.firstChunk
161
- contentType = await handleServerTiming('get-content-type', '', async () => getContentType({ filename: query.filename, bytes: firstChunk, path, contentTypeParser, log }), withServerTiming)
166
+ contentType = await context.serverTiming.time('get-content-type', '', getContentType({ filename: query.filename, bytes: firstChunk, path, contentTypeParser, log }))
162
167
 
163
168
  byteRangeContext.setBody(stream)
164
169
  }
@@ -177,17 +182,20 @@ export class DagPbPlugin extends BasePlugin {
177
182
  if (options?.signal?.aborted) {
178
183
  throw new AbortError(options?.signal?.reason)
179
184
  }
180
- log.error('error streaming %c/%s', cid, path, err)
185
+
186
+ log.error('error streaming %c/%s - %e', cid, path, err)
187
+
181
188
  if (byteRangeContext.isRangeRequest && err.code === 'ERR_INVALID_PARAMS') {
182
189
  return badRangeResponse(resource)
183
190
  }
184
- return badGatewayResponse(resource.toString(), 'Unable to stream content')
191
+
192
+ return badGatewayResponse(resource, 'Unable to stream content')
185
193
  }
186
194
  }
187
195
 
188
196
  private async handleRangeRequest (context: PluginContext & Required<Pick<PluginContext, 'byteRangeContext' | 'pathDetails'>>, entry: UnixFSEntry): Promise<string> {
189
- const { path, byteRangeContext, options, withServerTiming = false } = context
190
- const { handleServerTiming, contentTypeParser } = this.pluginOptions
197
+ const { path, byteRangeContext, options } = context
198
+ const { contentTypeParser } = this.pluginOptions
191
199
  const log = this.log
192
200
 
193
201
  // get the first chunk in order to determine the content type
@@ -203,7 +211,7 @@ export class DagPbPlugin extends BasePlugin {
203
211
  onProgress: options?.onProgress,
204
212
  signal: options?.signal
205
213
  })
206
- const contentType = await handleServerTiming('get-content-type', '', async () => getContentType({ bytes: firstChunk, path, contentTypeParser, log }), withServerTiming)
214
+ const contentType = await context.serverTiming.time('get-content-type', '', getContentType({ bytes: firstChunk, path, contentTypeParser, log }))
207
215
 
208
216
  byteRangeContext?.setBody((range): AsyncGenerator<Uint8Array, void, unknown> => {
209
217
  if (options?.signal?.aborted) {