@helia/verified-fetch 4.1.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +6 -40
  2. package/dist/index.min.js +73 -534
  3. package/dist/index.min.js.map +4 -4
  4. package/dist/src/constants.d.ts +2 -0
  5. package/dist/src/constants.d.ts.map +1 -1
  6. package/dist/src/constants.js +2 -0
  7. package/dist/src/constants.js.map +1 -1
  8. package/dist/src/index.d.ts +162 -68
  9. package/dist/src/index.d.ts.map +1 -1
  10. package/dist/src/index.js +11 -43
  11. package/dist/src/index.js.map +1 -1
  12. package/dist/src/plugins/index.d.ts +0 -5
  13. package/dist/src/plugins/index.d.ts.map +1 -1
  14. package/dist/src/plugins/index.js +0 -4
  15. package/dist/src/plugins/index.js.map +1 -1
  16. package/dist/src/plugins/plugin-base.d.ts +8 -9
  17. package/dist/src/plugins/plugin-base.d.ts.map +1 -1
  18. package/dist/src/plugins/plugin-base.js +5 -6
  19. package/dist/src/plugins/plugin-base.js.map +1 -1
  20. package/dist/src/plugins/plugin-handle-car.d.ts +3 -3
  21. package/dist/src/plugins/plugin-handle-car.d.ts.map +1 -1
  22. package/dist/src/plugins/plugin-handle-car.js +38 -39
  23. package/dist/src/plugins/plugin-handle-car.js.map +1 -1
  24. package/dist/src/plugins/plugin-handle-ipld.d.ts +12 -0
  25. package/dist/src/plugins/plugin-handle-ipld.d.ts.map +1 -0
  26. package/dist/src/plugins/plugin-handle-ipld.js +83 -0
  27. package/dist/src/plugins/plugin-handle-ipld.js.map +1 -0
  28. package/dist/src/plugins/plugin-handle-ipns-record.d.ts +3 -3
  29. package/dist/src/plugins/plugin-handle-ipns-record.d.ts.map +1 -1
  30. package/dist/src/plugins/plugin-handle-ipns-record.js +25 -34
  31. package/dist/src/plugins/plugin-handle-ipns-record.js.map +1 -1
  32. package/dist/src/plugins/plugin-handle-tar.d.ts +3 -3
  33. package/dist/src/plugins/plugin-handle-tar.d.ts.map +1 -1
  34. package/dist/src/plugins/plugin-handle-tar.js +20 -22
  35. package/dist/src/plugins/plugin-handle-tar.js.map +1 -1
  36. package/dist/src/plugins/plugin-handle-unixfs.d.ts +14 -0
  37. package/dist/src/plugins/plugin-handle-unixfs.d.ts.map +1 -0
  38. package/dist/src/plugins/plugin-handle-unixfs.js +180 -0
  39. package/dist/src/plugins/plugin-handle-unixfs.js.map +1 -0
  40. package/dist/src/plugins/types.d.ts +1 -77
  41. package/dist/src/plugins/types.d.ts.map +1 -1
  42. package/dist/src/url-resolver.d.ts +29 -11
  43. package/dist/src/url-resolver.d.ts.map +1 -1
  44. package/dist/src/url-resolver.js +152 -74
  45. package/dist/src/url-resolver.js.map +1 -1
  46. package/dist/src/utils/content-type-parser.d.ts.map +1 -1
  47. package/dist/src/utils/content-type-parser.js +4 -3
  48. package/dist/src/utils/content-type-parser.js.map +1 -1
  49. package/dist/src/utils/content-types.d.ts +26 -0
  50. package/dist/src/utils/content-types.d.ts.map +1 -0
  51. package/dist/src/utils/content-types.js +137 -0
  52. package/dist/src/utils/content-types.js.map +1 -0
  53. package/dist/src/utils/convert-output.d.ts +17 -0
  54. package/dist/src/utils/convert-output.d.ts.map +1 -0
  55. package/dist/src/utils/convert-output.js +176 -0
  56. package/dist/src/utils/convert-output.js.map +1 -0
  57. package/dist/src/utils/error-to-response.d.ts +3 -0
  58. package/dist/src/utils/error-to-response.d.ts.map +1 -0
  59. package/dist/src/utils/error-to-response.js +40 -0
  60. package/dist/src/utils/error-to-response.js.map +1 -0
  61. package/dist/src/utils/get-content-disposition-filename.d.ts +1 -1
  62. package/dist/src/utils/get-content-disposition-filename.d.ts.map +1 -1
  63. package/dist/src/utils/get-content-disposition-filename.js +4 -0
  64. package/dist/src/utils/get-content-disposition-filename.js.map +1 -1
  65. package/dist/src/utils/get-e-tag.d.ts +20 -15
  66. package/dist/src/utils/get-e-tag.d.ts.map +1 -1
  67. package/dist/src/utils/get-e-tag.js +8 -22
  68. package/dist/src/utils/get-e-tag.js.map +1 -1
  69. package/dist/src/utils/get-offset-and-length.d.ts +12 -2
  70. package/dist/src/utils/get-offset-and-length.d.ts.map +1 -1
  71. package/dist/src/utils/get-offset-and-length.js +63 -21
  72. package/dist/src/utils/get-offset-and-length.js.map +1 -1
  73. package/dist/src/utils/get-range-header.d.ts +22 -0
  74. package/dist/src/utils/get-range-header.d.ts.map +1 -0
  75. package/dist/src/utils/get-range-header.js +69 -0
  76. package/dist/src/utils/get-range-header.js.map +1 -0
  77. package/dist/src/utils/parse-url-string.d.ts +2 -1
  78. package/dist/src/utils/parse-url-string.d.ts.map +1 -1
  79. package/dist/src/utils/parse-url-string.js +46 -71
  80. package/dist/src/utils/parse-url-string.js.map +1 -1
  81. package/dist/src/utils/resource-to-cache-key.d.ts +3 -3
  82. package/dist/src/utils/resource-to-cache-key.js +5 -5
  83. package/dist/src/utils/resource-to-cache-key.js.map +1 -1
  84. package/dist/src/utils/response-headers.d.ts +4 -14
  85. package/dist/src/utils/response-headers.d.ts.map +1 -1
  86. package/dist/src/utils/response-headers.js +36 -36
  87. package/dist/src/utils/response-headers.js.map +1 -1
  88. package/dist/src/utils/responses.d.ts +30 -11
  89. package/dist/src/utils/responses.d.ts.map +1 -1
  90. package/dist/src/utils/responses.js +146 -39
  91. package/dist/src/utils/responses.js.map +1 -1
  92. package/dist/src/verified-fetch.d.ts +16 -15
  93. package/dist/src/verified-fetch.d.ts.map +1 -1
  94. package/dist/src/verified-fetch.js +307 -236
  95. package/dist/src/verified-fetch.js.map +1 -1
  96. package/dist/typedoc-urls.json +64 -45
  97. package/package.json +4 -3
  98. package/src/constants.ts +3 -0
  99. package/src/index.ts +203 -71
  100. package/src/plugins/index.ts +0 -6
  101. package/src/plugins/plugin-base.ts +8 -10
  102. package/src/plugins/plugin-handle-car.ts +48 -46
  103. package/src/plugins/plugin-handle-ipld.ts +93 -0
  104. package/src/plugins/plugin-handle-ipns-record.ts +31 -41
  105. package/src/plugins/plugin-handle-tar.ts +25 -29
  106. package/src/plugins/plugin-handle-unixfs.ts +217 -0
  107. package/src/plugins/types.ts +0 -86
  108. package/src/url-resolver.ts +197 -83
  109. package/src/utils/content-type-parser.ts +4 -3
  110. package/src/utils/content-types.ts +159 -0
  111. package/src/utils/convert-output.ts +187 -0
  112. package/src/utils/error-to-response.ts +49 -0
  113. package/src/utils/get-content-disposition-filename.ts +7 -1
  114. package/src/utils/get-e-tag.ts +26 -35
  115. package/src/utils/get-offset-and-length.ts +75 -21
  116. package/src/utils/get-range-header.ts +107 -0
  117. package/src/utils/parse-url-string.ts +51 -80
  118. package/src/utils/resource-to-cache-key.ts +5 -5
  119. package/src/utils/response-headers.ts +40 -41
  120. package/src/utils/responses.ts +186 -45
  121. package/src/verified-fetch.ts +359 -267
  122. package/dist/src/plugins/plugin-handle-byte-range-context.d.ts +0 -14
  123. package/dist/src/plugins/plugin-handle-byte-range-context.d.ts.map +0 -1
  124. package/dist/src/plugins/plugin-handle-byte-range-context.js +0 -25
  125. package/dist/src/plugins/plugin-handle-byte-range-context.js.map +0 -1
  126. package/dist/src/plugins/plugin-handle-cbor.d.ts +0 -17
  127. package/dist/src/plugins/plugin-handle-cbor.d.ts.map +0 -1
  128. package/dist/src/plugins/plugin-handle-cbor.js +0 -94
  129. package/dist/src/plugins/plugin-handle-cbor.js.map +0 -1
  130. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.d.ts +0 -27
  131. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.d.ts.map +0 -1
  132. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.js +0 -279
  133. package/dist/src/plugins/plugin-handle-dag-cbor-html-preview.js.map +0 -1
  134. package/dist/src/plugins/plugin-handle-dag-cbor.d.ts +0 -17
  135. package/dist/src/plugins/plugin-handle-dag-cbor.d.ts.map +0 -1
  136. package/dist/src/plugins/plugin-handle-dag-cbor.js +0 -66
  137. package/dist/src/plugins/plugin-handle-dag-cbor.js.map +0 -1
  138. package/dist/src/plugins/plugin-handle-dag-pb.d.ts +0 -17
  139. package/dist/src/plugins/plugin-handle-dag-pb.d.ts.map +0 -1
  140. package/dist/src/plugins/plugin-handle-dag-pb.js +0 -209
  141. package/dist/src/plugins/plugin-handle-dag-pb.js.map +0 -1
  142. package/dist/src/plugins/plugin-handle-dag-walk.d.ts +0 -21
  143. package/dist/src/plugins/plugin-handle-dag-walk.d.ts.map +0 -1
  144. package/dist/src/plugins/plugin-handle-dag-walk.js +0 -95
  145. package/dist/src/plugins/plugin-handle-dag-walk.js.map +0 -1
  146. package/dist/src/plugins/plugin-handle-dir-index-html.d.ts +0 -10
  147. package/dist/src/plugins/plugin-handle-dir-index-html.d.ts.map +0 -1
  148. package/dist/src/plugins/plugin-handle-dir-index-html.js +0 -59
  149. package/dist/src/plugins/plugin-handle-dir-index-html.js.map +0 -1
  150. package/dist/src/plugins/plugin-handle-json.d.ts +0 -12
  151. package/dist/src/plugins/plugin-handle-json.d.ts.map +0 -1
  152. package/dist/src/plugins/plugin-handle-json.js +0 -73
  153. package/dist/src/plugins/plugin-handle-json.js.map +0 -1
  154. package/dist/src/plugins/plugin-handle-raw.d.ts +0 -9
  155. package/dist/src/plugins/plugin-handle-raw.d.ts.map +0 -1
  156. package/dist/src/plugins/plugin-handle-raw.js +0 -92
  157. package/dist/src/plugins/plugin-handle-raw.js.map +0 -1
  158. package/dist/src/plugins/plugins.d.ts +0 -6
  159. package/dist/src/plugins/plugins.d.ts.map +0 -1
  160. package/dist/src/plugins/plugins.js +0 -6
  161. package/dist/src/plugins/plugins.js.map +0 -1
  162. package/dist/src/utils/byte-range-context.d.ts +0 -103
  163. package/dist/src/utils/byte-range-context.d.ts.map +0 -1
  164. package/dist/src/utils/byte-range-context.js +0 -504
  165. package/dist/src/utils/byte-range-context.js.map +0 -1
  166. package/dist/src/utils/dag-cbor-to-safe-json.d.ts +0 -15
  167. package/dist/src/utils/dag-cbor-to-safe-json.d.ts.map +0 -1
  168. package/dist/src/utils/dag-cbor-to-safe-json.js +0 -54
  169. package/dist/src/utils/dag-cbor-to-safe-json.js.map +0 -1
  170. package/dist/src/utils/dir-index-html.d.ts +0 -19
  171. package/dist/src/utils/dir-index-html.d.ts.map +0 -1
  172. package/dist/src/utils/dir-index-html.js +0 -438
  173. package/dist/src/utils/dir-index-html.js.map +0 -1
  174. package/dist/src/utils/get-peer-id-from-string.d.ts +0 -3
  175. package/dist/src/utils/get-peer-id-from-string.d.ts.map +0 -1
  176. package/dist/src/utils/get-peer-id-from-string.js +0 -10
  177. package/dist/src/utils/get-peer-id-from-string.js.map +0 -1
  178. package/dist/src/utils/get-resolved-accept-header.d.ts +0 -9
  179. package/dist/src/utils/get-resolved-accept-header.d.ts.map +0 -1
  180. package/dist/src/utils/get-resolved-accept-header.js +0 -27
  181. package/dist/src/utils/get-resolved-accept-header.js.map +0 -1
  182. package/dist/src/utils/get-stream-from-async-iterable.d.ts +0 -9
  183. package/dist/src/utils/get-stream-from-async-iterable.d.ts.map +0 -1
  184. package/dist/src/utils/get-stream-from-async-iterable.js +0 -43
  185. package/dist/src/utils/get-stream-from-async-iterable.js.map +0 -1
  186. package/dist/src/utils/handle-redirects.d.ts +0 -16
  187. package/dist/src/utils/handle-redirects.d.ts.map +0 -1
  188. package/dist/src/utils/handle-redirects.js +0 -84
  189. package/dist/src/utils/handle-redirects.js.map +0 -1
  190. package/dist/src/utils/is-accept-explicit.d.ts +0 -15
  191. package/dist/src/utils/is-accept-explicit.d.ts.map +0 -1
  192. package/dist/src/utils/is-accept-explicit.js +0 -26
  193. package/dist/src/utils/is-accept-explicit.js.map +0 -1
  194. package/dist/src/utils/request-headers.d.ts +0 -13
  195. package/dist/src/utils/request-headers.d.ts.map +0 -1
  196. package/dist/src/utils/request-headers.js +0 -63
  197. package/dist/src/utils/request-headers.js.map +0 -1
  198. package/dist/src/utils/select-output-type.d.ts +0 -17
  199. package/dist/src/utils/select-output-type.d.ts.map +0 -1
  200. package/dist/src/utils/select-output-type.js +0 -153
  201. package/dist/src/utils/select-output-type.js.map +0 -1
  202. package/dist/src/utils/tlru.d.ts +0 -15
  203. package/dist/src/utils/tlru.d.ts.map +0 -1
  204. package/dist/src/utils/tlru.js +0 -34
  205. package/dist/src/utils/tlru.js.map +0 -1
  206. package/dist/src/utils/walk-path.d.ts +0 -27
  207. package/dist/src/utils/walk-path.d.ts.map +0 -1
  208. package/dist/src/utils/walk-path.js +0 -45
  209. package/dist/src/utils/walk-path.js.map +0 -1
  210. package/src/plugins/plugin-handle-byte-range-context.ts +0 -30
  211. package/src/plugins/plugin-handle-cbor.ts +0 -107
  212. package/src/plugins/plugin-handle-dag-cbor-html-preview.ts +0 -295
  213. package/src/plugins/plugin-handle-dag-cbor.ts +0 -83
  214. package/src/plugins/plugin-handle-dag-pb.ts +0 -248
  215. package/src/plugins/plugin-handle-dag-walk.ts +0 -110
  216. package/src/plugins/plugin-handle-dir-index-html.ts +0 -72
  217. package/src/plugins/plugin-handle-json.ts +0 -80
  218. package/src/plugins/plugin-handle-raw.ts +0 -110
  219. package/src/plugins/plugins.ts +0 -5
  220. package/src/utils/byte-range-context.ts +0 -597
  221. package/src/utils/dag-cbor-to-safe-json.ts +0 -63
  222. package/src/utils/dir-index-html.ts +0 -505
  223. package/src/utils/get-peer-id-from-string.ts +0 -12
  224. package/src/utils/get-resolved-accept-header.ts +0 -42
  225. package/src/utils/get-stream-from-async-iterable.ts +0 -49
  226. package/src/utils/handle-redirects.ts +0 -109
  227. package/src/utils/is-accept-explicit.ts +0 -38
  228. package/src/utils/request-headers.ts +0 -65
  229. package/src/utils/select-output-type.ts +0 -175
  230. package/src/utils/tlru.ts +0 -42
  231. package/src/utils/walk-path.ts +0 -68
@@ -0,0 +1,217 @@
1
+ import { code as dagPbCode } from '@ipld/dag-pb'
2
+ import { isPromise } from '@libp2p/utils'
3
+ import { exporter } from 'ipfs-unixfs-exporter'
4
+ import first from 'it-first'
5
+ import itToBrowserReadableStream from 'it-to-browser-readablestream'
6
+ import toBuffer from 'it-to-buffer'
7
+ import * as raw from 'multiformats/codecs/raw'
8
+ import { MEDIA_TYPE_OCTET_STREAM, MEDIA_TYPE_DAG_PB } from '../utils/content-types.ts'
9
+ import { getContentDispositionFilename } from '../utils/get-content-disposition-filename.ts'
10
+ import { badGatewayResponse, movedPermanentlyResponse, partialContentResponse, okResponse } from '../utils/responses.js'
11
+ import { BasePlugin } from './plugin-base.js'
12
+ import type { PluginContext } from '../index.js'
13
+ import type { RangeHeader } from '../utils/get-range-header.ts'
14
+ import type { AbortOptions } from '@libp2p/interface'
15
+ import type { IdentityNode, RawNode, UnixFSEntry, UnixFSFile } from 'ipfs-unixfs-exporter'
16
+
17
+ /**
18
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#use-in-directory-url-normalization
19
+ */
20
+ function getRedirectUrl (resource: string, url: URL, terminalElement: UnixFSEntry): string | undefined {
21
+ let uri: URL
22
+
23
+ try {
24
+ // try the requested resource
25
+ uri = new URL(resource)
26
+ } catch {
27
+ // fall back to the canonical URL
28
+ uri = url
29
+ }
30
+
31
+ // directories must be requested with a trailing slash
32
+ if (terminalElement?.type === 'directory' && !uri.pathname.endsWith('/')) {
33
+ // make sure we append slash to end of the path
34
+ uri.pathname += '/'
35
+
36
+ return uri.toString()
37
+ }
38
+ }
39
+
40
+ /**
41
+ * Handles UnixFS content
42
+ */
43
+ export class UnixFSPlugin extends BasePlugin {
44
+ readonly id = 'unixfs-plugin'
45
+ readonly codes = [dagPbCode, raw.code]
46
+
47
+ canHandle ({ terminalElement, accept }: PluginContext): boolean {
48
+ const supportsCid = this.codes.includes(terminalElement.cid.code)
49
+ const supportsAccept = accept.length === 0 || accept.some(header => header.contentType.mediaType === MEDIA_TYPE_OCTET_STREAM ||
50
+ header.contentType.mediaType === MEDIA_TYPE_DAG_PB
51
+ )
52
+
53
+ return supportsCid && supportsAccept
54
+ }
55
+
56
+ async handle (context: PluginContext): Promise<Response> {
57
+ let { url, resource, terminalElement, ipfsRoots } = context
58
+ let filename = url.searchParams.get('filename') ?? terminalElement.name
59
+ let redirected: undefined | true
60
+
61
+ if (terminalElement.type === 'directory') {
62
+ const redirectUrl = getRedirectUrl(resource, url, terminalElement)
63
+
64
+ if (redirectUrl != null) {
65
+ this.log.trace('directory url normalization spec requires redirect')
66
+
67
+ if (context.options?.redirect === 'error') {
68
+ this.log('could not redirect to %s as redirect option was set to "error"', redirectUrl)
69
+ throw new TypeError('Failed to fetch')
70
+ } else if (context.options?.redirect === 'manual') {
71
+ this.log('returning 301 permanent redirect to %s', redirectUrl)
72
+ return movedPermanentlyResponse(context.resource, redirectUrl)
73
+ }
74
+
75
+ this.log('following redirect to %s', redirectUrl)
76
+
77
+ // fall-through simulates following the redirect?
78
+ resource = redirectUrl
79
+ redirected = true
80
+ }
81
+
82
+ const dirCid = terminalElement.cid
83
+
84
+ // if not disabled, search the directory for an index.html file
85
+ if (context.options?.supportDirectoryIndexes !== false) {
86
+ const rootFilePath = 'index.html'
87
+
88
+ try {
89
+ this.log.trace('found directory at %c/%s, looking for index.html', dirCid, url.pathname)
90
+
91
+ const entry = await context.serverTiming.time('exporter-dir', '', exporter(`/ipfs/${dirCid}/${rootFilePath}`, context.blockstore, context.options))
92
+
93
+ if (entry.type === 'directory' || entry.type === 'object') {
94
+ return badGatewayResponse(resource, 'Unable to stream content')
95
+ }
96
+
97
+ // use `index.html` as the file name to help with content types
98
+ filename = rootFilePath
99
+
100
+ this.log.trace('found directory index at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
101
+
102
+ return await this.streamFile(resource, entry, filename, redirected, context.range, context.options)
103
+ } catch (err: any) {
104
+ if (err.name !== 'NotFoundError') {
105
+ this.log.error('error loading path %c/%s - %e', dirCid, rootFilePath, err)
106
+ throw err
107
+ }
108
+ }
109
+ }
110
+
111
+ // no index file found, return the directory listing
112
+ const block = await toBuffer(context.blockstore.get(dirCid, context.options))
113
+
114
+ return okResponse(resource, block, {
115
+ headers: {
116
+ 'content-type': MEDIA_TYPE_DAG_PB,
117
+ 'content-length': `${block.byteLength}`,
118
+ 'content-disposition': `${url.searchParams.get('download') === 'true' ? 'attachment' : 'inline'}; ${
119
+ getContentDispositionFilename(`${dirCid}.dir`)
120
+ }`,
121
+ 'x-ipfs-roots': ipfsRoots.map(cid => cid.toV1()).join(','),
122
+ 'accept-ranges': 'bytes'
123
+ },
124
+ redirected
125
+ })
126
+ } else if (terminalElement.type === 'file' || terminalElement.type === 'raw' || terminalElement.type === 'identity') {
127
+ this.log('streaming file')
128
+ return this.streamFile(resource, terminalElement, filename, redirected, context.range, context.options)
129
+ } else {
130
+ this.log.error('cannot stream terminal element type %s', terminalElement.type)
131
+ return badGatewayResponse(resource, 'Unable to stream content')
132
+ }
133
+ }
134
+
135
+ private async streamFile (resource: string, entry: UnixFSFile | RawNode | IdentityNode, filename: string, redirected?: boolean, rangeHeader?: RangeHeader, options?: AbortOptions): Promise<Response> {
136
+ let contentType = MEDIA_TYPE_OCTET_STREAM
137
+
138
+ // only detect content type for non-range requests to avoid loading blocks
139
+ // we aren't going to stream to the user
140
+ if (rangeHeader == null) {
141
+ contentType = await this.detectContentType(entry, filename, options)
142
+ }
143
+
144
+ if (rangeHeader != null) {
145
+ return partialContentResponse(resource, (offset, length) => {
146
+ return entry.content({
147
+ ...(options ?? {}),
148
+ offset,
149
+ length
150
+ })
151
+ }, rangeHeader, entry.size, {
152
+ headers: {
153
+ 'content-type': contentType,
154
+ 'content-disposition': `inline; ${
155
+ getContentDispositionFilename(filename)
156
+ }`,
157
+ 'x-ipfs-roots': entry.cid.toString(),
158
+ 'accept-ranges': 'bytes'
159
+ },
160
+ redirected
161
+ })
162
+ }
163
+
164
+ // nb. if streaming the output fails (network error, unresolvable block,
165
+ // etc), a "TypeError: Failed to fetch" error will occur
166
+ return okResponse(resource, itToBrowserReadableStream(entry.content(options)), {
167
+ headers: {
168
+ 'content-type': contentType,
169
+ 'content-length': `${entry.size}`,
170
+ 'content-disposition': `inline; ${
171
+ getContentDispositionFilename(filename)
172
+ }`,
173
+ 'x-ipfs-roots': entry.cid.toString(),
174
+ 'accept-ranges': 'bytes'
175
+ },
176
+ redirected
177
+ })
178
+ }
179
+
180
+ private async detectContentType (entry: UnixFSFile | RawNode | IdentityNode, filename?: string, options?: AbortOptions): Promise<string> {
181
+ let buf: Uint8Array | undefined
182
+
183
+ if (entry.type === 'raw' || entry.type === 'identity') {
184
+ buf = entry.node
185
+ } else {
186
+ // read the first block of the file
187
+ buf = await first(entry.content(options))
188
+ }
189
+
190
+ if (buf == null) {
191
+ throw new Error('stream ended before first block was read')
192
+ }
193
+
194
+ let contentType: string | undefined
195
+
196
+ if (this.pluginOptions.contentTypeParser != null) {
197
+ try {
198
+ const parsed = this.pluginOptions.contentTypeParser(buf, filename)
199
+
200
+ if (isPromise(parsed)) {
201
+ const result = await parsed
202
+
203
+ if (result != null) {
204
+ contentType = result
205
+ }
206
+ } else if (parsed != null) {
207
+ contentType = parsed
208
+ }
209
+ this.log.trace('contentTypeParser returned %s for file with name %s', contentType, filename)
210
+ } catch (err) {
211
+ this.log.error('error parsing content type - %e', err)
212
+ }
213
+ }
214
+
215
+ return contentType ?? MEDIA_TYPE_OCTET_STREAM
216
+ }
217
+ }
@@ -1,86 +0,0 @@
1
- import type { ResolveURLResult, VerifiedFetchInit, ContentTypeParser, RequestFormatShorthand } from '../index.js'
2
- import type { ByteRangeContext } from '../utils/byte-range-context.js'
3
- import type { AcceptHeader } from '../utils/select-output-type.ts'
4
- import type { ServerTiming } from '../utils/server-timing.ts'
5
- import type { PathWalkerResponse } from '../utils/walk-path.js'
6
- import type { IPNSResolver } from '@helia/ipns'
7
- import type { AbortOptions, Logger } from '@libp2p/interface'
8
- import type { Helia } from 'helia'
9
- import type { Blockstore } from 'interface-blockstore'
10
- import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
11
- import type { CID } from 'multiformats/cid'
12
- import type { CustomProgressEvent } from 'progress-events'
13
-
14
- /**
15
- * Contains common components and functions required by plugins to handle a request.
16
- * - Read-Only: Plugins can read but shouldn't rewrite them.
17
- * - Persistent: Relevant even after the request completes (e.g., logging or metrics).
18
- */
19
- export interface PluginOptions {
20
- logger: Logger
21
- getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions): Blockstore
22
- contentTypeParser?: ContentTypeParser
23
- helia: Helia
24
- ipnsResolver: IPNSResolver
25
- }
26
-
27
- /**
28
- * Represents the ephemeral, modifiable state used by the pipeline.
29
- * - Mutable: Evolves as you walk the plugin chain.
30
- * - Shared Data: Allows plugins to communicate partial results, discovered data, or interim errors.
31
- * - Ephemeral: Typically discarded once fetch(...) completes.
32
- */
33
- export interface PluginContext extends ResolveURLResult {
34
- readonly resource: string
35
- readonly accept?: AcceptHeader
36
-
37
- /**
38
- * An array of plugin IDs that are all enabled. You can use this to check if a plugin is enabled and respond accordingly.
39
- */
40
- plugins: string[]
41
-
42
- /**
43
- * The last time the context is modified, so we know whether a plugin has modified it.
44
- * A plugin should increment this value if it modifies the context.
45
- */
46
- modified: number
47
- onProgress?(evt: CustomProgressEvent<any>): void
48
- options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
49
- isDirectory?: boolean
50
- directoryEntries?: UnixFSEntry[]
51
- reqFormat?: RequestFormatShorthand
52
- pathDetails?: PathWalkerResponse
53
-
54
- /**
55
- * ByteRangeContext contains information about the size of the content and range requests.
56
- * This can be used to set the Content-Length header without loading the entire body.
57
- *
58
- * This is set by the ByteRangeContextPlugin
59
- */
60
- byteRangeContext?: ByteRangeContext
61
- serverTiming: ServerTiming
62
- ipfsPath: string
63
-
64
- /**
65
- * Allow arbitrary keys/values
66
- */
67
- [key: string]: unknown
68
- }
69
-
70
- export interface VerifiedFetchPlugin {
71
- readonly id: string
72
- readonly codes: number[]
73
- readonly log: Logger
74
- canHandle (context: PluginContext): boolean
75
- handle (context: PluginContext): Promise<Response | null>
76
- }
77
-
78
- export interface VerifiedFetchPluginFactory {
79
- (options: PluginOptions): VerifiedFetchPlugin
80
- }
81
-
82
- export interface PluginErrorOptions {
83
- fatal?: boolean
84
- details?: Record<string, any>
85
- response?: Response
86
- }
@@ -1,100 +1,145 @@
1
- import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id'
1
+ import { DoesNotExistError } from '@helia/unixfs/errors'
2
+ import * as dagCbor from '@ipld/dag-cbor'
3
+ import * as dagJson from '@ipld/dag-json'
4
+ import * as dagPb from '@ipld/dag-pb'
5
+ import { peerIdFromString } from '@libp2p/peer-id'
6
+ import { InvalidParametersError, walkPath } from 'ipfs-unixfs-exporter'
7
+ import toBuffer from 'it-to-buffer'
2
8
  import { CID } from 'multiformats/cid'
3
- import { parseURLString } from './utils/parse-url-string.ts'
4
- import type { ResolveURLOptions, ResolveURLResult, Resource, URLResolver as URLResolverInterface } from './index.ts'
5
- import type { ParsedURL } from './utils/parse-url-string.ts'
6
- import type { ServerTiming } from './utils/server-timing.ts'
9
+ import * as json from 'multiformats/codecs/json'
10
+ import * as raw from 'multiformats/codecs/raw'
11
+ import QuickLRU from 'quick-lru'
12
+ import { SESSION_CACHE_MAX_SIZE, SESSION_CACHE_TTL_MS, CODEC_CBOR, CODEC_IDENTITY } from './constants.ts'
13
+ import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.ts'
14
+ import { ServerTiming } from './utils/server-timing.ts'
15
+ import type { ResolveURLResult, URLResolver as URLResolverInterface } from './index.ts'
7
16
  import type { DNSLink } from '@helia/dnslink'
8
17
  import type { IPNSResolver } from '@helia/ipns'
9
- import type { AbortOptions, PeerId } from '@libp2p/interface'
18
+ import type { AbortOptions } from '@libp2p/interface'
19
+ import type { Helia, SessionBlockstore } from 'helia'
20
+ import type { Blockstore } from 'interface-blockstore'
21
+ import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
22
+
23
+ // 1 year in seconds for ipfs content
24
+ const IPFS_CONTENT_TTL = 29030400
25
+
26
+ const ENTITY_CODECS = [
27
+ CODEC_CBOR,
28
+ json.code,
29
+ raw.code
30
+ ]
31
+
32
+ /**
33
+ * These are supported by the UnixFS exporter
34
+ */
35
+ const EXPORTABLE_CODECS = [
36
+ dagPb.code,
37
+ dagCbor.code,
38
+ dagJson.code,
39
+ raw.code
40
+ ]
41
+
42
+ interface GetBlockstoreOptions extends AbortOptions {
43
+ session?: boolean
44
+ }
10
45
 
11
- const CODEC_LIBP2P_KEY = 0x72
46
+ export interface WalkPathResult {
47
+ ipfsRoots: CID[]
48
+ terminalElement: UnixFSEntry
49
+ blockstore: Blockstore
50
+ }
51
+
52
+ function basicEntry (type: 'raw' | 'object', cid: CID, bytes: Uint8Array): UnixFSEntry {
53
+ return {
54
+ name: cid.toString(),
55
+ path: cid.toString(),
56
+ depth: 0,
57
+ type,
58
+ node: bytes,
59
+ cid,
60
+ size: BigInt(bytes.byteLength),
61
+ content: async function * () {
62
+ yield bytes
63
+ }
64
+ }
65
+ }
12
66
 
13
67
  export interface URLResolverComponents {
68
+ helia: Helia
14
69
  ipnsResolver: IPNSResolver
15
70
  dnsLink: DNSLink
16
- timing: ServerTiming
71
+ }
72
+
73
+ export interface URLResolverInit {
74
+ sessionCacheSize?: number
75
+ sessionTTLms?: number
76
+ }
77
+
78
+ export interface ResolveURLOptions extends AbortOptions {
79
+ session?: boolean
80
+ isRawBlockRequest?: boolean
81
+ onlyIfCached?: boolean
17
82
  }
18
83
 
19
84
  export class URLResolver implements URLResolverInterface {
20
85
  private readonly components: URLResolverComponents
86
+ private readonly blockstoreSessions: QuickLRU<string, SessionBlockstore>
21
87
 
22
- constructor (components: URLResolverComponents) {
88
+ constructor (components: URLResolverComponents, init: URLResolverInit = {}) {
23
89
  this.components = components
90
+
91
+ this.blockstoreSessions = new QuickLRU({
92
+ maxSize: init.sessionCacheSize ?? SESSION_CACHE_MAX_SIZE,
93
+ maxAge: init.sessionTTLms ?? SESSION_CACHE_TTL_MS,
94
+ onEviction: (key, store) => {
95
+ store.close()
96
+ }
97
+ })
24
98
  }
25
99
 
26
- async resolve (resource: Resource, options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
27
- if (typeof resource === 'string') {
28
- return this.parseUrlString(resource, options)
100
+ async resolve (url: URL, serverTiming: ServerTiming = new ServerTiming(), options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
101
+ if (url.protocol === 'ipfs:') {
102
+ return this.resolveIPFSPath(url, serverTiming, options)
29
103
  }
30
104
 
31
- const cid = CID.asCID(resource)
32
-
33
- if (cid != null) {
34
- return this.resolveCIDResource(cid, {
35
- url: new URL(`ipfs://${cid}`)
36
- }, options)
105
+ if (url.protocol === 'ipns:') {
106
+ return this.resolveIPNSName(url, serverTiming, options)
37
107
  }
38
108
 
39
- throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`)
40
- }
41
-
42
- async parseUrlString (urlString: string, options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
43
- const result = parseURLString(urlString)
44
-
45
- if (result.protocol === 'ipfs') {
46
- const cid = CID.parse(result.cidOrPeerIdOrDnsLink)
47
-
48
- return this.resolveCIDResource(cid, result, options)
109
+ if (url.protocol === 'dnslink:') {
110
+ return this.resolveDNSLink(url, serverTiming, options)
49
111
  }
50
112
 
51
- if (result.protocol === 'ipns') {
52
- // try to parse target as peer id
53
- let peerId: PeerId
54
-
55
- try {
56
- peerId = peerIdFromString(result.cidOrPeerIdOrDnsLink)
57
- } catch {
58
- // fall back to DNSLink (e.g. /ipns/example.com)
59
- return this.resolveDNSLink(result.cidOrPeerIdOrDnsLink, result, options)
60
- }
113
+ throw new InvalidParametersError(`Invalid resource. Unsupported protocol in URL, must be ipfs:, ipns:, or dnslink: ${url}`)
114
+ }
61
115
 
62
- // parse multihash from string (e.g. /ipns/QmFoo...)
63
- return this.resolveIPNSName(result.cidOrPeerIdOrDnsLink, peerId, result, options)
116
+ private getBlockstore (root: CID, options: GetBlockstoreOptions = {}): Blockstore {
117
+ if (options.session === false) {
118
+ return this.components.helia.blockstore
64
119
  }
65
120
 
66
- throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${urlString}`)
67
- }
121
+ const key = resourceToSessionCacheKey(root)
122
+ let session = this.blockstoreSessions.get(key)
68
123
 
69
- async resolveCIDResource (cid: CID, parsed: Partial<ParsedURL> & Pick<ParsedURL, 'url'>, options: ResolveURLOptions = {}): Promise<ResolveURLResult> {
70
- if (cid.code === CODEC_LIBP2P_KEY) {
71
- // special case - peer id encoded as a CID
72
- return this.resolveIPNSName(cid.toString(), peerIdFromCID(cid), parsed, options)
124
+ if (session == null) {
125
+ session = this.components.helia.blockstore.createSession(root, options)
126
+ this.blockstoreSessions.set(key, session)
73
127
  }
74
128
 
75
- return {
76
- url: parsed.url,
77
- cid,
78
- protocol: 'ipfs',
79
- query: parsed.query ?? {},
80
- path: parsed.path ?? [],
81
- fragment: parsed.fragment ?? '',
82
- ttl: 29030400, // 1 year for ipfs content
83
- ipfsPath: `/ipfs/${cid}${parsed.url.pathname}`
84
- }
129
+ return session
85
130
  }
86
131
 
87
- async resolveDNSLink (domain: string, parsed: ParsedURL, options?: ResolveURLOptions): Promise<ResolveURLResult> {
88
- const results = await this.components.timing.time('dnsLink.resolve', `Resolve DNSLink ${domain}`, this.components.dnsLink.resolve(domain, options))
132
+ private async resolveDNSLink (url: URL, serverTiming: ServerTiming, options?: ResolveURLOptions): Promise<ResolveURLResult> {
133
+ const results = await serverTiming.time('dnsLink.resolve', `Resolve DNSLink ${url.hostname}`, this.components.dnsLink.resolve(url.hostname, options))
89
134
  const result = results?.[0]
90
135
 
91
136
  if (result == null) {
92
- throw new TypeError(`Invalid resource. Cannot resolve DNSLink from domain: ${domain}`)
137
+ throw new TypeError(`Invalid resource. Cannot resolve DNSLink from domain: ${url.hostname}`)
93
138
  }
94
139
 
95
140
  // dnslink resolved to IPNS name
96
141
  if (result.namespace === 'ipns') {
97
- return this.resolveIPNSName(domain, result.peerId, parsed, options)
142
+ return this.resolveIPNSName(url, serverTiming, options)
98
143
  }
99
144
 
100
145
  // dnslink resolved to CID
@@ -103,38 +148,107 @@ export class URLResolver implements URLResolverInterface {
103
148
  throw new TypeError(`Invalid resource. Unexpected DNSLink namespace ${result.namespace} from domain: ${domain}`)
104
149
  }
105
150
 
151
+ if (result.path != null && url.pathname !== '') {
152
+ // path conflict?
153
+ }
154
+
155
+ const ipfsUrl = new URL(`ipfs://${result.cid}/${url.pathname}`)
156
+ const ipfsResult = await this.resolveIPFSPath(ipfsUrl, serverTiming, options)
157
+
106
158
  return {
107
- url: parsed.url,
108
- cid: result.cid,
109
- path: concatPaths(...(result.path ?? '').split('/'), ...(parsed.path ?? [])),
110
- fragment: parsed.fragment,
111
- // dnslink is mutable so return 'ipns' protocol so we do not include immutable in cache-control header
112
- protocol: 'ipns',
113
- ttl: result.answer.TTL,
114
- query: parsed.query,
115
- ipfsPath: `/ipns/${domain}${parsed.url.pathname}`
159
+ ...ipfsResult,
160
+ url,
161
+ ttl: result.answer.TTL
116
162
  }
117
163
  }
118
164
 
119
- async resolveIPNSName (resource: string, key: PeerId, parsed: Partial<ParsedURL> & Pick<ParsedURL, 'url'>, options?: AbortOptions): Promise<ResolveURLResult> {
120
- const result = await this.components.timing.time('ipns.resolve', `Resolve IPNS name ${key}`, this.components.ipnsResolver.resolve(key, options))
165
+ private async resolveIPNSName (url: URL, serverTiming: ServerTiming, options?: ResolveURLOptions): Promise<ResolveURLResult> {
166
+ const peerId = peerIdFromString(url.hostname)
167
+ const result = await serverTiming.time('ipns.resolve', `Resolve IPNS name ${peerId}`, this.components.ipnsResolver.resolve(peerId, options))
168
+
169
+ if (result.path != null && url.pathname !== '') {
170
+ // path conflict?
171
+ }
172
+
173
+ const ipfsUrl = new URL(`ipfs://${result.cid}/${url.pathname}`)
174
+ const ipfsResult = await this.resolveIPFSPath(ipfsUrl, serverTiming, options)
121
175
 
122
176
  return {
123
- url: parsed.url,
124
- cid: result.cid,
125
- path: concatPaths(...(result.path ?? '').split('/'), ...(parsed.path ?? [])),
126
- query: parsed.query ?? {},
127
- fragment: parsed.fragment ?? '',
128
- protocol: 'ipns',
177
+ ...ipfsResult,
178
+ url,
129
179
  // IPNS ttl is in nanoseconds, convert to seconds
130
- ttl: Number((result.record.ttl ?? 0n) / BigInt(1e9)),
131
- ipfsPath: `/ipns/${resource}${parsed.url.pathname}`
180
+ ttl: Number((result.record.ttl ?? 0n) / BigInt(1e9))
181
+ }
182
+ }
183
+
184
+ private async resolveIPFSPath (url: URL, serverTiming: ServerTiming, options?: ResolveURLOptions): Promise<ResolveURLResult> {
185
+ const walkPathResult = await serverTiming.time('ipfs.resolve', '', this.walkPath(url, options))
186
+
187
+ return {
188
+ ...walkPathResult,
189
+ url,
190
+ ttl: IPFS_CONTENT_TTL,
191
+ blockstore: walkPathResult.blockstore
192
+ }
193
+ }
194
+
195
+ private async walkPath (url: URL, options: ResolveURLOptions = {}): Promise<WalkPathResult> {
196
+ const cid = CID.parse(url.hostname)
197
+ const blockstore = this.getBlockstore(cid, options)
198
+
199
+ if (EXPORTABLE_CODECS.includes(cid.code)) {
200
+ const ipfsRoots: CID[] = []
201
+ let terminalElement: UnixFSEntry | undefined
202
+ const ipfsPath = toIPFSPath(url)
203
+
204
+ // @ts-expect-error offline is a helia option
205
+ for await (const entry of walkPath(ipfsPath, blockstore, {
206
+ ...options,
207
+ offline: options.onlyIfCached === true,
208
+ extended: options.isRawBlockRequest !== true
209
+ })) {
210
+ ipfsRoots.push(entry.cid)
211
+ terminalElement = entry
212
+ }
213
+
214
+ if (terminalElement == null) {
215
+ throw new DoesNotExistError('No terminal element found')
216
+ }
217
+
218
+ return {
219
+ ipfsRoots,
220
+ terminalElement,
221
+ blockstore
222
+ }
223
+ }
224
+
225
+ let bytes: Uint8Array
226
+
227
+ if (cid.multihash.code === CODEC_IDENTITY) {
228
+ bytes = cid.multihash.digest
229
+ } else {
230
+ bytes = await toBuffer(blockstore.get(cid, options))
231
+ }
232
+
233
+ // entity codecs contain all the bytes for an entity in one block and no
234
+ // path walking outside of that block is possible
235
+ if (ENTITY_CODECS.includes(cid.code)) {
236
+ return {
237
+ ipfsRoots: [cid],
238
+ terminalElement: basicEntry('object', cid, bytes),
239
+ blockstore
240
+ }
241
+ }
242
+
243
+ // may be an unknown codec
244
+ return {
245
+ ipfsRoots: [cid],
246
+ terminalElement: basicEntry('raw', cid, bytes),
247
+ blockstore
132
248
  }
133
249
  }
134
250
  }
135
251
 
136
- function concatPaths (...paths: Array<string | undefined>): string[] {
137
- // @ts-expect-error undefined is filtered out
138
- return paths
139
- .filter(p => p != null && p !== '')
252
+ function toIPFSPath (url: URL): string {
253
+ return `/ipfs/${url.hostname}${decodeURI(url.pathname)}`
140
254
  }
@@ -39,7 +39,8 @@ export async function contentTypeParser (bytes: Uint8Array, fileName?: string):
39
39
  return detectedType
40
40
  }
41
41
 
42
- if (fileName == null) {
42
+ // no filename or filename has no extension
43
+ if (fileName == null || fileName.includes('.') === false) {
43
44
  // it's likely text... no other way to determine file-type.
44
45
  const text = getText(bytes)
45
46
 
@@ -59,7 +60,7 @@ export async function contentTypeParser (bytes: Uint8Array, fileName?: string):
59
60
  return defaultMimeType
60
61
  }
61
62
 
62
- // no need to include file-types listed at https://github.com/SgtPooki/file-type#supported-file-types
63
+ // no need to include file-types listed at https://github.com/sindresorhus/file-type#supported-file-types
63
64
  switch (fileName.split('.').pop()) {
64
65
  case 'css':
65
66
  return 'text/css'
@@ -73,7 +74,7 @@ export async function contentTypeParser (bytes: Uint8Array, fileName?: string):
73
74
  return 'text/plain'
74
75
  case 'woff2':
75
76
  return 'font/woff2'
76
- // see bottom of https://github.com/SgtPooki/file-type#supported-file-types
77
+ // see bottom of https://github.com/sindresorhus/file-type#supported-file-types
77
78
  case 'svg':
78
79
  return 'image/svg+xml'
79
80
  case 'csv':