@j0hanz/superfetch 2.1.3 → 2.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (550) hide show
  1. package/dist/cache.d.ts +1 -0
  2. package/dist/cache.d.ts.map +1 -0
  3. package/dist/cache.js +37 -6
  4. package/dist/cache.js.map +1 -0
  5. package/dist/config.d.ts +1 -0
  6. package/dist/config.d.ts.map +1 -0
  7. package/dist/config.js +1 -0
  8. package/dist/config.js.map +1 -0
  9. package/dist/crypto.d.ts +1 -0
  10. package/dist/crypto.d.ts.map +1 -0
  11. package/dist/crypto.js +1 -0
  12. package/dist/crypto.js.map +1 -0
  13. package/dist/errors.d.ts +1 -0
  14. package/dist/errors.d.ts.map +1 -0
  15. package/dist/errors.js +1 -0
  16. package/dist/errors.js.map +1 -0
  17. package/dist/fetch.d.ts +1 -0
  18. package/dist/fetch.d.ts.map +1 -0
  19. package/dist/fetch.js +1 -0
  20. package/dist/fetch.js.map +1 -0
  21. package/dist/http.d.ts +2 -0
  22. package/dist/http.d.ts.map +1 -0
  23. package/dist/http.js +34 -5
  24. package/dist/http.js.map +1 -0
  25. package/dist/index.d.ts +1 -0
  26. package/dist/index.js +1 -0
  27. package/dist/index.js.map +1 -1
  28. package/dist/mcp.d.ts +1 -0
  29. package/dist/mcp.d.ts.map +1 -0
  30. package/dist/mcp.js +1 -0
  31. package/dist/mcp.js.map +1 -0
  32. package/dist/observability.d.ts +1 -0
  33. package/dist/observability.d.ts.map +1 -0
  34. package/dist/observability.js +1 -0
  35. package/dist/observability.js.map +1 -0
  36. package/dist/tools.d.ts +1 -0
  37. package/dist/tools.d.ts.map +1 -0
  38. package/dist/tools.js +1 -0
  39. package/dist/tools.js.map +1 -0
  40. package/dist/transform.d.ts +1 -0
  41. package/dist/transform.d.ts.map +1 -0
  42. package/dist/transform.js +1 -0
  43. package/dist/transform.js.map +1 -0
  44. package/dist/utils.d.ts +1 -0
  45. package/dist/utils.d.ts.map +1 -0
  46. package/dist/utils.js +1 -0
  47. package/dist/utils.js.map +1 -0
  48. package/dist/workers/transform-worker.d.ts +1 -0
  49. package/dist/workers/transform-worker.d.ts.map +1 -0
  50. package/dist/workers/transform-worker.js +1 -0
  51. package/dist/workers/transform-worker.js.map +1 -0
  52. package/package.json +8 -5
  53. package/dist/config/auth-config.d.ts +0 -16
  54. package/dist/config/auth-config.js +0 -53
  55. package/dist/config/constants.d.ts +0 -17
  56. package/dist/config/constants.d.ts.map +0 -1
  57. package/dist/config/constants.js +0 -22
  58. package/dist/config/constants.js.map +0 -1
  59. package/dist/config/env-parsers.d.ts +0 -8
  60. package/dist/config/env-parsers.js +0 -96
  61. package/dist/config/formatting.d.ts +0 -7
  62. package/dist/config/formatting.d.ts.map +0 -1
  63. package/dist/config/formatting.js +0 -9
  64. package/dist/config/formatting.js.map +0 -1
  65. package/dist/config/index.d.ts +0 -61
  66. package/dist/config/index.d.ts.map +0 -1
  67. package/dist/config/index.js +0 -107
  68. package/dist/config/index.js.map +0 -1
  69. package/dist/config/types/content.d.ts +0 -41
  70. package/dist/config/types/content.d.ts.map +0 -1
  71. package/dist/config/types/content.js +0 -1
  72. package/dist/config/types/content.js.map +0 -1
  73. package/dist/config/types/runtime.d.ts +0 -72
  74. package/dist/config/types/runtime.d.ts.map +0 -1
  75. package/dist/config/types/runtime.js +0 -1
  76. package/dist/config/types/runtime.js.map +0 -1
  77. package/dist/config/types/tools.d.ts +0 -28
  78. package/dist/config/types/tools.d.ts.map +0 -1
  79. package/dist/config/types/tools.js +0 -1
  80. package/dist/config/types/tools.js.map +0 -1
  81. package/dist/config/types.d.ts +0 -4
  82. package/dist/config/types.d.ts.map +0 -1
  83. package/dist/config/types.js +0 -2
  84. package/dist/config/types.js.map +0 -1
  85. package/dist/errors/app-error.d.ts +0 -7
  86. package/dist/errors/app-error.d.ts.map +0 -1
  87. package/dist/errors/app-error.js +0 -16
  88. package/dist/errors/app-error.js.map +0 -1
  89. package/dist/errors/index.d.ts +0 -2
  90. package/dist/errors/index.d.ts.map +0 -1
  91. package/dist/errors/index.js +0 -2
  92. package/dist/errors/index.js.map +0 -1
  93. package/dist/http/accept-policy.d.ts +0 -3
  94. package/dist/http/accept-policy.js +0 -45
  95. package/dist/http/async-handler.d.ts +0 -2
  96. package/dist/http/async-handler.js +0 -5
  97. package/dist/http/auth-introspection.d.ts +0 -2
  98. package/dist/http/auth-introspection.js +0 -141
  99. package/dist/http/auth-static.d.ts +0 -2
  100. package/dist/http/auth-static.js +0 -23
  101. package/dist/http/auth.d.ts +0 -3
  102. package/dist/http/auth.d.ts.map +0 -1
  103. package/dist/http/auth.js +0 -269
  104. package/dist/http/auth.js.map +0 -1
  105. package/dist/http/base-middleware.d.ts +0 -7
  106. package/dist/http/base-middleware.js +0 -143
  107. package/dist/http/cors.d.ts +0 -2
  108. package/dist/http/cors.d.ts.map +0 -1
  109. package/dist/http/cors.js +0 -9
  110. package/dist/http/cors.js.map +0 -1
  111. package/dist/http/download-routes.d.ts +0 -2
  112. package/dist/http/download-routes.d.ts.map +0 -1
  113. package/dist/http/download-routes.js +0 -104
  114. package/dist/http/download-routes.js.map +0 -1
  115. package/dist/http/error-handler.d.ts +0 -2
  116. package/dist/http/error-handler.js +0 -55
  117. package/dist/http/host-allowlist.d.ts +0 -3
  118. package/dist/http/host-allowlist.js +0 -117
  119. package/dist/http/jsonrpc-http.d.ts +0 -2
  120. package/dist/http/jsonrpc-http.js +0 -10
  121. package/dist/http/mcp-routes.d.ts +0 -9
  122. package/dist/http/mcp-routes.d.ts.map +0 -1
  123. package/dist/http/mcp-routes.js +0 -204
  124. package/dist/http/mcp-routes.js.map +0 -1
  125. package/dist/http/mcp-session-eviction.d.ts +0 -3
  126. package/dist/http/mcp-session-eviction.js +0 -24
  127. package/dist/http/mcp-session-helpers.d.ts +0 -12
  128. package/dist/http/mcp-session-helpers.d.ts.map +0 -1
  129. package/dist/http/mcp-session-helpers.js +0 -64
  130. package/dist/http/mcp-session-helpers.js.map +0 -1
  131. package/dist/http/mcp-session-init.d.ts +0 -7
  132. package/dist/http/mcp-session-init.js +0 -94
  133. package/dist/http/mcp-session-slots.d.ts +0 -17
  134. package/dist/http/mcp-session-slots.js +0 -55
  135. package/dist/http/mcp-session-transport-init.d.ts +0 -7
  136. package/dist/http/mcp-session-transport-init.js +0 -41
  137. package/dist/http/mcp-session-transport.d.ts +0 -7
  138. package/dist/http/mcp-session-transport.js +0 -57
  139. package/dist/http/mcp-session-types.d.ts +0 -5
  140. package/dist/http/mcp-session-types.js +0 -1
  141. package/dist/http/mcp-session.d.ts +0 -10
  142. package/dist/http/mcp-session.d.ts.map +0 -1
  143. package/dist/http/mcp-session.js +0 -23
  144. package/dist/http/mcp-session.js.map +0 -1
  145. package/dist/http/mcp-sessions.d.ts +0 -41
  146. package/dist/http/mcp-sessions.js +0 -392
  147. package/dist/http/mcp-validation.d.ts +0 -3
  148. package/dist/http/mcp-validation.d.ts.map +0 -1
  149. package/dist/http/mcp-validation.js +0 -14
  150. package/dist/http/mcp-validation.js.map +0 -1
  151. package/dist/http/protocol-policy.d.ts +0 -2
  152. package/dist/http/protocol-policy.js +0 -31
  153. package/dist/http/rate-limit.d.ts +0 -12
  154. package/dist/http/rate-limit.d.ts.map +0 -1
  155. package/dist/http/rate-limit.js +0 -93
  156. package/dist/http/rate-limit.js.map +0 -1
  157. package/dist/http/server-config.d.ts +0 -1
  158. package/dist/http/server-config.js +0 -40
  159. package/dist/http/server-middleware.d.ts +0 -7
  160. package/dist/http/server-middleware.d.ts.map +0 -1
  161. package/dist/http/server-middleware.js +0 -52
  162. package/dist/http/server-middleware.js.map +0 -1
  163. package/dist/http/server-shutdown.d.ts +0 -4
  164. package/dist/http/server-shutdown.js +0 -43
  165. package/dist/http/server-tuning.d.ts +0 -9
  166. package/dist/http/server-tuning.js +0 -45
  167. package/dist/http/server.d.ts +0 -3
  168. package/dist/http/server.d.ts.map +0 -1
  169. package/dist/http/server.js +0 -291
  170. package/dist/http/server.js.map +0 -1
  171. package/dist/http/session-cleanup.d.ts +0 -2
  172. package/dist/http/session-cleanup.d.ts.map +0 -1
  173. package/dist/http/session-cleanup.js +0 -40
  174. package/dist/http/session-cleanup.js.map +0 -1
  175. package/dist/http/sessions.d.ts +0 -14
  176. package/dist/http/sessions.d.ts.map +0 -1
  177. package/dist/http/sessions.js +0 -63
  178. package/dist/http/sessions.js.map +0 -1
  179. package/dist/middleware/error-handler.d.ts +0 -2
  180. package/dist/middleware/error-handler.d.ts.map +0 -1
  181. package/dist/middleware/error-handler.js +0 -56
  182. package/dist/middleware/error-handler.js.map +0 -1
  183. package/dist/middleware/rate-limiter.d.ts +0 -16
  184. package/dist/middleware/rate-limiter.d.ts.map +0 -1
  185. package/dist/middleware/rate-limiter.js +0 -111
  186. package/dist/middleware/rate-limiter.js.map +0 -1
  187. package/dist/parsers/base-html-element-parser.d.ts +0 -43
  188. package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
  189. package/dist/parsers/base-html-element-parser.js +0 -59
  190. package/dist/parsers/base-html-element-parser.js.map +0 -1
  191. package/dist/parsers/heading-element-parser.d.ts +0 -14
  192. package/dist/parsers/heading-element-parser.d.ts.map +0 -1
  193. package/dist/parsers/heading-element-parser.js +0 -26
  194. package/dist/parsers/heading-element-parser.js.map +0 -1
  195. package/dist/parsers/image-element-parser.d.ts +0 -16
  196. package/dist/parsers/image-element-parser.d.ts.map +0 -1
  197. package/dist/parsers/image-element-parser.js +0 -33
  198. package/dist/parsers/image-element-parser.js.map +0 -1
  199. package/dist/parsers/link-element-parser.d.ts +0 -15
  200. package/dist/parsers/link-element-parser.d.ts.map +0 -1
  201. package/dist/parsers/link-element-parser.js +0 -28
  202. package/dist/parsers/link-element-parser.js.map +0 -1
  203. package/dist/parsers/open-graph-parser.d.ts +0 -17
  204. package/dist/parsers/open-graph-parser.d.ts.map +0 -1
  205. package/dist/parsers/open-graph-parser.js +0 -41
  206. package/dist/parsers/open-graph-parser.js.map +0 -1
  207. package/dist/parsers/schema-org-parser.d.ts +0 -17
  208. package/dist/parsers/schema-org-parser.d.ts.map +0 -1
  209. package/dist/parsers/schema-org-parser.js +0 -32
  210. package/dist/parsers/schema-org-parser.js.map +0 -1
  211. package/dist/parsers/standard-meta-parser.d.ts +0 -18
  212. package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
  213. package/dist/parsers/standard-meta-parser.js +0 -32
  214. package/dist/parsers/standard-meta-parser.js.map +0 -1
  215. package/dist/parsers/twitter-card-parser.d.ts +0 -17
  216. package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
  217. package/dist/parsers/twitter-card-parser.js +0 -41
  218. package/dist/parsers/twitter-card-parser.js.map +0 -1
  219. package/dist/prompts/index.d.ts +0 -3
  220. package/dist/prompts/index.d.ts.map +0 -1
  221. package/dist/prompts/index.js +0 -73
  222. package/dist/prompts/index.js.map +0 -1
  223. package/dist/resources/cached-content-params.d.ts +0 -5
  224. package/dist/resources/cached-content-params.js +0 -36
  225. package/dist/resources/cached-content.d.ts +0 -2
  226. package/dist/resources/cached-content.d.ts.map +0 -1
  227. package/dist/resources/cached-content.js +0 -132
  228. package/dist/resources/cached-content.js.map +0 -1
  229. package/dist/resources/index.d.ts +0 -2
  230. package/dist/resources/index.d.ts.map +0 -1
  231. package/dist/resources/index.js +0 -4
  232. package/dist/resources/index.js.map +0 -1
  233. package/dist/server.d.ts +0 -3
  234. package/dist/server.d.ts.map +0 -1
  235. package/dist/server.js +0 -94
  236. package/dist/server.js.map +0 -1
  237. package/dist/services/cache-events.d.ts +0 -8
  238. package/dist/services/cache-events.js +0 -19
  239. package/dist/services/cache-keys.d.ts +0 -7
  240. package/dist/services/cache-keys.js +0 -57
  241. package/dist/services/cache.d.ts +0 -17
  242. package/dist/services/cache.d.ts.map +0 -1
  243. package/dist/services/cache.js +0 -145
  244. package/dist/services/cache.js.map +0 -1
  245. package/dist/services/cache.service.d.ts +0 -52
  246. package/dist/services/cache.service.d.ts.map +0 -1
  247. package/dist/services/cache.service.js +0 -113
  248. package/dist/services/cache.service.js.map +0 -1
  249. package/dist/services/card-extractor.d.ts +0 -6
  250. package/dist/services/card-extractor.d.ts.map +0 -1
  251. package/dist/services/card-extractor.js +0 -199
  252. package/dist/services/card-extractor.js.map +0 -1
  253. package/dist/services/context.d.ts +0 -10
  254. package/dist/services/context.d.ts.map +0 -1
  255. package/dist/services/context.js +0 -14
  256. package/dist/services/context.js.map +0 -1
  257. package/dist/services/extractor.d.ts +0 -5
  258. package/dist/services/extractor.d.ts.map +0 -1
  259. package/dist/services/extractor.js +0 -142
  260. package/dist/services/extractor.js.map +0 -1
  261. package/dist/services/extractor.service.d.ts +0 -18
  262. package/dist/services/extractor.service.d.ts.map +0 -1
  263. package/dist/services/extractor.service.js +0 -75
  264. package/dist/services/extractor.service.js.map +0 -1
  265. package/dist/services/fetcher/agents.d.ts +0 -3
  266. package/dist/services/fetcher/agents.d.ts.map +0 -1
  267. package/dist/services/fetcher/agents.js +0 -100
  268. package/dist/services/fetcher/agents.js.map +0 -1
  269. package/dist/services/fetcher/dns-selection.d.ts +0 -2
  270. package/dist/services/fetcher/dns-selection.js +0 -72
  271. package/dist/services/fetcher/errors.d.ts +0 -4
  272. package/dist/services/fetcher/errors.d.ts.map +0 -1
  273. package/dist/services/fetcher/errors.js +0 -70
  274. package/dist/services/fetcher/errors.js.map +0 -1
  275. package/dist/services/fetcher/headers.d.ts +0 -2
  276. package/dist/services/fetcher/headers.d.ts.map +0 -1
  277. package/dist/services/fetcher/headers.js +0 -6
  278. package/dist/services/fetcher/headers.js.map +0 -1
  279. package/dist/services/fetcher/interceptors.d.ts +0 -10
  280. package/dist/services/fetcher/interceptors.d.ts.map +0 -1
  281. package/dist/services/fetcher/interceptors.js +0 -108
  282. package/dist/services/fetcher/interceptors.js.map +0 -1
  283. package/dist/services/fetcher/redirects.d.ts +0 -4
  284. package/dist/services/fetcher/redirects.d.ts.map +0 -1
  285. package/dist/services/fetcher/redirects.js +0 -78
  286. package/dist/services/fetcher/redirects.js.map +0 -1
  287. package/dist/services/fetcher/response.d.ts +0 -4
  288. package/dist/services/fetcher/response.d.ts.map +0 -1
  289. package/dist/services/fetcher/response.js +0 -104
  290. package/dist/services/fetcher/response.js.map +0 -1
  291. package/dist/services/fetcher/retry-policy.d.ts +0 -1
  292. package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
  293. package/dist/services/fetcher/retry-policy.js +0 -131
  294. package/dist/services/fetcher/retry-policy.js.map +0 -1
  295. package/dist/services/fetcher.d.ts +0 -25
  296. package/dist/services/fetcher.d.ts.map +0 -1
  297. package/dist/services/fetcher.js +0 -607
  298. package/dist/services/fetcher.js.map +0 -1
  299. package/dist/services/fetcher.service.d.ts +0 -18
  300. package/dist/services/fetcher.service.d.ts.map +0 -1
  301. package/dist/services/fetcher.service.js +0 -122
  302. package/dist/services/fetcher.service.js.map +0 -1
  303. package/dist/services/fifo-queue.d.ts +0 -8
  304. package/dist/services/fifo-queue.js +0 -25
  305. package/dist/services/logger.d.ts +0 -5
  306. package/dist/services/logger.d.ts.map +0 -1
  307. package/dist/services/logger.js +0 -52
  308. package/dist/services/logger.js.map +0 -1
  309. package/dist/services/logger.service.d.ts +0 -5
  310. package/dist/services/logger.service.d.ts.map +0 -1
  311. package/dist/services/logger.service.js +0 -57
  312. package/dist/services/logger.service.js.map +0 -1
  313. package/dist/services/metadata-collector.d.ts +0 -2
  314. package/dist/services/metadata-collector.js +0 -80
  315. package/dist/services/parser.d.ts +0 -6
  316. package/dist/services/parser.d.ts.map +0 -1
  317. package/dist/services/parser.js +0 -278
  318. package/dist/services/parser.js.map +0 -1
  319. package/dist/services/parser.service.d.ts +0 -42
  320. package/dist/services/parser.service.d.ts.map +0 -1
  321. package/dist/services/parser.service.js +0 -209
  322. package/dist/services/parser.service.js.map +0 -1
  323. package/dist/services/session-manager.d.ts +0 -18
  324. package/dist/services/session-manager.d.ts.map +0 -1
  325. package/dist/services/session-manager.js +0 -73
  326. package/dist/services/session-manager.js.map +0 -1
  327. package/dist/services/telemetry.d.ts +0 -19
  328. package/dist/services/telemetry.js +0 -43
  329. package/dist/services/transform-worker-pool.d.ts +0 -11
  330. package/dist/services/transform-worker-pool.js +0 -244
  331. package/dist/services/transform-worker-types.d.ts +0 -32
  332. package/dist/services/transform-worker-types.js +0 -14
  333. package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
  334. package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
  335. package/dist/strategies/exponential-backoff-strategy.js +0 -32
  336. package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
  337. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
  338. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
  339. package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
  340. package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
  341. package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
  342. package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
  343. package/dist/tools/handlers/fetch-links.tool.js +0 -98
  344. package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
  345. package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
  346. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
  347. package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
  348. package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
  349. package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
  350. package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
  351. package/dist/tools/handlers/fetch-single.shared.js +0 -223
  352. package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
  353. package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
  354. package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
  355. package/dist/tools/handlers/fetch-url.tool.js +0 -119
  356. package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
  357. package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
  358. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
  359. package/dist/tools/handlers/fetch-urls/processor.js +0 -153
  360. package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
  361. package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
  362. package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
  363. package/dist/tools/handlers/fetch-urls/response.js +0 -79
  364. package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
  365. package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
  366. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
  367. package/dist/tools/handlers/fetch-urls/validation.js +0 -18
  368. package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
  369. package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
  370. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
  371. package/dist/tools/handlers/fetch-urls.tool.js +0 -124
  372. package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
  373. package/dist/tools/index.d.ts +0 -3
  374. package/dist/tools/index.d.ts.map +0 -1
  375. package/dist/tools/index.js +0 -37
  376. package/dist/tools/index.js.map +0 -1
  377. package/dist/tools/schemas.d.ts +0 -12
  378. package/dist/tools/schemas.d.ts.map +0 -1
  379. package/dist/tools/schemas.js +0 -21
  380. package/dist/tools/schemas.js.map +0 -1
  381. package/dist/tools/utils/cache-vary.d.ts +0 -1
  382. package/dist/tools/utils/cache-vary.d.ts.map +0 -1
  383. package/dist/tools/utils/cache-vary.js +0 -12
  384. package/dist/tools/utils/cache-vary.js.map +0 -1
  385. package/dist/tools/utils/cached-markdown.d.ts +0 -5
  386. package/dist/tools/utils/cached-markdown.js +0 -46
  387. package/dist/tools/utils/common.d.ts +0 -5
  388. package/dist/tools/utils/common.d.ts.map +0 -1
  389. package/dist/tools/utils/common.js +0 -42
  390. package/dist/tools/utils/common.js.map +0 -1
  391. package/dist/tools/utils/content-shaping.d.ts +0 -4
  392. package/dist/tools/utils/content-shaping.js +0 -67
  393. package/dist/tools/utils/content-transform-async.d.ts +0 -6
  394. package/dist/tools/utils/content-transform-async.js +0 -33
  395. package/dist/tools/utils/content-transform-core.d.ts +0 -5
  396. package/dist/tools/utils/content-transform-core.js +0 -180
  397. package/dist/tools/utils/content-transform-workers.d.ts +0 -1
  398. package/dist/tools/utils/content-transform-workers.js +0 -1
  399. package/dist/tools/utils/content-transform.d.ts +0 -3
  400. package/dist/tools/utils/content-transform.d.ts.map +0 -1
  401. package/dist/tools/utils/content-transform.js +0 -41
  402. package/dist/tools/utils/content-transform.js.map +0 -1
  403. package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
  404. package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
  405. package/dist/tools/utils/fetch-pipeline.js +0 -98
  406. package/dist/tools/utils/fetch-pipeline.js.map +0 -1
  407. package/dist/tools/utils/frontmatter.d.ts +0 -3
  408. package/dist/tools/utils/frontmatter.js +0 -73
  409. package/dist/tools/utils/index.d.ts +0 -4
  410. package/dist/tools/utils/index.d.ts.map +0 -1
  411. package/dist/tools/utils/index.js +0 -3
  412. package/dist/tools/utils/index.js.map +0 -1
  413. package/dist/tools/utils/inline-content.d.ts +0 -10
  414. package/dist/tools/utils/inline-content.d.ts.map +0 -1
  415. package/dist/tools/utils/inline-content.js +0 -35
  416. package/dist/tools/utils/inline-content.js.map +0 -1
  417. package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
  418. package/dist/tools/utils/markdown-heuristics.js +0 -19
  419. package/dist/tools/utils/markdown-signals.d.ts +0 -1
  420. package/dist/tools/utils/markdown-signals.js +0 -19
  421. package/dist/tools/utils/markdown-toc.d.ts +0 -3
  422. package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
  423. package/dist/tools/utils/markdown-toc.js +0 -35
  424. package/dist/tools/utils/markdown-toc.js.map +0 -1
  425. package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
  426. package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
  427. package/dist/tools/utils/raw-markdown.d.ts +0 -6
  428. package/dist/tools/utils/raw-markdown.js +0 -149
  429. package/dist/tools/utils/response-builder.d.ts +0 -3
  430. package/dist/tools/utils/response-builder.d.ts.map +0 -1
  431. package/dist/tools/utils/response-builder.js +0 -24
  432. package/dist/tools/utils/response-builder.js.map +0 -1
  433. package/dist/tools/utils/tool-response.d.ts +0 -9
  434. package/dist/tools/utils/tool-response.d.ts.map +0 -1
  435. package/dist/tools/utils/tool-response.js +0 -19
  436. package/dist/tools/utils/tool-response.js.map +0 -1
  437. package/dist/transformers/jsonl.transformer.d.ts +0 -2
  438. package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
  439. package/dist/transformers/jsonl.transformer.js +0 -75
  440. package/dist/transformers/jsonl.transformer.js.map +0 -1
  441. package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
  442. package/dist/transformers/markdown/fenced-code-rule.js +0 -38
  443. package/dist/transformers/markdown/frontmatter.d.ts +0 -2
  444. package/dist/transformers/markdown/frontmatter.js +0 -45
  445. package/dist/transformers/markdown/noise-rule.d.ts +0 -2
  446. package/dist/transformers/markdown/noise-rule.js +0 -80
  447. package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
  448. package/dist/transformers/markdown/turndown-instance.js +0 -19
  449. package/dist/transformers/markdown.d.ts +0 -5
  450. package/dist/transformers/markdown.js +0 -314
  451. package/dist/transformers/markdown.transformer.d.ts +0 -2
  452. package/dist/transformers/markdown.transformer.d.ts.map +0 -1
  453. package/dist/transformers/markdown.transformer.js +0 -14
  454. package/dist/transformers/markdown.transformer.js.map +0 -1
  455. package/dist/types/content.types.d.ts +0 -63
  456. package/dist/types/content.types.d.ts.map +0 -1
  457. package/dist/types/content.types.js +0 -2
  458. package/dist/types/content.types.js.map +0 -1
  459. package/dist/types/index.d.ts +0 -2
  460. package/dist/types/index.d.ts.map +0 -1
  461. package/dist/types/index.js +0 -2
  462. package/dist/types/index.js.map +0 -1
  463. package/dist/types/schemas.d.ts +0 -49
  464. package/dist/types/schemas.d.ts.map +0 -1
  465. package/dist/types/schemas.js +0 -5
  466. package/dist/types/schemas.js.map +0 -1
  467. package/dist/utils/cached-payload.d.ts +0 -7
  468. package/dist/utils/cached-payload.js +0 -36
  469. package/dist/utils/cancellation.d.ts +0 -1
  470. package/dist/utils/cancellation.js +0 -18
  471. package/dist/utils/code-language-bash.d.ts +0 -1
  472. package/dist/utils/code-language-bash.js +0 -48
  473. package/dist/utils/code-language-core.d.ts +0 -2
  474. package/dist/utils/code-language-core.js +0 -13
  475. package/dist/utils/code-language-detectors.d.ts +0 -5
  476. package/dist/utils/code-language-detectors.js +0 -142
  477. package/dist/utils/code-language-helpers.d.ts +0 -5
  478. package/dist/utils/code-language-helpers.js +0 -62
  479. package/dist/utils/code-language-parsing.d.ts +0 -5
  480. package/dist/utils/code-language-parsing.js +0 -62
  481. package/dist/utils/code-language.d.ts +0 -2
  482. package/dist/utils/code-language.d.ts.map +0 -1
  483. package/dist/utils/code-language.js +0 -260
  484. package/dist/utils/code-language.js.map +0 -1
  485. package/dist/utils/concurrency.d.ts +0 -3
  486. package/dist/utils/concurrency.d.ts.map +0 -1
  487. package/dist/utils/concurrency.js +0 -38
  488. package/dist/utils/concurrency.js.map +0 -1
  489. package/dist/utils/content-cleaner.d.ts +0 -5
  490. package/dist/utils/content-cleaner.d.ts.map +0 -1
  491. package/dist/utils/content-cleaner.js +0 -77
  492. package/dist/utils/content-cleaner.js.map +0 -1
  493. package/dist/utils/crypto.d.ts +0 -2
  494. package/dist/utils/crypto.d.ts.map +0 -1
  495. package/dist/utils/crypto.js +0 -32
  496. package/dist/utils/crypto.js.map +0 -1
  497. package/dist/utils/download-url.d.ts +0 -16
  498. package/dist/utils/download-url.d.ts.map +0 -1
  499. package/dist/utils/download-url.js +0 -30
  500. package/dist/utils/download-url.js.map +0 -1
  501. package/dist/utils/error-details.d.ts +0 -3
  502. package/dist/utils/error-details.js +0 -12
  503. package/dist/utils/error-utils.d.ts +0 -3
  504. package/dist/utils/error-utils.d.ts.map +0 -1
  505. package/dist/utils/error-utils.js +0 -12
  506. package/dist/utils/error-utils.js.map +0 -1
  507. package/dist/utils/filename-generator.d.ts +0 -1
  508. package/dist/utils/filename-generator.d.ts.map +0 -1
  509. package/dist/utils/filename-generator.js +0 -81
  510. package/dist/utils/filename-generator.js.map +0 -1
  511. package/dist/utils/guards.d.ts +0 -1
  512. package/dist/utils/guards.js +0 -3
  513. package/dist/utils/header-normalizer.d.ts +0 -5
  514. package/dist/utils/header-normalizer.d.ts.map +0 -1
  515. package/dist/utils/header-normalizer.js +0 -31
  516. package/dist/utils/header-normalizer.js.map +0 -1
  517. package/dist/utils/host-normalizer.d.ts +0 -1
  518. package/dist/utils/host-normalizer.js +0 -37
  519. package/dist/utils/html-truncator.d.ts +0 -1
  520. package/dist/utils/html-truncator.d.ts.map +0 -1
  521. package/dist/utils/html-truncator.js +0 -13
  522. package/dist/utils/html-truncator.js.map +0 -1
  523. package/dist/utils/ip-address.d.ts +0 -4
  524. package/dist/utils/ip-address.js +0 -6
  525. package/dist/utils/language-detector.d.ts +0 -2
  526. package/dist/utils/language-detector.d.ts.map +0 -1
  527. package/dist/utils/language-detector.js +0 -39
  528. package/dist/utils/language-detector.js.map +0 -1
  529. package/dist/utils/sanitizer.d.ts +0 -2
  530. package/dist/utils/sanitizer.d.ts.map +0 -1
  531. package/dist/utils/sanitizer.js +0 -20
  532. package/dist/utils/sanitizer.js.map +0 -1
  533. package/dist/utils/tool-error-handler.d.ts +0 -3
  534. package/dist/utils/tool-error-handler.d.ts.map +0 -1
  535. package/dist/utils/tool-error-handler.js +0 -31
  536. package/dist/utils/tool-error-handler.js.map +0 -1
  537. package/dist/utils/url-redactor.d.ts +0 -1
  538. package/dist/utils/url-redactor.js +0 -13
  539. package/dist/utils/url-sanitizer.d.ts +0 -2
  540. package/dist/utils/url-sanitizer.d.ts.map +0 -1
  541. package/dist/utils/url-sanitizer.js +0 -12
  542. package/dist/utils/url-sanitizer.js.map +0 -1
  543. package/dist/utils/url-transformer.d.ts +0 -7
  544. package/dist/utils/url-transformer.js +0 -147
  545. package/dist/utils/url-validator.d.ts +0 -6
  546. package/dist/utils/url-validator.d.ts.map +0 -1
  547. package/dist/utils/url-validator.js +0 -156
  548. package/dist/utils/url-validator.js.map +0 -1
  549. package/dist/workers/content-transform.worker.d.ts +0 -1
  550. package/dist/workers/content-transform.worker.js +0 -40
@@ -1,72 +0,0 @@
1
- import { createErrorWithCode } from '../../utils/error-details.js';
2
- import { isBlockedIp } from '../../utils/url-validator.js';
3
- function normalizeLookupResults(addresses, family) {
4
- if (Array.isArray(addresses)) {
5
- return addresses;
6
- }
7
- return [{ address: addresses, family: family ?? 4 }];
8
- }
9
- function findBlockedIpError(list, hostname) {
10
- for (const addr of list) {
11
- const ip = typeof addr === 'string' ? addr : addr.address;
12
- if (!isBlockedIp(ip)) {
13
- continue;
14
- }
15
- return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
16
- }
17
- return null;
18
- }
19
- function findInvalidFamilyError(list, hostname) {
20
- for (const addr of list) {
21
- const family = typeof addr === 'string' ? 0 : addr.family;
22
- if (family === 4 || family === 6)
23
- continue;
24
- return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
25
- }
26
- return null;
27
- }
28
- function createNoDnsResultsError(hostname) {
29
- return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
30
- }
31
- function createEmptySelection(hostname) {
32
- return {
33
- error: createNoDnsResultsError(hostname),
34
- fallback: [],
35
- address: [],
36
- };
37
- }
38
- function selectLookupResult(list, useAll, hostname) {
39
- if (list.length === 0)
40
- return createEmptySelection(hostname);
41
- if (useAll)
42
- return { address: list, fallback: list };
43
- const first = list.at(0);
44
- if (!first)
45
- return createEmptySelection(hostname);
46
- return {
47
- address: first.address,
48
- family: first.family,
49
- fallback: list,
50
- };
51
- }
52
- function findLookupError(list, hostname) {
53
- return (findInvalidFamilyError(list, hostname) ?? findBlockedIpError(list, hostname));
54
- }
55
- export function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
56
- if (error) {
57
- callback(error, addresses);
58
- return;
59
- }
60
- const list = normalizeLookupResults(addresses, resolvedFamily);
61
- const lookupError = findLookupError(list, hostname);
62
- if (lookupError) {
63
- callback(lookupError, list);
64
- return;
65
- }
66
- const selection = selectLookupResult(list, useAll, hostname);
67
- if (selection.error) {
68
- callback(selection.error, selection.fallback);
69
- return;
70
- }
71
- callback(null, selection.address, selection.family);
72
- }
@@ -1,4 +0,0 @@
1
- import { FetchError } from '../../errors/app-error.js';
2
- export declare function createRateLimitError(url: string, headerValue: string | null): FetchError;
3
- export declare function createHttpError(url: string, status: number, statusText: string): FetchError;
4
- export declare function mapFetchError(error: unknown, fallbackUrl: string, timeoutMs: number): FetchError;
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAoBvD,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,UAAU,CAGZ;AAED,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,GACjB,UAAU,CAEZ;AA4CD,wBAAgB,aAAa,CAC3B,KAAK,EAAE,OAAO,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,UAAU,CAiBZ"}
@@ -1,70 +0,0 @@
1
- import { FetchError } from '../../errors/app-error.js';
2
- function parseRetryAfter(header) {
3
- if (!header)
4
- return 60;
5
- const parsed = parseInt(header, 10);
6
- return Number.isNaN(parsed) ? 60 : parsed;
7
- }
8
- function createCanceledError(url) {
9
- return new FetchError('Request was canceled', url, 499, {
10
- reason: 'aborted',
11
- });
12
- }
13
- function createTimeoutError(url, timeoutMs) {
14
- return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
15
- timeout: timeoutMs,
16
- });
17
- }
18
- export function createRateLimitError(url, headerValue) {
19
- const retryAfter = parseRetryAfter(headerValue);
20
- return new FetchError('Too many requests', url, 429, { retryAfter });
21
- }
22
- export function createHttpError(url, status, statusText) {
23
- return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
24
- }
25
- function createNetworkError(url, message) {
26
- const details = message ? { message } : undefined;
27
- return new FetchError(`Network error: Could not reach ${url}`, url, undefined, details ?? {});
28
- }
29
- function createUnknownError(url, message) {
30
- return new FetchError(message, url);
31
- }
32
- function isAbortError(error) {
33
- return (error instanceof Error &&
34
- (error.name === 'AbortError' || error.name === 'TimeoutError'));
35
- }
36
- function isTimeoutError(error) {
37
- return error instanceof Error && error.name === 'TimeoutError';
38
- }
39
- function isRecord(value) {
40
- return value !== null && typeof value === 'object';
41
- }
42
- function getRequestUrl(record) {
43
- const value = record.requestUrl;
44
- return typeof value === 'string' ? value : null;
45
- }
46
- function resolveErrorUrl(error, fallback) {
47
- if (error instanceof FetchError)
48
- return error.url;
49
- if (!isRecord(error))
50
- return fallback;
51
- const requestUrl = getRequestUrl(error);
52
- if (requestUrl)
53
- return requestUrl;
54
- return fallback;
55
- }
56
- export function mapFetchError(error, fallbackUrl, timeoutMs) {
57
- if (error instanceof FetchError)
58
- return error;
59
- const url = resolveErrorUrl(error, fallbackUrl);
60
- if (isAbortError(error)) {
61
- if (isTimeoutError(error)) {
62
- return createTimeoutError(url, timeoutMs);
63
- }
64
- return createCanceledError(url);
65
- }
66
- if (error instanceof Error) {
67
- return createNetworkError(url, error.message);
68
- }
69
- return createUnknownError(url, 'Unexpected error');
70
- }
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../src/services/fetcher/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,SAAS,eAAe,CAAC,MAAqB;IAC5C,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,IAAI,UAAU,CAAC,sBAAsB,EAAE,GAAG,EAAE,GAAG,EAAE;QACtD,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,SAAiB;IACxD,OAAO,IAAI,UAAU,CAAC,yBAAyB,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE;QACtE,OAAO,EAAE,SAAS;KACnB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAW,EACX,WAA0B;IAE1B,MAAM,UAAU,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;IAChD,OAAO,IAAI,UAAU,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,MAAc,EACd,UAAkB;IAElB,OAAO,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,OAAgB;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,OAAO,IAAI,UAAU,CACnB,kCAAkC,GAAG,EAAE,EACvC,GAAG,EACH,SAAS,EACT,OAAO,IAAI,EAAE,CACd,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,OAAe;IACtD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,KAAK,YAAY,KAAK;QACtB,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAC/D,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,MAA+B;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC;IAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,QAAgB;IACvD,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC;IAClD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IACtC,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAClC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,KAAc,EACd,WAAmB,EACnB,SAAiB;IAEjB,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAE9C,MAAM,GAAG,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAEhD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,OAAO,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,kBAAkB,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;AACrD,CAAC"}
@@ -1,2 +0,0 @@
1
- export declare function sanitizeHeaders(headers?: Record<string, string>): Record<string, string> | undefined;
2
- //# sourceMappingURL=headers.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAIA,wBAAgB,eAAe,CAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAEpC"}
@@ -1,6 +0,0 @@
1
- import { config } from '../../config/index.js';
2
- import { normalizeHeaderRecord } from '../../utils/header-normalizer.js';
3
- export function sanitizeHeaders(headers) {
4
- return normalizeHeaderRecord(headers, config.security.blockedHeaders);
5
- }
6
- //# sourceMappingURL=headers.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,UAAU,eAAe,CAC7B,OAAgC;IAEhC,OAAO,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;AACxE,CAAC"}
@@ -1,10 +0,0 @@
1
- interface FetchTelemetryContext {
2
- requestId: string;
3
- startTime: number;
4
- url: string;
5
- method: string;
6
- }
7
- export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
8
- export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
9
- export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
10
- export {};
@@ -1 +0,0 @@
1
- {"version":3,"file":"interceptors.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAUA,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAwBvB;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAYN;AA4CD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAyBN"}
@@ -1,108 +0,0 @@
1
- import { randomUUID } from 'node:crypto';
2
- import diagnosticsChannel from 'node:diagnostics_channel';
3
- import { performance } from 'node:perf_hooks';
4
- import { isSystemError } from '../../utils/error-details.js';
5
- import { logDebug, logError, logWarn } from '../logger.js';
6
- const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
7
- function redactUrl(rawUrl) {
8
- try {
9
- const url = new URL(rawUrl);
10
- url.username = '';
11
- url.password = '';
12
- url.hash = '';
13
- url.search = '';
14
- return url.toString();
15
- }
16
- catch {
17
- return rawUrl;
18
- }
19
- }
20
- function publishFetchEvent(event) {
21
- if (!fetchChannel.hasSubscribers)
22
- return;
23
- try {
24
- fetchChannel.publish(event);
25
- }
26
- catch {
27
- // Avoid crashing the publisher if a subscriber throws.
28
- }
29
- }
30
- export function startFetchTelemetry(url, method) {
31
- const safeUrl = redactUrl(url);
32
- const context = {
33
- requestId: randomUUID(),
34
- startTime: performance.now(),
35
- url: safeUrl,
36
- method: method.toUpperCase(),
37
- };
38
- publishFetchEvent({
39
- v: 1,
40
- type: 'start',
41
- requestId: context.requestId,
42
- method: context.method,
43
- url: context.url,
44
- });
45
- logDebug('HTTP Request', {
46
- requestId: context.requestId,
47
- method: context.method,
48
- url: context.url,
49
- });
50
- return context;
51
- }
52
- export function recordFetchResponse(context, response, contentSize) {
53
- const duration = performance.now() - context.startTime;
54
- const durationLabel = `${Math.round(duration)}ms`;
55
- publishFetchEvent({
56
- v: 1,
57
- type: 'end',
58
- requestId: context.requestId,
59
- status: response.status,
60
- duration,
61
- });
62
- const contentType = response.headers.get('content-type');
63
- const contentLength = response.headers.get('content-length') ??
64
- (contentSize === undefined ? undefined : String(contentSize));
65
- logDebug('HTTP Response', {
66
- requestId: context.requestId,
67
- status: response.status,
68
- url: context.url,
69
- duration: durationLabel,
70
- ...(contentType ? { contentType } : {}),
71
- ...(contentLength ? { size: contentLength } : {}),
72
- });
73
- if (duration > 5000) {
74
- logWarn('Slow HTTP request detected', {
75
- requestId: context.requestId,
76
- url: context.url,
77
- duration: durationLabel,
78
- });
79
- }
80
- }
81
- export function recordFetchError(context, error, status) {
82
- const duration = performance.now() - context.startTime;
83
- const err = error instanceof Error ? error : new Error(String(error));
84
- const event = {
85
- v: 1,
86
- type: 'error',
87
- requestId: context.requestId,
88
- url: context.url,
89
- error: err.message,
90
- duration,
91
- };
92
- const code = isSystemError(err) ? err.code : undefined;
93
- if (code !== undefined) {
94
- event.code = code;
95
- }
96
- if (status !== undefined) {
97
- event.status = status;
98
- }
99
- publishFetchEvent(event);
100
- const log = status === 429 ? logWarn : logError;
101
- log('HTTP Request Error', {
102
- requestId: context.requestId,
103
- url: context.url,
104
- status,
105
- code,
106
- error: err.message,
107
- });
108
- }
@@ -1 +0,0 @@
1
- {"version":3,"file":"interceptors.js","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,kBAAkB,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE3D,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AASpE,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,MAAc;IAEd,MAAM,OAAO,GAA0B;QACrC,SAAS,EAAE,UAAU,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,GAAG,EAAE;QAC5B,GAAG;QACH,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;KAC7B,CAAC;IAEF,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA8B,EAC9B,QAAkB,EAClB,WAAoB;IAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEpD,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,iBAAiB,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC;KACtD,CAAC,CAAC;IAEH,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,eAAe,CACtB,OAA8B,EAC9B,MAAc,EACd,QAAgB;IAEhB,IAAI,CAAC,YAAY,CAAC,cAAc;QAAE,OAAO;IACzC,YAAY,CAAC,OAAO,CAAC;QACnB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM;QACN,QAAQ;KACT,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CACxB,QAAkB,EAClB,WAA+B,EAC/B,QAAgB;IAEhB,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC;IACtE,MAAM,aAAa,GACjB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,WAAW,EAAE,QAAQ,EAAE,CAAC;IAEpE,OAAO;QACL,WAAW;QACX,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI;QACrC,IAAI,EAAE,aAAa;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA8B,EAC9B,QAAgB;IAEhB,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO;IAC7B,OAAO,CAAC,4BAA4B,EAAE;QACpC,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI;KACtC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAA8B,EAC9B,KAAc,EACd,MAAe;IAEf,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,IAAI;YACJ,MAAM;YACN,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChD,GAAG,CAAC,oBAAoB,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM;QACN,IAAI;QACJ,KAAK,EAAE,GAAG,CAAC,OAAO;KACnB,CAAC,CAAC;AACL,CAAC"}
@@ -1,4 +0,0 @@
1
- export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
2
- response: Response;
3
- url: string;
4
- }>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"redirects.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"AA8EA,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAwB9C"}
@@ -1,78 +0,0 @@
1
- import { FetchError } from '../../errors/app-error.js';
2
- import { createErrorWithCode } from '../../utils/error-details.js';
3
- import { isRecord } from '../../utils/guards.js';
4
- import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
5
- const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
6
- function isRedirectStatus(status) {
7
- return REDIRECT_STATUSES.has(status);
8
- }
9
- function cancelResponseBody(response) {
10
- const cancelPromise = response.body?.cancel();
11
- if (cancelPromise) {
12
- cancelPromise.catch(() => {
13
- // Best-effort cancellation; ignore failures.
14
- });
15
- }
16
- }
17
- async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
18
- const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
19
- if (!isRedirectStatus(response.status)) {
20
- return { response };
21
- }
22
- assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
23
- const location = getRedirectLocation(response, currentUrl);
24
- cancelResponseBody(response);
25
- return {
26
- response,
27
- nextUrl: resolveRedirectTarget(currentUrl, location),
28
- };
29
- }
30
- function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
31
- if (redirectCount < redirectLimit)
32
- return;
33
- cancelResponseBody(response);
34
- throw new FetchError('Too many redirects', currentUrl);
35
- }
36
- function getRedirectLocation(response, currentUrl) {
37
- const location = response.headers.get('location');
38
- if (location)
39
- return location;
40
- cancelResponseBody(response);
41
- throw new FetchError('Redirect response missing Location header', currentUrl);
42
- }
43
- function annotateRedirectError(error, url) {
44
- if (!isRecord(error))
45
- return;
46
- error.requestUrl = url;
47
- }
48
- function resolveRedirectTarget(baseUrl, location) {
49
- if (!URL.canParse(location, baseUrl)) {
50
- throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
51
- }
52
- const resolved = new URL(location, baseUrl);
53
- if (resolved.username || resolved.password) {
54
- throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
55
- }
56
- return validateAndNormalizeUrl(resolved.href);
57
- }
58
- export async function fetchWithRedirects(url, init, maxRedirects) {
59
- let currentUrl = url;
60
- const redirectLimit = Math.max(0, maxRedirects);
61
- for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
62
- const { response, nextUrl } = await performFetchCycleSafely(currentUrl, init, redirectLimit, redirectCount);
63
- if (!nextUrl) {
64
- return { response, url: currentUrl };
65
- }
66
- currentUrl = nextUrl;
67
- }
68
- throw new FetchError('Too many redirects', currentUrl);
69
- }
70
- async function performFetchCycleSafely(currentUrl, init, redirectLimit, redirectCount) {
71
- try {
72
- return await performFetchCycle(currentUrl, init, redirectLimit, redirectCount);
73
- }
74
- catch (error) {
75
- annotateRedirectError(error, currentUrl);
76
- throw error;
77
- }
78
- }
@@ -1 +0,0 @@
1
- {"version":3,"file":"redirects.js","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC;AAOD,KAAK,UAAU,iBAAiB,CAC9B,UAAkB,EAClB,IAAiB,EACjB,aAAqB,EACrB,aAAqB;IAErB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,QAAQ,EAAE,CAAC;IACtB,CAAC;IAED,yBAAyB,CAAC,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE3D,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7B,OAAO;QACL,QAAQ;QACR,OAAO,EAAE,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAChC,QAAkB,EAClB,UAAkB,EAClB,aAAqB,EACrB,aAAqB;IAErB,IAAI,aAAa,GAAG,aAAa;QAAE,OAAO;IAC1C,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7B,MAAM,IAAI,UAAU,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAkB,EAAE,UAAkB;IACjE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAClD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7B,MAAM,IAAI,UAAU,CAAC,2CAA2C,EAAE,UAAU,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc,EAAE,GAAW;IACxD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO;IAC/C,KAAiC,CAAC,UAAU,GAAG,GAAG,CAAC;AACtD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,QAAgB;IAC9D,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,mBAAmB,CAAC,yBAAyB,EAAE,cAAc,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,mBAAmB,CACvB,sCAAsC,EACtC,cAAc,CACf,CAAC;IACJ,CAAC;IAED,OAAO,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,IAAiB,EACjB,YAAoB;IAEpB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAEhD,KACE,IAAI,aAAa,GAAG,CAAC,EACrB,aAAa,IAAI,aAAa,EAC9B,aAAa,IAAI,CAAC,EAClB,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,uBAAuB,CACzD,UAAU,EACV,IAAI,EACJ,aAAa,EACb,aAAa,CACd,CAAC;QAEF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;QACvC,CAAC;QAED,UAAU,GAAG,OAAO,CAAC;IACvB,CAAC;IAED,MAAM,IAAI,UAAU,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,UAAkB,EAClB,IAAiB,EACjB,aAAqB,EACrB,aAAqB;IAErB,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAC5B,UAAU,EACV,IAAI,EACJ,aAAa,EACb,aAAa,CACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,qBAAqB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
@@ -1,4 +0,0 @@
1
- export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
2
- text: string;
3
- size: number;
4
- }>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/response.ts"],"names":[],"mappings":"AAmDA,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CASzC"}
@@ -1,104 +0,0 @@
1
- import { FetchError } from '../../errors/app-error.js';
2
- function assertContentLengthWithinLimit(response, url, maxBytes) {
3
- const contentLengthHeader = response.headers.get('content-length');
4
- if (!contentLengthHeader)
5
- return;
6
- const contentLength = Number.parseInt(contentLengthHeader, 10);
7
- if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
8
- return;
9
- }
10
- cancelResponseBody(response);
11
- throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
12
- }
13
- function createReadState() {
14
- return {
15
- decoder: new TextDecoder(),
16
- parts: [],
17
- total: 0,
18
- };
19
- }
20
- function appendChunk(state, chunk, maxBytes, url) {
21
- state.total += chunk.byteLength;
22
- if (state.total > maxBytes) {
23
- throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
24
- }
25
- const decoded = state.decoder.decode(chunk, { stream: true });
26
- if (decoded)
27
- state.parts.push(decoded);
28
- }
29
- function finalizeRead(state) {
30
- const decoded = state.decoder.decode();
31
- if (decoded)
32
- state.parts.push(decoded);
33
- }
34
- function createAbortError(url) {
35
- return new FetchError('Request was aborted during response read', url, 499, {
36
- reason: 'aborted',
37
- });
38
- }
39
- function cancelResponseBody(response) {
40
- const cancelPromise = response.body?.cancel();
41
- if (cancelPromise) {
42
- cancelPromise.catch(() => {
43
- // Best-effort cancellation; ignore failures.
44
- });
45
- }
46
- }
47
- async function cancelReaderQuietly(reader) {
48
- try {
49
- await reader.cancel();
50
- }
51
- catch {
52
- // Ignore cancel errors; we're already failing this read.
53
- }
54
- }
55
- async function throwIfAborted(signal, url, reader) {
56
- if (!signal?.aborted)
57
- return;
58
- await cancelReaderQuietly(reader);
59
- throw createAbortError(url);
60
- }
61
- async function handleReadFailure(error, signal, url, reader) {
62
- const aborted = signal?.aborted ?? false;
63
- await cancelReaderQuietly(reader);
64
- if (aborted) {
65
- throw createAbortError(url);
66
- }
67
- throw error;
68
- }
69
- async function readAllChunks(reader, state, url, maxBytes, signal) {
70
- await throwIfAborted(signal, url, reader);
71
- let result = await reader.read();
72
- while (!result.done) {
73
- appendChunk(state, result.value, maxBytes, url);
74
- await throwIfAborted(signal, url, reader);
75
- result = await reader.read();
76
- }
77
- }
78
- async function readStreamWithLimit(stream, url, maxBytes, signal) {
79
- const state = createReadState();
80
- const reader = stream.getReader();
81
- try {
82
- await readAllChunks(reader, state, url, maxBytes, signal);
83
- }
84
- catch (error) {
85
- await handleReadFailure(error, signal, url, reader);
86
- }
87
- finally {
88
- reader.releaseLock();
89
- }
90
- finalizeRead(state);
91
- return { text: state.parts.join(''), size: state.total };
92
- }
93
- export async function readResponseText(response, url, maxBytes, signal) {
94
- assertContentLengthWithinLimit(response, url, maxBytes);
95
- if (!response.body) {
96
- const text = await response.text();
97
- const size = Buffer.byteLength(text);
98
- if (size > maxBytes) {
99
- throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
100
- }
101
- return { text, size };
102
- }
103
- return readStreamWithLimit(response.body, url, maxBytes, signal);
104
- }
@@ -1 +0,0 @@
1
- {"version":3,"file":"response.js","sourceRoot":"","sources":["../../../src/services/fetcher/response.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,SAAS,8BAA8B,CACrC,QAAkB,EAClB,GAAW,EACX,QAAgB;IAEhB,MAAM,mBAAmB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACnE,IAAI,CAAC,mBAAmB;QAAE,OAAO;IACjC,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,EAAE,CAAC;QAC7D,OAAO;IACT,CAAC;IAED,MAAM,IAAI,UAAU,CAClB,oCAAoC,QAAQ,QAAQ,EACpD,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,MAAkC,EAClC,GAAW,EACX,QAAgB;IAEhB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,SAAS,CAAC;QACR,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI;YAAE,MAAM;QAEhB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;QAE1B,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,UAAU,CAClB,oCAAoC,QAAQ,QAAQ,EACpD,GAAG,CACJ,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IACzB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAkB,EAClB,GAAW,EACX,QAAgB;IAEhB,8BAA8B,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;IAExD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;IACjD,CAAC;IAED,OAAO,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;AAC3D,CAAC"}
@@ -1 +0,0 @@
1
- export declare function executeWithRetry<T>(url: string, maxRetries: number, operation: () => Promise<T>, signal?: AbortSignal): Promise<T>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"retry-policy.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/retry-policy.ts"],"names":[],"mappings":"AAUA,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,CAAC,CAWZ"}
@@ -1,131 +0,0 @@
1
- import { setTimeout } from 'node:timers/promises';
2
- import { FetchError } from '../../errors/app-error.js';
3
- import { logDebug, logWarn } from '../logger.js';
4
- const BASE_DELAY_MS = 1000;
5
- const MAX_DELAY_MS = 10000;
6
- const JITTER_FACTOR = 0.25;
7
- export async function executeWithRetry(url, maxRetries, operation, signal) {
8
- let lastError = new Error(`Failed to fetch ${url}`);
9
- const retries = normalizeRetries(maxRetries);
10
- for (let attempt = 1; attempt <= retries; attempt++) {
11
- const result = await runAttempt(url, operation, attempt, retries, signal);
12
- if (result.done)
13
- return result.value;
14
- lastError = result.error;
15
- }
16
- throw buildFinalError(url, retries, lastError);
17
- }
18
- async function runAttempt(url, operation, attempt, retries, signal) {
19
- throwIfAborted(url, signal);
20
- try {
21
- const value = await operation();
22
- return { done: true, value };
23
- }
24
- catch (error) {
25
- const normalizedError = normalizeError(error);
26
- throwIfNotRetryable(attempt, retries, normalizedError);
27
- await wait(url, attempt, normalizedError, signal);
28
- return { done: false, error: normalizedError };
29
- }
30
- }
31
- function throwIfNotRetryable(attempt, retries, error) {
32
- if (!shouldRetry(attempt, retries, error)) {
33
- throw error;
34
- }
35
- }
36
- function shouldRetry(attempt, maxRetries, error) {
37
- if (attempt >= maxRetries)
38
- return false;
39
- if (!(error instanceof FetchError))
40
- return true;
41
- if (isAbortError(error))
42
- return false;
43
- if (isRateLimited(error))
44
- return true;
45
- return !isClientError(error);
46
- }
47
- async function wait(url, attempt, error, signal) {
48
- const delay = calculateDelay(attempt, error);
49
- logRetryDelay(url, attempt, delay, error);
50
- await sleep(url, delay, signal);
51
- }
52
- function calculateDelay(attempt, error) {
53
- const rateLimitDelay = getRateLimitDelay(error);
54
- if (rateLimitDelay !== null)
55
- return rateLimitDelay;
56
- const exponentialDelay = Math.min(BASE_DELAY_MS * Math.pow(2, attempt - 1), MAX_DELAY_MS);
57
- const jitter = exponentialDelay * JITTER_FACTOR * (Math.random() * 2 - 1);
58
- return Math.round(exponentialDelay + jitter);
59
- }
60
- function normalizeRetries(maxRetries) {
61
- return Math.min(Math.max(1, maxRetries), 10);
62
- }
63
- function throwIfAborted(url, signal) {
64
- if (!signal?.aborted)
65
- return;
66
- throw new FetchError('Request was aborted before execution', url);
67
- }
68
- function normalizeError(error) {
69
- return error instanceof Error ? error : new Error(String(error));
70
- }
71
- function buildFinalError(url, retries, error) {
72
- return new FetchError(`Failed after ${retries} attempts: ${error.message}`, url);
73
- }
74
- function isAbortError(error) {
75
- return error.details.reason === 'aborted';
76
- }
77
- function isRateLimited(error) {
78
- return error.details.httpStatus === 429;
79
- }
80
- function isClientError(error) {
81
- const status = error.details.httpStatus;
82
- return typeof status === 'number' && status >= 400 && status < 500;
83
- }
84
- function logRetryDelay(url, attempt, delay, error) {
85
- if (isRateLimitLog(error)) {
86
- logWarn('Rate limited, waiting before retry', {
87
- url,
88
- attempt,
89
- waitTime: `${delay}ms`,
90
- });
91
- return;
92
- }
93
- logDebug('Retrying request', {
94
- url,
95
- attempt,
96
- delay: `${delay}ms`,
97
- });
98
- }
99
- function isRateLimitLog(error) {
100
- return error instanceof FetchError && error.details.httpStatus === 429;
101
- }
102
- async function sleep(url, delay, signal) {
103
- try {
104
- await setTimeout(delay, undefined, { signal });
105
- }
106
- catch (timeoutError) {
107
- handleSleepError(url, timeoutError);
108
- }
109
- }
110
- function handleSleepError(url, error) {
111
- if (isAbortTimeout(error)) {
112
- throw new FetchError('Request was aborted during retry wait', url, 499, {
113
- reason: 'aborted',
114
- });
115
- }
116
- throw error;
117
- }
118
- function isAbortTimeout(error) {
119
- return (error instanceof Error &&
120
- (error.name === 'AbortError' || error.name === 'TimeoutError'));
121
- }
122
- function getRateLimitDelay(error) {
123
- if (!(error instanceof FetchError))
124
- return null;
125
- if (error.details.httpStatus !== 429)
126
- return null;
127
- const retryAfter = typeof error.details.retryAfter === 'number'
128
- ? error.details.retryAfter
129
- : 60;
130
- return Math.min(retryAfter * 1000, 30000);
131
- }