@j0hanz/superfetch 2.1.4 → 2.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (550) hide show
  1. package/dist/cache.d.ts +1 -0
  2. package/dist/cache.d.ts.map +1 -0
  3. package/dist/cache.js +1 -0
  4. package/dist/cache.js.map +1 -0
  5. package/dist/config.d.ts +1 -0
  6. package/dist/config.d.ts.map +1 -0
  7. package/dist/config.js +1 -0
  8. package/dist/config.js.map +1 -0
  9. package/dist/crypto.d.ts +1 -0
  10. package/dist/crypto.d.ts.map +1 -0
  11. package/dist/crypto.js +1 -0
  12. package/dist/crypto.js.map +1 -0
  13. package/dist/errors.d.ts +1 -0
  14. package/dist/errors.d.ts.map +1 -0
  15. package/dist/errors.js +1 -0
  16. package/dist/errors.js.map +1 -0
  17. package/dist/fetch.d.ts +1 -0
  18. package/dist/fetch.d.ts.map +1 -0
  19. package/dist/fetch.js +1 -0
  20. package/dist/fetch.js.map +1 -0
  21. package/dist/http.d.ts +1 -0
  22. package/dist/http.d.ts.map +1 -0
  23. package/dist/http.js +1 -0
  24. package/dist/http.js.map +1 -0
  25. package/dist/index.d.ts +1 -0
  26. package/dist/index.js +1 -0
  27. package/dist/index.js.map +1 -1
  28. package/dist/mcp.d.ts +1 -0
  29. package/dist/mcp.d.ts.map +1 -0
  30. package/dist/mcp.js +1 -0
  31. package/dist/mcp.js.map +1 -0
  32. package/dist/observability.d.ts +1 -0
  33. package/dist/observability.d.ts.map +1 -0
  34. package/dist/observability.js +1 -0
  35. package/dist/observability.js.map +1 -0
  36. package/dist/tools.d.ts +1 -0
  37. package/dist/tools.d.ts.map +1 -0
  38. package/dist/tools.js +1 -0
  39. package/dist/tools.js.map +1 -0
  40. package/dist/transform.d.ts +1 -0
  41. package/dist/transform.d.ts.map +1 -0
  42. package/dist/transform.js +1 -0
  43. package/dist/transform.js.map +1 -0
  44. package/dist/utils.d.ts +1 -0
  45. package/dist/utils.d.ts.map +1 -0
  46. package/dist/utils.js +1 -0
  47. package/dist/utils.js.map +1 -0
  48. package/dist/workers/transform-worker.d.ts +1 -0
  49. package/dist/workers/transform-worker.d.ts.map +1 -0
  50. package/dist/workers/transform-worker.js +1 -0
  51. package/dist/workers/transform-worker.js.map +1 -0
  52. package/package.json +5 -2
  53. package/dist/config/auth-config.d.ts +0 -16
  54. package/dist/config/auth-config.js +0 -53
  55. package/dist/config/constants.d.ts +0 -17
  56. package/dist/config/constants.d.ts.map +0 -1
  57. package/dist/config/constants.js +0 -22
  58. package/dist/config/constants.js.map +0 -1
  59. package/dist/config/env-parsers.d.ts +0 -8
  60. package/dist/config/env-parsers.js +0 -96
  61. package/dist/config/formatting.d.ts +0 -7
  62. package/dist/config/formatting.d.ts.map +0 -1
  63. package/dist/config/formatting.js +0 -9
  64. package/dist/config/formatting.js.map +0 -1
  65. package/dist/config/index.d.ts +0 -61
  66. package/dist/config/index.d.ts.map +0 -1
  67. package/dist/config/index.js +0 -107
  68. package/dist/config/index.js.map +0 -1
  69. package/dist/config/types/content.d.ts +0 -41
  70. package/dist/config/types/content.d.ts.map +0 -1
  71. package/dist/config/types/content.js +0 -1
  72. package/dist/config/types/content.js.map +0 -1
  73. package/dist/config/types/runtime.d.ts +0 -72
  74. package/dist/config/types/runtime.d.ts.map +0 -1
  75. package/dist/config/types/runtime.js +0 -1
  76. package/dist/config/types/runtime.js.map +0 -1
  77. package/dist/config/types/tools.d.ts +0 -28
  78. package/dist/config/types/tools.d.ts.map +0 -1
  79. package/dist/config/types/tools.js +0 -1
  80. package/dist/config/types/tools.js.map +0 -1
  81. package/dist/config/types.d.ts +0 -4
  82. package/dist/config/types.d.ts.map +0 -1
  83. package/dist/config/types.js +0 -2
  84. package/dist/config/types.js.map +0 -1
  85. package/dist/errors/app-error.d.ts +0 -7
  86. package/dist/errors/app-error.d.ts.map +0 -1
  87. package/dist/errors/app-error.js +0 -16
  88. package/dist/errors/app-error.js.map +0 -1
  89. package/dist/errors/index.d.ts +0 -2
  90. package/dist/errors/index.d.ts.map +0 -1
  91. package/dist/errors/index.js +0 -2
  92. package/dist/errors/index.js.map +0 -1
  93. package/dist/http/accept-policy.d.ts +0 -3
  94. package/dist/http/accept-policy.js +0 -45
  95. package/dist/http/async-handler.d.ts +0 -2
  96. package/dist/http/async-handler.js +0 -5
  97. package/dist/http/auth-introspection.d.ts +0 -2
  98. package/dist/http/auth-introspection.js +0 -141
  99. package/dist/http/auth-static.d.ts +0 -2
  100. package/dist/http/auth-static.js +0 -23
  101. package/dist/http/auth.d.ts +0 -3
  102. package/dist/http/auth.d.ts.map +0 -1
  103. package/dist/http/auth.js +0 -269
  104. package/dist/http/auth.js.map +0 -1
  105. package/dist/http/base-middleware.d.ts +0 -7
  106. package/dist/http/base-middleware.js +0 -143
  107. package/dist/http/cors.d.ts +0 -2
  108. package/dist/http/cors.d.ts.map +0 -1
  109. package/dist/http/cors.js +0 -9
  110. package/dist/http/cors.js.map +0 -1
  111. package/dist/http/download-routes.d.ts +0 -2
  112. package/dist/http/download-routes.d.ts.map +0 -1
  113. package/dist/http/download-routes.js +0 -104
  114. package/dist/http/download-routes.js.map +0 -1
  115. package/dist/http/error-handler.d.ts +0 -2
  116. package/dist/http/error-handler.js +0 -55
  117. package/dist/http/host-allowlist.d.ts +0 -3
  118. package/dist/http/host-allowlist.js +0 -117
  119. package/dist/http/jsonrpc-http.d.ts +0 -2
  120. package/dist/http/jsonrpc-http.js +0 -10
  121. package/dist/http/mcp-routes.d.ts +0 -9
  122. package/dist/http/mcp-routes.d.ts.map +0 -1
  123. package/dist/http/mcp-routes.js +0 -204
  124. package/dist/http/mcp-routes.js.map +0 -1
  125. package/dist/http/mcp-session-eviction.d.ts +0 -3
  126. package/dist/http/mcp-session-eviction.js +0 -24
  127. package/dist/http/mcp-session-helpers.d.ts +0 -12
  128. package/dist/http/mcp-session-helpers.d.ts.map +0 -1
  129. package/dist/http/mcp-session-helpers.js +0 -64
  130. package/dist/http/mcp-session-helpers.js.map +0 -1
  131. package/dist/http/mcp-session-init.d.ts +0 -7
  132. package/dist/http/mcp-session-init.js +0 -94
  133. package/dist/http/mcp-session-slots.d.ts +0 -17
  134. package/dist/http/mcp-session-slots.js +0 -55
  135. package/dist/http/mcp-session-transport-init.d.ts +0 -7
  136. package/dist/http/mcp-session-transport-init.js +0 -41
  137. package/dist/http/mcp-session-transport.d.ts +0 -7
  138. package/dist/http/mcp-session-transport.js +0 -57
  139. package/dist/http/mcp-session-types.d.ts +0 -5
  140. package/dist/http/mcp-session-types.js +0 -1
  141. package/dist/http/mcp-session.d.ts +0 -10
  142. package/dist/http/mcp-session.d.ts.map +0 -1
  143. package/dist/http/mcp-session.js +0 -23
  144. package/dist/http/mcp-session.js.map +0 -1
  145. package/dist/http/mcp-sessions.d.ts +0 -41
  146. package/dist/http/mcp-sessions.js +0 -392
  147. package/dist/http/mcp-validation.d.ts +0 -3
  148. package/dist/http/mcp-validation.d.ts.map +0 -1
  149. package/dist/http/mcp-validation.js +0 -14
  150. package/dist/http/mcp-validation.js.map +0 -1
  151. package/dist/http/protocol-policy.d.ts +0 -2
  152. package/dist/http/protocol-policy.js +0 -31
  153. package/dist/http/rate-limit.d.ts +0 -12
  154. package/dist/http/rate-limit.d.ts.map +0 -1
  155. package/dist/http/rate-limit.js +0 -93
  156. package/dist/http/rate-limit.js.map +0 -1
  157. package/dist/http/server-config.d.ts +0 -1
  158. package/dist/http/server-config.js +0 -40
  159. package/dist/http/server-middleware.d.ts +0 -7
  160. package/dist/http/server-middleware.d.ts.map +0 -1
  161. package/dist/http/server-middleware.js +0 -52
  162. package/dist/http/server-middleware.js.map +0 -1
  163. package/dist/http/server-shutdown.d.ts +0 -4
  164. package/dist/http/server-shutdown.js +0 -43
  165. package/dist/http/server-tuning.d.ts +0 -9
  166. package/dist/http/server-tuning.js +0 -45
  167. package/dist/http/server.d.ts +0 -3
  168. package/dist/http/server.d.ts.map +0 -1
  169. package/dist/http/server.js +0 -291
  170. package/dist/http/server.js.map +0 -1
  171. package/dist/http/session-cleanup.d.ts +0 -2
  172. package/dist/http/session-cleanup.d.ts.map +0 -1
  173. package/dist/http/session-cleanup.js +0 -40
  174. package/dist/http/session-cleanup.js.map +0 -1
  175. package/dist/http/sessions.d.ts +0 -14
  176. package/dist/http/sessions.d.ts.map +0 -1
  177. package/dist/http/sessions.js +0 -63
  178. package/dist/http/sessions.js.map +0 -1
  179. package/dist/middleware/error-handler.d.ts +0 -2
  180. package/dist/middleware/error-handler.d.ts.map +0 -1
  181. package/dist/middleware/error-handler.js +0 -56
  182. package/dist/middleware/error-handler.js.map +0 -1
  183. package/dist/middleware/rate-limiter.d.ts +0 -16
  184. package/dist/middleware/rate-limiter.d.ts.map +0 -1
  185. package/dist/middleware/rate-limiter.js +0 -111
  186. package/dist/middleware/rate-limiter.js.map +0 -1
  187. package/dist/parsers/base-html-element-parser.d.ts +0 -43
  188. package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
  189. package/dist/parsers/base-html-element-parser.js +0 -59
  190. package/dist/parsers/base-html-element-parser.js.map +0 -1
  191. package/dist/parsers/heading-element-parser.d.ts +0 -14
  192. package/dist/parsers/heading-element-parser.d.ts.map +0 -1
  193. package/dist/parsers/heading-element-parser.js +0 -26
  194. package/dist/parsers/heading-element-parser.js.map +0 -1
  195. package/dist/parsers/image-element-parser.d.ts +0 -16
  196. package/dist/parsers/image-element-parser.d.ts.map +0 -1
  197. package/dist/parsers/image-element-parser.js +0 -33
  198. package/dist/parsers/image-element-parser.js.map +0 -1
  199. package/dist/parsers/link-element-parser.d.ts +0 -15
  200. package/dist/parsers/link-element-parser.d.ts.map +0 -1
  201. package/dist/parsers/link-element-parser.js +0 -28
  202. package/dist/parsers/link-element-parser.js.map +0 -1
  203. package/dist/parsers/open-graph-parser.d.ts +0 -17
  204. package/dist/parsers/open-graph-parser.d.ts.map +0 -1
  205. package/dist/parsers/open-graph-parser.js +0 -41
  206. package/dist/parsers/open-graph-parser.js.map +0 -1
  207. package/dist/parsers/schema-org-parser.d.ts +0 -17
  208. package/dist/parsers/schema-org-parser.d.ts.map +0 -1
  209. package/dist/parsers/schema-org-parser.js +0 -32
  210. package/dist/parsers/schema-org-parser.js.map +0 -1
  211. package/dist/parsers/standard-meta-parser.d.ts +0 -18
  212. package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
  213. package/dist/parsers/standard-meta-parser.js +0 -32
  214. package/dist/parsers/standard-meta-parser.js.map +0 -1
  215. package/dist/parsers/twitter-card-parser.d.ts +0 -17
  216. package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
  217. package/dist/parsers/twitter-card-parser.js +0 -41
  218. package/dist/parsers/twitter-card-parser.js.map +0 -1
  219. package/dist/prompts/index.d.ts +0 -3
  220. package/dist/prompts/index.d.ts.map +0 -1
  221. package/dist/prompts/index.js +0 -73
  222. package/dist/prompts/index.js.map +0 -1
  223. package/dist/resources/cached-content-params.d.ts +0 -5
  224. package/dist/resources/cached-content-params.js +0 -36
  225. package/dist/resources/cached-content.d.ts +0 -2
  226. package/dist/resources/cached-content.d.ts.map +0 -1
  227. package/dist/resources/cached-content.js +0 -132
  228. package/dist/resources/cached-content.js.map +0 -1
  229. package/dist/resources/index.d.ts +0 -2
  230. package/dist/resources/index.d.ts.map +0 -1
  231. package/dist/resources/index.js +0 -4
  232. package/dist/resources/index.js.map +0 -1
  233. package/dist/server.d.ts +0 -3
  234. package/dist/server.d.ts.map +0 -1
  235. package/dist/server.js +0 -94
  236. package/dist/server.js.map +0 -1
  237. package/dist/services/cache-events.d.ts +0 -8
  238. package/dist/services/cache-events.js +0 -19
  239. package/dist/services/cache-keys.d.ts +0 -7
  240. package/dist/services/cache-keys.js +0 -57
  241. package/dist/services/cache.d.ts +0 -17
  242. package/dist/services/cache.d.ts.map +0 -1
  243. package/dist/services/cache.js +0 -145
  244. package/dist/services/cache.js.map +0 -1
  245. package/dist/services/cache.service.d.ts +0 -52
  246. package/dist/services/cache.service.d.ts.map +0 -1
  247. package/dist/services/cache.service.js +0 -113
  248. package/dist/services/cache.service.js.map +0 -1
  249. package/dist/services/card-extractor.d.ts +0 -6
  250. package/dist/services/card-extractor.d.ts.map +0 -1
  251. package/dist/services/card-extractor.js +0 -199
  252. package/dist/services/card-extractor.js.map +0 -1
  253. package/dist/services/context.d.ts +0 -10
  254. package/dist/services/context.d.ts.map +0 -1
  255. package/dist/services/context.js +0 -14
  256. package/dist/services/context.js.map +0 -1
  257. package/dist/services/extractor.d.ts +0 -5
  258. package/dist/services/extractor.d.ts.map +0 -1
  259. package/dist/services/extractor.js +0 -142
  260. package/dist/services/extractor.js.map +0 -1
  261. package/dist/services/extractor.service.d.ts +0 -18
  262. package/dist/services/extractor.service.d.ts.map +0 -1
  263. package/dist/services/extractor.service.js +0 -75
  264. package/dist/services/extractor.service.js.map +0 -1
  265. package/dist/services/fetcher/agents.d.ts +0 -3
  266. package/dist/services/fetcher/agents.d.ts.map +0 -1
  267. package/dist/services/fetcher/agents.js +0 -100
  268. package/dist/services/fetcher/agents.js.map +0 -1
  269. package/dist/services/fetcher/dns-selection.d.ts +0 -2
  270. package/dist/services/fetcher/dns-selection.js +0 -72
  271. package/dist/services/fetcher/errors.d.ts +0 -4
  272. package/dist/services/fetcher/errors.d.ts.map +0 -1
  273. package/dist/services/fetcher/errors.js +0 -70
  274. package/dist/services/fetcher/errors.js.map +0 -1
  275. package/dist/services/fetcher/headers.d.ts +0 -2
  276. package/dist/services/fetcher/headers.d.ts.map +0 -1
  277. package/dist/services/fetcher/headers.js +0 -6
  278. package/dist/services/fetcher/headers.js.map +0 -1
  279. package/dist/services/fetcher/interceptors.d.ts +0 -10
  280. package/dist/services/fetcher/interceptors.d.ts.map +0 -1
  281. package/dist/services/fetcher/interceptors.js +0 -108
  282. package/dist/services/fetcher/interceptors.js.map +0 -1
  283. package/dist/services/fetcher/redirects.d.ts +0 -4
  284. package/dist/services/fetcher/redirects.d.ts.map +0 -1
  285. package/dist/services/fetcher/redirects.js +0 -78
  286. package/dist/services/fetcher/redirects.js.map +0 -1
  287. package/dist/services/fetcher/response.d.ts +0 -4
  288. package/dist/services/fetcher/response.d.ts.map +0 -1
  289. package/dist/services/fetcher/response.js +0 -104
  290. package/dist/services/fetcher/response.js.map +0 -1
  291. package/dist/services/fetcher/retry-policy.d.ts +0 -1
  292. package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
  293. package/dist/services/fetcher/retry-policy.js +0 -131
  294. package/dist/services/fetcher/retry-policy.js.map +0 -1
  295. package/dist/services/fetcher.d.ts +0 -25
  296. package/dist/services/fetcher.d.ts.map +0 -1
  297. package/dist/services/fetcher.js +0 -607
  298. package/dist/services/fetcher.js.map +0 -1
  299. package/dist/services/fetcher.service.d.ts +0 -18
  300. package/dist/services/fetcher.service.d.ts.map +0 -1
  301. package/dist/services/fetcher.service.js +0 -122
  302. package/dist/services/fetcher.service.js.map +0 -1
  303. package/dist/services/fifo-queue.d.ts +0 -8
  304. package/dist/services/fifo-queue.js +0 -25
  305. package/dist/services/logger.d.ts +0 -5
  306. package/dist/services/logger.d.ts.map +0 -1
  307. package/dist/services/logger.js +0 -52
  308. package/dist/services/logger.js.map +0 -1
  309. package/dist/services/logger.service.d.ts +0 -5
  310. package/dist/services/logger.service.d.ts.map +0 -1
  311. package/dist/services/logger.service.js +0 -57
  312. package/dist/services/logger.service.js.map +0 -1
  313. package/dist/services/metadata-collector.d.ts +0 -2
  314. package/dist/services/metadata-collector.js +0 -80
  315. package/dist/services/parser.d.ts +0 -6
  316. package/dist/services/parser.d.ts.map +0 -1
  317. package/dist/services/parser.js +0 -278
  318. package/dist/services/parser.js.map +0 -1
  319. package/dist/services/parser.service.d.ts +0 -42
  320. package/dist/services/parser.service.d.ts.map +0 -1
  321. package/dist/services/parser.service.js +0 -209
  322. package/dist/services/parser.service.js.map +0 -1
  323. package/dist/services/session-manager.d.ts +0 -18
  324. package/dist/services/session-manager.d.ts.map +0 -1
  325. package/dist/services/session-manager.js +0 -73
  326. package/dist/services/session-manager.js.map +0 -1
  327. package/dist/services/telemetry.d.ts +0 -19
  328. package/dist/services/telemetry.js +0 -43
  329. package/dist/services/transform-worker-pool.d.ts +0 -11
  330. package/dist/services/transform-worker-pool.js +0 -244
  331. package/dist/services/transform-worker-types.d.ts +0 -32
  332. package/dist/services/transform-worker-types.js +0 -14
  333. package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
  334. package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
  335. package/dist/strategies/exponential-backoff-strategy.js +0 -32
  336. package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
  337. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
  338. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
  339. package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
  340. package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
  341. package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
  342. package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
  343. package/dist/tools/handlers/fetch-links.tool.js +0 -98
  344. package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
  345. package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
  346. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
  347. package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
  348. package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
  349. package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
  350. package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
  351. package/dist/tools/handlers/fetch-single.shared.js +0 -223
  352. package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
  353. package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
  354. package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
  355. package/dist/tools/handlers/fetch-url.tool.js +0 -119
  356. package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
  357. package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
  358. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
  359. package/dist/tools/handlers/fetch-urls/processor.js +0 -153
  360. package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
  361. package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
  362. package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
  363. package/dist/tools/handlers/fetch-urls/response.js +0 -79
  364. package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
  365. package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
  366. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
  367. package/dist/tools/handlers/fetch-urls/validation.js +0 -18
  368. package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
  369. package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
  370. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
  371. package/dist/tools/handlers/fetch-urls.tool.js +0 -124
  372. package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
  373. package/dist/tools/index.d.ts +0 -3
  374. package/dist/tools/index.d.ts.map +0 -1
  375. package/dist/tools/index.js +0 -37
  376. package/dist/tools/index.js.map +0 -1
  377. package/dist/tools/schemas.d.ts +0 -12
  378. package/dist/tools/schemas.d.ts.map +0 -1
  379. package/dist/tools/schemas.js +0 -21
  380. package/dist/tools/schemas.js.map +0 -1
  381. package/dist/tools/utils/cache-vary.d.ts +0 -1
  382. package/dist/tools/utils/cache-vary.d.ts.map +0 -1
  383. package/dist/tools/utils/cache-vary.js +0 -12
  384. package/dist/tools/utils/cache-vary.js.map +0 -1
  385. package/dist/tools/utils/cached-markdown.d.ts +0 -5
  386. package/dist/tools/utils/cached-markdown.js +0 -46
  387. package/dist/tools/utils/common.d.ts +0 -5
  388. package/dist/tools/utils/common.d.ts.map +0 -1
  389. package/dist/tools/utils/common.js +0 -42
  390. package/dist/tools/utils/common.js.map +0 -1
  391. package/dist/tools/utils/content-shaping.d.ts +0 -4
  392. package/dist/tools/utils/content-shaping.js +0 -67
  393. package/dist/tools/utils/content-transform-async.d.ts +0 -6
  394. package/dist/tools/utils/content-transform-async.js +0 -33
  395. package/dist/tools/utils/content-transform-core.d.ts +0 -5
  396. package/dist/tools/utils/content-transform-core.js +0 -180
  397. package/dist/tools/utils/content-transform-workers.d.ts +0 -1
  398. package/dist/tools/utils/content-transform-workers.js +0 -1
  399. package/dist/tools/utils/content-transform.d.ts +0 -3
  400. package/dist/tools/utils/content-transform.d.ts.map +0 -1
  401. package/dist/tools/utils/content-transform.js +0 -41
  402. package/dist/tools/utils/content-transform.js.map +0 -1
  403. package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
  404. package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
  405. package/dist/tools/utils/fetch-pipeline.js +0 -98
  406. package/dist/tools/utils/fetch-pipeline.js.map +0 -1
  407. package/dist/tools/utils/frontmatter.d.ts +0 -3
  408. package/dist/tools/utils/frontmatter.js +0 -73
  409. package/dist/tools/utils/index.d.ts +0 -4
  410. package/dist/tools/utils/index.d.ts.map +0 -1
  411. package/dist/tools/utils/index.js +0 -3
  412. package/dist/tools/utils/index.js.map +0 -1
  413. package/dist/tools/utils/inline-content.d.ts +0 -10
  414. package/dist/tools/utils/inline-content.d.ts.map +0 -1
  415. package/dist/tools/utils/inline-content.js +0 -35
  416. package/dist/tools/utils/inline-content.js.map +0 -1
  417. package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
  418. package/dist/tools/utils/markdown-heuristics.js +0 -19
  419. package/dist/tools/utils/markdown-signals.d.ts +0 -1
  420. package/dist/tools/utils/markdown-signals.js +0 -19
  421. package/dist/tools/utils/markdown-toc.d.ts +0 -3
  422. package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
  423. package/dist/tools/utils/markdown-toc.js +0 -35
  424. package/dist/tools/utils/markdown-toc.js.map +0 -1
  425. package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
  426. package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
  427. package/dist/tools/utils/raw-markdown.d.ts +0 -6
  428. package/dist/tools/utils/raw-markdown.js +0 -149
  429. package/dist/tools/utils/response-builder.d.ts +0 -3
  430. package/dist/tools/utils/response-builder.d.ts.map +0 -1
  431. package/dist/tools/utils/response-builder.js +0 -24
  432. package/dist/tools/utils/response-builder.js.map +0 -1
  433. package/dist/tools/utils/tool-response.d.ts +0 -9
  434. package/dist/tools/utils/tool-response.d.ts.map +0 -1
  435. package/dist/tools/utils/tool-response.js +0 -19
  436. package/dist/tools/utils/tool-response.js.map +0 -1
  437. package/dist/transformers/jsonl.transformer.d.ts +0 -2
  438. package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
  439. package/dist/transformers/jsonl.transformer.js +0 -75
  440. package/dist/transformers/jsonl.transformer.js.map +0 -1
  441. package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
  442. package/dist/transformers/markdown/fenced-code-rule.js +0 -38
  443. package/dist/transformers/markdown/frontmatter.d.ts +0 -2
  444. package/dist/transformers/markdown/frontmatter.js +0 -45
  445. package/dist/transformers/markdown/noise-rule.d.ts +0 -2
  446. package/dist/transformers/markdown/noise-rule.js +0 -80
  447. package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
  448. package/dist/transformers/markdown/turndown-instance.js +0 -19
  449. package/dist/transformers/markdown.d.ts +0 -5
  450. package/dist/transformers/markdown.js +0 -314
  451. package/dist/transformers/markdown.transformer.d.ts +0 -2
  452. package/dist/transformers/markdown.transformer.d.ts.map +0 -1
  453. package/dist/transformers/markdown.transformer.js +0 -14
  454. package/dist/transformers/markdown.transformer.js.map +0 -1
  455. package/dist/types/content.types.d.ts +0 -63
  456. package/dist/types/content.types.d.ts.map +0 -1
  457. package/dist/types/content.types.js +0 -2
  458. package/dist/types/content.types.js.map +0 -1
  459. package/dist/types/index.d.ts +0 -2
  460. package/dist/types/index.d.ts.map +0 -1
  461. package/dist/types/index.js +0 -2
  462. package/dist/types/index.js.map +0 -1
  463. package/dist/types/schemas.d.ts +0 -49
  464. package/dist/types/schemas.d.ts.map +0 -1
  465. package/dist/types/schemas.js +0 -5
  466. package/dist/types/schemas.js.map +0 -1
  467. package/dist/utils/cached-payload.d.ts +0 -7
  468. package/dist/utils/cached-payload.js +0 -36
  469. package/dist/utils/cancellation.d.ts +0 -1
  470. package/dist/utils/cancellation.js +0 -18
  471. package/dist/utils/code-language-bash.d.ts +0 -1
  472. package/dist/utils/code-language-bash.js +0 -48
  473. package/dist/utils/code-language-core.d.ts +0 -2
  474. package/dist/utils/code-language-core.js +0 -13
  475. package/dist/utils/code-language-detectors.d.ts +0 -5
  476. package/dist/utils/code-language-detectors.js +0 -142
  477. package/dist/utils/code-language-helpers.d.ts +0 -5
  478. package/dist/utils/code-language-helpers.js +0 -62
  479. package/dist/utils/code-language-parsing.d.ts +0 -5
  480. package/dist/utils/code-language-parsing.js +0 -62
  481. package/dist/utils/code-language.d.ts +0 -2
  482. package/dist/utils/code-language.d.ts.map +0 -1
  483. package/dist/utils/code-language.js +0 -260
  484. package/dist/utils/code-language.js.map +0 -1
  485. package/dist/utils/concurrency.d.ts +0 -3
  486. package/dist/utils/concurrency.d.ts.map +0 -1
  487. package/dist/utils/concurrency.js +0 -38
  488. package/dist/utils/concurrency.js.map +0 -1
  489. package/dist/utils/content-cleaner.d.ts +0 -5
  490. package/dist/utils/content-cleaner.d.ts.map +0 -1
  491. package/dist/utils/content-cleaner.js +0 -77
  492. package/dist/utils/content-cleaner.js.map +0 -1
  493. package/dist/utils/crypto.d.ts +0 -2
  494. package/dist/utils/crypto.d.ts.map +0 -1
  495. package/dist/utils/crypto.js +0 -32
  496. package/dist/utils/crypto.js.map +0 -1
  497. package/dist/utils/download-url.d.ts +0 -16
  498. package/dist/utils/download-url.d.ts.map +0 -1
  499. package/dist/utils/download-url.js +0 -30
  500. package/dist/utils/download-url.js.map +0 -1
  501. package/dist/utils/error-details.d.ts +0 -3
  502. package/dist/utils/error-details.js +0 -12
  503. package/dist/utils/error-utils.d.ts +0 -3
  504. package/dist/utils/error-utils.d.ts.map +0 -1
  505. package/dist/utils/error-utils.js +0 -12
  506. package/dist/utils/error-utils.js.map +0 -1
  507. package/dist/utils/filename-generator.d.ts +0 -1
  508. package/dist/utils/filename-generator.d.ts.map +0 -1
  509. package/dist/utils/filename-generator.js +0 -81
  510. package/dist/utils/filename-generator.js.map +0 -1
  511. package/dist/utils/guards.d.ts +0 -1
  512. package/dist/utils/guards.js +0 -3
  513. package/dist/utils/header-normalizer.d.ts +0 -5
  514. package/dist/utils/header-normalizer.d.ts.map +0 -1
  515. package/dist/utils/header-normalizer.js +0 -31
  516. package/dist/utils/header-normalizer.js.map +0 -1
  517. package/dist/utils/host-normalizer.d.ts +0 -1
  518. package/dist/utils/host-normalizer.js +0 -37
  519. package/dist/utils/html-truncator.d.ts +0 -1
  520. package/dist/utils/html-truncator.d.ts.map +0 -1
  521. package/dist/utils/html-truncator.js +0 -13
  522. package/dist/utils/html-truncator.js.map +0 -1
  523. package/dist/utils/ip-address.d.ts +0 -4
  524. package/dist/utils/ip-address.js +0 -6
  525. package/dist/utils/language-detector.d.ts +0 -2
  526. package/dist/utils/language-detector.d.ts.map +0 -1
  527. package/dist/utils/language-detector.js +0 -39
  528. package/dist/utils/language-detector.js.map +0 -1
  529. package/dist/utils/sanitizer.d.ts +0 -2
  530. package/dist/utils/sanitizer.d.ts.map +0 -1
  531. package/dist/utils/sanitizer.js +0 -20
  532. package/dist/utils/sanitizer.js.map +0 -1
  533. package/dist/utils/tool-error-handler.d.ts +0 -3
  534. package/dist/utils/tool-error-handler.d.ts.map +0 -1
  535. package/dist/utils/tool-error-handler.js +0 -31
  536. package/dist/utils/tool-error-handler.js.map +0 -1
  537. package/dist/utils/url-redactor.d.ts +0 -1
  538. package/dist/utils/url-redactor.js +0 -13
  539. package/dist/utils/url-sanitizer.d.ts +0 -2
  540. package/dist/utils/url-sanitizer.d.ts.map +0 -1
  541. package/dist/utils/url-sanitizer.js +0 -12
  542. package/dist/utils/url-sanitizer.js.map +0 -1
  543. package/dist/utils/url-transformer.d.ts +0 -7
  544. package/dist/utils/url-transformer.js +0 -147
  545. package/dist/utils/url-validator.d.ts +0 -6
  546. package/dist/utils/url-validator.d.ts.map +0 -1
  547. package/dist/utils/url-validator.js +0 -156
  548. package/dist/utils/url-validator.js.map +0 -1
  549. package/dist/workers/content-transform.worker.d.ts +0 -1
  550. package/dist/workers/content-transform.worker.js +0 -40
@@ -1,122 +0,0 @@
1
- import axios from 'axios';
2
- import { config } from '../config/index.js';
3
- import { FetchError, TimeoutError } from '../errors/app-error.js';
4
- // Headers that should not be overridden by custom headers (security)
5
- const BLOCKED_HEADERS = new Set([
6
- 'host',
7
- 'authorization',
8
- 'cookie',
9
- 'x-forwarded-for',
10
- 'x-real-ip',
11
- 'proxy-authorization',
12
- ]);
13
- /**
14
- * Sanitize custom headers to prevent injection of sensitive headers
15
- */
16
- function sanitizeHeaders(headers) {
17
- if (!headers || Object.keys(headers).length === 0) {
18
- return undefined;
19
- }
20
- const sanitized = {};
21
- for (const [key, value] of Object.entries(headers)) {
22
- if (!BLOCKED_HEADERS.has(key.toLowerCase())) {
23
- sanitized[key] = value;
24
- }
25
- }
26
- return Object.keys(sanitized).length > 0 ? sanitized : undefined;
27
- }
28
- /**
29
- * Calculates exponential backoff delay with jitter
30
- */
31
- function calculateBackoff(attempt, maxDelay = 10000) {
32
- const baseDelay = Math.min(1000 * Math.pow(2, attempt - 1), maxDelay);
33
- // Add jitter (±25%)
34
- const jitter = baseDelay * 0.25 * (Math.random() * 2 - 1);
35
- return Math.round(baseDelay + jitter);
36
- }
37
- class FetcherService {
38
- client;
39
- constructor() {
40
- this.client = axios.create({
41
- timeout: config.fetcher.timeout,
42
- maxRedirects: config.fetcher.maxRedirects,
43
- maxContentLength: config.fetcher.maxContentLength,
44
- headers: {
45
- 'User-Agent': config.fetcher.userAgent,
46
- Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
47
- 'Accept-Language': 'en-US,en;q=0.5',
48
- 'Accept-Encoding': 'gzip, deflate, br',
49
- Connection: 'keep-alive',
50
- },
51
- // Validate status - only 2xx responses are valid
52
- validateStatus: (status) => status >= 200 && status < 300,
53
- });
54
- }
55
- /**
56
- * Fetches HTML content from a URL
57
- * @throws {FetchError} on network or HTTP errors
58
- * @throws {TimeoutError} on request timeout
59
- */
60
- async fetchUrl(url, customHeaders) {
61
- const requestConfig = {
62
- method: 'GET',
63
- url,
64
- responseType: 'text',
65
- };
66
- // Add sanitized custom headers if provided
67
- const sanitized = sanitizeHeaders(customHeaders);
68
- if (sanitized) {
69
- requestConfig.headers = { ...requestConfig.headers, ...sanitized };
70
- }
71
- try {
72
- const response = await this.client.request(requestConfig);
73
- return response.data;
74
- }
75
- catch (error) {
76
- if (!axios.isAxiosError(error)) {
77
- throw new FetchError(`Unexpected error: ${error instanceof Error ? error.message : 'Unknown'}`, url);
78
- }
79
- const axiosError = error;
80
- if (axiosError.code === 'ECONNABORTED' || axiosError.code === 'ETIMEDOUT') {
81
- throw new TimeoutError(config.fetcher.timeout, true);
82
- }
83
- if (axiosError.response) {
84
- throw new FetchError(`HTTP ${axiosError.response.status}: ${axiosError.response.statusText}`, url, axiosError.response.status);
85
- }
86
- if (axiosError.request) {
87
- throw new FetchError(`Network error: Could not reach ${url}`, url);
88
- }
89
- throw new FetchError(axiosError.message, url);
90
- }
91
- }
92
- /**
93
- * Fetches URL with exponential backoff retry logic
94
- * @throws {FetchError} after all retries exhausted
95
- */
96
- async fetchUrlWithRetry(url, customHeaders, maxRetries = 3) {
97
- let lastError;
98
- for (let attempt = 1; attempt <= maxRetries; attempt++) {
99
- try {
100
- return await this.fetchUrl(url, customHeaders);
101
- }
102
- catch (error) {
103
- lastError = error instanceof Error ? error : new Error('Unknown error');
104
- // Don't retry on client errors (4xx) except 429 (rate limited)
105
- if (error instanceof FetchError && error.httpStatus) {
106
- const status = error.httpStatus;
107
- if (status >= 400 && status < 500 && status !== 429) {
108
- throw error;
109
- }
110
- }
111
- if (attempt < maxRetries) {
112
- const delay = calculateBackoff(attempt);
113
- await new Promise((resolve) => setTimeout(resolve, delay));
114
- }
115
- }
116
- }
117
- throw new FetchError(`Failed after ${maxRetries} attempts: ${lastError?.message ?? 'Unknown error'}`, url);
118
- }
119
- }
120
- // Singleton instance
121
- export const fetcherService = new FetcherService();
122
- //# sourceMappingURL=fetcher.service.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetcher.service.js","sourceRoot":"","sources":["../../src/services/fetcher.service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAwD,MAAM,OAAO,CAAC;AAC7E,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAElE,qEAAqE;AACrE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,MAAM;IACN,eAAe;IACf,QAAQ;IACR,iBAAiB;IACjB,WAAW;IACX,qBAAqB;CACtB,CAAC,CAAC;AAEH;;GAEG;AACH,SAAS,eAAe,CACtB,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,SAAS,GAA2B,EAAE,CAAC;IAE7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5C,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe,EAAE,QAAQ,GAAG,KAAK;IACzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtE,oBAAoB;IACpB,MAAM,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,cAAc;IACD,MAAM,CAAgB;IAEvC;QACE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;YAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;YACzC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB;YACjD,OAAO,EAAE;gBACP,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;gBACtC,MAAM,EACJ,4EAA4E;gBAC9E,iBAAiB,EAAE,gBAAgB;gBACnC,iBAAiB,EAAE,mBAAmB;gBACtC,UAAU,EAAE,YAAY;aACzB;YACD,iDAAiD;YACjD,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;SAC1D,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CACZ,GAAW,EACX,aAAsC;QAEtC,MAAM,aAAa,GAAuB;YACxC,MAAM,EAAE,KAAK;YACb,GAAG;YACH,YAAY,EAAE,MAAM;SACrB,CAAC;QAEF,2CAA2C;QAC3C,MAAM,SAAS,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;QACjD,IAAI,SAAS,EAAE,CAAC;YACd,aAAa,CAAC,OAAO,GAAG,EAAE,GAAG,aAAa,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;QACrE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAS,aAAa,CAAC,CAAC;YAClE,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,UAAU,CAClB,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,EACzE,GAAG,CACJ,CAAC;YACJ,CAAC;YAED,MAAM,UAAU,GAAG,KAAmB,CAAC;YAEvC,IAAI,UAAU,CAAC,IAAI,KAAK,cAAc,IAAI,UAAU,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC1E,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACvD,CAAC;YAED,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,MAAM,IAAI,UAAU,CAClB,QAAQ,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,EACvE,GAAG,EACH,UAAU,CAAC,QAAQ,CAAC,MAAM,CAC3B,CAAC;YACJ,CAAC;YAED,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACvB,MAAM,IAAI,UAAU,CAAC,kCAAkC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CACrB,GAAW,EACX,aAAsC,EACtC,UAAU,GAAG,CAAC;QAEd,IAAI,SAA4B,CAAC;QAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;gBAExE,+DAA+D;gBAC/D,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;oBACpD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;oBAChC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;wBACpD,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;oBACzB,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,UAAU,CAClB,gBAAgB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE,EAC/E,GAAG,CACJ,CAAC;IACJ,CAAC;CACF;AAED,qBAAqB;AACrB,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}
@@ -1,8 +0,0 @@
1
- export declare class FifoQueue<T> {
2
- private items;
3
- private head;
4
- get length(): number;
5
- push(item: T): void;
6
- shift(): T | undefined;
7
- clear(): void;
8
- }
@@ -1,25 +0,0 @@
1
- export class FifoQueue {
2
- items = [];
3
- head = 0;
4
- get length() {
5
- return this.items.length - this.head;
6
- }
7
- push(item) {
8
- this.items.push(item);
9
- }
10
- shift() {
11
- if (this.head >= this.items.length)
12
- return undefined;
13
- const item = this.items[this.head];
14
- this.head += 1;
15
- if (this.head > 64 && this.head * 2 >= this.items.length) {
16
- this.items = this.items.slice(this.head);
17
- this.head = 0;
18
- }
19
- return item;
20
- }
21
- clear() {
22
- this.items.length = 0;
23
- this.head = 0;
24
- }
25
- }
@@ -1,5 +0,0 @@
1
- import type { LogMetadata } from '../config/types/runtime.js';
2
- export declare function logInfo(message: string, meta?: LogMetadata): void;
3
- export declare function logDebug(message: string, meta?: LogMetadata): void;
4
- export declare function logWarn(message: string, meta?: LogMetadata): void;
5
- export declare function logError(message: string, error?: Error | LogMetadata): void;
@@ -1 +0,0 @@
1
- {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAkCxE,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIlE;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAIjE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAS3E"}
@@ -1,52 +0,0 @@
1
- import { config } from '../config/index.js';
2
- import { getOperationId, getRequestId, getSessionId } from './context.js';
3
- function formatMetadata(meta) {
4
- const requestId = getRequestId();
5
- const sessionId = getSessionId();
6
- const operationId = getOperationId();
7
- const contextMeta = {};
8
- if (requestId)
9
- contextMeta.requestId = requestId;
10
- if (sessionId)
11
- contextMeta.sessionId = sessionId;
12
- if (operationId)
13
- contextMeta.operationId = operationId;
14
- const merged = { ...contextMeta, ...meta };
15
- return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
16
- }
17
- function createTimestamp() {
18
- return new Date().toISOString();
19
- }
20
- function formatLogEntry(level, message, meta) {
21
- return `[${createTimestamp()}] ${level.toUpperCase()}: ${message}${formatMetadata(meta)}`;
22
- }
23
- function shouldLog(level) {
24
- // Debug logs only when LOG_LEVEL=debug
25
- if (level === 'debug')
26
- return config.logging.level === 'debug';
27
- // All other levels always log
28
- return true;
29
- }
30
- export function logInfo(message, meta) {
31
- if (shouldLog('info')) {
32
- process.stderr.write(`${formatLogEntry('info', message, meta)}\n`);
33
- }
34
- }
35
- export function logDebug(message, meta) {
36
- if (shouldLog('debug')) {
37
- process.stderr.write(`${formatLogEntry('debug', message, meta)}\n`);
38
- }
39
- }
40
- export function logWarn(message, meta) {
41
- if (shouldLog('warn')) {
42
- process.stderr.write(`${formatLogEntry('warn', message, meta)}\n`);
43
- }
44
- }
45
- export function logError(message, error) {
46
- if (!shouldLog('error'))
47
- return;
48
- const errorMeta = error instanceof Error
49
- ? { error: error.message, stack: error.stack }
50
- : (error ?? {});
51
- process.stderr.write(`${formatLogEntry('error', message, errorMeta)}\n`);
52
- }
@@ -1 +0,0 @@
1
- {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAgB,EAAE,CAAC;IACpC,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IACjD,IAAI,SAAS;QAAE,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC;IAEjD,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,OAAe,EACf,IAAkB;IAElB,OAAO,IAAI,eAAe,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,IAAkB;IAC1D,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,IAAkB;IACzD,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,KAA2B;IACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO;IAEhC,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;QAC9C,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC"}
@@ -1,5 +0,0 @@
1
- export declare const logInfo: (message: string, meta?: Record<string, unknown>) => void;
2
- export declare const logWarn: (message: string, meta?: Record<string, unknown>) => void;
3
- export declare const logDebug: (message: string, meta?: Record<string, unknown>) => void;
4
- export declare const logError: (message: string, error?: Error | Record<string, unknown>) => void;
5
- //# sourceMappingURL=logger.service.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"logger.service.d.ts","sourceRoot":"","sources":["../../src/services/logger.service.ts"],"names":[],"mappings":"AA+CA,eAAO,MAAM,OAAO,GAAI,SAAS,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAItE,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,SAAS,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAItE,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,SAAS,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAIvE,CAAC;AAEF,eAAO,MAAM,QAAQ,GACnB,SAAS,MAAM,EACf,QAAQ,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SASxC,CAAC"}
@@ -1,57 +0,0 @@
1
- import winston from 'winston';
2
- import { config } from '../config/index.js';
3
- import path from 'path';
4
- // Create logs directory path
5
- const logsDir = path.join(process.cwd(), 'logs');
6
- // Create logger instance
7
- const logger = winston.createLogger({
8
- level: config.logging.level,
9
- format: winston.format.combine(winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), winston.format.errors({ stack: true }), winston.format.splat(), winston.format.json()),
10
- defaultMeta: { service: 'superfetch' },
11
- transports: [
12
- // Write all logs to combined.log
13
- new winston.transports.File({
14
- filename: path.join(logsDir, 'combined.log'),
15
- maxsize: 5242880, // 5MB
16
- maxFiles: 5,
17
- }),
18
- // Write error logs to error.log
19
- new winston.transports.File({
20
- filename: path.join(logsDir, 'error.log'),
21
- level: 'error',
22
- maxsize: 5242880, // 5MB
23
- maxFiles: 5,
24
- }),
25
- ],
26
- });
27
- // Add console transport in development
28
- if (process.env.NODE_ENV !== 'production') {
29
- logger.add(new winston.transports.Console({
30
- format: winston.format.combine(winston.format.colorize(), winston.format.simple()),
31
- }));
32
- }
33
- // Export convenience methods
34
- export const logInfo = (message, meta) => {
35
- if (config.logging.enabled) {
36
- logger.info(message, meta);
37
- }
38
- };
39
- export const logWarn = (message, meta) => {
40
- if (config.logging.enabled) {
41
- logger.warn(message, meta);
42
- }
43
- };
44
- export const logDebug = (message, meta) => {
45
- if (config.logging.enabled) {
46
- logger.debug(message, meta);
47
- }
48
- };
49
- export const logError = (message, error) => {
50
- if (config.logging.enabled) {
51
- const errorMeta = error instanceof Error
52
- ? { error: error.message, stack: error.stack }
53
- : error;
54
- logger.error(message, errorMeta);
55
- }
56
- };
57
- //# sourceMappingURL=logger.service.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"logger.service.js","sourceRoot":"","sources":["../../src/services/logger.service.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,6BAA6B;AAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;AAEjD,yBAAyB;AACzB,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK;IAC3B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAC5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,EAC3D,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EACtB,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CACtB;IACD,WAAW,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE;IACtC,UAAU,EAAE;QACV,iCAAiC;QACjC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;YAC5C,OAAO,EAAE,OAAO,EAAE,MAAM;YACxB,QAAQ,EAAE,CAAC;SACZ,CAAC;QACF,gCAAgC;QAChC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC;YACzC,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,OAAO,EAAE,MAAM;YACxB,QAAQ,EAAE,CAAC;SACZ,CAAC;KACH;CACF,CAAC,CAAC;AAEH,uCAAuC;AACvC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;IAC1C,MAAM,CAAC,GAAG,CACR,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAC7B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,EACzB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CACxB;KACF,CAAC,CACH,CAAC;AACJ,CAAC;AAED,6BAA6B;AAC7B,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,EAAE;IACzE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,EAAE;IACzE,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,EAAE;IAC1E,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,OAAe,EACf,KAAuC,EACvC,EAAE;IACF,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;YACpB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;YAC9C,CAAC,CAAC,KAAK,CAAC;QACZ,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC,CAAC"}
@@ -1,2 +0,0 @@
1
- import type { ExtractedMetadata } from '../config/types/content.js';
2
- export declare function extractMetadata(document: Document): ExtractedMetadata;
@@ -1,80 +0,0 @@
1
- function createMetaCollectorState() {
2
- return {
3
- title: {},
4
- description: {},
5
- author: {},
6
- };
7
- }
8
- function resolveMetaField(state, field) {
9
- const sources = state[field];
10
- return sources.og ?? sources.twitter ?? sources.standard;
11
- }
12
- function parseOpenGraphKey(property) {
13
- if (!property?.startsWith('og:'))
14
- return null;
15
- const key = property.replace('og:', '');
16
- return key === 'title' || key === 'description' ? key : null;
17
- }
18
- function parseTwitterKey(name) {
19
- if (!name?.startsWith('twitter:'))
20
- return null;
21
- const key = name.replace('twitter:', '');
22
- return key === 'title' || key === 'description' ? key : null;
23
- }
24
- function parseStandardKey(name) {
25
- if (name === 'description')
26
- return 'description';
27
- if (name === 'author')
28
- return 'author';
29
- return null;
30
- }
31
- function collectMetaTag(state, tag) {
32
- const content = tag.getAttribute('content')?.trim();
33
- if (!content)
34
- return;
35
- const ogKey = parseOpenGraphKey(tag.getAttribute('property'));
36
- if (ogKey) {
37
- state[ogKey].og = content;
38
- return;
39
- }
40
- const name = tag.getAttribute('name');
41
- const twitterKey = parseTwitterKey(name);
42
- if (twitterKey) {
43
- state[twitterKey].twitter = content;
44
- return;
45
- }
46
- const standardKey = parseStandardKey(name);
47
- if (standardKey) {
48
- state[standardKey].standard = content;
49
- }
50
- }
51
- function scanMetaTags(document, state) {
52
- const metaTags = document.querySelectorAll('meta');
53
- for (const tag of metaTags) {
54
- collectMetaTag(state, tag);
55
- }
56
- }
57
- function ensureTitleFallback(document, state) {
58
- if (state.title.standard)
59
- return;
60
- const titleEl = document.querySelector('title');
61
- if (titleEl?.textContent) {
62
- state.title.standard = titleEl.textContent.trim();
63
- }
64
- }
65
- export function extractMetadata(document) {
66
- const state = createMetaCollectorState();
67
- scanMetaTags(document, state);
68
- ensureTitleFallback(document, state);
69
- const metadata = {};
70
- const title = resolveMetaField(state, 'title');
71
- const description = resolveMetaField(state, 'description');
72
- const author = resolveMetaField(state, 'author');
73
- if (title !== undefined)
74
- metadata.title = title;
75
- if (description !== undefined)
76
- metadata.description = description;
77
- if (author !== undefined)
78
- metadata.author = author;
79
- return metadata;
80
- }
@@ -1,6 +0,0 @@
1
- import type { ContentBlockUnion, ExtractedMetadata } from '../config/types/content.js';
2
- export declare function parseHtml(html: string): ContentBlockUnion[];
3
- export declare function parseHtmlWithMetadata(html: string): {
4
- blocks: ContentBlockUnion[];
5
- metadata: ExtractedMetadata;
6
- };
@@ -1 +0,0 @@
1
- {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,4BAA4B,CAAC;AAoPpC,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAS3D"}