@j0hanz/superfetch 2.1.4 → 2.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (550) hide show
  1. package/dist/cache.d.ts +1 -0
  2. package/dist/cache.d.ts.map +1 -0
  3. package/dist/cache.js +1 -0
  4. package/dist/cache.js.map +1 -0
  5. package/dist/config.d.ts +1 -0
  6. package/dist/config.d.ts.map +1 -0
  7. package/dist/config.js +1 -0
  8. package/dist/config.js.map +1 -0
  9. package/dist/crypto.d.ts +1 -0
  10. package/dist/crypto.d.ts.map +1 -0
  11. package/dist/crypto.js +1 -0
  12. package/dist/crypto.js.map +1 -0
  13. package/dist/errors.d.ts +1 -0
  14. package/dist/errors.d.ts.map +1 -0
  15. package/dist/errors.js +1 -0
  16. package/dist/errors.js.map +1 -0
  17. package/dist/fetch.d.ts +1 -0
  18. package/dist/fetch.d.ts.map +1 -0
  19. package/dist/fetch.js +1 -0
  20. package/dist/fetch.js.map +1 -0
  21. package/dist/http.d.ts +1 -0
  22. package/dist/http.d.ts.map +1 -0
  23. package/dist/http.js +1 -0
  24. package/dist/http.js.map +1 -0
  25. package/dist/index.d.ts +1 -0
  26. package/dist/index.js +1 -0
  27. package/dist/index.js.map +1 -1
  28. package/dist/mcp.d.ts +1 -0
  29. package/dist/mcp.d.ts.map +1 -0
  30. package/dist/mcp.js +1 -0
  31. package/dist/mcp.js.map +1 -0
  32. package/dist/observability.d.ts +1 -0
  33. package/dist/observability.d.ts.map +1 -0
  34. package/dist/observability.js +1 -0
  35. package/dist/observability.js.map +1 -0
  36. package/dist/tools.d.ts +1 -0
  37. package/dist/tools.d.ts.map +1 -0
  38. package/dist/tools.js +1 -0
  39. package/dist/tools.js.map +1 -0
  40. package/dist/transform.d.ts +1 -0
  41. package/dist/transform.d.ts.map +1 -0
  42. package/dist/transform.js +1 -0
  43. package/dist/transform.js.map +1 -0
  44. package/dist/utils.d.ts +1 -0
  45. package/dist/utils.d.ts.map +1 -0
  46. package/dist/utils.js +1 -0
  47. package/dist/utils.js.map +1 -0
  48. package/dist/workers/transform-worker.d.ts +1 -0
  49. package/dist/workers/transform-worker.d.ts.map +1 -0
  50. package/dist/workers/transform-worker.js +1 -0
  51. package/dist/workers/transform-worker.js.map +1 -0
  52. package/package.json +5 -2
  53. package/dist/config/auth-config.d.ts +0 -16
  54. package/dist/config/auth-config.js +0 -53
  55. package/dist/config/constants.d.ts +0 -17
  56. package/dist/config/constants.d.ts.map +0 -1
  57. package/dist/config/constants.js +0 -22
  58. package/dist/config/constants.js.map +0 -1
  59. package/dist/config/env-parsers.d.ts +0 -8
  60. package/dist/config/env-parsers.js +0 -96
  61. package/dist/config/formatting.d.ts +0 -7
  62. package/dist/config/formatting.d.ts.map +0 -1
  63. package/dist/config/formatting.js +0 -9
  64. package/dist/config/formatting.js.map +0 -1
  65. package/dist/config/index.d.ts +0 -61
  66. package/dist/config/index.d.ts.map +0 -1
  67. package/dist/config/index.js +0 -107
  68. package/dist/config/index.js.map +0 -1
  69. package/dist/config/types/content.d.ts +0 -41
  70. package/dist/config/types/content.d.ts.map +0 -1
  71. package/dist/config/types/content.js +0 -1
  72. package/dist/config/types/content.js.map +0 -1
  73. package/dist/config/types/runtime.d.ts +0 -72
  74. package/dist/config/types/runtime.d.ts.map +0 -1
  75. package/dist/config/types/runtime.js +0 -1
  76. package/dist/config/types/runtime.js.map +0 -1
  77. package/dist/config/types/tools.d.ts +0 -28
  78. package/dist/config/types/tools.d.ts.map +0 -1
  79. package/dist/config/types/tools.js +0 -1
  80. package/dist/config/types/tools.js.map +0 -1
  81. package/dist/config/types.d.ts +0 -4
  82. package/dist/config/types.d.ts.map +0 -1
  83. package/dist/config/types.js +0 -2
  84. package/dist/config/types.js.map +0 -1
  85. package/dist/errors/app-error.d.ts +0 -7
  86. package/dist/errors/app-error.d.ts.map +0 -1
  87. package/dist/errors/app-error.js +0 -16
  88. package/dist/errors/app-error.js.map +0 -1
  89. package/dist/errors/index.d.ts +0 -2
  90. package/dist/errors/index.d.ts.map +0 -1
  91. package/dist/errors/index.js +0 -2
  92. package/dist/errors/index.js.map +0 -1
  93. package/dist/http/accept-policy.d.ts +0 -3
  94. package/dist/http/accept-policy.js +0 -45
  95. package/dist/http/async-handler.d.ts +0 -2
  96. package/dist/http/async-handler.js +0 -5
  97. package/dist/http/auth-introspection.d.ts +0 -2
  98. package/dist/http/auth-introspection.js +0 -141
  99. package/dist/http/auth-static.d.ts +0 -2
  100. package/dist/http/auth-static.js +0 -23
  101. package/dist/http/auth.d.ts +0 -3
  102. package/dist/http/auth.d.ts.map +0 -1
  103. package/dist/http/auth.js +0 -269
  104. package/dist/http/auth.js.map +0 -1
  105. package/dist/http/base-middleware.d.ts +0 -7
  106. package/dist/http/base-middleware.js +0 -143
  107. package/dist/http/cors.d.ts +0 -2
  108. package/dist/http/cors.d.ts.map +0 -1
  109. package/dist/http/cors.js +0 -9
  110. package/dist/http/cors.js.map +0 -1
  111. package/dist/http/download-routes.d.ts +0 -2
  112. package/dist/http/download-routes.d.ts.map +0 -1
  113. package/dist/http/download-routes.js +0 -104
  114. package/dist/http/download-routes.js.map +0 -1
  115. package/dist/http/error-handler.d.ts +0 -2
  116. package/dist/http/error-handler.js +0 -55
  117. package/dist/http/host-allowlist.d.ts +0 -3
  118. package/dist/http/host-allowlist.js +0 -117
  119. package/dist/http/jsonrpc-http.d.ts +0 -2
  120. package/dist/http/jsonrpc-http.js +0 -10
  121. package/dist/http/mcp-routes.d.ts +0 -9
  122. package/dist/http/mcp-routes.d.ts.map +0 -1
  123. package/dist/http/mcp-routes.js +0 -204
  124. package/dist/http/mcp-routes.js.map +0 -1
  125. package/dist/http/mcp-session-eviction.d.ts +0 -3
  126. package/dist/http/mcp-session-eviction.js +0 -24
  127. package/dist/http/mcp-session-helpers.d.ts +0 -12
  128. package/dist/http/mcp-session-helpers.d.ts.map +0 -1
  129. package/dist/http/mcp-session-helpers.js +0 -64
  130. package/dist/http/mcp-session-helpers.js.map +0 -1
  131. package/dist/http/mcp-session-init.d.ts +0 -7
  132. package/dist/http/mcp-session-init.js +0 -94
  133. package/dist/http/mcp-session-slots.d.ts +0 -17
  134. package/dist/http/mcp-session-slots.js +0 -55
  135. package/dist/http/mcp-session-transport-init.d.ts +0 -7
  136. package/dist/http/mcp-session-transport-init.js +0 -41
  137. package/dist/http/mcp-session-transport.d.ts +0 -7
  138. package/dist/http/mcp-session-transport.js +0 -57
  139. package/dist/http/mcp-session-types.d.ts +0 -5
  140. package/dist/http/mcp-session-types.js +0 -1
  141. package/dist/http/mcp-session.d.ts +0 -10
  142. package/dist/http/mcp-session.d.ts.map +0 -1
  143. package/dist/http/mcp-session.js +0 -23
  144. package/dist/http/mcp-session.js.map +0 -1
  145. package/dist/http/mcp-sessions.d.ts +0 -41
  146. package/dist/http/mcp-sessions.js +0 -392
  147. package/dist/http/mcp-validation.d.ts +0 -3
  148. package/dist/http/mcp-validation.d.ts.map +0 -1
  149. package/dist/http/mcp-validation.js +0 -14
  150. package/dist/http/mcp-validation.js.map +0 -1
  151. package/dist/http/protocol-policy.d.ts +0 -2
  152. package/dist/http/protocol-policy.js +0 -31
  153. package/dist/http/rate-limit.d.ts +0 -12
  154. package/dist/http/rate-limit.d.ts.map +0 -1
  155. package/dist/http/rate-limit.js +0 -93
  156. package/dist/http/rate-limit.js.map +0 -1
  157. package/dist/http/server-config.d.ts +0 -1
  158. package/dist/http/server-config.js +0 -40
  159. package/dist/http/server-middleware.d.ts +0 -7
  160. package/dist/http/server-middleware.d.ts.map +0 -1
  161. package/dist/http/server-middleware.js +0 -52
  162. package/dist/http/server-middleware.js.map +0 -1
  163. package/dist/http/server-shutdown.d.ts +0 -4
  164. package/dist/http/server-shutdown.js +0 -43
  165. package/dist/http/server-tuning.d.ts +0 -9
  166. package/dist/http/server-tuning.js +0 -45
  167. package/dist/http/server.d.ts +0 -3
  168. package/dist/http/server.d.ts.map +0 -1
  169. package/dist/http/server.js +0 -291
  170. package/dist/http/server.js.map +0 -1
  171. package/dist/http/session-cleanup.d.ts +0 -2
  172. package/dist/http/session-cleanup.d.ts.map +0 -1
  173. package/dist/http/session-cleanup.js +0 -40
  174. package/dist/http/session-cleanup.js.map +0 -1
  175. package/dist/http/sessions.d.ts +0 -14
  176. package/dist/http/sessions.d.ts.map +0 -1
  177. package/dist/http/sessions.js +0 -63
  178. package/dist/http/sessions.js.map +0 -1
  179. package/dist/middleware/error-handler.d.ts +0 -2
  180. package/dist/middleware/error-handler.d.ts.map +0 -1
  181. package/dist/middleware/error-handler.js +0 -56
  182. package/dist/middleware/error-handler.js.map +0 -1
  183. package/dist/middleware/rate-limiter.d.ts +0 -16
  184. package/dist/middleware/rate-limiter.d.ts.map +0 -1
  185. package/dist/middleware/rate-limiter.js +0 -111
  186. package/dist/middleware/rate-limiter.js.map +0 -1
  187. package/dist/parsers/base-html-element-parser.d.ts +0 -43
  188. package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
  189. package/dist/parsers/base-html-element-parser.js +0 -59
  190. package/dist/parsers/base-html-element-parser.js.map +0 -1
  191. package/dist/parsers/heading-element-parser.d.ts +0 -14
  192. package/dist/parsers/heading-element-parser.d.ts.map +0 -1
  193. package/dist/parsers/heading-element-parser.js +0 -26
  194. package/dist/parsers/heading-element-parser.js.map +0 -1
  195. package/dist/parsers/image-element-parser.d.ts +0 -16
  196. package/dist/parsers/image-element-parser.d.ts.map +0 -1
  197. package/dist/parsers/image-element-parser.js +0 -33
  198. package/dist/parsers/image-element-parser.js.map +0 -1
  199. package/dist/parsers/link-element-parser.d.ts +0 -15
  200. package/dist/parsers/link-element-parser.d.ts.map +0 -1
  201. package/dist/parsers/link-element-parser.js +0 -28
  202. package/dist/parsers/link-element-parser.js.map +0 -1
  203. package/dist/parsers/open-graph-parser.d.ts +0 -17
  204. package/dist/parsers/open-graph-parser.d.ts.map +0 -1
  205. package/dist/parsers/open-graph-parser.js +0 -41
  206. package/dist/parsers/open-graph-parser.js.map +0 -1
  207. package/dist/parsers/schema-org-parser.d.ts +0 -17
  208. package/dist/parsers/schema-org-parser.d.ts.map +0 -1
  209. package/dist/parsers/schema-org-parser.js +0 -32
  210. package/dist/parsers/schema-org-parser.js.map +0 -1
  211. package/dist/parsers/standard-meta-parser.d.ts +0 -18
  212. package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
  213. package/dist/parsers/standard-meta-parser.js +0 -32
  214. package/dist/parsers/standard-meta-parser.js.map +0 -1
  215. package/dist/parsers/twitter-card-parser.d.ts +0 -17
  216. package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
  217. package/dist/parsers/twitter-card-parser.js +0 -41
  218. package/dist/parsers/twitter-card-parser.js.map +0 -1
  219. package/dist/prompts/index.d.ts +0 -3
  220. package/dist/prompts/index.d.ts.map +0 -1
  221. package/dist/prompts/index.js +0 -73
  222. package/dist/prompts/index.js.map +0 -1
  223. package/dist/resources/cached-content-params.d.ts +0 -5
  224. package/dist/resources/cached-content-params.js +0 -36
  225. package/dist/resources/cached-content.d.ts +0 -2
  226. package/dist/resources/cached-content.d.ts.map +0 -1
  227. package/dist/resources/cached-content.js +0 -132
  228. package/dist/resources/cached-content.js.map +0 -1
  229. package/dist/resources/index.d.ts +0 -2
  230. package/dist/resources/index.d.ts.map +0 -1
  231. package/dist/resources/index.js +0 -4
  232. package/dist/resources/index.js.map +0 -1
  233. package/dist/server.d.ts +0 -3
  234. package/dist/server.d.ts.map +0 -1
  235. package/dist/server.js +0 -94
  236. package/dist/server.js.map +0 -1
  237. package/dist/services/cache-events.d.ts +0 -8
  238. package/dist/services/cache-events.js +0 -19
  239. package/dist/services/cache-keys.d.ts +0 -7
  240. package/dist/services/cache-keys.js +0 -57
  241. package/dist/services/cache.d.ts +0 -17
  242. package/dist/services/cache.d.ts.map +0 -1
  243. package/dist/services/cache.js +0 -145
  244. package/dist/services/cache.js.map +0 -1
  245. package/dist/services/cache.service.d.ts +0 -52
  246. package/dist/services/cache.service.d.ts.map +0 -1
  247. package/dist/services/cache.service.js +0 -113
  248. package/dist/services/cache.service.js.map +0 -1
  249. package/dist/services/card-extractor.d.ts +0 -6
  250. package/dist/services/card-extractor.d.ts.map +0 -1
  251. package/dist/services/card-extractor.js +0 -199
  252. package/dist/services/card-extractor.js.map +0 -1
  253. package/dist/services/context.d.ts +0 -10
  254. package/dist/services/context.d.ts.map +0 -1
  255. package/dist/services/context.js +0 -14
  256. package/dist/services/context.js.map +0 -1
  257. package/dist/services/extractor.d.ts +0 -5
  258. package/dist/services/extractor.d.ts.map +0 -1
  259. package/dist/services/extractor.js +0 -142
  260. package/dist/services/extractor.js.map +0 -1
  261. package/dist/services/extractor.service.d.ts +0 -18
  262. package/dist/services/extractor.service.d.ts.map +0 -1
  263. package/dist/services/extractor.service.js +0 -75
  264. package/dist/services/extractor.service.js.map +0 -1
  265. package/dist/services/fetcher/agents.d.ts +0 -3
  266. package/dist/services/fetcher/agents.d.ts.map +0 -1
  267. package/dist/services/fetcher/agents.js +0 -100
  268. package/dist/services/fetcher/agents.js.map +0 -1
  269. package/dist/services/fetcher/dns-selection.d.ts +0 -2
  270. package/dist/services/fetcher/dns-selection.js +0 -72
  271. package/dist/services/fetcher/errors.d.ts +0 -4
  272. package/dist/services/fetcher/errors.d.ts.map +0 -1
  273. package/dist/services/fetcher/errors.js +0 -70
  274. package/dist/services/fetcher/errors.js.map +0 -1
  275. package/dist/services/fetcher/headers.d.ts +0 -2
  276. package/dist/services/fetcher/headers.d.ts.map +0 -1
  277. package/dist/services/fetcher/headers.js +0 -6
  278. package/dist/services/fetcher/headers.js.map +0 -1
  279. package/dist/services/fetcher/interceptors.d.ts +0 -10
  280. package/dist/services/fetcher/interceptors.d.ts.map +0 -1
  281. package/dist/services/fetcher/interceptors.js +0 -108
  282. package/dist/services/fetcher/interceptors.js.map +0 -1
  283. package/dist/services/fetcher/redirects.d.ts +0 -4
  284. package/dist/services/fetcher/redirects.d.ts.map +0 -1
  285. package/dist/services/fetcher/redirects.js +0 -78
  286. package/dist/services/fetcher/redirects.js.map +0 -1
  287. package/dist/services/fetcher/response.d.ts +0 -4
  288. package/dist/services/fetcher/response.d.ts.map +0 -1
  289. package/dist/services/fetcher/response.js +0 -104
  290. package/dist/services/fetcher/response.js.map +0 -1
  291. package/dist/services/fetcher/retry-policy.d.ts +0 -1
  292. package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
  293. package/dist/services/fetcher/retry-policy.js +0 -131
  294. package/dist/services/fetcher/retry-policy.js.map +0 -1
  295. package/dist/services/fetcher.d.ts +0 -25
  296. package/dist/services/fetcher.d.ts.map +0 -1
  297. package/dist/services/fetcher.js +0 -607
  298. package/dist/services/fetcher.js.map +0 -1
  299. package/dist/services/fetcher.service.d.ts +0 -18
  300. package/dist/services/fetcher.service.d.ts.map +0 -1
  301. package/dist/services/fetcher.service.js +0 -122
  302. package/dist/services/fetcher.service.js.map +0 -1
  303. package/dist/services/fifo-queue.d.ts +0 -8
  304. package/dist/services/fifo-queue.js +0 -25
  305. package/dist/services/logger.d.ts +0 -5
  306. package/dist/services/logger.d.ts.map +0 -1
  307. package/dist/services/logger.js +0 -52
  308. package/dist/services/logger.js.map +0 -1
  309. package/dist/services/logger.service.d.ts +0 -5
  310. package/dist/services/logger.service.d.ts.map +0 -1
  311. package/dist/services/logger.service.js +0 -57
  312. package/dist/services/logger.service.js.map +0 -1
  313. package/dist/services/metadata-collector.d.ts +0 -2
  314. package/dist/services/metadata-collector.js +0 -80
  315. package/dist/services/parser.d.ts +0 -6
  316. package/dist/services/parser.d.ts.map +0 -1
  317. package/dist/services/parser.js +0 -278
  318. package/dist/services/parser.js.map +0 -1
  319. package/dist/services/parser.service.d.ts +0 -42
  320. package/dist/services/parser.service.d.ts.map +0 -1
  321. package/dist/services/parser.service.js +0 -209
  322. package/dist/services/parser.service.js.map +0 -1
  323. package/dist/services/session-manager.d.ts +0 -18
  324. package/dist/services/session-manager.d.ts.map +0 -1
  325. package/dist/services/session-manager.js +0 -73
  326. package/dist/services/session-manager.js.map +0 -1
  327. package/dist/services/telemetry.d.ts +0 -19
  328. package/dist/services/telemetry.js +0 -43
  329. package/dist/services/transform-worker-pool.d.ts +0 -11
  330. package/dist/services/transform-worker-pool.js +0 -244
  331. package/dist/services/transform-worker-types.d.ts +0 -32
  332. package/dist/services/transform-worker-types.js +0 -14
  333. package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
  334. package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
  335. package/dist/strategies/exponential-backoff-strategy.js +0 -32
  336. package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
  337. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
  338. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
  339. package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
  340. package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
  341. package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
  342. package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
  343. package/dist/tools/handlers/fetch-links.tool.js +0 -98
  344. package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
  345. package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
  346. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
  347. package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
  348. package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
  349. package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
  350. package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
  351. package/dist/tools/handlers/fetch-single.shared.js +0 -223
  352. package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
  353. package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
  354. package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
  355. package/dist/tools/handlers/fetch-url.tool.js +0 -119
  356. package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
  357. package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
  358. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
  359. package/dist/tools/handlers/fetch-urls/processor.js +0 -153
  360. package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
  361. package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
  362. package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
  363. package/dist/tools/handlers/fetch-urls/response.js +0 -79
  364. package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
  365. package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
  366. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
  367. package/dist/tools/handlers/fetch-urls/validation.js +0 -18
  368. package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
  369. package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
  370. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
  371. package/dist/tools/handlers/fetch-urls.tool.js +0 -124
  372. package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
  373. package/dist/tools/index.d.ts +0 -3
  374. package/dist/tools/index.d.ts.map +0 -1
  375. package/dist/tools/index.js +0 -37
  376. package/dist/tools/index.js.map +0 -1
  377. package/dist/tools/schemas.d.ts +0 -12
  378. package/dist/tools/schemas.d.ts.map +0 -1
  379. package/dist/tools/schemas.js +0 -21
  380. package/dist/tools/schemas.js.map +0 -1
  381. package/dist/tools/utils/cache-vary.d.ts +0 -1
  382. package/dist/tools/utils/cache-vary.d.ts.map +0 -1
  383. package/dist/tools/utils/cache-vary.js +0 -12
  384. package/dist/tools/utils/cache-vary.js.map +0 -1
  385. package/dist/tools/utils/cached-markdown.d.ts +0 -5
  386. package/dist/tools/utils/cached-markdown.js +0 -46
  387. package/dist/tools/utils/common.d.ts +0 -5
  388. package/dist/tools/utils/common.d.ts.map +0 -1
  389. package/dist/tools/utils/common.js +0 -42
  390. package/dist/tools/utils/common.js.map +0 -1
  391. package/dist/tools/utils/content-shaping.d.ts +0 -4
  392. package/dist/tools/utils/content-shaping.js +0 -67
  393. package/dist/tools/utils/content-transform-async.d.ts +0 -6
  394. package/dist/tools/utils/content-transform-async.js +0 -33
  395. package/dist/tools/utils/content-transform-core.d.ts +0 -5
  396. package/dist/tools/utils/content-transform-core.js +0 -180
  397. package/dist/tools/utils/content-transform-workers.d.ts +0 -1
  398. package/dist/tools/utils/content-transform-workers.js +0 -1
  399. package/dist/tools/utils/content-transform.d.ts +0 -3
  400. package/dist/tools/utils/content-transform.d.ts.map +0 -1
  401. package/dist/tools/utils/content-transform.js +0 -41
  402. package/dist/tools/utils/content-transform.js.map +0 -1
  403. package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
  404. package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
  405. package/dist/tools/utils/fetch-pipeline.js +0 -98
  406. package/dist/tools/utils/fetch-pipeline.js.map +0 -1
  407. package/dist/tools/utils/frontmatter.d.ts +0 -3
  408. package/dist/tools/utils/frontmatter.js +0 -73
  409. package/dist/tools/utils/index.d.ts +0 -4
  410. package/dist/tools/utils/index.d.ts.map +0 -1
  411. package/dist/tools/utils/index.js +0 -3
  412. package/dist/tools/utils/index.js.map +0 -1
  413. package/dist/tools/utils/inline-content.d.ts +0 -10
  414. package/dist/tools/utils/inline-content.d.ts.map +0 -1
  415. package/dist/tools/utils/inline-content.js +0 -35
  416. package/dist/tools/utils/inline-content.js.map +0 -1
  417. package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
  418. package/dist/tools/utils/markdown-heuristics.js +0 -19
  419. package/dist/tools/utils/markdown-signals.d.ts +0 -1
  420. package/dist/tools/utils/markdown-signals.js +0 -19
  421. package/dist/tools/utils/markdown-toc.d.ts +0 -3
  422. package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
  423. package/dist/tools/utils/markdown-toc.js +0 -35
  424. package/dist/tools/utils/markdown-toc.js.map +0 -1
  425. package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
  426. package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
  427. package/dist/tools/utils/raw-markdown.d.ts +0 -6
  428. package/dist/tools/utils/raw-markdown.js +0 -149
  429. package/dist/tools/utils/response-builder.d.ts +0 -3
  430. package/dist/tools/utils/response-builder.d.ts.map +0 -1
  431. package/dist/tools/utils/response-builder.js +0 -24
  432. package/dist/tools/utils/response-builder.js.map +0 -1
  433. package/dist/tools/utils/tool-response.d.ts +0 -9
  434. package/dist/tools/utils/tool-response.d.ts.map +0 -1
  435. package/dist/tools/utils/tool-response.js +0 -19
  436. package/dist/tools/utils/tool-response.js.map +0 -1
  437. package/dist/transformers/jsonl.transformer.d.ts +0 -2
  438. package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
  439. package/dist/transformers/jsonl.transformer.js +0 -75
  440. package/dist/transformers/jsonl.transformer.js.map +0 -1
  441. package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
  442. package/dist/transformers/markdown/fenced-code-rule.js +0 -38
  443. package/dist/transformers/markdown/frontmatter.d.ts +0 -2
  444. package/dist/transformers/markdown/frontmatter.js +0 -45
  445. package/dist/transformers/markdown/noise-rule.d.ts +0 -2
  446. package/dist/transformers/markdown/noise-rule.js +0 -80
  447. package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
  448. package/dist/transformers/markdown/turndown-instance.js +0 -19
  449. package/dist/transformers/markdown.d.ts +0 -5
  450. package/dist/transformers/markdown.js +0 -314
  451. package/dist/transformers/markdown.transformer.d.ts +0 -2
  452. package/dist/transformers/markdown.transformer.d.ts.map +0 -1
  453. package/dist/transformers/markdown.transformer.js +0 -14
  454. package/dist/transformers/markdown.transformer.js.map +0 -1
  455. package/dist/types/content.types.d.ts +0 -63
  456. package/dist/types/content.types.d.ts.map +0 -1
  457. package/dist/types/content.types.js +0 -2
  458. package/dist/types/content.types.js.map +0 -1
  459. package/dist/types/index.d.ts +0 -2
  460. package/dist/types/index.d.ts.map +0 -1
  461. package/dist/types/index.js +0 -2
  462. package/dist/types/index.js.map +0 -1
  463. package/dist/types/schemas.d.ts +0 -49
  464. package/dist/types/schemas.d.ts.map +0 -1
  465. package/dist/types/schemas.js +0 -5
  466. package/dist/types/schemas.js.map +0 -1
  467. package/dist/utils/cached-payload.d.ts +0 -7
  468. package/dist/utils/cached-payload.js +0 -36
  469. package/dist/utils/cancellation.d.ts +0 -1
  470. package/dist/utils/cancellation.js +0 -18
  471. package/dist/utils/code-language-bash.d.ts +0 -1
  472. package/dist/utils/code-language-bash.js +0 -48
  473. package/dist/utils/code-language-core.d.ts +0 -2
  474. package/dist/utils/code-language-core.js +0 -13
  475. package/dist/utils/code-language-detectors.d.ts +0 -5
  476. package/dist/utils/code-language-detectors.js +0 -142
  477. package/dist/utils/code-language-helpers.d.ts +0 -5
  478. package/dist/utils/code-language-helpers.js +0 -62
  479. package/dist/utils/code-language-parsing.d.ts +0 -5
  480. package/dist/utils/code-language-parsing.js +0 -62
  481. package/dist/utils/code-language.d.ts +0 -2
  482. package/dist/utils/code-language.d.ts.map +0 -1
  483. package/dist/utils/code-language.js +0 -260
  484. package/dist/utils/code-language.js.map +0 -1
  485. package/dist/utils/concurrency.d.ts +0 -3
  486. package/dist/utils/concurrency.d.ts.map +0 -1
  487. package/dist/utils/concurrency.js +0 -38
  488. package/dist/utils/concurrency.js.map +0 -1
  489. package/dist/utils/content-cleaner.d.ts +0 -5
  490. package/dist/utils/content-cleaner.d.ts.map +0 -1
  491. package/dist/utils/content-cleaner.js +0 -77
  492. package/dist/utils/content-cleaner.js.map +0 -1
  493. package/dist/utils/crypto.d.ts +0 -2
  494. package/dist/utils/crypto.d.ts.map +0 -1
  495. package/dist/utils/crypto.js +0 -32
  496. package/dist/utils/crypto.js.map +0 -1
  497. package/dist/utils/download-url.d.ts +0 -16
  498. package/dist/utils/download-url.d.ts.map +0 -1
  499. package/dist/utils/download-url.js +0 -30
  500. package/dist/utils/download-url.js.map +0 -1
  501. package/dist/utils/error-details.d.ts +0 -3
  502. package/dist/utils/error-details.js +0 -12
  503. package/dist/utils/error-utils.d.ts +0 -3
  504. package/dist/utils/error-utils.d.ts.map +0 -1
  505. package/dist/utils/error-utils.js +0 -12
  506. package/dist/utils/error-utils.js.map +0 -1
  507. package/dist/utils/filename-generator.d.ts +0 -1
  508. package/dist/utils/filename-generator.d.ts.map +0 -1
  509. package/dist/utils/filename-generator.js +0 -81
  510. package/dist/utils/filename-generator.js.map +0 -1
  511. package/dist/utils/guards.d.ts +0 -1
  512. package/dist/utils/guards.js +0 -3
  513. package/dist/utils/header-normalizer.d.ts +0 -5
  514. package/dist/utils/header-normalizer.d.ts.map +0 -1
  515. package/dist/utils/header-normalizer.js +0 -31
  516. package/dist/utils/header-normalizer.js.map +0 -1
  517. package/dist/utils/host-normalizer.d.ts +0 -1
  518. package/dist/utils/host-normalizer.js +0 -37
  519. package/dist/utils/html-truncator.d.ts +0 -1
  520. package/dist/utils/html-truncator.d.ts.map +0 -1
  521. package/dist/utils/html-truncator.js +0 -13
  522. package/dist/utils/html-truncator.js.map +0 -1
  523. package/dist/utils/ip-address.d.ts +0 -4
  524. package/dist/utils/ip-address.js +0 -6
  525. package/dist/utils/language-detector.d.ts +0 -2
  526. package/dist/utils/language-detector.d.ts.map +0 -1
  527. package/dist/utils/language-detector.js +0 -39
  528. package/dist/utils/language-detector.js.map +0 -1
  529. package/dist/utils/sanitizer.d.ts +0 -2
  530. package/dist/utils/sanitizer.d.ts.map +0 -1
  531. package/dist/utils/sanitizer.js +0 -20
  532. package/dist/utils/sanitizer.js.map +0 -1
  533. package/dist/utils/tool-error-handler.d.ts +0 -3
  534. package/dist/utils/tool-error-handler.d.ts.map +0 -1
  535. package/dist/utils/tool-error-handler.js +0 -31
  536. package/dist/utils/tool-error-handler.js.map +0 -1
  537. package/dist/utils/url-redactor.d.ts +0 -1
  538. package/dist/utils/url-redactor.js +0 -13
  539. package/dist/utils/url-sanitizer.d.ts +0 -2
  540. package/dist/utils/url-sanitizer.d.ts.map +0 -1
  541. package/dist/utils/url-sanitizer.js +0 -12
  542. package/dist/utils/url-sanitizer.js.map +0 -1
  543. package/dist/utils/url-transformer.d.ts +0 -7
  544. package/dist/utils/url-transformer.js +0 -147
  545. package/dist/utils/url-validator.d.ts +0 -6
  546. package/dist/utils/url-validator.d.ts.map +0 -1
  547. package/dist/utils/url-validator.js +0 -156
  548. package/dist/utils/url-validator.js.map +0 -1
  549. package/dist/workers/content-transform.worker.d.ts +0 -1
  550. package/dist/workers/content-transform.worker.js +0 -40
@@ -1,278 +0,0 @@
1
- import * as cheerio from 'cheerio';
2
- import { config } from '../config/index.js';
3
- import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../utils/code-language.js';
4
- import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInlineTimestamps, } from '../utils/content-cleaner.js';
5
- import { getErrorMessage } from '../utils/error-utils.js';
6
- import { truncateHtml } from '../utils/html-truncator.js';
7
- import { sanitizeText } from '../utils/sanitizer.js';
8
- import { logWarn } from './logger.js';
9
- import { createMetaCollectorState, resolveMetaField, } from './metadata-collector.js';
10
- const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
11
- function extractMetadata($) {
12
- const state = createMetaCollectorState();
13
- $('meta').each((_, element) => {
14
- const content = $(element).attr('content')?.trim();
15
- if (!content)
16
- return;
17
- const property = $(element).attr('property');
18
- if (property?.startsWith('og:')) {
19
- const key = property.replace('og:', '');
20
- if (key === 'title')
21
- state.title.og = content;
22
- if (key === 'description')
23
- state.description.og = content;
24
- return;
25
- }
26
- const name = $(element).attr('name');
27
- if (name?.startsWith('twitter:')) {
28
- const key = name.replace('twitter:', '');
29
- if (key === 'title')
30
- state.title.twitter = content;
31
- if (key === 'description')
32
- state.description.twitter = content;
33
- return;
34
- }
35
- if (name === 'description') {
36
- state.description.standard = content;
37
- }
38
- if (name === 'author') {
39
- state.author.standard = content;
40
- }
41
- });
42
- if (!state.title.standard) {
43
- const titleText = $('title').first().text().trim();
44
- if (titleText) {
45
- state.title.standard = titleText;
46
- }
47
- }
48
- const metadata = {};
49
- const title = resolveMetaField(state, 'title');
50
- const description = resolveMetaField(state, 'description');
51
- const author = resolveMetaField(state, 'author');
52
- if (title !== undefined)
53
- metadata.title = title;
54
- if (description !== undefined)
55
- metadata.description = description;
56
- if (author !== undefined)
57
- metadata.author = author;
58
- return metadata;
59
- }
60
- function parseHeading($, element) {
61
- const rawText = sanitizeText($(element).text());
62
- const text = cleanHeading(rawText);
63
- if (!text)
64
- return null;
65
- return {
66
- type: 'heading',
67
- level: parseInt(element.tagName.substring(1), 10),
68
- text,
69
- };
70
- }
71
- function parseParagraph($, element) {
72
- let rawText = sanitizeText($(element).text());
73
- // Remove inline timestamps like "13 days ago" from paragraphs
74
- rawText = removeInlineTimestamps(rawText);
75
- const text = cleanParagraph(rawText);
76
- if (!text || text.length < config.extraction.minParagraphLength)
77
- return null;
78
- return { type: 'paragraph', text };
79
- }
80
- function parseList($, element) {
81
- const rawItems = [];
82
- $(element)
83
- .find('li')
84
- .each((_, li) => {
85
- const text = sanitizeText($(li).text());
86
- if (text)
87
- rawItems.push(text);
88
- });
89
- // Clean list items to remove noise
90
- const items = cleanListItems(rawItems);
91
- if (items.length === 0)
92
- return null;
93
- return {
94
- type: 'list',
95
- ordered: element.tagName.toLowerCase() === 'ol',
96
- items,
97
- };
98
- }
99
- function parseCode($, element) {
100
- const rawText = $(element).text().trim();
101
- const text = cleanCodeBlock(rawText);
102
- if (!text)
103
- return null;
104
- const className = $(element).attr('class') ?? '';
105
- const dataLang = $(element).attr('data-language') ?? '';
106
- const language = resolveLanguageFromAttributes(className, dataLang) ??
107
- detectLanguageFromCode(text);
108
- const block = {
109
- type: 'code',
110
- text,
111
- };
112
- if (language !== undefined) {
113
- block.language = language;
114
- }
115
- return block;
116
- }
117
- function parseTable($, element) {
118
- const headers = [];
119
- const rows = [];
120
- const $table = $(element);
121
- $table.find('thead th, thead td').each((_, cell) => {
122
- headers.push(sanitizeText($(cell).text()));
123
- });
124
- if (headers.length === 0) {
125
- $table
126
- .find('tr')
127
- .first()
128
- .find('th, td')
129
- .each((_, cell) => {
130
- headers.push(sanitizeText($(cell).text()));
131
- });
132
- }
133
- const rowsSelector = headers.length > 0 ? 'tbody tr, tr:not(:first)' : 'tbody tr, tr';
134
- $table.find(rowsSelector).each((_, row) => {
135
- const cells = [];
136
- $(row)
137
- .find('td, th')
138
- .each((_, cell) => {
139
- cells.push(sanitizeText($(cell).text()));
140
- });
141
- if (cells.length > 0)
142
- rows.push(cells);
143
- });
144
- if (rows.length === 0)
145
- return null;
146
- return headers.length > 0
147
- ? { type: 'table', headers, rows }
148
- : { type: 'table', rows };
149
- }
150
- function parseImage($, element) {
151
- const src = $(element).attr('src');
152
- if (!src)
153
- return null;
154
- const alt = $(element).attr('alt');
155
- const image = {
156
- type: 'image',
157
- src,
158
- };
159
- if (alt !== undefined) {
160
- image.alt = alt;
161
- }
162
- return image;
163
- }
164
- function parseBlockquote($, element) {
165
- const rawText = sanitizeText($(element).text());
166
- const text = cleanParagraph(rawText);
167
- if (!text || text.length < config.extraction.minParagraphLength)
168
- return null;
169
- return { type: 'blockquote', text };
170
- }
171
- const ELEMENT_PARSERS = {
172
- h1: parseHeading,
173
- h2: parseHeading,
174
- h3: parseHeading,
175
- h4: parseHeading,
176
- h5: parseHeading,
177
- h6: parseHeading,
178
- p: parseParagraph,
179
- ul: parseList,
180
- ol: parseList,
181
- pre: parseCode,
182
- code: parseCode,
183
- table: parseTable,
184
- img: parseImage,
185
- blockquote: parseBlockquote,
186
- };
187
- function isParseableTag(tag) {
188
- return tag in ELEMENT_PARSERS;
189
- }
190
- function parseElement($, node) {
191
- if (!('tagName' in node) || typeof node.tagName !== 'string')
192
- return null;
193
- const tagName = node.tagName.toLowerCase();
194
- if (!isParseableTag(tagName))
195
- return null;
196
- return ELEMENT_PARSERS[tagName]($, node);
197
- }
198
- function filterBlocks(blocks) {
199
- return blocks.filter(shouldIncludeBlock);
200
- }
201
- const TEXT_BLOCK_TYPES = new Set([
202
- 'paragraph',
203
- 'heading',
204
- 'code',
205
- 'blockquote',
206
- ]);
207
- function isTextBlock(block) {
208
- return TEXT_BLOCK_TYPES.has(block.type);
209
- }
210
- function shouldIncludeBlock(block) {
211
- if (isTextBlock(block)) {
212
- return block.text.length > 0;
213
- }
214
- if (block.type === 'list') {
215
- return block.items.length > 0;
216
- }
217
- return true;
218
- }
219
- function loadHtml(html) {
220
- try {
221
- return cheerio.load(html);
222
- }
223
- catch (error) {
224
- logWarn('Failed to parse HTML', {
225
- error: getErrorMessage(error),
226
- htmlLength: html.length,
227
- });
228
- return null;
229
- }
230
- }
231
- function prepareCheerio(html) {
232
- const processedHtml = truncateHtml(html);
233
- return loadHtml(processedHtml);
234
- }
235
- function removeNoiseElements($) {
236
- $('script, style, noscript, iframe, svg').remove();
237
- }
238
- function collectBlocks($) {
239
- const blocks = [];
240
- $('body')
241
- .find(CONTENT_SELECTOR)
242
- .each((_, element) => {
243
- const block = safeParseElement($, element);
244
- if (block)
245
- blocks.push(block);
246
- });
247
- return blocks;
248
- }
249
- function safeParseElement($, element) {
250
- try {
251
- return parseElement($, element);
252
- }
253
- catch {
254
- return null;
255
- }
256
- }
257
- export function parseHtml(html) {
258
- if (!html || typeof html !== 'string')
259
- return [];
260
- const $ = prepareCheerio(html);
261
- if (!$)
262
- return [];
263
- removeNoiseElements($);
264
- return filterBlocks(collectBlocks($));
265
- }
266
- export function parseHtmlWithMetadata(html) {
267
- if (!html || typeof html !== 'string') {
268
- return { blocks: [], metadata: {} };
269
- }
270
- const $ = prepareCheerio(html);
271
- if (!$) {
272
- return { blocks: [], metadata: {} };
273
- }
274
- const metadata = extractMetadata($);
275
- removeNoiseElements($);
276
- const blocks = filterBlocks(collectBlocks($));
277
- return { blocks, metadata };
278
- }
@@ -1 +0,0 @@
1
- {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,gBAAgB,GACpB,oFAAoF,CAAC;AACvF,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AACD,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AACD,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IACxD,MAAM,QAAQ,GACZ,6BAA6B,CAAC,SAAS,EAAE,QAAQ,CAAC;QAClD,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IAEnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;YAC7B,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAa;IACxC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,CAAa;IAClC,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,CAAC,CAAC,MAAM,CAAC;SACN,IAAI,CAAC,gBAAgB,CAAC;SACtB,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACnB,MAAM,KAAK,GAAG,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEL,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CACvB,CAAa,EACb,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IAClC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAElB,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACvB,OAAO,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC"}
@@ -1,42 +0,0 @@
1
- import type { ContentBlockUnion } from '../types/index.js';
2
- declare class ParserService {
3
- /**
4
- * Parses HTML content and extracts semantic blocks
5
- */
6
- parseHtml(html: string): ContentBlockUnion[];
7
- /**
8
- * Parses a single element into a content block
9
- */
10
- private parseElement;
11
- /**
12
- * Parses a heading element
13
- */
14
- private parseHeading;
15
- /**
16
- * Parses a paragraph element
17
- */
18
- private parseParagraph;
19
- /**
20
- * Parses a list element
21
- */
22
- private parseList;
23
- /**
24
- * Parses a code element
25
- */
26
- private parseCode;
27
- /**
28
- * Parses a table element
29
- */
30
- private parseTable;
31
- /**
32
- * Parses an image element
33
- */
34
- private parseImage;
35
- /**
36
- * Filters out empty or invalid blocks
37
- */
38
- private filterBlocks;
39
- }
40
- export declare const parserService: ParserService;
41
- export {};
42
- //# sourceMappingURL=parser.service.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"parser.service.d.ts","sourceRoot":"","sources":["../../src/services/parser.service.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAOV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAE3B,cAAM,aAAa;IACjB;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE;IAoB5C;;OAEG;IACH,OAAO,CAAC,YAAY;IAuCpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;OAEG;IACH,OAAO,CAAC,SAAS;IAwBjB;;OAEG;IACH,OAAO,CAAC,SAAS;IAkBjB;;OAEG;IACH,OAAO,CAAC,UAAU;IA+ClB;;OAEG;IACH,OAAO,CAAC,UAAU;IAelB;;OAEG;IACH,OAAO,CAAC,YAAY;CAiBrB;AAGD,eAAO,MAAM,aAAa,eAAsB,CAAC"}
@@ -1,209 +0,0 @@
1
- import * as cheerio from 'cheerio';
2
- import { sanitizeText } from '../utils/sanitizer.js';
3
- import { config } from '../config/index.js';
4
- class ParserService {
5
- /**
6
- * Parses HTML content and extracts semantic blocks
7
- */
8
- parseHtml(html) {
9
- const $ = cheerio.load(html);
10
- const blocks = [];
11
- // Remove script, style, and other non-content elements
12
- $('script, style, noscript, iframe, svg').remove();
13
- // Parse the body content
14
- $('body')
15
- .find('h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code, table, img')
16
- .each((_, element) => {
17
- const block = this.parseElement($, element);
18
- if (block) {
19
- blocks.push(block);
20
- }
21
- });
22
- return this.filterBlocks(blocks);
23
- }
24
- /**
25
- * Parses a single element into a content block
26
- */
27
- parseElement($, node) {
28
- // Check if node is an Element with tagName property
29
- if (!('tagName' in node) || typeof node.tagName !== 'string') {
30
- return null;
31
- }
32
- const element = node;
33
- const tagName = element.tagName.toLowerCase();
34
- switch (tagName) {
35
- case 'h1':
36
- case 'h2':
37
- case 'h3':
38
- case 'h4':
39
- case 'h5':
40
- case 'h6':
41
- return this.parseHeading($, element);
42
- case 'p':
43
- return this.parseParagraph($, element);
44
- case 'ul':
45
- case 'ol':
46
- return this.parseList($, element);
47
- case 'pre':
48
- case 'code':
49
- return this.parseCode($, element);
50
- case 'table':
51
- return this.parseTable($, element);
52
- case 'img':
53
- return this.parseImage($, element);
54
- default:
55
- return null;
56
- }
57
- }
58
- /**
59
- * Parses a heading element
60
- */
61
- parseHeading($, element) {
62
- const text = sanitizeText($(element).text());
63
- if (!text) {
64
- return null;
65
- }
66
- const level = parseInt(element.tagName.substring(1), 10);
67
- return {
68
- type: 'heading',
69
- level,
70
- text,
71
- };
72
- }
73
- /**
74
- * Parses a paragraph element
75
- */
76
- parseParagraph($, element) {
77
- const text = sanitizeText($(element).text());
78
- if (!text || text.length < config.extraction.minParagraphLength) {
79
- // Skip very short paragraphs
80
- return null;
81
- }
82
- return {
83
- type: 'paragraph',
84
- text,
85
- };
86
- }
87
- /**
88
- * Parses a list element
89
- */
90
- parseList($, element) {
91
- const items = [];
92
- $(element)
93
- .find('li')
94
- .each((_, li) => {
95
- const text = sanitizeText($(li).text());
96
- if (text) {
97
- items.push(text);
98
- }
99
- });
100
- if (items.length === 0) {
101
- return null;
102
- }
103
- const ordered = element.tagName.toLowerCase() === 'ol';
104
- return {
105
- type: 'list',
106
- ordered,
107
- items,
108
- };
109
- }
110
- /**
111
- * Parses a code element
112
- */
113
- parseCode($, element) {
114
- const text = $(element).text().trim();
115
- if (!text) {
116
- return null;
117
- }
118
- // Try to detect language from class name
119
- const className = $(element).attr('class') || '';
120
- const languageMatch = className.match(/language-(\w+)/);
121
- const language = languageMatch ? languageMatch[1] : undefined;
122
- return {
123
- type: 'code',
124
- language,
125
- text,
126
- };
127
- }
128
- /**
129
- * Parses a table element
130
- */
131
- parseTable($, element) {
132
- const headers = [];
133
- const rows = [];
134
- // Extract headers from thead or first row
135
- const $table = $(element);
136
- $table.find('thead th, thead td').each((_, cell) => {
137
- headers.push(sanitizeText($(cell).text()));
138
- });
139
- // If no thead, try first tr
140
- if (headers.length === 0) {
141
- $table
142
- .find('tr')
143
- .first()
144
- .find('th, td')
145
- .each((_, cell) => {
146
- headers.push(sanitizeText($(cell).text()));
147
- });
148
- }
149
- // Extract body rows
150
- const rowsSelector = headers.length > 0 ? 'tbody tr, tr:not(:first)' : 'tbody tr, tr';
151
- $table.find(rowsSelector).each((_, row) => {
152
- const cells = [];
153
- $(row)
154
- .find('td, th')
155
- .each((_, cell) => {
156
- cells.push(sanitizeText($(cell).text()));
157
- });
158
- if (cells.length > 0) {
159
- rows.push(cells);
160
- }
161
- });
162
- if (rows.length === 0) {
163
- return null;
164
- }
165
- return {
166
- type: 'table',
167
- headers: headers.length > 0 ? headers : undefined,
168
- rows,
169
- };
170
- }
171
- /**
172
- * Parses an image element
173
- */
174
- parseImage($, element) {
175
- const src = $(element).attr('src');
176
- if (!src) {
177
- return null;
178
- }
179
- const alt = $(element).attr('alt') || undefined;
180
- return {
181
- type: 'image',
182
- src,
183
- alt,
184
- };
185
- }
186
- /**
187
- * Filters out empty or invalid blocks
188
- */
189
- filterBlocks(blocks) {
190
- return blocks.filter((block) => {
191
- if (block.type === 'paragraph') {
192
- return block.text.length > 0;
193
- }
194
- if (block.type === 'heading') {
195
- return block.text.length > 0;
196
- }
197
- if (block.type === 'list') {
198
- return block.items.length > 0;
199
- }
200
- if (block.type === 'code') {
201
- return block.text.length > 0;
202
- }
203
- return true;
204
- });
205
- }
206
- }
207
- // Singleton instance
208
- export const parserService = new ParserService();
209
- //# sourceMappingURL=parser.service.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"parser.service.js","sourceRoot":"","sources":["../../src/services/parser.service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAW5C,MAAM,aAAa;IACjB;;OAEG;IACH,SAAS,CAAC,IAAY;QACpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,uDAAuD;QACvD,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,yBAAyB;QACzB,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,0DAA0D,CAAC;aAChE,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,CAAa,EAAE,IAAa;QAC/C,oDAAoD;QACpD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC;QACrB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAE9C,QAAQ,OAAO,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,IAAI;gBACP,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEvC,KAAK,GAAG;gBACN,OAAO,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEzC,KAAK,IAAI,CAAC;YACV,KAAK,IAAI;gBACP,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEpC,KAAK,KAAK,CAAC;YACX,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEpC,KAAK,OAAO;gBACV,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAErC,KAAK,KAAK;gBACR,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAErC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,CAAa,EAAE,OAAgB;QAClD,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK;YACL,IAAI;SACL,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,CAAa,EACb,OAAgB;QAEhB,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB,EAAE,CAAC;YAChE,6BAA6B;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,IAAI;SACL,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,CAAa,EAAE,OAAgB;QAC/C,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,OAAO,CAAC;aACP,IAAI,CAAC,IAAI,CAAC;aACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACxC,IAAI,IAAI,EAAE,CAAC;gBACT,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC;QAEvD,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,OAAO;YACP,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,CAAa,EAAE,OAAgB;QAC/C,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,yCAAyC;QACzC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACjD,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAE9D,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,QAAQ;YACR,IAAI;SACL,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,CAAa,EAAE,OAAgB;QAChD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,4BAA4B;QAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM;iBACH,IAAI,CAAC,IAAI,CAAC;iBACV,KAAK,EAAE;iBACP,IAAI,CAAC,QAAQ,CAAC;iBACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;gBAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;QACP,CAAC;QAED,oBAAoB;QACpB,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;YACxC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,CAAC,CAAC,GAAG,CAAC;iBACH,IAAI,CAAC,QAAQ,CAAC;iBACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;gBAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;YACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;YACjD,IAAI;SACL,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,CAAa,EAAE,OAAgB;QAChD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,OAAO;YACb,GAAG;YACH,GAAG;SACJ,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,MAA2B;QAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC/B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC7B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,qBAAqB;AACrB,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC"}
@@ -1,18 +0,0 @@
1
- import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
2
- import type { SessionEntry } from '../config/types.js';
3
- export declare class SessionManager {
4
- private readonly sessions;
5
- private readonly sessionTtlMs;
6
- private cleanupTimeout;
7
- constructor(sessionTtlMs?: number);
8
- get(sessionId: string): SessionEntry | undefined;
9
- set(sessionId: string, transport: StreamableHTTPServerTransport): void;
10
- update(sessionId: string): boolean;
11
- delete(sessionId: string): void;
12
- get size(): number;
13
- closeAll(): Promise<void>;
14
- private removeExpiredAndIdleSessions;
15
- private scheduleCleanup;
16
- destroy(): void;
17
- }
18
- //# sourceMappingURL=session-manager.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"session-manager.d.ts","sourceRoot":"","sources":["../../src/services/session-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAOvD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,cAAc,CAA+B;gBAEzC,YAAY,SAAyB;IAIjD,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIhD,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,6BAA6B,GAAG,IAAI;IAMtE,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAUlC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAK/B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEK,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YASjB,4BAA4B;IAiB1C,OAAO,CAAC,eAAe;IAiBvB,OAAO,IAAI,IAAI;CAMhB"}
@@ -1,73 +0,0 @@
1
- import { logInfo } from './logger.js';
2
- const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
3
- const CLEANUP_INTERVAL_MS = 2 * 60 * 1000;
4
- export class SessionManager {
5
- sessions = new Map();
6
- sessionTtlMs;
7
- cleanupTimeout = null;
8
- constructor(sessionTtlMs = DEFAULT_SESSION_TTL_MS) {
9
- this.sessionTtlMs = sessionTtlMs;
10
- }
11
- get(sessionId) {
12
- return this.sessions.get(sessionId);
13
- }
14
- set(sessionId, transport) {
15
- this.sessions.set(sessionId, { transport, createdAt: Date.now() });
16
- logInfo('Session initialized', { sessionId });
17
- this.scheduleCleanup();
18
- }
19
- update(sessionId) {
20
- const entry = this.sessions.get(sessionId);
21
- if (!entry) {
22
- return false;
23
- }
24
- entry.createdAt = Date.now();
25
- return true;
26
- }
27
- delete(sessionId) {
28
- this.sessions.delete(sessionId);
29
- logInfo('Session closed', { sessionId });
30
- }
31
- get size() {
32
- return this.sessions.size;
33
- }
34
- async closeAll() {
35
- const closePromises = Array.from(this.sessions.values()).map((session) => session.transport.close());
36
- await Promise.allSettled(closePromises);
37
- this.sessions.clear();
38
- }
39
- async removeExpiredAndIdleSessions() {
40
- const now = Date.now();
41
- const closePromises = [];
42
- for (const [sessionId, entry] of this.sessions) {
43
- const isExpired = now - entry.createdAt > this.sessionTtlMs;
44
- if (isExpired) {
45
- logInfo('Cleaning up stale session', { sessionId });
46
- closePromises.push(entry.transport.close());
47
- this.sessions.delete(sessionId);
48
- }
49
- }
50
- await Promise.allSettled(closePromises);
51
- }
52
- scheduleCleanup() {
53
- if (this.cleanupTimeout) {
54
- return;
55
- }
56
- this.cleanupTimeout = setTimeout(() => {
57
- void this.removeExpiredAndIdleSessions().then(() => {
58
- this.cleanupTimeout = null;
59
- if (this.sessions.size > 0) {
60
- this.scheduleCleanup();
61
- }
62
- });
63
- }, CLEANUP_INTERVAL_MS);
64
- this.cleanupTimeout.unref();
65
- }
66
- destroy() {
67
- if (this.cleanupTimeout) {
68
- clearTimeout(this.cleanupTimeout);
69
- this.cleanupTimeout = null;
70
- }
71
- }
72
- }
73
- //# sourceMappingURL=session-manager.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"session-manager.js","sourceRoot":"","sources":["../../src/services/session-manager.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,sBAAsB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9C,MAAM,mBAAmB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAE1C,MAAM,OAAO,cAAc;IACR,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC3C,YAAY,CAAS;IAC9B,cAAc,GAA0B,IAAI,CAAC;IAErD,YAAY,YAAY,GAAG,sBAAsB;QAC/C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,GAAG,CAAC,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,GAAG,CAAC,SAAiB,EAAE,SAAwC;QAC7D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,SAAiB;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACf,CAAC;QAED,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,CAAC,SAAiB;QACtB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACvE,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAC1B,CAAC;QAEF,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,4BAA4B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,aAAa,GAAoB,EAAE,CAAC;QAE1C,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;YAE5D,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,2BAA2B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;gBACpD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe;QACrB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;YACpC,KAAK,IAAI,CAAC,4BAA4B,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;gBACjD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAExB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}
@@ -1,19 +0,0 @@
1
- export interface TransformStageEvent {
2
- v: 1;
3
- type: 'stage';
4
- stage: string;
5
- durationMs: number;
6
- url: string;
7
- requestId?: string;
8
- operationId?: string;
9
- truncated?: boolean;
10
- }
11
- export interface TransformStageContext {
12
- readonly stage: string;
13
- readonly startTime: number;
14
- readonly url: string;
15
- }
16
- export declare function startTransformStage(url: string, stage: string): TransformStageContext | null;
17
- export declare function endTransformStage(context: TransformStageContext | null, options?: {
18
- truncated?: boolean;
19
- }): void;