@j0hanz/superfetch 1.2.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +18 -204
  2. package/dist/config/constants.d.ts +0 -1
  3. package/dist/config/constants.js +0 -1
  4. package/dist/config/formatting.d.ts +0 -1
  5. package/dist/config/formatting.d.ts.map +1 -1
  6. package/dist/config/formatting.js +1 -3
  7. package/dist/config/formatting.js.map +1 -1
  8. package/dist/config/index.d.ts +1 -2
  9. package/dist/config/index.d.ts.map +1 -1
  10. package/dist/config/index.js +5 -11
  11. package/dist/config/index.js.map +1 -1
  12. package/dist/config/types/content.d.ts +0 -1
  13. package/dist/config/types/content.js +0 -1
  14. package/dist/config/types/runtime.d.ts +0 -1
  15. package/dist/config/types/runtime.js +0 -1
  16. package/dist/config/types/tools.d.ts +0 -1
  17. package/dist/config/types/tools.js +0 -1
  18. package/dist/errors/app-error.d.ts +0 -1
  19. package/dist/errors/app-error.js +0 -1
  20. package/dist/http/auth.d.ts +0 -1
  21. package/dist/http/auth.js +0 -1
  22. package/dist/http/cors.d.ts +0 -1
  23. package/dist/http/cors.js +0 -1
  24. package/dist/http/download-routes.d.ts +1 -2
  25. package/dist/http/download-routes.d.ts.map +1 -1
  26. package/dist/http/download-routes.js +0 -1
  27. package/dist/http/mcp-routes.d.ts +0 -1
  28. package/dist/http/mcp-routes.js +0 -1
  29. package/dist/http/mcp-session-helpers.d.ts +0 -1
  30. package/dist/http/mcp-session-helpers.js +0 -1
  31. package/dist/http/mcp-session.d.ts +1 -2
  32. package/dist/http/mcp-session.d.ts.map +1 -1
  33. package/dist/http/mcp-session.js +0 -1
  34. package/dist/http/mcp-validation.d.ts +1 -2
  35. package/dist/http/mcp-validation.d.ts.map +1 -1
  36. package/dist/http/mcp-validation.js +6 -27
  37. package/dist/http/mcp-validation.js.map +1 -1
  38. package/dist/http/rate-limit.d.ts +1 -2
  39. package/dist/http/rate-limit.d.ts.map +1 -1
  40. package/dist/http/rate-limit.js +0 -1
  41. package/dist/http/rate-limit.js.map +1 -1
  42. package/dist/http/server-middleware.d.ts +0 -1
  43. package/dist/http/server-middleware.d.ts.map +1 -1
  44. package/dist/http/server-middleware.js +56 -1
  45. package/dist/http/server-middleware.js.map +1 -1
  46. package/dist/http/server.d.ts +0 -1
  47. package/dist/http/server.js +0 -1
  48. package/dist/http/session-cleanup.d.ts +0 -1
  49. package/dist/http/session-cleanup.js +0 -1
  50. package/dist/http/sessions.d.ts +1 -2
  51. package/dist/http/sessions.d.ts.map +1 -1
  52. package/dist/http/sessions.js +0 -1
  53. package/dist/index.d.ts +0 -1
  54. package/dist/index.js +0 -1
  55. package/dist/middleware/error-handler.d.ts +0 -1
  56. package/dist/middleware/error-handler.js +0 -1
  57. package/dist/resources/cached-content.d.ts +0 -1
  58. package/dist/resources/cached-content.js +0 -1
  59. package/dist/resources/index.d.ts +0 -1
  60. package/dist/resources/index.js +0 -1
  61. package/dist/server.d.ts +0 -1
  62. package/dist/server.d.ts.map +1 -1
  63. package/dist/server.js +8 -3
  64. package/dist/server.js.map +1 -1
  65. package/dist/services/cache.d.ts +1 -2
  66. package/dist/services/cache.d.ts.map +1 -1
  67. package/dist/services/cache.js +1 -2
  68. package/dist/services/context.d.ts +0 -1
  69. package/dist/services/context.js +0 -1
  70. package/dist/services/extractor.d.ts +1 -2
  71. package/dist/services/extractor.d.ts.map +1 -1
  72. package/dist/services/extractor.js +0 -1
  73. package/dist/services/fetcher/agents.d.ts +0 -1
  74. package/dist/services/fetcher/agents.js +0 -1
  75. package/dist/services/fetcher/errors.d.ts +0 -1
  76. package/dist/services/fetcher/errors.js +0 -1
  77. package/dist/services/fetcher/interceptors.d.ts +0 -1
  78. package/dist/services/fetcher/interceptors.js +0 -1
  79. package/dist/services/fetcher/redirects.d.ts +0 -1
  80. package/dist/services/fetcher/redirects.js +2 -3
  81. package/dist/services/fetcher/response.d.ts +0 -1
  82. package/dist/services/fetcher/response.js +4 -5
  83. package/dist/services/fetcher/retry-policy.d.ts +0 -1
  84. package/dist/services/fetcher/retry-policy.js +0 -1
  85. package/dist/services/fetcher.d.ts +1 -2
  86. package/dist/services/fetcher.d.ts.map +1 -1
  87. package/dist/services/fetcher.js +3 -4
  88. package/dist/services/fetcher.js.map +1 -1
  89. package/dist/services/logger.d.ts +1 -2
  90. package/dist/services/logger.d.ts.map +1 -1
  91. package/dist/services/logger.js +0 -1
  92. package/dist/services/parser.d.ts +1 -2
  93. package/dist/services/parser.d.ts.map +1 -1
  94. package/dist/services/parser.js +0 -1
  95. package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
  96. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  97. package/dist/tools/handlers/fetch-markdown.tool.js +34 -3
  98. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  99. package/dist/tools/handlers/fetch-single.shared.d.ts +5 -3
  100. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
  101. package/dist/tools/handlers/fetch-single.shared.js +8 -4
  102. package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
  103. package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
  104. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  105. package/dist/tools/handlers/fetch-url.tool.js +36 -3
  106. package/dist/tools/index.d.ts +0 -1
  107. package/dist/tools/index.js +0 -1
  108. package/dist/tools/schemas.d.ts +29 -30
  109. package/dist/tools/schemas.js +0 -1
  110. package/dist/tools/utils/cache-vary.d.ts +0 -1
  111. package/dist/tools/utils/cache-vary.d.ts.map +1 -1
  112. package/dist/tools/utils/cache-vary.js +7 -25
  113. package/dist/tools/utils/cache-vary.js.map +1 -1
  114. package/dist/tools/utils/common.d.ts +2 -3
  115. package/dist/tools/utils/common.d.ts.map +1 -1
  116. package/dist/tools/utils/common.js +6 -7
  117. package/dist/tools/utils/common.js.map +1 -1
  118. package/dist/tools/utils/content-transform.d.ts +1 -3
  119. package/dist/tools/utils/content-transform.d.ts.map +1 -1
  120. package/dist/tools/utils/content-transform.js +42 -6
  121. package/dist/tools/utils/content-transform.js.map +1 -1
  122. package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
  123. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  124. package/dist/tools/utils/fetch-pipeline.js +11 -18
  125. package/dist/tools/utils/inline-content.d.ts +0 -1
  126. package/dist/tools/utils/inline-content.js +0 -1
  127. package/dist/transformers/jsonl.transformer.d.ts +1 -2
  128. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  129. package/dist/transformers/jsonl.transformer.js +0 -1
  130. package/dist/transformers/jsonl.transformer.js.map +1 -1
  131. package/dist/transformers/markdown.transformer.d.ts +1 -2
  132. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  133. package/dist/transformers/markdown.transformer.js +8 -1
  134. package/dist/utils/code-language.d.ts +0 -1
  135. package/dist/utils/code-language.js +0 -1
  136. package/dist/utils/content-cleaner.d.ts +0 -1
  137. package/dist/utils/content-cleaner.js +0 -1
  138. package/dist/utils/crypto.d.ts +0 -1
  139. package/dist/utils/crypto.js +0 -1
  140. package/dist/utils/download-url.d.ts +1 -2
  141. package/dist/utils/download-url.d.ts.map +1 -1
  142. package/dist/utils/download-url.js +0 -1
  143. package/dist/utils/error-utils.d.ts +0 -1
  144. package/dist/utils/error-utils.js +1 -3
  145. package/dist/utils/filename-generator.d.ts +0 -1
  146. package/dist/utils/filename-generator.js +0 -1
  147. package/dist/utils/header-normalizer.d.ts +0 -1
  148. package/dist/utils/header-normalizer.js +0 -1
  149. package/dist/utils/html-truncator.d.ts +0 -1
  150. package/dist/utils/html-truncator.js +0 -1
  151. package/dist/utils/sanitizer.d.ts +0 -1
  152. package/dist/utils/sanitizer.js +0 -1
  153. package/dist/utils/tool-error-handler.d.ts +1 -2
  154. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  155. package/dist/utils/tool-error-handler.js +0 -1
  156. package/dist/utils/url-validator.d.ts +1 -3
  157. package/dist/utils/url-validator.d.ts.map +1 -1
  158. package/dist/utils/url-validator.js +44 -10
  159. package/dist/utils/url-validator.js.map +1 -1
  160. package/package.json +4 -4
@@ -1,10 +1,19 @@
1
1
  import { TRUNCATION_MARKER } from '../../config/formatting.js';
2
2
  import { extractContent } from '../../services/extractor.js';
3
3
  import { parseHtml } from '../../services/parser.js';
4
+ import { sanitizeText } from '../../utils/sanitizer.js';
4
5
  import { toJsonl } from '../../transformers/jsonl.transformer.js';
5
6
  import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
6
7
  import { createContentMetadataBlock, determineContentExtractionSource, truncateContent, } from './common.js';
8
+ const TITLE_PATTERN = /<title[^>]*>([\s\S]*?)<\/title>/i;
7
9
  function resolveContentSource(html, url, options) {
10
+ if (!options.extractMainContent && !options.includeMetadata) {
11
+ return {
12
+ sourceHtml: html,
13
+ title: extractTitleFromHtml(html),
14
+ metadata: undefined,
15
+ };
16
+ }
8
17
  const { article, metadata: extractedMeta } = extractContent(html, url, {
9
18
  extractArticle: options.extractMainContent,
10
19
  });
@@ -14,8 +23,36 @@ function resolveContentSource(html, url, options) {
14
23
  const title = shouldExtractFromArticle ? article.title : extractedMeta.title;
15
24
  return { sourceHtml, title, metadata };
16
25
  }
17
- function buildTransformContext(html, url, options) {
18
- return resolveContentSource(html, url, options);
26
+ function extractTitleFromHtml(html) {
27
+ const match = TITLE_PATTERN.exec(html);
28
+ if (!match?.[1])
29
+ return undefined;
30
+ const decoded = decodeHtmlEntities(match[1]);
31
+ const text = sanitizeText(decoded);
32
+ return text || undefined;
33
+ }
34
+ function decodeHtmlEntities(value) {
35
+ if (!value.includes('&'))
36
+ return value;
37
+ const basicDecoded = value
38
+ .replace(/&amp;/g, '&')
39
+ .replace(/&lt;/g, '<')
40
+ .replace(/&gt;/g, '>')
41
+ .replace(/&quot;/g, '"')
42
+ .replace(/&#39;/g, "'");
43
+ return basicDecoded
44
+ .replace(/&#(\d+);/g, (match, code) => {
45
+ const parsed = Number.parseInt(code, 10);
46
+ return Number.isFinite(parsed) && parsed >= 0 && parsed <= 0x10ffff
47
+ ? String.fromCodePoint(parsed)
48
+ : match;
49
+ })
50
+ .replace(/&#x([0-9a-fA-F]+);/g, (match, code) => {
51
+ const parsed = Number.parseInt(code, 16);
52
+ return Number.isFinite(parsed) && parsed >= 0 && parsed <= 0x10ffff
53
+ ? String.fromCodePoint(parsed)
54
+ : match;
55
+ });
19
56
  }
20
57
  function buildJsonlPayload(context, maxContentLength) {
21
58
  const contentBlocks = parseHtml(context.sourceHtml);
@@ -32,7 +69,7 @@ function buildMarkdownPayload(context, maxContentLength) {
32
69
  return { content, truncated };
33
70
  }
34
71
  export function transformHtmlToJsonl(html, url, options) {
35
- const context = buildTransformContext(html, url, options);
72
+ const context = resolveContentSource(html, url, options);
36
73
  const { content, contentBlocks, truncated } = buildJsonlPayload(context, options.maxContentLength);
37
74
  return {
38
75
  content,
@@ -42,7 +79,7 @@ export function transformHtmlToJsonl(html, url, options) {
42
79
  };
43
80
  }
44
81
  export function transformHtmlToMarkdown(html, url, options) {
45
- const context = buildTransformContext(html, url, options);
82
+ const context = resolveContentSource(html, url, options);
46
83
  const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
47
84
  return {
48
85
  markdown: content,
@@ -51,7 +88,7 @@ export function transformHtmlToMarkdown(html, url, options) {
51
88
  };
52
89
  }
53
90
  export function transformHtmlToMarkdownWithBlocks(html, url, options) {
54
- const context = buildTransformContext(html, url, options);
91
+ const context = resolveContentSource(html, url, options);
55
92
  const contentBlocks = parseHtml(context.sourceHtml);
56
93
  const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
57
94
  return {
@@ -61,4 +98,3 @@ export function transformHtmlToMarkdownWithBlocks(html, url, options) {
61
98
  ...(truncated && { truncated }),
62
99
  };
63
100
  }
64
- //# sourceMappingURL=content-transform.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;AA2BrB,SAAS,oBAAoB,CAC3B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,qBAAqB,CAC5B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,OAAO,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAyB,EACzB,gBAAyB;IAEzB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,EACxC,gBAAgB,CACjB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAyB,EACzB,gBAAyB;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAC7D,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa;QACb,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;AAmBrB,SAAS,oBAAoB,CAC3B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,EACxC,gBAAgB,CACjB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAC7D,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa;QACb,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC"}
@@ -1,4 +1,4 @@
1
- import type { FetchPipelineOptions, PipelineResult } from '../../config/types.js';
1
+ import type { FetchPipelineOptions, PipelineResult } from '../../config/types/runtime.js';
2
2
  /**
3
3
  * Unified fetch pipeline that handles caching, fetching, and transformation.
4
4
  * Implements cache-first strategy with automatic serialization.
@@ -8,4 +8,3 @@ import type { FetchPipelineOptions, PipelineResult } from '../../config/types.js
8
8
  * @returns Promise resolving to the pipeline result
9
9
  */
10
10
  export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
11
- //# sourceMappingURL=fetch-pipeline.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAuD/B;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAwB5B"}
1
+ {"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,+BAA+B,CAAC;AAuDvC;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAwB5B"}
@@ -1,29 +1,22 @@
1
1
  import * as cache from '../../services/cache.js';
2
2
  import { fetchUrlWithRetry } from '../../services/fetcher.js';
3
- import { logDebug, logWarn } from '../../services/logger.js';
3
+ import { logDebug } from '../../services/logger.js';
4
4
  import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
5
5
  import { appendHeaderVary } from './cache-vary.js';
6
- function safeJsonParse(cached, cacheKey) {
7
- try {
8
- return JSON.parse(cached);
9
- }
10
- catch {
11
- logWarn('Cache deserialize failed, treating as miss', {
12
- key: cacheKey.substring(0, 100),
13
- });
14
- return undefined;
15
- }
16
- }
17
6
  function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
18
7
  if (!cacheKey)
19
8
  return null;
20
9
  const cached = cache.get(cacheKey);
21
10
  if (!cached)
22
11
  return null;
23
- logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
24
- const data = deserialize
25
- ? deserialize(cached.content)
26
- : safeJsonParse(cached.content, cacheKey);
12
+ if (!deserialize) {
13
+ logDebug('Cache miss due to missing deserializer', {
14
+ namespace: cacheNamespace,
15
+ url: normalizedUrl,
16
+ });
17
+ return null;
18
+ }
19
+ const data = deserialize(cached.content);
27
20
  if (data === undefined) {
28
21
  logDebug('Cache miss due to deserialize failure', {
29
22
  namespace: cacheNamespace,
@@ -31,6 +24,7 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
31
24
  });
32
25
  return null;
33
26
  }
27
+ logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
34
28
  return {
35
29
  data,
36
30
  fromCache: true,
@@ -48,7 +42,7 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
48
42
  * @returns Promise resolving to the pipeline result
49
43
  */
50
44
  export async function executeFetchPipeline(options) {
51
- const normalizedUrl = validateAndNormalizeUrl(options.url);
45
+ const normalizedUrl = await validateAndNormalizeUrl(options.url);
52
46
  const cacheKey = resolveCacheKey(options, normalizedUrl);
53
47
  const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, normalizedUrl);
54
48
  if (cachedResult)
@@ -97,4 +91,3 @@ function buildPipelineResult(url, data, cacheKey) {
97
91
  cacheKey,
98
92
  };
99
93
  }
100
- //# sourceMappingURL=fetch-pipeline.js.map
@@ -9,4 +9,3 @@ interface InlineContentResult {
9
9
  }
10
10
  export declare function applyInlineContentLimit(content: string, cacheKey: string | null, format: InlineContentFormat): InlineContentResult;
11
11
  export {};
12
- //# sourceMappingURL=inline-content.d.ts.map
@@ -36,4 +36,3 @@ function buildTruncatedFallback(content, contentSize, inlineLimit) {
36
36
  truncated: true,
37
37
  };
38
38
  }
39
- //# sourceMappingURL=inline-content.js.map
@@ -1,3 +1,2 @@
1
- import type { ContentBlockUnion, MetadataBlock } from '../config/types.js';
1
+ import type { ContentBlockUnion, MetadataBlock } from '../config/types/content.js';
2
2
  export declare function toJsonl(blocks: readonly ContentBlockUnion[], metadata?: MetadataBlock): string;
3
- //# sourceMappingURL=jsonl.transformer.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAkE3E,wBAAgB,OAAO,CACrB,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,CAGR"}
1
+ {"version":3,"file":"jsonl.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,iBAAiB,EACjB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAkEpC,wBAAgB,OAAO,CACrB,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,CAGR"}
@@ -73,4 +73,3 @@ function serializeMetadata(metadata) {
73
73
  return null;
74
74
  }
75
75
  }
76
- //# sourceMappingURL=jsonl.transformer.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl.transformer.js","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,MAAM,IAAI,KAAK,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;AAC/B,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAmD,EACnD,SAAiB;IAEjB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACtD,OAAO,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAsD,EACtD,SAAiB;IAEjB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9C,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAC9B,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CACpC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAC7C,CAAC;IACF,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAClE,CAAC;AAED,SAAS,aAAa,CAAC,KAAwB;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC;IAEnD,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAwB;IAC9C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CACrB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"jsonl.transformer.js","sourceRoot":"","sources":["../../src/transformers/jsonl.transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAM5C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,SAAS;IACT,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,MAAM,IAAI,KAAK,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB;IAExB,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;AAC/B,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAmD,EACnD,SAAiB;IAEjB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACtD,OAAO,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAsD,EACtD,SAAiB;IAEjB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9C,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAC9B,CAAC;IACF,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CACpC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAC7C,CAAC;IACF,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAClE,CAAC;AAED,SAAS,aAAa,CAAC,KAAwB;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC;IAEnD,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAwB;IAC9C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CACrB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAoC,EACpC,QAAwB;IAExB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
@@ -1,3 +1,2 @@
1
- import type { MetadataBlock } from '../config/types.js';
1
+ import type { MetadataBlock } from '../config/types/content.js';
2
2
  export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock): string;
3
- //# sourceMappingURL=markdown.transformer.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"markdown.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/markdown.transformer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AA+HxD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,aAAa,GAAG,MAAM,CAa7E"}
1
+ {"version":3,"file":"markdown.transformer.d.ts","sourceRoot":"","sources":["../../src/transformers/markdown.transformer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AA+HhE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,aAAa,GAAG,MAAM,CAa7E"}
@@ -41,8 +41,16 @@ function isFencedCodeBlock(node, options) {
41
41
  return false;
42
42
  return firstChild.nodeName === 'CODE';
43
43
  }
44
+ function isElement(node) {
45
+ return (node !== null &&
46
+ typeof node === 'object' &&
47
+ 'getAttribute' in node &&
48
+ typeof node.getAttribute === 'function');
49
+ }
44
50
  function formatFencedCodeBlock(node) {
45
51
  const codeNode = node.firstChild;
52
+ if (!isElement(codeNode))
53
+ return '';
46
54
  const code = codeNode.textContent || '';
47
55
  const language = resolveCodeLanguage(codeNode, code);
48
56
  return CODE_BLOCK.format(code, language);
@@ -116,4 +124,3 @@ export function htmlToMarkdown(html, metadata) {
116
124
  function isValidHtmlInput(html) {
117
125
  return Boolean(html && typeof html === 'string');
118
126
  }
119
- //# sourceMappingURL=markdown.transformer.js.map
@@ -1,3 +1,2 @@
1
1
  export declare function detectLanguageFromCode(code: string): string | undefined;
2
2
  export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
3
- //# sourceMappingURL=code-language.d.ts.map
@@ -54,4 +54,3 @@ function resolveLanguageFromDataAttribute(dataLang) {
54
54
  const match = /^(\w+)$/.exec(dataLang);
55
55
  return match?.[1];
56
56
  }
57
- //# sourceMappingURL=code-language.js.map
@@ -3,4 +3,3 @@ export declare function cleanHeading(text: string): string | null;
3
3
  export declare function cleanListItems(items: string[]): string[];
4
4
  export declare function cleanCodeBlock(code: string): string | null;
5
5
  export declare function removeInlineTimestamps(text: string): string;
6
- //# sourceMappingURL=content-cleaner.d.ts.map
@@ -75,4 +75,3 @@ export function removeInlineTimestamps(text) {
75
75
  .replace(/\s{2,}/g, ' ')
76
76
  .trim();
77
77
  }
78
- //# sourceMappingURL=content-cleaner.js.map
@@ -1,3 +1,2 @@
1
1
  export declare function timingSafeEqualUtf8(a: string, b: string): boolean;
2
2
  export declare function sha256Hex(input: string | Uint8Array): string;
3
- //# sourceMappingURL=crypto.d.ts.map
@@ -30,4 +30,3 @@ function hashHex(algorithm, input) {
30
30
  export function sha256Hex(input) {
31
31
  return hashHex('sha256', input);
32
32
  }
33
- //# sourceMappingURL=crypto.js.map
@@ -1,4 +1,4 @@
1
- import type { FileDownloadInfo } from '../config/types.js';
1
+ import type { FileDownloadInfo } from '../config/types/tools.js';
2
2
  interface DownloadInfoOptions {
3
3
  cacheKey: string | null;
4
4
  url: string;
@@ -6,4 +6,3 @@ interface DownloadInfoOptions {
6
6
  }
7
7
  export declare function buildFileDownloadInfo(options: DownloadInfoOptions): FileDownloadInfo | null;
8
8
  export {};
9
- //# sourceMappingURL=download-url.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"download-url.d.ts","sourceRoot":"","sources":["../../src/utils/download-url.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAM3D,UAAU,mBAAmB;IAC3B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,mBAAmB,GAC3B,gBAAgB,GAAG,IAAI,CA0BzB"}
1
+ {"version":3,"file":"download-url.d.ts","sourceRoot":"","sources":["../../src/utils/download-url.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAMjE,UAAU,mBAAmB;IAC3B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,mBAAmB,GAC3B,gBAAgB,GAAG,IAAI,CA0BzB"}
@@ -25,4 +25,3 @@ function buildDownloadUrl(namespace, hash) {
25
25
  function resolveExtension(namespace) {
26
26
  return namespace === 'markdown' ? '.md' : '.jsonl';
27
27
  }
28
- //# sourceMappingURL=download-url.js.map
@@ -1,4 +1,3 @@
1
1
  export declare function getErrorMessage(error: unknown): string;
2
2
  export declare function createErrorWithCode(message: string, code: string): NodeJS.ErrnoException;
3
3
  export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;
4
- //# sourceMappingURL=error-utils.d.ts.map
@@ -3,12 +3,10 @@ export function getErrorMessage(error) {
3
3
  }
4
4
  export function createErrorWithCode(message, code) {
5
5
  const error = new Error(message);
6
- error.code = code;
7
- return error;
6
+ return Object.assign(error, { code });
8
7
  }
9
8
  export function isSystemError(error) {
10
9
  return (error instanceof Error &&
11
10
  'code' in error &&
12
11
  typeof error.code === 'string');
13
12
  }
14
- //# sourceMappingURL=error-utils.js.map
@@ -1,2 +1 @@
1
1
  export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
2
- //# sourceMappingURL=filename-generator.d.ts.map
@@ -57,4 +57,3 @@ function sanitizeFilename(name, extension) {
57
57
  }
58
58
  return `${sanitized}${extension}`;
59
59
  }
60
- //# sourceMappingURL=filename-generator.js.map
@@ -6,4 +6,3 @@ export declare function normalizeHeaderEntries(headers: Record<string, string>,
6
6
  export declare function hasHeaderEntries(headers: Headers): boolean;
7
7
  export declare function headersToRecord(headers: Headers): Record<string, string>;
8
8
  export {};
9
- //# sourceMappingURL=header-normalizer.d.ts.map
@@ -29,4 +29,3 @@ function setHeaderValue(headers, key, value, trimValue) {
29
29
  // Ignore invalid header values
30
30
  }
31
31
  }
32
- //# sourceMappingURL=header-normalizer.js.map
@@ -1,2 +1 @@
1
1
  export declare function truncateHtml(html: string): string;
2
- //# sourceMappingURL=html-truncator.d.ts.map
@@ -11,4 +11,3 @@ export function truncateHtml(html) {
11
11
  });
12
12
  return html.substring(0, maxSize);
13
13
  }
14
- //# sourceMappingURL=html-truncator.js.map
@@ -1,3 +1,2 @@
1
1
  export declare function sanitizeText(text: string | null | undefined): string;
2
2
  export declare function truncateText(text: string, maxLength: number): string;
3
- //# sourceMappingURL=sanitizer.d.ts.map
@@ -18,4 +18,3 @@ export function truncateText(text, maxLength) {
18
18
  const truncationPoint = maxLength - TRUNCATION_SUFFIX.length;
19
19
  return `${text.substring(0, truncationPoint)}${TRUNCATION_SUFFIX}`;
20
20
  }
21
- //# sourceMappingURL=sanitizer.js.map
@@ -1,4 +1,3 @@
1
- import type { ToolErrorResponse } from '../config/types.js';
1
+ import type { ToolErrorResponse } from '../config/types/tools.js';
2
2
  export declare function createToolErrorResponse(message: string, url: string, code: string): ToolErrorResponse;
3
3
  export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
4
- //# sourceMappingURL=tool-error-handler.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AA6B5D,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX,iBAAiB,CAanB;AAgBD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAoBnB"}
1
+ {"version":3,"file":"tool-error-handler.d.ts","sourceRoot":"","sources":["../../src/utils/tool-error-handler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AA6BlE,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,GACX,iBAAiB,CAanB;AAgBD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAoBnB"}
@@ -61,4 +61,3 @@ function isValidationError(error) {
61
61
  isSystemError(error) &&
62
62
  error.code === 'VALIDATION_ERROR');
63
63
  }
64
- //# sourceMappingURL=tool-error-handler.js.map
@@ -1,4 +1,2 @@
1
1
  export declare function isBlockedIp(ip: string): boolean;
2
- export declare function validateAndNormalizeUrl(urlString: string): string;
3
- export declare function isInternalUrl(url: string, baseUrl: string): boolean;
4
- //# sourceMappingURL=url-validator.d.ts.map
2
+ export declare function validateAndNormalizeUrl(urlString: string): Promise<string>;
@@ -1 +1 @@
1
- {"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AA8CA,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAS/C;AAcD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYjE;AAiFD,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAOnE"}
1
+ {"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AA8CA,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAS/C;AAcD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYjE"}
@@ -1,3 +1,4 @@
1
+ import { lookup } from 'node:dns/promises';
1
2
  import { BlockList, isIP } from 'node:net';
2
3
  import { config } from '../config/index.js';
3
4
  import { createErrorWithCode } from './error-utils.js';
@@ -30,6 +31,7 @@ for (const entry of BLOCKED_IPV4_SUBNETS) {
30
31
  for (const entry of BLOCKED_IPV6_SUBNETS) {
31
32
  BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
32
33
  }
34
+ const DNS_LOOKUP_TIMEOUT_MS = 5000;
33
35
  function matchesBlockedIpPatterns(resolvedIp) {
34
36
  for (const pattern of config.security.blockedIpPatterns) {
35
37
  if (pattern.test(resolvedIp)) {
@@ -60,7 +62,47 @@ function isBlockedByList(ip, ipType) {
60
62
  }
61
63
  return BLOCK_LIST.check(ip, 'ipv6');
62
64
  }
63
- export function validateAndNormalizeUrl(urlString) {
65
+ function lookupWithTimeout(hostname) {
66
+ return new Promise((resolve, reject) => {
67
+ const timer = setTimeout(() => {
68
+ reject(createValidationError(`DNS lookup timed out for ${hostname}`));
69
+ }, DNS_LOOKUP_TIMEOUT_MS);
70
+ lookup(hostname, { all: true })
71
+ .then((result) => {
72
+ clearTimeout(timer);
73
+ resolve(result);
74
+ })
75
+ .catch((error) => {
76
+ clearTimeout(timer);
77
+ reject(error instanceof Error ? error : createValidationError(String(error)));
78
+ });
79
+ });
80
+ }
81
+ async function assertResolvedAddressesAllowed(hostname) {
82
+ try {
83
+ const result = await lookupWithTimeout(hostname);
84
+ const addresses = Array.isArray(result) ? result : [result];
85
+ if (addresses.length === 0) {
86
+ throw createValidationError(`Unable to resolve hostname: ${hostname}`);
87
+ }
88
+ for (const { address } of addresses) {
89
+ if (isBlockedIp(address.toLowerCase())) {
90
+ throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
91
+ }
92
+ }
93
+ }
94
+ catch (error) {
95
+ const code = error?.code;
96
+ if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') {
97
+ throw createValidationError(`Unable to resolve hostname: ${hostname}`);
98
+ }
99
+ if (error instanceof Error) {
100
+ throw error;
101
+ }
102
+ throw createValidationError(String(error));
103
+ }
104
+ }
105
+ export async function validateAndNormalizeUrl(urlString) {
64
106
  const trimmedUrl = requireTrimmedUrl(urlString);
65
107
  assertUrlLength(trimmedUrl);
66
108
  const url = parseUrl(trimmedUrl);
@@ -68,6 +110,7 @@ export function validateAndNormalizeUrl(urlString) {
68
110
  assertNoCredentials(url);
69
111
  const hostname = normalizeHostname(url);
70
112
  assertHostnameAllowed(hostname);
113
+ await assertResolvedAddressesAllowed(hostname);
71
114
  return url.href;
72
115
  }
73
116
  const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
@@ -127,12 +170,3 @@ function assertHostnameAllowed(hostname) {
127
170
  function matchesBlockedSuffix(hostname) {
128
171
  return BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
129
172
  }
130
- export function isInternalUrl(url, baseUrl) {
131
- if (!URL.canParse(baseUrl) || !URL.canParse(url, baseUrl)) {
132
- return false;
133
- }
134
- const urlObj = new URL(url, baseUrl);
135
- const baseUrlObj = new URL(baseUrl);
136
- return urlObj.hostname === baseUrlObj.hostname;
137
- }
138
- //# sourceMappingURL=url-validator.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAEjD,SAAS,qBAAqB,CAAC,OAAe;IAC5C,OAAO,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY;QAAE,OAAO;IACxD,MAAM,qBAAqB,CACzB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAQ;IAClC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO;IAClE,MAAM,qBAAqB,CACzB,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO;IAC3C,MAAM,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,qBAAqB,CAAC,gCAAgC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAU,CAAC;AAE/D,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,qBAAqB,CACzB,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,qBAAqB,CACzB,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,qBAAqB,CACzB,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;AACjD,CAAC"}
1
+ {"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAEjD,SAAS,qBAAqB,CAAC,OAAe;IAC5C,OAAO,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY;QAAE,OAAO;IACxD,MAAM,qBAAqB,CACzB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAQ;IAClC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO;IAClE,MAAM,qBAAqB,CACzB,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO;IAC3C,MAAM,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,qBAAqB,CAAC,gCAAgC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAU,CAAC;AAE/D,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,qBAAqB,CACzB,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,qBAAqB,CACzB,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,qBAAqB,CACzB,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@j0hanz/superfetch",
3
- "version": "1.2.0",
3
+ "version": "1.2.1",
4
4
  "mcpName": "io.github.j0hanz/superfetch",
5
5
  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
6
6
  "type": "module",
@@ -35,7 +35,7 @@
35
35
  ],
36
36
  "scripts": {
37
37
  "dev": "tsx watch src/index.ts",
38
- "build": "tsc && shx chmod +x dist/*.js",
38
+ "build": "tsc -p tsconfig.build.json && shx chmod +x dist/*.js",
39
39
  "prepare": "npm run build",
40
40
  "prepublishOnly": "npm run build && npm run lint",
41
41
  "start": "node dist/index.js",
@@ -71,12 +71,12 @@
71
71
  "eslint": "^9.23.2",
72
72
  "eslint-config-prettier": "^10.1.8",
73
73
  "eslint-plugin-unused-imports": "^4.3.0",
74
- "knip": "^5.77.1",
74
+ "knip": "^5.78.0",
75
75
  "prettier": "^3.7.4",
76
76
  "shx": "^0.4.0",
77
77
  "tsx": "^4.21.0",
78
78
  "typescript": "^5.9.3",
79
- "typescript-eslint": "^8.50.1",
79
+ "typescript-eslint": "^8.51.0",
80
80
  "vitest": "^2.1.9"
81
81
  },
82
82
  "engines": {