@j0hanz/superfetch 1.1.2 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +52 -30
  2. package/dist/config/formatting.d.ts +9 -0
  3. package/dist/config/formatting.d.ts.map +1 -0
  4. package/dist/config/formatting.js +11 -0
  5. package/dist/config/formatting.js.map +1 -0
  6. package/dist/config/index.d.ts +11 -2
  7. package/dist/config/index.d.ts.map +1 -1
  8. package/dist/config/index.js +35 -13
  9. package/dist/config/index.js.map +1 -1
  10. package/dist/config/types/content.d.ts +107 -0
  11. package/dist/config/types/content.d.ts.map +1 -0
  12. package/dist/config/types/content.js +2 -0
  13. package/dist/config/types/content.js.map +1 -0
  14. package/dist/config/types/runtime.d.ts +78 -0
  15. package/dist/config/types/runtime.d.ts.map +1 -0
  16. package/dist/config/types/runtime.js +2 -0
  17. package/dist/config/types/runtime.js.map +1 -0
  18. package/dist/config/types/tools.d.ts +99 -0
  19. package/dist/config/types/tools.d.ts.map +1 -0
  20. package/dist/config/types/tools.js +2 -0
  21. package/dist/config/types/tools.js.map +1 -0
  22. package/dist/config/types.d.ts +3 -297
  23. package/dist/config/types.d.ts.map +1 -1
  24. package/dist/http/auth.d.ts +3 -0
  25. package/dist/http/auth.d.ts.map +1 -0
  26. package/dist/http/auth.js +34 -0
  27. package/dist/http/auth.js.map +1 -0
  28. package/dist/http/cors.d.ts +8 -0
  29. package/dist/http/cors.d.ts.map +1 -0
  30. package/dist/http/cors.js +47 -0
  31. package/dist/http/cors.js.map +1 -0
  32. package/dist/http/mcp-routes.d.ts +5 -0
  33. package/dist/http/mcp-routes.d.ts.map +1 -0
  34. package/dist/http/mcp-routes.js +110 -0
  35. package/dist/http/mcp-routes.js.map +1 -0
  36. package/dist/http/mcp-session.d.ts +12 -0
  37. package/dist/http/mcp-session.d.ts.map +1 -0
  38. package/dist/http/mcp-session.js +209 -0
  39. package/dist/http/mcp-session.js.map +1 -0
  40. package/dist/http/mcp-validation.d.ts +3 -0
  41. package/dist/http/mcp-validation.d.ts.map +1 -0
  42. package/dist/http/mcp-validation.js +34 -0
  43. package/dist/http/mcp-validation.js.map +1 -0
  44. package/dist/http/rate-limit.d.ts +13 -0
  45. package/dist/http/rate-limit.d.ts.map +1 -0
  46. package/dist/http/rate-limit.js +91 -0
  47. package/dist/http/rate-limit.js.map +1 -0
  48. package/dist/http/server.d.ts +4 -0
  49. package/dist/http/server.d.ts.map +1 -0
  50. package/dist/http/server.js +183 -0
  51. package/dist/http/server.js.map +1 -0
  52. package/dist/http/sessions.d.ts +15 -0
  53. package/dist/http/sessions.d.ts.map +1 -0
  54. package/dist/http/sessions.js +64 -0
  55. package/dist/http/sessions.js.map +1 -0
  56. package/dist/index.js +26 -341
  57. package/dist/index.js.map +1 -1
  58. package/dist/middleware/error-handler.d.ts +2 -2
  59. package/dist/middleware/error-handler.d.ts.map +1 -1
  60. package/dist/middleware/error-handler.js +46 -15
  61. package/dist/middleware/error-handler.js.map +1 -1
  62. package/dist/resources/cached-content.d.ts.map +1 -1
  63. package/dist/resources/cached-content.js +104 -44
  64. package/dist/resources/cached-content.js.map +1 -1
  65. package/dist/resources/index.d.ts.map +1 -1
  66. package/dist/resources/index.js +77 -69
  67. package/dist/resources/index.js.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +9 -3
  70. package/dist/server.js.map +1 -1
  71. package/dist/services/cache.d.ts +13 -1
  72. package/dist/services/cache.d.ts.map +1 -1
  73. package/dist/services/cache.js +89 -16
  74. package/dist/services/cache.js.map +1 -1
  75. package/dist/services/context.d.ts +1 -1
  76. package/dist/services/context.d.ts.map +1 -1
  77. package/dist/services/context.js +1 -1
  78. package/dist/services/context.js.map +1 -1
  79. package/dist/services/extractor.d.ts.map +1 -1
  80. package/dist/services/extractor.js +122 -87
  81. package/dist/services/extractor.js.map +1 -1
  82. package/dist/services/fetcher/agents.d.ts +4 -0
  83. package/dist/services/fetcher/agents.d.ts.map +1 -0
  84. package/dist/services/fetcher/agents.js +111 -0
  85. package/dist/services/fetcher/agents.js.map +1 -0
  86. package/dist/services/fetcher/errors.d.ts +5 -0
  87. package/dist/services/fetcher/errors.d.ts.map +1 -0
  88. package/dist/services/fetcher/errors.js +71 -0
  89. package/dist/services/fetcher/errors.js.map +1 -0
  90. package/dist/services/fetcher/headers.d.ts +2 -0
  91. package/dist/services/fetcher/headers.d.ts.map +1 -0
  92. package/dist/services/fetcher/headers.js +28 -0
  93. package/dist/services/fetcher/headers.js.map +1 -0
  94. package/dist/services/fetcher/interceptors.d.ts +10 -0
  95. package/dist/services/fetcher/interceptors.d.ts.map +1 -0
  96. package/dist/services/fetcher/interceptors.js +82 -0
  97. package/dist/services/fetcher/interceptors.js.map +1 -0
  98. package/dist/services/fetcher/redirects.d.ts +6 -0
  99. package/dist/services/fetcher/redirects.d.ts.map +1 -0
  100. package/dist/services/fetcher/redirects.js +67 -0
  101. package/dist/services/fetcher/redirects.js.map +1 -0
  102. package/dist/services/fetcher/response.d.ts +5 -0
  103. package/dist/services/fetcher/response.d.ts.map +1 -0
  104. package/dist/services/fetcher/response.js +39 -0
  105. package/dist/services/fetcher/response.js.map +1 -0
  106. package/dist/services/fetcher/retry-policy.d.ts +28 -0
  107. package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
  108. package/dist/services/fetcher/retry-policy.js +138 -0
  109. package/dist/services/fetcher/retry-policy.js.map +1 -0
  110. package/dist/services/fetcher.d.ts +2 -1
  111. package/dist/services/fetcher.d.ts.map +1 -1
  112. package/dist/services/fetcher.js +62 -315
  113. package/dist/services/fetcher.js.map +1 -1
  114. package/dist/services/logger.js +4 -4
  115. package/dist/services/logger.js.map +1 -1
  116. package/dist/services/parser.d.ts +1 -0
  117. package/dist/services/parser.d.ts.map +1 -1
  118. package/dist/services/parser.js +55 -35
  119. package/dist/services/parser.js.map +1 -1
  120. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
  121. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
  122. package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
  123. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
  124. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  125. package/dist/tools/handlers/fetch-links.tool.js +78 -116
  126. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  127. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
  128. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  129. package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
  130. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  131. package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
  132. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
  133. package/dist/tools/handlers/fetch-single.shared.js +49 -0
  134. package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
  135. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  136. package/dist/tools/handlers/fetch-url.tool.js +82 -54
  137. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  138. package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
  139. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
  140. package/dist/tools/handlers/fetch-urls/processor.js +153 -0
  141. package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
  142. package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
  143. package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
  144. package/dist/tools/handlers/fetch-urls/response.js +58 -0
  145. package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
  146. package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
  147. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
  148. package/dist/tools/handlers/fetch-urls/validation.js +18 -0
  149. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
  150. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  151. package/dist/tools/handlers/fetch-urls.tool.js +104 -202
  152. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  153. package/dist/tools/index.d.ts.map +1 -1
  154. package/dist/tools/index.js +36 -237
  155. package/dist/tools/index.js.map +1 -1
  156. package/dist/tools/schemas.d.ts +357 -0
  157. package/dist/tools/schemas.d.ts.map +1 -0
  158. package/dist/tools/schemas.js +272 -0
  159. package/dist/tools/schemas.js.map +1 -0
  160. package/dist/tools/utils/cache-vary.d.ts +3 -0
  161. package/dist/tools/utils/cache-vary.d.ts.map +1 -0
  162. package/dist/tools/utils/cache-vary.js +44 -0
  163. package/dist/tools/utils/cache-vary.js.map +1 -0
  164. package/dist/tools/utils/common.d.ts +2 -2
  165. package/dist/tools/utils/common.d.ts.map +1 -1
  166. package/dist/tools/utils/common.js +5 -1
  167. package/dist/tools/utils/common.js.map +1 -1
  168. package/dist/tools/utils/content-transform.d.ts +16 -0
  169. package/dist/tools/utils/content-transform.d.ts.map +1 -0
  170. package/dist/tools/utils/content-transform.js +49 -0
  171. package/dist/tools/utils/content-transform.js.map +1 -0
  172. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  173. package/dist/tools/utils/fetch-pipeline.js +32 -18
  174. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  175. package/dist/tools/utils/inline-content.d.ts +11 -0
  176. package/dist/tools/utils/inline-content.d.ts.map +1 -0
  177. package/dist/tools/utils/inline-content.js +39 -0
  178. package/dist/tools/utils/inline-content.js.map +1 -0
  179. package/dist/tools/utils/markdown-toc.d.ts +3 -0
  180. package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
  181. package/dist/tools/utils/markdown-toc.js +35 -0
  182. package/dist/tools/utils/markdown-toc.js.map +1 -0
  183. package/dist/tools/utils/tool-response.d.ts +9 -0
  184. package/dist/tools/utils/tool-response.d.ts.map +1 -0
  185. package/dist/tools/utils/tool-response.js +19 -0
  186. package/dist/tools/utils/tool-response.js.map +1 -0
  187. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  188. package/dist/transformers/jsonl.transformer.js +51 -28
  189. package/dist/transformers/jsonl.transformer.js.map +1 -1
  190. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  191. package/dist/transformers/markdown.transformer.js +82 -111
  192. package/dist/transformers/markdown.transformer.js.map +1 -1
  193. package/dist/utils/header-normalizer.d.ts +5 -0
  194. package/dist/utils/header-normalizer.d.ts.map +1 -0
  195. package/dist/utils/header-normalizer.js +25 -0
  196. package/dist/utils/header-normalizer.js.map +1 -0
  197. package/dist/utils/tool-error-handler.d.ts +1 -0
  198. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  199. package/dist/utils/tool-error-handler.js +29 -1
  200. package/dist/utils/tool-error-handler.js.map +1 -1
  201. package/dist/utils/url-validator.d.ts +0 -3
  202. package/dist/utils/url-validator.d.ts.map +1 -1
  203. package/dist/utils/url-validator.js +98 -18
  204. package/dist/utils/url-validator.js.map +1 -1
  205. package/package.json +11 -6
package/README.md CHANGED
@@ -307,6 +307,8 @@ node dist/index.js --stdio
307
307
 
308
308
  ## Available Tools
309
309
 
310
+ > **Note:** If extracted content exceeds `MAX_INLINE_CONTENT_CHARS`, the tool response includes a `resourceUri` and a `resource_link` content block instead of embedding the full text.
311
+
310
312
  ### `fetch-url`
311
313
 
312
314
  Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks.
@@ -635,6 +637,12 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
635
637
  | `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
636
638
  | `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
637
639
 
640
+ #### 📦 Output Settings
641
+
642
+ | Variable | Default | Valid Values | Description |
643
+ | -------------------------- | ------- | --------------- | --------------------------------------------------------------- |
644
+ | `MAX_INLINE_CONTENT_CHARS` | `20000` | `1000`-`200000` | Inline content limit before returning a `resource_link` instead |
645
+
638
646
  #### 📝 Logging Settings
639
647
 
640
648
  | Variable | Default | Valid Values | Description |
@@ -651,18 +659,26 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
651
659
 
652
660
  #### 🛡️ Security Settings
653
661
 
654
- | Variable | Default | Description |
655
- | -------------- | -------- | ---------------------------------------------------------------------------- |
656
- | `API_KEY` | | API Key for authentication (required if `REQUIRE_AUTH` is true) |
657
- | `REQUIRE_AUTH` | `true`\* | Require authentication (\*defaults to `false` on loopback unless configured) |
658
- | `ALLOW_REMOTE` | `false` | Allow binding to non-loopback interfaces |
662
+ | Variable | Default | Description |
663
+ | -------------- | ------- | -------------------------------------------------------- |
664
+ | `API_KEY` | - | API Key for HTTP authentication (required for HTTP mode) |
665
+ | `ALLOW_REMOTE` | `false` | Allow binding to non-loopback interfaces |
666
+
667
+ #### Rate Limiting
668
+
669
+ | Variable | Default | Valid Values | Description |
670
+ | ----------------------- | ------- | ----------------- | ------------------------------------ |
671
+ | `RATE_LIMIT_ENABLED` | `true` | `true` / `false` | Enable/disable HTTP rate limiting |
672
+ | `RATE_LIMIT_MAX` | `100` | `1`-`10000` | Max requests per window per IP |
673
+ | `RATE_LIMIT_WINDOW_MS` | `60000` | `1000`-`3600000` | Rate limit window in milliseconds |
674
+ | `RATE_LIMIT_CLEANUP_MS` | `60000` | `10000`-`3600000` | Cleanup interval for limiter entries |
659
675
 
660
676
  ### HTTP Mode Configuration
661
677
 
662
678
  <details>
663
679
  <summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
664
680
 
665
- SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
681
+ SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration and an `API_KEY` for authenticated access (send `Authorization: Bearer <key>` or `X-API-Key: <key>`).
666
682
 
667
683
  #### Start HTTP Server
668
684
 
@@ -673,14 +689,15 @@ npx -y @j0hanz/superfetch@latest
673
689
 
674
690
  #### HTTP-Specific Environment Variables
675
691
 
676
- | Variable | Default | Description |
677
- | ----------------- | ----------- | ------------------------------------------------ |
678
- | `PORT` | `3000` | HTTP server port |
679
- | `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
680
- | `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
681
- | `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
682
- | `SESSION_TTL_MS` | `1800000` | Session time-to-live in milliseconds (30 mins) |
683
- | `MAX_SESSIONS` | `200` | Maximum number of active sessions |
692
+ | Variable | Default | Description |
693
+ | ------------------------- | ----------- | ------------------------------------------------ |
694
+ | `PORT` | `3000` | HTTP server port |
695
+ | `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
696
+ | `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
697
+ | `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
698
+ | `SESSION_TTL_MS` | `1800000` | Session time-to-live in milliseconds (30 mins) |
699
+ | `SESSION_INIT_TIMEOUT_MS` | `10000` | Time allowed for session initialization (ms) |
700
+ | `MAX_SESSIONS` | `200` | Maximum number of active sessions |
684
701
 
685
702
  #### VS Code HTTP Mode Setup
686
703
 
@@ -749,7 +766,8 @@ Blocked headers: `host`, `authorization`, `cookie`, `x-forwarded-for`, `x-real-i
749
766
 
750
767
  ### Rate Limiting
751
768
 
752
- Default: **100 requests/minute** per IP (configurable)
769
+ Default: **100 requests/minute** per IP. Configure with `RATE_LIMIT_MAX` and
770
+ `RATE_LIMIT_WINDOW_MS`.
753
771
 
754
772
  ### HTTP Mode Endpoints
755
773
 
@@ -770,17 +788,20 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
770
788
 
771
789
  ### Scripts
772
790
 
773
- | Command | Description |
774
- | -------------------- | ---------------------------------- |
775
- | `npm run dev` | Development server with hot reload |
776
- | `npm run build` | Compile TypeScript |
777
- | `npm start` | Production server |
778
- | `npm run lint` | Run ESLint |
779
- | `npm run type-check` | TypeScript type checking |
780
- | `npm run format` | Format with Prettier |
781
- | `npm run release` | Create new release |
782
- | `npm run knip` | Find unused exports/dependencies |
783
- | `npm run knip:fix` | Auto-fix unused code |
791
+ | Command | Description |
792
+ | ----------------------- | ---------------------------------- |
793
+ | `npm run dev` | Development server with hot reload |
794
+ | `npm run build` | Compile TypeScript |
795
+ | `npm start` | Production server |
796
+ | `npm run lint` | Run ESLint |
797
+ | `npm run type-check` | TypeScript type checking |
798
+ | `npm run format` | Format with Prettier |
799
+ | `npm test` | Run Vitest tests |
800
+ | `npm run test:coverage` | Run tests with coverage |
801
+ | `npm run bench` | Run minimal performance benchmark |
802
+ | `npm run release` | Create new release |
803
+ | `npm run knip` | Find unused exports/dependencies |
804
+ | `npm run knip:fix` | Auto-fix unused code |
784
805
 
785
806
  ### Tech Stack
786
807
 
@@ -790,7 +811,7 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
790
811
  | Language | TypeScript 5.9 |
791
812
  | MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
792
813
  | Content Extraction | @mozilla/readability ^0.6.0 |
793
- | HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
814
+ | HTML Parsing | Cheerio ^1.1.2, LinkeDOM ^0.18.12 |
794
815
  | Markdown | Turndown ^7.2.2 |
795
816
  | HTTP | Express ^5.2.1, Axios ^1.7.9 |
796
817
  | Caching | node-cache ^5.1.2 |
@@ -803,8 +824,9 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
803
824
  1. Fork the repository
804
825
  2. Create a feature branch: `git checkout -b feature/amazing-feature`
805
826
  3. Ensure linting passes: `npm run lint`
806
- 4. Commit changes: `git commit -m 'Add amazing feature'`
807
- 5. Push: `git push origin feature/amazing-feature`
808
- 6. Open a Pull Request
827
+ 4. Run tests: `npm test`
828
+ 5. Commit changes: `git commit -m 'Add amazing feature'`
829
+ 6. Push: `git push origin feature/amazing-feature`
830
+ 7. Open a Pull Request
809
831
 
810
832
  For examples of other MCP servers, see: [github.com/modelcontextprotocol/servers](https://github.com/modelcontextprotocol/servers)
@@ -0,0 +1,9 @@
1
+ export declare const LINE_BREAK = "\n";
2
+ export declare const TRUNCATION_MARKER = "...[truncated]";
3
+ export declare const CODE_BLOCK: {
4
+ readonly fence: "```";
5
+ readonly format: (code: string, language?: string) => string;
6
+ };
7
+ export declare const FRONTMATTER_DELIMITER = "---";
8
+ export declare const joinLines: (lines: string[]) => string;
9
+ //# sourceMappingURL=formatting.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,OAAO,CAAC;AAE/B,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAElD,eAAO,MAAM,UAAU;;4BAEN,MAAM,wBAAkB,MAAM;CAGrC,CAAC;AAEX,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,GAAI,OAAO,MAAM,EAAE,KAAG,MAAgC,CAAC"}
@@ -0,0 +1,11 @@
1
+ export const LINE_BREAK = '\n';
2
+ export const TRUNCATION_MARKER = '...[truncated]';
3
+ export const CODE_BLOCK = {
4
+ fence: '```',
5
+ format: (code, language = '') => {
6
+ return `\`\`\`${language}\n${code}\n\`\`\``;
7
+ },
8
+ };
9
+ export const FRONTMATTER_DELIMITER = '---';
10
+ export const joinLines = (lines) => lines.join(LINE_BREAK);
11
+ //# sourceMappingURL=formatting.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"formatting.js","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAE/B,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;AAElD,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE,EAAU,EAAE;QAC9C,OAAO,SAAS,QAAQ,KAAK,IAAI,UAAU,CAAC;IAC9C,CAAC;CACO,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAE3C,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC"}
@@ -1,10 +1,12 @@
1
+ import type { LogLevel } from './types.js';
1
2
  export declare const config: {
2
3
  readonly server: {
3
4
  readonly name: "superFetch";
4
- readonly version: "1.1.2";
5
+ readonly version: string;
5
6
  readonly port: number;
6
7
  readonly host: string;
7
8
  readonly sessionTtlMs: number;
9
+ readonly sessionInitTimeoutMs: number;
8
10
  readonly maxSessions: number;
9
11
  };
10
12
  readonly fetcher: {
@@ -25,13 +27,14 @@ export declare const config: {
25
27
  readonly minParagraphLength: 10;
26
28
  };
27
29
  readonly logging: {
28
- readonly level: "debug" | "info" | "warn" | "error";
30
+ readonly level: LogLevel;
29
31
  readonly enabled: boolean;
30
32
  };
31
33
  readonly constants: {
32
34
  readonly maxHtmlSize: number;
33
35
  readonly maxContentSize: number;
34
36
  readonly maxUrlLength: 2048;
37
+ readonly maxInlineContentChars: number;
35
38
  };
36
39
  readonly security: {
37
40
  readonly blockedHosts: Set<string>;
@@ -41,5 +44,11 @@ export declare const config: {
41
44
  readonly allowRemote: boolean;
42
45
  readonly requireAuth: boolean;
43
46
  };
47
+ readonly rateLimit: {
48
+ readonly enabled: boolean;
49
+ readonly maxRequests: number;
50
+ readonly windowMs: number;
51
+ readonly cleanupIntervalMs: number;
52
+ };
44
53
  };
45
54
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAyCA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAiEV,SAAS,MAAM,EAAE;;;;;;CAahB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA6D3C,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAgFV,SAAS,MAAM,EAAE;;;;;;;;;;;;CA6BhB,CAAC"}
@@ -1,14 +1,20 @@
1
+ import packageJson from '../../package.json' with { type: 'json' };
1
2
  function parseInteger(envValue, defaultValue, min, max) {
3
+ const parsed = parseRawInteger(envValue, defaultValue);
4
+ return clampInteger(parsed, defaultValue, min, max);
5
+ }
6
+ function parseRawInteger(envValue, defaultValue) {
2
7
  if (!envValue)
3
8
  return defaultValue;
4
9
  const parsed = parseInt(envValue, 10);
5
- if (isNaN(parsed))
6
- return defaultValue;
7
- if (min !== undefined && parsed < min)
10
+ return Number.isNaN(parsed) ? defaultValue : parsed;
11
+ }
12
+ function clampInteger(value, defaultValue, min, max) {
13
+ if (min !== undefined && value < min)
8
14
  return defaultValue;
9
- if (max !== undefined && parsed > max)
15
+ if (max !== undefined && value > max)
10
16
  return defaultValue;
11
- return parsed;
17
+ return value;
12
18
  }
13
19
  function parseBoolean(envValue, defaultValue) {
14
20
  if (!envValue)
@@ -17,23 +23,29 @@ function parseBoolean(envValue, defaultValue) {
17
23
  }
18
24
  function parseLogLevel(envValue) {
19
25
  const level = envValue?.toLowerCase();
20
- if (level === 'debug' ||
21
- level === 'info' ||
22
- level === 'warn' ||
23
- level === 'error') {
24
- return level;
25
- }
26
- return 'info';
26
+ if (!level)
27
+ return 'info';
28
+ return isLogLevel(level) ? level : 'info';
29
+ }
30
+ const ALLOWED_LOG_LEVELS = new Set([
31
+ 'debug',
32
+ 'info',
33
+ 'warn',
34
+ 'error',
35
+ ]);
36
+ function isLogLevel(value) {
37
+ return ALLOWED_LOG_LEVELS.has(value);
27
38
  }
28
39
  const host = process.env.HOST ?? '127.0.0.1';
29
40
  const isLoopbackHost = host === '127.0.0.1' || host === '::1' || host === 'localhost';
30
41
  export const config = {
31
42
  server: {
32
43
  name: 'superFetch',
33
- version: '1.1.2',
44
+ version: packageJson.version,
34
45
  port: parseInteger(process.env.PORT, 3000, 1024, 65535),
35
46
  host,
36
47
  sessionTtlMs: parseInteger(process.env.SESSION_TTL_MS, 30 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000),
48
+ sessionInitTimeoutMs: parseInteger(process.env.SESSION_INIT_TIMEOUT_MS, 10000, 1000, 60000),
37
49
  maxSessions: parseInteger(process.env.MAX_SESSIONS, 200, 10, 10000),
38
50
  },
39
51
  fetcher: {
@@ -61,6 +73,7 @@ export const config = {
61
73
  maxHtmlSize: 10 * 1024 * 1024,
62
74
  maxContentSize: 5 * 1024 * 1024,
63
75
  maxUrlLength: 2048,
76
+ maxInlineContentChars: parseInteger(process.env.MAX_INLINE_CONTENT_CHARS, 20000, 1000, 200000),
64
77
  },
65
78
  security: {
66
79
  blockedHosts: new Set([
@@ -81,12 +94,15 @@ export const config = {
81
94
  /^127\./,
82
95
  /^0\./,
83
96
  /^169\.254\./,
97
+ /^100\.64\./,
84
98
  /^fc00:/i,
99
+ /^fd00:/i,
85
100
  /^fe80:/i,
86
101
  /^::ffff:127\./,
87
102
  /^::ffff:10\./,
88
103
  /^::ffff:172\.(1[6-9]|2\d|3[01])\./,
89
104
  /^::ffff:192\.168\./,
105
+ /^::ffff:169\.254\./,
90
106
  ],
91
107
  blockedHeaders: new Set([
92
108
  'host',
@@ -100,5 +116,11 @@ export const config = {
100
116
  allowRemote: parseBoolean(process.env.ALLOW_REMOTE, false),
101
117
  requireAuth: parseBoolean(process.env.REQUIRE_AUTH, !isLoopbackHost),
102
118
  },
119
+ rateLimit: {
120
+ enabled: parseBoolean(process.env.RATE_LIMIT_ENABLED, true),
121
+ maxRequests: parseInteger(process.env.RATE_LIMIT_MAX, 100, 1, 10000),
122
+ windowMs: parseInteger(process.env.RATE_LIMIT_WINDOW_MS, 60000, 1000, 3600000),
123
+ cleanupIntervalMs: parseInteger(process.env.RATE_LIMIT_CLEANUP_MS, 60000, 10000, 3600000),
124
+ },
103
125
  };
104
126
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B;IAE5B,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IACE,KAAK,KAAK,OAAO;QACjB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,OAAO,EACjB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;AAC7C,MAAM,cAAc,GAClB,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,CAAC;AAEjE,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI;QACJ,YAAY,EAAE,YAAY,CACxB,OAAO,CAAC,GAAG,CAAC,cAAc,EAC1B,EAAE,GAAG,EAAE,GAAG,IAAI,EACd,EAAE,GAAG,IAAI,EACT,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACpB;QACD,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACpE;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;KACnB;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;QAC3B,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC;QAC1D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC;KACrE;CACO,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAGnE,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvD,OAAO,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CACtB,QAA4B,EAC5B,YAAoB;IAEpB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;AACtD,CAAC;AAED,SAAS,YAAY,CACnB,KAAa,EACb,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,MAAM,kBAAkB,GAA0B,IAAI,GAAG,CAAC;IACxD,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,kBAAkB,CAAC,GAAG,CAAC,KAAiB,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;AAC7C,MAAM,cAAc,GAClB,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,CAAC;AAEjE,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI;QACJ,YAAY,EAAE,YAAY,CACxB,OAAO,CAAC,GAAG,CAAC,cAAc,EAC1B,EAAE,GAAG,EAAE,GAAG,IAAI,EACd,EAAE,GAAG,IAAI,EACT,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACpB;QACD,oBAAoB,EAAE,YAAY,CAChC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,KAAK,EACL,IAAI,EACJ,KAAK,CACN;QACD,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACpE;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;QAClB,qBAAqB,EAAE,YAAY,CACjC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EACpC,KAAK,EACL,IAAI,EACJ,MAAM,CACP;KACF;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,YAAY;YACZ,SAAS;YACT,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;YACpB,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;QAC3B,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC;QAC1D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC;KACrE;IACD,SAAS,EAAE;QACT,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC;QAC3D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC;QACpE,QAAQ,EAAE,YAAY,CACpB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,KAAK,EACL,IAAI,EACJ,OAAO,CACR;QACD,iBAAiB,EAAE,YAAY,CAC7B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EACjC,KAAK,EACL,KAAK,EACL,OAAO,CACR;KACF;CACO,CAAC"}
@@ -0,0 +1,107 @@
1
+ type ContentBlockType = 'metadata' | 'heading' | 'paragraph' | 'list' | 'code' | 'table' | 'image' | 'blockquote';
2
+ interface ContentBlock {
3
+ type: ContentBlockType;
4
+ }
5
+ export interface MetadataBlock extends ContentBlock {
6
+ type: 'metadata';
7
+ title?: string;
8
+ description?: string;
9
+ author?: string;
10
+ url: string;
11
+ fetchedAt: string;
12
+ }
13
+ export interface HeadingBlock extends ContentBlock {
14
+ type: 'heading';
15
+ level: number;
16
+ text: string;
17
+ }
18
+ export interface ParagraphBlock extends ContentBlock {
19
+ type: 'paragraph';
20
+ text: string;
21
+ }
22
+ export interface ListBlock extends ContentBlock {
23
+ type: 'list';
24
+ ordered: boolean;
25
+ items: string[];
26
+ }
27
+ export interface CodeBlock extends ContentBlock {
28
+ type: 'code';
29
+ language?: string;
30
+ text: string;
31
+ }
32
+ export interface TableBlock extends ContentBlock {
33
+ type: 'table';
34
+ headers?: string[];
35
+ rows: string[][];
36
+ }
37
+ export interface ImageBlock extends ContentBlock {
38
+ type: 'image';
39
+ src: string;
40
+ alt?: string;
41
+ }
42
+ export interface BlockquoteBlock extends ContentBlock {
43
+ type: 'blockquote';
44
+ text: string;
45
+ }
46
+ export type ContentBlockUnion = MetadataBlock | HeadingBlock | ParagraphBlock | ListBlock | CodeBlock | TableBlock | ImageBlock | BlockquoteBlock;
47
+ export interface ExtractedArticle {
48
+ title?: string;
49
+ byline?: string;
50
+ content: string;
51
+ textContent: string;
52
+ excerpt?: string;
53
+ siteName?: string;
54
+ }
55
+ export interface CacheEntry {
56
+ url: string;
57
+ content: string;
58
+ fetchedAt: string;
59
+ expiresAt: string;
60
+ }
61
+ export interface ExtractedLink {
62
+ href: string;
63
+ text: string;
64
+ type: LinkType;
65
+ }
66
+ export interface ExtractedMetadata {
67
+ title?: string;
68
+ description?: string;
69
+ author?: string;
70
+ }
71
+ export interface ExtractionResult {
72
+ article: ExtractedArticle | null;
73
+ metadata: ExtractedMetadata;
74
+ }
75
+ export type ParseableTagName = 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'p' | 'ul' | 'ol' | 'pre' | 'code' | 'table' | 'img' | 'blockquote';
76
+ export interface LinksTransformResult {
77
+ links: ExtractedLink[];
78
+ linkCount: number;
79
+ filtered: number;
80
+ truncated: boolean;
81
+ }
82
+ export interface ExtractLinksOptions {
83
+ includeInternal: boolean;
84
+ includeExternal: boolean;
85
+ includeImages: boolean;
86
+ maxLinks?: number;
87
+ filterPattern?: RegExp;
88
+ }
89
+ export interface MarkdownTransformResult {
90
+ markdown: string;
91
+ title: string | undefined;
92
+ truncated: boolean;
93
+ }
94
+ export interface TransformOptions {
95
+ extractMainContent: boolean;
96
+ includeMetadata: boolean;
97
+ maxContentLength?: number;
98
+ }
99
+ export interface JsonlTransformResult {
100
+ content: string;
101
+ contentBlocks: number;
102
+ title: string | undefined;
103
+ truncated?: boolean;
104
+ }
105
+ export type LinkType = 'internal' | 'external' | 'image';
106
+ export {};
107
+ //# sourceMappingURL=content.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":"AAAA,KAAK,gBAAgB,GACjB,UAAU,GACV,SAAS,GACT,WAAW,GACX,MAAM,GACN,MAAM,GACN,OAAO,GACP,OAAO,GACP,YAAY,CAAC;AAEjB,UAAU,YAAY;IACpB,IAAI,EAAE,gBAAgB,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,iBAAiB,GACzB,aAAa,GACb,YAAY,GACZ,cAAc,GACd,SAAS,GACT,SAAS,GACT,UAAU,GACV,UAAU,GACV,eAAe,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,gBAAgB,GACxB,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,MAAM,GACN,OAAO,GACP,KAAK,GACL,YAAY,CAAC;AAEjB,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=content.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"content.js","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":""}
@@ -0,0 +1,78 @@
1
+ import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
2
+ export interface RateLimitEntry {
3
+ count: number;
4
+ resetTime: number;
5
+ lastAccessed: number;
6
+ }
7
+ export interface RateLimiterOptions {
8
+ maxRequests: number;
9
+ windowMs: number;
10
+ cleanupIntervalMs: number;
11
+ }
12
+ export type ToolContentBlock = {
13
+ type: 'text';
14
+ text: string;
15
+ } | {
16
+ type: 'resource_link';
17
+ uri: string;
18
+ name: string;
19
+ title?: string;
20
+ description?: string;
21
+ mimeType?: string;
22
+ };
23
+ export interface FetchOptions {
24
+ customHeaders?: Record<string, string>;
25
+ signal?: AbortSignal;
26
+ timeout?: number;
27
+ }
28
+ export interface ContentTransformOptions {
29
+ readonly extractMainContent: boolean;
30
+ readonly includeMetadata: boolean;
31
+ }
32
+ export interface TruncationResult {
33
+ readonly content: string;
34
+ readonly truncated: boolean;
35
+ }
36
+ export interface SessionEntry {
37
+ readonly transport: StreamableHTTPServerTransport;
38
+ createdAt: number;
39
+ lastSeen: number;
40
+ }
41
+ export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
42
+ export type LogMetadata = Record<string, unknown>;
43
+ export interface McpRequestBody {
44
+ method?: string;
45
+ id?: string | number;
46
+ jsonrpc?: '2.0';
47
+ params?: unknown;
48
+ }
49
+ export interface FetchPipelineOptions<T> {
50
+ /** URL to fetch */
51
+ url: string;
52
+ /** Cache namespace (e.g., 'url', 'links', 'markdown') */
53
+ cacheNamespace: string;
54
+ /** Optional custom HTTP headers */
55
+ customHeaders?: Record<string, string>;
56
+ /** Optional: number of retry attempts (1-10, defaults to 3) */
57
+ retries?: number;
58
+ /** Optional: AbortSignal for request cancellation */
59
+ signal?: AbortSignal;
60
+ /** Optional: per-request timeout override in milliseconds */
61
+ timeout?: number;
62
+ /** Optional: cache variation input for headers/flags */
63
+ cacheVary?: Record<string, unknown> | string;
64
+ /** Transform function to process HTML into desired format */
65
+ transform: (html: string, url: string) => T;
66
+ /** Optional: serialize result for caching (defaults to JSON.stringify) */
67
+ serialize?: (result: T) => string;
68
+ /** Optional: deserialize cached content */
69
+ deserialize?: (cached: string) => T | undefined;
70
+ }
71
+ export interface PipelineResult<T> {
72
+ data: T;
73
+ fromCache: boolean;
74
+ url: string;
75
+ fetchedAt: string;
76
+ cacheKey?: string | null;
77
+ }
78
+ //# sourceMappingURL=runtime.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAGN,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,6BAA6B,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAC3D,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAGlD,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAGD,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,6DAA6D;IAC7D,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAC5C,0EAA0E;IAC1E,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":""}
@@ -0,0 +1,99 @@
1
+ import type { ToolContentBlock } from './runtime.js';
2
+ interface RequestOptions {
3
+ /** Custom HTTP headers for the request */
4
+ customHeaders?: Record<string, string>;
5
+ /** Request timeout in milliseconds (1000-60000) */
6
+ timeout?: number;
7
+ /** Number of retry attempts (1-10) */
8
+ retries?: number;
9
+ }
10
+ export interface FetchUrlInput extends RequestOptions {
11
+ url: string;
12
+ extractMainContent?: boolean;
13
+ includeMetadata?: boolean;
14
+ maxContentLength?: number;
15
+ format?: 'jsonl' | 'markdown';
16
+ }
17
+ export interface FetchLinksInput extends RequestOptions {
18
+ url: string;
19
+ includeExternal?: boolean;
20
+ includeInternal?: boolean;
21
+ maxLinks?: number;
22
+ filterPattern?: string;
23
+ includeImages?: boolean;
24
+ }
25
+ export interface FetchMarkdownInput extends RequestOptions {
26
+ url: string;
27
+ extractMainContent?: boolean;
28
+ includeMetadata?: boolean;
29
+ maxContentLength?: number;
30
+ }
31
+ export interface FetchUrlsInput extends RequestOptions {
32
+ urls: string[];
33
+ extractMainContent?: boolean;
34
+ includeMetadata?: boolean;
35
+ maxContentLength?: number;
36
+ format?: 'jsonl' | 'markdown';
37
+ concurrency?: number;
38
+ continueOnError?: boolean;
39
+ }
40
+ export interface ErrorResponse {
41
+ error: {
42
+ message: string;
43
+ code: string;
44
+ statusCode: number;
45
+ details?: Record<string, unknown>;
46
+ stack?: string;
47
+ };
48
+ }
49
+ export interface ToolResponse<T = Record<string, unknown>> {
50
+ [x: string]: unknown;
51
+ content: ToolContentBlock[];
52
+ structuredContent: T & Record<string, unknown>;
53
+ }
54
+ export interface BatchUrlResult {
55
+ url: string;
56
+ success: boolean;
57
+ title?: string;
58
+ content?: string;
59
+ contentBlocks?: number;
60
+ contentSize?: number;
61
+ resourceUri?: string;
62
+ resourceMimeType?: string;
63
+ cached?: boolean;
64
+ truncated?: boolean;
65
+ error?: string;
66
+ errorCode?: string;
67
+ }
68
+ export interface BatchSummary {
69
+ total: number;
70
+ successful: number;
71
+ failed: number;
72
+ cached: number;
73
+ totalContentBlocks: number;
74
+ }
75
+ export interface BatchResponseContent {
76
+ [x: string]: unknown;
77
+ results: BatchUrlResult[];
78
+ summary: BatchSummary;
79
+ fetchedAt: string;
80
+ }
81
+ export interface ToolErrorResponse {
82
+ [x: string]: unknown;
83
+ content: ToolContentBlock[];
84
+ structuredContent: {
85
+ [x: string]: unknown;
86
+ error: string;
87
+ url: string;
88
+ errorCode: string;
89
+ };
90
+ isError: true;
91
+ }
92
+ export interface ToolResponseBase {
93
+ [x: string]: unknown;
94
+ content: ToolContentBlock[];
95
+ structuredContent?: Record<string, unknown>;
96
+ isError?: boolean;
97
+ }
98
+ export {};
99
+ //# sourceMappingURL=tools.d.ts.map