@j0hanz/superfetch 1.1.1 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +57 -32
  2. package/dist/config/formatting.d.ts +9 -0
  3. package/dist/config/formatting.d.ts.map +1 -0
  4. package/dist/config/formatting.js +11 -0
  5. package/dist/config/formatting.js.map +1 -0
  6. package/dist/config/index.d.ts +16 -2
  7. package/dist/config/index.d.ts.map +1 -1
  8. package/dist/config/index.js +43 -14
  9. package/dist/config/index.js.map +1 -1
  10. package/dist/config/types/content.d.ts +107 -0
  11. package/dist/config/types/content.d.ts.map +1 -0
  12. package/dist/config/types/content.js +2 -0
  13. package/dist/config/types/content.js.map +1 -0
  14. package/dist/config/types/runtime.d.ts +78 -0
  15. package/dist/config/types/runtime.d.ts.map +1 -0
  16. package/dist/config/types/runtime.js +2 -0
  17. package/dist/config/types/runtime.js.map +1 -0
  18. package/dist/config/types/tools.d.ts +99 -0
  19. package/dist/config/types/tools.d.ts.map +1 -0
  20. package/dist/config/types/tools.js +2 -0
  21. package/dist/config/types/tools.js.map +1 -0
  22. package/dist/config/types.d.ts +3 -296
  23. package/dist/config/types.d.ts.map +1 -1
  24. package/dist/http/auth.d.ts +3 -0
  25. package/dist/http/auth.d.ts.map +1 -0
  26. package/dist/http/auth.js +34 -0
  27. package/dist/http/auth.js.map +1 -0
  28. package/dist/http/cors.d.ts +8 -0
  29. package/dist/http/cors.d.ts.map +1 -0
  30. package/dist/http/cors.js +47 -0
  31. package/dist/http/cors.js.map +1 -0
  32. package/dist/http/mcp-routes.d.ts +5 -0
  33. package/dist/http/mcp-routes.d.ts.map +1 -0
  34. package/dist/http/mcp-routes.js +110 -0
  35. package/dist/http/mcp-routes.js.map +1 -0
  36. package/dist/http/mcp-session.d.ts +12 -0
  37. package/dist/http/mcp-session.d.ts.map +1 -0
  38. package/dist/http/mcp-session.js +209 -0
  39. package/dist/http/mcp-session.js.map +1 -0
  40. package/dist/http/mcp-validation.d.ts +3 -0
  41. package/dist/http/mcp-validation.d.ts.map +1 -0
  42. package/dist/http/mcp-validation.js +34 -0
  43. package/dist/http/mcp-validation.js.map +1 -0
  44. package/dist/http/rate-limit.d.ts +13 -0
  45. package/dist/http/rate-limit.d.ts.map +1 -0
  46. package/dist/http/rate-limit.js +91 -0
  47. package/dist/http/rate-limit.js.map +1 -0
  48. package/dist/http/server.d.ts +4 -0
  49. package/dist/http/server.d.ts.map +1 -0
  50. package/dist/http/server.js +183 -0
  51. package/dist/http/server.js.map +1 -0
  52. package/dist/http/sessions.d.ts +15 -0
  53. package/dist/http/sessions.d.ts.map +1 -0
  54. package/dist/http/sessions.js +64 -0
  55. package/dist/http/sessions.js.map +1 -0
  56. package/dist/index.js +26 -223
  57. package/dist/index.js.map +1 -1
  58. package/dist/middleware/error-handler.d.ts +2 -2
  59. package/dist/middleware/error-handler.d.ts.map +1 -1
  60. package/dist/middleware/error-handler.js +46 -15
  61. package/dist/middleware/error-handler.js.map +1 -1
  62. package/dist/resources/cached-content.d.ts.map +1 -1
  63. package/dist/resources/cached-content.js +104 -44
  64. package/dist/resources/cached-content.js.map +1 -1
  65. package/dist/resources/index.d.ts.map +1 -1
  66. package/dist/resources/index.js +77 -69
  67. package/dist/resources/index.js.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +9 -3
  70. package/dist/server.js.map +1 -1
  71. package/dist/services/cache.d.ts +13 -1
  72. package/dist/services/cache.d.ts.map +1 -1
  73. package/dist/services/cache.js +90 -13
  74. package/dist/services/cache.js.map +1 -1
  75. package/dist/services/context.d.ts +9 -0
  76. package/dist/services/context.d.ts.map +1 -0
  77. package/dist/services/context.js +9 -0
  78. package/dist/services/context.js.map +1 -0
  79. package/dist/services/extractor.d.ts.map +1 -1
  80. package/dist/services/extractor.js +122 -87
  81. package/dist/services/extractor.js.map +1 -1
  82. package/dist/services/fetcher/agents.d.ts +4 -0
  83. package/dist/services/fetcher/agents.d.ts.map +1 -0
  84. package/dist/services/fetcher/agents.js +111 -0
  85. package/dist/services/fetcher/agents.js.map +1 -0
  86. package/dist/services/fetcher/errors.d.ts +5 -0
  87. package/dist/services/fetcher/errors.d.ts.map +1 -0
  88. package/dist/services/fetcher/errors.js +71 -0
  89. package/dist/services/fetcher/errors.js.map +1 -0
  90. package/dist/services/fetcher/headers.d.ts +2 -0
  91. package/dist/services/fetcher/headers.d.ts.map +1 -0
  92. package/dist/services/fetcher/headers.js +28 -0
  93. package/dist/services/fetcher/headers.js.map +1 -0
  94. package/dist/services/fetcher/interceptors.d.ts +10 -0
  95. package/dist/services/fetcher/interceptors.d.ts.map +1 -0
  96. package/dist/services/fetcher/interceptors.js +82 -0
  97. package/dist/services/fetcher/interceptors.js.map +1 -0
  98. package/dist/services/fetcher/redirects.d.ts +6 -0
  99. package/dist/services/fetcher/redirects.d.ts.map +1 -0
  100. package/dist/services/fetcher/redirects.js +67 -0
  101. package/dist/services/fetcher/redirects.js.map +1 -0
  102. package/dist/services/fetcher/response.d.ts +5 -0
  103. package/dist/services/fetcher/response.d.ts.map +1 -0
  104. package/dist/services/fetcher/response.js +39 -0
  105. package/dist/services/fetcher/response.js.map +1 -0
  106. package/dist/services/fetcher/retry-policy.d.ts +28 -0
  107. package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
  108. package/dist/services/fetcher/retry-policy.js +138 -0
  109. package/dist/services/fetcher/retry-policy.js.map +1 -0
  110. package/dist/services/fetcher.d.ts +2 -1
  111. package/dist/services/fetcher.d.ts.map +1 -1
  112. package/dist/services/fetcher.js +61 -254
  113. package/dist/services/fetcher.js.map +1 -1
  114. package/dist/services/logger.d.ts.map +1 -1
  115. package/dist/services/logger.js +14 -5
  116. package/dist/services/logger.js.map +1 -1
  117. package/dist/services/parser.d.ts +1 -0
  118. package/dist/services/parser.d.ts.map +1 -1
  119. package/dist/services/parser.js +55 -35
  120. package/dist/services/parser.js.map +1 -1
  121. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
  122. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
  123. package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
  124. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
  125. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  126. package/dist/tools/handlers/fetch-links.tool.js +78 -116
  127. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  128. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
  129. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  130. package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
  131. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  132. package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
  133. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
  134. package/dist/tools/handlers/fetch-single.shared.js +49 -0
  135. package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
  136. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  137. package/dist/tools/handlers/fetch-url.tool.js +82 -54
  138. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  139. package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
  140. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
  141. package/dist/tools/handlers/fetch-urls/processor.js +153 -0
  142. package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
  143. package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
  144. package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
  145. package/dist/tools/handlers/fetch-urls/response.js +58 -0
  146. package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
  147. package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
  148. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
  149. package/dist/tools/handlers/fetch-urls/validation.js +18 -0
  150. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
  151. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  152. package/dist/tools/handlers/fetch-urls.tool.js +104 -197
  153. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  154. package/dist/tools/index.d.ts.map +1 -1
  155. package/dist/tools/index.js +36 -237
  156. package/dist/tools/index.js.map +1 -1
  157. package/dist/tools/schemas.d.ts +357 -0
  158. package/dist/tools/schemas.d.ts.map +1 -0
  159. package/dist/tools/schemas.js +272 -0
  160. package/dist/tools/schemas.js.map +1 -0
  161. package/dist/tools/utils/cache-vary.d.ts +3 -0
  162. package/dist/tools/utils/cache-vary.d.ts.map +1 -0
  163. package/dist/tools/utils/cache-vary.js +44 -0
  164. package/dist/tools/utils/cache-vary.js.map +1 -0
  165. package/dist/tools/utils/common.d.ts +2 -2
  166. package/dist/tools/utils/common.d.ts.map +1 -1
  167. package/dist/tools/utils/common.js +5 -1
  168. package/dist/tools/utils/common.js.map +1 -1
  169. package/dist/tools/utils/content-transform.d.ts +16 -0
  170. package/dist/tools/utils/content-transform.d.ts.map +1 -0
  171. package/dist/tools/utils/content-transform.js +49 -0
  172. package/dist/tools/utils/content-transform.js.map +1 -0
  173. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  174. package/dist/tools/utils/fetch-pipeline.js +32 -18
  175. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  176. package/dist/tools/utils/inline-content.d.ts +11 -0
  177. package/dist/tools/utils/inline-content.d.ts.map +1 -0
  178. package/dist/tools/utils/inline-content.js +39 -0
  179. package/dist/tools/utils/inline-content.js.map +1 -0
  180. package/dist/tools/utils/markdown-toc.d.ts +3 -0
  181. package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
  182. package/dist/tools/utils/markdown-toc.js +35 -0
  183. package/dist/tools/utils/markdown-toc.js.map +1 -0
  184. package/dist/tools/utils/tool-response.d.ts +9 -0
  185. package/dist/tools/utils/tool-response.d.ts.map +1 -0
  186. package/dist/tools/utils/tool-response.js +19 -0
  187. package/dist/tools/utils/tool-response.js.map +1 -0
  188. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  189. package/dist/transformers/jsonl.transformer.js +51 -28
  190. package/dist/transformers/jsonl.transformer.js.map +1 -1
  191. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  192. package/dist/transformers/markdown.transformer.js +82 -111
  193. package/dist/transformers/markdown.transformer.js.map +1 -1
  194. package/dist/utils/header-normalizer.d.ts +5 -0
  195. package/dist/utils/header-normalizer.d.ts.map +1 -0
  196. package/dist/utils/header-normalizer.js +25 -0
  197. package/dist/utils/header-normalizer.js.map +1 -0
  198. package/dist/utils/tool-error-handler.d.ts +1 -0
  199. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  200. package/dist/utils/tool-error-handler.js +29 -1
  201. package/dist/utils/tool-error-handler.js.map +1 -1
  202. package/dist/utils/url-validator.d.ts +0 -3
  203. package/dist/utils/url-validator.d.ts.map +1 -1
  204. package/dist/utils/url-validator.js +98 -18
  205. package/dist/utils/url-validator.js.map +1 -1
  206. package/package.json +11 -6
package/README.md CHANGED
@@ -307,6 +307,8 @@ node dist/index.js --stdio
307
307
 
308
308
  ## Available Tools
309
309
 
310
+ > **Note:** If extracted content exceeds `MAX_INLINE_CONTENT_CHARS`, the tool response includes a `resourceUri` and a `resource_link` content block instead of embedding the full text.
311
+
310
312
  ### `fetch-url`
311
313
 
312
314
  Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks.
@@ -477,12 +479,6 @@ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing
477
479
  | `superfetch://health` | Real-time server health and dependency status |
478
480
  | Dynamic resources | Cached content available via resource subscriptions |
479
481
 
480
- ### Prompts
481
-
482
- - **`analyze-web-content`** — Analyze fetched content with optional focus area
483
- - **`summarize-page`** — Fetch and summarize a webpage concisely
484
- - **`extract-data`** — Extract structured data from a webpage
485
-
486
482
  ---
487
483
 
488
484
  ## Configuration
@@ -641,6 +637,12 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
641
637
  | `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
642
638
  | `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
643
639
 
640
+ #### 📦 Output Settings
641
+
642
+ | Variable | Default | Valid Values | Description |
643
+ | -------------------------- | ------- | --------------- | --------------------------------------------------------------- |
644
+ | `MAX_INLINE_CONTENT_CHARS` | `20000` | `1000`-`200000` | Inline content limit before returning a `resource_link` instead |
645
+
644
646
  #### 📝 Logging Settings
645
647
 
646
648
  | Variable | Default | Valid Values | Description |
@@ -655,12 +657,28 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
655
657
  | `EXTRACT_MAIN_CONTENT` | `true` | `true` / `false` | Use Mozilla Readability to extract main content |
656
658
  | `INCLUDE_METADATA` | `true` | `true` / `false` | Include page metadata (title, description, author) |
657
659
 
660
+ #### 🛡️ Security Settings
661
+
662
+ | Variable | Default | Description |
663
+ | -------------- | ------- | -------------------------------------------------------- |
664
+ | `API_KEY` | - | API Key for HTTP authentication (required for HTTP mode) |
665
+ | `ALLOW_REMOTE` | `false` | Allow binding to non-loopback interfaces |
666
+
667
+ #### Rate Limiting
668
+
669
+ | Variable | Default | Valid Values | Description |
670
+ | ----------------------- | ------- | ----------------- | ------------------------------------ |
671
+ | `RATE_LIMIT_ENABLED` | `true` | `true` / `false` | Enable/disable HTTP rate limiting |
672
+ | `RATE_LIMIT_MAX` | `100` | `1`-`10000` | Max requests per window per IP |
673
+ | `RATE_LIMIT_WINDOW_MS` | `60000` | `1000`-`3600000` | Rate limit window in milliseconds |
674
+ | `RATE_LIMIT_CLEANUP_MS` | `60000` | `10000`-`3600000` | Cleanup interval for limiter entries |
675
+
658
676
  ### HTTP Mode Configuration
659
677
 
660
678
  <details>
661
679
  <summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
662
680
 
663
- SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
681
+ SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration and an `API_KEY` for authenticated access (send `Authorization: Bearer <key>` or `X-API-Key: <key>`).
664
682
 
665
683
  #### Start HTTP Server
666
684
 
@@ -671,12 +689,15 @@ npx -y @j0hanz/superfetch@latest
671
689
 
672
690
  #### HTTP-Specific Environment Variables
673
691
 
674
- | Variable | Default | Description |
675
- | ----------------- | ----------- | ------------------------------------------------ |
676
- | `PORT` | `3000` | HTTP server port |
677
- | `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
678
- | `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
679
- | `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
692
+ | Variable | Default | Description |
693
+ | ------------------------- | ----------- | ------------------------------------------------ |
694
+ | `PORT` | `3000` | HTTP server port |
695
+ | `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
696
+ | `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
697
+ | `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
698
+ | `SESSION_TTL_MS` | `1800000` | Session time-to-live in milliseconds (30 mins) |
699
+ | `SESSION_INIT_TIMEOUT_MS` | `10000` | Time allowed for session initialization (ms) |
700
+ | `MAX_SESSIONS` | `200` | Maximum number of active sessions |
680
701
 
681
702
  #### VS Code HTTP Mode Setup
682
703
 
@@ -745,7 +766,8 @@ Blocked headers: `host`, `authorization`, `cookie`, `x-forwarded-for`, `x-real-i
745
766
 
746
767
  ### Rate Limiting
747
768
 
748
- Default: **100 requests/minute** per IP (configurable)
769
+ Default: **100 requests/minute** per IP. Configure with `RATE_LIMIT_MAX` and
770
+ `RATE_LIMIT_WINDOW_MS`.
749
771
 
750
772
  ### HTTP Mode Endpoints
751
773
 
@@ -766,17 +788,20 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
766
788
 
767
789
  ### Scripts
768
790
 
769
- | Command | Description |
770
- | -------------------- | ---------------------------------- |
771
- | `npm run dev` | Development server with hot reload |
772
- | `npm run build` | Compile TypeScript |
773
- | `npm start` | Production server |
774
- | `npm run lint` | Run ESLint |
775
- | `npm run type-check` | TypeScript type checking |
776
- | `npm run format` | Format with Prettier |
777
- | `npm run release` | Create new release |
778
- | `npm run knip` | Find unused exports/dependencies |
779
- | `npm run knip:fix` | Auto-fix unused code |
791
+ | Command | Description |
792
+ | ----------------------- | ---------------------------------- |
793
+ | `npm run dev` | Development server with hot reload |
794
+ | `npm run build` | Compile TypeScript |
795
+ | `npm start` | Production server |
796
+ | `npm run lint` | Run ESLint |
797
+ | `npm run type-check` | TypeScript type checking |
798
+ | `npm run format` | Format with Prettier |
799
+ | `npm test` | Run Vitest tests |
800
+ | `npm run test:coverage` | Run tests with coverage |
801
+ | `npm run bench` | Run minimal performance benchmark |
802
+ | `npm run release` | Create new release |
803
+ | `npm run knip` | Find unused exports/dependencies |
804
+ | `npm run knip:fix` | Auto-fix unused code |
780
805
 
781
806
  ### Tech Stack
782
807
 
@@ -786,12 +811,11 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
786
811
  | Language | TypeScript 5.9 |
787
812
  | MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
788
813
  | Content Extraction | @mozilla/readability ^0.6.0 |
789
- | HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
814
+ | HTML Parsing | Cheerio ^1.1.2, LinkeDOM ^0.18.12 |
790
815
  | Markdown | Turndown ^7.2.2 |
791
- | HTTP | Express ^5.2.1, Axios ^1.13.2 |
816
+ | HTTP | Express ^5.2.1, Axios ^1.7.9 |
792
817
  | Caching | node-cache ^5.1.2 |
793
- | Validation | Zod ^3.25.76 |
794
- | Logging | Winston ^3.19.0 |
818
+ | Validation | Zod ^3.24.1 |
795
819
 
796
820
  ---
797
821
 
@@ -800,8 +824,9 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
800
824
  1. Fork the repository
801
825
  2. Create a feature branch: `git checkout -b feature/amazing-feature`
802
826
  3. Ensure linting passes: `npm run lint`
803
- 4. Commit changes: `git commit -m 'Add amazing feature'`
804
- 5. Push: `git push origin feature/amazing-feature`
805
- 6. Open a Pull Request
827
+ 4. Run tests: `npm test`
828
+ 5. Commit changes: `git commit -m 'Add amazing feature'`
829
+ 6. Push: `git push origin feature/amazing-feature`
830
+ 7. Open a Pull Request
806
831
 
807
832
  For examples of other MCP servers, see: [github.com/modelcontextprotocol/servers](https://github.com/modelcontextprotocol/servers)
@@ -0,0 +1,9 @@
1
+ export declare const LINE_BREAK = "\n";
2
+ export declare const TRUNCATION_MARKER = "...[truncated]";
3
+ export declare const CODE_BLOCK: {
4
+ readonly fence: "```";
5
+ readonly format: (code: string, language?: string) => string;
6
+ };
7
+ export declare const FRONTMATTER_DELIMITER = "---";
8
+ export declare const joinLines: (lines: string[]) => string;
9
+ //# sourceMappingURL=formatting.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,OAAO,CAAC;AAE/B,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAElD,eAAO,MAAM,UAAU;;4BAEN,MAAM,wBAAkB,MAAM;CAGrC,CAAC;AAEX,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,GAAI,OAAO,MAAM,EAAE,KAAG,MAAgC,CAAC"}
@@ -0,0 +1,11 @@
1
+ export const LINE_BREAK = '\n';
2
+ export const TRUNCATION_MARKER = '...[truncated]';
3
+ export const CODE_BLOCK = {
4
+ fence: '```',
5
+ format: (code, language = '') => {
6
+ return `\`\`\`${language}\n${code}\n\`\`\``;
7
+ },
8
+ };
9
+ export const FRONTMATTER_DELIMITER = '---';
10
+ export const joinLines = (lines) => lines.join(LINE_BREAK);
11
+ //# sourceMappingURL=formatting.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"formatting.js","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAE/B,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;AAElD,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE,EAAU,EAAE;QAC9C,OAAO,SAAS,QAAQ,KAAK,IAAI,UAAU,CAAC;IAC9C,CAAC;CACO,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAE3C,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC"}
@@ -1,9 +1,13 @@
1
+ import type { LogLevel } from './types.js';
1
2
  export declare const config: {
2
3
  readonly server: {
3
4
  readonly name: "superFetch";
4
- readonly version: "1.0.0";
5
+ readonly version: string;
5
6
  readonly port: number;
6
7
  readonly host: string;
8
+ readonly sessionTtlMs: number;
9
+ readonly sessionInitTimeoutMs: number;
10
+ readonly maxSessions: number;
7
11
  };
8
12
  readonly fetcher: {
9
13
  readonly timeout: number;
@@ -23,18 +27,28 @@ export declare const config: {
23
27
  readonly minParagraphLength: 10;
24
28
  };
25
29
  readonly logging: {
26
- readonly level: "debug" | "info" | "warn" | "error";
30
+ readonly level: LogLevel;
27
31
  readonly enabled: boolean;
28
32
  };
29
33
  readonly constants: {
30
34
  readonly maxHtmlSize: number;
31
35
  readonly maxContentSize: number;
32
36
  readonly maxUrlLength: 2048;
37
+ readonly maxInlineContentChars: number;
33
38
  };
34
39
  readonly security: {
35
40
  readonly blockedHosts: Set<string>;
36
41
  readonly blockedIpPatterns: readonly RegExp[];
37
42
  readonly blockedHeaders: Set<string>;
43
+ readonly apiKey: string | undefined;
44
+ readonly allowRemote: boolean;
45
+ readonly requireAuth: boolean;
46
+ };
47
+ readonly rateLimit: {
48
+ readonly enabled: boolean;
49
+ readonly maxRequests: number;
50
+ readonly windowMs: number;
51
+ readonly cleanupIntervalMs: number;
38
52
  };
39
53
  };
40
54
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAqCA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCA0DV,SAAS,MAAM,EAAE;;;CAUhB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA6D3C,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAgFV,SAAS,MAAM,EAAE;;;;;;;;;;;;CA6BhB,CAAC"}
@@ -1,14 +1,20 @@
1
+ import packageJson from '../../package.json' with { type: 'json' };
1
2
  function parseInteger(envValue, defaultValue, min, max) {
3
+ const parsed = parseRawInteger(envValue, defaultValue);
4
+ return clampInteger(parsed, defaultValue, min, max);
5
+ }
6
+ function parseRawInteger(envValue, defaultValue) {
2
7
  if (!envValue)
3
8
  return defaultValue;
4
9
  const parsed = parseInt(envValue, 10);
5
- if (isNaN(parsed))
6
- return defaultValue;
7
- if (min !== undefined && parsed < min)
10
+ return Number.isNaN(parsed) ? defaultValue : parsed;
11
+ }
12
+ function clampInteger(value, defaultValue, min, max) {
13
+ if (min !== undefined && value < min)
8
14
  return defaultValue;
9
- if (max !== undefined && parsed > max)
15
+ if (max !== undefined && value > max)
10
16
  return defaultValue;
11
- return parsed;
17
+ return value;
12
18
  }
13
19
  function parseBoolean(envValue, defaultValue) {
14
20
  if (!envValue)
@@ -17,20 +23,30 @@ function parseBoolean(envValue, defaultValue) {
17
23
  }
18
24
  function parseLogLevel(envValue) {
19
25
  const level = envValue?.toLowerCase();
20
- if (level === 'debug' ||
21
- level === 'info' ||
22
- level === 'warn' ||
23
- level === 'error') {
24
- return level;
25
- }
26
- return 'info';
26
+ if (!level)
27
+ return 'info';
28
+ return isLogLevel(level) ? level : 'info';
27
29
  }
30
+ const ALLOWED_LOG_LEVELS = new Set([
31
+ 'debug',
32
+ 'info',
33
+ 'warn',
34
+ 'error',
35
+ ]);
36
+ function isLogLevel(value) {
37
+ return ALLOWED_LOG_LEVELS.has(value);
38
+ }
39
+ const host = process.env.HOST ?? '127.0.0.1';
40
+ const isLoopbackHost = host === '127.0.0.1' || host === '::1' || host === 'localhost';
28
41
  export const config = {
29
42
  server: {
30
43
  name: 'superFetch',
31
- version: '1.0.0',
44
+ version: packageJson.version,
32
45
  port: parseInteger(process.env.PORT, 3000, 1024, 65535),
33
- host: process.env.HOST ?? '127.0.0.1',
46
+ host,
47
+ sessionTtlMs: parseInteger(process.env.SESSION_TTL_MS, 30 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000),
48
+ sessionInitTimeoutMs: parseInteger(process.env.SESSION_INIT_TIMEOUT_MS, 10000, 1000, 60000),
49
+ maxSessions: parseInteger(process.env.MAX_SESSIONS, 200, 10, 10000),
34
50
  },
35
51
  fetcher: {
36
52
  timeout: parseInteger(process.env.FETCH_TIMEOUT, 30000, 5000, 120000),
@@ -57,6 +73,7 @@ export const config = {
57
73
  maxHtmlSize: 10 * 1024 * 1024,
58
74
  maxContentSize: 5 * 1024 * 1024,
59
75
  maxUrlLength: 2048,
76
+ maxInlineContentChars: parseInteger(process.env.MAX_INLINE_CONTENT_CHARS, 20000, 1000, 200000),
60
77
  },
61
78
  security: {
62
79
  blockedHosts: new Set([
@@ -77,12 +94,15 @@ export const config = {
77
94
  /^127\./,
78
95
  /^0\./,
79
96
  /^169\.254\./,
97
+ /^100\.64\./,
80
98
  /^fc00:/i,
99
+ /^fd00:/i,
81
100
  /^fe80:/i,
82
101
  /^::ffff:127\./,
83
102
  /^::ffff:10\./,
84
103
  /^::ffff:172\.(1[6-9]|2\d|3[01])\./,
85
104
  /^::ffff:192\.168\./,
105
+ /^::ffff:169\.254\./,
86
106
  ],
87
107
  blockedHeaders: new Set([
88
108
  'host',
@@ -92,6 +112,15 @@ export const config = {
92
112
  'x-real-ip',
93
113
  'proxy-authorization',
94
114
  ]),
115
+ apiKey: process.env.API_KEY,
116
+ allowRemote: parseBoolean(process.env.ALLOW_REMOTE, false),
117
+ requireAuth: parseBoolean(process.env.REQUIRE_AUTH, !isLoopbackHost),
118
+ },
119
+ rateLimit: {
120
+ enabled: parseBoolean(process.env.RATE_LIMIT_ENABLED, true),
121
+ maxRequests: parseInteger(process.env.RATE_LIMIT_MAX, 100, 1, 10000),
122
+ windowMs: parseInteger(process.env.RATE_LIMIT_WINDOW_MS, 60000, 1000, 3600000),
123
+ cleanupIntervalMs: parseInteger(process.env.RATE_LIMIT_CLEANUP_MS, 60000, 10000, 3600000),
95
124
  },
96
125
  };
97
126
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B;IAE5B,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IACE,KAAK,KAAK,OAAO;QACjB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,OAAO,EACjB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;KACnB;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;KACH;CACO,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAGnE,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvD,OAAO,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CACtB,QAA4B,EAC5B,YAAoB;IAEpB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;AACtD,CAAC;AAED,SAAS,YAAY,CACnB,KAAa,EACb,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,MAAM,kBAAkB,GAA0B,IAAI,GAAG,CAAC;IACxD,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,kBAAkB,CAAC,GAAG,CAAC,KAAiB,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;AAC7C,MAAM,cAAc,GAClB,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,CAAC;AAEjE,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI;QACJ,YAAY,EAAE,YAAY,CACxB,OAAO,CAAC,GAAG,CAAC,cAAc,EAC1B,EAAE,GAAG,EAAE,GAAG,IAAI,EACd,EAAE,GAAG,IAAI,EACT,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACpB;QACD,oBAAoB,EAAE,YAAY,CAChC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,KAAK,EACL,IAAI,EACJ,KAAK,CACN;QACD,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACpE;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;QAClB,qBAAqB,EAAE,YAAY,CACjC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EACpC,KAAK,EACL,IAAI,EACJ,MAAM,CACP;KACF;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,YAAY;YACZ,SAAS;YACT,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;YACpB,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;QAC3B,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC;QAC1D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC;KACrE;IACD,SAAS,EAAE;QACT,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC;QAC3D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC;QACpE,QAAQ,EAAE,YAAY,CACpB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,KAAK,EACL,IAAI,EACJ,OAAO,CACR;QACD,iBAAiB,EAAE,YAAY,CAC7B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EACjC,KAAK,EACL,KAAK,EACL,OAAO,CACR;KACF;CACO,CAAC"}
@@ -0,0 +1,107 @@
1
+ type ContentBlockType = 'metadata' | 'heading' | 'paragraph' | 'list' | 'code' | 'table' | 'image' | 'blockquote';
2
+ interface ContentBlock {
3
+ type: ContentBlockType;
4
+ }
5
+ export interface MetadataBlock extends ContentBlock {
6
+ type: 'metadata';
7
+ title?: string;
8
+ description?: string;
9
+ author?: string;
10
+ url: string;
11
+ fetchedAt: string;
12
+ }
13
+ export interface HeadingBlock extends ContentBlock {
14
+ type: 'heading';
15
+ level: number;
16
+ text: string;
17
+ }
18
+ export interface ParagraphBlock extends ContentBlock {
19
+ type: 'paragraph';
20
+ text: string;
21
+ }
22
+ export interface ListBlock extends ContentBlock {
23
+ type: 'list';
24
+ ordered: boolean;
25
+ items: string[];
26
+ }
27
+ export interface CodeBlock extends ContentBlock {
28
+ type: 'code';
29
+ language?: string;
30
+ text: string;
31
+ }
32
+ export interface TableBlock extends ContentBlock {
33
+ type: 'table';
34
+ headers?: string[];
35
+ rows: string[][];
36
+ }
37
+ export interface ImageBlock extends ContentBlock {
38
+ type: 'image';
39
+ src: string;
40
+ alt?: string;
41
+ }
42
+ export interface BlockquoteBlock extends ContentBlock {
43
+ type: 'blockquote';
44
+ text: string;
45
+ }
46
+ export type ContentBlockUnion = MetadataBlock | HeadingBlock | ParagraphBlock | ListBlock | CodeBlock | TableBlock | ImageBlock | BlockquoteBlock;
47
+ export interface ExtractedArticle {
48
+ title?: string;
49
+ byline?: string;
50
+ content: string;
51
+ textContent: string;
52
+ excerpt?: string;
53
+ siteName?: string;
54
+ }
55
+ export interface CacheEntry {
56
+ url: string;
57
+ content: string;
58
+ fetchedAt: string;
59
+ expiresAt: string;
60
+ }
61
+ export interface ExtractedLink {
62
+ href: string;
63
+ text: string;
64
+ type: LinkType;
65
+ }
66
+ export interface ExtractedMetadata {
67
+ title?: string;
68
+ description?: string;
69
+ author?: string;
70
+ }
71
+ export interface ExtractionResult {
72
+ article: ExtractedArticle | null;
73
+ metadata: ExtractedMetadata;
74
+ }
75
+ export type ParseableTagName = 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'p' | 'ul' | 'ol' | 'pre' | 'code' | 'table' | 'img' | 'blockquote';
76
+ export interface LinksTransformResult {
77
+ links: ExtractedLink[];
78
+ linkCount: number;
79
+ filtered: number;
80
+ truncated: boolean;
81
+ }
82
+ export interface ExtractLinksOptions {
83
+ includeInternal: boolean;
84
+ includeExternal: boolean;
85
+ includeImages: boolean;
86
+ maxLinks?: number;
87
+ filterPattern?: RegExp;
88
+ }
89
+ export interface MarkdownTransformResult {
90
+ markdown: string;
91
+ title: string | undefined;
92
+ truncated: boolean;
93
+ }
94
+ export interface TransformOptions {
95
+ extractMainContent: boolean;
96
+ includeMetadata: boolean;
97
+ maxContentLength?: number;
98
+ }
99
+ export interface JsonlTransformResult {
100
+ content: string;
101
+ contentBlocks: number;
102
+ title: string | undefined;
103
+ truncated?: boolean;
104
+ }
105
+ export type LinkType = 'internal' | 'external' | 'image';
106
+ export {};
107
+ //# sourceMappingURL=content.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":"AAAA,KAAK,gBAAgB,GACjB,UAAU,GACV,SAAS,GACT,WAAW,GACX,MAAM,GACN,MAAM,GACN,OAAO,GACP,OAAO,GACP,YAAY,CAAC;AAEjB,UAAU,YAAY;IACpB,IAAI,EAAE,gBAAgB,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,iBAAiB,GACzB,aAAa,GACb,YAAY,GACZ,cAAc,GACd,SAAS,GACT,SAAS,GACT,UAAU,GACV,UAAU,GACV,eAAe,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,gBAAgB,GACxB,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,MAAM,GACN,OAAO,GACP,KAAK,GACL,YAAY,CAAC;AAEjB,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=content.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"content.js","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":""}
@@ -0,0 +1,78 @@
1
+ import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
2
+ export interface RateLimitEntry {
3
+ count: number;
4
+ resetTime: number;
5
+ lastAccessed: number;
6
+ }
7
+ export interface RateLimiterOptions {
8
+ maxRequests: number;
9
+ windowMs: number;
10
+ cleanupIntervalMs: number;
11
+ }
12
+ export type ToolContentBlock = {
13
+ type: 'text';
14
+ text: string;
15
+ } | {
16
+ type: 'resource_link';
17
+ uri: string;
18
+ name: string;
19
+ title?: string;
20
+ description?: string;
21
+ mimeType?: string;
22
+ };
23
+ export interface FetchOptions {
24
+ customHeaders?: Record<string, string>;
25
+ signal?: AbortSignal;
26
+ timeout?: number;
27
+ }
28
+ export interface ContentTransformOptions {
29
+ readonly extractMainContent: boolean;
30
+ readonly includeMetadata: boolean;
31
+ }
32
+ export interface TruncationResult {
33
+ readonly content: string;
34
+ readonly truncated: boolean;
35
+ }
36
+ export interface SessionEntry {
37
+ readonly transport: StreamableHTTPServerTransport;
38
+ createdAt: number;
39
+ lastSeen: number;
40
+ }
41
+ export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
42
+ export type LogMetadata = Record<string, unknown>;
43
+ export interface McpRequestBody {
44
+ method?: string;
45
+ id?: string | number;
46
+ jsonrpc?: '2.0';
47
+ params?: unknown;
48
+ }
49
+ export interface FetchPipelineOptions<T> {
50
+ /** URL to fetch */
51
+ url: string;
52
+ /** Cache namespace (e.g., 'url', 'links', 'markdown') */
53
+ cacheNamespace: string;
54
+ /** Optional custom HTTP headers */
55
+ customHeaders?: Record<string, string>;
56
+ /** Optional: number of retry attempts (1-10, defaults to 3) */
57
+ retries?: number;
58
+ /** Optional: AbortSignal for request cancellation */
59
+ signal?: AbortSignal;
60
+ /** Optional: per-request timeout override in milliseconds */
61
+ timeout?: number;
62
+ /** Optional: cache variation input for headers/flags */
63
+ cacheVary?: Record<string, unknown> | string;
64
+ /** Transform function to process HTML into desired format */
65
+ transform: (html: string, url: string) => T;
66
+ /** Optional: serialize result for caching (defaults to JSON.stringify) */
67
+ serialize?: (result: T) => string;
68
+ /** Optional: deserialize cached content */
69
+ deserialize?: (cached: string) => T | undefined;
70
+ }
71
+ export interface PipelineResult<T> {
72
+ data: T;
73
+ fromCache: boolean;
74
+ url: string;
75
+ fetchedAt: string;
76
+ cacheKey?: string | null;
77
+ }
78
+ //# sourceMappingURL=runtime.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAGN,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,6BAA6B,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAC3D,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAGlD,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAGD,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,6DAA6D;IAC7D,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAC5C,0EAA0E;IAC1E,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":""}