@j0hanz/superfetch 1.1.2 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -30
- package/dist/config/formatting.d.ts +9 -0
- package/dist/config/formatting.d.ts.map +1 -0
- package/dist/config/formatting.js +11 -0
- package/dist/config/formatting.js.map +1 -0
- package/dist/config/index.d.ts +11 -2
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +35 -13
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +107 -0
- package/dist/config/types/content.d.ts.map +1 -0
- package/dist/config/types/content.js +2 -0
- package/dist/config/types/content.js.map +1 -0
- package/dist/config/types/runtime.d.ts +78 -0
- package/dist/config/types/runtime.d.ts.map +1 -0
- package/dist/config/types/runtime.js +2 -0
- package/dist/config/types/runtime.js.map +1 -0
- package/dist/config/types/tools.d.ts +99 -0
- package/dist/config/types/tools.d.ts.map +1 -0
- package/dist/config/types/tools.js +2 -0
- package/dist/config/types/tools.js.map +1 -0
- package/dist/config/types.d.ts +3 -297
- package/dist/config/types.d.ts.map +1 -1
- package/dist/http/auth.d.ts +3 -0
- package/dist/http/auth.d.ts.map +1 -0
- package/dist/http/auth.js +34 -0
- package/dist/http/auth.js.map +1 -0
- package/dist/http/cors.d.ts +8 -0
- package/dist/http/cors.d.ts.map +1 -0
- package/dist/http/cors.js +47 -0
- package/dist/http/cors.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +5 -0
- package/dist/http/mcp-routes.d.ts.map +1 -0
- package/dist/http/mcp-routes.js +110 -0
- package/dist/http/mcp-routes.js.map +1 -0
- package/dist/http/mcp-session.d.ts +12 -0
- package/dist/http/mcp-session.d.ts.map +1 -0
- package/dist/http/mcp-session.js +209 -0
- package/dist/http/mcp-session.js.map +1 -0
- package/dist/http/mcp-validation.d.ts +3 -0
- package/dist/http/mcp-validation.d.ts.map +1 -0
- package/dist/http/mcp-validation.js +34 -0
- package/dist/http/mcp-validation.js.map +1 -0
- package/dist/http/rate-limit.d.ts +13 -0
- package/dist/http/rate-limit.d.ts.map +1 -0
- package/dist/http/rate-limit.js +91 -0
- package/dist/http/rate-limit.js.map +1 -0
- package/dist/http/server.d.ts +4 -0
- package/dist/http/server.d.ts.map +1 -0
- package/dist/http/server.js +183 -0
- package/dist/http/server.js.map +1 -0
- package/dist/http/sessions.d.ts +15 -0
- package/dist/http/sessions.d.ts.map +1 -0
- package/dist/http/sessions.js +64 -0
- package/dist/http/sessions.js.map +1 -0
- package/dist/index.js +26 -341
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +46 -15
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +104 -44
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +77 -69
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +9 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +13 -1
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +89 -16
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +1 -1
- package/dist/services/context.d.ts.map +1 -1
- package/dist/services/context.js +1 -1
- package/dist/services/context.js.map +1 -1
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +122 -87
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +4 -0
- package/dist/services/fetcher/agents.d.ts.map +1 -0
- package/dist/services/fetcher/agents.js +111 -0
- package/dist/services/fetcher/agents.js.map +1 -0
- package/dist/services/fetcher/errors.d.ts +5 -0
- package/dist/services/fetcher/errors.d.ts.map +1 -0
- package/dist/services/fetcher/errors.js +71 -0
- package/dist/services/fetcher/errors.js.map +1 -0
- package/dist/services/fetcher/headers.d.ts +2 -0
- package/dist/services/fetcher/headers.d.ts.map +1 -0
- package/dist/services/fetcher/headers.js +28 -0
- package/dist/services/fetcher/headers.js.map +1 -0
- package/dist/services/fetcher/interceptors.d.ts +10 -0
- package/dist/services/fetcher/interceptors.d.ts.map +1 -0
- package/dist/services/fetcher/interceptors.js +82 -0
- package/dist/services/fetcher/interceptors.js.map +1 -0
- package/dist/services/fetcher/redirects.d.ts +6 -0
- package/dist/services/fetcher/redirects.d.ts.map +1 -0
- package/dist/services/fetcher/redirects.js +67 -0
- package/dist/services/fetcher/redirects.js.map +1 -0
- package/dist/services/fetcher/response.d.ts +5 -0
- package/dist/services/fetcher/response.d.ts.map +1 -0
- package/dist/services/fetcher/response.js +39 -0
- package/dist/services/fetcher/response.js.map +1 -0
- package/dist/services/fetcher/retry-policy.d.ts +28 -0
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
- package/dist/services/fetcher/retry-policy.js +138 -0
- package/dist/services/fetcher/retry-policy.js.map +1 -0
- package/dist/services/fetcher.d.ts +2 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +62 -315
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.js +4 -4
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -0
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +55 -35
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +78 -116
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-single.shared.js +49 -0
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +82 -54
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/processor.js +153 -0
- package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.js +58 -0
- package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.js +18 -0
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +104 -202
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +36 -237
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +357 -0
- package/dist/tools/schemas.d.ts.map +1 -0
- package/dist/tools/schemas.js +272 -0
- package/dist/tools/schemas.js.map +1 -0
- package/dist/tools/utils/cache-vary.d.ts +3 -0
- package/dist/tools/utils/cache-vary.d.ts.map +1 -0
- package/dist/tools/utils/cache-vary.js +44 -0
- package/dist/tools/utils/cache-vary.js.map +1 -0
- package/dist/tools/utils/common.d.ts +2 -2
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +5 -1
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +16 -0
- package/dist/tools/utils/content-transform.d.ts.map +1 -0
- package/dist/tools/utils/content-transform.js +49 -0
- package/dist/tools/utils/content-transform.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +32 -18
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +11 -0
- package/dist/tools/utils/inline-content.d.ts.map +1 -0
- package/dist/tools/utils/inline-content.js +39 -0
- package/dist/tools/utils/inline-content.js.map +1 -0
- package/dist/tools/utils/markdown-toc.d.ts +3 -0
- package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
- package/dist/tools/utils/markdown-toc.js +35 -0
- package/dist/tools/utils/markdown-toc.js.map +1 -0
- package/dist/tools/utils/tool-response.d.ts +9 -0
- package/dist/tools/utils/tool-response.d.ts.map +1 -0
- package/dist/tools/utils/tool-response.js +19 -0
- package/dist/tools/utils/tool-response.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +51 -28
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +82 -111
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/header-normalizer.d.ts +5 -0
- package/dist/utils/header-normalizer.d.ts.map +1 -0
- package/dist/utils/header-normalizer.js +25 -0
- package/dist/utils/header-normalizer.js.map +1 -0
- package/dist/utils/tool-error-handler.d.ts +1 -0
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +29 -1
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +97 -18
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +11 -6
package/README.md
CHANGED
|
@@ -307,6 +307,8 @@ node dist/index.js --stdio
|
|
|
307
307
|
|
|
308
308
|
## Available Tools
|
|
309
309
|
|
|
310
|
+
> **Note:** If extracted content exceeds `MAX_INLINE_CONTENT_CHARS`, the tool response includes a `resourceUri` and a `resource_link` content block instead of embedding the full text.
|
|
311
|
+
|
|
310
312
|
### `fetch-url`
|
|
311
313
|
|
|
312
314
|
Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks.
|
|
@@ -635,6 +637,12 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
|
|
|
635
637
|
| `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
|
|
636
638
|
| `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
|
|
637
639
|
|
|
640
|
+
#### 📦 Output Settings
|
|
641
|
+
|
|
642
|
+
| Variable | Default | Valid Values | Description |
|
|
643
|
+
| -------------------------- | ------- | --------------- | --------------------------------------------------------------- |
|
|
644
|
+
| `MAX_INLINE_CONTENT_CHARS` | `20000` | `1000`-`200000` | Inline content limit before returning a `resource_link` instead |
|
|
645
|
+
|
|
638
646
|
#### 📝 Logging Settings
|
|
639
647
|
|
|
640
648
|
| Variable | Default | Valid Values | Description |
|
|
@@ -651,18 +659,26 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
|
|
|
651
659
|
|
|
652
660
|
#### 🛡️ Security Settings
|
|
653
661
|
|
|
654
|
-
| Variable | Default
|
|
655
|
-
| -------------- |
|
|
656
|
-
| `API_KEY` |
|
|
657
|
-
| `
|
|
658
|
-
|
|
662
|
+
| Variable | Default | Description |
|
|
663
|
+
| -------------- | ------- | -------------------------------------------------------- |
|
|
664
|
+
| `API_KEY` | - | API Key for HTTP authentication (required for HTTP mode) |
|
|
665
|
+
| `ALLOW_REMOTE` | `false` | Allow binding to non-loopback interfaces |
|
|
666
|
+
|
|
667
|
+
#### Rate Limiting
|
|
668
|
+
|
|
669
|
+
| Variable | Default | Valid Values | Description |
|
|
670
|
+
| ----------------------- | ------- | ----------------- | ------------------------------------ |
|
|
671
|
+
| `RATE_LIMIT_ENABLED` | `true` | `true` / `false` | Enable/disable HTTP rate limiting |
|
|
672
|
+
| `RATE_LIMIT_MAX` | `100` | `1`-`10000` | Max requests per window per IP |
|
|
673
|
+
| `RATE_LIMIT_WINDOW_MS` | `60000` | `1000`-`3600000` | Rate limit window in milliseconds |
|
|
674
|
+
| `RATE_LIMIT_CLEANUP_MS` | `60000` | `10000`-`3600000` | Cleanup interval for limiter entries |
|
|
659
675
|
|
|
660
676
|
### HTTP Mode Configuration
|
|
661
677
|
|
|
662
678
|
<details>
|
|
663
679
|
<summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
|
|
664
680
|
|
|
665
|
-
SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
|
|
681
|
+
SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration and an `API_KEY` for authenticated access (send `Authorization: Bearer <key>` or `X-API-Key: <key>`).
|
|
666
682
|
|
|
667
683
|
#### Start HTTP Server
|
|
668
684
|
|
|
@@ -673,14 +689,15 @@ npx -y @j0hanz/superfetch@latest
|
|
|
673
689
|
|
|
674
690
|
#### HTTP-Specific Environment Variables
|
|
675
691
|
|
|
676
|
-
| Variable
|
|
677
|
-
|
|
|
678
|
-
| `PORT`
|
|
679
|
-
| `HOST`
|
|
680
|
-
| `ALLOWED_ORIGINS`
|
|
681
|
-
| `CORS_ALLOW_ALL`
|
|
682
|
-
| `SESSION_TTL_MS`
|
|
683
|
-
| `
|
|
692
|
+
| Variable | Default | Description |
|
|
693
|
+
| ------------------------- | ----------- | ------------------------------------------------ |
|
|
694
|
+
| `PORT` | `3000` | HTTP server port |
|
|
695
|
+
| `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
|
|
696
|
+
| `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
|
|
697
|
+
| `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
|
|
698
|
+
| `SESSION_TTL_MS` | `1800000` | Session time-to-live in milliseconds (30 mins) |
|
|
699
|
+
| `SESSION_INIT_TIMEOUT_MS` | `10000` | Time allowed for session initialization (ms) |
|
|
700
|
+
| `MAX_SESSIONS` | `200` | Maximum number of active sessions |
|
|
684
701
|
|
|
685
702
|
#### VS Code HTTP Mode Setup
|
|
686
703
|
|
|
@@ -749,7 +766,8 @@ Blocked headers: `host`, `authorization`, `cookie`, `x-forwarded-for`, `x-real-i
|
|
|
749
766
|
|
|
750
767
|
### Rate Limiting
|
|
751
768
|
|
|
752
|
-
Default: **100 requests/minute** per IP
|
|
769
|
+
Default: **100 requests/minute** per IP. Configure with `RATE_LIMIT_MAX` and
|
|
770
|
+
`RATE_LIMIT_WINDOW_MS`.
|
|
753
771
|
|
|
754
772
|
### HTTP Mode Endpoints
|
|
755
773
|
|
|
@@ -770,17 +788,20 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
770
788
|
|
|
771
789
|
### Scripts
|
|
772
790
|
|
|
773
|
-
| Command
|
|
774
|
-
|
|
|
775
|
-
| `npm run dev`
|
|
776
|
-
| `npm run build`
|
|
777
|
-
| `npm start`
|
|
778
|
-
| `npm run lint`
|
|
779
|
-
| `npm run type-check`
|
|
780
|
-
| `npm run format`
|
|
781
|
-
| `npm
|
|
782
|
-
| `npm run
|
|
783
|
-
| `npm run
|
|
791
|
+
| Command | Description |
|
|
792
|
+
| ----------------------- | ---------------------------------- |
|
|
793
|
+
| `npm run dev` | Development server with hot reload |
|
|
794
|
+
| `npm run build` | Compile TypeScript |
|
|
795
|
+
| `npm start` | Production server |
|
|
796
|
+
| `npm run lint` | Run ESLint |
|
|
797
|
+
| `npm run type-check` | TypeScript type checking |
|
|
798
|
+
| `npm run format` | Format with Prettier |
|
|
799
|
+
| `npm test` | Run Vitest tests |
|
|
800
|
+
| `npm run test:coverage` | Run tests with coverage |
|
|
801
|
+
| `npm run bench` | Run minimal performance benchmark |
|
|
802
|
+
| `npm run release` | Create new release |
|
|
803
|
+
| `npm run knip` | Find unused exports/dependencies |
|
|
804
|
+
| `npm run knip:fix` | Auto-fix unused code |
|
|
784
805
|
|
|
785
806
|
### Tech Stack
|
|
786
807
|
|
|
@@ -790,7 +811,7 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
790
811
|
| Language | TypeScript 5.9 |
|
|
791
812
|
| MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
|
|
792
813
|
| Content Extraction | @mozilla/readability ^0.6.0 |
|
|
793
|
-
| HTML Parsing | Cheerio ^1.1.2,
|
|
814
|
+
| HTML Parsing | Cheerio ^1.1.2, LinkeDOM ^0.18.12 |
|
|
794
815
|
| Markdown | Turndown ^7.2.2 |
|
|
795
816
|
| HTTP | Express ^5.2.1, Axios ^1.7.9 |
|
|
796
817
|
| Caching | node-cache ^5.1.2 |
|
|
@@ -803,8 +824,9 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
803
824
|
1. Fork the repository
|
|
804
825
|
2. Create a feature branch: `git checkout -b feature/amazing-feature`
|
|
805
826
|
3. Ensure linting passes: `npm run lint`
|
|
806
|
-
4.
|
|
807
|
-
5.
|
|
808
|
-
6.
|
|
827
|
+
4. Run tests: `npm test`
|
|
828
|
+
5. Commit changes: `git commit -m 'Add amazing feature'`
|
|
829
|
+
6. Push: `git push origin feature/amazing-feature`
|
|
830
|
+
7. Open a Pull Request
|
|
809
831
|
|
|
810
832
|
For examples of other MCP servers, see: [github.com/modelcontextprotocol/servers](https://github.com/modelcontextprotocol/servers)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const LINE_BREAK = "\n";
|
|
2
|
+
export declare const TRUNCATION_MARKER = "...[truncated]";
|
|
3
|
+
export declare const CODE_BLOCK: {
|
|
4
|
+
readonly fence: "```";
|
|
5
|
+
readonly format: (code: string, language?: string) => string;
|
|
6
|
+
};
|
|
7
|
+
export declare const FRONTMATTER_DELIMITER = "---";
|
|
8
|
+
export declare const joinLines: (lines: string[]) => string;
|
|
9
|
+
//# sourceMappingURL=formatting.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,OAAO,CAAC;AAE/B,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAElD,eAAO,MAAM,UAAU;;4BAEN,MAAM,wBAAkB,MAAM;CAGrC,CAAC;AAEX,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,GAAI,OAAO,MAAM,EAAE,KAAG,MAAgC,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export const LINE_BREAK = '\n';
|
|
2
|
+
export const TRUNCATION_MARKER = '...[truncated]';
|
|
3
|
+
export const CODE_BLOCK = {
|
|
4
|
+
fence: '```',
|
|
5
|
+
format: (code, language = '') => {
|
|
6
|
+
return `\`\`\`${language}\n${code}\n\`\`\``;
|
|
7
|
+
},
|
|
8
|
+
};
|
|
9
|
+
export const FRONTMATTER_DELIMITER = '---';
|
|
10
|
+
export const joinLines = (lines) => lines.join(LINE_BREAK);
|
|
11
|
+
//# sourceMappingURL=formatting.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"formatting.js","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAE/B,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;AAElD,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE,EAAU,EAAE;QAC9C,OAAO,SAAS,QAAQ,KAAK,IAAI,UAAU,CAAC;IAC9C,CAAC;CACO,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAE3C,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC"}
|
package/dist/config/index.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import type { LogLevel } from './types.js';
|
|
1
2
|
export declare const config: {
|
|
2
3
|
readonly server: {
|
|
3
4
|
readonly name: "superFetch";
|
|
4
|
-
readonly version:
|
|
5
|
+
readonly version: string;
|
|
5
6
|
readonly port: number;
|
|
6
7
|
readonly host: string;
|
|
7
8
|
readonly sessionTtlMs: number;
|
|
9
|
+
readonly sessionInitTimeoutMs: number;
|
|
8
10
|
readonly maxSessions: number;
|
|
9
11
|
};
|
|
10
12
|
readonly fetcher: {
|
|
@@ -25,13 +27,14 @@ export declare const config: {
|
|
|
25
27
|
readonly minParagraphLength: 10;
|
|
26
28
|
};
|
|
27
29
|
readonly logging: {
|
|
28
|
-
readonly level:
|
|
30
|
+
readonly level: LogLevel;
|
|
29
31
|
readonly enabled: boolean;
|
|
30
32
|
};
|
|
31
33
|
readonly constants: {
|
|
32
34
|
readonly maxHtmlSize: number;
|
|
33
35
|
readonly maxContentSize: number;
|
|
34
36
|
readonly maxUrlLength: 2048;
|
|
37
|
+
readonly maxInlineContentChars: number;
|
|
35
38
|
};
|
|
36
39
|
readonly security: {
|
|
37
40
|
readonly blockedHosts: Set<string>;
|
|
@@ -41,5 +44,11 @@ export declare const config: {
|
|
|
41
44
|
readonly allowRemote: boolean;
|
|
42
45
|
readonly requireAuth: boolean;
|
|
43
46
|
};
|
|
47
|
+
readonly rateLimit: {
|
|
48
|
+
readonly enabled: boolean;
|
|
49
|
+
readonly maxRequests: number;
|
|
50
|
+
readonly windowMs: number;
|
|
51
|
+
readonly cleanupIntervalMs: number;
|
|
52
|
+
};
|
|
44
53
|
};
|
|
45
54
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA6D3C,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAgFV,SAAS,MAAM,EAAE;;;;;;;;;;;;CA6BhB,CAAC"}
|
package/dist/config/index.js
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
|
+
import packageJson from '../../package.json' with { type: 'json' };
|
|
1
2
|
function parseInteger(envValue, defaultValue, min, max) {
|
|
3
|
+
const parsed = parseRawInteger(envValue, defaultValue);
|
|
4
|
+
return clampInteger(parsed, defaultValue, min, max);
|
|
5
|
+
}
|
|
6
|
+
function parseRawInteger(envValue, defaultValue) {
|
|
2
7
|
if (!envValue)
|
|
3
8
|
return defaultValue;
|
|
4
9
|
const parsed = parseInt(envValue, 10);
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
10
|
+
return Number.isNaN(parsed) ? defaultValue : parsed;
|
|
11
|
+
}
|
|
12
|
+
function clampInteger(value, defaultValue, min, max) {
|
|
13
|
+
if (min !== undefined && value < min)
|
|
8
14
|
return defaultValue;
|
|
9
|
-
if (max !== undefined &&
|
|
15
|
+
if (max !== undefined && value > max)
|
|
10
16
|
return defaultValue;
|
|
11
|
-
return
|
|
17
|
+
return value;
|
|
12
18
|
}
|
|
13
19
|
function parseBoolean(envValue, defaultValue) {
|
|
14
20
|
if (!envValue)
|
|
@@ -17,23 +23,29 @@ function parseBoolean(envValue, defaultValue) {
|
|
|
17
23
|
}
|
|
18
24
|
function parseLogLevel(envValue) {
|
|
19
25
|
const level = envValue?.toLowerCase();
|
|
20
|
-
if (level
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
if (!level)
|
|
27
|
+
return 'info';
|
|
28
|
+
return isLogLevel(level) ? level : 'info';
|
|
29
|
+
}
|
|
30
|
+
const ALLOWED_LOG_LEVELS = new Set([
|
|
31
|
+
'debug',
|
|
32
|
+
'info',
|
|
33
|
+
'warn',
|
|
34
|
+
'error',
|
|
35
|
+
]);
|
|
36
|
+
function isLogLevel(value) {
|
|
37
|
+
return ALLOWED_LOG_LEVELS.has(value);
|
|
27
38
|
}
|
|
28
39
|
const host = process.env.HOST ?? '127.0.0.1';
|
|
29
40
|
const isLoopbackHost = host === '127.0.0.1' || host === '::1' || host === 'localhost';
|
|
30
41
|
export const config = {
|
|
31
42
|
server: {
|
|
32
43
|
name: 'superFetch',
|
|
33
|
-
version:
|
|
44
|
+
version: packageJson.version,
|
|
34
45
|
port: parseInteger(process.env.PORT, 3000, 1024, 65535),
|
|
35
46
|
host,
|
|
36
47
|
sessionTtlMs: parseInteger(process.env.SESSION_TTL_MS, 30 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000),
|
|
48
|
+
sessionInitTimeoutMs: parseInteger(process.env.SESSION_INIT_TIMEOUT_MS, 10000, 1000, 60000),
|
|
37
49
|
maxSessions: parseInteger(process.env.MAX_SESSIONS, 200, 10, 10000),
|
|
38
50
|
},
|
|
39
51
|
fetcher: {
|
|
@@ -61,6 +73,7 @@ export const config = {
|
|
|
61
73
|
maxHtmlSize: 10 * 1024 * 1024,
|
|
62
74
|
maxContentSize: 5 * 1024 * 1024,
|
|
63
75
|
maxUrlLength: 2048,
|
|
76
|
+
maxInlineContentChars: parseInteger(process.env.MAX_INLINE_CONTENT_CHARS, 20000, 1000, 200000),
|
|
64
77
|
},
|
|
65
78
|
security: {
|
|
66
79
|
blockedHosts: new Set([
|
|
@@ -81,12 +94,15 @@ export const config = {
|
|
|
81
94
|
/^127\./,
|
|
82
95
|
/^0\./,
|
|
83
96
|
/^169\.254\./,
|
|
97
|
+
/^100\.64\./,
|
|
84
98
|
/^fc00:/i,
|
|
99
|
+
/^fd00:/i,
|
|
85
100
|
/^fe80:/i,
|
|
86
101
|
/^::ffff:127\./,
|
|
87
102
|
/^::ffff:10\./,
|
|
88
103
|
/^::ffff:172\.(1[6-9]|2\d|3[01])\./,
|
|
89
104
|
/^::ffff:192\.168\./,
|
|
105
|
+
/^::ffff:169\.254\./,
|
|
90
106
|
],
|
|
91
107
|
blockedHeaders: new Set([
|
|
92
108
|
'host',
|
|
@@ -100,5 +116,11 @@ export const config = {
|
|
|
100
116
|
allowRemote: parseBoolean(process.env.ALLOW_REMOTE, false),
|
|
101
117
|
requireAuth: parseBoolean(process.env.REQUIRE_AUTH, !isLoopbackHost),
|
|
102
118
|
},
|
|
119
|
+
rateLimit: {
|
|
120
|
+
enabled: parseBoolean(process.env.RATE_LIMIT_ENABLED, true),
|
|
121
|
+
maxRequests: parseInteger(process.env.RATE_LIMIT_MAX, 100, 1, 10000),
|
|
122
|
+
windowMs: parseInteger(process.env.RATE_LIMIT_WINDOW_MS, 60000, 1000, 3600000),
|
|
123
|
+
cleanupIntervalMs: parseInteger(process.env.RATE_LIMIT_CLEANUP_MS, 60000, 10000, 3600000),
|
|
124
|
+
},
|
|
103
125
|
};
|
|
104
126
|
//# sourceMappingURL=index.js.map
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAGnE,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvD,OAAO,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CACtB,QAA4B,EAC5B,YAAoB;IAEpB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;AACtD,CAAC;AAED,SAAS,YAAY,CACnB,KAAa,EACb,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,MAAM,kBAAkB,GAA0B,IAAI,GAAG,CAAC;IACxD,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,kBAAkB,CAAC,GAAG,CAAC,KAAiB,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;AAC7C,MAAM,cAAc,GAClB,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,CAAC;AAEjE,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI;QACJ,YAAY,EAAE,YAAY,CACxB,OAAO,CAAC,GAAG,CAAC,cAAc,EAC1B,EAAE,GAAG,EAAE,GAAG,IAAI,EACd,EAAE,GAAG,IAAI,EACT,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACpB;QACD,oBAAoB,EAAE,YAAY,CAChC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,KAAK,EACL,IAAI,EACJ,KAAK,CACN;QACD,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACpE;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;QAClB,qBAAqB,EAAE,YAAY,CACjC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EACpC,KAAK,EACL,IAAI,EACJ,MAAM,CACP;KACF;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,YAAY;YACZ,SAAS;YACT,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;YACpB,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;QAC3B,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC;QAC1D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC;KACrE;IACD,SAAS,EAAE;QACT,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC;QAC3D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC;QACpE,QAAQ,EAAE,YAAY,CACpB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,KAAK,EACL,IAAI,EACJ,OAAO,CACR;QACD,iBAAiB,EAAE,YAAY,CAC7B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EACjC,KAAK,EACL,KAAK,EACL,OAAO,CACR;KACF;CACO,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
type ContentBlockType = 'metadata' | 'heading' | 'paragraph' | 'list' | 'code' | 'table' | 'image' | 'blockquote';
|
|
2
|
+
interface ContentBlock {
|
|
3
|
+
type: ContentBlockType;
|
|
4
|
+
}
|
|
5
|
+
export interface MetadataBlock extends ContentBlock {
|
|
6
|
+
type: 'metadata';
|
|
7
|
+
title?: string;
|
|
8
|
+
description?: string;
|
|
9
|
+
author?: string;
|
|
10
|
+
url: string;
|
|
11
|
+
fetchedAt: string;
|
|
12
|
+
}
|
|
13
|
+
export interface HeadingBlock extends ContentBlock {
|
|
14
|
+
type: 'heading';
|
|
15
|
+
level: number;
|
|
16
|
+
text: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ParagraphBlock extends ContentBlock {
|
|
19
|
+
type: 'paragraph';
|
|
20
|
+
text: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ListBlock extends ContentBlock {
|
|
23
|
+
type: 'list';
|
|
24
|
+
ordered: boolean;
|
|
25
|
+
items: string[];
|
|
26
|
+
}
|
|
27
|
+
export interface CodeBlock extends ContentBlock {
|
|
28
|
+
type: 'code';
|
|
29
|
+
language?: string;
|
|
30
|
+
text: string;
|
|
31
|
+
}
|
|
32
|
+
export interface TableBlock extends ContentBlock {
|
|
33
|
+
type: 'table';
|
|
34
|
+
headers?: string[];
|
|
35
|
+
rows: string[][];
|
|
36
|
+
}
|
|
37
|
+
export interface ImageBlock extends ContentBlock {
|
|
38
|
+
type: 'image';
|
|
39
|
+
src: string;
|
|
40
|
+
alt?: string;
|
|
41
|
+
}
|
|
42
|
+
export interface BlockquoteBlock extends ContentBlock {
|
|
43
|
+
type: 'blockquote';
|
|
44
|
+
text: string;
|
|
45
|
+
}
|
|
46
|
+
export type ContentBlockUnion = MetadataBlock | HeadingBlock | ParagraphBlock | ListBlock | CodeBlock | TableBlock | ImageBlock | BlockquoteBlock;
|
|
47
|
+
export interface ExtractedArticle {
|
|
48
|
+
title?: string;
|
|
49
|
+
byline?: string;
|
|
50
|
+
content: string;
|
|
51
|
+
textContent: string;
|
|
52
|
+
excerpt?: string;
|
|
53
|
+
siteName?: string;
|
|
54
|
+
}
|
|
55
|
+
export interface CacheEntry {
|
|
56
|
+
url: string;
|
|
57
|
+
content: string;
|
|
58
|
+
fetchedAt: string;
|
|
59
|
+
expiresAt: string;
|
|
60
|
+
}
|
|
61
|
+
export interface ExtractedLink {
|
|
62
|
+
href: string;
|
|
63
|
+
text: string;
|
|
64
|
+
type: LinkType;
|
|
65
|
+
}
|
|
66
|
+
export interface ExtractedMetadata {
|
|
67
|
+
title?: string;
|
|
68
|
+
description?: string;
|
|
69
|
+
author?: string;
|
|
70
|
+
}
|
|
71
|
+
export interface ExtractionResult {
|
|
72
|
+
article: ExtractedArticle | null;
|
|
73
|
+
metadata: ExtractedMetadata;
|
|
74
|
+
}
|
|
75
|
+
export type ParseableTagName = 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'p' | 'ul' | 'ol' | 'pre' | 'code' | 'table' | 'img' | 'blockquote';
|
|
76
|
+
export interface LinksTransformResult {
|
|
77
|
+
links: ExtractedLink[];
|
|
78
|
+
linkCount: number;
|
|
79
|
+
filtered: number;
|
|
80
|
+
truncated: boolean;
|
|
81
|
+
}
|
|
82
|
+
export interface ExtractLinksOptions {
|
|
83
|
+
includeInternal: boolean;
|
|
84
|
+
includeExternal: boolean;
|
|
85
|
+
includeImages: boolean;
|
|
86
|
+
maxLinks?: number;
|
|
87
|
+
filterPattern?: RegExp;
|
|
88
|
+
}
|
|
89
|
+
export interface MarkdownTransformResult {
|
|
90
|
+
markdown: string;
|
|
91
|
+
title: string | undefined;
|
|
92
|
+
truncated: boolean;
|
|
93
|
+
}
|
|
94
|
+
export interface TransformOptions {
|
|
95
|
+
extractMainContent: boolean;
|
|
96
|
+
includeMetadata: boolean;
|
|
97
|
+
maxContentLength?: number;
|
|
98
|
+
}
|
|
99
|
+
export interface JsonlTransformResult {
|
|
100
|
+
content: string;
|
|
101
|
+
contentBlocks: number;
|
|
102
|
+
title: string | undefined;
|
|
103
|
+
truncated?: boolean;
|
|
104
|
+
}
|
|
105
|
+
export type LinkType = 'internal' | 'external' | 'image';
|
|
106
|
+
export {};
|
|
107
|
+
//# sourceMappingURL=content.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":"AAAA,KAAK,gBAAgB,GACjB,UAAU,GACV,SAAS,GACT,WAAW,GACX,MAAM,GACN,MAAM,GACN,OAAO,GACP,OAAO,GACP,YAAY,CAAC;AAEjB,UAAU,YAAY;IACpB,IAAI,EAAE,gBAAgB,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,iBAAiB,GACzB,aAAa,GACb,YAAY,GACZ,cAAc,GACd,SAAS,GACT,SAAS,GACT,UAAU,GACV,UAAU,GACV,eAAe,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,gBAAgB,GACxB,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,MAAM,GACN,OAAO,GACP,KAAK,GACL,YAAY,CAAC;AAEjB,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content.js","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
2
|
+
export interface RateLimitEntry {
|
|
3
|
+
count: number;
|
|
4
|
+
resetTime: number;
|
|
5
|
+
lastAccessed: number;
|
|
6
|
+
}
|
|
7
|
+
export interface RateLimiterOptions {
|
|
8
|
+
maxRequests: number;
|
|
9
|
+
windowMs: number;
|
|
10
|
+
cleanupIntervalMs: number;
|
|
11
|
+
}
|
|
12
|
+
export type ToolContentBlock = {
|
|
13
|
+
type: 'text';
|
|
14
|
+
text: string;
|
|
15
|
+
} | {
|
|
16
|
+
type: 'resource_link';
|
|
17
|
+
uri: string;
|
|
18
|
+
name: string;
|
|
19
|
+
title?: string;
|
|
20
|
+
description?: string;
|
|
21
|
+
mimeType?: string;
|
|
22
|
+
};
|
|
23
|
+
export interface FetchOptions {
|
|
24
|
+
customHeaders?: Record<string, string>;
|
|
25
|
+
signal?: AbortSignal;
|
|
26
|
+
timeout?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface ContentTransformOptions {
|
|
29
|
+
readonly extractMainContent: boolean;
|
|
30
|
+
readonly includeMetadata: boolean;
|
|
31
|
+
}
|
|
32
|
+
export interface TruncationResult {
|
|
33
|
+
readonly content: string;
|
|
34
|
+
readonly truncated: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface SessionEntry {
|
|
37
|
+
readonly transport: StreamableHTTPServerTransport;
|
|
38
|
+
createdAt: number;
|
|
39
|
+
lastSeen: number;
|
|
40
|
+
}
|
|
41
|
+
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
42
|
+
export type LogMetadata = Record<string, unknown>;
|
|
43
|
+
export interface McpRequestBody {
|
|
44
|
+
method?: string;
|
|
45
|
+
id?: string | number;
|
|
46
|
+
jsonrpc?: '2.0';
|
|
47
|
+
params?: unknown;
|
|
48
|
+
}
|
|
49
|
+
export interface FetchPipelineOptions<T> {
|
|
50
|
+
/** URL to fetch */
|
|
51
|
+
url: string;
|
|
52
|
+
/** Cache namespace (e.g., 'url', 'links', 'markdown') */
|
|
53
|
+
cacheNamespace: string;
|
|
54
|
+
/** Optional custom HTTP headers */
|
|
55
|
+
customHeaders?: Record<string, string>;
|
|
56
|
+
/** Optional: number of retry attempts (1-10, defaults to 3) */
|
|
57
|
+
retries?: number;
|
|
58
|
+
/** Optional: AbortSignal for request cancellation */
|
|
59
|
+
signal?: AbortSignal;
|
|
60
|
+
/** Optional: per-request timeout override in milliseconds */
|
|
61
|
+
timeout?: number;
|
|
62
|
+
/** Optional: cache variation input for headers/flags */
|
|
63
|
+
cacheVary?: Record<string, unknown> | string;
|
|
64
|
+
/** Transform function to process HTML into desired format */
|
|
65
|
+
transform: (html: string, url: string) => T;
|
|
66
|
+
/** Optional: serialize result for caching (defaults to JSON.stringify) */
|
|
67
|
+
serialize?: (result: T) => string;
|
|
68
|
+
/** Optional: deserialize cached content */
|
|
69
|
+
deserialize?: (cached: string) => T | undefined;
|
|
70
|
+
}
|
|
71
|
+
export interface PipelineResult<T> {
|
|
72
|
+
data: T;
|
|
73
|
+
fromCache: boolean;
|
|
74
|
+
url: string;
|
|
75
|
+
fetchedAt: string;
|
|
76
|
+
cacheKey?: string | null;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAGN,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,6BAA6B,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAC3D,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAGlD,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAGD,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,6DAA6D;IAC7D,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAC5C,0EAA0E;IAC1E,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import type { ToolContentBlock } from './runtime.js';
|
|
2
|
+
interface RequestOptions {
|
|
3
|
+
/** Custom HTTP headers for the request */
|
|
4
|
+
customHeaders?: Record<string, string>;
|
|
5
|
+
/** Request timeout in milliseconds (1000-60000) */
|
|
6
|
+
timeout?: number;
|
|
7
|
+
/** Number of retry attempts (1-10) */
|
|
8
|
+
retries?: number;
|
|
9
|
+
}
|
|
10
|
+
export interface FetchUrlInput extends RequestOptions {
|
|
11
|
+
url: string;
|
|
12
|
+
extractMainContent?: boolean;
|
|
13
|
+
includeMetadata?: boolean;
|
|
14
|
+
maxContentLength?: number;
|
|
15
|
+
format?: 'jsonl' | 'markdown';
|
|
16
|
+
}
|
|
17
|
+
export interface FetchLinksInput extends RequestOptions {
|
|
18
|
+
url: string;
|
|
19
|
+
includeExternal?: boolean;
|
|
20
|
+
includeInternal?: boolean;
|
|
21
|
+
maxLinks?: number;
|
|
22
|
+
filterPattern?: string;
|
|
23
|
+
includeImages?: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface FetchMarkdownInput extends RequestOptions {
|
|
26
|
+
url: string;
|
|
27
|
+
extractMainContent?: boolean;
|
|
28
|
+
includeMetadata?: boolean;
|
|
29
|
+
maxContentLength?: number;
|
|
30
|
+
}
|
|
31
|
+
export interface FetchUrlsInput extends RequestOptions {
|
|
32
|
+
urls: string[];
|
|
33
|
+
extractMainContent?: boolean;
|
|
34
|
+
includeMetadata?: boolean;
|
|
35
|
+
maxContentLength?: number;
|
|
36
|
+
format?: 'jsonl' | 'markdown';
|
|
37
|
+
concurrency?: number;
|
|
38
|
+
continueOnError?: boolean;
|
|
39
|
+
}
|
|
40
|
+
export interface ErrorResponse {
|
|
41
|
+
error: {
|
|
42
|
+
message: string;
|
|
43
|
+
code: string;
|
|
44
|
+
statusCode: number;
|
|
45
|
+
details?: Record<string, unknown>;
|
|
46
|
+
stack?: string;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export interface ToolResponse<T = Record<string, unknown>> {
|
|
50
|
+
[x: string]: unknown;
|
|
51
|
+
content: ToolContentBlock[];
|
|
52
|
+
structuredContent: T & Record<string, unknown>;
|
|
53
|
+
}
|
|
54
|
+
export interface BatchUrlResult {
|
|
55
|
+
url: string;
|
|
56
|
+
success: boolean;
|
|
57
|
+
title?: string;
|
|
58
|
+
content?: string;
|
|
59
|
+
contentBlocks?: number;
|
|
60
|
+
contentSize?: number;
|
|
61
|
+
resourceUri?: string;
|
|
62
|
+
resourceMimeType?: string;
|
|
63
|
+
cached?: boolean;
|
|
64
|
+
truncated?: boolean;
|
|
65
|
+
error?: string;
|
|
66
|
+
errorCode?: string;
|
|
67
|
+
}
|
|
68
|
+
export interface BatchSummary {
|
|
69
|
+
total: number;
|
|
70
|
+
successful: number;
|
|
71
|
+
failed: number;
|
|
72
|
+
cached: number;
|
|
73
|
+
totalContentBlocks: number;
|
|
74
|
+
}
|
|
75
|
+
export interface BatchResponseContent {
|
|
76
|
+
[x: string]: unknown;
|
|
77
|
+
results: BatchUrlResult[];
|
|
78
|
+
summary: BatchSummary;
|
|
79
|
+
fetchedAt: string;
|
|
80
|
+
}
|
|
81
|
+
export interface ToolErrorResponse {
|
|
82
|
+
[x: string]: unknown;
|
|
83
|
+
content: ToolContentBlock[];
|
|
84
|
+
structuredContent: {
|
|
85
|
+
[x: string]: unknown;
|
|
86
|
+
error: string;
|
|
87
|
+
url: string;
|
|
88
|
+
errorCode: string;
|
|
89
|
+
};
|
|
90
|
+
isError: true;
|
|
91
|
+
}
|
|
92
|
+
export interface ToolResponseBase {
|
|
93
|
+
[x: string]: unknown;
|
|
94
|
+
content: ToolContentBlock[];
|
|
95
|
+
structuredContent?: Record<string, unknown>;
|
|
96
|
+
isError?: boolean;
|
|
97
|
+
}
|
|
98
|
+
export {};
|
|
99
|
+
//# sourceMappingURL=tools.d.ts.map
|