@j0hanz/superfetch 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -32
- package/dist/config/formatting.d.ts +9 -0
- package/dist/config/formatting.d.ts.map +1 -0
- package/dist/config/formatting.js +11 -0
- package/dist/config/formatting.js.map +1 -0
- package/dist/config/index.d.ts +16 -2
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +43 -14
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +107 -0
- package/dist/config/types/content.d.ts.map +1 -0
- package/dist/config/types/content.js +2 -0
- package/dist/config/types/content.js.map +1 -0
- package/dist/config/types/runtime.d.ts +78 -0
- package/dist/config/types/runtime.d.ts.map +1 -0
- package/dist/config/types/runtime.js +2 -0
- package/dist/config/types/runtime.js.map +1 -0
- package/dist/config/types/tools.d.ts +99 -0
- package/dist/config/types/tools.d.ts.map +1 -0
- package/dist/config/types/tools.js +2 -0
- package/dist/config/types/tools.js.map +1 -0
- package/dist/config/types.d.ts +3 -296
- package/dist/config/types.d.ts.map +1 -1
- package/dist/http/auth.d.ts +3 -0
- package/dist/http/auth.d.ts.map +1 -0
- package/dist/http/auth.js +34 -0
- package/dist/http/auth.js.map +1 -0
- package/dist/http/cors.d.ts +8 -0
- package/dist/http/cors.d.ts.map +1 -0
- package/dist/http/cors.js +47 -0
- package/dist/http/cors.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +5 -0
- package/dist/http/mcp-routes.d.ts.map +1 -0
- package/dist/http/mcp-routes.js +110 -0
- package/dist/http/mcp-routes.js.map +1 -0
- package/dist/http/mcp-session.d.ts +12 -0
- package/dist/http/mcp-session.d.ts.map +1 -0
- package/dist/http/mcp-session.js +209 -0
- package/dist/http/mcp-session.js.map +1 -0
- package/dist/http/mcp-validation.d.ts +3 -0
- package/dist/http/mcp-validation.d.ts.map +1 -0
- package/dist/http/mcp-validation.js +34 -0
- package/dist/http/mcp-validation.js.map +1 -0
- package/dist/http/rate-limit.d.ts +13 -0
- package/dist/http/rate-limit.d.ts.map +1 -0
- package/dist/http/rate-limit.js +91 -0
- package/dist/http/rate-limit.js.map +1 -0
- package/dist/http/server.d.ts +4 -0
- package/dist/http/server.d.ts.map +1 -0
- package/dist/http/server.js +183 -0
- package/dist/http/server.js.map +1 -0
- package/dist/http/sessions.d.ts +15 -0
- package/dist/http/sessions.d.ts.map +1 -0
- package/dist/http/sessions.js +64 -0
- package/dist/http/sessions.js.map +1 -0
- package/dist/index.js +26 -223
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +46 -15
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +104 -44
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +77 -69
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +9 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +13 -1
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +90 -13
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +9 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +9 -0
- package/dist/services/context.js.map +1 -0
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +122 -87
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +4 -0
- package/dist/services/fetcher/agents.d.ts.map +1 -0
- package/dist/services/fetcher/agents.js +111 -0
- package/dist/services/fetcher/agents.js.map +1 -0
- package/dist/services/fetcher/errors.d.ts +5 -0
- package/dist/services/fetcher/errors.d.ts.map +1 -0
- package/dist/services/fetcher/errors.js +71 -0
- package/dist/services/fetcher/errors.js.map +1 -0
- package/dist/services/fetcher/headers.d.ts +2 -0
- package/dist/services/fetcher/headers.d.ts.map +1 -0
- package/dist/services/fetcher/headers.js +28 -0
- package/dist/services/fetcher/headers.js.map +1 -0
- package/dist/services/fetcher/interceptors.d.ts +10 -0
- package/dist/services/fetcher/interceptors.d.ts.map +1 -0
- package/dist/services/fetcher/interceptors.js +82 -0
- package/dist/services/fetcher/interceptors.js.map +1 -0
- package/dist/services/fetcher/redirects.d.ts +6 -0
- package/dist/services/fetcher/redirects.d.ts.map +1 -0
- package/dist/services/fetcher/redirects.js +67 -0
- package/dist/services/fetcher/redirects.js.map +1 -0
- package/dist/services/fetcher/response.d.ts +5 -0
- package/dist/services/fetcher/response.d.ts.map +1 -0
- package/dist/services/fetcher/response.js +39 -0
- package/dist/services/fetcher/response.js.map +1 -0
- package/dist/services/fetcher/retry-policy.d.ts +28 -0
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
- package/dist/services/fetcher/retry-policy.js +138 -0
- package/dist/services/fetcher/retry-policy.js.map +1 -0
- package/dist/services/fetcher.d.ts +2 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +61 -254
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +14 -5
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -0
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +55 -35
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +78 -116
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-single.shared.js +49 -0
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +82 -54
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/processor.js +153 -0
- package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.js +58 -0
- package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.js +18 -0
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +104 -197
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +36 -237
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +357 -0
- package/dist/tools/schemas.d.ts.map +1 -0
- package/dist/tools/schemas.js +272 -0
- package/dist/tools/schemas.js.map +1 -0
- package/dist/tools/utils/cache-vary.d.ts +3 -0
- package/dist/tools/utils/cache-vary.d.ts.map +1 -0
- package/dist/tools/utils/cache-vary.js +44 -0
- package/dist/tools/utils/cache-vary.js.map +1 -0
- package/dist/tools/utils/common.d.ts +2 -2
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +5 -1
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +16 -0
- package/dist/tools/utils/content-transform.d.ts.map +1 -0
- package/dist/tools/utils/content-transform.js +49 -0
- package/dist/tools/utils/content-transform.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +32 -18
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +11 -0
- package/dist/tools/utils/inline-content.d.ts.map +1 -0
- package/dist/tools/utils/inline-content.js +39 -0
- package/dist/tools/utils/inline-content.js.map +1 -0
- package/dist/tools/utils/markdown-toc.d.ts +3 -0
- package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
- package/dist/tools/utils/markdown-toc.js +35 -0
- package/dist/tools/utils/markdown-toc.js.map +1 -0
- package/dist/tools/utils/tool-response.d.ts +9 -0
- package/dist/tools/utils/tool-response.d.ts.map +1 -0
- package/dist/tools/utils/tool-response.js +19 -0
- package/dist/tools/utils/tool-response.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +51 -28
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +82 -111
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/header-normalizer.d.ts +5 -0
- package/dist/utils/header-normalizer.d.ts.map +1 -0
- package/dist/utils/header-normalizer.js +25 -0
- package/dist/utils/header-normalizer.js.map +1 -0
- package/dist/utils/tool-error-handler.d.ts +1 -0
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +29 -1
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +98 -18
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +11 -6
package/README.md
CHANGED
|
@@ -307,6 +307,8 @@ node dist/index.js --stdio
|
|
|
307
307
|
|
|
308
308
|
## Available Tools
|
|
309
309
|
|
|
310
|
+
> **Note:** If extracted content exceeds `MAX_INLINE_CONTENT_CHARS`, the tool response includes a `resourceUri` and a `resource_link` content block instead of embedding the full text.
|
|
311
|
+
|
|
310
312
|
### `fetch-url`
|
|
311
313
|
|
|
312
314
|
Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks.
|
|
@@ -477,12 +479,6 @@ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing
|
|
|
477
479
|
| `superfetch://health` | Real-time server health and dependency status |
|
|
478
480
|
| Dynamic resources | Cached content available via resource subscriptions |
|
|
479
481
|
|
|
480
|
-
### Prompts
|
|
481
|
-
|
|
482
|
-
- **`analyze-web-content`** — Analyze fetched content with optional focus area
|
|
483
|
-
- **`summarize-page`** — Fetch and summarize a webpage concisely
|
|
484
|
-
- **`extract-data`** — Extract structured data from a webpage
|
|
485
|
-
|
|
486
482
|
---
|
|
487
483
|
|
|
488
484
|
## Configuration
|
|
@@ -641,6 +637,12 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
|
|
|
641
637
|
| `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
|
|
642
638
|
| `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
|
|
643
639
|
|
|
640
|
+
#### 📦 Output Settings
|
|
641
|
+
|
|
642
|
+
| Variable | Default | Valid Values | Description |
|
|
643
|
+
| -------------------------- | ------- | --------------- | --------------------------------------------------------------- |
|
|
644
|
+
| `MAX_INLINE_CONTENT_CHARS` | `20000` | `1000`-`200000` | Inline content limit before returning a `resource_link` instead |
|
|
645
|
+
|
|
644
646
|
#### 📝 Logging Settings
|
|
645
647
|
|
|
646
648
|
| Variable | Default | Valid Values | Description |
|
|
@@ -655,12 +657,28 @@ Configure SuperFetch behavior by adding environment variables to your MCP client
|
|
|
655
657
|
| `EXTRACT_MAIN_CONTENT` | `true` | `true` / `false` | Use Mozilla Readability to extract main content |
|
|
656
658
|
| `INCLUDE_METADATA` | `true` | `true` / `false` | Include page metadata (title, description, author) |
|
|
657
659
|
|
|
660
|
+
#### 🛡️ Security Settings
|
|
661
|
+
|
|
662
|
+
| Variable | Default | Description |
|
|
663
|
+
| -------------- | ------- | -------------------------------------------------------- |
|
|
664
|
+
| `API_KEY` | - | API Key for HTTP authentication (required for HTTP mode) |
|
|
665
|
+
| `ALLOW_REMOTE` | `false` | Allow binding to non-loopback interfaces |
|
|
666
|
+
|
|
667
|
+
#### Rate Limiting
|
|
668
|
+
|
|
669
|
+
| Variable | Default | Valid Values | Description |
|
|
670
|
+
| ----------------------- | ------- | ----------------- | ------------------------------------ |
|
|
671
|
+
| `RATE_LIMIT_ENABLED` | `true` | `true` / `false` | Enable/disable HTTP rate limiting |
|
|
672
|
+
| `RATE_LIMIT_MAX` | `100` | `1`-`10000` | Max requests per window per IP |
|
|
673
|
+
| `RATE_LIMIT_WINDOW_MS` | `60000` | `1000`-`3600000` | Rate limit window in milliseconds |
|
|
674
|
+
| `RATE_LIMIT_CLEANUP_MS` | `60000` | `10000`-`3600000` | Cleanup interval for limiter entries |
|
|
675
|
+
|
|
658
676
|
### HTTP Mode Configuration
|
|
659
677
|
|
|
660
678
|
<details>
|
|
661
679
|
<summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
|
|
662
680
|
|
|
663
|
-
SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
|
|
681
|
+
SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration and an `API_KEY` for authenticated access (send `Authorization: Bearer <key>` or `X-API-Key: <key>`).
|
|
664
682
|
|
|
665
683
|
#### Start HTTP Server
|
|
666
684
|
|
|
@@ -671,12 +689,15 @@ npx -y @j0hanz/superfetch@latest
|
|
|
671
689
|
|
|
672
690
|
#### HTTP-Specific Environment Variables
|
|
673
691
|
|
|
674
|
-
| Variable
|
|
675
|
-
|
|
|
676
|
-
| `PORT`
|
|
677
|
-
| `HOST`
|
|
678
|
-
| `ALLOWED_ORIGINS`
|
|
679
|
-
| `CORS_ALLOW_ALL`
|
|
692
|
+
| Variable | Default | Description |
|
|
693
|
+
| ------------------------- | ----------- | ------------------------------------------------ |
|
|
694
|
+
| `PORT` | `3000` | HTTP server port |
|
|
695
|
+
| `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
|
|
696
|
+
| `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
|
|
697
|
+
| `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
|
|
698
|
+
| `SESSION_TTL_MS` | `1800000` | Session time-to-live in milliseconds (30 mins) |
|
|
699
|
+
| `SESSION_INIT_TIMEOUT_MS` | `10000` | Time allowed for session initialization (ms) |
|
|
700
|
+
| `MAX_SESSIONS` | `200` | Maximum number of active sessions |
|
|
680
701
|
|
|
681
702
|
#### VS Code HTTP Mode Setup
|
|
682
703
|
|
|
@@ -745,7 +766,8 @@ Blocked headers: `host`, `authorization`, `cookie`, `x-forwarded-for`, `x-real-i
|
|
|
745
766
|
|
|
746
767
|
### Rate Limiting
|
|
747
768
|
|
|
748
|
-
Default: **100 requests/minute** per IP
|
|
769
|
+
Default: **100 requests/minute** per IP. Configure with `RATE_LIMIT_MAX` and
|
|
770
|
+
`RATE_LIMIT_WINDOW_MS`.
|
|
749
771
|
|
|
750
772
|
### HTTP Mode Endpoints
|
|
751
773
|
|
|
@@ -766,17 +788,20 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
766
788
|
|
|
767
789
|
### Scripts
|
|
768
790
|
|
|
769
|
-
| Command
|
|
770
|
-
|
|
|
771
|
-
| `npm run dev`
|
|
772
|
-
| `npm run build`
|
|
773
|
-
| `npm start`
|
|
774
|
-
| `npm run lint`
|
|
775
|
-
| `npm run type-check`
|
|
776
|
-
| `npm run format`
|
|
777
|
-
| `npm
|
|
778
|
-
| `npm run
|
|
779
|
-
| `npm run
|
|
791
|
+
| Command | Description |
|
|
792
|
+
| ----------------------- | ---------------------------------- |
|
|
793
|
+
| `npm run dev` | Development server with hot reload |
|
|
794
|
+
| `npm run build` | Compile TypeScript |
|
|
795
|
+
| `npm start` | Production server |
|
|
796
|
+
| `npm run lint` | Run ESLint |
|
|
797
|
+
| `npm run type-check` | TypeScript type checking |
|
|
798
|
+
| `npm run format` | Format with Prettier |
|
|
799
|
+
| `npm test` | Run Vitest tests |
|
|
800
|
+
| `npm run test:coverage` | Run tests with coverage |
|
|
801
|
+
| `npm run bench` | Run minimal performance benchmark |
|
|
802
|
+
| `npm run release` | Create new release |
|
|
803
|
+
| `npm run knip` | Find unused exports/dependencies |
|
|
804
|
+
| `npm run knip:fix` | Auto-fix unused code |
|
|
780
805
|
|
|
781
806
|
### Tech Stack
|
|
782
807
|
|
|
@@ -786,12 +811,11 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
786
811
|
| Language | TypeScript 5.9 |
|
|
787
812
|
| MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
|
|
788
813
|
| Content Extraction | @mozilla/readability ^0.6.0 |
|
|
789
|
-
| HTML Parsing | Cheerio ^1.1.2,
|
|
814
|
+
| HTML Parsing | Cheerio ^1.1.2, LinkeDOM ^0.18.12 |
|
|
790
815
|
| Markdown | Turndown ^7.2.2 |
|
|
791
|
-
| HTTP | Express ^5.2.1, Axios ^1.
|
|
816
|
+
| HTTP | Express ^5.2.1, Axios ^1.7.9 |
|
|
792
817
|
| Caching | node-cache ^5.1.2 |
|
|
793
|
-
| Validation | Zod ^3.
|
|
794
|
-
| Logging | Winston ^3.19.0 |
|
|
818
|
+
| Validation | Zod ^3.24.1 |
|
|
795
819
|
|
|
796
820
|
---
|
|
797
821
|
|
|
@@ -800,8 +824,9 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
800
824
|
1. Fork the repository
|
|
801
825
|
2. Create a feature branch: `git checkout -b feature/amazing-feature`
|
|
802
826
|
3. Ensure linting passes: `npm run lint`
|
|
803
|
-
4.
|
|
804
|
-
5.
|
|
805
|
-
6.
|
|
827
|
+
4. Run tests: `npm test`
|
|
828
|
+
5. Commit changes: `git commit -m 'Add amazing feature'`
|
|
829
|
+
6. Push: `git push origin feature/amazing-feature`
|
|
830
|
+
7. Open a Pull Request
|
|
806
831
|
|
|
807
832
|
For examples of other MCP servers, see: [github.com/modelcontextprotocol/servers](https://github.com/modelcontextprotocol/servers)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const LINE_BREAK = "\n";
|
|
2
|
+
export declare const TRUNCATION_MARKER = "...[truncated]";
|
|
3
|
+
export declare const CODE_BLOCK: {
|
|
4
|
+
readonly fence: "```";
|
|
5
|
+
readonly format: (code: string, language?: string) => string;
|
|
6
|
+
};
|
|
7
|
+
export declare const FRONTMATTER_DELIMITER = "---";
|
|
8
|
+
export declare const joinLines: (lines: string[]) => string;
|
|
9
|
+
//# sourceMappingURL=formatting.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,OAAO,CAAC;AAE/B,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAElD,eAAO,MAAM,UAAU;;4BAEN,MAAM,wBAAkB,MAAM;CAGrC,CAAC;AAEX,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,GAAI,OAAO,MAAM,EAAE,KAAG,MAAgC,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export const LINE_BREAK = '\n';
|
|
2
|
+
export const TRUNCATION_MARKER = '...[truncated]';
|
|
3
|
+
export const CODE_BLOCK = {
|
|
4
|
+
fence: '```',
|
|
5
|
+
format: (code, language = '') => {
|
|
6
|
+
return `\`\`\`${language}\n${code}\n\`\`\``;
|
|
7
|
+
},
|
|
8
|
+
};
|
|
9
|
+
export const FRONTMATTER_DELIMITER = '---';
|
|
10
|
+
export const joinLines = (lines) => lines.join(LINE_BREAK);
|
|
11
|
+
//# sourceMappingURL=formatting.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"formatting.js","sourceRoot":"","sources":["../../src/config/formatting.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAE/B,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;AAElD,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE,EAAU,EAAE;QAC9C,OAAO,SAAS,QAAQ,KAAK,IAAI,UAAU,CAAC;IAC9C,CAAC;CACO,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAE3C,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC"}
|
package/dist/config/index.d.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import type { LogLevel } from './types.js';
|
|
1
2
|
export declare const config: {
|
|
2
3
|
readonly server: {
|
|
3
4
|
readonly name: "superFetch";
|
|
4
|
-
readonly version:
|
|
5
|
+
readonly version: string;
|
|
5
6
|
readonly port: number;
|
|
6
7
|
readonly host: string;
|
|
8
|
+
readonly sessionTtlMs: number;
|
|
9
|
+
readonly sessionInitTimeoutMs: number;
|
|
10
|
+
readonly maxSessions: number;
|
|
7
11
|
};
|
|
8
12
|
readonly fetcher: {
|
|
9
13
|
readonly timeout: number;
|
|
@@ -23,18 +27,28 @@ export declare const config: {
|
|
|
23
27
|
readonly minParagraphLength: 10;
|
|
24
28
|
};
|
|
25
29
|
readonly logging: {
|
|
26
|
-
readonly level:
|
|
30
|
+
readonly level: LogLevel;
|
|
27
31
|
readonly enabled: boolean;
|
|
28
32
|
};
|
|
29
33
|
readonly constants: {
|
|
30
34
|
readonly maxHtmlSize: number;
|
|
31
35
|
readonly maxContentSize: number;
|
|
32
36
|
readonly maxUrlLength: 2048;
|
|
37
|
+
readonly maxInlineContentChars: number;
|
|
33
38
|
};
|
|
34
39
|
readonly security: {
|
|
35
40
|
readonly blockedHosts: Set<string>;
|
|
36
41
|
readonly blockedIpPatterns: readonly RegExp[];
|
|
37
42
|
readonly blockedHeaders: Set<string>;
|
|
43
|
+
readonly apiKey: string | undefined;
|
|
44
|
+
readonly allowRemote: boolean;
|
|
45
|
+
readonly requireAuth: boolean;
|
|
46
|
+
};
|
|
47
|
+
readonly rateLimit: {
|
|
48
|
+
readonly enabled: boolean;
|
|
49
|
+
readonly maxRequests: number;
|
|
50
|
+
readonly windowMs: number;
|
|
51
|
+
readonly cleanupIntervalMs: number;
|
|
38
52
|
};
|
|
39
53
|
};
|
|
40
54
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA6D3C,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAgFV,SAAS,MAAM,EAAE;;;;;;;;;;;;CA6BhB,CAAC"}
|
package/dist/config/index.js
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
|
+
import packageJson from '../../package.json' with { type: 'json' };
|
|
1
2
|
function parseInteger(envValue, defaultValue, min, max) {
|
|
3
|
+
const parsed = parseRawInteger(envValue, defaultValue);
|
|
4
|
+
return clampInteger(parsed, defaultValue, min, max);
|
|
5
|
+
}
|
|
6
|
+
function parseRawInteger(envValue, defaultValue) {
|
|
2
7
|
if (!envValue)
|
|
3
8
|
return defaultValue;
|
|
4
9
|
const parsed = parseInt(envValue, 10);
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
10
|
+
return Number.isNaN(parsed) ? defaultValue : parsed;
|
|
11
|
+
}
|
|
12
|
+
function clampInteger(value, defaultValue, min, max) {
|
|
13
|
+
if (min !== undefined && value < min)
|
|
8
14
|
return defaultValue;
|
|
9
|
-
if (max !== undefined &&
|
|
15
|
+
if (max !== undefined && value > max)
|
|
10
16
|
return defaultValue;
|
|
11
|
-
return
|
|
17
|
+
return value;
|
|
12
18
|
}
|
|
13
19
|
function parseBoolean(envValue, defaultValue) {
|
|
14
20
|
if (!envValue)
|
|
@@ -17,20 +23,30 @@ function parseBoolean(envValue, defaultValue) {
|
|
|
17
23
|
}
|
|
18
24
|
function parseLogLevel(envValue) {
|
|
19
25
|
const level = envValue?.toLowerCase();
|
|
20
|
-
if (level
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
level === 'error') {
|
|
24
|
-
return level;
|
|
25
|
-
}
|
|
26
|
-
return 'info';
|
|
26
|
+
if (!level)
|
|
27
|
+
return 'info';
|
|
28
|
+
return isLogLevel(level) ? level : 'info';
|
|
27
29
|
}
|
|
30
|
+
const ALLOWED_LOG_LEVELS = new Set([
|
|
31
|
+
'debug',
|
|
32
|
+
'info',
|
|
33
|
+
'warn',
|
|
34
|
+
'error',
|
|
35
|
+
]);
|
|
36
|
+
function isLogLevel(value) {
|
|
37
|
+
return ALLOWED_LOG_LEVELS.has(value);
|
|
38
|
+
}
|
|
39
|
+
const host = process.env.HOST ?? '127.0.0.1';
|
|
40
|
+
const isLoopbackHost = host === '127.0.0.1' || host === '::1' || host === 'localhost';
|
|
28
41
|
export const config = {
|
|
29
42
|
server: {
|
|
30
43
|
name: 'superFetch',
|
|
31
|
-
version:
|
|
44
|
+
version: packageJson.version,
|
|
32
45
|
port: parseInteger(process.env.PORT, 3000, 1024, 65535),
|
|
33
|
-
host
|
|
46
|
+
host,
|
|
47
|
+
sessionTtlMs: parseInteger(process.env.SESSION_TTL_MS, 30 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000),
|
|
48
|
+
sessionInitTimeoutMs: parseInteger(process.env.SESSION_INIT_TIMEOUT_MS, 10000, 1000, 60000),
|
|
49
|
+
maxSessions: parseInteger(process.env.MAX_SESSIONS, 200, 10, 10000),
|
|
34
50
|
},
|
|
35
51
|
fetcher: {
|
|
36
52
|
timeout: parseInteger(process.env.FETCH_TIMEOUT, 30000, 5000, 120000),
|
|
@@ -57,6 +73,7 @@ export const config = {
|
|
|
57
73
|
maxHtmlSize: 10 * 1024 * 1024,
|
|
58
74
|
maxContentSize: 5 * 1024 * 1024,
|
|
59
75
|
maxUrlLength: 2048,
|
|
76
|
+
maxInlineContentChars: parseInteger(process.env.MAX_INLINE_CONTENT_CHARS, 20000, 1000, 200000),
|
|
60
77
|
},
|
|
61
78
|
security: {
|
|
62
79
|
blockedHosts: new Set([
|
|
@@ -77,12 +94,15 @@ export const config = {
|
|
|
77
94
|
/^127\./,
|
|
78
95
|
/^0\./,
|
|
79
96
|
/^169\.254\./,
|
|
97
|
+
/^100\.64\./,
|
|
80
98
|
/^fc00:/i,
|
|
99
|
+
/^fd00:/i,
|
|
81
100
|
/^fe80:/i,
|
|
82
101
|
/^::ffff:127\./,
|
|
83
102
|
/^::ffff:10\./,
|
|
84
103
|
/^::ffff:172\.(1[6-9]|2\d|3[01])\./,
|
|
85
104
|
/^::ffff:192\.168\./,
|
|
105
|
+
/^::ffff:169\.254\./,
|
|
86
106
|
],
|
|
87
107
|
blockedHeaders: new Set([
|
|
88
108
|
'host',
|
|
@@ -92,6 +112,15 @@ export const config = {
|
|
|
92
112
|
'x-real-ip',
|
|
93
113
|
'proxy-authorization',
|
|
94
114
|
]),
|
|
115
|
+
apiKey: process.env.API_KEY,
|
|
116
|
+
allowRemote: parseBoolean(process.env.ALLOW_REMOTE, false),
|
|
117
|
+
requireAuth: parseBoolean(process.env.REQUIRE_AUTH, !isLoopbackHost),
|
|
118
|
+
},
|
|
119
|
+
rateLimit: {
|
|
120
|
+
enabled: parseBoolean(process.env.RATE_LIMIT_ENABLED, true),
|
|
121
|
+
maxRequests: parseInteger(process.env.RATE_LIMIT_MAX, 100, 1, 10000),
|
|
122
|
+
windowMs: parseInteger(process.env.RATE_LIMIT_WINDOW_MS, 60000, 1000, 3600000),
|
|
123
|
+
cleanupIntervalMs: parseInteger(process.env.RATE_LIMIT_CLEANUP_MS, 60000, 10000, 3600000),
|
|
95
124
|
},
|
|
96
125
|
};
|
|
97
126
|
//# sourceMappingURL=index.js.map
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAGnE,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvD,OAAO,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CACtB,QAA4B,EAC5B,YAAoB;IAEpB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;AACtD,CAAC;AAED,SAAS,YAAY,CACnB,KAAa,EACb,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC1D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,MAAM,kBAAkB,GAA0B,IAAI,GAAG,CAAC;IACxD,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,kBAAkB,CAAC,GAAG,CAAC,KAAiB,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;AAC7C,MAAM,cAAc,GAClB,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW,CAAC;AAEjE,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI;QACJ,YAAY,EAAE,YAAY,CACxB,OAAO,CAAC,GAAG,CAAC,cAAc,EAC1B,EAAE,GAAG,EAAE,GAAG,IAAI,EACd,EAAE,GAAG,IAAI,EACT,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACpB;QACD,oBAAoB,EAAE,YAAY,CAChC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EACnC,KAAK,EACL,IAAI,EACJ,KAAK,CACN;QACD,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACpE;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;QAClB,qBAAqB,EAAE,YAAY,CACjC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EACpC,KAAK,EACL,IAAI,EACJ,MAAM,CACP;KACF;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,YAAY;YACZ,SAAS;YACT,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;YACpB,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;QAC3B,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC;QAC1D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC;KACrE;IACD,SAAS,EAAE;QACT,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC;QAC3D,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC;QACpE,QAAQ,EAAE,YAAY,CACpB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,KAAK,EACL,IAAI,EACJ,OAAO,CACR;QACD,iBAAiB,EAAE,YAAY,CAC7B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EACjC,KAAK,EACL,KAAK,EACL,OAAO,CACR;KACF;CACO,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
type ContentBlockType = 'metadata' | 'heading' | 'paragraph' | 'list' | 'code' | 'table' | 'image' | 'blockquote';
|
|
2
|
+
interface ContentBlock {
|
|
3
|
+
type: ContentBlockType;
|
|
4
|
+
}
|
|
5
|
+
export interface MetadataBlock extends ContentBlock {
|
|
6
|
+
type: 'metadata';
|
|
7
|
+
title?: string;
|
|
8
|
+
description?: string;
|
|
9
|
+
author?: string;
|
|
10
|
+
url: string;
|
|
11
|
+
fetchedAt: string;
|
|
12
|
+
}
|
|
13
|
+
export interface HeadingBlock extends ContentBlock {
|
|
14
|
+
type: 'heading';
|
|
15
|
+
level: number;
|
|
16
|
+
text: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ParagraphBlock extends ContentBlock {
|
|
19
|
+
type: 'paragraph';
|
|
20
|
+
text: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ListBlock extends ContentBlock {
|
|
23
|
+
type: 'list';
|
|
24
|
+
ordered: boolean;
|
|
25
|
+
items: string[];
|
|
26
|
+
}
|
|
27
|
+
export interface CodeBlock extends ContentBlock {
|
|
28
|
+
type: 'code';
|
|
29
|
+
language?: string;
|
|
30
|
+
text: string;
|
|
31
|
+
}
|
|
32
|
+
export interface TableBlock extends ContentBlock {
|
|
33
|
+
type: 'table';
|
|
34
|
+
headers?: string[];
|
|
35
|
+
rows: string[][];
|
|
36
|
+
}
|
|
37
|
+
export interface ImageBlock extends ContentBlock {
|
|
38
|
+
type: 'image';
|
|
39
|
+
src: string;
|
|
40
|
+
alt?: string;
|
|
41
|
+
}
|
|
42
|
+
export interface BlockquoteBlock extends ContentBlock {
|
|
43
|
+
type: 'blockquote';
|
|
44
|
+
text: string;
|
|
45
|
+
}
|
|
46
|
+
export type ContentBlockUnion = MetadataBlock | HeadingBlock | ParagraphBlock | ListBlock | CodeBlock | TableBlock | ImageBlock | BlockquoteBlock;
|
|
47
|
+
export interface ExtractedArticle {
|
|
48
|
+
title?: string;
|
|
49
|
+
byline?: string;
|
|
50
|
+
content: string;
|
|
51
|
+
textContent: string;
|
|
52
|
+
excerpt?: string;
|
|
53
|
+
siteName?: string;
|
|
54
|
+
}
|
|
55
|
+
export interface CacheEntry {
|
|
56
|
+
url: string;
|
|
57
|
+
content: string;
|
|
58
|
+
fetchedAt: string;
|
|
59
|
+
expiresAt: string;
|
|
60
|
+
}
|
|
61
|
+
export interface ExtractedLink {
|
|
62
|
+
href: string;
|
|
63
|
+
text: string;
|
|
64
|
+
type: LinkType;
|
|
65
|
+
}
|
|
66
|
+
export interface ExtractedMetadata {
|
|
67
|
+
title?: string;
|
|
68
|
+
description?: string;
|
|
69
|
+
author?: string;
|
|
70
|
+
}
|
|
71
|
+
export interface ExtractionResult {
|
|
72
|
+
article: ExtractedArticle | null;
|
|
73
|
+
metadata: ExtractedMetadata;
|
|
74
|
+
}
|
|
75
|
+
export type ParseableTagName = 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'p' | 'ul' | 'ol' | 'pre' | 'code' | 'table' | 'img' | 'blockquote';
|
|
76
|
+
export interface LinksTransformResult {
|
|
77
|
+
links: ExtractedLink[];
|
|
78
|
+
linkCount: number;
|
|
79
|
+
filtered: number;
|
|
80
|
+
truncated: boolean;
|
|
81
|
+
}
|
|
82
|
+
export interface ExtractLinksOptions {
|
|
83
|
+
includeInternal: boolean;
|
|
84
|
+
includeExternal: boolean;
|
|
85
|
+
includeImages: boolean;
|
|
86
|
+
maxLinks?: number;
|
|
87
|
+
filterPattern?: RegExp;
|
|
88
|
+
}
|
|
89
|
+
export interface MarkdownTransformResult {
|
|
90
|
+
markdown: string;
|
|
91
|
+
title: string | undefined;
|
|
92
|
+
truncated: boolean;
|
|
93
|
+
}
|
|
94
|
+
export interface TransformOptions {
|
|
95
|
+
extractMainContent: boolean;
|
|
96
|
+
includeMetadata: boolean;
|
|
97
|
+
maxContentLength?: number;
|
|
98
|
+
}
|
|
99
|
+
export interface JsonlTransformResult {
|
|
100
|
+
content: string;
|
|
101
|
+
contentBlocks: number;
|
|
102
|
+
title: string | undefined;
|
|
103
|
+
truncated?: boolean;
|
|
104
|
+
}
|
|
105
|
+
export type LinkType = 'internal' | 'external' | 'image';
|
|
106
|
+
export {};
|
|
107
|
+
//# sourceMappingURL=content.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":"AAAA,KAAK,gBAAgB,GACjB,UAAU,GACV,SAAS,GACT,WAAW,GACX,MAAM,GACN,MAAM,GACN,OAAO,GACP,OAAO,GACP,YAAY,CAAC;AAEjB,UAAU,YAAY;IACpB,IAAI,EAAE,gBAAgB,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,iBAAiB,GACzB,aAAa,GACb,YAAY,GACZ,cAAc,GACd,SAAS,GACT,SAAS,GACT,UAAU,GACV,UAAU,GACV,eAAe,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,MAAM,gBAAgB,GACxB,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,GAAG,GACH,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,MAAM,GACN,OAAO,GACP,KAAK,GACL,YAAY,CAAC;AAEjB,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content.js","sourceRoot":"","sources":["../../../src/config/types/content.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
2
|
+
export interface RateLimitEntry {
|
|
3
|
+
count: number;
|
|
4
|
+
resetTime: number;
|
|
5
|
+
lastAccessed: number;
|
|
6
|
+
}
|
|
7
|
+
export interface RateLimiterOptions {
|
|
8
|
+
maxRequests: number;
|
|
9
|
+
windowMs: number;
|
|
10
|
+
cleanupIntervalMs: number;
|
|
11
|
+
}
|
|
12
|
+
export type ToolContentBlock = {
|
|
13
|
+
type: 'text';
|
|
14
|
+
text: string;
|
|
15
|
+
} | {
|
|
16
|
+
type: 'resource_link';
|
|
17
|
+
uri: string;
|
|
18
|
+
name: string;
|
|
19
|
+
title?: string;
|
|
20
|
+
description?: string;
|
|
21
|
+
mimeType?: string;
|
|
22
|
+
};
|
|
23
|
+
export interface FetchOptions {
|
|
24
|
+
customHeaders?: Record<string, string>;
|
|
25
|
+
signal?: AbortSignal;
|
|
26
|
+
timeout?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface ContentTransformOptions {
|
|
29
|
+
readonly extractMainContent: boolean;
|
|
30
|
+
readonly includeMetadata: boolean;
|
|
31
|
+
}
|
|
32
|
+
export interface TruncationResult {
|
|
33
|
+
readonly content: string;
|
|
34
|
+
readonly truncated: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface SessionEntry {
|
|
37
|
+
readonly transport: StreamableHTTPServerTransport;
|
|
38
|
+
createdAt: number;
|
|
39
|
+
lastSeen: number;
|
|
40
|
+
}
|
|
41
|
+
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
42
|
+
export type LogMetadata = Record<string, unknown>;
|
|
43
|
+
export interface McpRequestBody {
|
|
44
|
+
method?: string;
|
|
45
|
+
id?: string | number;
|
|
46
|
+
jsonrpc?: '2.0';
|
|
47
|
+
params?: unknown;
|
|
48
|
+
}
|
|
49
|
+
export interface FetchPipelineOptions<T> {
|
|
50
|
+
/** URL to fetch */
|
|
51
|
+
url: string;
|
|
52
|
+
/** Cache namespace (e.g., 'url', 'links', 'markdown') */
|
|
53
|
+
cacheNamespace: string;
|
|
54
|
+
/** Optional custom HTTP headers */
|
|
55
|
+
customHeaders?: Record<string, string>;
|
|
56
|
+
/** Optional: number of retry attempts (1-10, defaults to 3) */
|
|
57
|
+
retries?: number;
|
|
58
|
+
/** Optional: AbortSignal for request cancellation */
|
|
59
|
+
signal?: AbortSignal;
|
|
60
|
+
/** Optional: per-request timeout override in milliseconds */
|
|
61
|
+
timeout?: number;
|
|
62
|
+
/** Optional: cache variation input for headers/flags */
|
|
63
|
+
cacheVary?: Record<string, unknown> | string;
|
|
64
|
+
/** Transform function to process HTML into desired format */
|
|
65
|
+
transform: (html: string, url: string) => T;
|
|
66
|
+
/** Optional: serialize result for caching (defaults to JSON.stringify) */
|
|
67
|
+
serialize?: (result: T) => string;
|
|
68
|
+
/** Optional: deserialize cached content */
|
|
69
|
+
deserialize?: (cached: string) => T | undefined;
|
|
70
|
+
}
|
|
71
|
+
export interface PipelineResult<T> {
|
|
72
|
+
data: T;
|
|
73
|
+
fromCache: boolean;
|
|
74
|
+
url: string;
|
|
75
|
+
fetchedAt: string;
|
|
76
|
+
cacheKey?: string | null;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAExG,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAGN,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,6BAA6B,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAGD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAC3D,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAGlD,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAGD,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,6DAA6D;IAC7D,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAC5C,0EAA0E;IAC1E,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../src/config/types/runtime.ts"],"names":[],"mappings":""}
|