@j0hanz/superfetch 1.2.1 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +60 -45
  2. package/dist/config/formatting.d.ts +1 -1
  3. package/dist/config/types/content.d.ts +3 -3
  4. package/dist/config/types/runtime.d.ts +1 -1
  5. package/dist/config/types/tools.d.ts +12 -12
  6. package/dist/http/cors.js +23 -23
  7. package/dist/http/download-routes.js +11 -5
  8. package/dist/http/mcp-routes.js +2 -13
  9. package/dist/http/mcp-validation.js +1 -1
  10. package/dist/http/server-middleware.js +5 -3
  11. package/dist/http/server.js +2 -0
  12. package/dist/index.js +5 -0
  13. package/dist/middleware/error-handler.js +1 -1
  14. package/dist/resources/cached-content.js +8 -4
  15. package/dist/server.js +2 -0
  16. package/dist/services/cache.d.ts +2 -1
  17. package/dist/services/cache.js +23 -7
  18. package/dist/services/context.d.ts +4 -4
  19. package/dist/services/context.js +11 -1
  20. package/dist/services/extractor.js +26 -21
  21. package/dist/services/fetcher/agents.js +55 -1
  22. package/dist/services/fetcher/interceptors.d.ts +22 -0
  23. package/dist/services/fetcher/interceptors.js +57 -26
  24. package/dist/services/fetcher/response.d.ts +1 -1
  25. package/dist/services/fetcher/response.js +37 -16
  26. package/dist/services/fetcher.d.ts +1 -1
  27. package/dist/services/fetcher.js +9 -8
  28. package/dist/services/metadata-collector.d.ts +10 -0
  29. package/dist/services/metadata-collector.js +11 -0
  30. package/dist/services/parser.d.ts +5 -1
  31. package/dist/services/parser.js +82 -11
  32. package/dist/services/transform-worker-pool.d.ts +14 -0
  33. package/dist/services/transform-worker-pool.js +167 -0
  34. package/dist/tools/handlers/fetch-markdown.tool.d.ts +9 -1
  35. package/dist/tools/handlers/fetch-markdown.tool.js +58 -30
  36. package/dist/tools/handlers/fetch-single.shared.d.ts +8 -3
  37. package/dist/tools/handlers/fetch-single.shared.js +42 -17
  38. package/dist/tools/handlers/fetch-url.tool.js +46 -16
  39. package/dist/tools/index.js +13 -0
  40. package/dist/tools/schemas.d.ts +19 -16
  41. package/dist/tools/schemas.js +25 -4
  42. package/dist/tools/utils/common.js +20 -16
  43. package/dist/tools/utils/content-transform-async.d.ts +6 -0
  44. package/dist/tools/utils/content-transform-async.js +33 -0
  45. package/dist/tools/utils/content-transform.d.ts +4 -1
  46. package/dist/tools/utils/content-transform.js +37 -3
  47. package/dist/tools/utils/fetch-pipeline.js +26 -15
  48. package/dist/utils/content-cleaner.d.ts +1 -1
  49. package/dist/utils/download-url.d.ts +9 -1
  50. package/dist/utils/download-url.js +9 -6
  51. package/dist/utils/tool-error-handler.d.ts +2 -2
  52. package/dist/utils/tool-error-handler.js +7 -7
  53. package/dist/utils/url-validator.d.ts +5 -0
  54. package/dist/utils/url-validator.js +45 -3
  55. package/dist/workers/transform-worker.d.ts +1 -0
  56. package/dist/workers/transform-worker.js +50 -0
  57. package/package.json +4 -6
@@ -0,0 +1,50 @@
1
+ import { parentPort } from 'node:worker_threads';
2
+ import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from '../tools/utils/content-transform.js';
3
+ function isTransformJob(value) {
4
+ if (!value || typeof value !== 'object')
5
+ return false;
6
+ const record = value;
7
+ return (typeof record.id === 'number' &&
8
+ typeof record.mode === 'string' &&
9
+ typeof record.html === 'string' &&
10
+ typeof record.url === 'string');
11
+ }
12
+ function resolveTransform(job) {
13
+ if (job.mode === 'markdown') {
14
+ return transformHtmlToMarkdown(job.html, job.url, job.options);
15
+ }
16
+ if (job.mode === 'markdown-blocks') {
17
+ return transformHtmlToMarkdownWithBlocks(job.html, job.url, {
18
+ ...job.options,
19
+ includeContentBlocks: job.options.includeContentBlocks ?? true,
20
+ });
21
+ }
22
+ return transformHtmlToJsonl(job.html, job.url, job.options);
23
+ }
24
+ function sendResponse(response) {
25
+ if (!parentPort)
26
+ return;
27
+ parentPort.postMessage(response);
28
+ }
29
+ function handleMessage(message) {
30
+ if (!isTransformJob(message)) {
31
+ sendResponse({
32
+ id: -1,
33
+ ok: false,
34
+ error: 'Invalid transform job payload',
35
+ });
36
+ return;
37
+ }
38
+ try {
39
+ const result = resolveTransform(message);
40
+ sendResponse({ id: message.id, ok: true, result });
41
+ }
42
+ catch (error) {
43
+ sendResponse({
44
+ id: message.id,
45
+ ok: false,
46
+ error: error instanceof Error ? error.message : String(error),
47
+ });
48
+ }
49
+ }
50
+ parentPort?.on('message', handleMessage);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@j0hanz/superfetch",
3
- "version": "1.2.1",
3
+ "version": "1.2.3",
4
4
  "mcpName": "io.github.j0hanz/superfetch",
5
5
  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
6
6
  "type": "module",
@@ -44,8 +44,8 @@
44
44
  "type-check": "tsc --noEmit",
45
45
  "lint": "eslint .",
46
46
  "lint:fix": "eslint . --fix",
47
- "test": "vitest run",
48
- "test:coverage": "vitest run --coverage",
47
+ "test": "npm run build --silent && node --test --experimental-transform-types",
48
+ "test:coverage": "npm run build --silent && node --test --experimental-transform-types --experimental-test-coverage",
49
49
  "bench": "npm run build && node scripts/bench.mjs",
50
50
  "knip": "knip",
51
51
  "knip:fix": "knip --fix"
@@ -67,7 +67,6 @@
67
67
  "@types/express": "^5.0.6",
68
68
  "@types/node": "^22.19.3",
69
69
  "@types/turndown": "^5.0.6",
70
- "@vitest/coverage-v8": "^2.1.9",
71
70
  "eslint": "^9.23.2",
72
71
  "eslint-config-prettier": "^10.1.8",
73
72
  "eslint-plugin-unused-imports": "^4.3.0",
@@ -76,8 +75,7 @@
76
75
  "shx": "^0.4.0",
77
76
  "tsx": "^4.21.0",
78
77
  "typescript": "^5.9.3",
79
- "typescript-eslint": "^8.51.0",
80
- "vitest": "^2.1.9"
78
+ "typescript-eslint": "^8.51.0"
81
79
  },
82
80
  "engines": {
83
81
  "node": ">=20.12.0"