@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/workers/transform-worker.js

@@ -1,50 +1,94 @@
 import { parentPort } from 'node:worker_threads';
-import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from '../tools/utils/content-transform.js';
-function isTransformJob(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const record = value;
-    return (typeof record.id === 'number' &&
-        typeof record.mode === 'string' &&
-        typeof record.html === 'string' &&
-        typeof record.url === 'string');
+import { FetchError, getErrorMessage } from '../errors.js';
+import { transformHtmlToMarkdownInProcess } from '../transform.js';
+const controllers = new Map();
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
 }
-function resolveTransform(job) {
-    if (job.mode === 'markdown') {
-        return transformHtmlToMarkdown(job.html, job.url, job.options);
+function post(message) {
+    parentPort?.postMessage(message);
+}
+function handleTransform(message) {
+    const controller = new AbortController();
+    controllers.set(message.id, controller);
+    try {
+        const result = transformHtmlToMarkdownInProcess(message.html, message.url, {
+            includeMetadata: message.includeMetadata,
+            signal: controller.signal,
+        });
+        post({
+            type: 'result',
+            id: message.id,
+            result: {
+                markdown: result.markdown,
+                ...(result.title === undefined ? {} : { title: result.title }),
+                truncated: result.truncated,
+            },
+        });
     }
-    if (job.mode === 'markdown-blocks') {
-        return transformHtmlToMarkdownWithBlocks(job.html, job.url, {
-            ...job.options,
-            includeContentBlocks: job.options.includeContentBlocks ?? true,
+    catch (error) {
+        if (error instanceof FetchError) {
+            post({
+                type: 'error',
+                id: message.id,
+                error: {
+                    name: error.name,
+                    message: error.message,
+                    url: error.url,
+                    statusCode: error.statusCode,
+                    details: { ...error.details },
+                },
+            });
+            return;
+        }
+        post({
+            type: 'error',
+            id: message.id,
+            error: {
+                name: error instanceof Error ? error.name : 'Error',
+                message: getErrorMessage(error),
+                url: message.url,
+            },
         });
     }
-    return transformHtmlToJsonl(job.html, job.url, job.options);
+    finally {
+        controllers.delete(message.id);
+    }
 }
-function sendResponse(response) {
-    if (!parentPort)
+function handleCancel(message) {
+    const controller = controllers.get(message.id);
+    if (!controller)
         return;
-    parentPort.postMessage(response);
+    controller.abort(new Error('Canceled'));
 }
-function handleMessage(message) {
-    if (!isTransformJob(message)) {
-        sendResponse({
-            id: -1,
-            ok: false,
-            error: 'Invalid transform job payload',
-        });
+if (!parentPort) {
+    throw new Error('transform-worker started without parentPort');
+}
+parentPort.on('message', (raw) => {
+    if (!isRecord(raw))
+        return;
+    const { type } = raw;
+    if (type === 'cancel') {
+        if (typeof raw.id !== 'string')
+            return;
+        handleCancel({ type: 'cancel', id: raw.id });
         return;
     }
-    try {
-        const result = resolveTransform(message);
-        sendResponse({ id: message.id, ok: true, result });
-    }
-    catch (error) {
-        sendResponse({
-            id: message.id,
-            ok: false,
-            error: error instanceof Error ? error.message : String(error),
+    if (type === 'transform') {
+        if (typeof raw.id !== 'string')
+            return;
+        if (typeof raw.html !== 'string')
+            return;
+        if (typeof raw.url !== 'string')
+            return;
+        if (typeof raw.includeMetadata !== 'boolean')
+            return;
+        handleTransform({
+            type: 'transform',
+            id: raw.id,
+            html: raw.html,
+            url: raw.url,
+            includeMetadata: raw.includeMetadata,
         });
     }
-}
-parentPort?.on('message', handleMessage);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",
@@ -40,6 +40,8 @@
     "start": "node dist/index.js",
     "format": "prettier --write .",
     "type-check": "tsc --noEmit",
+    "type-check:diagnostics": "tsc --noEmit --extendedDiagnostics",
+    "type-check:trace": "node -e \"require('fs').rmSync('.ts-trace',{recursive:true,force:true})\" && tsc --noEmit --generateTrace .ts-trace",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "test": "npm run build --silent && node --test --experimental-transform-types",
@@ -50,31 +52,32 @@
     "prepublishOnly": "npm run lint && npm run type-check && npm run build"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.25.1",
+    "@modelcontextprotocol/sdk": "^1.25.2",
     "@mozilla/readability": "^0.6.0",
     "express": "^5.2.1",
     "linkedom": "^0.18.12",
-    "turndown": "^7.2.2",
-    "undici": "^6.23.0",
+    "node-html-markdown": "^2.0.0",
+    "undici": "^7.18.2",
     "zod": "^4.3.5"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.2",
-    "@trivago/prettier-plugin-sort-imports": "^6.0.1",
+    "@trivago/prettier-plugin-sort-imports": "^6.0.2",
     "@types/express": "^5.0.6",
-    "@types/node": "^22.19.3",
-    "@types/turndown": "^5.0.6",
+    "@types/node": "^22.19.5",
     "eslint": "^9.23.2",
     "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-de-morgan": "^2.0.0",
+    "eslint-plugin-depend": "^1.4.0",
+    "eslint-plugin-sonarjs": "^3.0.5",
     "eslint-plugin-unused-imports": "^4.3.0",
-    "knip": "^5.80.0",
+    "knip": "^5.80.2",
     "prettier": "^3.7.4",
-    "shx": "^0.4.0",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.52.0"
   },
   "engines": {
-    "node": ">=20.12.0"
+    "node": ">=20.18.1"
   }
 }