@nanocollective/get-md 1.0.0

package/dist/utils/url-fetcher.js

@@ -0,0 +1,54 @@
+// src/utils/url-fetcher.ts
+import { DEFAULT_USER_AGENT, DEFAULT_FETCH_TIMEOUT } from "../config.js";
+/**
+ * Fetch HTML from a URL with timeout and redirect handling
+ */
+export async function fetchUrl(url, options = {}) {
+    const timeout = options.timeout ?? DEFAULT_FETCH_TIMEOUT;
+    const userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
+    const followRedirects = options.followRedirects ?? true;
+    const headers = {
+        "User-Agent": userAgent,
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate, br",
+        ...options.headers,
+    };
+    try {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeout);
+        const response = await fetch(url, {
+            headers,
+            signal: controller.signal,
+            redirect: followRedirects ? "follow" : "manual",
+        });
+        clearTimeout(timeoutId);
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+        const html = await response.text();
+        return html;
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            if (error.name === "AbortError") {
+                throw new Error(`Request timeout after ${timeout}ms`);
+            }
+            throw new Error(`Failed to fetch URL: ${error.message}`);
+        }
+        throw error;
+    }
+}
+/**
+ * Validate if a string is a valid URL
+ */
+export function isValidUrl(urlString) {
+    try {
+        const url = new URL(urlString);
+        return url.protocol === "http:" || url.protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=url-fetcher.js.map

package/dist/utils/url-fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-fetcher.js","sourceRoot":"","sources":["../../src/utils/url-fetcher.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAG3B,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAEzE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,UAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,qBAAqB,CAAC;IACzD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC;IAExD,MAAM,OAAO,GAA2B;QACtC,YAAY,EAAE,SAAS;QACvB,MAAM,EAAE,iEAAiE;QACzE,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,GAAG,OAAO,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QAEhE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO;YACP,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;SAChD,CAAC,CAAC;QAEH,YAAY,CAAC,SAAS,CAAC,CAAC;QAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,yBAAyB,OAAO,IAAI,CAAC,CAAC;YACxD,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/utils/validators.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Validate if HTML contains extractable content
+ */
+export declare function hasContent(html: string): boolean;
+//# sourceMappingURL=validators.d.ts.map

package/dist/utils/validators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../../src/utils/validators.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAmBhD"}

package/dist/utils/validators.js

@@ -0,0 +1,23 @@
+// src/utils/validators.ts
+import * as cheerio from "cheerio";
+/**
+ * Validate if HTML contains extractable content
+ */
+export function hasContent(html) {
+    if (!html || typeof html !== "string") {
+        return false;
+    }
+    try {
+        const $ = cheerio.load(html);
+        // Remove scripts, styles, and common noise
+        $("script, style, nav, header, footer").remove();
+        // Get text content
+        const text = $("body").text().trim();
+        // Consider it has content if there's at least 100 characters of text
+        return text.length >= 100;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=validators.js.map

package/dist/utils/validators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validators.js","sourceRoot":"","sources":["../../src/utils/validators.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAE1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,2CAA2C;QAC3C,CAAC,CAAC,oCAAoC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEjD,mBAAmB;QACnB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAErC,qEAAqE;QACrE,OAAO,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,104 @@
+{
+  "name": "@nanocollective/get-md",
+  "version": "1.0.0",
+  "description": "Fast HTML to Markdown converter optimized for LLM consumption",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "getmd": "./bin/get-md.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "bin",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsc && chmod +x bin/get-md.js",
+    "dev": "tsc --watch",
+    "test:all": "./scripts/test.sh",
+    "test:ava": "ava",
+    "test:ava:coverage": "c8 ava",
+    "test:format": "prettier --check .",
+    "test:types": "tsc --noEmit",
+    "test:lint": "eslint .",
+    "test:lint:fix": "eslint . --fix",
+    "test:knip": "knip",
+    "format": "prettier --write .",
+    "prepublishOnly": "pnpm run build && pnpm run test:all"
+  },
+  "keywords": [
+    "html",
+    "markdown",
+    "json",
+    "converter",
+    "parser",
+    "llm",
+    "readability",
+    "turndown",
+    "cheerio",
+    "fast",
+    "local"
+  ],
+  "dependencies": {
+    "@mozilla/readability": "^0.6.0",
+    "ajv": "^8.17.1",
+    "cheerio": "^1.1.2",
+    "commander": "^14.0.2",
+    "jsdom": "^24.1.3",
+    "turndown": "^7.2.2",
+    "turndown-plugin-gfm": "^1.0.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/nano-collective/get-md.git"
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "@ava/typescript": "^6.0.0",
+    "@eslint/js": "^9.38.0",
+    "@types/jsdom": "^27.0.0",
+    "@types/node": "^24.9.1",
+    "@types/turndown": "^5.0.6",
+    "@typescript-eslint/eslint-plugin": "^8.46.2",
+    "@typescript-eslint/parser": "^8.46.2",
+    "ava": "^6.4.1",
+    "c8": "^10.1.3",
+    "eslint": "^9.38.0",
+    "globals": "^16.4.0",
+    "knip": "^5.66.3",
+    "prettier": "^3.6.2",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.46.2"
+  },
+  "ava": {
+    "extensions": {
+      "ts": "module"
+    },
+    "nodeArguments": [
+      "--import=tsx",
+      "--no-warnings"
+    ],
+    "files": [
+      "src/**/*.spec.ts",
+      "!src/**/*-helpers.ts",
+      "!src/**/test-helpers.ts"
+    ],
+    "workerThreads": false,
+    "serial": true,
+    "environmentVariables": {
+      "NODE_ENV": "test"
+    }
+  }
+}