@lyhue1991/webfetch-cli 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,118 @@
+# webfetch-cli
+
+网页内容抓取工具，支持将网页转换为 Markdown、文本或 HTML 格式。可作为 CLI 工具或 OpenCode SKILL 使用。
+
+## 安装
+
+### 方式一：作为 SKILL 安装（推荐）
+
+```bash
+npx skills add lyhue1991/webfetch-cli
+```
+
+### 方式二：作为 CLI 工具安装
+
+```bash
+# 使用 npx 直接运行
+npx @lyhue1991/webfetch-cli https://example.com
+
+# 或全局安装
+npm install -g @lyhue1991/webfetch-cli
+webfetch https://example.com
+```
+
+## 使用方法
+
+```bash
+webfetch <url> [options]
+```
+
+### 参数说明
+
+| 参数 | 简写 | 说明 | 默认值 |
+|------|------|------|--------|
+| `--format` | `-f` | 输出格式：markdown, text, html | markdown |
+| `--timeout` | `-t` | 超时时间（秒），最大 120 | 30 |
+| `--output` | `-o` | 保存到指定文件 | - |
+| `--quiet` | `-q` | 静默模式，仅输出内容 | false |
+| `--proxy` | | 代理服务器地址 | - |
+| `--insecure` | | 跳过 TLS 证书验证 | false |
+| `--help` | `-h` | 显示帮助 | - |
+| `--version` | `-v` | 显示版本 | - |
+
+### 示例
+
+```bash
+# 抓取并转换为 Markdown（默认）
+webfetch https://example.com
+
+# 输出纯文本
+webfetch https://example.com -f text
+
+# 保存到文件
+webfetch https://example.com -o output.md
+
+# 静默模式（适合管道）
+webfetch https://example.com -q > content.md
+
+# 自定义超时
+webfetch https://example.com --timeout 60
+
+# 使用代理
+webfetch https://example.com --proxy http://proxy:8080
+
+# 跳过证书验证（自签名证书场景）
+webfetch https://example.com --insecure
+
+# 抓取微信公众号文章
+webfetch "https://mp.weixin.qq.com/s/xxx" -o article.md --insecure
+```
+
+## 代理支持
+
+webfetch 自动读取环境变量代理配置：
+
+- `HTTP_PROXY` / `http_proxy` - HTTP 请求代理
+- `HTTPS_PROXY` / `https_proxy` - HTTPS 请求代理
+- `NO_PROXY` / `no_proxy` - 跳过代理的主机列表
+
+```bash
+# 通过环境变量设置代理
+export HTTPS_PROXY=http://proxy.example.com:8080
+webfetch https://example.com
+
+# 命令行覆盖代理
+webfetch https://example.com --proxy http://custom-proxy:8080
+
+# 禁用本次请求的代理
+webfetch https://example.com --proxy ""
+
+# 处理代理证书问题
+webfetch https://example.com --insecure
+```
+
+## 功能特性
+
+- **多格式输出** - 转换 HTML 为 Markdown（默认）、纯文本或保留原始 HTML
+- **智能内容处理** - 自动检测 Content-Type，正确处理非 HTML 响应
+- **反爬虫策略** - Chrome UA 伪装，Cloudflare 挑战自动重试
+- **安全限制** - 最大 5MB 响应限制，可配置超时
+- **清洁输出** - 移除 script、style 等非内容元素
+- **代理支持** - 自动环境变量检测，支持手动覆盖
+
+## 退出码
+
+| 代码 | 说明 |
+|------|------|
+| 0 | 成功 |
+| 1 | 用户错误（无效 URL、参数错误） |
+| 2 | 网络错误（超时、DNS 解析失败） |
+| 3 | 服务器错误（4xx、5xx 响应） |
+
+## 系统要求
+
+- Node.js >= 18.0.0
+
+## 许可证
+
+MIT

package/dist/fetch.d.ts

@@ -0,0 +1,4 @@
+import { WebfetchOptions, WebfetchResult } from './types.js';
+export declare function validateUrl(url: string): void;
+export declare function webfetch(options: WebfetchOptions): Promise<WebfetchResult>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,cAAc,EAGf,MAAM,YAAY,CAAA;AAoBnB,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAI7C;AAED,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CA+FhF"}

package/dist/fetch.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateUrl = validateUrl;
+exports.webfetch = webfetch;
+const undici_1 = require("undici");
+const types_js_1 = require("./types.js");
+const proxy_js_1 = require("./proxy.js");
+const CHROME_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36';
+const HONEST_USER_AGENT = 'webfetch-cli';
+function buildAcceptHeader(format) {
+    switch (format) {
+        case 'markdown':
+            return 'text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1';
+        case 'text':
+            return 'text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1';
+        case 'html':
+            return 'text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1';
+        default:
+            return 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
+    }
+}
+function validateUrl(url) {
+    if (!url.startsWith('http://') && !url.startsWith('https://')) {
+        throw new Error('URL must start with http:// or https://');
+    }
+}
+async function webfetch(options) {
+    const { url, format, timeout, proxy: manualProxy, insecure } = options;
+    validateUrl(url);
+    const actualTimeout = Math.min(timeout, types_js_1.MAX_TIMEOUT) * 1000;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), actualTimeout);
+    // Configure proxy
+    const envProxyConfig = (0, proxy_js_1.detectProxyConfig)(url);
+    const proxyConfig = (0, proxy_js_1.mergeProxyConfig)(envProxyConfig, manualProxy, insecure);
+    const dispatcher = (0, proxy_js_1.createProxyAgent)(proxyConfig, url);
+    const headers = {
+        'User-Agent': CHROME_USER_AGENT,
+        Accept: buildAcceptHeader(format),
+        'Accept-Language': 'en-US,en;q=0.9',
+    };
+    let response;
+    try {
+        response = await (0, undici_1.fetch)(url, {
+            headers,
+            signal: controller.signal,
+            dispatcher,
+        });
+    }
+    catch (error) {
+        clearTimeout(timeoutId);
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw new Error(`Request timed out after ${timeout} seconds`);
+        }
+        if (error instanceof Error) {
+            if (error.message.includes('getaddrinfo') || error.message.includes('ENOTFOUND')) {
+                const urlObj = new URL(url);
+                throw new Error(`Failed to resolve host: ${urlObj.hostname}`);
+            }
+            if (error.message.includes('ECONNREFUSED')) {
+                throw new Error(`Connection refused: ${url}`);
+            }
+            if (error.message.includes('certificate') || error.message.includes('SSL') || error.message.includes('TLS')) {
+                throw new Error(`SSL/TLS certificate error. Try using --insecure flag to skip certificate verification`);
+            }
+            if (error.message.includes('Proxy')) {
+                throw new Error(`Proxy error: ${error.message}`);
+            }
+            throw new Error(`Network error: ${error.message}`);
+        }
+        throw error;
+    }
+    clearTimeout(timeoutId);
+    // Cloudflare challenge detection - retry with honest UA
+    if (response.status === 403 && response.headers.get('cf-mitigated') === 'challenge') {
+        const retryController = new AbortController();
+        const retryTimeoutId = setTimeout(() => retryController.abort(), actualTimeout);
+        headers['User-Agent'] = HONEST_USER_AGENT;
+        try {
+            response = await (0, undici_1.fetch)(url, {
+                headers,
+                signal: retryController.signal,
+                dispatcher,
+            });
+        }
+        catch (error) {
+            clearTimeout(retryTimeoutId);
+            throw error;
+        }
+        clearTimeout(retryTimeoutId);
+    }
+    if (!response.ok) {
+        throw new Error(`Request failed with status code: ${response.status}`);
+    }
+    const contentLength = response.headers.get('content-length');
+    if (contentLength && parseInt(contentLength) > types_js_1.MAX_RESPONSE_SIZE) {
+        throw new Error('Response too large (exceeds 5MB limit)');
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    if (arrayBuffer.byteLength > types_js_1.MAX_RESPONSE_SIZE) {
+        throw new Error('Response too large (exceeds 5MB limit)');
+    }
+    const contentType = response.headers.get('content-type') || 'text/plain';
+    return {
+        content: new TextDecoder().decode(arrayBuffer),
+        contentType,
+        url,
+    };
+}
+//# sourceMappingURL=fetch.js.map

package/dist/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":";;AA0BA,kCAIC;AAED,4BA+FC;AA/HD,mCAA6C;AAC7C,yCAKmB;AACnB,yCAAkF;AAElF,MAAM,iBAAiB,GACrB,iHAAiH,CAAA;AACnH,MAAM,iBAAiB,GAAG,cAAc,CAAA;AAExC,SAAS,iBAAiB,CAAC,MAAc;IACvC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU;YACb,OAAO,0FAA0F,CAAA;QACnG,KAAK,MAAM;YACT,OAAO,mEAAmE,CAAA;QAC5E,KAAK,MAAM;YACT,OAAO,gGAAgG,CAAA;QACzG;YACE,OAAO,iEAAiE,CAAA;IAC5E,CAAC;AACH,CAAC;AAED,SAAgB,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,QAAQ,CAAC,OAAwB;IACrD,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAA;IAEtE,WAAW,CAAC,GAAG,CAAC,CAAA;IAEhB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,sBAAW,CAAC,GAAG,IAAI,CAAA;IAC3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,aAAa,CAAC,CAAA;IAErE,kBAAkB;IAClB,MAAM,cAAc,GAAG,IAAA,4BAAiB,EAAC,GAAG,CAAC,CAAA;IAC7C,MAAM,WAAW,GAAG,IAAA,2BAAgB,EAAC,cAAc,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAA;IAC3E,MAAM,UAAU,GAAG,IAAA,2BAAgB,EAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAErD,MAAM,OAAO,GAA2B;QACtC,YAAY,EAAE,iBAAiB;QAC/B,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,iBAAiB,EAAE,gBAAgB;KACpC,CAAA;IAED,IAAI,QAAkB,CAAA;IACtB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,IAAA,cAAW,EAAC,GAAG,EAAE;YAChC,OAAO;YACP,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,UAAU;SACX,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,YAAY,CAAC,SAAS,CAAC,CAAA;QACvB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,2BAA2B,OAAO,UAAU,CAAC,CAAA;QAC/D,CAAC;QACD,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACjF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;gBAC3B,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;YAC/D,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;YAC/C,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5G,MAAM,IAAI,KAAK,CACb,uFAAuF,CACxF,CAAA;YACH,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;YAClD,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;QACpD,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,YAAY,CAAC,SAAS,CAAC,CAAA;IAEvB,wDAAwD;IACxD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,WAAW,EAAE,CAAC;QACpF,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;QAC7C,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,aAAa,CAAC,CAAA;QAE/E,OAAO,CAAC,YAAY,CAAC,GAAG,iBAAiB,CAAA;QACzC,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAA,cAAW,EAAC,GAAG,EAAE;gBAChC,OAAO;gBACP,MAAM,EAAE,eAAe,CAAC,MAAM;gBAC9B,UAAU;aACX,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,cAAc,CAAC,CAAA;YAC5B,MAAM,KAAK,CAAA;QACb,CAAC;QACD,YAAY,CAAC,cAAc,CAAC,CAAA;IAC9B,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;IACxE,CAAC;IAED,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;IAC5D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,CAAC,GAAG,4BAAiB,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAA;IAChD,IAAI,WAAW,CAAC,UAAU,GAAG,4BAAiB,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,YAAY,CAAA;IAExE,OAAO;QACL,OAAO,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC;QAC9C,WAAW;QACX,GAAG;KACJ,CAAA;AACH,CAAC"}

package/dist/format.d.ts

@@ -0,0 +1,6 @@
+import { OutputFormat, WebfetchResult } from './types.js';
+export declare function htmlToMarkdown(html: string): string;
+export declare function htmlToText(html: string): string;
+export declare function shouldConvert(contentType: string): boolean;
+export declare function convertResult(result: WebfetchResult, format: OutputFormat): string;
+//# sourceMappingURL=format.d.ts.map

package/dist/format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAczD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ/C;AAED,wBAAgB,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAG1D;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAkBlF"}

package/dist/format.js

@@ -0,0 +1,90 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.htmlToMarkdown = htmlToMarkdown;
+exports.htmlToText = htmlToText;
+exports.shouldConvert = shouldConvert;
+exports.convertResult = convertResult;
+const turndown_1 = __importDefault(require("turndown"));
+const cheerio = __importStar(require("cheerio"));
+function createTurndownService() {
+    const turndownService = new turndown_1.default({
+        headingStyle: 'atx',
+        hr: '---',
+        bulletListMarker: '-',
+        codeBlockStyle: 'fenced',
+        emDelimiter: '*',
+    });
+    turndownService.remove(['script', 'style', 'meta', 'link']);
+    return turndownService;
+}
+function htmlToMarkdown(html) {
+    const turndownService = createTurndownService();
+    return turndownService.turndown(html);
+}
+function htmlToText(html) {
+    const $ = cheerio.load(html);
+    // Remove script and style elements
+    $('script, style, noscript, iframe, object, embed').remove();
+    // Get text from body, normalize whitespace
+    const text = $('body').text();
+    // Normalize whitespace: collapse multiple spaces/newlines
+    return text.replace(/\s+/g, ' ').trim();
+}
+function shouldConvert(contentType) {
+    const mime = contentType.split(';')[0]?.trim().toLowerCase() || '';
+    return mime.includes('text/html') || mime.includes('application/xhtml+xml');
+}
+function convertResult(result, format) {
+    const { content, contentType } = result;
+    // If not HTML, return as-is
+    if (!shouldConvert(contentType)) {
+        return content;
+    }
+    switch (format) {
+        case 'markdown':
+            return htmlToMarkdown(content);
+        case 'text':
+            return htmlToText(content);
+        case 'html':
+            return content;
+        default:
+            return content;
+    }
+}
+//# sourceMappingURL=format.js.map

package/dist/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgBA,wCAGC;AAED,gCAQC;AAED,sCAGC;AAED,sCAkBC;AAtDD,wDAAsC;AACtC,iDAAkC;AAGlC,SAAS,qBAAqB;IAC5B,MAAM,eAAe,GAAG,IAAI,kBAAe,CAAC;QAC1C,YAAY,EAAE,KAAK;QACnB,EAAE,EAAE,KAAK;QACT,gBAAgB,EAAE,GAAG;QACrB,cAAc,EAAE,QAAQ;QACxB,WAAW,EAAE,GAAG;KACjB,CAAC,CAAA;IACF,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC3D,OAAO,eAAe,CAAA;AACxB,CAAC;AAED,SAAgB,cAAc,CAAC,IAAY;IACzC,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAA;IAC/C,OAAO,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AACvC,CAAC;AAED,SAAgB,UAAU,CAAC,IAAY;IACrC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,mCAAmC;IACnC,CAAC,CAAC,gDAAgD,CAAC,CAAC,MAAM,EAAE,CAAA;IAC5D,2CAA2C;IAC3C,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;IAC7B,0DAA0D;IAC1D,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACzC,CAAC;AAED,SAAgB,aAAa,CAAC,WAAmB;IAC/C,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAA;IAClE,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAA;AAC7E,CAAC;AAED,SAAgB,aAAa,CAAC,MAAsB,EAAE,MAAoB;IACxE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,CAAA;IAEvC,4BAA4B;IAC5B,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QAChC,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU;YACb,OAAO,cAAc,CAAC,OAAO,CAAC,CAAA;QAChC,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,MAAM;YACT,OAAO,OAAO,CAAA;QAChB;YACE,OAAO,OAAO,CAAA;IAClB,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}

package/dist/index.js

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const zod_1 = require("zod");
+const fetch_js_1 = require("./fetch.js");
+const format_js_1 = require("./format.js");
+const types_js_1 = require("./types.js");
+const fs_1 = require("fs");
+const formatSchema = zod_1.z.enum(['markdown', 'text', 'html']);
+const program = new commander_1.Command()
+    .name('webfetch')
+    .description('Fetch and convert web content to markdown, text, or html')
+    .version('0.1.0')
+    .argument('<url>', 'URL to fetch')
+    .option('-f, --format <format>', 'Output format: markdown, text, html', 'markdown')
+    .option('-t, --timeout <seconds>', 'Timeout in seconds (max 120)', String(types_js_1.DEFAULT_TIMEOUT))
+    .option('-o, --output <file>', 'Write output to file')
+    .option('-q, --quiet', 'Only output content, no metadata', false)
+    .option('--proxy <url>', 'Proxy server URL (e.g., http://proxy:8080)')
+    .option('--insecure', 'Skip TLS certificate verification', false)
+    .addHelpText('after', `
+Examples:
+  $ webfetch https://example.com
+  $ webfetch https://example.com -f text
+  $ webfetch https://example.com -o output.md
+  $ webfetch https://example.com --quiet > output.md
+
+Proxy:
+  $ webfetch https://example.com --proxy http://proxy:8080
+  $ webfetch https://example.com --insecure
+
+Environment variables:
+  HTTP_PROXY, HTTPS_PROXY - Proxy server URL
+  NO_PROXY - Comma-separated hosts to bypass proxy
+`);
+async function main() {
+    try {
+        program.parse(process.argv);
+        const options = program.opts();
+        const url = program.args[0];
+        if (!url) {
+            console.error('Error: URL is required');
+            process.exit(types_js_1.EXIT_CODES.USER_ERROR);
+        }
+        // Validate and parse options
+        const format = formatSchema.parse(options.format);
+        const timeout = Math.min(Math.max(1, parseInt(options.timeout)), 120);
+        const quiet = options.quiet;
+        const outputFile = options.output;
+        const proxy = options.proxy;
+        const insecure = options.insecure;
+        // Fetch the URL
+        const result = await (0, fetch_js_1.webfetch)({ url, format, timeout, proxy, insecure });
+        // Convert to requested format
+        const output = (0, format_js_1.convertResult)(result, format);
+        // Output
+        if (outputFile) {
+            (0, fs_1.writeFileSync)(outputFile, output);
+            if (!quiet) {
+                console.log(`Saved to ${outputFile}`);
+            }
+        }
+        else {
+            if (quiet) {
+                console.log(output);
+            }
+            else {
+                console.log(`URL: ${result.url}`);
+                console.log(`Content-Type: ${result.contentType}`);
+                console.log('---');
+                console.log(output);
+            }
+        }
+        process.exit(types_js_1.EXIT_CODES.SUCCESS);
+    }
+    catch (error) {
+        if (error instanceof zod_1.z.ZodError) {
+            console.error(`Error: Invalid format. Must be one of: markdown, text, html`);
+            process.exit(types_js_1.EXIT_CODES.USER_ERROR);
+        }
+        if (error instanceof Error) {
+            console.error(`Error: ${error.message}`);
+            if (error.cause) {
+                console.error(`Cause: ${error.cause}`);
+            }
+            // Determine exit code based on error type
+            if (error.message.includes('timed out') || error.message.includes('resolve host')) {
+                process.exit(types_js_1.EXIT_CODES.NETWORK_ERROR);
+            }
+            if (error.message.includes('status code')) {
+                process.exit(types_js_1.EXIT_CODES.SERVER_ERROR);
+            }
+            process.exit(types_js_1.EXIT_CODES.USER_ERROR);
+        }
+        console.error('An unexpected error occurred');
+        process.exit(types_js_1.EXIT_CODES.USER_ERROR);
+    }
+}
+main();
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAEA,yCAAmC;AACnC,6BAAuB;AACvB,yCAAqC;AACrC,2CAA2C;AAC3C,yCAImB;AACnB,2BAAkC;AAElC,MAAM,YAAY,GAAG,OAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;AAEzD,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE;KAC1B,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,0DAA0D,CAAC;KACvE,OAAO,CAAC,OAAO,CAAC;KAChB,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;KACjC,MAAM,CAAC,uBAAuB,EAAE,qCAAqC,EAAE,UAAU,CAAC;KAClF,MAAM,CAAC,yBAAyB,EAAE,8BAA8B,EAAE,MAAM,CAAC,0BAAe,CAAC,CAAC;KAC1F,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,CAAC;KACrD,MAAM,CAAC,aAAa,EAAE,kCAAkC,EAAE,KAAK,CAAC;KAChE,MAAM,CAAC,eAAe,EAAE,4CAA4C,CAAC;KACrE,MAAM,CAAC,YAAY,EAAE,mCAAmC,EAAE,KAAK,CAAC;KAChE,WAAW,CACV,OAAO,EACP;;;;;;;;;;;;;;CAcH,CACE,CAAA;AAEH,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC;QACH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;QAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE3B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;YACvC,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,UAAU,CAAC,CAAA;QACrC,CAAC;QAED,6BAA6B;QAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAiB,CAAA;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;QACrE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAgB,CAAA;QACtC,MAAM,UAAU,GAAG,OAAO,CAAC,MAA4B,CAAA;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAA2B,CAAA;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAmB,CAAA;QAE5C,gBAAgB;QAChB,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAQ,EAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;QAExE,8BAA8B;QAC9B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAE5C,SAAS;QACT,IAAI,UAAU,EAAE,CAAC;YACf,IAAA,kBAAa,EAAC,UAAU,EAAE,MAAM,CAAC,CAAA;YACjC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,GAAG,CAAC,YAAY,UAAU,EAAE,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;gBACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;gBAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;gBAClB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACrB,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,OAAO,CAAC,CAAA;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,OAAC,CAAC,QAAQ,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAA;YAC5E,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,UAAU,CAAC,CAAA;QACrC,CAAC;QACD,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;YACxC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;YACxC,CAAC;YACD,0CAA0C;YAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClF,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,aAAa,CAAC,CAAA;YACxC,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,YAAY,CAAC,CAAA;YACvC,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,UAAU,CAAC,CAAA;QACrC,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC7C,OAAO,CAAC,IAAI,CAAC,qBAAU,CAAC,UAAU,CAAC,CAAA;IACrC,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAA"}

package/dist/proxy.d.ts

@@ -0,0 +1,12 @@
+import type { Dispatcher } from 'undici';
+export interface ProxyConfig {
+    proxyUrl: string | null;
+    noProxy: string[];
+    insecure: boolean;
+}
+export declare function getProxyFromUrl(url: string): ProxyConfig;
+export declare function detectProxyConfig(targetUrl: string): ProxyConfig;
+export declare function shouldBypassProxy(targetUrl: string, noProxyList: string[]): boolean;
+export declare function createProxyAgent(config: ProxyConfig, targetUrl: string): Dispatcher | undefined;
+export declare function mergeProxyConfig(envConfig: ProxyConfig, manualProxy?: string, insecure?: boolean): ProxyConfig;
+//# sourceMappingURL=proxy.d.ts.map

package/dist/proxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../src/proxy.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAExC,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,QAAQ,EAAE,OAAO,CAAA;CAClB;AAWD,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,CAYxD;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAgChE;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAqBnF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAoB/F;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,WAAW,EACtB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,CAAC,EAAE,OAAO,GACjB,WAAW,CAMb"}

package/dist/proxy.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getProxyFromUrl = getProxyFromUrl;
+exports.detectProxyConfig = detectProxyConfig;
+exports.shouldBypassProxy = shouldBypassProxy;
+exports.createProxyAgent = createProxyAgent;
+exports.mergeProxyConfig = mergeProxyConfig;
+const undici_1 = require("undici");
+const PROXY_ENV_VARS = [
+    'HTTPS_PROXY',
+    'https_proxy',
+    'HTTP_PROXY',
+    'http_proxy',
+];
+const NO_PROXY_ENV_VARS = ['NO_PROXY', 'no_proxy'];
+function getProxyFromUrl(url) {
+    // Parse proxy URL
+    try {
+        const proxyUrl = new URL(url);
+        return {
+            proxyUrl: url,
+            noProxy: [],
+            insecure: false,
+        };
+    }
+    catch {
+        throw new Error(`Invalid proxy URL: ${url}`);
+    }
+}
+function detectProxyConfig(targetUrl) {
+    // Determine which proxy env var to use based on target URL
+    const isHttps = targetUrl.startsWith('https://');
+    let proxyUrl = null;
+    if (isHttps) {
+        proxyUrl = process.env.HTTPS_PROXY || process.env.https_proxy || null;
+    }
+    // Fallback to HTTP_PROXY if no HTTPS-specific proxy
+    if (!proxyUrl) {
+        proxyUrl =
+            process.env.HTTP_PROXY ||
+                process.env.http_proxy ||
+                process.env.HTTPS_PROXY ||
+                process.env.https_proxy ||
+                null;
+    }
+    // Parse NO_PROXY
+    const noProxyStr = NO_PROXY_ENV_VARS.map((v) => process.env[v]).find(Boolean) || '';
+    const noProxy = noProxyStr
+        .split(',')
+        .map((s) => s.trim())
+        .filter(Boolean);
+    return {
+        proxyUrl,
+        noProxy,
+        insecure: false,
+    };
+}
+function shouldBypassProxy(targetUrl, noProxyList) {
+    if (noProxyList.length === 0)
+        return false;
+    try {
+        const target = new URL(targetUrl);
+        const hostname = target.hostname;
+        for (const pattern of noProxyList) {
+            // Exact match
+            if (hostname === pattern)
+                return true;
+            // Wildcard match (e.g., *.example.com)
+            if (pattern.startsWith('.') && hostname.endsWith(pattern))
+                return true;
+            if (pattern.startsWith('*.') && hostname.endsWith(pattern.slice(1)))
+                return true;
+            // Simple suffix match
+            if (hostname.endsWith('.' + pattern))
+                return true;
+        }
+        return false;
+    }
+    catch {
+        return false;
+    }
+}
+function createProxyAgent(config, targetUrl) {
+    if (!config.proxyUrl)
+        return undefined;
+    if (shouldBypassProxy(targetUrl, config.noProxy)) {
+        return undefined;
+    }
+    return new undici_1.ProxyAgent({
+        uri: config.proxyUrl,
+        requestTls: config.insecure
+            ? {
+                rejectUnauthorized: false,
+            }
+            : undefined,
+        proxyTls: config.insecure
+            ? {
+                rejectUnauthorized: false,
+            }
+            : undefined,
+    });
+}
+function mergeProxyConfig(envConfig, manualProxy, insecure) {
+    return {
+        proxyUrl: manualProxy !== undefined ? (manualProxy || null) : envConfig.proxyUrl,
+        noProxy: envConfig.noProxy,
+        insecure: insecure ?? envConfig.insecure,
+    };
+}
+//# sourceMappingURL=proxy.js.map

package/dist/proxy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy.js","sourceRoot":"","sources":["../src/proxy.ts"],"names":[],"mappings":";;AAkBA,0CAYC;AAED,8CAgCC;AAED,8CAqBC;AAED,4CAoBC;AAED,4CAUC;AAzHD,mCAAmC;AASnC,MAAM,cAAc,GAAG;IACrB,aAAa;IACb,aAAa;IACb,YAAY;IACZ,YAAY;CACb,CAAA;AAED,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;AAElD,SAAgB,eAAe,CAAC,GAAW;IACzC,kBAAkB;IAClB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC7B,OAAO;YACL,QAAQ,EAAE,GAAG;YACb,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,KAAK;SAChB,CAAA;IACH,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAA;IAC9C,CAAC;AACH,CAAC;AAED,SAAgB,iBAAiB,CAAC,SAAiB;IACjD,2DAA2D;IAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;IAEhD,IAAI,QAAQ,GAAkB,IAAI,CAAA;IAElC,IAAI,OAAO,EAAE,CAAC;QACZ,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAA;IACvE,CAAC;IAED,oDAAoD;IACpD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ;YACN,OAAO,CAAC,GAAG,CAAC,UAAU;gBACtB,OAAO,CAAC,GAAG,CAAC,UAAU;gBACtB,OAAO,CAAC,GAAG,CAAC,WAAW;gBACvB,OAAO,CAAC,GAAG,CAAC,WAAW;gBACvB,IAAI,CAAA;IACR,CAAC;IAED,iBAAiB;IACjB,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACnF,MAAM,OAAO,GAAG,UAAU;SACvB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAA;IAElB,OAAO;QACL,QAAQ;QACR,OAAO;QACP,QAAQ,EAAE,KAAK;KAChB,CAAA;AACH,CAAC;AAED,SAAgB,iBAAiB,CAAC,SAAiB,EAAE,WAAqB;IACxE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAE1C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAA;QACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;QAEhC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAClC,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO;gBAAE,OAAO,IAAI,CAAA;YACrC,uCAAuC;YACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAA;YACtE,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAA;YAChF,sBAAsB;YACtB,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAA;QACnD,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,SAAgB,gBAAgB,CAAC,MAAmB,EAAE,SAAiB;IACrE,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAA;IAEtC,IAAI,iBAAiB,CAAC,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,mBAAU,CAAC;QACpB,GAAG,EAAE,MAAM,CAAC,QAAQ;QACpB,UAAU,EAAE,MAAM,CAAC,QAAQ;YACzB,CAAC,CAAC;gBACE,kBAAkB,EAAE,KAAK;aAC1B;YACH,CAAC,CAAC,SAAS;QACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACvB,CAAC,CAAC;gBACE,kBAAkB,EAAE,KAAK;aAC1B;YACH,CAAC,CAAC,SAAS;KACd,CAAC,CAAA;AACJ,CAAC;AAED,SAAgB,gBAAgB,CAC9B,SAAsB,EACtB,WAAoB,EACpB,QAAkB;IAElB,OAAO;QACL,QAAQ,EAAE,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ;QAChF,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,QAAQ,EAAE,QAAQ,IAAI,SAAS,CAAC,QAAQ;KACzC,CAAA;AACH,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,23 @@
+export type OutputFormat = 'markdown' | 'text' | 'html';
+export interface WebfetchOptions {
+    url: string;
+    format: OutputFormat;
+    timeout: number;
+    proxy?: string;
+    insecure?: boolean;
+}
+export interface WebfetchResult {
+    content: string;
+    contentType: string;
+    url: string;
+}
+export declare const DEFAULT_TIMEOUT = 30;
+export declare const MAX_TIMEOUT = 120;
+export declare const MAX_RESPONSE_SIZE: number;
+export declare const EXIT_CODES: {
+    readonly SUCCESS: 0;
+    readonly USER_ERROR: 1;
+    readonly NETWORK_ERROR: 2;
+    readonly SERVER_ERROR: 3;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAA;AAEvD,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,YAAY,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,eAAO,MAAM,eAAe,KAAK,CAAA;AACjC,eAAO,MAAM,WAAW,MAAM,CAAA;AAC9B,eAAO,MAAM,iBAAiB,QAAkB,CAAA;AAEhD,eAAO,MAAM,UAAU;;;;;CAKb,CAAA"}

package/dist/types.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXIT_CODES = exports.MAX_RESPONSE_SIZE = exports.MAX_TIMEOUT = exports.DEFAULT_TIMEOUT = void 0;
+exports.DEFAULT_TIMEOUT = 30;
+exports.MAX_TIMEOUT = 120;
+exports.MAX_RESPONSE_SIZE = 5 * 1024 * 1024; // 5MB
+exports.EXIT_CODES = {
+    SUCCESS: 0,
+    USER_ERROR: 1,
+    NETWORK_ERROR: 2,
+    SERVER_ERROR: 3,
+};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";;;AAgBa,QAAA,eAAe,GAAG,EAAE,CAAA;AACpB,QAAA,WAAW,GAAG,GAAG,CAAA;AACjB,QAAA,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA,CAAC,MAAM;AAE1C,QAAA,UAAU,GAAG;IACxB,OAAO,EAAE,CAAC;IACV,UAAU,EAAE,CAAC;IACb,aAAa,EAAE,CAAC;IAChB,YAAY,EAAE,CAAC;CACP,CAAA"}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@lyhue1991/webfetch-cli",
+  "version": "0.1.0",
+  "description": "Fetch and convert web content to markdown, text, or html from the command line",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "webfetch": "dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/index.ts",
+    "start": "node dist/index.js",
+    "test": "vitest",
+    "test:run": "vitest run",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [
+    "webfetch",
+    "cli",
+    "fetch",
+    "markdown",
+    "html",
+    "web",
+    "scraper",
+    "converter"
+  ],
+  "author": "",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": ""
+  },
+  "bugs": {
+    "url": ""
+  },
+  "homepage": "",
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@types/turndown": "^5.0.6",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^3.2.4"
+  },
+  "dependencies": {
+    "cheerio": "^1.2.0",
+    "commander": "^14.0.3",
+    "turndown": "^7.2.2",
+    "undici": "^7.24.5",
+    "zod": "^4.3.6"
+  }
+}