md-fetch 1.0.0

package/dist/screen.js

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+import { ScreenCLI } from './screen-cli.js';
+async function main() {
+    try {
+        const cli = new ScreenCLI();
+        await cli.run(process.argv);
+    }
+    catch (error) {
+        console.error('Fatal error:', error.message);
+        process.exit(1);
+    }
+}
+main();
+//# sourceMappingURL=screen.js.map

package/dist/screen.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen.js","sourceRoot":"","sources":["../src/screen.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}

package/dist/types/index.d.ts

@@ -0,0 +1,151 @@
+import type { PuppeteerLifeCycleEvent } from 'puppeteer-core';
+export interface FetchOptions {
+    headers?: Record<string, string>;
+    proxy?: string;
+    timeout?: number;
+    userAgent?: string;
+}
+export interface BrowserOptions {
+    executablePath?: string;
+    waitUntil?: PuppeteerLifeCycleEvent;
+    timeout?: number;
+    userAgent?: string;
+    proxy?: string;
+    headless?: boolean;
+}
+export interface ExtractOptions {
+    useReadability: boolean;
+    selector?: string;
+}
+export interface PageMetadata {
+    url: string;
+    title?: string;
+    description?: string;
+    author?: string;
+    publishedTime?: string;
+    modifiedTime?: string;
+    siteName?: string;
+    keywords?: string[];
+    image?: string;
+    lang?: string;
+}
+export interface ExtractedContent {
+    content: string;
+    metadata: PageMetadata;
+}
+export interface ConversionOptions {
+    headingStyle?: 'atx' | 'setext';
+    codeBlockStyle?: 'fenced' | 'indented';
+    bulletListMarker?: '-' | '+' | '*';
+}
+export interface ProcessOptions {
+    useBrowser: boolean;
+    useReadability: boolean;
+    selector?: string;
+    fetchOptions: FetchOptions;
+    browserOptions?: BrowserOptions;
+    conversionOptions?: ConversionOptions;
+    verbose?: boolean;
+}
+export interface FetchResult {
+    url: string;
+    markdown?: string;
+    error?: Error;
+    success: boolean;
+}
+export interface CLIOptions {
+    output?: string;
+    browser?: boolean;
+    browserPath?: string;
+    readability?: boolean;
+    selector?: string;
+    file?: string;
+    header?: string[];
+    proxy?: string;
+    timeout?: number;
+    config?: string;
+    userAgent?: string;
+    waitUntil?: string;
+    concurrent?: number;
+    verbose?: boolean;
+}
+export interface Config {
+    browser?: {
+        executablePath?: string;
+        waitUntil?: string;
+    };
+    fetch?: {
+        timeout?: number;
+        headers?: Record<string, string>;
+        proxy?: string;
+    };
+    conversion?: {
+        headingStyle?: 'atx' | 'setext';
+        codeBlockStyle?: 'fenced' | 'indented';
+        bulletListMarker?: '-' | '+' | '*';
+    };
+    defaults?: {
+        useReadability?: boolean;
+        concurrent?: number;
+    };
+}
+export declare class FetchError extends Error {
+    url: string;
+    statusCode?: number | undefined;
+    constructor(url: string, statusCode?: number | undefined, message?: string);
+}
+export declare class BrowserError extends Error {
+    url: string;
+    constructor(url: string, message: string);
+}
+export declare class ExtractionError extends Error {
+    url: string;
+    constructor(url: string, message: string);
+}
+export declare class ValidationError extends Error {
+    constructor(message: string);
+}
+export interface ScreenshotOptions {
+    fullPage: boolean;
+    width: number;
+    height: number;
+    deviceScaleFactor: number;
+    outputDir: string;
+    format: 'png' | 'jpeg' | 'webp';
+    quality?: number;
+    browserOptions: BrowserOptions;
+    delay?: number;
+    selector?: string;
+    hideSelectors?: string[];
+    verbose?: boolean;
+}
+export interface ScreenshotCLIOptions {
+    fullPage?: boolean;
+    viewport?: boolean;
+    width?: number;
+    height?: number;
+    scale?: number;
+    output?: string;
+    format?: string;
+    quality?: number;
+    browserPath?: string;
+    waitUntil?: string;
+    timeout?: number;
+    userAgent?: string;
+    proxy?: string;
+    delay?: number;
+    selector?: string;
+    hide?: string;
+    verbose?: boolean;
+}
+export interface ScreenshotResult {
+    url: string;
+    filepath?: string;
+    success: boolean;
+    error?: Error;
+}
+export declare class ScreenshotError extends Error {
+    url: string;
+    constructor(url: string, message: string);
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAG9D,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAGD,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAGD,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAGD,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAGD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,YAAY,CAAC;CACxB;AAGD,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAChC,cAAc,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IACvC,gBAAgB,CAAC,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;CACpC;AAGD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAGD,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;CAClB;AAGD,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAGD,MAAM,WAAW,MAAM;IACrB,OAAO,CAAC,EAAE;QACR,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,UAAU,CAAC,EAAE;QACX,YAAY,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;QAChC,cAAc,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;QACvC,gBAAgB,CAAC,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;KACpC,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAGD,qBAAa,UAAW,SAAQ,KAAK;IAE1B,GAAG,EAAE,MAAM;IACX,UAAU,CAAC,EAAE,MAAM;gBADnB,GAAG,EAAE,MAAM,EACX,UAAU,CAAC,EAAE,MAAM,YAAA,EAC1B,OAAO,CAAC,EAAE,MAAM;CAKnB;AAED,qBAAa,YAAa,SAAQ,KAAK;IAE5B,GAAG,EAAE,MAAM;gBAAX,GAAG,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM;CAKlB;AAED,qBAAa,eAAgB,SAAQ,KAAK;IAE/B,GAAG,EAAE,MAAM;gBAAX,GAAG,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM;CAKlB;AAED,qBAAa,eAAgB,SAAQ,KAAK;gBAC5B,OAAO,EAAE,MAAM;CAI5B;AAKD,MAAM,WAAW,iBAAiB;IAEhC,QAAQ,EAAE,OAAO,CAAC;IAGlB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;IAG1B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;IAChC,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,cAAc,EAAE,cAAc,CAAC;IAG/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IAGzB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAGD,MAAM,WAAW,oBAAoB;IAEnC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IAGd,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAGD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,qBAAa,eAAgB,SAAQ,KAAK;IAE/B,GAAG,EAAE,MAAM;gBAAX,GAAG,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM;CAKlB"}

package/dist/types/index.js

@@ -0,0 +1,42 @@
+// 自定义错误类型
+export class FetchError extends Error {
+    url;
+    statusCode;
+    constructor(url, statusCode, message) {
+        super(message || `Failed to fetch ${url}`);
+        this.url = url;
+        this.statusCode = statusCode;
+        this.name = 'FetchError';
+    }
+}
+export class BrowserError extends Error {
+    url;
+    constructor(url, message) {
+        super(message);
+        this.url = url;
+        this.name = 'BrowserError';
+    }
+}
+export class ExtractionError extends Error {
+    url;
+    constructor(url, message) {
+        super(message);
+        this.url = url;
+        this.name = 'ExtractionError';
+    }
+}
+export class ValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ValidationError';
+    }
+}
+export class ScreenshotError extends Error {
+    url;
+    constructor(url, message) {
+        super(message);
+        this.url = url;
+        this.name = 'ScreenshotError';
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAgHA,UAAU;AACV,MAAM,OAAO,UAAW,SAAQ,KAAK;IAE1B;IACA;IAFT,YACS,GAAW,EACX,UAAmB,EAC1B,OAAgB;QAEhB,KAAK,CAAC,OAAO,IAAI,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAJpC,QAAG,GAAH,GAAG,CAAQ;QACX,eAAU,GAAV,UAAU,CAAS;QAI1B,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;IAC3B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,KAAK;IAE5B;IADT,YACS,GAAW,EAClB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHR,QAAG,GAAH,GAAG,CAAQ;QAIlB,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAE/B;IADT,YACS,GAAW,EAClB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHR,QAAG,GAAH,GAAG,CAAQ;QAIlB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAqED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAE/B;IADT,YACS,GAAW,EAClB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHR,QAAG,GAAH,GAAG,CAAQ;QAIlB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF"}

package/dist/utils/filename-sanitizer.d.ts

@@ -0,0 +1,38 @@
+/**
+ * 文件名安全化工具
+ * 用于从 URL 生成安全的文件名
+ */
+/**
+ * 从 URL 生成安全的文件名
+ * 格式: <URL安全化前50字符>_<年月日时分秒>.png
+ *
+ * @param url - 原始URL
+ * @param extension - 文件扩展名（默认 'png'）
+ * @returns 安全的文件名
+ *
+ * @example
+ * generateScreenshotFilename('https://github.com/user/repo/issues/123')
+ * // => 'github.com_user_repo_issues_123_20251229143025.png'
+ */
+export declare function generateScreenshotFilename(url: string, extension?: string): string;
+/**
+ * 将 URL 转换为安全的文件名前缀
+ * - 保留域名
+ * - 替换非法字符为下划线
+ * - 截取前50个字符
+ *
+ * @param url - 原始URL
+ * @returns 安全化后的文件名前缀
+ */
+export declare function sanitizeUrlForFilename(url: string): string;
+/**
+ * 生成时间戳字符串
+ * 格式: YYYYMMDDHHMMSS
+ *
+ * @returns 时间戳字符串
+ *
+ * @example
+ * generateTimestamp() // => '20251229143025'
+ */
+export declare function generateTimestamp(): string;
+//# sourceMappingURL=filename-sanitizer.d.ts.map

package/dist/utils/filename-sanitizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filename-sanitizer.d.ts","sourceRoot":"","sources":["../../src/utils/filename-sanitizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AACH,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,MAAc,GAAG,MAAM,CAKzF;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAgC1D;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAW1C"}

package/dist/utils/filename-sanitizer.js

@@ -0,0 +1,79 @@
+/**
+ * 文件名安全化工具
+ * 用于从 URL 生成安全的文件名
+ */
+/**
+ * 从 URL 生成安全的文件名
+ * 格式: <URL安全化前50字符>_<年月日时分秒>.png
+ *
+ * @param url - 原始URL
+ * @param extension - 文件扩展名（默认 'png'）
+ * @returns 安全的文件名
+ *
+ * @example
+ * generateScreenshotFilename('https://github.com/user/repo/issues/123')
+ * // => 'github.com_user_repo_issues_123_20251229143025.png'
+ */
+export function generateScreenshotFilename(url, extension = 'png') {
+    const safeName = sanitizeUrlForFilename(url);
+    const timestamp = generateTimestamp();
+    return `${safeName}_${timestamp}.${extension}`;
+}
+/**
+ * 将 URL 转换为安全的文件名前缀
+ * - 保留域名
+ * - 替换非法字符为下划线
+ * - 截取前50个字符
+ *
+ * @param url - 原始URL
+ * @returns 安全化后的文件名前缀
+ */
+export function sanitizeUrlForFilename(url) {
+    try {
+        const urlObj = new URL(url);
+        // 构建完整的文件名部分：域名 + 路径 + 查询参数
+        let fullPart = urlObj.hostname + urlObj.pathname + urlObj.search;
+        // 安全化处理
+        const safeName = fullPart
+            // 替换非法文件名字符为下划线
+            .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_')
+            // 替换多个连续点为单个点（保留域名中的点）
+            .replace(/\.{2,}/g, '.')
+            // 替换空格为下划线
+            .replace(/\s+/g, '_')
+            // 替换多个连续斜杠和下划线为单个下划线
+            .replace(/[/_]+/g, '_')
+            // 去除开头和结尾的点和下划线
+            .replace(/^[._]+|[._]+$/g, '');
+        // 截取前50个字符
+        const truncated = safeName.slice(0, 50);
+        // 去除末尾的下划线或点
+        const cleaned = truncated.replace(/[._]+$/, '');
+        // 如果清理后为空，使用默认值
+        return cleaned || 'screenshot';
+    }
+    catch (error) {
+        // URL 解析失败，使用默认值
+        return 'screenshot';
+    }
+}
+/**
+ * 生成时间戳字符串
+ * 格式: YYYYMMDDHHMMSS
+ *
+ * @returns 时间戳字符串
+ *
+ * @example
+ * generateTimestamp() // => '20251229143025'
+ */
+export function generateTimestamp() {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
+    const hours = String(now.getHours()).padStart(2, '0');
+    const minutes = String(now.getMinutes()).padStart(2, '0');
+    const seconds = String(now.getSeconds()).padStart(2, '0');
+    return `${year}${month}${day}${hours}${minutes}${seconds}`;
+}
+//# sourceMappingURL=filename-sanitizer.js.map

package/dist/utils/filename-sanitizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filename-sanitizer.js","sourceRoot":"","sources":["../../src/utils/filename-sanitizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,0BAA0B,CAAC,GAAW,EAAE,YAAoB,KAAK;IAC/E,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAC;IAEtC,OAAO,GAAG,QAAQ,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;AACjD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAAW;IAChD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,4BAA4B;QAC5B,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;QAEjE,QAAQ;QACR,MAAM,QAAQ,GAAG,QAAQ;YACvB,gBAAgB;aACf,OAAO,CAAC,wBAAwB,EAAE,GAAG,CAAC;YACvC,uBAAuB;aACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;YACxB,WAAW;aACV,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YACrB,qBAAqB;aACpB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;YACvB,gBAAgB;aACf,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAEjC,WAAW;QACX,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAExC,aAAa;QACb,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAEhD,gBAAgB;QAChB,OAAO,OAAO,IAAI,YAAY,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,iBAAiB;QACjB,OAAO,YAAY,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAEvB,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAE1D,OAAO,GAAG,IAAI,GAAG,KAAK,GAAG,GAAG,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,EAAE,CAAC;AAC7D,CAAC"}

package/dist/utils/frontmatter.d.ts

@@ -0,0 +1,6 @@
+import type { PageMetadata } from '../types/index.js';
+/**
+ * Generate YAML frontmatter from page metadata
+ */
+export declare function generateFrontmatter(metadata: PageMetadata): string;
+//# sourceMappingURL=frontmatter.d.ts.map

package/dist/utils/frontmatter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.d.ts","sourceRoot":"","sources":["../../src/utils/frontmatter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,CAmDlE"}

package/dist/utils/frontmatter.js

@@ -0,0 +1,65 @@
+/**
+ * Generate YAML frontmatter from page metadata
+ */
+export function generateFrontmatter(metadata) {
+    const lines = ['---'];
+    // Add fields in a logical order
+    if (metadata.title) {
+        lines.push(`title: ${escapeYamlString(metadata.title)}`);
+    }
+    if (metadata.url) {
+        lines.push(`url: ${metadata.url}`);
+    }
+    if (metadata.description) {
+        lines.push(`description: ${escapeYamlString(metadata.description)}`);
+    }
+    if (metadata.author) {
+        lines.push(`author: ${escapeYamlString(metadata.author)}`);
+    }
+    if (metadata.siteName) {
+        lines.push(`siteName: ${escapeYamlString(metadata.siteName)}`);
+    }
+    if (metadata.publishedTime) {
+        lines.push(`publishedTime: ${metadata.publishedTime}`);
+    }
+    if (metadata.modifiedTime) {
+        lines.push(`modifiedTime: ${metadata.modifiedTime}`);
+    }
+    if (metadata.keywords && metadata.keywords.length > 0) {
+        lines.push('keywords:');
+        metadata.keywords.forEach(keyword => {
+            lines.push(`  - ${escapeYamlString(keyword)}`);
+        });
+    }
+    if (metadata.image) {
+        lines.push(`image: ${metadata.image}`);
+    }
+    if (metadata.lang) {
+        lines.push(`lang: ${metadata.lang}`);
+    }
+    lines.push('---');
+    lines.push(''); // Add blank line after frontmatter
+    return lines.join('\n');
+}
+/**
+ * Escape special characters in YAML strings
+ */
+function escapeYamlString(str) {
+    // If string contains special characters, quote it
+    if (str.includes(':') ||
+        str.includes('#') ||
+        str.includes('[') ||
+        str.includes(']') ||
+        str.includes('{') ||
+        str.includes('}') ||
+        str.includes('\n') ||
+        str.includes('"') ||
+        str.includes("'") ||
+        str.startsWith(' ') ||
+        str.endsWith(' ')) {
+        // Use double quotes and escape internal quotes
+        return `"${str.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+    }
+    return str;
+}
+//# sourceMappingURL=frontmatter.js.map

package/dist/utils/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../../src/utils/frontmatter.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAsB;IACxD,MAAM,KAAK,GAAa,CAAC,KAAK,CAAC,CAAC;IAEhC,gCAAgC;IAChC,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,UAAU,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,gBAAgB,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,WAAW,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,aAAa,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,kBAAkB,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,iBAAiB,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YAClC,KAAK,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mCAAmC;IAEnD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,kDAAkD;IAClD,IACE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;QAClB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;QACnB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EACjB,CAAC;QACD,+CAA+C;QAC/C,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC;IAChE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "md-fetch",
+  "version": "1.0.0",
+  "description": "Convert web pages to clean Markdown using fetch, readability, and turndown",
+  "type": "module",
+  "repository": {
+    "type": "github",
+    "url": "https://github.com/fairjm/md-fetch"
+  },
+  "bin": {
+    "md-fetch": "./dist/index.js",
+    "md-fetch-screen": "./dist/screen.js"
+  },
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/index.ts",
+    "test": "vitest",
+    "prepublishOnly": "pnpm run build"
+  },
+  "keywords": [
+    "markdown",
+    "cli",
+    "fetch",
+    "readability",
+    "web-scraping",
+    "html-to-markdown"
+  ],
+  "author": {
+    "name": "fairjm",
+    "url": "https://bingowith.me"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "packageManager": "pnpm@9.0.0",
+  "dependencies": {
+    "@mozilla/readability": "^0.5.0",
+    "commander": "^12.0.0",
+    "jsdom": "^24.0.0",
+    "puppeteer-core": "^22.0.0",
+    "turndown": "^7.2.0",
+    "undici": "^7.16.0"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^27.0.0",
+    "@types/node": "^20.0.0",
+    "@types/turndown": "^5.0.0",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.0",
+    "vitest": "^1.2.0"
+  }
+}

package/skills/md-fetch/SKILL.md

@@ -0,0 +1,133 @@
+---
+name: md-fetch
+description: Convert web pages to Markdown or take screenshots. Use when: (1) Converting web pages to Markdown with YAML frontmatter, (2) Taking screenshots of web pages, (3) Processing SPA/dynamic content, (4) Batch processing URLs. Triggers on "convert webpage", "save article", "fetch content", "take screenshot", or when mentioning md-fetch.
+---
+
+# md-fetch
+
+Convert web pages to Markdown or take high-quality screenshots.
+
+## Two Tools
+
+- **md-fetch** - Convert web pages to Markdown with YAML frontmatter
+- **md-fetch-screen** - Take screenshots
+
+## Common Usage
+
+### Convert to Markdown
+
+```bash
+# Basic usage
+md-fetch https://example.com -o article.md
+
+# Browser mode for SPA/dynamic pages
+md-fetch -b https://react-app.com -o app.md
+
+# Custom CSS selector
+md-fetch https://example.com -s "article.content" -o content.md
+
+# Disable readability (keep full content)
+md-fetch https://example.com -R -o full.md
+
+# Multiple URLs
+md-fetch https://site1.com https://site2.com
+```
+
+### Take Screenshots
+
+```bash
+# Full page screenshot
+md-fetch-screen https://example.com
+
+# Viewport-only (1440x900)
+md-fetch-screen https://example.com --viewport -W 1440 -H 900
+
+# High-DPI (2x scale for Retina)
+md-fetch-screen https://example.com --scale 2
+
+# Screenshot specific element
+md-fetch-screen https://example.com --selector "#main"
+
+# Hide unwanted elements
+md-fetch-screen https://example.com --hide ".ad,.popup"
+```
+
+## Key Parameters
+
+### md-fetch
+- `-o, --output <file>` - Save to file (otherwise stdout)
+- `-b, --browser` - Use headless browser for SPA pages
+- `-R, --no-readability` - Keep full HTML content
+- `-s, --selector <css>` - Custom CSS selector
+- `--wait-until <event>` - Browser wait: `load|networkidle0|networkidle2`
+- `--proxy <url>` - Proxy server
+- `--verbose` - Verbose logging
+
+### md-fetch-screen
+- `--viewport` - Viewport-only screenshot (vs full page)
+- `-W, --width <pixels>` - Viewport width (default: 1920)
+- `-H, --height <pixels>` - Viewport height (default: 1080)
+- `--scale <1|2|3>` - Device scale factor for high-DPI
+- `--selector <css>` - Screenshot specific element
+- `--hide <selectors>` - Hide elements (comma-separated)
+- `--format <png|jpeg|webp>` - Image format
+- `--delay <ms>` - Delay before screenshot
+- `--output <dir>` - Output directory (default: current)
+
+## Output Format
+
+**Markdown** includes YAML frontmatter:
+```markdown
+---
+title: "Page Title"
+url: https://example.com
+author: "Author Name"
+publishedTime: 2024-01-01T00:00:00Z
+---
+
+# Content here...
+```
+
+**Screenshots** named as: `domain_path_timestamp.png`
+
+## Common Scenarios
+
+**SPA pages (React/Vue/Angular):**
+```bash
+md-fetch -b https://app.com --wait-until networkidle0 -o app.md
+```
+
+**Specific content section:**
+```bash
+md-fetch https://blog.com/post -s "article.post-content" -o post.md
+```
+
+**High-quality screenshot without ads:**
+```bash
+md-fetch-screen https://site.com --scale 2 --hide ".ad,.banner"
+```
+
+**Using proxy:**
+```bash
+export HTTPS_PROXY=http://proxy.example.com:8080
+md-fetch https://example.com -o output.md
+```
+
+**Batch processing:**
+```bash
+for url in url1 url2 url3; do
+  md-fetch "$url" -o "${url##*/}.md"
+done
+```
+
+## Reference
+
+For complete CLI options and advanced usage, see [cli-reference.md](references/cli-reference.md).
+
+## Tips
+
+- Use `-b` (browser mode) only for SPA/dynamic pages
+- Test CSS selectors in browser DevTools first
+- `networkidle0` waits longest but is most reliable
+- Add `--verbose` for debugging
+- Screenshots auto-retry 3x with exponential backoff