@ariesfish/feedloom 0.1.2 → 0.1.4

package/README.md

@@ -1,5 +1,15 @@
 # Feedloom
 
+<div align="center">
+  <img src="assets/logo.png" alt="Feedloom logo" width="160">
+  <p><strong>Archive long-form web content as clean Markdown with local assets.</strong></p>
+  <p>
+    <a href="https://www.npmjs.com/package/@ariesfish/feedloom"><img alt="npm" src="https://img.shields.io/npm/v/@ariesfish/feedloom"></a>
+    <img alt="Node.js >= 24" src="https://img.shields.io/badge/node-%3E%3D24-339933">
+    <img alt="License MIT" src="https://img.shields.io/badge/license-MIT-blue">
+  </p>
+</div>
+
 Feedloom is a command-line tool for archiving long-form web content. It takes article URLs, URL list files, or RSS/Atom feeds, extracts readable article content, converts it to Markdown with YAML frontmatter, and saves page images as local assets. It is designed for personal knowledge bases, notebook vaults, and offline reading archives.
 
 ## Features
@@ -43,6 +53,14 @@ If you plan to use `browser`, `stealth`, or the browser fallback in `auto` mode,
 npx patchright install chromium
 ```
 
+You can verify or repair the runtime later with:
+
+```bash
+npm run dev -- doctor
+```
+
+If the Patchright Chromium executable is missing, `doctor` runs `npx patchright install chromium` automatically.
+
 ### 4. Build the CLI
 
 ```bash
@@ -261,6 +279,12 @@ Only use this on your own device and accounts. Always respect the target site's
 --site-rules-dir <dir>          Optional directory of private TOML site rules
 ```
 
+Run environment checks:
+
+```bash
+npm run dev -- doctor
+```
+
 For the full option list, run:
 
 ```bash
@@ -281,7 +305,8 @@ npm test
 - Respect robots.txt, website terms of service, copyright, and rate limits.
 - For dynamic pages, try `--fetch-mode browser` first.
 - For static blogs and news sites, `--fetch-mode static` is usually faster.
-- If article extraction is poor for a specific site, keep private TOML site rules outside the package and pass them with `--site-rules-dir <dir>`.
+- Feedloom ships bundled TOML site rules for common dynamic/structured sites such as WeChat official account articles and Zhihu. Site rules can define extraction, cleanup, and fetch preferences. For example, the bundled Zhihu rule uses browser fetch with copied Chrome state when `--chrome-user-data-dir`/`--chrome-profile` are configured.
+- If article extraction is poor for a specific site, keep private TOML site rules outside the package and pass them with `--site-rules-dir <dir>`. Private rules are loaded after bundled rules.
 - For large batches, test with `--limit` before running the full job.
 
 ## Acknowledgements

package/dist/cli.js

@@ -2,7 +2,9 @@
 
 // src/cli.ts
 import { readdir as readdir2 } from "fs/promises";
-import { join as join7, resolve as resolve2 } from "path";
+import { createRequire } from "module";
+import { dirname, join as join7, resolve as resolve2 } from "path";
+import { fileURLToPath } from "url";
 import { Command } from "commander";
 
 // src/cleaning/profiles.ts
@@ -39,10 +41,29 @@ function profileFromTomlRule(name, rule) {
     metadata: {
       fixedAuthor: rule.metadata?.fixed_author,
       titleSuffixPatterns: rule.metadata?.strip_title_regexes,
+      authorSuffixPatterns: rule.metadata?.strip_author_regexes,
       authorSelectors: rule.metadata?.author_selectors,
       authorMetaNames: rule.metadata?.author_meta_names,
       authorMetaItemprops: rule.metadata?.author_meta_itemprops,
       authorMetaProperties: rule.metadata?.author_meta_properties
+    },
+    fetch: {
+      mode: rule.fetch?.mode,
+      preferBrowserState: rule.fetch?.prefer_browser_state,
+      waitMs: rule.fetch?.wait_ms,
+      networkIdle: rule.fetch?.network_idle,
+      waitSelector: rule.fetch?.wait_selector,
+      waitSelectorState: rule.fetch?.wait_selector_state,
+      clickSelectors: rule.fetch?.click_selectors,
+      scrollToBottom: rule.fetch?.scroll_to_bottom,
+      useProxyEnv: rule.fetch?.use_proxy_env
+    },
+    media: {
+      includeMetaImages: rule.media?.include_meta_images,
+      imageMetaProperties: rule.media?.image_meta_properties
+    },
+    extraction: {
+      requireText: rule.extract?.require_text
     }
   };
 }
@@ -92,11 +113,172 @@ function firstContentSelector(profiles) {
   return void 0;
 }
 
+// src/doctor.ts
+import { spawn } from "child_process";
+import { access } from "fs/promises";
+import { chromium } from "patchright";
+var INSTALL_CHROMIUM_COMMAND = "npx patchright install chromium";
+var INSTALL_CHROMIUM_HINT = `Run: ${INSTALL_CHROMIUM_COMMAND}`;
+function errorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function chromiumExecutablePath() {
+  const browserType = chromium;
+  return browserType.executablePath();
+}
+function appendCheck(checks, check) {
+  checks.push(check);
+}
+async function installPatchrightChromium(events = {}) {
+  return new Promise((resolve3, reject) => {
+    const child = spawn("npx", ["patchright", "install", "chromium"], {
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => events.onStdout?.(chunk));
+    child.stderr.on("data", (chunk) => events.onStderr?.(chunk));
+    child.on("error", reject);
+    child.on("close", (code, signal) => resolve3({ code, signal }));
+  });
+}
+async function executableExists(path) {
+  try {
+    await access(path);
+    return { ok: true };
+  } catch (error) {
+    return { ok: false, error };
+  }
+}
+async function ensureChromiumInstalled(checks, executablePath, options) {
+  const firstCheck = await executableExists(executablePath);
+  if (firstCheck.ok) {
+    appendCheck(checks, {
+      name: "Patchright Chromium installation",
+      ok: true,
+      message: "Chromium executable exists.",
+      detail: executablePath
+    });
+    return true;
+  }
+  appendCheck(checks, {
+    name: "Patchright Chromium installation",
+    ok: false,
+    message: "Chromium executable was not found on disk. Installing Patchright Chromium...",
+    detail: `${executablePath}
+${errorMessage(firstCheck.error)}`
+  });
+  let output = "";
+  const appendOutput = (chunk) => {
+    output += chunk;
+    options.stderr.write(chunk);
+  };
+  try {
+    const result = await options.installChromium({ onStdout: appendOutput, onStderr: appendOutput });
+    if (result.code !== 0) {
+      appendCheck(checks, {
+        name: "Patchright Chromium auto-install",
+        ok: false,
+        message: `Installation command failed with exit code ${result.code ?? "null"}${result.signal ? ` and signal ${result.signal}` : ""}.`,
+        detail: output.trim() || void 0,
+        hint: INSTALL_CHROMIUM_HINT
+      });
+      return false;
+    }
+  } catch (error) {
+    appendCheck(checks, {
+      name: "Patchright Chromium auto-install",
+      ok: false,
+      message: "Installation command failed to start or crashed.",
+      detail: errorMessage(error),
+      hint: INSTALL_CHROMIUM_HINT
+    });
+    return false;
+  }
+  const secondCheck = await executableExists(executablePath);
+  appendCheck(checks, secondCheck.ok ? {
+    name: "Patchright Chromium auto-install",
+    ok: true,
+    message: "Chromium installed successfully.",
+    detail: executablePath
+  } : {
+    name: "Patchright Chromium auto-install",
+    ok: false,
+    message: "Installation finished, but Chromium executable is still missing.",
+    detail: `${executablePath}
+${errorMessage(secondCheck.error)}`,
+    hint: INSTALL_CHROMIUM_HINT
+  });
+  return secondCheck.ok;
+}
+async function runDoctor(options = {}) {
+  const checks = [];
+  const resolvedOptions = {
+    installChromium: options.installChromium ?? installPatchrightChromium,
+    stderr: options.stderr ?? process.stderr
+  };
+  let executablePath = "";
+  try {
+    executablePath = chromiumExecutablePath();
+    checks.push({
+      name: "Patchright Chromium executable path",
+      ok: true,
+      message: executablePath
+    });
+  } catch (error) {
+    checks.push({
+      name: "Patchright Chromium executable path",
+      ok: false,
+      message: "Patchright does not report a Chromium executable for this platform.",
+      detail: errorMessage(error),
+      hint: INSTALL_CHROMIUM_HINT
+    });
+  }
+  const installed = executablePath ? await ensureChromiumInstalled(checks, executablePath, resolvedOptions) : false;
+  if (installed) {
+    try {
+      const browser = await chromium.launch({ headless: true });
+      await browser.close();
+      checks.push({
+        name: "Patchright Chromium launch",
+        ok: true,
+        message: "Chromium launched successfully in headless mode."
+      });
+    } catch (error) {
+      checks.push({
+        name: "Patchright Chromium launch",
+        ok: false,
+        message: "Chromium executable exists but failed to launch.",
+        detail: errorMessage(error),
+        hint: INSTALL_CHROMIUM_HINT
+      });
+    }
+  }
+  return {
+    ok: checks.at(-1)?.ok === true,
+    checks
+  };
+}
+function formatDoctorResult(result) {
+  const lines = ["Feedloom doctor"];
+  for (const check of result.checks) {
+    lines.push(`${check.ok ? "\u2713" : "\u2717"} ${check.name}: ${check.message}`);
+    if (check.detail) {
+      lines.push(...check.detail.split("\n").map((line) => `  ${line}`));
+    }
+    if (check.hint) {
+      lines.push(`  ${check.hint}`);
+    }
+  }
+  lines.push(result.ok ? "OK" : "FAILED");
+  return lines.join("\n");
+}
+
 // src/fetch/browser.ts
 import { mkdtemp, rm } from "fs/promises";
 import { tmpdir } from "os";
 import { join } from "path";
-import { chromium } from "patchright";
+import { chromium as chromium2 } from "patchright";
 var SCRAPLING_DEFAULT_ARGS = [
   "--no-pings",
   "--no-first-run",
@@ -122,13 +304,13 @@ async function runPageActions(page, options) {
     await page.locator(selector).first().click({ timeout: 5e3 }).catch(() => void 0);
   }
   if (options.scrollToBottom) {
-    await page.evaluate(async () => {
-      const delay = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
+    await page.evaluate(`(async () => {
+      const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
       for (let i = 0; i < 8; i += 1) {
         window.scrollTo(0, document.body.scrollHeight);
         await delay(250);
       }
-    });
+    })()`);
   }
   if (options.waitSelector) {
     await page.locator(options.waitSelector).first().waitFor({
@@ -145,7 +327,7 @@ async function launchBrowserContext(options) {
   if (options.dnsOverHttps) {
     extraArgs.push("--dns-over-https-templates=https://cloudflare-dns.com/dns-query");
   }
-  const context = await chromium.launchPersistentContext(userDataDir, {
+  const context = await chromium2.launchPersistentContext(userDataDir, {
     channel: options.channel,
     headless: options.headless ?? true,
     args: extraArgs,
@@ -222,7 +404,7 @@ async function fetchBrowserHtml(url, options = {}) {
 import { mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
 import { tmpdir as tmpdir2 } from "os";
 import { join as join2 } from "path";
-import { chromium as chromium2 } from "patchright";
+import { chromium as chromium3 } from "patchright";
 var DEFAULT_ARGS = [
   "--no-pings",
   "--no-first-run",
@@ -365,7 +547,7 @@ async function solveCloudflare(page) {
 async function launchStealthContext(options) {
   const userDataDir = options.userDataDir ?? await mkdtemp2(join2(tmpdir2(), "feedloom-stealth-"));
   const ownsUserDataDir = options.userDataDir === void 0;
-  const context = await chromium2.launchPersistentContext(userDataDir, {
+  const context = await chromium3.launchPersistentContext(userDataDir, {
     channel: "chromium",
     headless: options.headless ?? true,
     args: stealthArgs(options),
@@ -409,13 +591,13 @@ async function fetchWithStealthContext(context, url, options) {
       await page.locator(selector).first().click({ timeout: 5e3 }).catch(() => void 0);
     }
     if (options.scrollToBottom) {
-      await page.evaluate(async () => {
-        const delay = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
+      await page.evaluate(`(async () => {
+        const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
         for (let i = 0; i < 8; i += 1) {
           window.scrollTo(0, document.body.scrollHeight);
           await delay(250);
         }
-      });
+      })()`);
     }
     if (options.waitSelector) {
       await page.locator(options.waitSelector).first().waitFor({ state: options.waitSelectorState ?? "attached", timeout: timeoutMs }).catch(() => void 0);
@@ -824,8 +1006,8 @@ function imageSource(img) {
   return first || null;
 }
 async function localizeImages(html, options) {
-  const { document: document2 } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
-  const images = Array.from(document2.querySelectorAll("img"));
+  const { document } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
+  const images = Array.from(document.querySelectorAll("img"));
   if (images.length === 0) return html;
   const fetchImage = options.fetchImage ?? fetch;
   const seen = /* @__PURE__ */ new Map();
@@ -864,7 +1046,7 @@ async function localizeImages(html, options) {
     img.removeAttribute("data-original");
     img.removeAttribute("data-src");
   }
-  return document2.body.innerHTML;
+  return document.body.innerHTML;
 }
 
 // src/cleaning/clean-html.ts
@@ -1004,6 +1186,18 @@ function cleanupTitle(metadata, profiles) {
   }
   metadata.title = title;
 }
+function cleanupAuthor(metadata, profiles) {
+  if (!metadata.author) {
+    return;
+  }
+  let author = metadata.author;
+  for (const profile of profiles) {
+    for (const pattern of profile.metadata?.authorSuffixPatterns ?? []) {
+      author = author.replace(new RegExp(pattern, "i"), "").trim();
+    }
+  }
+  metadata.author = author;
+}
 function applySiteProfiles(root, profiles, removals) {
   removeByExactSelectors(root, profiles, removals);
   removeByPartialAttributePatterns(root, profiles, removals);
@@ -1012,6 +1206,7 @@ function applySiteProfiles(root, profiles, removals) {
 function applyMetadataProfiles(metadata, profiles) {
   applyFixedAuthor(metadata, profiles);
   cleanupTitle(metadata, profiles);
+  cleanupAuthor(metadata, profiles);
 }
 
 // src/cleaning/clean-html.ts
@@ -1033,17 +1228,17 @@ var DEFAULT_FEEDLOOM_PROFILE = {
   }
 };
 var DefuddleClass = DefuddleModule.default ?? DefuddleModule.Defuddle;
-function firstMetaContent(document2, names) {
+function firstMetaContent(document, names) {
   for (const name of names) {
     const escaped = name.replace(/"/g, '\\"');
-    const element = document2.querySelector(`meta[property="${escaped}"], meta[name="${escaped}"], meta[itemprop="${escaped}"]`);
+    const element = document.querySelector(`meta[property="${escaped}"], meta[name="${escaped}"], meta[itemprop="${escaped}"]`);
     const content = element?.getAttribute("content")?.trim();
     if (content) return content;
   }
   return void 0;
 }
-function jsonLdValue(document2, keys) {
-  for (const script of Array.from(document2.querySelectorAll('script[type="application/ld+json"]'))) {
+function jsonLdValue(document, keys) {
+  for (const script of Array.from(document.querySelectorAll('script[type="application/ld+json"]'))) {
     const text = script.textContent?.trim();
     if (!text) continue;
     try {
@@ -1064,12 +1259,12 @@ function jsonLdValue(document2, keys) {
   }
   return void 0;
 }
-function profileAuthorFromDocument(document2, profiles) {
+function profileAuthorFromDocument(document, profiles) {
   for (const profile of profiles) {
     const metadata = profile.metadata;
     if (!metadata) continue;
     for (const selector of metadata.authorSelectors ?? []) {
-      const author = document2.querySelector(selector)?.textContent?.replace(/\s+/g, " ").trim();
+      const author = document.querySelector(selector)?.textContent?.replace(/\s+/g, " ").trim();
       if (author) return author;
     }
     const metaNames = [
@@ -1079,33 +1274,54 @@ function profileAuthorFromDocument(document2, profiles) {
     ];
     for (const entry of metaNames) {
       const escaped = entry.value.replace(/"/g, '\\"');
-      const author = document2.querySelector(`meta[${entry.attr}="${escaped}"]`)?.getAttribute("content")?.trim();
+      const author = document.querySelector(`meta[${entry.attr}="${escaped}"]`)?.getAttribute("content")?.trim();
       if (author) return author;
     }
   }
   return void 0;
 }
-function toMetadata(result, document2, profiles) {
+function toMetadata(result, document, profiles) {
   return {
-    title: result.title || firstMetaContent(document2, ["og:title", "twitter:title"]) || document2.querySelector("title")?.textContent?.trim() || void 0,
-    description: result.description || firstMetaContent(document2, ["description", "og:description", "twitter:description"]),
+    title: result.title || firstMetaContent(document, ["og:title", "twitter:title"]) || document.querySelector("title")?.textContent?.trim() || void 0,
+    description: result.description || firstMetaContent(document, ["description", "og:description", "twitter:description"]),
     domain: result.domain || void 0,
     favicon: result.favicon || void 0,
-    image: result.image || firstMetaContent(document2, ["og:image", "twitter:image"]),
-    language: result.language || document2.documentElement.getAttribute("lang") || void 0,
-    published: result.published || firstMetaContent(document2, ["article:published_time", "date", "datePublished", "pubdate", "publishdate"]) || jsonLdValue(document2, ["datePublished", "dateCreated"]),
-    author: result.author || profileAuthorFromDocument(document2, profiles) || firstMetaContent(document2, ["author", "article:author", "twitter:creator"]) || jsonLdValue(document2, ["author", "creator"]),
-    site: result.site || firstMetaContent(document2, ["og:site_name", "application-name"]),
+    image: result.image || firstMetaContent(document, ["og:image", "twitter:image"]),
+    language: result.language || document.documentElement.getAttribute("lang") || void 0,
+    published: result.published || firstMetaContent(document, ["article:published_time", "date", "datePublished", "pubdate", "publishdate"]) || jsonLdValue(document, ["datePublished", "dateCreated"]),
+    author: result.author || profileAuthorFromDocument(document, profiles) || firstMetaContent(document, ["author", "article:author", "twitter:creator"]) || jsonLdValue(document, ["author", "creator"]),
+    site: result.site || firstMetaContent(document, ["og:site_name", "application-name"]),
     schemaOrgData: result.schemaOrgData,
     wordCount: result.wordCount,
     parseTime: result.parseTime
   };
 }
-function serializeProfiledContent(content, profiles, removals) {
-  const { document: document2 } = parseHTML2(`<!doctype html><html><body><main data-feedloom-profile-root="true">${content}</main></body></html>`);
-  const root = document2.querySelector('[data-feedloom-profile-root="true"]') ?? document2.body;
+function appendMetaImages(document, root, profiles) {
+  const properties = profiles.flatMap((profile) => profile.media?.includeMetaImages ? profile.media.imageMetaProperties ?? ["og:image"] : []);
+  if (properties.length === 0) {
+    return;
+  }
+  const seen = new Set(Array.from(root.querySelectorAll("img")).map((img) => img.getAttribute("src") ?? ""));
+  for (const property of properties) {
+    const escaped = property.replace(/"/g, '\\"');
+    for (const meta of Array.from(document.querySelectorAll(`meta[property="${escaped}"], meta[name="${escaped}"], meta[itemprop="${escaped}"]`))) {
+      const src = meta.getAttribute("content")?.trim();
+      if (!src || seen.has(src)) continue;
+      const img = document.createElement("img");
+      img.setAttribute("src", src);
+      img.setAttribute("alt", "");
+      root.appendChild(document.createElement("p"));
+      root.lastElementChild?.appendChild(img);
+      seen.add(src);
+    }
+  }
+}
+function serializeProfiledContent(document, content, profiles, removals) {
+  const { document: contentDocument } = parseHTML2(`<!doctype html><html><body><main data-feedloom-profile-root="true">${content}</main></body></html>`);
+  const root = contentDocument.querySelector('[data-feedloom-profile-root="true"]') ?? contentDocument.body;
+  appendMetaImages(document, root, profiles);
   applySiteProfiles(root, profiles, removals);
-  const serialized = root.innerHTML || root.outerHTML || document2.body.innerHTML;
+  const serialized = root.innerHTML || root.outerHTML || contentDocument.body.innerHTML;
   return serialized.trim() ? `${serialized.trim()}
 ` : "";
 }
@@ -1120,9 +1336,9 @@ var HtmlCleaner = class {
     const preferredContentSelector = this.options.contentSelector ?? firstContentSelector(activeProfiles);
     const removals = [];
     const html = /<html[\s>]/i.test(rawHtml) ? rawHtml : `<!doctype html><html><body>${rawHtml}</body></html>`;
-    const { document: document2 } = parseHTML2(html);
-    const contentSelector = preferredContentSelector && document2.querySelector(preferredContentSelector) ? preferredContentSelector : void 0;
-    const doc = document2;
+    const { document } = parseHTML2(html);
+    const contentSelector = preferredContentSelector && document.querySelector(preferredContentSelector) ? preferredContentSelector : void 0;
+    const doc = document;
     if (this.options.baseUrl) {
       doc.URL = this.options.baseUrl;
     }
@@ -1139,12 +1355,14 @@ var HtmlCleaner = class {
       removeExactSelectors: this.options.removeExactSelectors,
       removePartialSelectors: this.options.removePartialSelectors,
       removeContentPatterns: this.options.removeContentPatterns,
-      standardize: this.options.standardize
+      standardize: this.options.standardize,
+      fetch: this.options.defuddleFetch,
+      language: this.options.language
     });
     const result = parser2.parseAsync ? await parser2.parseAsync() : parser2.parse();
-    const metadata = toMetadata(result, document2, activeProfiles);
+    const metadata = toMetadata(result, document, activeProfiles);
     applyMetadataProfiles(metadata, activeProfiles);
-    const content = serializeProfiledContent(result.content, postProfiles, removals);
+    const content = serializeProfiledContent(document, result.content, postProfiles, removals);
     return {
       content,
       contentMarkdown: result.contentMarkdown,
@@ -1161,6 +1379,212 @@ async function cleanHtml(rawHtml, options = {}) {
   return new HtmlCleaner(options).parse(rawHtml);
 }
 
+// src/fetch/proxy-fetch.ts
+import { request as httpRequest } from "http";
+import { connect as tlsConnect } from "tls";
+var REDIRECT_STATUSES = /* @__PURE__ */ new Set([301, 302, 303, 307, 308]);
+var DEFAULT_REDIRECT_LIMIT = 10;
+function envProxyForUrl(targetUrl) {
+  const raw = targetUrl.protocol === "https:" ? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy : process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy;
+  if (!raw || noProxyMatches(targetUrl.hostname)) {
+    return null;
+  }
+  try {
+    return new URL(raw);
+  } catch {
+    return null;
+  }
+}
+function noProxyMatches(hostname) {
+  const raw = process.env.NO_PROXY ?? process.env.no_proxy ?? "";
+  if (!raw) return false;
+  const host = hostname.toLowerCase();
+  return raw.split(",").map((entry) => entry.trim().toLowerCase()).some((entry) => {
+    if (!entry) return false;
+    if (entry === "*") return true;
+    if (entry.startsWith(".")) return host === entry.slice(1) || host.endsWith(entry);
+    return host === entry || host.endsWith(`.${entry}`);
+  });
+}
+function headersToRecord(headers) {
+  const record = {};
+  if (!headers) return record;
+  new Headers(headers).forEach((value, key) => {
+    record[key] = value;
+  });
+  return record;
+}
+function responseHeaders(headers) {
+  const result = new Headers();
+  for (const [key, value] of Object.entries(headers)) {
+    if (Array.isArray(value)) {
+      for (const item of value) result.append(key, item);
+    } else if (value !== void 0) {
+      result.set(key, String(value));
+    }
+  }
+  return result;
+}
+async function bodyToBuffer(body) {
+  if (body === void 0 || body === null) return void 0;
+  if (typeof ReadableStream !== "undefined" && body instanceof ReadableStream) {
+    throw new Error("proxy-aware fetch does not support streaming request bodies");
+  }
+  if (typeof body === "string") return Buffer.from(body);
+  if (body instanceof URLSearchParams) return Buffer.from(body.toString());
+  if (body instanceof ArrayBuffer) return Buffer.from(body);
+  if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
+  if (typeof Blob !== "undefined" && body instanceof Blob) return Buffer.from(await body.arrayBuffer());
+  throw new Error("proxy-aware fetch only supports buffered request bodies");
+}
+function collectResponse(res, done) {
+  const chunks = [];
+  res.on("data", (chunk) => {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  });
+  res.on("end", () => {
+    done(null, {
+      status: res.statusCode ?? 0,
+      statusText: res.statusMessage ?? "",
+      headers: Object.fromEntries(responseHeaders(res.headers)),
+      body: Buffer.concat(chunks)
+    });
+  });
+  res.on("error", (error) => done(error));
+}
+function proxyAuthorization(proxy) {
+  if (!proxy.username) return void 0;
+  return `Basic ${Buffer.from(`${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`).toString("base64")}`;
+}
+function requestViaHttpProxy(targetUrl, proxy, method, headers, body, signal) {
+  if (proxy.protocol !== "http:") {
+    throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
+  }
+  return new Promise((resolve3, reject) => {
+    let settled = false;
+    let active = null;
+    const done = (error, response) => {
+      if (settled) return;
+      settled = true;
+      signal?.removeEventListener("abort", abort);
+      if (error) reject(error);
+      else if (response) resolve3(response);
+      else reject(new Error("Proxy request ended without a response"));
+    };
+    const abort = () => active?.destroy(new Error("The operation was aborted"));
+    if (signal?.aborted) {
+      done(new Error("The operation was aborted"));
+      return;
+    }
+    signal?.addEventListener("abort", abort, { once: true });
+    const targetPort = targetUrl.port ? Number(targetUrl.port) : targetUrl.protocol === "https:" ? 443 : 80;
+    const proxyPort = proxy.port ? Number(proxy.port) : 8080;
+    const auth = proxyAuthorization(proxy);
+    const requestHeaders2 = {
+      ...headers,
+      host: targetUrl.host
+    };
+    if (body && !Object.keys(requestHeaders2).some((key) => key.toLowerCase() === "content-length")) {
+      requestHeaders2["content-length"] = String(body.byteLength);
+    }
+    if (targetUrl.protocol === "https:") {
+      const connectHeaders = { host: `${targetUrl.hostname}:${targetPort}` };
+      if (auth) connectHeaders["proxy-authorization"] = auth;
+      const connectReq = httpRequest({
+        host: proxy.hostname,
+        port: proxyPort,
+        method: "CONNECT",
+        path: `${targetUrl.hostname}:${targetPort}`,
+        headers: connectHeaders
+      });
+      active = connectReq;
+      connectReq.on("connect", (connectRes, socket) => {
+        if (connectRes.statusCode !== 200) {
+          socket.destroy();
+          done(new Error(`Proxy CONNECT failed: ${connectRes.statusCode ?? 0}`));
+          return;
+        }
+        const tlsSocket = tlsConnect({ socket, host: targetUrl.hostname, servername: targetUrl.hostname });
+        active = tlsSocket;
+        tlsSocket.on("error", (error) => done(error));
+        tlsSocket.on("secureConnect", () => {
+          const req2 = httpRequest({
+            method,
+            path: `${targetUrl.pathname}${targetUrl.search}`,
+            headers: requestHeaders2,
+            createConnection: () => tlsSocket
+          }, (res) => collectResponse(res, done));
+          active = req2;
+          req2.on("error", (error) => done(error));
+          if (body) req2.write(body);
+          req2.end();
+        });
+      });
+      connectReq.on("error", (error) => done(error));
+      connectReq.end();
+      return;
+    }
+    if (auth) requestHeaders2["proxy-authorization"] = auth;
+    const req = httpRequest({
+      host: proxy.hostname,
+      port: proxyPort,
+      method,
+      path: targetUrl.href,
+      headers: requestHeaders2
+    }, (res) => collectResponse(res, done));
+    active = req;
+    req.on("error", (error) => done(error));
+    if (body) req.write(body);
+    req.end();
+  });
+}
+function requestUrl(input) {
+  if (input instanceof URL) return input;
+  if (typeof input === "string") return new URL(input);
+  return new URL(input.url);
+}
+function requestMethod(input, init) {
+  if (init.method) return init.method.toUpperCase();
+  if (input instanceof Request) return input.method.toUpperCase();
+  return "GET";
+}
+function requestHeaders(input, init) {
+  return {
+    ...input instanceof Request ? headersToRecord(input.headers) : {},
+    ...headersToRecord(init.headers)
+  };
+}
+async function proxyAwareFetchInternal(input, init, redirectsLeft) {
+  const url = requestUrl(input);
+  const proxy = envProxyForUrl(url);
+  if (!proxy) {
+    return fetch(input, init);
+  }
+  const method = requestMethod(input, init);
+  const headers = requestHeaders(input, init);
+  const body = await bodyToBuffer(init.body ?? (input instanceof Request ? input.body : void 0));
+  const proxied = await requestViaHttpProxy(url, proxy, method, headers, body, init.signal ?? void 0);
+  const location = proxied.headers.location;
+  const redirectMode = init.redirect ?? "follow";
+  if (redirectMode !== "manual" && REDIRECT_STATUSES.has(proxied.status) && location && redirectsLeft > 0) {
+    const nextUrl = new URL(location, url);
+    const nextInit = { ...init };
+    if (proxied.status === 303) {
+      nextInit.method = "GET";
+      nextInit.body = void 0;
+    }
+    return proxyAwareFetchInternal(nextUrl, nextInit, redirectsLeft - 1);
+  }
+  return new Response(new Uint8Array(proxied.body), {
+    status: proxied.status,
+    statusText: proxied.statusText,
+    headers: proxied.headers
+  });
+}
+async function proxyAwareFetch(input, init = {}) {
+  return proxyAwareFetchInternal(input, init, DEFAULT_REDIRECT_LIMIT);
+}
+
 // src/fetch/strategy.ts
 import { writeFile as writeFile3 } from "fs/promises";
 
@@ -1175,8 +1599,8 @@ function extractPreloadedMarkdownUrl(html, baseUrl) {
   }
   return new URL(rawUrl, baseUrl).toString();
 }
-function removeNoise(document2) {
-  document2.querySelectorAll("script, style, noscript, svg, iframe").forEach((element) => element.remove());
+function removeNoise(document) {
+  document.querySelectorAll("script, style, noscript, svg, iframe").forEach((element) => element.remove());
 }
 function normalizedTextLength(element) {
   return (element?.textContent ?? "").replace(/\s+/g, " ").trim().length;
@@ -1185,12 +1609,12 @@ function htmlHasMeaningfulContent(url, html) {
   if (extractPreloadedMarkdownUrl(html, url) !== null) {
     return true;
   }
-  const { document: document2 } = parseHTML3(html);
-  removeNoise(document2);
+  const { document } = parseHTML3(html);
+  removeNoise(document);
   const selectors = ["#js_content", "article", "main", "section", "div", "body"];
   let bestLength = 0;
   for (const selector of selectors) {
-    document2.querySelectorAll(selector).forEach((element) => {
+    document.querySelectorAll(selector).forEach((element) => {
       bestLength = Math.max(bestLength, normalizedTextLength(element));
     });
     if (bestLength >= 600 && selector !== "div") {
@@ -1285,11 +1709,11 @@ async function fetchBrowserHtmlWithBrowserState(url, config) {
 }
 
 // src/fetch/static.ts
-async function fetchStaticHtml(url, timeoutMs = 6e4) {
+async function fetchStaticHtml(url, timeoutMs = 6e4, fetchImpl = fetch) {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), timeoutMs);
   try {
-    const response = await fetch(url, {
+    const response = await fetchImpl(url, {
       redirect: "follow",
       signal: controller.signal,
       headers: {
@@ -1318,7 +1742,7 @@ async function writeOutputIfRequested(outputPath, html) {
 }
 async function fetchHtmlResult(url, options = {}) {
   const isMeaningful = options.isMeaningful ?? htmlHasMeaningfulContent;
-  const staticFetch = options.staticFetch ?? (async (targetUrl) => (await fetchStaticHtml(targetUrl)).html);
+  const staticFetch = options.staticFetch ?? (async (targetUrl) => (await fetchStaticHtml(targetUrl, void 0, options.useProxyEnv ? proxyAwareFetch : void 0)).html);
   const browserFetch = options.browserFetch ?? ((targetUrl) => fetchBrowserHtml(targetUrl, {
     waitMs: options.waitMs,
     networkIdle: options.networkIdle,
@@ -1486,9 +1910,9 @@ ${code}
 \`\`\``).replace(/\[\s*\]\((?:#|javascript:void\(0\)|javascript:;)\)/gi, "").replace(/(^|[^\\])\$(?=\d)/g, "$1\\$").replace(/\n\s*\n\s*([-*+]\s)/g, "\n$1").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
 }
 function htmlFragmentText(fragment) {
-  const { document: document2 } = parseHTML4(`<!doctype html><html><body>${fragment}</body></html>`);
-  document2.querySelectorAll("br").forEach((br) => br.replaceWith(document2.createTextNode("\n")));
-  return document2.body.textContent ?? "";
+  const { document } = parseHTML4(`<!doctype html><html><body>${fragment}</body></html>`);
+  document.querySelectorAll("br").forEach((br) => br.replaceWith(document.createTextNode("\n")));
+  return document.body.textContent ?? "";
 }
 function fencedCodeHtml(text) {
   const escaped = text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
@@ -1584,10 +2008,45 @@ function resolveCreatedValue(item, published) {
   if (item.publishedAt) return createdFromItemDate(item.publishedAt);
   return (/* @__PURE__ */ new Date()).toISOString().replace(/\.\d{3}Z$/, "Z");
 }
+function mergeProfileFetchOptions(options, profiles) {
+  const merged = { ...options };
+  for (const profile of profiles) {
+    if (profile.fetch?.mode) merged.fetchMode = profile.fetch.mode;
+    if (profile.fetch?.waitMs !== void 0) merged.waitMs = profile.fetch.waitMs;
+    if (profile.fetch?.networkIdle !== void 0) merged.networkIdle = profile.fetch.networkIdle;
+    if (profile.fetch?.waitSelector) merged.waitSelector = profile.fetch.waitSelector;
+    if (profile.fetch?.waitSelectorState) merged.waitSelectorState = profile.fetch.waitSelectorState;
+    if (profile.fetch?.clickSelectors) merged.clickSelectors = profile.fetch.clickSelectors;
+    if (profile.fetch?.scrollToBottom !== void 0) merged.scrollToBottom = profile.fetch.scrollToBottom;
+    if (profile.fetch?.useProxyEnv !== void 0) merged.useProxyEnv = profile.fetch.useProxyEnv;
+    if (profile.fetch?.preferBrowserState && options.browserStateDefaults) {
+      merged.browserState = {
+        ...options.browserStateDefaults,
+        waitMs: merged.waitMs,
+        networkIdle: merged.networkIdle,
+        proxy: merged.proxy,
+        dnsOverHttps: merged.dnsOverHttps,
+        waitSelector: merged.waitSelector,
+        waitSelectorState: merged.waitSelectorState,
+        clickSelectors: merged.clickSelectors,
+        scrollToBottom: merged.scrollToBottom,
+        headless: merged.headless,
+        realChromeDefaults: merged.realChromeDefaults
+      };
+    }
+  }
+  return merged;
+}
 async function processItem(item, options) {
-  const html = await fetchHtml(item.url, options);
+  const urlProfiles = selectActiveProfiles(options.profiles, item.url, "");
+  const fetchOptions = mergeProfileFetchOptions(options, urlProfiles);
+  const html = await fetchHtml(item.url, fetchOptions);
   const activeProfiles = selectActiveProfiles(options.profiles, item.url, html);
-  const cleaned = await cleanHtml(html, { baseUrl: item.url, profiles: options.profiles, activeProfiles });
+  const defuddleFetch = activeProfiles.some((profile) => profile.fetch?.useProxyEnv) ? proxyAwareFetch : void 0;
+  const cleaned = await cleanHtml(html, { baseUrl: item.url, profiles: options.profiles, activeProfiles, defuddleFetch });
+  if (activeProfiles.some((profile) => profile.extraction?.requireText) && !cleaned.content.replace(/<[^>]*>/g, "").trim()) {
+    throw new Error("matched site rule requires extracted text, but no text content was extracted");
+  }
   const title = cleaned.metadata.title || item.sourceTitle || titleFromUrl(item.url);
   await cleanupExistingNote(options.outputDir, item.url);
   const contentHtml = options.localizeAssets === false ? cleaned.content : await localizeImages(cleaned.content, {
@@ -1656,10 +2115,15 @@ var ProgressTracker = class {
 };
 
 // src/cli.ts
+var require2 = createRequire(import.meta.url);
+var packageJson = require2("../package.json");
 var program = new Command();
 async function siteRulePathsFromDir(dir) {
   const names = await readdir2(dir);
-  return names.filter((name) => name.endsWith(".toml")).map((name) => join7(dir, name));
+  return names.filter((name) => name.endsWith(".toml")).sort().map((name) => join7(dir, name));
+}
+function builtinSiteRulesDir() {
+  return join7(dirname(fileURLToPath(import.meta.url)), "site-rules");
 }
 function positiveIntOption(value, fallback) {
   const parsed = Number(value ?? fallback);
@@ -1668,7 +2132,13 @@ function positiveIntOption(value, fallback) {
   }
   return parsed;
 }
-program.name("feedloom").description("Archive long-form web content as clean Markdown with local assets").version("0.1.0").option("--output-dir <dir>", "Output directory for markdown notes", "clippings").option("--source-kind <kind>", "auto, html-page, or rss-feed", "auto").option("--since <date>", "Only keep feed entries on or after YYYY-MM-DD", "").option("--limit <n>", "Process only first N deduplicated URLs", "0").option("--start <n>", "Start from 1-based index after deduplication", "1").option("--end <n>", "End at 1-based index after deduplication", "0").option("--prefer-browser-state", "Try copied local Chrome profile before regular browser fallback", false).option("--chrome-user-data-dir <path>", "Chrome user data directory used with --prefer-browser-state", "").option("--chrome-profile <name>", "Chrome profile directory name", "Default").option("--fetch-mode <mode>", "auto, static, browser, or stealth", "auto").option("--no-network-idle", "Do not wait for browser networkidle before reading HTML").option("--wait-ms <ms>", "Extra browser wait after load", "2500").option("--solve-cloudflare", "In stealth mode, attempt Cloudflare Turnstile/interstitial challenge handling", false).option("--disable-resources", "In stealth mode, block images/media/fonts/stylesheets for speed", false).option("--proxy <server>", "Proxy server for browser/stealth fetch, e.g. http://127.0.0.1:8080", "").option("--dns-over-https", "Use Chromium Cloudflare DNS-over-HTTPS flag for browser/stealth fetch", false).option("--wait-selector <selector>", "Wait for a CSS selector after page load", "").option("--wait-selector-state <state>", "attached, detached, visible, or hidden", "attached").option("--click-selector <selector...>", "Click one or more selectors after page load", []).option("--scroll-to-bottom", "Scroll to the bottom before reading HTML", false).option("--headful", "Run browser/browser-state fetches with a visible Chrome window", false).option("--site-rules-dir <dir>", "Optional directory of private TOML site extraction/cleaning rules", "").option("--no-real-chrome-defaults", "Disable Scrapling-inspired real Chrome context defaults").option("--no-reuse-browser", "Disable batch browser/stealth context reuse").argument("[inputs...]", "URLs or files containing URLs").action(async (inputs, options) => {
+program.name("feedloom").description("Archive long-form web content as clean Markdown with local assets").version(packageJson.version ?? "0.0.0");
+program.command("doctor").description("Check Feedloom runtime dependencies").action(async () => {
+  const result = await runDoctor();
+  console.error(formatDoctorResult(result));
+  process.exitCode = result.ok ? 0 : 1;
+});
+program.option("--output-dir <dir>", "Output directory for markdown notes", "clippings").option("--source-kind <kind>", "auto, html-page, or rss-feed", "auto").option("--since <date>", "Only keep feed entries on or after YYYY-MM-DD", "").option("--limit <n>", "Process only first N deduplicated URLs", "0").option("--start <n>", "Start from 1-based index after deduplication", "1").option("--end <n>", "End at 1-based index after deduplication", "0").option("--prefer-browser-state", "Try copied local Chrome profile before regular browser fallback", false).option("--chrome-user-data-dir <path>", "Chrome user data directory used with --prefer-browser-state", "").option("--chrome-profile <name>", "Chrome profile directory name", "Default").option("--fetch-mode <mode>", "auto, static, browser, or stealth", "auto").option("--no-network-idle", "Do not wait for browser networkidle before reading HTML").option("--wait-ms <ms>", "Extra browser wait after load", "2500").option("--solve-cloudflare", "In stealth mode, attempt Cloudflare Turnstile/interstitial challenge handling", false).option("--disable-resources", "In stealth mode, block images/media/fonts/stylesheets for speed", false).option("--proxy <server>", "Proxy server for browser/stealth fetch, e.g. http://127.0.0.1:8080", "").option("--dns-over-https", "Use Chromium Cloudflare DNS-over-HTTPS flag for browser/stealth fetch", false).option("--wait-selector <selector>", "Wait for a CSS selector after page load", "").option("--wait-selector-state <state>", "attached, detached, visible, or hidden", "attached").option("--click-selector <selector...>", "Click one or more selectors after page load", []).option("--scroll-to-bottom", "Scroll to the bottom before reading HTML", false).option("--headful", "Run browser/browser-state fetches with a visible Chrome window", false).option("--site-rules-dir <dir>", "Optional directory of private TOML site extraction/cleaning rules", "").option("--no-real-chrome-defaults", "Disable Scrapling-inspired real Chrome context defaults").option("--no-reuse-browser", "Disable batch browser/stealth context reuse").argument("[inputs...]", "URLs or files containing URLs").action(async (inputs, options) => {
   if (inputs.length === 0) {
     program.help({ error: true });
   }
@@ -1696,7 +2166,9 @@ program.name("feedloom").description("Archive long-form web content as clean Mar
       positiveIntOption(options.limit, 0)
     );
     const siteRulesDir = String(options.siteRulesDir || "");
-    const profiles = siteRulesDir ? await loadSiteProfiles(await siteRulePathsFromDir(resolve2(siteRulesDir))) : [];
+    const builtinRulePaths = await siteRulePathsFromDir(builtinSiteRulesDir());
+    const customRulePaths = siteRulesDir ? await siteRulePathsFromDir(resolve2(siteRulesDir)) : [];
+    const profiles = await loadSiteProfiles([...builtinRulePaths, ...customRulePaths]);
     const outputDir = String(options.outputDir ?? "clippings");
     let failures = 0;
     const tracker = new ProgressTracker(selected, outputDir);
@@ -1715,6 +2187,10 @@ program.name("feedloom").description("Archive long-form web content as clean Mar
       headless: !Boolean(options.headful),
       realChromeDefaults: options.realChromeDefaults !== false
     };
+    const browserStateDefaults = {
+      userDataDir: String(options.chromeUserDataDir || ""),
+      profile: String(options.chromeProfile || "Default")
+    };
     const sessions = options.reuseBrowser === false ? null : new BatchFetchSessions({
       browser: browserOptions,
       stealth: {
@@ -1727,15 +2203,11 @@ program.name("feedloom").description("Archive long-form web content as clean Mar
       for (const item of selected) {
         tracker.start(item.url);
         try {
-          const browserState = options.preferBrowserState ? {
-            userDataDir: String(options.chromeUserDataDir || ""),
-            profile: String(options.chromeProfile || "Default"),
-            ...browserOptions
-          } : null;
           const result = await processItem(item, {
             outputDir,
             profiles,
-            browserState,
+            browserState: options.preferBrowserState ? { ...browserStateDefaults, ...browserOptions } : null,
+            browserStateDefaults,
             fetchMode,
             ...browserOptions,
             solveCloudflare: Boolean(options.solveCloudflare),

package/dist/site-rules/wechat.toml

@@ -0,0 +1,5 @@
+[match]
+host_suffixes = ["mp.weixin.qq.com"]
+
+[extract]
+selectors = ["#js_content"]

package/dist/site-rules/xiaohongshu.toml

@@ -0,0 +1,55 @@
+[match]
+host_suffixes = ["xiaohongshu.com"]
+
+[fetch]
+mode = "auto"
+scroll_to_bottom = true
+wait_ms = 5000
+
+[extract]
+selectors = [".note-content", "#noteContainer", ".note-container", ".note-detail"]
+
+[metadata]
+strip_title_regexes = ["\\s*-\\s*小红书\\s*$"]
+strip_author_regexes = ["关注$"]
+author_selectors = [".author-container .username", ".author-wrapper .name", ".user-name"]
+author_meta_names = ["author"]
+author_meta_properties = ["article:author"]
+
+[media]
+include_meta_images = true
+image_meta_properties = ["og:image"]
+
+[clean.remove]
+selectors = [
+  ".side-bar",
+  ".left-container",
+  ".comments-el",
+  ".interactions",
+  ".engage-bar",
+  ".note-detail-dropdown",
+  ".close-circle",
+  ".close-box",
+  ".login-container",
+  ".bottom-container .notedetail-menu",
+  ".author",
+  ".author-container",
+  ".media-container",
+  ".fraction",
+  ".arrow-controller",
+  ".pagination-media-container"
+]
+text_contains = [
+  "创作中心",
+  "业务合作",
+  "沪ICP备",
+  "营业执照",
+  "违法不良信息举报电话",
+  "行吟信息科技",
+  "小红书网页版",
+  "登录后推荐更懂你的笔记"
+]
+exact_text = ["关注", "加载中", "更多", "发现", "直播", "发布", "通知"]
+
+[clean.truncate]
+after_regexes = ["^共 \\d+ 条评论$", "^相关推荐$", "^登录后推荐更懂你的笔记$"]

package/dist/site-rules/youtube.toml

@@ -0,0 +1,10 @@
+[match]
+host_suffixes = ["youtube.com", "youtu.be"]
+url_regexes = ["https?://(www\\.)?youtube\\.com/watch\\?", "https?://youtu\\.be/"]
+
+[fetch]
+mode = "auto"
+use_proxy_env = true
+
+[extract]
+require_text = true

package/dist/site-rules/zhihu.toml

@@ -0,0 +1,22 @@
+[match]
+host_suffixes = ["zhihu.com"]
+
+[extract]
+selectors = ["[class*=\"Post-RichTextContainer\"]", "[class*=\"RichText ztext\"]", "[class*=\"RichContent-inner\"]"]
+
+[metadata]
+strip_title_regexes = ["\\s*-\\s*知乎\\s*$"]
+
+[fetch]
+mode = "browser"
+prefer_browser_state = true
+scroll_to_bottom = true
+wait_ms = 8000
+
+[clean.remove]
+class_contains = ["RichText-LinkCardContainer"]
+text_regexes = ["^目录收起$", "^目录收起.*References$", "^.+\\d+ 赞同 · \\d+ 评论 文章$", "^.+\\d+ 赞同 · \\d+ 评论 文章$", "^\\d+ 赞同 · \\d+ 评论 文章$"]
+exact_text = ["目录", "收起"]
+
+[clean.truncate]
+after_regexes = ["^发布于 ", "^赞同 ", "^\\d+ 条评论$", "^分享$", "^申请转载$", ".*的广告$"]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ariesfish/feedloom",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "type": "module",
   "author": "ariesfish",
   "license": "MIT",
@@ -17,6 +17,7 @@
     "feedloom": "dist/cli.js"
   },
   "files": [
+    "assets",
     "dist",
     "skills",
     "README.md",
@@ -24,7 +25,7 @@
   ],
   "scripts": {
     "dev": "tsx src/cli.ts",
-    "build": "tsup src/cli.ts --format esm --dts --clean",
+    "build": "tsup src/cli.ts --format esm --dts --clean && rm -rf dist/site-rules && cp -R src/site-rules dist/site-rules",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
     "prepublishOnly": "npm run typecheck && npm test && npm run build"

package/skills/feedloom/SKILL.md

@@ -22,6 +22,12 @@ npx -y @ariesfish/feedloom <inputs...> [options]
 
 ## Common usage
 
+Before clipping with browser-based fetch modes, run `doctor` once to verify and repair the Patchright Chromium runtime. If Chromium is missing, `doctor` automatically runs `npx patchright install chromium`.
+
+```bash
+npx -y @ariesfish/feedloom doctor
+```
+
 Before running Feedloom, check whether this skill directory has a `site-rules/` directory. If it exists, always pass it with `--site-rules-dir $HOME/.agents/skills/feedloom/site-rules`; do not omit available site rules.
 
 ```bash
@@ -61,11 +67,13 @@ Use the least expensive mode that works:
 - `--site-rules-dir <dir>`: load optional private TOML extraction/cleaning rules from a local directory, for example `$HOME/.agents/skills/feedloom/site-rules/` reference folder.
 - `--solve-cloudflare`, `--proxy <server>`, `--dns-over-https`: use only when stealth fetching needs them.
 
-Run `npx -y @ariesfish/feedloom --help` for the complete option list. Do not invent unsupported options.
+Run `npx -y @ariesfish/feedloom doctor` when browser, stealth, or auto fallback fails because Chromium is missing or cannot launch. Run `npx -y @ariesfish/feedloom --help` for the complete option list. Do not invent unsupported options.
+
+## Site rules
 
-## Private site rules
+Feedloom ships built-in TOML site rules in the package for common sites such as WeChat and Zhihu. These are loaded automatically; do not pass a special option for built-in rules.
 
-Site-specific TOML rules are optional in the package, but mandatory to use when present next to this skill. Always check for `$HOME/.agents/skills/feedloom/site-rules/` before clipping. If that directory exists, pass it explicitly on every Feedloom command using the `$HOME`-prefixed path:
+Private skill rules are also supported and are mandatory to use when present next to this skill. Always check for `$HOME/.agents/skills/feedloom/site-rules/` before clipping. If that directory exists, pass it explicitly on every Feedloom command using the `$HOME`-prefixed path:
 
 ```bash
 npx -y @ariesfish/feedloom "https://example.com/article" --site-rules-dir $HOME/.agents/skills/feedloom/site-rules
@@ -73,6 +81,8 @@ npx -y @ariesfish/feedloom "https://example.com/article" --site-rules-dir $HOME/
 
 Treat rule files in `$HOME/.agents/skills/feedloom/site-rules/` as local reference material and use them whenever available; never skip an existing site-rules directory unless the user explicitly asks not to use it.
 
+For adding or editing private rules, read `references/site-rules.md`. It contains the TOML schema, examples, `[fetch]` behavior, and validation workflow.
+
 ## Output
 
 - Markdown files are written to `clippings/` by default, or to `--output-dir`.

package/skills/feedloom/references/site-rules.md

@@ -0,0 +1,104 @@
+# Feedloom site rules
+
+Use TOML site rules when Feedloom needs a narrow site-specific selector, cleanup overlay, metadata normalization, or conservative fetch preference. Do not write ad-hoc scrapers.
+
+## Locations
+
+Private skill rules live in:
+
+```text
+$HOME/.agents/skills/feedloom/site-rules/<site>.toml
+```
+
+When the private rules directory exists, pass it on every command:
+
+```bash
+npx -y @ariesfish/feedloom "https://example.com/article" --site-rules-dir $HOME/.agents/skills/feedloom/site-rules
+```
+
+## Add a private rule
+
+Create or edit one TOML file per site:
+
+```bash
+mkdir -p $HOME/.agents/skills/feedloom/site-rules
+$EDITOR $HOME/.agents/skills/feedloom/site-rules/example.toml
+```
+
+Minimal rule:
+
+```toml
+[match]
+host_suffixes = ["example.com"]
+
+[extract]
+selectors = ["article", "main"]
+```
+
+Rule with fetch preferences:
+
+```toml
+[match]
+host_suffixes = ["zhihu.com"]
+
+[fetch]
+mode = "browser"
+prefer_browser_state = true
+scroll_to_bottom = true
+wait_ms = 8000
+
+[extract]
+selectors = ["[class*=\"Post-RichTextContainer\"]", "[class*=\"RichText ztext\"]"]
+```
+
+## Schema
+
+Supported sections:
+
+- `[match]`: `host_suffixes`, `host_regexes`, `url_regexes`, `html_markers`.
+- `[fetch]`: `mode`, `prefer_browser_state`, `wait_ms`, `network_idle`, `wait_selector`, `wait_selector_state`, `click_selectors`, `scroll_to_bottom`, `use_proxy_env`.
+- `[extract]`: `selectors`, `require_text`.
+- `[metadata]`: `fixed_author`, `strip_title_regexes`, `strip_author_regexes`, `author_selectors`, `author_meta_names`, `author_meta_itemprops`, `author_meta_properties`.
+- `[clean.remove]`: `selectors`, `class_contains`, `id_contains`, `attr_contains`, `text_contains`, `text_regexes`, `exact_text`.
+- `[clean.truncate]`: `after_contains`, `after_regexes`.
+
+## Fetch rules
+
+Use `[fetch]` only when a site consistently needs browser rendering, local Chrome state, scrolling, waiting, clicking, or proxy-aware requests.
+
+`use_proxy_env = true` tells Feedloom to use `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` for static fetches and Defuddle async extractor fetches. Use this for YouTube transcript capture and similar extractor-backed pages that need the user's proxy settings.
+
+`prefer_browser_state = true` only tells Feedloom to use copied Chrome state for matching URLs. It does not store the local Chrome path. The command still needs Chrome state parameters when login state is required:
+
+```bash
+npx -y @ariesfish/feedloom \
+  --chrome-user-data-dir "$HOME/Library/Application Support/Google/Chrome" \
+  --chrome-profile Default \
+  --site-rules-dir $HOME/.agents/skills/feedloom/site-rules \
+  "https://zhuanlan.zhihu.com/p/..."
+```
+
+## Rules for writing rules
+
+- Prefer narrow domain-specific selectors over broad selectors.
+- Prefer content containers over page shells. Avoid `body` unless the HTML is already minimal.
+- Use `require_text = true` when a matched extractor-backed page should fail instead of writing an empty note.
+- Use cleanup only for repeated, stable noise inside otherwise correct content.
+- Use truncation only for stable tail markers where everything after the marker is non-article content.
+- Do not add aggressive crawling, high concurrency, repeated challenge solving, or broad stealth defaults.
+- Keep private rules outside project repos unless the user is working on Feedloom itself.
+
+## Validation
+
+After adding or editing a private rule, test one known URL and inspect the Markdown:
+
+```bash
+outdir=$(mktemp -d /tmp/feedloom-rule-test-XXXXXX)
+npx -y @ariesfish/feedloom \
+  --output-dir "$outdir" \
+  --site-rules-dir $HOME/.agents/skills/feedloom/site-rules \
+  "https://example.com/article"
+find "$outdir" -maxdepth 2 -type f | sort
+```
+
+For sites that require Chrome state, add the Chrome state options shown above.