@ariesfish/feedloom 0.1.0

package/dist/cli.js

@@ -0,0 +1,1745 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { readdir as readdir2 } from "fs/promises";
+import { dirname, join as join7, resolve as resolve2 } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+
+// src/cleaning/profiles.ts
+import { readFile } from "fs/promises";
+import { parse } from "@iarna/toml";
+function partialAttributePatterns(rule) {
+  return [
+    ...rule.clean?.remove?.class_contains ?? [],
+    ...rule.clean?.remove?.id_contains ?? [],
+    ...rule.clean?.remove?.attr_contains ?? []
+  ];
+}
+function profileFromTomlRule(name, rule) {
+  return {
+    name,
+    match: {
+      hostSuffixes: rule.match?.host_suffixes,
+      hostRegexes: rule.match?.host_regexes,
+      urlRegexes: rule.match?.url_regexes,
+      htmlMarkers: rule.match?.html_markers
+    },
+    content: {
+      selectors: rule.extract?.selectors
+    },
+    removals: {
+      exactSelectors: rule.clean?.remove?.selectors,
+      partialAttributePatterns: partialAttributePatterns(rule),
+      textContains: rule.clean?.remove?.text_contains,
+      textRegexes: rule.clean?.remove?.text_regexes,
+      cutAfterContains: rule.clean?.truncate?.after_contains,
+      cutAfterRegexes: rule.clean?.truncate?.after_regexes,
+      dropExactText: rule.clean?.remove?.exact_text
+    },
+    metadata: {
+      fixedAuthor: rule.metadata?.fixed_author,
+      titleSuffixPatterns: rule.metadata?.strip_title_regexes
+    }
+  };
+}
+async function loadSiteProfiles(paths) {
+  const profiles = [];
+  for (const path of paths) {
+    const text = await readFile(path, "utf8");
+    const raw = parse(text);
+    const name = path.split(/[\\/]/).pop()?.replace(/\.toml$/i, "") || path;
+    profiles.push(profileFromTomlRule(name, raw));
+  }
+  return profiles;
+}
+function profileMatches(profile, url, html) {
+  const match = profile.match;
+  if (!match) {
+    return true;
+  }
+  if (url) {
+    const parsed = new URL(url);
+    const host = parsed.hostname.toLowerCase();
+    if (match.hostSuffixes?.some((suffix) => host.endsWith(suffix.toLowerCase()))) {
+      return true;
+    }
+    if (match.hostRegexes?.some((pattern) => new RegExp(pattern, "i").test(host))) {
+      return true;
+    }
+    if (match.urlRegexes?.some((pattern) => new RegExp(pattern, "i").test(url))) {
+      return true;
+    }
+  }
+  if (match.htmlMarkers?.some((marker) => html.includes(marker))) {
+    return true;
+  }
+  return !match.hostSuffixes?.length && !match.hostRegexes?.length && !match.urlRegexes?.length && !match.htmlMarkers?.length;
+}
+function selectActiveProfiles(profiles, url, html) {
+  return profiles?.filter((profile) => profileMatches(profile, url, html)) ?? [];
+}
+function firstContentSelector(profiles) {
+  for (const profile of profiles) {
+    const selector = profile.content?.selectors?.find((candidate) => candidate.trim());
+    if (selector) {
+      return selector;
+    }
+  }
+  return void 0;
+}
+
+// src/fetch/browser.ts
+import { mkdtemp, rm } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { chromium } from "patchright";
+var SCRAPLING_DEFAULT_ARGS = [
+  "--no-pings",
+  "--no-first-run",
+  "--disable-infobars",
+  "--disable-breakpad",
+  "--no-service-autorun",
+  "--homepage=about:blank",
+  "--password-store=basic",
+  "--disable-hang-monitor",
+  "--no-default-browser-check",
+  "--disable-session-crashed-bubble",
+  "--disable-search-engine-choice-screen"
+];
+var SCRAPLING_HARMFUL_ARGS = [
+  "--enable-automation",
+  "--disable-popup-blocking",
+  "--disable-component-update",
+  "--disable-default-apps",
+  "--disable-extensions"
+];
+async function runPageActions(page, options) {
+  for (const selector of options.clickSelectors ?? []) {
+    await page.locator(selector).first().click({ timeout: 5e3 }).catch(() => void 0);
+  }
+  if (options.scrollToBottom) {
+    await page.evaluate(async () => {
+      const delay = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
+      for (let i = 0; i < 8; i += 1) {
+        window.scrollTo(0, document.body.scrollHeight);
+        await delay(250);
+      }
+    });
+  }
+  if (options.waitSelector) {
+    await page.locator(options.waitSelector).first().waitFor({
+      state: options.waitSelectorState ?? "attached",
+      timeout: options.timeoutMs ?? 9e4
+    }).catch(() => void 0);
+  }
+}
+async function launchBrowserContext(options) {
+  const userDataDir = options.userDataDir ?? await mkdtemp(join(tmpdir(), "feedloom-browser-"));
+  const ownsUserDataDir = options.userDataDir === void 0;
+  const realChromeDefaults = options.realChromeDefaults ?? false;
+  const extraArgs = realChromeDefaults ? [.../* @__PURE__ */ new Set([...options.extraArgs ?? [], ...SCRAPLING_DEFAULT_ARGS])] : [...options.extraArgs ?? []];
+  if (options.dnsOverHttps) {
+    extraArgs.push("--dns-over-https-templates=https://cloudflare-dns.com/dns-query");
+  }
+  const context = await chromium.launchPersistentContext(userDataDir, {
+    channel: options.channel,
+    headless: options.headless ?? true,
+    args: extraArgs,
+    ignoreDefaultArgs: realChromeDefaults ? SCRAPLING_HARMFUL_ARGS : void 0,
+    proxy: options.proxy ? { server: options.proxy } : void 0,
+    ignoreHTTPSErrors: true,
+    colorScheme: realChromeDefaults ? "dark" : void 0,
+    deviceScaleFactor: realChromeDefaults ? 2 : void 0,
+    locale: void 0,
+    timezoneId: realChromeDefaults ? "" : void 0,
+    userAgent: realChromeDefaults ? "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" : void 0,
+    viewport: { width: 1365, height: 900 },
+    screen: { width: 1365, height: 900 }
+  });
+  return { context, userDataDir, ownsUserDataDir };
+}
+async function fetchWithContext(context, url, options) {
+  const timeoutMs = options.timeoutMs ?? 9e4;
+  const waitMs = options.waitMs ?? 2500;
+  const page = await context.newPage();
+  try {
+    await page.goto(url, { waitUntil: "load", timeout: timeoutMs, referer: options.referer ?? (options.realChromeDefaults ? "https://www.google.com/" : void 0) });
+    await page.waitForLoadState("domcontentloaded", { timeout: timeoutMs }).catch(() => void 0);
+    if (options.networkIdle ?? true) {
+      await page.waitForLoadState("networkidle", { timeout: timeoutMs }).catch(() => void 0);
+    }
+    await runPageActions(page, options);
+    if (waitMs > 0) {
+      await page.waitForTimeout(waitMs);
+    }
+    return await page.content();
+  } finally {
+    await page.close().catch(() => void 0);
+  }
+}
+var BrowserFetchSession = class {
+  constructor(options = {}) {
+    this.options = options;
+  }
+  options;
+  context = null;
+  userDataDir = "";
+  ownsUserDataDir = false;
+  async start() {
+    if (this.context) return;
+    const launched = await launchBrowserContext(this.options);
+    this.context = launched.context;
+    this.userDataDir = launched.userDataDir;
+    this.ownsUserDataDir = launched.ownsUserDataDir;
+  }
+  async fetch(url) {
+    await this.start();
+    if (!this.context) throw new Error("Browser context was not initialized");
+    return fetchWithContext(this.context, url, this.options);
+  }
+  async close() {
+    await this.context?.close().catch(() => void 0);
+    this.context = null;
+    if (this.ownsUserDataDir && this.userDataDir) {
+      await rm(this.userDataDir, { recursive: true, force: true });
+    }
+  }
+};
+async function fetchBrowserHtml(url, options = {}) {
+  const session = new BrowserFetchSession(options);
+  try {
+    return await session.fetch(url);
+  } finally {
+    await session.close();
+  }
+}
+
+// src/fetch/stealth.ts
+import { mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join2 } from "path";
+import { chromium as chromium2 } from "patchright";
+var DEFAULT_ARGS = [
+  "--no-pings",
+  "--no-first-run",
+  "--disable-infobars",
+  "--disable-breakpad",
+  "--no-service-autorun",
+  "--homepage=about:blank",
+  "--password-store=basic",
+  "--disable-hang-monitor",
+  "--no-default-browser-check",
+  "--disable-session-crashed-bubble",
+  "--disable-search-engine-choice-screen"
+];
+var STEALTH_ARGS = [
+  "--test-type",
+  "--lang=en-US",
+  "--mute-audio",
+  "--disable-sync",
+  "--hide-scrollbars",
+  "--disable-logging",
+  "--start-maximized",
+  "--enable-async-dns",
+  "--accept-lang=en-US",
+  "--use-mock-keychain",
+  "--disable-translate",
+  "--disable-voice-input",
+  "--window-position=0,0",
+  "--disable-wake-on-wifi",
+  "--ignore-gpu-blocklist",
+  "--enable-tcp-fast-open",
+  "--enable-web-bluetooth",
+  "--disable-cloud-import",
+  "--disable-print-preview",
+  "--disable-dev-shm-usage",
+  "--metrics-recording-only",
+  "--disable-crash-reporter",
+  "--disable-partial-raster",
+  "--disable-gesture-typing",
+  "--disable-checker-imaging",
+  "--disable-prompt-on-repost",
+  "--force-color-profile=srgb",
+  "--font-render-hinting=none",
+  "--aggressive-cache-discard",
+  "--disable-cookie-encryption",
+  "--disable-domain-reliability",
+  "--disable-threaded-animation",
+  "--disable-threaded-scrolling",
+  "--enable-simple-cache-backend",
+  "--disable-background-networking",
+  "--enable-surface-synchronization",
+  "--disable-image-animation-resync",
+  "--disable-renderer-backgrounding",
+  "--disable-ipc-flooding-protection",
+  "--prerender-from-omnibox=disabled",
+  "--safebrowsing-disable-auto-update",
+  "--disable-offer-upload-credit-cards",
+  "--disable-background-timer-throttling",
+  "--disable-new-content-rendering-timeout",
+  "--run-all-compositor-stages-before-draw",
+  "--disable-client-side-phishing-detection",
+  "--disable-backgrounding-occluded-windows",
+  "--disable-layer-tree-host-memory-pressure",
+  "--autoplay-policy=user-gesture-required",
+  "--disable-offer-store-unmasked-wallet-cards",
+  "--disable-blink-features=AutomationControlled",
+  "--disable-component-extensions-with-background-pages",
+  "--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance",
+  "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
+  "--disable-features=AudioServiceOutOfProcess,TranslateUI,BlinkGenPropertyTrees"
+];
+var HARMFUL_ARGS = [
+  "--enable-automation",
+  "--disable-popup-blocking",
+  "--disable-component-update",
+  "--disable-default-apps",
+  "--disable-extensions"
+];
+var EXTRA_RESOURCES = /* @__PURE__ */ new Set([
+  "font",
+  "image",
+  "media",
+  "beacon",
+  "object",
+  "imageset",
+  "texttrack",
+  "websocket",
+  "csp_report",
+  "stylesheet"
+]);
+function stealthArgs(options) {
+  const args = [...DEFAULT_ARGS, ...STEALTH_ARGS, ...options.extraArgs ?? []];
+  if (options.blockWebrtc) {
+    args.push("--webrtc-ip-handling-policy=disable_non_proxied_udp", "--force-webrtc-ip-handling-policy");
+  }
+  if (options.allowWebgl === false) {
+    args.push("--disable-webgl", "--disable-webgl-image-chromium", "--disable-webgl2");
+  }
+  if (options.hideCanvas) {
+    args.push("--fingerprinting-canvas-image-data-noise");
+  }
+  if (options.dnsOverHttps) {
+    args.push("--dns-over-https-templates=https://cloudflare-dns.com/dns-query");
+  }
+  return [...new Set(args)];
+}
+function shouldBlock(route, options) {
+  const request = route.request();
+  if (options.disableResources && EXTRA_RESOURCES.has(request.resourceType())) return true;
+  const host = new URL(request.url()).hostname;
+  return options.blockedDomains?.some((domain) => host === domain || host.endsWith(`.${domain}`)) ?? false;
+}
+function cloudflareChallengeType(html) {
+  for (const type of ["non-interactive", "managed", "interactive"]) {
+    if (html.includes(`cType: '${type}'`)) return type;
+  }
+  if (/challenges\.cloudflare\.com\/turnstile\/v/i.test(html)) return "embedded";
+  if (html.includes("<title>Just a moment...</title>")) return "managed";
+  return null;
+}
+async function solveCloudflare(page) {
+  let html = await page.content();
+  let challenge = cloudflareChallengeType(html);
+  if (!challenge) return;
+  for (let attempt = 0; attempt < 3 && challenge; attempt += 1) {
+    if (challenge === "non-interactive") {
+      await page.waitForTimeout(1e3);
+    } else {
+      const box = await page.locator("#cf_turnstile div, #cf-turnstile div, .turnstile>div>div, .main-content p+div>div>div").last().boundingBox().catch(() => null);
+      if (box) {
+        await page.mouse.click(box.x + 27, box.y + 26, { delay: 150, button: "left" });
+      } else {
+        await page.waitForTimeout(1e3);
+      }
+    }
+    await page.waitForLoadState("networkidle", { timeout: 1e4 }).catch(() => void 0);
+    html = await page.content();
+    challenge = cloudflareChallengeType(html);
+  }
+}
+async function launchStealthContext(options) {
+  const userDataDir = options.userDataDir ?? await mkdtemp2(join2(tmpdir2(), "feedloom-stealth-"));
+  const ownsUserDataDir = options.userDataDir === void 0;
+  const context = await chromium2.launchPersistentContext(userDataDir, {
+    channel: "chromium",
+    headless: options.headless ?? true,
+    args: stealthArgs(options),
+    ignoreDefaultArgs: HARMFUL_ARGS,
+    proxy: options.proxy ? { server: options.proxy } : void 0,
+    ignoreHTTPSErrors: true,
+    colorScheme: "dark",
+    deviceScaleFactor: 2,
+    isMobile: false,
+    hasTouch: false,
+    serviceWorkers: "allow",
+    screen: { width: 1920, height: 1080 },
+    viewport: { width: 1920, height: 1080 },
+    permissions: ["geolocation", "notifications"],
+    locale: options.locale,
+    timezoneId: options.timezoneId,
+    userAgent: options.userAgent,
+    extraHTTPHeaders: options.extraHeaders
+  });
+  return { context, userDataDir, ownsUserDataDir };
+}
+async function fetchWithStealthContext(context, url, options) {
+  const timeoutMs = options.timeoutMs ?? (options.solveCloudflare ? 6e4 : 3e4);
+  const waitMs = options.waitMs ?? 0;
+  const page = await context.newPage();
+  try {
+    page.setDefaultNavigationTimeout(timeoutMs);
+    page.setDefaultTimeout(timeoutMs);
+    if (options.disableResources || options.blockedDomains?.length) {
+      await page.route("**/*", async (route) => shouldBlock(route, options) ? route.abort() : route.continue());
+    }
+    await page.goto(url, { waitUntil: "load", timeout: timeoutMs, referer: options.extraHeaders?.referer ?? "https://www.google.com/" });
+    await page.waitForLoadState("domcontentloaded", { timeout: timeoutMs }).catch(() => void 0);
+    if (options.networkIdle) {
+      await page.waitForLoadState("networkidle", { timeout: timeoutMs }).catch(() => void 0);
+    }
+    if (options.solveCloudflare) {
+      await solveCloudflare(page);
+    }
+    for (const selector of options.clickSelectors ?? []) {
+      await page.locator(selector).first().click({ timeout: 5e3 }).catch(() => void 0);
+    }
+    if (options.scrollToBottom) {
+      await page.evaluate(async () => {
+        const delay = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
+        for (let i = 0; i < 8; i += 1) {
+          window.scrollTo(0, document.body.scrollHeight);
+          await delay(250);
+        }
+      });
+    }
+    if (options.waitSelector) {
+      await page.locator(options.waitSelector).first().waitFor({ state: options.waitSelectorState ?? "attached", timeout: timeoutMs }).catch(() => void 0);
+    }
+    if (waitMs > 0) {
+      await page.waitForTimeout(waitMs);
+    }
+    return await page.content();
+  } finally {
+    await page.close().catch(() => void 0);
+  }
+}
+var StealthFetchSession = class {
+  constructor(options = {}) {
+    this.options = options;
+  }
+  options;
+  context = null;
+  userDataDir = "";
+  ownsUserDataDir = false;
+  async start() {
+    if (this.context) return;
+    const launched = await launchStealthContext(this.options);
+    this.context = launched.context;
+    this.userDataDir = launched.userDataDir;
+    this.ownsUserDataDir = launched.ownsUserDataDir;
+  }
+  async fetch(url) {
+    await this.start();
+    if (!this.context) throw new Error("Stealth context was not initialized");
+    return fetchWithStealthContext(this.context, url, this.options);
+  }
+  async close() {
+    await this.context?.close().catch(() => void 0);
+    this.context = null;
+    if (this.ownsUserDataDir && this.userDataDir) {
+      await rm2(this.userDataDir, { recursive: true, force: true });
+    }
+  }
+};
+async function fetchStealthHtml(url, options = {}) {
+  const session = new StealthFetchSession(options);
+  try {
+    return await session.fetch(url);
+  } finally {
+    await session.close();
+  }
+}
+
+// src/fetch/batch.ts
+var BatchFetchSessions = class {
+  constructor(options = {}) {
+    this.options = options;
+  }
+  options;
+  browserSession = null;
+  stealthSession = null;
+  async browserFetch(url) {
+    this.browserSession ??= new BrowserFetchSession(this.options.browser);
+    return this.browserSession.fetch(url);
+  }
+  async stealthFetch(url) {
+    this.stealthSession ??= new StealthFetchSession(this.options.stealth);
+    return this.stealthSession.fetch(url);
+  }
+  async close() {
+    await Promise.all([
+      this.browserSession?.close(),
+      this.stealthSession?.close()
+    ]);
+    this.browserSession = null;
+    this.stealthSession = null;
+  }
+};
+
+// src/input/inputs.ts
+import { readFile as readFile2, stat, writeFile } from "fs/promises";
+import { resolve } from "path";
+
+// src/constants.ts
+var URL_RE = /https?:\/\/[^\s)\]}>"']+/;
+
+// src/models.ts
+function makeUrlItem(url, overrides = {}) {
+  return {
+    url,
+    sourceKind: "html-page",
+    ...overrides
+  };
+}
+
+// src/input/inputs.ts
+var CheckboxFile = class _CheckboxFile {
+  path;
+  lines;
+  dirty = false;
+  constructor(path, lines) {
+    this.path = path;
+    this.lines = lines;
+  }
+  static async load(path) {
+    const text = await readFile2(path, "utf8");
+    return new _CheckboxFile(path, text.split(/\r?\n/).filter((_, index, lines) => index < lines.length - 1 || lines[index] !== ""));
+  }
+  markDone(lineNo, url) {
+    if (lineNo === void 0 || lineNo < 1 || lineNo > this.lines.length) {
+      return;
+    }
+    const line = this.lines[lineNo - 1] ?? "";
+    if (line.includes(url) && line.includes("- [ ] ")) {
+      this.lines[lineNo - 1] = line.replace("- [ ] ", "- [x] ");
+      this.dirty = true;
+    }
+  }
+  async save() {
+    if (this.dirty) {
+      await writeFile(this.path, `${this.lines.join("\n")}
+`, "utf8");
+    }
+  }
+};
+async function isFile(path) {
+  try {
+    return (await stat(path)).isFile();
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return false;
+    }
+    throw error;
+  }
+}
+async function parseInputs(inputs) {
+  const checkboxFiles = /* @__PURE__ */ new Map();
+  const items = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const raw of inputs) {
+    const path = resolve(raw.replace(/^~(?=$|\/)/, process.env.HOME ?? "~"));
+    if (await isFile(path)) {
+      const checkbox = await CheckboxFile.load(path);
+      checkboxFiles.set(path, checkbox);
+      checkbox.lines.forEach((line, index) => {
+        const match2 = URL_RE.exec(line);
+        if (!match2) {
+          return;
+        }
+        const url2 = match2[0];
+        if (seen.has(url2)) {
+          return;
+        }
+        seen.add(url2);
+        items.push(makeUrlItem(url2, { sourcePath: path, lineNo: index + 1 }));
+      });
+      continue;
+    }
+    const match = URL_RE.exec(raw);
+    if (!match) {
+      throw new Error(`Unsupported input: ${raw}`);
+    }
+    const url = match[0];
+    if (seen.has(url)) {
+      continue;
+    }
+    seen.add(url);
+    items.push(makeUrlItem(url));
+  }
+  if (items.length === 0) {
+    throw new Error("No URLs found in input");
+  }
+  return { items, checkboxFiles };
+}
+function sliceItems(items, start, end, limit) {
+  if (start < 1) {
+    throw new Error("--start must be at least 1");
+  }
+  if (end < 0) {
+    throw new Error("--end must be 0 or a positive 1-based index");
+  }
+  if (limit < 0) {
+    throw new Error("--limit must be 0 or a positive integer");
+  }
+  if (end !== 0 && end < start) {
+    throw new Error("--end must be greater than or equal to --start");
+  }
+  const begin = Math.max(start - 1, 0);
+  const result = items.slice(begin, end || void 0);
+  return limit > 0 ? result.slice(0, limit) : result;
+}
+
+// src/input/sources.ts
+import { XMLParser } from "fast-xml-parser";
+var FEED_HINT_RE = /(?:^|\/)(?:feed|rss|atom)(?:$|[./?])/i;
+function looksLikeFeedUrl(url) {
+  const parsed = new URL(url);
+  const path = parsed.pathname.toLowerCase();
+  return FEED_HINT_RE.test(path) || path.endsWith(".xml") || path.endsWith(".rss") || path.endsWith(".atom") || parsed.search.slice(1).toLowerCase().startsWith("feed=");
+}
+async function fetchSourceText(url, timeoutMs = 6e4) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const response = await fetch(url, {
+      redirect: "follow",
+      signal: controller.signal,
+      headers: { "user-agent": "Mozilla/5.0" }
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status} ${response.statusText}`);
+    }
+    return await response.text();
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+function parseDateTime(value) {
+  const raw = value?.trim();
+  if (!raw) {
+    return void 0;
+  }
+  const normalized = raw.endsWith("Z") ? raw : raw;
+  const timestamp = Date.parse(normalized);
+  if (Number.isNaN(timestamp)) {
+    return void 0;
+  }
+  return new Date(timestamp);
+}
+function parseSinceDate(value) {
+  const raw = value.trim();
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(raw)) {
+    throw new Error(`Invalid --since date: ${value}. Use YYYY-MM-DD.`);
+  }
+  const date = /* @__PURE__ */ new Date(`${raw}T00:00:00.000Z`);
+  if (Number.isNaN(date.getTime())) {
+    throw new Error(`Invalid --since date: ${value}. Use YYYY-MM-DD.`);
+  }
+  return date;
+}
+function asObject(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function asArray(value) {
+  if (value === void 0 || value === null) {
+    return [];
+  }
+  return Array.isArray(value) ? value : [value];
+}
+function textValue(value) {
+  if (value === void 0 || value === null) {
+    return "";
+  }
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return String(value).trim();
+  }
+  const object = asObject(value);
+  if (!object) {
+    return "";
+  }
+  const text = object["#text"] ?? object.text;
+  return textValue(text);
+}
+function childText(node, ...names) {
+  for (const name of names) {
+    const value = node[name];
+    const text = textValue(value);
+    if (text) {
+      return text;
+    }
+  }
+  return "";
+}
+function atomEntryLink(entry) {
+  for (const rawLink of asArray(entry.link)) {
+    if (typeof rawLink === "string") {
+      return rawLink.trim();
+    }
+    const link = asObject(rawLink);
+    if (!link) {
+      continue;
+    }
+    const href = textValue(link.href ?? link["@_href"]);
+    const rel = textValue(link.rel ?? link["@_rel"] ?? "alternate").toLowerCase();
+    if (href && (rel === "" || rel === "alternate")) {
+      return href;
+    }
+  }
+  return "";
+}
+function entryPublishedAt(node) {
+  return parseDateTime(childText(node, "published", "updated", "pubDate", "date"));
+}
+var parser = new XMLParser({
+  ignoreAttributes: false,
+  attributeNamePrefix: "@_",
+  removeNSPrefix: true,
+  trimValues: true
+});
+function parseFeedEntries(xmlText, sourceUrl) {
+  const parsed = parser.parse(xmlText);
+  if (asObject(parsed.rss)) {
+    const rss = asObject(parsed.rss);
+    const channel = asObject(rss.channel);
+    if (!channel) {
+      return [];
+    }
+    const feedTitle = childText(channel, "title");
+    return asArray(channel.item).map((rawItem) => asObject(rawItem)).filter((item) => item !== void 0).map((item) => {
+      const link = childText(item, "link");
+      if (!link) {
+        return void 0;
+      }
+      return makeUrlItem(new URL(link, sourceUrl).toString(), {
+        inputUrl: sourceUrl,
+        sourceKind: "html-page",
+        discoveredFrom: sourceUrl,
+        sourceTitle: feedTitle || childText(item, "title"),
+        publishedAt: entryPublishedAt(item)
+      });
+    }).filter((item) => item !== void 0);
+  }
+  if (asObject(parsed.feed)) {
+    const feed = asObject(parsed.feed);
+    const feedTitle = childText(feed, "title");
+    return asArray(feed.entry).map((rawEntry) => asObject(rawEntry)).filter((entry) => entry !== void 0).map((entry) => {
+      const link = atomEntryLink(entry);
+      if (!link) {
+        return void 0;
+      }
+      return makeUrlItem(new URL(link, sourceUrl).toString(), {
+        inputUrl: sourceUrl,
+        sourceKind: "html-page",
+        discoveredFrom: sourceUrl,
+        sourceTitle: feedTitle || childText(entry, "title"),
+        publishedAt: entryPublishedAt(entry)
+      });
+    }).filter((item) => item !== void 0);
+  }
+  throw new Error("Unsupported feed format");
+}
+async function expandSourceItems(items, sourceKind, since, options = {}) {
+  const expanded = [];
+  const seen = /* @__PURE__ */ new Set();
+  const fetchSource = options.fetchSource ?? fetchSourceText;
+  for (const item of items) {
+    const kinds = sourceKind === "auto" ? looksLikeFeedUrl(item.url) ? ["rss-feed", "html-page"] : ["html-page"] : [sourceKind];
+    let produced;
+    let lastError;
+    for (const kind of kinds) {
+      try {
+        if (kind === "html-page") {
+          produced = [item];
+          break;
+        }
+        if (kind === "rss-feed") {
+          if (sourceKind === "auto" && !looksLikeFeedUrl(item.url)) {
+            continue;
+          }
+          const xmlText = await fetchSource(item.url);
+          produced = parseFeedEntries(xmlText, item.url);
+          if (produced.length > 0) {
+            break;
+          }
+        }
+      } catch (error) {
+        lastError = error;
+      }
+    }
+    if (produced === void 0) {
+      if (sourceKind === "rss-feed" && lastError !== void 0) {
+        throw lastError;
+      }
+      produced = [item];
+    }
+    for (const producedItem of produced) {
+      producedItem.sourcePath = item.sourcePath;
+      producedItem.lineNo = item.lineNo;
+      producedItem.inputUrl = producedItem.inputUrl ?? item.url;
+      if (since && producedItem.publishedAt && producedItem.publishedAt < since) {
+        continue;
+      }
+      if (seen.has(producedItem.url)) {
+        continue;
+      }
+      seen.add(producedItem.url);
+      expanded.push(producedItem);
+    }
+  }
+  return expanded;
+}
+
+// src/assets.ts
+import { mkdir, writeFile as writeFile2 } from "fs/promises";
+import { extname, join as join3 } from "path";
+import { parseHTML } from "linkedom";
+function extensionFrom(contentType, url) {
+  const pathExt = extname(new URL(url).pathname).replace(/[^.a-z0-9]/gi, "");
+  if (pathExt && pathExt.length <= 8) return pathExt;
+  if (contentType?.includes("png")) return ".png";
+  if (contentType?.includes("webp")) return ".webp";
+  if (contentType?.includes("gif")) return ".gif";
+  return ".jpg";
+}
+function imageSource(img) {
+  const direct = img.getAttribute("data-original") || img.getAttribute("data-src") || img.getAttribute("src");
+  if (direct) return direct;
+  const srcset = img.getAttribute("data-srcset") || img.getAttribute("srcset");
+  const first = srcset?.split(",").map((part) => part.trim().split(/\s+/)[0]).find(Boolean);
+  return first || null;
+}
+async function localizeImages(html, options) {
+  const { document: document2 } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
+  const images = Array.from(document2.querySelectorAll("img"));
+  if (images.length === 0) return html;
+  const fetchImage = options.fetchImage ?? fetch;
+  const seen = /* @__PURE__ */ new Map();
+  let index = 1;
+  const assetDir = join3(options.outputDir, "assets", options.noteSlug);
+  for (const img of images) {
+    const raw = imageSource(img);
+    if (!raw || raw.startsWith("data:") || raw.startsWith("blob:")) continue;
+    let absolute;
+    try {
+      absolute = new URL(raw, options.baseUrl).toString();
+    } catch {
+      continue;
+    }
+    let rel = seen.get(absolute);
+    if (!rel) {
+      const response = await fetchImage(absolute);
+      if (!response.ok) continue;
+      const contentType = response.headers.get("content-type");
+      if (contentType && !contentType.toLowerCase().startsWith("image/")) continue;
+      const ext = extensionFrom(contentType, absolute);
+      const filename = `image-${String(index).padStart(3, "0")}${ext}`;
+      index += 1;
+      await mkdir(assetDir, { recursive: true });
+      await writeFile2(join3(assetDir, filename), new Uint8Array(await response.arrayBuffer()));
+      rel = `assets/${encodeURIComponent(options.noteSlug)}/${filename}`;
+      seen.set(absolute, rel);
+    }
+    img.setAttribute("src", rel);
+    const alt = img.getAttribute("alt")?.trim().toLowerCase();
+    if (alt === "image" || alt === "\u56FE\u50CF" || alt === "\u56FE\u7247") {
+      img.setAttribute("alt", "");
+    }
+    img.removeAttribute("srcset");
+    img.removeAttribute("data-srcset");
+    img.removeAttribute("data-original");
+    img.removeAttribute("data-src");
+  }
+  return document2.body.innerHTML;
+}
+
+// src/cleaning/clean-html.ts
+import * as DefuddleModule from "defuddle";
+import { parseHTML as parseHTML2 } from "linkedom";
+
+// src/cleaning/profile-dom.ts
+function textPreview(element) {
+  return (element.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, 160);
+}
+function recordRemoval(removals, step, reason, element, selector) {
+  removals.push({ step, reason, selector, text: textPreview(element) });
+}
+function removeElement(removals, step, reason, element, selector) {
+  recordRemoval(removals, step, reason, element, selector);
+  element.remove();
+}
+function removeByExactSelectors(root, profiles, removals) {
+  for (const profile of profiles) {
+    for (const selector of profile.removals?.exactSelectors ?? []) {
+      root.querySelectorAll(selector).forEach((element) => {
+        removeElement(removals, "site-profile:exact-selector", profile.name, element, selector);
+      });
+    }
+  }
+}
+function removeByPartialAttributePatterns(root, profiles, removals) {
+  const patterns = profiles.flatMap(
+    (profile) => (profile.removals?.partialAttributePatterns ?? []).map((pattern) => ({ profile: profile.name, regex: new RegExp(pattern, "i") }))
+  );
+  if (patterns.length === 0) {
+    return;
+  }
+  root.querySelectorAll("*").forEach((element) => {
+    if (element.closest("pre, code, table, figure")) {
+      return;
+    }
+    const attrs = [
+      element.id,
+      element.getAttribute("class") ?? "",
+      element.getAttribute("data-component") ?? "",
+      element.getAttribute("data-test") ?? "",
+      element.getAttribute("data-testid") ?? "",
+      element.getAttribute("data-qa") ?? "",
+      element.getAttribute("data-cy") ?? ""
+    ].join(" ");
+    const matched = patterns.find((pattern) => pattern.regex.test(attrs));
+    if (matched) {
+      removeElement(removals, "site-profile:partial-attribute", matched.profile, element);
+    }
+  });
+}
+function removeTrailingSiblings(element, removals, reason) {
+  let sibling = element.nextElementSibling;
+  while (sibling) {
+    const next = sibling.nextElementSibling;
+    removeElement(removals, "site-profile:content-pattern", reason, sibling);
+    sibling = next;
+  }
+}
+function compileProfileRegexes(profiles, key) {
+  return profiles.flatMap(
+    (profile) => (profile.removals?.[key] ?? []).map((pattern) => ({ profile: profile.name, regex: new RegExp(pattern, "i") }))
+  );
+}
+function removeByTextPatterns(root, profiles, removals) {
+  const textContains = profiles.flatMap(
+    (profile) => (profile.removals?.textContains ?? []).map((marker) => ({ profile: profile.name, marker }))
+  );
+  const cutContains = profiles.flatMap(
+    (profile) => (profile.removals?.cutAfterContains ?? []).map((marker) => ({ profile: profile.name, marker }))
+  );
+  const dropExact = /* @__PURE__ */ new Map();
+  for (const profile of profiles) {
+    for (const value of profile.removals?.dropExactText ?? []) {
+      dropExact.set(value.trim(), profile.name);
+    }
+  }
+  const textRegexes = compileProfileRegexes(profiles, "textRegexes");
+  const dropRegexes = compileProfileRegexes(profiles, "dropTextRegexes");
+  const cutRegexes = compileProfileRegexes(profiles, "cutAfterRegexes");
+  root.querySelectorAll("p, div, section, aside, footer, header, li, h1, h2, h3, h4, h5, h6").forEach((element) => {
+    if (element.closest("pre, code, table, figure")) {
+      return;
+    }
+    const text = (element.textContent ?? "").replace(/\s+/g, " ").trim();
+    if (!text) {
+      return;
+    }
+    const cut = text.length <= 240 ? cutContains.find((entry) => text.includes(entry.marker)) ?? cutRegexes.find((entry) => entry.regex.test(text)) : void 0;
+    if (cut) {
+      removeTrailingSiblings(element, removals, cut.profile);
+      removeElement(removals, "site-profile:content-pattern", cut.profile, element);
+      return;
+    }
+    const exactProfile = dropExact.get(text);
+    const matchedContains = textContains.find(
+      (entry) => text === entry.marker || text.length <= Math.max(entry.marker.length * 3, 120) && text.includes(entry.marker)
+    );
+    const matched = (exactProfile ? { profile: exactProfile } : void 0) ?? matchedContains ?? textRegexes.find((entry) => entry.regex.test(text)) ?? dropRegexes.find((entry) => entry.regex.test(text));
+    if (matched) {
+      removeElement(removals, "site-profile:content-pattern", matched.profile, element);
+    }
+  });
+}
+function applyFixedAuthor(metadata, profiles) {
+  for (const profile of profiles) {
+    if (profile.metadata?.fixedAuthor) {
+      metadata.author = profile.metadata.fixedAuthor;
+    }
+  }
+}
+function cleanupTitle(metadata, profiles) {
+  if (!metadata.title) {
+    return;
+  }
+  let title = metadata.title;
+  for (const profile of profiles) {
+    for (const pattern of profile.metadata?.titleSuffixPatterns ?? []) {
+      title = title.replace(new RegExp(pattern, "i"), "").trim();
+    }
+  }
+  metadata.title = title;
+}
+function applySiteProfiles(root, profiles, removals) {
+  removeByExactSelectors(root, profiles, removals);
+  removeByPartialAttributePatterns(root, profiles, removals);
+  removeByTextPatterns(root, profiles, removals);
+}
+function applyMetadataProfiles(metadata, profiles) {
+  applyFixedAuthor(metadata, profiles);
+  cleanupTitle(metadata, profiles);
+}
+
+// src/cleaning/clean-html.ts
+var DEFAULT_FEEDLOOM_PROFILE = {
+  name: "feedloom-default",
+  removals: {
+    exactSelectors: [
+      "script",
+      "style",
+      "noscript",
+      ".share-buttons",
+      ".social-share",
+      ".newsletter",
+      ".subscribe",
+      ".related",
+      ".comments"
+    ],
+    partialAttributePatterns: ["share", "newsletter", "subscribe", "related", "comment"]
+  }
+};
+var DefuddleClass = DefuddleModule.default ?? DefuddleModule.Defuddle;
+function firstMetaContent(document2, names) {
+  for (const name of names) {
+    const escaped = name.replace(/"/g, '\\"');
+    const element = document2.querySelector(`meta[property="${escaped}"], meta[name="${escaped}"], meta[itemprop="${escaped}"]`);
+    const content = element?.getAttribute("content")?.trim();
+    if (content) return content;
+  }
+  return void 0;
+}
+function jsonLdValue(document2, keys) {
+  for (const script of Array.from(document2.querySelectorAll('script[type="application/ld+json"]'))) {
+    const text = script.textContent?.trim();
+    if (!text) continue;
+    try {
+      const parsed = JSON.parse(text);
+      const nodes = Array.isArray(parsed) ? parsed : [parsed];
+      for (const node of nodes) {
+        if (!node || typeof node !== "object") continue;
+        for (const key of keys) {
+          const value = node[key];
+          if (typeof value === "string" && value.trim()) return value.trim();
+          if (value && typeof value === "object" && typeof value.name === "string") {
+            return String(value.name).trim();
+          }
+        }
+      }
+    } catch {
+    }
+  }
+  return void 0;
+}
+function toMetadata(result, document2) {
+  return {
+    title: result.title || firstMetaContent(document2, ["og:title", "twitter:title"]) || document2.querySelector("title")?.textContent?.trim() || void 0,
+    description: result.description || firstMetaContent(document2, ["description", "og:description", "twitter:description"]),
+    domain: result.domain || void 0,
+    favicon: result.favicon || void 0,
+    image: result.image || firstMetaContent(document2, ["og:image", "twitter:image"]),
+    language: result.language || document2.documentElement.getAttribute("lang") || void 0,
+    published: result.published || firstMetaContent(document2, ["article:published_time", "date", "datePublished", "pubdate", "publishdate"]) || jsonLdValue(document2, ["datePublished", "dateCreated"]),
+    author: result.author || firstMetaContent(document2, ["author", "article:author", "twitter:creator"]) || jsonLdValue(document2, ["author", "creator"]),
+    site: result.site || firstMetaContent(document2, ["og:site_name", "application-name"]),
+    schemaOrgData: result.schemaOrgData,
+    wordCount: result.wordCount,
+    parseTime: result.parseTime
+  };
+}
+function serializeProfiledContent(content, profiles, removals) {
+  const { document: document2 } = parseHTML2(`<!doctype html><html><body><main data-feedloom-profile-root="true">${content}</main></body></html>`);
+  const root = document2.querySelector('[data-feedloom-profile-root="true"]') ?? document2.body;
+  applySiteProfiles(root, profiles, removals);
+  const serialized = root.innerHTML || root.outerHTML || document2.body.innerHTML;
+  return serialized.trim() ? `${serialized.trim()}
+` : "";
+}
+var HtmlCleaner = class {
+  constructor(options = {}) {
+    this.options = options;
+  }
+  options;
+  async parse(rawHtml) {
+    const activeProfiles = this.options.activeProfiles ?? selectActiveProfiles(this.options.profiles, this.options.baseUrl, rawHtml);
+    const postProfiles = [DEFAULT_FEEDLOOM_PROFILE, ...activeProfiles];
+    const preferredContentSelector = this.options.contentSelector ?? firstContentSelector(activeProfiles);
+    const removals = [];
+    const html = /<html[\s>]/i.test(rawHtml) ? rawHtml : `<!doctype html><html><body>${rawHtml}</body></html>`;
+    const { document: document2 } = parseHTML2(html);
+    const contentSelector = preferredContentSelector && document2.querySelector(preferredContentSelector) ? preferredContentSelector : void 0;
+    const doc = document2;
+    if (this.options.baseUrl) {
+      doc.URL = this.options.baseUrl;
+    }
+    if (!doc.styleSheets) {
+      doc.styleSheets = [];
+    }
+    const parser2 = new DefuddleClass(doc, {
+      url: this.options.baseUrl,
+      debug: this.options.debug,
+      contentSelector,
+      removeSmallImages: this.options.removeSmallImages,
+      removeHiddenElements: this.options.removeHiddenElements,
+      removeLowScoring: this.options.removeLowScoring,
+      removeExactSelectors: this.options.removeExactSelectors,
+      removePartialSelectors: this.options.removePartialSelectors,
+      removeContentPatterns: this.options.removeContentPatterns,
+      standardize: this.options.standardize
+    });
+    const result = parser2.parseAsync ? await parser2.parseAsync() : parser2.parse();
+    const metadata = toMetadata(result, document2);
+    applyMetadataProfiles(metadata, activeProfiles);
+    const content = serializeProfiledContent(result.content, postProfiles, removals);
+    return {
+      content,
+      contentMarkdown: result.contentMarkdown,
+      metadata,
+      debug: this.options.debug ? {
+        contentSelector: result.debug?.contentSelector ?? contentSelector ?? preferredContentSelector,
+        activeProfiles: activeProfiles.map((profile) => profile.name),
+        removals: [...result.debug?.removals ?? [], ...removals]
+      } : void 0
+    };
+  }
+};
+async function cleanHtml(rawHtml, options = {}) {
+  return new HtmlCleaner(options).parse(rawHtml);
+}
+
+// src/fetch/strategy.ts
+import { writeFile as writeFile3 } from "fs/promises";
+
+// src/extract/meaningful.ts
+import { parseHTML as parseHTML3 } from "linkedom";
+var PRELOADED_MARKDOWN_RE = /window\.preloadPage\s*=\s*f\((['"])(.*?)\1\)/;
+function extractPreloadedMarkdownUrl(html, baseUrl) {
+  const match = PRELOADED_MARKDOWN_RE.exec(html);
+  const rawUrl = match?.[2]?.trim();
+  if (!rawUrl) {
+    return null;
+  }
+  return new URL(rawUrl, baseUrl).toString();
+}
+function removeNoise(document2) {
+  document2.querySelectorAll("script, style, noscript, svg, iframe").forEach((element) => element.remove());
+}
+function normalizedTextLength(element) {
+  return (element?.textContent ?? "").replace(/\s+/g, " ").trim().length;
+}
+function htmlHasMeaningfulContent(url, html) {
+  if (extractPreloadedMarkdownUrl(html, url) !== null) {
+    return true;
+  }
+  const { document: document2 } = parseHTML3(html);
+  removeNoise(document2);
+  const selectors = ["#js_content", "article", "main", "section", "div", "body"];
+  let bestLength = 0;
+  for (const selector of selectors) {
+    document2.querySelectorAll(selector).forEach((element) => {
+      bestLength = Math.max(bestLength, normalizedTextLength(element));
+    });
+    if (bestLength >= 600 && selector !== "div") {
+      return true;
+    }
+  }
+  return bestLength >= 600;
+}
+
+// src/fetch/browser-state.ts
+import { cp, mkdir as mkdir2, mkdtemp as mkdtemp3, stat as stat2, copyFile, rm as rm3 } from "fs/promises";
+import { tmpdir as tmpdir3 } from "os";
+import { basename, join as join4 } from "path";
+var ROOT_STATE_FILES = ["Local State", "First Run", "Last Version"];
+var IGNORED_NAMES = /* @__PURE__ */ new Set([
+  "Crashpad",
+  "Code Cache",
+  "GPUCache",
+  "ShaderCache",
+  "GrShaderCache",
+  "GraphiteDawnCache"
+]);
+function isIgnoredBrowserStatePath(path) {
+  const name = basename(path);
+  if (IGNORED_NAMES.has(name)) {
+    return true;
+  }
+  if (name.startsWith("Singleton")) {
+    return true;
+  }
+  if (name === "lockfile") {
+    return true;
+  }
+  return /\.(?:lock|tmp|log)$/i.test(name);
+}
+async function copyFileIfPresent(source, destination) {
+  try {
+    const info = await stat2(source);
+    if (info.isFile()) {
+      await copyFile(source, destination);
+    }
+  } catch (error) {
+    if (error.code !== "ENOENT") {
+      throw error;
+    }
+  }
+}
+async function copyBrowserState(sourceRoot, destRoot, profile) {
+  const profileDir = join4(sourceRoot, profile);
+  const profileInfo = await stat2(profileDir).catch((error) => {
+    if (error.code === "ENOENT") {
+      throw new Error(`Chrome profile not found: ${profileDir}`);
+    }
+    throw error;
+  });
+  if (!profileInfo.isDirectory()) {
+    throw new Error(`Chrome profile is not a directory: ${profileDir}`);
+  }
+  await mkdir2(destRoot, { recursive: true });
+  for (const filename of ROOT_STATE_FILES) {
+    await copyFileIfPresent(join4(sourceRoot, filename), join4(destRoot, filename));
+  }
+  await cp(profileDir, join4(destRoot, profile), {
+    recursive: true,
+    force: true,
+    filter: (source) => !isIgnoredBrowserStatePath(source)
+  });
+}
+async function fetchBrowserHtmlWithBrowserState(url, config) {
+  const stateCopy = await mkdtemp3(join4(tmpdir3(), "feedloom-browser-state-"));
+  try {
+    await copyBrowserState(config.userDataDir, stateCopy, config.profile);
+    return await fetchBrowserHtml(url, {
+      userDataDir: stateCopy,
+      channel: "chrome",
+      headless: config.headless ?? true,
+      timeoutMs: 9e4,
+      waitMs: config.waitMs ?? 2500,
+      networkIdle: config.networkIdle ?? true,
+      extraArgs: [`--profile-directory=${config.profile}`],
+      proxy: config.proxy,
+      dnsOverHttps: config.dnsOverHttps,
+      waitSelector: config.waitSelector,
+      waitSelectorState: config.waitSelectorState,
+      clickSelectors: config.clickSelectors,
+      scrollToBottom: config.scrollToBottom,
+      realChromeDefaults: config.realChromeDefaults ?? true
+    });
+  } finally {
+    await rm3(stateCopy, { recursive: true, force: true });
+  }
+}
+
+// src/fetch/static.ts
+async function fetchStaticHtml(url, timeoutMs = 6e4) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const response = await fetch(url, {
+      redirect: "follow",
+      signal: controller.signal,
+      headers: {
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+        accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+      }
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status} ${response.statusText}`);
+    }
+    return {
+      url: response.url || url,
+      html: await response.text(),
+      contentType: response.headers.get("content-type") ?? ""
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+// src/fetch/strategy.ts
+async function writeOutputIfRequested(outputPath, html) {
+  if (outputPath) {
+    await writeFile3(outputPath, html, "utf8");
+  }
+}
+async function fetchHtmlResult(url, options = {}) {
+  const isMeaningful = options.isMeaningful ?? htmlHasMeaningfulContent;
+  const staticFetch = options.staticFetch ?? (async (targetUrl) => (await fetchStaticHtml(targetUrl)).html);
+  const browserFetch = options.browserFetch ?? ((targetUrl) => fetchBrowserHtml(targetUrl, {
+    waitMs: options.waitMs,
+    networkIdle: options.networkIdle,
+    proxy: options.proxy,
+    dnsOverHttps: options.dnsOverHttps,
+    waitSelector: options.waitSelector,
+    waitSelectorState: options.waitSelectorState,
+    clickSelectors: options.clickSelectors,
+    scrollToBottom: options.scrollToBottom,
+    headless: options.headless,
+    realChromeDefaults: options.realChromeDefaults
+  }));
+  const stealthFetch = options.stealthFetch ?? ((targetUrl) => fetchStealthHtml(targetUrl, {
+    waitMs: options.waitMs,
+    networkIdle: options.networkIdle,
+    solveCloudflare: options.solveCloudflare,
+    disableResources: options.disableResources,
+    proxy: options.proxy,
+    dnsOverHttps: options.dnsOverHttps,
+    waitSelector: options.waitSelector,
+    waitSelectorState: options.waitSelectorState,
+    clickSelectors: options.clickSelectors,
+    scrollToBottom: options.scrollToBottom
+  }));
+  const browserStateFetch = options.browserStateFetch ?? fetchBrowserHtmlWithBrowserState;
+  const mode = options.fetchMode ?? "auto";
+  const attempts = [];
+  if (mode === "auto" || mode === "static") {
+    attempts.push({
+      label: "static",
+      fetch: () => staticFetch(url)
+    });
+  }
+  if (mode === "auto" || mode === "browser") {
+    if (options.browserState) {
+      attempts.push({
+        label: "browser-state",
+        fetch: () => browserStateFetch(url, options.browserState)
+      });
+    }
+    attempts.push({
+      label: "browser",
+      fetch: () => browserFetch(url)
+    });
+  }
+  if (mode === "auto" || mode === "stealth") {
+    attempts.push({
+      label: "stealth",
+      fetch: () => stealthFetch(url)
+    });
+  }
+  const errors = [];
+  for (const attempt of attempts) {
+    try {
+      const html = await attempt.fetch();
+      await writeOutputIfRequested(options.outputPath, html);
+      if (isMeaningful(url, html)) {
+        return { url, finalUrl: url, html, mode: attempt.label, diagnostics: errors };
+      }
+      errors.push(`${attempt.label} missing article content`);
+    } catch (error) {
+      errors.push(`${attempt.label} failed: ${error.message || String(error)}`);
+    }
+  }
+  throw new Error(errors.join("; "));
+}
+async function fetchHtml(url, options = {}) {
+  return (await fetchHtmlResult(url, options)).html;
+}
+
+// src/output.ts
+import { createHash } from "crypto";
+import { mkdir as mkdir3, readdir, readFile as readFile3, rm as rm4, stat as stat3, writeFile as writeFile4 } from "fs/promises";
+import { basename as basename2, join as join5 } from "path";
+var FRONTMATTER_ESCAPE_RE = /[\n\r]/g;
+function sanitizeFilename(title) {
+  return title.replace(/[\\/:*?"<>|]+/g, " ").replace(/\s+/g, " ").trim().replace(/^\.+|\.+$/g, "").slice(0, 180) || "Untitled";
+}
+function yamlString(value) {
+  return JSON.stringify(value.replace(FRONTMATTER_ESCAPE_RE, " "));
+}
+function renderFrontmatter(source, metadata, created) {
+  const lines = ["---", `source: ${yamlString(source)}`];
+  if (metadata.author) {
+    lines.push(`author: ${yamlString(metadata.author)}`);
+  }
+  lines.push(`created: ${yamlString(created)}`);
+  lines.push("---", "");
+  return `${lines.join("\n")}
+`;
+}
+function noteSource(text) {
+  const lines = text.split(/\r?\n/);
+  if (lines[0]?.trim() !== "---") return "";
+  for (const line of lines.slice(1)) {
+    if (line.trim() === "---") break;
+    const match = line.match(/^\s*source:\s*(.*)\s*$/);
+    if (!match) continue;
+    const raw = match[1].trim();
+    try {
+      return JSON.parse(raw);
+    } catch {
+      return raw.replace(/^['"]|['"]$/g, "");
+    }
+  }
+  return "";
+}
+async function cleanupExistingNote(outputDir, sourceUrl) {
+  await mkdir3(outputDir, { recursive: true });
+  const entries = await readdir(outputDir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
+    const path = join5(outputDir, entry.name);
+    let text = "";
+    try {
+      text = await readFile3(path, "utf8");
+    } catch {
+      continue;
+    }
+    if (noteSource(text) !== sourceUrl && !text.includes(`> Source: ${sourceUrl}`)) continue;
+    await rm4(path, { force: true });
+    await rm4(join5(outputDir, "assets", basename2(entry.name, ".md")), { recursive: true, force: true });
+    return;
+  }
+}
+function urlHash(url) {
+  return createHash("sha1").update(url).digest("hex").slice(0, 8);
+}
+async function writeMarkdownNote(outputDir, note) {
+  await mkdir3(outputDir, { recursive: true });
+  const base = sanitizeFilename(note.title);
+  let path = join5(outputDir, `${base}.md`);
+  try {
+    await stat3(path);
+    path = join5(outputDir, `${base}-${urlHash(note.sourceUrl)}.md`);
+  } catch (error) {
+    if (error.code !== "ENOENT") {
+      throw error;
+    }
+  }
+  const body = note.markdown.trim();
+  await writeFile4(path, `${renderFrontmatter(note.sourceUrl, note.metadata, note.created)}# ${note.title}
+
+${body}
+`, "utf8");
+  return path;
+}
+
+// src/render/markdown.ts
+import { parseHTML as parseHTML4 } from "linkedom";
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+function normalizeImageReferences(markdown) {
+  return markdown.replace(/!\[([^\]]*)\]\(<([^>]+)>\)(?:\{[^}]*\})?/g, (_match, alt, url) => {
+    return `![${alt}](${url})`;
+  });
+}
+function cleanupMarkdown(markdown) {
+  return normalizeImageReferences(markdown).replace(/^\s*content_copy\s*$/gim, "").replace(/^●●●\n\n```\n([\s\S]*?)\n```\n\n└$/gm, (_match, code) => `\`\`\`
+\u25CF\u25CF\u25CF
+
+${code}
+
+\u2514
+\`\`\``).replace(/\[\s*\]\((?:#|javascript:void\(0\)|javascript:;)\)/gi, "").replace(/(^|[^\\])\$(?=\d)/g, "$1\\$").replace(/\n\s*\n\s*([-*+]\s)/g, "\n$1").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+function htmlFragmentText(fragment) {
+  const { document: document2 } = parseHTML4(`<!doctype html><html><body>${fragment}</body></html>`);
+  document2.querySelectorAll("br").forEach((br) => br.replaceWith(document2.createTextNode("\n")));
+  return document2.body.textContent ?? "";
+}
+function fencedCodeHtml(text) {
+  const escaped = text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  return `<pre><code>${escaped}</code></pre>`;
+}
+function normalizeTableCellHtml(html) {
+  return html.replace(/<(t[hd])\b([^>]*)>\s*<section\b[^>]*>([\s\S]*?)<\/section>\s*<\/t[hd]>/gi, "<$1$2>$3</$1>").replace(/Input（<span\b[^>]*class=["']math inline["'][^>]*>\s*\/\s*<em>M<\/em>\s*<em>t<\/em>\s*<em>o<\/em>\s*<em>k<\/em>\s*<em>e<\/em>\s*<em>n<\/em>\s*<em>s<\/em>\s*）\s*\|\s*<em>O<\/em>\s*<em>u<\/em>\s*<em>t<\/em>\s*<em>p<\/em>\s*<em>u<\/em>\s*<em>t<\/em>\s*（<\/span>\s*\/M\s*tokens）/gi, 'Input\uFF08/M tokens\uFF09</th><th style="text-align: right;">Output\uFF08/M tokens\uFF09').replace(/<th\b[^>]*>\s*<\/th>/gi, "");
+}
+function normalizeBlockCodeHtml(html) {
+  return html.replace(/<code\b[^>]*>(((?:(?!<\/code>)[\s\S])*<br\b(?:(?!<\/code>)[\s\S])*))<\/code>/gi, (_match, codeInnerHtml) => {
+    return fencedCodeHtml(htmlFragmentText(codeInnerHtml).replace(/\n$/, ""));
+  });
+}
+function htmlToMarkdown(html) {
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-"
+  });
+  turndown.use(gfm);
+  turndown.addRule("dropEmptyLinks", {
+    filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
+    replacement: () => ""
+  });
+  turndown.addRule("blockCodeElement", {
+    filter: (node) => node.nodeName === "CODE" && node.parentNode?.nodeName !== "PRE" && Boolean(node.querySelector?.("br")),
+    replacement: (_content, node) => {
+      return `
+
+\`\`\`
+${node.textContent?.replace(/\n$/, "") ?? ""}
+\`\`\`
+
+`;
+    }
+  });
+  turndown.addRule("preserveCodeLanguage", {
+    filter: (node) => node.nodeName === "PRE" && node.firstChild?.nodeName === "CODE",
+    replacement: (_content, node) => {
+      const code = node.firstChild;
+      const className = code.getAttribute("class") ?? "";
+      const language = className.match(/language-([\w-]+)/)?.[1] ?? "";
+      return `
+
+\`\`\`${language}
+${code.textContent?.replace(/\n$/, "") ?? ""}
+\`\`\`
+
+`;
+    }
+  });
+  return `${cleanupMarkdown(turndown.turndown(normalizeBlockCodeHtml(normalizeTableCellHtml(html))))}
+`;
+}
+
+// src/pipeline.ts
+function titleFromUrl(url) {
+  const parsed = new URL(url);
+  const segment = parsed.pathname.split("/").filter(Boolean).pop();
+  return decodeURIComponent(segment || parsed.hostname || "Untitled").replace(/[-_]+/g, " ").trim() || "Untitled";
+}
+function stripDuplicateLeadingHeading(markdown, title) {
+  const normalizedTitle = title.replace(/\s+/g, " ").trim().toLowerCase();
+  return markdown.replace(/^#\s+(.+?)\s*\n+/, (match, heading) => {
+    return heading.replace(/\s+/g, " ").trim().toLowerCase() === normalizedTitle ? "" : match;
+  });
+}
+function stripLeadingDateLine(markdown) {
+  return markdown.replace(/^(?:Published\s+)?(?:\d{4}[-/.年]\d{1,2}[-/.月]\d{1,2}日?|[A-Z][a-z]+\s+\d{1,2},\s+\d{4})\s*\n+/i, "");
+}
+function demoteTopLevelHeadings(markdown) {
+  const lines = markdown.split("\n");
+  let inFence = false;
+  return lines.map((line) => {
+    if (/^```/.test(line.trim())) {
+      inFence = !inFence;
+      return line;
+    }
+    if (!inFence && /^#(?!#)\s+/.test(line)) {
+      return `#${line}`;
+    }
+    return line;
+  }).join("\n");
+}
+function createdFromItemDate(date) {
+  if (date.getUTCHours() === 0 && date.getUTCMinutes() === 0 && date.getUTCSeconds() === 0 && date.getUTCMilliseconds() === 0) {
+    return date.toISOString().slice(0, 10);
+  }
+  return date.toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+function resolveCreatedValue(item, published) {
+  if (published?.trim()) return published.trim();
+  if (item.publishedAt) return createdFromItemDate(item.publishedAt);
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+async function processItem(item, options) {
+  const html = await fetchHtml(item.url, options);
+  const activeProfiles = selectActiveProfiles(options.profiles, item.url, html);
+  const cleaned = await cleanHtml(html, { baseUrl: item.url, profiles: options.profiles, activeProfiles });
+  const title = cleaned.metadata.title || item.sourceTitle || titleFromUrl(item.url);
+  await cleanupExistingNote(options.outputDir, item.url);
+  const contentHtml = options.localizeAssets === false ? cleaned.content : await localizeImages(cleaned.content, {
+    outputDir: options.outputDir,
+    noteSlug: sanitizeFilename(title),
+    baseUrl: item.url,
+    fetchImage: options.fetchImage
+  });
+  const markdown = demoteTopLevelHeadings(stripLeadingDateLine(stripDuplicateLeadingHeading(htmlToMarkdown(contentHtml), title)));
+  const outputPath = await writeMarkdownNote(options.outputDir, {
+    sourceUrl: item.url,
+    title,
+    metadata: cleaned.metadata,
+    markdown,
+    created: resolveCreatedValue(item, cleaned.metadata.published)
+  });
+  return { item, outputPath, title };
+}
+
+// src/tracking.ts
+import { writeFileSync } from "fs";
+import { tmpdir as tmpdir4 } from "os";
+import { join as join6 } from "path";
+var ProgressTracker = class {
+  constructor(items, targetDir) {
+    this.targetDir = targetDir;
+    this.entries = items.map((item, index) => ({
+      index: index + 1,
+      url: item.url,
+      sourcePath: item.sourcePath,
+      lineNo: item.lineNo,
+      status: "pending"
+    }));
+    if (items.length > 1) {
+      this.path = join6(tmpdir4(), `feedloom-progress-${Date.now()}-${Math.random().toString(16).slice(2)}.json`);
+      this.write();
+    }
+  }
+  targetDir;
+  path;
+  entries;
+  start(url) {
+    this.update(url, { status: "in_progress", error: void 0 });
+  }
+  done(url, notePath) {
+    this.update(url, { status: "done", notePath, error: void 0 });
+  }
+  fail(url, error) {
+    this.update(url, { status: "failed", error });
+  }
+  update(url, patch) {
+    const entry = this.entries.find((item) => item.url === url);
+    if (entry) Object.assign(entry, patch);
+    this.write();
+  }
+  write() {
+    if (!this.path) return;
+    const payload = {
+      created_at: (/* @__PURE__ */ new Date()).toISOString(),
+      target_dir: this.targetDir,
+      items: this.entries
+    };
+    writeFileSync(this.path, `${JSON.stringify(payload, null, 2)}
+`, "utf8");
+  }
+};
+
+// src/cli.ts
+var program = new Command();
+async function standardSiteRulePaths() {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    resolve2(here, "../src/site-rules"),
+    resolve2(here, "../../src/site-rules"),
+    resolve2(process.cwd(), "src/site-rules"),
+    resolve2(here, "../../src/feedloom/site_rules"),
+    resolve2(process.cwd(), "../src/feedloom/site_rules"),
+    resolve2(process.cwd(), "src/feedloom/site_rules")
+  ];
+  for (const dir of candidates) {
+    try {
+      const names = await readdir2(dir);
+      return names.filter((name) => name.endsWith(".toml")).map((name) => join7(dir, name));
+    } catch (error) {
+      if (error.code !== "ENOENT") {
+        throw error;
+      }
+    }
+  }
+  return [];
+}
+function positiveIntOption(value, fallback) {
+  const parsed = Number(value ?? fallback);
+  if (!Number.isInteger(parsed)) {
+    throw new Error(`Expected integer option, got ${String(value)}`);
+  }
+  return parsed;
+}
+program.name("feedloom").description("Archive long-form web content as clean Markdown with local assets").version("0.1.0").option("--output-dir <dir>", "Output directory for markdown notes", "clippings").option("--source-kind <kind>", "auto, html-page, or rss-feed", "auto").option("--since <date>", "Only keep feed entries on or after YYYY-MM-DD", "").option("--limit <n>", "Process only first N deduplicated URLs", "0").option("--start <n>", "Start from 1-based index after deduplication", "1").option("--end <n>", "End at 1-based index after deduplication", "0").option("--prefer-browser-state", "Try copied local Chrome profile before regular browser fallback", false).option("--chrome-user-data-dir <path>", "Chrome user data directory used with --prefer-browser-state", "").option("--chrome-profile <name>", "Chrome profile directory name", "Default").option("--fetch-mode <mode>", "auto, static, browser, or stealth", "auto").option("--no-network-idle", "Do not wait for browser networkidle before reading HTML").option("--wait-ms <ms>", "Extra browser wait after load", "2500").option("--solve-cloudflare", "In stealth mode, attempt Cloudflare Turnstile/interstitial challenge handling", false).option("--disable-resources", "In stealth mode, block images/media/fonts/stylesheets for speed", false).option("--proxy <server>", "Proxy server for browser/stealth fetch, e.g. http://127.0.0.1:8080", "").option("--dns-over-https", "Use Chromium Cloudflare DNS-over-HTTPS flag for browser/stealth fetch", false).option("--wait-selector <selector>", "Wait for a CSS selector after page load", "").option("--wait-selector-state <state>", "attached, detached, visible, or hidden", "attached").option("--click-selector <selector...>", "Click one or more selectors after page load", []).option("--scroll-to-bottom", "Scroll to the bottom before reading HTML", false).option("--headful", "Run browser/browser-state fetches with a visible Chrome window", false).option("--no-real-chrome-defaults", "Disable Scrapling-inspired real Chrome context defaults").option("--no-reuse-browser", "Disable batch browser/stealth context reuse").argument("[inputs...]", "URLs or files containing URLs").action(async (inputs, options) => {
+  if (inputs.length === 0) {
+    program.help({ error: true });
+  }
+  try {
+    const sourceKind = String(options.sourceKind ?? "auto");
+    if (!["auto", "html-page", "rss-feed"].includes(sourceKind)) {
+      throw new Error("--source-kind must be auto, html-page, or rss-feed");
+    }
+    const fetchMode = String(options.fetchMode ?? "auto");
+    if (!["auto", "static", "browser", "stealth"].includes(fetchMode)) {
+      throw new Error("--fetch-mode must be auto, static, browser, or stealth");
+    }
+    const waitMs = positiveIntOption(options.waitMs, 2500);
+    const waitSelectorState = String(options.waitSelectorState ?? "attached");
+    if (!["attached", "detached", "visible", "hidden"].includes(waitSelectorState)) {
+      throw new Error("--wait-selector-state must be attached, detached, visible, or hidden");
+    }
+    const { items, checkboxFiles } = await parseInputs(inputs);
+    const since = options.since ? parseSinceDate(String(options.since)) : void 0;
+    const expanded = await expandSourceItems(items, sourceKind, since);
+    const selected = sliceItems(
+      expanded,
+      positiveIntOption(options.start, 1),
+      positiveIntOption(options.end, 0),
+      positiveIntOption(options.limit, 0)
+    );
+    const profiles = await loadSiteProfiles(await standardSiteRulePaths());
+    const outputDir = String(options.outputDir ?? "clippings");
+    let failures = 0;
+    const tracker = new ProgressTracker(selected, outputDir);
+    if (tracker.path) {
+      console.error(`Progress: ${tracker.path}`);
+    }
+    const browserOptions = {
+      waitMs,
+      networkIdle: Boolean(options.networkIdle),
+      proxy: String(options.proxy || "") || void 0,
+      dnsOverHttps: Boolean(options.dnsOverHttps),
+      waitSelector: String(options.waitSelector || "") || void 0,
+      waitSelectorState,
+      clickSelectors: Array.isArray(options.clickSelector) ? options.clickSelector.map(String) : [],
+      scrollToBottom: Boolean(options.scrollToBottom),
+      headless: !Boolean(options.headful),
+      realChromeDefaults: options.realChromeDefaults !== false
+    };
+    const sessions = options.reuseBrowser === false ? null : new BatchFetchSessions({
+      browser: browserOptions,
+      stealth: {
+        ...browserOptions,
+        solveCloudflare: Boolean(options.solveCloudflare),
+        disableResources: Boolean(options.disableResources)
+      }
+    });
+    try {
+      for (const item of selected) {
+        tracker.start(item.url);
+        try {
+          const browserState = options.preferBrowserState ? {
+            userDataDir: String(options.chromeUserDataDir || ""),
+            profile: String(options.chromeProfile || "Default"),
+            ...browserOptions
+          } : null;
+          const result = await processItem(item, {
+            outputDir,
+            profiles,
+            browserState,
+            fetchMode,
+            ...browserOptions,
+            solveCloudflare: Boolean(options.solveCloudflare),
+            disableResources: Boolean(options.disableResources),
+            browserFetch: sessions ? (targetUrl) => sessions.browserFetch(targetUrl) : void 0,
+            stealthFetch: sessions ? (targetUrl) => sessions.stealthFetch(targetUrl) : void 0
+          });
+          console.error(`Wrote ${result.outputPath}`);
+          tracker.done(item.url, result.outputPath);
+          const checkbox = item.sourcePath ? checkboxFiles.get(item.sourcePath) : void 0;
+          checkbox?.markDone(item.lineNo, item.url);
+        } catch (error) {
+          failures += 1;
+          const message = error.message || String(error);
+          tracker.fail(item.url, message);
+          console.error(`Failed ${item.url}: ${message}`);
+        }
+      }
+    } finally {
+      await sessions?.close();
+    }
+    await Promise.all([...checkboxFiles.values()].map((checkbox) => checkbox.save()));
+    process.exitCode = failures > 0 ? 1 : 0;
+  } catch (error) {
+    console.error(error.message || String(error));
+    process.exitCode = 2;
+  }
+});
+program.parseAsync();