into-md 0.1.0

package/src/fetcher.ts

@@ -0,0 +1,236 @@
+import { readFileSync } from "node:fs";
+import { basename } from "node:path";
+import { type CacheOptions, readFromCache, writeToCache } from "./cache";
+
+export interface FetchOptions {
+  useJs?: boolean;
+  cookiesPath?: string;
+  userAgent?: string;
+  encoding?: string;
+  timeoutMs?: number;
+  cache?: Partial<CacheOptions>;
+  noCache?: boolean;
+  verbose?: boolean;
+}
+
+export interface FetchResult {
+  html: string;
+  finalUrl: string;
+  fromCache: boolean;
+}
+
+interface CookieRecord {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  secure: boolean;
+  expires: number;
+}
+
+const DEFAULT_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0 Safari/537.36";
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+const logVerbose = (message: string, verbose?: boolean): void => {
+  if (verbose) {
+    console.error(message);
+  }
+};
+
+function parseNetscapeCookieLine(
+  line: string
+): { record: CookieRecord; headerPair: string } | null {
+  const trimmed = line.trim();
+  if (!trimmed || trimmed.startsWith("#")) {
+    return null;
+  }
+
+  const parts = trimmed.split("\t");
+  if (parts.length < 7) {
+    return null;
+  }
+
+  const domain = parts[0];
+  const path = parts[2];
+  const secureFlag = parts[3];
+  const expires = parts[4];
+  const name = parts[5];
+  const value = parts[6];
+  if (!(domain && path && secureFlag && expires && name && value)) {
+    return null;
+  }
+
+  return {
+    headerPair: `${name}=${value}`,
+    record: {
+      domain,
+      expires: Number(expires),
+      name,
+      path,
+      secure: secureFlag.toLowerCase() === "true",
+      value,
+    },
+  };
+}
+
+function parseCookiesFile(cookiesPath?: string): {
+  header: string | undefined;
+  playwrightCookies: CookieRecord[];
+} {
+  if (!cookiesPath) {
+    return { header: undefined, playwrightCookies: [] };
+  }
+  let content: string;
+  try {
+    content = readFileSync(cookiesPath, "utf8");
+  } catch (error) {
+    throw new Error(
+      `Unable to read cookies file "${basename(cookiesPath)}": ${String(error)}`,
+      { cause: error }
+    );
+  }
+
+  const entries: CookieRecord[] = [];
+  const headerPairs: string[] = [];
+  for (const line of content.split("\n")) {
+    const parsed = parseNetscapeCookieLine(line);
+    if (!parsed) {
+      continue;
+    }
+    entries.push(parsed.record);
+    headerPairs.push(parsed.headerPair);
+  }
+
+  return {
+    header: headerPairs.length ? headerPairs.join("; ") : undefined,
+    playwrightCookies: entries,
+  };
+}
+
+async function fetchWithHttp(
+  url: string,
+  options: FetchOptions
+): Promise<FetchResult> {
+  const controller = new AbortController();
+  const timeout = setTimeout(
+    () => controller.abort(),
+    options.timeoutMs ?? DEFAULT_TIMEOUT_MS
+  );
+
+  const { header: cookiesHeader } = parseCookiesFile(options.cookiesPath);
+  const headers = new Headers({
+    "User-Agent": options.userAgent ?? DEFAULT_USER_AGENT,
+  });
+  if (cookiesHeader) {
+    headers.set("Cookie", cookiesHeader);
+  }
+
+  try {
+    const response = await fetch(url, {
+      headers,
+      method: "GET",
+      redirect: "follow",
+      signal: controller.signal,
+    });
+    if (!response.ok) {
+      throw new Error(
+        `Request failed with status ${response.status}. If blocked, try --user-agent.`
+      );
+    }
+
+    const finalUrl = response.url;
+    const buffer = await response.arrayBuffer();
+    const decoder = new TextDecoder(options.encoding);
+    const html = decoder.decode(buffer);
+    return { finalUrl, fromCache: false, html };
+  } catch (error) {
+    const prefix =
+      error instanceof Error && error.name === "AbortError"
+        ? "Request timed out"
+        : "Request failed";
+    throw new Error(`${prefix}: ${String(error)}`, { cause: error });
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function fetchWithBrowser(
+  url: string,
+  options: FetchOptions
+): Promise<FetchResult> {
+  let playwright: typeof import("playwright") | null = null;
+  try {
+    playwright = await import("playwright");
+  } catch (error) {
+    throw new Error(
+      `JS mode requested but playwright is not installed. Install it and retry. (${String(
+        error
+      )})`,
+      { cause: error }
+    );
+  }
+
+  const { playwrightCookies } = parseCookiesFile(options.cookiesPath);
+  const browser = await playwright.chromium.launch({ headless: true });
+  const context = await browser.newContext({
+    userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
+  });
+
+  if (playwrightCookies.length) {
+    await context.addCookies(
+      playwrightCookies.map((cookie) => ({
+        ...cookie,
+        httpOnly: false,
+        sameSite: "Lax" as const,
+      }))
+    );
+  }
+
+  const page = await context.newPage();
+  await page.goto(url, {
+    timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    waitUntil: "networkidle",
+  });
+
+  const html = await page.content();
+  const finalUrl = page.url();
+
+  await browser.close();
+  return { finalUrl, fromCache: false, html };
+}
+
+export async function fetchPage(
+  url: string,
+  options: FetchOptions
+): Promise<FetchResult> {
+  const cacheEnabled = !options.noCache;
+  if (cacheEnabled) {
+    const cached = await readFromCache(url, {
+      enabled: cacheEnabled,
+      ...options.cache,
+    });
+    if (cached) {
+      logVerbose("Cache hit", options.verbose);
+      return { finalUrl: url, fromCache: true, html: cached.content };
+    }
+  }
+
+  logVerbose(
+    `Fetching ${url} ${options.useJs ? "(headless browser)" : "(http)"}`,
+    options.verbose
+  );
+  const result = options.useJs
+    ? await fetchWithBrowser(url, options)
+    : await fetchWithHttp(url, options);
+
+  if (cacheEnabled) {
+    await writeToCache(url, result.html, {
+      enabled: cacheEnabled,
+      ...options.cache,
+    });
+  }
+
+  return result;
+}

package/src/images.ts

@@ -0,0 +1,27 @@
+import { load } from "cheerio";
+
+import { getBodyHtml, toAbsoluteUrl } from "./utils";
+
+export function annotateImages(html: string, baseUrl: string): string {
+  const $ = load(html);
+
+  for (const img of $("img").toArray()) {
+    const $img = $(img);
+    const src = $img.attr("src");
+    const absoluteSrc = toAbsoluteUrl(src, baseUrl);
+    if (absoluteSrc) {
+      $img.attr("src", absoluteSrc);
+    }
+
+    const figure = $img.closest("figure");
+    const caption =
+      figure.find("figcaption").text().trim() ||
+      $img.attr("title")?.trim() ||
+      undefined;
+    if (caption) {
+      $img.attr("data-into-md-caption", caption);
+    }
+  }
+
+  return getBodyHtml($);
+}

package/src/index.ts

@@ -0,0 +1,143 @@
+import { writeFile } from "node:fs/promises";
+import { Command } from "commander";
+
+import { convertHtmlToMarkdown } from "./converter";
+import { extractContent } from "./extractor";
+import { fetchPage } from "./fetcher";
+import { annotateImages } from "./images";
+import { buildFrontmatter } from "./metadata";
+import { convertTablesToJson } from "./tables";
+
+const DEFAULT_TIMEOUT = 30_000;
+
+interface CliOptions {
+  output?: string;
+  js?: boolean;
+  raw?: boolean;
+  cookies?: string;
+  userAgent?: string;
+  encoding?: string;
+  stripLinks?: boolean;
+  exclude?: string;
+  timeout?: number;
+  noCache?: boolean;
+  verbose?: boolean;
+}
+
+async function run(url: string, options: CliOptions) {
+  const selectors =
+    options.exclude
+      ?.split(",")
+      .map((selector) => selector.trim())
+      .filter(Boolean) ?? [];
+
+  if (options.verbose) {
+    console.error("Starting into-md…");
+  }
+
+  const fetchResult = await fetchPage(url, {
+    cookiesPath: options.cookies,
+    encoding: options.encoding,
+    noCache: options.noCache,
+    timeoutMs: options.timeout ?? DEFAULT_TIMEOUT,
+    useJs: options.js,
+    userAgent: options.userAgent,
+    verbose: options.verbose,
+  });
+
+  const extracted = extractContent(fetchResult.html, {
+    baseUrl: fetchResult.finalUrl,
+    excludeSelectors: selectors,
+    raw: options.raw,
+  });
+
+  let workingHtml = extracted.html;
+  workingHtml = convertTablesToJson(workingHtml);
+  workingHtml = annotateImages(workingHtml, fetchResult.finalUrl);
+
+  const markdown = convertHtmlToMarkdown(workingHtml, {
+    baseUrl: fetchResult.finalUrl,
+    stripLinks: options.stripLinks,
+  });
+
+  const frontmatter = buildFrontmatter({
+    ...extracted.metadata,
+    source: fetchResult.finalUrl,
+  });
+
+  const output = `${frontmatter}\n\n${markdown}`.trim();
+
+  if (options.output) {
+    await writeFile(options.output, output, "utf8");
+    if (options.verbose) {
+      console.error(`Saved to ${options.output}`);
+    }
+  } else {
+    console.log(output);
+  }
+
+  const size = Buffer.byteLength(output, "utf8");
+  if (size > 100_000) {
+    console.error(
+      `Warning: Output is ${Math.round(size / 1024)}KB. Large documents may exceed LLM context limits.`
+    );
+  }
+}
+
+function buildProgram() {
+  const program = new Command()
+    .name("into-md")
+    .description("Fetch a web page and convert its content to markdown.")
+    .argument("<url>", "URL to fetch")
+    .option("-o, --output <file>", "Write output to file instead of stdout")
+    .option("--js", "Use headless browser (Playwright) for JS-rendered content")
+    .option("--raw", "Skip content extraction, convert entire HTML")
+    .option(
+      "--cookies <file>",
+      "Path to cookies file for authenticated requests"
+    )
+    .option("--user-agent <string>", "Custom User-Agent header")
+    .option(
+      "--encoding <encoding>",
+      "Force character encoding (auto-detected by default)"
+    )
+    .option("--strip-links", "Remove hyperlinks, keep only anchor text")
+    .option(
+      "--exclude <selectors>",
+      "CSS selectors to exclude (comma-separated)"
+    )
+    .option(
+      "--timeout <ms>",
+      "Request timeout in milliseconds",
+      `${DEFAULT_TIMEOUT}`
+    )
+    .option("--no-cache", "Bypass response cache")
+    .option("-v, --verbose", "Show detailed progress information");
+
+  program.version("0.1.0");
+  return program;
+}
+
+async function main() {
+  const program = buildProgram();
+  program.parse(process.argv);
+  const [url] = program.args;
+  if (!url) {
+    program.help();
+    return;
+  }
+
+  const opts = program.opts<CliOptions>();
+  try {
+    await run(url, {
+      ...opts,
+      timeout: opts.timeout ? Number(opts.timeout) : DEFAULT_TIMEOUT,
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(message);
+    process.exitCode = 1;
+  }
+}
+
+main();

package/src/metadata.ts

@@ -0,0 +1,30 @@
+export interface FrontmatterInput {
+  title?: string;
+  description?: string;
+  author?: string;
+  date?: string;
+  source: string;
+}
+
+export function buildFrontmatter(meta: FrontmatterInput): string {
+  const lines = ["---"];
+  if (meta.title) {
+    lines.push(`title: "${escapeFrontmatter(meta.title)}"`);
+  }
+  if (meta.description) {
+    lines.push(`description: "${escapeFrontmatter(meta.description)}"`);
+  }
+  if (meta.author) {
+    lines.push(`author: "${escapeFrontmatter(meta.author)}"`);
+  }
+  if (meta.date) {
+    lines.push(`date: "${escapeFrontmatter(meta.date)}"`);
+  }
+  lines.push(`source: "${escapeFrontmatter(meta.source)}"`);
+  lines.push("---");
+  return lines.join("\n");
+}
+
+function escapeFrontmatter(value: string): string {
+  return value.replaceAll('"', String.raw`\"`);
+}

package/src/tables.ts

@@ -0,0 +1,80 @@
+import { type Cheerio, type CheerioAPI, load } from "cheerio";
+import type { AnyNode } from "domhandler";
+
+import { getBodyHtml } from "./utils";
+
+interface TableJson {
+  caption?: string;
+  headers: string[];
+  rows: Record<string, string>[];
+}
+
+function extractHeaders($table: Cheerio<AnyNode>, $: CheerioAPI): string[] {
+  const explicitHeaders = $table.find("thead th");
+  if (explicitHeaders.length) {
+    return explicitHeaders
+      .toArray()
+      .map((th) => $(th).text().trim())
+      .filter(Boolean);
+  }
+
+  const firstRowHeaders = $table.find("tr").first().find("th, td");
+  if (firstRowHeaders.length) {
+    return firstRowHeaders
+      .toArray()
+      .map((cell, index) => $(cell).text().trim() || `Column ${index + 1}`);
+  }
+
+  return [];
+}
+
+function extractRows(
+  $table: Cheerio<AnyNode>,
+  headers: string[],
+  $: CheerioAPI
+): Record<string, string>[] {
+  const rows: Record<string, string>[] = [];
+  const dataRows =
+    $table.find("tbody tr").length > 0
+      ? $table.find("tbody tr")
+      : $table.find("tr").slice(1);
+
+  for (const row of dataRows.toArray()) {
+    const cells = $(row).find("td, th");
+    if (!cells.length) {
+      continue;
+    }
+    const record: Record<string, string> = {};
+    for (const [cellIndex, cell] of cells.toArray().entries()) {
+      const key = headers[cellIndex] ?? `Column ${cellIndex + 1}`;
+      record[key] = $(cell).text().trim();
+    }
+    rows.push(record);
+  }
+
+  return rows;
+}
+
+export function convertTablesToJson(html: string): string {
+  const $ = load(html);
+
+  for (const table of $("table").toArray()) {
+    const $table = $(table);
+    const caption = $table.find("caption").first().text().trim() || undefined;
+    const headers = extractHeaders($table, $);
+    const rows = extractRows($table, headers, $);
+
+    const json: TableJson = {
+      caption,
+      headers,
+      rows,
+    };
+
+    const pre = $("<pre>")
+      .attr("data-into-md-table", "true")
+      .text(JSON.stringify(json, null, 2));
+    $table.replaceWith(pre);
+  }
+
+  return getBodyHtml($);
+}

package/src/types/jsdom.d.ts

@@ -0,0 +1,10 @@
+declare module "jsdom" {
+  export interface JSDOMOptions {
+    url?: string;
+  }
+
+  export class JSDOM {
+    constructor(html?: string, options?: JSDOMOptions);
+    window: Window;
+  }
+}

package/src/utils.ts

@@ -0,0 +1,28 @@
+import type { CheerioAPI } from "cheerio";
+
+/**
+ * Converts a relative URL to an absolute URL using the provided base URL.
+ * Returns the original URL if it cannot be parsed.
+ */
+export const toAbsoluteUrl = (
+  url: string | undefined,
+  baseUrl: string
+): string | undefined => {
+  if (!url) {
+    return undefined;
+  }
+  try {
+    return new URL(url, baseUrl).toString();
+  } catch {
+    return url;
+  }
+};
+
+/**
+ * Extracts the inner HTML from the body element, or falls back to root HTML.
+ * Common pattern used across multiple cheerio-based transformations.
+ */
+export const getBodyHtml = ($: CheerioAPI): string => {
+  const body = $("body");
+  return body.length ? (body.html() ?? "") : ($.root().html() ?? "");
+};

package/tsconfig.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    // Environment setup & latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "Preserve",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

package/tsdown.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from "tsdown";
+
+export default defineConfig({
+  banner: "#!/usr/bin/env node\n",
+  clean: true,
+  dts: true,
+  entry: ["src/index.ts"],
+  format: ["esm"],
+  hash: false,
+  outDir: "dist",
+  platform: "node",
+  sourcemap: true,
+  target: "node18",
+});