@jenslys/curldown 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jens
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,80 @@
+# curldown
+
+Fetch a webpage and return clean Markdown.
+
+`curldown` is a CLI-first tool for AI agents and scripts:
+
+- Static mode: `fetch` HTML -> Cheerio cleanup -> Turndown markdown.
+- Dynamic mode: headless Chromium (Playwright) -> HTML -> markdown.
+
+## Install
+
+```bash
+npm install -g @jenslys/curldown
+```
+
+```bash
+bun add -g @jenslys/curldown
+```
+
+## Quick Start
+
+```bash
+# Print markdown to stdout
+curldown https://example.com
+
+# JS-heavy pages
+curldown https://example.com --dynamic
+
+# Write to file
+curldown https://example.com --output page.md
+```
+
+## CLI
+
+```bash
+curldown <url> [options]
+```
+
+## Options
+
+- `--dynamic` Use Playwright Chromium to render before extraction.
+- `-o, --output <path>` Write markdown to file instead of stdout.
+- `--timeout-ms <number>` Request/render timeout in milliseconds.
+- `--user-agent <string>` Override request user-agent.
+- `--header <key:value>` Custom request header (repeatable).
+- `--remove-selector <css>` Remove selector(s) before conversion (repeatable).
+- `--help` Show help.
+- `--version` Show version.
+
+## Local Development
+
+```bash
+bun install
+bun run build
+bun run test
+node dist/cli.js https://example.com
+```
+
+## AGENTS.md Snippet (Optional)
+
+Paste this into your `AGENTS.md` if you want agents to always use `curldown` for website content retrieval:
+
+```md
+## Website Content Retrieval
+
+- Always use `curldown` to fetch web pages for agent workflows.
+- Default command: `curldown <url>`.
+- If the page is JS-rendered or incomplete, retry with: `curldown <url> --dynamic`.
+- Prefer stdout output unless a task explicitly requires a file (`--output <path>`).
+- Do not use ad-hoc HTML scraping or direct browser automation when `curldown` can handle it.
+```
+
+## Exit Codes
+
+- `0` success
+- `1` input/usage error
+- `2` static fetch/network error
+- `3` dynamic render/browser error
+- `4` output write error
+- `5` conversion pipeline error

package/dist/cli.js

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+import { Command, CommanderError } from "commander";
+import { pathToFileURL } from "node:url";
+import { DEFAULT_DYNAMIC_TIMEOUT_MS, DEFAULT_STATIC_TIMEOUT_MS, DEFAULT_USER_AGENT, VERSION } from "./constants.js";
+import { asCurldownError, InputError } from "./errors.js";
+import { fetchDynamicHtml } from "./fetch-dynamic.js";
+import { fetchStaticHtml } from "./fetch-static.js";
+import { writeOutput } from "./output.js";
+import { transformHtmlToMarkdown } from "./transform.js";
+const defaultDependencies = {
+    fetchStatic: fetchStaticHtml,
+    fetchDynamic: fetchDynamicHtml,
+    transformHtmlToMarkdown,
+    writeOutput,
+    stderrWrite: (message) => process.stderr.write(message)
+};
+function collectRepeatable(value, previous = []) {
+    return [...previous, value];
+}
+function buildProgram() {
+    return new Command()
+        .name("curldown")
+        .description("Fetch URL content and convert it to markdown.")
+        .version(VERSION)
+        .argument("<url>", "The URL to fetch")
+        .option("--dynamic", "Use headless Chromium (Playwright) to render the page")
+        .option("-o, --output <path>", "Write markdown to a file instead of stdout")
+        .option("--timeout-ms <number>", "Timeout in milliseconds")
+        .option("--user-agent <string>", "Override request user-agent")
+        .option("--header <key:value>", "Set custom request header", collectRepeatable, [])
+        .option("--remove-selector <css>", "Remove matching selector(s) before markdown conversion", collectRepeatable, [])
+        .showHelpAfterError()
+        .exitOverride();
+}
+function parseHeaders(rawHeaders) {
+    const headers = {};
+    for (const rawHeader of rawHeaders) {
+        const separatorIndex = rawHeader.indexOf(":");
+        if (separatorIndex <= 0 || separatorIndex === rawHeader.length - 1) {
+            throw new InputError(`Invalid --header value \"${rawHeader}\". Use key:value format.`);
+        }
+        const key = rawHeader.slice(0, separatorIndex).trim();
+        const value = rawHeader.slice(separatorIndex + 1).trim();
+        if (!key || !value) {
+            throw new InputError(`Invalid --header value \"${rawHeader}\". Header key and value are required.`);
+        }
+        headers[key] = value;
+    }
+    return headers;
+}
+function parseTimeout(rawTimeout, dynamic) {
+    if (rawTimeout === undefined) {
+        return dynamic ? DEFAULT_DYNAMIC_TIMEOUT_MS : DEFAULT_STATIC_TIMEOUT_MS;
+    }
+    const parsed = Number.parseInt(rawTimeout, 10);
+    if (!Number.isInteger(parsed) || parsed <= 0) {
+        throw new InputError(`Invalid --timeout-ms value \"${rawTimeout}\". Must be a positive integer.`);
+    }
+    return parsed;
+}
+/**
+ * Validate and normalize parsed CLI arguments into the canonical runtime shape.
+ * Fails fast with {@link InputError} on malformed input.
+ */
+function normalizeArgs(urlInput, options) {
+    if (!urlInput) {
+        throw new InputError("A URL argument is required.");
+    }
+    let parsedUrl;
+    try {
+        parsedUrl = new URL(urlInput);
+    }
+    catch (error) {
+        throw new InputError(`Invalid URL \"${urlInput}\".`, {
+            cause: error instanceof Error ? error : undefined
+        });
+    }
+    if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
+        throw new InputError(`Unsupported URL protocol \"${parsedUrl.protocol}\". Only http:// and https:// are supported.`);
+    }
+    const dynamic = options.dynamic ?? false;
+    return {
+        url: parsedUrl.toString(),
+        dynamic,
+        outputPath: options.output,
+        timeoutMs: parseTimeout(options.timeoutMs, dynamic),
+        userAgent: options.userAgent?.trim() || DEFAULT_USER_AGENT,
+        headers: parseHeaders(options.header ?? []),
+        removeSelectors: (options.removeSelector ?? []).map((selector) => selector.trim()).filter(Boolean)
+    };
+}
+/**
+ * Execute one curldown CLI invocation and return process exit code.
+ * `argv` should not include the Node executable or script path.
+ */
+export async function run(argv, deps = defaultDependencies) {
+    const program = buildProgram();
+    let options;
+    let urlArg;
+    try {
+        const parsedProgram = program.parse(argv, { from: "user" });
+        options = parsedProgram.opts();
+        [urlArg] = parsedProgram.args;
+    }
+    catch (error) {
+        if (error instanceof CommanderError) {
+            if (error.code === "commander.helpDisplayed" || error.code === "commander.version") {
+                return 0;
+            }
+            deps.stderrWrite(`${error.message}\n`);
+            return 1;
+        }
+        const curldownError = asCurldownError(error);
+        deps.stderrWrite(`${curldownError.message}\n`);
+        return curldownError.exitCode;
+    }
+    try {
+        const args = normalizeArgs(urlArg, options);
+        const fetchInput = {
+            url: args.url,
+            timeoutMs: args.timeoutMs,
+            userAgent: args.userAgent,
+            headers: args.headers
+        };
+        const html = args.dynamic
+            ? await deps.fetchDynamic(fetchInput)
+            : await deps.fetchStatic(fetchInput);
+        const markdown = deps.transformHtmlToMarkdown({
+            html,
+            removeSelectors: args.removeSelectors
+        });
+        await deps.writeOutput({
+            markdown,
+            outputPath: args.outputPath
+        });
+        return 0;
+    }
+    catch (error) {
+        const curldownError = asCurldownError(error);
+        deps.stderrWrite(`${curldownError.message}\n`);
+        return curldownError.exitCode;
+    }
+}
+const isMain = process.argv[1] !== undefined && pathToFileURL(process.argv[1]).href === import.meta.url;
+if (isMain) {
+    void run(process.argv.slice(2)).then((exitCode) => {
+        process.exitCode = exitCode;
+    });
+}

package/dist/constants.js

@@ -0,0 +1,13 @@
+export const VERSION = "1.0.0";
+export const DEFAULT_STATIC_TIMEOUT_MS = 15_000;
+export const DEFAULT_DYNAMIC_TIMEOUT_MS = 30_000;
+export const DEFAULT_USER_AGENT = `curldown/${VERSION} (+https://www.npmjs.com/package/@jenslys/curldown)`;
+export const DEFAULT_REMOVE_SELECTORS = [
+    "script",
+    "style",
+    "noscript",
+    "template",
+    "svg",
+    "canvas",
+    "iframe"
+];

package/dist/errors.js

@@ -0,0 +1,55 @@
+/**
+ * Base error for all domain failures in curldown.
+ * Each subclass maps directly to a CLI exit code.
+ */
+export class CurldownError extends Error {
+    exitCode;
+    constructor(message, exitCode, options) {
+        super(message, options);
+        this.name = new.target.name;
+        this.exitCode = exitCode;
+    }
+}
+/** Invalid CLI usage or invalid input values. */
+export class InputError extends CurldownError {
+    constructor(message, options) {
+        super(message, 1, options);
+    }
+}
+/** Static network fetch failure from Node fetch. */
+export class FetchError extends CurldownError {
+    constructor(message, options) {
+        super(message, 2, options);
+    }
+}
+/** Browser-rendering failure in dynamic mode. */
+export class DynamicError extends CurldownError {
+    constructor(message, options) {
+        super(message, 3, options);
+    }
+}
+/** Failure while writing markdown to stdout or file output. */
+export class OutputError extends CurldownError {
+    constructor(message, options) {
+        super(message, 4, options);
+    }
+}
+/** Failure in HTML cleanup or markdown conversion. */
+export class ConversionError extends CurldownError {
+    constructor(message, options) {
+        super(message, 5, options);
+    }
+}
+/**
+ * Convert unknown thrown values into typed curldown errors so the CLI
+ * always returns a deterministic exit code.
+ */
+export function asCurldownError(error) {
+    if (error instanceof CurldownError) {
+        return error;
+    }
+    if (error instanceof Error) {
+        return new ConversionError(error.message, { cause: error });
+    }
+    return new ConversionError("Unknown error while processing page content.");
+}

package/dist/fetch-dynamic.js

@@ -0,0 +1,35 @@
+import { chromium } from "playwright";
+import { DynamicError } from "./errors.js";
+/**
+ * Render a page in headless Chromium and return the resulting HTML snapshot.
+ * Throws {@link DynamicError} if browser startup, navigation, or capture fails.
+ */
+export async function fetchDynamicHtml(input) {
+    let browser;
+    try {
+        browser = await chromium.launch({ headless: true });
+        const context = await browser.newContext({
+            userAgent: input.userAgent,
+            extraHTTPHeaders: input.headers
+        });
+        try {
+            const page = await context.newPage();
+            await page.goto(input.url, {
+                timeout: input.timeoutMs,
+                waitUntil: "domcontentloaded"
+            });
+            return await page.content();
+        }
+        finally {
+            await context.close();
+        }
+    }
+    catch (error) {
+        throw new DynamicError(`Dynamic fetch failed for ${input.url}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });
+    }
+    finally {
+        if (browser) {
+            await browser.close();
+        }
+    }
+}

package/dist/fetch-static.js

@@ -0,0 +1,31 @@
+import { FetchError } from "./errors.js";
+/**
+ * Fetch raw HTML using Node's native fetch implementation.
+ * Throws {@link FetchError} for transport, status, or body-read failures.
+ */
+export async function fetchStaticHtml(input) {
+    const headers = new Headers(input.headers);
+    if (input.userAgent) {
+        headers.set("user-agent", input.userAgent);
+    }
+    let response;
+    try {
+        response = await fetch(input.url, {
+            headers,
+            redirect: "follow",
+            signal: AbortSignal.timeout(input.timeoutMs)
+        });
+    }
+    catch (error) {
+        throw new FetchError(`Static fetch failed for ${input.url}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });
+    }
+    if (!response.ok) {
+        throw new FetchError(`Static fetch failed for ${input.url}: HTTP ${response.status} ${response.statusText}`);
+    }
+    try {
+        return await response.text();
+    }
+    catch (error) {
+        throw new FetchError(`Failed reading response body for ${input.url}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });
+    }
+}

package/dist/output.js

@@ -0,0 +1,23 @@
+import { writeFile } from "node:fs/promises";
+import { OutputError } from "./errors.js";
+/**
+ * Emit markdown to stdout (default) or to a target file.
+ * Throws {@link OutputError} when writing fails.
+ */
+export async function writeOutput(input) {
+    if (input.outputPath) {
+        try {
+            await writeFile(input.outputPath, input.markdown, "utf8");
+            return;
+        }
+        catch (error) {
+            throw new OutputError(`Failed writing markdown to ${input.outputPath}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });
+        }
+    }
+    try {
+        process.stdout.write(input.markdown);
+    }
+    catch (error) {
+        throw new OutputError(`Failed writing markdown to stdout: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });
+    }
+}

package/dist/transform.js

@@ -0,0 +1,38 @@
+import { load } from "cheerio";
+import TurndownService from "turndown";
+import { DEFAULT_REMOVE_SELECTORS } from "./constants.js";
+import { ConversionError } from "./errors.js";
+const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+    emDelimiter: "_"
+});
+/** Normalize selector input by trimming, dropping empties, and removing duplicates. */
+function uniqueSelectors(selectors) {
+    return [...new Set(selectors.map((selector) => selector.trim()).filter(Boolean))];
+}
+/**
+ * Convert fetched HTML into markdown.
+ * The function removes default non-content nodes and optional caller-provided
+ * selectors before running Turndown conversion.
+ */
+export function transformHtmlToMarkdown(input) {
+    const $ = load(input.html);
+    const selectorsToRemove = uniqueSelectors([
+        ...DEFAULT_REMOVE_SELECTORS,
+        ...input.removeSelectors
+    ]);
+    if (selectorsToRemove.length > 0) {
+        $(selectorsToRemove.join(",")).remove();
+    }
+    const bodyHtml = $("body").length > 0 ? $("body").html() ?? "" : $.root().html() ?? "";
+    if (bodyHtml.trim().length === 0) {
+        throw new ConversionError("No HTML body content found to convert.");
+    }
+    const markdown = turndown.turndown(bodyHtml).trim();
+    if (markdown.length === 0) {
+        throw new ConversionError("HTML was fetched but produced empty markdown output.");
+    }
+    return `${markdown}\n`;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@jenslys/curldown",
+  "version": "1.0.0",
+  "description": "Fetch URL content and convert it to markdown.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jenslys/curldown"
+  },
+  "homepage": "https://github.com/jenslys/curldown",
+  "bugs": {
+    "url": "https://github.com/jenslys/curldown/issues"
+  },
+  "type": "module",
+  "bin": {
+    "curldown": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "lint": "npm run typecheck",
+    "test": "vitest run",
+    "prepublishOnly": "npm run build && npm run test"
+  },
+  "dependencies": {
+    "cheerio": "^1.2.0",
+    "commander": "^14.0.3",
+    "playwright": "^1.58.2",
+    "turndown": "^7.2.2"
+  },
+  "devDependencies": {
+    "@types/node": "^25.3.3",
+    "@types/turndown": "^5.0.6",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.18"
+  }
+}