@jenslys/curldown 1.0.0 → 1.0.2

package/README.md

@@ -1,11 +1,13 @@
 # curldown
 
-Fetch a webpage and return clean Markdown.
+Fetch a webpage and return clean Markdown for AI workflows.
 
-`curldown` is a CLI-first tool for AI agents and scripts:
+`curldown` is CLI-first:
 
 - Static mode: `fetch` HTML -> Cheerio cleanup -> Turndown markdown.
 - Dynamic mode: headless Chromium (Playwright) -> HTML -> markdown.
+- `--auto` tries static first and falls back to dynamic when static output is thin.
+- `--format json` emits markdown plus metadata for agent pipelines.
 
 ## Install
 
@@ -13,10 +15,6 @@ Fetch a webpage and return clean Markdown.
 npm install -g @jenslys/curldown
 ```
 
-```bash
-bun add -g @jenslys/curldown
-```
-
 ## Quick Start
 
 ```bash
@@ -26,7 +24,13 @@ curldown https://example.com
 # JS-heavy pages
 curldown https://example.com --dynamic
 
-# Write to file
+# Auto fallback to dynamic when static output looks incomplete
+curldown https://example.com --auto
+
+# JSON output for AI pipelines
+curldown https://example.com --format json
+
+# Write output to a file
 curldown https://example.com --output page.md
 ```
 
@@ -38,15 +42,30 @@ curldown <url> [options]
 
 ## Options
 
+- `--auto` Try static first and fallback to dynamic when static output is thin.
 - `--dynamic` Use Playwright Chromium to render before extraction.
-- `-o, --output <path>` Write markdown to file instead of stdout.
+- `--format <type>` Output format: `markdown` (default) or `json`.
+- `-o, --output <path>` Write output to file instead of stdout.
 - `--timeout-ms <number>` Request/render timeout in milliseconds.
-- `--user-agent <string>` Override request user-agent.
 - `--header <key:value>` Custom request header (repeatable).
-- `--remove-selector <css>` Remove selector(s) before conversion (repeatable).
 - `--help` Show help.
 - `--version` Show version.
 
+## JSON Output Shape
+
+`--format json` returns:
+
+- `url`
+- `final_url`
+- `title`
+- `markdown`
+- `content_type`
+- `status`
+- `fetched_at`
+- `word_count`
+- `sha256`
+- `used_dynamic`
+
 ## Local Development
 
 ```bash
@@ -65,16 +84,8 @@ Paste this into your `AGENTS.md` if you want agents to always use `curldown` for
 
 - Always use `curldown` to fetch web pages for agent workflows.
 - Default command: `curldown <url>`.
-- If the page is JS-rendered or incomplete, retry with: `curldown <url> --dynamic`.
+- Prefer `curldown <url> --auto` when page rendering might be uncertain.
+- Use `curldown <url> --format json` when downstream steps need structured metadata.
 - Prefer stdout output unless a task explicitly requires a file (`--output <path>`).
 - Do not use ad-hoc HTML scraping or direct browser automation when `curldown` can handle it.
 ```
-
-## Exit Codes
-
-- `0` success
-- `1` input/usage error
-- `2` static fetch/network error
-- `3` dynamic render/browser error
-- `4` output write error
-- `5` conversion pipeline error

package/dist/cli.js

@@ -1,12 +1,20 @@
 #!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { realpathSync } from "node:fs";
 import { Command, CommanderError } from "commander";
-import { pathToFileURL } from "node:url";
-import { DEFAULT_DYNAMIC_TIMEOUT_MS, DEFAULT_STATIC_TIMEOUT_MS, DEFAULT_USER_AGENT, VERSION } from "./constants.js";
-import { asCurldownError, InputError } from "./errors.js";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { DEFAULT_DYNAMIC_TIMEOUT_MS, DEFAULT_STATIC_TIMEOUT_MS, VERSION } from "./constants.js";
+import { asCurldownError, ConversionError, InputError } from "./errors.js";
 import { fetchDynamicHtml } from "./fetch-dynamic.js";
 import { fetchStaticHtml } from "./fetch-static.js";
 import { writeOutput } from "./output.js";
-import { transformHtmlToMarkdown } from "./transform.js";
+import { extractHtmlTitle, transformHtmlToMarkdown } from "./transform.js";
+const MARKDOWN_CONTENT_TYPES = new Set([
+    "text/markdown",
+    "text/x-markdown",
+    "application/markdown",
+    "application/x-markdown"
+]);
 const defaultDependencies = {
     fetchStatic: fetchStaticHtml,
     fetchDynamic: fetchDynamicHtml,
@@ -24,11 +32,11 @@ function buildProgram() {
         .version(VERSION)
         .argument("<url>", "The URL to fetch")
         .option("--dynamic", "Use headless Chromium (Playwright) to render the page")
-        .option("-o, --output <path>", "Write markdown to a file instead of stdout")
+        .option("--auto", "Try static first and fallback to dynamic when static output is thin")
+        .option("--format <type>", "Output format: markdown|json", "markdown")
+        .option("-o, --output <path>", "Write output to a file instead of stdout")
         .option("--timeout-ms <number>", "Timeout in milliseconds")
-        .option("--user-agent <string>", "Override request user-agent")
         .option("--header <key:value>", "Set custom request header", collectRepeatable, [])
-        .option("--remove-selector <css>", "Remove matching selector(s) before markdown conversion", collectRepeatable, [])
         .showHelpAfterError()
         .exitOverride();
 }
@@ -48,15 +56,76 @@ function parseHeaders(rawHeaders) {
     }
     return headers;
 }
-function parseTimeout(rawTimeout, dynamic) {
+function parseFormat(rawFormat) {
+    if (rawFormat === "markdown" || rawFormat === "json") {
+        return rawFormat;
+    }
+    throw new InputError(`Invalid --format value \"${rawFormat}\". Use \"markdown\" or \"json\".`);
+}
+function parseTimeouts(rawTimeout, dynamic, auto) {
     if (rawTimeout === undefined) {
-        return dynamic ? DEFAULT_DYNAMIC_TIMEOUT_MS : DEFAULT_STATIC_TIMEOUT_MS;
+        if (dynamic) {
+            return {
+                timeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS,
+                dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
+            };
+        }
+        if (auto) {
+            return {
+                timeoutMs: DEFAULT_STATIC_TIMEOUT_MS,
+                dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
+            };
+        }
+        return {
+            timeoutMs: DEFAULT_STATIC_TIMEOUT_MS,
+            dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
+        };
     }
     const parsed = Number.parseInt(rawTimeout, 10);
     if (!Number.isInteger(parsed) || parsed <= 0) {
         throw new InputError(`Invalid --timeout-ms value \"${rawTimeout}\". Must be a positive integer.`);
     }
-    return parsed;
+    return {
+        timeoutMs: parsed,
+        dynamicTimeoutMs: parsed
+    };
+}
+function normalizeMarkdown(markdown) {
+    const trimmed = markdown.trim();
+    if (!trimmed) {
+        throw new ConversionError("Content was fetched but markdown output is empty.");
+    }
+    return `${trimmed}\n`;
+}
+function inferTitleFromMarkdown(markdown) {
+    const firstHeading = markdown.match(/^#\s+(.+)$/m)?.[1]?.trim();
+    return firstHeading || undefined;
+}
+function isMarkdownContentType(contentType) {
+    if (!contentType) {
+        return false;
+    }
+    const normalized = contentType.toLowerCase().split(";")[0]?.trim() ?? "";
+    return MARKDOWN_CONTENT_TYPES.has(normalized);
+}
+function countWords(value) {
+    const trimmed = value.trim();
+    if (!trimmed) {
+        return 0;
+    }
+    return trimmed.split(/\s+/).length;
+}
+function shouldAutoFallback(markdown) {
+    const trimmed = markdown.trim();
+    if (!trimmed) {
+        return true;
+    }
+    const lower = trimmed.toLowerCase();
+    if (/enable javascript|javascript is required|checking your browser|just a moment|please wait/.test(lower)) {
+        return true;
+    }
+    const nonEmptyLines = trimmed.split(/\r?\n/).filter((line) => line.trim().length > 0).length;
+    return countWords(trimmed) < 30 && nonEmptyLines <= 2;
 }
 /**
  * Validate and normalize parsed CLI arguments into the canonical runtime shape.
@@ -79,16 +148,58 @@ function normalizeArgs(urlInput, options) {
         throw new InputError(`Unsupported URL protocol \"${parsedUrl.protocol}\". Only http:// and https:// are supported.`);
     }
     const dynamic = options.dynamic ?? false;
+    const auto = options.auto ?? false;
+    if (dynamic && auto) {
+        throw new InputError("--dynamic and --auto cannot be used together.");
+    }
+    const timeouts = parseTimeouts(options.timeoutMs, dynamic, auto);
     return {
         url: parsedUrl.toString(),
+        auto,
         dynamic,
+        format: parseFormat(options.format ?? "markdown"),
         outputPath: options.output,
-        timeoutMs: parseTimeout(options.timeoutMs, dynamic),
-        userAgent: options.userAgent?.trim() || DEFAULT_USER_AGENT,
-        headers: parseHeaders(options.header ?? []),
-        removeSelectors: (options.removeSelector ?? []).map((selector) => selector.trim()).filter(Boolean)
+        timeoutMs: timeouts.timeoutMs,
+        dynamicTimeoutMs: timeouts.dynamicTimeoutMs,
+        headers: parseHeaders(options.header ?? [])
     };
 }
+function prepareContentFromFetchResult(result, deps) {
+    if (isMarkdownContentType(result.contentType)) {
+        const markdown = normalizeMarkdown(result.body);
+        return {
+            markdown,
+            title: inferTitleFromMarkdown(markdown),
+            source: result,
+            passthrough: true
+        };
+    }
+    const markdown = deps.transformHtmlToMarkdown({ html: result.body });
+    return {
+        markdown,
+        title: extractHtmlTitle(result.body),
+        source: result,
+        passthrough: false
+    };
+}
+function formatOutput(args, content, usedDynamic) {
+    if (args.format === "markdown") {
+        return content.markdown;
+    }
+    const payload = {
+        url: args.url,
+        final_url: content.source.finalUrl,
+        title: content.title ?? null,
+        markdown: content.markdown,
+        content_type: content.source.contentType ?? null,
+        status: content.source.status,
+        fetched_at: new Date().toISOString(),
+        word_count: countWords(content.markdown),
+        sha256: createHash("sha256").update(content.markdown).digest("hex"),
+        used_dynamic: usedDynamic
+    };
+    return `${JSON.stringify(payload, null, 2)}\n`;
+}
 /**
  * Execute one curldown CLI invocation and return process exit code.
  * `argv` should not include the Node executable or script path.
@@ -116,21 +227,45 @@ export async function run(argv, deps = defaultDependencies) {
     }
     try {
         const args = normalizeArgs(urlArg, options);
-        const fetchInput = {
-            url: args.url,
-            timeoutMs: args.timeoutMs,
-            userAgent: args.userAgent,
-            headers: args.headers
-        };
-        const html = args.dynamic
-            ? await deps.fetchDynamic(fetchInput)
-            : await deps.fetchStatic(fetchInput);
-        const markdown = deps.transformHtmlToMarkdown({
-            html,
-            removeSelectors: args.removeSelectors
-        });
+        let usedDynamic = false;
+        let content;
+        if (args.auto) {
+            const staticResult = await deps.fetchStatic({
+                url: args.url,
+                timeoutMs: args.timeoutMs,
+                headers: args.headers
+            });
+            content = prepareContentFromFetchResult(staticResult, deps);
+            if (!content.passthrough && shouldAutoFallback(content.markdown)) {
+                const dynamicResult = await deps.fetchDynamic({
+                    url: args.url,
+                    timeoutMs: args.dynamicTimeoutMs,
+                    headers: args.headers
+                });
+                content = prepareContentFromFetchResult(dynamicResult, deps);
+                usedDynamic = true;
+            }
+        }
+        else if (args.dynamic) {
+            const dynamicResult = await deps.fetchDynamic({
+                url: args.url,
+                timeoutMs: args.dynamicTimeoutMs,
+                headers: args.headers
+            });
+            content = prepareContentFromFetchResult(dynamicResult, deps);
+            usedDynamic = true;
+        }
+        else {
+            const staticResult = await deps.fetchStatic({
+                url: args.url,
+                timeoutMs: args.timeoutMs,
+                headers: args.headers
+            });
+            content = prepareContentFromFetchResult(staticResult, deps);
+        }
+        const output = formatOutput(args, content, usedDynamic);
         await deps.writeOutput({
-            markdown,
+            content: output,
             outputPath: args.outputPath
         });
         return 0;
@@ -141,7 +276,27 @@ export async function run(argv, deps = defaultDependencies) {
         return curldownError.exitCode;
     }
 }
-const isMain = process.argv[1] !== undefined && pathToFileURL(process.argv[1]).href === import.meta.url;
+function resolvePathStrict(pathInput) {
+    return realpathSync(pathInput);
+}
+/**
+ * Determine whether this module was invoked as the CLI entrypoint.
+ * Resolves symlinks for both paths so global installs that expose a symlinked bin still execute.
+ */
+export function isMainModule(argvPath = process.argv[1]) {
+    if (argvPath === undefined) {
+        return false;
+    }
+    try {
+        const invokedPath = resolvePathStrict(argvPath);
+        const modulePath = resolvePathStrict(fileURLToPath(import.meta.url));
+        return invokedPath === modulePath;
+    }
+    catch {
+        return pathToFileURL(argvPath).href === import.meta.url;
+    }
+}
+const isMain = isMainModule();
 if (isMain) {
     void run(process.argv.slice(2)).then((exitCode) => {
         process.exitCode = exitCode;

package/dist/constants.js

@@ -1,7 +1,6 @@
-export const VERSION = "1.0.0";
+export const VERSION = "1.0.1";
 export const DEFAULT_STATIC_TIMEOUT_MS = 15_000;
 export const DEFAULT_DYNAMIC_TIMEOUT_MS = 30_000;
-export const DEFAULT_USER_AGENT = `curldown/${VERSION} (+https://www.npmjs.com/package/@jenslys/curldown)`;
 export const DEFAULT_REMOVE_SELECTORS = [
     "script",
     "style",

package/dist/fetch-dynamic.js

@@ -9,16 +9,21 @@ export async function fetchDynamicHtml(input) {
     try {
         browser = await chromium.launch({ headless: true });
         const context = await browser.newContext({
-            userAgent: input.userAgent,
             extraHTTPHeaders: input.headers
         });
         try {
             const page = await context.newPage();
-            await page.goto(input.url, {
+            const response = await page.goto(input.url, {
                 timeout: input.timeoutMs,
                 waitUntil: "domcontentloaded"
             });
-            return await page.content();
+            const body = await page.content();
+            return {
+                body,
+                finalUrl: page.url(),
+                status: response?.status() ?? 200,
+                contentType: response?.headers()["content-type"]
+            };
         }
         finally {
             await context.close();

package/dist/fetch-static.js

@@ -5,9 +5,6 @@ import { FetchError } from "./errors.js";
  */
 export async function fetchStaticHtml(input) {
     const headers = new Headers(input.headers);
-    if (input.userAgent) {
-        headers.set("user-agent", input.userAgent);
-    }
     let response;
     try {
         response = await fetch(input.url, {
@@ -23,7 +20,13 @@ export async function fetchStaticHtml(input) {
         throw new FetchError(`Static fetch failed for ${input.url}: HTTP ${response.status} ${response.statusText}`);
     }
     try {
-        return await response.text();
+        const body = await response.text();
+        return {
+            body,
+            finalUrl: response.url || input.url,
+            status: response.status,
+            contentType: response.headers.get("content-type") ?? undefined
+        };
     }
     catch (error) {
         throw new FetchError(`Failed reading response body for ${input.url}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });

package/dist/output.js

@@ -7,7 +7,7 @@ import { OutputError } from "./errors.js";
 export async function writeOutput(input) {
     if (input.outputPath) {
         try {
-            await writeFile(input.outputPath, input.markdown, "utf8");
+            await writeFile(input.outputPath, input.content, "utf8");
             return;
         }
         catch (error) {
@@ -15,7 +15,7 @@ export async function writeOutput(input) {
         }
     }
     try {
-        process.stdout.write(input.markdown);
+        process.stdout.write(input.content);
     }
     catch (error) {
         throw new OutputError(`Failed writing markdown to stdout: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : undefined });

package/dist/transform.js

@@ -1,31 +1,25 @@
 import { load } from "cheerio";
+import { createRequire } from "node:module";
 import TurndownService from "turndown";
 import { DEFAULT_REMOVE_SELECTORS } from "./constants.js";
 import { ConversionError } from "./errors.js";
+const require = createRequire(import.meta.url);
+const turndownPluginGfm = require("@joplin/turndown-plugin-gfm");
 const turndown = new TurndownService({
     headingStyle: "atx",
     codeBlockStyle: "fenced",
     bulletListMarker: "-",
     emDelimiter: "_"
 });
-/** Normalize selector input by trimming, dropping empties, and removing duplicates. */
-function uniqueSelectors(selectors) {
-    return [...new Set(selectors.map((selector) => selector.trim()).filter(Boolean))];
-}
+turndown.use(turndownPluginGfm.gfm);
 /**
  * Convert fetched HTML into markdown.
- * The function removes default non-content nodes and optional caller-provided
- * selectors before running Turndown conversion.
+ * The function removes default non-content nodes before running Turndown
+ * with GitHub Flavored Markdown extensions.
  */
 export function transformHtmlToMarkdown(input) {
     const $ = load(input.html);
-    const selectorsToRemove = uniqueSelectors([
-        ...DEFAULT_REMOVE_SELECTORS,
-        ...input.removeSelectors
-    ]);
-    if (selectorsToRemove.length > 0) {
-        $(selectorsToRemove.join(",")).remove();
-    }
+    $(DEFAULT_REMOVE_SELECTORS.join(",")).remove();
     const bodyHtml = $("body").length > 0 ? $("body").html() ?? "" : $.root().html() ?? "";
     if (bodyHtml.trim().length === 0) {
         throw new ConversionError("No HTML body content found to convert.");
@@ -36,3 +30,9 @@ export function transformHtmlToMarkdown(input) {
     }
     return `${markdown}\n`;
 }
+/** Extract document title from HTML head when available. */
+export function extractHtmlTitle(html) {
+    const $ = load(html);
+    const title = $("title").first().text().trim();
+    return title || undefined;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jenslys/curldown",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Fetch URL content and convert it to markdown.",
   "repository": {
     "type": "git",
@@ -29,6 +29,7 @@
     "prepublishOnly": "npm run build && npm run test"
   },
   "dependencies": {
+    "@joplin/turndown-plugin-gfm": "^1.0.64",
     "cheerio": "^1.2.0",
     "commander": "^14.0.3",
     "playwright": "^1.58.2",