mdrip 0.1.0

package/README.md

@@ -0,0 +1,137 @@
+# mdrip
+
+Fetch markdown snapshots of web pages using Cloudflare's Markdown for Agents feature, so coding agents can consume clean structured content instead of HTML.
+
+## AI Skills
+
+This repo also includes an AI-consumable skills catalog in `skills/`, following the [agentskills](https://agentskills.io) format.
+
+- Skill index: `skills/README.md`
+- mdrip skill: `skills/mdrip/SKILL.md`
+
+## Why
+
+For agent workflows, markdown is often better than HTML:
+- cleaner structure
+- lower token overhead
+- easier chunking and context management
+
+`mdrip` requests pages with `Accept: text/markdown`, stores the markdown locally, and tracks fetched pages in an index.
+
+If a site does not return `text/markdown`, `mdrip` can automatically fall back to converting `text/html` into markdown.
+The fallback uses an in-project converter optimized for common documentation/blog content (headings, links, lists, code blocks, tables, blockquotes).
+
+## Installation
+
+```bash
+npm install -g mdrip
+```
+
+Or use with `npx`:
+
+```bash
+npx mdrip <url>
+```
+
+## Usage
+
+### Fetch pages
+
+```bash
+# Fetch one page
+mdrip https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/
+
+# Fetch multiple pages
+mdrip https://blog.cloudflare.com/markdown-for-agents/ https://developers.cloudflare.com/
+
+# Optional timeout override (ms)
+mdrip https://example.com --timeout 45000
+
+# Disable HTML fallback (strict Cloudflare markdown only)
+mdrip https://example.com --no-html-fallback
+```
+
+### List fetched pages
+
+```bash
+mdrip list
+mdrip list --json
+```
+
+### Remove pages
+
+```bash
+mdrip remove https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/
+```
+
+### Clean snapshots
+
+```bash
+# Remove all
+mdrip clean
+
+# Remove only one domain
+mdrip clean --domain developers.cloudflare.com
+```
+
+## File modifications
+
+On first run, mdrip can optionally update:
+- `.gitignore` (adds `mdrip/`)
+- `tsconfig.json` (excludes `mdrip`)
+- `AGENTS.md` (adds a section pointing agents to snapshots)
+
+Choice is stored in `mdrip/settings.json`.
+
+Use flags to skip prompt:
+
+```bash
+# allow updates
+mdrip https://example.com --modify
+
+# deny updates
+mdrip https://example.com --modify=false
+```
+
+## Output
+
+```text
+mdrip/
+├── settings.json
+├── sources.json
+└── pages/
+    └── developers.cloudflare.com/
+        └── fundamentals/
+            └── reference/
+                └── markdown-for-agents/
+                    └── index.md
+```
+
+## Requirements and notes
+
+- Node.js 18+
+- The target site must return markdown for `Accept: text/markdown` (Cloudflare Markdown for Agents enabled).
+- If a page does not return `text/markdown`, mdrip can convert `text/html` into markdown fallback unless `--no-html-fallback` is used.
+
+## Publishing to npm
+
+```bash
+# optional package check
+pnpm publish:dry-run
+
+# publish to npm
+pnpm publish:npm
+```
+
+`prepublishOnly` runs automatically before publish and executes:
+- `pnpm type-check`
+- `pnpm test`
+- `pnpm build`
+
+## Author
+
+Charl Kruger
+
+## License
+
+Apache-2.0

package/dist/commands/clean.d.ts

@@ -0,0 +1,6 @@
+export interface CleanOptions {
+    cwd?: string;
+    domain?: string;
+}
+export declare function cleanCommand(options?: CleanOptions): Promise<void>;
+//# sourceMappingURL=clean.d.ts.map

package/dist/commands/clean.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clean.d.ts","sourceRoot":"","sources":["../../src/commands/clean.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,YAAY,CAAC,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsD5E"}

package/dist/commands/clean.js

@@ -0,0 +1,49 @@
+import { listSources, removeStoredPage, cleanupEmptyPageDirs } from "../lib/storage.js";
+import { updateAgentsMd, updatePageIndex } from "../lib/agents.js";
+import { getFileModificationPermission } from "../lib/settings.js";
+import { getUrlHost } from "../lib/url.js";
+export async function cleanCommand(options = {}) {
+    const cwd = options.cwd || process.cwd();
+    const sources = await listSources(cwd);
+    if (sources.pages.length === 0) {
+        console.log("No pages to clean.");
+        return;
+    }
+    const targetDomain = options.domain?.trim().toLowerCase();
+    const pagesToRemove = targetDomain
+        ? sources.pages.filter((page) => getUrlHost(page.resolvedUrl || page.url) === targetDomain)
+        : sources.pages;
+    if (pagesToRemove.length === 0) {
+        console.log(`No pages found for domain: ${targetDomain}`);
+        return;
+    }
+    const removalSet = new Set(pagesToRemove.map((page) => page.url));
+    for (const page of pagesToRemove) {
+        await removeStoredPage(page.path, cwd);
+    }
+    await cleanupEmptyPageDirs(cwd);
+    const remainingPages = sources.pages.filter((page) => !removalSet.has(page.url));
+    const canModifyFiles = await getFileModificationPermission(cwd);
+    if (canModifyFiles) {
+        const agentsUpdated = await updateAgentsMd({ pages: remainingPages }, cwd);
+        if (agentsUpdated) {
+            if (remainingPages.length === 0) {
+                console.log("✓ Removed mdrip section from AGENTS.md");
+            }
+            else {
+                console.log("✓ Updated AGENTS.md");
+            }
+        }
+    }
+    else {
+        await updatePageIndex({ pages: remainingPages }, cwd);
+    }
+    if (targetDomain) {
+        console.log(`✓ Removed ${pagesToRemove.length} page(s) for ${targetDomain}`);
+    }
+    else {
+        console.log(`✓ Removed ${pagesToRemove.length} page(s)`);
+    }
+    console.log(`\nCleaned ${pagesToRemove.length} page(s)`);
+}
+//# sourceMappingURL=clean.js.map

package/dist/commands/clean.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clean.js","sourceRoot":"","sources":["../../src/commands/clean.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACxF,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAQ3C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAAwB,EAAE;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IAEvC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE1D,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,YAAY,CAAC;QAC3F,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAElB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAC1D,OAAO;IACT,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAElE,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAEhC,MAAM,cAAc,GAAgB,OAAO,CAAC,KAAK,CAAC,MAAM,CACtD,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CACpC,CAAC;IAEF,MAAM,cAAc,GAAG,MAAM,6BAA6B,CAAC,GAAG,CAAC,CAAC;IAEhE,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3E,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;YACxD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE,GAAG,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,MAAM,gBAAgB,YAAY,EAAE,CAAC,CAAC;IAC/E,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,MAAM,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,MAAM,UAAU,CAAC,CAAC;AAC3D,CAAC"}

package/dist/commands/fetch.d.ts

@@ -0,0 +1,9 @@
+import type { FetchResult } from "../types.js";
+export interface FetchOptions {
+    cwd?: string;
+    allowModifications?: boolean;
+    timeoutMs?: number;
+    htmlFallback?: boolean;
+}
+export declare function fetchCommand(urls: string[], options?: FetchOptions): Promise<FetchResult[]>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/commands/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/commands/fetch.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAa,MAAM,aAAa,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAmJD,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,EAAE,CAAC,CAmDxB"}

package/dist/commands/fetch.js

@@ -0,0 +1,163 @@
+import { fetchMarkdownPage } from "../lib/cloudflare.js";
+import { ensureGitignore } from "../lib/gitignore.js";
+import { ensureTsconfigExclude } from "../lib/tsconfig.js";
+import { updateAgentsMd, updatePageIndex, } from "../lib/agents.js";
+import { getFileModificationPermission, setFileModificationPermission } from "../lib/settings.js";
+import { confirm } from "../lib/prompt.js";
+import { listSources, getPageInfo, savePageMarkdown } from "../lib/storage.js";
+import { normalizeUrl } from "../lib/url.js";
+async function checkFileModificationPermission(cwd, cliOverride) {
+    if (cliOverride !== undefined) {
+        await setFileModificationPermission(cliOverride, cwd);
+        if (cliOverride) {
+            console.log("✓ File modifications enabled (--modify)");
+        }
+        else {
+            console.log("✗ File modifications disabled (--modify=false)");
+        }
+        return cliOverride;
+    }
+    const storedPermission = await getFileModificationPermission(cwd);
+    if (storedPermission !== undefined) {
+        return storedPermission;
+    }
+    console.log("\nmdrip can update the following files for better integration:");
+    console.log("  • .gitignore - add mdrip/ to ignore list");
+    console.log("  • tsconfig.json - exclude mdrip/ from compilation");
+    console.log("  • AGENTS.md - add markdown snapshot reference section\n");
+    const allowed = await confirm("Allow mdrip to modify these files?");
+    await setFileModificationPermission(allowed, cwd);
+    if (allowed) {
+        console.log("✓ Permission granted - saved to mdrip/settings.json\n");
+    }
+    else {
+        console.log("✗ Permission denied - saved to mdrip/settings.json\n");
+    }
+    return allowed;
+}
+async function fetchUrlInput(spec, cwd, timeoutMs, htmlFallback) {
+    let normalizedUrl;
+    try {
+        normalizedUrl = normalizeUrl(spec);
+    }
+    catch (err) {
+        return {
+            url: spec,
+            path: "",
+            success: false,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+    console.log(`\nFetching ${normalizedUrl}...`);
+    try {
+        const existing = await getPageInfo(normalizedUrl, cwd);
+        if (existing) {
+            console.log(`  → Updating existing snapshot at mdrip/${existing.path}`);
+        }
+        const response = await fetchMarkdownPage(normalizedUrl, {
+            timeoutMs,
+            htmlFallback,
+        });
+        const storageUrl = normalizeUrl(response.resolvedUrl || normalizedUrl);
+        const outputPath = await savePageMarkdown(storageUrl, response.markdown, cwd);
+        console.log(`  ✓ Saved to mdrip/${outputPath}`);
+        if (response.markdownTokens !== undefined) {
+            console.log(`  → x-markdown-tokens: ${response.markdownTokens}`);
+        }
+        if (response.source === "html-fallback") {
+            console.log("  → Converted from HTML (fallback mode)");
+        }
+        if (response.contentSignal) {
+            console.log(`  → Content-Signal: ${response.contentSignal}`);
+        }
+        if (response.resolvedUrl && response.resolvedUrl !== normalizedUrl) {
+            console.log(`  → Resolved URL: ${response.resolvedUrl}`);
+        }
+        return {
+            url: normalizedUrl,
+            resolvedUrl: response.resolvedUrl,
+            path: outputPath,
+            success: true,
+            status: response.status,
+            contentType: response.contentType,
+            markdownTokens: response.markdownTokens,
+            contentSignal: response.contentSignal,
+            source: response.source,
+        };
+    }
+    catch (err) {
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        console.log(`  ✗ Failed: ${errorMessage}`);
+        return {
+            url: normalizedUrl,
+            path: "",
+            success: false,
+            error: errorMessage,
+        };
+    }
+}
+function mergeResults(existing, results) {
+    const now = new Date().toISOString();
+    const merged = [...existing];
+    for (const result of results) {
+        if (!result.success) {
+            continue;
+        }
+        const entry = {
+            url: result.url,
+            resolvedUrl: result.resolvedUrl,
+            path: result.path,
+            fetchedAt: now,
+            status: result.status || 200,
+            contentType: result.contentType || "text/markdown",
+            markdownTokens: result.markdownTokens,
+            contentSignal: result.contentSignal,
+            source: result.source,
+        };
+        const index = merged.findIndex((page) => page.url === result.url);
+        if (index >= 0) {
+            merged[index] = entry;
+        }
+        else {
+            merged.push(entry);
+        }
+    }
+    return merged;
+}
+export async function fetchCommand(urls, options = {}) {
+    const cwd = options.cwd || process.cwd();
+    const results = [];
+    const canModifyFiles = await checkFileModificationPermission(cwd, options.allowModifications);
+    if (canModifyFiles) {
+        const gitignoreUpdated = await ensureGitignore(cwd);
+        if (gitignoreUpdated) {
+            console.log("✓ Added mdrip/ to .gitignore");
+        }
+        const tsconfigUpdated = await ensureTsconfigExclude(cwd);
+        if (tsconfigUpdated) {
+            console.log("✓ Added mdrip/ to tsconfig.json exclude");
+        }
+    }
+    for (const spec of urls) {
+        const result = await fetchUrlInput(spec, cwd, options.timeoutMs, options.htmlFallback);
+        results.push(result);
+    }
+    const successful = results.filter((result) => result.success).length;
+    const failed = results.length - successful;
+    console.log(`\nDone: ${successful} succeeded, ${failed} failed`);
+    if (successful > 0) {
+        const existing = await listSources(cwd);
+        const pages = mergeResults(existing.pages, results);
+        if (canModifyFiles) {
+            const agentsUpdated = await updateAgentsMd({ pages }, cwd);
+            if (agentsUpdated) {
+                console.log("✓ Updated AGENTS.md");
+            }
+        }
+        else {
+            await updatePageIndex({ pages }, cwd);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=fetch.js.map

package/dist/commands/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/commands/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,oBAAoB,CAAC;AAClG,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAU7C,KAAK,UAAU,+BAA+B,CAC5C,GAAW,EACX,WAAqB;IAErB,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,6BAA6B,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACtD,IAAI,WAAW,EAAE,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QACzD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,6BAA6B,CAAC,GAAG,CAAC,CAAC;IAClE,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;IAC9E,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;IAEzE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,oCAAoC,CAAC,CAAC;IAEpE,MAAM,6BAA6B,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAElD,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACvE,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,IAAY,EACZ,GAAW,EACX,SAAkB,EAClB,YAAsB;IAEtB,IAAI,aAAqB,CAAC;IAE1B,IAAI,CAAC;QACH,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,GAAG,EAAE,IAAI;YACT,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,cAAc,aAAa,KAAK,CAAC,CAAC;IAE9C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;QACvD,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,2CAA2C,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,aAAa,EAAE;YACtD,SAAS;YACT,YAAY;SACb,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,CAAC,WAAW,IAAI,aAAa,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE9E,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAEhD,IAAI,QAAQ,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,uBAAuB,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW,KAAK,aAAa,EAAE,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,qBAAqB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,MAAM,EAAE,QAAQ,CAAC,MAAM;SACxB,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,YAAY,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,eAAe,YAAY,EAAE,CAAC,CAAC;QAC3C,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,QAAqB,EAAE,OAAsB;IACjE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAE7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAc;YACvB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,GAAG;YACd,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG;YAC5B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,eAAe;YAClD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAc,EACd,UAAwB,EAAE;IAE1B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,MAAM,cAAc,GAAG,MAAM,+BAA+B,CAC1D,GAAG,EACH,OAAO,CAAC,kBAAkB,CAC3B,CAAC;IAEF,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;QACpD,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;QACzD,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,aAAa,CAChC,IAAI,EACJ,GAAG,EACH,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,YAAY,CACrB,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC;IAE3C,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,eAAe,MAAM,SAAS,CAAC,CAAC;IAEjE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEpD,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;YAC3D,IAAI,aAAa,EAAE,CAAC;gBAClB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/commands/list.d.ts

@@ -0,0 +1,6 @@
+export interface ListOptions {
+    cwd?: string;
+    json?: boolean;
+}
+export declare function listCommand(options?: ListOptions): Promise<void>;
+//# sourceMappingURL=list.d.ts.map

package/dist/commands/list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../src/commands/list.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,wBAAsB,WAAW,CAAC,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiE1E"}

package/dist/commands/list.js

@@ -0,0 +1,55 @@
+import { listSources } from "../lib/storage.js";
+import { getUrlHost } from "../lib/url.js";
+export async function listCommand(options = {}) {
+    const cwd = options.cwd || process.cwd();
+    const sources = await listSources(cwd);
+    if (sources.pages.length === 0) {
+        console.log("No markdown pages fetched yet.");
+        console.log("\nUse `mdrip <url>` to fetch markdown for a page.");
+        return;
+    }
+    if (options.json) {
+        console.log(JSON.stringify(sources, null, 2));
+        return;
+    }
+    const byHost = new Map();
+    for (const page of sources.pages) {
+        const host = getUrlHost(page.resolvedUrl || page.url);
+        const list = byHost.get(host) || [];
+        list.push(page);
+        byHost.set(host, list);
+    }
+    const hosts = [...byHost.keys()].sort((a, b) => a.localeCompare(b));
+    for (const host of hosts) {
+        console.log(`${host}:\n`);
+        const pages = byHost
+            .get(host)
+            .slice()
+            .sort((a, b) => b.fetchedAt.localeCompare(a.fetchedAt));
+        for (const page of pages) {
+            const date = new Date(page.fetchedAt);
+            const formattedDate = date.toLocaleDateString("en-US", {
+                year: "numeric",
+                month: "short",
+                day: "numeric",
+            });
+            console.log(`  ${page.url}`);
+            console.log(`    Path: mdrip/${page.path}`);
+            console.log(`    Fetched: ${formattedDate}`);
+            if (page.source) {
+                const sourceLabel = page.source === "cloudflare-markdown"
+                    ? "Cloudflare Markdown for Agents"
+                    : "HTML fallback conversion";
+                console.log(`    Source: ${sourceLabel}`);
+            }
+            if (page.markdownTokens !== undefined) {
+                console.log(`    Tokens: ${page.markdownTokens}`);
+            }
+            console.log("");
+        }
+    }
+    const totalTokens = sources.pages.reduce((acc, page) => acc + (page.markdownTokens || 0), 0);
+    const tokenSummary = totalTokens > 0 ? `, ~${totalTokens} tokens` : "";
+    console.log(`Total: ${sources.pages.length} page(s), ${hosts.length} domain(s)${tokenSummary}`);
+}
+//# sourceMappingURL=list.js.map

package/dist/commands/list.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.js","sourceRoot":"","sources":["../../src/commands/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAO3C,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAuB,EAAE;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IAEvC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;QACjE,OAAO;IACT,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgC,CAAC;IAEvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM;aACjB,GAAG,CAAC,IAAI,CAAE;aACV,KAAK,EAAE;aACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAE1D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE;gBACrD,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,OAAO;gBACd,GAAG,EAAE,SAAS;aACf,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,gBAAgB,aAAa,EAAE,CAAC,CAAC;YAC7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,WAAW,GACf,IAAI,CAAC,MAAM,KAAK,qBAAqB;oBACnC,CAAC,CAAC,gCAAgC;oBAClC,CAAC,CAAC,0BAA0B,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;YACpD,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC,EAC/C,CAAC,CACF,CAAC;IAEF,MAAM,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,WAAW,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,KAAK,CAAC,MAAM,aAAa,KAAK,CAAC,MAAM,aAAa,YAAY,EAAE,CAAC,CAAC;AAClG,CAAC"}

package/dist/commands/remove.d.ts

@@ -0,0 +1,5 @@
+export interface RemoveOptions {
+    cwd?: string;
+}
+export declare function removeCommand(urls: string[], options?: RemoveOptions): Promise<void>;
+//# sourceMappingURL=remove.d.ts.map

package/dist/commands/remove.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remove.d.ts","sourceRoot":"","sources":["../../src/commands/remove.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA2Df"}

package/dist/commands/remove.js

@@ -0,0 +1,53 @@
+import { listSources, removeStoredPage, cleanupEmptyPageDirs } from "../lib/storage.js";
+import { normalizeUrl } from "../lib/url.js";
+import { updateAgentsMd, updatePageIndex } from "../lib/agents.js";
+import { getFileModificationPermission } from "../lib/settings.js";
+export async function removeCommand(urls, options = {}) {
+    const cwd = options.cwd || process.cwd();
+    const sources = await listSources(cwd);
+    let pages = [...sources.pages];
+    let removed = 0;
+    let notFound = 0;
+    for (const spec of urls) {
+        let normalized;
+        try {
+            normalized = normalizeUrl(spec);
+        }
+        catch {
+            console.log(`  ⚠ Invalid URL: ${spec}`);
+            notFound++;
+            continue;
+        }
+        const index = pages.findIndex((page) => page.url === normalized || page.resolvedUrl === normalized);
+        if (index === -1) {
+            console.log(`  ⚠ ${normalized} not found`);
+            notFound++;
+            continue;
+        }
+        const entry = pages[index];
+        await removeStoredPage(entry.path, cwd);
+        pages = pages.filter((_, idx) => idx !== index);
+        console.log(`  ✓ Removed ${normalized}`);
+        removed++;
+    }
+    if (removed > 0) {
+        await cleanupEmptyPageDirs(cwd);
+        const canModifyFiles = await getFileModificationPermission(cwd);
+        if (canModifyFiles) {
+            const agentsUpdated = await updateAgentsMd({ pages }, cwd);
+            if (agentsUpdated) {
+                if (pages.length === 0) {
+                    console.log("✓ Removed mdrip section from AGENTS.md");
+                }
+                else {
+                    console.log("✓ Updated AGENTS.md");
+                }
+            }
+        }
+        else {
+            await updatePageIndex({ pages }, cwd);
+        }
+    }
+    console.log(`\nRemoved ${removed} page(s)${notFound > 0 ? `, ${notFound} not found` : ""}`);
+}
+//# sourceMappingURL=remove.js.map

package/dist/commands/remove.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remove.js","sourceRoot":"","sources":["../../src/commands/remove.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACxF,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oBAAoB,CAAC;AAOnE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAc,EACd,UAAyB,EAAE;IAE3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,KAAK,GAAgB,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAE5C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,UAAkB,CAAC;QAEvB,IAAI,CAAC;YACH,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;YACxC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAC3B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,IAAI,IAAI,CAAC,WAAW,KAAK,UAAU,CACrE,CAAC;QAEF,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,OAAO,UAAU,YAAY,CAAC,CAAC;YAC3C,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3B,MAAM,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACxC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC;QAEhD,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;QACzC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,cAAc,GAAG,MAAM,6BAA6B,CAAC,GAAG,CAAC,CAAC;QAEhE,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;YAC3D,IAAI,aAAa,EAAE,CAAC;gBAClB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;gBACxD,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CACT,aAAa,OAAO,WAAW,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/E,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}

package/dist/index.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { fetchCommand } from "./commands/fetch.js";
+import { listCommand } from "./commands/list.js";
+import { removeCommand } from "./commands/remove.js";
+import { cleanCommand } from "./commands/clean.js";
+const program = new Command();
+program
+    .name("mdrip")
+    .description("Fetch markdown snapshots for URLs using Cloudflare Markdown for Agents")
+    .version("0.1.0")
+    .option("--cwd <path>", "working directory (default: current directory)");
+program
+    .argument("[urls...]", "URLs to fetch as markdown")
+    .option("--modify [value]", "allow/deny modifying .gitignore, tsconfig.json, AGENTS.md", (val) => {
+    if (val === undefined || val === "" || val === "true")
+        return true;
+    if (val === "false")
+        return false;
+    return true;
+})
+    .option("--timeout <ms>", "request timeout in milliseconds", (value) => {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        throw new Error("--timeout must be a positive integer");
+    }
+    return parsed;
+})
+    .option("--no-html-fallback", "disable HTML->Markdown fallback when text/markdown is unavailable")
+    .action(async (urls, options, command) => {
+    const globalOptions = command.optsWithGlobals();
+    if (urls.length === 0) {
+        program.help();
+        return;
+    }
+    await fetchCommand(urls, {
+        cwd: globalOptions.cwd,
+        allowModifications: options.modify,
+        timeoutMs: options.timeout,
+        htmlFallback: options.htmlFallback,
+    });
+});
+program
+    .command("list")
+    .description("List all fetched markdown pages")
+    .option("--json", "output as JSON")
+    .action(async (options, command) => {
+    const globalOptions = command.optsWithGlobals();
+    await listCommand({
+        json: options.json,
+        cwd: globalOptions.cwd,
+    });
+});
+program
+    .command("remove <urls...>")
+    .alias("rm")
+    .description("Remove fetched markdown snapshots for one or more URLs")
+    .action(async (urls, _options, command) => {
+    const globalOptions = command.optsWithGlobals();
+    await removeCommand(urls, { cwd: globalOptions.cwd });
+});
+program
+    .command("clean")
+    .description("Remove all fetched markdown snapshots (or only one domain)")
+    .option("--domain <host>", "only clean snapshots for a specific domain")
+    .action(async (options, command) => {
+    const globalOptions = command.optsWithGlobals();
+    await cleanCommand({
+        domain: options.domain,
+        cwd: globalOptions.cwd,
+    });
+});
+program.parse();
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,OAAO,CAAC;KACb,WAAW,CACV,wEAAwE,CACzE;KACA,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,CAAC,cAAc,EAAE,gDAAgD,CAAC,CAAC;AAE5E,OAAO;KACJ,QAAQ,CAAC,WAAW,EAAE,2BAA2B,CAAC;KAClD,MAAM,CACL,kBAAkB,EAClB,2DAA2D,EAC3D,CAAC,GAAG,EAAE,EAAE;IACN,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACnE,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IAClC,OAAO,IAAI,CAAC;AACd,CAAC,CACF;KACA,MAAM,CAAC,gBAAgB,EAAE,iCAAiC,EAAE,CAAC,KAAK,EAAE,EAAE;IACrE,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;KACD,MAAM,CACL,oBAAoB,EACpB,mEAAmE,CACpE;KACA,MAAM,CACL,KAAK,EACH,IAAc,EACd,OAAuE,EACvE,OAAgB,EAChB,EAAE;IACF,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,EAAoB,CAAC;IAElE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,OAAO;IACT,CAAC;IAED,MAAM,YAAY,CAAC,IAAI,EAAE;QACvB,GAAG,EAAE,aAAa,CAAC,GAAG;QACtB,kBAAkB,EAAE,OAAO,CAAC,MAAM;QAClC,SAAS,EAAE,OAAO,CAAC,OAAO;QAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;KACnC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEJ,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,iCAAiC,CAAC;KAC9C,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAClC,MAAM,CAAC,KAAK,EAAE,OAA2B,EAAE,OAAgB,EAAE,EAAE;IAC9D,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,EAAoB,CAAC;IAClE,MAAM,WAAW,CAAC;QAChB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,GAAG,EAAE,aAAa,CAAC,GAAG;KACvB,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,kBAAkB,CAAC;KAC3B,KAAK,CAAC,IAAI,CAAC;KACX,WAAW,CAAC,wDAAwD,CAAC;KACrE,MAAM,CAAC,KAAK,EAAE,IAAc,EAAE,QAAiB,EAAE,OAAgB,EAAE,EAAE;IACpE,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,EAAoB,CAAC;IAClE,MAAM,aAAa,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,GAAG,EAAE,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,iBAAiB,EAAE,4CAA4C,CAAC;KACvE,MAAM,CAAC,KAAK,EAAE,OAA4B,EAAE,OAAgB,EAAE,EAAE;IAC/D,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,EAAoB,CAAC;IAClE,MAAM,YAAY,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG,EAAE,aAAa,CAAC,GAAG;KACvB,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/lib/agents.d.ts

@@ -0,0 +1,10 @@
+import type { PageEntry } from "../types.js";
+export declare function updatePageIndex(sources: {
+    pages: PageEntry[];
+}, cwd?: string): Promise<void>;
+export declare function ensureAgentsMd(cwd?: string): Promise<boolean>;
+export declare function removeMdripSection(cwd?: string): Promise<boolean>;
+export declare function updateAgentsMd(sources: {
+    pages: PageEntry[];
+}, cwd?: string): Promise<boolean>;
+//# sourceMappingURL=agents.d.ts.map

package/dist/lib/agents.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agents.d.ts","sourceRoot":"","sources":["../../src/lib/agents.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,aAAa,CAAC;AAgD3D,wBAAsB,eAAe,CACnC,OAAO,EAAE;IAAE,KAAK,EAAE,SAAS,EAAE,CAAA;CAAE,EAC/B,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,cAAc,CAClC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAwClB;AAED,wBAAsB,kBAAkB,CACtC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAoClB;AAED,wBAAsB,cAAc,CAClC,OAAO,EAAE;IAAE,KAAK,EAAE,SAAS,EAAE,CAAA;CAAE,EAC/B,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAQlB"}