mdrip 0.1.0

package/dist/lib/agents.js

@@ -0,0 +1,129 @@
+import { readFile, writeFile, rm } from "fs/promises";
+import { join } from "path";
+import { existsSync } from "fs";
+import { ensureMdripDir } from "./settings.js";
+const AGENTS_FILE = "AGENTS.md";
+const MDRIP_DIR = "mdrip";
+const SOURCES_FILE = "sources.json";
+const SECTION_TITLE = "## Website Markdown Reference";
+const SECTION_MARKER = "<!-- mdrip:start -->";
+const SECTION_END_MARKER = "<!-- mdrip:end -->";
+function getSectionContent() {
+    return `${SECTION_MARKER}
+
+${SECTION_TITLE}
+
+Markdown snapshots of web pages are available in \`mdrip/\` for deeper implementation context in AI workflows.
+
+See \`mdrip/sources.json\` for the list of available pages and metadata such as token estimates.
+
+Use these snapshots when your agent needs structured page content instead of raw HTML.
+
+### Fetching Additional Pages
+
+To fetch markdown for one or more pages, run:
+
+\`\`\`bash
+npx mdrip <url>
+npx mdrip https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/
+\`\`\`
+
+${SECTION_END_MARKER}`;
+}
+function getSourcesPath(cwd) {
+    return join(cwd, MDRIP_DIR, SOURCES_FILE);
+}
+function extractSection(content) {
+    const startIdx = content.indexOf(SECTION_MARKER);
+    const endIdx = content.indexOf(SECTION_END_MARKER);
+    if (startIdx === -1 || endIdx === -1) {
+        return null;
+    }
+    return content.slice(startIdx, endIdx + SECTION_END_MARKER.length);
+}
+export async function updatePageIndex(sources, cwd = process.cwd()) {
+    await ensureMdripDir(cwd);
+    const sourcesPath = getSourcesPath(cwd);
+    if (sources.pages.length === 0) {
+        if (existsSync(sourcesPath)) {
+            await rm(sourcesPath, { force: true });
+        }
+        return;
+    }
+    const index = {
+        updatedAt: new Date().toISOString(),
+        pages: sources.pages.map((page) => ({ ...page })),
+    };
+    await writeFile(sourcesPath, JSON.stringify(index, null, 2) + "\n", "utf-8");
+}
+export async function ensureAgentsMd(cwd = process.cwd()) {
+    const agentsPath = join(cwd, AGENTS_FILE);
+    const newSection = getSectionContent();
+    if (existsSync(agentsPath)) {
+        const content = await readFile(agentsPath, "utf-8");
+        if (content.includes(SECTION_MARKER)) {
+            const existingSection = extractSection(content);
+            if (existingSection === newSection) {
+                return false;
+            }
+            const startIdx = content.indexOf(SECTION_MARKER);
+            const endIdx = content.indexOf(SECTION_END_MARKER);
+            const before = content.slice(0, startIdx);
+            const after = content.slice(endIdx + SECTION_END_MARKER.length);
+            await writeFile(agentsPath, before + newSection + after, "utf-8");
+            return true;
+        }
+        let newContent = content;
+        if (newContent.length > 0 && !newContent.endsWith("\n")) {
+            newContent += "\n";
+        }
+        newContent += "\n" + newSection;
+        await writeFile(agentsPath, newContent, "utf-8");
+        return true;
+    }
+    const content = `# AGENTS.md
+
+Instructions for AI coding agents working with this codebase.
+
+${newSection}
+`;
+    await writeFile(agentsPath, content, "utf-8");
+    return true;
+}
+export async function removeMdripSection(cwd = process.cwd()) {
+    const agentsPath = join(cwd, AGENTS_FILE);
+    if (!existsSync(agentsPath)) {
+        return false;
+    }
+    try {
+        const content = await readFile(agentsPath, "utf-8");
+        if (!content.includes(SECTION_MARKER)) {
+            return false;
+        }
+        const startIdx = content.indexOf(SECTION_MARKER);
+        const endIdx = content.indexOf(SECTION_END_MARKER);
+        if (startIdx === -1 || endIdx === -1) {
+            return false;
+        }
+        const before = content.slice(0, startIdx).trimEnd();
+        const after = content.slice(endIdx + SECTION_END_MARKER.length).trimStart();
+        let newContent = before;
+        if (after) {
+            newContent += "\n\n" + after;
+        }
+        newContent = newContent.replace(/\n{3,}/g, "\n\n").trim() + "\n";
+        await writeFile(agentsPath, newContent, "utf-8");
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function updateAgentsMd(sources, cwd = process.cwd()) {
+    await updatePageIndex(sources, cwd);
+    if (sources.pages.length > 0) {
+        return ensureAgentsMd(cwd);
+    }
+    return removeMdripSection(cwd);
+}
+//# sourceMappingURL=agents.js.map

package/dist/lib/agents.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agents.js","sourceRoot":"","sources":["../../src/lib/agents.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,GAAG,WAAW,CAAC;AAChC,MAAM,SAAS,GAAG,OAAO,CAAC;AAC1B,MAAM,YAAY,GAAG,cAAc,CAAC;AACpC,MAAM,aAAa,GAAG,+BAA+B,CAAC;AACtD,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,kBAAkB,GAAG,oBAAoB,CAAC;AAEhD,SAAS,iBAAiB;IACxB,OAAO,GAAG,cAAc;;EAExB,aAAa;;;;;;;;;;;;;;;;;EAiBb,kBAAkB,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAEnD,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAA+B,EAC/B,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAiB;QAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;KAClD,CAAC;IAEF,MAAM,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,iBAAiB,EAAE,CAAC;IAEvC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAEpD,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YACrC,MAAM,eAAe,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;YAChD,IAAI,eAAe,KAAK,UAAU,EAAE,CAAC;gBACnC,OAAO,KAAK,CAAC;YACf,CAAC;YAED,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAEhE,MAAM,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,GAAG,KAAK,EAAE,OAAO,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,UAAU,GAAG,OAAO,CAAC;QACzB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,UAAU,IAAI,IAAI,CAAC;QACrB,CAAC;QACD,UAAU,IAAI,IAAI,GAAG,UAAU,CAAC;QAChC,MAAM,SAAS,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG;;;;EAIhB,UAAU;CACX,CAAC;IAEA,MAAM,SAAS,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAE1C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAEpD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAEnD,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YACrC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC;QAE5E,IAAI,UAAU,GAAG,MAAM,CAAC;QACxB,IAAI,KAAK,EAAE,CAAC;YACV,UAAU,IAAI,MAAM,GAAG,KAAK,CAAC;QAC/B,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;QAEjE,MAAM,SAAS,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA+B,EAC/B,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAEpC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC"}

package/dist/lib/cloudflare.d.ts

@@ -0,0 +1,17 @@
+export interface FetchMarkdownOptions {
+    timeoutMs?: number;
+    userAgent?: string;
+    htmlFallback?: boolean;
+    fetchImpl?: typeof fetch;
+}
+export interface MarkdownResponse {
+    markdown: string;
+    status: number;
+    contentType: string;
+    resolvedUrl: string;
+    markdownTokens?: number;
+    contentSignal?: string;
+    source: "cloudflare-markdown" | "html-fallback";
+}
+export declare function fetchMarkdownPage(url: string, options?: FetchMarkdownOptions): Promise<MarkdownResponse>;
+//# sourceMappingURL=cloudflare.d.ts.map

package/dist/lib/cloudflare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../../src/lib/cloudflare.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,qBAAqB,GAAG,eAAe,CAAC;CACjD;AAWD,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CA4E3B"}

package/dist/lib/cloudflare.js

@@ -0,0 +1,74 @@
+import { convertHtmlToMarkdown, estimateTokenCount } from "./html-to-markdown.js";
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_ACCEPT = "text/markdown, text/html;q=0.9, */*;q=0.1";
+function parseTokenHeader(value) {
+    if (!value) {
+        return undefined;
+    }
+    const parsed = Number.parseInt(value, 10);
+    return Number.isFinite(parsed) ? parsed : undefined;
+}
+export async function fetchMarkdownPage(url, options = {}) {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const htmlFallback = options.htmlFallback ?? true;
+    const controller = new AbortController();
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const response = await fetchImpl(url, {
+            method: "GET",
+            headers: {
+                Accept: DEFAULT_ACCEPT,
+                "User-Agent": options.userAgent ?? "mdrip-cli",
+            },
+            redirect: "follow",
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+        }
+        const contentType = response.headers.get("content-type") || "";
+        const normalizedType = contentType.toLowerCase();
+        if (normalizedType.includes("text/markdown")) {
+            const markdown = await response.text();
+            return {
+                markdown,
+                status: response.status,
+                contentType,
+                resolvedUrl: response.url || url,
+                markdownTokens: parseTokenHeader(response.headers.get("x-markdown-tokens")),
+                contentSignal: response.headers.get("content-signal") || undefined,
+                source: "cloudflare-markdown",
+            };
+        }
+        const isHtmlResponse = normalizedType.includes("text/html") ||
+            normalizedType.includes("application/xhtml+xml");
+        if (isHtmlResponse && htmlFallback) {
+            const html = await response.text();
+            const markdown = convertHtmlToMarkdown(html, response.url || url);
+            return {
+                markdown,
+                status: response.status,
+                contentType,
+                resolvedUrl: response.url || url,
+                markdownTokens: estimateTokenCount(markdown),
+                contentSignal: response.headers.get("content-signal") || undefined,
+                source: "html-fallback",
+            };
+        }
+        if (isHtmlResponse && !htmlFallback) {
+            throw new Error(`Expected text/markdown but received "${contentType || "unknown"}". Use HTML fallback (default) or remove --no-html-fallback.`);
+        }
+        throw new Error(`Expected text/markdown but received "${contentType || "unknown"}". Response type is not convertible by HTML fallback.`);
+    }
+    catch (err) {
+        if (err instanceof Error && err.name === "AbortError") {
+            throw new Error(`Request timed out after ${timeoutMs}ms`);
+        }
+        throw err;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+//# sourceMappingURL=cloudflare.js.map

package/dist/lib/cloudflare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.js","sourceRoot":"","sources":["../../src/lib/cloudflare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAElF,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,cAAc,GAAG,2CAA2C,CAAC;AAmBnE,SAAS,gBAAgB,CAAC,KAAoB;IAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IAClD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC7C,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YACpC,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,WAAW;aAC/C;YACD,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAE/D,MAAM,cAAc,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QAEjD,IAAI,cAAc,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEvC,OAAO;gBACL,QAAQ;gBACR,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,WAAW;gBACX,WAAW,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG;gBAChC,cAAc,EAAE,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;gBAC3E,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,SAAS;gBAClE,MAAM,EAAE,qBAAqB;aAC9B,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAClB,cAAc,CAAC,QAAQ,CAAC,WAAW,CAAC;YACpC,cAAc,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;QAEnD,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;YAElE,OAAO;gBACL,QAAQ;gBACR,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,WAAW;gBACX,WAAW,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG;gBAChC,cAAc,EAAE,kBAAkB,CAAC,QAAQ,CAAC;gBAC5C,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,SAAS;gBAClE,MAAM,EAAE,eAAe;aACxB,CAAC;QACJ,CAAC;QAED,IAAI,cAAc,IAAI,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,wCAAwC,WAAW,IAAI,SAAS,8DAA8D,CAC/H,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CACb,wCAAwC,WAAW,IAAI,SAAS,uDAAuD,CACxH,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC"}

package/dist/lib/cloudflare.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cloudflare.test.d.ts.map

package/dist/lib/cloudflare.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.test.d.ts","sourceRoot":"","sources":["../../src/lib/cloudflare.test.ts"],"names":[],"mappings":""}

package/dist/lib/cloudflare.test.js

@@ -0,0 +1,59 @@
+import { describe, it, expect } from "vitest";
+import { fetchMarkdownPage } from "./cloudflare.js";
+describe("fetchMarkdownPage", () => {
+    it("uses Cloudflare markdown response when available", async () => {
+        const mockFetch = async () => new Response("# Hello", {
+            status: 200,
+            headers: {
+                "content-type": "text/markdown; charset=utf-8",
+                "x-markdown-tokens": "22",
+                "content-signal": "ai-input=yes",
+            },
+        });
+        const result = await fetchMarkdownPage("https://example.com", {
+            fetchImpl: mockFetch,
+        });
+        expect(result.source).toBe("cloudflare-markdown");
+        expect(result.markdown).toBe("# Hello");
+        expect(result.markdownTokens).toBe(22);
+        expect(result.contentSignal).toBe("ai-input=yes");
+    });
+    it("falls back to HTML conversion when markdown is not returned", async () => {
+        const mockFetch = async () => new Response("<html><body><main><h1>Welcome</h1><p>Page</p></main></body></html>", {
+            status: 200,
+            headers: {
+                "content-type": "text/html; charset=utf-8",
+            },
+        });
+        const result = await fetchMarkdownPage("https://example.com", {
+            fetchImpl: mockFetch,
+        });
+        expect(result.source).toBe("html-fallback");
+        expect(result.markdown).toContain("# Welcome");
+        expect(result.markdownTokens).toBeGreaterThan(0);
+    });
+    it("errors on HTML when fallback is disabled", async () => {
+        const mockFetch = async () => new Response("<html><body>test</body></html>", {
+            status: 200,
+            headers: {
+                "content-type": "text/html; charset=utf-8",
+            },
+        });
+        await expect(fetchMarkdownPage("https://example.com", {
+            fetchImpl: mockFetch,
+            htmlFallback: false,
+        })).rejects.toThrow(/Expected text\/markdown/);
+    });
+    it("errors on unsupported non-markdown content types", async () => {
+        const mockFetch = async () => new Response('{"ok":true}', {
+            status: 200,
+            headers: {
+                "content-type": "application/json",
+            },
+        });
+        await expect(fetchMarkdownPage("https://example.com", {
+            fetchImpl: mockFetch,
+        })).rejects.toThrow(/not convertible by HTML fallback/);
+    });
+});
+//# sourceMappingURL=cloudflare.test.js.map

package/dist/lib/cloudflare.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.test.js","sourceRoot":"","sources":["../../src/lib/cloudflare.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,SAAS,EAAE;YACtB,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,8BAA8B;gBAC9C,mBAAmB,EAAE,IAAI;gBACzB,gBAAgB,EAAE,cAAc;aACjC;SACF,CAAC,CAAC;QAEL,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,qBAAqB,EAAE;YAC5D,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,oEAAoE,EAAE;YACjF,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,0BAA0B;aAC3C;SACF,CAAC,CAAC;QAEL,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,qBAAqB,EAAE;YAC5D,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,gCAAgC,EAAE;YAC7C,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,0BAA0B;aAC3C;SACF,CAAC,CAAC;QAEL,MAAM,MAAM,CACV,iBAAiB,CAAC,qBAAqB,EAAE;YACvC,SAAS,EAAE,SAAS;YACpB,YAAY,EAAE,KAAK;SACpB,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,aAAa,EAAE;YAC1B,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;aACnC;SACF,CAAC,CAAC;QAEL,MAAM,MAAM,CACV,iBAAiB,CAAC,qBAAqB,EAAE;YACvC,SAAS,EAAE,SAAS;SACrB,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/lib/gitignore.d.ts

@@ -0,0 +1,3 @@
+export declare function hasMdripEntry(cwd?: string): Promise<boolean>;
+export declare function ensureGitignore(cwd?: string): Promise<boolean>;
+//# sourceMappingURL=gitignore.d.ts.map

package/dist/lib/gitignore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAOA,wBAAsB,aAAa,CACjC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAkBlB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB"}

package/dist/lib/gitignore.js

@@ -0,0 +1,42 @@
+import { readFile, writeFile } from "fs/promises";
+import { join } from "path";
+import { existsSync } from "fs";
+const MDRIP_ENTRY = "mdrip/";
+const MARKER_COMMENT = "# mdrip - markdown snapshots for agents";
+export async function hasMdripEntry(cwd = process.cwd()) {
+    const gitignorePath = join(cwd, ".gitignore");
+    if (!existsSync(gitignorePath)) {
+        return false;
+    }
+    try {
+        const content = await readFile(gitignorePath, "utf-8");
+        const lines = content.split("\n");
+        return lines.some((line) => {
+            const trimmed = line.trim();
+            return trimmed === MDRIP_ENTRY || trimmed === "mdrip";
+        });
+    }
+    catch {
+        return false;
+    }
+}
+export async function ensureGitignore(cwd = process.cwd()) {
+    const gitignorePath = join(cwd, ".gitignore");
+    if (await hasMdripEntry(cwd)) {
+        return false;
+    }
+    let content = "";
+    if (existsSync(gitignorePath)) {
+        content = await readFile(gitignorePath, "utf-8");
+        if (content.length > 0 && !content.endsWith("\n")) {
+            content += "\n";
+        }
+        if (content.trim().length > 0) {
+            content += "\n";
+        }
+    }
+    content += `${MARKER_COMMENT}\n${MDRIP_ENTRY}\n`;
+    await writeFile(gitignorePath, content, "utf-8");
+    return true;
+}
+//# sourceMappingURL=gitignore.js.map

package/dist/lib/gitignore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,MAAM,WAAW,GAAG,QAAQ,CAAC;AAC7B,MAAM,cAAc,GAAG,yCAAyC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,GAAG,cAAc,KAAK,WAAW,IAAI,CAAC;IAEjD,MAAM,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/lib/html-to-markdown.d.ts

@@ -0,0 +1,3 @@
+export declare function estimateTokenCount(markdown: string): number;
+export declare function convertHtmlToMarkdown(html: string, baseUrl?: string): string;
+//# sourceMappingURL=html-to-markdown.d.ts.map

package/dist/lib/html-to-markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-markdown.d.ts","sourceRoot":"","sources":["../../src/lib/html-to-markdown.ts"],"names":[],"mappings":"AA2XA,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAO3D;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAiB5E"}