mdrip 0.1.2 → 0.1.3

package/README.md

@@ -58,6 +58,48 @@ Or use with `npx`:
 npx mdrip <url>
 ```
 
+For programmatic usage in Node.js or Workers:
+
+```bash
+npm install mdrip
+```
+
+## Programmatic API
+
+### Node.js (fetch and store)
+
+```ts
+import { fetchToStore, listStoredPages } from "mdrip/node";
+
+const result = await fetchToStore("https://developers.cloudflare.com/", {
+  cwd: process.cwd(),
+});
+
+if (!result.success) {
+  throw new Error(result.error || "Failed to fetch page");
+}
+
+const pages = await listStoredPages(process.cwd());
+console.log(pages.map((p) => p.path));
+```
+
+### Cloudflare Workers / Agent runtimes (raw in-memory markdown)
+
+```ts
+import { fetchMarkdown } from "mdrip";
+
+const page = await fetchMarkdown(
+  "https://blog.cloudflare.com/markdown-for-agents/",
+);
+
+console.log(page.markdownTokens);
+console.log(page.markdown);
+```
+
+Available programmatic methods:
+- `mdrip` (Workers-safe): `fetchMarkdown(url, options)`, `fetchRawMarkdown(url, options)`
+- `mdrip/node` (filesystem features): `fetchToStore(url, options)`, `fetchManyToStore(urls, options)`, `listStoredPages(cwd?)`
+
 ## Usage
 
 ### Fetch pages

package/dist/api.d.ts

@@ -0,0 +1,12 @@
+import { type FetchMarkdownOptions, type MarkdownResponse } from "./lib/cloudflare.js";
+import type { FetchResult, PageEntry, SourcesIndex } from "./types.js";
+export type { FetchMarkdownOptions, MarkdownResponse, FetchResult, PageEntry, SourcesIndex, };
+export interface StoreFetchOptions extends FetchMarkdownOptions {
+    cwd?: string;
+}
+export declare function fetchMarkdown(url: string, options?: FetchMarkdownOptions): Promise<MarkdownResponse>;
+export declare function fetchRawMarkdown(url: string, options?: FetchMarkdownOptions): Promise<string>;
+export declare function fetchToStore(url: string, options?: StoreFetchOptions): Promise<FetchResult>;
+export declare function fetchManyToStore(urls: string[], options?: StoreFetchOptions): Promise<FetchResult[]>;
+export declare function listStoredPages(cwd?: string): Promise<PageEntry[]>;
+//# sourceMappingURL=api.d.ts.map

package/dist/api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAO7B,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvE,YAAY,EACV,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,SAAS,EACT,YAAY,GACb,CAAC;AAEF,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB;IAC7D,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA8ED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CAG3B;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,WAAW,CAAC,CAatB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,WAAW,EAAE,CAAC,CAiBxB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,SAAS,EAAE,CAAC,CAGtB"}

package/dist/api.js

@@ -0,0 +1,109 @@
+import { fetchMarkdownPage, } from "./lib/cloudflare.js";
+import { listSources, savePageMarkdown, } from "./lib/storage.js";
+import { updatePageIndex } from "./lib/agents.js";
+import { normalizeUrl } from "./lib/url.js";
+function mergeResults(existing, results) {
+    const now = new Date().toISOString();
+    const merged = [...existing];
+    for (const result of results) {
+        if (!result.success || !result.path) {
+            continue;
+        }
+        const entry = {
+            url: result.url,
+            resolvedUrl: result.resolvedUrl,
+            path: result.path,
+            fetchedAt: now,
+            status: result.status || 200,
+            contentType: result.contentType || "text/markdown",
+            markdownTokens: result.markdownTokens,
+            contentSignal: result.contentSignal,
+            source: result.source,
+        };
+        const index = merged.findIndex((page) => page.url === result.url);
+        if (index >= 0) {
+            merged[index] = entry;
+        }
+        else {
+            merged.push(entry);
+        }
+    }
+    return merged;
+}
+async function fetchOneForStore(spec, cwd, options) {
+    let normalizedUrl;
+    try {
+        normalizedUrl = normalizeUrl(spec);
+    }
+    catch (err) {
+        return {
+            url: spec,
+            path: "",
+            success: false,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+    try {
+        const response = await fetchMarkdownPage(normalizedUrl, options);
+        const storageUrl = normalizeUrl(response.resolvedUrl || normalizedUrl);
+        const outputPath = await savePageMarkdown(storageUrl, response.markdown, cwd);
+        return {
+            url: normalizedUrl,
+            resolvedUrl: response.resolvedUrl,
+            path: outputPath,
+            success: true,
+            status: response.status,
+            contentType: response.contentType,
+            markdownTokens: response.markdownTokens,
+            contentSignal: response.contentSignal,
+            source: response.source,
+        };
+    }
+    catch (err) {
+        return {
+            url: normalizedUrl,
+            path: "",
+            success: false,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+export async function fetchMarkdown(url, options = {}) {
+    const normalizedUrl = normalizeUrl(url);
+    return fetchMarkdownPage(normalizedUrl, options);
+}
+export async function fetchRawMarkdown(url, options = {}) {
+    const response = await fetchMarkdown(url, options);
+    return response.markdown;
+}
+export async function fetchToStore(url, options = {}) {
+    const cwd = options.cwd || process.cwd();
+    const result = await fetchOneForStore(url, cwd, options);
+    if (!result.success) {
+        return result;
+    }
+    const existing = await listSources(cwd);
+    const pages = mergeResults(existing.pages, [result]);
+    await updatePageIndex({ pages }, cwd);
+    return result;
+}
+export async function fetchManyToStore(urls, options = {}) {
+    const cwd = options.cwd || process.cwd();
+    const results = [];
+    for (const spec of urls) {
+        const result = await fetchOneForStore(spec, cwd, options);
+        results.push(result);
+    }
+    const successful = results.filter((result) => result.success);
+    if (successful.length > 0) {
+        const existing = await listSources(cwd);
+        const pages = mergeResults(existing.pages, successful);
+        await updatePageIndex({ pages }, cwd);
+    }
+    return results;
+}
+export async function listStoredPages(cwd = process.cwd()) {
+    const sources = await listSources(cwd);
+    return sources.pages;
+}
+//# sourceMappingURL=api.js.map

package/dist/api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,GAGlB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,WAAW,EACX,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAe5C,SAAS,YAAY,CAAC,QAAqB,EAAE,OAAsB;IACjE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAE7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAc;YACvB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,GAAG;YACd,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG;YAC5B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,eAAe;YAClD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,IAAI,aAAqB,CAAC;IAE1B,IAAI,CAAC;QACH,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,GAAG,EAAE,IAAI;YACT,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjE,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,CAAC,WAAW,IAAI,aAAa,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE9E,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,MAAM,EAAE,QAAQ,CAAC,MAAM;SACxB,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnD,OAAO,QAAQ,CAAC,QAAQ,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UAA6B,EAAE;IAE/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;IAEtC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAc,EACd,UAA6B,EAAE;IAE/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,OAAO,CAAC,KAAK,CAAC;AACvB,CAAC"}

package/dist/api.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=api.test.d.ts.map

package/dist/api.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.test.d.ts","sourceRoot":"","sources":["../src/api.test.ts"],"names":[],"mappings":""}

package/dist/api.test.js

@@ -0,0 +1,89 @@
+import { describe, it, expect } from "vitest";
+import { mkdtemp, readFile, rm } from "fs/promises";
+import { join } from "path";
+import { tmpdir } from "os";
+import { fetchToStore, listStoredPages, fetchManyToStore, } from "./api.js";
+import { fetchMarkdown, fetchRawMarkdown } from "./web.js";
+describe("api", () => {
+    it("fetchMarkdown normalizes URL before requesting", async () => {
+        let requestedUrl = "";
+        const mockFetch = async (input) => {
+            requestedUrl = String(input);
+            return new Response("# Hello", {
+                status: 200,
+                headers: {
+                    "content-type": "text/markdown; charset=utf-8",
+                },
+            });
+        };
+        const result = await fetchMarkdown("example.com/docs", {
+            fetchImpl: mockFetch,
+        });
+        expect(requestedUrl).toBe("https://example.com/docs");
+        expect(result.markdown).toBe("# Hello");
+    });
+    it("fetchRawMarkdown returns markdown content only", async () => {
+        const mockFetch = async () => new Response("# Raw", {
+            status: 200,
+            headers: {
+                "content-type": "text/markdown; charset=utf-8",
+            },
+        });
+        const markdown = await fetchRawMarkdown("example.com", {
+            fetchImpl: mockFetch,
+        });
+        expect(markdown).toBe("# Raw");
+    });
+    it("fetchToStore writes snapshot and updates sources index", async () => {
+        const cwd = await mkdtemp(join(tmpdir(), "mdrip-api-"));
+        try {
+            const mockFetch = async () => new Response("# Stored", {
+                status: 200,
+                headers: {
+                    "content-type": "text/markdown; charset=utf-8",
+                    "x-markdown-tokens": "2",
+                },
+            });
+            const result = await fetchToStore("https://example.com/docs/page", {
+                cwd,
+                fetchImpl: mockFetch,
+            });
+            expect(result.success).toBe(true);
+            expect(result.path).toBe("pages/example.com/docs/page.md");
+            const pages = await listStoredPages(cwd);
+            expect(pages).toHaveLength(1);
+            expect(pages[0].url).toBe("https://example.com/docs/page");
+            const filePath = join(cwd, "mdrip", result.path);
+            const content = await readFile(filePath, "utf-8");
+            expect(content).toBe("# Stored");
+        }
+        finally {
+            await rm(cwd, { recursive: true, force: true });
+        }
+    });
+    it("fetchManyToStore records mixed success results", async () => {
+        const cwd = await mkdtemp(join(tmpdir(), "mdrip-api-"));
+        try {
+            const mockFetch = async () => new Response("# Multi", {
+                status: 200,
+                headers: {
+                    "content-type": "text/markdown; charset=utf-8",
+                },
+            });
+            const results = await fetchManyToStore(["https://example.com/a", "ftp://example.com/a"], {
+                cwd,
+                fetchImpl: mockFetch,
+            });
+            expect(results).toHaveLength(2);
+            expect(results[0].success).toBe(true);
+            expect(results[1].success).toBe(false);
+            const pages = await listStoredPages(cwd);
+            expect(pages).toHaveLength(1);
+            expect(pages[0].url).toBe("https://example.com/a");
+        }
+        finally {
+            await rm(cwd, { recursive: true, force: true });
+        }
+    });
+});
+//# sourceMappingURL=api.test.js.map

package/dist/api.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.test.js","sourceRoot":"","sources":["../src/api.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,eAAe,EACf,gBAAgB,GACjB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE3D,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE;IACnB,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,MAAM,SAAS,GAAiB,KAAK,EAAE,KAAK,EAAE,EAAE;YAC9C,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,OAAO,IAAI,QAAQ,CAAC,SAAS,EAAE;gBAC7B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,8BAA8B;iBAC/C;aACF,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE;YACrD,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACtD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,OAAO,EAAE;YACpB,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,8BAA8B;aAC/C;SACF,CAAC,CAAC;QAEL,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE;YACrD,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACvB,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,8BAA8B;oBAC9C,mBAAmB,EAAE,GAAG;iBACzB;aACF,CAAC,CAAC;YAEL,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,+BAA+B,EAAE;gBACjE,GAAG;gBACH,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YAE3D,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAE3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,SAAS,GAAiB,KAAK,IAAI,EAAE,CACzC,IAAI,QAAQ,CAAC,SAAS,EAAE;gBACtB,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,8BAA8B;iBAC/C;aACF,CAAC,CAAC;YAEL,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,EAChD;gBACE,GAAG;gBACH,SAAS,EAAE,SAAS;aACrB,CACF,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEvC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrD,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/index.js

@@ -8,7 +8,7 @@ const program = new Command();
 program
     .name("mdrip")
     .description("Fetch markdown snapshots for URLs using Cloudflare Markdown for Agents")
-    .version("0.1.2")
+    .version("0.1.3")
     .option("--cwd <path>", "working directory (default: current directory)");
 program
     .argument("[urls...]", "URLs to fetch as markdown")

package/dist/lib/gitignore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAOA,wBAAsB,aAAa,CACjC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAkBlB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB"}
+{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAOA,wBAAsB,aAAa,CACjC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB;AAED,wBAAsB,eAAe,CACnC,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAuBlB"}

package/dist/lib/gitignore.js

@@ -1,7 +1,7 @@
 import { readFile, writeFile } from "fs/promises";
 import { join } from "path";
 import { existsSync } from "fs";
-const MDRIP_ENTRY = "mdrip/";
+const MDRIP_ENTRY = "/mdrip/";
 const MARKER_COMMENT = "# mdrip - markdown snapshots for agents";
 export async function hasMdripEntry(cwd = process.cwd()) {
     const gitignorePath = join(cwd, ".gitignore");
@@ -13,7 +13,10 @@ export async function hasMdripEntry(cwd = process.cwd()) {
         const lines = content.split("\n");
         return lines.some((line) => {
             const trimmed = line.trim();
-            return trimmed === MDRIP_ENTRY || trimmed === "mdrip";
+            return (trimmed === MDRIP_ENTRY ||
+                trimmed === "mdrip/" ||
+                trimmed === "/mdrip" ||
+                trimmed === "mdrip");
         });
     }
     catch {

package/dist/lib/gitignore.js.map

@@ -1 +1 @@
-{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,MAAM,WAAW,GAAG,QAAQ,CAAC;AAC7B,MAAM,cAAc,GAAG,yCAAyC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,GAAG,cAAc,KAAK,WAAW,IAAI,CAAC;IAEjD,MAAM,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../../src/lib/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,MAAM,WAAW,GAAG,SAAS,CAAC;AAC9B,MAAM,cAAc,GAAG,yCAAyC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,CACL,OAAO,KAAK,WAAW;gBACvB,OAAO,KAAK,QAAQ;gBACpB,OAAO,KAAK,QAAQ;gBACpB,OAAO,KAAK,OAAO,CACpB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,GAAG,cAAc,KAAK,WAAW,IAAI,CAAC;IAEjD,MAAM,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/web.d.ts

@@ -0,0 +1,5 @@
+import { type FetchMarkdownOptions, type MarkdownResponse } from "./lib/cloudflare.js";
+export type { FetchMarkdownOptions, MarkdownResponse };
+export declare function fetchMarkdown(url: string, options?: FetchMarkdownOptions): Promise<MarkdownResponse>;
+export declare function fetchRawMarkdown(url: string, options?: FetchMarkdownOptions): Promise<string>;
+//# sourceMappingURL=web.d.ts.map

package/dist/web.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,CAAC;AAEvD,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CAG3B;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CAGjB"}

package/dist/web.js

@@ -0,0 +1,11 @@
+import { fetchMarkdownPage, } from "./lib/cloudflare.js";
+import { normalizeUrl } from "./lib/url.js";
+export async function fetchMarkdown(url, options = {}) {
+    const normalizedUrl = normalizeUrl(url);
+    return fetchMarkdownPage(normalizedUrl, options);
+}
+export async function fetchRawMarkdown(url, options = {}) {
+    const response = await fetchMarkdown(url, options);
+    return response.markdown;
+}
+//# sourceMappingURL=web.js.map

package/dist/web.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.js","sourceRoot":"","sources":["../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,GAGlB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnD,OAAO,QAAQ,CAAC,QAAQ,CAAC;AAC3B,CAAC"}

package/package.json

@@ -1,8 +1,24 @@
 {
   "name": "mdrip",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Fetch markdown snapshots of web pages using Cloudflare Markdown for Agents",
   "type": "module",
+  "main": "./dist/web.js",
+  "types": "./dist/web.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/web.d.ts",
+      "import": "./dist/web.js"
+    },
+    "./node": {
+      "types": "./dist/api.d.ts",
+      "import": "./dist/api.js"
+    },
+    "./cli": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
   "bin": {
     "mdrip": "./dist/index.js"
   },