mdrip 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +137 -0
  2. package/dist/commands/clean.d.ts +6 -0
  3. package/dist/commands/clean.d.ts.map +1 -0
  4. package/dist/commands/clean.js +49 -0
  5. package/dist/commands/clean.js.map +1 -0
  6. package/dist/commands/fetch.d.ts +9 -0
  7. package/dist/commands/fetch.d.ts.map +1 -0
  8. package/dist/commands/fetch.js +163 -0
  9. package/dist/commands/fetch.js.map +1 -0
  10. package/dist/commands/list.d.ts +6 -0
  11. package/dist/commands/list.d.ts.map +1 -0
  12. package/dist/commands/list.js +55 -0
  13. package/dist/commands/list.js.map +1 -0
  14. package/dist/commands/remove.d.ts +5 -0
  15. package/dist/commands/remove.d.ts.map +1 -0
  16. package/dist/commands/remove.js +53 -0
  17. package/dist/commands/remove.js.map +1 -0
  18. package/dist/index.d.ts +3 -0
  19. package/dist/index.d.ts.map +1 -0
  20. package/dist/index.js +74 -0
  21. package/dist/index.js.map +1 -0
  22. package/dist/lib/agents.d.ts +10 -0
  23. package/dist/lib/agents.d.ts.map +1 -0
  24. package/dist/lib/agents.js +129 -0
  25. package/dist/lib/agents.js.map +1 -0
  26. package/dist/lib/cloudflare.d.ts +17 -0
  27. package/dist/lib/cloudflare.d.ts.map +1 -0
  28. package/dist/lib/cloudflare.js +74 -0
  29. package/dist/lib/cloudflare.js.map +1 -0
  30. package/dist/lib/cloudflare.test.d.ts +2 -0
  31. package/dist/lib/cloudflare.test.d.ts.map +1 -0
  32. package/dist/lib/cloudflare.test.js +59 -0
  33. package/dist/lib/cloudflare.test.js.map +1 -0
  34. package/dist/lib/gitignore.d.ts +3 -0
  35. package/dist/lib/gitignore.d.ts.map +1 -0
  36. package/dist/lib/gitignore.js +42 -0
  37. package/dist/lib/gitignore.js.map +1 -0
  38. package/dist/lib/html-to-markdown.d.ts +3 -0
  39. package/dist/lib/html-to-markdown.d.ts.map +1 -0
  40. package/dist/lib/html-to-markdown.js +312 -0
  41. package/dist/lib/html-to-markdown.js.map +1 -0
  42. package/dist/lib/html-to-markdown.test.d.ts +2 -0
  43. package/dist/lib/html-to-markdown.test.d.ts.map +1 -0
  44. package/dist/lib/html-to-markdown.test.js +54 -0
  45. package/dist/lib/html-to-markdown.test.js.map +1 -0
  46. package/dist/lib/prompt.d.ts +5 -0
  47. package/dist/lib/prompt.d.ts.map +1 -0
  48. package/dist/lib/prompt.js +47 -0
  49. package/dist/lib/prompt.js.map +1 -0
  50. package/dist/lib/settings.d.ts +9 -0
  51. package/dist/lib/settings.d.ts.map +1 -0
  52. package/dist/lib/settings.js +42 -0
  53. package/dist/lib/settings.js.map +1 -0
  54. package/dist/lib/storage.d.ts +11 -0
  55. package/dist/lib/storage.d.ts.map +1 -0
  56. package/dist/lib/storage.js +87 -0
  57. package/dist/lib/storage.js.map +1 -0
  58. package/dist/lib/tsconfig.d.ts +4 -0
  59. package/dist/lib/tsconfig.d.ts.map +1 -0
  60. package/dist/lib/tsconfig.js +49 -0
  61. package/dist/lib/tsconfig.js.map +1 -0
  62. package/dist/lib/url.d.ts +5 -0
  63. package/dist/lib/url.d.ts.map +1 -0
  64. package/dist/lib/url.js +73 -0
  65. package/dist/lib/url.js.map +1 -0
  66. package/dist/lib/url.test.d.ts +2 -0
  67. package/dist/lib/url.test.d.ts.map +1 -0
  68. package/dist/lib/url.test.js +41 -0
  69. package/dist/lib/url.test.js.map +1 -0
  70. package/dist/types.d.ts +28 -0
  71. package/dist/types.d.ts.map +1 -0
  72. package/dist/types.js +2 -0
  73. package/dist/types.js.map +1 -0
  74. package/package.json +58 -0
@@ -0,0 +1,49 @@
1
+ import { readFile, writeFile } from "fs/promises";
2
+ import { join } from "path";
3
+ import { existsSync } from "fs";
4
+ const MDRIP_DIR = "mdrip";
5
+ export function hasTsConfig(cwd = process.cwd()) {
6
+ return existsSync(join(cwd, "tsconfig.json"));
7
+ }
8
+ export async function hasMdripExclude(cwd = process.cwd()) {
9
+ const tsconfigPath = join(cwd, "tsconfig.json");
10
+ if (!existsSync(tsconfigPath)) {
11
+ return false;
12
+ }
13
+ try {
14
+ const content = await readFile(tsconfigPath, "utf-8");
15
+ const config = JSON.parse(content);
16
+ if (!config.exclude) {
17
+ return false;
18
+ }
19
+ return config.exclude.some((entry) => entry === MDRIP_DIR ||
20
+ entry === `${MDRIP_DIR}/` ||
21
+ entry === `./${MDRIP_DIR}`);
22
+ }
23
+ catch {
24
+ return false;
25
+ }
26
+ }
27
+ export async function ensureTsconfigExclude(cwd = process.cwd()) {
28
+ const tsconfigPath = join(cwd, "tsconfig.json");
29
+ if (!existsSync(tsconfigPath)) {
30
+ return false;
31
+ }
32
+ if (await hasMdripExclude(cwd)) {
33
+ return false;
34
+ }
35
+ try {
36
+ const content = await readFile(tsconfigPath, "utf-8");
37
+ const config = JSON.parse(content);
38
+ if (!config.exclude) {
39
+ config.exclude = [];
40
+ }
41
+ config.exclude.push(MDRIP_DIR);
42
+ await writeFile(tsconfigPath, JSON.stringify(config, null, 2) + "\n", "utf-8");
43
+ return true;
44
+ }
45
+ catch {
46
+ return false;
47
+ }
48
+ }
49
+ //# sourceMappingURL=tsconfig.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tsconfig.js","sourceRoot":"","sources":["../../src/lib/tsconfig.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAEhC,MAAM,SAAS,GAAG,OAAO,CAAC;AAO1B,MAAM,UAAU,WAAW,CAAC,MAAc,OAAO,CAAC,GAAG,EAAE;IACrD,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC;QAE/C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CACxB,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,KAAK,SAAS;YACnB,KAAK,KAAK,GAAG,SAAS,GAAG;YACzB,KAAK,KAAK,KAAK,SAAS,EAAE,CAC7B,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,MAAM,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC;QAE/C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC;QACtB,CAAC;QAED,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE/B,MAAM,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/E,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare function parseUrlSpec(spec: string): URL;
2
+ export declare function normalizeUrl(spec: string): string;
3
+ export declare function getPageRelativePath(spec: string | URL): string;
4
+ export declare function getUrlHost(spec: string): string;
5
+ //# sourceMappingURL=url.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../../src/lib/url.ts"],"names":[],"mappings":"AA8BA,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAuB9C;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,MAAM,CA0B9D;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C"}
@@ -0,0 +1,73 @@
1
+ import { createHash } from "crypto";
2
+ import { posix } from "path";
3
+ const URL_PROTOCOL_RE = /^[a-zA-Z][a-zA-Z\d+.-]*:\/\//;
4
+ const SAFE_SEGMENT_RE = /[^a-zA-Z0-9._-]/g;
5
+ function sanitizeSegment(segment) {
6
+ let decoded = segment;
7
+ try {
8
+ decoded = decodeURIComponent(segment);
9
+ }
10
+ catch {
11
+ // Keep original segment when decoding fails.
12
+ }
13
+ const normalized = decoded
14
+ .replace(SAFE_SEGMENT_RE, "-")
15
+ .replace(/-+/g, "-")
16
+ .replace(/^-|-$/g, "");
17
+ return normalized || "segment";
18
+ }
19
+ function normalizeHost(url) {
20
+ if (url.port) {
21
+ return `${url.hostname.toLowerCase()}--${url.port}`;
22
+ }
23
+ return url.hostname.toLowerCase();
24
+ }
25
+ export function parseUrlSpec(spec) {
26
+ const trimmed = spec.trim();
27
+ if (!trimmed) {
28
+ throw new Error("URL cannot be empty");
29
+ }
30
+ const withProtocol = URL_PROTOCOL_RE.test(trimmed)
31
+ ? trimmed
32
+ : `https://${trimmed}`;
33
+ let url;
34
+ try {
35
+ url = new URL(withProtocol);
36
+ }
37
+ catch {
38
+ throw new Error(`Invalid URL: ${spec}`);
39
+ }
40
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
41
+ throw new Error(`Unsupported URL protocol: ${url.protocol}`);
42
+ }
43
+ url.hash = "";
44
+ return url;
45
+ }
46
+ export function normalizeUrl(spec) {
47
+ return parseUrlSpec(spec).toString();
48
+ }
49
+ export function getPageRelativePath(spec) {
50
+ const url = typeof spec === "string" ? parseUrlSpec(spec) : spec;
51
+ const hostPart = sanitizeSegment(normalizeHost(url));
52
+ const rawSegments = url.pathname.split("/").filter(Boolean);
53
+ const segments = rawSegments.map(sanitizeSegment).filter(Boolean);
54
+ const pathnameEndsWithSlash = url.pathname.endsWith("/") || segments.length === 0;
55
+ let directorySegments = segments;
56
+ let filename = "index.md";
57
+ if (!pathnameEndsWithSlash) {
58
+ const lastSegment = segments[segments.length - 1] || "index";
59
+ const parsed = posix.parse(lastSegment);
60
+ const baseName = sanitizeSegment(parsed.name || parsed.base || "index");
61
+ filename = `${baseName}.md`;
62
+ directorySegments = segments.slice(0, -1);
63
+ }
64
+ if (url.search) {
65
+ const queryHash = createHash("sha1").update(url.search).digest("hex").slice(0, 8);
66
+ filename = filename.replace(/\.md$/, `--q-${queryHash}.md`);
67
+ }
68
+ return posix.join("pages", hostPart, ...directorySegments, filename);
69
+ }
70
+ export function getUrlHost(spec) {
71
+ return parseUrlSpec(spec).hostname.toLowerCase();
72
+ }
73
+ //# sourceMappingURL=url.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/lib/url.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE7B,MAAM,eAAe,GAAG,8BAA8B,CAAC;AACvD,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C,SAAS,eAAe,CAAC,OAAe;IACtC,IAAI,OAAO,GAAG,OAAO,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;IAC/C,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACvB,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC;SAC7B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEzB,OAAO,UAAU,IAAI,SAAS,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CAAC,GAAQ;IAC7B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;IACtD,CAAC;IAED,OAAO,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;QAChD,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,WAAW,OAAO,EAAE,CAAC;IAEzB,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IACd,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAkB;IACpD,MAAM,GAAG,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,MAAM,QAAQ,GAAG,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAElE,MAAM,qBAAqB,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IAElF,IAAI,iBAAiB,GAAG,QAAQ,CAAC;IACjC,IAAI,QAAQ,GAAG,UAAU,CAAC;IAE1B,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAC7D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC;QACxE,QAAQ,GAAG,GAAG,QAAQ,KAAK,CAAC;QAC5B,iBAAiB,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QACf,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,SAAS,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,iBAAiB,EAAE,QAAQ,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;AACnD,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=url.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url.test.d.ts","sourceRoot":"","sources":["../../src/lib/url.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,41 @@
1
+ import { describe, it, expect } from "vitest";
2
+ import { parseUrlSpec, normalizeUrl, getPageRelativePath, getUrlHost } from "./url.js";
3
+ describe("parseUrlSpec", () => {
4
+ it("adds https when protocol is omitted", () => {
5
+ const url = parseUrlSpec("developers.cloudflare.com/fundamentals/");
6
+ expect(url.toString()).toBe("https://developers.cloudflare.com/fundamentals/");
7
+ });
8
+ it("supports http and strips hash fragments", () => {
9
+ const url = parseUrlSpec("http://example.com/docs#intro");
10
+ expect(url.toString()).toBe("http://example.com/docs");
11
+ });
12
+ it("rejects unsupported protocols", () => {
13
+ expect(() => parseUrlSpec("ftp://example.com/file")).toThrow("Unsupported URL protocol");
14
+ });
15
+ });
16
+ describe("normalizeUrl", () => {
17
+ it("returns canonical URL", () => {
18
+ expect(normalizeUrl("example.com")).toBe("https://example.com/");
19
+ });
20
+ });
21
+ describe("getPageRelativePath", () => {
22
+ it("stores site root as index.md", () => {
23
+ expect(getPageRelativePath("https://example.com/")).toBe("pages/example.com/index.md");
24
+ });
25
+ it("stores trailing-slash paths under index.md", () => {
26
+ expect(getPageRelativePath("https://example.com/docs/guides/")).toBe("pages/example.com/docs/guides/index.md");
27
+ });
28
+ it("stores leaf page as <name>.md", () => {
29
+ expect(getPageRelativePath("https://example.com/docs/page")).toBe("pages/example.com/docs/page.md");
30
+ });
31
+ it("adds deterministic query hash suffix", () => {
32
+ const path = getPageRelativePath("https://example.com/docs/page?lang=en");
33
+ expect(path).toMatch(/^pages\/example\.com\/docs\/page--q-[a-f0-9]{8}\.md$/);
34
+ });
35
+ });
36
+ describe("getUrlHost", () => {
37
+ it("extracts lowercase hostname", () => {
38
+ expect(getUrlHost("https://Developers.Cloudflare.com/path")).toBe("developers.cloudflare.com");
39
+ });
40
+ });
41
+ //# sourceMappingURL=url.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url.test.js","sourceRoot":"","sources":["../../src/lib/url.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEvF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,GAAG,GAAG,YAAY,CAAC,yCAAyC,CAAC,CAAC;QACpE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAC1D,0BAA0B,CAC3B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CACtD,4BAA4B,CAC7B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,CAAC,mBAAmB,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAClE,wCAAwC,CACzC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,CAAC,mBAAmB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAC/D,gCAAgC,CACjC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,IAAI,GAAG,mBAAmB,CAAC,uCAAuC,CAAC,CAAC;QAC1E,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,CAAC,UAAU,CAAC,wCAAwC,CAAC,CAAC,CAAC,IAAI,CAC/D,2BAA2B,CAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,28 @@
1
+ export interface FetchResult {
2
+ url: string;
3
+ resolvedUrl?: string;
4
+ path: string;
5
+ success: boolean;
6
+ error?: string;
7
+ status?: number;
8
+ contentType?: string;
9
+ markdownTokens?: number;
10
+ contentSignal?: string;
11
+ source?: "cloudflare-markdown" | "html-fallback";
12
+ }
13
+ export interface PageEntry {
14
+ url: string;
15
+ resolvedUrl?: string;
16
+ path: string;
17
+ fetchedAt: string;
18
+ status: number;
19
+ contentType: string;
20
+ markdownTokens?: number;
21
+ contentSignal?: string;
22
+ source?: "cloudflare-markdown" | "html-fallback";
23
+ }
24
+ export interface SourcesIndex {
25
+ pages?: PageEntry[];
26
+ updatedAt: string;
27
+ }
28
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,qBAAqB,GAAG,eAAe,CAAC;CAClD;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,qBAAqB,GAAG,eAAe,CAAC;CAClD;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB"}
package/dist/types.js ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
package/package.json ADDED
@@ -0,0 +1,58 @@
1
+ {
2
+ "name": "mdrip",
3
+ "version": "0.1.0",
4
+ "description": "Fetch markdown snapshots of web pages using Cloudflare Markdown for Agents",
5
+ "type": "module",
6
+ "bin": {
7
+ "mdrip": "./dist/index.js"
8
+ },
9
+ "files": [
10
+ "dist",
11
+ "README.md"
12
+ ],
13
+ "repository": {
14
+ "type": "git",
15
+ "url": "https://github.com/charl-kruger/mdrip"
16
+ },
17
+ "homepage": "https://github.com/charl-kruger/mdrip#readme",
18
+ "bugs": {
19
+ "url": "https://github.com/charl-kruger/mdrip/issues"
20
+ },
21
+ "scripts": {
22
+ "build": "tsc",
23
+ "dev": "tsc --watch",
24
+ "start": "node dist/index.js",
25
+ "test": "vitest run",
26
+ "test:watch": "vitest",
27
+ "test:coverage": "vitest run --coverage",
28
+ "format": "prettier --write \"**/*.{ts,tsx}\"",
29
+ "format:check": "prettier --check \"**/*.{ts,tsx}\"",
30
+ "type-check": "tsc --noEmit",
31
+ "prepublishOnly": "pnpm type-check && pnpm test && pnpm build",
32
+ "publish:dry-run": "npm publish --dry-run",
33
+ "publish:npm": "npm publish --access=public"
34
+ },
35
+ "keywords": [
36
+ "cli",
37
+ "markdown",
38
+ "cloudflare",
39
+ "agents",
40
+ "context"
41
+ ],
42
+ "license": "Apache-2.0",
43
+ "author": "Charl Kruger",
44
+ "dependencies": {
45
+ "commander": "^12.1.0",
46
+ "domhandler": "^5.0.3",
47
+ "htmlparser2": "^10.0.0"
48
+ },
49
+ "devDependencies": {
50
+ "@types/node": "^22.10.0",
51
+ "prettier": "^3.4.0",
52
+ "typescript": "^5.7.0",
53
+ "vitest": "^4.0.16"
54
+ },
55
+ "engines": {
56
+ "node": ">=18.0.0"
57
+ }
58
+ }