@mdream/crawl 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,7 +2,8 @@ import { existsSync, mkdirSync } from "node:fs";
|
|
|
2
2
|
import { writeFile } from "node:fs/promises";
|
|
3
3
|
import * as p from "@clack/prompts";
|
|
4
4
|
import { HttpCrawler, PlaywrightCrawler, log, purgeDefaultStorages } from "crawlee";
|
|
5
|
-
import {
|
|
5
|
+
import { htmlToMarkdown } from "mdream";
|
|
6
|
+
import { generateLlmsTxtArtifacts } from "mdream/llms-txt";
|
|
6
7
|
import { withMinimalPreset } from "mdream/preset/minimal";
|
|
7
8
|
import { dirname, join, normalize, resolve } from "pathe";
|
|
8
9
|
import { withHttps } from "ufo";
|
package/dist/cli.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-
|
|
1
|
+
import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-bReaLQf6.mjs";
|
|
2
2
|
import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
3
|
import * as p$1 from "@clack/prompts";
|
|
4
4
|
import * as p from "@clack/prompts";
|
package/dist/index.d.mts
CHANGED
|
@@ -74,4 +74,4 @@ declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress:
|
|
|
74
74
|
declare function generateLlmsTxt(options: LlmsTxtOptions): Promise<void>;
|
|
75
75
|
declare function generateLlmsFullTxt(options: LlmsTxtOptions): Promise<void>;
|
|
76
76
|
//#endregion
|
|
77
|
-
export { CrawlOptions, CrawlResult, LlmsTxtOptions, crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };
|
|
77
|
+
export { type CrawlOptions, type CrawlResult, type LlmsTxtOptions, crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };
|
package/dist/index.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.11.0",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -9,6 +9,11 @@
|
|
|
9
9
|
"url": "https://harlanzw.com/"
|
|
10
10
|
},
|
|
11
11
|
"license": "MIT",
|
|
12
|
+
"repository": {
|
|
13
|
+
"type": "git",
|
|
14
|
+
"url": "https://github.com/harlan-zw/mdream",
|
|
15
|
+
"directory": "packages/crawl"
|
|
16
|
+
},
|
|
12
17
|
"exports": {
|
|
13
18
|
".": {
|
|
14
19
|
"types": "./dist/index.d.mts",
|
|
@@ -46,12 +51,12 @@
|
|
|
46
51
|
},
|
|
47
52
|
"dependencies": {
|
|
48
53
|
"@clack/prompts": "^0.11.0",
|
|
49
|
-
"crawlee": "^3.
|
|
50
|
-
"nypm": "^0.6.
|
|
54
|
+
"crawlee": "^3.15.1",
|
|
55
|
+
"nypm": "^0.6.2",
|
|
51
56
|
"pathe": "^2.0.3",
|
|
52
57
|
"picomatch": "^4.0.3",
|
|
53
58
|
"ufo": "^1.6.1",
|
|
54
|
-
"mdream": "0.
|
|
59
|
+
"mdream": "0.11.0"
|
|
55
60
|
},
|
|
56
61
|
"devDependencies": {
|
|
57
62
|
"@types/picomatch": "^4.0.2"
|