@mdream/crawl 1.7.1 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +18 -13
- package/dist/cli.d.mts +1 -1
- package/dist/cli.mjs +2 -1
- package/dist/index.d.mts +7 -7
- package/package.json +7 -7
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -184,7 +184,10 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
|
184
184
|
if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern, allowSubdomains);
|
|
185
185
|
return url === pattern;
|
|
186
186
|
}
|
|
187
|
-
if (pattern.startsWith("/"))
|
|
187
|
+
if (pattern.startsWith("/")) {
|
|
188
|
+
const adjustedPattern = pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern;
|
|
189
|
+
return picomatch(adjustedPattern)(urlPath);
|
|
190
|
+
}
|
|
188
191
|
return picomatch(pattern)(urlPath) || picomatch(pattern)(urlPath.substring(1));
|
|
189
192
|
});
|
|
190
193
|
} catch {
|
|
@@ -651,7 +654,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
651
654
|
const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
|
|
652
655
|
const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
|
|
653
656
|
const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
|
|
654
|
-
const
|
|
657
|
+
const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
|
|
658
|
+
const safeFilename = normalize(`${filename}.md`);
|
|
655
659
|
filePath = join(outputDir, safeFilename);
|
|
656
660
|
}
|
|
657
661
|
const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
|
|
@@ -885,18 +889,19 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
885
889
|
return result.content.trim().replace(FRONTMATTER_BLOCK_RE, "").trim().length > 10;
|
|
886
890
|
});
|
|
887
891
|
const seenUrls = /* @__PURE__ */ new Set();
|
|
892
|
+
const processedFiles = contentResults.filter((result) => {
|
|
893
|
+
if (seenUrls.has(result.url)) return false;
|
|
894
|
+
seenUrls.add(result.url);
|
|
895
|
+
return true;
|
|
896
|
+
}).map((result) => ({
|
|
897
|
+
filePath: result.filePath,
|
|
898
|
+
title: result.title,
|
|
899
|
+
content: result.content,
|
|
900
|
+
url: result.url,
|
|
901
|
+
metadata: result.metadata
|
|
902
|
+
}));
|
|
888
903
|
const llmsResult = await generateLlmsTxtArtifacts({
|
|
889
|
-
files:
|
|
890
|
-
if (seenUrls.has(result.url)) return false;
|
|
891
|
-
seenUrls.add(result.url);
|
|
892
|
-
return true;
|
|
893
|
-
}).map((result) => ({
|
|
894
|
-
filePath: result.filePath,
|
|
895
|
-
title: result.title,
|
|
896
|
-
content: result.content,
|
|
897
|
-
url: result.url,
|
|
898
|
-
metadata: result.metadata
|
|
899
|
-
})),
|
|
904
|
+
files: processedFiles,
|
|
900
905
|
siteName,
|
|
901
906
|
description,
|
|
902
907
|
origin: originUrl || firstUrl.origin,
|
package/dist/cli.d.mts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export {}
|
package/dist/cli.mjs
CHANGED
|
@@ -21,7 +21,8 @@ function isSilentArgv(args) {
|
|
|
21
21
|
return args.some((arg) => QUIET_FLAGS.has(arg));
|
|
22
22
|
}
|
|
23
23
|
let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
|
|
24
|
-
const
|
|
24
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
25
|
+
const packageJsonPath = join(__dirname, "..", "package.json");
|
|
25
26
|
const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
|
|
26
27
|
function checkOutputDirectoryPermissions(outputDir) {
|
|
27
28
|
try {
|
package/dist/index.d.mts
CHANGED
|
@@ -28,7 +28,7 @@ interface BoilerplateOptions {
|
|
|
28
28
|
* Returns a new array aligned with `contents`. Documents that are unchanged (no
|
|
29
29
|
* detected chrome, or detection skipped) are returned byte-for-byte as given.
|
|
30
30
|
*/
|
|
31
|
-
declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
|
|
31
|
+
export declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
|
|
32
32
|
/**
|
|
33
33
|
* Minimal spinner surface used by the crawler. Mirrors the subset of
|
|
34
34
|
* `@clack/prompts` spinner methods the package relies on.
|
|
@@ -63,18 +63,18 @@ interface CrawlLogger {
|
|
|
63
63
|
* A logger that drops every message. Used when `silent` is set so a library
|
|
64
64
|
* consumer (e.g. an MCP server) gets a clean stdout.
|
|
65
65
|
*/
|
|
66
|
-
declare const silentLogger: CrawlLogger;
|
|
66
|
+
export declare const silentLogger: CrawlLogger;
|
|
67
67
|
/**
|
|
68
68
|
* The default logger, backed by `@clack/prompts` (writes to stdout). Preserves
|
|
69
69
|
* the existing CLI output exactly.
|
|
70
70
|
*/
|
|
71
|
-
declare function createClackLogger(): CrawlLogger;
|
|
71
|
+
export declare function createClackLogger(): CrawlLogger;
|
|
72
72
|
/**
|
|
73
73
|
* Resolve the effective logger from crawl options. An explicit `logger` wins;
|
|
74
74
|
* otherwise `silent` selects the no-op sink, and the default is the clack
|
|
75
75
|
* logger. Pure: same inputs, same logger.
|
|
76
76
|
*/
|
|
77
|
-
declare function resolveLogger(options: {
|
|
77
|
+
export declare function resolveLogger(options: {
|
|
78
78
|
silent?: boolean;
|
|
79
79
|
logger?: CrawlLogger;
|
|
80
80
|
}): CrawlLogger;
|
|
@@ -182,7 +182,7 @@ interface MdreamCrawlConfig {
|
|
|
182
182
|
artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
|
|
183
183
|
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
|
|
184
184
|
}
|
|
185
|
-
declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
|
|
185
|
+
export declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
|
|
186
186
|
interface ParsedUrlPattern {
|
|
187
187
|
baseUrl: string;
|
|
188
188
|
pattern: string;
|
|
@@ -231,5 +231,5 @@ interface CrawlProgress {
|
|
|
231
231
|
current?: string;
|
|
232
232
|
};
|
|
233
233
|
}
|
|
234
|
-
declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
235
|
-
export {
|
|
234
|
+
export declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
235
|
+
export type { BoilerplateOptions, CrawlHooks, CrawlLogger, CrawlOptions, CrawlResult, CrawlSpinner, MdreamCrawlConfig, PageData };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.7.
|
|
4
|
+
"version": "1.7.2",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -55,16 +55,16 @@
|
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"@clack/prompts": "^1.7.0",
|
|
58
|
-
"
|
|
58
|
+
"@mdream/js": "1.7.2",
|
|
59
|
+
"c12": "4.0.0-rc.1",
|
|
59
60
|
"hookable": "^6.1.1",
|
|
61
|
+
"mdream": "1.7.2",
|
|
60
62
|
"nypm": "^0.6.9",
|
|
61
63
|
"ofetch": "^1.5.1",
|
|
62
64
|
"pathe": "^2.0.3",
|
|
63
|
-
"picomatch": "^4.0.
|
|
64
|
-
"tldts": "^7.4.
|
|
65
|
-
"ufo": "^1.6.4"
|
|
66
|
-
"@mdream/js": "1.7.1",
|
|
67
|
-
"mdream": "1.7.1"
|
|
65
|
+
"picomatch": "^4.0.7",
|
|
66
|
+
"tldts": "^7.4.11",
|
|
67
|
+
"ufo": "^1.6.4"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/picomatch": "^4.0.3"
|