@mdream/crawl 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +21 -14
- package/dist/cli.d.mts +1 -1
- package/dist/cli.mjs +2 -1
- package/dist/index.d.mts +7 -7
- package/package.json +7 -7
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -184,7 +184,10 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
|
184
184
|
if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern, allowSubdomains);
|
|
185
185
|
return url === pattern;
|
|
186
186
|
}
|
|
187
|
-
if (pattern.startsWith("/"))
|
|
187
|
+
if (pattern.startsWith("/")) {
|
|
188
|
+
const adjustedPattern = pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern;
|
|
189
|
+
return picomatch(adjustedPattern)(urlPath);
|
|
190
|
+
}
|
|
188
191
|
return picomatch(pattern)(urlPath) || picomatch(pattern)(urlPath.substring(1));
|
|
189
192
|
});
|
|
190
193
|
} catch {
|
|
@@ -602,6 +605,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
602
605
|
const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
|
|
603
606
|
const createdDirs = /* @__PURE__ */ new Set();
|
|
604
607
|
const sharedOrigin = origin || "";
|
|
608
|
+
const sharedOriginBase = sharedOrigin.replace(URL_TRAILING_SLASH_RE, "");
|
|
605
609
|
const pendingUrls = [];
|
|
606
610
|
const queryVariantCounts = /* @__PURE__ */ new Map();
|
|
607
611
|
const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
|
|
@@ -624,9 +628,10 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
624
628
|
};
|
|
625
629
|
await hooks.callHook("crawl:html", htmlCtx);
|
|
626
630
|
content = htmlCtx.html;
|
|
631
|
+
const pageBaseUrl = sharedOriginBase ? `${sharedOriginBase}${parsedUrl.pathname}` : url;
|
|
627
632
|
const { extraction, getMetadata } = extractMetadataInline(parsedUrl, allowedRegistrableDomains);
|
|
628
633
|
md = htmlToMarkdown(content, {
|
|
629
|
-
origin:
|
|
634
|
+
origin: pageBaseUrl,
|
|
630
635
|
extraction
|
|
631
636
|
});
|
|
632
637
|
metadata = getMetadata();
|
|
@@ -649,7 +654,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
649
654
|
const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
|
|
650
655
|
const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
|
|
651
656
|
const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
|
|
652
|
-
const
|
|
657
|
+
const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
|
|
658
|
+
const safeFilename = normalize(`${filename}.md`);
|
|
653
659
|
filePath = join(outputDir, safeFilename);
|
|
654
660
|
}
|
|
655
661
|
const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
|
|
@@ -883,18 +889,19 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
883
889
|
return result.content.trim().replace(FRONTMATTER_BLOCK_RE, "").trim().length > 10;
|
|
884
890
|
});
|
|
885
891
|
const seenUrls = /* @__PURE__ */ new Set();
|
|
892
|
+
const processedFiles = contentResults.filter((result) => {
|
|
893
|
+
if (seenUrls.has(result.url)) return false;
|
|
894
|
+
seenUrls.add(result.url);
|
|
895
|
+
return true;
|
|
896
|
+
}).map((result) => ({
|
|
897
|
+
filePath: result.filePath,
|
|
898
|
+
title: result.title,
|
|
899
|
+
content: result.content,
|
|
900
|
+
url: result.url,
|
|
901
|
+
metadata: result.metadata
|
|
902
|
+
}));
|
|
886
903
|
const llmsResult = await generateLlmsTxtArtifacts({
|
|
887
|
-
files:
|
|
888
|
-
if (seenUrls.has(result.url)) return false;
|
|
889
|
-
seenUrls.add(result.url);
|
|
890
|
-
return true;
|
|
891
|
-
}).map((result) => ({
|
|
892
|
-
filePath: result.filePath,
|
|
893
|
-
title: result.title,
|
|
894
|
-
content: result.content,
|
|
895
|
-
url: result.url,
|
|
896
|
-
metadata: result.metadata
|
|
897
|
-
})),
|
|
904
|
+
files: processedFiles,
|
|
898
905
|
siteName,
|
|
899
906
|
description,
|
|
900
907
|
origin: originUrl || firstUrl.origin,
|
package/dist/cli.d.mts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export {}
|
package/dist/cli.mjs
CHANGED
|
@@ -21,7 +21,8 @@ function isSilentArgv(args) {
|
|
|
21
21
|
return args.some((arg) => QUIET_FLAGS.has(arg));
|
|
22
22
|
}
|
|
23
23
|
let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
|
|
24
|
-
const
|
|
24
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
25
|
+
const packageJsonPath = join(__dirname, "..", "package.json");
|
|
25
26
|
const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
|
|
26
27
|
function checkOutputDirectoryPermissions(outputDir) {
|
|
27
28
|
try {
|
package/dist/index.d.mts
CHANGED
|
@@ -28,7 +28,7 @@ interface BoilerplateOptions {
|
|
|
28
28
|
* Returns a new array aligned with `contents`. Documents that are unchanged (no
|
|
29
29
|
* detected chrome, or detection skipped) are returned byte-for-byte as given.
|
|
30
30
|
*/
|
|
31
|
-
declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
|
|
31
|
+
export declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
|
|
32
32
|
/**
|
|
33
33
|
* Minimal spinner surface used by the crawler. Mirrors the subset of
|
|
34
34
|
* `@clack/prompts` spinner methods the package relies on.
|
|
@@ -63,18 +63,18 @@ interface CrawlLogger {
|
|
|
63
63
|
* A logger that drops every message. Used when `silent` is set so a library
|
|
64
64
|
* consumer (e.g. an MCP server) gets a clean stdout.
|
|
65
65
|
*/
|
|
66
|
-
declare const silentLogger: CrawlLogger;
|
|
66
|
+
export declare const silentLogger: CrawlLogger;
|
|
67
67
|
/**
|
|
68
68
|
* The default logger, backed by `@clack/prompts` (writes to stdout). Preserves
|
|
69
69
|
* the existing CLI output exactly.
|
|
70
70
|
*/
|
|
71
|
-
declare function createClackLogger(): CrawlLogger;
|
|
71
|
+
export declare function createClackLogger(): CrawlLogger;
|
|
72
72
|
/**
|
|
73
73
|
* Resolve the effective logger from crawl options. An explicit `logger` wins;
|
|
74
74
|
* otherwise `silent` selects the no-op sink, and the default is the clack
|
|
75
75
|
* logger. Pure: same inputs, same logger.
|
|
76
76
|
*/
|
|
77
|
-
declare function resolveLogger(options: {
|
|
77
|
+
export declare function resolveLogger(options: {
|
|
78
78
|
silent?: boolean;
|
|
79
79
|
logger?: CrawlLogger;
|
|
80
80
|
}): CrawlLogger;
|
|
@@ -182,7 +182,7 @@ interface MdreamCrawlConfig {
|
|
|
182
182
|
artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
|
|
183
183
|
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
|
|
184
184
|
}
|
|
185
|
-
declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
|
|
185
|
+
export declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
|
|
186
186
|
interface ParsedUrlPattern {
|
|
187
187
|
baseUrl: string;
|
|
188
188
|
pattern: string;
|
|
@@ -231,5 +231,5 @@ interface CrawlProgress {
|
|
|
231
231
|
current?: string;
|
|
232
232
|
};
|
|
233
233
|
}
|
|
234
|
-
declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
235
|
-
export {
|
|
234
|
+
export declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
235
|
+
export type { BoilerplateOptions, CrawlHooks, CrawlLogger, CrawlOptions, CrawlResult, CrawlSpinner, MdreamCrawlConfig, PageData };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.7.
|
|
4
|
+
"version": "1.7.2",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -55,16 +55,16 @@
|
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"@clack/prompts": "^1.7.0",
|
|
58
|
-
"
|
|
58
|
+
"@mdream/js": "1.7.2",
|
|
59
|
+
"c12": "4.0.0-rc.1",
|
|
59
60
|
"hookable": "^6.1.1",
|
|
61
|
+
"mdream": "1.7.2",
|
|
60
62
|
"nypm": "^0.6.9",
|
|
61
63
|
"ofetch": "^1.5.1",
|
|
62
64
|
"pathe": "^2.0.3",
|
|
63
|
-
"picomatch": "^4.0.
|
|
64
|
-
"tldts": "^7.4.
|
|
65
|
-
"ufo": "^1.6.4"
|
|
66
|
-
"@mdream/js": "1.7.0",
|
|
67
|
-
"mdream": "1.7.0"
|
|
65
|
+
"picomatch": "^4.0.7",
|
|
66
|
+
"tldts": "^7.4.11",
|
|
67
|
+
"ufo": "^1.6.4"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/picomatch": "^4.0.3"
|