@mdream/crawl 1.7.0 → 1.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -184,7 +184,10 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
184
184
  if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern, allowSubdomains);
185
185
  return url === pattern;
186
186
  }
187
- if (pattern.startsWith("/")) return picomatch(pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern)(urlPath);
187
+ if (pattern.startsWith("/")) {
188
+ const adjustedPattern = pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern;
189
+ return picomatch(adjustedPattern)(urlPath);
190
+ }
188
191
  return picomatch(pattern)(urlPath) || picomatch(pattern)(urlPath.substring(1));
189
192
  });
190
193
  } catch {
@@ -602,6 +605,7 @@ async function crawlAndGenerate(options, onProgress) {
602
605
  const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
603
606
  const createdDirs = /* @__PURE__ */ new Set();
604
607
  const sharedOrigin = origin || "";
608
+ const sharedOriginBase = sharedOrigin.replace(URL_TRAILING_SLASH_RE, "");
605
609
  const pendingUrls = [];
606
610
  const queryVariantCounts = /* @__PURE__ */ new Map();
607
611
  const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
@@ -624,9 +628,10 @@ async function crawlAndGenerate(options, onProgress) {
624
628
  };
625
629
  await hooks.callHook("crawl:html", htmlCtx);
626
630
  content = htmlCtx.html;
631
+ const pageBaseUrl = sharedOriginBase ? `${sharedOriginBase}${parsedUrl.pathname}` : url;
627
632
  const { extraction, getMetadata } = extractMetadataInline(parsedUrl, allowedRegistrableDomains);
628
633
  md = htmlToMarkdown(content, {
629
- origin: pageOrigin,
634
+ origin: pageBaseUrl,
630
635
  extraction
631
636
  });
632
637
  metadata = getMetadata();
@@ -649,7 +654,8 @@ async function crawlAndGenerate(options, onProgress) {
649
654
  const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
650
655
  const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
651
656
  const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
652
- const safeFilename = normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`);
657
+ const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
658
+ const safeFilename = normalize(`${filename}.md`);
653
659
  filePath = join(outputDir, safeFilename);
654
660
  }
655
661
  const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
@@ -883,18 +889,19 @@ async function crawlAndGenerate(options, onProgress) {
883
889
  return result.content.trim().replace(FRONTMATTER_BLOCK_RE, "").trim().length > 10;
884
890
  });
885
891
  const seenUrls = /* @__PURE__ */ new Set();
892
+ const processedFiles = contentResults.filter((result) => {
893
+ if (seenUrls.has(result.url)) return false;
894
+ seenUrls.add(result.url);
895
+ return true;
896
+ }).map((result) => ({
897
+ filePath: result.filePath,
898
+ title: result.title,
899
+ content: result.content,
900
+ url: result.url,
901
+ metadata: result.metadata
902
+ }));
886
903
  const llmsResult = await generateLlmsTxtArtifacts({
887
- files: contentResults.filter((result) => {
888
- if (seenUrls.has(result.url)) return false;
889
- seenUrls.add(result.url);
890
- return true;
891
- }).map((result) => ({
892
- filePath: result.filePath,
893
- title: result.title,
894
- content: result.content,
895
- url: result.url,
896
- metadata: result.metadata
897
- })),
904
+ files: processedFiles,
898
905
  siteName,
899
906
  description,
900
907
  origin: originUrl || firstUrl.origin,
package/dist/cli.d.mts CHANGED
@@ -1 +1 @@
1
- export { };
1
+ export {}
package/dist/cli.mjs CHANGED
@@ -21,7 +21,8 @@ function isSilentArgv(args) {
21
21
  return args.some((arg) => QUIET_FLAGS.has(arg));
22
22
  }
23
23
  let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
24
- const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
24
+ const __dirname = dirname(fileURLToPath(import.meta.url));
25
+ const packageJsonPath = join(__dirname, "..", "package.json");
25
26
  const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
26
27
  function checkOutputDirectoryPermissions(outputDir) {
27
28
  try {
package/dist/index.d.mts CHANGED
@@ -28,7 +28,7 @@ interface BoilerplateOptions {
28
28
  * Returns a new array aligned with `contents`. Documents that are unchanged (no
29
29
  * detected chrome, or detection skipped) are returned byte-for-byte as given.
30
30
  */
31
- declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
31
+ export declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
32
32
  /**
33
33
  * Minimal spinner surface used by the crawler. Mirrors the subset of
34
34
  * `@clack/prompts` spinner methods the package relies on.
@@ -63,18 +63,18 @@ interface CrawlLogger {
63
63
  * A logger that drops every message. Used when `silent` is set so a library
64
64
  * consumer (e.g. an MCP server) gets a clean stdout.
65
65
  */
66
- declare const silentLogger: CrawlLogger;
66
+ export declare const silentLogger: CrawlLogger;
67
67
  /**
68
68
  * The default logger, backed by `@clack/prompts` (writes to stdout). Preserves
69
69
  * the existing CLI output exactly.
70
70
  */
71
- declare function createClackLogger(): CrawlLogger;
71
+ export declare function createClackLogger(): CrawlLogger;
72
72
  /**
73
73
  * Resolve the effective logger from crawl options. An explicit `logger` wins;
74
74
  * otherwise `silent` selects the no-op sink, and the default is the clack
75
75
  * logger. Pure: same inputs, same logger.
76
76
  */
77
- declare function resolveLogger(options: {
77
+ export declare function resolveLogger(options: {
78
78
  silent?: boolean;
79
79
  logger?: CrawlLogger;
80
80
  }): CrawlLogger;
@@ -182,7 +182,7 @@ interface MdreamCrawlConfig {
182
182
  artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
183
183
  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
184
184
  }
185
- declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
185
+ export declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
186
186
  interface ParsedUrlPattern {
187
187
  baseUrl: string;
188
188
  pattern: string;
@@ -231,5 +231,5 @@ interface CrawlProgress {
231
231
  current?: string;
232
232
  };
233
233
  }
234
- declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
235
- export { type BoilerplateOptions, type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger, stripBoilerplateFromCorpus };
234
+ export declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
235
+ export type { BoilerplateOptions, CrawlHooks, CrawlLogger, CrawlOptions, CrawlResult, CrawlSpinner, MdreamCrawlConfig, PageData };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "1.7.0",
4
+ "version": "1.7.2",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -55,16 +55,16 @@
55
55
  },
56
56
  "dependencies": {
57
57
  "@clack/prompts": "^1.7.0",
58
- "c12": "^3.3.4",
58
+ "@mdream/js": "1.7.2",
59
+ "c12": "4.0.0-rc.1",
59
60
  "hookable": "^6.1.1",
61
+ "mdream": "1.7.2",
60
62
  "nypm": "^0.6.9",
61
63
  "ofetch": "^1.5.1",
62
64
  "pathe": "^2.0.3",
63
- "picomatch": "^4.0.5",
64
- "tldts": "^7.4.10",
65
- "ufo": "^1.6.4",
66
- "@mdream/js": "1.7.0",
67
- "mdream": "1.7.0"
65
+ "picomatch": "^4.0.7",
66
+ "tldts": "^7.4.11",
67
+ "ufo": "^1.6.4"
68
68
  },
69
69
  "devDependencies": {
70
70
  "@types/picomatch": "^4.0.3"