@mdream/crawl 0.12.2 → 0.13.0

package/dist/_chunks/{crawl-bReaLQf6.mjs → crawl-DEZX9kH_.mjs}

@@ -216,7 +216,7 @@ async function loadSitemapWithoutRetries(sitemapUrl) {
 	}
 }
 async function crawlAndGenerate(options, onProgress) {
-	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false } = options;
+	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, onPage } = options;
 	const outputDir = resolve(normalize(rawOutputDir));
 	if (verbose) log.setLevel(log.LEVELS.INFO);
 	else log.setLevel(log.LEVELS.OFF);
@@ -431,8 +431,19 @@ async function crawlAndGenerate(options, onProgress) {
 			const metadata = extractMetadata(html, request.loadedUrl);
 			if (!title) title = metadata.title;
 			const shouldProcessMarkdown = shouldCrawlUrl(request.loadedUrl);
+			const pageOrigin = origin || new URL(request.loadedUrl).origin;
+			if (onPage && shouldProcessMarkdown) {
+				const pageData = {
+					url: request.loadedUrl,
+					html,
+					title,
+					metadata,
+					origin: pageOrigin
+				};
+				await onPage(pageData);
+			}
 			let md = "";
-			if (shouldProcessMarkdown) md = htmlToMarkdown(html, withMinimalPreset({ origin: origin || new URL(request.loadedUrl).origin }));
+			if (shouldProcessMarkdown && (!onPage || generateIndividualMd)) md = htmlToMarkdown(html, withMinimalPreset({ origin: pageOrigin }));
 			let filePath;
 			if (shouldProcessMarkdown) {
 				const urlObj = new URL(request.loadedUrl);

package/dist/cli.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-bReaLQf6.mjs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-DEZX9kH_.mjs";
 import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import * as p$1 from "@clack/prompts";
 import * as p from "@clack/prompts";

package/dist/index.d.mts

@@ -1,4 +1,11 @@
 //#region src/types.d.ts
+interface PageData {
+  url: string;
+  html: string;
+  title: string;
+  metadata: PageMetadata;
+  origin: string;
+}
 interface CrawlOptions {
   urls: string[];
   outputDir: string;
@@ -19,6 +26,7 @@ interface CrawlOptions {
   descriptionOverride?: string;
   verbose?: boolean;
   skipSitemap?: boolean;
+  onPage?: (page: PageData) => Promise<void> | void;
 }
 interface ParsedUrlPattern {
   baseUrl: string;
@@ -74,4 +82,4 @@ declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress:
 declare function generateLlmsTxt(options: LlmsTxtOptions): Promise<void>;
 declare function generateLlmsFullTxt(options: LlmsTxtOptions): Promise<void>;
 //#endregion
-export { type CrawlOptions, type CrawlResult, type LlmsTxtOptions, crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };
+export { type CrawlOptions, type CrawlResult, type LlmsTxtOptions, type PageData, crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-bReaLQf6.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-DEZX9kH_.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.12.2",
+  "version": "0.13.0",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -56,7 +56,7 @@
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
     "ufo": "^1.6.1",
-    "mdream": "0.12.2"
+    "mdream": "0.13.0"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.2"