@lenne.tech/cli 1.12.0 → 1.13.0

package/build/commands/config/validate.js

@@ -114,6 +114,22 @@ const KNOWN_KEYS = {
                 path: 'string',
             },
         },
+        tools: {
+            crawl: {
+                concurrency: 'number',
+                depth: 'number|all',
+                includeImages: 'boolean',
+                includeSitemap: 'boolean',
+                maxPages: 'number',
+                noConfirm: 'boolean',
+                out: 'string',
+                prune: 'boolean',
+                renderJs: 'boolean',
+                selector: 'string',
+                timeout: 'number',
+            },
+            noConfirm: 'boolean',
+        },
         typescript: {
             create: { author: 'string', noConfirm: 'boolean', updatePackages: 'boolean' },
         },
@@ -165,8 +181,26 @@ function validateConfig(config, knownKeys, path = '') {
         }
         // Validate type
         if (typeof expectedType === 'string') {
-            // Simple type check
-            if (expectedType === 'string' && typeof value !== 'string') {
+            // Simple type check. `'a|b'` means union (e.g. "number|all").
+            if (expectedType.includes('|')) {
+                const tokens = expectedType.split('|').map((t) => t.trim());
+                const ok = tokens.some((token) => {
+                    if (token === 'string')
+                        return typeof value === 'string';
+                    if (token === 'number')
+                        return typeof value === 'number';
+                    if (token === 'boolean')
+                        return typeof value === 'boolean';
+                    if (token === 'array')
+                        return Array.isArray(value);
+                    // Everything else is treated as a string literal enum member.
+                    return value === token;
+                });
+                if (!ok) {
+                    result.errors.push(`${currentPath}: expected ${tokens.join(' | ')}, got ${typeof value}`);
+                }
+            }
+            else if (expectedType === 'string' && typeof value !== 'string') {
                 result.errors.push(`${currentPath}: expected string, got ${typeof value}`);
             }
             else if (expectedType === 'boolean' && typeof value !== 'boolean') {

package/build/commands/tools/crawl.js

@@ -0,0 +1,307 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = require("path");
+const crawler_1 = require("../../lib/crawler");
+/**
+ * Crawl a website (optionally following same-origin links up to a
+ * configurable depth) and store the content as Markdown files for use
+ * as a Claude Code knowledge base. Inspired by ../../../../chrome-md:
+ * shares the defuddle + Turndown extraction pipeline but runs headless
+ * from Node and follows links / sitemaps automatically.
+ */
+const NewCommand = {
+    alias: ['cr'],
+    description: 'Crawl site to Markdown',
+    hidden: false,
+    name: 'crawl',
+    run: (toolbox) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, _b, _c, _d, _e;
+        const { config, filesystem, helper, parameters, print: { error, info, spin, success, warning }, prompt: { confirm }, tools, } = toolbox;
+        if (tools.helpJson({
+            aliases: ['cr'],
+            description: 'Crawl a website into Markdown files (for Claude Code knowledge bases)',
+            name: 'crawl',
+            options: [
+                {
+                    description: 'Start URL (absolute http/https URL)',
+                    flag: '--url',
+                    required: true,
+                    type: 'string',
+                },
+                {
+                    default: '.',
+                    description: 'Output directory (created if missing)',
+                    flag: '--out',
+                    type: 'string',
+                },
+                {
+                    default: 0,
+                    description: 'Link depth. 0 = only start page; 1 = + direct links; N = up to N hops; "all" (or -1) = follow every same-origin link until --max-pages is reached',
+                    flag: '--depth',
+                    type: 'number|all',
+                },
+                {
+                    default: true,
+                    description: 'Download images and inline them with local paths',
+                    flag: '--images',
+                    type: 'boolean',
+                },
+                {
+                    default: true,
+                    description: 'Also seed queue from <origin>/sitemap.xml',
+                    flag: '--sitemap',
+                    type: 'boolean',
+                },
+                {
+                    default: 4,
+                    description: 'Parallel HTTP requests',
+                    flag: '--concurrency',
+                    type: 'number',
+                },
+                {
+                    default: 200,
+                    description: 'Maximum number of pages to crawl (safety cap)',
+                    flag: '--max-pages',
+                    type: 'number',
+                },
+                {
+                    description: 'CSS selector for the main content container',
+                    flag: '--selector',
+                    type: 'string',
+                },
+                {
+                    default: 20000,
+                    description: 'HTTP request timeout in ms',
+                    flag: '--timeout',
+                    type: 'number',
+                },
+                {
+                    default: false,
+                    description: 'Shortcut for --depth all (follows every same-origin link until --max-pages)',
+                    flag: '--all',
+                    type: 'boolean',
+                },
+                {
+                    default: true,
+                    description: "Render pages through a headless browser before extracting (for SPAs like Vue/Nuxt/React/Angular). Uses playwright-core with system Chrome / Edge, falling back to Playwright's bundled chromium. Disable with --no-render for plain HTTP fetches.",
+                    flag: '--render',
+                    type: 'boolean',
+                },
+                {
+                    default: false,
+                    description: 'If --render cannot find any browser, auto-install Playwright chromium (one-time ~170 MB download).',
+                    flag: '--install-browser',
+                    type: 'boolean',
+                },
+                {
+                    default: true,
+                    description: 'After a multi-page crawl, remove any .md or image files inside <outDir>/pages and <outDir>/images that were not written by this run. Disable with --no-prune to preserve old files.',
+                    flag: '--prune',
+                    type: 'boolean',
+                },
+                {
+                    default: false,
+                    description: 'Skip confirmation prompts',
+                    flag: '--noConfirm',
+                    type: 'boolean',
+                },
+            ],
+        })) {
+            return 'crawl';
+        }
+        tools.nonInteractiveHint('lt tools crawl <url> --out <dir> --depth 1 --noConfirm');
+        const ltConfig = config.loadConfig();
+        const commandConfig = (_b = (_a = ltConfig === null || ltConfig === void 0 ? void 0 : ltConfig.commands) === null || _a === void 0 ? void 0 : _a.tools) === null || _b === void 0 ? void 0 : _b.crawl;
+        // URL: positional argument > --url > interactive prompt.
+        const urlInput = parameters.first ||
+            parameters.options.url ||
+            (yield helper.getInput(undefined, { name: 'Website URL', showError: false }));
+        if (!urlInput) {
+            error('No URL provided');
+            return;
+        }
+        const url = normalizeSeedUrl(urlInput);
+        try {
+            new URL(url);
+        }
+        catch (_f) {
+            error(`Invalid URL: ${urlInput}`);
+            return;
+        }
+        const depthRaw = config.getValue({
+            // `--all` is a convenience shortcut for `--depth all`. It wins
+            // over a numeric `--depth` so users can combine both.
+            cliValue: parameters.options.all === true ? 'all' : parameters.options.depth,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.depth,
+            defaultValue: 0,
+        });
+        const depth = parseDepth(depthRaw);
+        const includeImages = config.getValue({
+            cliValue: parameters.options.images === false ? false : undefined,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.includeImages,
+            defaultValue: true,
+        });
+        const includeSitemap = config.getValue({
+            cliValue: parameters.options.sitemap === false ? false : undefined,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.includeSitemap,
+            defaultValue: true,
+        });
+        const concurrency = Number(config.getValue({
+            cliValue: parameters.options.concurrency,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.concurrency,
+            defaultValue: 4,
+        }));
+        const maxPages = Number(config.getValue({
+            cliValue: (_c = parameters.options.maxPages) !== null && _c !== void 0 ? _c : parameters.options['max-pages'],
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.maxPages,
+            defaultValue: 200,
+        }));
+        const timeout = Number(config.getValue({
+            cliValue: parameters.options.timeout,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.timeout,
+            defaultValue: 20000,
+        }));
+        const selector = config.getValue({
+            cliValue: parameters.options.selector,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.selector,
+        });
+        // `--render` and `--prune` default ON — the common case is a
+        // full SPA-aware knowledge-base crawl that stays in sync on
+        // updates. `--no-render` / `--no-prune` opt out explicitly.
+        const renderJs = config.getValue({
+            cliValue: parameters.options.render === false ? false : undefined,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.renderJs,
+            defaultValue: true,
+        });
+        const installBrowser = parameters.options['install-browser'] === true || parameters.options.installBrowser === true;
+        const pruneOrphans = config.getValue({
+            cliValue: parameters.options.prune === false ? false : undefined,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.prune,
+            defaultValue: true,
+        });
+        const outDir = (0, path_1.resolve)(config.getValue({
+            cliValue: (_d = parameters.options.out) !== null && _d !== void 0 ? _d : parameters.options.output,
+            configValue: commandConfig === null || commandConfig === void 0 ? void 0 : commandConfig.out,
+            defaultValue: filesystem.cwd(),
+        }) || filesystem.cwd());
+        const noConfirm = config.getNoConfirm({
+            cliValue: parameters.options.noConfirm,
+            commandConfig,
+            config: ltConfig,
+            parentConfig: (_e = ltConfig === null || ltConfig === void 0 ? void 0 : ltConfig.commands) === null || _e === void 0 ? void 0 : _e.tools,
+        });
+        info('');
+        info(`Crawling: ${url}`);
+        info(`Output:   ${outDir}`);
+        info(`Depth:    ${depth === 'all' ? 'all (bounded by --max-pages)' : depth}`);
+        info(`Sitemap:  ${includeSitemap ? 'yes' : 'no'}`);
+        info(`Images:   ${includeImages ? 'yes' : 'no'}`);
+        info(`Parallel: ${concurrency}`);
+        info(`Max:      ${maxPages} pages`);
+        info(`Render:   ${renderJs ? 'yes (headless browser)' : 'no (raw HTTP)'}`);
+        info(`Prune:    ${pruneOrphans ? 'yes (remove orphaned pages/images)' : 'no'}`);
+        if (selector)
+            info(`Selector: ${selector}`);
+        info('');
+        if (!noConfirm && !(yield confirm('Start crawl?'))) {
+            return 'crawl cancelled';
+        }
+        const spinner = spin('Crawling...');
+        const result = yield (0, crawler_1.crawlSite)({
+            autoInstallBrowser: installBrowser,
+            concurrency,
+            depth,
+            includeImages,
+            includeSitemap,
+            maxPages,
+            onLog: (msg) => {
+                spinner.text = msg;
+            },
+            outDir,
+            prune: pruneOrphans,
+            renderJs,
+            selector,
+            timeout,
+            url,
+        }).catch((err) => {
+            spinner.fail('Crawl failed');
+            error(err.message);
+            return null;
+        });
+        if (!result) {
+            return;
+        }
+        spinner.succeed(`Crawl complete: ${result.pages.length} page(s)`);
+        info('');
+        if (result.indexFile) {
+            success(`Overview: ${result.indexFile}`);
+        }
+        for (const page of result.pages.slice(0, 10)) {
+            info(`  - ${page.relativePath}  (${page.url})`);
+        }
+        if (result.pages.length > 10) {
+            info(`  ... and ${result.pages.length - 10} more`);
+        }
+        if (result.pruned.length > 0) {
+            info(`Pruned ${result.pruned.length} orphaned file(s)`);
+            for (const path of result.pruned.slice(0, 5)) {
+                info(`  - ${path}`);
+            }
+            if (result.pruned.length > 5)
+                info(`  ... and ${result.pruned.length - 5} more`);
+        }
+        if (result.skipped.length > 0) {
+            warning(`Skipped ${result.skipped.length} URL(s) (non-HTML or foreign origin)`);
+        }
+        if (result.errors.length > 0) {
+            warning(`${result.errors.length} error(s):`);
+            for (const err of result.errors.slice(0, 5)) {
+                warning(`  - ${err.url}: ${err.reason}`);
+            }
+        }
+        if (!toolbox.parameters.options.fromGluegunMenu) {
+            process.exit();
+        }
+        return `crawled ${result.pages.length} pages`;
+    }),
+};
+function normalizeSeedUrl(raw) {
+    const trimmed = raw.trim();
+    if (/^https?:\/\//i.test(trimmed))
+        return trimmed;
+    return `https://${trimmed}`;
+}
+/**
+ * Parse the --depth parameter. Accepts positive integers, the string
+ * "all", and negative values (treated as "all"). Invalid values fall
+ * back to `0` so the crawl still runs against the seed URL.
+ */
+function parseDepth(raw) {
+    if (raw === undefined || raw === null)
+        return 0;
+    if (typeof raw === 'string') {
+        const normalized = raw.trim().toLowerCase();
+        if (normalized === 'all' || normalized === '-1')
+            return 'all';
+        const n = Number(normalized);
+        if (!Number.isFinite(n))
+            return 0;
+        return n < 0 ? 'all' : Math.floor(n);
+    }
+    if (typeof raw === 'number') {
+        if (!Number.isFinite(raw))
+            return 'all';
+        return raw < 0 ? 'all' : Math.floor(raw);
+    }
+    return 0;
+}
+exports.default = NewCommand;

package/build/lib/browser-fetcher.js

@@ -0,0 +1,139 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createBrowserFetcher = createBrowserFetcher;
+/**
+ * Headless-browser HTML fetcher for single-page applications.
+ *
+ * Mirrors the chrome-md content script's PageReadyDetector:
+ * waits for the network to settle, then returns the fully hydrated
+ * HTML so Defuddle can extract the real content instead of the
+ * pre-render shell.
+ *
+ * Uses `playwright-core` with a three-tier strategy:
+ *   1. System Chrome / Edge via `channel: 'chrome' | 'msedge'`.
+ *   2. Playwright's own bundled Chromium (if already installed).
+ *   3. Auto-install Playwright's Chromium (`npx playwright install
+ *      chromium`) and retry — opt-in via `autoInstall`.
+ */
+const child_process_1 = require("child_process");
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; lenneTech-CLI-Crawler/1.0; +https://lenne.tech)';
+/**
+ * Try to construct a browser fetcher. Prefers a system Chrome /
+ * Edge via Playwright channels, falls back to Playwright's bundled
+ * Chromium, and (optionally) auto-installs Chromium on demand.
+ */
+function createBrowserFetcher() {
+    return __awaiter(this, arguments, void 0, function* (options = {}) {
+        const log = options.onLog || (() => undefined);
+        const reasons = [];
+        const { chromium } = require('playwright-core');
+        // 1. System Chrome.
+        const chromeFetcher = yield launch(chromium, { channel: 'chrome' }, options, 'system-chrome').catch((error) => {
+            reasons.push(`channel:chrome: ${error.message}`);
+            return null;
+        });
+        if (chromeFetcher) {
+            log(`Browser engine: ${chromeFetcher.engine}`);
+            return chromeFetcher;
+        }
+        // 2. System Edge (Windows fallback, also common on macOS).
+        const edgeFetcher = yield launch(chromium, { channel: 'msedge' }, options, 'system-edge').catch((error) => {
+            reasons.push(`channel:msedge: ${error.message}`);
+            return null;
+        });
+        if (edgeFetcher) {
+            log(`Browser engine: ${edgeFetcher.engine}`);
+            return edgeFetcher;
+        }
+        // 3. Playwright's bundled Chromium.
+        const bundledFetcher = yield launch(chromium, {}, options, 'playwright-chromium').catch((error) => {
+            reasons.push(`playwright-chromium: ${error.message}`);
+            return null;
+        });
+        if (bundledFetcher) {
+            log(`Browser engine: ${bundledFetcher.engine}`);
+            return bundledFetcher;
+        }
+        // 4. Optional auto-install, then retry Playwright's chromium.
+        if (options.autoInstall) {
+            log('No browser available — installing Playwright chromium (one-time download, ~170 MB)…');
+            try {
+                yield runNpx(['playwright', 'install', 'chromium']);
+                const retry = yield launch(chromium, {}, options, 'playwright-chromium');
+                if (retry) {
+                    log(`Browser engine: ${retry.engine}`);
+                    return retry;
+                }
+            }
+            catch (error) {
+                reasons.push(`auto-install: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        throw new Error([
+            'Could not start a headless browser for SPA rendering.',
+            ...reasons.map((r) => `  - ${r}`),
+            '',
+            'Fix one of these:',
+            '  1. Install Google Chrome or Microsoft Edge (Playwright picks them up automatically).',
+            '  2. Install Playwright browsers manually: `npx playwright install chromium`.',
+            '  3. Re-run with --install-browser to let the CLI install them.',
+        ].join('\n'));
+    });
+}
+function launch(chromium, launchOptions, options, engineLabel) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const browser = yield chromium.launch(Object.assign(Object.assign({}, launchOptions), { headless: true }));
+        const context = yield browser.newContext({
+            userAgent: options.userAgent || DEFAULT_USER_AGENT,
+        });
+        return {
+            close: () => __awaiter(this, void 0, void 0, function* () {
+                yield context.close();
+                yield browser.close();
+            }),
+            engine: engineLabel,
+            fetch: (url) => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                const page = yield context.newPage();
+                try {
+                    yield page.goto(url, {
+                        timeout: (_a = options.maxWaitMs) !== null && _a !== void 0 ? _a : 20000,
+                        waitUntil: 'networkidle',
+                    });
+                    if (options.extraWaitMs) {
+                        yield page.waitForTimeout(options.extraWaitMs);
+                    }
+                    return yield page.content();
+                }
+                finally {
+                    yield page.close();
+                }
+            }),
+        };
+    });
+}
+/**
+ * Run an `npx` command, streaming its output to the current stdio.
+ * Resolves on exit code 0, rejects otherwise.
+ */
+function runNpx(args) {
+    return new Promise((resolve, reject) => {
+        const child = (0, child_process_1.spawn)('npx', args, { shell: false, stdio: 'inherit' });
+        child.on('error', reject);
+        child.on('exit', (code) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new Error(`npx ${args.join(' ')} exited with code ${code}`));
+        });
+    });
+}

package/build/lib/crawler.js

@@ -0,0 +1,661 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.crawlSite = crawlSite;
+/**
+ * Website crawler utilities.
+ *
+ * Fetches web pages (optionally guided by sitemap.xml), extracts the
+ * main content using the same defuddle + Turndown pipeline as the
+ * chrome-md browser extension (see ../../../chrome-md/content/content.js),
+ * converts it to Markdown, and writes one .md file per page plus an
+ * overview README when multiple pages are discovered. Designed for
+ * building Claude Code knowledge bases.
+ */
+const axios_1 = __importDefault(require("axios"));
+const crypto_1 = require("crypto");
+const defuddle_1 = __importDefault(require("defuddle"));
+const fs_1 = require("fs");
+const jsdom_1 = require("jsdom");
+const path_1 = require("path");
+const turndown_1 = __importDefault(require("turndown"));
+const turndown_plugin_gfm_1 = require("turndown-plugin-gfm");
+const browser_fetcher_1 = require("./browser-fetcher");
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; lenneTech-CLI-Crawler/1.0; +https://lenne.tech)';
+/**
+ * Crawl a website starting at `options.url` and write the collected
+ * pages as Markdown files beneath `options.outDir`.
+ */
+function crawlSite(options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { autoInstallBrowser = false, concurrency = 4, depth: rawDepth, includeImages = true, includeSitemap = true, maxPages = 200, onLog = () => undefined, outDir, prune = false, renderJs = false, selector, timeout = 20000, url: seedUrl, userAgent = DEFAULT_USER_AGENT, } = options;
+        // Normalize depth. `'all'` and negative numbers mean "follow every
+        // same-origin link we find" — bounded by `maxPages`.
+        const depth = rawDepth === 'all' || (typeof rawDepth === 'number' && rawDepth < 0) ? Number.POSITIVE_INFINITY : Number(rawDepth);
+        const http = axios_1.default.create({
+            headers: { 'User-Agent': userAgent },
+            maxRedirects: 5,
+            responseType: 'text',
+            timeout,
+            validateStatus: (status) => status >= 200 && status < 400,
+        });
+        // Headless browser only spun up when needed (SPA-mode).
+        let browserFetcher = null;
+        if (renderJs) {
+            browserFetcher = yield (0, browser_fetcher_1.createBrowserFetcher)({
+                autoInstall: autoInstallBrowser,
+                extraWaitMs: 500,
+                maxWaitMs: timeout,
+                onLog,
+                userAgent,
+            });
+        }
+        try {
+            const seed = new URL(seedUrl);
+            const origin = seed.origin;
+            if (!(0, fs_1.existsSync)(outDir)) {
+                (0, fs_1.mkdirSync)(outDir, { recursive: true });
+            }
+            // Queue preserves the depth at which a URL was discovered so children
+            // are only followed when `discovered.depth < options.depth`.
+            const queue = [{ depth: 0, url: normalizeUrl(seedUrl) }];
+            const seen = new Set([normalizeUrl(seedUrl)]);
+            if (includeSitemap) {
+                onLog(`Checking sitemap at ${origin}/sitemap.xml`);
+                const sitemapUrls = yield fetchSitemapUrls(http, origin, onLog);
+                for (const sitemapUrl of sitemapUrls) {
+                    const normalized = normalizeUrl(sitemapUrl);
+                    if (!seen.has(normalized) && sameOrigin(normalized, origin)) {
+                        seen.add(normalized);
+                        queue.push({ depth: 0, url: normalized });
+                    }
+                }
+                if (sitemapUrls.length > 0) {
+                    onLog(`Sitemap discovered ${sitemapUrls.length} URLs`);
+                }
+            }
+            const pages = [];
+            const errors = [];
+            const skipped = [];
+            // Shared deduplicated image map (content hash -> relative path under outDir).
+            const imageHashToPath = new Map();
+            // We can't know upfront whether the crawl is single- or multi-page,
+            // so we render pages into a buffer first and only materialize files
+            // once the queue drains.
+            const rendered = [];
+            const processPage = (item) => __awaiter(this, void 0, void 0, function* () {
+                var _a, _b;
+                if (pages.length + errors.length >= maxPages) {
+                    skipped.push(item.url);
+                    return;
+                }
+                onLog(`Fetching (depth ${item.depth}): ${item.url}`);
+                try {
+                    let html;
+                    let finalUrl = normalizeUrl(item.url);
+                    if (browserFetcher) {
+                        // In render mode we trust the URL we navigated to. We can't
+                        // cheaply detect redirects here, so assume same origin (the
+                        // crawler already filtered non-HTML URLs out of the queue).
+                        html = yield browserFetcher.fetch(item.url);
+                    }
+                    else {
+                        const response = yield http.get(item.url);
+                        finalUrl = normalizeUrl(((_b = (_a = response.request) === null || _a === void 0 ? void 0 : _a.res) === null || _b === void 0 ? void 0 : _b.responseUrl) || item.url);
+                        if (!sameOrigin(finalUrl, origin)) {
+                            skipped.push(item.url);
+                            return;
+                        }
+                        const contentType = String(response.headers['content-type'] || '');
+                        if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
+                            skipped.push(item.url);
+                            return;
+                        }
+                        html = String(response.data || '');
+                    }
+                    const extracted = yield extractContent(html, finalUrl, { selector });
+                    // Follow links when depth budget is left.
+                    if (item.depth < depth) {
+                        for (const link of extracted.links) {
+                            if (!sameOrigin(link, origin))
+                                continue;
+                            const normalized = normalizeUrl(link);
+                            if (seen.has(normalized))
+                                continue;
+                            seen.add(normalized);
+                            queue.push({ depth: item.depth + 1, url: normalized });
+                        }
+                    }
+                    // Download images and build a URL -> local path map for Turndown.
+                    const imageEntries = [];
+                    if (includeImages && extracted.images.length > 0) {
+                        for (const imgUrl of extracted.images) {
+                            try {
+                                const absolute = new URL(imgUrl, finalUrl).href;
+                                const result = yield fetchImage(http, absolute);
+                                if (!result)
+                                    continue;
+                                const hash = (0, crypto_1.createHash)('sha1').update(result.buffer).digest('hex');
+                                let relativeImagePath = imageHashToPath.get(hash);
+                                if (!relativeImagePath) {
+                                    // Filename uses a content-hash suffix so re-runs with
+                                    // identical bytes overwrite the same file instead of
+                                    // leaving orphans with rotating counter suffixes.
+                                    const filename = buildImageFilename(absolute, hash, result.contentType);
+                                    relativeImagePath = `images/${filename}`;
+                                    imageHashToPath.set(hash, relativeImagePath);
+                                    imageEntries.push({ data: result.buffer, filename });
+                                }
+                                extracted.imageMap.set(imgUrl, relativeImagePath);
+                                extracted.imageMap.set(absolute, relativeImagePath);
+                            }
+                            catch (_c) {
+                                // Skip image on error; continue with others.
+                            }
+                        }
+                    }
+                    const markdown = convertToMarkdown(extracted.contentHtml, finalUrl, extracted.imageMap);
+                    const filename = buildPageFilename(finalUrl, rendered.length === 0);
+                    rendered.push({
+                        filename,
+                        images: imageEntries,
+                        info: {
+                            author: extracted.meta.author,
+                            depth: item.depth,
+                            description: extracted.meta.description,
+                            downloadDate: new Date().toISOString(),
+                            firstDownloaded: new Date().toISOString(),
+                            imageCount: imageEntries.length,
+                            language: extracted.meta.language,
+                            ogImage: extracted.meta.ogImage,
+                            title: extracted.meta.title,
+                            url: finalUrl,
+                            wordCount: extracted.meta.wordCount || countWords(extracted.contentText),
+                        },
+                        markdown,
+                    });
+                }
+                catch (error) {
+                    errors.push({
+                        reason: error instanceof Error ? error.message : String(error),
+                        url: item.url,
+                    });
+                }
+            });
+            // Simple parallel worker pool. `queue` grows as pages are discovered,
+            // so workers pick new items until nothing is left.
+            let cursor = 0;
+            const worker = () => __awaiter(this, void 0, void 0, function* () {
+                while (cursor < queue.length && pages.length + errors.length < maxPages) {
+                    const item = queue[cursor++];
+                    yield processPage(item);
+                }
+            });
+            const workers = Array.from({ length: Math.max(1, concurrency) }, () => worker());
+            yield Promise.all(workers);
+            // Drain any late discoveries added after all initial workers exited.
+            while (cursor < queue.length) {
+                yield Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+            }
+            const multiPage = rendered.length > 1;
+            const pagesDir = multiPage ? (0, path_1.join)(outDir, 'pages') : outDir;
+            const imagesDir = (0, path_1.join)(outDir, 'images');
+            if (rendered.length === 0) {
+                onLog('No pages rendered');
+                return { errors, outDir, pages, pruned: [], skipped };
+            }
+            (0, fs_1.mkdirSync)(pagesDir, { recursive: true });
+            if (includeImages && imageHashToPath.size > 0) {
+                (0, fs_1.mkdirSync)(imagesDir, { recursive: true });
+            }
+            // Write deduplicated images.
+            const writtenImageFilenames = new Set();
+            for (const entry of rendered.flatMap((r) => r.images)) {
+                if (writtenImageFilenames.has(entry.filename))
+                    continue;
+                writtenImageFilenames.add(entry.filename);
+                (0, fs_1.writeFileSync)((0, path_1.join)(imagesDir, entry.filename), entry.data);
+            }
+            // Persist pages. When updating, preserve the original
+            // `first_downloaded` timestamp so history stays intact.
+            for (const entry of rendered) {
+                const outputPath = (0, path_1.join)(pagesDir, entry.filename);
+                const relativePath = (0, path_1.relative)(outDir, outputPath);
+                // Images live under `<outDir>/images/`. Each page rewrites the
+                // Turndown-emitted `images/<file>` placeholder to the correct
+                // relative path so nested URL slugs (`pages/ueber-uns/…`, or a
+                // single-page crawl that lands in `<outDir>/ueber-uns/…`) still
+                // render in Markdown previews.
+                const imagePrefix = `${(0, path_1.relative)((0, path_1.dirname)(outputPath), imagesDir).split(/[\\/]/).join('/')}/`;
+                const fixedMarkdown = entry.markdown.replace(/\]\(images\//g, `](${imagePrefix}`);
+                if ((0, fs_1.existsSync)(outputPath)) {
+                    const existing = (0, fs_1.readFileSync)(outputPath, 'utf8');
+                    const existingMeta = parseFrontmatter(existing);
+                    if (existingMeta === null || existingMeta === void 0 ? void 0 : existingMeta.first_downloaded) {
+                        entry.info.firstDownloaded = String(existingMeta.first_downloaded);
+                    }
+                }
+                const frontmatter = renderFrontmatter(entry.info);
+                (0, fs_1.mkdirSync)((0, path_1.dirname)(outputPath), { recursive: true });
+                (0, fs_1.writeFileSync)(outputPath, `${frontmatter}\n${fixedMarkdown.trim()}\n`);
+                pages.push(Object.assign(Object.assign({}, entry.info), { outputPath, relativePath }));
+            }
+            let indexFile;
+            if (multiPage) {
+                indexFile = (0, path_1.join)(outDir, 'README.md');
+                (0, fs_1.writeFileSync)(indexFile, renderOverview(seed.href, pages));
+            }
+            // Prune orphans (files left over from previous crawls). Scoped to
+            // `pages/` and `images/` so stray user files in outDir root never
+            // get touched. Only active in multi-page mode — a single-page
+            // crawl writes into `outDir` itself and has no page subfolder to
+            // sweep.
+            const pruned = [];
+            if (prune && multiPage) {
+                const keep = new Set(pages.map((p) => p.outputPath));
+                for (const entry of rendered.flatMap((r) => r.images)) {
+                    keep.add((0, path_1.join)(imagesDir, entry.filename));
+                }
+                pruned.push(...pruneOrphans(pagesDir, keep));
+                if ((0, fs_1.existsSync)(imagesDir)) {
+                    pruned.push(...pruneOrphans(imagesDir, keep));
+                }
+                if (pruned.length > 0) {
+                    onLog(`Pruned ${pruned.length} orphaned file(s)`);
+                }
+            }
+            return { errors, indexFile, outDir, pages, pruned, skipped };
+        }
+        finally {
+            // Guarantee the headless browser is shut down on every exit path,
+            // including thrown errors, so no orphan chromium processes linger.
+            if (browserFetcher) {
+                yield browserFetcher.close().catch(() => undefined);
+            }
+        }
+    });
+}
+function buildImageFilename(url, contentHash, contentType) {
+    let basename = 'image';
+    let extension = '';
+    try {
+        const u = new URL(url);
+        const last = u.pathname.split('/').filter(Boolean).pop() || '';
+        const parsedExt = (0, path_1.extname)(last).replace('.', '').toLowerCase();
+        if (parsedExt && /^(jpg|jpeg|png|gif|webp|svg|avif)$/.test(parsedExt)) {
+            extension = parsedExt;
+        }
+        basename =
+            last
+                .replace((0, path_1.extname)(last), '')
+                .replace(/[^a-zA-Z0-9-_]/g, '_')
+                .substring(0, 40) || 'image';
+    }
+    catch (_a) {
+        // fall through
+    }
+    if (!extension) {
+        const fromType = contentType.split(';')[0].split('/')[1];
+        if (fromType && /^(jpeg|jpg|png|gif|webp|svg\+xml|avif)$/.test(fromType)) {
+            extension = fromType === 'svg+xml' ? 'svg' : fromType;
+        }
+        else {
+            extension = 'png';
+        }
+    }
+    return `${basename}-${contentHash.slice(0, 8)}.${extension}`;
+}
+function buildPageFilename(url, isFirst) {
+    const u = new URL(url);
+    const segments = u.pathname.split('/').filter(Boolean);
+    if (segments.length === 0) {
+        return isFirst ? 'index.md' : 'home.md';
+    }
+    const slugged = segments
+        .map((s) => s
+        .toLowerCase()
+        .replace(/\.(html?|php|aspx?)$/, '')
+        .replace(/[^a-z0-9-_]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '') || 'page')
+        .join('/');
+    return `${slugged}.md`;
+}
+function convertToMarkdown(html, baseUrl, imageMap) {
+    const turndown = new turndown_1.default({
+        bulletListMarker: '-',
+        codeBlockStyle: 'fenced',
+        emDelimiter: '*',
+        headingStyle: 'atx',
+        linkStyle: 'inlined',
+        strongDelimiter: '**',
+    });
+    // Enable GFM so tables, strikethrough and task lists convert cleanly.
+    if (turndown_plugin_gfm_1.gfm) {
+        turndown.use(turndown_plugin_gfm_1.gfm);
+    }
+    turndown.addRule('absoluteLinks', {
+        filter: 'a',
+        replacement: (content, node) => {
+            var _a, _b, _c, _d;
+            const href = ((_b = (_a = node).getAttribute) === null || _b === void 0 ? void 0 : _b.call(_a, 'href')) || '';
+            if (!href || href === '#' || href.startsWith('javascript:')) {
+                return content;
+            }
+            let absolute = href;
+            try {
+                absolute = new URL(href, baseUrl).href;
+            }
+            catch (_e) {
+                // keep original
+            }
+            const title = (_d = (_c = node).getAttribute) === null || _d === void 0 ? void 0 : _d.call(_c, 'title');
+            return title ? `[${content}](${absolute} "${title}")` : `[${content}](${absolute})`;
+        },
+    });
+    turndown.addRule('localImages', {
+        filter: 'img',
+        replacement: (_content, node) => {
+            var _a, _b, _c, _d;
+            const src = ((_b = (_a = node).getAttribute) === null || _b === void 0 ? void 0 : _b.call(_a, 'src')) || '';
+            if (!src)
+                return '';
+            let absolute = src;
+            try {
+                absolute = new URL(src, baseUrl).href;
+            }
+            catch (_e) {
+                // keep original
+            }
+            const local = imageMap.get(src) || imageMap.get(absolute);
+            const alt = ((_d = (_c = node).getAttribute) === null || _d === void 0 ? void 0 : _d.call(_c, 'alt')) || '';
+            const target = local || absolute;
+            return `![${alt}](${target})`;
+        },
+    });
+    turndown.remove(['script', 'style', 'noscript', 'iframe']);
+    const markdown = turndown.turndown(html);
+    return markdown.replace(/\n{3,}/g, '\n\n').trim();
+}
+function countWords(text) {
+    return text.replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).length;
+}
+function escapeYaml(value) {
+    return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, ' ');
+}
+/**
+ * Extract main content + metadata using defuddle (the same engine as
+ * chrome-md). Falls back to a raw body dump if defuddle fails.
+ */
+function extractContent(html, pageUrl, options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, _b, _c, _d, _e, _f, _g;
+        const dom = new jsdom_1.JSDOM(html, { url: pageUrl });
+        const doc = dom.window.document;
+        const defuddleOptions = {
+            markdown: false,
+            removeHiddenElements: true,
+            removeLowScoring: true,
+            removeSmallImages: false,
+        };
+        if (options.selector) {
+            defuddleOptions.contentSelector = options.selector;
+        }
+        let parsed;
+        try {
+            // Same class-based API as chrome-md's content script.
+            const instance = new defuddle_1.default(doc, defuddleOptions);
+            parsed = instance.parse();
+        }
+        catch (_h) {
+            parsed = {
+                content: ((_a = doc.body) === null || _a === void 0 ? void 0 : _a.innerHTML) || html,
+                title: doc.title,
+            };
+        }
+        const contentHtml = parsed.content || ((_b = doc.body) === null || _b === void 0 ? void 0 : _b.innerHTML) || '';
+        // Collect images and links from the cleaned content.
+        const helperDom = new jsdom_1.JSDOM(`<!DOCTYPE html><html><body>${contentHtml}</body></html>`, {
+            url: pageUrl,
+        });
+        const contentDoc = helperDom.window.document;
+        const links = new Set();
+        contentDoc.querySelectorAll('a[href]').forEach((el) => {
+            const href = (el.getAttribute('href') || '').trim();
+            if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('javascript:')) {
+                return;
+            }
+            try {
+                links.add(new URL(href, pageUrl).href);
+            }
+            catch (_a) {
+                // ignore malformed URLs
+            }
+        });
+        const images = new Set();
+        contentDoc.querySelectorAll('img').forEach((el) => {
+            const src = (el.getAttribute('src') || el.getAttribute('data-src') || '').trim();
+            if (!src || src.startsWith('data:'))
+                return;
+            try {
+                images.add(new URL(src, pageUrl).href);
+            }
+            catch (_a) {
+                // ignore malformed URLs
+            }
+        });
+        // Some lazy-loading frameworks keep the real URL only in the source
+        // document (stripped out by defuddle), so also consult the original DOM.
+        doc.querySelectorAll('img[data-src], img[data-lazy-src]').forEach((el) => {
+            const src = (el.getAttribute('data-src') || el.getAttribute('data-lazy-src') || '').trim();
+            if (!src || src.startsWith('data:'))
+                return;
+            try {
+                images.add(new URL(src, pageUrl).href);
+            }
+            catch (_a) {
+                // ignore
+            }
+        });
+        const meta = {
+            author: parsed.author || ((_c = doc.querySelector('meta[name="author"]')) === null || _c === void 0 ? void 0 : _c.getAttribute('content')) || undefined,
+            description: parsed.description ||
+                ((_d = doc.querySelector('meta[name="description"]')) === null || _d === void 0 ? void 0 : _d.getAttribute('content')) ||
+                ((_e = doc.querySelector('meta[property="og:description"]')) === null || _e === void 0 ? void 0 : _e.getAttribute('content')) ||
+                '',
+            language: parsed.language || doc.documentElement.getAttribute('lang') || undefined,
+            ogImage: parsed.image || ((_f = doc.querySelector('meta[property="og:image"]')) === null || _f === void 0 ? void 0 : _f.getAttribute('content')) || undefined,
+            title: parsed.title || doc.title || pageUrl,
+            wordCount: parsed.wordCount,
+        };
+        return {
+            contentHtml,
+            contentText: ((_g = contentDoc.body) === null || _g === void 0 ? void 0 : _g.textContent) || '',
+            imageMap: new Map(),
+            images: [...images],
+            links: [...links],
+            meta,
+        };
+    });
+}
+function fetchImage(http, url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const response = yield http.get(url, { responseType: 'arraybuffer' });
+            const buffer = Buffer.from(response.data);
+            if (buffer.byteLength === 0)
+                return null;
+            return { buffer, contentType: String(response.headers['content-type'] || '') };
+        }
+        catch (_a) {
+            return null;
+        }
+    });
+}
+function fetchSitemapUrls(http, origin, onLog) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const urls = [];
+        const visited = new Set();
+        function walk(sitemapUrl) {
+            return __awaiter(this, void 0, void 0, function* () {
+                if (visited.has(sitemapUrl))
+                    return;
+                visited.add(sitemapUrl);
+                try {
+                    const response = yield http.get(sitemapUrl);
+                    const xml = String(response.data || '');
+                    // Nested sitemap index: follow each <sitemap><loc>...</loc></sitemap>.
+                    const nested = [...xml.matchAll(/<sitemap>[\s\S]*?<loc>\s*([^<\s]+)\s*<\/loc>[\s\S]*?<\/sitemap>/gi)].map((m) => m[1]);
+                    for (const child of nested) {
+                        yield walk(child);
+                    }
+                    const pageMatches = [...xml.matchAll(/<url>[\s\S]*?<loc>\s*([^<\s]+)\s*<\/loc>[\s\S]*?<\/url>/gi)].map((m) => m[1]);
+                    urls.push(...pageMatches);
+                }
+                catch (error) {
+                    onLog(`Sitemap fetch failed for ${sitemapUrl}: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            });
+        }
+        yield walk(`${origin}/sitemap.xml`);
+        return urls;
+    });
+}
+/**
+ * Normalize a URL for dedup: strip hash, drop default `index.html`,
+ * and remove trailing slashes (except root).
+ */
+function normalizeUrl(raw) {
+    try {
+        const u = new URL(raw);
+        u.hash = '';
+        u.pathname = u.pathname.replace(/\/index\.html?$/i, '/');
+        if (u.pathname.length > 1 && u.pathname.endsWith('/')) {
+            u.pathname = u.pathname.replace(/\/+$/, '');
+        }
+        return u.href;
+    }
+    catch (_a) {
+        return raw;
+    }
+}
+function parseFrontmatter(markdown) {
+    if (!markdown.startsWith('---'))
+        return null;
+    const end = markdown.indexOf('\n---', 3);
+    if (end === -1)
+        return null;
+    const block = markdown.slice(3, end);
+    const result = {};
+    for (const line of block.split('\n')) {
+        const match = line.match(/^([a-zA-Z0-9_]+):\s*(.*)$/);
+        if (!match)
+            continue;
+        result[match[1]] = match[2].replace(/^"(.*)"$/, '$1');
+    }
+    return result;
+}
+/**
+ * Walk `rootDir` recursively and delete every file whose absolute
+ * path is not in `keepPaths`. Empty directories left behind after
+ * the sweep are removed, too. Returns the absolute paths that were
+ * actually deleted.
+ */
+function pruneOrphans(rootDir, keepPaths) {
+    const removed = [];
+    if (!(0, fs_1.existsSync)(rootDir))
+        return removed;
+    const entries = (0, fs_1.readdirSync)(rootDir, { withFileTypes: true });
+    for (const entry of entries) {
+        const full = (0, path_1.join)(rootDir, entry.name);
+        if (entry.isDirectory()) {
+            removed.push(...pruneOrphans(full, keepPaths));
+            // Remove directory if now empty.
+            try {
+                if ((0, fs_1.readdirSync)(full).length === 0)
+                    (0, fs_1.rmdirSync)(full);
+            }
+            catch (_a) {
+                // Directory not empty or already gone — ignore.
+            }
+        }
+        else if (entry.isFile() && !keepPaths.has(full)) {
+            try {
+                (0, fs_1.unlinkSync)(full);
+                removed.push(full);
+            }
+            catch (_b) {
+                // File already removed or permission denied — skip.
+            }
+        }
+    }
+    return removed;
+}
+function renderFrontmatter(info) {
+    const lines = [
+        '---',
+        `title: "${escapeYaml(info.title)}"`,
+        `source_url: "${info.url}"`,
+        `source_domain: "${new URL(info.url).hostname}"`,
+        `crawl_depth: ${info.depth}`,
+        `download_date: "${info.downloadDate}"`,
+        `first_downloaded: "${info.firstDownloaded}"`,
+        info.description ? `description: "${escapeYaml(truncate(info.description, 500))}"` : null,
+        info.author ? `author: "${escapeYaml(info.author)}"` : null,
+        info.language ? `language: "${escapeYaml(info.language)}"` : null,
+        info.ogImage ? `og_image: "${escapeYaml(info.ogImage)}"` : null,
+        info.imageCount ? `image_count: ${info.imageCount}` : null,
+        `word_count: ${info.wordCount}`,
+        'content_type: "webpage"',
+        '---',
+    ].filter((l) => l !== null);
+    return lines.join('\n');
+}
+function renderOverview(startUrl, pages) {
+    const ordered = [...pages].sort((a, b) => a.url.localeCompare(b.url));
+    const host = new URL(startUrl).host;
+    const lines = [];
+    lines.push(`# ${host} — Knowledge Base`);
+    lines.push('');
+    lines.push(`Source: ${startUrl}`);
+    lines.push('');
+    lines.push(`Generated: ${new Date().toISOString()}`);
+    lines.push('');
+    lines.push(`Pages: ${ordered.length}`);
+    lines.push('');
+    lines.push('## Pages');
+    lines.push('');
+    for (const page of ordered) {
+        lines.push(`### [${page.title}](${page.relativePath.split(/[\\/]/).join('/')})`);
+        lines.push('');
+        lines.push(`- URL: ${page.url}`);
+        if (page.description) {
+            lines.push(`- ${truncate(page.description, 240)}`);
+        }
+        lines.push(`- Updated: ${page.downloadDate}`);
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+function sameOrigin(url, origin) {
+    try {
+        return new URL(url).origin === origin;
+    }
+    catch (_a) {
+        return false;
+    }
+}
+function truncate(value, max) {
+    return value.length <= max ? value : `${value.slice(0, max - 1)}…`;
+}

package/docs/commands.md

@@ -1409,6 +1409,58 @@ lt tools regex [pattern] [text]
 
 ---
 
+### `lt tools crawl`
+
+Crawls a website and stores each page as a Markdown file (with YAML frontmatter containing `source_url`, `download_date`, `first_downloaded`, `description`, language, word count, etc.) so it can be consumed as a Claude Code knowledge base. Optionally follows same-origin links up to a configurable depth, seeds the queue from `<origin>/sitemap.xml`, and downloads referenced images into a shared `images/` folder (deduplicated by content hash). Re-running the command against the same output directory updates existing pages while preserving their original `first_downloaded` timestamp.
+
+**Alias:** `cr`
+
+**Usage:**
+```bash
+lt tools crawl <url> [options]
+```
+
+**Options:**
+- `--out <dir>` — Output directory (default: current directory). Single-page crawls write the `.md` directly here; multi-page crawls generate `<out>/README.md` plus `<out>/pages/` and `<out>/images/`.
+- `--depth <n|all>` — Link depth (default `0`). `0` = only the start page, `1` = start page + direct same-origin links, `2` = and their links, ... Use `--depth all` (or `--depth -1`, or the shortcut flag `--all`) to follow every same-origin link transitively; the crawl then stops when `--max-pages` is reached.
+- `--all` — Shortcut for `--depth all`.
+- `--render` / `--no-render` — Render each page through a headless browser before extraction (default **on**). Required for SPAs (Vue/Nuxt/React/Angular) whose content is client-rendered. Uses `playwright-core` with system Chrome / Edge first, then Playwright's bundled Chromium. Use `--no-render` for a plain HTTP fetch when you know the site is static (faster, no browser needed).
+- `--install-browser` — If `--render` finds no browser, auto-install Playwright's Chromium (one-time ~170 MB download).
+- `--prune` / `--no-prune` — After a multi-page crawl, remove any `.md` or image files inside `<out>/pages` and `<out>/images` that were not written by the current run (default **on**). Keeps the knowledge base aligned with the live site on update runs. Empty subdirectories are cleaned up too. Ignored in single-page mode. Use `--no-prune` to preserve old files.
+- `--no-images` — Disable image downloads.
+- `--no-sitemap` — Skip discovery via `<origin>/sitemap.xml`.
+- `--concurrency <n>` — Parallel HTTP requests (default `4`).
+- `--max-pages <n>` — Safety cap on total pages (default `200`).
+- `--selector <css>` — CSS selector scoping the main content (e.g. `article`, `main`).
+- `--timeout <ms>` — HTTP request timeout in ms (default `20000`).
+- `--noConfirm` — Skip confirmation prompts.
+
+**Examples:**
+```bash
+# Single page into the current directory
+lt tools crawl https://example.com/article --noConfirm
+
+# Crawl start page + direct links into ./knowledge
+lt tools crawl https://example.com --out ./knowledge --depth 1 --noConfirm
+
+# Full mini-site with sitemap seeding and images
+lt tools crawl https://example.com --out ./kb --depth 2 --max-pages 100 --noConfirm
+
+# Crawl every reachable same-origin page (safety cap via --max-pages)
+lt tools crawl https://example.com --out ./kb --depth all --max-pages 500 --noConfirm
+
+# Same, using the --all shortcut
+lt tools crawl https://example.com --out ./kb --all --max-pages 500 --noConfirm
+
+# Full SPA-aware crawl (render + prune are on by default)
+lt tools crawl https://lenne.tech --all --noConfirm
+
+# Opt-out: plain HTTP fetch for a known-static site, keep orphans
+lt tools crawl https://example.com --all --no-render --no-prune --noConfirm
+```
+
+---
+
 ## Configuration Priority
 
 All configurable commands follow this priority order (highest to lowest):

package/docs/lt.config.md

@@ -923,6 +923,43 @@ Reinitializes npm packages.
 }
 ```
 
+#### `lt tools crawl`
+
+Crawls a website into Markdown files (knowledge base builder).
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `commands.tools.crawl.out` | `string` | `.` | Output directory |
+| `commands.tools.crawl.depth` | `number \| "all"` | `0` | Link depth (0 = only start page, 1 = direct links, ..., `"all"` or `-1` = follow every same-origin link, bounded by `maxPages`) |
+| `commands.tools.crawl.includeImages` | `boolean` | `true` | Download images and inline with local paths |
+| `commands.tools.crawl.includeSitemap` | `boolean` | `true` | Also seed queue from `<origin>/sitemap.xml` |
+| `commands.tools.crawl.concurrency` | `number` | `4` | Parallel HTTP requests |
+| `commands.tools.crawl.maxPages` | `number` | `200` | Safety cap on total pages |
+| `commands.tools.crawl.selector` | `string` | – | CSS selector for main content |
+| `commands.tools.crawl.timeout` | `number` | `20000` | HTTP request timeout in ms |
+| `commands.tools.crawl.renderJs` | `boolean` | `true` | Render pages through a headless browser (for SPAs). Uses playwright-core. Set to `false` for plain HTTP. |
+| `commands.tools.crawl.prune` | `boolean` | `true` | Remove orphaned `.md` / image files after a multi-page crawl (update-in-place). Set to `false` to preserve old files. |
+| `commands.tools.crawl.noConfirm` | `boolean` | `false` | Skip confirmation prompts |
+
+**Example:**
+```json
+{
+  "commands": {
+    "tools": {
+      "crawl": {
+        "out": "./knowledge",
+        "depth": 2,
+        "includeImages": true,
+        "includeSitemap": true,
+        "concurrency": 4,
+        "maxPages": 200,
+        "noConfirm": true
+      }
+    }
+  }
+}
+```
+
 ---
 
 ### Metadata

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lenne.tech/cli",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "lenne.Tech CLI: lt",
   "keywords": [
     "lenne.Tech",
@@ -61,14 +61,19 @@
     "@lenne.tech/cli-plugin-helper": "0.0.14",
     "axios": "1.15.0",
     "bcrypt": "6.0.0",
+    "defuddle": "0.17.0",
     "glob": "13.0.6",
     "gluegun": "5.2.2",
     "js-sha256": "0.11.1",
     "js-yaml": "4.1.1",
+    "jsdom": "29.0.2",
     "lodash": "4.18.1",
     "open": "11.0.0",
+    "playwright-core": "1.59.1",
     "ts-morph": "28.0.0",
     "ts-node": "10.9.2",
+    "turndown": "7.2.4",
+    "turndown-plugin-gfm": "1.0.2",
     "typescript": "6.0.3"
   },
   "devDependencies": {
@@ -77,8 +82,10 @@
     "@types/ejs": "3.1.5",
     "@types/jest": "30.0.0",
     "@types/js-yaml": "4.0.9",
+    "@types/jsdom": "28.0.1",
     "@types/lodash": "4.17.24",
     "@types/node": "25.6.0",
+    "@types/turndown": "5.0.6",
     "@typescript-eslint/eslint-plugin": "8.58.2",
     "@typescript-eslint/parser": "8.58.2",
     "ejs": "5.0.2",