@staticn0va/wigolo 0.1.0

package/dist/crawl/rate-limiter.js

@@ -0,0 +1,72 @@
+import { isPrivateUrl } from './url-utils.js';
+import { getConfig } from '../config.js';
+export class RateLimiter {
+    domains = new Map();
+    robotsDelays = new Map();
+    setRobotsCrawlDelay(domain, delaySeconds) {
+        this.robotsDelays.set(domain, delaySeconds * 1000);
+    }
+    async acquire(url) {
+        const domain = new URL(url).hostname;
+        const state = this.getOrCreateState(url, domain);
+        if (state.activeCount < state.maxConcurrency) {
+            // Enforce delay even when under concurrency limit
+            const elapsed = Date.now() - state.lastRequestTime;
+            const remaining = state.delayMs - elapsed;
+            if (remaining > 0 && state.lastRequestTime > 0) {
+                await new Promise((r) => setTimeout(r, remaining));
+            }
+            return this.startRequest(state);
+        }
+        // Wait in queue
+        return new Promise((resolve) => {
+            state.queue.push(() => resolve(this.startRequest(state)));
+        });
+    }
+    getOrCreateState(url, domain) {
+        if (!this.domains.has(domain)) {
+            const config = getConfig();
+            const isPrivate = isPrivateUrl(url);
+            const configDelay = isPrivate ? config.crawlPrivateDelayMs : config.crawlDelayMs;
+            // Use robots.txt delay if it's higher than configured delay
+            const robotsDelay = this.robotsDelays.get(domain) ?? 0;
+            const effectiveDelay = Math.max(configDelay, robotsDelay);
+            this.domains.set(domain, {
+                activeCount: 0,
+                lastRequestTime: 0,
+                queue: [],
+                maxConcurrency: isPrivate ? config.crawlPrivateConcurrency : config.crawlConcurrency,
+                delayMs: effectiveDelay,
+            });
+        }
+        const state = this.domains.get(domain);
+        // Update delay if robots delay was set after state creation
+        const robotsDelay = this.robotsDelays.get(domain);
+        if (robotsDelay !== undefined && robotsDelay > state.delayMs) {
+            state.delayMs = robotsDelay;
+        }
+        return state;
+    }
+    startRequest(state) {
+        state.activeCount++;
+        state.lastRequestTime = Date.now();
+        return () => {
+            state.activeCount--;
+            this.processQueue(state);
+        };
+    }
+    processQueue(state) {
+        if (state.queue.length === 0 || state.activeCount >= state.maxConcurrency)
+            return;
+        const next = state.queue.shift();
+        const elapsed = Date.now() - state.lastRequestTime;
+        const remaining = state.delayMs - elapsed;
+        if (remaining <= 0) {
+            next();
+        }
+        else {
+            setTimeout(next, remaining);
+        }
+    }
+}
+//# sourceMappingURL=rate-limiter.js.map

package/dist/crawl/rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../src/crawl/rate-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAUzC,MAAM,OAAO,WAAW;IACd,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IACzC,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEjD,mBAAmB,CAAC,MAAc,EAAE,YAAoB;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACvB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEjD,IAAI,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;YAC7C,kDAAkD;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC;YACnD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;YAC1C,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAED,gBAAgB;QAChB,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,gBAAgB,CAAC,GAAW,EAAE,MAAc;QAClD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;YAEjF,4DAA4D;YAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YAE1D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;gBACvB,WAAW,EAAE,CAAC;gBACd,eAAe,EAAE,CAAC;gBAClB,KAAK,EAAE,EAAE;gBACT,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB;gBACpF,OAAO,EAAE,cAAc;aACxB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;QACxC,4DAA4D;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC7D,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,YAAY,CAAC,KAAkB;QACrC,KAAK,CAAC,WAAW,EAAE,CAAC;QACpB,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,OAAO,GAAG,EAAE;YACV,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,KAAkB;QACrC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,cAAc;YAAE,OAAO;QAElF,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC;QACnD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;QAE1C,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,IAAI,EAAE,CAAC;QACT,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;CACF"}

package/dist/crawl/robots.d.ts

@@ -0,0 +1,9 @@
+export declare class RobotsParser {
+    private rules;
+    private crawlDelay;
+    constructor(robotsTxt: string);
+    private parse;
+    isAllowed(path: string): boolean;
+    getCrawlDelay(): number | null;
+}
+//# sourceMappingURL=robots.d.ts.map

package/dist/crawl/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/crawl/robots.ts"],"names":[],"mappings":"AAKA,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,UAAU,CAAuB;gBAE7B,SAAS,EAAE,MAAM;IAI7B,OAAO,CAAC,KAAK;IA4Cb,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAiBhC,aAAa,IAAI,MAAM,GAAG,IAAI;CAG/B"}

package/dist/crawl/robots.js

@@ -0,0 +1,63 @@
+export class RobotsParser {
+    rules = [];
+    crawlDelay = null;
+    constructor(robotsTxt) {
+        this.parse(robotsTxt);
+    }
+    parse(text) {
+        const lines = text.split('\n');
+        let inWildcardAgent = false;
+        for (const rawLine of lines) {
+            const line = rawLine.trim();
+            if (line.match(/^user-agent:\s*\*/i)) {
+                inWildcardAgent = true;
+                continue;
+            }
+            if (line.match(/^user-agent:/i) && !line.match(/^user-agent:\s*\*/i)) {
+                inWildcardAgent = false;
+                continue;
+            }
+            if (!inWildcardAgent)
+                continue;
+            const disallowMatch = line.match(/^disallow:\s*(.*)/i);
+            if (disallowMatch) {
+                const path = disallowMatch[1].trim();
+                if (path) {
+                    this.rules.push({ type: 'disallow', path });
+                }
+                continue;
+            }
+            const allowMatch = line.match(/^allow:\s*(.*)/i);
+            if (allowMatch) {
+                const path = allowMatch[1].trim();
+                if (path) {
+                    this.rules.push({ type: 'allow', path });
+                }
+                continue;
+            }
+            const delayMatch = line.match(/^crawl-delay:\s*(\d+(?:\.\d+)?)/i);
+            if (delayMatch) {
+                this.crawlDelay = parseFloat(delayMatch[1]);
+            }
+        }
+    }
+    isAllowed(path) {
+        let bestMatch = null;
+        let bestLength = -1;
+        for (const rule of this.rules) {
+            if (path.startsWith(rule.path)) {
+                if (rule.path.length > bestLength || (rule.path.length === bestLength && rule.type === 'allow')) {
+                    bestMatch = rule;
+                    bestLength = rule.path.length;
+                }
+            }
+        }
+        if (!bestMatch)
+            return true;
+        return bestMatch.type === 'allow';
+    }
+    getCrawlDelay() {
+        return this.crawlDelay;
+    }
+}
+//# sourceMappingURL=robots.js.map

package/dist/crawl/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/crawl/robots.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,YAAY;IACf,KAAK,GAAiB,EAAE,CAAC;IACzB,UAAU,GAAkB,IAAI,CAAC;IAEzC,YAAY,SAAiB;QAC3B,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,IAAY;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,eAAe,GAAG,KAAK,CAAC;QAE5B,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAI,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACrC,eAAe,GAAG,IAAI,CAAC;gBACvB,SAAS;YACX,CAAC;YAED,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACrE,eAAe,GAAG,KAAK,CAAC;gBACxB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,eAAe;gBAAE,SAAS;YAE/B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACvD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC9C,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACjD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAClC,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3C,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAClE,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,IAAI,SAAS,GAAsB,IAAI,CAAC;QACxC,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;QAEpB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,EAAE,CAAC;oBAChG,SAAS,GAAG,IAAI,CAAC;oBACjB,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;gBAChC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,SAAS,CAAC,IAAI,KAAK,OAAO,CAAC;IACpC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF"}

package/dist/crawl/sitemap.d.ts

@@ -0,0 +1,4 @@
+export declare function parseSitemap(xml: string): string[];
+export declare function parseSitemapIndex(xml: string): string[];
+export declare function extractSitemapUrlFromRobots(robotsTxt: string): string[];
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/crawl/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/crawl/sitemap.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAgBlD;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CASvD;AAED,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAYvE"}

package/dist/crawl/sitemap.js

@@ -0,0 +1,38 @@
+export function parseSitemap(xml) {
+    // A sitemapindex document should be parsed with parseSitemapIndex, not here
+    if (xml.includes('<sitemapindex'))
+        return [];
+    if (!xml.includes('<urlset') && !xml.includes('<loc>'))
+        return [];
+    const urls = [];
+    const locMatches = xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g);
+    for (const match of locMatches) {
+        const url = match[1].trim();
+        if (url) {
+            urls.push(url);
+        }
+    }
+    return urls;
+}
+export function parseSitemapIndex(xml) {
+    if (!xml.includes('<sitemapindex'))
+        return [];
+    const urls = [];
+    const locMatches = xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g);
+    for (const match of locMatches) {
+        urls.push(match[1].trim());
+    }
+    return urls;
+}
+export function extractSitemapUrlFromRobots(robotsTxt) {
+    const urls = [];
+    const lines = robotsTxt.split('\n');
+    for (const line of lines) {
+        const match = line.match(/^sitemap:\s*(.+)/i);
+        if (match) {
+            urls.push(match[1].trim());
+        }
+    }
+    return urls;
+}
+//# sourceMappingURL=sitemap.js.map

package/dist/crawl/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/crawl/sitemap.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,4EAA4E;IAC5E,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IAElE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;IAC/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAW;IAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC;QAAE,OAAO,EAAE,CAAC;IAE9C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;IAC/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,SAAiB;IAC3D,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC9C,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/crawl/url-utils.d.ts

@@ -0,0 +1,3 @@
+export declare function isPrivateUrl(url: string): boolean;
+export declare function matchesPatterns(url: string, includePatterns: string[] | undefined, excludePatterns: string[] | undefined): boolean;
+//# sourceMappingURL=url-utils.d.ts.map

package/dist/crawl/url-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-utils.d.ts","sourceRoot":"","sources":["../../src/crawl/url-utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAgCjD;AAED,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,MAAM,EAAE,GAAG,SAAS,EACrC,eAAe,EAAE,MAAM,EAAE,GAAG,SAAS,GACpC,OAAO,CAYT"}

package/dist/crawl/url-utils.js

@@ -0,0 +1,41 @@
+export function isPrivateUrl(url) {
+    const parsed = new URL(url);
+    const hostname = parsed.hostname.replace(/^\[|\]$/g, ''); // strip IPv6 brackets
+    if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1' || hostname === '0.0.0.0') {
+        return true;
+    }
+    if (hostname.endsWith('.local')) {
+        return true;
+    }
+    // 10.x.x.x
+    if (hostname.startsWith('10.')) {
+        return true;
+    }
+    // 192.168.x.x
+    if (hostname.startsWith('192.168.')) {
+        return true;
+    }
+    // 172.16.0.0/12 (172.16.x.x – 172.31.x.x)
+    if (hostname.startsWith('172.')) {
+        const parts = hostname.split('.');
+        const second = parseInt(parts[1], 10);
+        if (second >= 16 && second <= 31) {
+            return true;
+        }
+    }
+    return false;
+}
+export function matchesPatterns(url, includePatterns, excludePatterns) {
+    if (includePatterns && includePatterns.length > 0) {
+        const matches = includePatterns.some((p) => new RegExp(p).test(url));
+        if (!matches)
+            return false;
+    }
+    if (excludePatterns && excludePatterns.length > 0) {
+        const excluded = excludePatterns.some((p) => new RegExp(p).test(url));
+        if (excluded)
+            return false;
+    }
+    return true;
+}
+//# sourceMappingURL=url-utils.js.map

package/dist/crawl/url-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-utils.js","sourceRoot":"","sources":["../../src/crawl/url-utils.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB;IAEhF,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzG,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW;IACX,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc;IACd,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,eAAqC,EACrC,eAAqC;IAErC,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,IAAI,QAAQ;YAAE,OAAO,KAAK,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/extraction/defuddle.d.ts

@@ -0,0 +1,3 @@
+import type { ExtractionResult } from '../types.js';
+export declare function defuddleExtract(html: string, url: string): Promise<ExtractionResult | null>;
+//# sourceMappingURL=defuddle.d.ts.map

package/dist/extraction/defuddle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"defuddle.d.ts","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAoBjG"}

package/dist/extraction/defuddle.js

@@ -0,0 +1,26 @@
+import { Defuddle } from 'defuddle/node';
+const MIN_CONTENT_THRESHOLD = 100;
+export async function defuddleExtract(html, url) {
+    try {
+        const result = await Defuddle(html, url, { markdown: true });
+        if (!result.content || result.content.length < MIN_CONTENT_THRESHOLD)
+            return null;
+        return {
+            title: result.title ?? '',
+            markdown: result.content,
+            metadata: {
+                description: result.description || undefined,
+                author: result.author || undefined,
+                date: result.published || undefined,
+                language: result.language || undefined,
+            },
+            links: [],
+            images: [],
+            extractor: 'defuddle',
+        };
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=defuddle.js.map

package/dist/extraction/defuddle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defuddle.js","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAClF,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;YACzB,QAAQ,EAAE,MAAM,CAAC,OAAO;YACxB,QAAQ,EAAE;gBACR,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,SAAS;gBAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;gBAClC,IAAI,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;gBACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;aACvC;YACD,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,UAAU;SACtB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/extraction/extract.d.ts

@@ -0,0 +1,5 @@
+import type { MetadataData, TableData } from '../types.js';
+export declare function extractMetadata(html: string): MetadataData;
+export declare function extractSelector(html: string, selector: string, multiple: boolean): string | string[];
+export declare function extractTables(html: string): TableData[];
+//# sourceMappingURL=extract.d.ts.map

package/dist/extraction/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/extraction/extract.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAS3D,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CA2B1D;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,OAAO,GAChB,MAAM,GAAG,MAAM,EAAE,CAUnB;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,CA6CvD"}

package/dist/extraction/extract.js

@@ -0,0 +1,83 @@
+import { parseHTML } from 'linkedom';
+function getMetaContent(doc, nameOrProperty) {
+    const el = doc.querySelector(`meta[name="${nameOrProperty}"]`) ??
+        doc.querySelector(`meta[property="${nameOrProperty}"]`);
+    return el?.getAttribute('content') ?? undefined;
+}
+export function extractMetadata(html) {
+    const { document: doc } = parseHTML(html);
+    const result = {};
+    const title = doc.querySelector('title')?.textContent?.trim();
+    if (title)
+        result.title = title;
+    const description = getMetaContent(doc, 'description') ?? getMetaContent(doc, 'og:description');
+    if (description)
+        result.description = description;
+    const author = getMetaContent(doc, 'author');
+    if (author)
+        result.author = author;
+    const date = getMetaContent(doc, 'date') ?? getMetaContent(doc, 'article:published_time');
+    if (date)
+        result.date = date;
+    const keywords = getMetaContent(doc, 'keywords');
+    if (keywords) {
+        result.keywords = keywords.split(',').map((k) => k.trim()).filter(Boolean);
+    }
+    const ogImage = getMetaContent(doc, 'og:image');
+    if (ogImage)
+        result.og_image = ogImage;
+    return result;
+}
+export function extractSelector(html, selector, multiple) {
+    const { document: doc } = parseHTML(html);
+    if (multiple) {
+        const elements = doc.querySelectorAll(selector);
+        return Array.from(elements).map((el) => (el.textContent ?? '').trim());
+    }
+    const el = doc.querySelector(selector);
+    return el ? (el.textContent ?? '').trim() : '';
+}
+export function extractTables(html) {
+    const { document: doc } = parseHTML(html);
+    const tables = doc.querySelectorAll('table');
+    if (tables.length === 0)
+        return [];
+    return Array.from(tables).map((table) => {
+        const caption = table.querySelector('caption')?.textContent?.trim() || undefined;
+        const thElements = table.querySelectorAll('thead th');
+        let headers;
+        let bodyRows;
+        if (thElements.length > 0) {
+            headers = Array.from(thElements).map((th) => (th.textContent ?? '').trim());
+            bodyRows = Array.from(table.querySelectorAll('tbody tr'));
+            if (bodyRows.length === 0) {
+                const allRows = Array.from(table.querySelectorAll('tr'));
+                bodyRows = allRows.slice(1);
+            }
+        }
+        else {
+            const allRows = Array.from(table.querySelectorAll('tr'));
+            const firstRow = allRows[0];
+            const firstRowThs = firstRow ? Array.from(firstRow.querySelectorAll('th')) : [];
+            if (firstRowThs.length > 0) {
+                headers = firstRowThs.map((th) => (th.textContent ?? '').trim());
+                bodyRows = allRows.slice(1);
+            }
+            else {
+                const cellCount = firstRow ? firstRow.querySelectorAll('td').length : 0;
+                headers = Array.from({ length: cellCount }, (_, i) => `col_${i + 1}`);
+                bodyRows = allRows;
+            }
+        }
+        const rows = bodyRows.map((row) => {
+            const cells = Array.from(row.querySelectorAll('td'));
+            const obj = {};
+            headers.forEach((header, i) => {
+                obj[header] = (cells[i]?.textContent ?? '').trim();
+            });
+            return obj;
+        });
+        return { caption, headers, rows };
+    });
+}
+//# sourceMappingURL=extract.js.map

package/dist/extraction/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/extraction/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC,SAAS,cAAc,CAAC,GAAa,EAAE,cAAsB;IAC3D,MAAM,EAAE,GACN,GAAG,CAAC,aAAa,CAAC,cAAc,cAAc,IAAI,CAAC;QACnD,GAAG,CAAC,aAAa,CAAC,kBAAkB,cAAc,IAAI,CAAC,CAAC;IAC1D,OAAO,EAAE,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC9D,IAAI,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IAEhC,MAAM,WAAW,GACf,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,IAAI,cAAc,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAC9E,IAAI,WAAW;QAAE,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IAElD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC7C,IAAI,MAAM;QAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;IAEnC,MAAM,IAAI,GACR,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,cAAc,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC;IAC/E,IAAI,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;IAE7B,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACjD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAChD,IAAI,OAAO;QAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;IAEvC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,QAAgB,EAChB,QAAiB;IAEjB,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAE1C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,EAAE,GAAG,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QAEjF,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,OAAiB,CAAC;QACtB,IAAI,QAAmB,CAAC;QAExB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5E,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;YAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;gBACzD,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAEhF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjE,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxE,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtE,QAAQ,GAAG,OAAO,CAAC;YACrB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YACrD,MAAM,GAAG,GAA2B,EAAE,CAAC;YACvC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrD,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/extraction/jsonld.d.ts

@@ -0,0 +1,4 @@
+import type { JsonSchema } from './schema.js';
+export declare function extractJsonLd(html: string): Record<string, unknown>[];
+export declare function matchJsonLdToSchema(jsonLdBlocks: Record<string, unknown>[], schema: JsonSchema): Record<string, unknown>;
+//# sourceMappingURL=jsonld.d.ts.map

package/dist/extraction/jsonld.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonld.d.ts","sourceRoot":"","sources":["../../src/extraction/jsonld.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAyBrE;AAED,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EACvC,MAAM,EAAE,UAAU,GACjB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAazB"}

package/dist/extraction/jsonld.js

@@ -0,0 +1,64 @@
+import { parseHTML } from 'linkedom';
+import { createLogger } from '../logger.js';
+const log = createLogger('jsonld');
+export function extractJsonLd(html) {
+    const { document: doc } = parseHTML(html);
+    const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
+    const results = [];
+    for (const script of scripts) {
+        try {
+            const text = script.textContent?.trim();
+            if (!text)
+                continue;
+            const parsed = JSON.parse(text);
+            if (Array.isArray(parsed)) {
+                results.push(...parsed);
+            }
+            else if (parsed['@graph'] && Array.isArray(parsed['@graph'])) {
+                results.push(...parsed['@graph']);
+            }
+            else {
+                results.push(parsed);
+            }
+        }
+        catch (err) {
+            log.debug('Failed to parse JSON-LD block', { error: String(err) });
+        }
+    }
+    return results;
+}
+export function matchJsonLdToSchema(jsonLdBlocks, schema) {
+    if (!schema.properties || jsonLdBlocks.length === 0)
+        return {};
+    const result = {};
+    const flattened = flattenJsonLd(jsonLdBlocks);
+    for (const fieldName of Object.keys(schema.properties)) {
+        if (flattened[fieldName] !== undefined) {
+            result[fieldName] = flattened[fieldName];
+        }
+    }
+    return result;
+}
+function flattenJsonLd(blocks) {
+    const flat = {};
+    for (const block of blocks) {
+        flattenObject(block, flat);
+    }
+    return flat;
+}
+function flattenObject(obj, target) {
+    for (const [key, value] of Object.entries(obj)) {
+        if (key.startsWith('@'))
+            continue;
+        // First-wins: earlier blocks and shallower keys take priority
+        if (!(key in target)) {
+            if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+                flattenObject(value, target);
+            }
+            else {
+                target[key] = value;
+            }
+        }
+    }
+}
+//# sourceMappingURL=jsonld.js.map

package/dist/extraction/jsonld.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonld.js","sourceRoot":"","sources":["../../src/extraction/jsonld.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAA8B,EAAE,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,YAAuC,EACvC,MAAkB;IAElB,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE/D,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAE9C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CACpB,MAAiC;IAEjC,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,GAA4B,EAC5B,MAA+B;IAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElC,8DAA8D;QAC9D,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC;YACrB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzE,aAAa,CAAC,KAAgC,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/extraction/markdown.d.ts

@@ -0,0 +1,10 @@
+export declare function htmlToMarkdown(html: string): string;
+export declare function extractSection(markdown: string, section: string, sectionIndex?: number): {
+    content: string;
+    matched: boolean;
+};
+export declare function extractLinksAndImages(markdown: string): {
+    links: string[];
+    images: string[];
+};
+//# sourceMappingURL=markdown.d.ts.map

package/dist/extraction/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/extraction/markdown.ts"],"names":[],"mappings":"AAkDA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAmCD,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,SAAI,GACf;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CA2BvC;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAoB7F"}

package/dist/extraction/markdown.js

@@ -0,0 +1,107 @@
+import TurndownService from 'turndown';
+function buildTurndown() {
+    const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+    // Remove script and style tags entirely
+    td.remove(['script', 'style']);
+    // Custom rule: convert <table> to markdown table
+    td.addRule('table', {
+        filter: 'table',
+        replacement(_content, node) {
+            const el = node;
+            const rows = Array.from(el.querySelectorAll('tr'));
+            if (rows.length === 0)
+                return '';
+            const renderRow = (row) => {
+                const cells = Array.from(row.querySelectorAll('th, td'));
+                return '| ' + cells.map(c => c.textContent?.replace(/\n/g, ' ').trim() ?? '').join(' | ') + ' |';
+            };
+            const headerRow = rows[0];
+            const isHeaderRow = headerRow.querySelectorAll('th').length > 0;
+            const headerCells = Array.from(headerRow.querySelectorAll('th, td'));
+            const separator = '| ' + headerCells.map(() => '---').join(' | ') + ' |';
+            if (isHeaderRow) {
+                const bodyRows = rows.slice(1);
+                const lines = [renderRow(headerRow), separator, ...bodyRows.map(renderRow)];
+                return '\n\n' + lines.join('\n') + '\n\n';
+            }
+            const lines = [renderRow(headerRow), separator, ...rows.slice(1).map(renderRow)];
+            return '\n\n' + lines.join('\n') + '\n\n';
+        },
+    });
+    // Suppress thead/tbody/tr/th/td individually since table rule handles the whole node
+    td.addRule('tableCell', {
+        filter: ['thead', 'tbody', 'tfoot', 'tr', 'th', 'td'],
+        replacement(content) {
+            return content;
+        },
+    });
+    return td;
+}
+const turndown = buildTurndown();
+export function htmlToMarkdown(html) {
+    if (!html)
+        return '';
+    return turndown.turndown(html);
+}
+function parseHeadings(lines) {
+    const headings = [];
+    for (let i = 0; i < lines.length; i++) {
+        const match = lines[i].match(/^(#{1,6})\s+(.+)/);
+        if (match) {
+            headings.push({ level: match[1].length, text: match[2].trim(), lineIndex: i });
+        }
+    }
+    return headings;
+}
+function extractFromHeading(lines, headings, headingIdx) {
+    const heading = headings[headingIdx];
+    const start = heading.lineIndex;
+    // Find the next heading of equal or higher level (lower or equal # count)
+    let end = lines.length;
+    for (let i = headingIdx + 1; i < headings.length; i++) {
+        if (headings[i].level <= heading.level) {
+            end = headings[i].lineIndex;
+            break;
+        }
+    }
+    return lines.slice(start, end).join('\n');
+}
+export function extractSection(markdown, section, sectionIndex = 0) {
+    const lines = markdown.split('\n');
+    const headings = parseHeadings(lines);
+    if (headings.length === 0)
+        return { content: markdown, matched: false };
+    const lower = section.toLowerCase();
+    const indexed = headings.map((h, i) => ({ h, i }));
+    // Collect exact matches first
+    const exactMatches = indexed.filter(({ h }) => h.text.toLowerCase() === lower);
+    // If exact matches satisfy the requested index, use them
+    if (exactMatches.length > 0 && sectionIndex < exactMatches.length) {
+        const { i } = exactMatches[sectionIndex];
+        return { content: extractFromHeading(lines, headings, i), matched: true };
+    }
+    // Fall back to substring matches (includes exact headings and partial ones)
+    const substringMatches = indexed.filter(({ h }) => h.text.toLowerCase().includes(lower));
+    if (substringMatches.length === 0 || sectionIndex >= substringMatches.length) {
+        return { content: markdown, matched: false };
+    }
+    const { i } = substringMatches[sectionIndex];
+    return { content: extractFromHeading(lines, headings, i), matched: true };
+}
+export function extractLinksAndImages(markdown) {
+    const imagePattern = /!\[[^\]]*\]\(([^)]+)\)/g;
+    const linkPattern = /(?<!!)\[[^\]]*\]\(([^)]+)\)/g;
+    const images = new Set();
+    const links = new Set();
+    let match;
+    // Extract images first
+    while ((match = imagePattern.exec(markdown)) !== null) {
+        images.add(match[1]);
+    }
+    // Extract links (non-image)
+    while ((match = linkPattern.exec(markdown)) !== null) {
+        links.add(match[1]);
+    }
+    return { links: Array.from(links), images: Array.from(images) };
+}
+//# sourceMappingURL=markdown.js.map

package/dist/extraction/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/extraction/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,SAAS,aAAa;IACpB,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,wCAAwC;IACxC,EAAE,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IAE/B,iDAAiD;IACjD,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAClB,MAAM,EAAE,OAAO;QACf,WAAW,CAAC,QAAQ,EAAE,IAAI;YACxB,MAAM,EAAE,GAAG,IAAe,CAAC;YAC3B,MAAM,IAAI,GAAc,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAEjC,MAAM,SAAS,GAAG,CAAC,GAAY,EAAU,EAAE;gBACzC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACzD,OAAO,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACnG,CAAC,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,WAAW,GAAG,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;YACrE,MAAM,SAAS,GAAG,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YAEzE,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,KAAK,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC5E,OAAO,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;YAC5C,CAAC;YAED,MAAM,KAAK,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;YACjF,OAAO,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;QAC5C,CAAC;KACF,CAAC,CAAC;IAEH,qFAAqF;IACrF,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;QACrD,WAAW,CAAC,OAAO;YACjB,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC;AAEjC,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAQD,SAAS,aAAa,CAAC,KAAe;IACpC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACjD,IAAI,KAAK,EAAE,CAAC;YACV,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAe,EAAE,QAAmB,EAAE,UAAkB;IAClF,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC;IAEhC,0EAA0E;IAC1E,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,KAAK,IAAI,CAAC,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACvC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5B,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,QAAgB,EAChB,OAAe,EACf,YAAY,GAAG,CAAC;IAEhB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAExE,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEnD,8BAA8B;IAC9B,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,CAAC;IAE/E,yDAAyD;IACzD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;QACzC,OAAO,EAAE,OAAO,EAAE,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC5E,CAAC;IAED,4EAA4E;IAC5E,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAEzF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC7E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC/C,CAAC;IAED,MAAM,EAAE,CAAC,EAAE,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAC7C,OAAO,EAAE,OAAO,EAAE,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,YAAY,GAAG,yBAAyB,CAAC;IAC/C,MAAM,WAAW,GAAG,8BAA8B,CAAC;IAEnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,IAAI,KAA6B,CAAC;IAElC,uBAAuB;IACvB,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,4BAA4B;IAC5B,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrD,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;AAClE,CAAC"}