@dyyz1993/agent-browser 0.26.3 → 0.26.4

package/dist/actions/crawl.d.ts

@@ -2,7 +2,7 @@ import type { Page } from 'playwright-core';
 import type { BrowserManager } from '../browser/index.js';
 import type { CrawlCommand, CrawlResult, Response } from '../types.js';
 export declare function handleCrawl(command: CrawlCommand, browser: BrowserManager): Promise<Response<CrawlResult>>;
-export declare function discoverLinks(page: Page, baseOrigin: string, baseHostname: string, basePath: string): Promise<string[]>;
+export declare function discoverLinks(page: Page, baseOrigin: string, baseHostname: string, basePath: string, allowExternal?: boolean): Promise<string[]>;
 export declare function normalizeUrl(url: string): string;
 export declare function normalizeUrlFromUrl(u: URL): string;
 //# sourceMappingURL=crawl.d.ts.map

package/dist/actions/crawl.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/actions/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAa,WAAW,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAwGlF,wBAAsB,WAAW,CAC/B,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CA2GhC;AA4DD,wBAAsB,aAAa,CACjC,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAiDnB;AAyCD,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,GAAG,MAAM,CAelD"}
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/actions/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAa,WAAW,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AA+MlF,wBAAsB,WAAW,CAC/B,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAuLhC;AA4GD,wBAAsB,aAAa,CACjC,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,EAChB,aAAa,GAAE,OAAe,GAC7B,OAAO,CAAC,MAAM,EAAE,CAAC,CAoDnB;AAyCD,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,GAAG,MAAM,CAelD"}

package/dist/actions/crawl.js

@@ -5,31 +5,63 @@ const STATIC_EXTENSIONS = [
     '.jpg',
     '.jpeg',
     '.gif',
-    '.webp',
     '.svg',
     '.ico',
-    '.css',
-    '.js',
+    '.webp',
+    '.avif',
+    '.bmp',
+    '.tiff',
+    '.mp3',
+    '.mp4',
+    '.avi',
+    '.mov',
+    '.wmv',
+    '.flv',
+    '.webm',
+    '.ogg',
+    '.wav',
     '.woff',
     '.woff2',
     '.ttf',
     '.eot',
-    '.zip',
-    '.tar',
-    '.gz',
-    '.rar',
+    '.otf',
+    '.css',
+    '.js',
+    '.mjs',
     '.pdf',
     '.doc',
     '.docx',
     '.xls',
     '.xlsx',
-    '.mp3',
-    '.mp4',
-    '.avi',
-    '.mov',
+    '.ppt',
+    '.pptx',
+    '.odt',
+    '.ods',
+    '.odp',
+    '.rtf',
+    '.zip',
+    '.gz',
+    '.tar',
+    '.rar',
+    '.7z',
+    '.bz2',
+    '.exe',
+    '.dmg',
+    '.deb',
+    '.rpm',
+    '.msi',
+    '.xml',
+    '.json',
+    '.yaml',
+    '.yml',
+    '.csv',
+    '.bin',
+    '.iso',
+    '.img',
+    '.apk',
+    '.ipa',
     '.rss',
     '.atom',
-    '.xml',
 ];
 const SOCIAL_DOMAINS = [
     'facebook.com',
@@ -91,6 +123,71 @@ function filterUrlByPatterns(url, excludePatterns, includePatterns) {
     }
     return true;
 }
+async function fetchRobotsTxt(origin) {
+    try {
+        const res = await fetch(`${origin}/robots.txt`, {
+            signal: AbortSignal.timeout(5000),
+        });
+        return res.ok ? await res.text() : '';
+    }
+    catch {
+        return '';
+    }
+}
+function parseRobotsTxt(robotsTxt, userAgent = '*') {
+    const allowed = [];
+    const disallowed = [];
+    let crawlDelay;
+    const lines = robotsTxt.split('\n').map((l) => l.trim());
+    let matchAgent = false;
+    for (const line of lines) {
+        if (line.startsWith('#') || !line)
+            continue;
+        const colonIdx = line.indexOf(':');
+        if (colonIdx === -1)
+            continue;
+        const key = line.slice(0, colonIdx).trim().toLowerCase();
+        const value = line.slice(colonIdx + 1).trim();
+        if (key === 'user-agent') {
+            matchAgent = value === '*' || value.toLowerCase() === (userAgent || '*').toLowerCase();
+        }
+        else if (matchAgent) {
+            if (key === 'disallow' && value) {
+                disallowed.push(value);
+            }
+            else if (key === 'allow' && value) {
+                allowed.push(value);
+            }
+            else if (key === 'crawl-delay' && value) {
+                crawlDelay = parseFloat(value);
+            }
+        }
+    }
+    return { allowed, disallowed, crawlDelay };
+}
+function matchesRobotsPattern(pathname, pattern) {
+    const regex = pattern
+        .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+        .replace(/\*/g, '.*')
+        .replace(/\$$/, '');
+    try {
+        return new RegExp(`^${regex}`).test(pathname);
+    }
+    catch {
+        return false;
+    }
+}
+function isAllowedByRobots(pathname, rules) {
+    for (const pattern of rules.allowed) {
+        if (matchesRobotsPattern(pathname, pattern))
+            return true;
+    }
+    for (const pattern of rules.disallowed) {
+        if (matchesRobotsPattern(pathname, pattern))
+            return false;
+    }
+    return true;
+}
 export async function handleCrawl(command, browser) {
     const page = browser.getPage();
     if (!page) {
@@ -107,16 +204,73 @@ export async function handleCrawl(command, browser) {
     const allowExternal = command.allowExternal ?? false;
     const excludePatterns = command.excludePatterns;
     const includePatterns = command.includePatterns;
+    const concurrency = command.concurrency ?? 1;
+    const context = page.context();
+    if (command.cookies && command.cookies.length > 0) {
+        await context.addCookies(command.cookies);
+    }
     const startUrl = normalizeUrl(command.url);
     const parsedStart = new URL(startUrl);
     const baseOrigin = parsedStart.origin;
     const baseHostname = parsedStart.hostname.replace(/^www\./, '');
     const basePath = parsedStart.pathname.replace(/\/$/, '');
+    const robotsTxt = await fetchRobotsTxt(baseOrigin);
+    const robotsRules = parseRobotsTxt(robotsTxt);
+    const crawlDelay = robotsRules.crawlDelay ? robotsRules.crawlDelay * 1000 : 0;
     const visited = new Set();
     const pages = [];
     const pageUrls = new Set();
     let failed = 0;
     const queue = [{ url: startUrl, depth: 0, priority: 0 }];
+    function generateUrlPermutations(url) {
+        try {
+            const u = new URL(url);
+            const variants = [u.href];
+            if (u.hostname.startsWith('www.')) {
+                const stripped = new URL(u.href);
+                stripped.hostname = u.hostname.slice(4);
+                variants.push(stripped.href);
+            }
+            else {
+                const withWWW = new URL(u.href);
+                withWWW.hostname = 'www.' + u.hostname;
+                variants.push(withWWW.href);
+            }
+            if (u.pathname.endsWith('/')) {
+                const noSlash = new URL(u.href);
+                noSlash.pathname = noSlash.pathname.replace(/\/$/, '') || '/';
+                variants.push(noSlash.href);
+            }
+            else {
+                const withSlash = new URL(u.href);
+                withSlash.pathname += '/';
+                variants.push(withSlash.href);
+            }
+            const indexPattern = /\/(index\.(html|htm|php|aspx?))$/i;
+            if (indexPattern.test(u.pathname)) {
+                const noIndex = new URL(u.href);
+                noIndex.pathname = u.pathname.replace(indexPattern, '/') || '/';
+                variants.push(noIndex.href);
+            }
+            if (u.hash && !u.hash.startsWith('#/') && !u.hash.startsWith('#!')) {
+                const noHash = new URL(u.href);
+                noHash.hash = '';
+                variants.push(noHash.href);
+            }
+            return [...new Set(variants)];
+        }
+        catch {
+            return [url];
+        }
+    }
+    function isVisited(url) {
+        return generateUrlPermutations(url).some((perm) => visited.has(perm));
+    }
+    function markVisited(url) {
+        for (const perm of generateUrlPermutations(url)) {
+            visited.add(perm);
+        }
+    }
     function urlPriority(url) {
         for (const p of LOW_VALUE_PATTERNS) {
             if (p.test(url))
@@ -127,15 +281,17 @@ export async function handleCrawl(command, browser) {
     while (queue.length > 0 && pages.length < maxPages) {
         queue.sort((a, b) => a.priority - b.priority || a.depth - b.depth);
         const batch = [];
-        while (queue.length > 0 && batch.length < 1 && pages.length + batch.length < maxPages) {
+        while (queue.length > 0 &&
+            batch.length < concurrency &&
+            pages.length + batch.length < maxPages) {
             const entry = queue.shift();
             const normalized = normalizeUrl(entry.url);
-            if (visited.has(normalized))
+            if (isVisited(normalized))
                 continue;
-            visited.add(normalized);
+            markVisited(normalized);
             batch.push({ ...entry, url: normalized });
         }
-        const results = await Promise.allSettled(batch.map((entry) => crawlPage(page, entry.url, baseOrigin, baseHostname, basePath, format, command.selector, timeoutMs, allowExternal, excludePatterns, includePatterns)));
+        const results = await Promise.allSettled(batch.map((entry) => crawlPage(browser, page, entry.url, baseOrigin, baseHostname, basePath, format, command.selector, timeoutMs, allowExternal, excludePatterns, includePatterns, command.javaScriptEnabled, robotsRules, crawlDelay)));
         for (let i = 0; i < results.length; i++) {
             const result = results[i];
             const entry = batch[i];
@@ -145,11 +301,15 @@ export async function handleCrawl(command, browser) {
                 if (pageUrls.has(finalUrl))
                     continue;
                 pageUrls.add(finalUrl);
+                markVisited(finalUrl);
                 pages.push(crawlPageData);
+                if (crawlDelay > 0) {
+                    await new Promise((resolve) => setTimeout(resolve, crawlDelay));
+                }
                 if (entry.depth < maxDepth) {
                     for (const link of crawlPageData.links || []) {
                         const normalized = normalizeUrl(link);
-                        if (visited.has(normalized))
+                        if (isVisited(normalized))
                             continue;
                         if (!isAllowedUrl(normalized, baseOrigin, baseHostname, basePath, allowExternal))
                             continue;
@@ -178,9 +338,41 @@ export async function handleCrawl(command, browser) {
         failed,
     });
 }
-async function crawlPage(page, url, baseOrigin, baseHostname, basePath, format, selector, timeoutMs = 15000, allowExternal = false, excludePatterns, includePatterns) {
+async function crawlPage(browser, mainPage, url, baseOrigin, baseHostname, basePath, format, selector, timeoutMs = 15000, allowExternal = false, excludePatterns, includePatterns, javaScriptEnabled, robotsRules, crawlDelay) {
+    let page = mainPage;
+    let disposable = false;
+    if (javaScriptEnabled === false) {
+        const browserInstance = browser.getBrowser();
+        if (browserInstance) {
+            page = await browserInstance.newPage({ javaScriptEnabled: false });
+            disposable = true;
+        }
+    }
     try {
+        if (robotsRules) {
+            const urlPath = new URL(url).pathname;
+            if (!isAllowedByRobots(urlPath, robotsRules)) {
+                return null;
+            }
+        }
+        const currentUrl = page.url();
+        if (currentUrl !== 'about:blank' && currentUrl !== url) {
+            try {
+                const currentHost = new URL(currentUrl).hostname;
+                const targetHost = new URL(url).hostname;
+                if (currentHost !== targetHost) {
+                    await page.goto('about:blank').catch(() => { });
+                }
+            }
+            catch {
+                await page.goto('about:blank').catch(() => { });
+            }
+        }
         await page.goto(url, { timeout: timeoutMs, waitUntil: 'domcontentloaded' });
+        const contentType = await page.evaluate(() => document.contentType).catch(() => '');
+        if (contentType && !contentType.includes('html')) {
+            return null;
+        }
         await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => { });
         if (url.includes('#/') || url.includes('#!')) {
             await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => { });
@@ -197,7 +389,7 @@ async function crawlPage(page, url, baseOrigin, baseHostname, basePath, format,
         const [title, content, links] = await Promise.all([
             page.title(),
             extractContentFromPage(page, format, selector),
-            discoverLinks(page, baseOrigin, baseHostname, basePath),
+            discoverLinks(page, baseOrigin, baseHostname, basePath, allowExternal),
         ]);
         return { url: page.url(), title, content, links };
     }
@@ -209,26 +401,35 @@ async function crawlPage(page, url, baseOrigin, baseHostname, basePath, format,
                     page.title().catch(() => url),
                     extractContentFromPage(page, format, selector).catch(() => ''),
                 ]);
-                const links = await discoverLinks(page, baseOrigin, baseHostname, basePath).catch(() => []);
+                const links = await discoverLinks(page, baseOrigin, baseHostname, basePath, allowExternal).catch(() => []);
                 return { url: page.url(), title, content, links };
             }
             catch {
                 return null;
             }
+            finally {
+                if (disposable)
+                    await page.close().catch(() => { });
+            }
         }
         return null;
     }
+    finally {
+        if (disposable)
+            await page.close().catch(() => { });
+    }
 }
-export async function discoverLinks(page, baseOrigin, baseHostname, basePath) {
-    const hrefs = await page.evaluate((origin) => {
+export async function discoverLinks(page, baseOrigin, baseHostname, basePath, allowExternal = false) {
+    const hrefs = await page.evaluate(() => {
         const anchors = document.querySelectorAll('a[href]');
         const results = [];
+        const base = document.baseURI;
         anchors.forEach((a) => {
             const href = a.getAttribute('href');
             if (!href)
                 return;
             try {
-                const fullUrl = new URL(href, origin).href;
+                const fullUrl = new URL(href, base).href;
                 results.push(fullUrl);
             }
             catch {
@@ -236,7 +437,7 @@ export async function discoverLinks(page, baseOrigin, baseHostname, basePath) {
             }
         });
         return results;
-    }, baseOrigin);
+    });
     const filtered = new Set();
     for (const href of hrefs) {
         try {
@@ -249,9 +450,11 @@ export async function discoverLinks(page, baseOrigin, baseHostname, basePath) {
             const hostname = url.hostname.replace(/^www\./, '');
             if (SOCIAL_DOMAINS.some((d) => hostname === d || hostname.endsWith('.' + d)))
                 continue;
-            if (hostname !== baseHostname && !hostname.endsWith('.' + baseHostname)) {
-                if (url.origin !== baseOrigin)
-                    continue;
+            if (!allowExternal) {
+                if (hostname !== baseHostname && !hostname.endsWith('.' + baseHostname)) {
+                    if (url.origin !== baseOrigin)
+                        continue;
+                }
             }
             if (basePath && basePath !== '/') {
                 const normalizedBase = basePath.endsWith('/') ? basePath : basePath + '/';

package/dist/actions/crawl.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/actions/crawl.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEvE,MAAM,iBAAiB,GAAG;IACxB,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,cAAc;IACd,aAAa;IACb,OAAO;IACP,eAAe;IACf,cAAc;IACd,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,WAAW;IACX,WAAW;CACZ,CAAC;AAEF,MAAM,kBAAkB,GAAG;IACzB,aAAa;IACb,WAAW;IACX,WAAW;IACX,UAAU;IACV,eAAe;IACf,cAAc;IACd,SAAS;IACT,eAAe;IACf,aAAa;IACb,YAAY;IACZ,WAAW;IACX,aAAa;IACb,cAAc;IACd,cAAc;IACd,eAAe;IACf,WAAW;IACX,gBAAgB;CACjB,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW,EAAE,OAAe;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG;QACD,OAAO;aACJ,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC;aACpC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC;aAClC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC;aACvB,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;QACzB,GAAG,CACN,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAC1B,GAAW,EACX,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;QACjD,CAAC;IACH,CAAC;IACD,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAC;QAChD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAqB,EACrB,OAAuB;IAEvB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,8BAA8B;SACtC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC;IAC5C,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACrD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAEhD,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IAGf,MAAM,KAAK,GAAiB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAEvE,SAAS,WAAW,CAAC,GAAW;QAC9B,KAAK,MAAM,CAAC,IAAI,kBAAkB,EAAE,CAAC;YACnC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QACnD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEnE,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACtF,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC7B,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,SAAS;YACtC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClB,SAAS,CACP,IAAI,EACJ,KAAK,CAAC,GAAG,EACT,UAAU,EACV,YAAY,EACZ,QAAQ,EACR,MAAM,EACN,OAAO,CAAC,QAAQ,EAChB,SAAS,EACT,aAAa,EACb,eAAe,EACf,eAAe,CAChB,CACF,CACF,CAAC;QAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAClD,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;gBACnC,MAAM,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACjD,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBACrC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAE1B,IAAI,KAAK,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;oBAC3B,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;wBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;wBACtC,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;4BAAE,SAAS;wBACtC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,CAAC;4BAC9E,SAAS;wBACX,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,eAAe,EAAE,eAAe,CAAC;4BAAE,SAAS;wBACjF,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ;4BAAE,MAAM;wBACnD,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,EAAE,UAAU;4BACf,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC;4BACtB,QAAQ,EAAE,WAAW,CAAC,UAAU,CAAC;yBAClC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE;QACjC,GAAG,EAAE,QAAQ;QACb,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,IAAU,EACV,GAAW,EACX,UAAkB,EAClB,YAAoB,EACpB,QAAgB,EAChB,MAAoC,EACpC,QAAiB,EACjB,YAAoB,KAAK,EACzB,gBAAyB,KAAK,EAC9B,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAE5E,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9E,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC9E,MAAM,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACrF,CAAC;YAAC,MAAM,CAAC;gBACP,sCAAsC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE;YACZ,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC;YAC9C,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC;SACxD,CAAC,CAAC;QAEH,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;oBACzC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC;oBAC7B,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;iBAC/D,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,CAC/E,GAAG,EAAE,CAAC,EAAc,CACrB,CAAC;gBACF,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAU,EACV,UAAkB,EAClB,YAAoB,EACpB,QAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAc,EAAE,EAAE;QACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YACpB,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACP,cAAc;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC,EAAE,UAAU,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;YAE1B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;gBAAE,SAAS;YAEpE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC5C,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAAE,SAAS;YAEtE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACpD,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEvF,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,CAAC;gBACxE,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU;oBAAE,SAAS;YAC1C,CAAC;YAED,IAAI,QAAQ,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;gBACjC,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;gBAC1E,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC7B,IAAI,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC;oBAAE,SAAS;YAC5E,CAAC;YAED,MAAM,UAAU,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,UAAU;gBAAE,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,YAAY,CACnB,GAAW,EACX,UAAkB,EAClB,YAAoB,EACpB,QAAgB,EAChB,gBAAyB,KAAK;IAE9B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACpE,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1E,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClD,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAE3F,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IACE,QAAQ,KAAK,YAAY;gBACzB,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC;gBACtC,CAAC,CAAC,MAAM,KAAK,UAAU,EACvB,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;YACjC,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;YAC1E,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC3B,IAAI,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;gBAChE,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,CAAM;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC1B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;IACpB,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,aAAa,IAAI,aAAa,KAAK,GAAG,EAAE,CAAC;YAC5C,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAClC,CAAC;QACD,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,KAAK,aAAa,EAAE,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;AAClC,CAAC"}
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/actions/crawl.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEvE,MAAM,iBAAiB,GAAG;IACxB,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,cAAc;IACd,aAAa;IACb,OAAO;IACP,eAAe;IACf,cAAc;IACd,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,WAAW;IACX,WAAW;CACZ,CAAC;AAEF,MAAM,kBAAkB,GAAG;IACzB,aAAa;IACb,WAAW;IACX,WAAW;IACX,UAAU;IACV,eAAe;IACf,cAAc;IACd,SAAS;IACT,eAAe;IACf,aAAa;IACb,YAAY;IACZ,WAAW;IACX,aAAa;IACb,cAAc;IACd,cAAc;IACd,eAAe;IACf,WAAW;IACX,gBAAgB;CACjB,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW,EAAE,OAAe;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG;QACD,OAAO;aACJ,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC;aACpC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC;aAClC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC;aACvB,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;QACzB,GAAG,CACN,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAC1B,GAAW,EACX,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;QACjD,CAAC;IACH,CAAC;IACD,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAC;QAChD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAQD,KAAK,UAAU,cAAc,CAAC,MAAc;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,aAAa,EAAE;YAC9C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,YAAoB,GAAG;IAChE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,UAA8B,CAAC;IAEnC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI;YAAE,SAAS;QAE5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,QAAQ,KAAK,CAAC,CAAC;YAAE,SAAS;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE9C,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACzB,UAAU,GAAG,KAAK,KAAK,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QACzF,CAAC;aAAM,IAAI,UAAU,EAAE,CAAC;YACtB,IAAI,GAAG,KAAK,UAAU,IAAI,KAAK,EAAE,CAAC;gBAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,GAAG,KAAK,OAAO,IAAI,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,GAAG,KAAK,aAAa,IAAI,KAAK,EAAE,CAAC;gBAC1C,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB,EAAE,OAAe;IAC7D,MAAM,KAAK,GAAG,OAAO;SAClB,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACpC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAiB;IAC5D,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACpC,IAAI,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC;IAC3D,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACvC,IAAI,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;IAC5D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAqB,EACrB,OAAuB;IAEvB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,8BAA8B;SACtC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC;IAC5C,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACrD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAE7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IAGf,MAAM,KAAK,GAAiB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAEvE,SAAS,uBAAuB,CAAC,GAAW;QAC1C,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,MAAM,QAAQ,GAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAEpC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjC,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACxC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAChC,OAAO,CAAC,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC;gBACvC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAChC,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC;gBAC9D,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAClC,SAAS,CAAC,QAAQ,IAAI,GAAG,CAAC;gBAC1B,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;YAED,MAAM,YAAY,GAAG,mCAAmC,CAAC;YACzD,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAChC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,GAAG,CAAC;gBAChE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC;YAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,SAAS,SAAS,CAAC,GAAW;QAC5B,OAAO,uBAAuB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,SAAS,WAAW,CAAC,GAAW;QAC9B,KAAK,MAAM,IAAI,IAAI,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,SAAS,WAAW,CAAC,GAAW;QAC9B,KAAK,MAAM,CAAC,IAAI,kBAAkB,EAAE,CAAC;YACnC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QACnD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEnE,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,OACE,KAAK,CAAC,MAAM,GAAG,CAAC;YAChB,KAAK,CAAC,MAAM,GAAG,WAAW;YAC1B,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,EACtC,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC7B,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,SAAS,CAAC,UAAU,CAAC;gBAAE,SAAS;YACpC,WAAW,CAAC,UAAU,CAAC,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClB,SAAS,CACP,OAAO,EACP,IAAI,EACJ,KAAK,CAAC,GAAG,EACT,UAAU,EACV,YAAY,EACZ,QAAQ,EACR,MAAM,EACN,OAAO,CAAC,QAAQ,EAChB,SAAS,EACT,aAAa,EACb,eAAe,EACf,eAAe,EACf,OAAO,CAAC,iBAAiB,EACzB,WAAW,EACX,UAAU,CACX,CACF,CACF,CAAC;QAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAClD,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;gBACnC,MAAM,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACjD,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBACrC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACvB,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAE1B,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACnB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;gBAClE,CAAC;gBAED,IAAI,KAAK,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;oBAC3B,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;wBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;wBACtC,IAAI,SAAS,CAAC,UAAU,CAAC;4BAAE,SAAS;wBACpC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,CAAC;4BAC9E,SAAS;wBACX,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,eAAe,EAAE,eAAe,CAAC;4BAAE,SAAS;wBACjF,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ;4BAAE,MAAM;wBACnD,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,EAAE,UAAU;4BACf,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC;4BACtB,QAAQ,EAAE,WAAW,CAAC,UAAU,CAAC;yBAClC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE;QACjC,GAAG,EAAE,QAAQ;QACb,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,OAAuB,EACvB,QAAc,EACd,GAAW,EACX,UAAkB,EAClB,YAAoB,EACpB,QAAgB,EAChB,MAAoC,EACpC,QAAiB,EACjB,YAAoB,KAAK,EACzB,gBAAyB,KAAK,EAC9B,eAA0B,EAC1B,eAA0B,EAC1B,iBAA2B,EAC3B,WAAwB,EACxB,UAAmB;IAEnB,IAAI,IAAI,GAAG,QAAQ,CAAC;IACpB,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,IAAI,iBAAiB,KAAK,KAAK,EAAE,CAAC;QAChC,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,eAAe,EAAE,CAAC;YACpB,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;YACnE,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACtC,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;gBAC7C,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,UAAU,KAAK,aAAa,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC;gBACjD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;gBACzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAE5E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACjD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9E,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC9E,MAAM,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACrF,CAAC;YAAC,MAAM,CAAC;gBACP,sCAAsC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE;YACZ,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC;YAC9C,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,CAAC;SACvE,CAAC,CAAC;QAEH,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;oBACzC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC;oBAC7B,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;iBAC/D,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,aAAa,CAC/B,IAAI,EACJ,UAAU,EACV,YAAY,EACZ,QAAQ,EACR,aAAa,CACd,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;gBAC9B,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,IAAI,UAAU;oBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,UAAU;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAU,EACV,UAAkB,EAClB,YAAoB,EACpB,QAAgB,EAChB,gBAAyB,KAAK;IAE9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;QAC9B,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YACpB,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACP,cAAc;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;YAE1B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;gBAAE,SAAS;YAEpE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC5C,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAAE,SAAS;YAEtE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACpD,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEvF,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,CAAC;oBACxE,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU;wBAAE,SAAS;gBAC1C,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;gBACjC,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;gBAC1E,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC7B,IAAI,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC;oBAAE,SAAS;YAC5E,CAAC;YAED,MAAM,UAAU,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,UAAU;gBAAE,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,YAAY,CACnB,GAAW,EACX,UAAkB,EAClB,YAAoB,EACpB,QAAgB,EAChB,gBAAyB,KAAK;IAE9B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACpE,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1E,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClD,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAE3F,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IACE,QAAQ,KAAK,YAAY;gBACzB,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC;gBACtC,CAAC,CAAC,MAAM,KAAK,UAAU,EACvB,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;YACjC,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;YAC1E,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC3B,IAAI,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;gBAChE,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,CAAM;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC1B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;IACpB,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,aAAa,IAAI,aAAa,KAAK,GAAG,EAAE,CAAC;YAC5C,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAClC,CAAC;QACD,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,KAAK,aAAa,EAAE,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;AAClC,CAAC"}

package/dist/actions/map.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/actions/map.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAqCnE,wBAAsB,SAAS,CAC7B,OAAO,EAAE,UAAU,EACnB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAuE9B"}
+{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/actions/map.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AA8FnE,wBAAsB,SAAS,CAC7B,OAAO,EAAE,UAAU,EACnB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAuE9B"}

package/dist/actions/map.js

@@ -1,34 +1,93 @@
 import { successResponse } from '../protocol.js';
 import { discoverLinks } from './crawl.js';
-async function discoverFromSitemap(baseUrl) {
+function parseSitemapXml(xml) {
     const urls = [];
+    const urlMatches = xml.matchAll(/<loc>([^<]+)<\/loc>/g);
+    for (const match of urlMatches) {
+        urls.push(match[1].trim());
+    }
+    return urls;
+}
+function parseSitemapIndex(xml) {
+    const sitemapUrls = [];
+    const locRegex = /<sitemap[^>]*>[\s\S]*?<loc>([^<]+)<\/loc>[\s\S]*?<\/sitemap>/gi;
+    let match;
+    while ((match = locRegex.exec(xml)) !== null) {
+        sitemapUrls.push(match[1].trim());
+    }
+    return sitemapUrls;
+}
+async function discoverSitemapsFromRobots(origin) {
     try {
-        const base = new URL(baseUrl);
-        const sitemapUrl = `${base.origin}/sitemap.xml`;
-        const response = await fetch(sitemapUrl, {
+        const res = await fetch(`${origin}/robots.txt`, {
             signal: AbortSignal.timeout(5000),
         });
-        if (response.ok) {
-            const xml = await response.text();
-            const urlMatches = xml.matchAll(/<loc>([^<]+)<\/loc>/g);
-            for (const match of urlMatches) {
-                const url = match[1].trim();
-                try {
-                    const parsedUrl = new URL(url);
-                    if (parsedUrl.hostname === base.hostname) {
-                        urls.push(url);
+        if (!res.ok)
+            return [];
+        const text = await res.text();
+        return text
+            .split('\n')
+            .filter((l) => l.toLowerCase().startsWith('sitemap:'))
+            .map((l) => l.split(':').slice(1).join(':').trim());
+    }
+    catch {
+        return [];
+    }
+}
+async function fetchSitemapUrls(baseUrl) {
+    const allUrls = [];
+    const base = new URL(baseUrl);
+    const robotsSitemaps = await discoverSitemapsFromRobots(base.origin);
+    const sitemapUrls = [
+        ...robotsSitemaps,
+        new URL('/sitemap.xml', baseUrl).href,
+        new URL('/sitemap_index.xml', baseUrl).href,
+        new URL('/sitemap/', baseUrl).href,
+    ];
+    const tried = new Set();
+    for (const sitemapUrl of sitemapUrls) {
+        if (tried.has(sitemapUrl))
+            continue;
+        tried.add(sitemapUrl);
+        try {
+            const res = await fetch(sitemapUrl, { signal: AbortSignal.timeout(10000) });
+            if (!res.ok)
+                continue;
+            const xml = await res.text();
+            if (xml.includes('<sitemapindex')) {
+                const childUrls = parseSitemapIndex(xml);
+                for (const childUrl of childUrls) {
+                    try {
+                        const childRes = await fetch(childUrl, { signal: AbortSignal.timeout(10000) });
+                        if (childRes.ok) {
+                            const childXml = await childRes.text();
+                            allUrls.push(...parseSitemapXml(childXml));
+                        }
+                    }
+                    catch {
+                        // child sitemap fetch failed
                     }
-                }
-                catch {
-                    // invalid URL
                 }
             }
+            else {
+                allUrls.push(...parseSitemapXml(xml));
+            }
+            if (allUrls.length > 0)
+                break;
+        }
+        catch {
+            // sitemap not available
         }
     }
-    catch {
-        // Sitemap not available
-    }
-    return urls;
+    return allUrls.filter((url) => {
+        try {
+            const parsedUrl = new URL(url);
+            return parsedUrl.hostname === base.hostname;
+        }
+        catch {
+            return false;
+        }
+    });
 }
 export async function handleMap(command, browser) {
     if (!browser.isLaunched()) {
@@ -43,7 +102,7 @@ export async function handleMap(command, browser) {
     const baseUrl = command.url;
     const baseOrigin = new URL(baseUrl).origin;
     const baseHostname = new URL(baseUrl).hostname.replace(/^www\./, '');
-    const sitemapUrls = await discoverFromSitemap(baseUrl);
+    const sitemapUrls = await fetchSitemapUrls(baseUrl);
     const page = browser.getPage();
     await page.goto(baseUrl, {
         timeout: timeout * 1000,

package/dist/actions/map.js.map

@@ -1 +1 @@
-{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/actions/map.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,KAAK,UAAU,mBAAmB,CAAC,OAAe;IAChD,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,cAAc,CAAC;QAEhD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;YACvC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;YACxD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC/B,IAAI,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACzC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACjB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,cAAc;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAmB,EACnB,OAAuB;IAEvB,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QAC1B,MAAM,OAAO,CAAC,MAAM,CAAC;YACnB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;SACnC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC;IACnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;IAC5B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACvB,OAAO,EAAE,OAAO,GAAG,IAAI;QACvB,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI;aACD,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;aAC5E,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAClB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;KAC1B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IAEzE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEjD,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe;QAChD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,IAAI,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;gBACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC9C,IAAI,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;wBAAE,OAAO,KAAK,CAAC;gBAC5C,CAAC;YACH,CAAC;YACD,IAAI,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;gBACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC9C,IAAI,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;gBAC3C,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACJ,CAAC,CAAC,IAAI,CAAC;IAEX,OAAO,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE;QACjC,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,OAAe;IAC7C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG;QACD,OAAO;aACJ,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC;aACpC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC;aAClC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC;aACvB,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;QACzB,GAAG,CACN,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}
+{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/actions/map.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;IACxD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,gEAAgE,CAAC;IAClF,IAAI,KAAK,CAAC;IACV,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC7C,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,MAAc;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,aAAa,EAAE;YAC9C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,IAAI;aACR,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;aACrD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,OAAe;IAC7C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAE9B,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG;QAClB,GAAG,cAAc;QACjB,IAAI,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,IAAI;QACrC,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC,IAAI;QAC3C,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,IAAI;KACnC,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,SAAS;QACpC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,SAAS;YACtB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAE7B,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBACzC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAC/E,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;4BAChB,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;4BACvC,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAC7C,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,6BAA6B;oBAC/B,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC5B,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,OAAO,SAAS,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAmB,EACnB,OAAuB;IAEvB,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QAC1B,MAAM,OAAO,CAAC,MAAM,CAAC;YACnB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;SACnC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC;IACnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;IAC5B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAEpD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACvB,OAAO,EAAE,OAAO,GAAG,IAAI;QACvB,SAAS,EAAE,kBAAkB;KAC9B,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI;aACD,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;aAC5E,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAClB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;KAC1B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IAEzE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEjD,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe;QAChD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,IAAI,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;gBACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC9C,IAAI,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;wBAAE,OAAO,KAAK,CAAC;gBAC5C,CAAC;YACH,CAAC;YACD,IAAI,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;gBACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC9C,IAAI,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;gBAC3C,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACJ,CAAC,CAAC,IAAI,CAAC;IAEX,OAAO,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE;QACjC,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,OAAe;IAC7C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG;QACD,OAAO;aACJ,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC;aACpC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC;aAClC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC;aACvB,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;QACzB,GAAG,CACN,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/dist/actions/scrape.d.ts

@@ -1,10 +1,29 @@
 import type { BrowserManager } from '../browser/index.js';
 import type { ScrapeCommand, Response } from '../types.js';
+export interface ScrapeMetadata {
+    description?: string;
+    keywords?: string;
+    author?: string;
+    robots?: string;
+    canonical?: string;
+    favicon?: string;
+    lang?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogImage?: string;
+    ogUrl?: string;
+    ogSiteName?: string;
+    publishedTime?: string;
+    modifiedTime?: string;
+    articleTag?: string;
+    articleSection?: string;
+}
 export interface ScrapeResult {
     url: string;
     title: string;
     content: string;
     format: 'text' | 'html' | 'markdown';
+    metadata?: ScrapeMetadata;
 }
 export declare function handleScrape(command: ScrapeCommand, browser: BrowserManager): Promise<Response<ScrapeResult>>;
 //# sourceMappingURL=scrape.d.ts.map

package/dist/actions/scrape.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/actions/scrape.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI3D,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;CACtC;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAmDjC"}
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/actions/scrape.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI3D,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;IACrC,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAgIjC"}