@olib-ai/owl-browser-sdk 2.0.4 → 2.0.6

package/dist/extraction/pattern-detector.js

@@ -0,0 +1,390 @@
+/**
+ * Zero-config auto pattern discovery via pure DOM analysis.
+ *
+ * Finds repeating DOM structures without any AI — uses tag+class frequency,
+ * child consistency scoring, and semantic field inference.
+ */
+import { load } from 'cheerio';
+import { extractAll } from './selector-engine.js';
+import { analyzeStructure } from './list-extractor.js';
+/**
+ * Detect repeating patterns on the page.
+ */
+export function detect(html, options) {
+    const minCount = options?.minCount ?? 3;
+    const minConfidence = options?.minConfidence ?? 0.5;
+    const region = options?.region;
+    const $ = load(html);
+    const root = region ? $(region).first() : $('body');
+    if (root.length === 0)
+        return [];
+    // Find all elements that could be containers of repeating items
+    const candidates = findCandidates($, root, minCount);
+    // Score and rank candidates
+    const patterns = [];
+    for (const candidate of candidates) {
+        try {
+            const { containerSelector, itemSelector, items } = candidate;
+            if (items.length < minCount)
+                continue;
+            const confidence = scoreCandidate($, items);
+            if (confidence < minConfidence)
+                continue;
+            // Infer fields from first item
+            const fields = analyzeStructure(html, containerSelector, itemSelector);
+            if (Object.keys(fields).length === 0)
+                continue;
+            // Extract sample (first 3 items)
+            const fullSelector = `${containerSelector} ${itemSelector}`;
+            const sample = extractAll(html, fullSelector, fields).slice(0, 3);
+            patterns.push({
+                containerSelector,
+                itemSelector: fullSelector,
+                itemCount: items.length,
+                confidence,
+                fields,
+                sample,
+            });
+        }
+        catch {
+            // Skip candidates with invalid selectors (e.g. special chars in class names)
+            continue;
+        }
+    }
+    // Sort by confidence desc, then by item count desc
+    patterns.sort((a, b) => {
+        if (b.confidence !== a.confidence)
+            return b.confidence - a.confidence;
+        return b.itemCount - a.itemCount;
+    });
+    return patterns;
+}
+/**
+ * Detect patterns and immediately extract the best one.
+ */
+export function detectAndExtract(html, options) {
+    const patterns = detect(html, options);
+    if (patterns.length === 0)
+        return [];
+    const best = patterns[0];
+    return extractAll(html, best.itemSelector, best.fields);
+}
+// --- Utility class detection for multi-class matching ---
+const UTILITY_PREFIXES = /^(p|m|w|h|min|max|flex|grid|text|bg|border|rounded|shadow|gap|space|overflow|z|opacity|cursor|font|leading|tracking|transition|duration|ease|scale|rotate|translate|skew|origin|ring|outline|decoration|placeholder|divide|sr)-/;
+const UTILITY_WORDS = new Set(['flex', 'block', 'hidden', 'relative', 'absolute', 'fixed', 'sticky', 'inline', 'grid', 'table', 'contents', 'static', 'visible', 'invisible', 'isolate', 'truncate', 'antialiased', 'italic', 'underline', 'uppercase', 'lowercase', 'capitalize', 'ordinal']);
+function isSemanticClass(cls) {
+    if (cls.length <= 1)
+        return false;
+    if (UTILITY_PREFIXES.test(cls))
+        return false;
+    if (UTILITY_WORDS.has(cls))
+        return false;
+    return true;
+}
+function findCandidates($, root, minCount) {
+    const candidates = [];
+    const seen = new Set();
+    // Strategy 1: Find elements with multiple same-tag+class children
+    root.find('*').each((_i, el) => {
+        if (el.type !== 'tag')
+            return;
+        const $el = $(el);
+        const children = $el.children();
+        if (children.length < minCount)
+            return;
+        // Count tag+class combos (multi-class matching with utility filtering)
+        const comboCounts = new Map();
+        children.each((_j, child) => {
+            if (child.type !== 'tag')
+                return;
+            const tag = child.tagName.toLowerCase();
+            const allClasses = $(child).attr('class')?.split(/\s+/).filter(Boolean) ?? [];
+            const semanticClasses = allClasses.filter(isSemanticClass).slice(0, 3);
+            const key = semanticClasses.length > 0
+                ? `${tag}.${semanticClasses.map(cssEscape).join('.')}`
+                : (allClasses[0] ? `${tag}.${cssEscape(allClasses[0])}` : tag);
+            const existing = comboCounts.get(key);
+            if (existing) {
+                existing.count++;
+                existing.items.push(child);
+            }
+            else {
+                comboCounts.set(key, { count: 1, items: [child] });
+            }
+        });
+        for (const [itemKey, { count, items }] of comboCounts) {
+            if (count < minCount)
+                continue;
+            const containerSel = buildSelector($, $el);
+            const fullKey = `${containerSel}|${itemKey}`;
+            if (seen.has(fullKey))
+                continue;
+            seen.add(fullKey);
+            candidates.push({
+                containerSelector: containerSel,
+                itemSelector: `> ${itemKey}`,
+                items: items.map(i => $(i)),
+            });
+        }
+    });
+    // Strategy 2: Look for common list patterns (ul/ol with li, tbody with tr)
+    root.find('ul, ol').each((_i, el) => {
+        const $el = $(el);
+        const lis = $el.children('li');
+        if (lis.length >= minCount) {
+            const sel = buildSelector($, $el);
+            if (!seen.has(`${sel}|li`)) {
+                seen.add(`${sel}|li`);
+                const items = [];
+                lis.each((_j, li) => { items.push($(li)); });
+                candidates.push({
+                    containerSelector: sel,
+                    itemSelector: '> li',
+                    items,
+                });
+            }
+        }
+    });
+    root.find('tbody').each((_i, el) => {
+        const $el = $(el);
+        const trs = $el.children('tr');
+        if (trs.length >= minCount) {
+            const sel = buildSelector($, $el);
+            if (!seen.has(`${sel}|tr`)) {
+                seen.add(`${sel}|tr`);
+                const items = [];
+                trs.each((_j, tr) => { items.push($(tr)); });
+                candidates.push({
+                    containerSelector: sel,
+                    itemSelector: '> tr',
+                    items,
+                });
+            }
+        }
+    });
+    // Strategy 3: Deeper nesting — search grandchildren when direct children are wrapper divs
+    root.find('*').each((_i, el) => {
+        if (el.type !== 'tag')
+            return;
+        const $el = $(el);
+        const children = $el.children();
+        // Only try deeper search if children are generic wrappers (divs/spans/li with few classes)
+        const divChildren = children.filter((_j, child) => {
+            if (child.type !== 'tag')
+                return false;
+            const tag = child.tagName.toLowerCase();
+            return tag === 'div' || tag === 'span' || tag === 'li';
+        });
+        if (divChildren.length < minCount)
+            return;
+        // Look at grandchildren for consistent patterns
+        const grandComboCounts = new Map();
+        divChildren.each((_j, wrapper) => {
+            const $wrapper = $(wrapper);
+            const grandchildren = $wrapper.children();
+            if (grandchildren.length === 0)
+                return;
+            // Use the wrapper's first grandchild tag+class as the pattern
+            const firstGrand = grandchildren[0];
+            if (!firstGrand || firstGrand.type !== 'tag')
+                return;
+            const gTag = firstGrand.tagName.toLowerCase();
+            const gClasses = $(firstGrand).attr('class')?.split(/\s+/).filter(Boolean).filter(isSemanticClass).slice(0, 3) ?? [];
+            const gKey = gClasses.length > 0 ? `${gTag}.${gClasses.map(cssEscape).join('.')}` : gTag;
+            // Count the wrapper, not the grandchild — the wrapper IS the item
+            const wTag = wrapper.tagName.toLowerCase();
+            const wClasses = $(wrapper).attr('class')?.split(/\s+/).filter(Boolean).filter(isSemanticClass).slice(0, 3) ?? [];
+            const wKey = wClasses.length > 0 ? `${wTag}.${wClasses.map(cssEscape).join('.')}` : wTag;
+            const existing = grandComboCounts.get(wKey);
+            if (existing) {
+                existing.count++;
+                existing.items.push(wrapper);
+            }
+            else {
+                grandComboCounts.set(wKey, { count: 1, items: [wrapper] });
+            }
+        });
+        for (const [itemKey, { count, items }] of grandComboCounts) {
+            if (count < minCount)
+                continue;
+            const containerSel = buildSelector($, $el);
+            const fullKey = `${containerSel}|deep|${itemKey}`;
+            if (seen.has(fullKey))
+                continue;
+            seen.add(fullKey);
+            candidates.push({
+                containerSelector: containerSel,
+                itemSelector: `> ${itemKey}`,
+                items: items.map(i => $(i)),
+            });
+        }
+    });
+    // Strategy 4: ARIA role detection
+    const ariaRoles = ['listitem', 'article', 'row', 'option', 'menuitem', 'tab'];
+    for (const role of ariaRoles) {
+        root.find(`[role="${role}"]`).each((_i, el) => {
+            if (el.type !== 'tag')
+                return;
+            const $el = $(el);
+            const parent = $el.parent();
+            if (parent.length === 0)
+                return;
+            const siblings = parent.find(`> [role="${role}"]`);
+            if (siblings.length < minCount)
+                return;
+            const containerSel = buildSelector($, parent);
+            const itemKey = `[role="${role}"]`;
+            const fullKey = `${containerSel}|${itemKey}`;
+            if (seen.has(fullKey))
+                return;
+            seen.add(fullKey);
+            const items = [];
+            siblings.each((_j, sib) => { items.push($(sib)); });
+            candidates.push({
+                containerSelector: containerSel,
+                itemSelector: `> ${itemKey}`,
+                items,
+            });
+        });
+    }
+    // Strategy 5: Data-attribute patterns
+    const dataAttrs = ['data-testid', 'data-id', 'data-index', 'data-item', 'data-row'];
+    for (const attr of dataAttrs) {
+        const attrEls = root.find(`[${attr}]`);
+        if (attrEls.length < minCount)
+            continue;
+        // Group by parent
+        const parentGroups = new Map();
+        attrEls.each((_i, el) => {
+            if (el.type !== 'tag')
+                return;
+            const parent = $(el).parent();
+            const pKey = buildSelector($, parent);
+            const existing = parentGroups.get(pKey);
+            if (existing) {
+                existing.items.push(el);
+            }
+            else {
+                parentGroups.set(pKey, { parent, items: [el] });
+            }
+        });
+        for (const [containerSel, { items }] of parentGroups) {
+            if (items.length < minCount)
+                continue;
+            const itemKey = `[${attr}]`;
+            const fullKey = `${containerSel}|${itemKey}`;
+            if (seen.has(fullKey))
+                continue;
+            seen.add(fullKey);
+            candidates.push({
+                containerSelector: containerSel,
+                itemSelector: `> ${itemKey}`,
+                items: items.map(i => $(i)),
+            });
+        }
+    }
+    return candidates;
+}
+/**
+ * Score a candidate by child consistency and content richness.
+ */
+function scoreCandidate($, items) {
+    if (items.length === 0)
+        return 0;
+    let score = 0;
+    // 1. Child tag structure consistency (0-0.4)
+    const structures = items.map(item => {
+        const tags = [];
+        item.children().each((_i, el) => {
+            if (el.type === 'tag') {
+                tags.push(el.tagName.toLowerCase());
+            }
+        });
+        return tags.join(',');
+    });
+    const mostCommon = mode(structures);
+    const consistency = structures.filter(s => s === mostCommon).length / structures.length;
+    score += consistency * 0.4;
+    // 2. Average number of text nodes (0-0.2)
+    const avgTextNodes = items.reduce((sum, item) => {
+        const text = item.text().trim();
+        return sum + (text.length > 10 ? 1 : 0);
+    }, 0) / items.length;
+    score += Math.min(avgTextNodes, 1) * 0.2;
+    // 3. Presence of links (0-0.15)
+    const hasLinks = items.filter(item => item.find('a').length > 0).length / items.length;
+    score += hasLinks * 0.15;
+    // 4. Presence of images (0-0.1)
+    const hasImages = items.filter(item => item.find('img').length > 0).length / items.length;
+    score += hasImages * 0.1;
+    // 5. Item count bonus (0-0.15)
+    score += Math.min(items.length / 20, 1) * 0.15;
+    // 6. ARIA role consistency (0-0.1)
+    const hasRole = items.filter(item => item.attr('role')).length / items.length;
+    score += hasRole * 0.1;
+    // 7. Data-attribute consistency (0-0.1)
+    const hasDataAttr = items.filter(item => {
+        const node = item[0];
+        if (!node || node.type !== 'tag')
+            return false;
+        const attribs = node.attribs ?? {};
+        return Object.keys(attribs).some(k => k.startsWith('data-'));
+    }).length / items.length;
+    score += hasDataAttr * 0.1;
+    return Math.round(score * 100) / 100;
+}
+/**
+ * Escape special characters in a CSS identifier (class name, id).
+ * Characters like : [ ] . > + ~ need backslash escaping in CSS selectors.
+ */
+function cssEscape(value) {
+    return value.replace(/([!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1');
+}
+/**
+ * Build a CSS selector for an element.
+ */
+function buildSelector($, el) {
+    const node = el[0];
+    if (!node || node.type !== 'tag')
+        return '*';
+    const tag = node.tagName.toLowerCase();
+    const id = el.attr('id');
+    if (id)
+        return `${tag}#${cssEscape(id)}`;
+    const allClasses = el.attr('class')?.split(/\s+/).filter(Boolean) ?? [];
+    const semanticClasses = allClasses.filter(isSemanticClass).slice(0, 2);
+    const cls = semanticClasses.length > 0
+        ? semanticClasses.map(cssEscape).join('.')
+        : allClasses.slice(0, 2).map(cssEscape).join('.');
+    let selector = cls ? `${tag}.${cls}` : tag;
+    // Check uniqueness — if multiple matches, add nth-of-type
+    try {
+        const matches = $(selector);
+        if (matches.length > 1) {
+            const index = matches.index(el);
+            if (index >= 0) {
+                selector = `${selector}:nth-of-type(${index + 1})`;
+            }
+        }
+    }
+    catch {
+        // Skip uniqueness check on invalid selectors
+    }
+    return selector;
+}
+function mode(arr) {
+    const counts = new Map();
+    for (const item of arr) {
+        counts.set(item, (counts.get(item) ?? 0) + 1);
+    }
+    let best;
+    let bestCount = 0;
+    for (const [item, count] of counts) {
+        if (count > bestCount) {
+            bestCount = count;
+            best = item;
+        }
+    }
+    return best;
+}
+//# sourceMappingURL=pattern-detector.js.map

package/dist/extraction/pattern-detector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pattern-detector.js","sourceRoot":"","sources":["../../src/extraction/pattern-detector.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,IAAI,EAAiC,MAAM,SAAS,CAAC;AAG9D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD;;GAEG;AACH,MAAM,UAAU,MAAM,CACpB,IAAY,EACZ,OAAuB;IAEvB,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,GAAG,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAE/B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,gEAAgE;IAChE,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAErD,4BAA4B;IAC5B,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,EAAE,iBAAiB,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;YAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;gBAAE,SAAS;YAEtC,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC5C,IAAI,UAAU,GAAG,aAAa;gBAAE,SAAS;YAEzC,+BAA+B;YAC/B,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAE/C,iCAAiC;YACjC,MAAM,YAAY,GAAG,GAAG,iBAAiB,IAAI,YAAY,EAAE,CAAC;YAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAElE,QAAQ,CAAC,IAAI,CAAC;gBACZ,iBAAiB;gBACjB,YAAY,EAAE,YAAY;gBAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,UAAU;gBACV,MAAM;gBACN,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,6EAA6E;YAC7E,SAAS;QACX,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAAE,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;QACtE,OAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,IAAY,EACZ,OAAuB;IAEvB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC;IAC1B,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1D,CAAC;AAUD,2DAA2D;AAE3D,MAAM,gBAAgB,GAAG,iOAAiO,CAAC;AAC3P,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC;AAE/R,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACzC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,IAAsB,EACtB,QAAgB;IAEhB,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,kEAAkE;IAClE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAC7B,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO;QAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,QAAQ;YAAE,OAAO;QAEvC,uEAAuE;QACvE,MAAM,WAAW,GAAG,IAAI,GAAG,EAA+C,CAAC;QAC3E,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;YAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO;YACjC,MAAM,GAAG,GAAI,KAA6B,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACjE,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC9E,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC;gBACpC,CAAC,CAAC,GAAG,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACtD,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACtD,IAAI,KAAK,GAAG,QAAQ;gBAAE,SAAS;YAE/B,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAElB,UAAU,CAAC,IAAI,CAAC;gBACd,iBAAiB,EAAE,YAAY;gBAC/B,YAAY,EAAE,KAAK,OAAO,EAAE;gBAC5B,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAuB,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7C,UAAU,CAAC,IAAI,CAAC;oBACd,iBAAiB,EAAE,GAAG;oBACtB,YAAY,EAAE,MAAM;oBACpB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QACjC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAuB,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7C,UAAU,CAAC,IAAI,CAAC;oBACd,iBAAiB,EAAE,GAAG;oBACtB,YAAY,EAAE,MAAM;oBACpB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,0FAA0F;IAC1F,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAC7B,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO;QAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,2FAA2F;QAC3F,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;YAChD,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO,KAAK,CAAC;YACvC,MAAM,GAAG,GAAI,KAA6B,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACjE,OAAO,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,IAAI,WAAW,CAAC,MAAM,GAAG,QAAQ;YAAE,OAAO;QAE1C,gDAAgD;QAChD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA+C,CAAC;QAChF,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;YAC/B,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC1C,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACvC,8DAA8D;YAC9D,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO;YACrD,MAAM,IAAI,GAAI,UAAkC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACvE,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrH,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACzF,kEAAkE;YAClE,MAAM,IAAI,GAAI,OAA+B,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACpE,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAClH,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACzF,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YAC3D,IAAI,KAAK,GAAG,QAAQ;gBAAE,SAAS;YAC/B,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,GAAG,YAAY,SAAS,OAAO,EAAE,CAAC;YAClD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClB,UAAU,CAAC,IAAI,CAAC;gBACd,iBAAiB,EAAE,YAAY;gBAC/B,YAAY,EAAE,KAAK,OAAO,EAAE;gBAC5B,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,kCAAkC;IAClC,MAAM,SAAS,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;IAC9E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;YAC5C,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO;YAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;YACnD,IAAI,QAAQ,CAAC,MAAM,GAAG,QAAQ;gBAAE,OAAO;YAEvC,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,UAAU,IAAI,IAAI,CAAC;YACnC,MAAM,OAAO,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,OAAO;YAC9B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAElB,MAAM,KAAK,GAAuB,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpD,UAAU,CAAC,IAAI,CAAC;gBACd,iBAAiB,EAAE,YAAY;gBAC/B,YAAY,EAAE,KAAK,OAAO,EAAE;gBAC5B,KAAK;aACN,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,sCAAsC;IACtC,MAAM,SAAS,GAAG,CAAC,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IACpF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ;YAAE,SAAS;QAExC,kBAAkB;QAClB,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0D,CAAC;QACvF,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;YACtB,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO;YAC9B,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAClD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YACrD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;gBAAE,SAAS;YACtC,MAAM,OAAO,GAAG,IAAI,IAAI,GAAG,CAAC;YAC5B,MAAM,OAAO,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClB,UAAU,CAAC,IAAI,CAAC;gBACd,iBAAiB,EAAE,YAAY;gBAC/B,YAAY,EAAE,KAAK,OAAO,EAAE;gBAC5B,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,CAAa,EAAE,KAAyB;IAC9D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,6CAA6C;IAC7C,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QAClC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;YAC9B,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAE,EAA0B,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IACxF,KAAK,IAAI,WAAW,GAAG,GAAG,CAAC;IAE3B,0CAA0C;IAC1C,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACrB,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;IAEzC,gCAAgC;IAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACvF,KAAK,IAAI,QAAQ,GAAG,IAAI,CAAC;IAEzB,gCAAgC;IAChC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1F,KAAK,IAAI,SAAS,GAAG,GAAG,CAAC;IAEzB,+BAA+B;IAC/B,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;IAE/C,mCAAmC;IACnC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC9E,KAAK,IAAI,OAAO,GAAG,GAAG,CAAC;IAEvB,wCAAwC;IACxC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/C,MAAM,OAAO,GAAI,IAAY,CAAC,OAAO,IAAI,EAAE,CAAC;QAC5C,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACzB,KAAK,IAAI,WAAW,GAAG,GAAG,CAAC;IAE3B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,uCAAuC,EAAE,MAAM,CAAC,CAAC;AACxE,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,CAAa,EAAE,EAAoB;IACxD,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACnB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,GAAG,CAAC;IAE7C,MAAM,GAAG,GAAI,IAA4B,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAChE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,EAAE;QAAE,OAAO,GAAG,GAAG,IAAI,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;IAEzC,MAAM,UAAU,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACxE,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC;QACpC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpD,IAAI,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAE3C,0DAA0D;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,QAAQ,GAAG,GAAG,QAAQ,gBAAgB,KAAK,GAAG,CAAC,GAAG,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;IAC/C,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,IAAI,CAAI,GAAQ;IACvB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAa,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,IAAmB,CAAC;IACxB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/extraction/scrape-session.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Multi-page scraping orchestrator.
+ *
+ * Combines content cleaning, extraction, pagination, deduplication,
+ * and optional detail-page following into a single scrape() call.
+ */
+import type { ExtractionResult, ScrapeOptions } from './types.js';
+import type { HTMLProcessor } from './html-processor.js';
+export declare class ScrapeSession {
+    private readonly _proc;
+    private _aborted;
+    constructor(proc: HTMLProcessor);
+    /**
+     * Cancel a running scrape.
+     */
+    abort(): void;
+    /**
+     * Run a multi-page scrape.
+     */
+    scrape(containerSelector: string, options?: ScrapeOptions): Promise<ExtractionResult>;
+    private _followDetailPages;
+}
+//# sourceMappingURL=scrape-session.d.ts.map

package/dist/extraction/scrape-session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape-session.d.ts","sourceRoot":"","sources":["../../src/extraction/scrape-session.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,gBAAgB,EAGhB,aAAa,EAEd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAMzD,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgB;IACtC,OAAO,CAAC,QAAQ,CAAS;gBAEb,IAAI,EAAE,aAAa;IAI/B;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CACV,iBAAiB,EAAE,MAAM,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,gBAAgB,CAAC;YAkJd,kBAAkB;CA0DjC"}

package/dist/extraction/scrape-session.js

@@ -0,0 +1,192 @@
+/**
+ * Multi-page scraping orchestrator.
+ *
+ * Combines content cleaning, extraction, pagination, deduplication,
+ * and optional detail-page following into a single scrape() call.
+ */
+import { extractAll } from './selector-engine.js';
+import { removeObstructions } from './content-cleaner.js';
+import { detectPagination, hasNextPage, goToNextPage } from './pagination.js';
+import { resolveUrl } from './transforms.js';
+export class ScrapeSession {
+    _proc;
+    _aborted = false;
+    constructor(proc) {
+        this._proc = proc;
+    }
+    /**
+     * Cancel a running scrape.
+     */
+    abort() {
+        this._aborted = true;
+    }
+    /**
+     * Run a multi-page scrape.
+     */
+    async scrape(containerSelector, options) {
+        this._aborted = false;
+        const startTime = performance.now();
+        const maxPages = options?.maxPages ?? 10;
+        const maxItems = options?.maxItems ?? Infinity;
+        const shouldClean = options?.clean !== false;
+        const pageDelay = options?.pageDelay ?? 500;
+        const deduplicateBy = options?.deduplicateBy;
+        const fields = options?.fields ?? {};
+        const allItems = [];
+        const pageUrls = [];
+        const seenKeys = new Set();
+        let complete = true;
+        let prevPageSignatures = null;
+        // Detect or use provided pagination
+        let pagination = options?.pagination ?? (await detectPagination(this._proc));
+        for (let page = 0; page < maxPages; page++) {
+            if (this._aborted) {
+                complete = false;
+                break;
+            }
+            // Track page URL
+            const currentUrl = await this._proc.getUrl();
+            pageUrls.push(currentUrl);
+            // Clean obstructions
+            if (shouldClean && page === 0) {
+                await removeObstructions(this._proc);
+            }
+            // Wait for container to appear (handles dynamic content loading)
+            try {
+                await this._proc.waitForSelector(containerSelector, 5000);
+            }
+            catch {
+                // Timeout — try extracting anyway
+            }
+            // Get HTML and extract
+            const html = await this._proc.getHtml('basic');
+            let pageItems = extractAll(html, containerSelector, fields);
+            // Empty page guard
+            if (pageItems.length === 0) {
+                if (page > 0) {
+                    // Retry once after a short wait (handles slow-loading SPAs)
+                    await this._proc.wait(2000);
+                    const retryHtml = await this._proc.getHtml('basic');
+                    pageItems = extractAll(retryHtml, containerSelector, fields);
+                }
+                if (pageItems.length === 0) {
+                    // No items found — stop scraping
+                    if (page > 0)
+                        complete = false;
+                    break;
+                }
+            }
+            // Stale page guard — detect if page didn't actually change
+            const currentSignatures = new Set(pageItems.map(item => JSON.stringify(item)));
+            if (prevPageSignatures && prevPageSignatures.size > 0) {
+                let overlapCount = 0;
+                for (const sig of currentSignatures) {
+                    if (prevPageSignatures.has(sig))
+                        overlapCount++;
+                }
+                const overlapRatio = overlapCount / Math.max(currentSignatures.size, 1);
+                if (overlapRatio > 0.8) {
+                    // Page didn't change — stop
+                    complete = false;
+                    break;
+                }
+            }
+            prevPageSignatures = currentSignatures;
+            // Deduplicate
+            if (deduplicateBy) {
+                pageItems = pageItems.filter(item => {
+                    const key = String(item[deduplicateBy] ?? '');
+                    if (!key || seenKeys.has(key))
+                        return false;
+                    seenKeys.add(key);
+                    return true;
+                });
+            }
+            // Follow detail pages if configured
+            if (options?.follow) {
+                pageItems = await this._followDetailPages(html, containerSelector, pageItems, options.follow, currentUrl);
+            }
+            // Fire callbacks
+            for (const item of pageItems) {
+                if (allItems.length >= maxItems) {
+                    complete = false;
+                    break;
+                }
+                allItems.push(item);
+                options?.onItem?.(item);
+            }
+            options?.onPage?.(page + 1, pageItems);
+            // Check limits
+            if (allItems.length >= maxItems) {
+                complete = false;
+                break;
+            }
+            // Navigate to next page
+            if (page < maxPages - 1 && pagination) {
+                const currentPageNum = (pagination.startPage ?? 1) + page;
+                const hasMore = await hasNextPage(this._proc, pagination, currentPageNum);
+                if (!hasMore)
+                    break;
+                const success = await goToNextPage(this._proc, pagination, currentPageNum);
+                if (!success)
+                    break;
+                if (pageDelay > 0) {
+                    await this._proc.wait(pageDelay);
+                }
+            }
+            else if (!pagination) {
+                break; // No pagination detected
+            }
+        }
+        return {
+            items: allItems,
+            totalItems: allItems.length,
+            pagesScraped: pageUrls.length,
+            pageUrls,
+            complete,
+            durationMs: performance.now() - startTime,
+        };
+    }
+    async _followDetailPages(html, containerSelector, items, follow, baseUrl) {
+        const enrichedItems = [];
+        for (const item of items) {
+            if (this._aborted) {
+                enrichedItems.push(item);
+                continue;
+            }
+            // Look for URL in common field names
+            let url = null;
+            for (const key of ['link', 'url', 'href']) {
+                const val = item[key];
+                if (typeof val === 'string' && val) {
+                    url = resolveUrl(val, baseUrl);
+                    break;
+                }
+            }
+            if (!url) {
+                enrichedItems.push(item);
+                continue;
+            }
+            try {
+                // Navigate to detail page
+                await this._proc.goto(url);
+                await this._proc.wait(follow.waitAfter ?? 1000);
+                // Extract detail fields
+                const detailHtml = await this._proc.getHtml('basic');
+                const detailContainerSelector = follow.containerSelector ?? 'body';
+                const { extractFirst } = await import('./selector-engine.js');
+                const detailData = extractFirst(detailHtml, detailContainerSelector, follow.fields);
+                // Merge detail data into item
+                enrichedItems.push({ ...item, ...detailData });
+                // Navigate back
+                await this._proc.evaluate('window.history.back()');
+                await this._proc.wait(500);
+            }
+            catch {
+                enrichedItems.push(item);
+            }
+        }
+        return enrichedItems;
+    }
+}
+//# sourceMappingURL=scrape-session.js.map

package/dist/extraction/scrape-session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape-session.js","sourceRoot":"","sources":["../../src/extraction/scrape-session.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,MAAM,OAAO,aAAa;IACP,KAAK,CAAgB;IAC9B,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,IAAmB;QAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,iBAAyB,EACzB,OAAuB;QAEvB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,QAAQ,CAAC;QAC/C,MAAM,WAAW,GAAG,OAAO,EAAE,KAAK,KAAK,KAAK,CAAC;QAC7C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;QAC5C,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAErC,MAAM,QAAQ,GAAsB,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACnC,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,kBAAkB,GAAuB,IAAI,CAAC;QAElD,oCAAoC;QACpC,IAAI,UAAU,GACZ,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAE9D,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,QAAQ,GAAG,KAAK,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,iBAAiB;YACjB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7C,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE1B,qBAAqB;YACrB,IAAI,WAAW,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvC,CAAC;YAED,iEAAiE;YACjE,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;YAED,uBAAuB;YACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAE5D,mBAAmB;YACnB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBACb,4DAA4D;oBAC5D,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oBACpD,SAAS,GAAG,UAAU,CAAC,SAAS,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;gBAC/D,CAAC;gBACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3B,iCAAiC;oBACjC,IAAI,IAAI,GAAG,CAAC;wBAAE,QAAQ,GAAG,KAAK,CAAC;oBAC/B,MAAM;gBACR,CAAC;YACH,CAAC;YAED,2DAA2D;YAC3D,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAC/B,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAC5C,CAAC;YACF,IAAI,kBAAkB,IAAI,kBAAkB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACtD,IAAI,YAAY,GAAG,CAAC,CAAC;gBACrB,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;oBACpC,IAAI,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,YAAY,EAAE,CAAC;gBAClD,CAAC;gBACD,MAAM,YAAY,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACxE,IAAI,YAAY,GAAG,GAAG,EAAE,CAAC;oBACvB,4BAA4B;oBAC5B,QAAQ,GAAG,KAAK,CAAC;oBACjB,MAAM;gBACR,CAAC;YACH,CAAC;YACD,kBAAkB,GAAG,iBAAiB,CAAC;YAEvC,cAAc;YACd,IAAI,aAAa,EAAE,CAAC;gBAClB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;oBAClC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC9C,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAC5C,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClB,OAAO,IAAI,CAAC;gBACd,CAAC,CAAC,CAAC;YACL,CAAC;YAED,oCAAoC;YACpC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;gBACpB,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CACvC,IAAI,EACJ,iBAAiB,EACjB,SAAS,EACT,OAAO,CAAC,MAAM,EACd,UAAU,CACX,CAAC;YACJ,CAAC;YAED,iBAAiB;YACjB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;oBAChC,QAAQ,GAAG,KAAK,CAAC;oBACjB,MAAM;gBACR,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpB,OAAO,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,SAAS,CAAC,CAAC;YAEvC,eAAe;YACf,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAChC,QAAQ,GAAG,KAAK,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,wBAAwB;YACxB,IAAI,IAAI,GAAG,QAAQ,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;gBACtC,MAAM,cAAc,GAAG,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;gBAC1D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;gBAC1E,IAAI,CAAC,OAAO;oBAAE,MAAM;gBAEpB,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;gBAC3E,IAAI,CAAC,OAAO;oBAAE,MAAM;gBAEpB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAClB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,CAAC,yBAAyB;YAClC,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,YAAY,EAAE,QAAQ,CAAC,MAAM;YAC7B,QAAQ;YACR,QAAQ;YACR,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;SAC1C,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,IAAY,EACZ,iBAAyB,EACzB,KAAwB,EACxB,MAA4C,EAC5C,OAAe;QAEf,MAAM,aAAa,GAAsB,EAAE,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzB,SAAS;YACX,CAAC;YAED,qCAAqC;YACrC,IAAI,GAAG,GAAkB,IAAI,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,EAAE,CAAC;oBACnC,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBAC/B,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzB,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,0BAA0B;gBAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;gBAEhD,wBAAwB;gBACxB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrD,MAAM,uBAAuB,GAAG,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC;gBACnE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;gBAC9D,MAAM,UAAU,GAAG,YAAY,CAC7B,UAAU,EACV,uBAAuB,EACvB,MAAM,CAAC,MAAM,CACd,CAAC;gBAEF,8BAA8B;gBAC9B,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;gBAE/C,gBAAgB;gBAChB,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;gBACnD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}