khoji 2.1.4

package/dist/extractors/AnimationExtractor.js

@@ -0,0 +1,247 @@
+/**
+ * AnimationExtractor — 3-pass animation intelligence:
+ *
+ * Pass 1: CSS @keyframes + transitions (from document.styleSheets)
+ * Pass 2: JS animation library detection (GSAP, Framer Motion, AOS, Lottie, etc.)
+ * Pass 3: GIF purpose inference from context
+ */
+export async function extractAnimations(page, gifs) {
+    const [cssResult, jsResult] = await Promise.all([
+        extractCSSAnimations(page),
+        extractJSAnimations(page),
+    ]);
+    const gifAnimations = inferGifPurpose(gifs);
+    const summary = buildSummary(cssResult.animations, cssResult.transitions, jsResult.jsAnimations, jsResult.scrollAnimations, gifAnimations);
+    return {
+        cssAnimations: cssResult.animations,
+        cssTransitions: cssResult.transitions,
+        jsAnimations: jsResult.jsAnimations,
+        scrollAnimations: jsResult.scrollAnimations,
+        gifAnimations,
+        summary,
+    };
+}
+// ─── Pass 1: CSS ──────────────────────────────────────────────────────────────
+async function extractCSSAnimations(page) {
+    return page.evaluate(() => {
+        const keyframeMap = new Map(); // name → description of steps
+        const animationRules = [];
+        const transitionRules = [];
+        // Collect all @keyframes first
+        for (const sheet of Array.from(document.styleSheets)) {
+            let rules;
+            try {
+                rules = sheet.cssRules;
+            }
+            catch {
+                continue;
+            }
+            for (const rule of Array.from(rules)) {
+                if (rule instanceof CSSKeyframesRule) {
+                    const steps = Array.from(rule.cssRules)
+                        .map((r) => `${r.keyText}: ${r.style.cssText.slice(0, 80)}`)
+                        .join('; ');
+                    keyframeMap.set(rule.name, steps);
+                }
+            }
+        }
+        // Now map animation-name on elements back to keyframes
+        const processedSelectors = new Set();
+        for (const sheet of Array.from(document.styleSheets)) {
+            let rules;
+            try {
+                rules = sheet.cssRules;
+            }
+            catch {
+                continue;
+            }
+            for (const rule of Array.from(rules)) {
+                if (!(rule instanceof CSSStyleRule))
+                    continue;
+                const style = rule.style;
+                // CSS animations
+                const animName = style.getPropertyValue('animation-name') ||
+                    style.getPropertyValue('animation')?.split(' ')[0] || '';
+                if (animName && animName !== 'none' && !processedSelectors.has(`anim:${rule.selectorText}`)) {
+                    processedSelectors.add(`anim:${rule.selectorText}`);
+                    const keyframeDesc = keyframeMap.get(animName) ?? '';
+                    const trigger = rule.selectorText.includes(':hover') ? 'hover'
+                        : rule.selectorText.includes(':focus') ? 'focus'
+                            : 'page-load';
+                    animationRules.push({
+                        name: animName,
+                        selector: rule.selectorText.slice(0, 120),
+                        duration: style.getPropertyValue('animation-duration') || '1s',
+                        timingFunction: style.getPropertyValue('animation-timing-function') || 'ease',
+                        iterationCount: style.getPropertyValue('animation-iteration-count') || '1',
+                        delay: style.getPropertyValue('animation-delay') || '0s',
+                        trigger,
+                        description: keyframeDesc
+                            ? `"${animName}" — ${keyframeDesc.slice(0, 150)}`
+                            : `Animation "${animName}" applied`,
+                    });
+                }
+                // CSS transitions
+                const transitionProp = style.getPropertyValue('transition');
+                if (transitionProp && transitionProp !== 'none' && !processedSelectors.has(`trans:${rule.selectorText}`)) {
+                    processedSelectors.add(`trans:${rule.selectorText}`);
+                    const trigger = rule.selectorText.includes(':hover') ? 'hover'
+                        : rule.selectorText.includes(':focus') ? 'focus'
+                            : rule.selectorText.includes(':active') ? 'active'
+                                : 'unknown';
+                    const props = transitionProp.split(',').map((t) => t.trim().split(' ')[0]);
+                    transitionRules.push({
+                        selector: rule.selectorText.slice(0, 120),
+                        properties: props,
+                        duration: transitionProp.match(/[\d.]+s/)?.[0] ?? '0.3s',
+                        timingFunction: transitionProp.match(/ease[a-z-]*|linear|cubic-bezier\([^)]+\)/)?.[0] ?? 'ease',
+                        trigger: trigger,
+                        description: `Transition on ${props.join(', ')} — ${trigger}`,
+                    });
+                }
+            }
+        }
+        return {
+            animations: animationRules.slice(0, 50),
+            transitions: transitionRules.slice(0, 50),
+        };
+    });
+}
+// ─── Pass 2: JS Libraries ─────────────────────────────────────────────────────
+async function extractJSAnimations(page) {
+    return page.evaluate(() => {
+        const win = window;
+        const jsAnimations = [];
+        const scrollAnimations = [];
+        // GSAP
+        if (win['gsap']) {
+            jsAnimations.push({
+                library: 'gsap',
+                selector: 'document',
+                description: 'GSAP detected — timeline-based animation library in use',
+                trigger: 'page-load',
+            });
+        }
+        // Framer Motion
+        if (win['Motion'] || win['__framer_motion__'] || document.querySelector('[data-framer-motion]')) {
+            jsAnimations.push({
+                library: 'framer-motion',
+                selector: '[data-framer-motion]',
+                description: 'Framer Motion detected — declarative React animation library',
+                trigger: 'page-load',
+            });
+        }
+        // Anime.js
+        if (win['anime']) {
+            jsAnimations.push({
+                library: 'animejs',
+                selector: 'document',
+                description: 'Anime.js detected — lightweight JavaScript animation library',
+                trigger: 'page-load',
+            });
+        }
+        // Lottie
+        if (win['lottie'] || win['Lottie'] || document.querySelector('lottie-player, [data-lottie]')) {
+            jsAnimations.push({
+                library: 'lottie',
+                selector: 'lottie-player, [data-lottie]',
+                description: 'Lottie detected — JSON-based vector animations in use',
+                trigger: 'page-load',
+            });
+        }
+        // Web Animations API
+        if (typeof Element.prototype.animate === 'function') {
+            // Only flag if scripts are actually using it (heuristic: check for animate() on visible elements)
+            const hasWaapi = document.querySelectorAll('[data-animate], .animate').length > 0;
+            if (hasWaapi) {
+                jsAnimations.push({
+                    library: 'web-animations-api',
+                    selector: '[data-animate], .animate',
+                    description: 'Web Animations API in use',
+                    trigger: 'page-load',
+                });
+            }
+        }
+        // AOS (scroll animations)
+        if (win['AOS'] || document.querySelector('[data-aos]')) {
+            const aosEls = Array.from(document.querySelectorAll('[data-aos]')).slice(0, 20);
+            aosEls.forEach((el) => {
+                scrollAnimations.push({
+                    selector: el.tagName.toLowerCase() + (el.className ? `.${el.className.toString().split(' ')[0]}` : ''),
+                    library: 'aos',
+                    animationType: el.getAttribute('data-aos') ?? 'fade',
+                    description: `AOS scroll animation: ${el.getAttribute('data-aos') ?? 'fade'} — triggers when element enters viewport`,
+                });
+            });
+        }
+        // GSAP ScrollTrigger heuristic
+        if (win['ScrollTrigger'] || (win['gsap'] && document.querySelector('[data-scroll]'))) {
+            document.querySelectorAll('[data-scroll], [data-scroll-trigger]').forEach((el) => {
+                scrollAnimations.push({
+                    selector: el.tagName.toLowerCase(),
+                    library: 'gsap-scrolltrigger',
+                    animationType: el.getAttribute('data-scroll') ?? 'scroll-driven',
+                    description: 'GSAP ScrollTrigger — animation tied to scroll position',
+                });
+            });
+        }
+        return { jsAnimations, scrollAnimations };
+    });
+}
+// ─── Pass 3: GIF Purpose Inference ───────────────────────────────────────────
+function inferGifPurpose(gifs) {
+    return gifs.map((gif) => {
+        const combined = `${gif.alt} ${gif.selector} ${gif.url}`.toLowerCase();
+        let purpose = 'unknown';
+        if (/load|spin|wait|progress|preload|buffer/.test(combined)) {
+            purpose = 'loading-spinner';
+        }
+        else if (/demo|preview|product|feature|how|tutorial|guide|walkthrough|example/.test(combined)) {
+            purpose = gif.alt.toLowerCase().includes('tutorial') || gif.selector.includes('tutorial')
+                ? 'tutorial'
+                : 'product-demo';
+        }
+        else if (/bg|background|decoration|hero|banner|pattern|abstract/.test(combined)) {
+            purpose = 'decorative';
+        }
+        else if (gif.alt.length === 0 && gif.selector.includes('bg')) {
+            purpose = 'decorative';
+        }
+        return {
+            url: gif.url,
+            selector: gif.selector,
+            alt: gif.alt,
+            purpose,
+        };
+    });
+}
+// ─── Summary Builder ──────────────────────────────────────────────────────────
+function buildSummary(cssAnimations, cssTransitions, jsAnimations, scrollAnimations, gifAnimations) {
+    const parts = [];
+    if (cssAnimations.length > 0) {
+        const names = [...new Set(cssAnimations.map((a) => a.name))].slice(0, 3).join(', ');
+        parts.push(`CSS animations: ${names} (${cssAnimations.length} total)`);
+    }
+    if (cssTransitions.length > 0) {
+        const hoverCount = cssTransitions.filter((t) => t.trigger === 'hover').length;
+        parts.push(`${cssTransitions.length} CSS transitions${hoverCount > 0 ? ` (${hoverCount} on hover)` : ''}`);
+    }
+    if (jsAnimations.length > 0) {
+        const libs = [...new Set(jsAnimations.map((j) => j.library))].join(', ');
+        parts.push(`JS: ${libs}`);
+    }
+    if (scrollAnimations.length > 0) {
+        const lib = scrollAnimations[0]?.library ?? 'scroll';
+        parts.push(`${scrollAnimations.length} scroll animations (${lib})`);
+    }
+    if (gifAnimations.length > 0) {
+        const spinners = gifAnimations.filter((g) => g.purpose === 'loading-spinner').length;
+        const demos = gifAnimations.filter((g) => g.purpose === 'product-demo').length;
+        const detail = [spinners > 0 && `${spinners} spinner`, demos > 0 && `${demos} demo`]
+            .filter(Boolean)
+            .join(', ');
+        parts.push(`${gifAnimations.length} GIF${gifAnimations.length > 1 ? 's' : ''}${detail ? ` (${detail})` : ''}`);
+    }
+    return parts.length > 0 ? parts.join('. ') + '.' : 'No animations detected.';
+}
+//# sourceMappingURL=AnimationExtractor.js.map

package/dist/extractors/AnimationExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AnimationExtractor.js","sourceRoot":"","sources":["../../src/extractors/AnimationExtractor.ts"],"names":[],"mappings":"AAcA;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACnC,IAAU,EACV,IAAkB;IAElB,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,oBAAoB,CAAC,IAAI,CAAC;QAC1B,mBAAmB,CAAC,IAAI,CAAC;KAC5B,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,YAAY,CACxB,SAAS,CAAC,UAAU,EACpB,SAAS,CAAC,WAAW,EACrB,QAAQ,CAAC,YAAY,EACrB,QAAQ,CAAC,gBAAgB,EACzB,aAAa,CAChB,CAAC;IAEF,OAAO;QACH,aAAa,EAAE,SAAS,CAAC,UAAU;QACnC,cAAc,EAAE,SAAS,CAAC,WAAW;QACrC,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;QAC3C,aAAa;QACb,OAAO;KACV,CAAC;AACN,CAAC;AAED,iFAAiF;AAEjF,KAAK,UAAU,oBAAoB,CAC/B,IAAU;IAEV,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAiE,EAAE;QACpF,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAG,8BAA8B;QAC/E,MAAM,cAAc,GAAmB,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAoB,EAAE,CAAC;QAE5C,+BAA+B;QAC/B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnD,IAAI,KAAkB,CAAC;YACvB,IAAI,CAAC;gBAAC,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS;YAAC,CAAC;YAEnD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,IAAI,IAAI,YAAY,gBAAgB,EAAE,CAAC;oBACnC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;yBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAI,CAAqB,CAAC,OAAO,KAAM,CAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;yBACrG,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChB,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACtC,CAAC;YACL,CAAC;QACL,CAAC;QAED,uDAAuD;QACvD,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE7C,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnD,IAAI,KAAkB,CAAC;YACvB,IAAI,CAAC;gBAAC,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS;YAAC,CAAC;YAEnD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,CAAC,IAAI,YAAY,YAAY,CAAC;oBAAE,SAAS;gBAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBAEzB,iBAAiB;gBACjB,MAAM,QAAQ,GAAG,KAAK,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;oBACrD,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE7D,IAAI,QAAQ,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;oBAC1F,kBAAkB,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;oBACpD,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACrD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO;wBAC1D,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO;4BAC5C,CAAC,CAAC,WAAW,CAAC;oBAEtB,cAAc,CAAC,IAAI,CAAC;wBAChB,IAAI,EAAE,QAAQ;wBACd,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;wBACzC,QAAQ,EAAE,KAAK,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,IAAI;wBAC9D,cAAc,EAAE,KAAK,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,IAAI,MAAM;wBAC7E,cAAc,EAAE,KAAK,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,IAAI,GAAG;wBAC1E,KAAK,EAAE,KAAK,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,IAAI,IAAI;wBACxD,OAAO;wBACP,WAAW,EAAE,YAAY;4BACrB,CAAC,CAAC,IAAI,QAAQ,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;4BACjD,CAAC,CAAC,cAAc,QAAQ,WAAW;qBAC1C,CAAC,CAAC;gBACP,CAAC;gBAED,kBAAkB;gBAClB,MAAM,cAAc,GAAG,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;gBAC5D,IAAI,cAAc,IAAI,cAAc,KAAK,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;oBACvG,kBAAkB,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;oBACrD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO;wBAC1D,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO;4BAC5C,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ;gCAC9C,CAAC,CAAC,SAAS,CAAC;oBAExB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAE3E,eAAe,CAAC,IAAI,CAAC;wBACjB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;wBACzC,UAAU,EAAE,KAAK;wBACjB,QAAQ,EAAE,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM;wBACxD,cAAc,EAAE,cAAc,CAAC,KAAK,CAAC,0CAA0C,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM;wBAC/F,OAAO,EAAE,OAAmC;wBAC5C,WAAW,EAAE,iBAAiB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO,EAAE;qBAChE,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO;YACH,UAAU,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACvC,WAAW,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SAC5C,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC;AAED,iFAAiF;AAEjF,KAAK,UAAU,mBAAmB,CAC9B,IAAU;IAEV,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAyE,EAAE;QAC5F,MAAM,GAAG,GAAG,MAA4C,CAAC;QACzD,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,MAAM,gBAAgB,GAAsB,EAAE,CAAC;QAE/C,OAAO;QACP,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACd,YAAY,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE,UAAU;gBACpB,WAAW,EAAE,yDAAyD;gBACtE,OAAO,EAAE,WAAW;aACvB,CAAC,CAAC;QACP,CAAC;QAED,gBAAgB;QAChB,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,sBAAsB,CAAC,EAAE,CAAC;YAC9F,YAAY,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,eAAe;gBACxB,QAAQ,EAAE,sBAAsB;gBAChC,WAAW,EAAE,8DAA8D;gBAC3E,OAAO,EAAE,WAAW;aACvB,CAAC,CAAC;QACP,CAAC;QAED,WAAW;QACX,IAAI,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACf,YAAY,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,SAAS;gBAClB,QAAQ,EAAE,UAAU;gBACpB,WAAW,EAAE,8DAA8D;gBAC3E,OAAO,EAAE,WAAW;aACvB,CAAC,CAAC;QACP,CAAC;QAED,SAAS;QACT,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,8BAA8B,CAAC,EAAE,CAAC;YAC3F,YAAY,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,QAAQ;gBACjB,QAAQ,EAAE,8BAA8B;gBACxC,WAAW,EAAE,uDAAuD;gBACpE,OAAO,EAAE,WAAW;aACvB,CAAC,CAAC;QACP,CAAC;QAED,qBAAqB;QACrB,IAAI,OAAO,OAAO,CAAC,SAAS,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAClD,kGAAkG;YAClG,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAClF,IAAI,QAAQ,EAAE,CAAC;gBACX,YAAY,CAAC,IAAI,CAAC;oBACd,OAAO,EAAE,oBAAoB;oBAC7B,QAAQ,EAAE,0BAA0B;oBACpC,WAAW,EAAE,2BAA2B;oBACxC,OAAO,EAAE,WAAW;iBACvB,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,0BAA0B;QAC1B,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChF,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;gBAClB,gBAAgB,CAAC,IAAI,CAAC;oBAClB,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtG,OAAO,EAAE,KAAK;oBACd,aAAa,EAAE,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,MAAM;oBACpD,WAAW,EAAE,yBAAyB,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,MAAM,0CAA0C;iBACxH,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;QAED,+BAA+B;QAC/B,IAAI,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;YACnF,QAAQ,CAAC,gBAAgB,CAAC,sCAAsC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;gBAC7E,gBAAgB,CAAC,IAAI,CAAC;oBAClB,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE;oBAClC,OAAO,EAAE,oBAAoB;oBAC7B,aAAa,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,eAAe;oBAChE,WAAW,EAAE,wDAAwD;iBACxE,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;AACP,CAAC;AAED,gFAAgF;AAEhF,SAAS,eAAe,CAAC,IAAkB;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACpB,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QAEvE,IAAI,OAAO,GAAe,SAAS,CAAC;QAEpC,IAAI,wCAAwC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1D,OAAO,GAAG,iBAAiB,CAAC;QAChC,CAAC;aAAM,IAAI,qEAAqE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9F,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACrF,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,cAAc,CAAC;QACzB,CAAC;aAAM,IAAI,uDAAuD,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChF,OAAO,GAAG,YAAY,CAAC;QAC3B,CAAC;aAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7D,OAAO,GAAG,YAAY,CAAC;QAC3B,CAAC;QAED,OAAO;YACH,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,OAAO;SACV,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC;AAED,iFAAiF;AAEjF,SAAS,YAAY,CACjB,aAA6B,EAC7B,cAA+B,EAC/B,YAA2B,EAC3B,gBAAmC,EACnC,aAA6B;IAE7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpF,KAAK,CAAC,IAAI,CAAC,mBAAmB,KAAK,KAAK,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,mBAAmB,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/G,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzE,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,QAAQ,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,iBAAiB,CAAC,CAAC,MAAM,CAAC;QACrF,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;QAC/E,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,QAAQ,UAAU,EAAE,KAAK,GAAG,CAAC,IAAI,GAAG,KAAK,OAAO,CAAC;aAC/E,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,OAAO,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACnH,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,yBAAyB,CAAC;AACjF,CAAC"}

package/dist/extractors/AssetExtractor.d.ts

@@ -0,0 +1,11 @@
+import type { Page } from 'playwright';
+import type { AssetMap } from '../types/KhojContext.js';
+/**
+ * Collects all assets from the page:
+ * - Images and GIFs with metadata (alt, dimensions, lazy-load flag, CSS selector)
+ * - External scripts
+ * - Favicon and icon links
+ * - Font URLs from @font-face rules
+ */
+export declare function extractAssets(page: Page, baseUrl: string): Promise<AssetMap>;
+//# sourceMappingURL=AssetExtractor.d.ts.map

package/dist/extractors/AssetExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AssetExtractor.d.ts","sourceRoot":"","sources":["../../src/extractors/AssetExtractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAc,MAAM,yBAAyB,CAAC;AAEpE;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CA2HlF"}

package/dist/extractors/AssetExtractor.js

@@ -0,0 +1,124 @@
+/**
+ * Collects all assets from the page:
+ * - Images and GIFs with metadata (alt, dimensions, lazy-load flag, CSS selector)
+ * - External scripts
+ * - Favicon and icon links
+ * - Font URLs from @font-face rules
+ */
+export async function extractAssets(page, baseUrl) {
+    const result = await page.evaluate(({ base }) => {
+        function toAbsolute(url) {
+            if (!url || url.startsWith('data:'))
+                return url;
+            try {
+                return new URL(url, base).href;
+            }
+            catch {
+                return url;
+            }
+        }
+        function getCssSelector(el) {
+            if (el.id)
+                return `#${el.id}`;
+            const parts = [];
+            let current = el;
+            while (current && current !== document.body && parts.length < 4) {
+                let selector = current.tagName.toLowerCase();
+                if (current.className) {
+                    const firstClass = current.className.toString().trim().split(/\s+/)[0];
+                    if (firstClass)
+                        selector += `.${firstClass}`;
+                }
+                parts.unshift(selector);
+                current = current.parentElement;
+            }
+            return parts.join(' > ');
+        }
+        function isGif(url) {
+            return url.toLowerCase().includes('.gif') || url.toLowerCase().endsWith('.gif');
+        }
+        // ── Images ────────────────────────────────────────────────────────────
+        const rawImages = [];
+        document.querySelectorAll('img').forEach((img) => {
+            const src = img.getAttribute('src') ?? img.getAttribute('data-src') ?? '';
+            if (!src || src.startsWith('data:'))
+                return;
+            const absoluteSrc = toAbsolute(src);
+            rawImages.push({
+                url: absoluteSrc,
+                alt: img.getAttribute('alt') ?? '',
+                type: isGif(absoluteSrc) ? 'gif' : 'image',
+                width: img.naturalWidth || parseInt(img.getAttribute('width') ?? '0') || null,
+                height: img.naturalHeight || parseInt(img.getAttribute('height') ?? '0') || null,
+                isLazy: img.getAttribute('loading') === 'lazy' || !!img.getAttribute('data-src'),
+                selector: getCssSelector(img),
+            });
+        });
+        // ── Scripts ───────────────────────────────────────────────────────────
+        const scripts = [];
+        document.querySelectorAll('script[src]').forEach((s) => {
+            const src = s.src;
+            if (src && !src.startsWith(window.location.origin)) {
+                scripts.push(toAbsolute(src));
+            }
+        });
+        // ── Icons / Favicons ──────────────────────────────────────────────────
+        const icons = [];
+        document.querySelectorAll('link[rel~="icon"], link[rel~="apple-touch-icon"], link[rel~="shortcut"]').forEach((l) => {
+            const href = l.href;
+            if (href)
+                icons.push(toAbsolute(href));
+        });
+        // ── Fonts ─────────────────────────────────────────────────────────────
+        const fonts = [];
+        for (const sheet of Array.from(document.styleSheets)) {
+            try {
+                for (const rule of Array.from(sheet.cssRules)) {
+                    if (rule instanceof CSSFontFaceRule) {
+                        const src = rule.style.getPropertyValue('src');
+                        const matches = src.match(/url\(['"]?([^'")\s]+)['"]?\)/g) ?? [];
+                        matches.forEach((m) => {
+                            const urlMatch = m.match(/url\(['"]?([^'")\s]+)['"]?\)/);
+                            if (urlMatch?.[1])
+                                fonts.push(toAbsolute(urlMatch[1]));
+                        });
+                    }
+                }
+            }
+            catch {
+                continue;
+            }
+        }
+        return { rawImages, images: [], scripts: [...new Set(scripts)], icons: [...new Set(icons)], fonts: [...new Set(fonts)] };
+    }, { base: baseUrl });
+    // Retrieve GIFs intercepted from network responses
+    const networkGifs = await page.evaluate(() => {
+        return window.__khoj_gifs__ ?? [];
+    });
+    const allImages = result.rawImages;
+    // Merge network-discovered GIFs that weren't in <img> tags
+    const knownUrls = new Set(allImages.map((i) => i.url));
+    for (const gifUrl of networkGifs) {
+        if (!knownUrls.has(gifUrl)) {
+            allImages.push({
+                url: gifUrl,
+                alt: '',
+                type: 'gif',
+                width: null,
+                height: null,
+                isLazy: false,
+                selector: '',
+            });
+        }
+    }
+    const images = allImages.filter((i) => i.type === 'image');
+    const gifs = allImages.filter((i) => i.type === 'gif');
+    return {
+        images,
+        gifs,
+        fonts: result.fonts,
+        icons: result.icons,
+        scripts: result.scripts,
+    };
+}
+//# sourceMappingURL=AssetExtractor.js.map

package/dist/extractors/AssetExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AssetExtractor.js","sourceRoot":"","sources":["../../src/extractors/AssetExtractor.ts"],"names":[],"mappings":"AAGA;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,OAAe;IAC3D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAC9B,CAAC,EAAE,IAAI,EAAoB,EAAwD,EAAE;QACjF,SAAS,UAAU,CAAC,GAAW;YAC3B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO,GAAG,CAAC;YAChD,IAAI,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO,GAAG,CAAC;YACf,CAAC;QACL,CAAC;QAED,SAAS,cAAc,CAAC,EAAW;YAC/B,IAAI,EAAE,CAAC,EAAE;gBAAE,OAAO,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,OAAO,GAAmB,EAAE,CAAC;YACjC,OAAO,OAAO,IAAI,OAAO,KAAK,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9D,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC7C,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACpB,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBACvE,IAAI,UAAU;wBAAE,QAAQ,IAAI,IAAI,UAAU,EAAE,CAAC;gBACjD,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACxB,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC;YACpC,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QAED,SAAS,KAAK,CAAC,GAAW;YACtB,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpF,CAAC;QAED,yEAAyE;QACzE,MAAM,SAAS,GAAiB,EAAE,CAAC;QACnC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAC1E,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YACpC,SAAS,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,WAAW;gBAChB,GAAG,EAAE,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE;gBAClC,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;gBAC1C,KAAK,EAAE,GAAG,CAAC,YAAY,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI;gBAC7E,MAAM,EAAE,GAAG,CAAC,aAAa,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI;gBAChF,MAAM,EAAE,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,KAAK,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC;gBAChF,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;aAChC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,yEAAyE;QACzE,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,QAAQ,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YACnD,MAAM,GAAG,GAAI,CAAuB,CAAC,GAAG,CAAC;YACzC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,yEAAyE;QACzE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,QAAQ,CAAC,gBAAgB,CAAC,yEAAyE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/G,MAAM,IAAI,GAAI,CAAqB,CAAC,IAAI,CAAC;YACzC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,yEAAyE;QACzE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5C,IAAI,IAAI,YAAY,eAAe,EAAE,CAAC;wBAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;wBAC/C,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,IAAI,EAAE,CAAC;wBACjE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;4BAClB,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;4BACzD,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC;gCAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC3D,CAAC,CAAC,CAAC;oBACP,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;QACL,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;IAC7H,CAAC,EACD,EAAE,IAAI,EAAE,OAAO,EAAE,CACpB,CAAC;IAEF,mDAAmD;IACnD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAa,EAAE;QACnD,OAAQ,MAAgD,CAAC,aAAa,IAAI,EAAE,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IAEnC,2DAA2D;IAC3D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,MAAM;gBACX,GAAG,EAAE,EAAE;gBACP,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,EAAE;aACf,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IAEvD,OAAO;QACH,MAAM;QACN,IAAI;QACJ,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;KAC1B,CAAC;AACN,CAAC"}

package/dist/extractors/ContentExtractor.d.ts

@@ -0,0 +1,13 @@
+import type { Page } from 'playwright';
+import type { ContentBlock } from '../types/KhojContext.js';
+/**
+ * Extracts structured content blocks:
+ * - Headings h1–h6 with level
+ * - Paragraphs (truncated to 300 chars)
+ * - Button labels
+ * - Anchor links with href
+ * - Form labels
+ * Deduplicates by text.
+ */
+export declare function extractContent(page: Page): Promise<ContentBlock[]>;
+//# sourceMappingURL=ContentExtractor.d.ts.map

package/dist/extractors/ContentExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ContentExtractor.d.ts","sourceRoot":"","sources":["../../src/extractors/ContentExtractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAI5D;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAuDxE"}

package/dist/extractors/ContentExtractor.js

@@ -0,0 +1,60 @@
+const MAX_PARAGRAPH_LENGTH = 300;
+/**
+ * Extracts structured content blocks:
+ * - Headings h1–h6 with level
+ * - Paragraphs (truncated to 300 chars)
+ * - Button labels
+ * - Anchor links with href
+ * - Form labels
+ * Deduplicates by text.
+ */
+export async function extractContent(page) {
+    return page.evaluate(({ maxParaLen }) => {
+        const blocks = [];
+        const seen = new Set();
+        function add(block) {
+            const key = `${block.type}:${block.text}`;
+            if (!seen.has(key) && block.text.length > 0) {
+                seen.add(key);
+                blocks.push(block);
+            }
+        }
+        // Headings
+        for (let level = 1; level <= 6; level++) {
+            document.querySelectorAll(`h${level}`).forEach((el) => {
+                const text = el.textContent?.trim() ?? '';
+                if (text)
+                    add({ type: 'heading', text, level });
+            });
+        }
+        // Paragraphs
+        document.querySelectorAll('p').forEach((el) => {
+            const text = (el.textContent?.trim() ?? '').slice(0, maxParaLen);
+            if (text.split(' ').length >= 3) {
+                add({ type: 'paragraph', text });
+            }
+        });
+        // Buttons
+        document.querySelectorAll('button, [role="button"], input[type="submit"], input[type="button"]').forEach((el) => {
+            const text = el.textContent?.trim() ?? el.value ?? '';
+            if (text)
+                add({ type: 'button', text });
+        });
+        // Links
+        document.querySelectorAll('a[href]').forEach((el) => {
+            const text = el.textContent?.trim() ?? '';
+            const href = el.href;
+            if (text && href && !href.startsWith('javascript:')) {
+                add({ type: 'link', text, href });
+            }
+        });
+        // Labels
+        document.querySelectorAll('label').forEach((el) => {
+            const text = el.textContent?.trim() ?? '';
+            if (text)
+                add({ type: 'label', text });
+        });
+        return blocks;
+    }, { maxParaLen: MAX_PARAGRAPH_LENGTH });
+}
+//# sourceMappingURL=ContentExtractor.js.map

package/dist/extractors/ContentExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ContentExtractor.js","sourceRoot":"","sources":["../../src/extractors/ContentExtractor.ts"],"names":[],"mappings":"AAGA,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAU;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAChB,CAAC,EAAE,UAAU,EAA0B,EAAkB,EAAE;QACvD,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE/B,SAAS,GAAG,CAAC,KAAmB;YAC5B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;QAED,WAAW;QACX,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACtC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;gBAClD,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAC1C,IAAI,IAAI;oBAAE,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;QACP,CAAC;QAED,aAAa;QACb,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC1C,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YACjE,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC9B,GAAG,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YACrC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,QAAQ,CAAC,gBAAgB,CAAC,qEAAqE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC5G,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAK,EAAuB,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5E,IAAI,IAAI;gBAAE,GAAG,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,QAAQ;QACR,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAChD,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAI,EAAwB,CAAC,IAAI,CAAC;YAC5C,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBAClD,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,SAAS;QACT,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC9C,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC1C,IAAI,IAAI;gBAAE,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAClB,CAAC,EACD,EAAE,UAAU,EAAE,oBAAoB,EAAE,CACvC,CAAC;AACN,CAAC"}

package/dist/extractors/DomExtractor.d.ts

@@ -0,0 +1,11 @@
+import type { Page } from 'playwright';
+import type { DomNode } from '../types/KhojContext.js';
+/**
+ * Extracts a semantic DOM tree from the page body.
+ * - Skips noise tags (script, style, svg, etc.)
+ * - Caps depth at 10 levels
+ * - Trims text content to 200 characters
+ * - Captures: tag, id, classes, role, text
+ */
+export declare function extractDom(page: Page): Promise<DomNode[]>;
+//# sourceMappingURL=DomExtractor.d.ts.map

package/dist/extractors/DomExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DomExtractor.d.ts","sourceRoot":"","sources":["../../src/extractors/DomExtractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAYvD;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CA4D/D"}

package/dist/extractors/DomExtractor.js

@@ -0,0 +1,68 @@
+// Tags to skip entirely — they add noise, not signal
+const NOISE_TAGS = new Set([
+    'script', 'style', 'noscript', 'svg', 'path', 'defs',
+    'symbol', 'use', 'clippath', 'head', 'meta', 'link',
+    'br', 'hr', 'wbr', 'template', 'iframe',
+]);
+const MAX_DEPTH = 10;
+const MAX_TEXT_LENGTH = 200;
+/**
+ * Extracts a semantic DOM tree from the page body.
+ * - Skips noise tags (script, style, svg, etc.)
+ * - Caps depth at 10 levels
+ * - Trims text content to 200 characters
+ * - Captures: tag, id, classes, role, text
+ */
+export async function extractDom(page) {
+    return page.evaluate(({ noiseTags, maxDepth, maxTextLen }) => {
+        const noiseSet = new Set(noiseTags);
+        function walk(el, depth) {
+            if (depth > maxDepth)
+                return null;
+            const tag = el.tagName.toLowerCase();
+            if (noiseSet.has(tag))
+                return null;
+            const children = [];
+            for (const child of Array.from(el.children)) {
+                const node = walk(child, depth + 1);
+                if (node)
+                    children.push(node);
+            }
+            // Get direct text (not from children)
+            let text;
+            const directText = Array.from(el.childNodes)
+                .filter((n) => n.nodeType === Node.TEXT_NODE)
+                .map((n) => n.textContent?.trim() ?? '')
+                .join(' ')
+                .trim();
+            if (directText.length > 0) {
+                text = directText.slice(0, maxTextLen);
+            }
+            const node = {
+                tag,
+                classes: Array.from(el.classList),
+                children: children.length > 0 ? children : undefined,
+            };
+            const id = el.getAttribute('id');
+            if (id)
+                node.id = id;
+            const role = el.getAttribute('role');
+            if (role)
+                node.role = role;
+            if (text)
+                node.text = text;
+            // Prune leaf nodes with no info
+            if (!text && !node.id && node.classes.length === 0 && !node.role && !children.length) {
+                return null;
+            }
+            return node;
+        }
+        const body = document.body;
+        if (!body)
+            return [];
+        return Array.from(body.children)
+            .map((child) => walk(child, 1))
+            .filter((n) => n !== null);
+    }, { noiseTags: Array.from(NOISE_TAGS), maxDepth: MAX_DEPTH, maxTextLen: MAX_TEXT_LENGTH });
+}
+//# sourceMappingURL=DomExtractor.js.map

package/dist/extractors/DomExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DomExtractor.js","sourceRoot":"","sources":["../../src/extractors/DomExtractor.ts"],"names":[],"mappings":"AAGA,qDAAqD;AACrD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACvB,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IACpD,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACnD,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ;CAC1C,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAU;IACvC,OAAO,IAAI,CAAC,QAAQ,CAChB,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAiE,EAAa,EAAE;QAC9G,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAEpC,SAAS,IAAI,CAAC,EAAW,EAAE,KAAa;YACpC,IAAI,KAAK,GAAG,QAAQ;gBAAE,OAAO,IAAI,CAAC;YAElC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACrC,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAEnC,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBACpC,IAAI,IAAI;oBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,CAAC;YAED,sCAAsC;YACtC,IAAI,IAAwB,CAAC;YAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC;iBACvC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,SAAS,CAAC;iBAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;iBACvC,IAAI,CAAC,GAAG,CAAC;iBACT,IAAI,EAAE,CAAC;YAEZ,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,IAAI,GAAY;gBAClB,GAAG;gBACH,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC;gBACjC,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;aACvD,CAAC;YAEF,MAAM,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,EAAE;gBAAE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YAErB,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACrC,IAAI,IAAI;gBAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAE3B,IAAI,IAAI;gBAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAE3B,gCAAgC;YAChC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnF,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC3B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;aAC9B,MAAM,CAAC,CAAC,CAAC,EAAgB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACjD,CAAC,EACD,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,CAC1F,CAAC;AACN,CAAC"}

package/dist/extractors/InteractionExtractor.d.ts

@@ -0,0 +1,10 @@
+import type { Page } from 'playwright';
+import type { Interaction } from '../types/KhojContext.js';
+/**
+ * Maps interactive regions of the page:
+ * - Forms with all their input fields
+ * - Navigation elements
+ * - Interactive button-like elements
+ */
+export declare function extractInteractions(page: Page): Promise<Interaction[]>;
+//# sourceMappingURL=InteractionExtractor.d.ts.map

package/dist/extractors/InteractionExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"InteractionExtractor.d.ts","sourceRoot":"","sources":["../../src/extractors/InteractionExtractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,WAAW,EAAa,MAAM,yBAAyB,CAAC;AAEtE;;;;;GAKG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAmE5E"}