khoji 2.1.4

package/dist/extractors/InteractionExtractor.js

@@ -0,0 +1,64 @@
+/**
+ * Maps interactive regions of the page:
+ * - Forms with all their input fields
+ * - Navigation elements
+ * - Interactive button-like elements
+ */
+export async function extractInteractions(page) {
+    return page.evaluate(() => {
+        const interactions = [];
+        function getCssSelector(el) {
+            if (el.id)
+                return `#${el.id}`;
+            const tag = el.tagName.toLowerCase();
+            const cls = el.className?.toString().trim().split(/\s+/)[0];
+            return cls ? `${tag}.${cls}` : tag;
+        }
+        // ── Forms ───────────────────────────────────────────────────────────────
+        document.querySelectorAll('form').forEach((form) => {
+            const fields = [];
+            form.querySelectorAll('input, select, textarea').forEach((field) => {
+                const el = field;
+                fields.push({
+                    inputType: el.tagName === 'SELECT' ? 'select' : el.tagName === 'TEXTAREA' ? 'textarea' : el.type || 'text',
+                    name: el.name || el.id || '',
+                    placeholder: el.placeholder || '',
+                    required: el.required ?? false,
+                });
+            });
+            const label = form.getAttribute('aria-label') ??
+                form.querySelector('legend, h1, h2, h3, h4')?.textContent?.trim() ??
+                form.id ??
+                'Form';
+            interactions.push({
+                type: 'form',
+                selector: getCssSelector(form),
+                label,
+                fields,
+            });
+        });
+        // ── Navigation ──────────────────────────────────────────────────────────
+        document.querySelectorAll('nav, [role="navigation"]').forEach((nav) => {
+            const label = nav.getAttribute('aria-label') ??
+                nav.getAttribute('id') ??
+                'Navigation';
+            interactions.push({
+                type: 'nav',
+                selector: getCssSelector(nav),
+                label,
+            });
+        });
+        // ── Interactive Buttons ─────────────────────────────────────────────────
+        const buttonEls = document.querySelectorAll('[role="button"]:not(button), [data-action], [data-click], [onclick]');
+        buttonEls.forEach((el) => {
+            const label = el.getAttribute('aria-label') ?? el.textContent?.trim() ?? 'Button';
+            interactions.push({
+                type: 'button',
+                selector: getCssSelector(el),
+                label,
+            });
+        });
+        return interactions;
+    });
+}
+//# sourceMappingURL=InteractionExtractor.js.map

package/dist/extractors/InteractionExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InteractionExtractor.js","sourceRoot":"","sources":["../../src/extractors/InteractionExtractor.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAU;IAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAkB,EAAE;QACrC,MAAM,YAAY,GAAkB,EAAE,CAAC;QAEvC,SAAS,cAAc,CAAC,EAAW;YAC/B,IAAI,EAAE,CAAC,EAAE;gBAAE,OAAO,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACvC,CAAC;QAED,2EAA2E;QAC3E,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/C,MAAM,MAAM,GAAgB,EAAE,CAAC;YAC/B,IAAI,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/D,MAAM,EAAE,GAAG,KAAmE,CAAC;gBAC/E,MAAM,CAAC,IAAI,CAAC;oBACR,SAAS,EAAE,EAAE,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE,EAAuB,CAAC,IAAI,IAAI,MAAM;oBAChI,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE;oBAC5B,WAAW,EAAG,EAAuB,CAAC,WAAW,IAAI,EAAE;oBACvD,QAAQ,EAAG,EAAuB,CAAC,QAAQ,IAAI,KAAK;iBACvD,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;YAEH,MAAM,KAAK,GACP,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC;gBAC/B,IAAI,CAAC,aAAa,CAAC,wBAAwB,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE;gBACjE,IAAI,CAAC,EAAE;gBACP,MAAM,CAAC;YAEX,YAAY,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,cAAc,CAAC,IAAI,CAAC;gBAC9B,KAAK;gBACL,MAAM;aACT,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,2EAA2E;QAC3E,QAAQ,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClE,MAAM,KAAK,GACP,GAAG,CAAC,YAAY,CAAC,YAAY,CAAC;gBAC9B,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC;gBACtB,YAAY,CAAC;YAEjB,YAAY,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;gBAC7B,KAAK;aACR,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,2EAA2E;QAC3E,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CACvC,qEAAqE,CACxE,CAAC;QACF,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YACrB,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;YAClF,YAAY,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,cAAc,CAAC,EAAE,CAAC;gBAC5B,KAAK;aACR,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/extractors/MetaExtractor.d.ts

@@ -0,0 +1,8 @@
+import type { Page } from 'playwright';
+import type { PageMeta } from '../types/KhojContext.js';
+/**
+ * Extracts page metadata: title, description, Open Graph tags, canonical URL,
+ * theme-color, and embedded JSON-LD structured data.
+ */
+export declare function extractMeta(page: Page): Promise<PageMeta>;
+//# sourceMappingURL=MetaExtractor.d.ts.map

package/dist/extractors/MetaExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetaExtractor.d.ts","sourceRoot":"","sources":["../../src/extractors/MetaExtractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAExD;;;GAGG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CA+B/D"}

package/dist/extractors/MetaExtractor.js

@@ -0,0 +1,33 @@
+/**
+ * Extracts page metadata: title, description, Open Graph tags, canonical URL,
+ * theme-color, and embedded JSON-LD structured data.
+ */
+export async function extractMeta(page) {
+    return page.evaluate(() => {
+        function getMeta(name) {
+            const el = document.querySelector(`meta[name="${name}"]`) ??
+                document.querySelector(`meta[property="${name}"]`);
+            return el?.content ?? '';
+        }
+        // Collect JSON-LD scripts
+        const jsonLd = [];
+        document.querySelectorAll('script[type="application/ld+json"]').forEach((el) => {
+            try {
+                jsonLd.push(JSON.parse(el.textContent ?? ''));
+            }
+            catch {
+                // Malformed JSON-LD — skip silently
+            }
+        });
+        const canonical = document.querySelector('link[rel="canonical"]')?.href ?? null;
+        return {
+            title: document.title?.trim() ?? '',
+            description: getMeta('description') || getMeta('og:description') || '',
+            ogImage: getMeta('og:image') || null,
+            canonical: canonical || null,
+            themeColor: getMeta('theme-color') || null,
+            jsonLd,
+        };
+    });
+}
+//# sourceMappingURL=MetaExtractor.js.map

package/dist/extractors/MetaExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetaExtractor.js","sourceRoot":"","sources":["../../src/extractors/MetaExtractor.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAU;IACxC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAa,EAAE;QAChC,SAAS,OAAO,CAAC,IAAY;YACzB,MAAM,EAAE,GACJ,QAAQ,CAAC,aAAa,CAAC,cAAc,IAAI,IAAI,CAAC;gBAC9C,QAAQ,CAAC,aAAa,CAAC,kBAAkB,IAAI,IAAI,CAAC,CAAC;YACvD,OAAQ,EAA6B,EAAE,OAAO,IAAI,EAAE,CAAC;QACzD,CAAC;QAED,0BAA0B;QAC1B,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,QAAQ,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC3E,IAAI,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACL,oCAAoC;YACxC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GACV,QAAQ,CAAC,aAAa,CAAC,uBAAuB,CAA4B,EAAE,IAAI,IAAI,IAAI,CAAC;QAE9F,OAAO;YACH,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE;YACnC,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE;YACtE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI;YACpC,SAAS,EAAE,SAAS,IAAI,IAAI;YAC5B,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,IAAI;YAC1C,MAAM;SACT,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/extractors/StyleExtractor.d.ts

@@ -0,0 +1,10 @@
+import type { Page } from 'playwright';
+import type { DesignTokens } from '../types/KhojContext.js';
+/**
+ * Extracts design tokens from the page's stylesheets:
+ * - CSS custom properties (--color-*, --space-*, etc.)
+ * - Font families in use
+ * - Media query breakpoints
+ */
+export declare function extractStyles(page: Page): Promise<DesignTokens>;
+//# sourceMappingURL=StyleExtractor.d.ts.map

package/dist/extractors/StyleExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StyleExtractor.d.ts","sourceRoot":"","sources":["../../src/extractors/StyleExtractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAiFrE"}

package/dist/extractors/StyleExtractor.js

@@ -0,0 +1,87 @@
+/**
+ * Extracts design tokens from the page's stylesheets:
+ * - CSS custom properties (--color-*, --space-*, etc.)
+ * - Font families in use
+ * - Media query breakpoints
+ */
+export async function extractStyles(page) {
+    return page.evaluate(() => {
+        const colors = {};
+        const spacing = {};
+        const typography = {};
+        const fontSet = new Set();
+        const breakpointSet = new Set();
+        const COLOR_KEYWORDS = ['color', 'bg', 'background', 'fill', 'stroke', 'text', 'border', 'shadow', 'accent', 'primary', 'secondary'];
+        const SPACING_KEYWORDS = ['space', 'gap', 'margin', 'padding', 'indent', 'inset', 'offset', 'size', 'width', 'height'];
+        const TYPO_KEYWORDS = ['font', 'text', 'line-height', 'letter', 'weight', 'size'];
+        function categorise(name, value) {
+            const lower = name.toLowerCase();
+            if (COLOR_KEYWORDS.some((k) => lower.includes(k))) {
+                colors[name] = value;
+            }
+            else if (SPACING_KEYWORDS.some((k) => lower.includes(k))) {
+                spacing[name] = value;
+            }
+            else if (TYPO_KEYWORDS.some((k) => lower.includes(k))) {
+                typography[name] = value;
+            }
+        }
+        // Walk all CSSStyleSheets
+        for (const sheet of Array.from(document.styleSheets)) {
+            let rules;
+            try {
+                rules = sheet.cssRules;
+            }
+            catch {
+                // Cross-origin stylesheet — skip
+                continue;
+            }
+            for (const rule of Array.from(rules)) {
+                // CSS custom properties on :root or html
+                if (rule instanceof CSSStyleRule) {
+                    const sel = rule.selectorText ?? '';
+                    if (sel === ':root' || sel === 'html') {
+                        for (const prop of Array.from(rule.style)) {
+                            if (prop.startsWith('--')) {
+                                const val = rule.style.getPropertyValue(prop).trim();
+                                categorise(prop, val);
+                            }
+                        }
+                    }
+                }
+                // @media breakpoints
+                if (rule instanceof CSSMediaRule) {
+                    const condition = rule.conditionText ?? rule.media.mediaText;
+                    const match = condition.match(/\d+px/);
+                    if (match)
+                        breakpointSet.add(match[0]);
+                }
+            }
+        }
+        // Font families from computed styles
+        const targets = ['body', 'h1', 'h2', 'p', 'button', 'a'];
+        for (const selector of targets) {
+            const el = document.querySelector(selector);
+            if (!el)
+                continue;
+            const computed = window.getComputedStyle(el);
+            const family = computed.fontFamily;
+            if (family) {
+                family.split(',').forEach((f) => {
+                    const clean = f.trim().replace(/['"]/g, '');
+                    if (clean && clean !== 'inherit' && clean !== 'initial') {
+                        fontSet.add(clean);
+                    }
+                });
+            }
+        }
+        return {
+            colors,
+            spacing,
+            typography,
+            fonts: Array.from(fontSet).slice(0, 12),
+            breakpoints: Array.from(breakpointSet).sort((a, b) => parseInt(a) - parseInt(b)),
+        };
+    });
+}
+//# sourceMappingURL=StyleExtractor.js.map

package/dist/extractors/StyleExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StyleExtractor.js","sourceRoot":"","sources":["../../src/extractors/StyleExtractor.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU;IAC1C,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAiB,EAAE;QACpC,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,MAAM,UAAU,GAA2B,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QAExC,MAAM,cAAc,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QACrI,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACvH,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElF,SAAS,UAAU,CAAC,IAAY,EAAE,KAAa;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChD,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YACzB,CAAC;iBAAM,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC1B,CAAC;iBAAM,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtD,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,0BAA0B;QAC1B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnD,IAAI,KAAkB,CAAC;YACvB,IAAI,CAAC;gBACD,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC;YAC3B,CAAC;YAAC,MAAM,CAAC;gBACL,iCAAiC;gBACjC,SAAS;YACb,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,yCAAyC;gBACzC,IAAI,IAAI,YAAY,YAAY,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;oBACpC,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;wBACpC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;4BACxC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gCACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gCACrD,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAC1B,CAAC;wBACL,CAAC;oBACL,CAAC;gBACL,CAAC;gBAED,qBAAqB;gBACrB,IAAI,IAAI,YAAY,YAAY,EAAE,CAAC;oBAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;oBAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBACvC,IAAI,KAAK;wBAAE,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACL,CAAC;QACL,CAAC;QAED,qCAAqC;QACrC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACzD,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC5C,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBAC5C,IAAI,KAAK,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACtD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACvB,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,OAAO;YACH,MAAM;YACN,OAAO;YACP,UAAU;YACV,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACvC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;SACnF,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { runExtraction } from './pipeline/runner.js';
+export type { KhojContext, PageMeta, DomNode, DesignTokens, DetectedComponent, AssetMap, ImageAsset, ContentBlock, ContentType, Interaction, FormField, InteractionType, AnimationMap, CSSAnimation, CSSTransition, JSAnimation, JSAnimationLibrary, ScrollAnimation, ScrollAnimationLibrary, GifAnimation, GifAnimationPurpose, ExtractionOptions, } from './types/KhojContext.js';
+export { estimateTokens } from './utils/tokenEstimator.js';
+export { serializeJson } from './serializer/JsonSerializer.js';
+export { serializeMarkdown } from './serializer/MarkdownSerializer.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,YAAY,EACR,WAAW,EACX,QAAQ,EACR,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,WAAW,EACX,WAAW,EACX,SAAS,EACT,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,GACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC"}

package/dist/index.js

@@ -0,0 +1,6 @@
+// Public API — import khoj as a library in your own Node.js/TypeScript project
+export { runExtraction } from './pipeline/runner.js';
+export { estimateTokens } from './utils/tokenEstimator.js';
+export { serializeJson } from './serializer/JsonSerializer.js';
+export { serializeMarkdown } from './serializer/MarkdownSerializer.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAyBrD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC"}

package/dist/output/Writer.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Writes content to a file, creating directories as needed.
+ */
+export declare function writeOutput(filePath: string, content: string): Promise<void>;
+//# sourceMappingURL=Writer.d.ts.map

package/dist/output/Writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Writer.d.ts","sourceRoot":"","sources":["../../src/output/Writer.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKlF"}

package/dist/output/Writer.js

@@ -0,0 +1,13 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { logger } from '../utils/logger.js';
+/**
+ * Writes content to a file, creating directories as needed.
+ */
+export async function writeOutput(filePath, content) {
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    await fs.writeFile(filePath, content, 'utf-8');
+    const sizeKb = (Buffer.byteLength(content, 'utf-8') / 1024).toFixed(1);
+    logger.success(`Written: ${filePath} (${sizeKb} KB)`);
+}
+//# sourceMappingURL=Writer.js.map

package/dist/output/Writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Writer.js","sourceRoot":"","sources":["../../src/output/Writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,OAAe;IAC/D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,CAAC,OAAO,CAAC,YAAY,QAAQ,KAAK,MAAM,MAAM,CAAC,CAAC;AAC1D,CAAC"}

package/dist/pipeline/Cleaner.d.ts

@@ -0,0 +1,12 @@
+import type { ContentBlock, DomNode, ImageAsset } from '../types/KhojContext.js';
+/**
+ * Post-processing pipeline:
+ * - Removes duplicate content blocks
+ * - Deduplicates asset URLs
+ * - Prunes empty DOM nodes
+ */
+export declare function cleanContent(blocks: ContentBlock[]): ContentBlock[];
+export declare function cleanImages(images: ImageAsset[]): ImageAsset[];
+export declare function cleanStringList(list: string[]): string[];
+export declare function pruneEmptyDomNodes(nodes: DomNode[]): DomNode[];
+//# sourceMappingURL=Cleaner.d.ts.map

package/dist/pipeline/Cleaner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Cleaner.d.ts","sourceRoot":"","sources":["../../src/pipeline/Cleaner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAEjF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAQnE;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,UAAU,EAAE,CAO9D;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAExD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAc9D"}

package/dist/pipeline/Cleaner.js

@@ -0,0 +1,41 @@
+/**
+ * Post-processing pipeline:
+ * - Removes duplicate content blocks
+ * - Deduplicates asset URLs
+ * - Prunes empty DOM nodes
+ */
+export function cleanContent(blocks) {
+    const seen = new Set();
+    return blocks.filter((b) => {
+        const key = `${b.type}:${b.text.toLowerCase()}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+export function cleanImages(images) {
+    const seen = new Set();
+    return images.filter((img) => {
+        if (seen.has(img.url))
+            return false;
+        seen.add(img.url);
+        return true;
+    });
+}
+export function cleanStringList(list) {
+    return [...new Set(list.filter((s) => s.trim().length > 0))];
+}
+export function pruneEmptyDomNodes(nodes) {
+    return nodes
+        .map((node) => {
+        const children = node.children ? pruneEmptyDomNodes(node.children) : undefined;
+        return { ...node, children: children?.length ? children : undefined };
+    })
+        .filter((node) => node.text ||
+        node.id ||
+        node.classes.length > 0 ||
+        node.role ||
+        (node.children && node.children.length > 0));
+}
+//# sourceMappingURL=Cleaner.js.map

package/dist/pipeline/Cleaner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Cleaner.js","sourceRoot":"","sources":["../../src/pipeline/Cleaner.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,MAAsB;IAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAoB;IAC5C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAc;IAC1C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAgB;IAC/C,OAAO,KAAK;SACP,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/E,OAAO,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC1E,CAAC,CAAC;SACD,MAAM,CACH,CAAC,IAAI,EAAE,EAAE,CACL,IAAI,CAAC,IAAI;QACT,IAAI,CAAC,EAAE;QACP,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACvB,IAAI,CAAC,IAAI;QACT,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAClD,CAAC;AACV,CAAC"}

package/dist/pipeline/ComponentDetector.d.ts

@@ -0,0 +1,8 @@
+import type { Page } from 'playwright';
+import type { DetectedComponent } from '../types/KhojContext.js';
+/**
+ * Detects repeated DOM structural patterns that likely represent UI components.
+ * Examples: card grids, list items, nav links, table rows.
+ */
+export declare function detectComponents(page: Page): Promise<DetectedComponent[]>;
+//# sourceMappingURL=ComponentDetector.d.ts.map

package/dist/pipeline/ComponentDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ComponentDetector.d.ts","sourceRoot":"","sources":["../../src/pipeline/ComponentDetector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjE;;;GAGG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAwC/E"}

package/dist/pipeline/ComponentDetector.js

@@ -0,0 +1,43 @@
+/**
+ * Detects repeated DOM structural patterns that likely represent UI components.
+ * Examples: card grids, list items, nav links, table rows.
+ */
+export async function detectComponents(page) {
+    return page.evaluate(() => {
+        const MIN_REPETITIONS = 3;
+        const components = [];
+        // Candidate selectors — common repeating component patterns
+        const candidates = [
+            { name: 'Card', pattern: '.card, [class*="card"]' },
+            { name: 'ListItem', pattern: 'ul > li, ol > li' },
+            { name: 'GridItem', pattern: '[class*="grid"] > *, [class*="col-"]' },
+            { name: 'NavLink', pattern: 'nav a, header a' },
+            { name: 'TableRow', pattern: 'tbody tr' },
+            { name: 'Article', pattern: 'article, [class*="post"], [class*="article"]' },
+            { name: 'Testimonial', pattern: '[class*="testimonial"], [class*="review"]' },
+            { name: 'Feature', pattern: '[class*="feature"], [class*="benefit"]' },
+            { name: 'Pricing', pattern: '[class*="pricing"], [class*="plan"], [class*="tier"]' },
+            { name: 'MenuItem', pattern: '[role="menuitem"], [class*="menu-item"]' },
+            { name: 'Tag', pattern: '[class*="tag"], [class*="badge"], [class*="chip"]' },
+        ];
+        for (const { name, pattern } of candidates) {
+            let els;
+            try {
+                els = document.querySelectorAll(pattern);
+            }
+            catch {
+                continue;
+            }
+            if (els.length < MIN_REPETITIONS)
+                continue;
+            components.push({
+                name,
+                pattern,
+                count: els.length,
+                sampleHTML: els[0].outerHTML.slice(0, 400),
+            });
+        }
+        return components;
+    });
+}
+//# sourceMappingURL=ComponentDetector.js.map

package/dist/pipeline/ComponentDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ComponentDetector.js","sourceRoot":"","sources":["../../src/pipeline/ComponentDetector.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU;IAC7C,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAwB,EAAE;QAC3C,MAAM,eAAe,GAAG,CAAC,CAAC;QAC1B,MAAM,UAAU,GAAwB,EAAE,CAAC;QAE3C,4DAA4D;QAC5D,MAAM,UAAU,GAAG;YACf,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,EAAE;YACnD,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,kBAAkB,EAAE;YACjD,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,sCAAsC,EAAE;YACrE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,iBAAiB,EAAE;YAC/C,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE;YACzC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,8CAA8C,EAAE;YAC5E,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,2CAA2C,EAAE;YAC7E,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,wCAAwC,EAAE;YACtE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,sDAAsD,EAAE;YACpF,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,yCAAyC,EAAE;YACxE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,mDAAmD,EAAE;SAChF,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC;YACzC,IAAI,GAAwB,CAAC;YAC7B,IAAI,CAAC;gBACD,GAAG,GAAG,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,eAAe;gBAAE,SAAS;YAE3C,UAAU,CAAC,IAAI,CAAC;gBACZ,IAAI;gBACJ,OAAO;gBACP,KAAK,EAAE,GAAG,CAAC,MAAM;gBACjB,UAAU,EAAG,GAAG,CAAC,CAAC,CAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aAC9D,CAAC,CAAC;QACP,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/pipeline/runner.d.ts

@@ -0,0 +1,3 @@
+import type { ExtractionOptions } from '../types/KhojContext.js';
+export declare function runExtraction(opts: ExtractionOptions): Promise<void>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/pipeline/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/pipeline/runner.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,iBAAiB,EAAe,MAAM,yBAAyB,CAAC;AAE9E,wBAAsB,aAAa,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA4L1E"}

package/dist/pipeline/runner.js

@@ -0,0 +1,182 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { PromptGenerator } from '../prompting/PromptGenerator.js';
+import { BrowserManager } from '../browser/BrowserManager.js';
+import { PageLoader } from '../browser/PageLoader.js';
+import { extractMeta } from '../extractors/MetaExtractor.js';
+import { extractDom } from '../extractors/DomExtractor.js';
+import { extractStyles } from '../extractors/StyleExtractor.js';
+import { extractAssets } from '../extractors/AssetExtractor.js';
+import { extractContent } from '../extractors/ContentExtractor.js';
+import { extractInteractions } from '../extractors/InteractionExtractor.js';
+import { extractAnimations } from '../extractors/AnimationExtractor.js';
+import { detectComponents } from './ComponentDetector.js';
+import { cleanContent, cleanImages, cleanStringList, pruneEmptyDomNodes } from './Cleaner.js';
+import { serializeJson } from '../serializer/JsonSerializer.js';
+import { serializeMarkdown } from '../serializer/MarkdownSerializer.js';
+import { writeOutput } from '../output/Writer.js';
+import { estimateTokens } from '../utils/tokenEstimator.js';
+import { logger } from '../utils/logger.js';
+export async function runExtraction(opts) {
+    const browserManager = new BrowserManager();
+    const mode = opts.fast ? 'fast' : 'full';
+    try {
+        const context = await browserManager.launch(mode);
+        const loader = new PageLoader(context, opts.timeout);
+        logger.step('🚀', `Loading page (${mode} mode)...`);
+        const { page, finalUrl, loadTime } = await loader.load(opts.url, opts.clickSelector);
+        logger.step('⚙️', 'Running extraction pipeline (parallel)...');
+        logger.divider();
+        // Run all extractors in parallel for performance
+        const [meta, structure, designTokens, assets, content, interactions, components] = await Promise.all([
+            extractMeta(page).catch(() => ({ title: '', description: '', ogImage: null, canonical: null, themeColor: null, jsonLd: [] })),
+            extractDom(page).catch(() => []),
+            extractStyles(page).catch(() => ({ colors: {}, spacing: {}, typography: {}, fonts: [], breakpoints: [] })),
+            extractAssets(page, finalUrl).catch(() => ({ images: [], gifs: [], fonts: [], icons: [], scripts: [] })),
+            extractContent(page).catch(() => []),
+            extractInteractions(page).catch(() => []),
+            detectComponents(page).catch(() => []),
+        ]);
+        logger.step('🎬', 'Analysing animations...');
+        const animations = await extractAnimations(page, assets.gifs).catch(() => ({
+            cssAnimations: [],
+            cssTransitions: [],
+            jsAnimations: [],
+            scrollAnimations: [],
+            gifAnimations: [],
+            summary: 'Animation extraction failed',
+        }));
+        // Post-processing
+        const cleanedContent = cleanContent(content);
+        const cleanedImages = cleanImages(assets.images);
+        const cleanedGifs = cleanImages(assets.gifs);
+        const cleanedStructure = pruneEmptyDomNodes(structure);
+        const ctx = {
+            schemaVersion: '1.0',
+            url: opts.url,
+            finalUrl,
+            extractedAt: new Date().toISOString(),
+            loadTimeMs: loadTime,
+            tokenEstimate: 0, // calculated below
+            meta,
+            structure: cleanedStructure,
+            designTokens,
+            components,
+            assets: {
+                images: cleanedImages,
+                gifs: cleanedGifs,
+                fonts: cleanStringList(assets.fonts),
+                icons: cleanStringList(assets.icons),
+                scripts: cleanStringList(assets.scripts),
+            },
+            content: cleanedContent,
+            interactions,
+            animations,
+        };
+        // Calculate token estimate on the final object
+        const jsonString = JSON.stringify(ctx, null, 2);
+        ctx.tokenEstimate = estimateTokens(jsonString);
+        // Serialize
+        logger.step('📦', 'Serializing output...');
+        logger.divider();
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+        // Create a domain-specific subdirectory
+        const hostname = new URL(opts.url).hostname.replace(/^www\./, '');
+        const outputDir = path.resolve(opts.outputDir, hostname);
+        await fs.mkdir(outputDir, { recursive: true });
+        if (opts.format === 'json' || opts.format === 'both') {
+            const json = serializeJson(ctx);
+            await writeOutput(path.join(outputDir, `khoj-context-${timestamp}.json`), json);
+        }
+        if (opts.format === 'markdown' || opts.format === 'both') {
+            const md = serializeMarkdown(ctx);
+            await writeOutput(path.join(outputDir, `khoj-context-${timestamp}.md`), md);
+        }
+        // Clone Mode Extraction
+        if (opts.clone) {
+            logger.step('📸', 'Extracting clone assets (Screenshot, HTML, CSS)...');
+            logger.divider();
+            // 1. Full Page Screenshot
+            const screenshotPath = path.join(outputDir, `khoj-clone-${timestamp}.png`);
+            await page.screenshot({ path: screenshotPath, fullPage: true });
+            // 2. Raw HTML
+            const rawHtml = await page.content();
+            const htmlPath = path.join(outputDir, `khoj-clone-${timestamp}.html`);
+            await writeOutput(htmlPath, rawHtml);
+            // 3. Raw CSS (Inline + External)
+            const cssPath = path.join(outputDir, `khoj-clone-${timestamp}.css`);
+            // First get all inline styles and external URLs from the browser context
+            const { inlineStyles, externalUrls } = await page.evaluate(() => {
+                const styles = Array.from(document.querySelectorAll('style')).map(s => s.textContent || '');
+                const links = Array.from(document.querySelectorAll('link[rel="stylesheet"]'))
+                    .map(l => l.getAttribute('href'))
+                    .filter((href) => href !== null)
+                    .map(href => new URL(href, document.baseURI).href);
+                return { inlineStyles: styles, externalUrls: links };
+            });
+            let rawCss = '';
+            // Render inline styles
+            for (const style of inlineStyles) {
+                if (style.trim()) {
+                    rawCss += `/* Inline Style */\n${style}\n\n`;
+                }
+            }
+            // Fetch external styles from Node context to bypass browser CORS policies
+            for (const url of externalUrls) {
+                try {
+                    const response = await fetch(url);
+                    if (response.ok) {
+                        const cssText = await response.text();
+                        rawCss += `/* External Style: ${url} */\n${cssText}\n\n`;
+                    }
+                    else {
+                        rawCss += `/* Failed to fetch external style (HTTP ${response.status}): ${url} */\n\n`;
+                    }
+                }
+                catch (e) {
+                    rawCss += `/* Failed to fetch external style (Network Error): ${url} */\n\n`;
+                }
+            }
+            await writeOutput(cssPath, rawCss);
+            // 4. Generate AI Prompt (if requested)
+            if (opts.cloneSkills !== undefined) {
+                // We assert the type here because PromptGenerator expects CloneSkill[] 
+                // and the CLI ensures only valid values are passed.
+                const promptContent = PromptGenerator.generate(opts.cloneSkills);
+                const promptPath = path.join(outputDir, `prompt.md`);
+                await writeOutput(promptPath, promptContent);
+            }
+        }
+        // Final report
+        logger.divider();
+        logger.step('📊', 'Extraction complete');
+        logger.stat('URL', ctx.url);
+        logger.stat('Images', `${cleanedImages.length} images, ${cleanedGifs.length} GIFs`);
+        logger.stat('Animations', `${ctx.animations.cssAnimations.length} CSS, ${ctx.animations.jsAnimations.length} JS`);
+        logger.stat('Token estimate', `~${ctx.tokenEstimate.toLocaleString()} tokens`);
+        logger.stat('Load time', `${(loadTime / 1000).toFixed(2)}s`);
+        logger.divider();
+        if (opts.clone) {
+            logger.step('📸', 'Clone artifacts saved:');
+            logger.stat('Screenshot', `khoj-clone-${timestamp}.png`);
+            logger.stat('HTML Source', `khoj-clone-${timestamp}.html`);
+            logger.stat('CSS Source', `khoj-clone-${timestamp}.css`);
+            if (opts.cloneSkills !== undefined) {
+                logger.stat('System Prompt', `prompt.md`);
+            }
+            logger.divider();
+        }
+        if (ctx.animations.summary) {
+            logger.step('🎬', `Animations: ${ctx.animations.summary}`);
+        }
+        // Optional Gemini integration
+        if (opts.sendToGemini) {
+            const { sendToGemini } = await import('../ai/GeminiAdapter.js');
+            await sendToGemini(ctx, opts.prompt);
+        }
+    }
+    finally {
+        await browserManager.close();
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/pipeline/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/pipeline/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC9F,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAuB;IACvD,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;IAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEzC,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAErD,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,IAAI,WAAW,CAAC,CAAC;QACpD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAErF,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,2CAA2C,CAAC,CAAC;QAC/D,MAAM,CAAC,OAAO,EAAE,CAAC;QAEjB,iDAAiD;QACjD,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,CAAC,GAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7H,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;YAChC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1G,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACxG,cAAc,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;YACpC,mBAAmB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;YACzC,gBAAgB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;SACzC,CAAC,CAAC;QAEP,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,aAAa,EAAE,EAAE;YACjB,cAAc,EAAE,EAAE;YAClB,YAAY,EAAE,EAAE;YAChB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,EAAE;YACjB,OAAO,EAAE,6BAA6B;SACzC,CAAC,CAAC,CAAC;QAEJ,kBAAkB;QAClB,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEvD,MAAM,GAAG,GAAgB;YACrB,aAAa,EAAE,KAAK;YACpB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ;YACR,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,UAAU,EAAE,QAAQ;YACpB,aAAa,EAAE,CAAC,EAAE,mBAAmB;YACrC,IAAI;YACJ,SAAS,EAAE,gBAAgB;YAC3B,YAAY;YACZ,UAAU;YACV,MAAM,EAAE;gBACJ,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;gBACpC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC;gBACpC,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;aAC3C;YACD,OAAO,EAAE,cAAc;YACvB,YAAY;YACZ,UAAU;SACb,CAAC;QAEF,+CAA+C;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,aAAa,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;QAE/C,YAAY;QACZ,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,EAAE,CAAC;QAEjB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAE9E,wCAAwC;QACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,SAAS,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACpF,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACvD,MAAM,EAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,SAAS,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,wBAAwB;QACxB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,oDAAoD,CAAC,CAAC;YACxE,MAAM,CAAC,OAAO,EAAE,CAAC;YAEjB,0BAA0B;YAC1B,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,SAAS,MAAM,CAAC,CAAC;YAC3E,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAEhE,cAAc;YACd,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,SAAS,OAAO,CAAC,CAAC;YACtE,MAAM,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAErC,iCAAiC;YACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,SAAS,MAAM,CAAC,CAAC;YAEpE,yEAAyE;YACzE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;gBAC5F,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC;qBACxE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;qBAChC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC;qBAC/C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEvD,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;YACzD,CAAC,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,uBAAuB;YACvB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBAC/B,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;oBACf,MAAM,IAAI,uBAAuB,KAAK,MAAM,CAAC;gBACjD,CAAC;YACL,CAAC;YAED,0EAA0E;YAC1E,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACd,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,sBAAsB,GAAG,QAAQ,OAAO,MAAM,CAAC;oBAC7D,CAAC;yBAAM,CAAC;wBACJ,MAAM,IAAI,2CAA2C,QAAQ,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC;oBAC3F,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,MAAM,IAAI,sDAAsD,GAAG,SAAS,CAAC;gBACjF,CAAC;YACL,CAAC;YAED,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAEnC,uCAAuC;YACvC,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACjC,wEAAwE;gBACxE,oDAAoD;gBACpD,MAAM,aAAa,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAkB,CAAC,CAAC;gBACxE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;gBACrD,MAAM,WAAW,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;YACjD,CAAC;QACL,CAAC;QAED,eAAe;QACf,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,aAAa,CAAC,MAAM,YAAY,WAAW,CAAC,MAAM,OAAO,CAAC,CAAC;QACpF,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC;QAClH,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,GAAG,CAAC,aAAa,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,CAAC,OAAO,EAAE,CAAC;QAEjB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,cAAc,SAAS,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,SAAS,OAAO,CAAC,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,cAAc,SAAS,MAAM,CAAC,CAAC;YACzD,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;YAC9C,CAAC;YACD,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,8BAA8B;QAC9B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YAChE,MAAM,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;IAEL,CAAC;YAAS,CAAC;QACP,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;AACL,CAAC"}