@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/js-coverage.d.ts

@@ -0,0 +1,29 @@
+/**
+ * JS/CSS coverage collector via Playwright's built-in coverage API.
+ *
+ * For each crawled page, records what percentage of loaded JS/CSS was actually
+ * executed/applied. High unused code % → performance opportunity (code splitting, lazy loading).
+ *
+ * Uses Playwright's page.coverage API (wraps CDP Coverage domain).
+ */
+import type { Page } from 'playwright';
+export interface CoverageEntry {
+    url: string;
+    type: 'js' | 'css';
+    totalBytes: number;
+    usedBytes: number;
+    unusedPercent: number;
+}
+export interface CoverageReport {
+    pageUrl: string;
+    entries: CoverageEntry[];
+    totalJsBytes: number;
+    usedJsBytes: number;
+    totalCssBytes: number;
+    usedCssBytes: number;
+    jsUnusedPercent: number;
+    cssUnusedPercent: number;
+    collectedAt: string;
+}
+export declare function startCoverage(page: Page): Promise<void>;
+export declare function stopAndCollectCoverage(page: Page, pageUrl: string): Promise<CoverageReport | null>;

package/dist/js-coverage.js

@@ -0,0 +1,56 @@
+async function startCoverage(page) {
+  try {
+    await Promise.all([
+      page.coverage.startJSCoverage({ resetOnNavigation: false }),
+      page.coverage.startCSSCoverage({ resetOnNavigation: false })
+    ]);
+  } catch {
+  }
+}
+async function stopAndCollectCoverage(page, pageUrl) {
+  try {
+    const [jsCoverage, cssCoverage] = await Promise.all([
+      page.coverage.stopJSCoverage().catch(() => []),
+      page.coverage.stopCSSCoverage().catch(() => [])
+    ]);
+    const entries = [];
+    for (const entry of jsCoverage) {
+      const totalBytes = entry.text?.length ?? 0;
+      const usedBytes = (entry.ranges ?? []).reduce((sum, r) => sum + (r.end - r.start), 0);
+      if (totalBytes > 0) {
+        entries.push({ url: entry.url, type: "js", totalBytes, usedBytes, unusedPercent: Math.round((1 - usedBytes / totalBytes) * 100) });
+      }
+    }
+    for (const entry of cssCoverage) {
+      const totalBytes = entry.text?.length ?? 0;
+      const usedBytes = (entry.ranges ?? []).reduce((sum, r) => sum + (r.end - r.start), 0);
+      if (totalBytes > 0) {
+        entries.push({ url: entry.url, type: "css", totalBytes, usedBytes, unusedPercent: Math.round((1 - usedBytes / totalBytes) * 100) });
+      }
+    }
+    const jsEntries = entries.filter((e) => e.type === "js");
+    const cssEntries = entries.filter((e) => e.type === "css");
+    const totalJsBytes = jsEntries.reduce((s, e) => s + e.totalBytes, 0);
+    const usedJsBytes = jsEntries.reduce((s, e) => s + e.usedBytes, 0);
+    const totalCssBytes = cssEntries.reduce((s, e) => s + e.totalBytes, 0);
+    const usedCssBytes = cssEntries.reduce((s, e) => s + e.usedBytes, 0);
+    return {
+      pageUrl,
+      entries,
+      totalJsBytes,
+      usedJsBytes,
+      totalCssBytes,
+      usedCssBytes,
+      jsUnusedPercent: totalJsBytes > 0 ? Math.round((1 - usedJsBytes / totalJsBytes) * 100) : 0,
+      cssUnusedPercent: totalCssBytes > 0 ? Math.round((1 - usedCssBytes / totalCssBytes) * 100) : 0,
+      collectedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  } catch {
+    return null;
+  }
+}
+export {
+  startCoverage,
+  stopAndCollectCoverage
+};
+//# sourceMappingURL=js-coverage.js.map

package/dist/js-coverage.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/js-coverage.ts"],
+  "sourcesContent": ["/**\n * JS/CSS coverage collector via Playwright's built-in coverage API.\n *\n * For each crawled page, records what percentage of loaded JS/CSS was actually\n * executed/applied. High unused code % \u2192 performance opportunity (code splitting, lazy loading).\n *\n * Uses Playwright's page.coverage API (wraps CDP Coverage domain).\n */\n\nimport type { Page } from 'playwright';\n\nexport interface CoverageEntry {\n  url: string;\n  type: 'js' | 'css';\n  totalBytes: number;\n  usedBytes: number;\n  unusedPercent: number;\n}\n\nexport interface CoverageReport {\n  pageUrl: string;\n  entries: CoverageEntry[];\n  totalJsBytes: number;\n  usedJsBytes: number;\n  totalCssBytes: number;\n  usedCssBytes: number;\n  jsUnusedPercent: number;\n  cssUnusedPercent: number;\n  collectedAt: string;\n}\n\nexport async function startCoverage(page: Page): Promise<void> {\n  try {\n    await Promise.all([\n      page.coverage.startJSCoverage({ resetOnNavigation: false }),\n      page.coverage.startCSSCoverage({ resetOnNavigation: false }),\n    ]);\n  } catch { /* coverage not available in all environments */ }\n}\n\nexport async function stopAndCollectCoverage(page: Page, pageUrl: string): Promise<CoverageReport | null> {\n  try {\n    const [jsCoverage, cssCoverage] = await Promise.all([\n      page.coverage.stopJSCoverage().catch(() => [] as any[]),\n      page.coverage.stopCSSCoverage().catch(() => [] as any[]),\n    ]);\n\n    const entries: CoverageEntry[] = [];\n\n    for (const entry of jsCoverage) {\n      const totalBytes = entry.text?.length ?? 0;\n      const usedBytes = (entry.ranges ?? []).reduce((sum: number, r: any) => sum + (r.end - r.start), 0);\n      if (totalBytes > 0) {\n        entries.push({ url: entry.url, type: 'js', totalBytes, usedBytes, unusedPercent: Math.round((1 - usedBytes / totalBytes) * 100) });\n      }\n    }\n\n    for (const entry of cssCoverage) {\n      const totalBytes = entry.text?.length ?? 0;\n      const usedBytes = (entry.ranges ?? []).reduce((sum: number, r: any) => sum + (r.end - r.start), 0);\n      if (totalBytes > 0) {\n        entries.push({ url: entry.url, type: 'css', totalBytes, usedBytes, unusedPercent: Math.round((1 - usedBytes / totalBytes) * 100) });\n      }\n    }\n\n    const jsEntries = entries.filter(e => e.type === 'js');\n    const cssEntries = entries.filter(e => e.type === 'css');\n    const totalJsBytes = jsEntries.reduce((s, e) => s + e.totalBytes, 0);\n    const usedJsBytes = jsEntries.reduce((s, e) => s + e.usedBytes, 0);\n    const totalCssBytes = cssEntries.reduce((s, e) => s + e.totalBytes, 0);\n    const usedCssBytes = cssEntries.reduce((s, e) => s + e.usedBytes, 0);\n\n    return {\n      pageUrl,\n      entries,\n      totalJsBytes,\n      usedJsBytes,\n      totalCssBytes,\n      usedCssBytes,\n      jsUnusedPercent: totalJsBytes > 0 ? Math.round((1 - usedJsBytes / totalJsBytes) * 100) : 0,\n      cssUnusedPercent: totalCssBytes > 0 ? Math.round((1 - usedCssBytes / totalCssBytes) * 100) : 0,\n      collectedAt: new Date().toISOString(),\n    };\n  } catch {\n    return null;\n  }\n}\n"],
+  "mappings": "AA+BA,eAAsB,cAAc,MAA2B;AAC7D,MAAI;AACF,UAAM,QAAQ,IAAI;AAAA,MAChB,KAAK,SAAS,gBAAgB,EAAE,mBAAmB,MAAM,CAAC;AAAA,MAC1D,KAAK,SAAS,iBAAiB,EAAE,mBAAmB,MAAM,CAAC;AAAA,IAC7D,CAAC;AAAA,EACH,QAAQ;AAAA,EAAmD;AAC7D;AAEA,eAAsB,uBAAuB,MAAY,SAAiD;AACxG,MAAI;AACF,UAAM,CAAC,YAAY,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MAClD,KAAK,SAAS,eAAe,EAAE,MAAM,MAAM,CAAC,CAAU;AAAA,MACtD,KAAK,SAAS,gBAAgB,EAAE,MAAM,MAAM,CAAC,CAAU;AAAA,IACzD,CAAC;AAED,UAAM,UAA2B,CAAC;AAElC,eAAW,SAAS,YAAY;AAC9B,YAAM,aAAa,MAAM,MAAM,UAAU;AACzC,YAAM,aAAa,MAAM,UAAU,CAAC,GAAG,OAAO,CAAC,KAAa,MAAW,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC;AACjG,UAAI,aAAa,GAAG;AAClB,gBAAQ,KAAK,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,YAAY,WAAW,eAAe,KAAK,OAAO,IAAI,YAAY,cAAc,GAAG,EAAE,CAAC;AAAA,MACnI;AAAA,IACF;AAEA,eAAW,SAAS,aAAa;AAC/B,YAAM,aAAa,MAAM,MAAM,UAAU;AACzC,YAAM,aAAa,MAAM,UAAU,CAAC,GAAG,OAAO,CAAC,KAAa,MAAW,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC;AACjG,UAAI,aAAa,GAAG;AAClB,gBAAQ,KAAK,EAAE,KAAK,MAAM,KAAK,MAAM,OAAO,YAAY,WAAW,eAAe,KAAK,OAAO,IAAI,YAAY,cAAc,GAAG,EAAE,CAAC;AAAA,MACpI;AAAA,IACF;AAEA,UAAM,YAAY,QAAQ,OAAO,OAAK,EAAE,SAAS,IAAI;AACrD,UAAM,aAAa,QAAQ,OAAO,OAAK,EAAE,SAAS,KAAK;AACvD,UAAM,eAAe,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,YAAY,CAAC;AACnE,UAAM,cAAc,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,WAAW,CAAC;AACjE,UAAM,gBAAgB,WAAW,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,YAAY,CAAC;AACrE,UAAM,eAAe,WAAW,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,WAAW,CAAC;AAEnE,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,eAAe,IAAI,KAAK,OAAO,IAAI,cAAc,gBAAgB,GAAG,IAAI;AAAA,MACzF,kBAAkB,gBAAgB,IAAI,KAAK,OAAO,IAAI,eAAe,iBAAiB,GAAG,IAAI;AAAA,MAC7F,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,IACtC;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/js-router-detector.d.ts

@@ -0,0 +1,17 @@
+/**
+ * P2: JS Router Detection
+ *
+ * Scans the loaded page for client-side route tables declared by React Router v6,
+ * Vue Router, Angular Router, or custom window globals. Supplements the existing
+ * history.pushState interception (which only captures runtime navigation) with a
+ * static analysis of routes declared in the JS bundle.
+ *
+ * Called once per page after goto(). Runs entirely in-browser via page.evaluate().
+ * Adds discovered routes to the BFS frontier so the crawler visits them.
+ */
+/**
+ * Extract absolute page URLs from the current page's JS router config.
+ * Returns deduplicated absolute URLs (same origin). Dynamic segments (:id, *)
+ * are skipped — the crawler can't resolve them without knowing valid IDs.
+ */
+export declare function detectJsRoutes(page: any, origin: string): Promise<string[]>;

package/dist/js-router-detector.js

@@ -0,0 +1,90 @@
+import { logger as rootLogger } from "./logger.js";
+const logger = rootLogger.child({ module: "js-router-detector" });
+async function detectJsRoutes(page, origin) {
+  try {
+    const raw = await page.evaluate(() => {
+      const found = [];
+      const windowGlobals = [
+        "__routes__",
+        "__ROUTER__",
+        "__vue_router__",
+        "__router__",
+        "router",
+        "_router",
+        "__reactRouter",
+        "__NEXT_DATA__"
+      ];
+      for (const g of windowGlobals) {
+        try {
+          const val = window[g];
+          if (!val || typeof val !== "object") continue;
+          if (g === "__NEXT_DATA__" && Array.isArray(val.buildManifest?.pages)) {
+            for (const p of val.buildManifest.pages) if (typeof p === "string") found.push(p);
+            continue;
+          }
+          const routeArr = (Array.isArray(val.options?.routes) ? val.options.routes : null) ?? (Array.isArray(val.routes) ? val.routes : null) ?? (typeof val.getRoutes === "function" ? val.getRoutes() : null);
+          if (!Array.isArray(routeArr)) continue;
+          const flatRoutes = (arr, prefix = "") => {
+            for (const r of arr) {
+              if (!r || typeof r !== "object") continue;
+              const path = r.path ?? r.href ?? r.url ?? "";
+              const full = path.startsWith("/") ? path : `${prefix}/${path}`.replace(/\/\//g, "/");
+              if (full && full !== "/") found.push(full);
+              if (Array.isArray(r.children)) flatRoutes(r.children, full);
+            }
+          };
+          flatRoutes(routeArr);
+        } catch {
+        }
+      }
+      const routePatterns = [
+        /\bpath\s*:\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /\broute\s*\(\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /<Route[^>]+path\s*=\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /to\s*:\s*['"](\/?[\w\-\/\.~@]+)['"]/g
+      ];
+      const scripts = Array.from(document.querySelectorAll("script:not([src])"));
+      for (const el of scripts.slice(0, 15)) {
+        const src = el.textContent ?? "";
+        if (src.length < 100 || src.length > 8e5) continue;
+        for (const pat of routePatterns) {
+          const hits = [];
+          let m;
+          pat.lastIndex = 0;
+          while ((m = pat.exec(src)) !== null && hits.length < 100) {
+            if (m[1] && m[1].startsWith("/") && m[1].length > 1) hits.push(m[1]);
+          }
+          if (hits.length > 2) for (const h of hits) found.push(h);
+        }
+      }
+      return found;
+    });
+    const DYNAMIC = /[*:?]/;
+    const ASSET = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|map|json|xml)$/i;
+    const seen = /* @__PURE__ */ new Set();
+    const result = [];
+    for (const r of raw) {
+      if (!r || typeof r !== "string" || DYNAMIC.test(r) || ASSET.test(r)) continue;
+      const abs = r.startsWith("http") ? r : `${origin}${r.startsWith("/") ? "" : "/"}${r}`;
+      try {
+        const parsed = new URL(abs);
+        if (parsed.origin !== origin) continue;
+      } catch {
+        continue;
+      }
+      if (!seen.has(abs)) {
+        seen.add(abs);
+        result.push(abs);
+      }
+    }
+    if (result.length > 0) logger.info({ count: result.length }, "[P2] JS router: routes detected");
+    return result;
+  } catch (err) {
+    logger.warn({ err: String(err) }, "[P2] JS router detection failed \u2014 non-fatal");
+    return [];
+  }
+}
+export {
+  detectJsRoutes
+};
+//# sourceMappingURL=js-router-detector.js.map

package/dist/js-router-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/js-router-detector.ts"],
+  "sourcesContent": ["/**\n * P2: JS Router Detection\n *\n * Scans the loaded page for client-side route tables declared by React Router v6,\n * Vue Router, Angular Router, or custom window globals. Supplements the existing\n * history.pushState interception (which only captures runtime navigation) with a\n * static analysis of routes declared in the JS bundle.\n *\n * Called once per page after goto(). Runs entirely in-browser via page.evaluate().\n * Adds discovered routes to the BFS frontier so the crawler visits them.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'js-router-detector' });\n\n/**\n * Extract absolute page URLs from the current page's JS router config.\n * Returns deduplicated absolute URLs (same origin). Dynamic segments (:id, *)\n * are skipped \u2014 the crawler can't resolve them without knowing valid IDs.\n */\nexport async function detectJsRoutes(page: any, origin: string): Promise<string[]> {\n  try {\n    const raw: string[] = await page.evaluate(() => {\n      const found: string[] = [];\n\n      // 1. Check common framework globals \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n      const windowGlobals = [\n        '__routes__', '__ROUTER__', '__vue_router__', '__router__',\n        'router', '_router', '__reactRouter', '__NEXT_DATA__',\n      ];\n      for (const g of windowGlobals) {\n        try {\n          const val = (window as any)[g];\n          if (!val || typeof val !== 'object') continue;\n\n          // Next.js page manifest\n          if (g === '__NEXT_DATA__' && Array.isArray(val.buildManifest?.pages)) {\n            for (const p of val.buildManifest.pages) if (typeof p === 'string') found.push(p);\n            continue;\n          }\n\n          // Router objects: options.routes, routes, getRoutes()\n          const routeArr =\n            (Array.isArray(val.options?.routes) ? val.options.routes : null) ??\n            (Array.isArray(val.routes) ? val.routes : null) ??\n            (typeof val.getRoutes === 'function' ? val.getRoutes() : null);\n\n          if (!Array.isArray(routeArr)) continue;\n\n          const flatRoutes = (arr: any[], prefix = ''): void => {\n            for (const r of arr) {\n              if (!r || typeof r !== 'object') continue;\n              const path: string = r.path ?? r.href ?? r.url ?? '';\n              const full = path.startsWith('/') ? path : `${prefix}/${path}`.replace(/\\/\\//g, '/');\n              if (full && full !== '/') found.push(full);\n              if (Array.isArray(r.children)) flatRoutes(r.children, full);\n            }\n          };\n          flatRoutes(routeArr);\n        } catch { /* skip corrupt globals */ }\n      }\n\n      // 2. Scan inline <script> content for route patterns \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n      // Two-pass: regex first, if >3 found trust them. Below 3 \u2192 skip (too noisy).\n      const routePatterns = [\n        /\\bpath\\s*:\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n        /\\broute\\s*\\(\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n        /<Route[^>]+path\\s*=\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n        /to\\s*:\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n      ];\n      const scripts = Array.from(document.querySelectorAll('script:not([src])'));\n      for (const el of scripts.slice(0, 15)) {\n        const src = el.textContent ?? '';\n        if (src.length < 100 || src.length > 800_000) continue;\n        for (const pat of routePatterns) {\n          const hits: string[] = [];\n          let m: RegExpExecArray | null;\n          pat.lastIndex = 0;\n          while ((m = pat.exec(src)) !== null && hits.length < 100) {\n            if (m[1] && m[1].startsWith('/') && m[1].length > 1) hits.push(m[1]);\n          }\n          // Only trust if we found >2 hits (reduces false positives)\n          if (hits.length > 2) for (const h of hits) found.push(h);\n        }\n      }\n\n      return found;\n    });\n\n    // Post-process: resolve to absolute, dedup, drop dynamic segments\n    const DYNAMIC = /[*:?]/;\n    const ASSET = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|map|json|xml)$/i;\n    const seen = new Set<string>();\n    const result: string[] = [];\n    for (const r of raw) {\n      if (!r || typeof r !== 'string' || DYNAMIC.test(r) || ASSET.test(r)) continue;\n      const abs = r.startsWith('http') ? r : `${origin}${r.startsWith('/') ? '' : '/'}${r}`;\n      try {\n        const parsed = new URL(abs);\n        if (parsed.origin !== origin) continue;\n      } catch { continue; }\n      if (!seen.has(abs)) { seen.add(abs); result.push(abs); }\n    }\n\n    if (result.length > 0) logger.info({ count: result.length }, '[P2] JS router: routes detected');\n    return result;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[P2] JS router detection failed \u2014 non-fatal');\n    return [];\n  }\n}\n"],
+  "mappings": "AAYA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,qBAAqB,CAAC;AAOhE,eAAsB,eAAe,MAAW,QAAmC;AACjF,MAAI;AACF,UAAM,MAAgB,MAAM,KAAK,SAAS,MAAM;AAC9C,YAAM,QAAkB,CAAC;AAGzB,YAAM,gBAAgB;AAAA,QACpB;AAAA,QAAc;AAAA,QAAc;AAAA,QAAkB;AAAA,QAC9C;AAAA,QAAU;AAAA,QAAW;AAAA,QAAiB;AAAA,MACxC;AACA,iBAAW,KAAK,eAAe;AAC7B,YAAI;AACF,gBAAM,MAAO,OAAe,CAAC;AAC7B,cAAI,CAAC,OAAO,OAAO,QAAQ,SAAU;AAGrC,cAAI,MAAM,mBAAmB,MAAM,QAAQ,IAAI,eAAe,KAAK,GAAG;AACpE,uBAAW,KAAK,IAAI,cAAc,MAAO,KAAI,OAAO,MAAM,SAAU,OAAM,KAAK,CAAC;AAChF;AAAA,UACF;AAGA,gBAAM,YACH,MAAM,QAAQ,IAAI,SAAS,MAAM,IAAI,IAAI,QAAQ,SAAS,UAC1D,MAAM,QAAQ,IAAI,MAAM,IAAI,IAAI,SAAS,UACzC,OAAO,IAAI,cAAc,aAAa,IAAI,UAAU,IAAI;AAE3D,cAAI,CAAC,MAAM,QAAQ,QAAQ,EAAG;AAE9B,gBAAM,aAAa,CAAC,KAAY,SAAS,OAAa;AACpD,uBAAW,KAAK,KAAK;AACnB,kBAAI,CAAC,KAAK,OAAO,MAAM,SAAU;AACjC,oBAAM,OAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;AAClD,oBAAM,OAAO,KAAK,WAAW,GAAG,IAAI,OAAO,GAAG,MAAM,IAAI,IAAI,GAAG,QAAQ,SAAS,GAAG;AACnF,kBAAI,QAAQ,SAAS,IAAK,OAAM,KAAK,IAAI;AACzC,kBAAI,MAAM,QAAQ,EAAE,QAAQ,EAAG,YAAW,EAAE,UAAU,IAAI;AAAA,YAC5D;AAAA,UACF;AACA,qBAAW,QAAQ;AAAA,QACrB,QAAQ;AAAA,QAA6B;AAAA,MACvC;AAIA,YAAM,gBAAgB;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,YAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,mBAAmB,CAAC;AACzE,iBAAW,MAAM,QAAQ,MAAM,GAAG,EAAE,GAAG;AACrC,cAAM,MAAM,GAAG,eAAe;AAC9B,YAAI,IAAI,SAAS,OAAO,IAAI,SAAS,IAAS;AAC9C,mBAAW,OAAO,eAAe;AAC/B,gBAAM,OAAiB,CAAC;AACxB,cAAI;AACJ,cAAI,YAAY;AAChB,kBAAQ,IAAI,IAAI,KAAK,GAAG,OAAO,QAAQ,KAAK,SAAS,KAAK;AACxD,gBAAI,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,WAAW,GAAG,KAAK,EAAE,CAAC,EAAE,SAAS,EAAG,MAAK,KAAK,EAAE,CAAC,CAAC;AAAA,UACrE;AAEA,cAAI,KAAK,SAAS,EAAG,YAAW,KAAK,KAAM,OAAM,KAAK,CAAC;AAAA,QACzD;AAAA,MACF;AAEA,aAAO;AAAA,IACT,CAAC;AAGD,UAAM,UAAU;AAChB,UAAM,QAAQ;AACd,UAAM,OAAO,oBAAI,IAAY;AAC7B,UAAM,SAAmB,CAAC;AAC1B,eAAW,KAAK,KAAK;AACnB,UAAI,CAAC,KAAK,OAAO,MAAM,YAAY,QAAQ,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,EAAG;AACrE,YAAM,MAAM,EAAE,WAAW,MAAM,IAAI,IAAI,GAAG,MAAM,GAAG,EAAE,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,CAAC;AACnF,UAAI;AACF,cAAM,SAAS,IAAI,IAAI,GAAG;AAC1B,YAAI,OAAO,WAAW,OAAQ;AAAA,MAChC,QAAQ;AAAE;AAAA,MAAU;AACpB,UAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAAE,aAAK,IAAI,GAAG;AAAG,eAAO,KAAK,GAAG;AAAA,MAAG;AAAA,IACzD;AAEA,QAAI,OAAO,SAAS,EAAG,QAAO,KAAK,EAAE,OAAO,OAAO,OAAO,GAAG,iCAAiC;AAC9F,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,kDAA6C;AAC/E,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/language-detector.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Language detector — identify page language from HTML attributes and content analysis.
+ *
+ * Sources (in priority order):
+ * 1. <html lang="..."> attribute
+ * 2. <meta http-equiv="Content-Language"> header
+ * 3. hreflang alternate links
+ * 4. Content-based n-gram fingerprint (detects top 20 languages)
+ *
+ * Used to: route pages to the correct markdown processor, group screens by locale,
+ * and detect missing hreflang implementations.
+ */
+import type { Page } from 'playwright';
+export interface LanguageDetectionResult {
+    url: string;
+    declaredLang: string;
+    detectedLang: string;
+    confidence: 'high' | 'medium' | 'low';
+    alternates: Record<string, string>;
+    isMultilingual: boolean;
+}
+export declare function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult>;

package/dist/language-detector.js

@@ -0,0 +1,57 @@
+const LANG_SIGNATURES = {
+  en: /\b(the|and|for|are|but|not|you|all|can|had|her|was|one|our|out|day|get|has|him|his|how)\b/gi,
+  es: /\b(que|con|una|por|del|las|los|para|como|pero|más|este|esto|sus|ser|hay|sobre|entre|también)\b/gi,
+  fr: /\b(les|des|une|est|dans|pour|qui|par|sur|avec|que|pas|son|tout|mais|bien|comme|vous)\b/gi,
+  de: /\b(die|der|und|den|das|ein|ist|von|mit|dem|auf|sich|des|aber|auch|bei|aus|nach)\b/gi,
+  pt: /\b(que|não|uma|com|por|dos|para|como|mais|mas|seu|sua|nas|nos|ele|ela|são)\b/gi,
+  it: /\b(che|con|del|per|una|nel|non|sono|gli|dei|alla|delle|questa|questo|come|più)\b/gi,
+  nl: /\b(van|het|een|zijn|maar|ook|dan|door|op|uit|aan|bij|om|al|veel|haar|hun)\b/gi,
+  ja: /[぀-ゟ゠-ヿ]/g,
+  zh: /[一-鿿]/g,
+  ar: /[؀-ۿ]/g,
+  hi: /[ऀ-ॿ]/g,
+  ko: /[가-힯ᄀ-ᇿ]/g
+};
+function detectFromContent(text) {
+  if (!text || text.length < 50) return "en";
+  let best = "en";
+  let bestScore = 0;
+  for (const [lang, re] of Object.entries(LANG_SIGNATURES)) {
+    const matches = (text.match(re) ?? []).length;
+    const score = matches / Math.max(text.length / 100, 1);
+    if (score > bestScore) {
+      bestScore = score;
+      best = lang;
+    }
+  }
+  return best;
+}
+async function detectLanguage(page, url) {
+  const data = await page.evaluate(() => {
+    const htmlLang = document.documentElement.lang?.toLowerCase().split("-")[0] ?? "";
+    const metaLang = document.querySelector('meta[http-equiv="Content-Language"]')?.content?.toLowerCase().split("-")[0] ?? "";
+    const alternates = {};
+    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
+      const lang = el.getAttribute("hreflang")?.toLowerCase();
+      const href = el.getAttribute("href") ?? "";
+      if (lang && lang !== "x-default") alternates[lang] = href;
+    });
+    const bodyText = (document.body?.innerText ?? "").slice(0, 2e3);
+    return { htmlLang, metaLang, alternates, bodyText };
+  }).catch(() => ({ htmlLang: "", metaLang: "", alternates: {}, bodyText: "" }));
+  const declaredLang = data.htmlLang || data.metaLang || "";
+  const detectedLang = detectFromContent(data.bodyText);
+  const confidence = declaredLang ? "high" : data.bodyText.length > 200 ? "medium" : "low";
+  return {
+    url,
+    declaredLang,
+    detectedLang: declaredLang || detectedLang,
+    confidence,
+    alternates: data.alternates,
+    isMultilingual: Object.keys(data.alternates).length > 1
+  };
+}
+export {
+  detectLanguage
+};
+//# sourceMappingURL=language-detector.js.map

package/dist/language-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/language-detector.ts"],
+  "sourcesContent": ["/**\n * Language detector \u2014 identify page language from HTML attributes and content analysis.\n *\n * Sources (in priority order):\n * 1. <html lang=\"...\"> attribute\n * 2. <meta http-equiv=\"Content-Language\"> header\n * 3. hreflang alternate links\n * 4. Content-based n-gram fingerprint (detects top 20 languages)\n *\n * Used to: route pages to the correct markdown processor, group screens by locale,\n * and detect missing hreflang implementations.\n */\n\nimport type { Page } from 'playwright';\n\n// Frequency signatures for n-gram based detection\n// Common trigrams by language (simplified \u2014 not production-grade)\nconst LANG_SIGNATURES: Record<string, RegExp> = {\n  en: /\\b(the|and|for|are|but|not|you|all|can|had|her|was|one|our|out|day|get|has|him|his|how)\\b/gi,\n  es: /\\b(que|con|una|por|del|las|los|para|como|pero|m\u00E1s|este|esto|sus|ser|hay|sobre|entre|tambi\u00E9n)\\b/gi,\n  fr: /\\b(les|des|une|est|dans|pour|qui|par|sur|avec|que|pas|son|tout|mais|bien|comme|vous)\\b/gi,\n  de: /\\b(die|der|und|den|das|ein|ist|von|mit|dem|auf|sich|des|aber|auch|bei|aus|nach)\\b/gi,\n  pt: /\\b(que|n\u00E3o|uma|com|por|dos|para|como|mais|mas|seu|sua|nas|nos|ele|ela|s\u00E3o)\\b/gi,\n  it: /\\b(che|con|del|per|una|nel|non|sono|gli|dei|alla|delle|questa|questo|come|pi\u00F9)\\b/gi,\n  nl: /\\b(van|het|een|zijn|maar|ook|dan|door|op|uit|aan|bij|om|al|veel|haar|hun)\\b/gi,\n  ja: /[\u3040-\u309F\u30A0-\u30FF]/g,\n  zh: /[\u4E00-\u9FFF]/g,\n  ar: /[\u0600-\u06FF]/g,\n  hi: /[\u0900-\u097F]/g,\n  ko: /[\uAC00-\uD7AF\u1100-\u11FF]/g,\n};\n\nfunction detectFromContent(text: string): string {\n  if (!text || text.length < 50) return 'en';\n  let best = 'en';\n  let bestScore = 0;\n  for (const [lang, re] of Object.entries(LANG_SIGNATURES)) {\n    const matches = (text.match(re) ?? []).length;\n    const score = matches / Math.max(text.length / 100, 1);\n    if (score > bestScore) { bestScore = score; best = lang; }\n  }\n  return best;\n}\n\nexport interface LanguageDetectionResult {\n  url: string;\n  declaredLang: string;   // from <html lang> or meta\n  detectedLang: string;   // from content analysis\n  confidence: 'high' | 'medium' | 'low';\n  alternates: Record<string, string>; // lang \u2192 url from hreflang\n  isMultilingual: boolean;\n}\n\nexport async function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult> {\n  const data = await page.evaluate(() => {\n    const htmlLang = document.documentElement.lang?.toLowerCase().split('-')[0] ?? '';\n    const metaLang = (document.querySelector('meta[http-equiv=\"Content-Language\"]') as any)?.content?.toLowerCase().split('-')[0] ?? '';\n    const alternates: Record<string, string> = {};\n    document.querySelectorAll('link[rel=\"alternate\"][hreflang]').forEach((el: any) => {\n      const lang = el.getAttribute('hreflang')?.toLowerCase();\n      const href = el.getAttribute('href') ?? '';\n      if (lang && lang !== 'x-default') alternates[lang] = href;\n    });\n    const bodyText = (document.body?.innerText ?? '').slice(0, 2000);\n    return { htmlLang, metaLang, alternates, bodyText };\n  }).catch(() => ({ htmlLang: '', metaLang: '', alternates: {}, bodyText: '' }));\n\n  const declaredLang = data.htmlLang || data.metaLang || '';\n  const detectedLang = detectFromContent(data.bodyText);\n\n  const confidence: LanguageDetectionResult['confidence'] =\n    declaredLang ? 'high' : data.bodyText.length > 200 ? 'medium' : 'low';\n\n  return {\n    url,\n    declaredLang,\n    detectedLang: declaredLang || detectedLang,\n    confidence,\n    alternates: data.alternates,\n    isMultilingual: Object.keys(data.alternates).length > 1,\n  };\n}\n"],
+  "mappings": "AAiBA,MAAM,kBAA0C;AAAA,EAC9C,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AACN;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,MAAI,CAAC,QAAQ,KAAK,SAAS,GAAI,QAAO;AACtC,MAAI,OAAO;AACX,MAAI,YAAY;AAChB,aAAW,CAAC,MAAM,EAAE,KAAK,OAAO,QAAQ,eAAe,GAAG;AACxD,UAAM,WAAW,KAAK,MAAM,EAAE,KAAK,CAAC,GAAG;AACvC,UAAM,QAAQ,UAAU,KAAK,IAAI,KAAK,SAAS,KAAK,CAAC;AACrD,QAAI,QAAQ,WAAW;AAAE,kBAAY;AAAO,aAAO;AAAA,IAAM;AAAA,EAC3D;AACA,SAAO;AACT;AAWA,eAAsB,eAAe,MAAY,KAA+C;AAC9F,QAAM,OAAO,MAAM,KAAK,SAAS,MAAM;AACrC,UAAM,WAAW,SAAS,gBAAgB,MAAM,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,KAAK;AAC/E,UAAM,WAAY,SAAS,cAAc,qCAAqC,GAAW,SAAS,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,KAAK;AACjI,UAAM,aAAqC,CAAC;AAC5C,aAAS,iBAAiB,iCAAiC,EAAE,QAAQ,CAAC,OAAY;AAChF,YAAM,OAAO,GAAG,aAAa,UAAU,GAAG,YAAY;AACtD,YAAM,OAAO,GAAG,aAAa,MAAM,KAAK;AACxC,UAAI,QAAQ,SAAS,YAAa,YAAW,IAAI,IAAI;AAAA,IACvD,CAAC;AACD,UAAM,YAAY,SAAS,MAAM,aAAa,IAAI,MAAM,GAAG,GAAI;AAC/D,WAAO,EAAE,UAAU,UAAU,YAAY,SAAS;AAAA,EACpD,CAAC,EAAE,MAAM,OAAO,EAAE,UAAU,IAAI,UAAU,IAAI,YAAY,CAAC,GAAG,UAAU,GAAG,EAAE;AAE7E,QAAM,eAAe,KAAK,YAAY,KAAK,YAAY;AACvD,QAAM,eAAe,kBAAkB,KAAK,QAAQ;AAEpD,QAAM,aACJ,eAAe,SAAS,KAAK,SAAS,SAAS,MAAM,WAAW;AAElE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,gBAAgB;AAAA,IAC9B;AAAA,IACA,YAAY,KAAK;AAAA,IACjB,gBAAgB,OAAO,KAAK,KAAK,UAAU,EAAE,SAAS;AAAA,EACxD;AACF;",
+  "names": []
+}

package/dist/link-classifier.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Link classifier — assigns priority and category to discovered URLs.
+ *
+ * Categories:
+ * - 'nav'       : site navigation (homepage, top-level pages) — highest priority
+ * - 'auth'      : login/register/password flows — high priority
+ * - 'app'       : authenticated app screens — high priority
+ * - 'content'   : blog/docs/articles — medium priority
+ * - 'api'       : REST/GraphQL API endpoints — detected, not crawled as screens
+ * - 'resource'  : static assets (JS/CSS/img/fonts) — skip
+ * - 'external'  : different domain — skip unless cross-domain crawl enabled
+ * - 'low-value' : pagination, sort, filter query strings — deprioritize
+ */
+export type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';
+export interface ClassifiedLink {
+    url: string;
+    category: LinkCategory;
+    priority: number;
+    reason: string;
+}
+export declare function classifyLink(url: string, baseUrl: string): ClassifiedLink;
+export declare function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[];
+export declare function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[];

package/dist/link-classifier.js

@@ -0,0 +1,40 @@
+const RESOURCE_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|mp4|mp3|pdf|zip|gz|tar|exe|dmg|apk)(\?|$)/i;
+const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i;
+const AUTH_PATTERN = /\/(login|signin|sign-in|register|signup|sign-up|auth|oauth|sso|password|forgot|reset|logout|signout)\b/i;
+const LOW_VALUE_PATTERN = /[?&](page|p|offset|sort|order|filter|q|search|tag|category|color|size|brand)=/i;
+const PAGINATION_PATTERN = /\/page\/\d+|\/p\/\d+|\bpage=\d+/i;
+function classifyLink(url, baseUrl) {
+  let parsed;
+  let baseHost;
+  try {
+    parsed = new URL(url);
+    baseHost = new URL(baseUrl).hostname;
+  } catch {
+    return { url, category: "low-value", priority: 0, reason: "invalid URL" };
+  }
+  const path = parsed.pathname + parsed.search;
+  const isExternal = parsed.hostname !== baseHost;
+  if (isExternal) return { url, category: "external", priority: 0, reason: "different domain" };
+  if (RESOURCE_EXT.test(path)) return { url, category: "resource", priority: 0, reason: "static asset extension" };
+  if (API_PATTERN.test(path)) return { url, category: "api", priority: 10, reason: "API path pattern" };
+  if (AUTH_PATTERN.test(path)) return { url, category: "auth", priority: 90, reason: "auth flow pattern" };
+  if (PAGINATION_PATTERN.test(path)) return { url, category: "low-value", priority: 15, reason: "pagination" };
+  if (LOW_VALUE_PATTERN.test(path)) return { url, category: "low-value", priority: 20, reason: "filter/sort query param" };
+  const segments = parsed.pathname.split("/").filter(Boolean);
+  if (segments.length === 0) return { url, category: "nav", priority: 100, reason: "homepage" };
+  if (segments.length === 1) return { url, category: "nav", priority: 80, reason: "top-level nav" };
+  if (segments.length <= 3) return { url, category: "app", priority: 60, reason: "app screen" };
+  return { url, category: "content", priority: 40, reason: "deep content page" };
+}
+function classifyLinks(urls, baseUrl) {
+  return urls.map((u) => classifyLink(u, baseUrl)).sort((a, b) => b.priority - a.priority);
+}
+function filterCrawlable(classified) {
+  return classified.filter((l) => l.category !== "resource" && l.category !== "external" && l.category !== "api");
+}
+export {
+  classifyLink,
+  classifyLinks,
+  filterCrawlable
+};
+//# sourceMappingURL=link-classifier.js.map

package/dist/link-classifier.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/link-classifier.ts"],
+  "sourcesContent": ["/**\n * Link classifier \u2014 assigns priority and category to discovered URLs.\n *\n * Categories:\n * - 'nav'       : site navigation (homepage, top-level pages) \u2014 highest priority\n * - 'auth'      : login/register/password flows \u2014 high priority\n * - 'app'       : authenticated app screens \u2014 high priority\n * - 'content'   : blog/docs/articles \u2014 medium priority\n * - 'api'       : REST/GraphQL API endpoints \u2014 detected, not crawled as screens\n * - 'resource'  : static assets (JS/CSS/img/fonts) \u2014 skip\n * - 'external'  : different domain \u2014 skip unless cross-domain crawl enabled\n * - 'low-value' : pagination, sort, filter query strings \u2014 deprioritize\n */\n\nexport type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';\n\nexport interface ClassifiedLink {\n  url: string;\n  category: LinkCategory;\n  priority: number; // 0-100, higher = crawl sooner\n  reason: string;\n}\n\nconst RESOURCE_EXT = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|mp4|mp3|pdf|zip|gz|tar|exe|dmg|apk)(\\?|$)/i;\nconst API_PATTERN = /\\/(api|graphql|gql|rest|v\\d+)\\//i;\nconst AUTH_PATTERN = /\\/(login|signin|sign-in|register|signup|sign-up|auth|oauth|sso|password|forgot|reset|logout|signout)\\b/i;\nconst LOW_VALUE_PATTERN = /[?&](page|p|offset|sort|order|filter|q|search|tag|category|color|size|brand)=/i;\nconst PAGINATION_PATTERN = /\\/page\\/\\d+|\\/p\\/\\d+|\\bpage=\\d+/i;\n\nexport function classifyLink(url: string, baseUrl: string): ClassifiedLink {\n  let parsed: URL;\n  let baseHost: string;\n  try {\n    parsed = new URL(url);\n    baseHost = new URL(baseUrl).hostname;\n  } catch {\n    return { url, category: 'low-value', priority: 0, reason: 'invalid URL' };\n  }\n\n  const path = parsed.pathname + parsed.search;\n  const isExternal = parsed.hostname !== baseHost;\n\n  if (isExternal) return { url, category: 'external', priority: 0, reason: 'different domain' };\n  if (RESOURCE_EXT.test(path)) return { url, category: 'resource', priority: 0, reason: 'static asset extension' };\n  if (API_PATTERN.test(path)) return { url, category: 'api', priority: 10, reason: 'API path pattern' };\n  if (AUTH_PATTERN.test(path)) return { url, category: 'auth', priority: 90, reason: 'auth flow pattern' };\n  if (PAGINATION_PATTERN.test(path)) return { url, category: 'low-value', priority: 15, reason: 'pagination' };\n  if (LOW_VALUE_PATTERN.test(path)) return { url, category: 'low-value', priority: 20, reason: 'filter/sort query param' };\n\n  const segments = parsed.pathname.split('/').filter(Boolean);\n  if (segments.length === 0) return { url, category: 'nav', priority: 100, reason: 'homepage' };\n  if (segments.length === 1) return { url, category: 'nav', priority: 80, reason: 'top-level nav' };\n  if (segments.length <= 3) return { url, category: 'app', priority: 60, reason: 'app screen' };\n\n  return { url, category: 'content', priority: 40, reason: 'deep content page' };\n}\n\nexport function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[] {\n  return urls.map(u => classifyLink(u, baseUrl))\n    .sort((a, b) => b.priority - a.priority);\n}\n\nexport function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[] {\n  return classified.filter(l => l.category !== 'resource' && l.category !== 'external' && l.category !== 'api');\n}\n"],
+  "mappings": "AAuBA,MAAM,eAAe;AACrB,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAEpB,SAAS,aAAa,KAAa,SAAiC;AACzE,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,GAAG;AACpB,eAAW,IAAI,IAAI,OAAO,EAAE;AAAA,EAC9B,QAAQ;AACN,WAAO,EAAE,KAAK,UAAU,aAAa,UAAU,GAAG,QAAQ,cAAc;AAAA,EAC1E;AAEA,QAAM,OAAO,OAAO,WAAW,OAAO;AACtC,QAAM,aAAa,OAAO,aAAa;AAEvC,MAAI,WAAY,QAAO,EAAE,KAAK,UAAU,YAAY,UAAU,GAAG,QAAQ,mBAAmB;AAC5F,MAAI,aAAa,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,YAAY,UAAU,GAAG,QAAQ,yBAAyB;AAC/G,MAAI,YAAY,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,IAAI,QAAQ,mBAAmB;AACpG,MAAI,aAAa,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,QAAQ,UAAU,IAAI,QAAQ,oBAAoB;AACvG,MAAI,mBAAmB,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,aAAa,UAAU,IAAI,QAAQ,aAAa;AAC3G,MAAI,kBAAkB,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,aAAa,UAAU,IAAI,QAAQ,0BAA0B;AAEvH,QAAM,WAAW,OAAO,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AAC1D,MAAI,SAAS,WAAW,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,KAAK,QAAQ,WAAW;AAC5F,MAAI,SAAS,WAAW,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,IAAI,QAAQ,gBAAgB;AAChG,MAAI,SAAS,UAAU,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,IAAI,QAAQ,aAAa;AAE5F,SAAO,EAAE,KAAK,UAAU,WAAW,UAAU,IAAI,QAAQ,oBAAoB;AAC/E;AAEO,SAAS,cAAc,MAAgB,SAAmC;AAC/E,SAAO,KAAK,IAAI,OAAK,aAAa,GAAG,OAAO,CAAC,EAC1C,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAC3C;AAEO,SAAS,gBAAgB,YAAgD;AAC9E,SAAO,WAAW,OAAO,OAAK,EAAE,aAAa,cAAc,EAAE,aAAa,cAAc,EAAE,aAAa,KAAK;AAC9G;",
+  "names": []
+}

package/dist/live-view.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export declare function startLiveView(appUrl: string, storageState?: string): Promise<string>;
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export declare function getFrame(id: string): string | null;
+/**
+ * Navigate the live view browser to a URL.
+ */
+export declare function liveViewNavigate(id: string, url: string): Promise<void>;
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export declare function completeLiveView(id: string): Promise<string>;
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export declare function sendInput(id: string, event: {
+    type: 'click' | 'type' | 'key';
+    x?: number;
+    y?: number;
+    button?: 'left' | 'right' | 'middle';
+    text?: string;
+    key?: string;
+}): Promise<void>;
+/**
+ * Close a live view session without capturing.
+ */
+export declare function closeLiveView(id: string): Promise<void>;
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export declare function listLiveViews(): string[];

package/dist/live-view.js

@@ -0,0 +1,105 @@
+import { chromium } from "playwright";
+import { randomBytes } from "node:crypto";
+import { logger } from "./logger.js";
+import { getNonHeadlessUserAgent } from "./crawler.js";
+const sessions = /* @__PURE__ */ new Map();
+const SESSION_TTL_MS = 15 * 60 * 1e3;
+function generateId() {
+  return randomBytes(16).toString("hex");
+}
+async function startLiveView(appUrl, storageState) {
+  const id = generateId();
+  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === "linux";
+  const browser = await chromium.launch({
+    headless: true,
+    args: isDocker ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu"] : []
+  });
+  const context = await browser.newContext({
+    viewport: { width: 1280, height: 800 },
+    userAgent: await getNonHeadlessUserAgent(chromium.executablePath())
+  });
+  if (storageState) {
+    try {
+      const state = JSON.parse(storageState);
+      if (state.cookies?.length) await context.addCookies(state.cookies);
+    } catch {
+    }
+  }
+  const page = await context.newPage();
+  await page.goto(appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
+  const session = {
+    id,
+    browser,
+    context,
+    page,
+    appUrl,
+    lastFrame: null,
+    streaming: true,
+    frameInterval: null,
+    createdAt: Date.now()
+  };
+  session.frameInterval = setInterval(async () => {
+    if (!session.streaming) return;
+    try {
+      const buf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false });
+      session.lastFrame = buf.toString("base64");
+    } catch {
+      session.streaming = false;
+    }
+  }, 500);
+  sessions.set(id, session);
+  setTimeout(() => closeLiveView(id).catch(() => {
+  }), SESSION_TTL_MS);
+  logger.info({ id, appUrl }, "[live-view] Session started");
+  return id;
+}
+function getFrame(id) {
+  return sessions.get(id)?.lastFrame ?? null;
+}
+async function liveViewNavigate(id, url) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  await session.page.goto(url, { waitUntil: "domcontentloaded" });
+}
+async function completeLiveView(id) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found or expired`);
+  const state = await session.context.storageState();
+  const json = JSON.stringify(state);
+  await closeLiveView(id);
+  return json;
+}
+async function sendInput(id, event) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  if (event.type === "click" && event.x !== void 0 && event.y !== void 0) {
+    await session.page.mouse.click(event.x, event.y, { button: event.button ?? "left" });
+  } else if (event.type === "type" && event.text) {
+    await session.page.keyboard.type(event.text);
+  } else if (event.type === "key" && event.key) {
+    await session.page.keyboard.press(event.key);
+  }
+}
+async function closeLiveView(id) {
+  const session = sessions.get(id);
+  if (!session) return;
+  session.streaming = false;
+  if (session.frameInterval) clearInterval(session.frameInterval);
+  sessions.delete(id);
+  await session.browser.close().catch(() => {
+  });
+  logger.info({ id }, "[live-view] Session closed");
+}
+function listLiveViews() {
+  return [...sessions.keys()];
+}
+export {
+  closeLiveView,
+  completeLiveView,
+  getFrame,
+  listLiveViews,
+  liveViewNavigate,
+  sendInput,
+  startLiveView
+};
+//# sourceMappingURL=live-view.js.map

package/dist/live-view.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/live-view.ts"],
+  "sourcesContent": ["import { chromium, type Browser, type BrowserContext, type Page } from 'playwright';\nimport { randomBytes } from 'node:crypto';\nimport { logger } from './logger.js';\nimport { getNonHeadlessUserAgent } from './crawler.js';\n\ninterface LiveViewSession {\n  id: string;\n  browser: Browser;\n  context: BrowserContext;\n  page: Page;\n  appUrl: string;\n  lastFrame: string | null; // base64 JPEG\n  streaming: boolean;\n  frameInterval: ReturnType<typeof setInterval> | null;\n  createdAt: number;\n}\n\nconst sessions = new Map<string, LiveViewSession>();\nconst SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min\n\nfunction generateId(): string {\n  return randomBytes(16).toString('hex');\n}\n\n/**\n * Start a live view session: launches a headless browser, navigates to appUrl,\n * begins capturing frames. Returns the session ID.\n */\nexport async function startLiveView(appUrl: string, storageState?: string): Promise<string> {\n  const id = generateId();\n  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === 'linux';\n\n  const browser = await chromium.launch({\n    headless: true,\n    args: isDocker ? ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] : [],\n  });\n\n  const context = await browser.newContext({\n    viewport: { width: 1280, height: 800 },\n    userAgent: await getNonHeadlessUserAgent(chromium.executablePath()),\n  });\n\n  // Restore session if provided\n  if (storageState) {\n    try {\n      const state = JSON.parse(storageState);\n      if (state.cookies?.length) await context.addCookies(state.cookies);\n    } catch { /* ignore */ }\n  }\n\n  const page = await context.newPage();\n  await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });\n\n  const session: LiveViewSession = {\n    id, browser, context, page, appUrl,\n    lastFrame: null, streaming: true, frameInterval: null,\n    createdAt: Date.now(),\n  };\n\n  // Start capturing frames (every 500ms)\n  session.frameInterval = setInterval(async () => {\n    if (!session.streaming) return;\n    try {\n      const buf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false });\n      session.lastFrame = buf.toString('base64');\n    } catch { session.streaming = false; }\n  }, 500);\n\n  sessions.set(id, session);\n\n  // Auto-cleanup after TTL\n  setTimeout(() => closeLiveView(id).catch(() => {}), SESSION_TTL_MS);\n\n  logger.info({ id, appUrl }, '[live-view] Session started');\n  return id;\n}\n\n/**\n * Get the latest frame for a session (base64 JPEG).\n */\nexport function getFrame(id: string): string | null {\n  return sessions.get(id)?.lastFrame ?? null;\n}\n\n/**\n * Navigate the live view browser to a URL.\n */\nexport async function liveViewNavigate(id: string, url: string): Promise<void> {\n  const session = sessions.get(id);\n  if (!session) throw new Error(`Live view session ${id} not found`);\n  await session.page.goto(url, { waitUntil: 'domcontentloaded' });\n}\n\n/**\n * Capture storageState from the live view session, close the browser, return JSON string.\n */\nexport async function completeLiveView(id: string): Promise<string> {\n  const session = sessions.get(id);\n  if (!session) throw new Error(`Live view session ${id} not found or expired`);\n\n  const state = await session.context.storageState();\n  const json = JSON.stringify(state);\n  await closeLiveView(id);\n  return json;\n}\n\n/**\n * Send a mouse or keyboard input event to the live view browser.\n */\nexport async function sendInput(id: string, event: {\n  type: 'click' | 'type' | 'key';\n  x?: number; y?: number;\n  button?: 'left' | 'right' | 'middle';\n  text?: string;\n  key?: string;\n}): Promise<void> {\n  const session = sessions.get(id);\n  if (!session) throw new Error(`Live view session ${id} not found`);\n  if (event.type === 'click' && event.x !== undefined && event.y !== undefined) {\n    await session.page.mouse.click(event.x, event.y, { button: event.button ?? 'left' });\n  } else if (event.type === 'type' && event.text) {\n    await session.page.keyboard.type(event.text);\n  } else if (event.type === 'key' && event.key) {\n    await session.page.keyboard.press(event.key);\n  }\n}\n\n/**\n * Close a live view session without capturing.\n */\nexport async function closeLiveView(id: string): Promise<void> {\n  const session = sessions.get(id);\n  if (!session) return;\n  session.streaming = false;\n  if (session.frameInterval) clearInterval(session.frameInterval);\n  sessions.delete(id);\n  await session.browser.close().catch(() => {});\n  logger.info({ id }, '[live-view] Session closed');\n}\n\n/**\n * List all active session IDs (for health checks / cleanup).\n */\nexport function listLiveViews(): string[] {\n  return [...sessions.keys()];\n}\n"],
+  "mappings": "AAAA,SAAS,gBAA8D;AACvE,SAAS,mBAAmB;AAC5B,SAAS,cAAc;AACvB,SAAS,+BAA+B;AAcxC,MAAM,WAAW,oBAAI,IAA6B;AAClD,MAAM,iBAAiB,KAAK,KAAK;AAEjC,SAAS,aAAqB;AAC5B,SAAO,YAAY,EAAE,EAAE,SAAS,KAAK;AACvC;AAMA,eAAsB,cAAc,QAAgB,cAAwC;AAC1F,QAAM,KAAK,WAAW;AACtB,QAAM,WAAW,CAAC,CAAC,QAAQ,IAAI,wBAAwB,QAAQ,aAAa;AAE5E,QAAM,UAAU,MAAM,SAAS,OAAO;AAAA,IACpC,UAAU;AAAA,IACV,MAAM,WAAW,CAAC,gBAAgB,4BAA4B,2BAA2B,eAAe,IAAI,CAAC;AAAA,EAC/G,CAAC;AAED,QAAM,UAAU,MAAM,QAAQ,WAAW;AAAA,IACvC,UAAU,EAAE,OAAO,MAAM,QAAQ,IAAI;AAAA,IACrC,WAAW,MAAM,wBAAwB,SAAS,eAAe,CAAC;AAAA,EACpE,CAAC;AAGD,MAAI,cAAc;AAChB,QAAI;AACF,YAAM,QAAQ,KAAK,MAAM,YAAY;AACrC,UAAI,MAAM,SAAS,OAAQ,OAAM,QAAQ,WAAW,MAAM,OAAO;AAAA,IACnE,QAAQ;AAAA,IAAe;AAAA,EACzB;AAEA,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,QAAM,KAAK,KAAK,QAAQ,EAAE,WAAW,oBAAoB,SAAS,IAAO,CAAC;AAE1E,QAAM,UAA2B;AAAA,IAC/B;AAAA,IAAI;AAAA,IAAS;AAAA,IAAS;AAAA,IAAM;AAAA,IAC5B,WAAW;AAAA,IAAM,WAAW;AAAA,IAAM,eAAe;AAAA,IACjD,WAAW,KAAK,IAAI;AAAA,EACtB;AAGA,UAAQ,gBAAgB,YAAY,YAAY;AAC9C,QAAI,CAAC,QAAQ,UAAW;AACxB,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AAChF,cAAQ,YAAY,IAAI,SAAS,QAAQ;AAAA,IAC3C,QAAQ;AAAE,cAAQ,YAAY;AAAA,IAAO;AAAA,EACvC,GAAG,GAAG;AAEN,WAAS,IAAI,IAAI,OAAO;AAGxB,aAAW,MAAM,cAAc,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC,GAAG,cAAc;AAElE,SAAO,KAAK,EAAE,IAAI,OAAO,GAAG,6BAA6B;AACzD,SAAO;AACT;AAKO,SAAS,SAAS,IAA2B;AAClD,SAAO,SAAS,IAAI,EAAE,GAAG,aAAa;AACxC;AAKA,eAAsB,iBAAiB,IAAY,KAA4B;AAC7E,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,qBAAqB,EAAE,YAAY;AACjE,QAAM,QAAQ,KAAK,KAAK,KAAK,EAAE,WAAW,mBAAmB,CAAC;AAChE;AAKA,eAAsB,iBAAiB,IAA6B;AAClE,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,qBAAqB,EAAE,uBAAuB;AAE5E,QAAM,QAAQ,MAAM,QAAQ,QAAQ,aAAa;AACjD,QAAM,OAAO,KAAK,UAAU,KAAK;AACjC,QAAM,cAAc,EAAE;AACtB,SAAO;AACT;AAKA,eAAsB,UAAU,IAAY,OAM1B;AAChB,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,qBAAqB,EAAE,YAAY;AACjE,MAAI,MAAM,SAAS,WAAW,MAAM,MAAM,UAAa,MAAM,MAAM,QAAW;AAC5E,UAAM,QAAQ,KAAK,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,EAAE,QAAQ,MAAM,UAAU,OAAO,CAAC;AAAA,EACrF,WAAW,MAAM,SAAS,UAAU,MAAM,MAAM;AAC9C,UAAM,QAAQ,KAAK,SAAS,KAAK,MAAM,IAAI;AAAA,EAC7C,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK;AAC5C,UAAM,QAAQ,KAAK,SAAS,MAAM,MAAM,GAAG;AAAA,EAC7C;AACF;AAKA,eAAsB,cAAc,IAA2B;AAC7D,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS;AACd,UAAQ,YAAY;AACpB,MAAI,QAAQ,cAAe,eAAc,QAAQ,aAAa;AAC9D,WAAS,OAAO,EAAE;AAClB,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,SAAO,KAAK,EAAE,GAAG,GAAG,4BAA4B;AAClD;AAKO,SAAS,gBAA0B;AACxC,SAAO,CAAC,GAAG,SAAS,KAAK,CAAC;AAC5B;",
+  "names": []
+}

package/dist/locator-extractor.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    ariaState?: Record<string, unknown> | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+    ariaState?: Record<string, unknown> | null;
+}>): PlaywrightLocator[];
+/**
+ * Detect whether a set of locators was extracted from an auth-wall page.
+ * Returns a warning message if auth-wall detected, null otherwise.
+ * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
+ */
+export declare function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null;
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;