@detiq/crawler 0.1.3

package/dist/js-router-detector.js

@@ -0,0 +1,90 @@
+import { logger as rootLogger } from "./logger.js";
+const logger = rootLogger.child({ module: "js-router-detector" });
+async function detectJsRoutes(page, origin) {
+  try {
+    const raw = await page.evaluate(() => {
+      const found = [];
+      const windowGlobals = [
+        "__routes__",
+        "__ROUTER__",
+        "__vue_router__",
+        "__router__",
+        "router",
+        "_router",
+        "__reactRouter",
+        "__NEXT_DATA__"
+      ];
+      for (const g of windowGlobals) {
+        try {
+          const val = window[g];
+          if (!val || typeof val !== "object") continue;
+          if (g === "__NEXT_DATA__" && Array.isArray(val.buildManifest?.pages)) {
+            for (const p of val.buildManifest.pages) if (typeof p === "string") found.push(p);
+            continue;
+          }
+          const routeArr = (Array.isArray(val.options?.routes) ? val.options.routes : null) ?? (Array.isArray(val.routes) ? val.routes : null) ?? (typeof val.getRoutes === "function" ? val.getRoutes() : null);
+          if (!Array.isArray(routeArr)) continue;
+          const flatRoutes = (arr, prefix = "") => {
+            for (const r of arr) {
+              if (!r || typeof r !== "object") continue;
+              const path = r.path ?? r.href ?? r.url ?? "";
+              const full = path.startsWith("/") ? path : `${prefix}/${path}`.replace(/\/\//g, "/");
+              if (full && full !== "/") found.push(full);
+              if (Array.isArray(r.children)) flatRoutes(r.children, full);
+            }
+          };
+          flatRoutes(routeArr);
+        } catch {
+        }
+      }
+      const routePatterns = [
+        /\bpath\s*:\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /\broute\s*\(\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /<Route[^>]+path\s*=\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /to\s*:\s*['"](\/?[\w\-\/\.~@]+)['"]/g
+      ];
+      const scripts = Array.from(document.querySelectorAll("script:not([src])"));
+      for (const el of scripts.slice(0, 15)) {
+        const src = el.textContent ?? "";
+        if (src.length < 100 || src.length > 8e5) continue;
+        for (const pat of routePatterns) {
+          const hits = [];
+          let m;
+          pat.lastIndex = 0;
+          while ((m = pat.exec(src)) !== null && hits.length < 100) {
+            if (m[1] && m[1].startsWith("/") && m[1].length > 1) hits.push(m[1]);
+          }
+          if (hits.length > 2) for (const h of hits) found.push(h);
+        }
+      }
+      return found;
+    });
+    const DYNAMIC = /[*:?]/;
+    const ASSET = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|map|json|xml)$/i;
+    const seen = /* @__PURE__ */ new Set();
+    const result = [];
+    for (const r of raw) {
+      if (!r || typeof r !== "string" || DYNAMIC.test(r) || ASSET.test(r)) continue;
+      const abs = r.startsWith("http") ? r : `${origin}${r.startsWith("/") ? "" : "/"}${r}`;
+      try {
+        const parsed = new URL(abs);
+        if (parsed.origin !== origin) continue;
+      } catch {
+        continue;
+      }
+      if (!seen.has(abs)) {
+        seen.add(abs);
+        result.push(abs);
+      }
+    }
+    if (result.length > 0) logger.info({ count: result.length }, "[P2] JS router: routes detected");
+    return result;
+  } catch (err) {
+    logger.warn({ err: String(err) }, "[P2] JS router detection failed \u2014 non-fatal");
+    return [];
+  }
+}
+export {
+  detectJsRoutes
+};
+//# sourceMappingURL=js-router-detector.js.map

package/dist/js-router-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/js-router-detector.ts"],
+  "sourcesContent": ["/**\n * P2: JS Router Detection\n *\n * Scans the loaded page for client-side route tables declared by React Router v6,\n * Vue Router, Angular Router, or custom window globals. Supplements the existing\n * history.pushState interception (which only captures runtime navigation) with a\n * static analysis of routes declared in the JS bundle.\n *\n * Called once per page after goto(). Runs entirely in-browser via page.evaluate().\n * Adds discovered routes to the BFS frontier so the crawler visits them.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'js-router-detector' });\n\n/**\n * Extract absolute page URLs from the current page's JS router config.\n * Returns deduplicated absolute URLs (same origin). Dynamic segments (:id, *)\n * are skipped \u2014 the crawler can't resolve them without knowing valid IDs.\n */\nexport async function detectJsRoutes(page: any, origin: string): Promise<string[]> {\n  try {\n    const raw: string[] = await page.evaluate(() => {\n      const found: string[] = [];\n\n      // 1. Check common framework globals \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n      const windowGlobals = [\n        '__routes__', '__ROUTER__', '__vue_router__', '__router__',\n        'router', '_router', '__reactRouter', '__NEXT_DATA__',\n      ];\n      for (const g of windowGlobals) {\n        try {\n          const val = (window as any)[g];\n          if (!val || typeof val !== 'object') continue;\n\n          // Next.js page manifest\n          if (g === '__NEXT_DATA__' && Array.isArray(val.buildManifest?.pages)) {\n            for (const p of val.buildManifest.pages) if (typeof p === 'string') found.push(p);\n            continue;\n          }\n\n          // Router objects: options.routes, routes, getRoutes()\n          const routeArr =\n            (Array.isArray(val.options?.routes) ? val.options.routes : null) ??\n            (Array.isArray(val.routes) ? val.routes : null) ??\n            (typeof val.getRoutes === 'function' ? val.getRoutes() : null);\n\n          if (!Array.isArray(routeArr)) continue;\n\n          const flatRoutes = (arr: any[], prefix = ''): void => {\n            for (const r of arr) {\n              if (!r || typeof r !== 'object') continue;\n              const path: string = r.path ?? r.href ?? r.url ?? '';\n              const full = path.startsWith('/') ? path : `${prefix}/${path}`.replace(/\\/\\//g, '/');\n              if (full && full !== '/') found.push(full);\n              if (Array.isArray(r.children)) flatRoutes(r.children, full);\n            }\n          };\n          flatRoutes(routeArr);\n        } catch { /* skip corrupt globals */ }\n      }\n\n      // 2. Scan inline <script> content for route patterns \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n      // Two-pass: regex first, if >3 found trust them. Below 3 \u2192 skip (too noisy).\n      const routePatterns = [\n        /\\bpath\\s*:\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n        /\\broute\\s*\\(\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n        /<Route[^>]+path\\s*=\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n        /to\\s*:\\s*['\"](\\/?[\\w\\-\\/\\.~@]+)['\"]/g,\n      ];\n      const scripts = Array.from(document.querySelectorAll('script:not([src])'));\n      for (const el of scripts.slice(0, 15)) {\n        const src = el.textContent ?? '';\n        if (src.length < 100 || src.length > 800_000) continue;\n        for (const pat of routePatterns) {\n          const hits: string[] = [];\n          let m: RegExpExecArray | null;\n          pat.lastIndex = 0;\n          while ((m = pat.exec(src)) !== null && hits.length < 100) {\n            if (m[1] && m[1].startsWith('/') && m[1].length > 1) hits.push(m[1]);\n          }\n          // Only trust if we found >2 hits (reduces false positives)\n          if (hits.length > 2) for (const h of hits) found.push(h);\n        }\n      }\n\n      return found;\n    });\n\n    // Post-process: resolve to absolute, dedup, drop dynamic segments\n    const DYNAMIC = /[*:?]/;\n    const ASSET = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|map|json|xml)$/i;\n    const seen = new Set<string>();\n    const result: string[] = [];\n    for (const r of raw) {\n      if (!r || typeof r !== 'string' || DYNAMIC.test(r) || ASSET.test(r)) continue;\n      const abs = r.startsWith('http') ? r : `${origin}${r.startsWith('/') ? '' : '/'}${r}`;\n      try {\n        const parsed = new URL(abs);\n        if (parsed.origin !== origin) continue;\n      } catch { continue; }\n      if (!seen.has(abs)) { seen.add(abs); result.push(abs); }\n    }\n\n    if (result.length > 0) logger.info({ count: result.length }, '[P2] JS router: routes detected');\n    return result;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[P2] JS router detection failed \u2014 non-fatal');\n    return [];\n  }\n}\n"],
+  "mappings": "AAYA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,qBAAqB,CAAC;AAOhE,eAAsB,eAAe,MAAW,QAAmC;AACjF,MAAI;AACF,UAAM,MAAgB,MAAM,KAAK,SAAS,MAAM;AAC9C,YAAM,QAAkB,CAAC;AAGzB,YAAM,gBAAgB;AAAA,QACpB;AAAA,QAAc;AAAA,QAAc;AAAA,QAAkB;AAAA,QAC9C;AAAA,QAAU;AAAA,QAAW;AAAA,QAAiB;AAAA,MACxC;AACA,iBAAW,KAAK,eAAe;AAC7B,YAAI;AACF,gBAAM,MAAO,OAAe,CAAC;AAC7B,cAAI,CAAC,OAAO,OAAO,QAAQ,SAAU;AAGrC,cAAI,MAAM,mBAAmB,MAAM,QAAQ,IAAI,eAAe,KAAK,GAAG;AACpE,uBAAW,KAAK,IAAI,cAAc,MAAO,KAAI,OAAO,MAAM,SAAU,OAAM,KAAK,CAAC;AAChF;AAAA,UACF;AAGA,gBAAM,YACH,MAAM,QAAQ,IAAI,SAAS,MAAM,IAAI,IAAI,QAAQ,SAAS,UAC1D,MAAM,QAAQ,IAAI,MAAM,IAAI,IAAI,SAAS,UACzC,OAAO,IAAI,cAAc,aAAa,IAAI,UAAU,IAAI;AAE3D,cAAI,CAAC,MAAM,QAAQ,QAAQ,EAAG;AAE9B,gBAAM,aAAa,CAAC,KAAY,SAAS,OAAa;AACpD,uBAAW,KAAK,KAAK;AACnB,kBAAI,CAAC,KAAK,OAAO,MAAM,SAAU;AACjC,oBAAM,OAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;AAClD,oBAAM,OAAO,KAAK,WAAW,GAAG,IAAI,OAAO,GAAG,MAAM,IAAI,IAAI,GAAG,QAAQ,SAAS,GAAG;AACnF,kBAAI,QAAQ,SAAS,IAAK,OAAM,KAAK,IAAI;AACzC,kBAAI,MAAM,QAAQ,EAAE,QAAQ,EAAG,YAAW,EAAE,UAAU,IAAI;AAAA,YAC5D;AAAA,UACF;AACA,qBAAW,QAAQ;AAAA,QACrB,QAAQ;AAAA,QAA6B;AAAA,MACvC;AAIA,YAAM,gBAAgB;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,YAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,mBAAmB,CAAC;AACzE,iBAAW,MAAM,QAAQ,MAAM,GAAG,EAAE,GAAG;AACrC,cAAM,MAAM,GAAG,eAAe;AAC9B,YAAI,IAAI,SAAS,OAAO,IAAI,SAAS,IAAS;AAC9C,mBAAW,OAAO,eAAe;AAC/B,gBAAM,OAAiB,CAAC;AACxB,cAAI;AACJ,cAAI,YAAY;AAChB,kBAAQ,IAAI,IAAI,KAAK,GAAG,OAAO,QAAQ,KAAK,SAAS,KAAK;AACxD,gBAAI,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,WAAW,GAAG,KAAK,EAAE,CAAC,EAAE,SAAS,EAAG,MAAK,KAAK,EAAE,CAAC,CAAC;AAAA,UACrE;AAEA,cAAI,KAAK,SAAS,EAAG,YAAW,KAAK,KAAM,OAAM,KAAK,CAAC;AAAA,QACzD;AAAA,MACF;AAEA,aAAO;AAAA,IACT,CAAC;AAGD,UAAM,UAAU;AAChB,UAAM,QAAQ;AACd,UAAM,OAAO,oBAAI,IAAY;AAC7B,UAAM,SAAmB,CAAC;AAC1B,eAAW,KAAK,KAAK;AACnB,UAAI,CAAC,KAAK,OAAO,MAAM,YAAY,QAAQ,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,EAAG;AACrE,YAAM,MAAM,EAAE,WAAW,MAAM,IAAI,IAAI,GAAG,MAAM,GAAG,EAAE,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,CAAC;AACnF,UAAI;AACF,cAAM,SAAS,IAAI,IAAI,GAAG;AAC1B,YAAI,OAAO,WAAW,OAAQ;AAAA,MAChC,QAAQ;AAAE;AAAA,MAAU;AACpB,UAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAAE,aAAK,IAAI,GAAG;AAAG,eAAO,KAAK,GAAG;AAAA,MAAG;AAAA,IACzD;AAEA,QAAI,OAAO,SAAS,EAAG,QAAO,KAAK,EAAE,OAAO,OAAO,OAAO,GAAG,iCAAiC;AAC9F,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,kDAA6C;AAC/E,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/language-detector.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Language detector — identify page language from HTML attributes and content analysis.
+ *
+ * Sources (in priority order):
+ * 1. <html lang="..."> attribute
+ * 2. <meta http-equiv="Content-Language"> header
+ * 3. hreflang alternate links
+ * 4. Content-based n-gram fingerprint (detects top 20 languages)
+ *
+ * Used to: route pages to the correct markdown processor, group screens by locale,
+ * and detect missing hreflang implementations.
+ */
+import type { Page } from 'playwright';
+export interface LanguageDetectionResult {
+    url: string;
+    declaredLang: string;
+    detectedLang: string;
+    confidence: 'high' | 'medium' | 'low';
+    alternates: Record<string, string>;
+    isMultilingual: boolean;
+}
+export declare function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult>;

package/dist/language-detector.js

@@ -0,0 +1,57 @@
+const LANG_SIGNATURES = {
+  en: /\b(the|and|for|are|but|not|you|all|can|had|her|was|one|our|out|day|get|has|him|his|how)\b/gi,
+  es: /\b(que|con|una|por|del|las|los|para|como|pero|más|este|esto|sus|ser|hay|sobre|entre|también)\b/gi,
+  fr: /\b(les|des|une|est|dans|pour|qui|par|sur|avec|que|pas|son|tout|mais|bien|comme|vous)\b/gi,
+  de: /\b(die|der|und|den|das|ein|ist|von|mit|dem|auf|sich|des|aber|auch|bei|aus|nach)\b/gi,
+  pt: /\b(que|não|uma|com|por|dos|para|como|mais|mas|seu|sua|nas|nos|ele|ela|são)\b/gi,
+  it: /\b(che|con|del|per|una|nel|non|sono|gli|dei|alla|delle|questa|questo|come|più)\b/gi,
+  nl: /\b(van|het|een|zijn|maar|ook|dan|door|op|uit|aan|bij|om|al|veel|haar|hun)\b/gi,
+  ja: /[぀-ゟ゠-ヿ]/g,
+  zh: /[一-鿿]/g,
+  ar: /[؀-ۿ]/g,
+  hi: /[ऀ-ॿ]/g,
+  ko: /[가-힯ᄀ-ᇿ]/g
+};
+function detectFromContent(text) {
+  if (!text || text.length < 50) return "en";
+  let best = "en";
+  let bestScore = 0;
+  for (const [lang, re] of Object.entries(LANG_SIGNATURES)) {
+    const matches = (text.match(re) ?? []).length;
+    const score = matches / Math.max(text.length / 100, 1);
+    if (score > bestScore) {
+      bestScore = score;
+      best = lang;
+    }
+  }
+  return best;
+}
+async function detectLanguage(page, url) {
+  const data = await page.evaluate(() => {
+    const htmlLang = document.documentElement.lang?.toLowerCase().split("-")[0] ?? "";
+    const metaLang = document.querySelector('meta[http-equiv="Content-Language"]')?.content?.toLowerCase().split("-")[0] ?? "";
+    const alternates = {};
+    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
+      const lang = el.getAttribute("hreflang")?.toLowerCase();
+      const href = el.getAttribute("href") ?? "";
+      if (lang && lang !== "x-default") alternates[lang] = href;
+    });
+    const bodyText = (document.body?.innerText ?? "").slice(0, 2e3);
+    return { htmlLang, metaLang, alternates, bodyText };
+  }).catch(() => ({ htmlLang: "", metaLang: "", alternates: {}, bodyText: "" }));
+  const declaredLang = data.htmlLang || data.metaLang || "";
+  const detectedLang = detectFromContent(data.bodyText);
+  const confidence = declaredLang ? "high" : data.bodyText.length > 200 ? "medium" : "low";
+  return {
+    url,
+    declaredLang,
+    detectedLang: declaredLang || detectedLang,
+    confidence,
+    alternates: data.alternates,
+    isMultilingual: Object.keys(data.alternates).length > 1
+  };
+}
+export {
+  detectLanguage
+};
+//# sourceMappingURL=language-detector.js.map

package/dist/language-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/language-detector.ts"],
+  "sourcesContent": ["/**\n * Language detector \u2014 identify page language from HTML attributes and content analysis.\n *\n * Sources (in priority order):\n * 1. <html lang=\"...\"> attribute\n * 2. <meta http-equiv=\"Content-Language\"> header\n * 3. hreflang alternate links\n * 4. Content-based n-gram fingerprint (detects top 20 languages)\n *\n * Used to: route pages to the correct markdown processor, group screens by locale,\n * and detect missing hreflang implementations.\n */\n\nimport type { Page } from 'playwright';\n\n// Frequency signatures for n-gram based detection\n// Common trigrams by language (simplified \u2014 not production-grade)\nconst LANG_SIGNATURES: Record<string, RegExp> = {\n  en: /\\b(the|and|for|are|but|not|you|all|can|had|her|was|one|our|out|day|get|has|him|his|how)\\b/gi,\n  es: /\\b(que|con|una|por|del|las|los|para|como|pero|m\u00E1s|este|esto|sus|ser|hay|sobre|entre|tambi\u00E9n)\\b/gi,\n  fr: /\\b(les|des|une|est|dans|pour|qui|par|sur|avec|que|pas|son|tout|mais|bien|comme|vous)\\b/gi,\n  de: /\\b(die|der|und|den|das|ein|ist|von|mit|dem|auf|sich|des|aber|auch|bei|aus|nach)\\b/gi,\n  pt: /\\b(que|n\u00E3o|uma|com|por|dos|para|como|mais|mas|seu|sua|nas|nos|ele|ela|s\u00E3o)\\b/gi,\n  it: /\\b(che|con|del|per|una|nel|non|sono|gli|dei|alla|delle|questa|questo|come|pi\u00F9)\\b/gi,\n  nl: /\\b(van|het|een|zijn|maar|ook|dan|door|op|uit|aan|bij|om|al|veel|haar|hun)\\b/gi,\n  ja: /[\u3040-\u309F\u30A0-\u30FF]/g,\n  zh: /[\u4E00-\u9FFF]/g,\n  ar: /[\u0600-\u06FF]/g,\n  hi: /[\u0900-\u097F]/g,\n  ko: /[\uAC00-\uD7AF\u1100-\u11FF]/g,\n};\n\nfunction detectFromContent(text: string): string {\n  if (!text || text.length < 50) return 'en';\n  let best = 'en';\n  let bestScore = 0;\n  for (const [lang, re] of Object.entries(LANG_SIGNATURES)) {\n    const matches = (text.match(re) ?? []).length;\n    const score = matches / Math.max(text.length / 100, 1);\n    if (score > bestScore) { bestScore = score; best = lang; }\n  }\n  return best;\n}\n\nexport interface LanguageDetectionResult {\n  url: string;\n  declaredLang: string;   // from <html lang> or meta\n  detectedLang: string;   // from content analysis\n  confidence: 'high' | 'medium' | 'low';\n  alternates: Record<string, string>; // lang \u2192 url from hreflang\n  isMultilingual: boolean;\n}\n\nexport async function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult> {\n  const data = await page.evaluate(() => {\n    const htmlLang = document.documentElement.lang?.toLowerCase().split('-')[0] ?? '';\n    const metaLang = (document.querySelector('meta[http-equiv=\"Content-Language\"]') as any)?.content?.toLowerCase().split('-')[0] ?? '';\n    const alternates: Record<string, string> = {};\n    document.querySelectorAll('link[rel=\"alternate\"][hreflang]').forEach((el: any) => {\n      const lang = el.getAttribute('hreflang')?.toLowerCase();\n      const href = el.getAttribute('href') ?? '';\n      if (lang && lang !== 'x-default') alternates[lang] = href;\n    });\n    const bodyText = (document.body?.innerText ?? '').slice(0, 2000);\n    return { htmlLang, metaLang, alternates, bodyText };\n  }).catch(() => ({ htmlLang: '', metaLang: '', alternates: {}, bodyText: '' }));\n\n  const declaredLang = data.htmlLang || data.metaLang || '';\n  const detectedLang = detectFromContent(data.bodyText);\n\n  const confidence: LanguageDetectionResult['confidence'] =\n    declaredLang ? 'high' : data.bodyText.length > 200 ? 'medium' : 'low';\n\n  return {\n    url,\n    declaredLang,\n    detectedLang: declaredLang || detectedLang,\n    confidence,\n    alternates: data.alternates,\n    isMultilingual: Object.keys(data.alternates).length > 1,\n  };\n}\n"],
+  "mappings": "AAiBA,MAAM,kBAA0C;AAAA,EAC9C,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AACN;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,MAAI,CAAC,QAAQ,KAAK,SAAS,GAAI,QAAO;AACtC,MAAI,OAAO;AACX,MAAI,YAAY;AAChB,aAAW,CAAC,MAAM,EAAE,KAAK,OAAO,QAAQ,eAAe,GAAG;AACxD,UAAM,WAAW,KAAK,MAAM,EAAE,KAAK,CAAC,GAAG;AACvC,UAAM,QAAQ,UAAU,KAAK,IAAI,KAAK,SAAS,KAAK,CAAC;AACrD,QAAI,QAAQ,WAAW;AAAE,kBAAY;AAAO,aAAO;AAAA,IAAM;AAAA,EAC3D;AACA,SAAO;AACT;AAWA,eAAsB,eAAe,MAAY,KAA+C;AAC9F,QAAM,OAAO,MAAM,KAAK,SAAS,MAAM;AACrC,UAAM,WAAW,SAAS,gBAAgB,MAAM,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,KAAK;AAC/E,UAAM,WAAY,SAAS,cAAc,qCAAqC,GAAW,SAAS,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,KAAK;AACjI,UAAM,aAAqC,CAAC;AAC5C,aAAS,iBAAiB,iCAAiC,EAAE,QAAQ,CAAC,OAAY;AAChF,YAAM,OAAO,GAAG,aAAa,UAAU,GAAG,YAAY;AACtD,YAAM,OAAO,GAAG,aAAa,MAAM,KAAK;AACxC,UAAI,QAAQ,SAAS,YAAa,YAAW,IAAI,IAAI;AAAA,IACvD,CAAC;AACD,UAAM,YAAY,SAAS,MAAM,aAAa,IAAI,MAAM,GAAG,GAAI;AAC/D,WAAO,EAAE,UAAU,UAAU,YAAY,SAAS;AAAA,EACpD,CAAC,EAAE,MAAM,OAAO,EAAE,UAAU,IAAI,UAAU,IAAI,YAAY,CAAC,GAAG,UAAU,GAAG,EAAE;AAE7E,QAAM,eAAe,KAAK,YAAY,KAAK,YAAY;AACvD,QAAM,eAAe,kBAAkB,KAAK,QAAQ;AAEpD,QAAM,aACJ,eAAe,SAAS,KAAK,SAAS,SAAS,MAAM,WAAW;AAElE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,gBAAgB;AAAA,IAC9B;AAAA,IACA,YAAY,KAAK;AAAA,IACjB,gBAAgB,OAAO,KAAK,KAAK,UAAU,EAAE,SAAS;AAAA,EACxD;AACF;",
+  "names": []
+}

package/dist/link-classifier.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Link classifier — assigns priority and category to discovered URLs.
+ *
+ * Categories:
+ * - 'nav'       : site navigation (homepage, top-level pages) — highest priority
+ * - 'auth'      : login/register/password flows — high priority
+ * - 'app'       : authenticated app screens — high priority
+ * - 'content'   : blog/docs/articles — medium priority
+ * - 'api'       : REST/GraphQL API endpoints — detected, not crawled as screens
+ * - 'resource'  : static assets (JS/CSS/img/fonts) — skip
+ * - 'external'  : different domain — skip unless cross-domain crawl enabled
+ * - 'low-value' : pagination, sort, filter query strings — deprioritize
+ */
+export type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';
+export interface ClassifiedLink {
+    url: string;
+    category: LinkCategory;
+    priority: number;
+    reason: string;
+}
+export declare function classifyLink(url: string, baseUrl: string): ClassifiedLink;
+export declare function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[];
+export declare function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[];

package/dist/link-classifier.js

@@ -0,0 +1,40 @@
+const RESOURCE_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|mp4|mp3|pdf|zip|gz|tar|exe|dmg|apk)(\?|$)/i;
+const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i;
+const AUTH_PATTERN = /\/(login|signin|sign-in|register|signup|sign-up|auth|oauth|sso|password|forgot|reset|logout|signout)\b/i;
+const LOW_VALUE_PATTERN = /[?&](page|p|offset|sort|order|filter|q|search|tag|category|color|size|brand)=/i;
+const PAGINATION_PATTERN = /\/page\/\d+|\/p\/\d+|\bpage=\d+/i;
+function classifyLink(url, baseUrl) {
+  let parsed;
+  let baseHost;
+  try {
+    parsed = new URL(url);
+    baseHost = new URL(baseUrl).hostname;
+  } catch {
+    return { url, category: "low-value", priority: 0, reason: "invalid URL" };
+  }
+  const path = parsed.pathname + parsed.search;
+  const isExternal = parsed.hostname !== baseHost;
+  if (isExternal) return { url, category: "external", priority: 0, reason: "different domain" };
+  if (RESOURCE_EXT.test(path)) return { url, category: "resource", priority: 0, reason: "static asset extension" };
+  if (API_PATTERN.test(path)) return { url, category: "api", priority: 10, reason: "API path pattern" };
+  if (AUTH_PATTERN.test(path)) return { url, category: "auth", priority: 90, reason: "auth flow pattern" };
+  if (PAGINATION_PATTERN.test(path)) return { url, category: "low-value", priority: 15, reason: "pagination" };
+  if (LOW_VALUE_PATTERN.test(path)) return { url, category: "low-value", priority: 20, reason: "filter/sort query param" };
+  const segments = parsed.pathname.split("/").filter(Boolean);
+  if (segments.length === 0) return { url, category: "nav", priority: 100, reason: "homepage" };
+  if (segments.length === 1) return { url, category: "nav", priority: 80, reason: "top-level nav" };
+  if (segments.length <= 3) return { url, category: "app", priority: 60, reason: "app screen" };
+  return { url, category: "content", priority: 40, reason: "deep content page" };
+}
+function classifyLinks(urls, baseUrl) {
+  return urls.map((u) => classifyLink(u, baseUrl)).sort((a, b) => b.priority - a.priority);
+}
+function filterCrawlable(classified) {
+  return classified.filter((l) => l.category !== "resource" && l.category !== "external" && l.category !== "api");
+}
+export {
+  classifyLink,
+  classifyLinks,
+  filterCrawlable
+};
+//# sourceMappingURL=link-classifier.js.map

package/dist/link-classifier.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/link-classifier.ts"],
+  "sourcesContent": ["/**\n * Link classifier \u2014 assigns priority and category to discovered URLs.\n *\n * Categories:\n * - 'nav'       : site navigation (homepage, top-level pages) \u2014 highest priority\n * - 'auth'      : login/register/password flows \u2014 high priority\n * - 'app'       : authenticated app screens \u2014 high priority\n * - 'content'   : blog/docs/articles \u2014 medium priority\n * - 'api'       : REST/GraphQL API endpoints \u2014 detected, not crawled as screens\n * - 'resource'  : static assets (JS/CSS/img/fonts) \u2014 skip\n * - 'external'  : different domain \u2014 skip unless cross-domain crawl enabled\n * - 'low-value' : pagination, sort, filter query strings \u2014 deprioritize\n */\n\nexport type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';\n\nexport interface ClassifiedLink {\n  url: string;\n  category: LinkCategory;\n  priority: number; // 0-100, higher = crawl sooner\n  reason: string;\n}\n\nconst RESOURCE_EXT = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|mp4|mp3|pdf|zip|gz|tar|exe|dmg|apk)(\\?|$)/i;\nconst API_PATTERN = /\\/(api|graphql|gql|rest|v\\d+)\\//i;\nconst AUTH_PATTERN = /\\/(login|signin|sign-in|register|signup|sign-up|auth|oauth|sso|password|forgot|reset|logout|signout)\\b/i;\nconst LOW_VALUE_PATTERN = /[?&](page|p|offset|sort|order|filter|q|search|tag|category|color|size|brand)=/i;\nconst PAGINATION_PATTERN = /\\/page\\/\\d+|\\/p\\/\\d+|\\bpage=\\d+/i;\n\nexport function classifyLink(url: string, baseUrl: string): ClassifiedLink {\n  let parsed: URL;\n  let baseHost: string;\n  try {\n    parsed = new URL(url);\n    baseHost = new URL(baseUrl).hostname;\n  } catch {\n    return { url, category: 'low-value', priority: 0, reason: 'invalid URL' };\n  }\n\n  const path = parsed.pathname + parsed.search;\n  const isExternal = parsed.hostname !== baseHost;\n\n  if (isExternal) return { url, category: 'external', priority: 0, reason: 'different domain' };\n  if (RESOURCE_EXT.test(path)) return { url, category: 'resource', priority: 0, reason: 'static asset extension' };\n  if (API_PATTERN.test(path)) return { url, category: 'api', priority: 10, reason: 'API path pattern' };\n  if (AUTH_PATTERN.test(path)) return { url, category: 'auth', priority: 90, reason: 'auth flow pattern' };\n  if (PAGINATION_PATTERN.test(path)) return { url, category: 'low-value', priority: 15, reason: 'pagination' };\n  if (LOW_VALUE_PATTERN.test(path)) return { url, category: 'low-value', priority: 20, reason: 'filter/sort query param' };\n\n  const segments = parsed.pathname.split('/').filter(Boolean);\n  if (segments.length === 0) return { url, category: 'nav', priority: 100, reason: 'homepage' };\n  if (segments.length === 1) return { url, category: 'nav', priority: 80, reason: 'top-level nav' };\n  if (segments.length <= 3) return { url, category: 'app', priority: 60, reason: 'app screen' };\n\n  return { url, category: 'content', priority: 40, reason: 'deep content page' };\n}\n\nexport function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[] {\n  return urls.map(u => classifyLink(u, baseUrl))\n    .sort((a, b) => b.priority - a.priority);\n}\n\nexport function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[] {\n  return classified.filter(l => l.category !== 'resource' && l.category !== 'external' && l.category !== 'api');\n}\n"],
+  "mappings": "AAuBA,MAAM,eAAe;AACrB,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAEpB,SAAS,aAAa,KAAa,SAAiC;AACzE,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,GAAG;AACpB,eAAW,IAAI,IAAI,OAAO,EAAE;AAAA,EAC9B,QAAQ;AACN,WAAO,EAAE,KAAK,UAAU,aAAa,UAAU,GAAG,QAAQ,cAAc;AAAA,EAC1E;AAEA,QAAM,OAAO,OAAO,WAAW,OAAO;AACtC,QAAM,aAAa,OAAO,aAAa;AAEvC,MAAI,WAAY,QAAO,EAAE,KAAK,UAAU,YAAY,UAAU,GAAG,QAAQ,mBAAmB;AAC5F,MAAI,aAAa,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,YAAY,UAAU,GAAG,QAAQ,yBAAyB;AAC/G,MAAI,YAAY,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,IAAI,QAAQ,mBAAmB;AACpG,MAAI,aAAa,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,QAAQ,UAAU,IAAI,QAAQ,oBAAoB;AACvG,MAAI,mBAAmB,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,aAAa,UAAU,IAAI,QAAQ,aAAa;AAC3G,MAAI,kBAAkB,KAAK,IAAI,EAAG,QAAO,EAAE,KAAK,UAAU,aAAa,UAAU,IAAI,QAAQ,0BAA0B;AAEvH,QAAM,WAAW,OAAO,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AAC1D,MAAI,SAAS,WAAW,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,KAAK,QAAQ,WAAW;AAC5F,MAAI,SAAS,WAAW,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,IAAI,QAAQ,gBAAgB;AAChG,MAAI,SAAS,UAAU,EAAG,QAAO,EAAE,KAAK,UAAU,OAAO,UAAU,IAAI,QAAQ,aAAa;AAE5F,SAAO,EAAE,KAAK,UAAU,WAAW,UAAU,IAAI,QAAQ,oBAAoB;AAC/E;AAEO,SAAS,cAAc,MAAgB,SAAmC;AAC/E,SAAO,KAAK,IAAI,OAAK,aAAa,GAAG,OAAO,CAAC,EAC1C,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAC3C;AAEO,SAAS,gBAAgB,YAAgD;AAC9E,SAAO,WAAW,OAAO,OAAK,EAAE,aAAa,cAAc,EAAE,aAAa,cAAc,EAAE,aAAa,KAAK;AAC9G;",
+  "names": []
+}

package/dist/live-view.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export declare function startLiveView(appUrl: string, storageState?: string): Promise<string>;
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export declare function getFrame(id: string): string | null;
+/**
+ * Navigate the live view browser to a URL.
+ */
+export declare function liveViewNavigate(id: string, url: string): Promise<void>;
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export declare function completeLiveView(id: string): Promise<string>;
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export declare function sendInput(id: string, event: {
+    type: 'click' | 'type' | 'key';
+    x?: number;
+    y?: number;
+    button?: 'left' | 'right' | 'middle';
+    text?: string;
+    key?: string;
+}): Promise<void>;
+/**
+ * Close a live view session without capturing.
+ */
+export declare function closeLiveView(id: string): Promise<void>;
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export declare function listLiveViews(): string[];

package/dist/live-view.js

@@ -0,0 +1,100 @@
+import { chromium } from "playwright";
+import { logger } from "./logger.js";
+const sessions = /* @__PURE__ */ new Map();
+const SESSION_TTL_MS = 15 * 60 * 1e3;
+function generateId() {
+  return Math.random().toString(36).slice(2) + Date.now().toString(36);
+}
+async function startLiveView(appUrl, storageState) {
+  const id = generateId();
+  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === "linux";
+  const browser = await chromium.launch({
+    headless: true,
+    args: isDocker ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu"] : []
+  });
+  const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+  if (storageState) {
+    try {
+      const state = JSON.parse(storageState);
+      if (state.cookies?.length) await context.addCookies(state.cookies);
+    } catch {
+    }
+  }
+  const page = await context.newPage();
+  await page.goto(appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
+  const session = {
+    id,
+    browser,
+    context,
+    page,
+    appUrl,
+    lastFrame: null,
+    streaming: true,
+    frameInterval: null,
+    createdAt: Date.now()
+  };
+  session.frameInterval = setInterval(async () => {
+    if (!session.streaming) return;
+    try {
+      const buf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false });
+      session.lastFrame = buf.toString("base64");
+    } catch {
+      session.streaming = false;
+    }
+  }, 500);
+  sessions.set(id, session);
+  setTimeout(() => closeLiveView(id).catch(() => {
+  }), SESSION_TTL_MS);
+  logger.info({ id, appUrl }, "[live-view] Session started");
+  return id;
+}
+function getFrame(id) {
+  return sessions.get(id)?.lastFrame ?? null;
+}
+async function liveViewNavigate(id, url) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  await session.page.goto(url, { waitUntil: "domcontentloaded" });
+}
+async function completeLiveView(id) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found or expired`);
+  const state = await session.context.storageState();
+  const json = JSON.stringify(state);
+  await closeLiveView(id);
+  return json;
+}
+async function sendInput(id, event) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  if (event.type === "click" && event.x !== void 0 && event.y !== void 0) {
+    await session.page.mouse.click(event.x, event.y, { button: event.button ?? "left" });
+  } else if (event.type === "type" && event.text) {
+    await session.page.keyboard.type(event.text);
+  } else if (event.type === "key" && event.key) {
+    await session.page.keyboard.press(event.key);
+  }
+}
+async function closeLiveView(id) {
+  const session = sessions.get(id);
+  if (!session) return;
+  session.streaming = false;
+  if (session.frameInterval) clearInterval(session.frameInterval);
+  sessions.delete(id);
+  await session.browser.close().catch(() => {
+  });
+  logger.info({ id }, "[live-view] Session closed");
+}
+function listLiveViews() {
+  return [...sessions.keys()];
+}
+export {
+  closeLiveView,
+  completeLiveView,
+  getFrame,
+  listLiveViews,
+  liveViewNavigate,
+  sendInput,
+  startLiveView
+};
+//# sourceMappingURL=live-view.js.map

package/dist/live-view.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/live-view.ts"],
+  "sourcesContent": ["import { chromium, type Browser, type BrowserContext, type Page } from 'playwright';\nimport { logger } from './logger.js';\n\ninterface LiveViewSession {\n  id: string;\n  browser: Browser;\n  context: BrowserContext;\n  page: Page;\n  appUrl: string;\n  lastFrame: string | null; // base64 JPEG\n  streaming: boolean;\n  frameInterval: ReturnType<typeof setInterval> | null;\n  createdAt: number;\n}\n\nconst sessions = new Map<string, LiveViewSession>();\nconst SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min\n\nfunction generateId(): string {\n  return Math.random().toString(36).slice(2) + Date.now().toString(36);\n}\n\n/**\n * Start a live view session: launches a headless browser, navigates to appUrl,\n * begins capturing frames. Returns the session ID.\n */\nexport async function startLiveView(appUrl: string, storageState?: string): Promise<string> {\n  const id = generateId();\n  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === 'linux';\n\n  const browser = await chromium.launch({\n    headless: true,\n    args: isDocker ? ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] : [],\n  });\n\n  const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });\n\n  // Restore session if provided\n  if (storageState) {\n    try {\n      const state = JSON.parse(storageState);\n      if (state.cookies?.length) await context.addCookies(state.cookies);\n    } catch { /* ignore */ }\n  }\n\n  const page = await context.newPage();\n  await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });\n\n  const session: LiveViewSession = {\n    id, browser, context, page, appUrl,\n    lastFrame: null, streaming: true, frameInterval: null,\n    createdAt: Date.now(),\n  };\n\n  // Start capturing frames (every 500ms)\n  session.frameInterval = setInterval(async () => {\n    if (!session.streaming) return;\n    try {\n      const buf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false });\n      session.lastFrame = buf.toString('base64');\n    } catch { session.streaming = false; }\n  }, 500);\n\n  sessions.set(id, session);\n\n  // Auto-cleanup after TTL\n  setTimeout(() => closeLiveView(id).catch(() => {}), SESSION_TTL_MS);\n\n  logger.info({ id, appUrl }, '[live-view] Session started');\n  return id;\n}\n\n/**\n * Get the latest frame for a session (base64 JPEG).\n */\nexport function getFrame(id: string): string | null {\n  return sessions.get(id)?.lastFrame ?? null;\n}\n\n/**\n * Navigate the live view browser to a URL.\n */\nexport async function liveViewNavigate(id: string, url: string): Promise<void> {\n  const session = sessions.get(id);\n  if (!session) throw new Error(`Live view session ${id} not found`);\n  await session.page.goto(url, { waitUntil: 'domcontentloaded' });\n}\n\n/**\n * Capture storageState from the live view session, close the browser, return JSON string.\n */\nexport async function completeLiveView(id: string): Promise<string> {\n  const session = sessions.get(id);\n  if (!session) throw new Error(`Live view session ${id} not found or expired`);\n\n  const state = await session.context.storageState();\n  const json = JSON.stringify(state);\n  await closeLiveView(id);\n  return json;\n}\n\n/**\n * Send a mouse or keyboard input event to the live view browser.\n */\nexport async function sendInput(id: string, event: {\n  type: 'click' | 'type' | 'key';\n  x?: number; y?: number;\n  button?: 'left' | 'right' | 'middle';\n  text?: string;\n  key?: string;\n}): Promise<void> {\n  const session = sessions.get(id);\n  if (!session) throw new Error(`Live view session ${id} not found`);\n  if (event.type === 'click' && event.x !== undefined && event.y !== undefined) {\n    await session.page.mouse.click(event.x, event.y, { button: event.button ?? 'left' });\n  } else if (event.type === 'type' && event.text) {\n    await session.page.keyboard.type(event.text);\n  } else if (event.type === 'key' && event.key) {\n    await session.page.keyboard.press(event.key);\n  }\n}\n\n/**\n * Close a live view session without capturing.\n */\nexport async function closeLiveView(id: string): Promise<void> {\n  const session = sessions.get(id);\n  if (!session) return;\n  session.streaming = false;\n  if (session.frameInterval) clearInterval(session.frameInterval);\n  sessions.delete(id);\n  await session.browser.close().catch(() => {});\n  logger.info({ id }, '[live-view] Session closed');\n}\n\n/**\n * List all active session IDs (for health checks / cleanup).\n */\nexport function listLiveViews(): string[] {\n  return [...sessions.keys()];\n}\n"],
+  "mappings": "AAAA,SAAS,gBAA8D;AACvE,SAAS,cAAc;AAcvB,MAAM,WAAW,oBAAI,IAA6B;AAClD,MAAM,iBAAiB,KAAK,KAAK;AAEjC,SAAS,aAAqB;AAC5B,SAAO,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,SAAS,EAAE;AACrE;AAMA,eAAsB,cAAc,QAAgB,cAAwC;AAC1F,QAAM,KAAK,WAAW;AACtB,QAAM,WAAW,CAAC,CAAC,QAAQ,IAAI,wBAAwB,QAAQ,aAAa;AAE5E,QAAM,UAAU,MAAM,SAAS,OAAO;AAAA,IACpC,UAAU;AAAA,IACV,MAAM,WAAW,CAAC,gBAAgB,4BAA4B,2BAA2B,eAAe,IAAI,CAAC;AAAA,EAC/G,CAAC;AAED,QAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,UAAU,EAAE,OAAO,MAAM,QAAQ,IAAI,EAAE,CAAC;AAGnF,MAAI,cAAc;AAChB,QAAI;AACF,YAAM,QAAQ,KAAK,MAAM,YAAY;AACrC,UAAI,MAAM,SAAS,OAAQ,OAAM,QAAQ,WAAW,MAAM,OAAO;AAAA,IACnE,QAAQ;AAAA,IAAe;AAAA,EACzB;AAEA,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,QAAM,KAAK,KAAK,QAAQ,EAAE,WAAW,oBAAoB,SAAS,IAAO,CAAC;AAE1E,QAAM,UAA2B;AAAA,IAC/B;AAAA,IAAI;AAAA,IAAS;AAAA,IAAS;AAAA,IAAM;AAAA,IAC5B,WAAW;AAAA,IAAM,WAAW;AAAA,IAAM,eAAe;AAAA,IACjD,WAAW,KAAK,IAAI;AAAA,EACtB;AAGA,UAAQ,gBAAgB,YAAY,YAAY;AAC9C,QAAI,CAAC,QAAQ,UAAW;AACxB,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AAChF,cAAQ,YAAY,IAAI,SAAS,QAAQ;AAAA,IAC3C,QAAQ;AAAE,cAAQ,YAAY;AAAA,IAAO;AAAA,EACvC,GAAG,GAAG;AAEN,WAAS,IAAI,IAAI,OAAO;AAGxB,aAAW,MAAM,cAAc,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC,GAAG,cAAc;AAElE,SAAO,KAAK,EAAE,IAAI,OAAO,GAAG,6BAA6B;AACzD,SAAO;AACT;AAKO,SAAS,SAAS,IAA2B;AAClD,SAAO,SAAS,IAAI,EAAE,GAAG,aAAa;AACxC;AAKA,eAAsB,iBAAiB,IAAY,KAA4B;AAC7E,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,qBAAqB,EAAE,YAAY;AACjE,QAAM,QAAQ,KAAK,KAAK,KAAK,EAAE,WAAW,mBAAmB,CAAC;AAChE;AAKA,eAAsB,iBAAiB,IAA6B;AAClE,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,qBAAqB,EAAE,uBAAuB;AAE5E,QAAM,QAAQ,MAAM,QAAQ,QAAQ,aAAa;AACjD,QAAM,OAAO,KAAK,UAAU,KAAK;AACjC,QAAM,cAAc,EAAE;AACtB,SAAO;AACT;AAKA,eAAsB,UAAU,IAAY,OAM1B;AAChB,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,qBAAqB,EAAE,YAAY;AACjE,MAAI,MAAM,SAAS,WAAW,MAAM,MAAM,UAAa,MAAM,MAAM,QAAW;AAC5E,UAAM,QAAQ,KAAK,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,EAAE,QAAQ,MAAM,UAAU,OAAO,CAAC;AAAA,EACrF,WAAW,MAAM,SAAS,UAAU,MAAM,MAAM;AAC9C,UAAM,QAAQ,KAAK,SAAS,KAAK,MAAM,IAAI;AAAA,EAC7C,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK;AAC5C,UAAM,QAAQ,KAAK,SAAS,MAAM,MAAM,GAAG;AAAA,EAC7C;AACF;AAKA,eAAsB,cAAc,IAA2B;AAC7D,QAAM,UAAU,SAAS,IAAI,EAAE;AAC/B,MAAI,CAAC,QAAS;AACd,UAAQ,YAAY;AACpB,MAAI,QAAQ,cAAe,eAAc,QAAQ,aAAa;AAC9D,WAAS,OAAO,EAAE;AAClB,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,SAAO,KAAK,EAAE,GAAG,GAAG,4BAA4B;AAClD;AAKO,SAAS,gBAA0B;AACxC,SAAO,CAAC,GAAG,SAAS,KAAK,CAAC;AAC5B;",
+  "names": []
+}

package/dist/logger.d.ts

@@ -0,0 +1,2 @@
+import pino from 'pino';
+export declare const logger: pino.Logger<never, boolean>;

package/dist/logger.js

@@ -0,0 +1,10 @@
+import pino from "pino";
+const logger = pino({
+  level: process.env.LOG_LEVEL ?? "info",
+  ...process.env.NODE_ENV === "development" ? { transport: { target: "pino-pretty", options: { colorize: true } } } : {},
+  base: { service: "crawler" }
+});
+export {
+  logger
+};
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/logger.ts"],
+  "sourcesContent": ["import pino from 'pino';\n\nexport const logger = pino({\n  level: process.env.LOG_LEVEL ?? 'info',\n  ...(process.env.NODE_ENV === 'development'\n    ? { transport: { target: 'pino-pretty', options: { colorize: true } } }\n    : {}),\n  base: { service: 'crawler' },\n});\n"],
+  "mappings": "AAAA,OAAO,UAAU;AAEV,MAAM,SAAS,KAAK;AAAA,EACzB,OAAO,QAAQ,IAAI,aAAa;AAAA,EAChC,GAAI,QAAQ,IAAI,aAAa,gBACzB,EAAE,WAAW,EAAE,QAAQ,eAAe,SAAS,EAAE,UAAU,KAAK,EAAE,EAAE,IACpE,CAAC;AAAA,EACL,MAAM,EAAE,SAAS,UAAU;AAC7B,CAAC;",
+  "names": []
+}

package/dist/markdown-extract.d.ts

@@ -0,0 +1,8 @@
+export interface ExtractedMarkdown {
+    markdown: string;
+    method: 'readability' | 'full-page';
+    title: string | null;
+    wordCount: number;
+}
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export declare function extractMarkdown(html: string, url: string): ExtractedMarkdown | null;

package/dist/markdown-extract.js

@@ -0,0 +1,54 @@
+import { parseHTML } from "linkedom";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+turndown.remove(["script", "style", "noscript", "svg", "iframe"]);
+function stripNoiseElements(document) {
+  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+  noisy.forEach((el) => el.remove());
+}
+function extractMarkdown(html, url) {
+  try {
+    const { document } = parseHTML(html);
+    stripNoiseElements(document);
+    let readabilityResult = null;
+    try {
+      const { document: readerDoc } = parseHTML(html);
+      stripNoiseElements(readerDoc);
+      readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
+    } catch {
+      readabilityResult = null;
+    }
+    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+      const markdown2 = turndown.turndown(readabilityResult.content);
+      return {
+        markdown: `# ${readabilityResult.title || document.title || url}
+
+${markdown2}`,
+        method: "readability",
+        title: readabilityResult.title || document.title || null,
+        wordCount: readabilityResult.textContent.trim().split(/\s+/).length
+      };
+    }
+    const body = document.body;
+    if (!body) return null;
+    const markdown = turndown.turndown(body.innerHTML || "");
+    const text = markdown.trim();
+    if (!text) return null;
+    const title = document.title || null;
+    return {
+      markdown: title ? `# ${title}
+
+${markdown}` : markdown,
+      method: "full-page",
+      title,
+      wordCount: text.split(/\s+/).length
+    };
+  } catch {
+    return null;
+  }
+}
+export {
+  extractMarkdown
+};
+//# sourceMappingURL=markdown-extract.js.map

package/dist/markdown-extract.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/markdown-extract.ts"],
+  "sourcesContent": ["/**\n * HTML \u2192 Markdown extraction for crawled pages \u2014 matches crawl4AI's markdown\n * output capability. Two-tier, mirroring crawl4AI's \"markdown\" vs \"fit markdown\":\n *\n *  - \"fit\" markdown: Readability's best guess at the primary content (title +\n *    article body), stripped of nav/ads/chrome. Ideal for content/marketing\n *    pages and RAG chunking. null when Readability finds no article (most\n *    app UI screens \u2014 dashboards, forms \u2014 legitimately have none).\n *  - \"full\" markdown: the whole visible page converted straight to Markdown,\n *    always present. The fallback for app screens where there's no \"article\".\n *\n * Runs entirely on an already-captured `page.content()` string \u2014 no extra\n * network fetch, no extra page navigation.\n */\nimport { parseHTML } from 'linkedom';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\nturndown.remove(['script', 'style', 'noscript', 'svg', 'iframe'] as unknown as (keyof HTMLElementTagNameMap)[]);\n\nexport interface ExtractedMarkdown {\n  markdown: string;\n  method: 'readability' | 'full-page';\n  title: string | null;\n  wordCount: number;\n}\n\nfunction stripNoiseElements(document: Document): void {\n  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden=\"true\"]');\n  noisy.forEach((el: any) => el.remove());\n}\n\n/** Best-effort HTML\u2192Markdown for one crawled page. Never throws. */\nexport function extractMarkdown(html: string, url: string): ExtractedMarkdown | null {\n  try {\n    const { document } = parseHTML(html);\n    stripNoiseElements(document);\n\n    let readabilityResult: { title: string; content: string; textContent: string } | null = null;\n    try {\n      // Readability mutates the DOM it's given \u2014 clone via re-parsing so the\n      // full-page fallback below still has an intact document.\n      const { document: readerDoc } = parseHTML(html);\n      stripNoiseElements(readerDoc);\n      readabilityResult = new Readability(readerDoc as any, { charThreshold: 50 }).parse() as any;\n    } catch {\n      readabilityResult = null;\n    }\n\n    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {\n      const markdown = turndown.turndown(readabilityResult.content);\n      return {\n        markdown: `# ${readabilityResult.title || document.title || url}\\n\\n${markdown}`,\n        method: 'readability',\n        title: readabilityResult.title || document.title || null,\n        wordCount: readabilityResult.textContent.trim().split(/\\s+/).length,\n      };\n    }\n\n    const body = document.body;\n    if (!body) return null;\n    const markdown = turndown.turndown(body.innerHTML || '');\n    const text = markdown.trim();\n    if (!text) return null;\n    const title = document.title || null;\n    return {\n      markdown: title ? `# ${title}\\n\\n${markdown}` : markdown,\n      method: 'full-page',\n      title,\n      wordCount: text.split(/\\s+/).length,\n    };\n  } catch {\n    return null;\n  }\n}\n"],
+  "mappings": "AAcA,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAE5B,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AACtF,SAAS,OAAO,CAAC,UAAU,SAAS,YAAY,OAAO,QAAQ,CAA+C;AAS9G,SAAS,mBAAmB,UAA0B;AACpD,QAAM,QAAQ,SAAS,iBAAiB,4DAA4D;AACpG,QAAM,QAAQ,CAAC,OAAY,GAAG,OAAO,CAAC;AACxC;AAGO,SAAS,gBAAgB,MAAc,KAAuC;AACnF,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,uBAAmB,QAAQ;AAE3B,QAAI,oBAAoF;AACxF,QAAI;AAGF,YAAM,EAAE,UAAU,UAAU,IAAI,UAAU,IAAI;AAC9C,yBAAmB,SAAS;AAC5B,0BAAoB,IAAI,YAAY,WAAkB,EAAE,eAAe,GAAG,CAAC,EAAE,MAAM;AAAA,IACrF,QAAQ;AACN,0BAAoB;AAAA,IACtB;AAEA,QAAI,mBAAmB,WAAW,kBAAkB,YAAY,KAAK,EAAE,UAAU,IAAI;AACnF,YAAMA,YAAW,SAAS,SAAS,kBAAkB,OAAO;AAC5D,aAAO;AAAA,QACL,UAAU,KAAK,kBAAkB,SAAS,SAAS,SAAS,GAAG;AAAA;AAAA,EAAOA,SAAQ;AAAA,QAC9E,QAAQ;AAAA,QACR,OAAO,kBAAkB,SAAS,SAAS,SAAS;AAAA,QACpD,WAAW,kBAAkB,YAAY,KAAK,EAAE,MAAM,KAAK,EAAE;AAAA,MAC/D;AAAA,IACF;AAEA,UAAM,OAAO,SAAS;AACtB,QAAI,CAAC,KAAM,QAAO;AAClB,UAAM,WAAW,SAAS,SAAS,KAAK,aAAa,EAAE;AACvD,UAAM,OAAO,SAAS,KAAK;AAC3B,QAAI,CAAC,KAAM,QAAO;AAClB,UAAM,QAAQ,SAAS,SAAS;AAChC,WAAO;AAAA,MACL,UAAU,QAAQ,KAAK,KAAK;AAAA;AAAA,EAAO,QAAQ,KAAK;AAAA,MAChD,QAAQ;AAAA,MACR;AAAA,MACA,WAAW,KAAK,MAAM,KAAK,EAAE;AAAA,IAC/B;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": ["markdown"]
+}

package/dist/native-crawler.d.ts

@@ -0,0 +1,69 @@
+/**
+ * ZeTa Native App Crawler — Appium-based crawler for Android APK and iOS IPA.
+ *
+ * Architecture:
+ *   Appium Server (running at APPIUM_SERVER_URL) → WebDriverIO session
+ *   → UiAutomator2 (Android) / XCUITest (iOS)
+ *   → accessibility tree XML → ZeTa Element records
+ *
+ * TODO: requires Appium server running at APPIUM_SERVER_URL env var
+ * TODO: requires `webdriverio` package — add "webdriverio": "^9.x" to package.json
+ *
+ * Device farm support:
+ *   - local_emulator: Appium server + local emulator/simulator
+ *   - aws_device_farm: presigned S3 URL uploaded to Device Farm, session via WDIO
+ *   - browserstack: BrowserStack Automate Appium endpoint
+ */
+export interface NativeCrawlerOptions {
+    tenantId: string;
+    projectId: string;
+    jobId: string;
+    platform: 'android' | 'ios' | 'windows' | 'macos';
+    appS3Path: string;
+    packageName?: string;
+    deviceModel?: string;
+    osVersion?: string;
+    appActivity?: string;
+    appWaitActivity?: string;
+    deviceFarm: 'local_emulator' | 'aws_device_farm' | 'browserstack';
+    appiumServerUrl?: string;
+    browserstackUsername?: string;
+    browserstackAccessKey?: string;
+    awsAccessKeyId?: string;
+    awsSecretAccessKey?: string;
+    awsRegion?: string;
+    awsDeviceFarmProjectArn?: string;
+    credentials?: {
+        username?: string;
+        password?: string;
+        usernameFieldLabel?: string;
+        passwordFieldLabel?: string;
+        loginButtonLabel?: string;
+    };
+    visionLlmConfig?: {
+        provider: string;
+        model: string;
+        apiKey?: string;
+        baseUrl?: string;
+    } | null;
+    gestureConfig?: {
+        enableSwipe?: boolean;
+        enableLongPress?: boolean;
+    };
+    mitmProxyPort?: number;
+}
+/**
+ * Run a native app crawl session via Appium.
+ *
+ * Flow:
+ *   1. Download APK/IPA from S3 to local temp dir
+ *   2. Build Appium WebDriverIO capabilities for the target platform
+ *   3. Create WebDriverIO remote session (Appium server at APPIUM_SERVER_URL)
+ *   4. For each discovered screen:
+ *      a. Get page source XML → parse elements
+ *      b. Take screenshot → save to screenshotDir → persist via AppBrain
+ *      c. Save screen + elements to DB
+ *      d. Tap nav elements to discover next screens
+ *   5. Update PlatformJob as DONE (or FAILED on error)
+ */
+export declare function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>;