@detiq/crawler 0.1.3

package/dist/stealth/human-behavior.js

@@ -0,0 +1,168 @@
+function gaussian(mean, sigma) {
+  const u1 = Math.random();
+  const u2 = Math.random();
+  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+  return mean + z * sigma;
+}
+async function humanDelay(min = 80, max = 450) {
+  const mean = (min + max) / 2;
+  const sigma = (max - min) / 6;
+  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
+  await new Promise((r) => setTimeout(r, Math.round(delay)));
+}
+async function simulateMouseMovement(page, targetX, targetY) {
+  try {
+    const mouse = page.mouse;
+    const steps = 8 + Math.floor(Math.random() * 8);
+    const startX = targetX - 100 + Math.random() * 200;
+    const startY = targetY - 100 + Math.random() * 200;
+    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
+    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
+    for (let i = 0; i <= steps; i++) {
+      const t = i / steps;
+      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
+      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
+      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
+      if (i < steps) await humanDelay(8, 25);
+    }
+    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
+    await humanDelay(30, 80);
+    await mouse.move(targetX, targetY);
+  } catch {
+  }
+}
+async function simulateMouseHover(page, selector) {
+  try {
+    const el = await page.$(selector);
+    if (!el) return;
+    const box = await el.boundingBox();
+    if (!box) return;
+    const cx = box.x + box.width / 2;
+    const cy = box.y + box.height / 2;
+    await simulateMouseMovement(page, cx, cy);
+    await humanDelay(100, 300);
+  } catch {
+  }
+}
+async function simulateClick(page, selector) {
+  try {
+    await simulateMouseHover(page, selector);
+    await humanDelay(50, 150);
+    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
+  } catch {
+  }
+}
+async function injectFingerprintOverrides(page) {
+  await page.addInitScript(() => {
+    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
+    HTMLCanvasElement.prototype.toDataURL = function(...args) {
+      const ctx = this.getContext("2d");
+      if (ctx) {
+        const imageData = ctx.getImageData(0, 0, this.width, this.height);
+        for (let i = 0; i < imageData.data.length; i += 4) {
+          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? Math.random() > 0.5 ? 1 : -1 : 0)));
+        }
+        ctx.putImageData(imageData, 0, 0);
+      }
+      return origToDataURL.apply(this, args);
+    };
+    const origGetParam = WebGLRenderingContext.prototype.getParameter;
+    WebGLRenderingContext.prototype.getParameter = function(parameter) {
+      const UNMASKED_VENDOR_WEBGL = 37445;
+      const UNMASKED_RENDERER_WEBGL = 37446;
+      if (parameter === UNMASKED_VENDOR_WEBGL) return "Intel Inc.";
+      if (parameter === UNMASKED_RENDERER_WEBGL) return "Intel Iris OpenGL Engine";
+      return origGetParam.call(this, parameter);
+    };
+    const cpuCounts = [4, 6, 8, 8];
+    Object.defineProperty(navigator, "hardwareConcurrency", {
+      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
+      configurable: true
+    });
+    Object.defineProperty(navigator, "deviceMemory", {
+      get: () => [4, 8][Math.floor(Math.random() * 2)],
+      configurable: true
+    });
+  }).catch(() => {
+  });
+}
+async function infiniteScrollToBottom(page, opts = {}) {
+  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
+  let scrollCount = 0;
+  let sameHeightCount = 0;
+  let lastHeight = 0;
+  try {
+    while (scrollCount < maxScrolls) {
+      const currentHeight = await page.evaluate(() => document.body.scrollHeight).catch(() => 0);
+      if (currentHeight === lastHeight) {
+        sameHeightCount++;
+        if (sameHeightCount >= 3) break;
+      } else {
+        sameHeightCount = 0;
+        lastHeight = currentHeight;
+      }
+      const scrollBy = 500 + Math.floor(Math.random() * 300);
+      await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => {
+      });
+      await humanDelay(scrollDelay - 100, scrollDelay + 200);
+      await page.waitForTimeout(stabilizeMs).catch(() => {
+      });
+      scrollCount++;
+    }
+  } catch {
+  }
+  return { scrollCount, heightReached: lastHeight };
+}
+async function extractIframeContent(page) {
+  const results = [];
+  try {
+    const iframeSrcs = await page.evaluate(
+      () => Array.from(document.querySelectorAll("iframe")).map((f) => f.getAttribute("src") ?? f.getAttribute("data-src") ?? "")
+    ).catch(() => []);
+    for (const frame of page.frames()) {
+      const src = frame.url();
+      if (!src || src === "about:blank" || src === page.url()) continue;
+      try {
+        const content = await frame.content().catch(() => "");
+        if (!content || content.length < 50) continue;
+        const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+        results.push({ src, content: content.slice(0, 1e4), markdown: text.slice(0, 5e3) });
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return results;
+}
+async function pierceShadowDom(page) {
+  try {
+    return await page.evaluate(() => {
+      const parts = [];
+      function walkShadow(root) {
+        const shadow = root.shadowRoot;
+        if (shadow) {
+          parts.push(shadow.innerHTML ?? "");
+          for (const child of [...shadow.querySelectorAll("*")]) walkShadow(child);
+        }
+        for (const child of Array.from(root.querySelectorAll(":not(script):not(style)"))) {
+          if (child.shadowRoot) walkShadow(child);
+        }
+      }
+      walkShadow(document.body);
+      return parts.join("\n").replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 1e4);
+    }).catch(() => "");
+  } catch {
+    return "";
+  }
+}
+export {
+  extractIframeContent,
+  humanDelay,
+  infiniteScrollToBottom,
+  injectFingerprintOverrides,
+  pierceShadowDom,
+  simulateClick,
+  simulateMouseHover,
+  simulateMouseMovement
+};
+//# sourceMappingURL=human-behavior.js.map

package/dist/stealth/human-behavior.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/stealth/human-behavior.ts"],
+  "sourcesContent": ["import type { Page } from 'playwright';\n\n// Gaussian random using Box-Muller transform\nfunction gaussian(mean: number, sigma: number): number {\n  const u1 = Math.random();\n  const u2 = Math.random();\n  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);\n  return mean + z * sigma;\n}\n\nexport async function humanDelay(min = 80, max = 450): Promise<void> {\n  const mean = (min + max) / 2;\n  const sigma = (max - min) / 6;\n  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));\n  await new Promise(r => setTimeout(r, Math.round(delay)));\n}\n\nexport async function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void> {\n  try {\n    const mouse = page.mouse;\n    const steps = 8 + Math.floor(Math.random() * 8);\n    // Generate Bezier control points with slight perpendicular offset\n    const startX = targetX - 100 + Math.random() * 200;\n    const startY = targetY - 100 + Math.random() * 200;\n    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;\n    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;\n\n    for (let i = 0; i <= steps; i++) {\n      const t = i / steps;\n      // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2\n      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;\n      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;\n      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);\n      if (i < steps) await humanDelay(8, 25);\n    }\n    // Small overshoot correction\n    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);\n    await humanDelay(30, 80);\n    await mouse.move(targetX, targetY);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateMouseHover(page: Page, selector: string): Promise<void> {\n  try {\n    const el = await page.$(selector);\n    if (!el) return;\n    const box = await el.boundingBox();\n    if (!box) return;\n    const cx = box.x + box.width / 2;\n    const cy = box.y + box.height / 2;\n    await simulateMouseMovement(page, cx, cy);\n    await humanDelay(100, 300);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateClick(page: Page, selector: string): Promise<void> {\n  try {\n    await simulateMouseHover(page, selector);\n    await humanDelay(50, 150);\n    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });\n  } catch { /* non-fatal */ }\n}\n\nexport async function injectFingerprintOverrides(page: Page): Promise<void> {\n  await page.addInitScript(() => {\n    // Canvas noise \u2014 subtle per-pixel perturbation\n    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;\n    HTMLCanvasElement.prototype.toDataURL = function(this: HTMLCanvasElement, ...args: any[]) {\n      const ctx = this.getContext('2d');\n      if (ctx) {\n        const imageData = ctx.getImageData(0, 0, this.width, this.height);\n        for (let i = 0; i < imageData.data.length; i += 4) {\n          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));\n        }\n        ctx.putImageData(imageData, 0, 0);\n      }\n      return origToDataURL.apply(this, args as [type?: string, quality?: number]);\n    };\n\n    // WebGL parameter spoofing\n    const origGetParam = WebGLRenderingContext.prototype.getParameter;\n    WebGLRenderingContext.prototype.getParameter = function(this: WebGLRenderingContext, parameter: number) {\n      const UNMASKED_VENDOR_WEBGL = 0x9245;\n      const UNMASKED_RENDERER_WEBGL = 0x9246;\n      if (parameter === UNMASKED_VENDOR_WEBGL) return 'Intel Inc.';\n      if (parameter === UNMASKED_RENDERER_WEBGL) return 'Intel Iris OpenGL Engine';\n      return origGetParam.call(this, parameter);\n    };\n\n    // Hardware concurrency \u2014 pick from common values\n    const cpuCounts = [4, 6, 8, 8];\n    Object.defineProperty(navigator, 'hardwareConcurrency', {\n      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],\n      configurable: true,\n    });\n\n    // Device memory\n    Object.defineProperty(navigator, 'deviceMemory', {\n      get: () => [4, 8][Math.floor(Math.random() * 2)],\n      configurable: true,\n    });\n  }).catch(() => {});\n}\n\nexport async function infiniteScrollToBottom(\n  page: Page,\n  opts: { maxScrolls?: number; scrollDelay?: number; stabilizeMs?: number } = {}\n): Promise<{ scrollCount: number; heightReached: number }> {\n  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;\n  let scrollCount = 0;\n  let sameHeightCount = 0;\n  let lastHeight = 0;\n\n  try {\n    while (scrollCount < maxScrolls) {\n      const currentHeight: number = await page.evaluate(() => document.body.scrollHeight).catch(() => 0);\n      if (currentHeight === lastHeight) {\n        sameHeightCount++;\n        if (sameHeightCount >= 3) break; // no new content\n      } else {\n        sameHeightCount = 0;\n        lastHeight = currentHeight;\n      }\n\n      const scrollBy = 500 + Math.floor(Math.random() * 300);\n      await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});\n      await humanDelay(scrollDelay - 100, scrollDelay + 200);\n\n      // Wait for potential lazy-loaded content\n      await page.waitForTimeout(stabilizeMs).catch(() => {});\n      scrollCount++;\n    }\n  } catch { /* non-fatal */ }\n\n  return { scrollCount, heightReached: lastHeight };\n}\n\nexport async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {\n  const results: Array<{ src: string; content: string; markdown: string }> = [];\n  try {\n    const iframeSrcs: string[] = await page.evaluate(() =>\n      Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')\n    ).catch(() => []);\n\n    for (const frame of page.frames()) {\n      const src = frame.url();\n      if (!src || src === 'about:blank' || src === page.url()) continue;\n      try {\n        const content = await frame.content().catch(() => '');\n        if (!content || content.length < 50) continue;\n        // Simple text extraction from iframe HTML\n        const text = content.replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n          .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n          .replace(/<[^>]+>/g, ' ')\n          .replace(/\\s+/g, ' ')\n          .trim();\n        results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });\n      } catch { /* skip this frame */ }\n    }\n  } catch { /* non-fatal */ }\n  return results;\n}\n\nexport async function pierceShadowDom(page: Page): Promise<string> {\n  try {\n    return await page.evaluate(() => {\n      const parts: string[] = [];\n      function walkShadow(root: Element | ShadowRoot): void {\n        const shadow = (root as any).shadowRoot;\n        if (shadow) {\n          parts.push(shadow.innerHTML ?? '');\n          for (const child of [...shadow.querySelectorAll('*')]) walkShadow(child);\n        }\n        for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {\n          if ((child as any).shadowRoot) walkShadow(child);\n        }\n      }\n      walkShadow(document.body);\n      return parts\n        .join('\\n')\n        .replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n        .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n        .replace(/<[^>]+>/g, ' ')\n        .replace(/\\s+/g, ' ')\n        .trim()\n        .slice(0, 10_000);\n    }).catch(() => '');\n  } catch {\n    return '';\n  }\n}\n"],
+  "mappings": "AAGA,SAAS,SAAS,MAAc,OAAuB;AACrD,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,IAAI,KAAK,KAAK,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;AAClE,SAAO,OAAO,IAAI;AACpB;AAEA,eAAsB,WAAW,MAAM,IAAI,MAAM,KAAoB;AACnE,QAAM,QAAQ,MAAM,OAAO;AAC3B,QAAM,SAAS,MAAM,OAAO;AAC5B,QAAM,QAAQ,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,SAAS,MAAM,KAAK,CAAC,CAAC;AAChE,QAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,KAAK,MAAM,KAAK,CAAC,CAAC;AACzD;AAEA,eAAsB,sBAAsB,MAAY,SAAiB,SAAgC;AACvG,MAAI;AACF,UAAM,QAAQ,KAAK;AACnB,UAAM,QAAQ,IAAI,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC;AAE9C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAC7D,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAE7D,aAAS,IAAI,GAAG,KAAK,OAAO,KAAK;AAC/B,YAAM,IAAI,IAAI;AAEd,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,MAAM,KAAK,KAAK,KAAK,OAAO,IAAI,OAAO,GAAG,KAAK,KAAK,OAAO,IAAI,OAAO,CAAC;AAC7E,UAAI,IAAI,MAAO,OAAM,WAAW,GAAG,EAAE;AAAA,IACvC;AAEA,UAAM,MAAM,KAAK,WAAW,KAAK,OAAO,IAAI,OAAO,GAAG,WAAW,KAAK,OAAO,IAAI,OAAO,CAAC;AACzF,UAAM,WAAW,IAAI,EAAE;AACvB,UAAM,MAAM,KAAK,SAAS,OAAO;AAAA,EACnC,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,mBAAmB,MAAY,UAAiC;AACpF,MAAI;AACF,UAAM,KAAK,MAAM,KAAK,EAAE,QAAQ;AAChC,QAAI,CAAC,GAAI;AACT,UAAM,MAAM,MAAM,GAAG,YAAY;AACjC,QAAI,CAAC,IAAK;AACV,UAAM,KAAK,IAAI,IAAI,IAAI,QAAQ;AAC/B,UAAM,KAAK,IAAI,IAAI,IAAI,SAAS;AAChC,UAAM,sBAAsB,MAAM,IAAI,EAAE;AACxC,UAAM,WAAW,KAAK,GAAG;AAAA,EAC3B,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,cAAc,MAAY,UAAiC;AAC/E,MAAI;AACF,UAAM,mBAAmB,MAAM,QAAQ;AACvC,UAAM,WAAW,IAAI,GAAG;AACxB,UAAM,KAAK,MAAM,UAAU,EAAE,OAAO,KAAK,MAAM,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;AAAA,EACpE,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,2BAA2B,MAA2B;AAC1E,QAAM,KAAK,cAAc,MAAM;AAE7B,UAAM,gBAAgB,kBAAkB,UAAU;AAClD,sBAAkB,UAAU,YAAY,YAAqC,MAAa;AACxF,YAAM,MAAM,KAAK,WAAW,IAAI;AAChC,UAAI,KAAK;AACP,cAAM,YAAY,IAAI,aAAa,GAAG,GAAG,KAAK,OAAO,KAAK,MAAM;AAChE,iBAAS,IAAI,GAAG,IAAI,UAAU,KAAK,QAAQ,KAAK,GAAG;AACjD,oBAAU,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,OAAO,IAAI,OAAQ,KAAK,OAAO,IAAI,MAAM,IAAI,KAAM,EAAE,CAAC;AAAA,QAChI;AACA,YAAI,aAAa,WAAW,GAAG,CAAC;AAAA,MAClC;AACA,aAAO,cAAc,MAAM,MAAM,IAAyC;AAAA,IAC5E;AAGA,UAAM,eAAe,sBAAsB,UAAU;AACrD,0BAAsB,UAAU,eAAe,SAAsC,WAAmB;AACtG,YAAM,wBAAwB;AAC9B,YAAM,0BAA0B;AAChC,UAAI,cAAc,sBAAuB,QAAO;AAChD,UAAI,cAAc,wBAAyB,QAAO;AAClD,aAAO,aAAa,KAAK,MAAM,SAAS;AAAA,IAC1C;AAGA,UAAM,YAAY,CAAC,GAAG,GAAG,GAAG,CAAC;AAC7B,WAAO,eAAe,WAAW,uBAAuB;AAAA,MACtD,KAAK,MAAM,UAAU,KAAK,MAAM,KAAK,OAAO,IAAI,UAAU,MAAM,CAAC;AAAA,MACjE,cAAc;AAAA,IAChB,CAAC;AAGD,WAAO,eAAe,WAAW,gBAAgB;AAAA,MAC/C,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC;AAAA,MAC/C,cAAc;AAAA,IAChB,CAAC;AAAA,EACH,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACnB;AAEA,eAAsB,uBACpB,MACA,OAA4E,CAAC,GACpB;AACzD,QAAM,EAAE,aAAa,IAAI,cAAc,KAAK,cAAc,IAAI,IAAI;AAClE,MAAI,cAAc;AAClB,MAAI,kBAAkB;AACtB,MAAI,aAAa;AAEjB,MAAI;AACF,WAAO,cAAc,YAAY;AAC/B,YAAM,gBAAwB,MAAM,KAAK,SAAS,MAAM,SAAS,KAAK,YAAY,EAAE,MAAM,MAAM,CAAC;AACjG,UAAI,kBAAkB,YAAY;AAChC;AACA,YAAI,mBAAmB,EAAG;AAAA,MAC5B,OAAO;AACL,0BAAkB;AAClB,qBAAa;AAAA,MACf;AAEA,YAAM,WAAW,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACrD,YAAM,KAAK,SAAS,CAAC,OAAe,OAAO,SAAS,GAAG,EAAE,GAAG,QAAQ,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACpF,YAAM,WAAW,cAAc,KAAK,cAAc,GAAG;AAGrD,YAAM,KAAK,eAAe,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACrD;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAAkB;AAE1B,SAAO,EAAE,aAAa,eAAe,WAAW;AAClD;AAEA,eAAsB,qBAAqB,MAAgF;AACzH,QAAM,UAAqE,CAAC;AAC5E,MAAI;AACF,UAAM,aAAuB,MAAM,KAAK;AAAA,MAAS,MAC/C,MAAM,KAAK,SAAS,iBAAiB,QAAQ,CAAC,EAAE,IAAI,OAAK,EAAE,aAAa,KAAK,KAAK,EAAE,aAAa,UAAU,KAAK,EAAE;AAAA,IACpH,EAAE,MAAM,MAAM,CAAC,CAAC;AAEhB,eAAW,SAAS,KAAK,OAAO,GAAG;AACjC,YAAM,MAAM,MAAM,IAAI;AACtB,UAAI,CAAC,OAAO,QAAQ,iBAAiB,QAAQ,KAAK,IAAI,EAAG;AACzD,UAAI;AACF,cAAM,UAAU,MAAM,MAAM,QAAQ,EAAE,MAAM,MAAM,EAAE;AACpD,YAAI,CAAC,WAAW,QAAQ,SAAS,GAAI;AAErC,cAAM,OAAO,QAAQ,QAAQ,qCAAqC,EAAE,EACjE,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACR,gBAAQ,KAAK,EAAE,KAAK,SAAS,QAAQ,MAAM,GAAG,GAAM,GAAG,UAAU,KAAK,MAAM,GAAG,GAAK,EAAE,CAAC;AAAA,MACzF,QAAQ;AAAA,MAAwB;AAAA,IAClC;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,eAAsB,gBAAgB,MAA6B;AACjE,MAAI;AACF,WAAO,MAAM,KAAK,SAAS,MAAM;AAC/B,YAAM,QAAkB,CAAC;AACzB,eAAS,WAAW,MAAkC;AACpD,cAAM,SAAU,KAAa;AAC7B,YAAI,QAAQ;AACV,gBAAM,KAAK,OAAO,aAAa,EAAE;AACjC,qBAAW,SAAS,CAAC,GAAG,OAAO,iBAAiB,GAAG,CAAC,EAAG,YAAW,KAAK;AAAA,QACzE;AACA,mBAAW,SAAS,MAAM,KAAK,KAAK,iBAAiB,yBAAyB,CAAC,GAAG;AAChF,cAAK,MAAc,WAAY,YAAW,KAAK;AAAA,QACjD;AAAA,MACF;AACA,iBAAW,SAAS,IAAI;AACxB,aAAO,MACJ,KAAK,IAAI,EACT,QAAQ,qCAAqC,EAAE,EAC/C,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK,EACL,MAAM,GAAG,GAAM;AAAA,IACpB,CAAC,EAAE,MAAM,MAAM,EAAE;AAAA,EACnB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/telemetry.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Lightweight OTel-compatible tracing for Zeta crawler.
+ *
+ * Emits spans as structured JSON logs compatible with OpenTelemetry format.
+ * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
+ *
+ * Usage:
+ *   const span = startSpan('crawl.url', { url, engine });
+ *   // ... do work ...
+ *   endSpan(span, { screensFound: 3 });
+ */
+export interface Span {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startTimeMs: number;
+    attributes: Record<string, string | number | boolean>;
+}
+export interface FinishedSpan extends Span {
+    endTimeMs: number;
+    durationMs: number;
+    status: 'ok' | 'error';
+    error?: string;
+}
+/** Start a new span. If a trace is already active, the new span is a child. */
+export declare function startSpan(name: string, attributes?: Record<string, string | number | boolean>): Span;
+/** End a span and emit it as a structured log line. */
+export declare function endSpan(span: Span, extraAttributes?: Record<string, string | number | boolean>, error?: Error): FinishedSpan;
+/** Convenience: wrap an async function in a span. */
+export declare function withSpan<T>(name: string, attributes: Record<string, string | number | boolean>, fn: (span: Span) => Promise<T>): Promise<T>;
+export declare function increment(metric: string, value?: number, tags?: Record<string, string>): void;
+/** Flush counters as a structured log — call periodically (e.g. at job end). */
+export declare function flushMetrics(): Record<string, number>;

package/dist/telemetry.js

@@ -0,0 +1,77 @@
+import { randomBytes } from "node:crypto";
+let currentTraceId = null;
+let currentSpanId = null;
+function genId(bytes) {
+  return randomBytes(bytes).toString("hex");
+}
+function startSpan(name, attributes = {}) {
+  const traceId = currentTraceId ?? genId(16);
+  const spanId = genId(8);
+  const parentSpanId = currentSpanId ?? void 0;
+  if (!currentTraceId) currentTraceId = traceId;
+  currentSpanId = spanId;
+  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
+}
+function endSpan(span, extraAttributes = {}, error) {
+  const endTimeMs = Date.now();
+  const durationMs = endTimeMs - span.startTimeMs;
+  const finished = {
+    ...span,
+    attributes: { ...span.attributes, ...extraAttributes },
+    endTimeMs,
+    durationMs,
+    status: error ? "error" : "ok",
+    error: error?.message
+  };
+  if (!span.parentSpanId) {
+    currentTraceId = null;
+    currentSpanId = span.parentSpanId ?? null;
+  } else {
+    currentSpanId = span.parentSpanId ?? null;
+  }
+  console.log(JSON.stringify({
+    ts: new Date(endTimeMs).toISOString(),
+    level: error ? "error" : "info",
+    msg: `span:${finished.name}`,
+    trace_id: finished.traceId,
+    span_id: finished.spanId,
+    parent_span_id: finished.parentSpanId,
+    duration_ms: durationMs,
+    status: finished.status,
+    error: finished.error,
+    ...finished.attributes
+  }));
+  return finished;
+}
+async function withSpan(name, attributes, fn) {
+  const span = startSpan(name, attributes);
+  try {
+    const result = await fn(span);
+    endSpan(span);
+    return result;
+  } catch (err) {
+    endSpan(span, {}, err);
+    throw err;
+  }
+}
+const counters = /* @__PURE__ */ new Map();
+function increment(metric, value = 1, tags = {}) {
+  const key = metric + (Object.keys(tags).length ? ":" + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(",") : "");
+  counters.set(key, (counters.get(key) ?? 0) + value);
+}
+function flushMetrics() {
+  const snapshot = Object.fromEntries(counters);
+  if (Object.keys(snapshot).length > 0) {
+    console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "metrics:flush", ...snapshot }));
+    counters.clear();
+  }
+  return snapshot;
+}
+export {
+  endSpan,
+  flushMetrics,
+  increment,
+  startSpan,
+  withSpan
+};
+//# sourceMappingURL=telemetry.js.map

package/dist/telemetry.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/telemetry.ts"],
+  "sourcesContent": ["/**\n * Lightweight OTel-compatible tracing for Zeta crawler.\n *\n * Emits spans as structured JSON logs compatible with OpenTelemetry format.\n * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.\n *\n * Usage:\n *   const span = startSpan('crawl.url', { url, engine });\n *   // ... do work ...\n *   endSpan(span, { screensFound: 3 });\n */\n\nimport { randomBytes } from 'node:crypto';\n\nexport interface Span {\n  traceId: string;\n  spanId: string;\n  parentSpanId?: string;\n  name: string;\n  startTimeMs: number;\n  attributes: Record<string, string | number | boolean>;\n}\n\nexport interface FinishedSpan extends Span {\n  endTimeMs: number;\n  durationMs: number;\n  status: 'ok' | 'error';\n  error?: string;\n}\n\n// Active trace context \u2014 per async execution context (simplified: per-process global)\nlet currentTraceId: string | null = null;\nlet currentSpanId: string | null = null;\n\nfunction genId(bytes: number): string {\n  return randomBytes(bytes).toString('hex');\n}\n\n/** Start a new span. If a trace is already active, the new span is a child. */\nexport function startSpan(name: string, attributes: Record<string, string | number | boolean> = {}): Span {\n  const traceId = currentTraceId ?? genId(16);\n  const spanId = genId(8);\n  const parentSpanId = currentSpanId ?? undefined;\n\n  if (!currentTraceId) currentTraceId = traceId;\n  currentSpanId = spanId;\n\n  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };\n}\n\n/** End a span and emit it as a structured log line. */\nexport function endSpan(span: Span, extraAttributes: Record<string, string | number | boolean> = {}, error?: Error): FinishedSpan {\n  const endTimeMs = Date.now();\n  const durationMs = endTimeMs - span.startTimeMs;\n  const finished: FinishedSpan = {\n    ...span,\n    attributes: { ...span.attributes, ...extraAttributes },\n    endTimeMs,\n    durationMs,\n    status: error ? 'error' : 'ok',\n    error: error?.message,\n  };\n\n  // Reset context when root span ends\n  if (!span.parentSpanId) {\n    currentTraceId = null;\n    currentSpanId = span.parentSpanId ?? null;\n  } else {\n    currentSpanId = span.parentSpanId ?? null;\n  }\n\n  // Emit as structured JSON \u2014 compatible with Datadog/Grafana log parsing\n  console.log(JSON.stringify({\n    ts: new Date(endTimeMs).toISOString(),\n    level: error ? 'error' : 'info',\n    msg: `span:${finished.name}`,\n    trace_id: finished.traceId,\n    span_id: finished.spanId,\n    parent_span_id: finished.parentSpanId,\n    duration_ms: durationMs,\n    status: finished.status,\n    error: finished.error,\n    ...finished.attributes,\n  }));\n\n  return finished;\n}\n\n/** Convenience: wrap an async function in a span. */\nexport async function withSpan<T>(\n  name: string,\n  attributes: Record<string, string | number | boolean>,\n  fn: (span: Span) => Promise<T>,\n): Promise<T> {\n  const span = startSpan(name, attributes);\n  try {\n    const result = await fn(span);\n    endSpan(span);\n    return result;\n  } catch (err: any) {\n    endSpan(span, {}, err);\n    throw err;\n  }\n}\n\n/** Counter metrics \u2014 increment and log on flush. */\nconst counters: Map<string, number> = new Map();\n\nexport function increment(metric: string, value = 1, tags: Record<string, string> = {}): void {\n  const key = metric + (Object.keys(tags).length ? ':' + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(',') : '');\n  counters.set(key, (counters.get(key) ?? 0) + value);\n}\n\n/** Flush counters as a structured log \u2014 call periodically (e.g. at job end). */\nexport function flushMetrics(): Record<string, number> {\n  const snapshot = Object.fromEntries(counters);\n  if (Object.keys(snapshot).length > 0) {\n    console.log(JSON.stringify({ ts: new Date().toISOString(), level: 'info', msg: 'metrics:flush', ...snapshot }));\n    counters.clear();\n  }\n  return snapshot;\n}\n"],
+  "mappings": "AAYA,SAAS,mBAAmB;AAmB5B,IAAI,iBAAgC;AACpC,IAAI,gBAA+B;AAEnC,SAAS,MAAM,OAAuB;AACpC,SAAO,YAAY,KAAK,EAAE,SAAS,KAAK;AAC1C;AAGO,SAAS,UAAU,MAAc,aAAwD,CAAC,GAAS;AACxG,QAAM,UAAU,kBAAkB,MAAM,EAAE;AAC1C,QAAM,SAAS,MAAM,CAAC;AACtB,QAAM,eAAe,iBAAiB;AAEtC,MAAI,CAAC,eAAgB,kBAAiB;AACtC,kBAAgB;AAEhB,SAAO,EAAE,SAAS,QAAQ,cAAc,MAAM,aAAa,KAAK,IAAI,GAAG,WAAW;AACpF;AAGO,SAAS,QAAQ,MAAY,kBAA6D,CAAC,GAAG,OAA6B;AAChI,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,aAAa,YAAY,KAAK;AACpC,QAAM,WAAyB;AAAA,IAC7B,GAAG;AAAA,IACH,YAAY,EAAE,GAAG,KAAK,YAAY,GAAG,gBAAgB;AAAA,IACrD;AAAA,IACA;AAAA,IACA,QAAQ,QAAQ,UAAU;AAAA,IAC1B,OAAO,OAAO;AAAA,EAChB;AAGA,MAAI,CAAC,KAAK,cAAc;AACtB,qBAAiB;AACjB,oBAAgB,KAAK,gBAAgB;AAAA,EACvC,OAAO;AACL,oBAAgB,KAAK,gBAAgB;AAAA,EACvC;AAGA,UAAQ,IAAI,KAAK,UAAU;AAAA,IACzB,IAAI,IAAI,KAAK,SAAS,EAAE,YAAY;AAAA,IACpC,OAAO,QAAQ,UAAU;AAAA,IACzB,KAAK,QAAQ,SAAS,IAAI;AAAA,IAC1B,UAAU,SAAS;AAAA,IACnB,SAAS,SAAS;AAAA,IAClB,gBAAgB,SAAS;AAAA,IACzB,aAAa;AAAA,IACb,QAAQ,SAAS;AAAA,IACjB,OAAO,SAAS;AAAA,IAChB,GAAG,SAAS;AAAA,EACd,CAAC,CAAC;AAEF,SAAO;AACT;AAGA,eAAsB,SACpB,MACA,YACA,IACY;AACZ,QAAM,OAAO,UAAU,MAAM,UAAU;AACvC,MAAI;AACF,UAAM,SAAS,MAAM,GAAG,IAAI;AAC5B,YAAQ,IAAI;AACZ,WAAO;AAAA,EACT,SAAS,KAAU;AACjB,YAAQ,MAAM,CAAC,GAAG,GAAG;AACrB,UAAM;AAAA,EACR;AACF;AAGA,MAAM,WAAgC,oBAAI,IAAI;AAEvC,SAAS,UAAU,QAAgB,QAAQ,GAAG,OAA+B,CAAC,GAAS;AAC5F,QAAM,MAAM,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,MAAM,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;AACrH,WAAS,IAAI,MAAM,SAAS,IAAI,GAAG,KAAK,KAAK,KAAK;AACpD;AAGO,SAAS,eAAuC;AACrD,QAAM,WAAW,OAAO,YAAY,QAAQ;AAC5C,MAAI,OAAO,KAAK,QAAQ,EAAE,SAAS,GAAG;AACpC,YAAQ,IAAI,KAAK,UAAU,EAAE,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,OAAO,QAAQ,KAAK,iBAAiB,GAAG,SAAS,CAAC,CAAC;AAC9G,aAAS,MAAM;AAAA,EACjB;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/test-executor.d.ts

@@ -0,0 +1,67 @@
+import type { ExecutionStep } from '@qip/agents';
+export interface StepResult {
+    stepNumber: number;
+    description: string;
+    status: 'PASSED' | 'FAILED' | 'SKIPPED';
+    duration: number;
+    error?: string;
+    screenshotPath?: string;
+    healed?: boolean;
+    /** 'mechanical' = same selector text, different Playwright locator strategy.
+     *  'ai' = AI Self-Healing Agent matched a semantically different element
+     *  (the app's selector text itself changed, e.g. "Sign In" -> "Log In"). */
+    healedVia?: 'mechanical' | 'ai';
+    healReasoning?: string;
+    healConfidence?: number;
+}
+export interface ExecutionResult {
+    stepResults: StepResult[];
+    status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';
+    errorStep?: number;
+    totalDuration: number;
+    finalScreenshotPath?: string;
+    retryCount?: number;
+    consoleLogs?: Array<{
+        type: string;
+        text: string;
+        timestamp: number;
+    }>;
+    networkErrors?: Array<{
+        url: string;
+        status: number;
+        method: string;
+        timestamp: number;
+    }>;
+    rootCauseSummary?: string;
+    networkTimings?: Array<{
+        url: string;
+        method: string;
+        status: number;
+        duration: number;
+    }>;
+}
+export declare function executeTestPlan(plan: ExecutionStep[], options: {
+    screenshotDir: string;
+    testRunId: string;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    onStepComplete?: (result: StepResult) => void;
+    /** Run the test up to this many times to detect flakiness (default 1). */
+    maxRuns?: number;
+    browser?: 'chromium' | 'firefox' | 'webkit';
+    /** Data row for parameterized execution: substitutes {{varname}} in step fields. */
+    dataRow?: Record<string, string>;
+    /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved
+     *  even after trying alternate locator strategies. Omit to keep mechanical-only
+     *  healing (no LLM calls during execution). */
+    healCtx?: {
+        tenantId: string;
+        projectId: string;
+    };
+    /** Decrypted Playwright storageState JSON — restores a logged-in session so
+     *  steps don't have to re-authenticate. Without this, any test case behind a
+     *  login wall fails at the first authenticated step. */
+    storageState?: string;
+}): Promise<ExecutionResult>;