@detiq/crawler 1.1.9 → 1.1.10

package/dist/stealth/human-behavior.js

@@ -1,186 +1,169 @@
-// Gaussian random using Box-Muller transform
 function gaussian(mean, sigma) {
-    const u1 = Math.random();
-    const u2 = Math.random();
-    const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
-    return mean + z * sigma;
+  const u1 = Math.random();
+  const u2 = Math.random();
+  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+  return mean + z * sigma;
 }
-export async function humanDelay(min = 80, max = 450) {
-    const mean = (min + max) / 2;
-    const sigma = (max - min) / 6;
-    const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
-    await new Promise(r => setTimeout(r, Math.round(delay)));
+async function humanDelay(min = 80, max = 450) {
+  const mean = (min + max) / 2;
+  const sigma = (max - min) / 6;
+  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
+  await new Promise((r) => setTimeout(r, Math.round(delay)));
 }
-export async function simulateMouseMovement(page, targetX, targetY) {
-    try {
-        const mouse = page.mouse;
-        const steps = 8 + Math.floor(Math.random() * 8);
-        // Generate Bezier control points with slight perpendicular offset
-        const startX = targetX - 100 + Math.random() * 200;
-        const startY = targetY - 100 + Math.random() * 200;
-        const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
-        const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
-        for (let i = 0; i <= steps; i++) {
-            const t = i / steps;
-            // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2
-            const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
-            const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
-            await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
-            if (i < steps)
-                await humanDelay(8, 25);
-        }
-        // Small overshoot correction
-        await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
-        await humanDelay(30, 80);
-        await mouse.move(targetX, targetY);
+async function simulateMouseMovement(page, targetX, targetY) {
+  try {
+    const mouse = page.mouse;
+    const steps = 8 + Math.floor(Math.random() * 8);
+    const startX = targetX - 100 + Math.random() * 200;
+    const startY = targetY - 100 + Math.random() * 200;
+    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
+    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
+    for (let i = 0; i <= steps; i++) {
+      const t = i / steps;
+      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
+      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
+      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
+      if (i < steps) await humanDelay(8, 25);
     }
-    catch { /* non-fatal */ }
+    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
+    await humanDelay(30, 80);
+    await mouse.move(targetX, targetY);
+  } catch {
+  }
 }
-export async function simulateMouseHover(page, selector) {
-    try {
-        const el = await page.$(selector);
-        if (!el)
-            return;
-        const box = await el.boundingBox();
-        if (!box)
-            return;
-        const cx = box.x + box.width / 2;
-        const cy = box.y + box.height / 2;
-        await simulateMouseMovement(page, cx, cy);
-        await humanDelay(100, 300);
-    }
-    catch { /* non-fatal */ }
+async function simulateMouseHover(page, selector) {
+  try {
+    const el = await page.$(selector);
+    if (!el) return;
+    const box = await el.boundingBox();
+    if (!box) return;
+    const cx = box.x + box.width / 2;
+    const cy = box.y + box.height / 2;
+    await simulateMouseMovement(page, cx, cy);
+    await humanDelay(100, 300);
+  } catch {
+  }
 }
-export async function simulateClick(page, selector) {
-    try {
-        await simulateMouseHover(page, selector);
-        await humanDelay(50, 150);
-        await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
-    }
-    catch { /* non-fatal */ }
+async function simulateClick(page, selector) {
+  try {
+    await simulateMouseHover(page, selector);
+    await humanDelay(50, 150);
+    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
+  } catch {
+  }
 }
-export async function injectFingerprintOverrides(page) {
-    await page.addInitScript(() => {
-        // Canvas noise — subtle per-pixel perturbation
-        const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
-        HTMLCanvasElement.prototype.toDataURL = function (...args) {
-            const ctx = this.getContext('2d');
-            if (ctx) {
-                const imageData = ctx.getImageData(0, 0, this.width, this.height);
-                for (let i = 0; i < imageData.data.length; i += 4) {
-                    imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));
-                }
-                ctx.putImageData(imageData, 0, 0);
-            }
-            return origToDataURL.apply(this, args);
-        };
-        // WebGL parameter spoofing
-        const origGetParam = WebGLRenderingContext.prototype.getParameter;
-        WebGLRenderingContext.prototype.getParameter = function (parameter) {
-            const UNMASKED_VENDOR_WEBGL = 0x9245;
-            const UNMASKED_RENDERER_WEBGL = 0x9246;
-            if (parameter === UNMASKED_VENDOR_WEBGL)
-                return 'Intel Inc.';
-            if (parameter === UNMASKED_RENDERER_WEBGL)
-                return 'Intel Iris OpenGL Engine';
-            return origGetParam.call(this, parameter);
-        };
-        // Hardware concurrency — pick from common values
-        const cpuCounts = [4, 6, 8, 8];
-        Object.defineProperty(navigator, 'hardwareConcurrency', {
-            get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
-            configurable: true,
-        });
-        // Device memory
-        Object.defineProperty(navigator, 'deviceMemory', {
-            get: () => [4, 8][Math.floor(Math.random() * 2)],
-            configurable: true,
-        });
-    }).catch(() => { });
-}
-export async function infiniteScrollToBottom(page, opts = {}) {
-    const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
-    let scrollCount = 0;
-    let stableScrollCount = 0;
-    let previousHeight = 0;
-    try {
-        while (scrollCount < maxScrolls) {
-            previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
-            const scrollBy = 500 + Math.floor(Math.random() * 300);
-            await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => { });
-            await humanDelay(scrollDelay - 100, scrollDelay + 200);
-            // Wait for potential lazy-loaded content
-            await page.waitForTimeout(stabilizeMs).catch(() => { });
-            scrollCount++;
-            const newHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
-            if (newHeight - previousHeight < 10) {
-                stableScrollCount++;
-                if (stableScrollCount >= 3)
-                    break; // no meaningful new content after 3 consecutive non-growing scrolls
-            }
-            else {
-                stableScrollCount = 0;
-            }
-            previousHeight = newHeight;
+async function injectFingerprintOverrides(page) {
+  await page.addInitScript(() => {
+    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
+    HTMLCanvasElement.prototype.toDataURL = function(...args) {
+      const ctx = this.getContext("2d");
+      if (ctx) {
+        const imageData = ctx.getImageData(0, 0, this.width, this.height);
+        for (let i = 0; i < imageData.data.length; i += 4) {
+          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? Math.random() > 0.5 ? 1 : -1 : 0)));
         }
-    }
-    catch { /* non-fatal */ }
-    return { scrollCount, heightReached: previousHeight };
+        ctx.putImageData(imageData, 0, 0);
+      }
+      return origToDataURL.apply(this, args);
+    };
+    const origGetParam = WebGLRenderingContext.prototype.getParameter;
+    WebGLRenderingContext.prototype.getParameter = function(parameter) {
+      const UNMASKED_VENDOR_WEBGL = 37445;
+      const UNMASKED_RENDERER_WEBGL = 37446;
+      if (parameter === UNMASKED_VENDOR_WEBGL) return "Intel Inc.";
+      if (parameter === UNMASKED_RENDERER_WEBGL) return "Intel Iris OpenGL Engine";
+      return origGetParam.call(this, parameter);
+    };
+    const cpuCounts = [4, 6, 8, 8];
+    Object.defineProperty(navigator, "hardwareConcurrency", {
+      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
+      configurable: true
+    });
+    Object.defineProperty(navigator, "deviceMemory", {
+      get: () => [4, 8][Math.floor(Math.random() * 2)],
+      configurable: true
+    });
+  }).catch(() => {
+  });
 }
-export async function extractIframeContent(page) {
-    const results = [];
-    try {
-        const iframeSrcs = await page.evaluate(() => Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')).catch(() => []);
-        for (const frame of page.frames()) {
-            const src = frame.url();
-            if (!src || src === 'about:blank' || src === page.url())
-                continue;
-            try {
-                const content = await frame.content().catch(() => '');
-                if (!content || content.length < 50)
-                    continue;
-                // Simple text extraction from iframe HTML
-                const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
-                    .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
-                    .replace(/<[^>]+>/g, ' ')
-                    .replace(/\s+/g, ' ')
-                    .trim();
-                results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });
-            }
-            catch { /* skip this frame */ }
-        }
+async function infiniteScrollToBottom(page, opts = {}) {
+  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
+  let scrollCount = 0;
+  let stableScrollCount = 0;
+  let previousHeight = 0;
+  try {
+    while (scrollCount < maxScrolls) {
+      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+      const scrollBy = 500 + Math.floor(Math.random() * 300);
+      await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => {
+      });
+      await humanDelay(scrollDelay - 100, scrollDelay + 200);
+      await page.waitForTimeout(stabilizeMs).catch(() => {
+      });
+      scrollCount++;
+      const newHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+      if (newHeight - previousHeight < 10) {
+        stableScrollCount++;
+        if (stableScrollCount >= 3) break;
+      } else {
+        stableScrollCount = 0;
+      }
+      previousHeight = newHeight;
     }
-    catch { /* non-fatal */ }
-    return results;
+  } catch {
+  }
+  return { scrollCount, heightReached: previousHeight };
 }
-export async function pierceShadowDom(page) {
-    try {
-        return await page.evaluate(() => {
-            const parts = [];
-            function walkShadow(root) {
-                const shadow = root.shadowRoot;
-                if (shadow) {
-                    parts.push(shadow.innerHTML ?? '');
-                    for (const child of [...shadow.querySelectorAll('*')])
-                        walkShadow(child);
-                }
-                for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {
-                    if (child.shadowRoot)
-                        walkShadow(child);
-                }
-            }
-            walkShadow(document.body);
-            return parts
-                .join('\n')
-                .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
-                .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
-                .replace(/<[^>]+>/g, ' ')
-                .replace(/\s+/g, ' ')
-                .trim()
-                .slice(0, 10_000);
-        }).catch(() => '');
-    }
-    catch {
-        return '';
+async function extractIframeContent(page) {
+  const results = [];
+  try {
+    const iframeSrcs = await page.evaluate(
+      () => Array.from(document.querySelectorAll("iframe")).map((f) => f.getAttribute("src") ?? f.getAttribute("data-src") ?? "")
+    ).catch(() => []);
+    for (const frame of page.frames()) {
+      const src = frame.url();
+      if (!src || src === "about:blank" || src === page.url()) continue;
+      try {
+        const content = await frame.content().catch(() => "");
+        if (!content || content.length < 50) continue;
+        const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+        results.push({ src, content: content.slice(0, 1e4), markdown: text.slice(0, 5e3) });
+      } catch {
+      }
     }
+  } catch {
+  }
+  return results;
+}
+async function pierceShadowDom(page) {
+  try {
+    return await page.evaluate(() => {
+      const parts = [];
+      function walkShadow(root) {
+        const shadow = root.shadowRoot;
+        if (shadow) {
+          parts.push(shadow.innerHTML ?? "");
+          for (const child of [...shadow.querySelectorAll("*")]) walkShadow(child);
+        }
+        for (const child of Array.from(root.querySelectorAll(":not(script):not(style)"))) {
+          if (child.shadowRoot) walkShadow(child);
+        }
+      }
+      walkShadow(document.body);
+      return parts.join("\n").replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 1e4);
+    }).catch(() => "");
+  } catch {
+    return "";
+  }
 }
+export {
+  extractIframeContent,
+  humanDelay,
+  infiniteScrollToBottom,
+  injectFingerprintOverrides,
+  pierceShadowDom,
+  simulateClick,
+  simulateMouseHover,
+  simulateMouseMovement
+};
+//# sourceMappingURL=human-behavior.js.map

package/dist/telemetry.js

@@ -1,91 +1,77 @@
-/**
- * Lightweight OTel-compatible tracing for Zeta crawler.
- *
- * Emits spans as structured JSON logs compatible with OpenTelemetry format.
- * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
- *
- * Usage:
- *   const span = startSpan('crawl.url', { url, engine });
- *   // ... do work ...
- *   endSpan(span, { screensFound: 3 });
- */
-import { randomBytes } from 'node:crypto';
-// Active trace context — per async execution context (simplified: per-process global)
+import { randomBytes } from "node:crypto";
 let currentTraceId = null;
 let currentSpanId = null;
 function genId(bytes) {
-    return randomBytes(bytes).toString('hex');
+  return randomBytes(bytes).toString("hex");
 }
-/** Start a new span. If a trace is already active, the new span is a child. */
-export function startSpan(name, attributes = {}) {
-    const traceId = currentTraceId ?? genId(16);
-    const spanId = genId(8);
-    const parentSpanId = currentSpanId ?? undefined;
-    if (!currentTraceId)
-        currentTraceId = traceId;
-    currentSpanId = spanId;
-    return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
+function startSpan(name, attributes = {}) {
+  const traceId = currentTraceId ?? genId(16);
+  const spanId = genId(8);
+  const parentSpanId = currentSpanId ?? void 0;
+  if (!currentTraceId) currentTraceId = traceId;
+  currentSpanId = spanId;
+  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
 }
-/** End a span and emit it as a structured log line. */
-export function endSpan(span, extraAttributes = {}, error) {
-    const endTimeMs = Date.now();
-    const durationMs = endTimeMs - span.startTimeMs;
-    const finished = {
-        ...span,
-        attributes: { ...span.attributes, ...extraAttributes },
-        endTimeMs,
-        durationMs,
-        status: error ? 'error' : 'ok',
-        error: error?.message,
-    };
-    // Reset context when root span ends
-    if (!span.parentSpanId) {
-        currentTraceId = null;
-        currentSpanId = span.parentSpanId ?? null;
-    }
-    else {
-        currentSpanId = span.parentSpanId ?? null;
-    }
-    // Emit as structured JSON — compatible with Datadog/Grafana log parsing
-    console.log(JSON.stringify({
-        ts: new Date(endTimeMs).toISOString(),
-        level: error ? 'error' : 'info',
-        msg: `span:${finished.name}`,
-        trace_id: finished.traceId,
-        span_id: finished.spanId,
-        parent_span_id: finished.parentSpanId,
-        duration_ms: durationMs,
-        status: finished.status,
-        error: finished.error,
-        ...finished.attributes,
-    }));
-    return finished;
+function endSpan(span, extraAttributes = {}, error) {
+  const endTimeMs = Date.now();
+  const durationMs = endTimeMs - span.startTimeMs;
+  const finished = {
+    ...span,
+    attributes: { ...span.attributes, ...extraAttributes },
+    endTimeMs,
+    durationMs,
+    status: error ? "error" : "ok",
+    error: error?.message
+  };
+  if (!span.parentSpanId) {
+    currentTraceId = null;
+    currentSpanId = span.parentSpanId ?? null;
+  } else {
+    currentSpanId = span.parentSpanId ?? null;
+  }
+  console.log(JSON.stringify({
+    ts: new Date(endTimeMs).toISOString(),
+    level: error ? "error" : "info",
+    msg: `span:${finished.name}`,
+    trace_id: finished.traceId,
+    span_id: finished.spanId,
+    parent_span_id: finished.parentSpanId,
+    duration_ms: durationMs,
+    status: finished.status,
+    error: finished.error,
+    ...finished.attributes
+  }));
+  return finished;
 }
-/** Convenience: wrap an async function in a span. */
-export async function withSpan(name, attributes, fn) {
-    const span = startSpan(name, attributes);
-    try {
-        const result = await fn(span);
-        endSpan(span);
-        return result;
-    }
-    catch (err) {
-        endSpan(span, {}, err);
-        throw err;
-    }
+async function withSpan(name, attributes, fn) {
+  const span = startSpan(name, attributes);
+  try {
+    const result = await fn(span);
+    endSpan(span);
+    return result;
+  } catch (err) {
+    endSpan(span, {}, err);
+    throw err;
+  }
 }
-/** Counter metrics — increment and log on flush. */
-const counters = new Map();
-export function increment(metric, value = 1, tags = {}) {
-    const key = metric + (Object.keys(tags).length ? ':' + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(',') : '');
-    counters.set(key, (counters.get(key) ?? 0) + value);
+const counters = /* @__PURE__ */ new Map();
+function increment(metric, value = 1, tags = {}) {
+  const key = metric + (Object.keys(tags).length ? ":" + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(",") : "");
+  counters.set(key, (counters.get(key) ?? 0) + value);
 }
-/** Flush counters as a structured log — call periodically (e.g. at job end). */
-export function flushMetrics() {
-    const snapshot = Object.fromEntries(counters);
-    if (Object.keys(snapshot).length > 0) {
-        console.log(JSON.stringify({ ts: new Date().toISOString(), level: 'info', msg: 'metrics:flush', ...snapshot }));
-        counters.clear();
-    }
-    return snapshot;
+function flushMetrics() {
+  const snapshot = Object.fromEntries(counters);
+  if (Object.keys(snapshot).length > 0) {
+    console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "metrics:flush", ...snapshot }));
+    counters.clear();
+  }
+  return snapshot;
 }
+export {
+  endSpan,
+  flushMetrics,
+  increment,
+  startSpan,
+  withSpan
+};
+//# sourceMappingURL=telemetry.js.map