@detiq/crawler 1.1.7 → 1.1.9

package/dist/stealth/human-behavior.js

@@ -1,169 +1,186 @@
+// Gaussian random using Box-Muller transform
 function gaussian(mean, sigma) {
-  const u1 = Math.random();
-  const u2 = Math.random();
-  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
-  return mean + z * sigma;
+    const u1 = Math.random();
+    const u2 = Math.random();
+    const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+    return mean + z * sigma;
 }
-async function humanDelay(min = 80, max = 450) {
-  const mean = (min + max) / 2;
-  const sigma = (max - min) / 6;
-  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
-  await new Promise((r) => setTimeout(r, Math.round(delay)));
+export async function humanDelay(min = 80, max = 450) {
+    const mean = (min + max) / 2;
+    const sigma = (max - min) / 6;
+    const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
+    await new Promise(r => setTimeout(r, Math.round(delay)));
 }
-async function simulateMouseMovement(page, targetX, targetY) {
-  try {
-    const mouse = page.mouse;
-    const steps = 8 + Math.floor(Math.random() * 8);
-    const startX = targetX - 100 + Math.random() * 200;
-    const startY = targetY - 100 + Math.random() * 200;
-    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
-    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
-    for (let i = 0; i <= steps; i++) {
-      const t = i / steps;
-      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
-      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
-      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
-      if (i < steps) await humanDelay(8, 25);
+export async function simulateMouseMovement(page, targetX, targetY) {
+    try {
+        const mouse = page.mouse;
+        const steps = 8 + Math.floor(Math.random() * 8);
+        // Generate Bezier control points with slight perpendicular offset
+        const startX = targetX - 100 + Math.random() * 200;
+        const startY = targetY - 100 + Math.random() * 200;
+        const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
+        const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
+        for (let i = 0; i <= steps; i++) {
+            const t = i / steps;
+            // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2
+            const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
+            const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
+            await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
+            if (i < steps)
+                await humanDelay(8, 25);
+        }
+        // Small overshoot correction
+        await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
+        await humanDelay(30, 80);
+        await mouse.move(targetX, targetY);
     }
-    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
-    await humanDelay(30, 80);
-    await mouse.move(targetX, targetY);
-  } catch {
-  }
+    catch { /* non-fatal */ }
 }
-async function simulateMouseHover(page, selector) {
-  try {
-    const el = await page.$(selector);
-    if (!el) return;
-    const box = await el.boundingBox();
-    if (!box) return;
-    const cx = box.x + box.width / 2;
-    const cy = box.y + box.height / 2;
-    await simulateMouseMovement(page, cx, cy);
-    await humanDelay(100, 300);
-  } catch {
-  }
+export async function simulateMouseHover(page, selector) {
+    try {
+        const el = await page.$(selector);
+        if (!el)
+            return;
+        const box = await el.boundingBox();
+        if (!box)
+            return;
+        const cx = box.x + box.width / 2;
+        const cy = box.y + box.height / 2;
+        await simulateMouseMovement(page, cx, cy);
+        await humanDelay(100, 300);
+    }
+    catch { /* non-fatal */ }
 }
-async function simulateClick(page, selector) {
-  try {
-    await simulateMouseHover(page, selector);
-    await humanDelay(50, 150);
-    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
-  } catch {
-  }
+export async function simulateClick(page, selector) {
+    try {
+        await simulateMouseHover(page, selector);
+        await humanDelay(50, 150);
+        await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
+    }
+    catch { /* non-fatal */ }
 }
-async function injectFingerprintOverrides(page) {
-  await page.addInitScript(() => {
-    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
-    HTMLCanvasElement.prototype.toDataURL = function(...args) {
-      const ctx = this.getContext("2d");
-      if (ctx) {
-        const imageData = ctx.getImageData(0, 0, this.width, this.height);
-        for (let i = 0; i < imageData.data.length; i += 4) {
-          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? Math.random() > 0.5 ? 1 : -1 : 0)));
-        }
-        ctx.putImageData(imageData, 0, 0);
-      }
-      return origToDataURL.apply(this, args);
-    };
-    const origGetParam = WebGLRenderingContext.prototype.getParameter;
-    WebGLRenderingContext.prototype.getParameter = function(parameter) {
-      const UNMASKED_VENDOR_WEBGL = 37445;
-      const UNMASKED_RENDERER_WEBGL = 37446;
-      if (parameter === UNMASKED_VENDOR_WEBGL) return "Intel Inc.";
-      if (parameter === UNMASKED_RENDERER_WEBGL) return "Intel Iris OpenGL Engine";
-      return origGetParam.call(this, parameter);
-    };
-    const cpuCounts = [4, 6, 8, 8];
-    Object.defineProperty(navigator, "hardwareConcurrency", {
-      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
-      configurable: true
-    });
-    Object.defineProperty(navigator, "deviceMemory", {
-      get: () => [4, 8][Math.floor(Math.random() * 2)],
-      configurable: true
-    });
-  }).catch(() => {
-  });
+export async function injectFingerprintOverrides(page) {
+    await page.addInitScript(() => {
+        // Canvas noise — subtle per-pixel perturbation
+        const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
+        HTMLCanvasElement.prototype.toDataURL = function (...args) {
+            const ctx = this.getContext('2d');
+            if (ctx) {
+                const imageData = ctx.getImageData(0, 0, this.width, this.height);
+                for (let i = 0; i < imageData.data.length; i += 4) {
+                    imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));
+                }
+                ctx.putImageData(imageData, 0, 0);
+            }
+            return origToDataURL.apply(this, args);
+        };
+        // WebGL parameter spoofing
+        const origGetParam = WebGLRenderingContext.prototype.getParameter;
+        WebGLRenderingContext.prototype.getParameter = function (parameter) {
+            const UNMASKED_VENDOR_WEBGL = 0x9245;
+            const UNMASKED_RENDERER_WEBGL = 0x9246;
+            if (parameter === UNMASKED_VENDOR_WEBGL)
+                return 'Intel Inc.';
+            if (parameter === UNMASKED_RENDERER_WEBGL)
+                return 'Intel Iris OpenGL Engine';
+            return origGetParam.call(this, parameter);
+        };
+        // Hardware concurrency — pick from common values
+        const cpuCounts = [4, 6, 8, 8];
+        Object.defineProperty(navigator, 'hardwareConcurrency', {
+            get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
+            configurable: true,
+        });
+        // Device memory
+        Object.defineProperty(navigator, 'deviceMemory', {
+            get: () => [4, 8][Math.floor(Math.random() * 2)],
+            configurable: true,
+        });
+    }).catch(() => { });
 }
-async function infiniteScrollToBottom(page, opts = {}) {
-  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
-  let scrollCount = 0;
-  let stableScrollCount = 0;
-  let previousHeight = 0;
-  try {
-    while (scrollCount < maxScrolls) {
-      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
-      const scrollBy = 500 + Math.floor(Math.random() * 300);
-      await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => {
-      });
-      await humanDelay(scrollDelay - 100, scrollDelay + 200);
-      await page.waitForTimeout(stabilizeMs).catch(() => {
-      });
-      scrollCount++;
-      const newHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
-      if (newHeight - previousHeight < 10) {
-        stableScrollCount++;
-        if (stableScrollCount >= 3) break;
-      } else {
-        stableScrollCount = 0;
-      }
-      previousHeight = newHeight;
+export async function infiniteScrollToBottom(page, opts = {}) {
+    const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
+    let scrollCount = 0;
+    let stableScrollCount = 0;
+    let previousHeight = 0;
+    try {
+        while (scrollCount < maxScrolls) {
+            previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+            const scrollBy = 500 + Math.floor(Math.random() * 300);
+            await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => { });
+            await humanDelay(scrollDelay - 100, scrollDelay + 200);
+            // Wait for potential lazy-loaded content
+            await page.waitForTimeout(stabilizeMs).catch(() => { });
+            scrollCount++;
+            const newHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+            if (newHeight - previousHeight < 10) {
+                stableScrollCount++;
+                if (stableScrollCount >= 3)
+                    break; // no meaningful new content after 3 consecutive non-growing scrolls
+            }
+            else {
+                stableScrollCount = 0;
+            }
+            previousHeight = newHeight;
+        }
     }
-  } catch {
-  }
-  return { scrollCount, heightReached: previousHeight };
+    catch { /* non-fatal */ }
+    return { scrollCount, heightReached: previousHeight };
 }
-async function extractIframeContent(page) {
-  const results = [];
-  try {
-    const iframeSrcs = await page.evaluate(
-      () => Array.from(document.querySelectorAll("iframe")).map((f) => f.getAttribute("src") ?? f.getAttribute("data-src") ?? "")
-    ).catch(() => []);
-    for (const frame of page.frames()) {
-      const src = frame.url();
-      if (!src || src === "about:blank" || src === page.url()) continue;
-      try {
-        const content = await frame.content().catch(() => "");
-        if (!content || content.length < 50) continue;
-        const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
-        results.push({ src, content: content.slice(0, 1e4), markdown: text.slice(0, 5e3) });
-      } catch {
-      }
+export async function extractIframeContent(page) {
+    const results = [];
+    try {
+        const iframeSrcs = await page.evaluate(() => Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')).catch(() => []);
+        for (const frame of page.frames()) {
+            const src = frame.url();
+            if (!src || src === 'about:blank' || src === page.url())
+                continue;
+            try {
+                const content = await frame.content().catch(() => '');
+                if (!content || content.length < 50)
+                    continue;
+                // Simple text extraction from iframe HTML
+                const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+                    .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+                    .replace(/<[^>]+>/g, ' ')
+                    .replace(/\s+/g, ' ')
+                    .trim();
+                results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });
+            }
+            catch { /* skip this frame */ }
+        }
     }
-  } catch {
-  }
-  return results;
+    catch { /* non-fatal */ }
+    return results;
 }
-async function pierceShadowDom(page) {
-  try {
-    return await page.evaluate(() => {
-      const parts = [];
-      function walkShadow(root) {
-        const shadow = root.shadowRoot;
-        if (shadow) {
-          parts.push(shadow.innerHTML ?? "");
-          for (const child of [...shadow.querySelectorAll("*")]) walkShadow(child);
-        }
-        for (const child of Array.from(root.querySelectorAll(":not(script):not(style)"))) {
-          if (child.shadowRoot) walkShadow(child);
-        }
-      }
-      walkShadow(document.body);
-      return parts.join("\n").replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 1e4);
-    }).catch(() => "");
-  } catch {
-    return "";
-  }
+export async function pierceShadowDom(page) {
+    try {
+        return await page.evaluate(() => {
+            const parts = [];
+            function walkShadow(root) {
+                const shadow = root.shadowRoot;
+                if (shadow) {
+                    parts.push(shadow.innerHTML ?? '');
+                    for (const child of [...shadow.querySelectorAll('*')])
+                        walkShadow(child);
+                }
+                for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {
+                    if (child.shadowRoot)
+                        walkShadow(child);
+                }
+            }
+            walkShadow(document.body);
+            return parts
+                .join('\n')
+                .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+                .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+                .replace(/<[^>]+>/g, ' ')
+                .replace(/\s+/g, ' ')
+                .trim()
+                .slice(0, 10_000);
+        }).catch(() => '');
+    }
+    catch {
+        return '';
+    }
 }
-export {
-  extractIframeContent,
-  humanDelay,
-  infiniteScrollToBottom,
-  injectFingerprintOverrides,
-  pierceShadowDom,
-  simulateClick,
-  simulateMouseHover,
-  simulateMouseMovement
-};
-//# sourceMappingURL=human-behavior.js.map

package/dist/telemetry.js

@@ -1,77 +1,91 @@
-import { randomBytes } from "node:crypto";
+/**
+ * Lightweight OTel-compatible tracing for Zeta crawler.
+ *
+ * Emits spans as structured JSON logs compatible with OpenTelemetry format.
+ * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
+ *
+ * Usage:
+ *   const span = startSpan('crawl.url', { url, engine });
+ *   // ... do work ...
+ *   endSpan(span, { screensFound: 3 });
+ */
+import { randomBytes } from 'node:crypto';
+// Active trace context — per async execution context (simplified: per-process global)
 let currentTraceId = null;
 let currentSpanId = null;
 function genId(bytes) {
-  return randomBytes(bytes).toString("hex");
+    return randomBytes(bytes).toString('hex');
 }
-function startSpan(name, attributes = {}) {
-  const traceId = currentTraceId ?? genId(16);
-  const spanId = genId(8);
-  const parentSpanId = currentSpanId ?? void 0;
-  if (!currentTraceId) currentTraceId = traceId;
-  currentSpanId = spanId;
-  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
+/** Start a new span. If a trace is already active, the new span is a child. */
+export function startSpan(name, attributes = {}) {
+    const traceId = currentTraceId ?? genId(16);
+    const spanId = genId(8);
+    const parentSpanId = currentSpanId ?? undefined;
+    if (!currentTraceId)
+        currentTraceId = traceId;
+    currentSpanId = spanId;
+    return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
 }
-function endSpan(span, extraAttributes = {}, error) {
-  const endTimeMs = Date.now();
-  const durationMs = endTimeMs - span.startTimeMs;
-  const finished = {
-    ...span,
-    attributes: { ...span.attributes, ...extraAttributes },
-    endTimeMs,
-    durationMs,
-    status: error ? "error" : "ok",
-    error: error?.message
-  };
-  if (!span.parentSpanId) {
-    currentTraceId = null;
-    currentSpanId = span.parentSpanId ?? null;
-  } else {
-    currentSpanId = span.parentSpanId ?? null;
-  }
-  console.log(JSON.stringify({
-    ts: new Date(endTimeMs).toISOString(),
-    level: error ? "error" : "info",
-    msg: `span:${finished.name}`,
-    trace_id: finished.traceId,
-    span_id: finished.spanId,
-    parent_span_id: finished.parentSpanId,
-    duration_ms: durationMs,
-    status: finished.status,
-    error: finished.error,
-    ...finished.attributes
-  }));
-  return finished;
+/** End a span and emit it as a structured log line. */
+export function endSpan(span, extraAttributes = {}, error) {
+    const endTimeMs = Date.now();
+    const durationMs = endTimeMs - span.startTimeMs;
+    const finished = {
+        ...span,
+        attributes: { ...span.attributes, ...extraAttributes },
+        endTimeMs,
+        durationMs,
+        status: error ? 'error' : 'ok',
+        error: error?.message,
+    };
+    // Reset context when root span ends
+    if (!span.parentSpanId) {
+        currentTraceId = null;
+        currentSpanId = span.parentSpanId ?? null;
+    }
+    else {
+        currentSpanId = span.parentSpanId ?? null;
+    }
+    // Emit as structured JSON — compatible with Datadog/Grafana log parsing
+    console.log(JSON.stringify({
+        ts: new Date(endTimeMs).toISOString(),
+        level: error ? 'error' : 'info',
+        msg: `span:${finished.name}`,
+        trace_id: finished.traceId,
+        span_id: finished.spanId,
+        parent_span_id: finished.parentSpanId,
+        duration_ms: durationMs,
+        status: finished.status,
+        error: finished.error,
+        ...finished.attributes,
+    }));
+    return finished;
 }
-async function withSpan(name, attributes, fn) {
-  const span = startSpan(name, attributes);
-  try {
-    const result = await fn(span);
-    endSpan(span);
-    return result;
-  } catch (err) {
-    endSpan(span, {}, err);
-    throw err;
-  }
+/** Convenience: wrap an async function in a span. */
+export async function withSpan(name, attributes, fn) {
+    const span = startSpan(name, attributes);
+    try {
+        const result = await fn(span);
+        endSpan(span);
+        return result;
+    }
+    catch (err) {
+        endSpan(span, {}, err);
+        throw err;
+    }
 }
-const counters = /* @__PURE__ */ new Map();
-function increment(metric, value = 1, tags = {}) {
-  const key = metric + (Object.keys(tags).length ? ":" + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(",") : "");
-  counters.set(key, (counters.get(key) ?? 0) + value);
+/** Counter metrics — increment and log on flush. */
+const counters = new Map();
+export function increment(metric, value = 1, tags = {}) {
+    const key = metric + (Object.keys(tags).length ? ':' + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(',') : '');
+    counters.set(key, (counters.get(key) ?? 0) + value);
 }
-function flushMetrics() {
-  const snapshot = Object.fromEntries(counters);
-  if (Object.keys(snapshot).length > 0) {
-    console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "metrics:flush", ...snapshot }));
-    counters.clear();
-  }
-  return snapshot;
+/** Flush counters as a structured log — call periodically (e.g. at job end). */
+export function flushMetrics() {
+    const snapshot = Object.fromEntries(counters);
+    if (Object.keys(snapshot).length > 0) {
+        console.log(JSON.stringify({ ts: new Date().toISOString(), level: 'info', msg: 'metrics:flush', ...snapshot }));
+        counters.clear();
+    }
+    return snapshot;
 }
-export {
-  endSpan,
-  flushMetrics,
-  increment,
-  startSpan,
-  withSpan
-};
-//# sourceMappingURL=telemetry.js.map