@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/state-graph.js

@@ -0,0 +1,193 @@
+function fnv1a32(token) {
+  let h = 2166136261;
+  for (let i = 0; i < token.length; i++) {
+    h = Math.imul(h ^ token.charCodeAt(i), 16777619);
+    h = h >>> 0;
+  }
+  return h;
+}
+function simHash32(text) {
+  const tokens = text.split(/\W+/).filter((t) => t.length > 2).slice(0, 500);
+  const vec = new Array(32).fill(0);
+  for (const token of tokens) {
+    const h = fnv1a32(token);
+    for (let b = 0; b < 32; b++) {
+      vec[b] += h >>> b & 1 ? 1 : -1;
+    }
+  }
+  let result = 0;
+  for (let b = 0; b < 32; b++) {
+    if (vec[b] > 0) result |= 1 << b;
+  }
+  return result >>> 0;
+}
+function hammingDistance32(a, b) {
+  let n = (a ^ b) >>> 0;
+  n = n - (n >>> 1 & 1431655765);
+  n = (n & 858993459) + (n >>> 2 & 858993459);
+  n = n + (n >>> 4) & 252645135;
+  return Math.imul(n, 16843009) >>> 24;
+}
+const STRIP_PARAMS = /* @__PURE__ */ new Set([
+  "utm_source",
+  "utm_medium",
+  "utm_campaign",
+  "ref",
+  "_ga",
+  "fbclid",
+  "session_id"
+]);
+function normalizeUrl(raw) {
+  try {
+    const u = new URL(raw);
+    u.hash = "";
+    const keep = [];
+    u.searchParams.forEach((v, k) => {
+      if (!STRIP_PARAMS.has(k)) keep.push([k, v]);
+    });
+    keep.sort((a, b) => a[0].localeCompare(b[0]));
+    u.search = keep.length ? "?" + keep.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&") : "";
+    return u.toString();
+  } catch {
+    return raw;
+  }
+}
+function computeStateId(normalizedUrl, hash) {
+  let path = normalizedUrl.slice(0, 40).replace(/[^a-zA-Z0-9]/g, "_");
+  const hex = hash.toString(16).padStart(8, "0");
+  return `${path}_${hex}`;
+}
+async function fingerprintPage(page, visitedAt) {
+  let visibleText = "";
+  let interactiveCount = 0;
+  try {
+    visibleText = await page.evaluate(() => {
+      const el = document.querySelector("main") ?? document.body;
+      return el.innerText ?? "";
+    });
+  } catch {
+    visibleText = "";
+  }
+  try {
+    interactiveCount = await page.evaluate(
+      () => document.querySelectorAll(
+        'a[href], button, input, select, textarea, [role="button"], [role="link"], [tabindex]'
+      ).length
+    );
+  } catch {
+    interactiveCount = 0;
+  }
+  const simHashValue = simHash32(visibleText + visitedAt.slice(0, 10));
+  const fingerprint = simHashValue.toString(16).padStart(8, "0");
+  return { fingerprint, visibleText, interactiveCount, simHashValue };
+}
+class ScreenTransitionGraph {
+  states = /* @__PURE__ */ new Map();
+  transitions = [];
+  /** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
+  hashIndex = /* @__PURE__ */ new Map();
+  // -------------------------------------------------------------------------
+  // State management
+  // -------------------------------------------------------------------------
+  addState(node) {
+    if (this.states.has(node.id)) {
+      return { added: false, duplicateOf: node.id };
+    }
+    for (const [indexedHash, ids] of this.hashIndex) {
+      if (hammingDistance32(node.simHashValue, indexedHash) < 3) {
+        return { added: false, duplicateOf: ids[0] };
+      }
+    }
+    this.states.set(node.id, node);
+    const bucket = this.hashIndex.get(node.simHashValue);
+    if (bucket) {
+      bucket.push(node.id);
+    } else {
+      this.hashIndex.set(node.simHashValue, [node.id]);
+    }
+    return { added: true };
+  }
+  addTransition(fromId, toId, action, label, triggeredAt) {
+    this.transitions.push({ fromId, toId, action, label, triggeredAt });
+  }
+  isDuplicate(url, visibleText, threshold = 3) {
+    const normalized = normalizeUrl(url);
+    const hash = simHash32(visibleText);
+    for (const [indexedHash] of this.hashIndex) {
+      if (hammingDistance32(hash, indexedHash) < threshold) return true;
+    }
+    for (const node of this.states.values()) {
+      if (node.normalizedUrl === normalized) return true;
+    }
+    return false;
+  }
+  // -------------------------------------------------------------------------
+  // Reporting
+  // -------------------------------------------------------------------------
+  getCoverageReport() {
+    const totalStates = this.states.size;
+    const totalTransitions = this.transitions.length;
+    const uniqueUrls = new Set(
+      Array.from(this.states.values()).map((n) => n.normalizedUrl)
+    ).size;
+    const interactiveElementsTotal = Array.from(this.states.values()).reduce(
+      (sum, n) => sum + n.interactiveElementCount,
+      0
+    );
+    const estimatedCoveragePercent = Math.min(
+      100,
+      totalTransitions / Math.max(1, totalStates) * 50
+    );
+    return {
+      totalStates,
+      totalTransitions,
+      uniqueUrls,
+      interactiveElementsTotal,
+      estimatedCoveragePercent
+    };
+  }
+  // -------------------------------------------------------------------------
+  // Serialisation
+  // -------------------------------------------------------------------------
+  toJSON() {
+    return {
+      states: Array.from(this.states.values()),
+      transitions: this.transitions
+    };
+  }
+  static fromJSON(data) {
+    const graph = new ScreenTransitionGraph();
+    for (const node of data.states) {
+      graph.states.set(node.id, node);
+      const bucket = graph.hashIndex.get(node.simHashValue);
+      if (bucket) {
+        bucket.push(node.id);
+      } else {
+        graph.hashIndex.set(node.simHashValue, [node.id]);
+      }
+    }
+    graph.transitions = [...data.transitions];
+    return graph;
+  }
+  // -------------------------------------------------------------------------
+  // Accessors
+  // -------------------------------------------------------------------------
+  getState(id) {
+    return this.states.get(id);
+  }
+  getAllStates() {
+    return Array.from(this.states.values());
+  }
+  getTransitionsFrom(id) {
+    return this.transitions.filter((t) => t.fromId === id);
+  }
+}
+export {
+  ScreenTransitionGraph,
+  computeStateId,
+  fingerprintPage,
+  hammingDistance32,
+  normalizeUrl,
+  simHash32
+};
+//# sourceMappingURL=state-graph.js.map

package/dist/state-graph.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/state-graph.ts"],
+  "sourcesContent": ["import type { Page } from 'playwright';\n\n// ---------------------------------------------------------------------------\n// Internal: FNV-1a 32-bit SimHash\n// ---------------------------------------------------------------------------\n\nfunction fnv1a32(token: string): number {\n  let h = 2166136261;\n  for (let i = 0; i < token.length; i++) {\n    h = Math.imul(h ^ token.charCodeAt(i), 16777619);\n    h = h >>> 0;\n  }\n  return h;\n}\n\nexport function simHash32(text: string): number {\n  const tokens = text\n    .split(/\\W+/)\n    .filter((t) => t.length > 2)\n    .slice(0, 500);\n\n  const vec = new Array<number>(32).fill(0);\n\n  for (const token of tokens) {\n    const h = fnv1a32(token);\n    for (let b = 0; b < 32; b++) {\n      vec[b] += (h >>> b) & 1 ? 1 : -1;\n    }\n  }\n\n  let result = 0;\n  for (let b = 0; b < 32; b++) {\n    if (vec[b] > 0) result |= 1 << b;\n  }\n  return result >>> 0;\n}\n\nexport function hammingDistance32(a: number, b: number): number {\n  let n = (a ^ b) >>> 0;\n  n = n - ((n >>> 1) & 0x55555555);\n  n = (n & 0x33333333) + ((n >>> 2) & 0x33333333);\n  n = (n + (n >>> 4)) & 0x0f0f0f0f;\n  return Math.imul(n, 0x01010101) >>> 24;\n}\n\n// ---------------------------------------------------------------------------\n// URL normalisation\n// ---------------------------------------------------------------------------\n\nconst STRIP_PARAMS = new Set([\n  'utm_source', 'utm_medium', 'utm_campaign',\n  'ref', '_ga', 'fbclid', 'session_id',\n]);\n\nexport function normalizeUrl(raw: string): string {\n  try {\n    const u = new URL(raw);\n    u.hash = '';\n    const keep: Array<[string, string]> = [];\n    u.searchParams.forEach((v, k) => {\n      if (!STRIP_PARAMS.has(k)) keep.push([k, v]);\n    });\n    keep.sort((a, b) => a[0].localeCompare(b[0]));\n    u.search = keep.length\n      ? '?' + keep.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')\n      : '';\n    return u.toString();\n  } catch {\n    return raw;\n  }\n}\n\nexport function computeStateId(normalizedUrl: string, hash: number): string {\n  let path = normalizedUrl.slice(0, 40).replace(/[^a-zA-Z0-9]/g, '_');\n  const hex = hash.toString(16).padStart(8, '0');\n  return `${path}_${hex}`;\n}\n\n// ---------------------------------------------------------------------------\n// Public types\n// ---------------------------------------------------------------------------\n\nexport interface UIStateNode {\n  id: string;\n  url: string;\n  normalizedUrl: string;\n  title: string;\n  interactiveElementCount: number;\n  domFingerprint: string;\n  simHashValue: number;\n  visitedAt: string;\n}\n\nexport interface StateTransition {\n  fromId: string;\n  toId: string;\n  action: string;\n  label: string;\n  triggeredAt: string;\n}\n\nexport interface CoverageReport {\n  totalStates: number;\n  totalTransitions: number;\n  uniqueUrls: number;\n  interactiveElementsTotal: number;\n  estimatedCoveragePercent: number;\n}\n\n// ---------------------------------------------------------------------------\n// Page fingerprinting\n// ---------------------------------------------------------------------------\n\nexport async function fingerprintPage(\n  page: Page,\n  visitedAt: string,\n): Promise<{\n  fingerprint: string;\n  visibleText: string;\n  interactiveCount: number;\n  simHashValue: number;\n}> {\n  let visibleText = '';\n  let interactiveCount = 0;\n\n  try {\n    visibleText = await page.evaluate(() => {\n      const el = document.querySelector('main') ?? document.body;\n      return (el as HTMLElement).innerText ?? '';\n    });\n  } catch {\n    visibleText = '';\n  }\n\n  try {\n    interactiveCount = await page.evaluate(() =>\n      document.querySelectorAll(\n        'a[href], button, input, select, textarea, [role=\"button\"], [role=\"link\"], [tabindex]',\n      ).length,\n    );\n  } catch {\n    interactiveCount = 0;\n  }\n\n  const simHashValue = simHash32(visibleText + visitedAt.slice(0, 10));\n  const fingerprint = simHashValue.toString(16).padStart(8, '0');\n\n  return { fingerprint, visibleText, interactiveCount, simHashValue };\n}\n\n// ---------------------------------------------------------------------------\n// ScreenTransitionGraph\n// ---------------------------------------------------------------------------\n\nexport class ScreenTransitionGraph {\n  private states = new Map<string, UIStateNode>();\n  private transitions: StateTransition[] = [];\n  /** Maps simHash value \u2192 array of state ids with that hash (for near-dup lookup) */\n  private hashIndex = new Map<number, string[]>();\n\n  // -------------------------------------------------------------------------\n  // State management\n  // -------------------------------------------------------------------------\n\n  addState(node: UIStateNode): { added: boolean; duplicateOf?: string } {\n    if (this.states.has(node.id)) {\n      return { added: false, duplicateOf: node.id };\n    }\n\n    // Check near-duplicate via hamming distance across all indexed hashes\n    for (const [indexedHash, ids] of this.hashIndex) {\n      if (hammingDistance32(node.simHashValue, indexedHash) < 3) {\n        return { added: false, duplicateOf: ids[0] };\n      }\n    }\n\n    this.states.set(node.id, node);\n\n    const bucket = this.hashIndex.get(node.simHashValue);\n    if (bucket) {\n      bucket.push(node.id);\n    } else {\n      this.hashIndex.set(node.simHashValue, [node.id]);\n    }\n\n    return { added: true };\n  }\n\n  addTransition(\n    fromId: string,\n    toId: string,\n    action: string,\n    label: string,\n    triggeredAt: string,\n  ): void {\n    this.transitions.push({ fromId, toId, action, label, triggeredAt });\n  }\n\n  isDuplicate(url: string, visibleText: string, threshold = 3): boolean {\n    const normalized = normalizeUrl(url);\n    const hash = simHash32(visibleText);\n    for (const [indexedHash] of this.hashIndex) {\n      if (hammingDistance32(hash, indexedHash) < threshold) return true;\n    }\n    // Also check exact normalized URL match\n    for (const node of this.states.values()) {\n      if (node.normalizedUrl === normalized) return true;\n    }\n    return false;\n  }\n\n  // -------------------------------------------------------------------------\n  // Reporting\n  // -------------------------------------------------------------------------\n\n  getCoverageReport(): CoverageReport {\n    const totalStates = this.states.size;\n    const totalTransitions = this.transitions.length;\n    const uniqueUrls = new Set(\n      Array.from(this.states.values()).map((n) => n.normalizedUrl),\n    ).size;\n    const interactiveElementsTotal = Array.from(this.states.values()).reduce(\n      (sum, n) => sum + n.interactiveElementCount,\n      0,\n    );\n    const estimatedCoveragePercent = Math.min(\n      100,\n      (totalTransitions / Math.max(1, totalStates)) * 50,\n    );\n    return {\n      totalStates,\n      totalTransitions,\n      uniqueUrls,\n      interactiveElementsTotal,\n      estimatedCoveragePercent,\n    };\n  }\n\n  // -------------------------------------------------------------------------\n  // Serialisation\n  // -------------------------------------------------------------------------\n\n  toJSON(): { states: UIStateNode[]; transitions: StateTransition[] } {\n    return {\n      states: Array.from(this.states.values()),\n      transitions: this.transitions,\n    };\n  }\n\n  static fromJSON(data: { states: UIStateNode[]; transitions: StateTransition[] }): ScreenTransitionGraph {\n    const graph = new ScreenTransitionGraph();\n    for (const node of data.states) {\n      graph.states.set(node.id, node);\n      const bucket = graph.hashIndex.get(node.simHashValue);\n      if (bucket) {\n        bucket.push(node.id);\n      } else {\n        graph.hashIndex.set(node.simHashValue, [node.id]);\n      }\n    }\n    graph.transitions = [...data.transitions];\n    return graph;\n  }\n\n  // -------------------------------------------------------------------------\n  // Accessors\n  // -------------------------------------------------------------------------\n\n  getState(id: string): UIStateNode | undefined {\n    return this.states.get(id);\n  }\n\n  getAllStates(): UIStateNode[] {\n    return Array.from(this.states.values());\n  }\n\n  getTransitionsFrom(id: string): StateTransition[] {\n    return this.transitions.filter((t) => t.fromId === id);\n  }\n}\n"],
+  "mappings": "AAMA,SAAS,QAAQ,OAAuB;AACtC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,QAAI,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC,GAAG,QAAQ;AAC/C,QAAI,MAAM;AAAA,EACZ;AACA,SAAO;AACT;AAEO,SAAS,UAAU,MAAsB;AAC9C,QAAM,SAAS,KACZ,MAAM,KAAK,EACX,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAC1B,MAAM,GAAG,GAAG;AAEf,QAAM,MAAM,IAAI,MAAc,EAAE,EAAE,KAAK,CAAC;AAExC,aAAW,SAAS,QAAQ;AAC1B,UAAM,IAAI,QAAQ,KAAK;AACvB,aAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC3B,UAAI,CAAC,KAAM,MAAM,IAAK,IAAI,IAAI;AAAA,IAChC;AAAA,EACF;AAEA,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC3B,QAAI,IAAI,CAAC,IAAI,EAAG,WAAU,KAAK;AAAA,EACjC;AACA,SAAO,WAAW;AACpB;AAEO,SAAS,kBAAkB,GAAW,GAAmB;AAC9D,MAAI,KAAK,IAAI,OAAO;AACpB,MAAI,KAAM,MAAM,IAAK;AACrB,OAAK,IAAI,cAAgB,MAAM,IAAK;AACpC,MAAK,KAAK,MAAM,KAAM;AACtB,SAAO,KAAK,KAAK,GAAG,QAAU,MAAM;AACtC;AAMA,MAAM,eAAe,oBAAI,IAAI;AAAA,EAC3B;AAAA,EAAc;AAAA,EAAc;AAAA,EAC5B;AAAA,EAAO;AAAA,EAAO;AAAA,EAAU;AAC1B,CAAC;AAEM,SAAS,aAAa,KAAqB;AAChD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AACT,UAAM,OAAgC,CAAC;AACvC,MAAE,aAAa,QAAQ,CAAC,GAAG,MAAM;AAC/B,UAAI,CAAC,aAAa,IAAI,CAAC,EAAG,MAAK,KAAK,CAAC,GAAG,CAAC,CAAC;AAAA,IAC5C,CAAC;AACD,SAAK,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;AAC5C,MAAE,SAAS,KAAK,SACZ,MAAM,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,mBAAmB,CAAC,CAAC,IAAI,mBAAmB,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,IACxF;AACJ,WAAO,EAAE,SAAS;AAAA,EACpB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,eAAe,eAAuB,MAAsB;AAC1E,MAAI,OAAO,cAAc,MAAM,GAAG,EAAE,EAAE,QAAQ,iBAAiB,GAAG;AAClE,QAAM,MAAM,KAAK,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAC7C,SAAO,GAAG,IAAI,IAAI,GAAG;AACvB;AAqCA,eAAsB,gBACpB,MACA,WAMC;AACD,MAAI,cAAc;AAClB,MAAI,mBAAmB;AAEvB,MAAI;AACF,kBAAc,MAAM,KAAK,SAAS,MAAM;AACtC,YAAM,KAAK,SAAS,cAAc,MAAM,KAAK,SAAS;AACtD,aAAQ,GAAmB,aAAa;AAAA,IAC1C,CAAC;AAAA,EACH,QAAQ;AACN,kBAAc;AAAA,EAChB;AAEA,MAAI;AACF,uBAAmB,MAAM,KAAK;AAAA,MAAS,MACrC,SAAS;AAAA,QACP;AAAA,MACF,EAAE;AAAA,IACJ;AAAA,EACF,QAAQ;AACN,uBAAmB;AAAA,EACrB;AAEA,QAAM,eAAe,UAAU,cAAc,UAAU,MAAM,GAAG,EAAE,CAAC;AACnE,QAAM,cAAc,aAAa,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAE7D,SAAO,EAAE,aAAa,aAAa,kBAAkB,aAAa;AACpE;AAMO,MAAM,sBAAsB;AAAA,EACzB,SAAS,oBAAI,IAAyB;AAAA,EACtC,cAAiC,CAAC;AAAA;AAAA,EAElC,YAAY,oBAAI,IAAsB;AAAA;AAAA;AAAA;AAAA,EAM9C,SAAS,MAA6D;AACpE,QAAI,KAAK,OAAO,IAAI,KAAK,EAAE,GAAG;AAC5B,aAAO,EAAE,OAAO,OAAO,aAAa,KAAK,GAAG;AAAA,IAC9C;AAGA,eAAW,CAAC,aAAa,GAAG,KAAK,KAAK,WAAW;AAC/C,UAAI,kBAAkB,KAAK,cAAc,WAAW,IAAI,GAAG;AACzD,eAAO,EAAE,OAAO,OAAO,aAAa,IAAI,CAAC,EAAE;AAAA,MAC7C;AAAA,IACF;AAEA,SAAK,OAAO,IAAI,KAAK,IAAI,IAAI;AAE7B,UAAM,SAAS,KAAK,UAAU,IAAI,KAAK,YAAY;AACnD,QAAI,QAAQ;AACV,aAAO,KAAK,KAAK,EAAE;AAAA,IACrB,OAAO;AACL,WAAK,UAAU,IAAI,KAAK,cAAc,CAAC,KAAK,EAAE,CAAC;AAAA,IACjD;AAEA,WAAO,EAAE,OAAO,KAAK;AAAA,EACvB;AAAA,EAEA,cACE,QACA,MACA,QACA,OACA,aACM;AACN,SAAK,YAAY,KAAK,EAAE,QAAQ,MAAM,QAAQ,OAAO,YAAY,CAAC;AAAA,EACpE;AAAA,EAEA,YAAY,KAAa,aAAqB,YAAY,GAAY;AACpE,UAAM,aAAa,aAAa,GAAG;AACnC,UAAM,OAAO,UAAU,WAAW;AAClC,eAAW,CAAC,WAAW,KAAK,KAAK,WAAW;AAC1C,UAAI,kBAAkB,MAAM,WAAW,IAAI,UAAW,QAAO;AAAA,IAC/D;AAEA,eAAW,QAAQ,KAAK,OAAO,OAAO,GAAG;AACvC,UAAI,KAAK,kBAAkB,WAAY,QAAO;AAAA,IAChD;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,oBAAoC;AAClC,UAAM,cAAc,KAAK,OAAO;AAChC,UAAM,mBAAmB,KAAK,YAAY;AAC1C,UAAM,aAAa,IAAI;AAAA,MACrB,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa;AAAA,IAC7D,EAAE;AACF,UAAM,2BAA2B,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC,EAAE;AAAA,MAChE,CAAC,KAAK,MAAM,MAAM,EAAE;AAAA,MACpB;AAAA,IACF;AACA,UAAM,2BAA2B,KAAK;AAAA,MACpC;AAAA,MACC,mBAAmB,KAAK,IAAI,GAAG,WAAW,IAAK;AAAA,IAClD;AACA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,SAAoE;AAClE,WAAO;AAAA,MACL,QAAQ,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC;AAAA,MACvC,aAAa,KAAK;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,OAAO,SAAS,MAAwF;AACtG,UAAM,QAAQ,IAAI,sBAAsB;AACxC,eAAW,QAAQ,KAAK,QAAQ;AAC9B,YAAM,OAAO,IAAI,KAAK,IAAI,IAAI;AAC9B,YAAM,SAAS,MAAM,UAAU,IAAI,KAAK,YAAY;AACpD,UAAI,QAAQ;AACV,eAAO,KAAK,KAAK,EAAE;AAAA,MACrB,OAAO;AACL,cAAM,UAAU,IAAI,KAAK,cAAc,CAAC,KAAK,EAAE,CAAC;AAAA,MAClD;AAAA,IACF;AACA,UAAM,cAAc,CAAC,GAAG,KAAK,WAAW;AACxC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,IAAqC;AAC5C,WAAO,KAAK,OAAO,IAAI,EAAE;AAAA,EAC3B;AAAA,EAEA,eAA8B;AAC5B,WAAO,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC;AAAA,EACxC;AAAA,EAEA,mBAAmB,IAA+B;AAChD,WAAO,KAAK,YAAY,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE;AAAA,EACvD;AACF;",
+  "names": []
+}

package/dist/stealth/human-behavior.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright';
+export declare function humanDelay(min?: number, max?: number): Promise<void>;
+export declare function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void>;
+export declare function simulateMouseHover(page: Page, selector: string): Promise<void>;
+export declare function simulateClick(page: Page, selector: string): Promise<void>;
+export declare function injectFingerprintOverrides(page: Page): Promise<void>;
+export declare function infiniteScrollToBottom(page: Page, opts?: {
+    maxScrolls?: number;
+    scrollDelay?: number;
+    stabilizeMs?: number;
+}, rawPage?: any): Promise<{
+    scrollCount: number;
+    heightReached: number;
+}>;
+export declare function extractIframeContent(page: Page): Promise<Array<{
+    src: string;
+    content: string;
+    markdown: string;
+}>>;
+export declare function pierceShadowDom(page: Page): Promise<string>;

package/dist/stealth/human-behavior.js

@@ -0,0 +1,202 @@
+function gaussian(mean, sigma) {
+  const u1 = Math.random();
+  const u2 = Math.random();
+  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+  return mean + z * sigma;
+}
+async function humanDelay(min = 80, max = 450) {
+  const mean = (min + max) / 2;
+  const sigma = (max - min) / 6;
+  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
+  await new Promise((r) => setTimeout(r, Math.round(delay)));
+}
+async function simulateMouseMovement(page, targetX, targetY) {
+  try {
+    const mouse = page.mouse;
+    const steps = 8 + Math.floor(Math.random() * 8);
+    const startX = targetX - 100 + Math.random() * 200;
+    const startY = targetY - 100 + Math.random() * 200;
+    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
+    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
+    for (let i = 0; i <= steps; i++) {
+      const t = i / steps;
+      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
+      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
+      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
+      if (i < steps) await humanDelay(8, 25);
+    }
+    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
+    await humanDelay(30, 80);
+    await mouse.move(targetX, targetY);
+  } catch {
+  }
+}
+async function simulateMouseHover(page, selector) {
+  try {
+    const el = await page.$(selector);
+    if (!el) return;
+    const box = await el.boundingBox();
+    if (!box) return;
+    const cx = box.x + box.width / 2;
+    const cy = box.y + box.height / 2;
+    await simulateMouseMovement(page, cx, cy);
+    await humanDelay(100, 300);
+  } catch {
+  }
+}
+async function simulateClick(page, selector) {
+  try {
+    await simulateMouseHover(page, selector);
+    await humanDelay(50, 150);
+    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
+  } catch {
+  }
+}
+async function injectFingerprintOverrides(page) {
+  await page.addInitScript(() => {
+    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
+    HTMLCanvasElement.prototype.toDataURL = function(...args) {
+      const ctx = this.getContext("2d");
+      if (ctx) {
+        const imageData = ctx.getImageData(0, 0, this.width, this.height);
+        for (let i = 0; i < imageData.data.length; i += 4) {
+          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? Math.random() > 0.5 ? 1 : -1 : 0)));
+        }
+        ctx.putImageData(imageData, 0, 0);
+      }
+      return origToDataURL.apply(this, args);
+    };
+    const origGetParam = WebGLRenderingContext.prototype.getParameter;
+    WebGLRenderingContext.prototype.getParameter = function(parameter) {
+      const UNMASKED_VENDOR_WEBGL = 37445;
+      const UNMASKED_RENDERER_WEBGL = 37446;
+      if (parameter === UNMASKED_VENDOR_WEBGL) return "Intel Inc.";
+      if (parameter === UNMASKED_RENDERER_WEBGL) return "Intel Iris OpenGL Engine";
+      return origGetParam.call(this, parameter);
+    };
+    const cpuCounts = [4, 6, 8, 8];
+    Object.defineProperty(navigator, "hardwareConcurrency", {
+      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
+      configurable: true
+    });
+    Object.defineProperty(navigator, "deviceMemory", {
+      get: () => [4, 8][Math.floor(Math.random() * 2)],
+      configurable: true
+    });
+  }).catch(() => {
+  });
+}
+async function infiniteScrollToBottom(page, opts = {}, rawPage) {
+  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
+  let scrollCount = 0;
+  let stableScrollCount = 0;
+  let previousHeight = 0;
+  let cdpForWheel = null;
+  let viewW = 1280;
+  let viewH = 800;
+  if (rawPage && typeof rawPage.context === "function") {
+    try {
+      const ctx = rawPage.context();
+      if (typeof ctx?.newCDPSession === "function") {
+        cdpForWheel = await ctx.newCDPSession(rawPage);
+        const { result } = await cdpForWheel.send("Runtime.evaluate", {
+          expression: `({"w":window.innerWidth,"h":window.innerHeight})`,
+          returnByValue: true
+        }).catch(() => ({ result: { value: null } }));
+        if (result?.value) {
+          viewW = result.value.w ?? 1280;
+          viewH = result.value.h ?? 800;
+        }
+      }
+    } catch {
+      cdpForWheel = null;
+    }
+  }
+  try {
+    while (scrollCount < maxScrolls) {
+      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+      const scrollBy = 500 + Math.floor(Math.random() * 300);
+      await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => {
+      });
+      if (cdpForWheel) {
+        await cdpForWheel.send("Input.dispatchMouseEvent", {
+          type: "mouseWheel",
+          x: Math.floor(viewW / 2),
+          y: Math.floor(viewH / 2),
+          deltaX: 0,
+          deltaY: scrollBy,
+          modifiers: 0
+        }).catch(() => {
+        });
+      }
+      await humanDelay(scrollDelay - 100, scrollDelay + 200);
+      await new Promise((r) => setTimeout(r, stabilizeMs));
+      scrollCount++;
+      const newHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+      if (newHeight - previousHeight < 10) {
+        stableScrollCount++;
+        if (stableScrollCount >= 3) break;
+      } else {
+        stableScrollCount = 0;
+      }
+      previousHeight = newHeight;
+    }
+  } catch {
+  }
+  if (cdpForWheel) cdpForWheel.detach().catch(() => {
+  });
+  return { scrollCount, heightReached: previousHeight };
+}
+async function extractIframeContent(page) {
+  const results = [];
+  try {
+    const iframeSrcs = await page.evaluate(
+      () => Array.from(document.querySelectorAll("iframe")).map((f) => f.getAttribute("src") ?? f.getAttribute("data-src") ?? "")
+    ).catch(() => []);
+    for (const frame of page.frames()) {
+      const src = frame.url();
+      if (!src || src === "about:blank" || src === page.url()) continue;
+      try {
+        const content = await frame.content().catch(() => "");
+        if (!content || content.length < 50) continue;
+        const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+        results.push({ src, content: content.slice(0, 1e4), markdown: text.slice(0, 5e3) });
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return results;
+}
+async function pierceShadowDom(page) {
+  try {
+    return await page.evaluate(() => {
+      const parts = [];
+      function walkShadow(root) {
+        const shadow = root.shadowRoot;
+        if (shadow) {
+          parts.push(shadow.innerHTML ?? "");
+          for (const child of [...shadow.querySelectorAll("*")]) walkShadow(child);
+        }
+        for (const child of Array.from(root.querySelectorAll(":not(script):not(style)"))) {
+          if (child.shadowRoot) walkShadow(child);
+        }
+      }
+      walkShadow(document.body);
+      return parts.join("\n").replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 1e4);
+    }).catch(() => "");
+  } catch {
+    return "";
+  }
+}
+export {
+  extractIframeContent,
+  humanDelay,
+  infiniteScrollToBottom,
+  injectFingerprintOverrides,
+  pierceShadowDom,
+  simulateClick,
+  simulateMouseHover,
+  simulateMouseMovement
+};
+//# sourceMappingURL=human-behavior.js.map

package/dist/stealth/human-behavior.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/stealth/human-behavior.ts"],
+  "sourcesContent": ["import type { Page } from 'playwright';\n\n// Gaussian random using Box-Muller transform\nfunction gaussian(mean: number, sigma: number): number {\n  const u1 = Math.random();\n  const u2 = Math.random();\n  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);\n  return mean + z * sigma;\n}\n\nexport async function humanDelay(min = 80, max = 450): Promise<void> {\n  const mean = (min + max) / 2;\n  const sigma = (max - min) / 6;\n  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));\n  await new Promise(r => setTimeout(r, Math.round(delay)));\n}\n\nexport async function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void> {\n  try {\n    const mouse = page.mouse;\n    const steps = 8 + Math.floor(Math.random() * 8);\n    // Generate Bezier control points with slight perpendicular offset\n    const startX = targetX - 100 + Math.random() * 200;\n    const startY = targetY - 100 + Math.random() * 200;\n    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;\n    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;\n\n    for (let i = 0; i <= steps; i++) {\n      const t = i / steps;\n      // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2\n      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;\n      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;\n      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);\n      if (i < steps) await humanDelay(8, 25);\n    }\n    // Small overshoot correction\n    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);\n    await humanDelay(30, 80);\n    await mouse.move(targetX, targetY);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateMouseHover(page: Page, selector: string): Promise<void> {\n  try {\n    const el = await page.$(selector);\n    if (!el) return;\n    const box = await el.boundingBox();\n    if (!box) return;\n    const cx = box.x + box.width / 2;\n    const cy = box.y + box.height / 2;\n    await simulateMouseMovement(page, cx, cy);\n    await humanDelay(100, 300);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateClick(page: Page, selector: string): Promise<void> {\n  try {\n    await simulateMouseHover(page, selector);\n    await humanDelay(50, 150);\n    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });\n  } catch { /* non-fatal */ }\n}\n\nexport async function injectFingerprintOverrides(page: Page): Promise<void> {\n  await page.addInitScript(() => {\n    // Canvas noise \u2014 subtle per-pixel perturbation\n    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;\n    HTMLCanvasElement.prototype.toDataURL = function(this: HTMLCanvasElement, ...args: any[]) {\n      const ctx = this.getContext('2d');\n      if (ctx) {\n        const imageData = ctx.getImageData(0, 0, this.width, this.height);\n        for (let i = 0; i < imageData.data.length; i += 4) {\n          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));\n        }\n        ctx.putImageData(imageData, 0, 0);\n      }\n      return origToDataURL.apply(this, args as [type?: string, quality?: number]);\n    };\n\n    // WebGL parameter spoofing\n    const origGetParam = WebGLRenderingContext.prototype.getParameter;\n    WebGLRenderingContext.prototype.getParameter = function(this: WebGLRenderingContext, parameter: number) {\n      const UNMASKED_VENDOR_WEBGL = 0x9245;\n      const UNMASKED_RENDERER_WEBGL = 0x9246;\n      if (parameter === UNMASKED_VENDOR_WEBGL) return 'Intel Inc.';\n      if (parameter === UNMASKED_RENDERER_WEBGL) return 'Intel Iris OpenGL Engine';\n      return origGetParam.call(this, parameter);\n    };\n\n    // Hardware concurrency \u2014 pick from common values\n    const cpuCounts = [4, 6, 8, 8];\n    Object.defineProperty(navigator, 'hardwareConcurrency', {\n      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],\n      configurable: true,\n    });\n\n    // Device memory\n    Object.defineProperty(navigator, 'deviceMemory', {\n      get: () => [4, 8][Math.floor(Math.random() * 2)],\n      configurable: true,\n    });\n  }).catch(() => {});\n}\n\nexport async function infiniteScrollToBottom(\n  page: Page,\n  opts: { maxScrolls?: number; scrollDelay?: number; stabilizeMs?: number } = {},\n  rawPage?: any,\n): Promise<{ scrollCount: number; heightReached: number }> {\n  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;\n  let scrollCount = 0;\n  let stableScrollCount = 0;\n  let previousHeight = 0;\n\n  // One CDP session for wheel events (triggers IntersectionObserver-based virtual scrollers).\n  // React-window / react-virtualized / TanStack Virtual ignore window.scrollBy but respond to\n  // wheel events because they attach their own scroll listeners to the list container.\n  let cdpForWheel: any = null;\n  let viewW = 1280;\n  let viewH = 800;\n  if (rawPage && typeof rawPage.context === 'function') {\n    try {\n      const ctx = rawPage.context();\n      if (typeof ctx?.newCDPSession === 'function') {\n        cdpForWheel = await ctx.newCDPSession(rawPage);\n        const { result } = await cdpForWheel.send('Runtime.evaluate', {\n          expression: `({\"w\":window.innerWidth,\"h\":window.innerHeight})`,\n          returnByValue: true,\n        }).catch(() => ({ result: { value: null } }));\n        if (result?.value) { viewW = result.value.w ?? 1280; viewH = result.value.h ?? 800; }\n      }\n    } catch { cdpForWheel = null; }\n  }\n\n  try {\n    while (scrollCount < maxScrolls) {\n      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);\n\n      const scrollBy = 500 + Math.floor(Math.random() * 300);\n      // window.scrollBy handles standard scroll containers\n      await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});\n      // CDP wheel event triggers IntersectionObserver + virtual list scroll listeners\n      if (cdpForWheel) {\n        await cdpForWheel.send('Input.dispatchMouseEvent', {\n          type: 'mouseWheel',\n          x: Math.floor(viewW / 2),\n          y: Math.floor(viewH / 2),\n          deltaX: 0,\n          deltaY: scrollBy,\n          modifiers: 0,\n        }).catch(() => {});\n      }\n      await humanDelay(scrollDelay - 100, scrollDelay + 200);\n\n      // Use Node.js setTimeout \u2014 page.waitForTimeout may not be available on Stagehand proxy\n      await new Promise<void>(r => setTimeout(r, stabilizeMs));\n      scrollCount++;\n\n      const newHeight: number = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);\n      if (newHeight - previousHeight < 10) {\n        stableScrollCount++;\n        if (stableScrollCount >= 3) break; // no new content after 3 consecutive non-growing scrolls\n      } else {\n        stableScrollCount = 0;\n      }\n      previousHeight = newHeight;\n    }\n  } catch { /* non-fatal */ }\n\n  if (cdpForWheel) cdpForWheel.detach().catch(() => {});\n  return { scrollCount, heightReached: previousHeight };\n}\n\nexport async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {\n  const results: Array<{ src: string; content: string; markdown: string }> = [];\n  try {\n    const iframeSrcs: string[] = await page.evaluate(() =>\n      Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')\n    ).catch(() => []);\n\n    for (const frame of page.frames()) {\n      const src = frame.url();\n      if (!src || src === 'about:blank' || src === page.url()) continue;\n      try {\n        const content = await frame.content().catch(() => '');\n        if (!content || content.length < 50) continue;\n        // Simple text extraction from iframe HTML\n        const text = content.replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n          .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n          .replace(/<[^>]+>/g, ' ')\n          .replace(/\\s+/g, ' ')\n          .trim();\n        results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });\n      } catch { /* skip this frame */ }\n    }\n  } catch { /* non-fatal */ }\n  return results;\n}\n\nexport async function pierceShadowDom(page: Page): Promise<string> {\n  try {\n    return await page.evaluate(() => {\n      const parts: string[] = [];\n      function walkShadow(root: Element | ShadowRoot): void {\n        const shadow = (root as any).shadowRoot;\n        if (shadow) {\n          parts.push(shadow.innerHTML ?? '');\n          for (const child of [...shadow.querySelectorAll('*')]) walkShadow(child);\n        }\n        for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {\n          if ((child as any).shadowRoot) walkShadow(child);\n        }\n      }\n      walkShadow(document.body);\n      return parts\n        .join('\\n')\n        .replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n        .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n        .replace(/<[^>]+>/g, ' ')\n        .replace(/\\s+/g, ' ')\n        .trim()\n        .slice(0, 10_000);\n    }).catch(() => '');\n  } catch {\n    return '';\n  }\n}\n"],
+  "mappings": "AAGA,SAAS,SAAS,MAAc,OAAuB;AACrD,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,IAAI,KAAK,KAAK,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;AAClE,SAAO,OAAO,IAAI;AACpB;AAEA,eAAsB,WAAW,MAAM,IAAI,MAAM,KAAoB;AACnE,QAAM,QAAQ,MAAM,OAAO;AAC3B,QAAM,SAAS,MAAM,OAAO;AAC5B,QAAM,QAAQ,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,SAAS,MAAM,KAAK,CAAC,CAAC;AAChE,QAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,KAAK,MAAM,KAAK,CAAC,CAAC;AACzD;AAEA,eAAsB,sBAAsB,MAAY,SAAiB,SAAgC;AACvG,MAAI;AACF,UAAM,QAAQ,KAAK;AACnB,UAAM,QAAQ,IAAI,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC;AAE9C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAC7D,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAE7D,aAAS,IAAI,GAAG,KAAK,OAAO,KAAK;AAC/B,YAAM,IAAI,IAAI;AAEd,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,MAAM,KAAK,KAAK,KAAK,OAAO,IAAI,OAAO,GAAG,KAAK,KAAK,OAAO,IAAI,OAAO,CAAC;AAC7E,UAAI,IAAI,MAAO,OAAM,WAAW,GAAG,EAAE;AAAA,IACvC;AAEA,UAAM,MAAM,KAAK,WAAW,KAAK,OAAO,IAAI,OAAO,GAAG,WAAW,KAAK,OAAO,IAAI,OAAO,CAAC;AACzF,UAAM,WAAW,IAAI,EAAE;AACvB,UAAM,MAAM,KAAK,SAAS,OAAO;AAAA,EACnC,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,mBAAmB,MAAY,UAAiC;AACpF,MAAI;AACF,UAAM,KAAK,MAAM,KAAK,EAAE,QAAQ;AAChC,QAAI,CAAC,GAAI;AACT,UAAM,MAAM,MAAM,GAAG,YAAY;AACjC,QAAI,CAAC,IAAK;AACV,UAAM,KAAK,IAAI,IAAI,IAAI,QAAQ;AAC/B,UAAM,KAAK,IAAI,IAAI,IAAI,SAAS;AAChC,UAAM,sBAAsB,MAAM,IAAI,EAAE;AACxC,UAAM,WAAW,KAAK,GAAG;AAAA,EAC3B,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,cAAc,MAAY,UAAiC;AAC/E,MAAI;AACF,UAAM,mBAAmB,MAAM,QAAQ;AACvC,UAAM,WAAW,IAAI,GAAG;AACxB,UAAM,KAAK,MAAM,UAAU,EAAE,OAAO,KAAK,MAAM,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;AAAA,EACpE,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,2BAA2B,MAA2B;AAC1E,QAAM,KAAK,cAAc,MAAM;AAE7B,UAAM,gBAAgB,kBAAkB,UAAU;AAClD,sBAAkB,UAAU,YAAY,YAAqC,MAAa;AACxF,YAAM,MAAM,KAAK,WAAW,IAAI;AAChC,UAAI,KAAK;AACP,cAAM,YAAY,IAAI,aAAa,GAAG,GAAG,KAAK,OAAO,KAAK,MAAM;AAChE,iBAAS,IAAI,GAAG,IAAI,UAAU,KAAK,QAAQ,KAAK,GAAG;AACjD,oBAAU,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,OAAO,IAAI,OAAQ,KAAK,OAAO,IAAI,MAAM,IAAI,KAAM,EAAE,CAAC;AAAA,QAChI;AACA,YAAI,aAAa,WAAW,GAAG,CAAC;AAAA,MAClC;AACA,aAAO,cAAc,MAAM,MAAM,IAAyC;AAAA,IAC5E;AAGA,UAAM,eAAe,sBAAsB,UAAU;AACrD,0BAAsB,UAAU,eAAe,SAAsC,WAAmB;AACtG,YAAM,wBAAwB;AAC9B,YAAM,0BAA0B;AAChC,UAAI,cAAc,sBAAuB,QAAO;AAChD,UAAI,cAAc,wBAAyB,QAAO;AAClD,aAAO,aAAa,KAAK,MAAM,SAAS;AAAA,IAC1C;AAGA,UAAM,YAAY,CAAC,GAAG,GAAG,GAAG,CAAC;AAC7B,WAAO,eAAe,WAAW,uBAAuB;AAAA,MACtD,KAAK,MAAM,UAAU,KAAK,MAAM,KAAK,OAAO,IAAI,UAAU,MAAM,CAAC;AAAA,MACjE,cAAc;AAAA,IAChB,CAAC;AAGD,WAAO,eAAe,WAAW,gBAAgB;AAAA,MAC/C,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC;AAAA,MAC/C,cAAc;AAAA,IAChB,CAAC;AAAA,EACH,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACnB;AAEA,eAAsB,uBACpB,MACA,OAA4E,CAAC,GAC7E,SACyD;AACzD,QAAM,EAAE,aAAa,IAAI,cAAc,KAAK,cAAc,IAAI,IAAI;AAClE,MAAI,cAAc;AAClB,MAAI,oBAAoB;AACxB,MAAI,iBAAiB;AAKrB,MAAI,cAAmB;AACvB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,MAAI,WAAW,OAAO,QAAQ,YAAY,YAAY;AACpD,QAAI;AACF,YAAM,MAAM,QAAQ,QAAQ;AAC5B,UAAI,OAAO,KAAK,kBAAkB,YAAY;AAC5C,sBAAc,MAAM,IAAI,cAAc,OAAO;AAC7C,cAAM,EAAE,OAAO,IAAI,MAAM,YAAY,KAAK,oBAAoB;AAAA,UAC5D,YAAY;AAAA,UACZ,eAAe;AAAA,QACjB,CAAC,EAAE,MAAM,OAAO,EAAE,QAAQ,EAAE,OAAO,KAAK,EAAE,EAAE;AAC5C,YAAI,QAAQ,OAAO;AAAE,kBAAQ,OAAO,MAAM,KAAK;AAAM,kBAAQ,OAAO,MAAM,KAAK;AAAA,QAAK;AAAA,MACtF;AAAA,IACF,QAAQ;AAAE,oBAAc;AAAA,IAAM;AAAA,EAChC;AAEA,MAAI;AACF,WAAO,cAAc,YAAY;AAC/B,uBAAiB,MAAM,KAAK,SAAS,MAAM,SAAS,gBAAgB,YAAY,EAAE,MAAM,MAAM,CAAC;AAE/F,YAAM,WAAW,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAErD,YAAM,KAAK,SAAS,CAAC,OAAe,OAAO,SAAS,GAAG,EAAE,GAAG,QAAQ,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAEpF,UAAI,aAAa;AACf,cAAM,YAAY,KAAK,4BAA4B;AAAA,UACjD,MAAM;AAAA,UACN,GAAG,KAAK,MAAM,QAAQ,CAAC;AAAA,UACvB,GAAG,KAAK,MAAM,QAAQ,CAAC;AAAA,UACvB,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,WAAW;AAAA,QACb,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM,WAAW,cAAc,KAAK,cAAc,GAAG;AAGrD,YAAM,IAAI,QAAc,OAAK,WAAW,GAAG,WAAW,CAAC;AACvD;AAEA,YAAM,YAAoB,MAAM,KAAK,SAAS,MAAM,SAAS,gBAAgB,YAAY,EAAE,MAAM,MAAM,CAAC;AACxG,UAAI,YAAY,iBAAiB,IAAI;AACnC;AACA,YAAI,qBAAqB,EAAG;AAAA,MAC9B,OAAO;AACL,4BAAoB;AAAA,MACtB;AACA,uBAAiB;AAAA,IACnB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAE1B,MAAI,YAAa,aAAY,OAAO,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACpD,SAAO,EAAE,aAAa,eAAe,eAAe;AACtD;AAEA,eAAsB,qBAAqB,MAAgF;AACzH,QAAM,UAAqE,CAAC;AAC5E,MAAI;AACF,UAAM,aAAuB,MAAM,KAAK;AAAA,MAAS,MAC/C,MAAM,KAAK,SAAS,iBAAiB,QAAQ,CAAC,EAAE,IAAI,OAAK,EAAE,aAAa,KAAK,KAAK,EAAE,aAAa,UAAU,KAAK,EAAE;AAAA,IACpH,EAAE,MAAM,MAAM,CAAC,CAAC;AAEhB,eAAW,SAAS,KAAK,OAAO,GAAG;AACjC,YAAM,MAAM,MAAM,IAAI;AACtB,UAAI,CAAC,OAAO,QAAQ,iBAAiB,QAAQ,KAAK,IAAI,EAAG;AACzD,UAAI;AACF,cAAM,UAAU,MAAM,MAAM,QAAQ,EAAE,MAAM,MAAM,EAAE;AACpD,YAAI,CAAC,WAAW,QAAQ,SAAS,GAAI;AAErC,cAAM,OAAO,QAAQ,QAAQ,qCAAqC,EAAE,EACjE,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACR,gBAAQ,KAAK,EAAE,KAAK,SAAS,QAAQ,MAAM,GAAG,GAAM,GAAG,UAAU,KAAK,MAAM,GAAG,GAAK,EAAE,CAAC;AAAA,MACzF,QAAQ;AAAA,MAAwB;AAAA,IAClC;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,eAAsB,gBAAgB,MAA6B;AACjE,MAAI;AACF,WAAO,MAAM,KAAK,SAAS,MAAM;AAC/B,YAAM,QAAkB,CAAC;AACzB,eAAS,WAAW,MAAkC;AACpD,cAAM,SAAU,KAAa;AAC7B,YAAI,QAAQ;AACV,gBAAM,KAAK,OAAO,aAAa,EAAE;AACjC,qBAAW,SAAS,CAAC,GAAG,OAAO,iBAAiB,GAAG,CAAC,EAAG,YAAW,KAAK;AAAA,QACzE;AACA,mBAAW,SAAS,MAAM,KAAK,KAAK,iBAAiB,yBAAyB,CAAC,GAAG;AAChF,cAAK,MAAc,WAAY,YAAW,KAAK;AAAA,QACjD;AAAA,MACF;AACA,iBAAW,SAAS,IAAI;AACxB,aAAO,MACJ,KAAK,IAAI,EACT,QAAQ,qCAAqC,EAAE,EAC/C,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK,EACL,MAAM,GAAG,GAAM;AAAA,IACpB,CAAC,EAAE,MAAM,MAAM,EAAE;AAAA,EACnB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/telemetry.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Lightweight OTel-compatible tracing for Zeta crawler.
+ *
+ * Emits spans as structured JSON logs compatible with OpenTelemetry format.
+ * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
+ *
+ * Usage:
+ *   const span = startSpan('crawl.url', { url, engine });
+ *   // ... do work ...
+ *   endSpan(span, { screensFound: 3 });
+ */
+export interface Span {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startTimeMs: number;
+    attributes: Record<string, string | number | boolean>;
+}
+export interface FinishedSpan extends Span {
+    endTimeMs: number;
+    durationMs: number;
+    status: 'ok' | 'error';
+    error?: string;
+}
+/** Start a new span. If a trace is already active, the new span is a child. */
+export declare function startSpan(name: string, attributes?: Record<string, string | number | boolean>): Span;
+/** End a span and emit it as a structured log line. */
+export declare function endSpan(span: Span, extraAttributes?: Record<string, string | number | boolean>, error?: Error): FinishedSpan;
+/** Convenience: wrap an async function in a span. */
+export declare function withSpan<T>(name: string, attributes: Record<string, string | number | boolean>, fn: (span: Span) => Promise<T>): Promise<T>;
+export declare function increment(metric: string, value?: number, tags?: Record<string, string>): void;
+/** Flush counters as a structured log — call periodically (e.g. at job end). */
+export declare function flushMetrics(): Record<string, number>;

package/dist/telemetry.js

@@ -0,0 +1,77 @@
+import { randomBytes } from "node:crypto";
+let currentTraceId = null;
+let currentSpanId = null;
+function genId(bytes) {
+  return randomBytes(bytes).toString("hex");
+}
+function startSpan(name, attributes = {}) {
+  const traceId = currentTraceId ?? genId(16);
+  const spanId = genId(8);
+  const parentSpanId = currentSpanId ?? void 0;
+  if (!currentTraceId) currentTraceId = traceId;
+  currentSpanId = spanId;
+  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
+}
+function endSpan(span, extraAttributes = {}, error) {
+  const endTimeMs = Date.now();
+  const durationMs = endTimeMs - span.startTimeMs;
+  const finished = {
+    ...span,
+    attributes: { ...span.attributes, ...extraAttributes },
+    endTimeMs,
+    durationMs,
+    status: error ? "error" : "ok",
+    error: error?.message
+  };
+  if (!span.parentSpanId) {
+    currentTraceId = null;
+    currentSpanId = span.parentSpanId ?? null;
+  } else {
+    currentSpanId = span.parentSpanId ?? null;
+  }
+  console.log(JSON.stringify({
+    ts: new Date(endTimeMs).toISOString(),
+    level: error ? "error" : "info",
+    msg: `span:${finished.name}`,
+    trace_id: finished.traceId,
+    span_id: finished.spanId,
+    parent_span_id: finished.parentSpanId,
+    duration_ms: durationMs,
+    status: finished.status,
+    error: finished.error,
+    ...finished.attributes
+  }));
+  return finished;
+}
+async function withSpan(name, attributes, fn) {
+  const span = startSpan(name, attributes);
+  try {
+    const result = await fn(span);
+    endSpan(span);
+    return result;
+  } catch (err) {
+    endSpan(span, {}, err);
+    throw err;
+  }
+}
+const counters = /* @__PURE__ */ new Map();
+function increment(metric, value = 1, tags = {}) {
+  const key = metric + (Object.keys(tags).length ? ":" + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(",") : "");
+  counters.set(key, (counters.get(key) ?? 0) + value);
+}
+function flushMetrics() {
+  const snapshot = Object.fromEntries(counters);
+  if (Object.keys(snapshot).length > 0) {
+    console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "metrics:flush", ...snapshot }));
+    counters.clear();
+  }
+  return snapshot;
+}
+export {
+  endSpan,
+  flushMetrics,
+  increment,
+  startSpan,
+  withSpan
+};
+//# sourceMappingURL=telemetry.js.map

package/dist/telemetry.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/telemetry.ts"],
+  "sourcesContent": ["/**\n * Lightweight OTel-compatible tracing for Zeta crawler.\n *\n * Emits spans as structured JSON logs compatible with OpenTelemetry format.\n * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.\n *\n * Usage:\n *   const span = startSpan('crawl.url', { url, engine });\n *   // ... do work ...\n *   endSpan(span, { screensFound: 3 });\n */\n\nimport { randomBytes } from 'node:crypto';\n\nexport interface Span {\n  traceId: string;\n  spanId: string;\n  parentSpanId?: string;\n  name: string;\n  startTimeMs: number;\n  attributes: Record<string, string | number | boolean>;\n}\n\nexport interface FinishedSpan extends Span {\n  endTimeMs: number;\n  durationMs: number;\n  status: 'ok' | 'error';\n  error?: string;\n}\n\n// Active trace context \u2014 per async execution context (simplified: per-process global)\nlet currentTraceId: string | null = null;\nlet currentSpanId: string | null = null;\n\nfunction genId(bytes: number): string {\n  return randomBytes(bytes).toString('hex');\n}\n\n/** Start a new span. If a trace is already active, the new span is a child. */\nexport function startSpan(name: string, attributes: Record<string, string | number | boolean> = {}): Span {\n  const traceId = currentTraceId ?? genId(16);\n  const spanId = genId(8);\n  const parentSpanId = currentSpanId ?? undefined;\n\n  if (!currentTraceId) currentTraceId = traceId;\n  currentSpanId = spanId;\n\n  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };\n}\n\n/** End a span and emit it as a structured log line. */\nexport function endSpan(span: Span, extraAttributes: Record<string, string | number | boolean> = {}, error?: Error): FinishedSpan {\n  const endTimeMs = Date.now();\n  const durationMs = endTimeMs - span.startTimeMs;\n  const finished: FinishedSpan = {\n    ...span,\n    attributes: { ...span.attributes, ...extraAttributes },\n    endTimeMs,\n    durationMs,\n    status: error ? 'error' : 'ok',\n    error: error?.message,\n  };\n\n  // Reset context when root span ends\n  if (!span.parentSpanId) {\n    currentTraceId = null;\n    currentSpanId = span.parentSpanId ?? null;\n  } else {\n    currentSpanId = span.parentSpanId ?? null;\n  }\n\n  // Emit as structured JSON \u2014 compatible with Datadog/Grafana log parsing\n  console.log(JSON.stringify({\n    ts: new Date(endTimeMs).toISOString(),\n    level: error ? 'error' : 'info',\n    msg: `span:${finished.name}`,\n    trace_id: finished.traceId,\n    span_id: finished.spanId,\n    parent_span_id: finished.parentSpanId,\n    duration_ms: durationMs,\n    status: finished.status,\n    error: finished.error,\n    ...finished.attributes,\n  }));\n\n  return finished;\n}\n\n/** Convenience: wrap an async function in a span. */\nexport async function withSpan<T>(\n  name: string,\n  attributes: Record<string, string | number | boolean>,\n  fn: (span: Span) => Promise<T>,\n): Promise<T> {\n  const span = startSpan(name, attributes);\n  try {\n    const result = await fn(span);\n    endSpan(span);\n    return result;\n  } catch (err: any) {\n    endSpan(span, {}, err);\n    throw err;\n  }\n}\n\n/** Counter metrics \u2014 increment and log on flush. */\nconst counters: Map<string, number> = new Map();\n\nexport function increment(metric: string, value = 1, tags: Record<string, string> = {}): void {\n  const key = metric + (Object.keys(tags).length ? ':' + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(',') : '');\n  counters.set(key, (counters.get(key) ?? 0) + value);\n}\n\n/** Flush counters as a structured log \u2014 call periodically (e.g. at job end). */\nexport function flushMetrics(): Record<string, number> {\n  const snapshot = Object.fromEntries(counters);\n  if (Object.keys(snapshot).length > 0) {\n    console.log(JSON.stringify({ ts: new Date().toISOString(), level: 'info', msg: 'metrics:flush', ...snapshot }));\n    counters.clear();\n  }\n  return snapshot;\n}\n"],
+  "mappings": "AAYA,SAAS,mBAAmB;AAmB5B,IAAI,iBAAgC;AACpC,IAAI,gBAA+B;AAEnC,SAAS,MAAM,OAAuB;AACpC,SAAO,YAAY,KAAK,EAAE,SAAS,KAAK;AAC1C;AAGO,SAAS,UAAU,MAAc,aAAwD,CAAC,GAAS;AACxG,QAAM,UAAU,kBAAkB,MAAM,EAAE;AAC1C,QAAM,SAAS,MAAM,CAAC;AACtB,QAAM,eAAe,iBAAiB;AAEtC,MAAI,CAAC,eAAgB,kBAAiB;AACtC,kBAAgB;AAEhB,SAAO,EAAE,SAAS,QAAQ,cAAc,MAAM,aAAa,KAAK,IAAI,GAAG,WAAW;AACpF;AAGO,SAAS,QAAQ,MAAY,kBAA6D,CAAC,GAAG,OAA6B;AAChI,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,aAAa,YAAY,KAAK;AACpC,QAAM,WAAyB;AAAA,IAC7B,GAAG;AAAA,IACH,YAAY,EAAE,GAAG,KAAK,YAAY,GAAG,gBAAgB;AAAA,IACrD;AAAA,IACA;AAAA,IACA,QAAQ,QAAQ,UAAU;AAAA,IAC1B,OAAO,OAAO;AAAA,EAChB;AAGA,MAAI,CAAC,KAAK,cAAc;AACtB,qBAAiB;AACjB,oBAAgB,KAAK,gBAAgB;AAAA,EACvC,OAAO;AACL,oBAAgB,KAAK,gBAAgB;AAAA,EACvC;AAGA,UAAQ,IAAI,KAAK,UAAU;AAAA,IACzB,IAAI,IAAI,KAAK,SAAS,EAAE,YAAY;AAAA,IACpC,OAAO,QAAQ,UAAU;AAAA,IACzB,KAAK,QAAQ,SAAS,IAAI;AAAA,IAC1B,UAAU,SAAS;AAAA,IACnB,SAAS,SAAS;AAAA,IAClB,gBAAgB,SAAS;AAAA,IACzB,aAAa;AAAA,IACb,QAAQ,SAAS;AAAA,IACjB,OAAO,SAAS;AAAA,IAChB,GAAG,SAAS;AAAA,EACd,CAAC,CAAC;AAEF,SAAO;AACT;AAGA,eAAsB,SACpB,MACA,YACA,IACY;AACZ,QAAM,OAAO,UAAU,MAAM,UAAU;AACvC,MAAI;AACF,UAAM,SAAS,MAAM,GAAG,IAAI;AAC5B,YAAQ,IAAI;AACZ,WAAO;AAAA,EACT,SAAS,KAAU;AACjB,YAAQ,MAAM,CAAC,GAAG,GAAG;AACrB,UAAM;AAAA,EACR;AACF;AAGA,MAAM,WAAgC,oBAAI,IAAI;AAEvC,SAAS,UAAU,QAAgB,QAAQ,GAAG,OAA+B,CAAC,GAAS;AAC5F,QAAM,MAAM,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,MAAM,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;AACrH,WAAS,IAAI,MAAM,SAAS,IAAI,GAAG,KAAK,KAAK,KAAK;AACpD;AAGO,SAAS,eAAuC;AACrD,QAAM,WAAW,OAAO,YAAY,QAAQ;AAC5C,MAAI,OAAO,KAAK,QAAQ,EAAE,SAAS,GAAG;AACpC,YAAQ,IAAI,KAAK,UAAU,EAAE,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,OAAO,QAAQ,KAAK,iBAAiB,GAAG,SAAS,CAAC,CAAC;AAC9G,aAAS,MAAM;AAAA,EACjB;AACA,SAAO;AACT;",
+  "names": []
+}