crawlio-browser 1.5.5 → 1.5.8

package/dist/mcp-server/index.js

@@ -9,7 +9,7 @@ import {
   WS_PORT_MAX,
   WS_RECONNECT_GRACE,
   WS_STALE_THRESHOLD
-} from "./chunk-A4EQCKHH.js";
+} from "./chunk-YEKQAHYW.js";
 
 // src/mcp-server/index.ts
 import { randomBytes as randomBytes2 } from "crypto";
@@ -1408,7 +1408,11 @@ var TOOL_TIMEOUTS = {
   stop_recording: 1e4,
   get_recording_status: 5e3,
   compile_recording: 5e3,
-  ocr_screenshot: 3e4
+  ocr_screenshot: 3e4,
+  wait_for_network_idle: 35e3,
+  detect_tables: 15e3,
+  extract_table: 15e3,
+  extract_data: 3e4
 };
 function toolSuccess(content) {
   return { content: [{ type: "text", text: JSON.stringify(content ?? {}) }], isError: false };
@@ -1418,6 +1422,38 @@ function toolError(message) {
 }
 var ACTIONABILITY_BACKOFF = [0, 20, 100, 100, 500];
 var smartObjectCache = null;
+var sessionFindings = [];
+var sessionGaps = [];
+function summarizeAccessibility(raw) {
+  const tree = raw.tree ?? raw.nodes ?? [];
+  let nodeCount = typeof raw.nodeCount === "number" ? raw.nodeCount : 0;
+  let landmarkCount = 0;
+  let imagesWithoutAlt = 0;
+  const headingStructure = [];
+  const landmarkRoles = /* @__PURE__ */ new Set(["banner", "navigation", "main", "contentinfo", "complementary", "search", "form", "region"]);
+  function walk(nodes) {
+    for (const node of nodes) {
+      nodeCount++;
+      const role = String(node.role ?? "");
+      if (landmarkRoles.has(role)) landmarkCount++;
+      if (role === "img" || role === "image") {
+        const name = String(node.name ?? "");
+        if (!name) imagesWithoutAlt++;
+      }
+      if (role === "heading") {
+        const level = typeof node.level === "number" ? node.level : 0;
+        const text = String(node.name ?? "").substring(0, 120);
+        if (level > 0) headingStructure.push({ level, text });
+      }
+      if (Array.isArray(node.children)) walk(node.children);
+    }
+  }
+  if (tree.length > 0) {
+    nodeCount = 0;
+    walk(tree);
+  }
+  return { nodeCount, landmarkCount, imagesWithoutAlt, headingStructure: headingStructure.slice(0, 30) };
+}
 async function checkActionability(bridge2, selector) {
   const result = await bridge2.send({
     type: "browser_evaluate",
@@ -1444,7 +1480,7 @@ async function checkActionability(bridge2, selector) {
       return { actionable: true };
     })()`
   }, 5e3);
-  return result;
+  return result.result ?? result;
 }
 async function pollActionability(bridge2, selector, timeoutMs = 3e3) {
   const start = Date.now();
@@ -2497,7 +2533,7 @@ function createTools(bridge2, crawlio2) {
         return toolSuccess(data);
       }
     },
-    // Network replay (PiecesOS Heist Ph3)
+    // Network replay
     {
       name: "replay_request",
       description: "Re-fire a previously captured network request with optional modifications. Requires active network capture. Specify the request by URL. Optionally override headers, body, or method. Returns the new response status, headers, and body. Useful for API testing, auth token replay, and form submission testing.",
@@ -3706,17 +3742,531 @@ function createTools(bridge2, crawlio2) {
           });
         }
       }
+    },
+    // --- Network idle detection (Phase 11) ---
+    {
+      name: "wait_for_network_idle",
+      description: "Wait for all network requests to complete (idle detection). Uses CDP Network domain event tracking \u2014 catches ALL request types (fetch, XHR, images, CSS, fonts, scripts). Returns when no requests are pending for the specified idle window, or on timeout.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          timeout: { type: "number", description: "Max wait in ms (default 15000, max 30000)" },
+          idleTime: { type: "number", description: "Quiet window before declaring idle in ms (default 500, max 5000)" }
+        }
+      },
+      handler: async (args) => {
+        const schema = z.object({
+          timeout: z.number().int().min(100).max(3e4).default(15e3),
+          idleTime: z.number().int().min(100).max(5e3).default(500)
+        });
+        const parsed = schema.parse(args);
+        const data = await bridge2.send({
+          type: "wait_for_network_idle",
+          timeout: parsed.timeout,
+          idleTime: parsed.idleTime
+        }, TOOL_TIMEOUTS.wait_for_network_idle);
+        return toolSuccess(data);
+      }
+    },
+    // --- Structured data extraction (Phase 11 — full-mode tools) ---
+    {
+      name: "detect_tables",
+      description: "Detect repeating data patterns (tables, lists, grids, card layouts) on the page using class-frequency scoring. Returns ranked candidates with selectors, row counts, and sample text. Use extract_table to extract data from a candidate.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          maxCandidates: { type: "number", description: "Maximum candidates to return (default 5, max 20)" }
+        }
+      },
+      handler: async (args) => {
+        const schema = z.object({
+          maxCandidates: z.number().int().min(1).max(20).default(5)
+        });
+        const parsed = schema.parse(args);
+        const maxCandidates = parsed.maxCandidates;
+        const result = await bridge2.send({
+          type: "browser_evaluate",
+          expression: `(() => {
+      function getClasses(el) {
+        return (el.className || '').toString().trim().split(/\\s+/).filter(c => c && !c.match(/\\d/));
+      }
+      function getMatchingChildren(parent) {
+        const children = [...parent.children].filter(c =>
+          !['SCRIPT','IMG','STYLE','SVG','NOSCRIPT'].includes(c.tagName) &&
+          c.textContent.trim().length > 0
+        );
+        if (children.length < 2) return [];
+        const freq = {};
+        children.forEach(c => {
+          const key = getClasses(c).sort().join(' ');
+          freq[key] = (freq[key] || 0) + 1;
+        });
+        const threshold = children.length / 2 - 2;
+        let patterns = Object.keys(freq).filter(k => k && freq[k] >= threshold);
+        if (!patterns.length) {
+          const indiv = {};
+          children.forEach(c => getClasses(c).forEach(cls => { indiv[cls] = (indiv[cls] || 0) + 1; }));
+          patterns = Object.keys(indiv).filter(k => indiv[k] >= threshold);
+        }
+        return children.filter(c => {
+          const cls = getClasses(c);
+          return patterns.some(p => p.split(' ').every(pc => !pc || cls.includes(pc)));
+        });
+      }
+      function buildSelector(el) {
+        const parts = [];
+        let node = el;
+        while (node && node !== document.body && node !== document.documentElement) {
+          let tag = node.tagName.toLowerCase();
+          if (node.id && !/\\d/.test(node.id)) tag += '#' + CSS.escape(node.id);
+          else if (node.className && typeof node.className === 'string') {
+            const cls = node.className.trim().split(/\\s+/).filter(Boolean).slice(0, 3);
+            if (cls.length) tag += '.' + cls.map(c => CSS.escape(c)).join('.');
+          }
+          parts.unshift(tag);
+          node = node.parentElement;
+        }
+        return parts.join(' > ');
+      }
+      const candidates = [];
+      document.querySelectorAll('body *').forEach(el => {
+        const w = el.offsetWidth, h = el.offsetHeight;
+        if (w < 100 || h < 50) return;
+        const matching = getMatchingChildren(el);
+        if (matching.length < 2) return;
+        const score = w * h * matching.length * matching.length;
+        const text = matching.map(c => c.textContent.trim()).join(' ').substring(0, 200);
+        candidates.push({ selector: buildSelector(el), score, rowCount: matching.length, sampleText: text, area: w * h });
+      });
+      candidates.sort((a, b) => b.score - a.score);
+      return candidates.slice(0, ${maxCandidates});
+    })()`
+        }, TOOL_TIMEOUTS.detect_tables);
+        const data = result;
+        return toolSuccess(data?.result ?? []);
+      }
+    },
+    {
+      name: "extract_table",
+      description: "Extract structured data from a container element using IDS-inspired recursive path-keyed extraction. Pass a selector from detect_tables. Returns columns (with fill rates) and rows as key-value objects.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          selector: { type: "string", description: "CSS selector of the container element" },
+          maxRows: { type: "number", description: "Maximum rows to extract (default 200, max 1000)" }
+        },
+        required: ["selector"]
+      },
+      handler: async (args) => {
+        const schema = z.object({
+          selector: z.string().min(1),
+          maxRows: z.number().int().min(1).max(1e3).default(200)
+        });
+        const parsed = schema.parse(args);
+        const selector = parsed.selector;
+        const maxRows = parsed.maxRows;
+        const result = await bridge2.send({
+          type: "browser_evaluate",
+          expression: `(() => {
+      function getClasses(el) {
+        return (el.className || '').toString().trim().split(/\\s+/).filter(c => c && !c.match(/\\d/));
+      }
+      function getMatchingChildren(parent) {
+        const children = [...parent.children].filter(c =>
+          !['SCRIPT','IMG','STYLE','SVG','NOSCRIPT'].includes(c.tagName) &&
+          c.textContent.trim().length > 0
+        );
+        if (children.length < 2) return [];
+        const freq = {};
+        children.forEach(c => {
+          const key = getClasses(c).sort().join(' ');
+          freq[key] = (freq[key] || 0) + 1;
+        });
+        const threshold = children.length / 2 - 2;
+        let patterns = Object.keys(freq).filter(k => k && freq[k] >= threshold);
+        if (!patterns.length) {
+          const indiv = {};
+          children.forEach(c => getClasses(c).forEach(cls => { indiv[cls] = (indiv[cls] || 0) + 1; }));
+          patterns = Object.keys(indiv).filter(k => indiv[k] >= threshold);
+        }
+        return children.filter(c => {
+          const cls = getClasses(c);
+          return patterns.some(p => p.split(' ').every(pc => !pc || cls.includes(pc)));
+        });
+      }
+      function directText(el) {
+        let text = '';
+        for (const n of el.childNodes) {
+          if (n.nodeType === 3) text += n.textContent;
+        }
+        return text.trim();
+      }
+      function extractRow(el, prefix) {
+        const data = {};
+        const tag = el.tagName.toLowerCase();
+        const cls = getClasses(el).slice(0, 2).join('.');
+        const key = prefix + '/' + (cls ? tag + '.' + cls : tag);
+        const dt = directText(el);
+        if (dt) data[key] = dt;
+        if (el.href) data[key + ' href'] = el.href;
+        if (el.src) data[key + ' src'] = el.src;
+        for (const child of el.children) {
+          Object.assign(data, extractRow(child, key));
+        }
+        return data;
+      }
+      const container = document.querySelector(${JSON.stringify(selector)});
+      if (!container) return { selector: ${JSON.stringify(selector)}, columns: [], rows: [], totalRows: 0, truncated: false };
+      const matching = getMatchingChildren(container);
+      const totalRows = matching.length;
+      const limited = matching.slice(0, ${maxRows});
+      const rawRows = limited.map(el => extractRow(el, ''));
+      const allKeys = new Set();
+      rawRows.forEach(r => Object.keys(r).forEach(k => allKeys.add(k)));
+      const columns = [];
+      const keyToName = {};
+      for (const key of allKeys) {
+        const values = rawRows.map(r => r[key] || '');
+        const filled = values.filter(v => v).length;
+        const fillRate = filled / rawRows.length;
+        if (fillRate < 0.2) continue;
+        const unique = new Set(values.filter(v => v));
+        if (unique.size <= 1 && rawRows.length > 2) continue;
+        const parts = key.split('/').filter(Boolean);
+        const last = parts[parts.length - 1] || key;
+        let name = last.replace(/^[a-z]+\\./, '').replace(/ (href|src)$/, ' $1').replace(/\\./g, ' ').trim() || last;
+        keyToName[key] = name;
+        columns.push({ name, path: key, fillRate: Math.round(fillRate * 100) / 100 });
+      }
+      const nameCount = {};
+      columns.forEach(c => { nameCount[c.name] = (nameCount[c.name] || 0) + 1; });
+      const nameSeen = {};
+      columns.forEach(c => {
+        if (nameCount[c.name] > 1) {
+          nameSeen[c.name] = (nameSeen[c.name] || 0) + 1;
+          if (nameSeen[c.name] > 1) c.name = c.name + ' ' + nameSeen[c.name];
+        }
+      });
+      const rows = rawRows.map(raw => {
+        const row = {};
+        columns.forEach(c => { row[c.name] = raw[c.path] || ''; });
+        return row;
+      });
+      return { selector: ${JSON.stringify(selector)}, columns, rows, totalRows, truncated: totalRows > ${maxRows} };
+    })()`
+        }, TOOL_TIMEOUTS.extract_table);
+        const data = result;
+        return toolSuccess(data?.result ?? { selector, columns: [], rows: [], totalRows: 0, truncated: false });
+      }
+    },
+    {
+      name: "extract_data",
+      description: "Compound extraction: detect tables + extract each + collect JSON-LD structured data from the page. Returns all tables with columns/rows plus any schema.org/JSON-LD data. Combines detect_tables, extract_table, and JSON-LD extraction in one call.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          maxTables: { type: "number", description: "Maximum tables to extract (default 3, max 10)" },
+          maxRowsPerTable: { type: "number", description: "Maximum rows per table (default 200, max 1000)" }
+        }
+      },
+      handler: async (args) => {
+        const schema = z.object({
+          maxTables: z.number().int().min(1).max(10).default(3),
+          maxRowsPerTable: z.number().int().min(1).max(1e3).default(200)
+        });
+        const parsed = schema.parse(args);
+        const maxTables = parsed.maxTables;
+        const maxRowsPerTable = parsed.maxRowsPerTable;
+        const status = await bridge2.send({ type: "get_connection_status" }, 5e3);
+        const url = status?.connectedTab?.url || "";
+        const detectResult = await bridge2.send({
+          type: "browser_evaluate",
+          expression: `(() => {
+      function getClasses(el) {
+        return (el.className || '').toString().trim().split(/\\s+/).filter(c => c && !c.match(/\\d/));
+      }
+      function getMatchingChildren(parent) {
+        const children = [...parent.children].filter(c =>
+          !['SCRIPT','IMG','STYLE','SVG','NOSCRIPT'].includes(c.tagName) &&
+          c.textContent.trim().length > 0
+        );
+        if (children.length < 2) return [];
+        const freq = {};
+        children.forEach(c => {
+          const key = getClasses(c).sort().join(' ');
+          freq[key] = (freq[key] || 0) + 1;
+        });
+        const threshold = children.length / 2 - 2;
+        let patterns = Object.keys(freq).filter(k => k && freq[k] >= threshold);
+        if (!patterns.length) {
+          const indiv = {};
+          children.forEach(c => getClasses(c).forEach(cls => { indiv[cls] = (indiv[cls] || 0) + 1; }));
+          patterns = Object.keys(indiv).filter(k => indiv[k] >= threshold);
+        }
+        return children.filter(c => {
+          const cls = getClasses(c);
+          return patterns.some(p => p.split(' ').every(pc => !pc || cls.includes(pc)));
+        });
+      }
+      function buildSelector(el) {
+        const parts = [];
+        let node = el;
+        while (node && node !== document.body && node !== document.documentElement) {
+          let tag = node.tagName.toLowerCase();
+          if (node.id && !/\\d/.test(node.id)) tag += '#' + CSS.escape(node.id);
+          else if (node.className && typeof node.className === 'string') {
+            const cls = node.className.trim().split(/\\s+/).filter(Boolean).slice(0, 3);
+            if (cls.length) tag += '.' + cls.map(c => CSS.escape(c)).join('.');
+          }
+          parts.unshift(tag);
+          node = node.parentElement;
+        }
+        return parts.join(' > ');
+      }
+      const candidates = [];
+      document.querySelectorAll('body *').forEach(el => {
+        const w = el.offsetWidth, h = el.offsetHeight;
+        if (w < 100 || h < 50) return;
+        const matching = getMatchingChildren(el);
+        if (matching.length < 2) return;
+        const score = w * h * matching.length * matching.length;
+        const text = matching.map(c => c.textContent.trim()).join(' ').substring(0, 200);
+        candidates.push({ selector: buildSelector(el), score, rowCount: matching.length, sampleText: text, area: w * h });
+      });
+      candidates.sort((a, b) => b.score - a.score);
+      return candidates.slice(0, ${maxTables});
+    })()`
+        }, TOOL_TIMEOUTS.detect_tables);
+        const candidates = detectResult?.result ?? [];
+        const tables = [];
+        for (const candidate of candidates) {
+          const sel = candidate.selector;
+          const extractResult = await bridge2.send({
+            type: "browser_evaluate",
+            expression: `(() => {
+      function getClasses(el) {
+        return (el.className || '').toString().trim().split(/\\s+/).filter(c => c && !c.match(/\\d/));
+      }
+      function getMatchingChildren(parent) {
+        const children = [...parent.children].filter(c =>
+          !['SCRIPT','IMG','STYLE','SVG','NOSCRIPT'].includes(c.tagName) &&
+          c.textContent.trim().length > 0
+        );
+        if (children.length < 2) return [];
+        const freq = {};
+        children.forEach(c => {
+          const key = getClasses(c).sort().join(' ');
+          freq[key] = (freq[key] || 0) + 1;
+        });
+        const threshold = children.length / 2 - 2;
+        let patterns = Object.keys(freq).filter(k => k && freq[k] >= threshold);
+        if (!patterns.length) {
+          const indiv = {};
+          children.forEach(c => getClasses(c).forEach(cls => { indiv[cls] = (indiv[cls] || 0) + 1; }));
+          patterns = Object.keys(indiv).filter(k => indiv[k] >= threshold);
+        }
+        return children.filter(c => {
+          const cls = getClasses(c);
+          return patterns.some(p => p.split(' ').every(pc => !pc || cls.includes(pc)));
+        });
+      }
+      function directText(el) {
+        let text = '';
+        for (const n of el.childNodes) {
+          if (n.nodeType === 3) text += n.textContent;
+        }
+        return text.trim();
+      }
+      function extractRow(el, prefix) {
+        const data = {};
+        const tag = el.tagName.toLowerCase();
+        const cls = getClasses(el).slice(0, 2).join('.');
+        const key = prefix + '/' + (cls ? tag + '.' + cls : tag);
+        const dt = directText(el);
+        if (dt) data[key] = dt;
+        if (el.href) data[key + ' href'] = el.href;
+        if (el.src) data[key + ' src'] = el.src;
+        for (const child of el.children) {
+          Object.assign(data, extractRow(child, key));
+        }
+        return data;
+      }
+      const container = document.querySelector(${JSON.stringify(sel)});
+      if (!container) return { selector: ${JSON.stringify(sel)}, columns: [], rows: [], totalRows: 0, truncated: false };
+      const matching = getMatchingChildren(container);
+      const totalRows = matching.length;
+      const limited = matching.slice(0, ${maxRowsPerTable});
+      const rawRows = limited.map(el => extractRow(el, ''));
+      const allKeys = new Set();
+      rawRows.forEach(r => Object.keys(r).forEach(k => allKeys.add(k)));
+      const columns = [];
+      for (const key of allKeys) {
+        const values = rawRows.map(r => r[key] || '');
+        const filled = values.filter(v => v).length;
+        const fillRate = filled / rawRows.length;
+        if (fillRate < 0.2) continue;
+        const unique = new Set(values.filter(v => v));
+        if (unique.size <= 1 && rawRows.length > 2) continue;
+        const parts = key.split('/').filter(Boolean);
+        const last = parts[parts.length - 1] || key;
+        let name = last.replace(/^[a-z]+\\./, '').replace(/ (href|src)$/, ' $1').replace(/\\./g, ' ').trim() || last;
+        columns.push({ name, path: key, fillRate: Math.round(fillRate * 100) / 100 });
+      }
+      const nameCount = {};
+      columns.forEach(c => { nameCount[c.name] = (nameCount[c.name] || 0) + 1; });
+      const nameSeen = {};
+      columns.forEach(c => {
+        if (nameCount[c.name] > 1) {
+          nameSeen[c.name] = (nameSeen[c.name] || 0) + 1;
+          if (nameSeen[c.name] > 1) c.name = c.name + ' ' + nameSeen[c.name];
+        }
+      });
+      const rows = rawRows.map(raw => {
+        const row = {};
+        columns.forEach(c => { row[c.name] = raw[c.path] || ''; });
+        return row;
+      });
+      return { selector: ${JSON.stringify(sel)}, columns, rows, totalRows, truncated: totalRows > ${maxRowsPerTable} };
+    })()`
+          }, TOOL_TIMEOUTS.extract_table);
+          const extraction = extractResult?.result;
+          if (extraction && (extraction.columns?.length ?? 0) >= 2 && (extraction.rows?.length ?? 0) >= 2) {
+            tables.push(extraction);
+          }
+        }
+        const jsonLdResult = await bridge2.send({
+          type: "browser_evaluate",
+          expression: `(() => {
+      const scripts = document.querySelectorAll('script[type="application/ld+json"]');
+      const result = [];
+      scripts.forEach(s => { try { result.push(JSON.parse(s.textContent)); } catch {} });
+      return result;
+    })()`
+        }, 5e3);
+        return toolSuccess({
+          tables,
+          structuredData: jsonLdResult?.result ?? [],
+          url
+        });
+      }
     }
   ];
 }
+var COMPARISON_DIMENSIONS = [
+  "framework",
+  "performance",
+  "security",
+  "seo",
+  "accessibility",
+  "error-surface",
+  "third-party-load",
+  "architecture",
+  "content-delivery",
+  "mobile-readiness",
+  "data-structure"
+];
+function observeField(data, dimension) {
+  switch (dimension) {
+    case "framework":
+      return data.capture?.framework ? { type: "present", dimension, value: data.capture.framework } : { type: "absent", dimension, gap: { dimension, reason: "No framework detected", impact: "data-absent", reducesConfidence: false } };
+    case "performance":
+      return data.performance ? { type: "present", dimension, value: data.performance } : { type: "absent", dimension, gap: data.gaps?.find((g) => g.dimension === "performance") ?? { dimension, reason: "Performance data unavailable", impact: "data-absent", reducesConfidence: false } };
+    case "security":
+      return data.security ? { type: "present", dimension, value: data.security } : { type: "absent", dimension, gap: data.gaps?.find((g) => g.dimension === "security") ?? { dimension, reason: "Security data unavailable", impact: "data-absent", reducesConfidence: false } };
+    case "seo":
+      return data.meta?._title || data.meta?._canonical || (data.meta?._structuredData?.length ?? 0) > 0 ? { type: "present", dimension, value: { title: data.meta?._title, canonical: data.meta?._canonical, structuredData: data.meta?._structuredData?.length ?? 0 } } : { type: "absent", dimension };
+    case "accessibility":
+      return data.accessibility ? { type: "present", dimension, value: data.accessibility } : { type: "absent", dimension, gap: data.gaps?.find((g) => g.dimension === "accessibility") ?? { dimension, reason: "Accessibility data unavailable", impact: "data-absent", reducesConfidence: false } };
+    case "error-surface": {
+      const consoleLogs = data.capture?.console;
+      return consoleLogs ? { type: "present", dimension, value: consoleLogs } : { type: "absent", dimension };
+    }
+    case "third-party-load": {
+      const network = data.capture?.network;
+      return network ? { type: "present", dimension, value: network } : { type: "absent", dimension };
+    }
+    case "architecture":
+      return data.capture?.framework ? { type: "present", dimension, value: data.capture.framework } : { type: "absent", dimension };
+    case "content-delivery":
+      return data.security ? { type: "present", dimension, value: data.security } : { type: "absent", dimension };
+    case "mobile-readiness": {
+      const mr = data.mobileReadiness;
+      return mr ? { type: mr.hasViewportMeta ? "present" : "degraded", dimension, value: mr } : { type: "absent", dimension, gap: data.gaps?.find((g) => g.dimension === "mobile-readiness") ?? { dimension, reason: "Mobile readiness data unavailable", impact: "data-absent", reducesConfidence: false } };
+    }
+    case "data-structure":
+      return (data.meta?._structuredData?.length ?? 0) > 0 ? { type: "present", dimension, value: { structuredDataCount: data.meta._structuredData.length } } : { type: "absent", dimension };
+    default:
+      return { type: "absent", dimension };
+  }
+}
+function extractMetrics(data) {
+  const metrics = {};
+  const perf = data.performance;
+  if (perf) {
+    const wv = perf.webVitals;
+    if (wv) {
+      if (typeof wv.lcp === "number") metrics["LCP"] = wv.lcp;
+      if (typeof wv.cls === "number") metrics["CLS"] = wv.cls;
+      if (typeof wv.fid === "number") metrics["FID"] = wv.fid;
+    }
+    const timing = perf.timing;
+    if (timing) {
+      if (typeof timing.domContentLoaded === "number") metrics["domContentLoaded"] = timing.domContentLoaded;
+      if (typeof timing.load === "number") metrics["loadTime"] = timing.load;
+      if (typeof timing.firstByte === "number") metrics["TTFB"] = timing.firstByte;
+    }
+    const chrome = perf.chrome;
+    if (chrome) {
+      if (typeof chrome.taskDuration === "number") metrics["taskDuration"] = chrome.taskDuration;
+      if (typeof chrome.scriptDuration === "number") metrics["scriptDuration"] = chrome.scriptDuration;
+      if (typeof chrome.jsHeapUsedSize === "number") metrics["jsHeapUsedSize"] = chrome.jsHeapUsedSize;
+    }
+    const m = perf.metrics;
+    if (m) {
+      if (typeof m.LCP === "number" && !metrics["LCP"]) metrics["LCP"] = m.LCP;
+      if (typeof m.FCP === "number") metrics["FCP"] = m.FCP;
+      if (typeof m.CLS === "number" && !metrics["CLS"]) metrics["CLS"] = m.CLS;
+    }
+  }
+  const net = data.capture?.network;
+  if (net && typeof net.total === "number") metrics["networkRequests"] = net.total;
+  const dom = data.capture?.dom;
+  if (dom && typeof dom.nodeCount === "number") metrics["domNodeCount"] = dom.nodeCount;
+  return metrics;
+}
+function buildComparisonScaffold(a, b) {
+  const dimensions = COMPARISON_DIMENSIONS.map((name) => {
+    const obsA = observeField(a, name);
+    const obsB = observeField(b, name);
+    return { name, siteA: obsA, siteB: obsB, comparable: obsA.type === "present" && obsB.type === "present" };
+  });
+  const fieldsA = new Set(Object.keys(a.capture));
+  const fieldsB = new Set(Object.keys(b.capture));
+  const sharedFields = [...fieldsA].filter((f) => fieldsB.has(f));
+  const missingFields = {
+    siteA: [...fieldsB].filter((f) => !fieldsA.has(f)),
+    siteB: [...fieldsA].filter((f) => !fieldsB.has(f))
+  };
+  const metricsA = extractMetrics(a);
+  const metricsB = extractMetrics(b);
+  const allMetricKeys = /* @__PURE__ */ new Set([...Object.keys(metricsA), ...Object.keys(metricsB)]);
+  const metrics = [...allMetricKeys].map((name) => ({
+    name,
+    siteA: metricsA[name] ?? null,
+    siteB: metricsB[name] ?? null
+  }));
+  return { dimensions, sharedFields, missingFields, metrics };
+}
 function parseSnapshotRef(selector) {
   const m = /^\[ref=([a-zA-Z0-9]+)\]$/.exec(selector.trim());
   return m ? m[1] : null;
 }
 async function buildSmartObject(bridge2) {
   const evaluate = (expression) => {
+    const trimmed = expression.trim();
     const hasReturn = /(?:^|[;\n{])\s*return\s/m.test(expression);
-    const expr = hasReturn ? `(async () => { ${expression} })()` : expression;
+    const isExpression = trimmed.startsWith("(");
+    const expr = hasReturn && !isExpression ? `(async () => { ${expression} })()` : expression;
     return bridge2.send({ type: "browser_evaluate", expression: expr }, 5e3);
   };
   const smart = {
@@ -3894,6 +4444,245 @@ async function buildSmartObject(bridge2) {
       getVersion: () => evaluate(`window.jQuery?.fn?.jquery`)
     };
   }
+  smart.finding = (input) => {
+    const { claim, evidence, sourceUrl, confidence, method, dimension } = input;
+    if (typeof claim !== "string" || !claim) throw new Error("finding.claim is required (string)");
+    if (!Array.isArray(evidence) || evidence.length === 0) throw new Error("finding.evidence is required (non-empty string[])");
+    if (typeof sourceUrl !== "string" || !sourceUrl) throw new Error("finding.sourceUrl is required (string)");
+    if (!["high", "medium", "low"].includes(confidence)) throw new Error("finding.confidence must be 'high' | 'medium' | 'low'");
+    if (typeof method !== "string" || !method) throw new Error("finding.method is required (string)");
+    for (const e of evidence) {
+      if (typeof e !== "string") throw new Error("finding.evidence[] must be strings");
+    }
+    let finalConfidence = confidence;
+    let confidenceCapped = false;
+    let cappedBy;
+    if (typeof dimension === "string") {
+      const gap = sessionGaps.find((g) => g.dimension === dimension && g.reducesConfidence);
+      if (gap) {
+        const capMap = { high: "medium", medium: "low", low: "low" };
+        const capped = capMap[finalConfidence];
+        if (capped && capped !== finalConfidence) {
+          finalConfidence = capped;
+          confidenceCapped = true;
+          cappedBy = gap.dimension;
+        }
+      }
+    }
+    const finding = {
+      claim,
+      evidence,
+      sourceUrl,
+      confidence: finalConfidence,
+      method,
+      dimension: typeof dimension === "string" ? dimension : void 0
+    };
+    if (confidenceCapped) {
+      finding.confidenceCapped = true;
+      finding.cappedBy = cappedBy;
+    }
+    sessionFindings.push(finding);
+    return finding;
+  };
+  smart.findings = () => [...sessionFindings];
+  smart.clearFindings = () => {
+    sessionFindings = [];
+    sessionGaps = [];
+  };
+  smart.detectTables = async (opts) => {
+    const maxCandidates = opts?.maxCandidates ?? 5;
+    const result = await evaluate(`(() => {
+      function getClasses(el) {
+        return (el.className || '').toString().trim().split(/\\s+/).filter(c => c && !c.match(/\\d/));
+      }
+      function getMatchingChildren(parent) {
+        const children = [...parent.children].filter(c =>
+          !['SCRIPT','IMG','STYLE','SVG','NOSCRIPT'].includes(c.tagName) &&
+          c.textContent.trim().length > 0
+        );
+        if (children.length < 2) return [];
+        const freq = {};
+        children.forEach(c => {
+          const key = getClasses(c).sort().join(' ');
+          freq[key] = (freq[key] || 0) + 1;
+        });
+        const threshold = children.length / 2 - 2;
+        let patterns = Object.keys(freq).filter(k => k && freq[k] >= threshold);
+        if (!patterns.length) {
+          const indiv = {};
+          children.forEach(c => getClasses(c).forEach(cls => { indiv[cls] = (indiv[cls] || 0) + 1; }));
+          patterns = Object.keys(indiv).filter(k => indiv[k] >= threshold);
+        }
+        return children.filter(c => {
+          const cls = getClasses(c);
+          return patterns.some(p => p.split(' ').every(pc => !pc || cls.includes(pc)));
+        });
+      }
+      function buildSelector(el) {
+        const parts = [];
+        let node = el;
+        while (node && node !== document.body && node !== document.documentElement) {
+          let tag = node.tagName.toLowerCase();
+          if (node.id && !/\\d/.test(node.id)) tag += '#' + CSS.escape(node.id);
+          else if (node.className && typeof node.className === 'string') {
+            const cls = node.className.trim().split(/\\s+/).filter(Boolean).slice(0, 3);
+            if (cls.length) tag += '.' + cls.map(c => CSS.escape(c)).join('.');
+          }
+          parts.unshift(tag);
+          node = node.parentElement;
+        }
+        return parts.join(' > ');
+      }
+      const candidates = [];
+      document.querySelectorAll('body *').forEach(el => {
+        const w = el.offsetWidth, h = el.offsetHeight;
+        if (w < 100 || h < 50) return;
+        const matching = getMatchingChildren(el);
+        if (matching.length < 2) return;
+        const score = w * h * matching.length * matching.length;
+        const text = matching.map(c => c.textContent.trim()).join(' ').substring(0, 200);
+        candidates.push({ selector: buildSelector(el), score, rowCount: matching.length, sampleText: text, area: w * h });
+      });
+      candidates.sort((a, b) => b.score - a.score);
+      return candidates.slice(0, ${maxCandidates});
+    })()`);
+    return result.result || [];
+  };
+  smart.extractTable = async (selector, opts) => {
+    const maxRows = opts?.maxRows ?? 200;
+    const result = await evaluate(`(() => {
+      function getClasses(el) {
+        return (el.className || '').toString().trim().split(/\\s+/).filter(c => c && !c.match(/\\d/));
+      }
+      function getMatchingChildren(parent) {
+        const children = [...parent.children].filter(c =>
+          !['SCRIPT','IMG','STYLE','SVG','NOSCRIPT'].includes(c.tagName) &&
+          c.textContent.trim().length > 0
+        );
+        if (children.length < 2) return [];
+        const freq = {};
+        children.forEach(c => {
+          const key = getClasses(c).sort().join(' ');
+          freq[key] = (freq[key] || 0) + 1;
+        });
+        const threshold = children.length / 2 - 2;
+        let patterns = Object.keys(freq).filter(k => k && freq[k] >= threshold);
+        if (!patterns.length) {
+          const indiv = {};
+          children.forEach(c => getClasses(c).forEach(cls => { indiv[cls] = (indiv[cls] || 0) + 1; }));
+          patterns = Object.keys(indiv).filter(k => indiv[k] >= threshold);
+        }
+        return children.filter(c => {
+          const cls = getClasses(c);
+          return patterns.some(p => p.split(' ').every(pc => !pc || cls.includes(pc)));
+        });
+      }
+      function directText(el) {
+        let text = '';
+        for (const n of el.childNodes) {
+          if (n.nodeType === 3) text += n.textContent;
+        }
+        return text.trim();
+      }
+      function extractRow(el, prefix) {
+        const data = {};
+        const tag = el.tagName.toLowerCase();
+        const cls = getClasses(el).slice(0, 2).join('.');
+        const key = prefix + '/' + (cls ? tag + '.' + cls : tag);
+        const dt = directText(el);
+        if (dt) data[key] = dt;
+        if (el.href) data[key + ' href'] = el.href;
+        if (el.src) data[key + ' src'] = el.src;
+        for (const child of el.children) {
+          Object.assign(data, extractRow(child, key));
+        }
+        return data;
+      }
+      const container = document.querySelector(${JSON.stringify(selector)});
+      if (!container) return { selector: ${JSON.stringify(selector)}, columns: [], rows: [], totalRows: 0, truncated: false };
+      const matching = getMatchingChildren(container);
+      const totalRows = matching.length;
+      const limited = matching.slice(0, ${maxRows});
+      const rawRows = limited.map(el => extractRow(el, ''));
+      // Collect all keys
+      const allKeys = new Set();
+      rawRows.forEach(r => Object.keys(r).forEach(k => allKeys.add(k)));
+      // Build columns: filter sparse + constant
+      const columns = [];
+      const keyToName = {};
+      for (const key of allKeys) {
+        const values = rawRows.map(r => r[key] || '');
+        const filled = values.filter(v => v).length;
+        const fillRate = filled / rawRows.length;
+        if (fillRate < 0.2) continue;
+        // Drop constant columns
+        const unique = new Set(values.filter(v => v));
+        if (unique.size <= 1 && rawRows.length > 2) continue;
+        // Smart name: shortest class or last path segment
+        const parts = key.split('/').filter(Boolean);
+        const last = parts[parts.length - 1] || key;
+        let name = last.replace(/^[a-z]+\\./, '').replace(/ (href|src)$/, ' $1').replace(/\\./g, ' ').trim() || last;
+        keyToName[key] = name;
+        columns.push({ name, path: key, fillRate: Math.round(fillRate * 100) / 100 });
+      }
+      // Deduplicate names
+      const nameCount = {};
+      columns.forEach(c => { nameCount[c.name] = (nameCount[c.name] || 0) + 1; });
+      const nameSeen = {};
+      columns.forEach(c => {
+        if (nameCount[c.name] > 1) {
+          nameSeen[c.name] = (nameSeen[c.name] || 0) + 1;
+          if (nameSeen[c.name] > 1) c.name = c.name + ' ' + nameSeen[c.name];
+        }
+      });
+      // Build final rows
+      const rows = rawRows.map(raw => {
+        const row = {};
+        columns.forEach(c => { row[c.name] = raw[c.path] || ''; });
+        return row;
+      });
+      return { selector: ${JSON.stringify(selector)}, columns, rows, totalRows, truncated: totalRows > ${maxRows} };
+    })()`);
+    return result.result || { selector, columns: [], rows: [], totalRows: 0, truncated: false };
+  };
+  smart.waitForNetworkIdle = async (opts) => {
+    const timeout = Math.min(opts?.timeout ?? 15e3, 3e4);
+    const idleTime = Math.min(opts?.idleTime ?? 500, 5e3);
+    const result = await bridge2.send({
+      type: "wait_for_network_idle",
+      timeout,
+      idleTime
+    }, timeout + 5e3);
+    return result;
+  };
+  smart.extractData = async (opts) => {
+    const maxTables = opts?.maxTables ?? 3;
+    const maxRowsPerTable = opts?.maxRowsPerTable ?? 200;
+    const status = await bridge2.send({ type: "get_connection_status" }, 5e3);
+    const url = status?.connectedTab?.url || "";
+    const candidates = await smart.detectTables({ maxCandidates: maxTables });
+    const tables = [];
+    for (const candidate of candidates) {
+      const extraction = await smart.extractTable(
+        candidate.selector,
+        { maxRows: maxRowsPerTable }
+      );
+      if (extraction.columns.length >= 2 && extraction.rows.length >= 2) {
+        tables.push(extraction);
+      }
+    }
+    const jsonLd = await evaluate(`(() => {
+      const scripts = document.querySelectorAll('script[type="application/ld+json"]');
+      const result = [];
+      scripts.forEach(s => { try { result.push(JSON.parse(s.textContent)); } catch {} });
+      return result;
+    })()`);
+    return {
+      tables,
+      structuredData: jsonLd.result || [],
+      url
+    };
+  };
   smart.scrollCapture = async (opts) => {
     const max = opts?.maxSections ?? 10;
     const px = opts?.pixelsPerScroll ?? 800;
@@ -3916,7 +4705,7 @@ async function buildSmartObject(bridge2) {
   smart.waitForIdle = async (timeout) => {
     const ms = Math.min(timeout ?? 5e3, 15e3);
     const result = await evaluate(`
-      await new Promise((resolve) => {
+      new Promise((resolve) => {
         let timer;
         const reset = () => { clearTimeout(timer); timer = setTimeout(() => resolve('idle'), 500); };
         const observer = new MutationObserver(reset);
@@ -3927,13 +4716,38 @@ async function buildSmartObject(bridge2) {
     `);
     return { status: result.result };
   };
-  smart.extractPage = async () => {
-    const [capture, perf, security, fonts, meta] = await Promise.all([
-      bridge2.send({ type: "capture_page" }, 6e4),
-      bridge2.send({ type: "get_performance_metrics" }, 5e3).catch(() => null),
-      bridge2.send({ type: "get_security_state" }, 5e3).catch(() => null),
-      bridge2.send({ type: "detect_fonts" }, 1e4).catch(() => null),
-      evaluate(`(() => {
+  smart.extractPage = async (opts) => {
+    const gaps = [];
+    const tracing = opts?.trace === true;
+    const traceSteps = [];
+    const traceStart = tracing ? Date.now() : 0;
+    const traced = async (name, fn) => {
+      if (!tracing) return fn();
+      const s = Date.now();
+      try {
+        const r = await fn();
+        traceSteps.push({ name, elapsed: Date.now() - s, success: true });
+        return r;
+      } catch (e) {
+        traceSteps.push({ name, elapsed: Date.now() - s, success: false });
+        throw e;
+      }
+    };
+    const [capture, perf, security, fonts, meta, rawA11y, rawMobile] = await Promise.all([
+      traced("capture_page", () => bridge2.send({ type: "capture_page" }, 6e4)),
+      traced("get_performance_metrics", () => bridge2.send({ type: "get_performance_metrics" }, 5e3)).catch((e) => {
+        gaps.push({ dimension: "performance", reason: e instanceof Error ? e.message : String(e), impact: "method-failed", reducesConfidence: true });
+        return null;
+      }),
+      traced("get_security_state", () => bridge2.send({ type: "get_security_state" }, 5e3)).catch((e) => {
+        gaps.push({ dimension: "security", reason: e instanceof Error ? e.message : String(e), impact: "method-failed", reducesConfidence: true });
+        return null;
+      }),
+      traced("detect_fonts", () => bridge2.send({ type: "detect_fonts" }, 1e4)).catch((e) => {
+        gaps.push({ dimension: "fonts", reason: e instanceof Error ? e.message : String(e), impact: "method-failed", reducesConfidence: false });
+        return null;
+      }),
+      traced("meta_extraction", () => evaluate(`(() => {
         const m = {}; document.querySelectorAll('meta').forEach(el => {
           const k = el.getAttribute('property') || el.getAttribute('name');
           if (k) m[k] = el.getAttribute('content');
@@ -3944,22 +4758,66 @@ async function buildSmartObject(bridge2) {
         m._headings = [...document.querySelectorAll('h1,h2,h3')].slice(0, 30).map(h => ({ level: h.tagName, text: h.textContent?.trim()?.substring(0, 120) }));
         m._nav = [...new Set([...document.querySelectorAll('nav a, header a')].map(a => a.href))].slice(0, 50);
         return m;
-      })()`)
+      })()`)),
+      traced("accessibility_tree", () => bridge2.send({ type: "get_accessibility_tree", depth: 3 }, 1e4)).catch((e) => {
+        gaps.push({ dimension: "accessibility", reason: e instanceof Error ? e.message : String(e), impact: "method-failed", reducesConfidence: false });
+        return null;
+      }),
+      traced("mobile_readiness", () => evaluate(`(() => {
+        const vp = document.querySelector('meta[name="viewport"]');
+        let mediaQueryCount = 0;
+        try { [...document.styleSheets].forEach(s => { try { [...s.cssRules].forEach(r => { if (r instanceof CSSMediaRule) mediaQueryCount++; }); } catch {} }); } catch {}
+        return {
+          hasViewportMeta: !!vp,
+          viewportContent: vp?.getAttribute('content') || null,
+          mediaQueryCount,
+          bodyScrollWidth: document.body?.scrollWidth ?? 0,
+          windowInnerWidth: window.innerWidth,
+          isOverflowing: (document.body?.scrollWidth ?? 0) > window.innerWidth,
+        };
+      })()`)).catch((e) => {
+        gaps.push({ dimension: "mobile-readiness", reason: e instanceof Error ? e.message : String(e), impact: "method-failed", reducesConfidence: false });
+        return null;
+      })
     ]);
-    return {
+    const a11ySummary = rawA11y ? summarizeAccessibility(rawA11y) : null;
+    const mobileResult = rawMobile;
+    const mobileReadiness = mobileResult?.result ? { hasViewportMeta: !!mobileResult.result.hasViewportMeta, viewportContent: mobileResult.result.viewportContent ?? null, mediaQueryCount: mobileResult.result.mediaQueryCount ?? 0, isOverflowing: !!mobileResult.result.isOverflowing } : null;
+    for (const gap of gaps) {
+      if (!sessionGaps.some((g) => g.dimension === gap.dimension)) sessionGaps.push(gap);
+    }
+    const evidence = {
       capture,
       performance: perf,
       security,
       fonts,
-      meta: meta?.result || null
+      meta: meta?.result || null,
+      accessibility: a11ySummary,
+      mobileReadiness,
+      gaps
     };
+    if (tracing) {
+      evidence._trace = {
+        method: "extractPage",
+        startedAt: traceStart,
+        elapsed: Date.now() - traceStart,
+        steps: traceSteps,
+        outcome: gaps.length === 0 ? "success" : "partial"
+      };
+    }
+    return evidence;
   };
-  smart.comparePages = async (urlA, urlB) => {
+  smart.comparePages = async (urlA, urlB, opts) => {
     await smart.navigate(urlA);
-    const a = await smart.extractPage();
+    const a = await smart.extractPage(opts);
     await smart.navigate(urlB);
-    const b = await smart.extractPage();
-    return { siteA: { url: urlA, ...a }, siteB: { url: urlB, ...b } };
+    const b = await smart.extractPage(opts);
+    const scaffold = buildComparisonScaffold(a, b);
+    return {
+      siteA: { url: urlA, ...a },
+      siteB: { url: urlB, ...b },
+      scaffold
+    };
   };
   return smart;
 }
@@ -4102,8 +4960,15 @@ function createCodeModeTools(bridge2, crawlio2) {
         "  smart.snapshot() \u2014 capture accessibility snapshot",
         "  smart.scrollCapture(opts?) \u2014 state-aware page scroll with screenshots, stops at page bottom",
         "  smart.waitForIdle(timeout?) \u2014 wait for DOM mutations to settle (500ms quiet window)",
-        "  smart.extractPage() \u2014 capture_page + perf metrics + security + fonts + meta/headings/structured data",
-        "  smart.comparePages(urlA, urlB) \u2014 navigate to each URL, run extractPage(), return structured pair",
+        "  smart.extractPage(opts?) \u2014 capture_page + perf + security + fonts + meta + accessibility + mobileReadiness. Returns { capture, performance, security, fonts, meta, accessibility, mobileReadiness, gaps[] }. opts: { trace: true } adds _trace.",
+        "  smart.comparePages(urlA, urlB, opts?) \u2014 navigate to each URL, run extractPage(), return { siteA, siteB, scaffold }. scaffold has dimensions[], sharedFields, missingFields, metrics.",
+        "  smart.finding({ claim, evidence, sourceUrl, confidence, method, dimension? }) \u2014 create validated Finding, accumulate in session. Confidence auto-capped if dimension has active gap with reducesConfidence.",
+        "  smart.findings() \u2014 return all accumulated Finding[] from current session.",
+        "  smart.clearFindings() \u2014 reset accumulated findings and session gaps.",
+        "  smart.detectTables(opts?) \u2014 find repeating data patterns in the page. Returns TableCandidate[] (selector, score, rowCount, sampleText). Uses class-frequency scoring.",
+        "  smart.extractTable(selector, opts?) \u2014 extract structured data from a container. Returns { columns, rows, totalRows, truncated }. opts: { maxRows: 200 }.",
+        "  smart.waitForNetworkIdle(opts?) \u2014 wait for all network requests to settle (CDP-level, catches fetch/XHR/images/CSS/fonts). Returns { status, elapsed }. opts: { timeout: 15000, idleTime: 500 }.",
+        "  smart.extractData(opts?) \u2014 compound: detectTables + extractTable + JSON-LD. Returns { tables, structuredData, url }.",
         "  Framework namespaces (injected based on detected framework):",
         "  smart.react.{getVersion,getRootCount,hasProfiler,isHookInstalled}",
         "  smart.vue.{getVersion,getAppCount,getConfig,isDevMode}",
@@ -4211,7 +5076,7 @@ function createCodeModeTools(bridge2, crawlio2) {
 process.title = "Crawlio Agent";
 var initMode = process.argv.includes("init") || process.argv.includes("--setup") || process.argv.includes("setup");
 if (initMode) {
-  const { runInit } = await import("./init-77AO6DDJ.js");
+  const { runInit } = await import("./init-ZLXCKEQB.js");
   await runInit(process.argv.slice(2));
   process.exit(0);
 }