@wenathlan/extension 1.1.37 → 1.1.38

package/extension/dist/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnOEjO8Z0PDgQyfvawGcaO2j+o0GLCFTLNj7TkYC/Avo9l2NenMRq7gp90Nfd7E9MViv/OMcCKSYZ5unv12QPRtv31C+a5UQWDFAOP/cH5mwMd6hsayElrSoW8ta+FwFqmr9dIFkn7cQEU3YhZr4Gcbs+ycUHOxVgDA4NBKB0rQ6e9VW5LvTw0isRYUrqM+M72vKxHk9zUIYYn/LGPvottKBYi2GLr0PHSeC2UE+Shmq7vcFIXj6hDjvD4kLJ5sKoUllEcZ1TPuBcnHUQ9ndKA5iktXDQOIJCUJmi7a0YJ2PGg7fvpYfT9k0ai/qZ+pIoRfoOEwE01bPoDn7NjeYnNQIDAQAB",
   "name": "Devthink",
-  "version": "1.1.37",
+  "version": "1.1.38",
   "description": "A consent-first bridge for reviewed browser-agent tasks.",
   "permissions": [
     "activeTab",

package/extension/dist/pagebridge.js

@@ -1,9 +1,9 @@
 "use strict";
 (() => {
   // policy.ts
-  var sensitiveactions = /* @__PURE__ */ new Set(["click", "type", "navigate", "select", "presskey", "drag", "drop", "upload", "clear", "check", "uncheck", "toggle", "submit", "reload", "back", "forward", "writestorage", "setattribute", "removeattribute", "evaluate", "tabcreate", "tabactivate", "tabclose", "tabreload", "windowcreate", "windowclose", "windowresize", "downloadfile", "clickpoint", "shiftclick", "dismissdialog", "enterframe", "typetime", "appendtext", "setvalue", "typeedit", "keyhold", "keyrelease", "submitsearch", "selectmulti", "chooseradio", "setslider", "setdate", "setcolor", "openlink", "openprivate", "reloadcache", "stopnav", "followlink", "spanav", "rewritequery", "setfragment", "navlist", "navprofile", "handleauth", "printpdf", "prefetch", "preconnect", "deeplink", "reopentab", "pausenav", "navrate", "openclipboard", "batchopen", "duplicatetab", "closepattern", "pintab", "mutetab", "movetab", "movetabwindow", "grouptabs", "colorgroup", "collapsegroup", "discardtab", "reloadtabs", "zoomin", "zoomout", "switchtab", "maximizewindow", "minimizewindow", "restorewindow", "focuswindow", "scratchwindow", "incognitowindow", "restoretab", "restorelayout", "reopenrun", "badgetab", "fillform", "filllabel", "fillplaceholder", "submitform", "retryform", "runwizard", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcard", "fillcode", "consentpassword"]);
+  var sensitiveactions = /* @__PURE__ */ new Set(["click", "type", "navigate", "select", "presskey", "drag", "drop", "upload", "clear", "check", "uncheck", "toggle", "submit", "reload", "back", "forward", "writestorage", "setattribute", "removeattribute", "evaluate", "tabcreate", "tabactivate", "tabclose", "tabreload", "windowcreate", "windowclose", "windowresize", "downloadfile", "clickpoint", "shiftclick", "dismissdialog", "enterframe", "typetime", "appendtext", "setvalue", "typeedit", "keyhold", "keyrelease", "submitsearch", "selectmulti", "chooseradio", "setslider", "setdate", "setcolor", "openlink", "openprivate", "reloadcache", "stopnav", "followlink", "spanav", "rewritequery", "setfragment", "navlist", "navprofile", "handleauth", "printpdf", "prefetch", "preconnect", "deeplink", "reopentab", "pausenav", "navrate", "openclipboard", "batchopen", "duplicatetab", "closepattern", "pintab", "mutetab", "movetab", "movetabwindow", "grouptabs", "colorgroup", "collapsegroup", "discardtab", "reloadtabs", "zoomin", "zoomout", "switchtab", "maximizewindow", "minimizewindow", "restorewindow", "focuswindow", "scratchwindow", "incognitowindow", "restoretab", "restorelayout", "reopenrun", "badgetab", "fillform", "filllabel", "fillplaceholder", "submitform", "retryform", "runwizard", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcard", "fillcode", "consentpassword", "exportcsv", "exportjson", "exportexcel", "copytable", "pushsheets", "streamdisk", "paginateextract", "resumeextract"]);
   var interactionactions = /* @__PURE__ */ new Set(["focus", "scroll", "hover", "clickdeep", "rightclick", "doubleclick", "scrollpage", "scrollby", "scrollend", "scrolltop", "fullscreen", "zoomset", "movepointer", "clicktext", "clickaria", "clickname", "expanddetails", "pierceshadow", "retryaction"]);
-  var readactions = /* @__PURE__ */ new Set(["observe", "inspect", "extract", "wait", "waitfor", "waittext", "readattribute", "readstyle", "readgeometry", "readvalue", "readtext", "readhtml", "countelements", "readtable", "readlinks", "readimages", "readmeta", "readforms", "readstorage", "highlight", "tablist", "windowlist", "tabsnapshot", "mapclicks", "verifyvisible", "verifyenabled", "resolvexpath", "a11ytree", "readvisible", "readertree", "detectlists", "detecttables", "readjson", "watchmutate", "waitquiet", "watchbanner", "detectinfinitescroll", "detectvirtual", "detectlazy", "readscrollpos", "readlang", "readoutline", "countpages", "listshadow", "listframes", "classifypage", "fingerprintsection", "diffsnapshots", "readselection", "watchfocus", "detectsticky", "detectscrolllock", "readopengraph", "detectlanguage", "deriveselector", "waitload", "waiturl", "spawait", "detecthttp", "readredirects", "readfinalurl", "trailaudit", "navintent", "checksafe", "querytabs", "watchtab", "findclones", "searchtabs", "listaudio", "snapshotsession", "savelayout", "attachmeta", "detectfields", "generatevalues", "saveprofiles", "asksubmit", "readerrors", "skiphoneypot", "detectlogin", "detecttemplate", "handoffcaptcha"]);
+  var readactions = /* @__PURE__ */ new Set(["observe", "inspect", "extract", "wait", "waitfor", "waittext", "readattribute", "readstyle", "readgeometry", "readvalue", "readtext", "readhtml", "countelements", "readtable", "readlinks", "readimages", "readmeta", "readforms", "readstorage", "highlight", "tablist", "windowlist", "tabsnapshot", "mapclicks", "verifyvisible", "verifyenabled", "resolvexpath", "a11ytree", "readvisible", "readertree", "detectlists", "detecttables", "readjson", "watchmutate", "waitquiet", "watchbanner", "detectinfinitescroll", "detectvirtual", "detectlazy", "readscrollpos", "readlang", "readoutline", "countpages", "listshadow", "listframes", "classifypage", "fingerprintsection", "diffsnapshots", "readselection", "watchfocus", "detectsticky", "detectscrolllock", "readopengraph", "detectlanguage", "deriveselector", "waitload", "waiturl", "spawait", "detecthttp", "readredirects", "readfinalurl", "trailaudit", "navintent", "checksafe", "querytabs", "watchtab", "findclones", "searchtabs", "listaudio", "snapshotsession", "savelayout", "attachmeta", "detectfields", "generatevalues", "saveprofiles", "asksubmit", "readerrors", "skiphoneypot", "detectlogin", "detecttemplate", "handoffcaptcha", "scrapetable", "importcsv", "looprows", "transformvalues", "deduperows", "mergepages", "stamplerows", "previewgrid", "logprovenance"]);
   var allowedactions = /* @__PURE__ */ new Set([...sensitiveactions, ...interactionactions, ...readactions]);
   function parseoptions(step) {
     if (step.options === void 0) return {};
@@ -2934,6 +2934,189 @@
     }
   }
 
+  // extension/pagedata.ts
+  function normalizeheader(label) {
+    const slug = label.trim().toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "-").replace(/^-+|-+$/g, "");
+    return slug || "column";
+  }
+  function columnspecof(label, used = /* @__PURE__ */ new Set()) {
+    const base = normalizeheader(label);
+    let key = base;
+    let suffix = 2;
+    while (used.has(key)) {
+      key = `${base}${suffix}`;
+      suffix += 1;
+    }
+    used.add(key);
+    return { key, label: label.trim(), kind: "text", normalized: label.trim().toLowerCase() };
+  }
+  function classifycolumn(values) {
+    const present = values.filter((value) => value.trim().length > 0);
+    if (present.length === 0) return "text";
+    return present.every((value) => Number.isFinite(Number(value.replace(/,/g, ".")))) ? "number" : "text";
+  }
+  function expandspans(rows) {
+    const filled = [];
+    const pending = /* @__PURE__ */ new Map();
+    for (let index = 0; index < rows.length; index += 1) {
+      const row = filled[index] ?? (filled[index] = []);
+      let column = 0;
+      for (const cell of rows[index].cells) {
+        while (row[column] !== void 0 || pending.has(`${index},${column}`)) {
+          if (row[column] === void 0) row[column] = pending.get(`${index},${column}`);
+          column += 1;
+        }
+        row[column] = cell.text;
+        for (let spanrow = 0; spanrow < Math.max(1, cell.rowspan); spanrow += 1) {
+          for (let spancol = 0; spancol < Math.max(1, cell.colspan); spancol += 1) {
+            if (spanrow === 0 && spancol === 0) continue;
+            pending.set(`${index + spanrow},${column + spancol}`, cell.text);
+          }
+        }
+        column += Math.max(1, cell.colspan);
+      }
+    }
+    for (const [key, value] of pending) {
+      const [rowpart, columnpart] = key.split(",");
+      const rowindex = Number.parseInt(rowpart ?? "0", 10);
+      const columnindex = Number.parseInt(columnpart ?? "0", 10);
+      const target = filled[rowindex] ?? (filled[rowindex] = []);
+      if (target[columnindex] === void 0) target[columnindex] = value;
+    }
+    const width = filled.reduce((largest, row) => Math.max(largest, row.length), 0);
+    return filled.map((row) => Array.from({ length: width }, (_, column) => row[column] ?? ""));
+  }
+  function readgrid(rows) {
+    const headerindex = rows.findIndex((row) => row.header || (row.cells[0]?.header ?? false));
+    const useheader = headerindex !== -1 ? headerindex : 0;
+    const hasheader = headerindex !== -1 || rows.length > 0;
+    const grid = expandspans(rows);
+    const headercells = hasheader ? grid[useheader] ?? [] : [];
+    const used = /* @__PURE__ */ new Set();
+    const columns = Array.from({ length: headercells.length || (grid[0]?.length ?? 0) }, (_, index) => columnspecof(headercells[index] ?? `column${index + 1}`, used));
+    const bodyrows = grid.filter((_, index) => hasheader ? index !== useheader : true).filter((line) => line.some((value) => value.trim().length > 0)).map((line) => {
+      const row = {};
+      columns.forEach((column, index) => {
+        row[column.key] = line[index] ?? "";
+      });
+      return row;
+    });
+    for (const column of columns) column.kind = classifycolumn(bodyrows.map((row) => row[column.key] ?? ""));
+    const children = [];
+    rows.forEach((row, rowindex) => {
+      if (hasheader && rowindex === useheader) return;
+      const parentrow = hasheader && rowindex > useheader ? rowindex - 1 : rowindex;
+      row.cells.forEach((cell) => {
+        if (!cell.nested) return;
+        const child = readgrid(cell.nested.rows);
+        children.push({ parentrow, selector: cell.nested.selector, columns: child.columns, rows: child.rows });
+      });
+    });
+    return { columns, rows: bodyrows, children };
+  }
+  function rowsfresh(previous, current) {
+    if (current.length === 0) return false;
+    const known = new Set(previous.map((row) => JSON.stringify(row)));
+    return current.some((row) => !known.has(JSON.stringify(row)));
+  }
+  function mergedatasets(datasets) {
+    const columns = [];
+    const seen = /* @__PURE__ */ new Set();
+    for (const dataset of datasets) {
+      for (const column of dataset.columns) {
+        if (seen.has(column.key)) continue;
+        seen.add(column.key);
+        columns.push(column);
+      }
+    }
+    const rows = datasets.flatMap((dataset) => dataset.rows.map((row) => {
+      const merged = {};
+      for (const column of columns) merged[column.key] = row[column.key] ?? "";
+      return merged;
+    }));
+    return { columns, rows };
+  }
+  function collectrowshapes(table) {
+    return [...table.rows].map((row) => ({
+      header: [...row.cells].every((cell) => cell.tagName === "TH") && row.cells.length > 0,
+      cells: [...row.cells].map((cell) => {
+        const nested = cell.querySelector("table");
+        return {
+          text: clean(nested ? `${nested.rows.length} rows` : cell.textContent ?? ""),
+          header: cell.tagName === "TH",
+          rowspan: cell.rowSpan,
+          colspan: cell.colSpan,
+          ...nested instanceof HTMLTableElement ? { nested: { selector: elementselector(nested), rows: collectrowshapes(nested) } } : {}
+        };
+      })
+    }));
+  }
+  function readtable(target, root, fallbackselector) {
+    const table = target instanceof HTMLTableElement ? target : fallbackselector ? root.querySelector(fallbackselector) : root.querySelector("table");
+    if (!table) return null;
+    return readgrid(collectrowshapes(table));
+  }
+  async function runpagedata(step, target, root = document) {
+    let options = {};
+    try {
+      options = parseoptions(step);
+    } catch {
+      options = {};
+    }
+    if (step.kind === "scrapetable") {
+      const grid = readtable(target, root, step.target);
+      if (!grid) return { ok: false, summary: "The reviewed table selector matches no table element." };
+      const rowlimit = typeof options.rowlimit === "number" && Number.isInteger(options.rowlimit) && options.rowlimit > 0 ? options.rowlimit : grid.rows.length;
+      const limited = grid.rows.slice(0, rowlimit);
+      return {
+        ok: true,
+        summary: `Scraped ${limited.length} row${limited.length === 1 ? "" : "s"} into ${grid.columns.length} normalized column${grid.columns.length === 1 ? "" : "s"}${grid.children.length > 0 ? ` with ${grid.children.length} nested child table${grid.children.length === 1 ? "" : "s"}` : ""}.`,
+        details: { grid: { columns: grid.columns, rows: limited, children: grid.children }, rows: limited.length, columns: grid.columns.length }
+      };
+    }
+    if (step.kind === "paginateextract") {
+      const nextselector = typeof options.next === "string" ? options.next : "";
+      const pages = typeof options.pages === "number" && Number.isInteger(options.pages) && options.pages > 0 ? options.pages : 1;
+      const wait5 = typeof options.wait === "number" && Number.isFinite(options.wait) && options.wait > 0 ? options.wait : 0;
+      const cursor = typeof options.cursor === "number" && Number.isInteger(options.cursor) && options.cursor > 0 ? options.cursor : 0;
+      const grids = [];
+      let previous = [];
+      for (let page = 0; page < pages + cursor; page += 1) {
+        if (page < cursor) {
+          const control = root.querySelector(nextselector);
+          if (!control) break;
+          control.click();
+          await new Promise((resolve) => window.setTimeout(resolve, 0));
+          continue;
+        }
+        if (page > cursor) {
+          const control = root.querySelector(nextselector);
+          if (!control) break;
+          control.click();
+          const deadline = Date.now() + wait5;
+          let fresh = false;
+          while (!fresh && Date.now() < deadline) {
+            await new Promise((resolve) => window.setTimeout(resolve, Math.min(100, Math.max(16, deadline - Date.now()))));
+            const probe = readtable(target, root, step.target);
+            fresh = probe !== null && rowsfresh(previous, probe.rows);
+          }
+        }
+        const grid = readtable(target, root, step.target);
+        if (!grid) return { ok: false, summary: "The reviewed table selector matches no table element." };
+        grids.push(grid);
+        previous = grid.rows;
+      }
+      const merged = mergedatasets(grids);
+      const hasnext = Boolean(root.querySelector(nextselector));
+      return {
+        ok: grids.length > 0,
+        summary: grids.length > 0 ? `Followed ${grids.length} page${grids.length === 1 ? "" : "s"} of the reviewed table into ${merged.rows.length} row${merged.rows.length === 1 ? "" : "s"}${hasnext ? "; a next control remains" : ""}.` : "No page of the reviewed table was extracted.",
+        details: { grid: { columns: merged.columns, rows: merged.rows, children: grids.flatMap((grid) => grid.children) }, pages: grids.length, rows: merged.rows.length, next: hasnext }
+      };
+    }
+    return { ok: false, summary: "Unsupported data step." };
+  }
+
   // extension/pagebridge.ts
   function stepoptions3(step) {
     if (!step.options) return {};
@@ -3043,6 +3226,7 @@
   var navstepkinds = /* @__PURE__ */ new Set(["waitload", "waiturl", "followlink", "spanav", "spawait", "rewritequery", "setfragment", "stopnav", "prefetch", "preconnect", "printpdf"]);
   var formkinds = /* @__PURE__ */ new Set(["fillform", "filllabel", "fillplaceholder", "detectfields", "generatevalues", "readerrors", "skiphoneypot", "detectlogin", "detecttemplate", "handoffcaptcha", "asksubmit", "submitform", "consentpassword", "attachfile"]);
   var wizardkinds = /* @__PURE__ */ new Set(["runwizard", "selectchain", "picktypeahead", "pickdate", "fillcard", "fillcode"]);
+  var datastepkinds = /* @__PURE__ */ new Set(["scrapetable", "paginateextract"]);
   async function performstep(step, expectedorigin, rootdocument = document) {
     if (location.origin !== expectedorigin) return { ok: false, summary: "Page origin changed before action." };
     if (step.kind === "observe") return { ok: true, summary: "Observation completed." };
@@ -3091,6 +3275,7 @@
     let result;
     if (formkinds.has(step.kind)) result = runpageform(step, element, rootdocument);
     else if (wizardkinds.has(step.kind)) result = runpagewizard(step, element, rootdocument);
+    else if (datastepkinds.has(step.kind)) result = runpagedata(step, element, rootdocument);
     else if (readkinds.has(step.kind)) result = runpageread(step, element, rootdocument);
     else if (controlkinds.has(step.kind)) result = runpagecontrol(step, element, rootdocument);
     else if (interactkinds.has(step.kind)) return await runinteractstep(step, expectedorigin, performstep);