@wenathlan/extension 1.1.37 → 1.1.38

package/extension/dist/background.js

@@ -520,21 +520,122 @@ var sessionmemory = class {
   async getcodevalue() {
     return this.adapter.get("codevalue");
   }
+  /** Stores one dataset with its column specs and rows, replacing the previous record of that id. */
+  async setdataset(value) {
+    await this.adapter.set(`dataset${value.id}`, value);
+    const ids = (await this.adapter.get("datasets") ?? []).filter((id) => id !== value.id);
+    await this.adapter.set("datasets", [value.id, ...ids]);
+  }
+  /** Returns one stored dataset by its id with its column specs and row count. */
+  async getdataset(id) {
+    return this.adapter.get(`dataset${id}`);
+  }
+  /** Returns every stored dataset id, newest first. */
+  async getdatasets() {
+    const ids = await this.adapter.get("datasets") ?? [];
+    const records = [];
+    for (const id of ids) {
+      const record2 = await this.adapter.get(`dataset${id}`);
+      if (record2) records.push(record2);
+    }
+    return records;
+  }
+  /** Stores one imported csv dataset for fill loops beside the dataset store. */
+  async addimport(value) {
+    await this.setdataset(value);
+    const ids = (await this.adapter.get("imports") ?? []).filter((id) => id !== value.id);
+    await this.adapter.set("imports", [value.id, ...ids]);
+  }
+  /** Returns every imported csv dataset for fill loops, newest first. */
+  async getimports() {
+    const ids = await this.adapter.get("imports") ?? [];
+    const records = [];
+    for (const id of ids) {
+      const record2 = await this.adapter.get(`dataset${id}`);
+      if (record2) records.push(record2);
+    }
+    return records;
+  }
+  /** Stores one extraction session with its cursor and page history, replacing the previous session of that id. */
+  async setextractsession(value) {
+    await this.adapter.set(`extract${value.id}`, value);
+    const ids = (await this.adapter.get("extracts") ?? []).filter((id) => id !== value.id);
+    await this.adapter.set("extracts", [value.id, ...ids]);
+  }
+  /** Returns every extraction session with its cursor and page history, newest first. */
+  async getextractsessions() {
+    const ids = await this.adapter.get("extracts") ?? [];
+    const records = [];
+    for (const id of ids) {
+      const record2 = await this.adapter.get(`extract${id}`);
+      if (record2) records.push(record2);
+    }
+    return records;
+  }
+  /** Records one provenance record of an exported artifact. */
+  async addprovenance(record2) {
+    const records = await this.getprovenances();
+    await this.adapter.set("provenances", [record2, ...records]);
+  }
+  /** Returns every provenance record per exported artifact, newest first. */
+  async getprovenances() {
+    return await this.adapter.get("provenances") ?? [];
+  }
+  /** Stores the transform rules and dedupe keys of one task, replacing the previous record of that task. */
+  async settaskrules(value) {
+    const records = (await this.adapter.get("taskrules") ?? []).filter((item) => item.taskid !== value.taskid);
+    await this.adapter.set("taskrules", [value, ...records]);
+  }
+  /** Returns the transform rules and dedupe keys per task, newest first. */
+  async gettaskrules() {
+    return await this.adapter.get("taskrules") ?? [];
+  }
+  /** Stores one stream chunk state for resume, replacing the previous state of that dataset. */
+  async setstream(state) {
+    const records = (await this.adapter.get("streams") ?? []).filter((item) => item.datasetid !== state.datasetid);
+    await this.adapter.set("streams", [state, ...records]);
+  }
+  /** Returns every stream chunk state persisted for resume, newest first. */
+  async getstreams() {
+    return await this.adapter.get("streams") ?? [];
+  }
+  /** Stores one reviewed sheet endpoint config behind its origin grant, replacing the previous config of that origin. */
+  async setsheetendpoint(config) {
+    const records = (await this.adapter.get("sheetendpoints") ?? []).filter((item) => item.origin !== config.origin);
+    await this.adapter.set("sheetendpoints", [config, ...records]);
+  }
+  /** Returns every reviewed sheet endpoint config, newest first. */
+  async getsheetendpoints() {
+    return await this.adapter.get("sheetendpoints") ?? [];
+  }
+  /** Records one exported data artifact; artifact retention is a user setting and an absent setting keeps every artifact. */
+  async addexport(artifact) {
+    const records = await this.getexports();
+    const combined = [artifact, ...records.filter((item) => item.id !== artifact.id)];
+    const retention = (await this.getsettings())?.artifactretention;
+    await this.adapter.set("exports", retention === void 0 ? combined : combined.slice(0, retention));
+  }
+  /** Returns every exported data artifact with its content and checksum, newest first. */
+  async getexports() {
+    return await this.adapter.get("exports") ?? [];
+  }
 };
 function randomid() {
   return crypto.randomUUID();
 }
 
 // policy.ts
-var sensitiveactions = /* @__PURE__ */ new Set(["click", "type", "navigate", "select", "presskey", "drag", "drop", "upload", "clear", "check", "uncheck", "toggle", "submit", "reload", "back", "forward", "writestorage", "setattribute", "removeattribute", "evaluate", "tabcreate", "tabactivate", "tabclose", "tabreload", "windowcreate", "windowclose", "windowresize", "downloadfile", "clickpoint", "shiftclick", "dismissdialog", "enterframe", "typetime", "appendtext", "setvalue", "typeedit", "keyhold", "keyrelease", "submitsearch", "selectmulti", "chooseradio", "setslider", "setdate", "setcolor", "openlink", "openprivate", "reloadcache", "stopnav", "followlink", "spanav", "rewritequery", "setfragment", "navlist", "navprofile", "handleauth", "printpdf", "prefetch", "preconnect", "deeplink", "reopentab", "pausenav", "navrate", "openclipboard", "batchopen", "duplicatetab", "closepattern", "pintab", "mutetab", "movetab", "movetabwindow", "grouptabs", "colorgroup", "collapsegroup", "discardtab", "reloadtabs", "zoomin", "zoomout", "switchtab", "maximizewindow", "minimizewindow", "restorewindow", "focuswindow", "scratchwindow", "incognitowindow", "restoretab", "restorelayout", "reopenrun", "badgetab", "fillform", "filllabel", "fillplaceholder", "submitform", "retryform", "runwizard", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcard", "fillcode", "consentpassword"]);
+var sensitiveactions = /* @__PURE__ */ new Set(["click", "type", "navigate", "select", "presskey", "drag", "drop", "upload", "clear", "check", "uncheck", "toggle", "submit", "reload", "back", "forward", "writestorage", "setattribute", "removeattribute", "evaluate", "tabcreate", "tabactivate", "tabclose", "tabreload", "windowcreate", "windowclose", "windowresize", "downloadfile", "clickpoint", "shiftclick", "dismissdialog", "enterframe", "typetime", "appendtext", "setvalue", "typeedit", "keyhold", "keyrelease", "submitsearch", "selectmulti", "chooseradio", "setslider", "setdate", "setcolor", "openlink", "openprivate", "reloadcache", "stopnav", "followlink", "spanav", "rewritequery", "setfragment", "navlist", "navprofile", "handleauth", "printpdf", "prefetch", "preconnect", "deeplink", "reopentab", "pausenav", "navrate", "openclipboard", "batchopen", "duplicatetab", "closepattern", "pintab", "mutetab", "movetab", "movetabwindow", "grouptabs", "colorgroup", "collapsegroup", "discardtab", "reloadtabs", "zoomin", "zoomout", "switchtab", "maximizewindow", "minimizewindow", "restorewindow", "focuswindow", "scratchwindow", "incognitowindow", "restoretab", "restorelayout", "reopenrun", "badgetab", "fillform", "filllabel", "fillplaceholder", "submitform", "retryform", "runwizard", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcard", "fillcode", "consentpassword", "exportcsv", "exportjson", "exportexcel", "copytable", "pushsheets", "streamdisk", "paginateextract", "resumeextract"]);
 var interactionactions = /* @__PURE__ */ new Set(["focus", "scroll", "hover", "clickdeep", "rightclick", "doubleclick", "scrollpage", "scrollby", "scrollend", "scrolltop", "fullscreen", "zoomset", "movepointer", "clicktext", "clickaria", "clickname", "expanddetails", "pierceshadow", "retryaction"]);
-var readactions = /* @__PURE__ */ new Set(["observe", "inspect", "extract", "wait", "waitfor", "waittext", "readattribute", "readstyle", "readgeometry", "readvalue", "readtext", "readhtml", "countelements", "readtable", "readlinks", "readimages", "readmeta", "readforms", "readstorage", "highlight", "tablist", "windowlist", "tabsnapshot", "mapclicks", "verifyvisible", "verifyenabled", "resolvexpath", "a11ytree", "readvisible", "readertree", "detectlists", "detecttables", "readjson", "watchmutate", "waitquiet", "watchbanner", "detectinfinitescroll", "detectvirtual", "detectlazy", "readscrollpos", "readlang", "readoutline", "countpages", "listshadow", "listframes", "classifypage", "fingerprintsection", "diffsnapshots", "readselection", "watchfocus", "detectsticky", "detectscrolllock", "readopengraph", "detectlanguage", "deriveselector", "waitload", "waiturl", "spawait", "detecthttp", "readredirects", "readfinalurl", "trailaudit", "navintent", "checksafe", "querytabs", "watchtab", "findclones", "searchtabs", "listaudio", "snapshotsession", "savelayout", "attachmeta", "detectfields", "generatevalues", "saveprofiles", "asksubmit", "readerrors", "skiphoneypot", "detectlogin", "detecttemplate", "handoffcaptcha"]);
+var readactions = /* @__PURE__ */ new Set(["observe", "inspect", "extract", "wait", "waitfor", "waittext", "readattribute", "readstyle", "readgeometry", "readvalue", "readtext", "readhtml", "countelements", "readtable", "readlinks", "readimages", "readmeta", "readforms", "readstorage", "highlight", "tablist", "windowlist", "tabsnapshot", "mapclicks", "verifyvisible", "verifyenabled", "resolvexpath", "a11ytree", "readvisible", "readertree", "detectlists", "detecttables", "readjson", "watchmutate", "waitquiet", "watchbanner", "detectinfinitescroll", "detectvirtual", "detectlazy", "readscrollpos", "readlang", "readoutline", "countpages", "listshadow", "listframes", "classifypage", "fingerprintsection", "diffsnapshots", "readselection", "watchfocus", "detectsticky", "detectscrolllock", "readopengraph", "detectlanguage", "deriveselector", "waitload", "waiturl", "spawait", "detecthttp", "readredirects", "readfinalurl", "trailaudit", "navintent", "checksafe", "querytabs", "watchtab", "findclones", "searchtabs", "listaudio", "snapshotsession", "savelayout", "attachmeta", "detectfields", "generatevalues", "saveprofiles", "asksubmit", "readerrors", "skiphoneypot", "detectlogin", "detecttemplate", "handoffcaptcha", "scrapetable", "importcsv", "looprows", "transformvalues", "deduperows", "mergepages", "stamplerows", "previewgrid", "logprovenance"]);
 var allowedactions = /* @__PURE__ */ new Set([...sensitiveactions, ...interactionactions, ...readactions]);
 var watchactions = /* @__PURE__ */ new Set(["watchmutate", "watchbanner", "watchfocus", "watchtab"]);
-var targetactions = /* @__PURE__ */ new Set(["inspect", "focus", "click", "type", "scroll", "select", "hover", "clickdeep", "rightclick", "doubleclick", "drag", "drop", "upload", "clear", "check", "uncheck", "toggle", "submit", "readattribute", "readstyle", "readgeometry", "readvalue", "readtext", "readhtml", "countelements", "readtable", "highlight", "setattribute", "removeattribute", "waitfor", "shiftclick", "typetime", "appendtext", "setvalue", "typeedit", "submitsearch", "selectmulti", "chooseradio", "setslider", "setdate", "setcolor", "expanddetails", "verifyvisible", "verifyenabled", "pierceshadow", "deriveselector", "fingerprintsection", "submitform", "retryform", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcode", "consentpassword"]);
+var targetactions = /* @__PURE__ */ new Set(["inspect", "focus", "click", "type", "scroll", "select", "hover", "clickdeep", "rightclick", "doubleclick", "drag", "drop", "upload", "clear", "check", "uncheck", "toggle", "submit", "readattribute", "readstyle", "readgeometry", "readvalue", "readtext", "readhtml", "countelements", "readtable", "highlight", "setattribute", "removeattribute", "waitfor", "shiftclick", "typetime", "appendtext", "setvalue", "typeedit", "submitsearch", "selectmulti", "chooseradio", "setslider", "setdate", "setcolor", "expanddetails", "verifyvisible", "verifyenabled", "pierceshadow", "deriveselector", "fingerprintsection", "submitform", "retryform", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcode", "consentpassword", "scrapetable", "paginateextract"]);
 var valueactions = /* @__PURE__ */ new Set(["presskey", "drag", "drop", "upload", "readattribute", "removeattribute", "waittext", "evaluate", "zoomset", "tabactivate", "tabclose", "tabreload", "windowclose", "windowresize", "tabcreate", "windowcreate", "downloadfile", "typetime", "appendtext", "setvalue", "typeedit", "keyhold", "keyrelease", "chooseradio", "setslider", "setdate", "setcolor", "followlink", "setfragment", "handleauth", "navintent", "openclipboard", "checksafe", "reopentab", "spanav", "duplicatetab", "pintab", "mutetab", "movetab", "movetabwindow", "searchtabs", "badgetab", "attachmeta", "focuswindow", "maximizewindow", "minimizewindow", "restorewindow", "incognitowindow", "asksubmit", "selectchain", "picktypeahead", "pickdate", "attachfile", "fillcode", "consentpassword"]);
 var tabscommandactions = /* @__PURE__ */ new Set(["querytabs", "duplicatetab", "closepattern", "pintab", "mutetab", "movetab", "movetabwindow", "grouptabs", "colorgroup", "collapsegroup", "discardtab", "reloadtabs", "zoomin", "zoomout", "watchtab", "switchtab", "maximizewindow", "minimizewindow", "restorewindow", "focuswindow", "scratchwindow", "incognitowindow", "restoretab", "savelayout", "restorelayout", "findclones", "searchtabs", "badgetab", "attachmeta", "listaudio", "reopenrun", "snapshotsession"]);
 var formactions = /* @__PURE__ */ new Set(["fillform", "filllabel", "fillplaceholder", "detectfields", "generatevalues", "saveprofiles", "asksubmit", "submitform", "readerrors", "retryform", "runwizard", "selectchain", "picktypeahead", "pickdate", "attachfile", "handoffcaptcha", "fillcard", "fillcode", "consentpassword", "skiphoneypot", "detectlogin", "detecttemplate"]);
+var datasetactions = /* @__PURE__ */ new Set(["scrapetable", "exportcsv", "exportjson", "exportexcel", "copytable", "pushsheets", "importcsv", "looprows", "transformvalues", "deduperows", "paginateextract", "mergepages", "stamplerows", "previewgrid", "streamdisk", "resumeextract", "logprovenance"]);
+var exportactions = /* @__PURE__ */ new Set(["exportcsv", "exportjson", "exportexcel", "copytable", "pushsheets", "streamdisk"]);
 var fieldkinds = ["text", "email", "phone", "date", "number", "select", "check", "radio", "file", "password", "card", "code"];
 var layoutmutationactions = /* @__PURE__ */ new Set(["grouptabs", "colorgroup", "collapsegroup", "savelayout", "restorelayout"]);
 var groupcolors = ["grey", "blue", "red", "yellow", "green", "pink", "purple", "cyan", "orange"];
@@ -569,6 +670,7 @@ function requiredcapability(kind) {
   if (kind === "tablist") return "tabs";
   if (kind === "downloadfile") return "downloads";
   if (kind === "openclipboard") return "clipboardRead";
+  if (kind === "copytable") return "clipboardWrite";
   if (kind === "openlink" || kind === "openprivate" || kind === "navlist" || kind === "batchopen" || kind === "reopentab" || kind === "deeplink") return "tabs";
   if (tabscommandactions.has(kind)) return "tabs";
   return void 0;
@@ -582,6 +684,16 @@ function islayoutkind(kind) {
 function isformkind(kind) {
   return formactions.has(kind);
 }
+function isdatasetkind(kind) {
+  return datasetactions.has(kind);
+}
+function isexportkind(kind) {
+  return exportactions.has(kind);
+}
+function exportgranted(session, origin) {
+  if (!origingranted(session, origin)) return { allowed: false, reason: `The export of extracted data from ${origin} needs the session origin grants before it leaves local memory.` };
+  return { allowed: true };
+}
 function validatefieldmatch(value) {
   if (!value || typeof value !== "object" || Array.isArray(value)) return { allowed: false, reason: "A reviewed field match is required in options." };
   const match = value;
@@ -679,6 +791,89 @@ function validateformgrammar(step, options) {
   if (kind === "consentpassword" && !isnonempty(options.consentref)) return { allowed: false, reason: "A reviewed consent ref is required in options before any password is filled." };
   return { allowed: true };
 }
+function validatetransformrule(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return { allowed: false, reason: "A reviewed transform rule with an expression, sources and a target is required in options." };
+  const rule = value;
+  const expression = rule.expression;
+  if (typeof expression !== "string" || !/^(trim|upper|lower|number|prefix|suffix|replace)(?::.+)?$/.test(expression)) return { allowed: false, reason: "The reviewed transform expression must be trim, upper, lower, number, prefix, suffix or replace with an optional argument." };
+  if (expression.startsWith("replace") && !expression.slice("replace".length).includes("=>")) return { allowed: false, reason: "The reviewed replace expression needs the from=>to separator." };
+  if (expression.startsWith("replace") && expression.slice("replace:".length).split("=>")[0] === "") return { allowed: false, reason: "The reviewed replace expression needs a non-empty from part." };
+  if (!Array.isArray(rule.sources) || rule.sources.length === 0 || !rule.sources.every((source) => isnonempty(source))) return { allowed: false, reason: "Every reviewed transform rule needs a non-empty list of source columns." };
+  if (!isnonempty(rule.target)) return { allowed: false, reason: "Every reviewed transform rule needs a non-empty target column." };
+  return { allowed: true };
+}
+function validatedatasetids(options, key) {
+  const ids = options[key];
+  if (!Array.isArray(ids) || ids.length === 0 || !ids.every((id) => isnonempty(id))) return { allowed: false, reason: `A reviewed non-empty list of dataset ids is required in options as ${key}.` };
+  return { allowed: true };
+}
+function validatedatagrammar(step, options, origin) {
+  const kind = step.kind;
+  if (kind === "scrapetable") {
+    if (options.name !== void 0 && !isnonempty(options.name)) return { allowed: false, reason: "The reviewed dataset name must be a non-empty string." };
+    if (options.rowlimit !== void 0 && (typeof options.rowlimit !== "number" || !Number.isInteger(options.rowlimit) || options.rowlimit < 1)) return { allowed: false, reason: "The reviewed row limit must be a positive integer with no code ceiling." };
+  }
+  if (kind === "paginateextract") {
+    if (!isnonempty(options.next)) return { allowed: false, reason: "A reviewed next control selector is required in options." };
+    if (options.pages !== void 0 && (typeof options.pages !== "number" || !Number.isInteger(options.pages) || options.pages < 1)) return { allowed: false, reason: "The reviewed page count must be a positive integer with no code ceiling." };
+    if (!nonnegativeoption(options, "wait")) return { allowed: false, reason: "The reviewed row freshness wait must be zero or a positive number of milliseconds." };
+  }
+  if (kind === "exportcsv" || kind === "exportjson" || kind === "exportexcel" || kind === "copytable" || kind === "streamdisk") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    if (options.name !== void 0 && !isnonempty(options.name)) return { allowed: false, reason: "The reviewed artifact name must be a non-empty string." };
+  }
+  if (kind === "exportcsv" && options.delimiter !== void 0 && (typeof options.delimiter !== "string" || options.delimiter.length !== 1)) return { allowed: false, reason: "The reviewed csv delimiter must be a single character." };
+  if (kind === "streamdisk" && (typeof options.chunk !== "number" || !Number.isInteger(options.chunk) || options.chunk < 1)) return { allowed: false, reason: "The reviewed streaming chunk size must be a positive integer with no code ceiling." };
+  if (kind === "pushsheets") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    if (!isnonempty(options.sheet)) return { allowed: false, reason: "A reviewed sheet endpoint url is required in options." };
+    if (!ishttpsurl(options.sheet)) return { allowed: false, reason: "The reviewed sheet endpoint url must use HTTPS." };
+    if (options.reviewed !== true) return { allowed: false, reason: "The sheet push needs the explicit reviewed flag before any data leaves local memory." };
+  }
+  if (kind === "importcsv") {
+    if (typeof options.csv !== "string" || !options.csv.trim()) return { allowed: false, reason: "Reviewed csv content is required in options." };
+    if (options.name !== void 0 && !isnonempty(options.name)) return { allowed: false, reason: "The reviewed dataset name must be a non-empty string." };
+    if (options.mapping !== void 0) {
+      const mapping = options.mapping;
+      if (!mapping || typeof mapping !== "object" || Array.isArray(mapping) || !Object.values(mapping).every((item) => typeof item === "string")) return { allowed: false, reason: "The reviewed csv column mapping must be an object of string values." };
+    }
+  }
+  if (kind === "looprows") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    if (options.variable !== void 0 && !isnonempty(options.variable)) return { allowed: false, reason: "The reviewed row variable name must be a non-empty string." };
+    const inner = validateinnerstep(options, origin);
+    if (!inner.allowed) return inner;
+  }
+  if (kind === "transformvalues") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    const rules = options.rules;
+    if (!Array.isArray(rules) || rules.length === 0) return { allowed: false, reason: "A reviewed non-empty list of transform rules is required in options." };
+    for (const item of rules) {
+      const rulecheck = validatetransformrule(item);
+      if (!rulecheck.allowed) return rulecheck;
+    }
+  }
+  if (kind === "deduperows") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    const keys = options.keys;
+    if (!Array.isArray(keys) || keys.length === 0 || !keys.every((key) => isnonempty(key))) return { allowed: false, reason: "A reviewed non-empty list of dedupe column keys is required in options." };
+  }
+  if (kind === "mergepages") {
+    const listcheck = validatedatasetids(options, "datasets");
+    if (!listcheck.allowed) return listcheck;
+  }
+  if (kind === "stamplerows") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    if (options.url !== void 0 && !ishttpsurl(options.url)) return { allowed: false, reason: "The reviewed source url must use HTTPS." };
+  }
+  if (kind === "previewgrid") {
+    if (!isnonempty(options.dataset)) return { allowed: false, reason: "A reviewed dataset id is required in options." };
+    if (options.sample !== void 0 && (typeof options.sample !== "number" || !Number.isInteger(options.sample) || options.sample < 1)) return { allowed: false, reason: "The reviewed sample row count must be a positive integer with no code ceiling." };
+  }
+  if (kind === "resumeextract" && !isnonempty(options.session)) return { allowed: false, reason: "A reviewed extract session id is required in options." };
+  if (kind === "logprovenance" && !isnonempty(options.artifact)) return { allowed: false, reason: "A reviewed artifact id or name is required in options." };
+  return { allowed: true };
+}
 function submitreviewgranted(steps, submitid) {
   const position = steps.findIndex((candidate) => candidate.id === submitid);
   const asked = steps.some((candidate, index) => candidate.kind === "asksubmit" && (position === -1 || index < position));
@@ -817,7 +1012,7 @@ function validateinnerstep(options, origin) {
     return { allowed: true };
   }
   if (typeof kind !== "string" || !kind.trim()) return { allowed: false, reason: "A reviewed step id or inline step kind is required in options." };
-  if (kind === "retryaction" || kind === "enterframe") return { allowed: false, reason: "The reviewed inner step cannot be another wrapper kind." };
+  if (kind === "retryaction" || kind === "enterframe" || kind === "looprows") return { allowed: false, reason: "The reviewed inner step cannot be another wrapper kind." };
   if (!allowedactions.has(kind)) return { allowed: false, reason: "The reviewed inner step kind is unsupported." };
   const inneroptions = options.options;
   if (inneroptions !== void 0 && (!inneroptions || typeof inneroptions !== "object" || Array.isArray(inneroptions))) return { allowed: false, reason: "The reviewed inner step options must be an object." };
@@ -1201,6 +1396,10 @@ function validatestep(step, origin) {
     const formcheck = validateformgrammar(step, options);
     if (!formcheck.allowed) return formcheck;
   }
+  if (isdatasetkind(step.kind)) {
+    const datacheck = validatedatagrammar(step, options, origin);
+    if (!datacheck.allowed) return datacheck;
+  }
   if (step.kind === "tabcreate") {
     if (options.background !== void 0 && typeof options.background !== "boolean") return { allowed: false, reason: "The reviewed background flag must be a boolean." };
     if (options.window !== void 0 && (typeof options.window !== "number" || !Number.isInteger(options.window) || options.window < 0)) return { allowed: false, reason: "The reviewed target window id must be a non-negative integer." };
@@ -1228,6 +1427,10 @@ function canexecute(input) {
   if (input.plan.expiresat <= now) return { allowed: false, reason: "The approved plan has expired." };
   if ((input.step.kind === "pierceshadow" || input.step.kind === "enterframe") && !origingranted(input.session, input.origin)) return { allowed: false, reason: "The shadow or frame step is outside the session origin grants." };
   if (input.step.kind === "readjson" && !origingranted(input.session, input.origin)) return { allowed: false, reason: "The json state read is outside the session origin grants." };
+  if (isexportkind(input.step.kind)) {
+    const exportgate = exportgranted(input.session, input.origin);
+    if (!exportgate.allowed) return exportgate;
+  }
   if (input.step.kind === "navlist") {
     let options = {};
     try {
@@ -1336,6 +1539,15 @@ function wizardcompletion(state) {
   if (state.steps <= 0) return 0;
   return Math.min(1, state.completed.filter(Boolean).length / state.steps);
 }
+function extractionshare(rowscollected, estimatedtotal) {
+  if (!Number.isFinite(estimatedtotal) || estimatedtotal <= 0) return 0;
+  return Math.min(1, Math.max(0, rowscollected) / estimatedtotal);
+}
+function recordextraction(progress, planid, stepid, entry, now) {
+  const base = progress && progress.planid === planid ? progress : emptyprogress(planid, now);
+  const outcome = { stepid, ok: true, summary: `Extraction page ${entry.page} collected ${entry.rows} row${entry.rows === 1 ? "" : "s"} at cursor ${entry.cursor}.`, details: { extraction: entry }, at: now };
+  return recordoutcome(base, planid, outcome, now);
+}
 function recordwizardstep(progress, planid, stepid, state, now) {
   const base = progress && progress.planid === planid ? progress : emptyprogress(planid, now);
   const executed = Math.min(state.index, state.steps);
@@ -1345,7 +1557,7 @@ function recordwizardstep(progress, planid, stepid, state, now) {
 }
 
 // version.ts
-var packageversion = "1.1.37";
+var packageversion = "1.1.38";
 
 // types.ts
 var protocolversion = packageversion;
@@ -1382,9 +1594,9 @@ function parseproposal(value, origin) {
     return step;
   });
   for (const step of steps) {
-    if (step.kind !== "retryaction" && step.kind !== "enterframe") continue;
+    if (step.kind !== "retryaction" && step.kind !== "enterframe" && step.kind !== "looprows") continue;
     const options = parseoptions(step);
-    if (typeof options.stepid === "string" && !steps.some((candidate) => candidate.id === options.stepid)) throw new Error("A retry or frame wrapper references an unknown step id.");
+    if (typeof options.stepid === "string" && !steps.some((candidate) => candidate.id === options.stepid)) throw new Error("A retry, frame or loop wrapper references an unknown step id.");
   }
   for (const step of steps) {
     if (step.kind !== "submitform" && step.kind !== "retryform") continue;
@@ -1451,6 +1663,17 @@ function errorreportresponse(input) {
 function wizardreport(input) {
   return { version: protocolversion, ...input.sessionid ? { sessionid: input.sessionid } : {}, wizards: input.wizards, picks: input.picks };
 }
+function datasetresponse(input) {
+  const sample = Math.max(0, Math.floor(input.sample ?? 10));
+  const payload = { ...input.dataset, rows: input.dataset.rows.slice(0, sample), totalrows: input.dataset.rows.length };
+  return JSON.stringify({ version: protocolversion, planid: input.plan.id, planstate: input.plan.state, dataset: payload });
+}
+function extractionreport(input) {
+  return { version: protocolversion, sessions: input.sessions };
+}
+function provenancereport(input) {
+  return { version: protocolversion, records: input.records };
+}
 
 // extension/browsertabs.ts
 var browserkinds = /* @__PURE__ */ new Set(["tablist", "tabcreate", "tabactivate", "tabclose", "tabreload", "tabsnapshot", "windowlist", "windowcreate", "windowclose", "zoomset", "windowresize", "downloadfile"]);
@@ -2213,6 +2436,261 @@ function backoffwaits(attempts, wait, factor) {
   return windows;
 }
 
+// extension/pagedata.ts
+function normalizeheader(label) {
+  const slug = label.trim().toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "-").replace(/^-+|-+$/g, "");
+  return slug || "column";
+}
+function columnspecof(label, used = /* @__PURE__ */ new Set()) {
+  const base = normalizeheader(label);
+  let key = base;
+  let suffix = 2;
+  while (used.has(key)) {
+    key = `${base}${suffix}`;
+    suffix += 1;
+  }
+  used.add(key);
+  return { key, label: label.trim(), kind: "text", normalized: label.trim().toLowerCase() };
+}
+function rowhash(row, keys) {
+  const source = (keys.length > 0 ? keys : Object.keys(row).sort()).map((key) => `${key}=${row[key] ?? ""}`).join("|");
+  let hash = 5381;
+  for (let index = 0; index < source.length; index += 1) hash = (hash * 33 ^ source.charCodeAt(index)) >>> 0;
+  return hash.toString(16);
+}
+function dedupebykeys(rows, keys) {
+  const seen = /* @__PURE__ */ new Set();
+  const kept = [];
+  for (const row of rows) {
+    const hash = rowhash(row, keys);
+    if (seen.has(hash)) continue;
+    seen.add(hash);
+    kept.push(row);
+  }
+  return { kept, removed: rows.length - kept.length };
+}
+function applyexpression(value, expression) {
+  const split = expression.indexOf(":");
+  const op = split === -1 ? expression : expression.slice(0, split);
+  const argument = split === -1 ? void 0 : expression.slice(split + 1);
+  if (op === "trim") return value.trim();
+  if (op === "upper") return value.toUpperCase();
+  if (op === "lower") return value.toLowerCase();
+  if (op === "number") return value.replace(/[^\d.\-]/g, "");
+  if (op === "prefix") return `${argument ?? ""}${value}`;
+  if (op === "suffix") return `${value}${argument ?? ""}`;
+  if (op === "replace") {
+    const separator = argument?.indexOf("=>") ?? -1;
+    if (separator === -1 || separator === 0) throw new Error(`The reviewed transform expression ${expression} needs the from=>to separator.`);
+    const from = argument.slice(0, separator);
+    const to = argument.slice(separator + 2);
+    return value.split(from).join(to);
+  }
+  throw new Error(`The reviewed transform expression ${op} is not supported.`);
+}
+function transformrows(rows, rules) {
+  const errors = [];
+  const output = rows.map((row) => ({ ...row }));
+  for (const rule of rules) {
+    const updated = [];
+    try {
+      for (const row of output) updated.push({ ...row, [rule.target]: applyexpression(rule.sources.map((source) => row[source] ?? "").join(" "), rule.expression) });
+    } catch (error) {
+      errors.push(`${rule.target}: ${error instanceof Error ? error.message : String(error)}`);
+      continue;
+    }
+    output.splice(0, output.length, ...updated);
+  }
+  return { rows: output, errors };
+}
+function mergedatasets(datasets) {
+  const columns = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const dataset of datasets) {
+    for (const column of dataset.columns) {
+      if (seen.has(column.key)) continue;
+      seen.add(column.key);
+      columns.push(column);
+    }
+  }
+  const rows = datasets.flatMap((dataset) => dataset.rows.map((row) => {
+    const merged = {};
+    for (const column of columns) merged[column.key] = row[column.key] ?? "";
+    return merged;
+  }));
+  return { columns, rows };
+}
+function samplerows(rows, url, stepid, at) {
+  const stamped = rows.map((row) => ({ ...row, source: url, capturedat: String(at), step: stepid }));
+  const sources = stamped.map((row, index) => ({ row: index, url, at, stepid }));
+  return { rows: stamped, sources };
+}
+function csvfield(value, delimiter) {
+  return value.includes(delimiter) || value.includes('"') || value.includes("\n") ? `"${value.replace(/"/g, '""')}"` : value;
+}
+function tocsv(columns, rows, delimiter = ",") {
+  const lines = [columns.map((column) => csvfield(column.label || column.key, delimiter)).join(delimiter)];
+  for (const row of rows) lines.push(columns.map((column) => csvfield(row[column.key] ?? "", delimiter)).join(delimiter));
+  return lines.join("\n");
+}
+function parsecsv(text2, delimiter = ",") {
+  const records = [];
+  let field = "";
+  let record2 = [];
+  let quoted = false;
+  for (let index = 0; index < text2.length; index += 1) {
+    const character = text2[index];
+    if (quoted) {
+      if (character === '"') {
+        if (text2[index + 1] === '"') {
+          field += '"';
+          index += 1;
+        } else quoted = false;
+      } else field += character;
+      continue;
+    }
+    if (character === '"') {
+      quoted = true;
+      continue;
+    }
+    if (character === delimiter) {
+      record2.push(field);
+      field = "";
+      continue;
+    }
+    if (character === "\n" || character === "\r") {
+      if (character === "\r" && text2[index + 1] === "\n") index += 1;
+      record2.push(field);
+      field = "";
+      if (record2.some((value) => value.length > 0) || record2.length > 1) records.push(record2);
+      record2 = [];
+      continue;
+    }
+    field += character;
+  }
+  record2.push(field);
+  if (record2.some((value) => value.length > 0) || record2.length > 1) records.push(record2);
+  const [headers = [], ...rows] = records;
+  return { headers, rows };
+}
+function mapcolumns(headers, mapping = {}) {
+  const used = /* @__PURE__ */ new Set();
+  return headers.map((header) => {
+    const target = mapping[header] ?? mapping[normalizeheader(header)] ?? header;
+    return columnspecof(target, used);
+  });
+}
+function tojson(columns, rows) {
+  return JSON.stringify({ columns, rows });
+}
+function xmltext(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function toexcel(columns, rows, name) {
+  const head = columns.map((column) => `<Cell ss:StyleID="head"><Data ss:Type="String">${xmltext(column.label || column.key)}</Data></Cell>`).join("");
+  const body = rows.map((row) => `<Row>${columns.map((column) => {
+    const value = row[column.key] ?? "";
+    const numeric = column.kind === "number" && value.trim() !== "" && Number.isFinite(Number(value));
+    return numeric ? `<Cell><Data ss:Type="Number">${xmltext(value)}</Data></Cell>` : `<Cell><Data ss:Type="String">${xmltext(value)}</Data></Cell>`;
+  }).join("")}</Row>`).join("");
+  return `<?xml version="1.0"?><?mso-application progid="Excel.Sheet"?><Workbook xmlns="urn:schemas-microsoft-com:office:spreadsheet" xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet"><Styles><Style ss:ID="head"><Font ss:Bold="1"/></Style></Styles><Worksheet ss:Name="${xmltext(name || "dataset").slice(0, 31)}"><Table><Row>${head}</Row>${body}</Table></Worksheet></Workbook>`;
+}
+
+// extension/datacommand.ts
+function builddataset(id, name, grid, at) {
+  return { id, name: name || id, columns: grid.columns, rows: grid.rows, sources: [], at };
+}
+function checksum(value) {
+  let hash = 5381;
+  for (let index = 0; index < value.length; index += 1) hash = (hash * 33 ^ value.charCodeAt(index)) >>> 0;
+  return `fnv1a-${hash.toString(16)}`;
+}
+function exportcontent(datasetvalue, format, delimiter = ",") {
+  if (format === "json") return tojson(datasetvalue.columns, datasetvalue.rows);
+  if (format === "excel") return toexcel(datasetvalue.columns, datasetvalue.rows, datasetvalue.name);
+  return tocsv(datasetvalue.columns, datasetvalue.rows, delimiter);
+}
+function exportartifact(id, datasetvalue, format, stepid, content, at) {
+  const extension = format === "excel" ? "xml" : format;
+  return { id, kind: format, name: `${datasetvalue.name || datasetvalue.id}.${extension}`, stepid, rowcount: datasetvalue.rows.length, content, checksum: checksum(content), at };
+}
+function artifactrecordof(artifact) {
+  return { id: artifact.id, kind: artifact.kind, name: artifact.name, stepid: artifact.stepid, at: artifact.at };
+}
+function chunkplan(rows, chunk) {
+  const size = Math.max(1, Math.floor(chunk));
+  const chunks = [];
+  for (let from = 0; from < rows || chunks.length === 0; from += size) {
+    const to = Math.min(rows, from + size);
+    chunks.push({ index: chunks.length, from, to });
+    if (to >= rows) break;
+  }
+  return chunks;
+}
+function backpressure(written, acknowledged) {
+  return written - acknowledged >= 1;
+}
+function advancestream(state, chunk, at, done) {
+  return { datasetid: state.datasetid, name: state.name, chunk: chunk.index + 1, chunks: state.chunks, written: chunk.to, ...done ? { done: true } : {}, at };
+}
+function newstream(datasetvalue, chunks, at) {
+  return { datasetid: datasetvalue.id, name: datasetvalue.name, chunk: 0, chunks, written: 0, at };
+}
+function advancecursor(sessionvalue, page, rows, at, done) {
+  return {
+    id: sessionvalue.id,
+    datasetid: sessionvalue.datasetid,
+    name: sessionvalue.name,
+    target: sessionvalue.target,
+    next: sessionvalue.next,
+    planned: sessionvalue.planned,
+    pages: [...sessionvalue.pages, page],
+    rows: sessionvalue.rows + rows,
+    cursor: sessionvalue.cursor + 1,
+    ...done || sessionvalue.cursor + 1 >= sessionvalue.planned ? { done: true } : {},
+    startedat: sessionvalue.startedat,
+    updatedat: at
+  };
+}
+function newextractsession(id, datasetid, name, target, next, planned, at) {
+  return { id, datasetid, name, target, next, planned, pages: [], rows: 0, cursor: 0, startedat: at, updatedat: at };
+}
+function remainingpages(sessionvalue, planned) {
+  if (sessionvalue.done) return 0;
+  return Math.max(0, Math.max(sessionvalue.planned, planned) - sessionvalue.cursor);
+}
+function provenancefor(artifact, url, stepid, at) {
+  return { artifact: artifact.id, name: artifact.name, url, stepid, rowstart: artifact.rowcount > 0 ? 1 : 0, rowend: artifact.rowcount, checksum: artifact.checksum, at };
+}
+function interpolate(text2, row) {
+  return text2.replace(/\{\{([^}]+)\}\}/g, (_, key) => row[key.trim()] ?? "");
+}
+function loopstep(step, row) {
+  return {
+    ...step,
+    ...step.target !== void 0 ? { target: interpolate(step.target, row) } : {},
+    ...step.value !== void 0 ? { value: interpolate(step.value, row) } : {},
+    ...step.options !== void 0 ? { options: interpolate(step.options, row) } : {}
+  };
+}
+function loopvariables(row) {
+  return { ...row };
+}
+function gridpreview(datasetvalue, sample) {
+  return { datasetid: datasetvalue.id, columns: datasetvalue.columns.map((column) => column.key), rows: datasetvalue.rows.length, sample: datasetvalue.rows.slice(0, Math.max(0, Math.floor(sample))) };
+}
+function sheetpayload(datasetvalue, sheet) {
+  return { sheet, columns: datasetvalue.columns.map((column) => column.key), rows: datasetvalue.rows };
+}
+function mergetaskrules(existing, taskid, transforms, dedupekeys, at) {
+  return {
+    taskid,
+    transforms: transforms.length > 0 ? transforms : existing?.transforms ?? [],
+    dedupekeys: dedupekeys.length > 0 ? dedupekeys : existing?.dedupekeys ?? [],
+    at
+  };
+}
+
 // extension/background.ts
 var sessionduration = 15 * 60 * 1e3;
 var freshcheckkinds = /* @__PURE__ */ new Set(["focus", "inspect", "click", "type", "scroll", "select", "hover"]);
@@ -2403,6 +2881,13 @@ function stepauditkind(step, ok) {
   if (step.kind === "submitform" || step.kind === "asksubmit") return "submit";
   if (step.kind === "consentpassword") return "consent";
   if (step.kind === "handoffcaptcha") return "handoff";
+  if (isdatasetkind(step.kind)) {
+    if (step.kind === "exportcsv" || step.kind === "exportjson" || step.kind === "exportexcel" || step.kind === "copytable" || step.kind === "pushsheets") return "export";
+    if (step.kind === "streamdisk") return "stream";
+    if (step.kind === "resumeextract") return "resume";
+    if (step.kind === "logprovenance") return "provenance";
+    return "scrape";
+  }
   if (formfillkinds.has(step.kind)) return "fill";
   if (pointerkinds.has(step.kind)) return "pointer";
   if (watchstepkinds.has(step.kind)) return "watch";
@@ -3585,6 +4070,220 @@ async function executeformstep(step, session, plan, tabid2, origin) {
     }
   }
 }
+async function loaddataset(datasetid) {
+  const record2 = await memory.getdataset(datasetid);
+  if (!record2) throw new Error(`No dataset ${datasetid} exists yet; scrape or import it first.`);
+  return record2;
+}
+function readgridoutput(output) {
+  const grid = output?.details?.grid;
+  if (!grid || typeof grid !== "object") return null;
+  return grid;
+}
+async function storeexport(stepid, datasetvalue, format, delimiter, session, planid, origin) {
+  const content = exportcontent(datasetvalue, format, delimiter);
+  const artifact = exportartifact(randomid(), datasetvalue, format, stepid, content, Date.now());
+  await memory.addexport(artifact);
+  await memory.addartifact(artifactrecordof(artifact));
+  await memory.addprovenance(provenancefor(artifact, datasetvalue.sources[0]?.url ?? origin, stepid, Date.now()));
+  await audit("export", `Exported ${artifact.rowcount} row${artifact.rowcount === 1 ? "" : "s"} of dataset ${datasetvalue.name} into the ${format} artifact ${artifact.name} with checksum ${artifact.checksum}; the provenance record keeps the source url, step ref and row range.`, { ...session ? { sessionid: session.id } : {}, planid, stepid });
+  await refreshbadge();
+  return artifact;
+}
+async function executedatastep(step, session, plan, tabid2, origin) {
+  const options = stepoptions2(step);
+  const extra = { ...session ? { sessionid: session.id } : {}, planid: plan.id, stepid: step.id };
+  switch (step.kind) {
+    case "scrapetable": {
+      const output = await dispatchpagestep(step, tabid2, origin, plan);
+      const grid = readgridoutput(output);
+      if (!output?.ok || !grid) return output ?? { ok: false, summary: "The table scrape returned no result." };
+      const id = randomid();
+      const name = typeof options.name === "string" && options.name ? options.name : `dataset-${step.id}`;
+      const datasetvalue = builddataset(id, name, grid, Date.now());
+      await memory.setdataset(datasetvalue);
+      const extract = advancecursor(newextractsession(randomid(), id, name, step.target ?? "", "", 1, Date.now()), origin, datasetvalue.rows.length, Date.now(), true);
+      await memory.setextractsession(extract);
+      await memory.setprogress(recordextraction(await memory.getprogress(), plan.id, step.id, { page: 1, rows: datasetvalue.rows.length, cursor: 1 }, Date.now()));
+      await refreshbadge();
+      return { ok: true, summary: output.summary, details: { ...output.details ?? {}, dataset: { id, name, rows: datasetvalue.rows.length, columns: grid.columns.length } } };
+    }
+    case "paginateextract": {
+      const output = await dispatchpagestep(step, tabid2, origin, plan);
+      const grid = readgridoutput(output);
+      if (!output?.ok || !grid) return output ?? { ok: false, summary: "The paginated extraction returned no result." };
+      const id = randomid();
+      const name = typeof options.name === "string" && options.name ? options.name : `dataset-${step.id}`;
+      const datasetvalue = builddataset(id, name, grid, Date.now());
+      await memory.setdataset(datasetvalue);
+      const planned = typeof options.pages === "number" && Number.isInteger(options.pages) && options.pages > 0 ? options.pages : Number(output.details?.pages ?? 1);
+      const next = output.details?.next === true;
+      const extract = advancecursor(newextractsession(randomid(), id, name, step.target ?? "", typeof options.next === "string" ? options.next : "", planned, Date.now()), origin, datasetvalue.rows.length, Date.now(), !next);
+      await memory.setextractsession(extract);
+      await memory.setprogress(recordextraction(await memory.getprogress(), plan.id, step.id, { page: extract.cursor, rows: datasetvalue.rows.length, cursor: extract.cursor }, Date.now()));
+      await refreshbadge();
+      const extractedpages = Number(output.details?.pages ?? 1);
+      const estimated = Math.max(1, Math.round(datasetvalue.rows.length / Math.max(1, extractedpages) * planned));
+      return { ok: true, summary: output.summary, details: { ...output.details ?? {}, dataset: { id, name, rows: datasetvalue.rows.length, columns: grid.columns.length }, extractsession: { id: extract.id, cursor: extract.cursor, planned, done: extract.done === true }, progressshare: extractionshare(datasetvalue.rows.length, estimated) } };
+    }
+    case "resumeextract": {
+      const sessionid = typeof options.session === "string" ? options.session : "";
+      const extract = (await memory.getextractsessions()).find((item) => item.id === sessionid);
+      if (!extract) throw new Error(`No extract session ${sessionid} is stored yet.`);
+      if (extract.done) return { ok: true, summary: `Extraction ${extract.name} already completed at cursor ${extract.cursor}.`, details: { cursor: extract.cursor, done: true } };
+      const datasetvalue = await loaddataset(extract.datasetid);
+      const remaining = remainingpages(extract, extract.planned);
+      const derived = { id: step.id, kind: "paginateextract", target: extract.target, summary: step.summary, risk: "sensitive", options: JSON.stringify({ next: extract.next, pages: remaining, cursor: extract.cursor, name: extract.name }) };
+      const output = await dispatchpagestep(derived, tabid2, origin, plan);
+      const grid = readgridoutput(output);
+      if (!output?.ok || !grid) return output ?? { ok: false, summary: "The resumed extraction returned no result." };
+      const merged = mergedatasets([{ columns: datasetvalue.columns, rows: datasetvalue.rows }, { columns: grid.columns, rows: grid.rows }]);
+      const updated = { ...datasetvalue, columns: merged.columns, rows: merged.rows, at: Date.now() };
+      await memory.setdataset(updated);
+      const next = output.details?.next === true;
+      const resumed = advancecursor({ ...extract, updatedat: Date.now() }, origin, grid.rows.length, Date.now(), !next);
+      await memory.setextractsession(resumed);
+      await memory.setprogress(recordextraction(await memory.getprogress(), plan.id, step.id, { page: resumed.cursor, rows: grid.rows.length, cursor: resumed.cursor }, Date.now()));
+      await audit("resume", `Extraction ${extract.name} resumed from cursor ${extract.cursor} and collected ${grid.rows.length} further row${grid.rows.length === 1 ? "" : "s"} across ${Number(output.details?.pages ?? 1)} page${Number(output.details?.pages ?? 1) === 1 ? "" : "s"}; the dataset now holds ${updated.rows.length} rows.`, extra);
+      return { ok: true, summary: `Resumed extraction ${extract.name} from cursor ${extract.cursor}; the dataset now holds ${updated.rows.length} rows.`, details: { dataset: { id: updated.id, rows: updated.rows.length }, extractsession: { id: resumed.id, cursor: resumed.cursor, planned: resumed.planned, done: resumed.done === true } } };
+    }
+    case "mergepages": {
+      const ids = Array.isArray(options.datasets) ? options.datasets.filter((item) => typeof item === "string" && item.trim().length > 0) : [];
+      const records = [];
+      for (const id2 of ids) records.push(await loaddataset(id2));
+      const merged = mergedatasets(records);
+      const id = randomid();
+      const datasetvalue = { id, name: typeof options.name === "string" && options.name ? options.name : `merged-${step.id}`, columns: merged.columns, rows: merged.rows, sources: records.flatMap((record2) => record2.sources), at: Date.now() };
+      await memory.setdataset(datasetvalue);
+      await refreshbadge();
+      return { ok: true, summary: `Merged ${records.length} dataset${records.length === 1 ? "" : "s"} into ${datasetvalue.name} with ${merged.columns.length} aligned column${merged.columns.length === 1 ? "" : "s"} and ${merged.rows.length} row${merged.rows.length === 1 ? "" : "s"}.`, details: { dataset: { id, name: datasetvalue.name, rows: merged.rows.length, columns: merged.columns.length }, merged: records.map((record2) => record2.id) } };
+    }
+    case "transformvalues": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const rules = (Array.isArray(options.rules) ? options.rules : []).filter((item) => Boolean(item) && typeof item === "object" && !Array.isArray(item));
+      const applied = transformrows(datasetvalue.rows, rules);
+      await memory.setdataset({ ...datasetvalue, rows: applied.rows, at: Date.now() });
+      const existing = (await memory.gettaskrules()).find((item) => item.taskid === plan.id);
+      await memory.settaskrules(mergetaskrules(existing, plan.id, rules, [], Date.now()));
+      if (applied.errors.length > 0) return { ok: false, summary: `Transform rules surfaced ${applied.errors.length} error${applied.errors.length === 1 ? "" : "s"}: ${applied.errors.join("; ")}.`, details: { errors: applied.errors, rules: rules.length } };
+      return { ok: true, summary: `Applied ${rules.length} reviewed transform rule${rules.length === 1 ? "" : "s"} to ${applied.rows.length} row${applied.rows.length === 1 ? "" : "s"}.`, details: { rules: rules.length, rows: applied.rows.length } };
+    }
+    case "deduperows": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const keys = Array.isArray(options.keys) ? options.keys.filter((item) => typeof item === "string" && item.trim().length > 0) : [];
+      const result = dedupebykeys(datasetvalue.rows, keys);
+      await memory.setdataset({ ...datasetvalue, rows: result.kept, at: Date.now() });
+      const existing = (await memory.gettaskrules()).find((item) => item.taskid === plan.id);
+      await memory.settaskrules(mergetaskrules(existing, plan.id, [], keys, Date.now()));
+      return { ok: true, summary: `Deduplicated ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} by ${keys.join(", ")}: removed ${result.removed} duplicate${result.removed === 1 ? "" : "s"}, kept ${result.kept.length}.`, details: { dedupe: { removed: result.removed, kept: result.kept.length, keys } } };
+    }
+    case "stamplerows": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const url = typeof options.url === "string" && options.url ? options.url : origin;
+      const stamped = samplerows(datasetvalue.rows, url, step.id, Date.now());
+      await memory.setdataset({ ...datasetvalue, rows: stamped.rows, sources: stamped.sources, at: Date.now() });
+      return { ok: true, summary: `Stamped ${stamped.rows.length} row${stamped.rows.length === 1 ? "" : "s"} with the source url, timestamp and step ref.`, details: { rows: stamped.rows.length, url } };
+    }
+    case "previewgrid": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const sample = typeof options.sample === "number" && Number.isInteger(options.sample) && options.sample > 0 ? options.sample : 10;
+      const preview = gridpreview(datasetvalue, sample);
+      return { ok: true, summary: `Previewed ${preview.rows} row${preview.rows === 1 ? "" : "s"} of dataset ${datasetvalue.name} in the grid with ${preview.columns.length} column${preview.columns.length === 1 ? "" : "s"}.`, details: { preview } };
+    }
+    case "importcsv": {
+      const csv = typeof options.csv === "string" ? options.csv : "";
+      const parsed = parsecsv(csv, typeof options.delimiter === "string" && options.delimiter.length === 1 ? options.delimiter : ",");
+      const mapping = options.mapping && typeof options.mapping === "object" && !Array.isArray(options.mapping) ? options.mapping : {};
+      const columns = mapcolumns(parsed.headers, mapping);
+      const rows = parsed.rows.map((line) => {
+        const row = {};
+        columns.forEach((column, index) => {
+          row[column.key] = line[index] ?? "";
+        });
+        return row;
+      });
+      const id = randomid();
+      const datasetvalue = { id, name: typeof options.name === "string" && options.name ? options.name : `import-${step.id}`, columns, rows, sources: [], at: Date.now() };
+      await memory.addimport(datasetvalue);
+      await refreshbadge();
+      return { ok: true, summary: `Imported ${rows.length} row${rows.length === 1 ? "" : "s"} and ${columns.length} mapped column${columns.length === 1 ? "" : "s"} from the reviewed csv for fill loops.`, details: { dataset: { id, name: datasetvalue.name, rows: rows.length, columns: columns.length } } };
+    }
+    case "looprows": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const inner = resolvedinnerstep(step, plan);
+      if (!inner) throw new Error("A reviewed inner step or step id is required in options.");
+      const variable = typeof options.variable === "string" && options.variable ? options.variable : "row";
+      let completed = 0;
+      let failed = 0;
+      for (const [index, row] of datasetvalue.rows.entries()) {
+        const derived = loopstep(inner, row);
+        const output = await dispatchpagestep(derived, tabid2, origin, plan);
+        if (output?.ok) completed += 1;
+        else failed += 1;
+        await memory.setprogress(recordoutcome(await memory.getprogress(), plan.id, { stepid: step.id, ok: Boolean(output?.ok), summary: `Iteration ${index + 1} of ${datasetvalue.rows.length}: ${output?.summary ?? "no result"}`, details: { iteration: index + 1, variable, variables: loopvariables(row) }, at: Date.now() }, Date.now()));
+      }
+      return { ok: failed === 0, summary: `Looped ${datasetvalue.rows.length} dataset row${datasetvalue.rows.length === 1 ? "" : "s"} as ${variable} variables: ${completed} iteration${completed === 1 ? "" : "s"} completed${failed > 0 ? `, ${failed} failed` : ""}.`, details: { iterations: datasetvalue.rows.length, completed, failed, variable } };
+    }
+    case "exportcsv":
+    case "exportjson":
+    case "exportexcel": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const format = step.kind === "exportcsv" ? "csv" : step.kind === "exportjson" ? "json" : "excel";
+      const delimiter = typeof options.delimiter === "string" && options.delimiter.length === 1 ? options.delimiter : ",";
+      const artifact = await storeexport(step.id, { ...datasetvalue, ...typeof options.name === "string" && options.name ? { name: options.name } : {} }, format, delimiter, session, plan.id, origin);
+      return { ok: true, summary: `Exported ${artifact.rowcount} row${artifact.rowcount === 1 ? "" : "s"} of dataset ${datasetvalue.name} to ${artifact.name} with checksum ${artifact.checksum}.`, details: { artifact: { id: artifact.id, name: artifact.name, kind: artifact.kind, checksum: artifact.checksum, rowcount: artifact.rowcount } } };
+    }
+    case "copytable": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const content = exportcontent(datasetvalue, "csv", typeof options.delimiter === "string" && options.delimiter.length === 1 ? options.delimiter : ",");
+      await navigator.clipboard.writeText(content);
+      await audit("export", `Copied ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} of dataset ${datasetvalue.name} to the clipboard under the clipboardWrite capability.`, extra);
+      return { ok: true, summary: `Copied ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} to the clipboard.`, details: { rows: datasetvalue.rows.length } };
+    }
+    case "pushsheets": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const sheet = typeof options.sheet === "string" ? options.sheet : "";
+      const config = (await memory.getsheetendpoints()).find((item) => item.endpoint === sheet || item.origin === new URL(sheet).origin);
+      if (!config) throw new Error(`The reviewed sheet endpoint has not been configured; configure ${sheet} from the review panel first.`);
+      const granted = await chrome.permissions.contains({ origins: [hostpattern(config.origin)] });
+      if (!granted) throw new Error(`The sheet endpoint origin ${config.origin} has not received optional permission.`);
+      const payload = sheetpayload(datasetvalue, sheet);
+      const response = await fetch(config.endpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) });
+      await audit("export", `Pushed ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} of dataset ${datasetvalue.name} to the reviewed sheet endpoint ${config.origin} with response ${response.status}.`, extra);
+      return { ok: response.ok, summary: response.ok ? `Pushed ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} to the reviewed sheet endpoint ${config.origin}.` : `The reviewed sheet endpoint answered ${response.status}; the push failed.`, details: { endpoint: config.endpoint, status: response.status, rows: datasetvalue.rows.length } };
+    }
+    case "streamdisk": {
+      const datasetvalue = await loaddataset(typeof options.dataset === "string" ? options.dataset : "");
+      const chunk = typeof options.chunk === "number" && Number.isInteger(options.chunk) && options.chunk > 0 ? options.chunk : datasetvalue.rows.length || 1;
+      const planchunks = chunkplan(datasetvalue.rows.length, chunk);
+      const previous = (await memory.getstreams()).find((item) => item.datasetid === datasetvalue.id && item.done !== true);
+      let state = previous && previous.chunks === planchunks.length ? previous : newstream(datasetvalue, planchunks.length, Date.now());
+      let written = 0;
+      let acknowledged = 0;
+      for (const part of planchunks.slice(state.chunk)) {
+        written += 1;
+        if (backpressure(written, acknowledged)) await new Promise((resolve) => setTimeout(resolve, 0));
+        state = advancestream(state, part, Date.now(), part.to >= datasetvalue.rows.length);
+        await memory.setstream(state);
+        acknowledged += 1;
+      }
+      const artifact = await storeexport(step.id, datasetvalue, "csv", ",", session, plan.id, origin);
+      await audit("stream", `Streamed ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} of dataset ${datasetvalue.name} to disk in ${planchunks.length} reviewed chunk${planchunks.length === 1 ? "" : "s"} of ${chunk} row${chunk === 1 ? "" : "s"} with backpressure; the stream state stays persisted for resume.`, extra);
+      return { ok: true, summary: `Streamed ${datasetvalue.rows.length} row${datasetvalue.rows.length === 1 ? "" : "s"} to disk in ${planchunks.length} chunk${planchunks.length === 1 ? "" : "s"} and landed the artifact ${artifact.name}.`, details: { chunks: planchunks.length, chunk, rows: datasetvalue.rows.length, artifact: { id: artifact.id, name: artifact.name, checksum: artifact.checksum } } };
+    }
+    case "logprovenance": {
+      const reference = typeof options.artifact === "string" ? options.artifact : "";
+      const artifact = (await memory.getexports()).find((item) => item.id === reference || item.name === reference);
+      if (!artifact) throw new Error(`No exported artifact ${reference} exists yet.`);
+      const record2 = provenancefor(artifact, origin, step.id, Date.now());
+      await memory.addprovenance(record2);
+      await audit("provenance", `Provenance of artifact ${artifact.name}: rows ${record2.rowstart} to ${record2.rowend}, checksum ${record2.checksum}, source ${record2.url}.`, extra);
+      return { ok: true, summary: `Logged the provenance of ${artifact.name} for audit: rows ${record2.rowstart} to ${record2.rowend} with checksum ${record2.checksum}.`, details: { provenance: record2 } };
+    }
+    default:
+      return { ok: false, summary: "Unsupported forms and data step." };
+  }
+}
 async function enforcewindowreview(step, session, plan) {
   const windowid = step.value && /^\d+$/.test(step.value) ? Number.parseInt(step.value, 10) : 0;
   const progress = plan ? await memory.getprogress() : void 0;
@@ -3620,8 +4319,9 @@ async function refreshbadge() {
   const queues = await memory.getnavqueues();
   const badges = await memory.getbadges();
   const prompts = (await memory.gettickets()).filter((ticket) => ticket.approved === void 0).length;
+  const datasets = (await memory.getdatasets()).length;
   const tasktabs2 = new Set(badges.map((badge) => badge.tabid)).size;
-  const total = (queues?.prefetch ?? 0) + (queues?.batchopen ?? 0) + tasktabs2 + prompts;
+  const total = (queues?.prefetch ?? 0) + (queues?.batchopen ?? 0) + tasktabs2 + prompts + datasets;
   await chrome.action.setBadgeText({ text: total > 0 ? String(total) : "" }).catch(() => {
   });
 }
@@ -3645,6 +4345,8 @@ async function executestep(stepid) {
   }
   if (istabscommandkind(step.kind)) {
     output = await executetabscommand(step, session, plan, tab.id);
+  } else if (isdatasetkind(step.kind)) {
+    output = await executedatastep(step, session, plan, tab.id, origin);
   } else if (isformkind(step.kind)) {
     output = await executeformstep(step, session, plan, tab.id, origin);
   } else if (isbrowserkind(step.kind)) {
@@ -3752,6 +4454,12 @@ async function grantcapability(permission) {
   await audit("capability", `Capability ${permission} granted by the user.`);
   return refreshcapabilities();
 }
+async function extractionreportValue() {
+  return extractionreport({ sessions: await memory.getextractsessions() });
+}
+async function provenancereportValue() {
+  return provenancereport({ records: await memory.getprovenances() });
+}
 async function handlerequest(message, sender) {
   if (!extensionpage(sender)) throw new Error("Requests are accepted only from Devthink extension pages.");
   const input = message;
@@ -3804,13 +4512,25 @@ async function handlerequest(message, sender) {
       const captchas = await memory.getcaptchas();
       const detections = await memory.getdetections();
       const codeentry = await memory.getcodevalue();
+      const datasets = await memory.getdatasets();
+      const imports = await memory.getimports();
+      const extractsessions = await memory.getextractsessions();
+      const streams = await memory.getstreams();
+      const exports = (await memory.getexports()).map((artifact) => ({ id: artifact.id, kind: artifact.kind, name: artifact.name, stepid: artifact.stepid, rowcount: artifact.rowcount, checksum: artifact.checksum, at: artifact.at }));
+      const provenances = await memory.getprovenances();
+      const taskrules = await memory.gettaskrules();
+      const sheetendpoints = await memory.getsheetendpoints();
+      const sheetgrants = [];
+      for (const config of sheetendpoints) {
+        sheetgrants.push({ ...config, granted: await chrome.permissions.contains({ origins: [hostpattern(config.origin)] }).catch(() => false) });
+      }
       const clones = clonetabs(tabs);
       const taskgauge = tasktabgauge(tabs.filter((tab) => badges.some((badge) => badge.tabid === tab.tabid)).length, tasktabceiling(await memory.getsettings()));
       const report = await buildtabreport(tabs);
       const livetab = session ? await chrome.tabs.get(session.tabid).catch(() => void 0) : void 0;
       const waitprofile = session ? waitprofiles.find((record2) => record2.origin === session.origin) : void 0;
       const livestate = { phase: livetab?.status === "loading" ? "loading" : "complete", ...navrecords[0] ? { finalurl: navrecords[0].finalurl, redirects: navrecords[0].chain } : {} };
-      return { config: await memory.getconfig(), session, plan, progress: plan && progress?.planid === plan.id ? progress : void 0, diagnostic: await memory.getdiagnostic(), audit: await memory.getaudit(), capabilities: await refreshcapabilities(), outcomes: await memory.getoutcomes(), holds: heldkeysreport({ tabid: session?.tabid ?? 0, holds }), dialogs: await memory.getdialogs(), retries: await memory.getretries(), ...signals ? { signals: signalsreport({ signals }) } : { signals: signalsreport({}) }, banners: await memory.getbanners(), mutationevents: await memory.getmutationevents(), focusevents: await memory.getfocusevents(), diffs: await memory.getdiffs(), selectors: await memory.getselectors(), ...a11y ? { a11y } : {}, ...reader ? { reader } : {}, ...map ? { map } : {}, trail: trailreport({ ...session ? { sessionid: session.id } : {}, trail }), navrecords, ratestates, safeties, curated, waitprofiles, auths, navcontrol, navqueues, artifacts, navstate: livestate, ...waitprofile ? { waitprofile } : {}, offline: !navigator.onLine, tabs, windows, layouts: layoutreport({ layouts }), tabgroups, tabmetas, badges, snapshots, closedtabs, tabwatchevents, clones, tasktabgauge: taskgauge, ...controltab ? { controltab } : {}, tabreport: report, profiles, tickets, wizards: wizardreport({ ...session ? { sessionid: session.id } : {}, wizards, picks }), picks, errorreports, captchas, detections, ...codeentry !== void 0 ? { codeentry: true } : {} };
+      return { config: await memory.getconfig(), session, plan, progress: plan && progress?.planid === plan.id ? progress : void 0, diagnostic: await memory.getdiagnostic(), audit: await memory.getaudit(), capabilities: await refreshcapabilities(), outcomes: await memory.getoutcomes(), holds: heldkeysreport({ tabid: session?.tabid ?? 0, holds }), dialogs: await memory.getdialogs(), retries: await memory.getretries(), ...signals ? { signals: signalsreport({ signals }) } : { signals: signalsreport({}) }, banners: await memory.getbanners(), mutationevents: await memory.getmutationevents(), focusevents: await memory.getfocusevents(), diffs: await memory.getdiffs(), selectors: await memory.getselectors(), ...a11y ? { a11y } : {}, ...reader ? { reader } : {}, ...map ? { map } : {}, trail: trailreport({ ...session ? { sessionid: session.id } : {}, trail }), navrecords, ratestates, safeties, curated, waitprofiles, auths, navcontrol, navqueues, artifacts, navstate: livestate, ...waitprofile ? { waitprofile } : {}, offline: !navigator.onLine, tabs, windows, layouts: layoutreport({ layouts }), tabgroups, tabmetas, badges, snapshots, closedtabs, tabwatchevents, clones, tasktabgauge: taskgauge, ...controltab ? { controltab } : {}, tabreport: report, profiles, tickets, wizards: wizardreport({ ...session ? { sessionid: session.id } : {}, wizards, picks }), picks, errorreports, captchas, detections, ...codeentry !== void 0 ? { codeentry: true } : {}, datasets, imports, extractsessions, streams, exports, provenances, taskrules, sheetendpoints: sheetgrants };
     }
     case "capabilities":
       return refreshcapabilities();
@@ -4086,6 +4806,61 @@ async function handlerequest(message, sender) {
       if (!report) throw new Error("No error report has been collected yet.");
       return JSON.parse(errorreportresponse({ report, plan }));
     }
+    case "configuresheet": {
+      const inputsheet = message;
+      const config = normalizeendpoint(inputsheet.endpoint ?? "");
+      const granted = await chrome.permissions.contains({ origins: [hostpattern(config.origin)] });
+      if (!granted) throw new Error("The sheet endpoint origin has not received optional permission.");
+      const record2 = { endpoint: config.endpoint, origin: config.origin, configuredat: Date.now() };
+      await memory.setsheetendpoint(record2);
+      await audit("configure", `Configured the reviewed sheet endpoint ${config.origin} for data pushes; pushes need the explicit reviewed flag.`);
+      return record2;
+    }
+    case "exportdataset": {
+      const session = await memory.getsession();
+      if (!session || session.stoppedat || session.expiresat <= Date.now()) throw new Error("Data exports stay behind the consent gate of an active session.");
+      const { tab, origin } = await activecontext();
+      const gate = exportgranted(session, origin);
+      if (!gate.allowed) throw new Error(gate.reason);
+      const inputexport = message;
+      const format = inputexport.format === "json" ? "json" : inputexport.format === "excel" ? "excel" : "csv";
+      const datasetvalue = await loaddataset(inputexport.datasetid ?? "");
+      const artifact = await storeexport("panel", { ...datasetvalue, ...inputexport.name?.trim() ? { name: inputexport.name.trim() } : {} }, format, ",", session, "", origin);
+      void tab;
+      return { id: artifact.id, kind: artifact.kind, name: artifact.name, rowcount: artifact.rowcount, checksum: artifact.checksum, at: artifact.at };
+    }
+    case "importcsv": {
+      const session = await memory.getsession();
+      if (!session || session.stoppedat || session.expiresat <= Date.now()) throw new Error("Csv imports stay behind the consent gate of an active session.");
+      const inputimport = message;
+      const parsed = parsecsv(inputimport.csv ?? "");
+      if (parsed.headers.length === 0) throw new Error("The reviewed csv needs a header line.");
+      const columns = mapcolumns(parsed.headers, inputimport.mapping ?? {});
+      const rows = parsed.rows.map((line) => {
+        const row = {};
+        columns.forEach((column, index) => {
+          row[column.key] = line[index] ?? "";
+        });
+        return row;
+      });
+      const datasetvalue = { id: randomid(), name: inputimport.name?.trim() || `import-${Date.now()}`, columns, rows, sources: [], at: Date.now() };
+      await memory.addimport(datasetvalue);
+      await audit("scrape", `The review panel imported ${rows.length} row${rows.length === 1 ? "" : "s"} from the reviewed csv as dataset ${datasetvalue.name} for fill loops.`, { sessionid: session.id });
+      await refreshbadge();
+      return { id: datasetvalue.id, name: datasetvalue.name, rows: rows.length, columns: columns.length };
+    }
+    case "dataset": {
+      const plan = await memory.getplan();
+      if (!plan) throw new Error("No plan is available for a dataset envelope.");
+      const inputdataset = message;
+      const datasetvalue = await memory.getdataset(inputdataset.datasetid ?? "");
+      if (!datasetvalue) throw new Error("No dataset has been captured yet.");
+      return JSON.parse(datasetresponse({ dataset: datasetvalue, plan, ...typeof inputdataset.sample === "number" ? { sample: inputdataset.sample } : {} }));
+    }
+    case "extraction":
+      return extractionreportValue();
+    case "provenance":
+      return provenancereportValue();
     case "stop": {
       const session = await memory.getsession();
       if (session) await memory.setsession({ ...session, stoppedat: Date.now() });