@fusengine/browser-mcp 0.1.20 → 0.1.21

package/README.md

@@ -167,6 +167,9 @@ Key agentic patterns:
   rows by stable key** until the list ends. Returns the full set of rows (text / url /
   optional prices) — data, not refs (virtualization recycles nodes), so to act on a found
   row use `browser_act` by text or `browser_scroll` then `browser_snapshot`.
+  Pass an optional **`pipeline`** to post-process rows in one call:
+  `clean` (normalize strings, parse numbers) → `validate` (per-field rules, rejects
+  to `invalidCount`) → `dedupeBy` → `columns` (subset) → `emit:"csv"`.
 - **`browser_run`** — execute an ordered plan (navigate/act/wait/extract) in one call,
   stopping at the first failure. Guardrails apply to the whole plan.
 

package/dist/extraction/pipeline/clean.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Clean stage: normalize string fields (decode HTML entities, strip zero-width
+ * chars, collapse whitespace) and coerce chosen fields to numbers. Pure.
+ * @module extraction/pipeline/clean
+ */
+/** Decode common HTML entities, strip zero-width chars, collapse whitespace. */
+export declare function normalizeString(s: string): string;
+/**
+ * Parse a number from messy text ('€1,234.56', "1'234,50", '1 234,56'). The
+ * decimal separator is the last `.`/`,` followed by 1-2 digits; the rest are
+ * thousands separators. Returns null when no number is present.
+ */
+export declare function parseNumber(raw: string): number | null;
+/** Apply string normalization to every string field; coerce `numericFields`. */
+export declare function cleanRows(rows: Record<string, unknown>[], numericFields?: string[]): Record<string, unknown>[];
+//# sourceMappingURL=clean.d.ts.map

package/dist/extraction/pipeline/clean.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clean.d.ts","sourceRoot":"","sources":["../../../src/extraction/pipeline/clean.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CASjD;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAStD;AAED,gFAAgF;AAChF,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,aAAa,GAAE,MAAM,EAAO,GAC3B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAY3B"}

package/dist/extraction/pipeline/clean.js

@@ -0,0 +1,52 @@
+/**
+ * Clean stage: normalize string fields (decode HTML entities, strip zero-width
+ * chars, collapse whitespace) and coerce chosen fields to numbers. Pure.
+ * @module extraction/pipeline/clean
+ */
+const ENTITIES = {
+    amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", "#39": "'", nbsp: " ",
+};
+/** Decode common HTML entities, strip zero-width chars, collapse whitespace. */
+export function normalizeString(s) {
+    const decoded = s.replace(/&(#x?[0-9a-f]+|\w+);/gi, (m, body) => {
+        if (body[0] === "#") {
+            const code = body[1] === "x" || body[1] === "X" ? parseInt(body.slice(2), 16) : Number(body.slice(1));
+            return Number.isFinite(code) ? String.fromCodePoint(code) : m;
+        }
+        return ENTITIES[body.toLowerCase()] ?? m;
+    });
+    return decoded.replace(/[​-‍­]/g, "").replace(/\s+/g, " ").trim();
+}
+/**
+ * Parse a number from messy text ('€1,234.56', "1'234,50", '1 234,56'). The
+ * decimal separator is the last `.`/`,` followed by 1-2 digits; the rest are
+ * thousands separators. Returns null when no number is present.
+ */
+export function parseNumber(raw) {
+    const s = raw.replace(/[^0-9.,'\s-]/g, "").trim();
+    if (!s)
+        return null;
+    const frac = s.match(/[.,](\d{1,2})$/)?.[1];
+    const normalized = frac
+        ? `${s.slice(0, s.length - frac.length - 1).replace(/[.,'\s]/g, "")}.${frac}`
+        : s.replace(/[.,'\s]/g, "");
+    const n = Number(normalized);
+    return Number.isFinite(n) ? n : null;
+}
+/** Apply string normalization to every string field; coerce `numericFields`. */
+export function cleanRows(rows, numericFields = []) {
+    return rows.map((row) => {
+        const out = {};
+        for (const [k, v] of Object.entries(row))
+            out[k] = typeof v === "string" ? normalizeString(v) : v;
+        for (const f of numericFields) {
+            if (out[f] != null) {
+                const n = parseNumber(String(out[f]));
+                if (n !== null)
+                    out[f] = n;
+            }
+        }
+        return out;
+    });
+}
+//# sourceMappingURL=clean.js.map

package/dist/extraction/pipeline/clean.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clean.js","sourceRoot":"","sources":["../../../src/extraction/pipeline/clean.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,QAAQ,GAA2B;IACvC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG;CACxE,CAAC;AAEF,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,CAAS;IACvC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC,CAAC,EAAE,IAAY,EAAE,EAAE;QACtE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtG,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI;QACrB,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE;QAC7E,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CACvB,IAA+B,EAC/B,gBAA0B,EAAE;IAE5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9B,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,IAAI;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/extraction/pipeline/dedupe.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Dedupe stage: collapse rows sharing a composite key built from `keyFields`
+ * (lowercased + trimmed, NUL-separated to avoid field-boundary collisions).
+ * Pure.
+ * @module extraction/pipeline/dedupe
+ */
+/** Remove duplicate rows by `keyFields`, keeping the first or last occurrence. */
+export declare function dedupeRows(rows: Record<string, unknown>[], keyFields: string[], keep?: "first" | "last"): Record<string, unknown>[];
+//# sourceMappingURL=dedupe.d.ts.map

package/dist/extraction/pipeline/dedupe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedupe.d.ts","sourceRoot":"","sources":["../../../src/extraction/pipeline/dedupe.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,kFAAkF;AAClF,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,GAAE,OAAO,GAAG,MAAgB,GAC/B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAS3B"}

package/dist/extraction/pipeline/dedupe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedupe.js","sourceRoot":"","sources":["../../../src/extraction/pipeline/dedupe.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,oDAAoD;AACpD,SAAS,KAAK,CAAC,GAA4B,EAAE,SAAmB;IAC9D,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnF,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,UAAU,CACxB,IAA+B,EAC/B,SAAmB,EACnB,OAAyB,OAAO;IAEhC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAmC,CAAC;IACxD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAClC,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAChD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC5B,CAAC"}

package/dist/extraction/pipeline/run.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Run the composable extraction pipeline: clean → validate → dedupe → pick →
+ * emit. Each stage is optional and pure; returns the processed rows, rejected
+ * rows (with reasons), and an optional CSV rendering.
+ * @module extraction/pipeline/run
+ */
+import type { PipelineSpec, PipelineResult } from "../../interfaces/pipeline.js";
+/** Apply the declarative `spec` to `rows`. */
+export declare function runPipeline(rows: Record<string, unknown>[], spec: PipelineSpec): PipelineResult;
+//# sourceMappingURL=run.d.ts.map

package/dist/extraction/pipeline/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/extraction/pipeline/run.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAc,YAAY,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAW7F,8CAA8C;AAC9C,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,CAY/F"}

package/dist/extraction/pipeline/run.js

@@ -0,0 +1,25 @@
+import { toCsv } from "../../lib/serp-csv.js";
+import { cleanRows } from "./clean.js";
+import { dedupeRows } from "./dedupe.js";
+import { validateRows } from "./validate.js";
+/** Restrict each row to `columns` (preserving order). */
+function pickColumns(rows, columns) {
+    return rows.map((row) => Object.fromEntries(columns.map((c) => [c, row[c]])));
+}
+/** Apply the declarative `spec` to `rows`. */
+export function runPipeline(rows, spec) {
+    let out = spec.clean ? cleanRows(rows, spec.clean.numericFields) : rows;
+    let invalid = [];
+    if (spec.validate) {
+        const res = validateRows(out, spec.validate);
+        out = res.valid;
+        invalid = res.invalid;
+    }
+    if (spec.dedupeBy?.length)
+        out = dedupeRows(out, spec.dedupeBy, spec.keep ?? "first");
+    if (spec.columns?.length)
+        out = pickColumns(out, spec.columns);
+    const csv = spec.emit === "csv" ? toCsv(out, spec.columns ?? Object.keys(out[0] ?? {})) : undefined;
+    return { rows: out, invalid, csv };
+}
+//# sourceMappingURL=run.js.map

package/dist/extraction/pipeline/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/extraction/pipeline/run.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,yDAAyD;AACzD,SAAS,WAAW,CAAC,IAA+B,EAAE,OAAiB;IACrE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,WAAW,CAAC,IAA+B,EAAE,IAAkB;IAC7E,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxE,IAAI,OAAO,GAAiB,EAAE,CAAC;IAC/B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC;QAChB,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IACxB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM;QAAE,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC;IACtF,IAAI,IAAI,CAAC,OAAO,EAAE,MAAM;QAAE,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AACrC,CAAC"}

package/dist/extraction/pipeline/validate.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Validate stage: apply per-field rules to rows, partitioning into valid and
+ * invalid (with reasons). Zero-dep and pure — no schema library.
+ * @module extraction/pipeline/validate
+ */
+import type { FieldRule, InvalidRow } from "../../interfaces/pipeline.js";
+/** Partition `rows` by `schema`; failing rows go to `invalid` with reasons. */
+export declare function validateRows(rows: Record<string, unknown>[], schema: Record<string, FieldRule>): {
+    valid: Record<string, unknown>[];
+    invalid: InvalidRow[];
+};
+//# sourceMappingURL=validate.d.ts.map

package/dist/extraction/pipeline/validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../../src/extraction/pipeline/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAoB1E,+EAA+E;AAC/E,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAChC;IAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAAC,OAAO,EAAE,UAAU,EAAE,CAAA;CAAE,CAS7D"}

package/dist/extraction/pipeline/validate.js

@@ -0,0 +1,37 @@
+/** Collect rule violations for one field value (empty array = ok). */
+function checkField(field, value, rule) {
+    const errors = [];
+    const missing = value === undefined || value === null || value === "";
+    if (rule.required && missing)
+        return [`${field}: required`];
+    if (missing)
+        return errors;
+    if (rule.type && typeof value !== rule.type)
+        errors.push(`${field}: expected ${rule.type}, got ${typeof value}`);
+    if (rule.regex && typeof value === "string" && !new RegExp(rule.regex).test(value)) {
+        errors.push(`${field}: failed regex ${rule.regex}`);
+    }
+    if (typeof value === "number") {
+        if (rule.min !== undefined && value < rule.min)
+            errors.push(`${field}: ${value} < min ${rule.min}`);
+        if (rule.max !== undefined && value > rule.max)
+            errors.push(`${field}: ${value} > max ${rule.max}`);
+    }
+    if (rule.enum && !rule.enum.includes(value))
+        errors.push(`${field}: "${String(value)}" not allowed`);
+    return errors;
+}
+/** Partition `rows` by `schema`; failing rows go to `invalid` with reasons. */
+export function validateRows(rows, schema) {
+    const valid = [];
+    const invalid = [];
+    for (const row of rows) {
+        const errors = Object.entries(schema).flatMap(([field, rule]) => checkField(field, row[field], rule));
+        if (errors.length)
+            invalid.push({ row, errors });
+        else
+            valid.push(row);
+    }
+    return { valid, invalid };
+}
+//# sourceMappingURL=validate.js.map

package/dist/extraction/pipeline/validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../src/extraction/pipeline/validate.ts"],"names":[],"mappings":"AAOA,sEAAsE;AACtE,SAAS,UAAU,CAAC,KAAa,EAAE,KAAc,EAAE,IAAe;IAChE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;IACtE,IAAI,IAAI,CAAC,QAAQ,IAAI,OAAO;QAAE,OAAO,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;IAC5D,IAAI,OAAO;QAAE,OAAO,MAAM,CAAC;IAC3B,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO,KAAK,KAAK,IAAI,CAAC,IAAI;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,cAAc,IAAI,CAAC,IAAI,SAAS,OAAO,KAAK,EAAE,CAAC,CAAC;IACjH,IAAI,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,KAAK,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACpG,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,KAAK,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACtG,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACrG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,YAAY,CAC1B,IAA+B,EAC/B,MAAiC;IAEjC,MAAM,KAAK,GAA8B,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACtG,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;;YAC5C,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC"}

package/dist/interfaces/pipeline.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Types for the composable extraction pipeline (clean → validate → dedupe →
+ * emit) applied to extracted records.
+ * @module interfaces/pipeline
+ */
+/** Expected primitive type of a field. */
+export type FieldType = "string" | "number" | "boolean";
+/** Per-field validation rule (regex is a string, compiled at run time). */
+export interface FieldRule {
+    required?: boolean;
+    type?: FieldType;
+    regex?: string;
+    min?: number;
+    max?: number;
+    enum?: unknown[];
+}
+/** A row rejected by validation, with its failure reasons. */
+export interface InvalidRow {
+    row: Record<string, unknown>;
+    errors: string[];
+}
+/** Declarative pipeline specification (all stages optional). */
+export interface PipelineSpec {
+    /** Normalize strings; coerce listed fields to numbers. */
+    clean?: {
+        numericFields?: string[];
+    };
+    /** Per-field rules; failing rows move to `invalid`. */
+    validate?: Record<string, FieldRule>;
+    /** Dedupe by these fields (composite key, normalized). */
+    dedupeBy?: string[];
+    /** Keep the first or last duplicate (default `first`). */
+    keep?: "first" | "last";
+    /** Restrict output to these columns. */
+    columns?: string[];
+    /** Output format: `json` rows (default) or a `csv` string. */
+    emit?: "json" | "csv";
+}
+/** Outcome of running a pipeline over rows. */
+export interface PipelineResult {
+    rows: Record<string, unknown>[];
+    invalid: InvalidRow[];
+    csv?: string;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/interfaces/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/interfaces/pipeline.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,0CAA0C;AAC1C,MAAM,MAAM,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAExD,2EAA2E;AAC3E,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;CAClB;AAED,8DAA8D;AAC9D,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,gEAAgE;AAChE,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,KAAK,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IACrC,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACrC,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,wCAAwC;IACxC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,8DAA8D;IAC9D,IAAI,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;CACvB;AAED,+CAA+C;AAC/C,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd"}

package/dist/interfaces/pipeline.js

@@ -0,0 +1,7 @@
+/**
+ * Types for the composable extraction pipeline (clean → validate → dedupe →
+ * emit) applied to extracted records.
+ * @module interfaces/pipeline
+ */
+export {};
+//# sourceMappingURL=pipeline.js.map

package/dist/interfaces/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/interfaces/pipeline.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/server/tools/collect.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"collect.d.ts","sourceRoot":"","sources":["../../../src/server/tools/collect.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAI/D,kCAAkC;AAClC,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI,CAiCrF"}
+{"version":3,"file":"collect.d.ts","sourceRoot":"","sources":["../../../src/server/tools/collect.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAKzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAyB/D,kCAAkC;AAClC,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI,CA8CrF"}

package/dist/server/tools/collect.js

@@ -1,7 +1,27 @@
 import { z } from "zod";
+import { runPipeline } from "../../extraction/pipeline/run.js";
 import { scrollCollect } from "../../state/scroll-collect.js";
 import { jsonResult } from "../result.js";
 import { withSession } from "./with-session.js";
+const FIELD_RULE = z.object({
+    required: z.boolean().optional(),
+    type: z.enum(["string", "number", "boolean"]).optional(),
+    regex: z.string().optional(),
+    min: z.number().optional(),
+    max: z.number().optional(),
+    enum: z.array(z.unknown()).optional(),
+});
+/** Declarative clean→validate→dedupe→emit spec applied to collected rows. */
+const PIPELINE_SCHEMA = z
+    .object({
+    clean: z.object({ numericFields: z.array(z.string()).optional() }).optional(),
+    validate: z.record(z.string(), FIELD_RULE).optional(),
+    dedupeBy: z.array(z.string()).optional(),
+    keep: z.enum(["first", "last"]).optional(),
+    columns: z.array(z.string()).optional(),
+    emit: z.enum(["json", "csv"]).optional(),
+})
+    .optional();
 /** Register `browser_collect`. */
 export function registerCollectTool(server, sessions) {
     server.registerTool("browser_collect", {
@@ -13,6 +33,7 @@ export function registerCollectTool(server, sessions) {
             container: z.string().optional(),
             maxSteps: z.number().int().optional(),
             extractPrices: z.boolean().optional(),
+            pipeline: PIPELINE_SCHEMA,
         },
     }, async (args) => {
         const a = args;
@@ -23,6 +44,18 @@ export function registerCollectTool(server, sessions) {
                 maxSteps: typeof a.maxSteps === "number" ? a.maxSteps : undefined,
                 extractPrices: a.extractPrices === true,
             });
+            if (a.pipeline) {
+                const rows = result.items;
+                const pr = runPipeline(rows, a.pipeline);
+                return jsonResult({
+                    count: pr.rows.length,
+                    steps: result.steps,
+                    reachedEnd: result.reachedEnd,
+                    invalidCount: pr.invalid.length,
+                    items: pr.rows,
+                    csv: pr.csv,
+                });
+            }
             return jsonResult({
                 count: result.items.length,
                 steps: result.steps,

package/dist/server/tools/collect.js.map

@@ -1 +1 @@
-{"version":3,"file":"collect.js","sourceRoot":"","sources":["../../../src/server/tools/collect.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAE9D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,kCAAkC;AAClC,MAAM,UAAU,mBAAmB,CAAC,MAAiB,EAAE,QAAwB;IAC7E,MAAM,CAAC,YAAY,CACjB,iBAAiB,EACjB;QACE,KAAK,EAAE,0BAA0B;QACjC,WAAW,EACT,0WAA0W;QAC5W,WAAW,EAAE;YACX,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;YAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YACrC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SACtC;KACF,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;QACb,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,OAAO,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YAC5D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE;gBACzC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;gBACpB,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;gBACpE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;gBACjE,aAAa,EAAE,CAAC,CAAC,aAAa,KAAK,IAAI;aACxC,CAAC,CAAC;YACH,OAAO,UAAU,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;gBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"collect.js","sourceRoot":"","sources":["../../../src/server/tools/collect.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAE9D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAChC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAEH,6EAA6E;AAC7E,MAAM,eAAe,GAAG,CAAC;KACtB,MAAM,CAAC;IACN,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7E,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC1C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE;CACzC,CAAC;KACD,QAAQ,EAAE,CAAC;AAEd,kCAAkC;AAClC,MAAM,UAAU,mBAAmB,CAAC,MAAiB,EAAE,QAAwB;IAC7E,MAAM,CAAC,YAAY,CACjB,iBAAiB,EACjB;QACE,KAAK,EAAE,0BAA0B;QACjC,WAAW,EACT,0WAA0W;QAC5W,WAAW,EAAE;YACX,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;YAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YACrC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YACrC,QAAQ,EAAE,eAAe;SAC1B;KACF,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;QACb,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,OAAO,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YAC5D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE;gBACzC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;gBACpB,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;gBACpE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;gBACjE,aAAa,EAAE,CAAC,CAAC,aAAa,KAAK,IAAI;aACxC,CAAC,CAAC;YACH,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,MAAM,CAAC,KAA6C,CAAC;gBAClE,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,QAAwB,CAAC,CAAC;gBACzD,OAAO,UAAU,CAAC;oBAChB,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM;oBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,YAAY,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM;oBAC/B,KAAK,EAAE,EAAE,CAAC,IAAI;oBACd,GAAG,EAAE,EAAE,CAAC,GAAG;iBACZ,CAAC,CAAC;YACL,CAAC;YACD,OAAO,UAAU,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;gBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CACF,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fusengine/browser-mcp",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "MCP server + CLI giving AI agents a real, stealth browser (Patchright/Playwright) — per-country identity, self-healing actions, snapshots, multi-step plans, structured extraction, CDP attach.",
   "license": "MIT",
   "author": "Fusengine",
@@ -54,7 +54,7 @@
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "test": "bun test tests/unit",
-    "test:integration": "node --test --import tsx tests/integration/mcp.test.ts tests/integration/probe.test.ts tests/integration/snapshot.test.ts tests/integration/snapshot-frames.test.ts tests/integration/collect.test.ts tests/integration/selectors.test.ts tests/integration/visual-diff.test.ts tests/integration/session-state.test.ts tests/integration/run.test.ts tests/integration/extract-schema.test.ts",
+    "test:integration": "node --test --import tsx tests/integration/mcp.test.ts tests/integration/probe.test.ts tests/integration/snapshot.test.ts tests/integration/snapshot-frames.test.ts tests/integration/collect.test.ts tests/integration/selectors.test.ts tests/integration/visual-diff.test.ts tests/integration/session-state.test.ts tests/integration/pipeline.test.ts tests/integration/run.test.ts tests/integration/extract-schema.test.ts",
     "browsers": "patchright install chromium",
     "mcp": "node --import tsx src/bin/mcp.ts",
     "cli": "node --import tsx src/bin/cli.ts"