@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/phash.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Composites neutral grey (128,128,128) over each ignoredRegion (fractional
+ * 0-1 coordinates) so masked content — timestamps, live counters, ads —
+ * contributes nothing to any downstream pixel comparison. Shared by every
+ * comparison path (whole-image hash, per-cell grid hash, AI vision) so a
+ * region masked once is actually ignored everywhere, not just in the
+ * cheapest check.
+ */
+export declare function applyMask(imagePath: string, ignoredRegions: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Buffer>;
+/**
+ * Generates a pixelmatch diff image highlighting changed pixels in red.
+ * Applies the same ignoredRegions mask as screenshotDiffPercent so masked
+ * zones don't show as "changed" in the visual output.
+ * Returns a PNG Buffer suitable for upload to S3.
+ */
+export declare function generatePixelmatchDiff(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Buffer>;
+export declare function screenshotDiffPercent(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<{
+    pHashDistance: number;
+    changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT';
+}>;
+/**
+ * Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
+ * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
+ * Uses sharp's `extract` to crop each cell before hashing.
+ */
+export declare function gridLayoutDiff(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>>;

package/dist/phash.js

@@ -0,0 +1,136 @@
+import sharp from "sharp";
+async function applyMask(imagePath, ignoredRegions) {
+  const meta = await sharp(imagePath).metadata();
+  const W = meta.width ?? 1200;
+  const H = meta.height ?? 800;
+  const overlays = await Promise.all(
+    ignoredRegions.map(async (r) => {
+      const rW = Math.max(1, Math.round(r.width * W));
+      const rH = Math.max(1, Math.round(r.height * H));
+      const buf = await sharp({
+        create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } }
+      }).png().toBuffer();
+      return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
+    })
+  );
+  return sharp(imagePath).composite(overlays).png().toBuffer();
+}
+async function hashWithMasks(imagePath, ignoredRegions = []) {
+  const pipeline = ignoredRegions.length > 0 ? sharp(await applyMask(imagePath, ignoredRegions)) : sharp(imagePath);
+  const buf = await pipeline.resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer();
+  const pixels = Array.from(buf);
+  const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
+  return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
+}
+function hammingDistance(a, b) {
+  let xor = a ^ b;
+  let count = 0;
+  while (xor !== 0n) {
+    count += Number(xor & 1n);
+    xor >>= 1n;
+  }
+  return count;
+}
+async function generatePixelmatchDiff(pathA, pathB, ignoredRegions = []) {
+  const metaA = await sharp(pathA).metadata();
+  const W = metaA.width ?? 1200;
+  const H = metaA.height ?? 800;
+  const toRawRGBA = async (path) => {
+    const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
+    const { data } = await base.resize(W, H, { fit: "fill" }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+    return data;
+  };
+  const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
+  const { default: pixelmatch } = await import("pixelmatch");
+  const diffData = Buffer.alloc(W * H * 4);
+  pixelmatch(rawA, rawB, diffData, W, H, {
+    threshold: 0.1,
+    alpha: 0.3,
+    diffColor: [255, 0, 0],
+    aaColor: [255, 255, 0]
+  });
+  return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
+}
+async function screenshotDiffPercent(pathA, pathB, ignoredRegions = []) {
+  const [hashA, hashB] = await Promise.all([
+    hashWithMasks(pathA, ignoredRegions),
+    hashWithMasks(pathB, ignoredRegions)
+  ]);
+  let xor = hashA ^ hashB;
+  let dist = 0;
+  while (xor !== 0n) {
+    dist += Number(xor & 1n);
+    xor >>= 1n;
+  }
+  const pct = parseFloat((dist / 64 * 100).toFixed(1));
+  const changeLevel = pct === 0 ? "IDENTICAL" : pct < 10 ? "MINOR" : pct < 30 ? "MODERATE" : "SIGNIFICANT";
+  return { pHashDistance: pct, changeLevel };
+}
+async function gridLayoutDiff(pathA, pathB, ignoredRegions = []) {
+  const GRID_LABELS = [
+    "top-left",
+    "top-center",
+    "top-right",
+    "mid-left",
+    "mid-center",
+    "mid-right",
+    "bottom-left",
+    "bottom-center",
+    "bottom-right"
+  ];
+  try {
+    const [sourceA, sourceB] = ignoredRegions.length > 0 ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)]) : [pathA, pathB];
+    const [metaA, metaB] = await Promise.all([
+      sharp(sourceA).metadata(),
+      sharp(sourceB).metadata()
+    ]);
+    const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
+    const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
+    const cW = Math.floor(W / 3);
+    const cH = Math.floor(H / 3);
+    const changes = [];
+    for (let row = 0; row < 3; row++) {
+      for (let col = 0; col < 3; col++) {
+        const region = { left: col * cW, top: row * cH, width: cW, height: cH };
+        const label = GRID_LABELS[row * 3 + col];
+        const [hashA, hashB] = await Promise.all([
+          sharp(sourceA).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
+            const pixels = Array.from(buf);
+            const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
+            return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
+          }),
+          sharp(sourceB).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
+            const pixels = Array.from(buf);
+            const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
+            return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
+          })
+        ]);
+        let xor = hashA ^ hashB;
+        let dist = 0;
+        while (xor !== 0n) {
+          dist += Number(xor & 1n);
+          xor >>= 1n;
+        }
+        const pct = dist / 64 * 100;
+        if (pct >= 5) {
+          const severity = pct >= 20 ? "HIGH" : pct >= 10 ? "MEDIUM" : "LOW";
+          changes.push({
+            location: label,
+            severity,
+            description: `${Math.round(pct)}% visual change in ${label.replace("-", " ")} region`
+          });
+        }
+      }
+    }
+    return changes;
+  } catch {
+    return [];
+  }
+}
+export {
+  applyMask,
+  generatePixelmatchDiff,
+  gridLayoutDiff,
+  screenshotDiffPercent
+};
+//# sourceMappingURL=phash.js.map

package/dist/phash.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/phash.ts"],
+  "sourcesContent": ["import sharp from 'sharp';\nimport type { OverlayOptions } from 'sharp';\n\n/**\n * Composites neutral grey (128,128,128) over each ignoredRegion (fractional\n * 0-1 coordinates) so masked content \u2014 timestamps, live counters, ads \u2014\n * contributes nothing to any downstream pixel comparison. Shared by every\n * comparison path (whole-image hash, per-cell grid hash, AI vision) so a\n * region masked once is actually ignored everywhere, not just in the\n * cheapest check.\n */\nexport async function applyMask(\n  imagePath: string,\n  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }>,\n): Promise<Buffer> {\n  const meta = await sharp(imagePath).metadata();\n  const W = meta.width ?? 1200;\n  const H = meta.height ?? 800;\n\n  const overlays: OverlayOptions[] = await Promise.all(\n    ignoredRegions.map(async (r) => {\n      const rW = Math.max(1, Math.round(r.width * W));\n      const rH = Math.max(1, Math.round(r.height * H));\n      const buf = await sharp({\n        create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } },\n      }).png().toBuffer();\n      return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) } as OverlayOptions;\n    }),\n  );\n\n  return sharp(imagePath).composite(overlays).png().toBuffer();\n}\n\n/**\n * Compute an 8x8 average hash (aHash) of an image file, with optional mask overlays.\n * ignoredRegions are fractional (0-1 range), converted to pixels here.\n * Masked regions are filled with neutral grey (128,128,128) before hashing.\n */\nasync function hashWithMasks(\n  imagePath: string,\n  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<bigint> {\n  const pipeline = ignoredRegions.length > 0\n    ? sharp(await applyMask(imagePath, ignoredRegions))\n    : sharp(imagePath);\n\n  const buf = await pipeline.resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer();\n  const pixels = Array.from(buf);\n  const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;\n  return pixels.reduce((h, p, i) => (p >= mean ? h | (1n << BigInt(63 - i)) : h), 0n);\n}\n\nfunction hammingDistance(a: bigint, b: bigint): number {\n  let xor = a ^ b;\n  let count = 0;\n  while (xor !== 0n) { count += Number(xor & 1n); xor >>= 1n; }\n  return count;\n}\n\n/**\n * Generates a pixelmatch diff image highlighting changed pixels in red.\n * Applies the same ignoredRegions mask as screenshotDiffPercent so masked\n * zones don't show as \"changed\" in the visual output.\n * Returns a PNG Buffer suitable for upload to S3.\n */\nexport async function generatePixelmatchDiff(\n  pathA: string,\n  pathB: string,\n  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<Buffer> {\n  const metaA = await sharp(pathA).metadata();\n  const W = metaA.width ?? 1200;\n  const H = metaA.height ?? 800;\n\n  const toRawRGBA = async (path: string): Promise<Buffer> => {\n    const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);\n    const { data } = await base.resize(W, H, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });\n    return data;\n  };\n\n  const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);\n  const { default: pixelmatch } = await import('pixelmatch');\n  const diffData = Buffer.alloc(W * H * 4);\n  pixelmatch(rawA, rawB, diffData, W, H, {\n    threshold: 0.1,\n    alpha: 0.3,\n    diffColor: [255, 0, 0],\n    aaColor: [255, 255, 0],\n  });\n  return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();\n}\n\nexport async function screenshotDiffPercent(\n  pathA: string,\n  pathB: string,\n  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<{ pHashDistance: number; changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT' }> {\n  const [hashA, hashB] = await Promise.all([\n    hashWithMasks(pathA, ignoredRegions),\n    hashWithMasks(pathB, ignoredRegions),\n  ]);\n  let xor = hashA ^ hashB;\n  let dist = 0;\n  while (xor !== 0n) { dist += Number(xor & 1n); xor >>= 1n; }\n  const pct = parseFloat(((dist / 64) * 100).toFixed(1));\n  const changeLevel = pct === 0 ? 'IDENTICAL' : pct < 10 ? 'MINOR' : pct < 30 ? 'MODERATE' : 'SIGNIFICANT';\n  return { pHashDistance: pct, changeLevel };\n}\n\n/**\n * Grid-based layout analysis: divides screenshots into a 3\u00D73 grid and computes\n * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.\n * Uses sharp's `extract` to crop each cell before hashing.\n */\nexport async function gridLayoutDiff(\n  pathA: string,\n  pathB: string,\n  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<Array<{ location: string; severity: string; description: string }>> {\n  const GRID_LABELS = [\n    'top-left',    'top-center',    'top-right',\n    'mid-left',    'mid-center',    'mid-right',\n    'bottom-left', 'bottom-center', 'bottom-right',\n  ];\n\n  try {\n    // Mask before cropping into cells \u2014 a masked region (e.g. a timestamp)\n    // then contributes nothing to any cell's hash, instead of only being\n    // ignored by the separate whole-image hash in screenshotDiffPercent().\n    const [sourceA, sourceB] = ignoredRegions.length > 0\n      ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)])\n      : [pathA, pathB];\n\n    const [metaA, metaB] = await Promise.all([\n      sharp(sourceA).metadata(),\n      sharp(sourceB).metadata(),\n    ]);\n    const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);\n    const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);\n    const cW = Math.floor(W / 3);\n    const cH = Math.floor(H / 3);\n\n    const changes: Array<{ location: string; severity: string; description: string }> = [];\n\n    for (let row = 0; row < 3; row++) {\n      for (let col = 0; col < 3; col++) {\n        const region = { left: col * cW, top: row * cH, width: cW, height: cH };\n        const label = GRID_LABELS[row * 3 + col];\n\n        const [hashA, hashB] = await Promise.all([\n          sharp(sourceA).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()\n            .then((buf) => {\n              const pixels = Array.from(buf);\n              const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;\n              return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);\n            }),\n          sharp(sourceB).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()\n            .then((buf) => {\n              const pixels = Array.from(buf);\n              const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;\n              return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);\n            }),\n        ]);\n\n        let xor = hashA ^ hashB;\n        let dist = 0;\n        while (xor !== 0n) { dist += Number(xor & 1n); xor >>= 1n; }\n        const pct = (dist / 64) * 100;\n\n        if (pct >= 5) {\n          const severity = pct >= 20 ? 'HIGH' : pct >= 10 ? 'MEDIUM' : 'LOW';\n          changes.push({\n            location: label,\n            severity,\n            description: `${Math.round(pct)}% visual change in ${label.replace('-', ' ')} region`,\n          });\n        }\n      }\n    }\n    return changes;\n  } catch {\n    return [];\n  }\n}\n"],
+  "mappings": "AAAA,OAAO,WAAW;AAWlB,eAAsB,UACpB,WACA,gBACiB;AACjB,QAAM,OAAO,MAAM,MAAM,SAAS,EAAE,SAAS;AAC7C,QAAM,IAAI,KAAK,SAAS;AACxB,QAAM,IAAI,KAAK,UAAU;AAEzB,QAAM,WAA6B,MAAM,QAAQ;AAAA,IAC/C,eAAe,IAAI,OAAO,MAAM;AAC9B,YAAM,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC9C,YAAM,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,SAAS,CAAC,CAAC;AAC/C,YAAM,MAAM,MAAM,MAAM;AAAA,QACtB,QAAQ,EAAE,OAAO,IAAI,QAAQ,IAAI,UAAU,GAAG,YAAY,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,IAAI,EAAE;AAAA,MACvF,CAAC,EAAE,IAAI,EAAE,SAAS;AAClB,aAAO,EAAE,OAAO,KAAK,KAAK,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,MAAM,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE;AAAA,IAC3E,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,SAAS,EAAE,UAAU,QAAQ,EAAE,IAAI,EAAE,SAAS;AAC7D;AAOA,eAAe,cACb,WACA,iBAAiF,CAAC,GACjE;AACjB,QAAM,WAAW,eAAe,SAAS,IACrC,MAAM,MAAM,UAAU,WAAW,cAAc,CAAC,IAChD,MAAM,SAAS;AAEnB,QAAM,MAAM,MAAM,SAAS,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS;AACpF,QAAM,SAAS,MAAM,KAAK,GAAG;AAC7B,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,SAAO,OAAO,OAAO,CAAC,GAAG,GAAG,MAAO,KAAK,OAAO,IAAK,MAAM,OAAO,KAAK,CAAC,IAAK,GAAI,EAAE;AACpF;AAEA,SAAS,gBAAgB,GAAW,GAAmB;AACrD,MAAI,MAAM,IAAI;AACd,MAAI,QAAQ;AACZ,SAAO,QAAQ,IAAI;AAAE,aAAS,OAAO,MAAM,EAAE;AAAG,YAAQ;AAAA,EAAI;AAC5D,SAAO;AACT;AAQA,eAAsB,uBACpB,OACA,OACA,iBAAiF,CAAC,GACjE;AACjB,QAAM,QAAQ,MAAM,MAAM,KAAK,EAAE,SAAS;AAC1C,QAAM,IAAI,MAAM,SAAS;AACzB,QAAM,IAAI,MAAM,UAAU;AAE1B,QAAM,YAAY,OAAO,SAAkC;AACzD,UAAM,OAAO,eAAe,SAAS,IAAI,MAAM,MAAM,UAAU,MAAM,cAAc,CAAC,IAAI,MAAM,IAAI;AAClG,UAAM,EAAE,KAAK,IAAI,MAAM,KAAK,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,mBAAmB,KAAK,CAAC;AAClH,WAAO;AAAA,EACT;AAEA,QAAM,CAAC,MAAM,IAAI,IAAI,MAAM,QAAQ,IAAI,CAAC,UAAU,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC;AAC3E,QAAM,EAAE,SAAS,WAAW,IAAI,MAAM,OAAO,YAAY;AACzD,QAAM,WAAW,OAAO,MAAM,IAAI,IAAI,CAAC;AACvC,aAAW,MAAM,MAAM,UAAU,GAAG,GAAG;AAAA,IACrC,WAAW;AAAA,IACX,OAAO;AAAA,IACP,WAAW,CAAC,KAAK,GAAG,CAAC;AAAA,IACrB,SAAS,CAAC,KAAK,KAAK,CAAC;AAAA,EACvB,CAAC;AACD,SAAO,MAAM,UAAU,EAAE,KAAK,EAAE,OAAO,GAAG,QAAQ,GAAG,UAAU,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS;AACvF;AAEA,eAAsB,sBACpB,OACA,OACA,iBAAiF,CAAC,GACmB;AACrG,QAAM,CAAC,OAAO,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,IACvC,cAAc,OAAO,cAAc;AAAA,IACnC,cAAc,OAAO,cAAc;AAAA,EACrC,CAAC;AACD,MAAI,MAAM,QAAQ;AAClB,MAAI,OAAO;AACX,SAAO,QAAQ,IAAI;AAAE,YAAQ,OAAO,MAAM,EAAE;AAAG,YAAQ;AAAA,EAAI;AAC3D,QAAM,MAAM,YAAa,OAAO,KAAM,KAAK,QAAQ,CAAC,CAAC;AACrD,QAAM,cAAc,QAAQ,IAAI,cAAc,MAAM,KAAK,UAAU,MAAM,KAAK,aAAa;AAC3F,SAAO,EAAE,eAAe,KAAK,YAAY;AAC3C;AAOA,eAAsB,eACpB,OACA,OACA,iBAAiF,CAAC,GACL;AAC7E,QAAM,cAAc;AAAA,IAClB;AAAA,IAAe;AAAA,IAAiB;AAAA,IAChC;AAAA,IAAe;AAAA,IAAiB;AAAA,IAChC;AAAA,IAAe;AAAA,IAAiB;AAAA,EAClC;AAEA,MAAI;AAIF,UAAM,CAAC,SAAS,OAAO,IAAI,eAAe,SAAS,IAC/C,MAAM,QAAQ,IAAI,CAAC,UAAU,OAAO,cAAc,GAAG,UAAU,OAAO,cAAc,CAAC,CAAC,IACtF,CAAC,OAAO,KAAK;AAEjB,UAAM,CAAC,OAAO,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,MACvC,MAAM,OAAO,EAAE,SAAS;AAAA,MACxB,MAAM,OAAO,EAAE,SAAS;AAAA,IAC1B,CAAC;AACD,UAAM,IAAI,KAAK,IAAI,MAAM,SAAS,MAAM,MAAM,SAAS,IAAI;AAC3D,UAAM,IAAI,KAAK,IAAI,MAAM,UAAU,KAAK,MAAM,UAAU,GAAG;AAC3D,UAAM,KAAK,KAAK,MAAM,IAAI,CAAC;AAC3B,UAAM,KAAK,KAAK,MAAM,IAAI,CAAC;AAE3B,UAAM,UAA8E,CAAC;AAErF,aAAS,MAAM,GAAG,MAAM,GAAG,OAAO;AAChC,eAAS,MAAM,GAAG,MAAM,GAAG,OAAO;AAChC,cAAM,SAAS,EAAE,MAAM,MAAM,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,QAAQ,GAAG;AACtE,cAAM,QAAQ,YAAY,MAAM,IAAI,GAAG;AAEvC,cAAM,CAAC,OAAO,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,UACvC,MAAM,OAAO,EAAE,QAAQ,MAAM,EAAE,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EACrF,KAAK,CAAC,QAAQ;AACb,kBAAM,SAAS,MAAM,KAAK,GAAG;AAC7B,kBAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,mBAAO,OAAO,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,OAAO,IAAK,MAAM,OAAO,KAAK,CAAC,IAAK,GAAG,EAAE;AAAA,UAClF,CAAC;AAAA,UACH,MAAM,OAAO,EAAE,QAAQ,MAAM,EAAE,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EACrF,KAAK,CAAC,QAAQ;AACb,kBAAM,SAAS,MAAM,KAAK,GAAG;AAC7B,kBAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,mBAAO,OAAO,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,OAAO,IAAK,MAAM,OAAO,KAAK,CAAC,IAAK,GAAG,EAAE;AAAA,UAClF,CAAC;AAAA,QACL,CAAC;AAED,YAAI,MAAM,QAAQ;AAClB,YAAI,OAAO;AACX,eAAO,QAAQ,IAAI;AAAE,kBAAQ,OAAO,MAAM,EAAE;AAAG,kBAAQ;AAAA,QAAI;AAC3D,cAAM,MAAO,OAAO,KAAM;AAE1B,YAAI,OAAO,GAAG;AACZ,gBAAM,WAAW,OAAO,KAAK,SAAS,OAAO,KAAK,WAAW;AAC7D,kBAAQ,KAAK;AAAA,YACX,UAAU;AAAA,YACV;AAAA,YACA,aAAa,GAAG,KAAK,MAAM,GAAG,CAAC,sBAAsB,MAAM,QAAQ,KAAK,GAAG,CAAC;AAAA,UAC9E,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/pii-scrubber.d.ts

@@ -0,0 +1,21 @@
+/**
+ * PII scrubber — detects and redacts Personally Identifiable Information
+ * from crawled markdown, HTML, and extracted text before storage.
+ *
+ * GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
+ */
+export interface PiiMatch {
+    type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';
+    value: string;
+    start: number;
+    end: number;
+}
+export interface ScrubResult {
+    text: string;
+    matches: PiiMatch[];
+    scrubbed: boolean;
+}
+export declare function detectPii(text: string): PiiMatch[];
+export declare function scrubPii(text: string): ScrubResult;
+export declare function hasPii(text: string): boolean;
+export declare function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T;

package/dist/pii-scrubber.js

@@ -0,0 +1,75 @@
+const PATTERNS = [
+  // Email — RFC 5321 simplified
+  { type: "email", re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
+  // US SSN — 9 digits with dashes or spaces
+  { type: "ssn", re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
+  // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
+  { type: "credit-card", re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
+  // IBAN — international bank account (15-34 chars)
+  { type: "iban", re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
+  // Phone — international or US formats
+  { type: "phone", re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
+  // Date of birth — common formats
+  { type: "dob", re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
+  // IPv4 private/internal addresses — only flag private-looking ones in content
+  { type: "ip-address", re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g }
+];
+const REDACT_MAP = {
+  email: "[EMAIL]",
+  phone: "[PHONE]",
+  ssn: "[SSN]",
+  "credit-card": "[CARD]",
+  "ip-address": "[IP]",
+  passport: "[PASSPORT]",
+  iban: "[IBAN]",
+  dob: "[DOB]"
+};
+function detectPii(text) {
+  const matches = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const { type, re } of PATTERNS) {
+    const r = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g");
+    let m;
+    while ((m = r.exec(text)) !== null) {
+      const key = `${type}:${m.index}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
+      }
+    }
+  }
+  return matches.sort((a, b) => a.start - b.start);
+}
+function scrubPii(text) {
+  const matches = detectPii(text);
+  if (matches.length === 0) return { text, matches: [], scrubbed: false };
+  let scrubbed = text;
+  for (let i = matches.length - 1; i >= 0; i--) {
+    const { type, start, end } = matches[i];
+    scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
+  }
+  return { text: scrubbed, matches, scrubbed: true };
+}
+function hasPii(text) {
+  for (const { re } of PATTERNS) {
+    const r = new RegExp(re.source, re.flags);
+    if (r.test(text)) return true;
+  }
+  return false;
+}
+function scrubObject(obj, fields) {
+  const result = { ...obj };
+  for (const field of fields) {
+    if (typeof result[field] === "string") {
+      result[field] = scrubPii(result[field]).text;
+    }
+  }
+  return result;
+}
+export {
+  detectPii,
+  hasPii,
+  scrubObject,
+  scrubPii
+};
+//# sourceMappingURL=pii-scrubber.js.map

package/dist/pii-scrubber.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/pii-scrubber.ts"],
+  "sourcesContent": ["/**\n * PII scrubber \u2014 detects and redacts Personally Identifiable Information\n * from crawled markdown, HTML, and extracted text before storage.\n *\n * GDPR Article 4(1): \"any information relating to an identified or identifiable natural person\"\n */\n\nexport interface PiiMatch {\n  type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';\n  value: string;\n  start: number;\n  end: number;\n}\n\nexport interface ScrubResult {\n  text: string;\n  matches: PiiMatch[];\n  scrubbed: boolean;\n}\n\n// Patterns \u2014 ordered from most-specific to least-specific\nconst PATTERNS: Array<{ type: PiiMatch['type']; re: RegExp }> = [\n  // Email \u2014 RFC 5321 simplified\n  { type: 'email', re: /\\b[a-zA-Z0-9._%+\\-]+@[a-zA-Z0-9.\\-]+\\.[a-zA-Z]{2,}\\b/g },\n  // US SSN \u2014 9 digits with dashes or spaces\n  { type: 'ssn', re: /\\b(?!000|666|9\\d\\d)\\d{3}[-\\s](?!00)\\d{2}[-\\s](?!0000)\\d{4}\\b/g },\n  // Credit card \u2014 Visa/MC/Amex/Discover with optional spaces/dashes\n  { type: 'credit-card', re: /\\b(?:4\\d{3}|5[1-5]\\d{2}|3[47]\\d{2}|6(?:011|5\\d{2}))[-\\s]?\\d{4}[-\\s]?\\d{4}[-\\s]?\\d{3,4}\\b/g },\n  // IBAN \u2014 international bank account (15-34 chars)\n  { type: 'iban', re: /\\b[A-Z]{2}\\d{2}[A-Z0-9]{1,30}\\b/g },\n  // Phone \u2014 international or US formats\n  { type: 'phone', re: /(?:\\+?1[-.\\s]?)?\\(?\\d{3}\\)?[-.\\s]\\d{3}[-.\\s]\\d{4}\\b|\\+\\d{1,3}[-.\\s]\\d{2,4}[-.\\s]\\d{3,4}[-.\\s]\\d{3,4}/g },\n  // Date of birth \u2014 common formats\n  { type: 'dob', re: /\\b(?:0?[1-9]|1[0-2])\\/(?:0?[1-9]|[12]\\d|3[01])\\/(?:19|20)\\d{2}\\b/g },\n  // IPv4 private/internal addresses \u2014 only flag private-looking ones in content\n  { type: 'ip-address', re: /\\b(?:192\\.168\\.\\d{1,3}\\.\\d{1,3}|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|172\\.(?:1[6-9]|2\\d|3[01])\\.\\d{1,3}\\.\\d{1,3})\\b/g },\n];\n\nconst REDACT_MAP: Record<PiiMatch['type'], string> = {\n  email: '[EMAIL]',\n  phone: '[PHONE]',\n  ssn: '[SSN]',\n  'credit-card': '[CARD]',\n  'ip-address': '[IP]',\n  passport: '[PASSPORT]',\n  iban: '[IBAN]',\n  dob: '[DOB]',\n};\n\nexport function detectPii(text: string): PiiMatch[] {\n  const matches: PiiMatch[] = [];\n  const seen = new Set<string>();\n\n  for (const { type, re } of PATTERNS) {\n    const r = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');\n    let m: RegExpExecArray | null;\n    while ((m = r.exec(text)) !== null) {\n      const key = `${type}:${m.index}`;\n      if (!seen.has(key)) {\n        seen.add(key);\n        matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });\n      }\n    }\n  }\n\n  return matches.sort((a, b) => a.start - b.start);\n}\n\nexport function scrubPii(text: string): ScrubResult {\n  const matches = detectPii(text);\n  if (matches.length === 0) return { text, matches: [], scrubbed: false };\n\n  // Replace from end to start so indices stay valid\n  let scrubbed = text;\n  for (let i = matches.length - 1; i >= 0; i--) {\n    const { type, start, end } = matches[i];\n    scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);\n  }\n\n  return { text: scrubbed, matches, scrubbed: true };\n}\n\nexport function hasPii(text: string): boolean {\n  for (const { re } of PATTERNS) {\n    const r = new RegExp(re.source, re.flags);\n    if (r.test(text)) return true;\n  }\n  return false;\n}\n\nexport function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T {\n  const result = { ...obj };\n  for (const field of fields) {\n    if (typeof result[field] === 'string') {\n      (result as any)[field] = scrubPii(result[field] as string).text;\n    }\n  }\n  return result;\n}\n"],
+  "mappings": "AAqBA,MAAM,WAA0D;AAAA;AAAA,EAE9D,EAAE,MAAM,SAAS,IAAI,wDAAwD;AAAA;AAAA,EAE7E,EAAE,MAAM,OAAO,IAAI,gEAAgE;AAAA;AAAA,EAEnF,EAAE,MAAM,eAAe,IAAI,4FAA4F;AAAA;AAAA,EAEvH,EAAE,MAAM,QAAQ,IAAI,mCAAmC;AAAA;AAAA,EAEvD,EAAE,MAAM,SAAS,IAAI,wGAAwG;AAAA;AAAA,EAE7H,EAAE,MAAM,OAAO,IAAI,oEAAoE;AAAA;AAAA,EAEvF,EAAE,MAAM,cAAc,IAAI,gHAAgH;AAC5I;AAEA,MAAM,aAA+C;AAAA,EACnD,OAAO;AAAA,EACP,OAAO;AAAA,EACP,KAAK;AAAA,EACL,eAAe;AAAA,EACf,cAAc;AAAA,EACd,UAAU;AAAA,EACV,MAAM;AAAA,EACN,KAAK;AACP;AAEO,SAAS,UAAU,MAA0B;AAClD,QAAM,UAAsB,CAAC;AAC7B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,EAAE,MAAM,GAAG,KAAK,UAAU;AACnC,UAAM,IAAI,IAAI,OAAO,GAAG,QAAQ,GAAG,MAAM,SAAS,GAAG,IAAI,GAAG,QAAQ,GAAG,QAAQ,GAAG;AAClF,QAAI;AACJ,YAAQ,IAAI,EAAE,KAAK,IAAI,OAAO,MAAM;AAClC,YAAM,MAAM,GAAG,IAAI,IAAI,EAAE,KAAK;AAC9B,UAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAClB,aAAK,IAAI,GAAG;AACZ,gBAAQ,KAAK,EAAE,MAAM,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,OAAO,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,QAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACjD;AAEO,SAAS,SAAS,MAA2B;AAClD,QAAM,UAAU,UAAU,IAAI;AAC9B,MAAI,QAAQ,WAAW,EAAG,QAAO,EAAE,MAAM,SAAS,CAAC,GAAG,UAAU,MAAM;AAGtE,MAAI,WAAW;AACf,WAAS,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KAAK;AAC5C,UAAM,EAAE,MAAM,OAAO,IAAI,IAAI,QAAQ,CAAC;AACtC,eAAW,SAAS,MAAM,GAAG,KAAK,IAAI,WAAW,IAAI,IAAI,SAAS,MAAM,GAAG;AAAA,EAC7E;AAEA,SAAO,EAAE,MAAM,UAAU,SAAS,UAAU,KAAK;AACnD;AAEO,SAAS,OAAO,MAAuB;AAC5C,aAAW,EAAE,GAAG,KAAK,UAAU;AAC7B,UAAM,IAAI,IAAI,OAAO,GAAG,QAAQ,GAAG,KAAK;AACxC,QAAI,EAAE,KAAK,IAAI,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAEO,SAAS,YAA+C,KAAQ,QAAwB;AAC7F,QAAM,SAAS,EAAE,GAAG,IAAI;AACxB,aAAW,SAAS,QAAQ;AAC1B,QAAI,OAAO,OAAO,KAAK,MAAM,UAAU;AACrC,MAAC,OAAe,KAAK,IAAI,SAAS,OAAO,KAAK,CAAW,EAAE;AAAA,IAC7D;AAAA,EACF;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/playwright-recorder.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Playwright test recorder — generate Playwright test code from crawl data.
+ *
+ * Given a list of crawled screens, generates a minimal Playwright test file that:
+ * 1. Navigates to each screen URL
+ * 2. Verifies the page title
+ * 3. Checks key elements visible on the page
+ * 4. Verifies no console errors (optional)
+ *
+ * This is a "smoke test" generator — not a full interaction recorder.
+ * Output: valid TypeScript Playwright test file ready to run with npx playwright test.
+ */
+export interface RecorderScreen {
+    id: string;
+    name: string;
+    url: string;
+    title?: string;
+    heading?: string;
+    markdownWordCount?: number;
+    smartLocator?: {
+        tag: string;
+        text?: string;
+        role?: string;
+        placeholder?: string;
+        nearText?: string;
+    };
+}
+export interface RecorderOpts {
+    baseUrl?: string;
+    includeConsoleCheck?: boolean;
+    testTimeout?: number;
+    projectName?: string;
+    useSmartLocators?: boolean;
+}
+export declare function generatePlaywrightTest(screens: RecorderScreen[], opts?: RecorderOpts): string;
+export declare function generatePlaywrightConfig(opts?: {
+    baseUrl?: string;
+    reportDir?: string;
+}): string;

package/dist/playwright-recorder.js

@@ -0,0 +1,123 @@
+function buildSmartLocatorHelper() {
+  return `
+// smartLocator \u2014 falls back to similarity search when primary selector fails.
+// Generated by ZeTa Crawler. Do not edit manually.
+async function smartLocator(page: any, primary: string, ref: { tag?: string; text?: string; role?: string; placeholder?: string }) {
+  try {
+    const el = page.locator(primary);
+    if (await el.count() > 0) return el;
+  } catch { /* fall through */ }
+  // Fallback: find by ARIA role + text
+  if (ref.role && ref.text) {
+    const byRole = page.getByRole(ref.role as any, { name: ref.text, exact: false });
+    if (await byRole.count() > 0) return byRole;
+  }
+  // Fallback: find by text content
+  if (ref.text) {
+    const byText = page.getByText(ref.text, { exact: false });
+    if (await byText.count() > 0) return byText;
+  }
+  // Fallback: find by placeholder
+  if (ref.placeholder) {
+    const byPlaceholder = page.getByPlaceholder(ref.placeholder, { exact: false });
+    if (await byPlaceholder.count() > 0) return byPlaceholder;
+  }
+  // Last resort: return the primary (will fail with original error)
+  return page.locator(primary);
+}`.trim();
+}
+function sanitizeTestName(name) {
+  return name.replace(/[^a-zA-Z0-9 _-]/g, "").trim().slice(0, 80) || "Screen";
+}
+function escapeString(s) {
+  return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/`/g, "\\`");
+}
+function generatePlaywrightTest(screens, opts = {}) {
+  const {
+    includeConsoleCheck = false,
+    testTimeout = 3e4,
+    projectName = "ZeTa Crawl",
+    useSmartLocators = true
+  } = opts;
+  const needsSmartLocator = useSmartLocators && screens.some((s) => s.smartLocator != null);
+  const lines = [
+    `// Generated by ZeTa Crawler \u2014 ${(/* @__PURE__ */ new Date()).toISOString()}`,
+    `// Project: ${escapeString(projectName)}`,
+    `// Screens: ${screens.length}`,
+    `// Run: npx playwright test`,
+    "",
+    "import { test, expect } from '@playwright/test';",
+    ""
+  ];
+  if (needsSmartLocator) {
+    lines.push(buildSmartLocatorHelper(), "");
+  }
+  lines.push(`test.setTimeout(${testTimeout});`, "");
+  if (includeConsoleCheck) {
+    lines.push(
+      `test.beforeEach(async ({ page }) => {`,
+      `  const errors: string[] = [];`,
+      `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`,
+      `  page.on('pageerror', err => errors.push(err.message));`,
+      `  (page as any).__consoleErrors = errors;`,
+      `});`,
+      ""
+    );
+  }
+  for (const screen of screens) {
+    const testName = sanitizeTestName(screen.name);
+    lines.push(
+      `test('${escapeString(testName)}', async ({ page }) => {`,
+      `  await page.goto('${escapeString(screen.url)}');`,
+      `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))}/);`
+    );
+    if (screen.title) {
+      lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))}/i);`);
+    }
+    if (screen.heading) {
+      const h = escapeString(screen.heading.slice(0, 60));
+      if (needsSmartLocator && screen.smartLocator) {
+        const tag = escapeString(screen.smartLocator.tag);
+        const text = escapeString(screen.smartLocator.text ?? "");
+        lines.push(
+          `  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`,
+          `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`
+        );
+      } else {
+        lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);
+      }
+    }
+    if (includeConsoleCheck) {
+      lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);
+    }
+    lines.push(`});`, "");
+  }
+  return lines.join("\n");
+}
+function generatePlaywrightConfig(opts = {}) {
+  const { baseUrl = "http://localhost:3000", reportDir = "playwright-report" } = opts;
+  return `// playwright.config.ts \u2014 generated by ZeTa Crawler
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  reporter: [['html', { outputFolder: '${reportDir}' }]],
+  use: {
+    baseURL: '${baseUrl}',
+    headless: true,
+    screenshot: 'only-on-failure',
+    video: 'retain-on-failure',
+  },
+  projects: [
+    { name: 'chromium', use: { browserName: 'chromium' } },
+    { name: 'firefox', use: { browserName: 'firefox' } },
+    { name: 'webkit', use: { browserName: 'webkit' } },
+  ],
+});
+`;
+}
+export {
+  generatePlaywrightConfig,
+  generatePlaywrightTest
+};
+//# sourceMappingURL=playwright-recorder.js.map

package/dist/playwright-recorder.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/playwright-recorder.ts"],
+  "sourcesContent": ["/**\n * Playwright test recorder \u2014 generate Playwright test code from crawl data.\n *\n * Given a list of crawled screens, generates a minimal Playwright test file that:\n * 1. Navigates to each screen URL\n * 2. Verifies the page title\n * 3. Checks key elements visible on the page\n * 4. Verifies no console errors (optional)\n *\n * This is a \"smoke test\" generator \u2014 not a full interaction recorder.\n * Output: valid TypeScript Playwright test file ready to run with npx playwright test.\n */\n\nexport interface RecorderScreen {\n  id: string;\n  name: string;\n  url: string;\n  title?: string;\n  heading?: string;\n  markdownWordCount?: number;\n  smartLocator?: {\n    tag: string;          // e.g. 'button', 'a', 'input'\n    text?: string;        // visible text content\n    role?: string;        // ARIA role\n    placeholder?: string; // for inputs\n    nearText?: string;    // nearby label text\n  };\n}\n\nexport interface RecorderOpts {\n  baseUrl?: string;\n  includeConsoleCheck?: boolean;\n  testTimeout?: number; // ms, default 30000\n  projectName?: string;\n  useSmartLocators?: boolean; // default true\n}\n\nfunction buildSmartLocatorHelper(): string {\n  return `\n// smartLocator \u2014 falls back to similarity search when primary selector fails.\n// Generated by ZeTa Crawler. Do not edit manually.\nasync function smartLocator(page: any, primary: string, ref: { tag?: string; text?: string; role?: string; placeholder?: string }) {\n  try {\n    const el = page.locator(primary);\n    if (await el.count() > 0) return el;\n  } catch { /* fall through */ }\n  // Fallback: find by ARIA role + text\n  if (ref.role && ref.text) {\n    const byRole = page.getByRole(ref.role as any, { name: ref.text, exact: false });\n    if (await byRole.count() > 0) return byRole;\n  }\n  // Fallback: find by text content\n  if (ref.text) {\n    const byText = page.getByText(ref.text, { exact: false });\n    if (await byText.count() > 0) return byText;\n  }\n  // Fallback: find by placeholder\n  if (ref.placeholder) {\n    const byPlaceholder = page.getByPlaceholder(ref.placeholder, { exact: false });\n    if (await byPlaceholder.count() > 0) return byPlaceholder;\n  }\n  // Last resort: return the primary (will fail with original error)\n  return page.locator(primary);\n}`.trim();\n}\n\nfunction sanitizeTestName(name: string): string {\n  return name.replace(/[^a-zA-Z0-9 _-]/g, '').trim().slice(0, 80) || 'Screen';\n}\n\nfunction escapeString(s: string): string {\n  return s.replace(/\\\\/g, '\\\\\\\\').replace(/'/g, \"\\\\'\").replace(/`/g, '\\\\`');\n}\n\nexport function generatePlaywrightTest(screens: RecorderScreen[], opts: RecorderOpts = {}): string {\n  const {\n    includeConsoleCheck = false,\n    testTimeout = 30_000,\n    projectName = 'ZeTa Crawl',\n    useSmartLocators = true,\n  } = opts;\n\n  const needsSmartLocator = useSmartLocators && screens.some(s => s.smartLocator != null);\n\n  const lines: string[] = [\n    `// Generated by ZeTa Crawler \u2014 ${new Date().toISOString()}`,\n    `// Project: ${escapeString(projectName)}`,\n    `// Screens: ${screens.length}`,\n    `// Run: npx playwright test`,\n    '',\n    \"import { test, expect } from '@playwright/test';\",\n    '',\n  ];\n\n  if (needsSmartLocator) {\n    lines.push(buildSmartLocatorHelper(), '');\n  }\n\n  lines.push(`test.setTimeout(${testTimeout});`, '');\n\n  if (includeConsoleCheck) {\n    lines.push(\n      `test.beforeEach(async ({ page }) => {`,\n      `  const errors: string[] = [];`,\n      `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`,\n      `  page.on('pageerror', err => errors.push(err.message));`,\n      `  (page as any).__consoleErrors = errors;`,\n      `});`,\n      '',\n    );\n  }\n\n  for (const screen of screens) {\n    const testName = sanitizeTestName(screen.name);\n    lines.push(\n      `test('${escapeString(testName)}', async ({ page }) => {`,\n      `  await page.goto('${escapeString(screen.url)}');`,\n      `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'))}/);`,\n    );\n\n    if (screen.title) {\n      lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'))}/i);`);\n    }\n\n    if (screen.heading) {\n      const h = escapeString(screen.heading.slice(0, 60));\n      if (needsSmartLocator && screen.smartLocator) {\n        const tag = escapeString(screen.smartLocator.tag);\n        const text = escapeString(screen.smartLocator.text ?? '');\n        lines.push(\n          `  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`,\n          `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`,\n        );\n      } else {\n        lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);\n      }\n    }\n\n    if (includeConsoleCheck) {\n      lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);\n    }\n\n    lines.push(`});`, '');\n  }\n\n  return lines.join('\\n');\n}\n\nexport function generatePlaywrightConfig(opts: { baseUrl?: string; reportDir?: string } = {}): string {\n  const { baseUrl = 'http://localhost:3000', reportDir = 'playwright-report' } = opts;\n  return `// playwright.config.ts \u2014 generated by ZeTa Crawler\nimport { defineConfig } from '@playwright/test';\n\nexport default defineConfig({\n  testDir: './tests',\n  reporter: [['html', { outputFolder: '${reportDir}' }]],\n  use: {\n    baseURL: '${baseUrl}',\n    headless: true,\n    screenshot: 'only-on-failure',\n    video: 'retain-on-failure',\n  },\n  projects: [\n    { name: 'chromium', use: { browserName: 'chromium' } },\n    { name: 'firefox', use: { browserName: 'firefox' } },\n    { name: 'webkit', use: { browserName: 'webkit' } },\n  ],\n});\n`;\n}\n"],
+  "mappings": "AAqCA,SAAS,0BAAkC;AACzC,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAyBN,KAAK;AACR;AAEA,SAAS,iBAAiB,MAAsB;AAC9C,SAAO,KAAK,QAAQ,oBAAoB,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE,KAAK;AACrE;AAEA,SAAS,aAAa,GAAmB;AACvC,SAAO,EAAE,QAAQ,OAAO,MAAM,EAAE,QAAQ,MAAM,KAAK,EAAE,QAAQ,MAAM,KAAK;AAC1E;AAEO,SAAS,uBAAuB,SAA2B,OAAqB,CAAC,GAAW;AACjG,QAAM;AAAA,IACJ,sBAAsB;AAAA,IACtB,cAAc;AAAA,IACd,cAAc;AAAA,IACd,mBAAmB;AAAA,EACrB,IAAI;AAEJ,QAAM,oBAAoB,oBAAoB,QAAQ,KAAK,OAAK,EAAE,gBAAgB,IAAI;AAEtF,QAAM,QAAkB;AAAA,IACtB,wCAAkC,oBAAI,KAAK,GAAE,YAAY,CAAC;AAAA,IAC1D,eAAe,aAAa,WAAW,CAAC;AAAA,IACxC,eAAe,QAAQ,MAAM;AAAA,IAC7B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,MAAI,mBAAmB;AACrB,UAAM,KAAK,wBAAwB,GAAG,EAAE;AAAA,EAC1C;AAEA,QAAM,KAAK,mBAAmB,WAAW,MAAM,EAAE;AAEjD,MAAI,qBAAqB;AACvB,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,aAAW,UAAU,SAAS;AAC5B,UAAM,WAAW,iBAAiB,OAAO,IAAI;AAC7C,UAAM;AAAA,MACJ,SAAS,aAAa,QAAQ,CAAC;AAAA,MAC/B,sBAAsB,aAAa,OAAO,GAAG,CAAC;AAAA,MAC9C,mCAAmC,aAAa,OAAO,IAAI,QAAQ,uBAAuB,MAAM,CAAC,CAAC;AAAA,IACpG;AAEA,QAAI,OAAO,OAAO;AAChB,YAAM,KAAK,qCAAqC,aAAa,OAAO,MAAM,MAAM,GAAG,EAAE,EAAE,QAAQ,uBAAuB,MAAM,CAAC,CAAC,MAAM;AAAA,IACtI;AAEA,QAAI,OAAO,SAAS;AAClB,YAAM,IAAI,aAAa,OAAO,QAAQ,MAAM,GAAG,EAAE,CAAC;AAClD,UAAI,qBAAqB,OAAO,cAAc;AAC5C,cAAM,MAAM,aAAa,OAAO,aAAa,GAAG;AAChD,cAAM,OAAO,aAAa,OAAO,aAAa,QAAQ,EAAE;AACxD,cAAM;AAAA,UACJ,sEAAsE,GAAG,aAAa,IAAI;AAAA,UAC1F,oDAAoD,CAAC;AAAA,QACvD;AAAA,MACF,OAAO;AACL,cAAM,KAAK,qEAAqE,CAAC,2BAA2B;AAAA,MAC9G;AAAA,IACF;AAEA,QAAI,qBAAqB;AACvB,YAAM,KAAK,gEAAgE;AAAA,IAC7E;AAEA,UAAM,KAAK,OAAO,EAAE;AAAA,EACtB;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEO,SAAS,yBAAyB,OAAiD,CAAC,GAAW;AACpG,QAAM,EAAE,UAAU,yBAAyB,YAAY,oBAAoB,IAAI;AAC/E,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA,yCAKgC,SAAS;AAAA;AAAA,gBAElC,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAYvB;",
+  "names": []
+}

package/dist/policies/crawl-safety.d.ts

@@ -0,0 +1,9 @@
+import type { DomainProfile } from '../profiles/domain-profile.js';
+export interface CrawlSafetyDecision {
+    allowed: boolean;
+    reason?: string;
+    severity: 'info' | 'warn' | 'block';
+}
+export declare function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision;
+export declare function redactSensitiveValue(value: string): string;
+export declare function redactJson(value: unknown): unknown;

package/dist/policies/crawl-safety.js

@@ -0,0 +1,41 @@
+const _extraDestructive = process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS ? process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS.split(",").map((p) => p.trim()).filter(Boolean) : [];
+const DESTRUCTIVE_ACTION = new RegExp(
+  `\\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop${_extraDestructive.length ? `|${_extraDestructive.join("|")}` : ""})\\b`,
+  "i"
+);
+const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
+function assessActionSafety(label, profile) {
+  const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));
+  if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {
+    return {
+      allowed: false,
+      severity: "block",
+      reason: `Blocked risky action "${label}" for ${profile.industry} profile.`
+    };
+  }
+  return { allowed: true, severity: "info" };
+}
+function redactSensitiveValue(value) {
+  let redacted = value.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[email]").replace(/\b(?:\+?\d[\d -]{8,}\d)\b/g, "[phone-or-id]").replace(/\b(?:\d[ -]*?){13,19}\b/g, "[card-like-number]").replace(/\b[A-Za-z0-9_-]{24,}\b/g, "[token-like-value]");
+  if (SECRET_LIKE.test(redacted)) redacted = redacted.replace(/:\s*[^,\s}]+/g, ": [redacted]");
+  return redacted;
+}
+function redactJson(value) {
+  if (typeof value === "string") return redactSensitiveValue(value);
+  if (Array.isArray(value)) return value.map(redactJson);
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [key, item] of Object.entries(value)) {
+      if (SECRET_LIKE.test(key)) out[key] = "[redacted]";
+      else out[key] = redactJson(item);
+    }
+    return out;
+  }
+  return value;
+}
+export {
+  assessActionSafety,
+  redactJson,
+  redactSensitiveValue
+};
+//# sourceMappingURL=crawl-safety.js.map

package/dist/policies/crawl-safety.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/policies/crawl-safety.ts"],
+  "sourcesContent": ["import type { DomainProfile } from '../profiles/domain-profile.js';\n\nexport interface CrawlSafetyDecision {\n  allowed: boolean;\n  reason?: string;\n  severity: 'info' | 'warn' | 'block';\n}\n\nconst _extraDestructive = process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS\n  ? process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS.split(',').map((p) => p.trim()).filter(Boolean)\n  : [];\nconst DESTRUCTIVE_ACTION = new RegExp(\n  `\\\\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop${_extraDestructive.length ? `|${_extraDestructive.join('|')}` : ''})\\\\b`,\n  'i',\n);\nconst SECRET_LIKE = /\\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\\b/i;\n\nexport function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision {\n  const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));\n  if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {\n    return {\n      allowed: false,\n      severity: 'block',\n      reason: `Blocked risky action \"${label}\" for ${profile.industry} profile.`,\n    };\n  }\n  return { allowed: true, severity: 'info' };\n}\n\nexport function redactSensitiveValue(value: string): string {\n  let redacted = value\n    .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}/gi, '[email]')\n    .replace(/\\b(?:\\+?\\d[\\d -]{8,}\\d)\\b/g, '[phone-or-id]')\n    .replace(/\\b(?:\\d[ -]*?){13,19}\\b/g, '[card-like-number]')\n    .replace(/\\b[A-Za-z0-9_-]{24,}\\b/g, '[token-like-value]');\n  if (SECRET_LIKE.test(redacted)) redacted = redacted.replace(/:\\s*[^,\\s}]+/g, ': [redacted]');\n  return redacted;\n}\n\nexport function redactJson(value: unknown): unknown {\n  if (typeof value === 'string') return redactSensitiveValue(value);\n  if (Array.isArray(value)) return value.map(redactJson);\n  if (value && typeof value === 'object') {\n    const out: Record<string, unknown> = {};\n    for (const [key, item] of Object.entries(value)) {\n      if (SECRET_LIKE.test(key)) out[key] = '[redacted]';\n      else out[key] = redactJson(item);\n    }\n    return out;\n  }\n  return value;\n}\n"],
+  "mappings": "AAQA,MAAM,oBAAoB,QAAQ,IAAI,mCAClC,QAAQ,IAAI,iCAAiC,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO,IAC3F,CAAC;AACL,MAAM,qBAAqB,IAAI;AAAA,EAC7B,wLAAwL,kBAAkB,SAAS,IAAI,kBAAkB,KAAK,GAAG,CAAC,KAAK,EAAE;AAAA,EACzP;AACF;AACA,MAAM,cAAc;AAEb,SAAS,mBAAmB,OAAe,SAA6C;AAC7F,QAAM,iBAAiB,QAAQ,aAAa,KAAK,CAAC,SAAS,MAAM,YAAY,EAAE,SAAS,KAAK,YAAY,CAAC,CAAC;AAC3G,MAAI,mBAAmB,KAAK,KAAK,KAAK,gBAAgB;AACpD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,UAAU;AAAA,MACV,QAAQ,yBAAyB,KAAK,SAAS,QAAQ,QAAQ;AAAA,IACjE;AAAA,EACF;AACA,SAAO,EAAE,SAAS,MAAM,UAAU,OAAO;AAC3C;AAEO,SAAS,qBAAqB,OAAuB;AAC1D,MAAI,WAAW,MACZ,QAAQ,2CAA2C,SAAS,EAC5D,QAAQ,8BAA8B,eAAe,EACrD,QAAQ,4BAA4B,oBAAoB,EACxD,QAAQ,2BAA2B,oBAAoB;AAC1D,MAAI,YAAY,KAAK,QAAQ,EAAG,YAAW,SAAS,QAAQ,iBAAiB,cAAc;AAC3F,SAAO;AACT;AAEO,SAAS,WAAW,OAAyB;AAClD,MAAI,OAAO,UAAU,SAAU,QAAO,qBAAqB,KAAK;AAChE,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,UAAU;AACrD,MAAI,SAAS,OAAO,UAAU,UAAU;AACtC,UAAM,MAA+B,CAAC;AACtC,eAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,KAAK,GAAG;AAC/C,UAAI,YAAY,KAAK,GAAG,EAAG,KAAI,GAAG,IAAI;AAAA,UACjC,KAAI,GAAG,IAAI,WAAW,IAAI;AAAA,IACjC;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/profiles/domain-profile.d.ts

@@ -0,0 +1,19 @@
+export type Industry = 'ecommerce' | 'banking' | 'fintech' | 'healthcare' | 'pharma' | 'agriculture' | 'legacy-enterprise' | 'saas' | 'unknown';
+export interface DomainProfile {
+    industry: Industry;
+    confidence: number;
+    importantScreenPatterns: string[];
+    workflowGoals: string[];
+    piiRules: string[];
+    riskyActions: string[];
+    safeProbeRules: string[];
+    requiredArtifacts: string[];
+    evidence: string[];
+}
+export declare function classifyDomainProfile(input: {
+    appUrl: string;
+    projectName?: string | null;
+    appType?: string | null;
+    seedText?: string;
+}): DomainProfile;
+export declare function getProfile(industry: Industry): DomainProfile;