@detiq/crawler 1.1.7 → 1.1.9

package/dist/pii-scrubber.js

@@ -1,75 +1,78 @@
+/**
+ * PII scrubber — detects and redacts Personally Identifiable Information
+ * from crawled markdown, HTML, and extracted text before storage.
+ *
+ * GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
+ */
+// Patterns — ordered from most-specific to least-specific
 const PATTERNS = [
-  // Email — RFC 5321 simplified
-  { type: "email", re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
-  // US SSN — 9 digits with dashes or spaces
-  { type: "ssn", re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
-  // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
-  { type: "credit-card", re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
-  // IBAN — international bank account (15-34 chars)
-  { type: "iban", re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
-  // Phone — international or US formats
-  { type: "phone", re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
-  // Date of birth — common formats
-  { type: "dob", re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
-  // IPv4 private/internal addresses — only flag private-looking ones in content
-  { type: "ip-address", re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g }
+    // Email — RFC 5321 simplified
+    { type: 'email', re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
+    // US SSN — 9 digits with dashes or spaces
+    { type: 'ssn', re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
+    // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
+    { type: 'credit-card', re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
+    // IBAN — international bank account (15-34 chars)
+    { type: 'iban', re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
+    // Phone — international or US formats
+    { type: 'phone', re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
+    // Date of birth — common formats
+    { type: 'dob', re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
+    // IPv4 private/internal addresses — only flag private-looking ones in content
+    { type: 'ip-address', re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g },
 ];
 const REDACT_MAP = {
-  email: "[EMAIL]",
-  phone: "[PHONE]",
-  ssn: "[SSN]",
-  "credit-card": "[CARD]",
-  "ip-address": "[IP]",
-  passport: "[PASSPORT]",
-  iban: "[IBAN]",
-  dob: "[DOB]"
+    email: '[EMAIL]',
+    phone: '[PHONE]',
+    ssn: '[SSN]',
+    'credit-card': '[CARD]',
+    'ip-address': '[IP]',
+    passport: '[PASSPORT]',
+    iban: '[IBAN]',
+    dob: '[DOB]',
 };
-function detectPii(text) {
-  const matches = [];
-  const seen = /* @__PURE__ */ new Set();
-  for (const { type, re } of PATTERNS) {
-    const r = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g");
-    let m;
-    while ((m = r.exec(text)) !== null) {
-      const key = `${type}:${m.index}`;
-      if (!seen.has(key)) {
-        seen.add(key);
-        matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
-      }
+export function detectPii(text) {
+    const matches = [];
+    const seen = new Set();
+    for (const { type, re } of PATTERNS) {
+        const r = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');
+        let m;
+        while ((m = r.exec(text)) !== null) {
+            const key = `${type}:${m.index}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
+            }
+        }
     }
-  }
-  return matches.sort((a, b) => a.start - b.start);
+    return matches.sort((a, b) => a.start - b.start);
 }
-function scrubPii(text) {
-  const matches = detectPii(text);
-  if (matches.length === 0) return { text, matches: [], scrubbed: false };
-  let scrubbed = text;
-  for (let i = matches.length - 1; i >= 0; i--) {
-    const { type, start, end } = matches[i];
-    scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
-  }
-  return { text: scrubbed, matches, scrubbed: true };
+export function scrubPii(text) {
+    const matches = detectPii(text);
+    if (matches.length === 0)
+        return { text, matches: [], scrubbed: false };
+    // Replace from end to start so indices stay valid
+    let scrubbed = text;
+    for (let i = matches.length - 1; i >= 0; i--) {
+        const { type, start, end } = matches[i];
+        scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
+    }
+    return { text: scrubbed, matches, scrubbed: true };
 }
-function hasPii(text) {
-  for (const { re } of PATTERNS) {
-    const r = new RegExp(re.source, re.flags);
-    if (r.test(text)) return true;
-  }
-  return false;
+export function hasPii(text) {
+    for (const { re } of PATTERNS) {
+        const r = new RegExp(re.source, re.flags);
+        if (r.test(text))
+            return true;
+    }
+    return false;
 }
-function scrubObject(obj, fields) {
-  const result = { ...obj };
-  for (const field of fields) {
-    if (typeof result[field] === "string") {
-      result[field] = scrubPii(result[field]).text;
+export function scrubObject(obj, fields) {
+    const result = { ...obj };
+    for (const field of fields) {
+        if (typeof result[field] === 'string') {
+            result[field] = scrubPii(result[field]).text;
+        }
     }
-  }
-  return result;
+    return result;
 }
-export {
-  detectPii,
-  hasPii,
-  scrubObject,
-  scrubPii
-};
-//# sourceMappingURL=pii-scrubber.js.map

package/dist/playwright-recorder.js

@@ -1,6 +1,18 @@
+/**
+ * Playwright test recorder — generate Playwright test code from crawl data.
+ *
+ * Given a list of crawled screens, generates a minimal Playwright test file that:
+ * 1. Navigates to each screen URL
+ * 2. Verifies the page title
+ * 3. Checks key elements visible on the page
+ * 4. Verifies no console errors (optional)
+ *
+ * This is a "smoke test" generator — not a full interaction recorder.
+ * Output: valid TypeScript Playwright test file ready to run with npx playwright test.
+ */
 function buildSmartLocatorHelper() {
-  return `
-// smartLocator \u2014 falls back to similarity search when primary selector fails.
+    return `
+// smartLocator — falls back to similarity search when primary selector fails.
 // Generated by ZeTa Crawler. Do not edit manually.
 async function smartLocator(page: any, primary: string, ref: { tag?: string; text?: string; role?: string; placeholder?: string }) {
   try {
@@ -27,76 +39,57 @@ async function smartLocator(page: any, primary: string, ref: { tag?: string; tex
 }`.trim();
 }
 function sanitizeTestName(name) {
-  return name.replace(/[^a-zA-Z0-9 _-]/g, "").trim().slice(0, 80) || "Screen";
+    return name.replace(/[^a-zA-Z0-9 _-]/g, '').trim().slice(0, 80) || 'Screen';
 }
 function escapeString(s) {
-  return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/`/g, "\\`");
+    return s.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/`/g, '\\`');
 }
-function generatePlaywrightTest(screens, opts = {}) {
-  const {
-    includeConsoleCheck = false,
-    testTimeout = 3e4,
-    projectName = "ZeTa Crawl",
-    useSmartLocators = true
-  } = opts;
-  const needsSmartLocator = useSmartLocators && screens.some((s) => s.smartLocator != null);
-  const lines = [
-    `// Generated by ZeTa Crawler \u2014 ${(/* @__PURE__ */ new Date()).toISOString()}`,
-    `// Project: ${escapeString(projectName)}`,
-    `// Screens: ${screens.length}`,
-    `// Run: npx playwright test`,
-    "",
-    "import { test, expect } from '@playwright/test';",
-    ""
-  ];
-  if (needsSmartLocator) {
-    lines.push(buildSmartLocatorHelper(), "");
-  }
-  lines.push(`test.setTimeout(${testTimeout});`, "");
-  if (includeConsoleCheck) {
-    lines.push(
-      `test.beforeEach(async ({ page }) => {`,
-      `  const errors: string[] = [];`,
-      `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`,
-      `  page.on('pageerror', err => errors.push(err.message));`,
-      `  (page as any).__consoleErrors = errors;`,
-      `});`,
-      ""
-    );
-  }
-  for (const screen of screens) {
-    const testName = sanitizeTestName(screen.name);
-    lines.push(
-      `test('${escapeString(testName)}', async ({ page }) => {`,
-      `  await page.goto('${escapeString(screen.url)}');`,
-      `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))}/);`
-    );
-    if (screen.title) {
-      lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))}/i);`);
-    }
-    if (screen.heading) {
-      const h = escapeString(screen.heading.slice(0, 60));
-      if (needsSmartLocator && screen.smartLocator) {
-        const tag = escapeString(screen.smartLocator.tag);
-        const text = escapeString(screen.smartLocator.text ?? "");
-        lines.push(
-          `  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`,
-          `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`
-        );
-      } else {
-        lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);
-      }
+export function generatePlaywrightTest(screens, opts = {}) {
+    const { includeConsoleCheck = false, testTimeout = 30_000, projectName = 'ZeTa Crawl', useSmartLocators = true, } = opts;
+    const needsSmartLocator = useSmartLocators && screens.some(s => s.smartLocator != null);
+    const lines = [
+        `// Generated by ZeTa Crawler — ${new Date().toISOString()}`,
+        `// Project: ${escapeString(projectName)}`,
+        `// Screens: ${screens.length}`,
+        `// Run: npx playwright test`,
+        '',
+        "import { test, expect } from '@playwright/test';",
+        '',
+    ];
+    if (needsSmartLocator) {
+        lines.push(buildSmartLocatorHelper(), '');
     }
+    lines.push(`test.setTimeout(${testTimeout});`, '');
     if (includeConsoleCheck) {
-      lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);
+        lines.push(`test.beforeEach(async ({ page }) => {`, `  const errors: string[] = [];`, `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`, `  page.on('pageerror', err => errors.push(err.message));`, `  (page as any).__consoleErrors = errors;`, `});`, '');
     }
-    lines.push(`});`, "");
-  }
-  return lines.join("\n");
+    for (const screen of screens) {
+        const testName = sanitizeTestName(screen.name);
+        lines.push(`test('${escapeString(testName)}', async ({ page }) => {`, `  await page.goto('${escapeString(screen.url)}');`, `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))}/);`);
+        if (screen.title) {
+            lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))}/i);`);
+        }
+        if (screen.heading) {
+            const h = escapeString(screen.heading.slice(0, 60));
+            if (needsSmartLocator && screen.smartLocator) {
+                const tag = escapeString(screen.smartLocator.tag);
+                const text = escapeString(screen.smartLocator.text ?? '');
+                lines.push(`  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`, `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`);
+            }
+            else {
+                lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);
+            }
+        }
+        if (includeConsoleCheck) {
+            lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);
+        }
+        lines.push(`});`, '');
+    }
+    return lines.join('\n');
 }
-function generatePlaywrightConfig(opts = {}) {
-  const { baseUrl = "http://localhost:3000", reportDir = "playwright-report" } = opts;
-  return `// playwright.config.ts \u2014 generated by ZeTa Crawler
+export function generatePlaywrightConfig(opts = {}) {
+    const { baseUrl = 'http://localhost:3000', reportDir = 'playwright-report' } = opts;
+    return `// playwright.config.ts — generated by ZeTa Crawler
 import { defineConfig } from '@playwright/test';
 
 export default defineConfig({
@@ -116,8 +109,3 @@ export default defineConfig({
 });
 `;
 }
-export {
-  generatePlaywrightConfig,
-  generatePlaywrightTest
-};
-//# sourceMappingURL=playwright-recorder.js.map

package/dist/policies/crawl-safety.js

@@ -1,37 +1,40 @@
 const DESTRUCTIVE_ACTION = /\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop)\b/i;
 const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
-function assessActionSafety(label, profile) {
-  const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));
-  if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {
-    return {
-      allowed: false,
-      severity: "block",
-      reason: `Blocked risky action "${label}" for ${profile.industry} profile.`
-    };
-  }
-  return { allowed: true, severity: "info" };
+export function assessActionSafety(label, profile) {
+    const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));
+    if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {
+        return {
+            allowed: false,
+            severity: 'block',
+            reason: `Blocked risky action "${label}" for ${profile.industry} profile.`,
+        };
+    }
+    return { allowed: true, severity: 'info' };
 }
-function redactSensitiveValue(value) {
-  let redacted = value.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[email]").replace(/\b(?:\+?\d[\d -]{8,}\d)\b/g, "[phone-or-id]").replace(/\b(?:\d[ -]*?){13,19}\b/g, "[card-like-number]").replace(/\b[A-Za-z0-9_-]{24,}\b/g, "[token-like-value]");
-  if (SECRET_LIKE.test(redacted)) redacted = redacted.replace(/:\s*[^,\s}]+/g, ": [redacted]");
-  return redacted;
+export function redactSensitiveValue(value) {
+    let redacted = value
+        .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]')
+        .replace(/\b(?:\+?\d[\d -]{8,}\d)\b/g, '[phone-or-id]')
+        .replace(/\b(?:\d[ -]*?){13,19}\b/g, '[card-like-number]')
+        .replace(/\b[A-Za-z0-9_-]{24,}\b/g, '[token-like-value]');
+    if (SECRET_LIKE.test(redacted))
+        redacted = redacted.replace(/:\s*[^,\s}]+/g, ': [redacted]');
+    return redacted;
 }
-function redactJson(value) {
-  if (typeof value === "string") return redactSensitiveValue(value);
-  if (Array.isArray(value)) return value.map(redactJson);
-  if (value && typeof value === "object") {
-    const out = {};
-    for (const [key, item] of Object.entries(value)) {
-      if (SECRET_LIKE.test(key)) out[key] = "[redacted]";
-      else out[key] = redactJson(item);
+export function redactJson(value) {
+    if (typeof value === 'string')
+        return redactSensitiveValue(value);
+    if (Array.isArray(value))
+        return value.map(redactJson);
+    if (value && typeof value === 'object') {
+        const out = {};
+        for (const [key, item] of Object.entries(value)) {
+            if (SECRET_LIKE.test(key))
+                out[key] = '[redacted]';
+            else
+                out[key] = redactJson(item);
+        }
+        return out;
     }
-    return out;
-  }
-  return value;
+    return value;
 }
-export {
-  assessActionSafety,
-  redactJson,
-  redactSensitiveValue
-};
-//# sourceMappingURL=crawl-safety.js.map