@detiq/crawler 1.1.9 → 1.1.10

package/dist/pii-scrubber.js

@@ -1,78 +1,75 @@
-/**
- * PII scrubber — detects and redacts Personally Identifiable Information
- * from crawled markdown, HTML, and extracted text before storage.
- *
- * GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
- */
-// Patterns — ordered from most-specific to least-specific
 const PATTERNS = [
-    // Email — RFC 5321 simplified
-    { type: 'email', re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
-    // US SSN — 9 digits with dashes or spaces
-    { type: 'ssn', re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
-    // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
-    { type: 'credit-card', re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
-    // IBAN — international bank account (15-34 chars)
-    { type: 'iban', re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
-    // Phone — international or US formats
-    { type: 'phone', re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
-    // Date of birth — common formats
-    { type: 'dob', re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
-    // IPv4 private/internal addresses — only flag private-looking ones in content
-    { type: 'ip-address', re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g },
+  // Email — RFC 5321 simplified
+  { type: "email", re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
+  // US SSN — 9 digits with dashes or spaces
+  { type: "ssn", re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
+  // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
+  { type: "credit-card", re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
+  // IBAN — international bank account (15-34 chars)
+  { type: "iban", re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
+  // Phone — international or US formats
+  { type: "phone", re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
+  // Date of birth — common formats
+  { type: "dob", re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
+  // IPv4 private/internal addresses — only flag private-looking ones in content
+  { type: "ip-address", re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g }
 ];
 const REDACT_MAP = {
-    email: '[EMAIL]',
-    phone: '[PHONE]',
-    ssn: '[SSN]',
-    'credit-card': '[CARD]',
-    'ip-address': '[IP]',
-    passport: '[PASSPORT]',
-    iban: '[IBAN]',
-    dob: '[DOB]',
+  email: "[EMAIL]",
+  phone: "[PHONE]",
+  ssn: "[SSN]",
+  "credit-card": "[CARD]",
+  "ip-address": "[IP]",
+  passport: "[PASSPORT]",
+  iban: "[IBAN]",
+  dob: "[DOB]"
 };
-export function detectPii(text) {
-    const matches = [];
-    const seen = new Set();
-    for (const { type, re } of PATTERNS) {
-        const r = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');
-        let m;
-        while ((m = r.exec(text)) !== null) {
-            const key = `${type}:${m.index}`;
-            if (!seen.has(key)) {
-                seen.add(key);
-                matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
-            }
-        }
+function detectPii(text) {
+  const matches = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const { type, re } of PATTERNS) {
+    const r = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g");
+    let m;
+    while ((m = r.exec(text)) !== null) {
+      const key = `${type}:${m.index}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
+      }
     }
-    return matches.sort((a, b) => a.start - b.start);
+  }
+  return matches.sort((a, b) => a.start - b.start);
 }
-export function scrubPii(text) {
-    const matches = detectPii(text);
-    if (matches.length === 0)
-        return { text, matches: [], scrubbed: false };
-    // Replace from end to start so indices stay valid
-    let scrubbed = text;
-    for (let i = matches.length - 1; i >= 0; i--) {
-        const { type, start, end } = matches[i];
-        scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
-    }
-    return { text: scrubbed, matches, scrubbed: true };
+function scrubPii(text) {
+  const matches = detectPii(text);
+  if (matches.length === 0) return { text, matches: [], scrubbed: false };
+  let scrubbed = text;
+  for (let i = matches.length - 1; i >= 0; i--) {
+    const { type, start, end } = matches[i];
+    scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
+  }
+  return { text: scrubbed, matches, scrubbed: true };
 }
-export function hasPii(text) {
-    for (const { re } of PATTERNS) {
-        const r = new RegExp(re.source, re.flags);
-        if (r.test(text))
-            return true;
-    }
-    return false;
+function hasPii(text) {
+  for (const { re } of PATTERNS) {
+    const r = new RegExp(re.source, re.flags);
+    if (r.test(text)) return true;
+  }
+  return false;
 }
-export function scrubObject(obj, fields) {
-    const result = { ...obj };
-    for (const field of fields) {
-        if (typeof result[field] === 'string') {
-            result[field] = scrubPii(result[field]).text;
-        }
+function scrubObject(obj, fields) {
+  const result = { ...obj };
+  for (const field of fields) {
+    if (typeof result[field] === "string") {
+      result[field] = scrubPii(result[field]).text;
     }
-    return result;
+  }
+  return result;
 }
+export {
+  detectPii,
+  hasPii,
+  scrubObject,
+  scrubPii
+};
+//# sourceMappingURL=pii-scrubber.js.map

package/dist/playwright-recorder.js

@@ -1,18 +1,6 @@
-/**
- * Playwright test recorder — generate Playwright test code from crawl data.
- *
- * Given a list of crawled screens, generates a minimal Playwright test file that:
- * 1. Navigates to each screen URL
- * 2. Verifies the page title
- * 3. Checks key elements visible on the page
- * 4. Verifies no console errors (optional)
- *
- * This is a "smoke test" generator — not a full interaction recorder.
- * Output: valid TypeScript Playwright test file ready to run with npx playwright test.
- */
 function buildSmartLocatorHelper() {
-    return `
-// smartLocator — falls back to similarity search when primary selector fails.
+  return `
+// smartLocator \u2014 falls back to similarity search when primary selector fails.
 // Generated by ZeTa Crawler. Do not edit manually.
 async function smartLocator(page: any, primary: string, ref: { tag?: string; text?: string; role?: string; placeholder?: string }) {
   try {
@@ -39,57 +27,76 @@ async function smartLocator(page: any, primary: string, ref: { tag?: string; tex
 }`.trim();
 }
 function sanitizeTestName(name) {
-    return name.replace(/[^a-zA-Z0-9 _-]/g, '').trim().slice(0, 80) || 'Screen';
+  return name.replace(/[^a-zA-Z0-9 _-]/g, "").trim().slice(0, 80) || "Screen";
 }
 function escapeString(s) {
-    return s.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/`/g, '\\`');
+  return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/`/g, "\\`");
 }
-export function generatePlaywrightTest(screens, opts = {}) {
-    const { includeConsoleCheck = false, testTimeout = 30_000, projectName = 'ZeTa Crawl', useSmartLocators = true, } = opts;
-    const needsSmartLocator = useSmartLocators && screens.some(s => s.smartLocator != null);
-    const lines = [
-        `// Generated by ZeTa Crawler — ${new Date().toISOString()}`,
-        `// Project: ${escapeString(projectName)}`,
-        `// Screens: ${screens.length}`,
-        `// Run: npx playwright test`,
-        '',
-        "import { test, expect } from '@playwright/test';",
-        '',
-    ];
-    if (needsSmartLocator) {
-        lines.push(buildSmartLocatorHelper(), '');
+function generatePlaywrightTest(screens, opts = {}) {
+  const {
+    includeConsoleCheck = false,
+    testTimeout = 3e4,
+    projectName = "ZeTa Crawl",
+    useSmartLocators = true
+  } = opts;
+  const needsSmartLocator = useSmartLocators && screens.some((s) => s.smartLocator != null);
+  const lines = [
+    `// Generated by ZeTa Crawler \u2014 ${(/* @__PURE__ */ new Date()).toISOString()}`,
+    `// Project: ${escapeString(projectName)}`,
+    `// Screens: ${screens.length}`,
+    `// Run: npx playwright test`,
+    "",
+    "import { test, expect } from '@playwright/test';",
+    ""
+  ];
+  if (needsSmartLocator) {
+    lines.push(buildSmartLocatorHelper(), "");
+  }
+  lines.push(`test.setTimeout(${testTimeout});`, "");
+  if (includeConsoleCheck) {
+    lines.push(
+      `test.beforeEach(async ({ page }) => {`,
+      `  const errors: string[] = [];`,
+      `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`,
+      `  page.on('pageerror', err => errors.push(err.message));`,
+      `  (page as any).__consoleErrors = errors;`,
+      `});`,
+      ""
+    );
+  }
+  for (const screen of screens) {
+    const testName = sanitizeTestName(screen.name);
+    lines.push(
+      `test('${escapeString(testName)}', async ({ page }) => {`,
+      `  await page.goto('${escapeString(screen.url)}');`,
+      `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))}/);`
+    );
+    if (screen.title) {
+      lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))}/i);`);
     }
-    lines.push(`test.setTimeout(${testTimeout});`, '');
-    if (includeConsoleCheck) {
-        lines.push(`test.beforeEach(async ({ page }) => {`, `  const errors: string[] = [];`, `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`, `  page.on('pageerror', err => errors.push(err.message));`, `  (page as any).__consoleErrors = errors;`, `});`, '');
+    if (screen.heading) {
+      const h = escapeString(screen.heading.slice(0, 60));
+      if (needsSmartLocator && screen.smartLocator) {
+        const tag = escapeString(screen.smartLocator.tag);
+        const text = escapeString(screen.smartLocator.text ?? "");
+        lines.push(
+          `  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`,
+          `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`
+        );
+      } else {
+        lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);
+      }
     }
-    for (const screen of screens) {
-        const testName = sanitizeTestName(screen.name);
-        lines.push(`test('${escapeString(testName)}', async ({ page }) => {`, `  await page.goto('${escapeString(screen.url)}');`, `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))}/);`);
-        if (screen.title) {
-            lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))}/i);`);
-        }
-        if (screen.heading) {
-            const h = escapeString(screen.heading.slice(0, 60));
-            if (needsSmartLocator && screen.smartLocator) {
-                const tag = escapeString(screen.smartLocator.tag);
-                const text = escapeString(screen.smartLocator.text ?? '');
-                lines.push(`  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`, `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`);
-            }
-            else {
-                lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);
-            }
-        }
-        if (includeConsoleCheck) {
-            lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);
-        }
-        lines.push(`});`, '');
+    if (includeConsoleCheck) {
+      lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);
     }
-    return lines.join('\n');
+    lines.push(`});`, "");
+  }
+  return lines.join("\n");
 }
-export function generatePlaywrightConfig(opts = {}) {
-    const { baseUrl = 'http://localhost:3000', reportDir = 'playwright-report' } = opts;
-    return `// playwright.config.ts — generated by ZeTa Crawler
+function generatePlaywrightConfig(opts = {}) {
+  const { baseUrl = "http://localhost:3000", reportDir = "playwright-report" } = opts;
+  return `// playwright.config.ts \u2014 generated by ZeTa Crawler
 import { defineConfig } from '@playwright/test';
 
 export default defineConfig({
@@ -109,3 +116,8 @@ export default defineConfig({
 });
 `;
 }
+export {
+  generatePlaywrightConfig,
+  generatePlaywrightTest
+};
+//# sourceMappingURL=playwright-recorder.js.map

package/dist/policies/crawl-safety.js

@@ -1,40 +1,37 @@
 const DESTRUCTIVE_ACTION = /\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop)\b/i;
 const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
-export function assessActionSafety(label, profile) {
-    const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));
-    if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {
-        return {
-            allowed: false,
-            severity: 'block',
-            reason: `Blocked risky action "${label}" for ${profile.industry} profile.`,
-        };
-    }
-    return { allowed: true, severity: 'info' };
+function assessActionSafety(label, profile) {
+  const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));
+  if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {
+    return {
+      allowed: false,
+      severity: "block",
+      reason: `Blocked risky action "${label}" for ${profile.industry} profile.`
+    };
+  }
+  return { allowed: true, severity: "info" };
 }
-export function redactSensitiveValue(value) {
-    let redacted = value
-        .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]')
-        .replace(/\b(?:\+?\d[\d -]{8,}\d)\b/g, '[phone-or-id]')
-        .replace(/\b(?:\d[ -]*?){13,19}\b/g, '[card-like-number]')
-        .replace(/\b[A-Za-z0-9_-]{24,}\b/g, '[token-like-value]');
-    if (SECRET_LIKE.test(redacted))
-        redacted = redacted.replace(/:\s*[^,\s}]+/g, ': [redacted]');
-    return redacted;
+function redactSensitiveValue(value) {
+  let redacted = value.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[email]").replace(/\b(?:\+?\d[\d -]{8,}\d)\b/g, "[phone-or-id]").replace(/\b(?:\d[ -]*?){13,19}\b/g, "[card-like-number]").replace(/\b[A-Za-z0-9_-]{24,}\b/g, "[token-like-value]");
+  if (SECRET_LIKE.test(redacted)) redacted = redacted.replace(/:\s*[^,\s}]+/g, ": [redacted]");
+  return redacted;
 }
-export function redactJson(value) {
-    if (typeof value === 'string')
-        return redactSensitiveValue(value);
-    if (Array.isArray(value))
-        return value.map(redactJson);
-    if (value && typeof value === 'object') {
-        const out = {};
-        for (const [key, item] of Object.entries(value)) {
-            if (SECRET_LIKE.test(key))
-                out[key] = '[redacted]';
-            else
-                out[key] = redactJson(item);
-        }
-        return out;
+function redactJson(value) {
+  if (typeof value === "string") return redactSensitiveValue(value);
+  if (Array.isArray(value)) return value.map(redactJson);
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [key, item] of Object.entries(value)) {
+      if (SECRET_LIKE.test(key)) out[key] = "[redacted]";
+      else out[key] = redactJson(item);
     }
-    return value;
+    return out;
+  }
+  return value;
 }
+export {
+  assessActionSafety,
+  redactJson,
+  redactSensitiveValue
+};
+//# sourceMappingURL=crawl-safety.js.map