@detiq/crawler 1.1.7 → 1.1.9

package/dist/failure-analysis-agent.js

@@ -1,41 +1,37 @@
-import { readFileSync } from "node:fs";
-import { askTrackedWithImages, askTracked, deterministicSeed } from "@detiq/agents";
-import { AppBrain } from "@detiq/app-brain";
-async function analyzeFailure(ctx) {
-  try {
-    let llmCfg;
-    if (ctx.tenantId) {
-      llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, "testRunner").catch(() => void 0) ?? void 0;
-    }
-    if (!llmCfg) return null;
-    const consoleSection = ctx.consoleLogs?.length ? `
-
-Browser console (last ${ctx.consoleLogs.length} entries):
-${ctx.consoleLogs.map((l) => `[${l.type}] ${l.text}`).join("\n")}` : "";
-    const networkSection = ctx.networkErrors?.length ? `
+import { readFileSync } from 'node:fs';
+import { askTrackedWithImages, askTracked, deterministicSeed } from '@detiq/agents';
+import { AppBrain } from '@detiq/app-brain';
+export async function analyzeFailure(ctx) {
+    try {
+        let llmCfg;
+        if (ctx.tenantId) {
+            llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, 'testRunner').catch(() => undefined) ?? undefined;
+        }
+        if (!llmCfg)
+            return null;
+        const consoleSection = ctx.consoleLogs?.length
+            ? `\n\nBrowser console (last ${ctx.consoleLogs.length} entries):\n${ctx.consoleLogs.map(l => `[${l.type}] ${l.text}`).join('\n')}`
+            : '';
+        const networkSection = ctx.networkErrors?.length
+            ? `\n\nFailed network requests:\n${ctx.networkErrors.map(n => `${n.method} ${n.url} → ${n.status}`).join('\n')}`
+            : '';
+        const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific — name the failing element, API endpoint, or error type. Do not suggest fixes.
 
-Failed network requests:
-${ctx.networkErrors.map((n) => `${n.method} ${n.url} \u2192 ${n.status}`).join("\n")}` : "";
-    const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific \u2014 name the failing element, API endpoint, or error type. Do not suggest fixes.
-
-Test step: ${ctx.stepDescription ?? "unknown"}
+Test step: ${ctx.stepDescription ?? 'unknown'}
 Error: ${ctx.error}${consoleSection}${networkSection}`;
-    if (ctx.screenshotPath) {
-      try {
-        const img = readFileSync(ctx.screenshotPath);
-        const result2 = await askTrackedWithImages("", userText, [{ base64: img.toString("base64"), mediaType: "image/png" }], llmCfg);
-        return result2.text || null;
-      } catch {
-      }
+        if (ctx.screenshotPath) {
+            try {
+                const img = readFileSync(ctx.screenshotPath);
+                const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg);
+                return result.text || null;
+            }
+            catch { /* fall through to text-only */ }
+        }
+        const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+        const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });
+        return result.text || null;
+    }
+    catch {
+        return null;
     }
-    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ""}:failure:${(ctx.stepDescription ?? "").slice(0, 100)}:${ctx.error.slice(0, 200)}`);
-    const result = await askTracked("", userText, llmCfg, 256, void 0, { temperature: 0, seed: failureSeed });
-    return result.text || null;
-  } catch {
-    return null;
-  }
 }
-export {
-  analyzeFailure
-};
-//# sourceMappingURL=failure-analysis-agent.js.map

package/dist/feature-flag-detector.js

@@ -1,55 +1,70 @@
+/**
+ * Feature flag + A/B test detector.
+ *
+ * Detects popular feature flag / experiment SDKs loaded on the page:
+ * LaunchDarkly, Optimizely, Split.io, Unleash, GrowthBook, Statsig, Flagsmith, Amplitude Experiment.
+ *
+ * Also detects common A/B test patterns: data-testid attributes, gtm.js experiment layers,
+ * Optimizely variation IDs, VWO, etc.
+ *
+ * Why: ZeTa needs to know when a page shows different content to different users
+ * so test generation can account for variants and not generate flaky tests.
+ */
+// SDK detectors — checked against window object + script URLs
 const FLAG_SDK_CHECKS = [
-  { sdk: "LaunchDarkly", windowKeys: ["LDClient", "LaunchDarkly", "ldClient"], scriptPattern: /launchdarkly/i },
-  { sdk: "Optimizely", windowKeys: ["optimizely", "optimizelySdk"], scriptPattern: /optimizely/i },
-  { sdk: "Split.io", windowKeys: ["splitio", "SplitFactory"], scriptPattern: /split\.io/i },
-  { sdk: "GrowthBook", windowKeys: ["growthbook", "GrowthBook"], scriptPattern: /growthbook/i },
-  { sdk: "Statsig", windowKeys: ["statsig", "StatsigClient"], scriptPattern: /statsig/i },
-  { sdk: "Flagsmith", windowKeys: ["flagsmith"], scriptPattern: /flagsmith/i },
-  { sdk: "Unleash", windowKeys: ["unleash", "UnleashClient"], scriptPattern: /unleash/i },
-  { sdk: "VWO", windowKeys: ["vwo", "_vwo"], scriptPattern: /vwo\.com/i },
-  { sdk: "Google Optimize", windowKeys: ["gaData", "google_optimize"], scriptPattern: /googleoptimize/i },
-  { sdk: "AB Tasty", windowKeys: ["ABTasty"], scriptPattern: /abtasty/i }
+    { sdk: 'LaunchDarkly', windowKeys: ['LDClient', 'LaunchDarkly', 'ldClient'], scriptPattern: /launchdarkly/i },
+    { sdk: 'Optimizely', windowKeys: ['optimizely', 'optimizelySdk'], scriptPattern: /optimizely/i },
+    { sdk: 'Split.io', windowKeys: ['splitio', 'SplitFactory'], scriptPattern: /split\.io/i },
+    { sdk: 'GrowthBook', windowKeys: ['growthbook', 'GrowthBook'], scriptPattern: /growthbook/i },
+    { sdk: 'Statsig', windowKeys: ['statsig', 'StatsigClient'], scriptPattern: /statsig/i },
+    { sdk: 'Flagsmith', windowKeys: ['flagsmith'], scriptPattern: /flagsmith/i },
+    { sdk: 'Unleash', windowKeys: ['unleash', 'UnleashClient'], scriptPattern: /unleash/i },
+    { sdk: 'VWO', windowKeys: ['vwo', '_vwo'], scriptPattern: /vwo\.com/i },
+    { sdk: 'Google Optimize', windowKeys: ['gaData', 'google_optimize'], scriptPattern: /googleoptimize/i },
+    { sdk: 'AB Tasty', windowKeys: ['ABTasty'], scriptPattern: /abtasty/i },
 ];
-async function detectFeatureFlags(page, url) {
-  const detectedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const result = await page.evaluate((checks) => {
-    const flagSdks = [];
-    const scripts = Array.from(document.querySelectorAll("script[src]")).map((s) => s.src ?? "");
-    for (const check of checks) {
-      let found = false;
-      let confidence = "low";
-      if (check.windowKeys?.some((k) => k in window)) {
-        found = true;
-        confidence = "high";
-      }
-      if (!found && check.scriptPattern) {
-        const re = new RegExp(check.scriptPattern.source, "i");
-        const matchingSrc = scripts.find((s) => re.test(s));
-        if (matchingSrc) {
-          found = true;
-          confidence = "medium";
+export async function detectFeatureFlags(page, url) {
+    const detectedAt = new Date().toISOString();
+    const result = await page.evaluate((checks) => {
+        const flagSdks = [];
+        const scripts = Array.from(document.querySelectorAll('script[src]')).map((s) => s.src ?? '');
+        for (const check of checks) {
+            let found = false;
+            let confidence = 'low';
+            // Check window keys
+            if (check.windowKeys?.some((k) => k in window)) {
+                found = true;
+                confidence = 'high';
+            }
+            // Check script URLs
+            if (!found && check.scriptPattern) {
+                const re = new RegExp(check.scriptPattern.source, 'i');
+                const matchingSrc = scripts.find(s => re.test(s));
+                if (matchingSrc) {
+                    found = true;
+                    confidence = 'medium';
+                }
+            }
+            if (found)
+                flagSdks.push({ sdk: check.sdk, confidence, details: undefined });
         }
-      }
-      if (found) flagSdks.push({ sdk: check.sdk, confidence, details: void 0 });
-    }
-    const abTools = [];
-    if ("_vwo_code" in window || "_vis_opt_queue" in window) abTools.push({ tool: "VWO", experiments: [] });
-    if ("dataLayer" in window) {
-      const dl = window.dataLayer ?? [];
-      const expEvents = dl.filter((e) => e?.event?.includes("experiment") || e?.experimentId);
-      if (expEvents.length > 0) abTools.push({ tool: "Google Tag Manager Experiments", experiments: expEvents.slice(0, 5).map((e) => e.experimentId ?? "") });
-    }
-    return { flagSdks, abTools };
-  }, FLAG_SDK_CHECKS).catch(() => ({ flagSdks: [], abTools: [] }));
-  return {
-    url,
-    flagSdks: result.flagSdks,
-    abTools: result.abTools,
-    hasAnyFlags: result.flagSdks.length > 0 || result.abTools.length > 0,
-    detectedAt
-  };
+        // A/B tools
+        const abTools = [];
+        if ('_vwo_code' in window || '_vis_opt_queue' in window)
+            abTools.push({ tool: 'VWO', experiments: [] });
+        if ('dataLayer' in window) {
+            const dl = window.dataLayer ?? [];
+            const expEvents = dl.filter((e) => e?.event?.includes('experiment') || e?.experimentId);
+            if (expEvents.length > 0)
+                abTools.push({ tool: 'Google Tag Manager Experiments', experiments: expEvents.slice(0, 5).map((e) => e.experimentId ?? '') });
+        }
+        return { flagSdks, abTools };
+    }, FLAG_SDK_CHECKS).catch(() => ({ flagSdks: [], abTools: [] }));
+    return {
+        url,
+        flagSdks: result.flagSdks,
+        abTools: result.abTools,
+        hasAnyFlags: result.flagSdks.length > 0 || result.abTools.length > 0,
+        detectedAt,
+    };
 }
-export {
-  detectFeatureFlags
-};
-//# sourceMappingURL=feature-flag-detector.js.map

package/dist/form-detector.js

@@ -1,58 +1,78 @@
+/**
+ * Form detector — extract form metadata from crawled pages.
+ *
+ * For each form found on a page, captures: action URL, method, all input fields
+ * with their types/names/required/validation constraints, and submit button text.
+ *
+ * This data feeds ZeTa's test generator to create form validation test cases
+ * automatically (happy path + boundary/error cases per field type).
+ */
 function guessPurpose(fields, action) {
-  const names = fields.map((f) => f.name?.toLowerCase() + (f.label ?? "").toLowerCase()).join(" ");
-  if (/password/.test(names) && /email|username|user/.test(names) && fields.length <= 4) return "login";
-  if (/password/.test(names) && /confirm|repeat/.test(names)) return "registration";
-  if (/search|query|q\b/.test(names) && fields.length <= 3) return "search";
-  if (/card|payment|cvv|expir/.test(names)) return "checkout";
-  if (/message|subject|contact/.test(names)) return "contact";
-  if (/setting|preference|profile/.test(names)) return "settings";
-  return "unknown";
+    const names = fields.map(f => f.name?.toLowerCase() + (f.label ?? '').toLowerCase()).join(' ');
+    if (/password/.test(names) && /email|username|user/.test(names) && fields.length <= 4)
+        return 'login';
+    if (/password/.test(names) && /confirm|repeat/.test(names))
+        return 'registration';
+    if (/search|query|q\b/.test(names) && fields.length <= 3)
+        return 'search';
+    if (/card|payment|cvv|expir/.test(names))
+        return 'checkout';
+    if (/message|subject|contact/.test(names))
+        return 'contact';
+    if (/setting|preference|profile/.test(names))
+        return 'settings';
+    return 'unknown';
 }
-async function detectForms(page) {
-  try {
-    return await page.$$eval("form", (forms) => forms.map((form, index) => {
-      const inputs = Array.from(form.querySelectorAll('input,select,textarea,button[type="submit"]'));
-      const fields = [];
-      for (const input of inputs) {
-        const tag = input.tagName.toLowerCase();
-        const type = tag === "button" ? "submit" : input.type || tag;
-        if (["submit", "reset", "button", "hidden"].includes(type)) continue;
-        let label = "";
-        if (input.id) {
-          const lbl = form.querySelector(`label[for="${input.id}"]`);
-          if (lbl) label = (lbl.textContent ?? "").trim();
-        }
-        if (!label) {
-          const parent = input.closest("label");
-          if (parent) label = (parent.textContent ?? "").replace(input.value ?? "", "").trim().slice(0, 80);
-        }
-        const options = tag === "select" ? Array.from(input.options ?? []).map((o) => o.text?.trim()).filter(Boolean).slice(0, 20) : tag === "input" && type === "radio" ? [input.value] : void 0;
-        fields.push({
-          name: input.name || input.id || "",
-          type,
-          label: label || void 0,
-          required: !!input.required,
-          placeholder: input.placeholder || void 0,
-          minLength: input.minLength > 0 ? input.minLength : void 0,
-          maxLength: input.maxLength > 0 && input.maxLength < 524288 ? input.maxLength : void 0,
-          min: input.min || void 0,
-          max: input.max || void 0,
-          pattern: input.pattern || void 0,
-          options,
-          autocomplete: input.autocomplete || void 0
-        });
-      }
-      const submitBtn = form.querySelector('button[type="submit"],input[type="submit"]');
-      const hasFileUpload = !!form.querySelector('input[type="file"]');
-      const action = form.action || void 0;
-      const method = (form.method || "get").toUpperCase();
-      return { index, action, method, fields, submitText: submitBtn?.textContent?.trim() || submitBtn?.value || void 0, hasFileUpload };
-    })).then((forms) => forms.map((f) => ({ ...f, purpose: guessPurpose(f.fields, f.action) })));
-  } catch {
-    return [];
-  }
+export async function detectForms(page) {
+    try {
+        return await page.$$eval('form', (forms) => forms.map((form, index) => {
+            const inputs = Array.from(form.querySelectorAll('input,select,textarea,button[type="submit"]'));
+            const fields = [];
+            for (const input of inputs) {
+                const tag = input.tagName.toLowerCase();
+                const type = tag === 'button' ? 'submit' : (input.type || tag);
+                if (['submit', 'reset', 'button', 'hidden'].includes(type))
+                    continue;
+                // Find associated label
+                let label = '';
+                if (input.id) {
+                    const lbl = form.querySelector(`label[for="${input.id}"]`);
+                    if (lbl)
+                        label = (lbl.textContent ?? '').trim();
+                }
+                if (!label) {
+                    const parent = input.closest('label');
+                    if (parent)
+                        label = (parent.textContent ?? '').replace(input.value ?? '', '').trim().slice(0, 80);
+                }
+                const options = tag === 'select'
+                    ? Array.from(input.options ?? []).map((o) => o.text?.trim()).filter(Boolean).slice(0, 20)
+                    : tag === 'input' && type === 'radio'
+                        ? [input.value]
+                        : undefined;
+                fields.push({
+                    name: input.name || input.id || '',
+                    type,
+                    label: label || undefined,
+                    required: !!input.required,
+                    placeholder: input.placeholder || undefined,
+                    minLength: input.minLength > 0 ? input.minLength : undefined,
+                    maxLength: input.maxLength > 0 && input.maxLength < 524288 ? input.maxLength : undefined,
+                    min: input.min || undefined,
+                    max: input.max || undefined,
+                    pattern: input.pattern || undefined,
+                    options,
+                    autocomplete: input.autocomplete || undefined,
+                });
+            }
+            const submitBtn = form.querySelector('button[type="submit"],input[type="submit"]');
+            const hasFileUpload = !!form.querySelector('input[type="file"]');
+            const action = form.action || undefined;
+            const method = (form.method || 'get').toUpperCase();
+            return { index, action, method, fields, submitText: submitBtn?.textContent?.trim() || submitBtn?.value || undefined, hasFileUpload };
+        })).then(forms => forms.map((f) => ({ ...f, purpose: guessPurpose(f.fields, f.action) })));
+    }
+    catch {
+        return [];
+    }
 }
-export {
-  detectForms
-};
-//# sourceMappingURL=form-detector.js.map

package/dist/frontier.js

@@ -1,83 +1,108 @@
+/**
+ * Configurable crawl traversal strategy — matches crawl4AI's BFS/DFS/
+ * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded
+ * FIFO queue, i.e. always BFS).
+ *
+ * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it
+ * drops into crawler.ts's existing queue call sites with a rename, not a
+ * rewrite — BFS's `shift()` behavior is byte-for-byte the same as the plain
+ * array it replaces, so BFS (the default) has zero behavior change.
+ */
+/**
+ * Best-First scoring heuristic: shallower paths score higher (breadth-like
+ * bias toward top-level sections before deep sub-pages), boosted further by
+ * any user-supplied keyword appearing in the URL. Simple and transparent by
+ * design — crawl4AI's own default scorer is a similarly simple keyword-
+ * relevance heuristic, not a learned model.
+ */
 function scoreUrl(url, keywords) {
-  try {
-    const u = new URL(url);
-    const segments = u.pathname.split("/").filter(Boolean);
-    let score = -segments.length;
-    if (keywords.length > 0) {
-      const lower = url.toLowerCase();
-      for (const kw of keywords) {
-        if (kw && lower.includes(kw)) score += 10;
-      }
+    try {
+        const u = new URL(url);
+        const segments = u.pathname.split('/').filter(Boolean);
+        let score = -segments.length;
+        if (keywords.length > 0) {
+            const lower = url.toLowerCase();
+            for (const kw of keywords) {
+                if (kw && lower.includes(kw))
+                    score += 10;
+            }
+        }
+        return score;
+    }
+    catch {
+        return 0;
     }
-    return score;
-  } catch {
-    return 0;
-  }
 }
-class CrawlFrontier {
-  items = [];
-  strategy;
-  keywords;
-  urlFilter;
-  constructor(strategy, seedUrls, keywords = [], urlFilter) {
-    this.strategy = strategy;
-    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
-    this.urlFilter = urlFilter ? (() => {
-      try {
-        return new RegExp(urlFilter, "i");
-      } catch {
-        return null;
-      }
-    })() : null;
-    for (const url of seedUrls) this.push(url);
-  }
-  get length() {
-    return this.items.length;
-  }
-  /**
-   * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
-   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
-   * the frontier revisits sparse branches before deep-linked content.
-   */
-  push(url, pushOpts) {
-    if (this.urlFilter && !this.urlFilter.test(url)) return;
-    const sparseBoost = this.strategy === "BEST_FIRST" && pushOpts?.parentElements !== void 0 && pushOpts.parentElements < 5 ? 1 : 0;
-    this.items.push({ url, score: this.strategy === "BEST_FIRST" ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
-  }
-  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
-  shift() {
-    if (this.items.length === 0) return void 0;
-    switch (this.strategy) {
-      case "DFS":
-        return this.items.pop().url;
-      case "BEST_FIRST": {
-        let bestIdx = 0;
-        for (let i = 1; i < this.items.length; i++) {
-          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;
+export class CrawlFrontier {
+    items = [];
+    strategy;
+    keywords;
+    urlFilter;
+    constructor(strategy, seedUrls, keywords = [], urlFilter) {
+        this.strategy = strategy;
+        this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+        this.urlFilter = urlFilter
+            ? (() => { try {
+                return new RegExp(urlFilter, 'i');
+            }
+            catch {
+                return null;
+            } })()
+            : null;
+        for (const url of seedUrls)
+            this.push(url);
+    }
+    get length() {
+        return this.items.length;
+    }
+    /**
+     * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+     * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+     * the frontier revisits sparse branches before deep-linked content.
+     */
+    push(url, pushOpts) {
+        if (this.urlFilter && !this.urlFilter.test(url))
+            return;
+        const sparseBoost = this.strategy === 'BEST_FIRST' &&
+            pushOpts?.parentElements !== undefined &&
+            pushOpts.parentElements < 5
+            ? 1
+            : 0;
+        this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
+    }
+    /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+    shift() {
+        if (this.items.length === 0)
+            return undefined;
+        switch (this.strategy) {
+            case 'DFS':
+                return this.items.pop().url;
+            case 'BEST_FIRST': {
+                let bestIdx = 0;
+                for (let i = 1; i < this.items.length; i++) {
+                    if (this.items[i].score > this.items[bestIdx].score)
+                        bestIdx = i;
+                }
+                return this.items.splice(bestIdx, 1)[0].url;
+            }
+            case 'BFS':
+            default:
+                return this.items.shift().url;
         }
-        return this.items.splice(bestIdx, 1)[0].url;
-      }
-      case "BFS":
-      default:
-        return this.items.shift().url;
     }
-  }
-  filter(predicate) {
-    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);
-  }
-  /** Remove queued URLs not matching the regex pattern. Returns count removed. */
-  filterByPattern(pattern) {
-    try {
-      const re = new RegExp(pattern, "i");
-      const before = this.items.length;
-      this.items = this.items.filter((e) => re.test(e.url));
-      return before - this.items.length;
-    } catch {
-      return 0;
+    filter(predicate) {
+        return this.items.filter((e) => predicate(e.url)).map((e) => e.url);
+    }
+    /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+    filterByPattern(pattern) {
+        try {
+            const re = new RegExp(pattern, 'i');
+            const before = this.items.length;
+            this.items = this.items.filter((e) => re.test(e.url));
+            return before - this.items.length;
+        }
+        catch {
+            return 0;
+        }
     }
-  }
 }
-export {
-  CrawlFrontier
-};
-//# sourceMappingURL=frontier.js.map