@detiq/crawler 1.1.9 → 1.1.11

package/dist/frontier.js

@@ -1,108 +1,83 @@
-/**
- * Configurable crawl traversal strategy — matches crawl4AI's BFS/DFS/
- * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded
- * FIFO queue, i.e. always BFS).
- *
- * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it
- * drops into crawler.ts's existing queue call sites with a rename, not a
- * rewrite — BFS's `shift()` behavior is byte-for-byte the same as the plain
- * array it replaces, so BFS (the default) has zero behavior change.
- */
-/**
- * Best-First scoring heuristic: shallower paths score higher (breadth-like
- * bias toward top-level sections before deep sub-pages), boosted further by
- * any user-supplied keyword appearing in the URL. Simple and transparent by
- * design — crawl4AI's own default scorer is a similarly simple keyword-
- * relevance heuristic, not a learned model.
- */
 function scoreUrl(url, keywords) {
-    try {
-        const u = new URL(url);
-        const segments = u.pathname.split('/').filter(Boolean);
-        let score = -segments.length;
-        if (keywords.length > 0) {
-            const lower = url.toLowerCase();
-            for (const kw of keywords) {
-                if (kw && lower.includes(kw))
-                    score += 10;
-            }
-        }
-        return score;
-    }
-    catch {
-        return 0;
+  try {
+    const u = new URL(url);
+    const segments = u.pathname.split("/").filter(Boolean);
+    let score = -segments.length;
+    if (keywords.length > 0) {
+      const lower = url.toLowerCase();
+      for (const kw of keywords) {
+        if (kw && lower.includes(kw)) score += 10;
+      }
     }
+    return score;
+  } catch {
+    return 0;
+  }
 }
-export class CrawlFrontier {
-    items = [];
-    strategy;
-    keywords;
-    urlFilter;
-    constructor(strategy, seedUrls, keywords = [], urlFilter) {
-        this.strategy = strategy;
-        this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
-        this.urlFilter = urlFilter
-            ? (() => { try {
-                return new RegExp(urlFilter, 'i');
-            }
-            catch {
-                return null;
-            } })()
-            : null;
-        for (const url of seedUrls)
-            this.push(url);
-    }
-    get length() {
-        return this.items.length;
-    }
-    /**
-     * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
-     * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
-     * the frontier revisits sparse branches before deep-linked content.
-     */
-    push(url, pushOpts) {
-        if (this.urlFilter && !this.urlFilter.test(url))
-            return;
-        const sparseBoost = this.strategy === 'BEST_FIRST' &&
-            pushOpts?.parentElements !== undefined &&
-            pushOpts.parentElements < 5
-            ? 1
-            : 0;
-        this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
-    }
-    /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
-    shift() {
-        if (this.items.length === 0)
-            return undefined;
-        switch (this.strategy) {
-            case 'DFS':
-                return this.items.pop().url;
-            case 'BEST_FIRST': {
-                let bestIdx = 0;
-                for (let i = 1; i < this.items.length; i++) {
-                    if (this.items[i].score > this.items[bestIdx].score)
-                        bestIdx = i;
-                }
-                return this.items.splice(bestIdx, 1)[0].url;
-            }
-            case 'BFS':
-            default:
-                return this.items.shift().url;
+class CrawlFrontier {
+  items = [];
+  strategy;
+  keywords;
+  urlFilter;
+  constructor(strategy, seedUrls, keywords = [], urlFilter) {
+    this.strategy = strategy;
+    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+    this.urlFilter = urlFilter ? (() => {
+      try {
+        return new RegExp(urlFilter, "i");
+      } catch {
+        return null;
+      }
+    })() : null;
+    for (const url of seedUrls) this.push(url);
+  }
+  get length() {
+    return this.items.length;
+  }
+  /**
+   * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+   * the frontier revisits sparse branches before deep-linked content.
+   */
+  push(url, pushOpts) {
+    if (this.urlFilter && !this.urlFilter.test(url)) return;
+    const sparseBoost = this.strategy === "BEST_FIRST" && pushOpts?.parentElements !== void 0 && pushOpts.parentElements < 5 ? 1 : 0;
+    this.items.push({ url, score: this.strategy === "BEST_FIRST" ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
+  }
+  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+  shift() {
+    if (this.items.length === 0) return void 0;
+    switch (this.strategy) {
+      case "DFS":
+        return this.items.pop().url;
+      case "BEST_FIRST": {
+        let bestIdx = 0;
+        for (let i = 1; i < this.items.length; i++) {
+          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;
         }
+        return this.items.splice(bestIdx, 1)[0].url;
+      }
+      case "BFS":
+      default:
+        return this.items.shift().url;
     }
-    filter(predicate) {
-        return this.items.filter((e) => predicate(e.url)).map((e) => e.url);
-    }
-    /** Remove queued URLs not matching the regex pattern. Returns count removed. */
-    filterByPattern(pattern) {
-        try {
-            const re = new RegExp(pattern, 'i');
-            const before = this.items.length;
-            this.items = this.items.filter((e) => re.test(e.url));
-            return before - this.items.length;
-        }
-        catch {
-            return 0;
-        }
+  }
+  filter(predicate) {
+    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);
+  }
+  /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+  filterByPattern(pattern) {
+    try {
+      const re = new RegExp(pattern, "i");
+      const before = this.items.length;
+      this.items = this.items.filter((e) => re.test(e.url));
+      return before - this.items.length;
+    } catch {
+      return 0;
     }
+  }
 }
+export {
+  CrawlFrontier
+};
+//# sourceMappingURL=frontier.js.map

package/dist/gap-pass.js

@@ -1,44 +1,23 @@
-/**
- * P6: Gap Pass — LLM-guided second crawl pass
- *
- * After the primary BFS crawl completes, this module:
- * 1. Sends the discovered screen list to an LLM for gap analysis
- * 2. HEAD-validates each suggested URL
- * 3. Returns confirmed gap URLs for targeted re-crawl
- *
- * The caller (crawler.ts) is responsible for actually visiting the confirmed
- * URLs with the existing browser — this module only identifies them.
- */
-import { logger as rootLogger } from './logger.js';
-const logger = rootLogger.child({ module: 'gap-pass' });
-/**
- * Run LLM gap analysis + HEAD validation.
- * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.
- */
-export async function runGapAnalysis(screens, appUrl, llmCfg) {
-    if (!llmCfg?.apiKey || screens.length < 2)
-        return [];
-    const origin = (() => { try {
-        return new URL(appUrl).origin;
+import { logger as rootLogger } from "./logger.js";
+const logger = rootLogger.child({ module: "gap-pass" });
+async function runGapAnalysis(screens, appUrl, llmCfg) {
+  if (!llmCfg?.apiKey || screens.length < 2) return [];
+  const origin = (() => {
+    try {
+      return new URL(appUrl).origin;
+    } catch {
+      return "";
     }
-    catch {
-        return '';
-    } })();
-    if (!origin)
-        return [];
-    // Cap context — only send url + name, not full element data
-    const screenList = screens
-        .slice(0, 80)
-        .map(s => {
-        try {
-            return new URL(s.url).pathname + (s.name ? ` (${s.name})` : '');
-        }
-        catch {
-            return s.url;
-        }
-    })
-        .join('\n');
-    const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
+  })();
+  if (!origin) return [];
+  const screenList = screens.slice(0, 80).map((s) => {
+    try {
+      return new URL(s.url).pathname + (s.name ? ` (${s.name})` : "");
+    } catch {
+      return s.url;
+    }
+  }).join("\n");
+  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
 
 Discovered screens (${screens.length} total):
 ${screenList}
@@ -49,55 +28,55 @@ Based on these screens and common patterns for this type of web application, ide
 
 Respond with ONLY valid JSON, no markdown:
 {"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`;
-    try {
-        const Anthropic = (await import('@anthropic-ai/sdk')).default;
-        const client = new Anthropic({ apiKey: llmCfg.apiKey });
-        const response = await client.messages.create({
-            model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
-            max_tokens: 1024,
-            temperature: 0,
-            // @ts-ignore
-            system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
-            messages: [{ role: 'user', content: prompt }],
-        });
-        const text = response.content[0]?.text ?? '';
-        const jsonMatch = text.match(/\{[\s\S]*\}/);
-        if (!jsonMatch)
-            return [];
-        const parsed = JSON.parse(jsonMatch[0]);
-        const rawGaps = parsed.gaps ?? [];
-        if (!Array.isArray(rawGaps) || rawGaps.length === 0)
-            return [];
-        logger.info({ gaps: rawGaps.length }, '[P6] Gap analysis: LLM returned candidates');
-        // HEAD-validate each URL pattern, take the first 200
-        const results = [];
-        for (const gap of rawGaps.slice(0, 15)) {
-            let confirmedUrl = null;
-            let headStatus = null;
-            for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {
-                const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`;
-                try {
-                    const controller = new AbortController();
-                    const timeout = setTimeout(() => controller.abort(), 6_000);
-                    const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);
-                    clearTimeout(timeout);
-                    headStatus = res?.status ?? null;
-                    if (res?.ok) {
-                        confirmedUrl = fullUrl;
-                        break;
-                    }
-                }
-                catch { /* non-fatal */ }
-            }
-            results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });
-            logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked');
+  try {
+    const Anthropic = (await import("@anthropic-ai/sdk")).default;
+    const client = new Anthropic({ apiKey: llmCfg.apiKey });
+    const response = await client.messages.create({
+      model: llmCfg.model ?? "claude-haiku-4-5-20251001",
+      max_tokens: 1024,
+      temperature: 0,
+      // @ts-ignore
+      system: [{ type: "text", text: "You are a web application coverage analyst. Respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
+      messages: [{ role: "user", content: prompt }]
+    });
+    const text = response.content[0]?.text ?? "";
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return [];
+    const parsed = JSON.parse(jsonMatch[0]);
+    const rawGaps = parsed.gaps ?? [];
+    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];
+    logger.info({ gaps: rawGaps.length }, "[P6] Gap analysis: LLM returned candidates");
+    const results = [];
+    for (const gap of rawGaps.slice(0, 15)) {
+      let confirmedUrl = null;
+      let headStatus = null;
+      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {
+        const fullUrl = pattern.startsWith("http") ? pattern : `${origin}${pattern.startsWith("/") ? "" : "/"}${pattern}`;
+        try {
+          const controller = new AbortController();
+          const timeout = setTimeout(() => controller.abort(), 6e3);
+          const res = await fetch(fullUrl, { method: "HEAD", signal: controller.signal, redirect: "follow" }).catch(() => null);
+          clearTimeout(timeout);
+          headStatus = res?.status ?? null;
+          if (res?.ok) {
+            confirmedUrl = fullUrl;
+            break;
+          }
+        } catch {
         }
-        const confirmed = results.filter(r => r.confirmedUrl).length;
-        logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete');
-        return results;
-    }
-    catch (err) {
-        logger.warn({ err: String(err) }, '[P6] Gap analysis failed — non-fatal');
-        return [];
+      }
+      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });
+      logger.info({ category: gap.category, confirmedUrl, headStatus }, "[P6] Gap candidate checked");
     }
+    const confirmed = results.filter((r) => r.confirmedUrl).length;
+    logger.info({ total: results.length, confirmed }, "[P6] Gap pass analysis complete");
+    return results;
+  } catch (err) {
+    logger.warn({ err: String(err) }, "[P6] Gap analysis failed \u2014 non-fatal");
+    return [];
+  }
 }
+export {
+  runGapAnalysis
+};
+//# sourceMappingURL=gap-pass.js.map

package/dist/graphql-detector.js

@@ -1,10 +1,3 @@
-/**
- * GraphQL detection + introspection.
- *
- * During crawl, if a GraphQL endpoint is discovered (via network observer or
- * known-URL heuristics), fire a minimal introspection query to get the schema.
- * The schema is stored in the crawl artifact for use during test generation.
- */
 const INTROSPECTION_QUERY = `
   query IntrospectionQuery {
     __schema {
@@ -22,75 +15,72 @@ const INTROSPECTION_QUERY = `
   }
 `;
 function flattenType(t) {
-    if (!t)
-        return 'unknown';
-    if (t.kind === 'NON_NULL' || t.kind === 'LIST')
-        return flattenType(t.ofType);
-    return t.name ?? 'unknown';
+  if (!t) return "unknown";
+  if (t.kind === "NON_NULL" || t.kind === "LIST") return flattenType(t.ofType);
+  return t.name ?? "unknown";
 }
 function parseSchema(raw) {
-    const s = raw?.__schema ?? raw?.data?.__schema ?? raw;
-    const types = (s?.types ?? [])
-        .filter((t) => !t.name?.startsWith('__'))
-        .map((t) => ({
-        name: t.name,
-        kind: t.kind,
-        fields: (t.fields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
-        inputFields: (t.inputFields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
-        enumValues: (t.enumValues ?? []).map((e) => e.name),
-    }));
-    return {
-        queryType: s?.queryType?.name,
-        mutationType: s?.mutationType?.name,
-        subscriptionType: s?.subscriptionType?.name,
-        types,
-    };
+  const s = raw?.__schema ?? raw?.data?.__schema ?? raw;
+  const types = (s?.types ?? []).filter((t) => !t.name?.startsWith("__")).map((t) => ({
+    name: t.name,
+    kind: t.kind,
+    fields: (t.fields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
+    inputFields: (t.inputFields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
+    enumValues: (t.enumValues ?? []).map((e) => e.name)
+  }));
+  return {
+    queryType: s?.queryType?.name,
+    mutationType: s?.mutationType?.name,
+    subscriptionType: s?.subscriptionType?.name,
+    types
+  };
 }
-export async function introspectGraphQL(endpoint, opts = {}) {
-    const { headers = {}, timeoutMs = 10_000, cookies } = opts;
-    const t0 = Date.now();
-    try {
-        const reqHeaders = {
-            'Content-Type': 'application/json',
-            'Accept': 'application/json',
-            ...headers,
-        };
-        if (cookies)
-            reqHeaders['Cookie'] = cookies;
-        const res = await fetch(endpoint, {
-            method: 'POST',
-            headers: reqHeaders,
-            body: JSON.stringify({ query: INTROSPECTION_QUERY }),
-            signal: AbortSignal.timeout(timeoutMs),
-        });
-        if (!res.ok) {
-            return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };
-        }
-        const json = await res.json();
-        if (json?.errors?.length) {
-            const msg = json.errors[0]?.message ?? 'introspection disabled';
-            return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };
-        }
-        return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };
+async function introspectGraphQL(endpoint, opts = {}) {
+  const { headers = {}, timeoutMs = 1e4, cookies } = opts;
+  const t0 = Date.now();
+  try {
+    const reqHeaders = {
+      "Content-Type": "application/json",
+      "Accept": "application/json",
+      ...headers
+    };
+    if (cookies) reqHeaders["Cookie"] = cookies;
+    const res = await fetch(endpoint, {
+      method: "POST",
+      headers: reqHeaders,
+      body: JSON.stringify({ query: INTROSPECTION_QUERY }),
+      signal: AbortSignal.timeout(timeoutMs)
+    });
+    if (!res.ok) {
+      return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };
     }
-    catch (e) {
-        return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };
+    const json = await res.json();
+    if (json?.errors?.length) {
+      const msg = json.errors[0]?.message ?? "introspection disabled";
+      return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };
     }
+    return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };
+  } catch (e) {
+    return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };
+  }
 }
-/** Guess GraphQL endpoint URLs from a known app URL. */
-export function guessGraphQLEndpoints(appUrl) {
-    try {
-        const u = new URL(appUrl);
-        const base = `${u.protocol}//${u.host}`;
-        return [
-            `${base}/graphql`,
-            `${base}/api/graphql`,
-            `${base}/v1/graphql`,
-            `${base}/gql`,
-            `${base}/query`,
-        ];
-    }
-    catch {
-        return [];
-    }
+function guessGraphQLEndpoints(appUrl) {
+  try {
+    const u = new URL(appUrl);
+    const base = `${u.protocol}//${u.host}`;
+    return [
+      `${base}/graphql`,
+      `${base}/api/graphql`,
+      `${base}/v1/graphql`,
+      `${base}/gql`,
+      `${base}/query`
+    ];
+  } catch {
+    return [];
+  }
 }
+export {
+  guessGraphQLEndpoints,
+  introspectGraphQL
+};
+//# sourceMappingURL=graphql-detector.js.map