@detiq/crawler 1.1.7 → 1.1.9

package/dist/gap-pass.js

@@ -1,23 +1,44 @@
-import { logger as rootLogger } from "./logger.js";
-const logger = rootLogger.child({ module: "gap-pass" });
-async function runGapAnalysis(screens, appUrl, llmCfg) {
-  if (!llmCfg?.apiKey || screens.length < 2) return [];
-  const origin = (() => {
-    try {
-      return new URL(appUrl).origin;
-    } catch {
-      return "";
-    }
-  })();
-  if (!origin) return [];
-  const screenList = screens.slice(0, 80).map((s) => {
-    try {
-      return new URL(s.url).pathname + (s.name ? ` (${s.name})` : "");
-    } catch {
-      return s.url;
+/**
+ * P6: Gap Pass — LLM-guided second crawl pass
+ *
+ * After the primary BFS crawl completes, this module:
+ * 1. Sends the discovered screen list to an LLM for gap analysis
+ * 2. HEAD-validates each suggested URL
+ * 3. Returns confirmed gap URLs for targeted re-crawl
+ *
+ * The caller (crawler.ts) is responsible for actually visiting the confirmed
+ * URLs with the existing browser — this module only identifies them.
+ */
+import { logger as rootLogger } from './logger.js';
+const logger = rootLogger.child({ module: 'gap-pass' });
+/**
+ * Run LLM gap analysis + HEAD validation.
+ * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.
+ */
+export async function runGapAnalysis(screens, appUrl, llmCfg) {
+    if (!llmCfg?.apiKey || screens.length < 2)
+        return [];
+    const origin = (() => { try {
+        return new URL(appUrl).origin;
     }
-  }).join("\n");
-  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
+    catch {
+        return '';
+    } })();
+    if (!origin)
+        return [];
+    // Cap context — only send url + name, not full element data
+    const screenList = screens
+        .slice(0, 80)
+        .map(s => {
+        try {
+            return new URL(s.url).pathname + (s.name ? ` (${s.name})` : '');
+        }
+        catch {
+            return s.url;
+        }
+    })
+        .join('\n');
+    const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
 
 Discovered screens (${screens.length} total):
 ${screenList}
@@ -28,55 +49,55 @@ Based on these screens and common patterns for this type of web application, ide
 
 Respond with ONLY valid JSON, no markdown:
 {"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`;
-  try {
-    const Anthropic = (await import("@anthropic-ai/sdk")).default;
-    const client = new Anthropic({ apiKey: llmCfg.apiKey });
-    const response = await client.messages.create({
-      model: llmCfg.model ?? "claude-haiku-4-5-20251001",
-      max_tokens: 1024,
-      temperature: 0,
-      // @ts-ignore
-      system: [{ type: "text", text: "You are a web application coverage analyst. Respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
-      messages: [{ role: "user", content: prompt }]
-    });
-    const text = response.content[0]?.text ?? "";
-    const jsonMatch = text.match(/\{[\s\S]*\}/);
-    if (!jsonMatch) return [];
-    const parsed = JSON.parse(jsonMatch[0]);
-    const rawGaps = parsed.gaps ?? [];
-    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];
-    logger.info({ gaps: rawGaps.length }, "[P6] Gap analysis: LLM returned candidates");
-    const results = [];
-    for (const gap of rawGaps.slice(0, 15)) {
-      let confirmedUrl = null;
-      let headStatus = null;
-      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {
-        const fullUrl = pattern.startsWith("http") ? pattern : `${origin}${pattern.startsWith("/") ? "" : "/"}${pattern}`;
-        try {
-          const controller = new AbortController();
-          const timeout = setTimeout(() => controller.abort(), 6e3);
-          const res = await fetch(fullUrl, { method: "HEAD", signal: controller.signal, redirect: "follow" }).catch(() => null);
-          clearTimeout(timeout);
-          headStatus = res?.status ?? null;
-          if (res?.ok) {
-            confirmedUrl = fullUrl;
-            break;
-          }
-        } catch {
+    try {
+        const Anthropic = (await import('@anthropic-ai/sdk')).default;
+        const client = new Anthropic({ apiKey: llmCfg.apiKey });
+        const response = await client.messages.create({
+            model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+            max_tokens: 1024,
+            temperature: 0,
+            // @ts-ignore
+            system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
+            messages: [{ role: 'user', content: prompt }],
+        });
+        const text = response.content[0]?.text ?? '';
+        const jsonMatch = text.match(/\{[\s\S]*\}/);
+        if (!jsonMatch)
+            return [];
+        const parsed = JSON.parse(jsonMatch[0]);
+        const rawGaps = parsed.gaps ?? [];
+        if (!Array.isArray(rawGaps) || rawGaps.length === 0)
+            return [];
+        logger.info({ gaps: rawGaps.length }, '[P6] Gap analysis: LLM returned candidates');
+        // HEAD-validate each URL pattern, take the first 200
+        const results = [];
+        for (const gap of rawGaps.slice(0, 15)) {
+            let confirmedUrl = null;
+            let headStatus = null;
+            for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {
+                const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`;
+                try {
+                    const controller = new AbortController();
+                    const timeout = setTimeout(() => controller.abort(), 6_000);
+                    const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);
+                    clearTimeout(timeout);
+                    headStatus = res?.status ?? null;
+                    if (res?.ok) {
+                        confirmedUrl = fullUrl;
+                        break;
+                    }
+                }
+                catch { /* non-fatal */ }
+            }
+            results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });
+            logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked');
         }
-      }
-      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });
-      logger.info({ category: gap.category, confirmedUrl, headStatus }, "[P6] Gap candidate checked");
+        const confirmed = results.filter(r => r.confirmedUrl).length;
+        logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete');
+        return results;
+    }
+    catch (err) {
+        logger.warn({ err: String(err) }, '[P6] Gap analysis failed — non-fatal');
+        return [];
     }
-    const confirmed = results.filter((r) => r.confirmedUrl).length;
-    logger.info({ total: results.length, confirmed }, "[P6] Gap pass analysis complete");
-    return results;
-  } catch (err) {
-    logger.warn({ err: String(err) }, "[P6] Gap analysis failed \u2014 non-fatal");
-    return [];
-  }
 }
-export {
-  runGapAnalysis
-};
-//# sourceMappingURL=gap-pass.js.map

package/dist/graphql-detector.js

@@ -1,3 +1,10 @@
+/**
+ * GraphQL detection + introspection.
+ *
+ * During crawl, if a GraphQL endpoint is discovered (via network observer or
+ * known-URL heuristics), fire a minimal introspection query to get the schema.
+ * The schema is stored in the crawl artifact for use during test generation.
+ */
 const INTROSPECTION_QUERY = `
   query IntrospectionQuery {
     __schema {
@@ -15,72 +22,75 @@ const INTROSPECTION_QUERY = `
   }
 `;
 function flattenType(t) {
-  if (!t) return "unknown";
-  if (t.kind === "NON_NULL" || t.kind === "LIST") return flattenType(t.ofType);
-  return t.name ?? "unknown";
+    if (!t)
+        return 'unknown';
+    if (t.kind === 'NON_NULL' || t.kind === 'LIST')
+        return flattenType(t.ofType);
+    return t.name ?? 'unknown';
 }
 function parseSchema(raw) {
-  const s = raw?.__schema ?? raw?.data?.__schema ?? raw;
-  const types = (s?.types ?? []).filter((t) => !t.name?.startsWith("__")).map((t) => ({
-    name: t.name,
-    kind: t.kind,
-    fields: (t.fields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
-    inputFields: (t.inputFields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
-    enumValues: (t.enumValues ?? []).map((e) => e.name)
-  }));
-  return {
-    queryType: s?.queryType?.name,
-    mutationType: s?.mutationType?.name,
-    subscriptionType: s?.subscriptionType?.name,
-    types
-  };
-}
-async function introspectGraphQL(endpoint, opts = {}) {
-  const { headers = {}, timeoutMs = 1e4, cookies } = opts;
-  const t0 = Date.now();
-  try {
-    const reqHeaders = {
-      "Content-Type": "application/json",
-      "Accept": "application/json",
-      ...headers
+    const s = raw?.__schema ?? raw?.data?.__schema ?? raw;
+    const types = (s?.types ?? [])
+        .filter((t) => !t.name?.startsWith('__'))
+        .map((t) => ({
+        name: t.name,
+        kind: t.kind,
+        fields: (t.fields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
+        inputFields: (t.inputFields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
+        enumValues: (t.enumValues ?? []).map((e) => e.name),
+    }));
+    return {
+        queryType: s?.queryType?.name,
+        mutationType: s?.mutationType?.name,
+        subscriptionType: s?.subscriptionType?.name,
+        types,
     };
-    if (cookies) reqHeaders["Cookie"] = cookies;
-    const res = await fetch(endpoint, {
-      method: "POST",
-      headers: reqHeaders,
-      body: JSON.stringify({ query: INTROSPECTION_QUERY }),
-      signal: AbortSignal.timeout(timeoutMs)
-    });
-    if (!res.ok) {
-      return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };
+}
+export async function introspectGraphQL(endpoint, opts = {}) {
+    const { headers = {}, timeoutMs = 10_000, cookies } = opts;
+    const t0 = Date.now();
+    try {
+        const reqHeaders = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            ...headers,
+        };
+        if (cookies)
+            reqHeaders['Cookie'] = cookies;
+        const res = await fetch(endpoint, {
+            method: 'POST',
+            headers: reqHeaders,
+            body: JSON.stringify({ query: INTROSPECTION_QUERY }),
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+        if (!res.ok) {
+            return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };
+        }
+        const json = await res.json();
+        if (json?.errors?.length) {
+            const msg = json.errors[0]?.message ?? 'introspection disabled';
+            return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };
+        }
+        return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };
     }
-    const json = await res.json();
-    if (json?.errors?.length) {
-      const msg = json.errors[0]?.message ?? "introspection disabled";
-      return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };
+    catch (e) {
+        return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };
     }
-    return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };
-  } catch (e) {
-    return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };
-  }
 }
-function guessGraphQLEndpoints(appUrl) {
-  try {
-    const u = new URL(appUrl);
-    const base = `${u.protocol}//${u.host}`;
-    return [
-      `${base}/graphql`,
-      `${base}/api/graphql`,
-      `${base}/v1/graphql`,
-      `${base}/gql`,
-      `${base}/query`
-    ];
-  } catch {
-    return [];
-  }
+/** Guess GraphQL endpoint URLs from a known app URL. */
+export function guessGraphQLEndpoints(appUrl) {
+    try {
+        const u = new URL(appUrl);
+        const base = `${u.protocol}//${u.host}`;
+        return [
+            `${base}/graphql`,
+            `${base}/api/graphql`,
+            `${base}/v1/graphql`,
+            `${base}/gql`,
+            `${base}/query`,
+        ];
+    }
+    catch {
+        return [];
+    }
 }
-export {
-  guessGraphQLEndpoints,
-  introspectGraphQL
-};
-//# sourceMappingURL=graphql-detector.js.map