@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/frontier.ts

@@ -0,0 +1,144 @@
+/**
+ * Configurable crawl traversal strategy — matches crawl4AI's BFS/DFS/
+ * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded
+ * FIFO queue, i.e. always BFS).
+ *
+ * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it
+ * drops into crawler.ts's existing queue call sites with a rename, not a
+ * rewrite — BFS's `shift()` behavior is byte-for-byte the same as the plain
+ * array it replaces, so BFS (the default) has zero behavior change.
+ */
+
+export type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';
+
+/**
+ * Simple glob-like pattern matching against a URL's pathname.
+ * Supports `*` (matches any single path segment characters, not `/`)
+ * and `**` (matches anything including `/`).
+ */
+export function matchesPattern(url: string, patterns: string[]): boolean {
+  try {
+    const path = new URL(url).pathname;
+    return patterns.some(pattern => {
+      const regex = new RegExp('^' + pattern.replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*') + '$');
+      return regex.test(path);
+    });
+  } catch {
+    return false;
+  }
+}
+
+interface FrontierEntry {
+  url: string;
+  score: number;
+}
+
+/**
+ * Best-First scoring heuristic: shallower paths score higher (breadth-like
+ * bias toward top-level sections before deep sub-pages), boosted further by
+ * any user-supplied keyword appearing in the URL. Simple and transparent by
+ * design — crawl4AI's own default scorer is a similarly simple keyword-
+ * relevance heuristic, not a learned model.
+ */
+function scoreUrl(url: string, keywords: string[]): number {
+  try {
+    const u = new URL(url);
+    const segments = u.pathname.split('/').filter(Boolean);
+    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;
+    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;
+    let score = -segments.length * depthPenaltyWeight;
+    if (keywords.length > 0) {
+      const lower = url.toLowerCase();
+      for (const kw of keywords) {
+        if (kw && lower.includes(kw)) score += keywordBoostWeight;
+      }
+    }
+    return score;
+  } catch {
+    return 0;
+  }
+}
+
+export class CrawlFrontier {
+  private items: FrontierEntry[] = [];
+  private readonly strategy: TraversalStrategy;
+  private readonly keywords: string[];
+  private readonly urlFilter: RegExp | null;
+  private readonly includePatterns: string[];
+  private readonly excludePatterns: string[];
+
+  constructor(
+    strategy: TraversalStrategy,
+    seedUrls: string[],
+    keywords: string[] = [],
+    urlFilter?: string,
+    includePatterns?: string[],
+    excludePatterns?: string[],
+  ) {
+    this.strategy = strategy;
+    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+    this.urlFilter = urlFilter
+      ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()
+      : null;
+    this.includePatterns = includePatterns ?? [];
+    this.excludePatterns = excludePatterns ?? [];
+    for (const url of seedUrls) this.push(url);
+  }
+
+  get length(): number {
+    return this.items.length;
+  }
+
+  /**
+   * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+   * the frontier revisits sparse branches before deep-linked content.
+   */
+  push(url: string, pushOpts?: { parentElements?: number }): void {
+    if (this.urlFilter && !this.urlFilter.test(url)) return;
+    if (this.includePatterns.length && !matchesPattern(url, this.includePatterns)) return;
+    if (this.excludePatterns.length && matchesPattern(url, this.excludePatterns)) return;
+    const sparseBoost =
+      this.strategy === 'BEST_FIRST' &&
+      pushOpts?.parentElements !== undefined &&
+      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)
+        ? 1
+        : 0;
+    this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
+  }
+
+  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+  shift(): string | undefined {
+    if (this.items.length === 0) return undefined;
+    switch (this.strategy) {
+      case 'DFS':
+        return this.items.pop()!.url;
+      case 'BEST_FIRST': {
+        let bestIdx = 0;
+        for (let i = 1; i < this.items.length; i++) {
+          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;
+        }
+        return this.items.splice(bestIdx, 1)[0].url;
+      }
+      case 'BFS':
+      default:
+        return this.items.shift()!.url;
+    }
+  }
+
+  filter(predicate: (url: string) => boolean): string[] {
+    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);
+  }
+
+  /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+  filterByPattern(pattern: string): number {
+    try {
+      const re = new RegExp(pattern, 'i');
+      const before = this.items.length;
+      this.items = this.items.filter((e) => re.test(e.url));
+      return before - this.items.length;
+    } catch {
+      return 0;
+    }
+  }
+}

package/src/gap-pass.ts

@@ -0,0 +1,165 @@
+/**
+ * P6: Gap Pass — LLM-guided second crawl pass
+ *
+ * After the primary BFS crawl completes, this module:
+ * 1. Sends the discovered screen list to an LLM for gap analysis
+ * 2. HEAD-validates each suggested URL
+ * 3. Returns confirmed gap URLs for targeted re-crawl
+ *
+ * The caller (crawler.ts) is responsible for actually visiting the confirmed
+ * URLs with the existing browser — this module only identifies them.
+ */
+
+import { logger as rootLogger } from './logger.js';
+
+const logger = rootLogger.child({ module: 'gap-pass' });
+
+export interface GapCandidate {
+  category: string;
+  urlPatterns: string[];
+  confidence: number;
+  rationale: string;
+  confirmedUrl: string | null;
+  headStatus: number | null;
+}
+
+interface DiscoveredScreen {
+  url: string;
+  name?: string;
+  elements?: number;
+}
+
+interface LlmCfg {
+  provider: string;
+  model: string;
+  apiKey?: string;
+  baseUrl?: string;
+}
+
+// Only openai and anthropic are supported here. NVIDIA, groq, ollama, etc. → skip, fall to env.
+/**
+ * Resolve effective LLM. Priority: openai → anthropic → skip unsupported providers.
+ * Falls back to env vars when platform provider is not supported.
+ */
+function resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null {
+  if (llmCfg?.provider === 'openai' && llmCfg.apiKey) {
+    return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };
+  }
+  if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) {
+    return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model };
+  }
+  // Unsupported provider (nvidia, groq, etc.) or no key — env fallback, openai first.
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' };
+  }
+  return null;
+}
+
+/**
+ * Run LLM gap analysis + HEAD validation.
+ * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.
+ */
+export async function runGapAnalysis(
+  screens: DiscoveredScreen[],
+  appUrl: string,
+  llmCfg: LlmCfg | null | undefined,
+): Promise<GapCandidate[]> {
+  const effective = resolveTextLLM(llmCfg);
+  if (!effective || screens.length < 2) return [];
+
+  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();
+  if (!origin) return [];
+
+  // Cap context — only send url + name, not full element data
+  const screenList = screens
+    .slice(0, 80)
+    .map(s => {
+      try { return new URL(s.url).pathname + (s.name ? ` (${s.name})` : ''); } catch { return s.url; }
+    })
+    .join('\n');
+
+  const systemPrompt = 'You are a web application coverage analyst. Respond with valid JSON only.';
+  const userPrompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
+
+Discovered screens (${screens.length} total):
+${screenList}
+
+App URL: ${appUrl}
+
+Based on these screens and common patterns for this type of web application, identify screen categories that are likely present but missing. Focus on pages that would exist in a typical SaaS or web app but are not represented above.
+
+Respond with ONLY valid JSON, no markdown:
+{"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`;
+
+  try {
+    let text = '';
+
+    if (effective.provider === 'openai') {
+      const { OpenAI } = await import('openai');
+      const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) });
+      const response = await client.chat.completions.create({
+        model: effective.model ?? 'gpt-4o-mini',
+        max_tokens: 1024,
+        temperature: 0,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      });
+      text = response.choices[0]?.message?.content ?? '';
+    } else {
+      const Anthropic = (await import('@anthropic-ai/sdk')).default;
+      const client = new Anthropic({ apiKey: effective.apiKey });
+      const response = await client.messages.create({
+        model: effective.model ?? 'claude-haiku-4-5-20251001',
+        max_tokens: 1024,
+        temperature: 0,
+        // @ts-ignore
+        system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
+        messages: [{ role: 'user', content: userPrompt }],
+      });
+      text = (response.content[0] as any)?.text ?? '';
+    }
+
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return [];
+    const parsed = JSON.parse(jsonMatch[0]);
+    const rawGaps: Array<{ category: string; urlPatterns: string[]; confidence: number; rationale: string }> =
+      parsed.gaps ?? [];
+
+    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];
+    logger.info({ gaps: rawGaps.length, provider: effective.provider }, '[P6] Gap analysis: LLM returned candidates');
+
+    // HEAD-validate each URL pattern
+    const results: GapCandidate[] = [];
+    for (const gap of rawGaps.slice(0, 15)) {
+      let confirmedUrl: string | null = null;
+      let headStatus: number | null = null;
+
+      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {
+        const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`;
+        try {
+          const controller = new AbortController();
+          const timeout = setTimeout(() => controller.abort(), 6_000);
+          const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);
+          clearTimeout(timeout);
+          headStatus = res?.status ?? null;
+          if (res?.ok) { confirmedUrl = fullUrl; break; }
+        } catch { /* non-fatal */ }
+      }
+
+      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });
+      logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked');
+    }
+
+    const confirmed = results.filter(r => r.confirmedUrl).length;
+    logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete');
+    return results;
+  } catch (err) {
+    logger.warn({ err: String(err) }, '[P6] Gap analysis failed — non-fatal');
+    return [];
+  }
+}

package/src/graphql-detector.ts

@@ -0,0 +1,129 @@
+/**
+ * GraphQL detection + introspection.
+ *
+ * During crawl, if a GraphQL endpoint is discovered (via network observer or
+ * known-URL heuristics), fire a minimal introspection query to get the schema.
+ * The schema is stored in the crawl artifact for use during test generation.
+ */
+
+export interface GqlType {
+  name: string;
+  kind: string;
+  fields?: Array<{ name: string; type: string }>;
+  inputFields?: Array<{ name: string; type: string }>;
+  enumValues?: string[];
+}
+
+export interface GqlSchema {
+  queryType?: string;
+  mutationType?: string;
+  subscriptionType?: string;
+  types: GqlType[];
+}
+
+export interface GqlIntrospectionResult {
+  endpoint: string;
+  schema: GqlSchema | null;
+  error?: string;
+  durationMs: number;
+}
+
+const INTROSPECTION_QUERY = `
+  query IntrospectionQuery {
+    __schema {
+      queryType { name }
+      mutationType { name }
+      subscriptionType { name }
+      types {
+        name
+        kind
+        fields(includeDeprecated: false) { name type { name kind ofType { name kind } } }
+        inputFields { name type { name kind ofType { name kind } } }
+        enumValues(includeDeprecated: false) { name }
+      }
+    }
+  }
+`;
+
+function flattenType(t: any): string {
+  if (!t) return 'unknown';
+  if (t.kind === 'NON_NULL' || t.kind === 'LIST') return flattenType(t.ofType);
+  return t.name ?? 'unknown';
+}
+
+function parseSchema(raw: any): GqlSchema {
+  const s = raw?.__schema ?? raw?.data?.__schema ?? raw;
+  const types: GqlType[] = (s?.types ?? [])
+    .filter((t: any) => !t.name?.startsWith('__'))
+    .map((t: any) => ({
+      name: t.name,
+      kind: t.kind,
+      fields: (t.fields ?? []).map((f: any) => ({ name: f.name, type: flattenType(f.type) })),
+      inputFields: (t.inputFields ?? []).map((f: any) => ({ name: f.name, type: flattenType(f.type) })),
+      enumValues: (t.enumValues ?? []).map((e: any) => e.name),
+    }));
+  return {
+    queryType: s?.queryType?.name,
+    mutationType: s?.mutationType?.name,
+    subscriptionType: s?.subscriptionType?.name,
+    types,
+  };
+}
+
+export async function introspectGraphQL(
+  endpoint: string,
+  opts: {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    cookies?: string;
+  } = {},
+): Promise<GqlIntrospectionResult> {
+  const { headers = {}, timeoutMs = 10_000, cookies } = opts;
+  const t0 = Date.now();
+  try {
+    const reqHeaders: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'Accept': 'application/json',
+      ...headers,
+    };
+    if (cookies) reqHeaders['Cookie'] = cookies;
+
+    const res = await fetch(endpoint, {
+      method: 'POST',
+      headers: reqHeaders,
+      body: JSON.stringify({ query: INTROSPECTION_QUERY }),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+
+    if (!res.ok) {
+      return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };
+    }
+
+    const json = await res.json();
+    if (json?.errors?.length) {
+      const msg = json.errors[0]?.message ?? 'introspection disabled';
+      return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };
+    }
+
+    return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };
+  } catch (e: any) {
+    return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };
+  }
+}
+
+/** Guess GraphQL endpoint URLs from a known app URL. */
+export function guessGraphQLEndpoints(appUrl: string): string[] {
+  try {
+    const u = new URL(appUrl);
+    const base = `${u.protocol}//${u.host}`;
+    return [
+      `${base}/graphql`,
+      `${base}/api/graphql`,
+      `${base}/v1/graphql`,
+      `${base}/gql`,
+      `${base}/query`,
+    ];
+  } catch {
+    return [];
+  }
+}