@detiq/crawler 1.1.22 → 1.1.24

package/src/extractors/summary-extractor.ts

@@ -0,0 +1,49 @@
+export interface SummaryResult {
+  summary: string;
+  wordCount: number;
+  method: 'llm' | 'truncated';
+  model?: string;
+}
+
+export async function extractSummary(
+  markdown: string,
+  options?: { model?: string; maxTokens?: number },
+): Promise<SummaryResult> {
+  // Truncate to 8000 chars to stay within token budget
+  const input = markdown.slice(0, 8000);
+
+  try {
+    // Dynamic import to avoid hard dependency if agents package unavailable
+    const { ask } = await import('@detiq/agents').catch(() => ({ ask: null as any }));
+
+    if (!ask) {
+      return makeTruncated(input);
+    }
+
+    const summary: string = await ask(
+      'Summarize the following web page content in 2-4 concise sentences. Focus on the main purpose, key features, and primary value proposition. Return only the summary text, no preamble.',
+      input,
+      options?.model ? { provider: 'anthropic', model: options.model } : undefined,
+      options?.maxTokens ?? 300,
+    );
+
+    return {
+      summary: summary.trim(),
+      wordCount: markdown.split(/\s+/).length,
+      method: 'llm',
+      model: options?.model,
+    };
+  } catch (err) {
+    console.warn('[summary-extractor] LLM call failed, using truncated fallback:', err);
+    return makeTruncated(input);
+  }
+}
+
+function makeTruncated(input: string): SummaryResult {
+  const excerpt = input.slice(0, 500).trim();
+  return {
+    summary: excerpt + (input.length > 500 ? '...' : ''),
+    wordCount: excerpt.split(/\s+/).length,
+    method: 'truncated',
+  };
+}

package/src/frontier.ts

@@ -11,6 +11,23 @@
 
 export type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';
 
+/**
+ * Simple glob-like pattern matching against a URL's pathname.
+ * Supports `*` (matches any single path segment characters, not `/`)
+ * and `**` (matches anything including `/`).
+ */
+export function matchesPattern(url: string, patterns: string[]): boolean {
+  try {
+    const path = new URL(url).pathname;
+    return patterns.some(pattern => {
+      const regex = new RegExp('^' + pattern.replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*') + '$');
+      return regex.test(path);
+    });
+  } catch {
+    return false;
+  }
+}
+
 interface FrontierEntry {
   url: string;
   score: number;
@@ -47,13 +64,24 @@ export class CrawlFrontier {
   private readonly strategy: TraversalStrategy;
   private readonly keywords: string[];
   private readonly urlFilter: RegExp | null;
+  private readonly includePatterns: string[];
+  private readonly excludePatterns: string[];
 
-  constructor(strategy: TraversalStrategy, seedUrls: string[], keywords: string[] = [], urlFilter?: string) {
+  constructor(
+    strategy: TraversalStrategy,
+    seedUrls: string[],
+    keywords: string[] = [],
+    urlFilter?: string,
+    includePatterns?: string[],
+    excludePatterns?: string[],
+  ) {
     this.strategy = strategy;
     this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
     this.urlFilter = urlFilter
       ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()
       : null;
+    this.includePatterns = includePatterns ?? [];
+    this.excludePatterns = excludePatterns ?? [];
     for (const url of seedUrls) this.push(url);
   }
 
@@ -68,6 +96,8 @@ export class CrawlFrontier {
    */
   push(url: string, pushOpts?: { parentElements?: number }): void {
     if (this.urlFilter && !this.urlFilter.test(url)) return;
+    if (this.includePatterns.length && !matchesPattern(url, this.includePatterns)) return;
+    if (this.excludePatterns.length && matchesPattern(url, this.excludePatterns)) return;
     const sparseBoost =
       this.strategy === 'BEST_FIRST' &&
       pushOpts?.parentElements !== undefined &&

package/src/incremental-diff.ts

@@ -119,3 +119,141 @@ export function partitionByChange(
   }
   return { toProcess, toSkip };
 }
+
+// ─── Typed monitor diffs (GAP-01) ───────────────────────────────────────────
+
+/** A single line-level change in text content (markdown, html, summary). */
+export interface TextChange {
+  type: 'added' | 'removed';
+  line: number;
+  content: string;
+}
+
+/** A change to a specific JSON path between two scraped JSON objects. */
+export interface JsonChange {
+  path: string;
+  old: unknown;
+  new: unknown;
+}
+
+/** Set-based change for collections like links or images. */
+export interface SetChange {
+  added: string[];
+  removed: string[];
+}
+
+/** Screenshot pixel-level change. changed is percentage (0-100). */
+export interface ImageChange {
+  size: number;
+  changed: number;
+  mask?: string; // base64 diff mask
+}
+
+/** All possible diffs for a single monitor tick. */
+export interface MonitorDiffs {
+  markdown?: TextChange[];
+  html?: TextChange[];
+  json?: JsonChange[];
+  summary?: TextChange[];
+  links?: SetChange;
+  images?: SetChange;
+  branding?: JsonChange[];
+  screenshot?: ImageChange;
+}
+
+/**
+ * Compute line-level text diffs between two strings.
+ * Returns TextChange[] showing exactly which lines were added or removed.
+ * No external dependencies — pure JS Myers-inspired algorithm.
+ */
+export function computeTextDiffs(previous: string, current: string): TextChange[] {
+  const prevLines = previous.split('\n');
+  const currLines = current.split('\n');
+  const changes: TextChange[] = [];
+
+  const prevSet = new Set(prevLines);
+  const currSet = new Set(currLines);
+
+  let lineNum = 0;
+  for (const line of prevLines) {
+    if (!currSet.has(line)) {
+      changes.push({ type: 'removed', line: lineNum, content: line });
+    }
+    lineNum++;
+  }
+  lineNum = 0;
+  for (const line of currLines) {
+    if (!prevSet.has(line)) {
+      changes.push({ type: 'added', line: lineNum, content: line });
+    }
+    lineNum++;
+  }
+
+  return changes;
+}
+
+/**
+ * Compute set-based diffs for URL collections (links, images).
+ */
+export function computeSetDiffs(previous: string[], current: string[]): SetChange {
+  const prevSet = new Set(previous);
+  const currSet = new Set(current);
+  return {
+    added: current.filter(u => !prevSet.has(u)),
+    removed: previous.filter(u => !currSet.has(u)),
+  };
+}
+
+/**
+ * Flatten a JSON object into dot-notation path/value pairs.
+ * Used to compute JsonChange[] between two JSON extraction results.
+ */
+export function flattenJson(obj: unknown, prefix = ''): Record<string, unknown> {
+  if (typeof obj !== 'object' || obj === null) {
+    return { [prefix]: obj };
+  }
+  const result: Record<string, unknown> = {};
+  for (const [k, v] of Object.entries(obj as Record<string, unknown>)) {
+    const path = prefix ? `${prefix}.${k}` : k;
+    if (typeof v === 'object' && v !== null && !Array.isArray(v)) {
+      Object.assign(result, flattenJson(v, path));
+    } else {
+      result[path] = v;
+    }
+  }
+  return result;
+}
+
+/**
+ * Compute JSON path-level diffs between two extraction results.
+ */
+export function computeJsonDiffs(previous: unknown, current: unknown): JsonChange[] {
+  const prevFlat = flattenJson(previous);
+  const currFlat = flattenJson(current);
+  const allKeys = new Set([...Object.keys(prevFlat), ...Object.keys(currFlat)]);
+  const changes: JsonChange[] = [];
+  for (const path of allKeys) {
+    const oldVal = prevFlat[path];
+    const newVal = currFlat[path];
+    if (JSON.stringify(oldVal) !== JSON.stringify(newVal)) {
+      changes.push({ path, old: oldVal, new: newVal });
+    }
+  }
+  return changes;
+}
+
+/**
+ * Count total number of changes across all diff types.
+ */
+export function countMonitorDiffs(diffs: MonitorDiffs): number {
+  let count = 0;
+  if (diffs.markdown) count += diffs.markdown.length;
+  if (diffs.html) count += diffs.html.length;
+  if (diffs.json) count += diffs.json.length;
+  if (diffs.summary) count += diffs.summary.length;
+  if (diffs.links) count += diffs.links.added.length + diffs.links.removed.length;
+  if (diffs.images) count += diffs.images.added.length + diffs.images.removed.length;
+  if (diffs.branding) count += diffs.branding.length;
+  if (diffs.screenshot) count += diffs.screenshot.changed > 0 ? 1 : 0;
+  return count;
+}

package/src/index.ts

@@ -3,7 +3,7 @@ export * from './visual-diff-verdict.js';
 export * from './auth.js';
 export * from './screen-reader.js';
 export * from './job-queue.js';
-export { startSqsWorker } from './sqs-queue.js';
+export { startSqsWorker, getDlqStats, redriveFromDlq } from './sqs-queue.js';
 export * from './session-capture.js';
 export * from './session-export.js';
 export * from './test-executor.js';

package/src/interactive-probe.ts

@@ -43,12 +43,16 @@ export interface ModalObservation {
   title: string;
   content: string;
   fields: string[];
+  stateId?: string;
+  domSnapshot?: string;
 }
 
 export interface ConditionalReveal {
   triggerElement: string;
   triggerValue: string;
   revealedText: string;
+  stateId?: string;
+  domSnapshot?: string;
 }
 
 export interface TooltipCapture {
@@ -196,27 +200,52 @@ async function findSubmitButton(page: Page): Promise<string | null> {
   });
 }
 
-/** Find all modal-trigger buttons (non-submit buttons). */
+/** Find modal-trigger buttons and clickable row-level items (e.g. test-case rows opening side panels). */
 async function findModalTriggers(page: Page): Promise<Array<{ selector: string; label: string }>> {
   return page.evaluate(() => {
     const SKIP = /submit|send|save|create|log.?in|sign.?in|register|continue|checkout|buy|order|search|close|cancel|dismiss/i;
     const NAV_PARENTS = new Set(['NAV', 'HEADER', 'FOOTER']);
-    return Array.from(document.querySelectorAll('button, [role="button"]'))
-      .filter((el) => {
-        const style = window.getComputedStyle(el as HTMLElement);
-        if (style.display === 'none' || style.visibility === 'hidden') return false;
-        const label = (el as HTMLElement).innerText?.trim() || el.getAttribute('aria-label') || '';
-        if (!label || SKIP.test(label)) return false;
-        // Skip nav chrome
-        let p = el.parentElement;
-        while (p) { if (NAV_PARENTS.has(p.tagName)) return false; p = p.parentElement; }
-        return true;
-      })
-      .slice(0, 20) // probe at most 20 buttons per screen
-      .map((el) => {
-        const label = (el as HTMLElement).innerText?.trim() || el.getAttribute('aria-label') || 'button';
-        return { selector: `[aria-label="${label}"]`, label: label.slice(0, 60) };
-      });
+
+    function isVisible(el: Element): boolean {
+      const s = window.getComputedStyle(el as HTMLElement);
+      return s.display !== 'none' && s.visibility !== 'hidden' && (s.opacity !== '0');
+    }
+
+    const seen = new Set<string>();
+    const results: Array<{ selector: string; label: string }> = [];
+
+    // Standard button / role=button elements
+    for (const el of Array.from(document.querySelectorAll('button, [role="button"]'))) {
+      if (!isVisible(el)) continue;
+      const label = ((el as HTMLElement).innerText?.trim() || el.getAttribute('aria-label') || '').slice(0, 60);
+      if (!label || SKIP.test(label) || seen.has(label)) continue;
+      let p = el.parentElement; let inNav = false;
+      while (p) { if (NAV_PARENTS.has(p.tagName)) { inNav = true; break; } p = p.parentElement; }
+      if (inNav) continue;
+      seen.add(label);
+      results.push({ selector: `button:has-text("${label.slice(0, 40)}")`, label });
+      if (results.length >= 15) break;
+    }
+
+    // Row-level clickable elements — table/list rows that open side panels on click
+    // (e.g. test-case rows → ScreenPreviewPanel, ScriptPopup)
+    const rowCandidates = Array.from(document.querySelectorAll(
+      '[role="row"][tabindex="0"], tr[tabindex="0"], [role="listitem"][tabindex="0"], [class*="list-item"][tabindex="0"], [class*="table-row"][tabindex="0"]'
+    )).filter(isVisible).slice(0, 3);
+    for (const row of rowCandidates) {
+      const ariaLabel = row.getAttribute('aria-label');
+      const textLabel = (row as HTMLElement).innerText?.trim().split('\n')[0].trim().slice(0, 60);
+      const label = (ariaLabel || textLabel || '').slice(0, 60);
+      if (!label || seen.has(label)) continue;
+      seen.add(label);
+      // Use aria-label selector when available; fall back to row role + text content
+      const selector = ariaLabel
+        ? `[aria-label="${ariaLabel}"]`
+        : `[role="row"]:has-text("${label.slice(0, 40)}")`;
+      results.push({ selector, label });
+    }
+
+    return results.slice(0, 20);
   });
 }
 
@@ -245,7 +274,17 @@ async function probeConditionals(page: Page): Promise<ConditionalReveal[]> {
               .join(' | ');
           });
           if (revealed) {
-            results.push({ triggerElement: label, triggerValue: opt.text || opt.value, revealedText: revealed.slice(0, 200) });
+            const domSnapshot = await page.content();
+            const stateId = await page.evaluate(() => {
+              let hash = 0;
+              const str = document.body.innerText;
+              for (let i = 0; i < str.length; i++) {
+                hash = ((hash << 5) - hash) + str.charCodeAt(i);
+                hash = hash & hash;
+              }
+              return Math.abs(hash).toString(16);
+            });
+            results.push({ triggerElement: label, triggerValue: opt.text || opt.value, revealedText: revealed.slice(0, 200), stateId, domSnapshot });
           }
         }
       }
@@ -366,11 +405,21 @@ export async function probeScreen(
     const triggers = await findModalTriggers(page);
     for (const trigger of triggers) {
       try {
-        const btn = page.locator(`button:has-text("${trigger.label}")`).first();
+        const btn = page.locator(trigger.selector).first();
         const domBefore = await page.evaluate(() => document.body.innerText.length);
 
         await btn.click({ timeout: 1500 });
-        await page.waitForTimeout(1000);
+        // Adaptive wait: some triggers launch async jobs (e.g. script generation 5–30s).
+        // Wait for a visible dialog/panel rather than a fixed sleep.
+        try {
+          await page.waitForSelector(
+            '[role="dialog"],[role="alertdialog"],[aria-modal="true"],[data-state="open"],[class*="modal" i]:not([class*="backdrop" i]),[class*="Dialog"]:not([class*="Overlay"])',
+            { timeout: 8000, state: 'visible' }
+          );
+        } catch {
+          // No dialog appeared — wait briefly for accordion/panel DOM settle
+          await page.waitForTimeout(300);
+        }
 
         // Check if a dialog/modal appeared
         const dialogText = await page.evaluate(() => {
@@ -387,12 +436,26 @@ export async function probeScreen(
           return { title, content, fields };
         });
 
+        const domSnapshot = await page.content();
+        const stateId = await page.evaluate(() => {
+          // simple simHash placeholder
+          let hash = 0;
+          const str = document.body.innerText;
+          for (let i = 0; i < str.length; i++) {
+            hash = ((hash << 5) - hash) + str.charCodeAt(i);
+            hash = hash & hash;
+          }
+          return Math.abs(hash).toString(16);
+        });
+
         if (dialogText && dialogText.content.length > 20) {
           result.modals.push({
             triggerElement: trigger.label,
             title: dialogText.title,
             content: dialogText.content,
             fields: dialogText.fields.slice(0, 10),
+            stateId,
+            domSnapshot,
           });
         } else {
           // No modal appeared — check for accordion/disclosure panel reveals
@@ -416,6 +479,8 @@ export async function probeScreen(
               title: accordionText.heading || `${trigger.label} (revealed)`,
               content: accordionText.content,
               fields: [],
+              stateId,
+              domSnapshot,
             });
           }
         }
@@ -640,6 +705,7 @@ export async function probeAllScreens(
   projectId: string,
   discovered: Array<{ url: string; screenId?: string; elements?: any[] }>,
   onProgress?: (pct: number, step: string) => Promise<void>,
+  jobId?: string,
 ): Promise<void> {
   const probeable = discovered.filter((d) => d.screenId && d.url && !d.url.includes('#'));
   if (probeable.length === 0) return;
@@ -677,6 +743,49 @@ export async function probeAllScreens(
         await AppBrain.saveKnowledgeChunk(tenantId, projectId, 'probe', screen.screenId!, probeMarkdown, `Probe — ${screen.url}`).catch(console.warn);
       }
 
+      // Save Virtual Screens — synthetic URL: parent_url + #probe:<stateId>
+      const virtualScreensToSave: Array<{ stateId: string; triggerAction: string; overlayType: string }> = [];
+      for (const m of result.modals) {
+        if (m.stateId) {
+          virtualScreensToSave.push({
+            stateId: m.stateId,
+            triggerAction: `click ${m.triggerElement}`,
+            overlayType: 'modal',
+          });
+          // Also save as a first-class Screen so it appears in the screen library
+          // and participates in test case generation. Use a synthetic URL.
+          const modalUrl = `${screen.url}#probe:${m.stateId}`;
+          const modalTitle = m.title || `${m.triggerElement} Modal`;
+          AppBrain.saveScreen(tenantId, projectId, modalTitle, modalUrl, jobId, 'INTERACTIVE_PROBE')
+            .then((modalScreen: any) => {
+              // Link modal back to parent via DiscoveredLink
+              if (modalScreen?.id && screen.screenId) {
+                AppBrain.saveDiscoveredLinks(tenantId, projectId, screen.screenId!, [{
+                  href: modalUrl,
+                  text: m.triggerElement,
+                  linkType: 'BUTTON_LINK',
+                  source: 'INTERACTIVE',
+                  toScreenId: modalScreen.id,
+                  jobId,
+                }]).catch(() => {});
+              }
+            })
+            .catch(() => {});
+        }
+      }
+      for (const c of result.conditionals) {
+        if ((c as any).stateId) {
+          virtualScreensToSave.push({
+            stateId: (c as any).stateId,
+            triggerAction: `change ${c.triggerElement} to ${c.triggerValue}`,
+            overlayType: 'panel',
+          });
+        }
+      }
+      if (virtualScreensToSave.length > 0) {
+        await (AppBrain as any).saveVirtualScreens(tenantId, projectId, screen.screenId!, virtualScreensToSave).catch(console.warn);
+      }
+
       if (result.validationErrors.length > 0) {
         console.log(`[probe] ${screen.url}: ${result.validationErrors.length} validation errors, ${result.modals.length} modals`);
       }