@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/extractors/artifact-envelope.ts

@@ -0,0 +1,111 @@
+import crypto from 'node:crypto';
+import type { RawElement } from '../screen-reader.js';
+
+export interface CrawlEvidenceFact {
+  type: string;
+  payload: unknown;
+}
+
+export interface CrawlArtifactEnvelope {
+  tenantId: string;
+  projectId: string;
+  jobId?: string;
+  url: string;
+  canonicalUrl: string;
+  screenId?: string;
+  status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';
+  outputs: {
+    screenshot?: string;
+    html?: string;
+    markdown?: string;
+    fitMarkdown?: string;
+    accessibilityTree?: string;
+    structuredJson?: unknown;
+    networkTrace?: string;
+    consoleLog?: string;
+  };
+  fingerprints: {
+    textHash?: string;
+    domHash?: string;
+    accessibilityHash?: string;
+    urlCanonicalHash: string;
+  };
+  evidence: CrawlEvidenceFact[];
+}
+
+export function sha256(input: string): string {
+  return crypto.createHash('sha256').update(input).digest('hex');
+}
+
+export function buildFitMarkdown(markdown: string, keywords: string[] = []): string {
+  const lines = markdown
+    .split('\n')
+    .map((line) => line.trimEnd())
+    .filter((line) => line.trim().length > 0);
+  const lowerKeywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+  const filtered = lines.filter((line) => {
+    const plain = line.replace(/[#*_`>\-[\]()]/g, '').trim();
+    if (plain.length < 3) return false;
+    if (/^(cookie|privacy|terms|copyright|all rights reserved|subscribe|newsletter)$/i.test(plain)) return false;
+    if (lowerKeywords.length === 0) return true;
+    const lower = plain.toLowerCase();
+    return lowerKeywords.some((kw) => lower.includes(kw)) || /^#{1,4}\s/.test(line);
+  });
+  const candidate = filtered.join('\n');
+  return candidate.length >= 200 ? candidate : markdown;
+}
+
+export function summarizeStructuredElements(elements: RawElement[]) {
+  const forms = elements.filter((e) => ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role));
+  const actions = elements.filter((e) => e.role === 'button' || e.role === 'submit' || e.role === 'link' || e.role === 'a');
+  return {
+    elementCount: elements.length,
+    formFieldCount: forms.length,
+    actionCount: actions.length,
+    fields: forms.slice(0, 50).map((e) => ({ label: e.label, role: e.role, expectedData: e.expectedData })),
+    actions: actions.slice(0, 50).map((e) => ({ label: e.label, role: e.role })),
+  };
+}
+
+export function buildCrawlArtifactEnvelope(input: {
+  tenantId: string;
+  projectId: string;
+  jobId?: string;
+  url: string;
+  canonicalUrl: string;
+  screenId?: string;
+  screenshotPath?: string;
+  domPath?: string;
+  domContent?: string;
+  markdown?: string;
+  fitMarkdown?: string;
+  accessibilityElements?: RawElement[];
+  structuredJson?: unknown;
+  evidence?: CrawlEvidenceFact[];
+}): CrawlArtifactEnvelope {
+  const accessibilityTree = input.accessibilityElements ? JSON.stringify(input.accessibilityElements) : undefined;
+  return {
+    tenantId: input.tenantId,
+    projectId: input.projectId,
+    jobId: input.jobId,
+    url: input.url,
+    canonicalUrl: input.canonicalUrl,
+    screenId: input.screenId,
+    status: 'ok',
+    outputs: {
+      screenshot: input.screenshotPath,
+      html: input.domPath,
+      markdown: input.markdown,
+      fitMarkdown: input.fitMarkdown,
+      accessibilityTree,
+      structuredJson: input.structuredJson,
+    },
+    fingerprints: {
+      textHash: input.markdown ? sha256(input.markdown) : undefined,
+      domHash: input.domContent ? sha256(input.domContent) : undefined,
+      accessibilityHash: accessibilityTree ? sha256(accessibilityTree) : undefined,
+      urlCanonicalHash: sha256(input.canonicalUrl),
+    },
+    evidence: input.evidence ?? [],
+  };
+}

package/src/extractors/branding-extractor.ts

@@ -0,0 +1,159 @@
+import type { Page } from 'playwright';
+
+export interface BrandingColors {
+  primary?: string;
+  accent?: string;
+  background?: string;
+  textPrimary?: string;
+  link?: string;
+}
+
+export interface BrandingFont {
+  family: string;
+  role: 'heading' | 'body';
+}
+
+export interface BrandingImages {
+  logo?: string;
+  favicon?: string;
+  ogImage?: string;
+}
+
+export interface BrandingPersonality {
+  tone?: string;
+  energy?: 'high' | 'medium' | 'low';
+  targetAudience?: string;
+}
+
+export interface BrandingResult {
+  colorScheme: 'light' | 'dark';
+  colors: BrandingColors;
+  fonts: BrandingFont[];
+  images: BrandingImages;
+  personality?: BrandingPersonality;
+  designSystem?: { framework?: string; componentLibrary?: string };
+  metadata?: { title?: string; description?: string; themeColor?: string; language?: string };
+  confidence: { colors: number; fonts: number; images: number; overall: number };
+}
+
+export async function extractBranding(page: Page): Promise<BrandingResult> {
+  try {
+    const result = await page.evaluate(() => {
+      const bodyStyle = window.getComputedStyle(document.body);
+      const isDark = window.matchMedia('(prefers-color-scheme: dark)').matches ||
+        document.documentElement.getAttribute('data-theme') === 'dark' ||
+        document.body.classList.contains('dark');
+
+      // Extract colors from computed styles
+      const bg = bodyStyle.backgroundColor;
+      const text = bodyStyle.color;
+
+      // Find primary button color
+      const primaryBtn = document.querySelector(
+        'button[class*="primary"], .btn-primary, [class*="btn-primary"], [class*="button-primary"]'
+      );
+      const primaryColor = primaryBtn ? window.getComputedStyle(primaryBtn).backgroundColor : undefined;
+
+      // Find link color
+      const firstLink = document.querySelector('a');
+      const linkColor = firstLink ? window.getComputedStyle(firstLink).color : undefined;
+
+      // Extract font families
+      const headingEl = document.querySelector('h1, h2');
+      const bodyEl = document.body;
+      const headingFont = headingEl ? window.getComputedStyle(headingEl).fontFamily : bodyStyle.fontFamily;
+      const bodyFont = bodyStyle.fontFamily;
+
+      // Extract images
+      const logoEl = document.querySelector(
+        'header img, img[alt*="logo" i], img[class*="logo" i], [class*="logo"] img, [id*="logo"] img'
+      ) as HTMLImageElement | null;
+      const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]') as HTMLLinkElement | null;
+      const ogImageEl = document.querySelector('meta[property="og:image"]') as HTMLMetaElement | null;
+
+      // Extract metadata
+      const titleEl = document.querySelector('title');
+      const descEl = document.querySelector('meta[name="description"]') as HTMLMetaElement | null;
+      const themeColorEl = document.querySelector('meta[name="theme-color"]') as HTMLMetaElement | null;
+
+      // Detect design framework
+      const hasBootstrap = !!document.querySelector('[class*="btn btn-"], .container-fluid, .navbar-expand');
+      const hasTailwind = !!document.querySelector('[class*="flex "], [class*="text-"], [class*="bg-"]');
+      const hasChakra = !!(document as any).__chakra_factory__ || !!document.querySelector('[class*="chakra-"]');
+      const hasMUI = !!document.querySelector('.MuiButton-root, .MuiTypography-root');
+
+      const framework = hasBootstrap ? 'Bootstrap' : hasTailwind ? 'Tailwind CSS' : hasChakra ? 'Chakra UI' : hasMUI ? 'Material UI' : null;
+
+      return {
+        isDark,
+        bg,
+        text,
+        primaryColor,
+        linkColor,
+        headingFont,
+        bodyFont,
+        logo: logoEl?.src || logoEl?.getAttribute('src') || undefined,
+        favicon: faviconEl?.href || undefined,
+        ogImage: ogImageEl?.content || undefined,
+        title: titleEl?.textContent || undefined,
+        description: descEl?.content || undefined,
+        themeColor: themeColorEl?.content || undefined,
+        language: document.documentElement.lang || undefined,
+        framework,
+      };
+    });
+
+    const fonts: BrandingFont[] = [];
+    if (result.headingFont && result.headingFont !== result.bodyFont) {
+      fonts.push({ family: result.headingFont.split(',')[0].replace(/['"]/g, '').trim(), role: 'heading' });
+    }
+    if (result.bodyFont) {
+      fonts.push({ family: result.bodyFont.split(',')[0].replace(/['"]/g, '').trim(), role: 'body' });
+    }
+
+    const colorConfidence = (result.bg || result.text || result.primaryColor) ? 0.8 : 0.3;
+    const fontConfidence = fonts.length > 0 ? 0.7 : 0.2;
+    const imageConfidence = (result.logo || result.ogImage) ? 0.9 : 0.2;
+
+    return {
+      colorScheme: result.isDark ? 'dark' : 'light',
+      colors: {
+        background: result.bg || undefined,
+        textPrimary: result.text || undefined,
+        primary: result.primaryColor || undefined,
+        link: result.linkColor || undefined,
+      },
+      fonts,
+      images: {
+        logo: result.logo,
+        favicon: result.favicon,
+        ogImage: result.ogImage,
+      },
+      designSystem: {
+        framework: result.framework || undefined,
+        componentLibrary: undefined,
+      },
+      metadata: {
+        title: result.title,
+        description: result.description,
+        themeColor: result.themeColor,
+        language: result.language,
+      },
+      confidence: {
+        colors: colorConfidence,
+        fonts: fontConfidence,
+        images: imageConfidence,
+        overall: (colorConfidence + fontConfidence + imageConfidence) / 3,
+      },
+    };
+  } catch (err) {
+    console.warn('[branding-extractor] Extraction failed:', err);
+    return {
+      colorScheme: 'light',
+      colors: {},
+      fonts: [],
+      images: {},
+      confidence: { colors: 0, fonts: 0, images: 0, overall: 0 },
+    };
+  }
+}

package/src/extractors/summary-extractor.ts

@@ -0,0 +1,49 @@
+export interface SummaryResult {
+  summary: string;
+  wordCount: number;
+  method: 'llm' | 'truncated';
+  model?: string;
+}
+
+export async function extractSummary(
+  markdown: string,
+  options?: { model?: string; maxTokens?: number },
+): Promise<SummaryResult> {
+  // Truncate to 8000 chars to stay within token budget
+  const input = markdown.slice(0, 8000);
+
+  try {
+    // Dynamic import to avoid hard dependency if agents package unavailable
+    const { ask } = await import('@taodigital/agents').catch(() => ({ ask: null as any }));
+
+    if (!ask) {
+      return makeTruncated(input);
+    }
+
+    const summary: string = await ask(
+      'Summarize the following web page content in 2-4 concise sentences. Focus on the main purpose, key features, and primary value proposition. Return only the summary text, no preamble.',
+      input,
+      options?.model ? { provider: 'anthropic', model: options.model } : undefined,
+      options?.maxTokens ?? 300,
+    );
+
+    return {
+      summary: summary.trim(),
+      wordCount: markdown.split(/\s+/).length,
+      method: 'llm',
+      model: options?.model,
+    };
+  } catch (err) {
+    console.warn('[summary-extractor] LLM call failed, using truncated fallback:', err);
+    return makeTruncated(input);
+  }
+}
+
+function makeTruncated(input: string): SummaryResult {
+  const excerpt = input.slice(0, 500).trim();
+  return {
+    summary: excerpt + (input.length > 500 ? '...' : ''),
+    wordCount: excerpt.split(/\s+/).length,
+    method: 'truncated',
+  };
+}

package/src/failure-analysis-agent.ts

@@ -0,0 +1,51 @@
+import { readFileSync } from 'node:fs';
+import { askTrackedWithImages, askTracked, deterministicSeed } from '@taodigital/agents';
+import type { LLMConfig } from '@taodigital/agents';
+import { AppBrain } from '@taodigital/app-brain';
+
+export interface FailureContext {
+  tenantId?: string;
+  error: string;
+  stepDescription?: string;
+  consoleLogs?: Array<{ type: string; text: string }>;
+  networkErrors?: Array<{ url: string; status: number; method: string }>;
+  screenshotPath?: string;
+}
+
+export async function analyzeFailure(ctx: FailureContext): Promise<string | null> {
+  try {
+    let llmCfg: LLMConfig | undefined;
+    if (ctx.tenantId) {
+      llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, 'testRunner').catch(() => undefined) as LLMConfig | undefined ?? undefined;
+    }
+    if (!llmCfg) return null;
+
+    const consoleSection = ctx.consoleLogs?.length
+      ? `\n\nBrowser console (last ${ctx.consoleLogs.length} entries):\n${ctx.consoleLogs.map(l => `[${l.type}] ${l.text}`).join('\n')}`
+      : '';
+
+    const networkSection = ctx.networkErrors?.length
+      ? `\n\nFailed network requests:\n${ctx.networkErrors.map(n => `${n.method} ${n.url} → ${n.status}`).join('\n')}`
+      : '';
+
+    const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific — name the failing element, API endpoint, or error type. Do not suggest fixes.
+
+Test step: ${ctx.stepDescription ?? 'unknown'}
+Error: ${ctx.error}${consoleSection}${networkSection}`;
+
+    if (ctx.screenshotPath) {
+      try {
+        const img = readFileSync(ctx.screenshotPath);
+        const imgSeed = deterministicSeed(`failure-img:${ctx.tenantId ?? ''}:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+        const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg, { temperature: 0, seed: imgSeed });
+        return result.text || null;
+      } catch { /* fall through to text-only */ }
+    }
+
+    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+    const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });
+    return result.text || null;
+  } catch {
+    return null;
+  }
+}

package/src/feature-flag-detector.ts

@@ -0,0 +1,97 @@
+/**
+ * Feature flag + A/B test detector.
+ *
+ * Detects popular feature flag / experiment SDKs loaded on the page:
+ * LaunchDarkly, Optimizely, Split.io, Unleash, GrowthBook, Statsig, Flagsmith, Amplitude Experiment.
+ *
+ * Also detects common A/B test patterns: data-testid attributes, gtm.js experiment layers,
+ * Optimizely variation IDs, VWO, etc.
+ *
+ * Why: ZeTa needs to know when a page shows different content to different users
+ * so test generation can account for variants and not generate flaky tests.
+ */
+
+import type { Page } from 'playwright';
+
+export interface FeatureFlagInfo {
+  sdk: string;
+  confidence: 'high' | 'medium' | 'low';
+  details?: string;
+}
+
+export interface AbTestInfo {
+  tool: string;
+  experiments?: string[];
+}
+
+export interface FeatureFlagDetectionResult {
+  url: string;
+  flagSdks: FeatureFlagInfo[];
+  abTools: AbTestInfo[];
+  hasAnyFlags: boolean;
+  detectedAt: string;
+}
+
+// SDK detectors — checked against window object + script URLs
+const FLAG_SDK_CHECKS: Array<{
+  sdk: string;
+  windowKeys?: string[];
+  scriptPattern?: RegExp;
+}> = [
+  { sdk: 'LaunchDarkly', windowKeys: ['LDClient', 'LaunchDarkly', 'ldClient'], scriptPattern: /launchdarkly/i },
+  { sdk: 'Optimizely', windowKeys: ['optimizely', 'optimizelySdk'], scriptPattern: /optimizely/i },
+  { sdk: 'Split.io', windowKeys: ['splitio', 'SplitFactory'], scriptPattern: /split\.io/i },
+  { sdk: 'GrowthBook', windowKeys: ['growthbook', 'GrowthBook'], scriptPattern: /growthbook/i },
+  { sdk: 'Statsig', windowKeys: ['statsig', 'StatsigClient'], scriptPattern: /statsig/i },
+  { sdk: 'Flagsmith', windowKeys: ['flagsmith'], scriptPattern: /flagsmith/i },
+  { sdk: 'Unleash', windowKeys: ['unleash', 'UnleashClient'], scriptPattern: /unleash/i },
+  { sdk: 'VWO', windowKeys: ['vwo', '_vwo'], scriptPattern: /vwo\.com/i },
+  { sdk: 'Google Optimize', windowKeys: ['gaData', 'google_optimize'], scriptPattern: /googleoptimize/i },
+  { sdk: 'AB Tasty', windowKeys: ['ABTasty'], scriptPattern: /abtasty/i },
+];
+
+export async function detectFeatureFlags(page: Page, url: string): Promise<FeatureFlagDetectionResult> {
+  const detectedAt = new Date().toISOString();
+
+  const result = await page.evaluate((checks) => {
+    const flagSdks: any[] = [];
+    const scripts = Array.from(document.querySelectorAll('script[src]')).map((s: any) => s.src ?? '');
+
+    for (const check of checks) {
+      let found = false;
+      let confidence = 'low';
+
+      // Check window keys
+      if (check.windowKeys?.some((k: string) => k in (window as any))) {
+        found = true;
+        confidence = 'high';
+      }
+      // Check script URLs
+      if (!found && check.scriptPattern) {
+        const re = new RegExp(check.scriptPattern.source, 'i');
+        const matchingSrc = scripts.find(s => re.test(s));
+        if (matchingSrc) { found = true; confidence = 'medium'; }
+      }
+      if (found) flagSdks.push({ sdk: check.sdk, confidence, details: undefined });
+    }
+
+    // A/B tools
+    const abTools: any[] = [];
+    if ('_vwo_code' in window || '_vis_opt_queue' in (window as any)) abTools.push({ tool: 'VWO', experiments: [] });
+    if ('dataLayer' in window) {
+      const dl = (window as any).dataLayer ?? [];
+      const expEvents = dl.filter((e: any) => e?.event?.includes('experiment') || e?.experimentId);
+      if (expEvents.length > 0) abTools.push({ tool: 'Google Tag Manager Experiments', experiments: expEvents.slice(0, 5).map((e: any) => e.experimentId ?? '') });
+    }
+
+    return { flagSdks, abTools };
+  }, FLAG_SDK_CHECKS as any).catch(() => ({ flagSdks: [], abTools: [] }));
+
+  return {
+    url,
+    flagSdks: result.flagSdks,
+    abTools: result.abTools,
+    hasAnyFlags: result.flagSdks.length > 0 || result.abTools.length > 0,
+    detectedAt,
+  };
+}