outcome-cli 1.0.0

package/src/modules/omnibridge/core/semantic-normalizer.ts

@@ -0,0 +1,702 @@
+/**
+ * Semantic Normalizer
+ *
+ * Converts raw DOM to Intent-Tagged JSON, stripping 90% of non-functional code.
+ * This is Layer 1 of OmniBridge: DOM → Intent
+ *
+ * Requirements: 1.1, 1.2, 1.3, 1.5, 1.6
+ */
+
+import * as cheerio from 'cheerio';
+import type {
+  IntentDocument,
+  IntentElement,
+  IntentId,
+  IntentForm,
+  IntentFormField,
+  IntentNavigation,
+} from './types.js';
+
+/**
+ * Elements to strip from HTML (noise reduction)
+ */
+const NOISE_SELECTORS = [
+  'script',
+  'style',
+  'noscript',
+  'iframe',
+  'svg',
+  'link[rel="stylesheet"]',
+  'meta',
+  'head > *:not(title)',
+  '[data-tracking]',
+  '[data-analytics]',
+  '[data-gtm]',
+  '.tracking',
+  '.analytics',
+  '#google_ads_iframe',
+  '[id*="google_ads"]',
+  '[class*="ad-"]',
+  '[class*="advertisement"]',
+  '[data-ad]',
+  'ins.adsbygoogle',
+];
+
+/**
+ * ARIA role to Intent category mapping
+ */
+const ARIA_TO_CATEGORY: Record<string, IntentId['category']> = {
+  button: 'ACTION',
+  link: 'NAV',
+  textbox: 'INPUT',
+  searchbox: 'INPUT',
+  checkbox: 'INPUT',
+  radio: 'INPUT',
+  combobox: 'INPUT',
+  listbox: 'INPUT',
+  slider: 'INPUT',
+  spinbutton: 'INPUT',
+  switch: 'INPUT',
+  navigation: 'NAV',
+  menu: 'NAV',
+  menuitem: 'NAV',
+  tab: 'NAV',
+  tablist: 'NAV',
+  heading: 'DISPLAY',
+  img: 'DISPLAY',
+  figure: 'DISPLAY',
+  table: 'DISPLAY',
+  grid: 'DISPLAY',
+  list: 'DISPLAY',
+  listitem: 'DISPLAY',
+  article: 'DISPLAY',
+  region: 'DISPLAY',
+  main: 'DISPLAY',
+  banner: 'DISPLAY',
+  contentinfo: 'DISPLAY',
+  complementary: 'DISPLAY',
+  form: 'INPUT',
+  search: 'INPUT',
+  alert: 'DISPLAY',
+  alertdialog: 'ACTION',
+  dialog: 'ACTION',
+  progressbar: 'DISPLAY',
+  status: 'DISPLAY',
+  tooltip: 'DISPLAY',
+};
+
+
+/**
+ * Tag to Intent category mapping (fallback when no ARIA role)
+ */
+const TAG_TO_CATEGORY: Record<string, IntentId['category']> = {
+  button: 'ACTION',
+  a: 'NAV',
+  input: 'INPUT',
+  textarea: 'INPUT',
+  select: 'INPUT',
+  form: 'INPUT',
+  nav: 'NAV',
+  header: 'DISPLAY',
+  footer: 'DISPLAY',
+  main: 'DISPLAY',
+  article: 'DISPLAY',
+  section: 'DISPLAY',
+  aside: 'DISPLAY',
+  h1: 'DISPLAY',
+  h2: 'DISPLAY',
+  h3: 'DISPLAY',
+  h4: 'DISPLAY',
+  h5: 'DISPLAY',
+  h6: 'DISPLAY',
+  p: 'DISPLAY',
+  span: 'DISPLAY',
+  div: 'DISPLAY',
+  img: 'DISPLAY',
+  table: 'DISPLAY',
+  ul: 'DISPLAY',
+  ol: 'DISPLAY',
+  li: 'DISPLAY',
+  label: 'DISPLAY',
+};
+
+/**
+ * Semantic Normalizer class
+ */
+export class SemanticNormalizer {
+  /**
+   * Strip noise from HTML (CSS, scripts, tracking, metadata)
+   */
+  stripNoise(html: string): string {
+    const $ = cheerio.load(html);
+
+    // Remove all noise elements
+    for (const selector of NOISE_SELECTORS) {
+      $(selector).remove();
+    }
+
+    // Remove inline styles
+    $('[style]').removeAttr('style');
+
+    // Remove class attributes (CSS-only, not semantic)
+    // Keep classes that might have semantic meaning
+    $('[class]').each((_, el) => {
+      const $el = $(el);
+      const classes = $el.attr('class') || '';
+      // Keep only semantic-looking classes
+      const semanticClasses = classes
+        .split(/\s+/)
+        .filter(
+          (c) =>
+            c.includes('btn') ||
+            c.includes('button') ||
+            c.includes('nav') ||
+            c.includes('menu') ||
+            c.includes('form') ||
+            c.includes('input') ||
+            c.includes('search') ||
+            c.includes('submit') ||
+            c.includes('login') ||
+            c.includes('signup') ||
+            c.includes('cart') ||
+            c.includes('checkout')
+        );
+      if (semanticClasses.length > 0) {
+        $el.attr('class', semanticClasses.join(' '));
+      } else {
+        $el.removeAttr('class');
+      }
+    });
+
+    // Remove data attributes except semantic ones
+    $('*').each((_, el) => {
+      const $el = $(el);
+      const attrs = $el.attr() || {};
+      for (const attr of Object.keys(attrs)) {
+        if (
+          attr.startsWith('data-') &&
+          !attr.includes('action') &&
+          !attr.includes('target') &&
+          !attr.includes('value') &&
+          !attr.includes('id') &&
+          !attr.includes('name')
+        ) {
+          $el.removeAttr(attr);
+        }
+      }
+    });
+
+    // Remove comments
+    $('*')
+      .contents()
+      .filter(function () {
+        return this.type === 'comment';
+      })
+      .remove();
+
+    // Remove empty elements (except semantic ones)
+    $('div:empty, span:empty, p:empty').remove();
+
+    return $.html();
+  }
+
+
+  /**
+   * Assign an Intent ID to an element based on its semantic meaning.
+   * Prioritizes ARIA roles when present (Requirement 1.5)
+   */
+  assignIntentId(
+    tagName: string,
+    ariaRole: string | undefined,
+    attributes: Record<string, string>,
+    textContent: string,
+    contextHints: string[]
+  ): IntentId {
+    // Priority 1: ARIA role (Requirement 1.5)
+    if (ariaRole && ARIA_TO_CATEGORY[ariaRole]) {
+      const purpose = this.inferPurpose(ariaRole, attributes, textContent, contextHints);
+      return {
+        category: ARIA_TO_CATEGORY[ariaRole],
+        purpose,
+        confidence: 0.95, // High confidence when ARIA role is present
+      };
+    }
+
+    // Priority 2: Tag-based inference
+    const category = TAG_TO_CATEGORY[tagName.toLowerCase()] || 'DISPLAY';
+    const purpose = this.inferPurpose(tagName, attributes, textContent, contextHints);
+
+    // Calculate confidence based on available signals
+    let confidence = 0.7; // Base confidence
+    if (attributes.id || attributes.name) confidence += 0.1;
+    if (textContent.trim().length > 0) confidence += 0.1;
+    if (contextHints.length > 0) confidence += 0.05;
+
+    return {
+      category,
+      purpose: purpose.toUpperCase().replace(/\s+/g, '_'),
+      confidence: Math.min(confidence, 1),
+    };
+  }
+
+  /**
+   * Infer the purpose of an element from available signals
+   */
+  private inferPurpose(
+    roleOrTag: string,
+    attributes: Record<string, string>,
+    textContent: string,
+    contextHints: string[]
+  ): string {
+    const text = textContent.toLowerCase().trim();
+    const id = (attributes.id || '').toLowerCase();
+    const name = (attributes.name || '').toLowerCase();
+    const type = (attributes.type || '').toLowerCase();
+    const placeholder = (attributes.placeholder || '').toLowerCase();
+    const ariaLabel = (attributes['aria-label'] || '').toLowerCase();
+
+    // Combine all signals
+    const signals = [text, id, name, type, placeholder, ariaLabel, ...contextHints]
+      .join(' ')
+      .toLowerCase();
+
+    // Common purpose patterns
+    if (signals.includes('login') || signals.includes('sign in') || signals.includes('signin')) {
+      return 'LOGIN';
+    }
+    if (signals.includes('signup') || signals.includes('sign up') || signals.includes('register')) {
+      return 'SIGNUP';
+    }
+    if (signals.includes('search')) {
+      return 'SEARCH';
+    }
+    if (signals.includes('submit') || signals.includes('send')) {
+      return 'SUBMIT';
+    }
+    if (signals.includes('cancel') || signals.includes('close')) {
+      return 'CANCEL';
+    }
+    if (signals.includes('delete') || signals.includes('remove')) {
+      return 'DELETE';
+    }
+    if (signals.includes('edit') || signals.includes('modify')) {
+      return 'EDIT';
+    }
+    if (signals.includes('save')) {
+      return 'SAVE';
+    }
+    if (signals.includes('cart') || signals.includes('basket')) {
+      return 'CART';
+    }
+    if (signals.includes('checkout') || signals.includes('purchase') || signals.includes('buy')) {
+      return 'PURCHASE';
+    }
+    if (signals.includes('pay') || signals.includes('payment')) {
+      return 'PAYMENT';
+    }
+    if (signals.includes('email') || type === 'email') {
+      return 'EMAIL_INPUT';
+    }
+    if (signals.includes('password') || type === 'password') {
+      return 'PASSWORD_INPUT';
+    }
+    if (signals.includes('phone') || type === 'tel') {
+      return 'PHONE_INPUT';
+    }
+    if (signals.includes('name') && !signals.includes('username')) {
+      return 'NAME_INPUT';
+    }
+    if (signals.includes('username')) {
+      return 'USERNAME_INPUT';
+    }
+    if (signals.includes('address')) {
+      return 'ADDRESS_INPUT';
+    }
+    if (signals.includes('next') || signals.includes('continue')) {
+      return 'NEXT';
+    }
+    if (signals.includes('back') || signals.includes('previous')) {
+      return 'BACK';
+    }
+    if (signals.includes('menu') || signals.includes('navigation')) {
+      return 'NAVIGATION';
+    }
+    if (signals.includes('home')) {
+      return 'HOME';
+    }
+    if (signals.includes('profile') || signals.includes('account')) {
+      return 'PROFILE';
+    }
+    if (signals.includes('settings') || signals.includes('preferences')) {
+      return 'SETTINGS';
+    }
+    if (signals.includes('help') || signals.includes('support')) {
+      return 'HELP';
+    }
+    if (signals.includes('download')) {
+      return 'DOWNLOAD';
+    }
+    if (signals.includes('upload')) {
+      return 'UPLOAD';
+    }
+
+    // Fallback to role/tag based purpose
+    return roleOrTag.toUpperCase();
+  }
+
+
+  /**
+   * Extract context hints from surrounding elements
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private extractContextHints($: cheerio.CheerioAPI, el: any): string[] {
+    const hints: string[] = [];
+    const $el = $(el);
+
+    // Get parent text
+    const parentText = $el.parent().clone().children().remove().end().text().trim();
+    if (parentText) {
+      hints.push(parentText.slice(0, 100)); // Limit length
+    }
+
+    // Get sibling labels
+    const prevSibling = $el.prev('label').text().trim();
+    if (prevSibling) {
+      hints.push(prevSibling);
+    }
+
+    // Get associated label by 'for' attribute
+    const id = $el.attr('id');
+    if (id) {
+      const labelText = $(`label[for="${id}"]`).text().trim();
+      if (labelText) {
+        hints.push(labelText);
+      }
+    }
+
+    // Get aria-describedby text
+    const describedBy = $el.attr('aria-describedby');
+    if (describedBy) {
+      const descText = $(`#${describedBy}`).text().trim();
+      if (descText) {
+        hints.push(descText);
+      }
+    }
+
+    return hints.filter((h) => h.length > 0);
+  }
+
+  /**
+   * Extract forms from the document (Requirement 1.6)
+   */
+  private extractForms($: cheerio.CheerioAPI): IntentForm[] {
+    const forms: IntentForm[] = [];
+
+    $('form').each((_, formEl) => {
+      const $form = $(formEl);
+      const fields: IntentFormField[] = [];
+
+      // Extract all form fields
+      $form.find('input, textarea, select').each((_, fieldEl) => {
+        const $field = $(fieldEl);
+        const tagName = fieldEl.tagName.toLowerCase();
+        const type = $field.attr('type') || (tagName === 'textarea' ? 'textarea' : 'text');
+        const name = $field.attr('name') || '';
+        const id = $field.attr('id') || '';
+        const required = $field.attr('required') !== undefined || $field.attr('aria-required') === 'true';
+        const placeholder = $field.attr('placeholder') || '';
+
+        // Find label
+        let label = '';
+        if (id) {
+          label = $(`label[for="${id}"]`).text().trim();
+        }
+        if (!label) {
+          label = $field.closest('label').text().trim();
+        }
+        if (!label) {
+          label = placeholder || name || type;
+        }
+
+        // Extract validation rules
+        const validationRules: string[] = [];
+        if ($field.attr('pattern')) {
+          validationRules.push(`pattern:${$field.attr('pattern')}`);
+        }
+        if ($field.attr('minlength')) {
+          validationRules.push(`minlength:${$field.attr('minlength')}`);
+        }
+        if ($field.attr('maxlength')) {
+          validationRules.push(`maxlength:${$field.attr('maxlength')}`);
+        }
+        if ($field.attr('min')) {
+          validationRules.push(`min:${$field.attr('min')}`);
+        }
+        if ($field.attr('max')) {
+          validationRules.push(`max:${$field.attr('max')}`);
+        }
+
+        const intentId = this.assignIntentId(
+          tagName,
+          $field.attr('role'),
+          {
+            id,
+            name,
+            type,
+            placeholder,
+            'aria-label': $field.attr('aria-label') || '',
+          },
+          label,
+          []
+        );
+
+        fields.push({
+          intentId: `${intentId.category}_ID:${intentId.purpose}`,
+          name,
+          type,
+          label,
+          required,
+          validationRules: validationRules.length > 0 ? validationRules : undefined,
+          placeholder: placeholder || undefined,
+        });
+      });
+
+      // Find submit button
+      const $submitBtn = $form.find('button[type="submit"], input[type="submit"]').first();
+      const submitButtonId = $submitBtn.length > 0 ? 'ACTION_ID:SUBMIT' : undefined;
+
+      const formIntentId = this.assignIntentId(
+        'form',
+        $form.attr('role'),
+        {
+          id: $form.attr('id') || '',
+          name: $form.attr('name') || '',
+          action: $form.attr('action') || '',
+        },
+        '',
+        []
+      );
+
+      forms.push({
+        intentId: `${formIntentId.category}_ID:${formIntentId.purpose}`,
+        action: $form.attr('action') || undefined,
+        method: ($form.attr('method')?.toUpperCase() as 'GET' | 'POST') || 'POST',
+        fields,
+        submitButtonId,
+      });
+    });
+
+    return forms;
+  }
+
+
+  /**
+   * Extract navigation structure from the document
+   */
+  private extractNavigation($: cheerio.CheerioAPI): IntentNavigation {
+    const primaryLinks: IntentNavigation['primaryLinks'] = [];
+    const breadcrumbs: IntentNavigation['breadcrumbs'] = [];
+
+    // Extract primary navigation links
+    $('nav a, header a, [role="navigation"] a').each((_, el) => {
+      const $el = $(el);
+      const href = $el.attr('href');
+      const label = $el.text().trim();
+
+      if (href && label && !href.startsWith('#') && !href.startsWith('javascript:')) {
+        const intentId = this.assignIntentId(
+          'a',
+          $el.attr('role'),
+          { href, id: $el.attr('id') || '' },
+          label,
+          []
+        );
+
+        primaryLinks.push({
+          intentId: `${intentId.category}_ID:${intentId.purpose}`,
+          label,
+          href,
+        });
+      }
+    });
+
+    // Extract breadcrumbs
+    $('[aria-label="breadcrumb"] a, .breadcrumb a, nav[aria-label*="breadcrumb"] a').each((_, el) => {
+      const $el = $(el);
+      breadcrumbs.push({
+        label: $el.text().trim(),
+        href: $el.attr('href'),
+      });
+    });
+
+    // Add current page to breadcrumbs if present
+    $('[aria-label="breadcrumb"] [aria-current], .breadcrumb [aria-current]').each((_, el) => {
+      breadcrumbs.push({
+        label: $(el).text().trim(),
+      });
+    });
+
+    return {
+      primaryLinks,
+      breadcrumbs: breadcrumbs.length > 0 ? breadcrumbs : undefined,
+    };
+  }
+
+  /**
+   * Extract semantic elements from the document
+   */
+  private extractElements($: cheerio.CheerioAPI): IntentElement[] {
+    const elements: IntentElement[] = [];
+    const processedIds = new Set<string>();
+
+    // Selectors for interactive/semantic elements
+    const selectors = [
+      'button',
+      'a[href]',
+      'input',
+      'textarea',
+      'select',
+      '[role]',
+      '[onclick]',
+      '[data-action]',
+      'h1, h2, h3, h4, h5, h6',
+      'label',
+      'nav',
+      'main',
+      'article',
+      'section[aria-label]',
+      'aside',
+      'header',
+      'footer',
+    ];
+
+    $(selectors.join(', ')).each((_, el) => {
+      const $el = $(el);
+      // Type guard for element nodes
+      if (!('tagName' in el)) {
+        return;
+      }
+      const tagName = (el.tagName as string).toLowerCase();
+
+      // Skip hidden elements
+      if ($el.attr('hidden') !== undefined || $el.attr('aria-hidden') === 'true') {
+        return;
+      }
+
+      // Skip elements inside forms (handled separately)
+      if (tagName !== 'form' && $el.closest('form').length > 0 && ['input', 'textarea', 'select'].includes(tagName)) {
+        return;
+      }
+
+      const id = $el.attr('id') || '';
+      const uniqueKey = id || `${tagName}-${$el.text().trim().slice(0, 50)}`;
+
+      // Skip duplicates
+      if (processedIds.has(uniqueKey)) {
+        return;
+      }
+      processedIds.add(uniqueKey);
+
+      const ariaRole = $el.attr('role');
+      const textContent = $el.text().trim().slice(0, 200); // Limit text length
+      const contextHints = this.extractContextHints($, el);
+
+      const attributes: Record<string, string> = {};
+      const attrNames = ['id', 'name', 'type', 'href', 'placeholder', 'aria-label', 'title', 'value'];
+      for (const attr of attrNames) {
+        const val = $el.attr(attr);
+        if (val) {
+          attributes[attr] = val;
+        }
+      }
+
+      const intentId = this.assignIntentId(tagName, ariaRole, attributes, textContent, contextHints);
+
+      // Determine role
+      let role: IntentElement['role'] = 'display';
+      if (intentId.category === 'ACTION') role = 'action';
+      else if (intentId.category === 'INPUT') role = 'input';
+      else if (intentId.category === 'NAV') role = 'navigation';
+
+      elements.push({
+        intentId: `${intentId.category}_ID:${intentId.purpose}`,
+        role,
+        label: textContent || attributes['aria-label'] || attributes.placeholder || tagName,
+        ariaRole,
+        contextHints,
+        tagName,
+      });
+    });
+
+    return elements;
+  }
+
+
+  /**
+   * Count tokens in a string (approximate)
+   * Uses a simple word-based approximation
+   */
+  countTokens(text: string): number {
+    // Approximate token count: ~4 characters per token on average
+    return Math.ceil(text.length / 4);
+  }
+
+  /**
+   * Normalize HTML to Intent Document
+   * Main entry point for the Semantic Normalizer
+   *
+   * Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 1.6
+   */
+  normalize(html: string, sourceUrl: string = ''): IntentDocument {
+    const startTime = performance.now();
+
+    // Count original tokens
+    const originalTokens = this.countTokens(html);
+
+    // Strip noise (Requirement 1.2)
+    const cleanedHtml = this.stripNoise(html);
+
+    // Parse cleaned HTML
+    const $ = cheerio.load(cleanedHtml);
+
+    // Extract semantic elements (Requirement 1.1, 1.3, 1.5)
+    const elements = this.extractElements($);
+
+    // Extract forms (Requirement 1.6)
+    const forms = this.extractForms($);
+
+    // Extract navigation
+    const navigation = this.extractNavigation($);
+
+    // Build the Intent Document
+    const document: IntentDocument = {
+      elements,
+      forms,
+      navigation,
+      buildTimeMs: 0, // Will be set below
+      tokenReduction: 0, // Will be set below
+      sourceUrl,
+      createdAt: Date.now(),
+    };
+
+    // Calculate metrics
+    const resultJson = JSON.stringify(document);
+    const resultTokens = this.countTokens(resultJson);
+    const buildTimeMs = performance.now() - startTime;
+
+    // Calculate token reduction percentage
+    const tokenReduction = originalTokens > 0 ? ((originalTokens - resultTokens) / originalTokens) * 100 : 0;
+
+    document.buildTimeMs = Math.round(buildTimeMs * 100) / 100;
+    document.tokenReduction = Math.round(tokenReduction * 100) / 100;
+
+    return document;
+  }
+}
+
+/**
+ * Create a new Semantic Normalizer instance
+ */
+export function createSemanticNormalizer(): SemanticNormalizer {
+  return new SemanticNormalizer();
+}