@zibby/core 0.1.0

package/src/enrichment/base.js

@@ -0,0 +1,49 @@
+/**
+ * Base Enrichment Pipeline Component
+ * Each enricher adds specific data to events without affecting others
+ */
+export class EventEnricher {
+  constructor(config = {}) {
+    this.config = config;
+    this.enabled = config.enabled !== false; // Default enabled
+    this.priority = config.priority || 50; // Default priority
+  }
+
+  /**
+   * Get enricher name for logging
+   */
+  getName() {
+    throw new Error('EventEnricher.getName() must be implemented');
+  }
+
+  /**
+   * Check if this enricher can run (dependencies met, etc)
+   * @param {Object} context - { page, element, event, session }
+   * @returns {boolean}
+   */
+  canEnrich(_context) {
+    return this.enabled;
+  }
+
+  /**
+   * Enrich the event with additional data
+   * @param {Object} event - The MCP event to enrich
+   * @param {Object} context - { page, element, ref }
+   * @returns {Promise<Object>} - Enriched data to merge into event
+   */
+  async enrich(_event, _context) {
+    throw new Error('EventEnricher.enrich() must be implemented');
+  }
+
+  /**
+   * Handle errors gracefully (never break the pipeline)
+   * @param {Error} error
+   * @param {Object} event
+   */
+  handleError(error, event) {
+    console.warn(`[${this.getName()}] Enrichment failed for event ${event.type}:`, error.message);
+    return null; // Return null = skip this enrichment
+  }
+}
+
+export default EventEnricher;

package/src/enrichment/enrichers/accessibility-enricher.js

@@ -0,0 +1,197 @@
+/**
+ * Accessibility Enricher - Captures accessibility tree data and structural hash
+ */
+import { EventEnricher } from '../base.js';
+import crypto from 'crypto';
+
+export class AccessibilityEnricher extends EventEnricher {
+  getName() {
+    return 'AccessibilityEnricher';
+  }
+
+  getPriority() {
+    return 100; // Highest priority - most stable identifier
+  }
+
+  canEnrich(context) {
+    if (!this.enabled) return false;
+    if (!context.element) return false;
+    if (!context.event) return false;
+    
+    return ['click', 'fill', 'type', 'selectOption', 'hover'].includes(context.event.type);
+  }
+
+  async enrich(event, context) {
+    try {
+      const { page, element } = context;
+      
+      // Get accessibility snapshot of entire page
+      const snapshot = await page.accessibility.snapshot();
+      
+      // Find the target element in AX tree
+      const axNode = await this.findAxNode(element, snapshot);
+      if (!axNode) return null;
+      
+      // Get parent and siblings
+      const axContext = await this.getAxContext(axNode, snapshot);
+      
+      // Calculate structural hash (hash of AX subtree)
+      const axTreeHash = this.hashAxSubtree(axNode);
+      const axPathHash = this.hashAxPath(axContext.path);
+      
+      return {
+        accessibility: {
+          role: axNode.role,
+          name: axNode.name || '',
+          level: axContext.level,
+          parent: axContext.parent,
+          siblings: axContext.siblings,
+          axTreeHash,
+          axPathHash
+        }
+      };
+    } catch (error) {
+      return this.handleError(error, event);
+    }
+  }
+
+  /**
+   * Find AX node for given element
+   */
+  async findAxNode(element, snapshot) {
+    // Get element's aria attributes for matching
+    const ariaProps = await element.evaluate(el => ({
+      role: el.getAttribute('role') || el.tagName.toLowerCase(),
+      name: el.getAttribute('aria-label') || el.textContent?.trim() || '',
+      tagName: el.tagName.toLowerCase()
+    }));
+    
+    // Search AX tree for matching node
+    return this.searchAxTree(snapshot, ariaProps);
+  }
+
+  /**
+   * Recursively search AX tree
+   */
+  searchAxTree(node, props) {
+    if (!node) return null;
+    
+    // Match by role and name
+    if (node.role === props.role && 
+        (node.name || '').includes(props.name.substring(0, 20))) {
+      return node;
+    }
+    
+    // Search children
+    if (node.children) {
+      for (const child of node.children) {
+        const found = this.searchAxTree(child, props);
+        if (found) return found;
+      }
+    }
+    
+    return null;
+  }
+
+  /**
+   * Get context (parent, siblings, path)
+   */
+  getAxContext(node, snapshot) {
+    const context = {
+      level: 0,
+      parent: null,
+      siblings: [],
+      path: []
+    };
+    
+    // Walk up tree to find parent and siblings
+    const parent = this.findParent(node, snapshot);
+    if (parent) {
+      context.parent = { role: parent.role, name: parent.name };
+      context.siblings = (parent.children || [])
+        .filter(c => c !== node)
+        .map(c => ({ role: c.role, name: c.name }))
+        .slice(0, 3); // Limit siblings
+    }
+    
+    // Calculate level (depth in tree)
+    context.level = this.calculateLevel(node, snapshot);
+    
+    // Build path from root to node
+    context.path = this.buildPath(node, snapshot);
+    
+    return context;
+  }
+
+  findParent(node, tree, current = tree) {
+    if (!current || !current.children) return null;
+    
+    if (current.children.includes(node)) {
+      return current;
+    }
+    
+    for (const child of current.children) {
+      const found = this.findParent(node, tree, child);
+      if (found) return found;
+    }
+    
+    return null;
+  }
+
+  calculateLevel(node, tree, current = tree, level = 0) {
+    if (current === node) return level;
+    
+    if (current.children) {
+      for (const child of current.children) {
+        const found = this.calculateLevel(node, tree, child, level + 1);
+        if (found >= 0) return found;
+      }
+    }
+    
+    return -1;
+  }
+
+  buildPath(node, tree, current = tree, path = []) {
+    if (current === node) {
+      return [...path, { role: current.role, name: current.name }];
+    }
+    
+    if (current.children) {
+      for (const child of current.children) {
+        const found = this.buildPath(node, tree, child, [
+          ...path,
+          { role: current.role, name: current.name }
+        ]);
+        if (found) return found;
+      }
+    }
+    
+    return null;
+  }
+
+  /**
+   * Hash AX subtree for stability detection
+   */
+  hashAxSubtree(node) {
+    const structure = JSON.stringify({
+      role: node.role,
+      name: node.name,
+      children: (node.children || []).map(c => ({
+        role: c.role,
+        name: c.name
+      }))
+    });
+    
+    return crypto.createHash('md5').update(structure).digest('hex').substring(0, 12);
+  }
+
+  /**
+   * Hash AX path from root to node
+   */
+  hashAxPath(path) {
+    const pathStr = path.map(p => `${p.role}:${p.name}`).join('/');
+    return crypto.createHash('md5').update(pathStr).digest('hex').substring(0, 12);
+  }
+}
+
+export default AccessibilityEnricher;

package/src/enrichment/enrichers/dom-enricher.js

@@ -0,0 +1,171 @@
+/**
+ * DOM Enricher - Captures DOM path, XPath, and element attributes
+ * This file contains browser code executed via element.evaluate() where browser globals are available
+ */
+/* global document, window */
+import { EventEnricher } from '../base.js';
+
+export class DOMEnricher extends EventEnricher {
+  getName() {
+    return 'DOMEnricher';
+  }
+
+  getPriority() {
+    return 85; // High priority - structural data
+  }
+
+  canEnrich(context) {
+    if (!this.enabled) return false;
+    if (!context.element) return false;
+    if (!context.event) return false;
+    
+    return ['click', 'fill', 'type', 'selectOption', 'hover'].includes(context.event.type);
+  }
+
+  async enrich(event, context) {
+    try {
+      const { element } = context;
+      
+      // Get DOM path, XPath, and attributes
+      const domData = await element.evaluate(el => {
+        // Build CSS path
+        const getCssPath = (innerEl) => {
+          const path = [];
+          let current = innerEl;
+          
+          while (current && current !== document.body) {
+            let selector = current.tagName.toLowerCase();
+            
+            // Add nth-child if there are siblings of same type
+            const parent = current.parentElement;
+            if (parent) {
+              const siblings = Array.from(parent.children).filter(
+                child => child.tagName === current.tagName
+              );
+              if (siblings.length > 1) {
+                const index = siblings.indexOf(current) + 1;
+                selector += `:nth-child(${index})`;
+              }
+            }
+            
+            path.unshift(selector);
+            current = current.parentElement;
+          }
+          
+          return `body > ${path.join(' > ')}`;
+        };
+        
+        // Build XPath
+        const getXPath = (innerEl) => {
+          const path = [];
+          let current = innerEl;
+          
+          while (current && current !== document.body) {
+            let index = 1;
+            let sibling = current.previousSibling;
+            
+            while (sibling) {
+              if (sibling.nodeType === 1 && sibling.tagName === current.tagName) {
+                index++;
+              }
+              sibling = sibling.previousSibling;
+            }
+            
+            const tag = current.tagName.toLowerCase();
+            path.unshift(`${tag}[${index}]`);
+            current = current.parentElement;
+          }
+          
+          return `/html/body/${path.join('/')}`;
+        };
+        
+        // Get all attributes
+        const attributes = {};
+        for (const attr of el.attributes) {
+          attributes[attr.name] = attr.value;
+        }
+        
+        // Get computed style (only essential properties)
+        const computed = window.getComputedStyle(el);
+        const state = {
+          display: computed.display,
+          visibility: computed.visibility,
+          opacity: computed.opacity,
+          pointerEvents: computed.pointerEvents
+        };
+        
+        // Calculate depth
+        let depth = 0;
+        let parent = el.parentElement;
+        while (parent) {
+          depth++;
+          parent = parent.parentElement;
+        }
+        
+        return {
+          path: getCssPath(el),
+          xpath: getXPath(el),
+          depth,
+          parent: el.parentElement ? el.parentElement.tagName.toLowerCase() : null,
+          tagName: el.tagName.toLowerCase(),
+          attributes,
+          state: {
+            visible: computed.display !== 'none' && computed.visibility !== 'hidden',
+            enabled: !el.disabled,
+            focused: document.activeElement === el,
+            ...state
+          }
+        };
+      });
+      
+      return {
+        dom: {
+          path: domData.path,
+          xpath: domData.xpath,
+          depth: domData.depth,
+          parent: domData.parent,
+          selector: this.buildSmartSelector(domData)
+        },
+        attributes: domData.attributes,
+        state: domData.state
+      };
+    } catch (error) {
+      return this.handleError(error, event);
+    }
+  }
+
+  /**
+   * Build a smart CSS selector from DOM data
+   */
+  buildSmartSelector(domData) {
+    let selector = domData.tagName;
+    
+    // Add ID if available
+    if (domData.attributes.id) {
+      return `#${domData.attributes.id}`;
+    }
+    
+    // Add data-test-id if available
+    if (domData.attributes['data-test-id']) {
+      return `[data-test-id="${domData.attributes['data-test-id']}"]`;
+    }
+    
+    // Add class if available
+    if (domData.attributes.class) {
+      const classes = domData.attributes.class.split(' ')
+        .filter(c => c && !c.match(/^(active|focus|hover|disabled)/)); // Skip state classes
+      if (classes.length > 0) {
+        selector += `.${classes.slice(0, 2).join('.')}`;
+      }
+    }
+    
+    // Add parent context
+    if (domData.parent) {
+      selector = `${domData.parent} > ${selector}`;
+    }
+    
+    return selector;
+  }
+}
+
+export default DOMEnricher;

package/src/enrichment/enrichers/page-state-enricher.js

@@ -0,0 +1,129 @@
+/**
+ * Page State Enricher - Captures network and page stability state
+ * This file contains browser code executed via page.evaluate() where browser globals are available
+ */
+/* global document, MutationObserver */
+import { EventEnricher } from '../base.js';
+
+export class PageStateEnricher extends EventEnricher {
+  constructor(config = {}) {
+    super(config);
+    this.pendingRequests = new Set();
+    this.setupNetworkTracking = false;
+  }
+
+  getName() {
+    return 'PageStateEnricher';
+  }
+
+  getPriority() {
+    return 95; // Very high priority - critical for timing
+  }
+
+  canEnrich(context) {
+    return this.enabled && context.page;
+  }
+
+  /**
+   * Setup network request tracking (call once per page)
+   */
+  async setupTracking(page) {
+    if (this.setupNetworkTracking) return;
+    
+    page.on('request', (request) => {
+      if (['document', 'xhr', 'fetch'].includes(request.resourceType())) {
+        this.pendingRequests.add(request.url());
+      }
+    });
+    
+    page.on('requestfinished', (request) => {
+      this.pendingRequests.delete(request.url());
+    });
+    
+    page.on('requestfailed', (request) => {
+      this.pendingRequests.delete(request.url());
+    });
+    
+    this.setupNetworkTracking = true;
+  }
+
+  async enrich(event, context) {
+    try {
+      const { page } = context;
+      
+      // Setup tracking if not done
+      await this.setupTracking(page);
+      
+      // Get page load state
+      const pageState = await page.evaluate(() => ({
+        readyState: document.readyState,
+        domContentLoaded: document.readyState !== 'loading',
+        loadComplete: document.readyState === 'complete',
+        url: document.location.href
+      }));
+      
+      // Check if DOM is stable (no mutations for 500ms)
+      const domStable = await this.checkDOMStability(page);
+      
+      return {
+        page: {
+          networkIdle: this.pendingRequests.size === 0,
+          pendingRequests: this.pendingRequests.size,
+          domStable,
+          ...pageState
+        }
+      };
+    } catch (error) {
+      return this.handleError(error, event);
+    }
+  }
+
+  /**
+   * Check if DOM hasn't mutated for 500ms
+   */
+  async checkDOMStability(page, timeoutMs = 500) {
+    try {
+      const stable = await page.evaluate((timeout) => {
+        return new Promise((resolve) => {
+          let timer;
+          let mutations = 0;
+          
+          const observer = new MutationObserver(() => {
+            mutations++;
+            clearTimeout(timer);
+            timer = setTimeout(() => {
+              observer.disconnect();
+              resolve(mutations === 0);
+            }, timeout);
+          });
+          
+          observer.observe(document.body, {
+            childList: true,
+            subtree: true,
+            attributes: true
+          });
+          
+          // Initial timer
+          timer = setTimeout(() => {
+            observer.disconnect();
+            resolve(true);
+          }, timeout);
+        });
+      }, timeoutMs);
+      
+      return stable;
+    } catch (_error) {
+      return false; // Assume not stable on error
+    }
+  }
+
+  /**
+   * Reset tracking for new page navigation
+   */
+  reset() {
+    this.pendingRequests.clear();
+    this.setupNetworkTracking = false;
+  }
+}
+
+export default PageStateEnricher;

package/src/enrichment/enrichers/position-enricher.js

@@ -0,0 +1,67 @@
+/**
+ * Position Enricher - Captures element bounding box and viewport state
+ * This file contains browser code executed via element.evaluate() where browser globals are available
+ */
+/* global window */
+import { EventEnricher } from '../base.js';
+
+export class PositionEnricher extends EventEnricher {
+  getName() {
+    return 'PositionEnricher';
+  }
+
+  getPriority() {
+    return 90; // High priority - cheap and useful
+  }
+
+  canEnrich(context) {
+    // Only enrich interactive actions that have an element
+    if (!this.enabled) return false;
+    if (!context.element) return false;
+    if (!context.event) return false;
+    
+    return ['click', 'fill', 'type', 'selectOption', 'hover'].includes(context.event.type);
+  }
+
+  async enrich(event, context) {
+    try {
+      const { page, element } = context;
+      
+      // Get bounding box
+      const box = await element.boundingBox();
+      if (!box) return null; // Element not visible
+      
+      // Get viewport state
+      const viewport = await page.evaluate(() => ({
+        scrollX: window.scrollX,
+        scrollY: window.scrollY,
+        width: window.innerWidth,
+        height: window.innerHeight
+      }));
+      
+      // Check if in viewport
+      const inViewport = (
+        box.y >= viewport.scrollY &&
+        box.y + box.height <= viewport.scrollY + viewport.height &&
+        box.x >= 0 &&
+        box.x + box.width <= viewport.width
+      );
+      
+      return {
+        position: {
+          boundingBox: box,
+          viewport,
+          inViewport,
+          centerPoint: {
+            x: Math.round(box.x + box.width / 2),
+            y: Math.round(box.y + box.height / 2)
+          }
+        }
+      };
+    } catch (error) {
+      return this.handleError(error, event);
+    }
+  }
+}
+
+export default PositionEnricher;

package/src/enrichment/index.js

@@ -0,0 +1,96 @@
+/**
+ * Event Enrichment System
+ * Modular pipeline for adding data to MCP events
+ */
+
+export { EventEnricher } from './base.js';
+export { EnrichmentPipeline } from './pipeline.js';
+
+// Enrichers
+export { PositionEnricher } from './enrichers/position-enricher.js';
+export { AccessibilityEnricher } from './enrichers/accessibility-enricher.js';
+export { PageStateEnricher } from './enrichers/page-state-enricher.js';
+export { DOMEnricher } from './enrichers/dom-enricher.js';
+export { MCPRefEnricher } from './mcp-ref-enricher.js';
+export { TraceTextEnricher } from './trace-text-enricher.js';
+
+import { EnrichmentPipeline } from './pipeline.js';
+import { PositionEnricher } from './enrichers/position-enricher.js';
+import { AccessibilityEnricher } from './enrichers/accessibility-enricher.js';
+import { PageStateEnricher } from './enrichers/page-state-enricher.js';
+import { DOMEnricher } from './enrichers/dom-enricher.js';
+import { MCPRefEnricher } from './mcp-ref-enricher.js';
+import { TraceTextEnricher } from './trace-text-enricher.js';
+
+/**
+ * Create a default enrichment pipeline with all enrichers
+ * @param {Object} config - Configuration options
+ * @param {boolean} config.enablePosition - Enable position enricher (default: true)
+ * @param {boolean} config.enableAccessibility - Enable accessibility enricher (default: true)
+ * @param {boolean} config.enablePageState - Enable page state enricher (default: true)
+ * @param {boolean} config.enableDOM - Enable DOM enricher (default: true)
+ * @returns {EnrichmentPipeline}
+ */
+export function createDefaultPipeline(config = {}) {
+  const pipeline = new EnrichmentPipeline(config);
+  
+  // Register enrichers (they'll auto-sort by priority)
+  // MCPRefEnricher has highest priority (200) - captures exact MCP refs
+  if (config.enableMCPRef !== false) {
+    pipeline.register(new MCPRefEnricher(config));
+  }
+  
+  // TraceTextEnricher (190) - extracts ACTUAL text from trace (for Chinese/multi-language)
+  if (config.enableTraceText !== false) {
+    pipeline.register(new TraceTextEnricher(config));
+  }
+  
+  if (config.enableAccessibility !== false) {
+    pipeline.register(new AccessibilityEnricher(config));
+  }
+  
+  if (config.enablePageState !== false) {
+    pipeline.register(new PageStateEnricher(config));
+  }
+  
+  if (config.enablePosition !== false) {
+    pipeline.register(new PositionEnricher(config));
+  }
+  
+  if (config.enableDOM !== false) {
+    pipeline.register(new DOMEnricher(config));
+  }
+  
+  return pipeline;
+}
+
+/**
+ * Create a minimal pipeline (only critical enrichers)
+ * @param {Object} config
+ * @returns {EnrichmentPipeline}
+ */
+export function createMinimalPipeline(config = {}) {
+  const pipeline = new EnrichmentPipeline(config);
+  
+  // Only most critical enrichers
+  pipeline.register(new AccessibilityEnricher(config));
+  pipeline.register(new PageStateEnricher(config));
+  
+  return pipeline;
+}
+
+/**
+ * Create a custom pipeline
+ * @param {Array} enrichers - Array of enricher instances
+ * @param {Object} config
+ * @returns {EnrichmentPipeline}
+ */
+export function createCustomPipeline(enrichers, config = {}) {
+  const pipeline = new EnrichmentPipeline(config);
+  
+  for (const enricher of enrichers) {
+    pipeline.register(enricher);
+  }
+  
+  return pipeline;
+}