agent-web-interface 4.0.0

package/dist/src/observation/eid-linker.js

@@ -0,0 +1,403 @@
+/**
+ * EID Linker
+ *
+ * Links DOM observations to snapshot nodes by semantic matching.
+ * Since observations are captured in browser context (no access to CDP backend_node_id),
+ * we perform post-hoc matching after snapshot capture using semantic signals:
+ * - Tag → Kind mapping
+ * - Role matching
+ * - Label/text fuzzy matching
+ */
+import { fuzzyTokensMatch, tokenizeForMatching } from '../lib/text-utils.js';
+// ============================================================================
+// Scoring Constants
+// ============================================================================
+/** Score contribution for tag/kind match (required for any match) */
+const SCORE_TAG_MATCH = 0.3;
+/** Score contribution for ARIA role match */
+const SCORE_ROLE_MATCH = 0.25;
+/** Score contribution for text/label match */
+const SCORE_TEXT_MATCH = 0.3;
+/** Score contribution for dialog context match */
+const SCORE_DIALOG_CONTEXT = 0.15;
+/**
+ * Estimated delay in ms for elements that appeared after initial page load.
+ * Used when appearedAfterDelay signal is true but exact timing is unavailable.
+ * Based on typical async content loading patterns (API responses, lazy loading).
+ */
+const ESTIMATED_DELAY_MS = 200;
+// ============================================================================
+// Tag to Kind Mapping
+// ============================================================================
+/**
+ * Map HTML tags from observations to possible NodeKind values in snapshots.
+ * A tag can map to multiple kinds since role can change the semantic meaning.
+ */
+const TAG_TO_KINDS = {
+    // Links
+    a: ['link'],
+    // Buttons
+    button: ['button'],
+    // Inputs
+    input: ['input', 'checkbox', 'radio', 'switch', 'slider'],
+    textarea: ['textarea'],
+    select: ['select', 'combobox'],
+    // Structural with role variants
+    div: ['generic', 'dialog', 'section', 'button', 'link'],
+    span: ['generic', 'text'],
+    section: ['section', 'generic'],
+    // Dialog-specific
+    dialog: ['dialog'],
+    // Navigation
+    nav: ['navigation'],
+    form: ['form'],
+    // Content
+    h1: ['heading'],
+    h2: ['heading'],
+    h3: ['heading'],
+    h4: ['heading'],
+    h5: ['heading'],
+    h6: ['heading'],
+    p: ['paragraph', 'text'],
+    img: ['image'],
+    // Lists
+    ul: ['list'],
+    ol: ['list'],
+    li: ['listitem'],
+    // Media
+    video: ['media'],
+    audio: ['media'],
+    // Table
+    table: ['table'],
+};
+/**
+ * Role to Kind mapping - when observation has explicit ARIA role.
+ */
+const ROLE_TO_KIND = {
+    alert: 'generic', // alerts are typically generic divs with role
+    alertdialog: 'dialog',
+    dialog: 'dialog',
+    button: 'button',
+    link: 'link',
+    textbox: 'input',
+    checkbox: 'checkbox',
+    radio: 'radio',
+    combobox: 'combobox',
+    listbox: 'combobox',
+    menu: 'navigation',
+    menuitem: 'menuitem',
+    tab: 'tab',
+    tabpanel: 'section',
+    navigation: 'navigation',
+    form: 'form',
+    search: 'form',
+};
+// ============================================================================
+// Core Functions
+// ============================================================================
+/**
+ * Link observations to snapshot nodes by computing eids.
+ *
+ * Mutates observations in-place, setting the `eid` field for matched observations.
+ * Only processes 'appeared' observations (disappeared elements don't exist in snapshot).
+ *
+ * @param observations - Observation groups to link
+ * @param snapshot - Current snapshot to match against
+ * @param registry - Element registry for eid lookup
+ * @param options - Matching configuration
+ * @returns Statistics about linking operation
+ */
+export function linkObservationsToSnapshot(observations, snapshot, registry, options = {}) {
+    // Build index once for efficient lookup
+    const nodeIndex = buildNodeIndex(snapshot);
+    // Pre-filter interactive nodes once (O(n)) instead of per-observation (O(m*n))
+    const interactiveNodes = buildInteractiveNodeList(snapshot);
+    let linked = 0;
+    let unlinked = 0;
+    // Collect all appeared observations from both groups
+    const toProcess = [
+        ...observations.duringAction.filter((o) => o.type === 'appeared'),
+        ...observations.sincePrevious.filter((o) => o.type === 'appeared'),
+    ];
+    for (const observation of toProcess) {
+        const match = findBestMatch(observation, nodeIndex, options);
+        if (match) {
+            // Look up eid from registry using backend_node_id
+            const eid = registry.getEidByBackendNodeId(match.backend_node_id);
+            if (eid) {
+                observation.eid = eid;
+                // Extract children from the matched container using pre-filtered list
+                const children = extractObservationChildren(match, interactiveNodes, registry);
+                if (children.length > 0) {
+                    observation.children = children;
+                }
+                // Extract delay from appearedAfterDelay signal
+                if (observation.signals.appearedAfterDelay) {
+                    observation.delayMs = ESTIMATED_DELAY_MS;
+                }
+                linked++;
+            }
+            else {
+                unlinked++;
+            }
+        }
+        else {
+            unlinked++;
+        }
+    }
+    return {
+        linked,
+        unlinked,
+        total: toProcess.length,
+    };
+}
+/**
+ * Build an index of snapshot nodes by kind for efficient lookup.
+ *
+ * @param snapshot - Snapshot to index
+ * @returns Map from NodeKind to array of nodes
+ */
+export function buildNodeIndex(snapshot) {
+    const index = new Map();
+    for (const node of snapshot.nodes) {
+        const kind = node.kind;
+        const nodes = index.get(kind) ?? [];
+        nodes.push(node);
+        index.set(kind, nodes);
+    }
+    return index;
+}
+/**
+ * Find the best matching node for an observation.
+ *
+ * @param observation - The observation to match
+ * @param nodeIndex - Pre-built index of nodes by kind
+ * @param options - Matching options
+ * @returns Best matching node or undefined if no match meets threshold
+ */
+export function findBestMatch(observation, nodeIndex, options = {}) {
+    const { minMatchScore = 0.5 } = options;
+    // Get candidate kinds based on tag and role
+    const candidateKinds = getCandidateKinds(observation);
+    // Collect all candidate nodes
+    const candidates = [];
+    for (const kind of candidateKinds) {
+        const nodesOfKind = nodeIndex.get(kind);
+        if (nodesOfKind) {
+            candidates.push(...nodesOfKind);
+        }
+    }
+    if (candidates.length === 0) {
+        return undefined;
+    }
+    // Score each candidate and find best match
+    let bestMatch;
+    let bestScore = minMatchScore; // Must exceed threshold
+    for (const node of candidates) {
+        const score = computeMatchScore(observation, node, options);
+        if (score > bestScore) {
+            bestScore = score;
+            bestMatch = node;
+        }
+    }
+    return bestMatch;
+}
+/**
+ * Get candidate NodeKind values for an observation based on tag and role.
+ */
+export function getCandidateKinds(observation) {
+    const tag = observation.content.tag.toLowerCase();
+    const role = observation.content.role?.toLowerCase();
+    const kinds = new Set();
+    // Add kinds from tag mapping
+    const tagKinds = TAG_TO_KINDS[tag];
+    if (tagKinds) {
+        for (const kind of tagKinds) {
+            kinds.add(kind);
+        }
+    }
+    // Add kind from role mapping (higher priority when present)
+    if (role && role in ROLE_TO_KIND) {
+        kinds.add(ROLE_TO_KIND[role]);
+    }
+    // Fallback to generic if nothing matched
+    if (kinds.size === 0) {
+        kinds.add('generic');
+    }
+    return Array.from(kinds);
+}
+/**
+ * Compute match score between an observation and a snapshot node.
+ *
+ * Scoring breakdown:
+ * - Tag/Kind match: 0.3 (required - if no match, return 0)
+ * - Role match: 0.25
+ * - Label fuzzy match: 0.3
+ * - Dialog context match: 0.15
+ *
+ * @param observation - DOM observation
+ * @param node - Snapshot node
+ * @param options - Matching options
+ * @returns Score from 0-1 (0 = no match, 1 = perfect match)
+ */
+export function computeMatchScore(observation, node, options = {}) {
+    const { fuzzyTextMatch = true, minTokenOverlap = 0.5 } = options;
+    const content = observation.content;
+    let score = 0;
+    // Tag/Kind match check (required)
+    const candidateKinds = getCandidateKinds(observation);
+    if (!candidateKinds.includes(node.kind)) {
+        return 0; // No match possible
+    }
+    score += SCORE_TAG_MATCH;
+    // Role match (when observation has explicit role that matches node kind)
+    const role = content.role?.toLowerCase();
+    if (role &&
+        role in ROLE_TO_KIND &&
+        ROLE_TO_KIND[role] === node.kind) {
+        score += SCORE_ROLE_MATCH;
+    }
+    // Label/text matching
+    if (content.text && node.label) {
+        const normalizedObsText = content.text.toLowerCase().trim();
+        const normalizedNodeLabel = node.label.toLowerCase().trim();
+        // Exact match
+        if (normalizedObsText === normalizedNodeLabel) {
+            score += SCORE_TEXT_MATCH;
+        }
+        else if (fuzzyTextMatch) {
+            // Fuzzy match
+            const obsTokens = tokenizeForMatching(normalizedObsText, 10, 2);
+            const nodeTokens = tokenizeForMatching(normalizedNodeLabel, 10, 2);
+            const result = fuzzyTokensMatch(nodeTokens, obsTokens, { minTokenOverlap });
+            if (result.isMatch) {
+                score += SCORE_TEXT_MATCH * result.score;
+            }
+        }
+    }
+    // Dialog context bonus - if observation is dialog and node is in dialog region
+    if (observation.signals.isDialog && node.where.region === 'dialog') {
+        score += SCORE_DIALOG_CONTEXT;
+    }
+    // Cap at 1.0
+    return Math.min(score, 1.0);
+}
+// ============================================================================
+// Child Extraction
+// ============================================================================
+/** Maximum number of children to extract per observation */
+const MAX_OBSERVATION_CHILDREN = 5;
+/** Interactive node kinds that qualify as observation children */
+const INTERACTIVE_KINDS = [
+    'button',
+    'link',
+    'input',
+    'textarea',
+    'select',
+    'combobox',
+    'checkbox',
+    'radio',
+    'switch',
+    'slider',
+    'tab',
+    'menuitem',
+];
+/**
+ * Check if a node kind is interactive (actionable).
+ */
+function isInteractiveKind(kind) {
+    return INTERACTIVE_KINDS.includes(kind);
+}
+/**
+ * Check if a node qualifies as an extractable child (interactive or heading with label).
+ */
+function isExtractableChild(node) {
+    // Must have a visible label
+    if (!node.label || node.label.trim() === '')
+        return false;
+    // Must be interactive or a heading
+    return isInteractiveKind(node.kind) || node.kind === 'heading';
+}
+/**
+ * Build a pre-filtered list of interactive nodes for child extraction.
+ * This is O(n) and done once per snapshot, avoiding O(m*n) when processing m observations.
+ *
+ * @param snapshot - Snapshot to filter
+ * @returns Array of nodes that qualify as observation children
+ */
+export function buildInteractiveNodeList(snapshot) {
+    return snapshot.nodes.filter(isExtractableChild);
+}
+/**
+ * Check if a bounding box is contained within another.
+ */
+function isNodeInsideContainer(nodeBbox, containerBbox) {
+    return (nodeBbox.x >= containerBbox.x &&
+        nodeBbox.y >= containerBbox.y &&
+        nodeBbox.x + nodeBbox.w <= containerBbox.x + containerBbox.w &&
+        nodeBbox.y + nodeBbox.h <= containerBbox.y + containerBbox.h);
+}
+/**
+ * Map NodeKind to child tag name for XML output.
+ */
+function mapKindToChildTag(kind) {
+    switch (kind) {
+        case 'button':
+            return 'button';
+        case 'link':
+            return 'link';
+        case 'heading':
+            return 'heading';
+        case 'input':
+        case 'textarea':
+            return 'input';
+        case 'select':
+        case 'combobox':
+            return 'select';
+        case 'checkbox':
+        case 'radio':
+        case 'switch':
+            return 'checkbox';
+        case 'slider':
+            return 'slider';
+        case 'tab':
+        case 'menuitem':
+            return 'tab';
+        default:
+            return 'text';
+    }
+}
+/**
+ * Extract notable children from an observation's matched node.
+ *
+ * Finds interactive elements and headings that are visually contained within
+ * the observation container, up to MAX_OBSERVATION_CHILDREN limit.
+ *
+ * @param matchedNode - The snapshot node that matched the observation
+ * @param interactiveNodes - Pre-filtered list of interactive nodes (from buildInteractiveNodeList)
+ * @param registry - Element registry for eid lookup
+ * @returns Array of observation children (interactive elements and headings)
+ */
+export function extractObservationChildren(matchedNode, interactiveNodes, registry) {
+    const children = [];
+    const containerBbox = matchedNode.layout.bbox;
+    for (const node of interactiveNodes) {
+        // Skip the container itself
+        if (node.backend_node_id === matchedNode.backend_node_id)
+            continue;
+        // Skip nodes not inside the container
+        if (!isNodeInsideContainer(node.layout.bbox, containerBbox))
+            continue;
+        const eid = registry.getEidByBackendNodeId(node.backend_node_id);
+        children.push({
+            tag: mapKindToChildTag(node.kind),
+            eid: eid,
+            text: node.label,
+        });
+        // Limit children to prevent bloat
+        if (children.length >= MAX_OBSERVATION_CHILDREN)
+            break;
+    }
+    return children;
+}
+//# sourceMappingURL=eid-linker.js.map

package/dist/src/observation/eid-linker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eid-linker.js","sourceRoot":"","sources":["../../../src/observation/eid-linker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAkC7E,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E,qEAAqE;AACrE,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,6CAA6C;AAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,8CAA8C;AAC9C,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,kDAAkD;AAClD,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,+EAA+E;AAC/E,sBAAsB;AACtB,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,YAAY,GAA+B;IAC/C,QAAQ;IACR,CAAC,EAAE,CAAC,MAAM,CAAC;IAEX,UAAU;IACV,MAAM,EAAE,CAAC,QAAQ,CAAC;IAElB,SAAS;IACT,KAAK,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC;IACzD,QAAQ,EAAE,CAAC,UAAU,CAAC;IACtB,MAAM,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC;IAE9B,gCAAgC;IAChC,GAAG,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC;IACvD,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;IAE/B,kBAAkB;IAClB,MAAM,EAAE,CAAC,QAAQ,CAAC;IAElB,aAAa;IACb,GAAG,EAAE,CAAC,YAAY,CAAC;IACnB,IAAI,EAAE,CAAC,MAAM,CAAC;IAEd,UAAU;IACV,EAAE,EAAE,CAAC,SAAS,CAAC;IACf,EAAE,EAAE,CAAC,SAAS,CAAC;IACf,EAAE,EAAE,CAAC,SAAS,CAAC;IACf,EAAE,EAAE,CAAC,SAAS,CAAC;IACf,EAAE,EAAE,CAAC,SAAS,CAAC;IACf,EAAE,EAAE,CAAC,SAAS,CAAC;IACf,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC;IACxB,GAAG,EAAE,CAAC,OAAO,CAAC;IAEd,QAAQ;IACR,EAAE,EAAE,CAAC,MAAM,CAAC;IACZ,EAAE,EAAE,CAAC,MAAM,CAAC;IACZ,EAAE,EAAE,CAAC,UAAU,CAAC;IAEhB,QAAQ;IACR,KAAK,EAAE,CAAC,OAAO,CAAC;IAChB,KAAK,EAAE,CAAC,OAAO,CAAC;IAEhB,QAAQ;IACR,KAAK,EAAE,CAAC,OAAO,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,MAAM,YAAY,GAAG;IACnB,KAAK,EAAE,SAAS,EAAE,8CAA8C;IAChE,WAAW,EAAE,QAAQ;IACrB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,UAAU;IACpB,KAAK,EAAE,OAAO;IACd,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE,UAAU;IACnB,IAAI,EAAE,YAAY;IAClB,QAAQ,EAAE,UAAU;IACpB,GAAG,EAAE,KAAK;IACV,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,YAAY;IACxB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,MAAM;CAC6B,CAAC;AAE9C,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,0BAA0B,CACxC,YAA+B,EAC/B,QAAsB,EACtB,QAAyB,EACzB,UAA4B,EAAE;IAE9B,wCAAwC;IACxC,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAE3C,+EAA+E;IAC/E,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IAE5D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,qDAAqD;IACrD,MAAM,SAAS,GAAG;QAChB,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;QACjE,GAAG,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;KACnE,CAAC;IAEF,KAAK,MAAM,WAAW,IAAI,SAAS,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAE7D,IAAI,KAAK,EAAE,CAAC;YACV,kDAAkD;YAClD,MAAM,GAAG,GAAG,QAAQ,CAAC,qBAAqB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAClE,IAAI,GAAG,EAAE,CAAC;gBACR,WAAW,CAAC,GAAG,GAAG,GAAG,CAAC;gBAEtB,sEAAsE;gBACtE,MAAM,QAAQ,GAAG,0BAA0B,CAAC,KAAK,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAAC;gBAC/E,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,WAAW,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBAClC,CAAC;gBAED,+CAA+C;gBAC/C,IAAI,WAAW,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,kBAAkB,CAAC;gBAC3C,CAAC;gBAED,MAAM,EAAE,CAAC;YACX,CAAC;iBAAM,CAAC;gBACN,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;aAAM,CAAC;YACN,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM;QACN,QAAQ;QACR,KAAK,EAAE,SAAS,CAAC,MAAM;KACxB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,QAAsB;IACnD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAC3B,WAA2B,EAC3B,SAAwC,EACxC,UAA4B,EAAE;IAE9B,MAAM,EAAE,aAAa,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;IAExC,4CAA4C;IAC5C,MAAM,cAAc,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAEtD,8BAA8B;IAC9B,MAAM,UAAU,GAAmB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,WAAW,EAAE,CAAC;YAChB,UAAU,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,2CAA2C;IAC3C,IAAI,SAAmC,CAAC;IACxC,IAAI,SAAS,GAAG,aAAa,CAAC,CAAC,wBAAwB;IAEvD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,iBAAiB,CAAC,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,WAA2B;IAC3D,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAErD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAY,CAAC;IAElC,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,4DAA4D;IAC5D,IAAI,IAAI,IAAI,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAiC,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,yCAAyC;IACzC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrB,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,iBAAiB,CAC/B,WAA2B,EAC3B,IAAkB,EAClB,UAA4B,EAAE;IAE9B,MAAM,EAAE,cAAc,GAAG,IAAI,EAAE,eAAe,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;IAEjE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,kCAAkC;IAClC,MAAM,cAAc,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IACtD,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,CAAC,CAAC,oBAAoB;IAChC,CAAC;IACD,KAAK,IAAI,eAAe,CAAC;IAEzB,yEAAyE;IACzE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IACzC,IACE,IAAI;QACJ,IAAI,IAAI,YAAY;QACpB,YAAY,CAAC,IAAiC,CAAC,KAAK,IAAI,CAAC,IAAI,EAC7D,CAAC;QACD,KAAK,IAAI,gBAAgB,CAAC;IAC5B,CAAC;IAED,sBAAsB;IACtB,IAAI,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAE5D,cAAc;QACd,IAAI,iBAAiB,KAAK,mBAAmB,EAAE,CAAC;YAC9C,KAAK,IAAI,gBAAgB,CAAC;QAC5B,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,cAAc;YACd,MAAM,SAAS,GAAG,mBAAmB,CAAC,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,UAAU,GAAG,mBAAmB,CAAC,mBAAmB,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAEnE,MAAM,MAAM,GAAG,gBAAgB,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC;YAC5E,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,KAAK,IAAI,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,IAAI,WAAW,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACnE,KAAK,IAAI,oBAAoB,CAAC;IAChC,CAAC;IAED,aAAa;IACb,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,4DAA4D;AAC5D,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAEnC,kEAAkE;AAClE,MAAM,iBAAiB,GAAe;IACpC,QAAQ;IACR,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,UAAU;IACV,UAAU;IACV,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;CACX,CAAC;AAEF;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAc;IACvC,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAkB;IAC5C,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAC1D,mCAAmC;IACnC,OAAO,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC;AACjE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,QAAsB;IAC7D,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAC5B,QAAwD,EACxD,aAA6D;IAE7D,OAAO,CACL,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC;QAC7B,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC;QAC7B,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC;QAC5D,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAc;IACvC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC;QAClB,KAAK,MAAM;YACT,OAAO,MAAM,CAAC;QAChB,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;QACnB,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,OAAO,CAAC;QACjB,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,QAAQ,CAAC;QAClB,KAAK,UAAU,CAAC;QAChB,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ;YACX,OAAO,UAAU,CAAC;QACpB,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC;QAClB,KAAK,KAAK,CAAC;QACX,KAAK,UAAU;YACb,OAAO,KAAK,CAAC;QACf;YACE,OAAO,MAAM,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,0BAA0B,CACxC,WAAyB,EACzB,gBAAgC,EAChC,QAAyB;IAEzB,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC;IAE9C,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,4BAA4B;QAC5B,IAAI,IAAI,CAAC,eAAe,KAAK,WAAW,CAAC,eAAe;YAAE,SAAS;QAEnE,sCAAsC;QACtC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC;YAAE,SAAS;QAEtE,MAAM,GAAG,GAAG,QAAQ,CAAC,qBAAqB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC;YACZ,GAAG,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,kCAAkC;QAClC,IAAI,QAAQ,CAAC,MAAM,IAAI,wBAAwB;YAAE,MAAM;IACzD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/observation/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * DOM Observation module.
+ *
+ * Captures significant DOM mutations (toasts, dialogs, banners, overlays)
+ * that appear during or between actions.
+ */
+export type { SignificanceSignals, ObservedContent, DOMObservation, RawMutationEntry, ObservationGroups, } from './observation.types.js';
+export { SIGNIFICANCE_WEIGHTS, SIGNIFICANCE_THRESHOLD, computeSignificance, } from './observation.types.js';
+export { ObservationAccumulator, observationAccumulator } from './observation-accumulator.js';
+export { OBSERVATION_OBSERVER_SCRIPT } from './observer-script.js';
+export { linkObservationsToSnapshot, type EidLinkerOptions, type EidLinkingResult, } from './eid-linker.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/observation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/observation/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAG9F,OAAO,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EACL,0BAA0B,EAC1B,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,GACtB,MAAM,iBAAiB,CAAC"}

package/dist/src/observation/index.js

@@ -0,0 +1,15 @@
+/**
+ * DOM Observation module.
+ *
+ * Captures significant DOM mutations (toasts, dialogs, banners, overlays)
+ * that appear during or between actions.
+ */
+// Constants and functions
+export { SIGNIFICANCE_WEIGHTS, SIGNIFICANCE_THRESHOLD, computeSignificance, } from './observation.types.js';
+// Accumulator
+export { ObservationAccumulator, observationAccumulator } from './observation-accumulator.js';
+// Browser script
+export { OBSERVATION_OBSERVER_SCRIPT } from './observer-script.js';
+// EID Linker
+export { linkObservationsToSnapshot, } from './eid-linker.js';
+//# sourceMappingURL=index.js.map

package/dist/src/observation/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/observation/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,0BAA0B;AAC1B,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,wBAAwB,CAAC;AAEhC,cAAc;AACd,OAAO,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAE9F,iBAAiB;AACjB,OAAO,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AAEnE,aAAa;AACb,OAAO,EACL,0BAA0B,GAG3B,MAAM,iBAAiB,CAAC"}

package/dist/src/observation/observation-accumulator.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Manages persistent DOM observation for a page.
+ *
+ * Lifecycle:
+ * - inject() on page creation
+ * - ensureInjected() after navigation (document context changes)
+ * - getObservations() to retrieve significant mutations
+ */
+import type { Page } from 'puppeteer-core';
+import type { ObservationGroups } from './observation.types.js';
+/**
+ * Manages persistent DOM observation for a page.
+ */
+export declare class ObservationAccumulator {
+    /**
+     * Inject the persistent observer into the page.
+     * Safe to call multiple times - checks for existing injection.
+     */
+    inject(page: Page): Promise<void>;
+    /**
+     * Ensure observer is injected and valid (re-inject if needed after navigation).
+     *
+     * After navigation or page content changes, the observer might be stale
+     * (observing a detached body). We check if the observer's body reference
+     * matches the current document.body and only re-inject if stale.
+     *
+     * This preserves accumulated observations when the observer is still valid.
+     */
+    ensureInjected(page: Page): Promise<void>;
+    /**
+     * Get observations for an action.
+     * Returns both action-scoped and unreported accumulated observations.
+     */
+    getObservations(page: Page, actionStartTime: number): Promise<ObservationGroups>;
+    /**
+     * Get accumulated observations without action context.
+     * Used by capture_snapshot to report accumulated changes.
+     */
+    getAccumulatedObservations(page: Page): Promise<ObservationGroups>;
+    /**
+     * Convert raw mutation entry to DOMObservation.
+     */
+    private rawToObservation;
+    /**
+     * Reset observation log (call after navigation).
+     */
+    reset(page: Page): Promise<void>;
+    /**
+     * Check if there are any unreported observations.
+     */
+    hasUnreported(page: Page): Promise<boolean>;
+    /**
+     * Filter observations by significance threshold.
+     */
+    filterBySignificance(observations: ObservationGroups, threshold?: number): ObservationGroups;
+}
+export declare const observationAccumulator: ObservationAccumulator;
+//# sourceMappingURL=observation-accumulator.d.ts.map

package/dist/src/observation/observation-accumulator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observation-accumulator.d.ts","sourceRoot":"","sources":["../../../src/observation/observation-accumulator.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAIV,iBAAiB,EAClB,MAAM,wBAAwB,CAAC;AAIhC;;GAEG;AACH,qBAAa,sBAAsB;IACjC;;;OAGG;IACG,MAAM,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IASvC;;;;;;;;OAQG;IACG,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAiC/C;;;OAGG;IACG,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAsCtF;;;OAGG;IACG,0BAA0B,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA6BxE;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmCxB;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAYtC;;OAEG;IACG,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;IAcjD;;OAEG;IACH,oBAAoB,CAClB,YAAY,EAAE,iBAAiB,EAC/B,SAAS,GAAE,MAA+B,GACzC,iBAAiB;CAMrB;AAGD,eAAO,MAAM,sBAAsB,wBAA+B,CAAC"}