@xoxoai/checkmate 0.4.0

package/dist/tools/browser/snapshot-filter/semantic-scorer.js

@@ -0,0 +1,99 @@
+import { cos_sim, pipeline } from '@huggingface/transformers';
+const EMBEDDING_MODEL = 'snowflake/snowflake-arctic-embed-xs';
+const SNAPSHOT_METADATA_REGEX = /\s*\[[^\]]*\]/g;
+let featureExtractionPipelinePromise;
+const embeddingCache = new Map();
+export async function scoreSnapshotElements(json, query) {
+    const normalizedQuery = normalizeText(query);
+    if (!normalizedQuery) {
+        return [];
+    }
+    const candidates = [];
+    collectCandidates(json, [], candidates);
+    if (candidates.length === 0) {
+        return [];
+    }
+    const queryEmbedding = await getEmbedding(normalizedQuery);
+    const scoredElements = await Promise.all(candidates.map(async (candidate) => {
+        const normalizedText = normalizeText(candidate.text);
+        if (!normalizedText) {
+            return undefined;
+        }
+        const elementEmbedding = await getEmbedding(normalizedText);
+        const score = Math.max(0, cos_sim(queryEmbedding, elementEmbedding));
+        if (score <= 0) {
+            return undefined;
+        }
+        return {
+            score,
+            path: candidate.path,
+            element: candidate.element,
+            key: candidate.key,
+        };
+    }));
+    return scoredElements.filter((candidate) => candidate !== undefined);
+}
+function collectCandidates(value, currentPath, results) {
+    if (value === null || value === undefined) {
+        return;
+    }
+    if (typeof value === 'string') {
+        results.push({ path: [...currentPath], element: value, key: value, text: value });
+        return;
+    }
+    if (Array.isArray(value)) {
+        value.forEach((item, index) => {
+            collectCandidates(item, [...currentPath, index], results);
+        });
+        return;
+    }
+    const objectValue = value;
+    for (const key of Object.keys(objectValue)) {
+        results.push({ path: [...currentPath, key], element: objectValue[key], key, text: key });
+        collectCandidates(objectValue[key], [...currentPath, key], results);
+    }
+}
+function normalizeText(text) {
+    return text.replaceAll(SNAPSHOT_METADATA_REGEX, ' ').replaceAll(/\s+/g, ' ').trim().toLowerCase();
+}
+async function getFeatureExtractionPipeline() {
+    featureExtractionPipelinePromise ??= pipeline('feature-extraction', EMBEDDING_MODEL, {
+        dtype: 'int8',
+    });
+    return featureExtractionPipelinePromise;
+}
+async function getEmbedding(text) {
+    const normalizedText = normalizeText(text);
+    if (!normalizedText) {
+        return [];
+    }
+    const cachedEmbedding = embeddingCache.get(normalizedText);
+    if (cachedEmbedding) {
+        return cachedEmbedding;
+    }
+    const embeddingPromise = (async () => {
+        const extractor = await getFeatureExtractionPipeline();
+        const output = await extractor(normalizedText, { pooling: 'mean', normalize: true });
+        return Array.from(output.data);
+    })();
+    embeddingCache.set(normalizedText, embeddingPromise);
+    try {
+        return await embeddingPromise;
+    }
+    catch (error) {
+        embeddingCache.delete(normalizedText);
+        throw error;
+    }
+}
+export function filterByThreshold(scoredElements, threshold) {
+    return scoredElements.filter((element) => element.score >= threshold);
+}
+export function filterTopPercent(scoredElements, topPercent) {
+    if (scoredElements.length === 0 || topPercent <= 0 || topPercent > 1) {
+        return [];
+    }
+    const sortedElements = [...scoredElements].sort((a, b) => b.score - a.score);
+    const selectedCount = Math.max(1, Math.ceil(sortedElements.length * topPercent));
+    return sortedElements.slice(0, selectedCount);
+}
+//# sourceMappingURL=semantic-scorer.js.map

package/dist/tools/browser/snapshot-filter/semantic-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-scorer.js","sourceRoot":"","sources":["../../../../src/tools/browser/snapshot-filter/semantic-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAoB7D,MAAM,eAAe,GAAG,qCAAqC,CAAA;AAC7D,MAAM,uBAAuB,GAAG,gBAAgB,CAAA;AAOhD,IAAI,gCAAyE,CAAA;AAC7E,MAAM,cAAc,GAAG,IAAI,GAAG,EAA6B,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAe,EAAE,KAAa;IACzE,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,CAAC,CAAA;IAC5C,IAAI,CAAC,eAAe,EAAE,CAAC;QACtB,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,UAAU,GAAuB,EAAE,CAAA;IACzC,iBAAiB,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,CAAA;IACvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,CAAA;IAC1D,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;QAClC,MAAM,cAAc,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QACpD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAA;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAA;QACpE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YAChB,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO;YACN,KAAK;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,GAAG,EAAE,SAAS,CAAC,GAAG;SAClB,CAAA;IACF,CAAC,CAAC,CACF,CAAA;IAED,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,SAAS,EAA8B,EAAE,CAAC,SAAS,KAAK,SAAS,CAAC,CAAA;AACjG,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAgB,EAAE,WAAgC,EAAE,OAA2B;IACzG,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC3C,OAAM;IACP,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,WAAW,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QACjF,OAAM;IACP,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7B,iBAAiB,CAAC,IAAI,EAAE,CAAC,GAAG,WAAW,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,CAAA;QAC1D,CAAC,CAAC,CAAA;QACF,OAAM;IACP,CAAC;IAED,MAAM,WAAW,GAAG,KAAmB,CAAA;IACvC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,WAAW,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAA;QACxF,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;AACF,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,UAAU,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;AAClG,CAAC;AAED,KAAK,UAAU,4BAA4B;IAC1C,gCAAgC,KAAK,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE;QACpF,KAAK,EAAE,MAAM;KACb,CAAgC,CAAA;IACjC,OAAO,gCAAgC,CAAA;AACxC,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;IAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;QACrB,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,eAAe,GAAG,cAAc,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAC1D,IAAI,eAAe,EAAE,CAAC;QACrB,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,MAAM,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;QACpC,MAAM,SAAS,GAAG,MAAM,4BAA4B,EAAE,CAAA;QACtD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACpF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAA+B,CAAC,CAAA;IAC1D,CAAC,CAAC,EAAE,CAAA;IAEJ,cAAc,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAA;IAEpD,IAAI,CAAC;QACJ,OAAO,MAAM,gBAAgB,CAAA;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,cAAc,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACrC,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,cAA+B,EAAE,SAAiB;IACnF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,SAAS,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,cAA+B,EAAE,UAAkB;IACnF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACtE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,cAAc,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAA;IAChF,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAA;AAC9C,CAAC"}

package/dist/tools/browser/snapshot-filter/snapshot-filter.d.ts

@@ -0,0 +1,4 @@
+import { Step } from '../../../runtime/types';
+import { JsonValue } from './semantic-scorer';
+export declare function filterSnapshot(json: JsonValue, step?: Step): Promise<JsonValue>;
+//# sourceMappingURL=snapshot-filter.d.ts.map

package/dist/tools/browser/snapshot-filter/snapshot-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-filter.d.ts","sourceRoot":"","sources":["../../../../src/tools/browser/snapshot-filter/snapshot-filter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAA;AAC7C,OAAO,EAAuC,SAAS,EAAyB,MAAM,mBAAmB,CAAA;AAMzG,wBAAsB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CA6CrF"}

package/dist/tools/browser/snapshot-filter/snapshot-filter.js

@@ -0,0 +1,57 @@
+import { logger } from '../../../logging';
+import { filterByThreshold, filterTopPercent, scoreSnapshotElements } from './semantic-scorer';
+import { reconstructTree } from './tree-reconstructor';
+const DEFAULT_SCORE_THRESHOLD = 0.3;
+const DEFAULT_TOP_PERCENT = 10;
+export async function filterSnapshot(json, step) {
+    if (!step) {
+        logger.debug('filterSnapshot: No step provided, returning original snapshot');
+        return json;
+    }
+    const searchQuery = resolveSearchQuery(step);
+    logger.debug(`filterSnapshot: Resolved search query: ${JSON.stringify(searchQuery)}`);
+    if (!searchQuery) {
+        logger.debug('filterSnapshot: No search query found, returning original snapshot');
+        return json;
+    }
+    const scoredElements = await scoreSnapshotElements(json, searchQuery);
+    logger.debug(`filterSnapshot: Scored ${scoredElements.length} elements`);
+    if (scoredElements.length === 0) {
+        logger.debug('filterSnapshot: No scored elements, returning original snapshot');
+        return json;
+    }
+    const topPercent = resolveTopPercent(step);
+    const selectedByPrimaryRule = filterTopPercent(scoredElements, topPercent / 100);
+    logger.debug(`filterSnapshot: Filtered to ${selectedByPrimaryRule.length} elements from top ${topPercent}%`);
+    const selectedElements = selectedByPrimaryRule.length > 0
+        ? selectedByPrimaryRule
+        : filterByThreshold(scoredElements, DEFAULT_SCORE_THRESHOLD);
+    if (selectedElements !== selectedByPrimaryRule) {
+        logger.debug(`filterSnapshot: Top-percent selection was empty, falling back to threshold ${DEFAULT_SCORE_THRESHOLD}`);
+    }
+    const filtered = reconstructTree(json, selectedElements);
+    const originalSize = JSON.stringify(json).length;
+    const filteredSize = JSON.stringify(filtered).length;
+    logger.info(`filterSnapshot: Reduced snapshot from ${originalSize} to ${filteredSize} chars (${Math.round((1 - filteredSize / originalSize) * 100)}% reduction)`);
+    return filtered;
+}
+function resolveTopPercent(step) {
+    const candidate = step.topPercent;
+    if (typeof candidate === 'number' && candidate > 0 && candidate <= 100) {
+        return candidate;
+    }
+    return DEFAULT_TOP_PERCENT;
+}
+function resolveSearchQuery(step) {
+    if (step.search && step.search.length > 0) {
+        logger.info(`filterSnapshot: using search keywords: ${JSON.stringify(step.search)}`);
+        return step.search.join(' ');
+    }
+    const semanticQuery = `${step.action} ${step.expect}`.trim();
+    if (semanticQuery) {
+        logger.info(`filterSnapshot: using step semantics`);
+        return semanticQuery;
+    }
+    return '';
+}
+//# sourceMappingURL=snapshot-filter.js.map

package/dist/tools/browser/snapshot-filter/snapshot-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-filter.js","sourceRoot":"","sources":["../../../../src/tools/browser/snapshot-filter/snapshot-filter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAa,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AACzG,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAEtD,MAAM,uBAAuB,GAAG,GAAG,CAAA;AACnC,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAe,EAAE,IAAW;IAChE,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAA;QAC7E,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IAC5C,MAAM,CAAC,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;IAErF,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAA;QAClF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IACrE,MAAM,CAAC,KAAK,CAAC,0BAA0B,cAAc,CAAC,MAAM,WAAW,CAAC,CAAA;IAExE,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAA;QAC/E,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,cAAc,EAAE,UAAU,GAAG,GAAG,CAAC,CAAA;IAChF,MAAM,CAAC,KAAK,CAAC,+BAA+B,qBAAqB,CAAC,MAAM,sBAAsB,UAAU,GAAG,CAAC,CAAA;IAE5G,MAAM,gBAAgB,GACrB,qBAAqB,CAAC,MAAM,GAAG,CAAC;QAC/B,CAAC,CAAC,qBAAqB;QACvB,CAAC,CAAC,iBAAiB,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAA;IAE9D,IAAI,gBAAgB,KAAK,qBAAqB,EAAE,CAAC;QAChD,MAAM,CAAC,KAAK,CACX,8EAA8E,uBAAuB,EAAE,CACvG,CAAA;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA;IACxD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAA;IACpD,MAAM,CAAC,IAAI,CACV,yCAAyC,YAAY,OAAO,YAAY,WAAW,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,YAAY,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,cAAc,CACpJ,CAAA;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAU;IACpC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAA;IACjC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QACxE,OAAO,SAAS,CAAA;IACjB,CAAC;IAED,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAU;IACrC,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACpF,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAA;IAC5D,IAAI,aAAa,EAAE,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAA;QACnD,OAAO,aAAa,CAAA;IACrB,CAAC;IAED,OAAO,EAAE,CAAA;AACV,CAAC"}

package/dist/tools/browser/snapshot-filter/tree-reconstructor.d.ts

@@ -0,0 +1,3 @@
+import { JsonValue, ScoredElement } from './semantic-scorer';
+export declare function reconstructTree(originalJson: JsonValue, selectedElements: ScoredElement[]): JsonValue;
+//# sourceMappingURL=tree-reconstructor.d.ts.map

package/dist/tools/browser/snapshot-filter/tree-reconstructor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree-reconstructor.d.ts","sourceRoot":"","sources":["../../../../src/tools/browser/snapshot-filter/tree-reconstructor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,SAAS,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAEnF,wBAAgB,eAAe,CAAC,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,SAAS,CAOrG"}

package/dist/tools/browser/snapshot-filter/tree-reconstructor.js

@@ -0,0 +1,85 @@
+export function reconstructTree(originalJson, selectedElements) {
+    if (selectedElements.length === 0) {
+        return originalJson;
+    }
+    const allPaths = collectAllRequiredPaths(selectedElements);
+    return buildFilteredTree(originalJson, allPaths, []) ?? originalJson;
+}
+function collectAllRequiredPaths(elements) {
+    const paths = new Set();
+    for (const element of elements) {
+        for (let i = 0; i <= element.path.length; i++) {
+            paths.add(JSON.stringify(element.path.slice(0, i)));
+        }
+        paths.add(JSON.stringify(element.path));
+    }
+    return paths;
+}
+function buildFilteredTree(value, requiredPaths, currentPath) {
+    if (!isPathOrAncestorRequired(currentPath, requiredPaths)) {
+        return undefined;
+    }
+    if (value === null || typeof value !== 'object') {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        return buildFilteredArray(value, requiredPaths, currentPath);
+    }
+    return buildFilteredObject(value, requiredPaths, currentPath);
+}
+function isPathOrAncestorRequired(path, requiredPaths) {
+    const pathKey = JSON.stringify(path);
+    if (requiredPaths.has(pathKey)) {
+        return true;
+    }
+    for (const requiredPath of requiredPaths) {
+        const parsed = JSON.parse(requiredPath);
+        if (isAncestorOf(path, parsed)) {
+            return true;
+        }
+    }
+    return false;
+}
+function isAncestorOf(potentialAncestor, descendant) {
+    return (potentialAncestor.length < descendant.length &&
+        potentialAncestor.every((segment, index) => segment === descendant[index]));
+}
+function buildFilteredArray(value, requiredPaths, currentPath) {
+    const filteredItems = [];
+    for (let index = 0; index < value.length; index++) {
+        const filteredChild = buildFilteredTree(value[index], requiredPaths, [...currentPath, index]);
+        if (filteredChild !== undefined) {
+            filteredItems.push(filteredChild);
+        }
+    }
+    if (filteredItems.length === 0) {
+        return includeDirectChildren(requiredPaths, currentPath) ? value : undefined;
+    }
+    return filteredItems;
+}
+function buildFilteredObject(value, requiredPaths, currentPath) {
+    const filteredObject = {};
+    let hasContent = false;
+    for (const key of Object.keys(value)) {
+        const filteredChild = buildFilteredTree(value[key], requiredPaths, [...currentPath, key]);
+        if (filteredChild !== undefined) {
+            filteredObject[key] = filteredChild;
+            hasContent = true;
+        }
+    }
+    if (!hasContent && includeDirectChildren(requiredPaths, currentPath)) {
+        return value;
+    }
+    return hasContent ? filteredObject : undefined;
+}
+function includeDirectChildren(requiredPaths, currentPath) {
+    const currentPathKey = JSON.stringify(currentPath);
+    for (const requiredPath of requiredPaths) {
+        const parsed = JSON.parse(requiredPath);
+        if (parsed.length === currentPath.length && currentPathKey === requiredPath) {
+            return true;
+        }
+    }
+    return false;
+}
+//# sourceMappingURL=tree-reconstructor.js.map

package/dist/tools/browser/snapshot-filter/tree-reconstructor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree-reconstructor.js","sourceRoot":"","sources":["../../../../src/tools/browser/snapshot-filter/tree-reconstructor.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,eAAe,CAAC,YAAuB,EAAE,gBAAiC;IACzF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,MAAM,QAAQ,GAAG,uBAAuB,CAAC,gBAAgB,CAAC,CAAA;IAC1D,OAAO,iBAAiB,CAAC,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,YAAY,CAAA;AACrE,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAyB;IACzD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IACxC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,iBAAiB,CACzB,KAAgB,EAChB,aAA0B,EAC1B,WAAgC;IAEhC,IAAI,CAAC,wBAAwB,CAAC,WAAW,EAAE,aAAa,CAAC,EAAE,CAAC;QAC3D,OAAO,SAAS,CAAA;IACjB,CAAC;IAED,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,KAAK,CAAA;IACb,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,kBAAkB,CAAC,KAAK,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;IAC7D,CAAC;IAED,OAAO,mBAAmB,CAAC,KAAmB,EAAE,aAAa,EAAE,WAAW,CAAC,CAAA;AAC5E,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAyB,EAAE,aAA0B;IACtF,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IACpC,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAwB,CAAA;QAC9D,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CAAC,iBAAsC,EAAE,UAA+B;IAC5F,OAAO,CACN,iBAAiB,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;QAC5C,iBAAiB,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,UAAU,CAAC,KAAK,CAAC,CAAC,CAC1E,CAAA;AACF,CAAC;AAED,SAAS,kBAAkB,CAC1B,KAAgB,EAChB,aAA0B,EAC1B,WAAgC;IAEhC,MAAM,aAAa,GAAgB,EAAE,CAAA;IAErC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC,GAAG,WAAW,EAAE,KAAK,CAAC,CAAC,CAAA;QAC7F,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACjC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAClC,CAAC;IACF,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,qBAAqB,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAA;IAC7E,CAAC;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAED,SAAS,mBAAmB,CAC3B,KAAiB,EACjB,aAA0B,EAC1B,WAAgC;IAEhC,MAAM,cAAc,GAAe,EAAE,CAAA;IACrC,IAAI,UAAU,GAAG,KAAK,CAAA;IAEtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,CAAC,GAAG,WAAW,EAAE,GAAG,CAAC,CAAC,CAAA;QACzF,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACjC,cAAc,CAAC,GAAG,CAAC,GAAG,aAAa,CAAA;YACnC,UAAU,GAAG,IAAI,CAAA;QAClB,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,qBAAqB,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAA;AAC/C,CAAC;AAED,SAAS,qBAAqB,CAAC,aAA0B,EAAE,WAAgC;IAC1F,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAA;IAElD,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAwB,CAAA;QAC9D,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,IAAI,cAAc,KAAK,YAAY,EAAE,CAAC;YAC7E,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC"}

package/dist/tools/browser/snapshot-service.d.ts

@@ -0,0 +1,19 @@
+import { Page } from '@playwright/test';
+import { Step } from '../../runtime/types';
+export type BrowserSnapshot = string | null;
+export interface SnapshotServiceOptions {
+    skipFilter?: boolean;
+}
+export declare class SnapshotService {
+    private readonly page;
+    private readonly step?;
+    private readonly options;
+    private readonly runtimeConfig;
+    constructor(page: Page | null, step?: Step, options?: SnapshotServiceOptions);
+    private getHeader;
+    private minify;
+    private redactAds;
+    private compress;
+    get(): Promise<BrowserSnapshot>;
+}
+//# sourceMappingURL=snapshot-service.d.ts.map

package/dist/tools/browser/snapshot-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-service.d.ts","sourceRoot":"","sources":["../../../src/tools/browser/snapshot-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAIvC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAA;AAG1C,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,IAAI,CAAA;AAE3C,MAAM,WAAW,sBAAsB;IACtC,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,qBAAa,eAAe;IAI1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IALzB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAsB;gBAGlC,IAAI,EAAE,IAAI,GAAG,IAAI,EACjB,IAAI,CAAC,EAAE,IAAI,EACX,OAAO,GAAE,sBAA2B;YAGxC,SAAS;IAQvB,OAAO,CAAC,MAAM;IASd,OAAO,CAAC,SAAS;YAkBH,QAAQ;IAQhB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;CAcrC"}

package/dist/tools/browser/snapshot-service.js

@@ -0,0 +1,67 @@
+import { parse } from 'yaml';
+import { RuntimeConfig } from '../../config/runtime-config';
+import { logger } from '../../logging';
+import { filterSnapshot } from './snapshot-filter';
+export class SnapshotService {
+    page;
+    step;
+    options;
+    runtimeConfig = new RuntimeConfig();
+    constructor(page, step, options = {}) {
+        this.page = page;
+        this.step = step;
+        this.options = options;
+    }
+    async getHeader() {
+        if (!this.page) {
+            throw new Error('Page is not initialized');
+        }
+        return ['page data:', `url: '${this.page.url()}'`, `title: '${await this.page.title()}'`].join('\n');
+    }
+    minify(snapshot) {
+        return snapshot
+            .replaceAll('  ', '')
+            .replaceAll('"', '')
+            .replaceAll('\\', '')
+            .replaceAll(' [', '[')
+            .replaceAll('] ', ']');
+    }
+    redactAds(snapshot) {
+        const adPatterns = [
+            /googleadservices\.com/i,
+            /doubleclick\.net/i,
+            /googlesyndication\.com/i,
+            /googleads/i,
+            /pagead\/aclk/i,
+            /adurl=/i,
+            /\bgad_source\b/i,
+            /\bgclid\b/i,
+        ];
+        const snapshotUrlWrapperRegex = /\{\/url:([^}]+)\}/g;
+        return snapshot.replace(snapshotUrlWrapperRegex, (match, urlContent) => {
+            return adPatterns.some((pattern) => pattern.test(urlContent)) ? '{/url:ADVERT}' : match;
+        });
+    }
+    async compress(snapshot) {
+        const snapshotTree = parse(snapshot)?.[0] ?? { state: 'page is blank - navigate to a relevant page url' };
+        const shouldSkipFilter = this.options.skipFilter || !this.runtimeConfig.isSnapshotFilteringEnabled();
+        const processed = shouldSkipFilter ? snapshotTree : await filterSnapshot(snapshotTree, this.step);
+        const minifiedSnapshot = `page snapshot:\n${this.minify(JSON.stringify(processed))}`;
+        return this.redactAds(`${await this.getHeader()}\n${minifiedSnapshot}`);
+    }
+    async get() {
+        try {
+            if (!this.page) {
+                throw new Error('Page is not initialized');
+            }
+            const rawSnapshot = await this.page.ariaSnapshot({ mode: 'ai' });
+            const compressedSnapshot = await this.compress(rawSnapshot);
+            logger.debug(`created aria page snapshot:\n${compressedSnapshot}`);
+            return compressedSnapshot;
+        }
+        catch (error) {
+            throw new Error(`Failed to create aria page snapshot:\n${error}`, { cause: error });
+        }
+    }
+}
+//# sourceMappingURL=snapshot-service.js.map

package/dist/tools/browser/snapshot-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-service.js","sourceRoot":"","sources":["../../../src/tools/browser/snapshot-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAEtC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAA;AAQlD,MAAM,OAAO,eAAe;IAIT;IACA;IACA;IALD,aAAa,GAAG,IAAI,aAAa,EAAE,CAAA;IAEpD,YACkB,IAAiB,EACjB,IAAW,EACX,UAAkC,EAAE;QAFpC,SAAI,GAAJ,IAAI,CAAa;QACjB,SAAI,GAAJ,IAAI,CAAO;QACX,YAAO,GAAP,OAAO,CAA6B;IACnD,CAAC;IAEI,KAAK,CAAC,SAAS;QACtB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;QAC3C,CAAC;QAED,OAAO,CAAC,YAAY,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrG,CAAC;IAEO,MAAM,CAAC,QAAgB;QAC9B,OAAO,QAAQ;aACb,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;aACpB,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC;aACnB,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;aACpB,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC;aACrB,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACxB,CAAC;IAEO,SAAS,CAAC,QAAgB;QACjC,MAAM,UAAU,GAAG;YAClB,wBAAwB;YACxB,mBAAmB;YACnB,yBAAyB;YACzB,YAAY;YACZ,eAAe;YACf,SAAS;YACT,iBAAiB;YACjB,YAAY;SACZ,CAAA;QACD,MAAM,uBAAuB,GAAG,oBAAoB,CAAA;QAEpD,OAAO,QAAQ,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YACtE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,CAAA;QACxF,CAAC,CAAC,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACtC,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,iDAAiD,EAAE,CAAA;QACzG,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,0BAA0B,EAAE,CAAA;QACpG,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QACjG,MAAM,gBAAgB,GAAG,mBAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,CAAA;QACpF,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,KAAK,gBAAgB,EAAE,CAAC,CAAA;IACxE,CAAC;IAED,KAAK,CAAC,GAAG;QACR,IAAI,CAAC;YACJ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;YAC3C,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YAChE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;YAC3D,MAAM,CAAC,KAAK,CAAC,gCAAgC,kBAAkB,EAAE,CAAC,CAAA;YAClE,OAAO,kBAAkB,CAAA;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAA;QACpF,CAAC;IACF,CAAC;CACD"}

package/dist/tools/browser/tool.d.ts

@@ -0,0 +1,34 @@
+import { Page } from '@playwright/test';
+import { Step } from '../../runtime/types';
+import { AgentTool, AgentToolResponse } from '../types';
+type BrowserInputElement = {
+    ref: string;
+    text: string;
+    name: string;
+    clear: boolean;
+    select: boolean;
+};
+export declare const BrowserTool: {
+    readonly TOOL_NAVIGATE: "browser_navigate";
+    readonly TOOL_CLICK_OR_HOVER: "browser_click_or_hover";
+    readonly TOOL_TYPE_OR_SELECT: "browser_type_or_select";
+    readonly TOOL_PRESS_KEY: "browser_press_key";
+    readonly TOOL_SNAPSHOT: "browser_snapshot";
+    readonly TOOL_WAIT: "browser_wait";
+};
+export declare class BrowserToolRuntime {
+    private readonly page;
+    constructor(page: Page);
+    navigateToUrl(url: string, step: Step): Promise<AgentToolResponse | string>;
+    clickElement(ref: string, hover: boolean, step: Step): Promise<AgentToolResponse | string>;
+    typeOrSelectInElement(elements: BrowserInputElement[], step: Step): Promise<AgentToolResponse | string>;
+    pressKey(key: string, step: Step): Promise<AgentToolResponse | string>;
+    captureCurrentSnapshot(step: Step, options?: {
+        skipFilter?: boolean;
+    }): Promise<string>;
+    wait(seconds: number, step: Step): Promise<AgentToolResponse | string>;
+    private wrapWithTracker;
+}
+export declare function createBrowserTools(runtime: BrowserToolRuntime): AgentTool[];
+export {};
+//# sourceMappingURL=tool.d.ts.map

package/dist/tools/browser/tool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool.d.ts","sourceRoot":"","sources":["../../../src/tools/browser/tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAG/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAA;AAE1C,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAA;AAIvD,KAAK,mBAAmB,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;CACf,CAAA;AAED,eAAO,MAAM,WAAW;;;;;;;CAOd,CAAA;AAcV,qBAAa,kBAAkB;IAClB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,EAAE,IAAI;IAEjC,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,GAAG,MAAM,CAAC;IAkB3E,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,GAAG,MAAM,CAAC;IAmB1F,qBAAqB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,GAAG,MAAM,CAAC;IAsCvG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,GAAG,MAAM,CAAC;IAmBtE,sBAAsB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAqB3F,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,iBAAiB,GAAG,MAAM,CAAC;YAmB9D,eAAe;CAwB7B;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,kBAAkB,GAAG,SAAS,EAAE,CA2E3E"}

package/dist/tools/browser/tool.js

@@ -0,0 +1,226 @@
+import { expect } from '@playwright/test';
+import { z } from 'zod/v4';
+import { logger } from '../../logging';
+import { defineAgentTool } from '../define-agent-tool';
+import { SnapshotService } from './snapshot-service';
+import { TransientStateTracker } from './transient-state-tracker';
+export const BrowserTool = {
+    TOOL_NAVIGATE: 'browser_navigate',
+    TOOL_CLICK_OR_HOVER: 'browser_click_or_hover',
+    TOOL_TYPE_OR_SELECT: 'browser_type_or_select',
+    TOOL_PRESS_KEY: 'browser_press_key',
+    TOOL_SNAPSHOT: 'browser_snapshot',
+    TOOL_WAIT: 'browser_wait',
+};
+const browserInputElementSchema = z
+    .object({
+    ref: z.string().describe('ref value of the element from the snapshot, example: e123'),
+    name: z.string().describe('name of the element to type into, example: Username Input'),
+    text: z.string().describe('The text to type into the element, example: Hello World'),
+    clear: z
+        .boolean()
+        .describe('If true, clears existing text before typing. Use false with dropdowns and selects'),
+    select: z.boolean().describe('If true, selects the option matching "text" from a dropdown or select element'),
+})
+    .strict();
+export class BrowserToolRuntime {
+    page;
+    constructor(page) {
+        this.page = page;
+    }
+    async navigateToUrl(url, step) {
+        return this.wrapWithTracker(async () => {
+            if (!url) {
+                throw new Error(`valid URL is required for ${BrowserTool.TOOL_NAVIGATE} but received: '${url}'`);
+            }
+            try {
+                await this.page.goto(url);
+            }
+            catch (error) {
+                throw new Error(`Failed to navigate to URL ${url}`, { cause: error });
+            }
+        }, `Navigated to: ${url}`, step);
+    }
+    async clickElement(ref, hover, step) {
+        return this.wrapWithTracker(async () => {
+            try {
+                if (hover) {
+                    await this.page.hover(`aria-ref=${ref}`);
+                }
+                else {
+                    await this.page.click(`aria-ref=${ref}`);
+                }
+            }
+            catch (error) {
+                logger.error(`error clicking element with ref '${ref}' due to:\n${error}`);
+                return `failed to click element with ref '${ref}':\n${error} try with different element type or ref`;
+            }
+        }, hover ? `Hovered element with ref '${ref}'.` : `Clicked element with ref '${ref}'.`, step);
+    }
+    async typeOrSelectInElement(elements, step) {
+        return this.wrapWithTracker(async () => {
+            for (const element of elements) {
+                try {
+                    if (!element.ref || element.text === undefined || element.text === null) {
+                        throw new Error(`both 'ref' and 'text' are required for ${BrowserTool.TOOL_TYPE_OR_SELECT} but received ref='${element.ref}' and text='${element.text}'`);
+                    }
+                    if (!element.select && element.clear) {
+                        await this.page.locator(`aria-ref=${element.ref}`).clear();
+                    }
+                    if (element.select && element.text.length > 0) {
+                        await this.page.locator(`aria-ref=${element.ref}`).selectOption(element.text);
+                        continue;
+                    }
+                    if (!element.select && element.text.length > 0) {
+                        await this.page
+                            .locator(`aria-ref=${element.ref}`)
+                            .pressSequentially(element.text, { delay: 50 });
+                    }
+                }
+                catch (error) {
+                    logger.error(`error ${element.select ? 'selecting' : 'typing'} '${element.text}' in element with ref '${element.ref}' due to:\n${error}`);
+                    return `failed to ${element.select ? 'select' : 'type'} '${element.text}' in element with ref '${element.ref}':\n${error}\n try with different element type or ref`;
+                }
+            }
+        }, `Updated ${elements.length} page element${elements.length === 1 ? '' : 's'}.`, step);
+    }
+    async pressKey(key, step) {
+        return this.wrapWithTracker(async () => {
+            try {
+                if (!key) {
+                    throw new Error(`'key' is required for ${BrowserTool.TOOL_PRESS_KEY} but received: '${key}'`);
+                }
+                await this.page.keyboard.press(key);
+            }
+            catch (error) {
+                logger.error(`error pressing key '${key}' due to:\n${error}`);
+                return `failed to press key '${key}':\n${error}`;
+            }
+        }, `Pressed key '${key}'.`, step);
+    }
+    async captureCurrentSnapshot(step, options = {}) {
+        try {
+            await expect
+                .poll(async () => {
+                const readHtml = async () => this.page.locator('html').innerHTML();
+                const first = await readHtml();
+                await this.page.waitForTimeout(500);
+                const second = await readHtml();
+                return first !== second ? 'not stable' : 'stable';
+            }, { timeout: 30_000, message: 'page snapshots should be stable' })
+                .toEqual('stable');
+            return new SnapshotService(this.page, step, { skipFilter: options.skipFilter }).get();
+        }
+        catch (error) {
+            throw new Error(`Failed to capture page snapshot:\n${error}`, { cause: error });
+        }
+    }
+    async wait(seconds, step) {
+        return this.wrapWithTracker(async () => {
+            try {
+                if (!Number.isFinite(seconds) || seconds <= 0) {
+                    throw new Error(`invalid seconds value received: ${seconds}. It should be a positive number.`);
+                }
+                await this.page.waitForTimeout(seconds * 1000);
+            }
+            catch (error) {
+                logger.error(`error waiting for ${seconds} seconds due to:\n${error}`);
+                return `failed to wait for ${seconds} seconds:\n${error}`;
+            }
+        }, `Waited ${seconds} seconds.`, step);
+    }
+    async wrapWithTracker(action, fallbackResponse, step) {
+        const tracker = new TransientStateTracker(this.page);
+        await tracker.start();
+        try {
+            const actionResult = await action();
+            if (typeof actionResult === 'string') {
+                await tracker.stop();
+                return actionResult;
+            }
+            const snapshot = await this.captureCurrentSnapshot(step);
+            await tracker.stop();
+            const formattedTimeline = tracker.formatTimeline();
+            return { response: formattedTimeline || fallbackResponse, snapshot };
+        }
+        catch (error) {
+            await tracker.stop();
+            throw error;
+        }
+    }
+}
+export function createBrowserTools(runtime) {
+    return [
+        defineAgentTool({
+            name: BrowserTool.TOOL_NAVIGATE,
+            description: 'Navigate to a specified URL in the browser, example: https://www.example.com',
+            schema: z
+                .object({
+                url: z.string().describe('The URL to navigate to'),
+                goal: z.string().describe('The goal or purpose of navigating to this URL'),
+            })
+                .strict(),
+            handler: ({ url }, context) => runtime.navigateToUrl(url, context.step),
+        }),
+        defineAgentTool({
+            name: BrowserTool.TOOL_CLICK_OR_HOVER,
+            description: 'Click or hover a specified element reference in the browser',
+            schema: z
+                .object({
+                ref: z.string().describe('ref value of the element from the snapshot, example: e123'),
+                name: z.string().describe('name of the element to click or hover, example: Submit Button'),
+                hover: z.boolean().describe('If true: hover the element, if false: click element'),
+                goal: z.string().describe('The goal or purpose of clicking this element'),
+            })
+                .strict(),
+            handler: ({ ref, hover }, context) => runtime.clickElement(ref, hover, context.step),
+        }),
+        defineAgentTool({
+            name: BrowserTool.TOOL_TYPE_OR_SELECT,
+            description: 'Type text into a specified element reference in the browser or select an option from a dropdown.',
+            schema: z
+                .object({
+                elements: z.array(browserInputElementSchema).min(1).describe('array of elements to type into'),
+                goal: z.string().describe('The goal or purpose of typing this text into the element'),
+            })
+                .strict(),
+            handler: ({ elements }, context) => runtime.typeOrSelectInElement(elements, context.step),
+        }),
+        defineAgentTool({
+            name: BrowserTool.TOOL_PRESS_KEY,
+            description: 'Press a specified key in the browser',
+            schema: z
+                .object({
+                key: z.string().describe('The key to press, example: Enter, Escape, ArrowDown'),
+                goal: z.string().describe('The goal or purpose of pressing this key'),
+            })
+                .strict(),
+            handler: ({ key }, context) => runtime.pressKey(key, context.step),
+        }),
+        defineAgentTool({
+            name: BrowserTool.TOOL_SNAPSHOT,
+            description: 'Capture the ARIA snapshot of the current page - use only if you miss information from previous snapshot.',
+            schema: z
+                .object({
+                goal: z.string().describe('The goal or purpose of capturing the snapshot'),
+            })
+                .strict(),
+            handler: async (_args, context) => ({
+                response: 'Captured latest page snapshot.',
+                snapshot: await runtime.captureCurrentSnapshot(context.step, { skipFilter: true }),
+            }),
+        }),
+        defineAgentTool({
+            name: BrowserTool.TOOL_WAIT,
+            description: 'Wait for a timeout in seconds',
+            schema: z
+                .object({
+                seconds: z.number().describe('Number of seconds to wait, example: 5'),
+                goal: z.string().describe('The goal or purpose of waiting'),
+            })
+                .strict(),
+            handler: ({ seconds }, context) => runtime.wait(seconds, context.step),
+        }),
+    ];
+}
+//# sourceMappingURL=tool.js.map