autokap 1.9.9 → 2.0.0

package/dist/opcode-runner.js

@@ -56,13 +56,18 @@ export class NoOpRecoveryChain {
 const MIN_CLIP_FINALIZATION_TIMEOUT_MS = 30000;
 const DEFAULT_VIDEO_RECORDING_RESOLUTION = { width: 1920, height: 1080 };
 /**
- * Per-call bounds for the CAPTURE_SCREENSHOT action, kept well under the ~30s
- * global capture deadline so a hung Playwright call fails early and leaves budget
- * for recovery/recapture instead of voiding the variant. Fixed constants because
- * executeOpcodeAction has no access to the deadline; 12s + 3×3s worst case still
- * leaves >=12s of the 30s budget for the recovery path.
+ * Recovery budget reserved from the global capture deadline. The screenshot
+ * buffer (the artifact-critical call) may use the REST of the deadline, so a
+ * slow-but-completing high-res capture (e.g. a 5K device frame legitimately
+ * needing 15-30s under load) succeeds, while a truly hung one still fails with
+ * this slice left for the recovery/recapture path. Replaces the old flat 12s
+ * buffer cap, which was smaller than legitimate heavy captures and killed them
+ * (and killed the recapture too, since it re-applied the same 12s wall).
  */
-const CAPTURE_BUFFER_TIMEOUT_MS = 12000;
+const CAPTURE_RECOVERY_RESERVE_MS = 10000;
+/** Floor for the first-attempt capture budget when little global time remains. */
+const MIN_CAPTURE_ACTION_BUDGET_MS = 8000;
+/** Best-effort bound for browser-mockup metadata (URL / favicon / title). */
 const CAPTURE_METADATA_TIMEOUT_MS = 3000;
 /**
  * The compiled per-opcode action budget. For CAPTURE_SCREENSHOT this governs
@@ -510,7 +515,15 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
         const isPureWait = opcode.kind === 'WAIT_FOR';
         const usesGlobalDeadline = isPureWait || isArtifactProducingOpcode(opcode.kind);
         const actionDeadlineMs = usesGlobalDeadline ? globalDeadlineMs : deadlineMs;
-        const actionBudgetMs = getRemainingTimeMs(actionDeadlineMs);
+        let actionBudgetMs = getRemainingTimeMs(actionDeadlineMs);
+        // Reserve a slice of the global deadline for recovery/recapture on the
+        // artifact-critical CAPTURE_SCREENSHOT: the capture uses the rest (so a slow
+        // high-res shot completes) yet a hung one still leaves room to recover.
+        // END_CLIP keeps its full budget — its finalization hard-fails without a
+        // recapture, so reserving would only shorten it.
+        if (opcode.kind === 'CAPTURE_SCREENSHOT') {
+            actionBudgetMs = Math.max(MIN_CAPTURE_ACTION_BUDGET_MS, actionBudgetMs - CAPTURE_RECOVERY_RESERVE_MS);
+        }
         if (actionBudgetMs <= 0) {
             const reason = `timeout after ${effectiveTimeoutMs}ms`;
             logger.debug(`[opcode ${index}] no budget left after captureBeforeState (deadline=${actionDeadlineMs}, now=${Date.now()})`);
@@ -891,13 +904,13 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
                     lowConfidenceReasons.push(`captured before visual stability: ${stability.reason}`);
                 }
                 const captureLowConfidenceReason = lowConfidenceReasons.join('; ') || undefined;
-                // Capture the screenshot buffer FIRST and bound it — this is the only
-                // artifact-critical call. Under CPU contention (many cloud variants per
-                // container) a Playwright screenshot can otherwise block the full ~30s
-                // global deadline, leaving zero budget for recovery/recapture and voiding
-                // the variant. Failing at ~12s keeps ≥12s of the 30s budget for the
-                // recovery path (handleFailure → recapture in `failWithRecovery`).
-                const buffer = await withTimeout(() => takeCaptureBuffer(adapter, opcode), CAPTURE_BUFFER_TIMEOUT_MS);
+                // Capture the screenshot buffer FIRST — the only artifact-critical call.
+                // It is bounded by the reserved action budget (the withTimeout around this
+                // whole action in executeOpcode) and the screenshot primitive's own
+                // timeout — NOT a flat sub-cap — so a legitimately slow high-res capture
+                // can use most of the deadline while a hung one still frees the reserved
+                // recovery slice (handleFailure → recapture in `failWithRecovery`).
+                const buffer = await takeCaptureBuffer(adapter, opcode);
                 // Everything below is best-effort browser-mockup metadata. The buffer is
                 // already in hand, so a slow/hung metadata call must NEVER throw out of
                 // the action and discard a good screenshot. Each is individually bounded

package/dist/program-signing.d.ts

@@ -277,13 +277,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"ASSERT_SURFACE">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             button: z.ZodOptional<z.ZodEnum<{
@@ -322,13 +326,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"CLICK">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             text: z.ZodString;
@@ -400,10 +408,14 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             state: z.ZodEnum<{
@@ -579,10 +591,14 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             targetSelector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             description: z.ZodString;
@@ -722,13 +738,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"END_CLIP">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             fingerprint: z.ZodOptional<z.ZodString>;
@@ -763,13 +783,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"HOVER">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             optionLabel: z.ZodOptional<z.ZodString>;
@@ -807,13 +831,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"SELECT_OPTION">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             checked: z.ZodBoolean;
@@ -849,13 +877,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"CHECK">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             fingerprint: z.ZodOptional<z.ZodString>;
@@ -890,13 +922,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             stepId: z.ZodOptional<z.ZodString>;
             kind: z.ZodLiteral<"DOUBLE_CLICK">;
         }, z.core.$strict>, z.ZodObject<{
-            selector: z.ZodString;
+            selector: z.ZodOptional<z.ZodString>;
             target: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             fingerprint: z.ZodOptional<z.ZodString>;
@@ -904,10 +940,14 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
             toSelector: z.ZodOptional<z.ZodString>;
             toTarget: z.ZodOptional<z.ZodObject<{
                 text: z.ZodOptional<z.ZodString>;
+                textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 role: z.ZodOptional<z.ZodString>;
                 label: z.ZodOptional<z.ZodString>;
+                labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 near: z.ZodOptional<z.ZodString>;
+                nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 placeholder: z.ZodOptional<z.ZodString>;
+                placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
                 exact: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             toSelectorAlternates: z.ZodOptional<z.ZodArray<z.ZodString>>;

package/dist/recovery-chain.js

@@ -8,7 +8,7 @@
  * 4. Targeted reload (reload page and retry)
  * 5. LLM Healer (last resort)
  */
-import { resolveSelector } from './selector-resolver.js';
+import { resolveSelector, resolveSelectorByFuzzyText, semanticTargetFuzzyText, } from './selector-resolver.js';
 import { evaluatePostcondition, evaluatePostconditionWithProgress } from './postcondition.js';
 import { LLMHealer } from './llm-healer.js';
 import { serializeAKTree } from './ak-tree.js';
@@ -41,7 +41,7 @@ export class RecoveryChainImpl {
                 return result;
         }
         // Strategy 2: Selector memory + alternate selectors
-        if (recovery.useSelectorMemory && hasSelector(failedOpcode)) {
+        if (recovery.useSelectorMemory && (hasSelector(failedOpcode) || hasSemanticTarget(failedOpcode))) {
             logger.debug(`[recovery ${opcodeIndex}] strategy 2 (selector memory)`);
             const result = await trySelectorAlternatives(failedOpcode, opcodeIndex, adapter, this.selectorMemory, this.credentials, options.currentVariant);
             logger.debug(`[recovery ${opcodeIndex}] strategy 2 → recovered=${result.recovered}, reason=${result.reason}`);
@@ -49,7 +49,7 @@ export class RecoveryChainImpl {
                 return result;
         }
         // Strategy 3: Alternative interaction methods
-        if (recovery.useAltInteraction && hasSelector(failedOpcode)) {
+        if (recovery.useAltInteraction && (hasSelector(failedOpcode) || hasSemanticTarget(failedOpcode))) {
             logger.debug(`[recovery ${opcodeIndex}] strategy 3 (alt interaction: keyboard, js_dispatch, coords)`);
             const result = await tryAltInteraction(failedOpcode, adapter, this.credentials);
             logger.debug(`[recovery ${opcodeIndex}] strategy 3 → recovered=${result.recovered}, reason=${result.reason}`);
@@ -114,14 +114,14 @@ async function retryOpcode(opcode, adapter, maxRetries, remainingTimeMs, current
 }
 // ── Strategy 2: Selector memory ─────────────────────────────────────
 async function trySelectorAlternatives(opcode, opcodeIndex, adapter, selectorMemory, credentials, currentVariant) {
-    if (!hasSelector(opcode)) {
-        return { recovered: false, reason: 'opcode has no selector' };
+    if (!hasSelector(opcode) && !hasSemanticTarget(opcode)) {
+        return { recovered: false, reason: 'opcode has no selector or target' };
     }
     const selectorOpcode = opcode;
-    const primarySelector = selectorOpcode.selector ?? '';
-    const stepSignature = `${opcode.kind}:${primarySelector}`;
+    const stepSignature = buildStepSignature(opcode);
     const resolved = await resolveSelector(adapter, {
-        primary: primarySelector,
+        primary: selectorOpcode.selector,
+        target: selectorOpcode.target,
         fingerprint: selectorOpcode.fingerprint,
         alternates: selectorOpcode.selectorAlternates,
         selectorMemory,
@@ -150,11 +150,26 @@ async function trySelectorAlternatives(opcode, opcodeIndex, adapter, selectorMem
 }
 // ── Strategy 3: Alternative interaction ─────────────────────────────
 async function tryAltInteraction(opcode, adapter, _credentials) {
-    if (opcode.kind !== 'CLICK' || !opcode.selector) {
-        return { recovered: false, reason: 'alt interaction only for CLICK opcodes with selectors' };
+    if (opcode.kind !== 'CLICK') {
+        return { recovered: false, reason: 'alt interaction only for CLICK opcodes' };
     }
     const clickOpcode = opcode;
-    const sel = clickOpcode.selector;
+    let sel = clickOpcode.selector;
+    if (!sel && clickOpcode.target) {
+        // Target-only opcode: resolve a concrete selector from the AKTree using
+        // the target's text/label so the alternative interaction methods have
+        // something to act on.
+        try {
+            const tree = await adapter.getAKTree();
+            sel = resolveSelectorByFuzzyText(tree, semanticTargetFuzzyText(clickOpcode.target)) ?? undefined;
+        }
+        catch {
+            // fall through to the guard below
+        }
+    }
+    if (!sel) {
+        return { recovered: false, reason: 'alt interaction needs a selector or a target resolvable in the AKTree' };
+    }
     const methods = [
         { name: 'keyboard', options: { useKeyboard: true } },
         { name: 'js_dispatch', options: { useJsDispatch: true } },
@@ -291,6 +306,30 @@ async function tryHealer(failedOpcode, opcodeIndex, adapter, healer, programStep
 function hasSelector(opcode) {
     return 'selector' in opcode && typeof opcode.selector === 'string' && !!opcode.selector;
 }
+function hasSemanticTarget(opcode) {
+    return 'target' in opcode && Boolean(opcode.target);
+}
+/**
+ * Stable signature for selector-memory lookups. Selector-based opcodes keep
+ * the legacy `KIND:selector` form; target-only opcodes derive a signature
+ * from the semantic descriptor so memory survives across runs.
+ */
+function buildStepSignature(opcode) {
+    const { selector, target } = opcode;
+    if (selector)
+        return `${opcode.kind}:${selector}`;
+    if (target) {
+        const name = target.label
+            ?? target.text
+            ?? target.placeholder
+            ?? Object.values(target.labelByLocale ?? {})[0]
+            ?? Object.values(target.textByLocale ?? {})[0]
+            ?? Object.values(target.placeholderByLocale ?? {})[0]
+            ?? '';
+        return `${opcode.kind}:target:${target.role ?? ''}/${name}`;
+    }
+    return `${opcode.kind}:`;
+}
 async function executeRawAction(opcode, adapter, currentVariant, credentials, suppressPageReloads = false) {
     const result = await executeOpcodeCoreAction(opcode, adapter, {
         currentVariant,
@@ -302,7 +341,7 @@ async function executeRawAction(opcode, adapter, currentVariant, credentials, su
     }
 }
 async function executeHealerPatchedAction(opcode, adapter, interactionMode, currentVariant, credentials, suppressPageReloads = false) {
-    if (opcode.kind === 'CLICK' && interactionMode && interactionMode !== 'default') {
+    if (opcode.kind === 'CLICK' && opcode.selector && interactionMode && interactionMode !== 'default') {
         const options = interactionMode === 'keyboard'
             ? { useKeyboard: true }
             : interactionMode === 'js_dispatch'

package/dist/selector-resolver.d.ts

@@ -7,15 +7,18 @@
  * 3. Selector memory candidates (from Supabase)
  * 4. Fuzzy text match on AKTree nodes
  */
-import type { RuntimeAdapter } from './execution-types.js';
+import type { AKTree } from './types.js';
+import type { RuntimeAdapter, SemanticTarget } from './execution-types.js';
 export interface ResolvedSelector {
     selector: string;
-    strategy: 'primary' | 'fingerprint' | 'memory' | 'fuzzy_text' | 'alternate';
+    strategy: 'primary' | 'fingerprint' | 'memory' | 'fuzzy_text' | 'alternate' | 'target_text';
     confidence: 'high' | 'medium' | 'low';
 }
 export interface SelectorResolverOptions {
-    /** Primary selector from the compiled opcode */
-    primary: string;
+    /** Primary selector from the compiled opcode. Absent for target-only opcodes. */
+    primary?: string;
+    /** Semantic target from the compiled opcode (for text-based AKTree matching) */
+    target?: SemanticTarget;
     /** AKTree fingerprint for fuzzy matching */
     fingerprint?: string;
     /** Alternative selectors from the compiled opcode */
@@ -32,3 +35,15 @@ export interface SelectorResolverOptions {
  * Returns the first working selector, or null if nothing matches.
  */
 export declare function resolveSelector(adapter: RuntimeAdapter, options: SelectorResolverOptions): Promise<ResolvedSelector | null>;
+/**
+ * Flattens a SemanticTarget into the text used for AKTree fuzzy matching:
+ * base label/text/placeholder plus every locale variant. Returns '' when the
+ * target carries no text at all (e.g. role-only).
+ */
+export declare function semanticTargetFuzzyText(target: SemanticTarget): string;
+/**
+ * Resolves a CSS selector for a visible interactive AKTree node matching the
+ * given text. Used by recovery strategies that need a concrete selector for a
+ * target-only opcode (e.g. coordinate clicks).
+ */
+export declare function resolveSelectorByFuzzyText(tree: AKTree, text: string): string | null;

package/dist/selector-resolver.js

@@ -14,7 +14,7 @@
 export async function resolveSelector(adapter, options) {
     const tree = await adapter.getAKTree();
     // 1. Try primary selector
-    if (await selectorExists(adapter, options.primary)) {
+    if (options.primary && await selectorExists(adapter, options.primary)) {
         return { selector: options.primary, strategy: 'primary', confidence: 'high' };
     }
     // 2. Try alternate selectors
@@ -32,7 +32,17 @@ export async function resolveSelector(adapter, options) {
             return { selector: match, strategy: 'fingerprint', confidence: 'medium' };
         }
     }
-    // 4. Try selector memory
+    // 4. Try the semantic target's text/label against the AKTree
+    if (options.target) {
+        const targetText = semanticTargetFuzzyText(options.target);
+        if (targetText) {
+            const match = findNodeByFuzzyText(tree, targetText);
+            if (match) {
+                return { selector: match, strategy: 'target_text', confidence: 'medium' };
+            }
+        }
+    }
+    // 5. Try selector memory
     if (options.selectorMemory && options.stepSignature) {
         const candidates = options.selectorMemory[options.stepSignature];
         if (candidates) {
@@ -43,7 +53,7 @@ export async function resolveSelector(adapter, options) {
             }
         }
     }
-    // 5. Try fuzzy text match
+    // 6. Try fuzzy text match on the opcode description
     if (options.description) {
         const match = findNodeByFuzzyText(tree, options.description);
         if (match) {
@@ -52,6 +62,32 @@ export async function resolveSelector(adapter, options) {
     }
     return null;
 }
+/**
+ * Flattens a SemanticTarget into the text used for AKTree fuzzy matching:
+ * base label/text/placeholder plus every locale variant. Returns '' when the
+ * target carries no text at all (e.g. role-only).
+ */
+export function semanticTargetFuzzyText(target) {
+    const parts = [
+        target.label,
+        target.text,
+        target.placeholder,
+        ...Object.values(target.labelByLocale ?? {}),
+        ...Object.values(target.textByLocale ?? {}),
+        ...Object.values(target.placeholderByLocale ?? {}),
+    ].filter((value) => Boolean(value && value.trim()));
+    return parts.join(' ');
+}
+/**
+ * Resolves a CSS selector for a visible interactive AKTree node matching the
+ * given text. Used by recovery strategies that need a concrete selector for a
+ * target-only opcode (e.g. coordinate clicks).
+ */
+export function resolveSelectorByFuzzyText(tree, text) {
+    if (!text.trim())
+        return null;
+    return findNodeByFuzzyText(tree, text);
+}
 // ── Helpers ─────────────────────────────────────────────────────────
 async function selectorExists(adapter, selector) {
     try {

package/dist/semantic-resolver.d.ts

@@ -25,6 +25,8 @@ export interface ResolveOptions {
     target?: SemanticTarget;
     /** Fallback selectors */
     selectorAlternates?: string[];
+    /** Active variant locale used to resolve the target's `*ByLocale` maps */
+    locale?: string;
     /** Timeout for visibility check (ms). Default: 3000 */
     timeoutMs?: number;
 }
@@ -33,3 +35,9 @@ export interface ResolveOptions {
  * Returns the first visible locator found, or null if nothing matches.
  */
 export declare function resolveTarget(page: Page, options: ResolveOptions): Promise<ResolvedTarget | null>;
+/**
+ * Projects a SemanticTarget onto a single locale: each `*ByLocale` map is
+ * collapsed into its base field. Lookup order: exact locale tag → primary
+ * language subtag → base field → first non-empty map value.
+ */
+export declare function localizeSemanticTarget(target: SemanticTarget, locale?: string): SemanticTarget;