autokap 1.9.10 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. package/dist/execution-schema.d.ts +382 -182
  2. package/dist/execution-schema.js +40 -13
  3. package/dist/execution-types.d.ts +64 -33
  4. package/dist/llm-healer.js +28 -16
  5. package/dist/opcode-actions.js +139 -42
  6. package/dist/program-signing.d.ts +47 -7
  7. package/dist/recovery-chain.js +51 -12
  8. package/dist/selector-resolver.d.ts +19 -4
  9. package/dist/selector-resolver.js +39 -3
  10. package/dist/semantic-resolver.d.ts +8 -0
  11. package/dist/semantic-resolver.js +0 -0
  12. package/dist/video-narration-schema.d.ts +47 -7
  13. package/dist/web-playwright-local.d.ts +12 -0
  14. package/dist/web-playwright-local.js +53 -0
  15. package/package.json +1 -1
package/dist/program-signing.d.ts CHANGED
@@ -277,13 +277,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
277
277
  stepId: z.ZodOptional<z.ZodString>;
278
278
  kind: z.ZodLiteral<"ASSERT_SURFACE">;
279
279
  }, z.core.$strict>, z.ZodObject<{
280
- selector: z.ZodString;
280
+ selector: z.ZodOptional<z.ZodString>;
281
281
  target: z.ZodOptional<z.ZodObject<{
282
282
  text: z.ZodOptional<z.ZodString>;
283
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
283
284
  role: z.ZodOptional<z.ZodString>;
284
285
  label: z.ZodOptional<z.ZodString>;
286
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
285
287
  near: z.ZodOptional<z.ZodString>;
288
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
286
289
  placeholder: z.ZodOptional<z.ZodString>;
290
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
287
291
  exact: z.ZodOptional<z.ZodBoolean>;
288
292
  }, z.core.$strict>>;
289
293
  button: z.ZodOptional<z.ZodEnum<{
@@ -322,13 +326,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
322
326
  stepId: z.ZodOptional<z.ZodString>;
323
327
  kind: z.ZodLiteral<"CLICK">;
324
328
  }, z.core.$strict>, z.ZodObject<{
325
- selector: z.ZodString;
329
+ selector: z.ZodOptional<z.ZodString>;
326
330
  target: z.ZodOptional<z.ZodObject<{
327
331
  text: z.ZodOptional<z.ZodString>;
332
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
328
333
  role: z.ZodOptional<z.ZodString>;
329
334
  label: z.ZodOptional<z.ZodString>;
335
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
330
336
  near: z.ZodOptional<z.ZodString>;
337
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
331
338
  placeholder: z.ZodOptional<z.ZodString>;
339
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
332
340
  exact: z.ZodOptional<z.ZodBoolean>;
333
341
  }, z.core.$strict>>;
334
342
  text: z.ZodString;
@@ -400,10 +408,14 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
400
408
  selector: z.ZodOptional<z.ZodString>;
401
409
  target: z.ZodOptional<z.ZodObject<{
402
410
  text: z.ZodOptional<z.ZodString>;
411
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
403
412
  role: z.ZodOptional<z.ZodString>;
404
413
  label: z.ZodOptional<z.ZodString>;
414
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
405
415
  near: z.ZodOptional<z.ZodString>;
416
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
406
417
  placeholder: z.ZodOptional<z.ZodString>;
418
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
407
419
  exact: z.ZodOptional<z.ZodBoolean>;
408
420
  }, z.core.$strict>>;
409
421
  state: z.ZodEnum<{
@@ -579,10 +591,14 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
579
591
  targetSelector: z.ZodOptional<z.ZodString>;
580
592
  target: z.ZodOptional<z.ZodObject<{
581
593
  text: z.ZodOptional<z.ZodString>;
594
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
582
595
  role: z.ZodOptional<z.ZodString>;
583
596
  label: z.ZodOptional<z.ZodString>;
597
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
584
598
  near: z.ZodOptional<z.ZodString>;
599
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
585
600
  placeholder: z.ZodOptional<z.ZodString>;
601
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
586
602
  exact: z.ZodOptional<z.ZodBoolean>;
587
603
  }, z.core.$strict>>;
588
604
  description: z.ZodString;
@@ -722,13 +738,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
722
738
  stepId: z.ZodOptional<z.ZodString>;
723
739
  kind: z.ZodLiteral<"END_CLIP">;
724
740
  }, z.core.$strict>, z.ZodObject<{
725
- selector: z.ZodString;
741
+ selector: z.ZodOptional<z.ZodString>;
726
742
  target: z.ZodOptional<z.ZodObject<{
727
743
  text: z.ZodOptional<z.ZodString>;
744
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
728
745
  role: z.ZodOptional<z.ZodString>;
729
746
  label: z.ZodOptional<z.ZodString>;
747
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
730
748
  near: z.ZodOptional<z.ZodString>;
749
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
731
750
  placeholder: z.ZodOptional<z.ZodString>;
751
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
732
752
  exact: z.ZodOptional<z.ZodBoolean>;
733
753
  }, z.core.$strict>>;
734
754
  fingerprint: z.ZodOptional<z.ZodString>;
@@ -763,13 +783,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
763
783
  stepId: z.ZodOptional<z.ZodString>;
764
784
  kind: z.ZodLiteral<"HOVER">;
765
785
  }, z.core.$strict>, z.ZodObject<{
766
- selector: z.ZodString;
786
+ selector: z.ZodOptional<z.ZodString>;
767
787
  target: z.ZodOptional<z.ZodObject<{
768
788
  text: z.ZodOptional<z.ZodString>;
789
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
769
790
  role: z.ZodOptional<z.ZodString>;
770
791
  label: z.ZodOptional<z.ZodString>;
792
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
771
793
  near: z.ZodOptional<z.ZodString>;
794
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
772
795
  placeholder: z.ZodOptional<z.ZodString>;
796
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
773
797
  exact: z.ZodOptional<z.ZodBoolean>;
774
798
  }, z.core.$strict>>;
775
799
  optionLabel: z.ZodOptional<z.ZodString>;
@@ -807,13 +831,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
807
831
  stepId: z.ZodOptional<z.ZodString>;
808
832
  kind: z.ZodLiteral<"SELECT_OPTION">;
809
833
  }, z.core.$strict>, z.ZodObject<{
810
- selector: z.ZodString;
834
+ selector: z.ZodOptional<z.ZodString>;
811
835
  target: z.ZodOptional<z.ZodObject<{
812
836
  text: z.ZodOptional<z.ZodString>;
837
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
813
838
  role: z.ZodOptional<z.ZodString>;
814
839
  label: z.ZodOptional<z.ZodString>;
840
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
815
841
  near: z.ZodOptional<z.ZodString>;
842
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
816
843
  placeholder: z.ZodOptional<z.ZodString>;
844
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
817
845
  exact: z.ZodOptional<z.ZodBoolean>;
818
846
  }, z.core.$strict>>;
819
847
  checked: z.ZodBoolean;
@@ -849,13 +877,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
849
877
  stepId: z.ZodOptional<z.ZodString>;
850
878
  kind: z.ZodLiteral<"CHECK">;
851
879
  }, z.core.$strict>, z.ZodObject<{
852
- selector: z.ZodString;
880
+ selector: z.ZodOptional<z.ZodString>;
853
881
  target: z.ZodOptional<z.ZodObject<{
854
882
  text: z.ZodOptional<z.ZodString>;
883
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
855
884
  role: z.ZodOptional<z.ZodString>;
856
885
  label: z.ZodOptional<z.ZodString>;
886
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
857
887
  near: z.ZodOptional<z.ZodString>;
888
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
858
889
  placeholder: z.ZodOptional<z.ZodString>;
890
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
859
891
  exact: z.ZodOptional<z.ZodBoolean>;
860
892
  }, z.core.$strict>>;
861
893
  fingerprint: z.ZodOptional<z.ZodString>;
@@ -890,13 +922,17 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
890
922
  stepId: z.ZodOptional<z.ZodString>;
891
923
  kind: z.ZodLiteral<"DOUBLE_CLICK">;
892
924
  }, z.core.$strict>, z.ZodObject<{
893
- selector: z.ZodString;
925
+ selector: z.ZodOptional<z.ZodString>;
894
926
  target: z.ZodOptional<z.ZodObject<{
895
927
  text: z.ZodOptional<z.ZodString>;
928
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
896
929
  role: z.ZodOptional<z.ZodString>;
897
930
  label: z.ZodOptional<z.ZodString>;
931
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
898
932
  near: z.ZodOptional<z.ZodString>;
933
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
899
934
  placeholder: z.ZodOptional<z.ZodString>;
935
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
900
936
  exact: z.ZodOptional<z.ZodBoolean>;
901
937
  }, z.core.$strict>>;
902
938
  fingerprint: z.ZodOptional<z.ZodString>;
@@ -904,10 +940,14 @@ export declare const SignedExecutionProgramEnvelopeSchema: z.ZodObject<{
904
940
  toSelector: z.ZodOptional<z.ZodString>;
905
941
  toTarget: z.ZodOptional<z.ZodObject<{
906
942
  text: z.ZodOptional<z.ZodString>;
943
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
907
944
  role: z.ZodOptional<z.ZodString>;
908
945
  label: z.ZodOptional<z.ZodString>;
946
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
909
947
  near: z.ZodOptional<z.ZodString>;
948
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
910
949
  placeholder: z.ZodOptional<z.ZodString>;
950
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
911
951
  exact: z.ZodOptional<z.ZodBoolean>;
912
952
  }, z.core.$strict>>;
913
953
  toSelectorAlternates: z.ZodOptional<z.ZodArray<z.ZodString>>;
package/dist/recovery-chain.js CHANGED
@@ -8,7 +8,7 @@
8
8
  * 4. Targeted reload (reload page and retry)
9
9
  * 5. LLM Healer (last resort)
10
10
  */
11
- import { resolveSelector } from './selector-resolver.js';
11
+ import { resolveSelector, resolveSelectorByFuzzyText, semanticTargetFuzzyText, } from './selector-resolver.js';
12
12
  import { evaluatePostcondition, evaluatePostconditionWithProgress } from './postcondition.js';
13
13
  import { LLMHealer } from './llm-healer.js';
14
14
  import { serializeAKTree } from './ak-tree.js';
@@ -41,7 +41,7 @@ export class RecoveryChainImpl {
41
41
  return result;
42
42
  }
43
43
  // Strategy 2: Selector memory + alternate selectors
44
- if (recovery.useSelectorMemory && hasSelector(failedOpcode)) {
44
+ if (recovery.useSelectorMemory && (hasSelector(failedOpcode) || hasSemanticTarget(failedOpcode))) {
45
45
  logger.debug(`[recovery ${opcodeIndex}] strategy 2 (selector memory)`);
46
46
  const result = await trySelectorAlternatives(failedOpcode, opcodeIndex, adapter, this.selectorMemory, this.credentials, options.currentVariant);
47
47
  logger.debug(`[recovery ${opcodeIndex}] strategy 2 → recovered=${result.recovered}, reason=${result.reason}`);
@@ -49,7 +49,7 @@ export class RecoveryChainImpl {
49
49
  return result;
50
50
  }
51
51
  // Strategy 3: Alternative interaction methods
52
- if (recovery.useAltInteraction && hasSelector(failedOpcode)) {
52
+ if (recovery.useAltInteraction && (hasSelector(failedOpcode) || hasSemanticTarget(failedOpcode))) {
53
53
  logger.debug(`[recovery ${opcodeIndex}] strategy 3 (alt interaction: keyboard, js_dispatch, coords)`);
54
54
  const result = await tryAltInteraction(failedOpcode, adapter, this.credentials);
55
55
  logger.debug(`[recovery ${opcodeIndex}] strategy 3 → recovered=${result.recovered}, reason=${result.reason}`);
@@ -114,14 +114,14 @@ async function retryOpcode(opcode, adapter, maxRetries, remainingTimeMs, current
114
114
  }
115
115
  // ── Strategy 2: Selector memory ─────────────────────────────────────
116
116
  async function trySelectorAlternatives(opcode, opcodeIndex, adapter, selectorMemory, credentials, currentVariant) {
117
- if (!hasSelector(opcode)) {
118
- return { recovered: false, reason: 'opcode has no selector' };
117
+ if (!hasSelector(opcode) && !hasSemanticTarget(opcode)) {
118
+ return { recovered: false, reason: 'opcode has no selector or target' };
119
119
  }
120
120
  const selectorOpcode = opcode;
121
- const primarySelector = selectorOpcode.selector ?? '';
122
- const stepSignature = `${opcode.kind}:${primarySelector}`;
121
+ const stepSignature = buildStepSignature(opcode);
123
122
  const resolved = await resolveSelector(adapter, {
124
- primary: primarySelector,
123
+ primary: selectorOpcode.selector,
124
+ target: selectorOpcode.target,
125
125
  fingerprint: selectorOpcode.fingerprint,
126
126
  alternates: selectorOpcode.selectorAlternates,
127
127
  selectorMemory,
@@ -150,11 +150,26 @@ async function trySelectorAlternatives(opcode, opcodeIndex, adapter, selectorMem
150
150
  }
151
151
  // ── Strategy 3: Alternative interaction ─────────────────────────────
152
152
  async function tryAltInteraction(opcode, adapter, _credentials) {
153
- if (opcode.kind !== 'CLICK' || !opcode.selector) {
154
- return { recovered: false, reason: 'alt interaction only for CLICK opcodes with selectors' };
153
+ if (opcode.kind !== 'CLICK') {
154
+ return { recovered: false, reason: 'alt interaction only for CLICK opcodes' };
155
155
  }
156
156
  const clickOpcode = opcode;
157
- const sel = clickOpcode.selector;
157
+ let sel = clickOpcode.selector;
158
+ if (!sel && clickOpcode.target) {
159
+ // Target-only opcode: resolve a concrete selector from the AKTree using
160
+ // the target's text/label so the alternative interaction methods have
161
+ // something to act on.
162
+ try {
163
+ const tree = await adapter.getAKTree();
164
+ sel = resolveSelectorByFuzzyText(tree, semanticTargetFuzzyText(clickOpcode.target)) ?? undefined;
165
+ }
166
+ catch {
167
+ // fall through to the guard below
168
+ }
169
+ }
170
+ if (!sel) {
171
+ return { recovered: false, reason: 'alt interaction needs a selector or a target resolvable in the AKTree' };
172
+ }
158
173
  const methods = [
159
174
  { name: 'keyboard', options: { useKeyboard: true } },
160
175
  { name: 'js_dispatch', options: { useJsDispatch: true } },
@@ -291,6 +306,30 @@ async function tryHealer(failedOpcode, opcodeIndex, adapter, healer, programStep
291
306
  function hasSelector(opcode) {
292
307
  return 'selector' in opcode && typeof opcode.selector === 'string' && !!opcode.selector;
293
308
  }
309
+ function hasSemanticTarget(opcode) {
310
+ return 'target' in opcode && Boolean(opcode.target);
311
+ }
312
+ /**
313
+ * Stable signature for selector-memory lookups. Selector-based opcodes keep
314
+ * the legacy `KIND:selector` form; target-only opcodes derive a signature
315
+ * from the semantic descriptor so memory survives across runs.
316
+ */
317
+ function buildStepSignature(opcode) {
318
+ const { selector, target } = opcode;
319
+ if (selector)
320
+ return `${opcode.kind}:${selector}`;
321
+ if (target) {
322
+ const name = target.label
323
+ ?? target.text
324
+ ?? target.placeholder
325
+ ?? Object.values(target.labelByLocale ?? {})[0]
326
+ ?? Object.values(target.textByLocale ?? {})[0]
327
+ ?? Object.values(target.placeholderByLocale ?? {})[0]
328
+ ?? '';
329
+ return `${opcode.kind}:target:${target.role ?? ''}/${name}`;
330
+ }
331
+ return `${opcode.kind}:`;
332
+ }
294
333
  async function executeRawAction(opcode, adapter, currentVariant, credentials, suppressPageReloads = false) {
295
334
  const result = await executeOpcodeCoreAction(opcode, adapter, {
296
335
  currentVariant,
@@ -302,7 +341,7 @@ async function executeRawAction(opcode, adapter, currentVariant, credentials, su
302
341
  }
303
342
  }
304
343
  async function executeHealerPatchedAction(opcode, adapter, interactionMode, currentVariant, credentials, suppressPageReloads = false) {
305
- if (opcode.kind === 'CLICK' && interactionMode && interactionMode !== 'default') {
344
+ if (opcode.kind === 'CLICK' && opcode.selector && interactionMode && interactionMode !== 'default') {
306
345
  const options = interactionMode === 'keyboard'
307
346
  ? { useKeyboard: true }
308
347
  : interactionMode === 'js_dispatch'
package/dist/selector-resolver.d.ts CHANGED
@@ -7,15 +7,18 @@
7
7
  * 3. Selector memory candidates (from Supabase)
8
8
  * 4. Fuzzy text match on AKTree nodes
9
9
  */
10
- import type { RuntimeAdapter } from './execution-types.js';
10
+ import type { AKTree } from './types.js';
11
+ import type { RuntimeAdapter, SemanticTarget } from './execution-types.js';
11
12
  export interface ResolvedSelector {
12
13
  selector: string;
13
- strategy: 'primary' | 'fingerprint' | 'memory' | 'fuzzy_text' | 'alternate';
14
+ strategy: 'primary' | 'fingerprint' | 'memory' | 'fuzzy_text' | 'alternate' | 'target_text';
14
15
  confidence: 'high' | 'medium' | 'low';
15
16
  }
16
17
  export interface SelectorResolverOptions {
17
- /** Primary selector from the compiled opcode */
18
- primary: string;
18
+ /** Primary selector from the compiled opcode. Absent for target-only opcodes. */
19
+ primary?: string;
20
+ /** Semantic target from the compiled opcode (for text-based AKTree matching) */
21
+ target?: SemanticTarget;
19
22
  /** AKTree fingerprint for fuzzy matching */
20
23
  fingerprint?: string;
21
24
  /** Alternative selectors from the compiled opcode */
@@ -32,3 +35,15 @@ export interface SelectorResolverOptions {
32
35
  * Returns the first working selector, or null if nothing matches.
33
36
  */
34
37
  export declare function resolveSelector(adapter: RuntimeAdapter, options: SelectorResolverOptions): Promise<ResolvedSelector | null>;
38
+ /**
39
+ * Flattens a SemanticTarget into the text used for AKTree fuzzy matching:
40
+ * base label/text/placeholder plus every locale variant. Returns '' when the
41
+ * target carries no text at all (e.g. role-only).
42
+ */
43
+ export declare function semanticTargetFuzzyText(target: SemanticTarget): string;
44
+ /**
45
+ * Resolves a CSS selector for a visible interactive AKTree node matching the
46
+ * given text. Used by recovery strategies that need a concrete selector for a
47
+ * target-only opcode (e.g. coordinate clicks).
48
+ */
49
+ export declare function resolveSelectorByFuzzyText(tree: AKTree, text: string): string | null;
package/dist/selector-resolver.js CHANGED
@@ -14,7 +14,7 @@
14
14
  export async function resolveSelector(adapter, options) {
15
15
  const tree = await adapter.getAKTree();
16
16
  // 1. Try primary selector
17
- if (await selectorExists(adapter, options.primary)) {
17
+ if (options.primary && await selectorExists(adapter, options.primary)) {
18
18
  return { selector: options.primary, strategy: 'primary', confidence: 'high' };
19
19
  }
20
20
  // 2. Try alternate selectors
@@ -32,7 +32,17 @@ export async function resolveSelector(adapter, options) {
32
32
  return { selector: match, strategy: 'fingerprint', confidence: 'medium' };
33
33
  }
34
34
  }
35
- // 4. Try selector memory
35
+ // 4. Try the semantic target's text/label against the AKTree
36
+ if (options.target) {
37
+ const targetText = semanticTargetFuzzyText(options.target);
38
+ if (targetText) {
39
+ const match = findNodeByFuzzyText(tree, targetText);
40
+ if (match) {
41
+ return { selector: match, strategy: 'target_text', confidence: 'medium' };
42
+ }
43
+ }
44
+ }
45
+ // 5. Try selector memory
36
46
  if (options.selectorMemory && options.stepSignature) {
37
47
  const candidates = options.selectorMemory[options.stepSignature];
38
48
  if (candidates) {
@@ -43,7 +53,7 @@ export async function resolveSelector(adapter, options) {
43
53
  }
44
54
  }
45
55
  }
46
- // 5. Try fuzzy text match
56
+ // 6. Try fuzzy text match on the opcode description
47
57
  if (options.description) {
48
58
  const match = findNodeByFuzzyText(tree, options.description);
49
59
  if (match) {
@@ -52,6 +62,32 @@ export async function resolveSelector(adapter, options) {
52
62
  }
53
63
  return null;
54
64
  }
65
+ /**
66
+ * Flattens a SemanticTarget into the text used for AKTree fuzzy matching:
67
+ * base label/text/placeholder plus every locale variant. Returns '' when the
68
+ * target carries no text at all (e.g. role-only).
69
+ */
70
+ export function semanticTargetFuzzyText(target) {
71
+ const parts = [
72
+ target.label,
73
+ target.text,
74
+ target.placeholder,
75
+ ...Object.values(target.labelByLocale ?? {}),
76
+ ...Object.values(target.textByLocale ?? {}),
77
+ ...Object.values(target.placeholderByLocale ?? {}),
78
+ ].filter((value) => Boolean(value && value.trim()));
79
+ return parts.join(' ');
80
+ }
81
+ /**
82
+ * Resolves a CSS selector for a visible interactive AKTree node matching the
83
+ * given text. Used by recovery strategies that need a concrete selector for a
84
+ * target-only opcode (e.g. coordinate clicks).
85
+ */
86
+ export function resolveSelectorByFuzzyText(tree, text) {
87
+ if (!text.trim())
88
+ return null;
89
+ return findNodeByFuzzyText(tree, text);
90
+ }
55
91
  // ── Helpers ─────────────────────────────────────────────────────────
56
92
  async function selectorExists(adapter, selector) {
57
93
  try {
package/dist/semantic-resolver.d.ts CHANGED
@@ -25,6 +25,8 @@ export interface ResolveOptions {
25
25
  target?: SemanticTarget;
26
26
  /** Fallback selectors */
27
27
  selectorAlternates?: string[];
28
+ /** Active variant locale used to resolve the target's `*ByLocale` maps */
29
+ locale?: string;
28
30
  /** Timeout for visibility check (ms). Default: 3000 */
29
31
  timeoutMs?: number;
30
32
  }
@@ -33,3 +35,9 @@ export interface ResolveOptions {
33
35
  * Returns the first visible locator found, or null if nothing matches.
34
36
  */
35
37
  export declare function resolveTarget(page: Page, options: ResolveOptions): Promise<ResolvedTarget | null>;
38
+ /**
39
+ * Projects a SemanticTarget onto a single locale: each `*ByLocale` map is
40
+ * collapsed into its base field. Lookup order: exact locale tag → primary
41
+ * language subtag → base field → first non-empty map value.
42
+ */
43
+ export declare function localizeSemanticTarget(target: SemanticTarget, locale?: string): SemanticTarget;
package/dist/semantic-resolver.js CHANGED
Binary file
package/dist/video-narration-schema.d.ts CHANGED
@@ -296,13 +296,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
296
296
  stepId: z.ZodOptional<z.ZodString>;
297
297
  kind: z.ZodLiteral<"ASSERT_SURFACE">;
298
298
  }, z.core.$strict>, z.ZodObject<{
299
- selector: z.ZodString;
299
+ selector: z.ZodOptional<z.ZodString>;
300
300
  target: z.ZodOptional<z.ZodObject<{
301
301
  text: z.ZodOptional<z.ZodString>;
302
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
302
303
  role: z.ZodOptional<z.ZodString>;
303
304
  label: z.ZodOptional<z.ZodString>;
305
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
304
306
  near: z.ZodOptional<z.ZodString>;
307
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
305
308
  placeholder: z.ZodOptional<z.ZodString>;
309
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
306
310
  exact: z.ZodOptional<z.ZodBoolean>;
307
311
  }, z.core.$strict>>;
308
312
  button: z.ZodOptional<z.ZodEnum<{
@@ -341,13 +345,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
341
345
  stepId: z.ZodOptional<z.ZodString>;
342
346
  kind: z.ZodLiteral<"CLICK">;
343
347
  }, z.core.$strict>, z.ZodObject<{
344
- selector: z.ZodString;
348
+ selector: z.ZodOptional<z.ZodString>;
345
349
  target: z.ZodOptional<z.ZodObject<{
346
350
  text: z.ZodOptional<z.ZodString>;
351
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
347
352
  role: z.ZodOptional<z.ZodString>;
348
353
  label: z.ZodOptional<z.ZodString>;
354
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
349
355
  near: z.ZodOptional<z.ZodString>;
356
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
350
357
  placeholder: z.ZodOptional<z.ZodString>;
358
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
351
359
  exact: z.ZodOptional<z.ZodBoolean>;
352
360
  }, z.core.$strict>>;
353
361
  text: z.ZodString;
@@ -419,10 +427,14 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
419
427
  selector: z.ZodOptional<z.ZodString>;
420
428
  target: z.ZodOptional<z.ZodObject<{
421
429
  text: z.ZodOptional<z.ZodString>;
430
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
422
431
  role: z.ZodOptional<z.ZodString>;
423
432
  label: z.ZodOptional<z.ZodString>;
433
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
424
434
  near: z.ZodOptional<z.ZodString>;
435
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
425
436
  placeholder: z.ZodOptional<z.ZodString>;
437
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
426
438
  exact: z.ZodOptional<z.ZodBoolean>;
427
439
  }, z.core.$strict>>;
428
440
  state: z.ZodEnum<{
@@ -598,10 +610,14 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
598
610
  targetSelector: z.ZodOptional<z.ZodString>;
599
611
  target: z.ZodOptional<z.ZodObject<{
600
612
  text: z.ZodOptional<z.ZodString>;
613
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
601
614
  role: z.ZodOptional<z.ZodString>;
602
615
  label: z.ZodOptional<z.ZodString>;
616
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
603
617
  near: z.ZodOptional<z.ZodString>;
618
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
604
619
  placeholder: z.ZodOptional<z.ZodString>;
620
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
605
621
  exact: z.ZodOptional<z.ZodBoolean>;
606
622
  }, z.core.$strict>>;
607
623
  description: z.ZodString;
@@ -741,13 +757,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
741
757
  stepId: z.ZodOptional<z.ZodString>;
742
758
  kind: z.ZodLiteral<"END_CLIP">;
743
759
  }, z.core.$strict>, z.ZodObject<{
744
- selector: z.ZodString;
760
+ selector: z.ZodOptional<z.ZodString>;
745
761
  target: z.ZodOptional<z.ZodObject<{
746
762
  text: z.ZodOptional<z.ZodString>;
763
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
747
764
  role: z.ZodOptional<z.ZodString>;
748
765
  label: z.ZodOptional<z.ZodString>;
766
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
749
767
  near: z.ZodOptional<z.ZodString>;
768
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
750
769
  placeholder: z.ZodOptional<z.ZodString>;
770
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
751
771
  exact: z.ZodOptional<z.ZodBoolean>;
752
772
  }, z.core.$strict>>;
753
773
  fingerprint: z.ZodOptional<z.ZodString>;
@@ -782,13 +802,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
782
802
  stepId: z.ZodOptional<z.ZodString>;
783
803
  kind: z.ZodLiteral<"HOVER">;
784
804
  }, z.core.$strict>, z.ZodObject<{
785
- selector: z.ZodString;
805
+ selector: z.ZodOptional<z.ZodString>;
786
806
  target: z.ZodOptional<z.ZodObject<{
787
807
  text: z.ZodOptional<z.ZodString>;
808
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
788
809
  role: z.ZodOptional<z.ZodString>;
789
810
  label: z.ZodOptional<z.ZodString>;
811
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
790
812
  near: z.ZodOptional<z.ZodString>;
813
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
791
814
  placeholder: z.ZodOptional<z.ZodString>;
815
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
792
816
  exact: z.ZodOptional<z.ZodBoolean>;
793
817
  }, z.core.$strict>>;
794
818
  optionLabel: z.ZodOptional<z.ZodString>;
@@ -826,13 +850,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
826
850
  stepId: z.ZodOptional<z.ZodString>;
827
851
  kind: z.ZodLiteral<"SELECT_OPTION">;
828
852
  }, z.core.$strict>, z.ZodObject<{
829
- selector: z.ZodString;
853
+ selector: z.ZodOptional<z.ZodString>;
830
854
  target: z.ZodOptional<z.ZodObject<{
831
855
  text: z.ZodOptional<z.ZodString>;
856
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
832
857
  role: z.ZodOptional<z.ZodString>;
833
858
  label: z.ZodOptional<z.ZodString>;
859
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
834
860
  near: z.ZodOptional<z.ZodString>;
861
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
835
862
  placeholder: z.ZodOptional<z.ZodString>;
863
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
836
864
  exact: z.ZodOptional<z.ZodBoolean>;
837
865
  }, z.core.$strict>>;
838
866
  checked: z.ZodBoolean;
@@ -868,13 +896,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
868
896
  stepId: z.ZodOptional<z.ZodString>;
869
897
  kind: z.ZodLiteral<"CHECK">;
870
898
  }, z.core.$strict>, z.ZodObject<{
871
- selector: z.ZodString;
899
+ selector: z.ZodOptional<z.ZodString>;
872
900
  target: z.ZodOptional<z.ZodObject<{
873
901
  text: z.ZodOptional<z.ZodString>;
902
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
874
903
  role: z.ZodOptional<z.ZodString>;
875
904
  label: z.ZodOptional<z.ZodString>;
905
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
876
906
  near: z.ZodOptional<z.ZodString>;
907
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
877
908
  placeholder: z.ZodOptional<z.ZodString>;
909
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
878
910
  exact: z.ZodOptional<z.ZodBoolean>;
879
911
  }, z.core.$strict>>;
880
912
  fingerprint: z.ZodOptional<z.ZodString>;
@@ -909,13 +941,17 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
909
941
  stepId: z.ZodOptional<z.ZodString>;
910
942
  kind: z.ZodLiteral<"DOUBLE_CLICK">;
911
943
  }, z.core.$strict>, z.ZodObject<{
912
- selector: z.ZodString;
944
+ selector: z.ZodOptional<z.ZodString>;
913
945
  target: z.ZodOptional<z.ZodObject<{
914
946
  text: z.ZodOptional<z.ZodString>;
947
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
915
948
  role: z.ZodOptional<z.ZodString>;
916
949
  label: z.ZodOptional<z.ZodString>;
950
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
917
951
  near: z.ZodOptional<z.ZodString>;
952
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
918
953
  placeholder: z.ZodOptional<z.ZodString>;
954
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
919
955
  exact: z.ZodOptional<z.ZodBoolean>;
920
956
  }, z.core.$strict>>;
921
957
  fingerprint: z.ZodOptional<z.ZodString>;
@@ -923,10 +959,14 @@ export declare const VideoIngestPayloadSchema: z.ZodObject<{
923
959
  toSelector: z.ZodOptional<z.ZodString>;
924
960
  toTarget: z.ZodOptional<z.ZodObject<{
925
961
  text: z.ZodOptional<z.ZodString>;
962
+ textByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
926
963
  role: z.ZodOptional<z.ZodString>;
927
964
  label: z.ZodOptional<z.ZodString>;
965
+ labelByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
928
966
  near: z.ZodOptional<z.ZodString>;
967
+ nearByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
929
968
  placeholder: z.ZodOptional<z.ZodString>;
969
+ placeholderByLocale: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
930
970
  exact: z.ZodOptional<z.ZodBoolean>;
931
971
  }, z.core.$strict>>;
932
972
  toSelectorAlternates: z.ZodOptional<z.ZodArray<z.ZodString>>;
package/dist/web-playwright-local.d.ts CHANGED
@@ -100,6 +100,17 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
100
100
  value?: string;
101
101
  index?: number;
102
102
  }): Promise<void>;
103
+ selectOptionByTarget(opts: ResolveOptions, option: {
104
+ label?: string;
105
+ value?: string;
106
+ index?: number;
107
+ }): Promise<void>;
108
+ checkByTarget(opts: ResolveOptions, checked: boolean, actionOpts?: {
109
+ onClick?: (timestampMs: number) => void;
110
+ }): Promise<void>;
111
+ doubleClickByTarget(opts: ResolveOptions, actionOpts?: {
112
+ onClick?: (timestampMs: number) => void;
113
+ }): Promise<void>;
103
114
  check(selector: string, checked: boolean, actionOpts?: {
104
115
  onClick?: (timestampMs: number) => void;
105
116
  }): Promise<void>;
@@ -117,6 +128,7 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
117
128
  dx: number;
118
129
  dy: number;
119
130
  };
131
+ locale?: string;
120
132
  }): Promise<void>;
121
133
  cloneElement(opts: {
122
134
  sourceSelector: string;