donobu 5.60.8 → 5.61.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-side-scripts/smart-selector-generator.js +27 -1
- package/dist/esm/browser-side-scripts/smart-selector-generator.js +27 -1
- package/dist/esm/lib/ai/locate/locateElement.js +11 -15
- package/dist/esm/managers/DonobuFlow.d.ts +11 -4
- package/dist/esm/managers/DonobuFlow.js +44 -10
- package/dist/esm/managers/DonobuFlowsManager.js +1 -7
- package/dist/esm/tools/AssertTool.js +14 -18
- package/dist/esm/tools/ReplayableInteraction.d.ts +12 -2
- package/dist/esm/tools/ReplayableInteraction.js +184 -139
- package/dist/esm/utils/AdvancedSelectorGenerator.d.ts +130 -0
- package/dist/esm/utils/AdvancedSelectorGenerator.js +485 -0
- package/dist/esm/utils/EnvVarSensitivity.d.ts +37 -0
- package/dist/esm/utils/EnvVarSensitivity.js +64 -0
- package/dist/esm/utils/TemplateInterpolator.d.ts +50 -0
- package/dist/esm/utils/TemplateInterpolator.js +146 -5
- package/dist/esm/utils/envReferencePromptSection.d.ts +25 -0
- package/dist/esm/utils/envReferencePromptSection.js +42 -0
- package/dist/lib/ai/locate/locateElement.js +11 -15
- package/dist/managers/DonobuFlow.d.ts +11 -4
- package/dist/managers/DonobuFlow.js +44 -10
- package/dist/managers/DonobuFlowsManager.js +1 -7
- package/dist/tools/AssertTool.js +14 -18
- package/dist/tools/ReplayableInteraction.d.ts +12 -2
- package/dist/tools/ReplayableInteraction.js +184 -139
- package/dist/utils/AdvancedSelectorGenerator.d.ts +130 -0
- package/dist/utils/AdvancedSelectorGenerator.js +485 -0
- package/dist/utils/EnvVarSensitivity.d.ts +37 -0
- package/dist/utils/EnvVarSensitivity.js +64 -0
- package/dist/utils/TemplateInterpolator.d.ts +50 -0
- package/dist/utils/TemplateInterpolator.js +146 -5
- package/dist/utils/envReferencePromptSection.d.ts +25 -0
- package/dist/utils/envReferencePromptSection.js +42 -0
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ const v4_1 = require("zod/v4");
|
|
|
5
5
|
const PageClosedException_1 = require("../exceptions/PageClosedException");
|
|
6
6
|
const ElementSelector_1 = require("../models/ElementSelector");
|
|
7
7
|
const ToolSchema_1 = require("../models/ToolSchema");
|
|
8
|
+
const AdvancedSelectorGenerator_1 = require("../utils/AdvancedSelectorGenerator");
|
|
8
9
|
const Logger_1 = require("../utils/Logger");
|
|
9
10
|
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
|
|
10
11
|
const TargetUtils_1 = require("../utils/TargetUtils");
|
|
@@ -412,53 +413,87 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
412
413
|
},
|
|
413
414
|
};
|
|
414
415
|
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
416
|
+
// Try each candidate locator in priority order, and within a candidate each
|
|
417
|
+
// matched element, returning on the first successful interaction and
|
|
418
|
+
// otherwise summarizing what was tried.
|
|
419
|
+
const coreParameters = this.coreSchema.parse(parameters);
|
|
420
|
+
const selectorAttempts = [];
|
|
421
|
+
for (const selectorLocator of locators) {
|
|
422
|
+
const attemptSummary = {
|
|
423
|
+
selector: selectorLocator.selector,
|
|
424
|
+
matchCount: null,
|
|
425
|
+
errors: [],
|
|
426
|
+
};
|
|
427
|
+
selectorAttempts.push(attemptSummary);
|
|
428
|
+
try {
|
|
429
|
+
const count = await selectorLocator.locator.count();
|
|
430
|
+
attemptSummary.matchCount = count;
|
|
431
|
+
if (count === 0) {
|
|
432
|
+
attemptSummary.errors.push('Locator resolved to 0 elements before interaction.');
|
|
433
|
+
}
|
|
434
|
+
for (let i = 0; i < count; ++i) {
|
|
435
|
+
try {
|
|
436
|
+
const forLlm = await this.interactWithLocator(context, coreParameters, selectorLocator.locator.nth(i), page);
|
|
437
|
+
// Be careful to not report back the internal, ephemeral, Donobu selector
|
|
438
|
+
// used for autonomous runs. If we are in autonomous mode, just report
|
|
439
|
+
// back the top-resolved selector.
|
|
440
|
+
const reportedSelector = !selectorLocator.selector.includes((0, TargetUtils_1.webInspector)(context).interactableElementAttribute)
|
|
441
|
+
? selectorLocator.selector
|
|
442
|
+
: parameters.selector.element[0];
|
|
443
|
+
return {
|
|
444
|
+
isSuccessful: true,
|
|
445
|
+
forLlm: forLlm,
|
|
446
|
+
metadata: {
|
|
447
|
+
...parameters.selector,
|
|
448
|
+
usedSelector: reportedSelector,
|
|
449
|
+
},
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
catch (elementError) {
|
|
453
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(elementError)) {
|
|
454
|
+
throw elementError;
|
|
455
|
+
}
|
|
456
|
+
Logger_1.appLogger.error(`Failed to interact with element '${i}' for selector '${selectorLocator.selector}' due to exception, will fail over to remaining elements (if any)`, elementError);
|
|
457
|
+
attemptSummary.errors.push(`element ${i}: ${ReplayableInteraction.describeError(elementError)}`);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
catch (locatorError) {
|
|
462
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(locatorError)) {
|
|
463
|
+
throw locatorError;
|
|
464
|
+
}
|
|
465
|
+
Logger_1.appLogger.error(`Failed to interact with selector '${selectorLocator.selector}' due to exception, will fail over to remaining selectors (if any)`, locatorError);
|
|
466
|
+
attemptSummary.errors.push(`selector error: ${ReplayableInteraction.describeError(locatorError)}`);
|
|
467
|
+
}
|
|
428
468
|
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
469
|
+
return {
|
|
470
|
+
isSuccessful: false,
|
|
471
|
+
forLlm: `FAILED! Unable to apply operation. ${ReplayableInteraction.summarizeAttemptsForLlm(selectorAttempts)}`,
|
|
472
|
+
metadata: {
|
|
473
|
+
selectorForReplay: parameters.selector,
|
|
474
|
+
selectorAttempts: selectorAttempts,
|
|
475
|
+
},
|
|
476
|
+
};
|
|
436
477
|
}
|
|
437
478
|
async callFromGpt(context, parameters) {
|
|
438
479
|
const page = (0, TargetUtils_1.webPage)(context);
|
|
439
480
|
const elementSelector = `[${(0, TargetUtils_1.webInspector)(context).interactableElementAttribute}="${parameters.annotation}"]`;
|
|
440
|
-
|
|
481
|
+
// Find the annotated element in whichever frame holds it.
|
|
482
|
+
let locator = null;
|
|
441
483
|
for (const frame of page.frames()) {
|
|
442
484
|
if (frame.isDetached()) {
|
|
443
485
|
continue;
|
|
444
486
|
}
|
|
445
|
-
const
|
|
446
|
-
if ((await
|
|
447
|
-
|
|
448
|
-
const frameSelector = frame.parentFrame() === null
|
|
449
|
-
? null
|
|
450
|
-
: (await PlaywrightUtils_1.PlaywrightUtils.generateSelectors(await frame.frameElement()))[0];
|
|
451
|
-
locatorData = {
|
|
452
|
-
locators: [locator],
|
|
453
|
-
selectorForReplay: {
|
|
454
|
-
element: selectorCandidates,
|
|
455
|
-
frame: frameSelector,
|
|
456
|
-
},
|
|
457
|
-
};
|
|
487
|
+
const candidate = frame.locator(elementSelector);
|
|
488
|
+
if ((await candidate.count()) > 0) {
|
|
489
|
+
locator = candidate;
|
|
458
490
|
break;
|
|
459
491
|
}
|
|
460
492
|
}
|
|
461
|
-
|
|
493
|
+
const targetHandle = locator
|
|
494
|
+
? await locator.elementHandle({ timeout: 1000 }).catch(() => null)
|
|
495
|
+
: null;
|
|
496
|
+
if (!locator || !targetHandle) {
|
|
462
497
|
return {
|
|
463
498
|
isSuccessful: false,
|
|
464
499
|
forLlm: `FAILED! Unable to resolve HTML element for annotation '${parameters.annotation}'.`,
|
|
@@ -468,12 +503,60 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
468
503
|
},
|
|
469
504
|
};
|
|
470
505
|
}
|
|
471
|
-
|
|
506
|
+
// Turn the live element into the best replayable selector — generated from
|
|
507
|
+
// the DOM and, for env-parameterized actions, re-parameterized + pruned so it
|
|
508
|
+
// reflects intent rather than incidental position. This runs BEFORE the
|
|
509
|
+
// interaction: the generation reads the pre-interaction DOM.
|
|
510
|
+
const selectorForReplay = await (0, AdvancedSelectorGenerator_1.generateAdvancedSelectors)(targetHandle, context);
|
|
511
|
+
// Resolve the annotation to its single element and act on it directly — no
|
|
512
|
+
// failover loop, since the annotation selector is unique.
|
|
513
|
+
const coreParameters = this.coreSchema.parse(parameters);
|
|
514
|
+
try {
|
|
515
|
+
const forLlm = await this.interactWithLocator(context, coreParameters, locator, page);
|
|
516
|
+
return {
|
|
517
|
+
isSuccessful: true,
|
|
518
|
+
forLlm,
|
|
519
|
+
// Report the top generated selector (the annotation attribute is
|
|
520
|
+
// internal/ephemeral).
|
|
521
|
+
metadata: {
|
|
522
|
+
...selectorForReplay,
|
|
523
|
+
usedSelector: selectorForReplay.element[0],
|
|
524
|
+
},
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
catch (error) {
|
|
528
|
+
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
529
|
+
throw error;
|
|
530
|
+
}
|
|
531
|
+
Logger_1.appLogger.error(`Failed to interact with annotation '${parameters.annotation}'`, error);
|
|
472
532
|
return {
|
|
473
|
-
|
|
474
|
-
|
|
533
|
+
isSuccessful: false,
|
|
534
|
+
forLlm: `FAILED! Unable to apply operation. ${ReplayableInteraction.describeError(error)}`,
|
|
535
|
+
metadata: {
|
|
536
|
+
selectorForReplay,
|
|
537
|
+
},
|
|
475
538
|
};
|
|
476
|
-
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* SUPERVISED-mode preview: resolve the element this interaction *would*
|
|
543
|
+
* target (from either an annotation- or selector-based proposal) and move the
|
|
544
|
+
* on-screen cursor to it, without performing the interaction. Best-effort —
|
|
545
|
+
* an unresolvable element simply leaves the cursor where it is.
|
|
546
|
+
*/
|
|
547
|
+
async previewInteraction(context, parameters) {
|
|
548
|
+
const page = (0, TargetUtils_1.webPage)(context);
|
|
549
|
+
const locator = await this.resolvePreviewLocator(context, page, parameters);
|
|
550
|
+
if (!locator) {
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
// Point at the same visible target (or its label) the real interaction
|
|
554
|
+
// would, so the preview matches what approval will touch.
|
|
555
|
+
const pointTarget = await ReplayableInteraction.getLocatorOrItsLabel(locator.first());
|
|
556
|
+
// Only reveal the cursor now that we have a real target to point at, so a
|
|
557
|
+
// non-interactive proposal never pops a stationary cursor.
|
|
558
|
+
await (0, TargetUtils_1.webInspector)(context).showInteractionCursor();
|
|
559
|
+
await context.interactionVisualizer.pointAt(page, pointTarget.first(), ReplayableInteraction.PREVIEW_CURSOR_DURATION_MILLIS);
|
|
477
560
|
}
|
|
478
561
|
/**
|
|
479
562
|
* Transform a historical {@link ToolCall} into a replay-ready
|
|
@@ -525,6 +608,14 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
525
608
|
? ElementSelector_1.ElementSelectorSchema.parse(toolCall.parameters.selector)
|
|
526
609
|
: ElementSelector_1.ElementSelectorSchema.parse(toolCall.outcome.metadata);
|
|
527
610
|
let selectorCandidates = [...originalSelector.element];
|
|
611
|
+
// 0) Drop ephemeral-id selectors (e.g. React useId() values like
|
|
612
|
+
// `#\:r18\:`) outright. They regenerate every render, so replaying or
|
|
613
|
+
// (below) promoting one only re-pins a value that is already stale. Older
|
|
614
|
+
// recordings may still carry them; keep the list non-empty as a backstop.
|
|
615
|
+
const nonEphemeral = selectorCandidates.filter((sel) => !(0, AdvancedSelectorGenerator_1.isEphemeralIdSelector)(sel));
|
|
616
|
+
if (nonEphemeral.length > 0) {
|
|
617
|
+
selectorCandidates = nonEphemeral;
|
|
618
|
+
}
|
|
528
619
|
// 1) Drop ID-based selectors if requested --------------------------------
|
|
529
620
|
if (options.areElementIdsVolatile) {
|
|
530
621
|
const nonId = selectorCandidates.filter((sel) => {
|
|
@@ -542,11 +633,18 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
542
633
|
const firstSelectorIsAria = selectorCandidates.length > 0 &&
|
|
543
634
|
ReplayableInteraction.isAriaBasedSelector(selectorCandidates[0]);
|
|
544
635
|
if (!firstSelectorIsAria) {
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
//
|
|
548
|
-
|
|
549
|
-
|
|
636
|
+
// Promote only STABLE ids. A dynamic id (a per-render React useId()
|
|
637
|
+
// value, a hash/UUID, an over-long id) is one the generator already
|
|
638
|
+
// deprioritized; promoting it would re-pin a value that is guaranteed
|
|
639
|
+
// stale at replay — exactly over the robust env/text selector that
|
|
640
|
+
// actually works. Such ids stay where generation ranked them (failover).
|
|
641
|
+
const stableIdSelectors = selectorCandidates.filter((sel) => ReplayableInteraction.isIdBasedSelector(sel) &&
|
|
642
|
+
!(0, AdvancedSelectorGenerator_1.isDynamicIdSelector)(sel));
|
|
643
|
+
const otherSelectors = selectorCandidates.filter((sel) => !ReplayableInteraction.isIdBasedSelector(sel) ||
|
|
644
|
+
(0, AdvancedSelectorGenerator_1.isDynamicIdSelector)(sel));
|
|
645
|
+
// Reorder: stable ID-based selectors first, the rest in generated order.
|
|
646
|
+
if (stableIdSelectors.length > 0) {
|
|
647
|
+
selectorCandidates = [...stableIdSelectors, ...otherSelectors];
|
|
550
648
|
}
|
|
551
649
|
}
|
|
552
650
|
// If first selector is aria-based, leave the order unchanged
|
|
@@ -608,105 +706,52 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
608
706
|
}
|
|
609
707
|
return null;
|
|
610
708
|
}
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
709
|
+
/**
|
|
710
|
+
* Resolve a single, already-located element and perform the interaction on it:
|
|
711
|
+
* grab the target handle (and its label, if any), verify it is still attached,
|
|
712
|
+
* capture an HTML snippet, run the concrete {@link invoke}, and wait for the
|
|
713
|
+
* page to settle. Returns the LLM-facing string (action result + snippet) and
|
|
714
|
+
* throws on any failure so callers can fail over or record it.
|
|
715
|
+
*
|
|
716
|
+
* `locator` must already point at the intended element (callers narrow with
|
|
717
|
+
* `.nth(i)` or a unique annotation selector); only its first match is used.
|
|
718
|
+
*/
|
|
719
|
+
async interactWithLocator(context, coreParameters, locator, page) {
|
|
614
720
|
const timeoutOpt = {
|
|
615
721
|
timeout: 1000, // Millis
|
|
616
722
|
};
|
|
617
|
-
const
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
const attemptSummary = {
|
|
621
|
-
selector: selectorLocator.selector,
|
|
622
|
-
matchCount: null,
|
|
623
|
-
errors: [],
|
|
624
|
-
};
|
|
625
|
-
selectorAttempts.push(attemptSummary);
|
|
626
|
-
try {
|
|
627
|
-
const count = await selectorLocator.locator.count();
|
|
628
|
-
attemptSummary.matchCount = count;
|
|
629
|
-
if (count === 0) {
|
|
630
|
-
attemptSummary.errors.push('Locator resolved to 0 elements before interaction.');
|
|
631
|
-
}
|
|
632
|
-
for (let i = 0; i < count; ++i) {
|
|
633
|
-
try {
|
|
634
|
-
const originalLocator = selectorLocator.locator.nth(i);
|
|
635
|
-
const targetHandle = await originalLocator
|
|
636
|
-
.first()
|
|
637
|
-
.elementHandle(timeoutOpt);
|
|
638
|
-
if (!targetHandle) {
|
|
639
|
-
throw new Error(`Failed to resolve element handle for selector '${selectorLocator.selector}'`);
|
|
640
|
-
}
|
|
641
|
-
const labelLocator = await ReplayableInteraction.getLocatorOrItsLabel(originalLocator);
|
|
642
|
-
let labelHandle;
|
|
643
|
-
try {
|
|
644
|
-
labelHandle = await labelLocator
|
|
645
|
-
.first()
|
|
646
|
-
.elementHandle(timeoutOpt);
|
|
647
|
-
}
|
|
648
|
-
catch {
|
|
649
|
-
labelHandle = null;
|
|
650
|
-
}
|
|
651
|
-
const chosenHandle = labelHandle ?? targetHandle;
|
|
652
|
-
// Check if element is still attached before attempting to do anything meaningful.
|
|
653
|
-
const isAttached = await chosenHandle.evaluate((el) => el.isConnected);
|
|
654
|
-
if (!isAttached) {
|
|
655
|
-
throw new Error(`Element '${i}' for selector '${selectorLocator.selector}' does not exist in the DOM`);
|
|
656
|
-
}
|
|
657
|
-
let htmlSnippet = '';
|
|
658
|
-
try {
|
|
659
|
-
htmlSnippet +=
|
|
660
|
-
await (0, TargetUtils_1.webInspector)(context).pageInspector.getHtmlSnippet(chosenHandle);
|
|
661
|
-
}
|
|
662
|
-
catch (error) {
|
|
663
|
-
Logger_1.appLogger.warn('Failed to get HTML snippet', error);
|
|
664
|
-
}
|
|
665
|
-
const forLlm = await this.invoke(context, coreParameters, {
|
|
666
|
-
target: targetHandle,
|
|
667
|
-
label: labelHandle ?? undefined,
|
|
668
|
-
});
|
|
669
|
-
await PlaywrightUtils_1.PlaywrightUtils.waitForPageStability(page);
|
|
670
|
-
// Be careful to not report back the internal, ephemeral, Donobu selector
|
|
671
|
-
// used for autonomous runs. If we are in autonomous mode, just report
|
|
672
|
-
// back the top-resolved selector.
|
|
673
|
-
const reportedSelector = !selectorLocator.selector.includes((0, TargetUtils_1.webInspector)(context).interactableElementAttribute)
|
|
674
|
-
? selectorLocator.selector
|
|
675
|
-
: selectorForReplay.element[0];
|
|
676
|
-
return {
|
|
677
|
-
isSuccessful: true,
|
|
678
|
-
forLlm: forLlm + htmlSnippet,
|
|
679
|
-
metadata: {
|
|
680
|
-
...selectorForReplay,
|
|
681
|
-
usedSelector: reportedSelector,
|
|
682
|
-
},
|
|
683
|
-
};
|
|
684
|
-
}
|
|
685
|
-
catch (elementError) {
|
|
686
|
-
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(elementError)) {
|
|
687
|
-
throw elementError;
|
|
688
|
-
}
|
|
689
|
-
Logger_1.appLogger.error(`Failed to interact with element '${i}' for selector '${selectorLocator.selector}' due to exception, will fail over to remaining elements (if any)`, elementError);
|
|
690
|
-
attemptSummary.errors.push(`element ${i}: ${ReplayableInteraction.describeError(elementError)}`);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
catch (locatorError) {
|
|
695
|
-
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(locatorError)) {
|
|
696
|
-
throw locatorError;
|
|
697
|
-
}
|
|
698
|
-
Logger_1.appLogger.error(`Failed to interact with selector '${selectorLocator.selector}' due to exception, will fail over to remaining selectors (if any)`, locatorError);
|
|
699
|
-
attemptSummary.errors.push(`selector error: ${ReplayableInteraction.describeError(locatorError)}`);
|
|
700
|
-
}
|
|
723
|
+
const targetHandle = await locator.first().elementHandle(timeoutOpt);
|
|
724
|
+
if (!targetHandle) {
|
|
725
|
+
throw new Error('Failed to resolve an element handle for the locator.');
|
|
701
726
|
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
}
|
|
727
|
+
const labelLocator = await ReplayableInteraction.getLocatorOrItsLabel(locator);
|
|
728
|
+
let labelHandle;
|
|
729
|
+
try {
|
|
730
|
+
labelHandle = await labelLocator.first().elementHandle(timeoutOpt);
|
|
731
|
+
}
|
|
732
|
+
catch {
|
|
733
|
+
labelHandle = null;
|
|
734
|
+
}
|
|
735
|
+
const chosenHandle = labelHandle ?? targetHandle;
|
|
736
|
+
// Check if element is still attached before attempting to do anything meaningful.
|
|
737
|
+
const isAttached = await chosenHandle.evaluate((el) => el.isConnected);
|
|
738
|
+
if (!isAttached) {
|
|
739
|
+
throw new Error('The resolved element is no longer attached to the DOM.');
|
|
740
|
+
}
|
|
741
|
+
let htmlSnippet = '';
|
|
742
|
+
try {
|
|
743
|
+
htmlSnippet +=
|
|
744
|
+
await (0, TargetUtils_1.webInspector)(context).pageInspector.getHtmlSnippet(chosenHandle);
|
|
745
|
+
}
|
|
746
|
+
catch (error) {
|
|
747
|
+
Logger_1.appLogger.warn('Failed to get HTML snippet', error);
|
|
748
|
+
}
|
|
749
|
+
const forLlm = await this.invoke(context, coreParameters, {
|
|
750
|
+
target: targetHandle,
|
|
751
|
+
label: labelHandle ?? undefined,
|
|
752
|
+
});
|
|
753
|
+
await PlaywrightUtils_1.PlaywrightUtils.waitForPageStability(page);
|
|
754
|
+
return forLlm + htmlSnippet;
|
|
710
755
|
}
|
|
711
756
|
}
|
|
712
757
|
exports.ReplayableInteraction = ReplayableInteraction;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import type { ElementHandle } from 'playwright';
|
|
2
|
+
import { z } from 'zod/v4';
|
|
3
|
+
import type { ElementSelector } from '../models/ElementSelector';
|
|
4
|
+
import type { GptMessage } from '../models/GptMessage';
|
|
5
|
+
import type { ToolCallContext } from '../models/ToolCallContext';
|
|
6
|
+
/**
|
|
7
|
+
* The model's decision for advancing a generated selector: the SAME candidate
|
|
8
|
+
* list, in the same order and count, each rewritten so env-derived substrings are
|
|
9
|
+
* expressed as `{{$.env.NAME}}` (optionally with a `| filter`) placeholders, or
|
|
10
|
+
* returned unchanged. The deterministic core ({@link advanceSelectorCandidates})
|
|
11
|
+
* validates and applies it.
|
|
12
|
+
*/
|
|
13
|
+
export declare const AdvancedSelectorDecisionSchema: z.ZodObject<{
|
|
14
|
+
element: z.ZodArray<z.ZodString>;
|
|
15
|
+
}, z.core.$strip>;
|
|
16
|
+
export type AdvancedSelectorDecision = z.infer<typeof AdvancedSelectorDecisionSchema>;
|
|
17
|
+
/**
|
|
18
|
+
* The model's synthesized selector: one that locates the target by anchoring on
|
|
19
|
+
* nearby env-derived text (e.g. a card's title), or null if it can't. Validated
|
|
20
|
+
* against the live page before being accepted.
|
|
21
|
+
*/
|
|
22
|
+
export declare const SynthesizedSelectorSchema: z.ZodObject<{
|
|
23
|
+
selector: z.ZodNullable<z.ZodString>;
|
|
24
|
+
}, z.core.$strip>;
|
|
25
|
+
/**
|
|
26
|
+
* The slice of {@link ToolCallContext} this routine needs: the env data and the
|
|
27
|
+
* action's references (to know what was baked in), plus the GPT client and flow
|
|
28
|
+
* metadata to run + account for the one model call. Callers pass their
|
|
29
|
+
* `ToolCallContext` directly.
|
|
30
|
+
*/
|
|
31
|
+
export type AdvancedSelectorContext = Pick<ToolCallContext, 'gptClient' | 'metadata' | 'envData' | 'rawParameters'>;
|
|
32
|
+
/** A referenced env var and the forms of its value that appear in the candidates. */
|
|
33
|
+
type BakedEnvForm = {
|
|
34
|
+
name: string;
|
|
35
|
+
value: string;
|
|
36
|
+
forms: Array<{
|
|
37
|
+
filter: string | null;
|
|
38
|
+
form: string;
|
|
39
|
+
}>;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Turn the element the agent acted on into the best *replayable* selector:
|
|
43
|
+
* generate raw candidates from the DOM, then (only for env-parameterized actions
|
|
44
|
+
* whose value got baked in) re-parameterize and prune them so the cached selector
|
|
45
|
+
* reflects the action's semantic intent rather than incidental DOM position.
|
|
46
|
+
*
|
|
47
|
+
* The DOM read and the model call are the only impurities and live here; the
|
|
48
|
+
* decision logic is delegated to the pure helpers below. Never returns null — it
|
|
49
|
+
* falls back to the raw generated selector whenever there is nothing to advance
|
|
50
|
+
* or the model's rewrite can't be validated.
|
|
51
|
+
*/
|
|
52
|
+
export declare function generateAdvancedSelectors(target: ElementHandle<HTMLElement | SVGElement>, context: AdvancedSelectorContext): Promise<ElementSelector>;
|
|
53
|
+
/**
|
|
54
|
+
* For each referenced env var, the distinct forms of its value (verbatim or via a
|
|
55
|
+
* transform — e.g. "contact" / "Contact" / "/contact") that actually appear in
|
|
56
|
+
* the generated candidates. PURE.
|
|
57
|
+
*/
|
|
58
|
+
export declare function computeBakedEnvForms(candidates: string[], envData: Record<string, string>, referencedEnvVars: string[]): BakedEnvForm[];
|
|
59
|
+
/**
|
|
60
|
+
* Build the model prompt: deterministic `form -> placeholder` hints (with the
|
|
61
|
+
* `| filter` to use for each transformed form) plus the candidate list. PURE.
|
|
62
|
+
*/
|
|
63
|
+
export declare function buildAdvancedSelectorRequest(candidates: string[], baked: BakedEnvForm[]): GptMessage[];
|
|
64
|
+
/**
|
|
65
|
+
* Build the synthesis prompt: the target's neighborhood HTML (with the target
|
|
66
|
+
* marked) and the env values in play. PURE.
|
|
67
|
+
*/
|
|
68
|
+
export declare function buildSynthesisRequest(neighborhoodHtml: string, envMapping: string, markerAttribute: string): GptMessage[];
|
|
69
|
+
/**
|
|
70
|
+
* Apply the model's decision and prune the result. PURE:
|
|
71
|
+
*
|
|
72
|
+
* 1. **Validate + templatize.** A rewrite is accepted only if interpolating it
|
|
73
|
+
* with the current env values reproduces the exact original candidate — so the
|
|
74
|
+
* model is bound to placeholder substitution; any other edit is rejected and
|
|
75
|
+
* that candidate keeps its literal. A wrong-shaped decision is ignored wholesale.
|
|
76
|
+
* 2. **Elide.** Once a candidate tracks an env value, drop any purely-positional
|
|
77
|
+
* candidate that does not: for a data-driven action, position is a coincidence
|
|
78
|
+
* of the recorded data and would mislocate when the value (or order) changes.
|
|
79
|
+
*
|
|
80
|
+
* Never returns empty — falls back to the raw candidates if pruning would remove
|
|
81
|
+
* everything (and elision never fires unless a semantic, env-bearing candidate
|
|
82
|
+
* survives to take over).
|
|
83
|
+
*/
|
|
84
|
+
export declare function advanceSelectorCandidates(rawCandidates: string[], envData: Record<string, string>, decision: AdvancedSelectorDecision): string[];
|
|
85
|
+
/**
|
|
86
|
+
* Once a candidate tracks an env value, drop purely-positional candidates that
|
|
87
|
+
* don't — for a data-driven action, position is a coincidence of the recorded
|
|
88
|
+
* data and would mislocate when the value (or order) changes. Returns the input
|
|
89
|
+
* unchanged when nothing is env-bearing (no semantic anchor to take over) or when
|
|
90
|
+
* pruning would remove everything. PURE.
|
|
91
|
+
*/
|
|
92
|
+
export declare function pruneCoincidentalPositional(candidates: string[]): string[];
|
|
93
|
+
/** A selector that carries an env reference (so it tracks the value at replay). */
|
|
94
|
+
export declare function isEnvBearingSelector(selector: string): boolean;
|
|
95
|
+
/**
|
|
96
|
+
* A selector anchored on an *ephemeral* id — one regenerated on every render, so
|
|
97
|
+
* pinning it guarantees the selector goes stale at replay. The canonical case is
|
|
98
|
+
* a React `useId()` value: a colon-wrapped id (`#:r18:`, often CSS-escaped as
|
|
99
|
+
* `#\:r18\:`), possibly scoped (`#\:rd\: > li:nth-of-type(7)`). Mirrors the
|
|
100
|
+
* `isEphemeralId` guard in the browser-side generator; kept here as a Node-side
|
|
101
|
+
* funnel guard for candidates from any source. PURE.
|
|
102
|
+
*/
|
|
103
|
+
export declare function isEphemeralIdSelector(selector: string): boolean;
|
|
104
|
+
/**
|
|
105
|
+
* Drop candidates anchored on an ephemeral id, but never empty the list: if every
|
|
106
|
+
* candidate is ephemeral, keep them (a stale selector still beats no selector).
|
|
107
|
+
* PURE.
|
|
108
|
+
*/
|
|
109
|
+
export declare function pruneEphemeralIdSelectors(candidates: string[]): string[];
|
|
110
|
+
/**
|
|
111
|
+
* A selector anchored on a *dynamic* id — one the generator deliberately
|
|
112
|
+
* deprioritizes because it is machine-generated and unstable: an ephemeral
|
|
113
|
+
* (per-render) id like React's `useId()`, a hash/UUID-style id, or an over-long
|
|
114
|
+
* id. Such an id may legitimately survive as a low-priority failover, but must
|
|
115
|
+
* never be *promoted* to primary (doing so re-pins a value that is stale at
|
|
116
|
+
* replay). Mirrors the browser-side ranking heuristics. PURE.
|
|
117
|
+
*/
|
|
118
|
+
export declare function isDynamicIdSelector(selector: string): boolean;
|
|
119
|
+
/**
|
|
120
|
+
* Heuristic for a selector that locates an element *purely by DOM position* — an
|
|
121
|
+
* `:nth-*` pseudo, an XPath numeric index `[N]`, or a bare combinator chain — with
|
|
122
|
+
* no meaningful anchor (id, class, attribute, or text predicate). PURE.
|
|
123
|
+
*
|
|
124
|
+
* The "meaningful anchor" guard is what keeps a text/attribute selector that also
|
|
125
|
+
* happens to carry an index (e.g. `(.//a[normalize-space(.)='X'])[2]`) from being
|
|
126
|
+
* treated as positional.
|
|
127
|
+
*/
|
|
128
|
+
export declare function isPurelyPositionalSelector(selector: string): boolean;
|
|
129
|
+
export {};
|
|
130
|
+
//# sourceMappingURL=AdvancedSelectorGenerator.d.ts.map
|