donobu 5.55.0 → 5.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/GptConfigsApi.d.ts +5 -5
- package/dist/apis/GptConfigsApi.js +14 -14
- package/dist/bindings/PageInteractionTracker.d.ts +1 -1
- package/dist/bindings/PageInteractionTracker.js +3 -3
- package/dist/bindings/SetDonobuAnnotations.d.ts +1 -1
- package/dist/bindings/SetDonobuAnnotations.js +3 -3
- package/dist/clients/AnthropicGptClient.d.ts +2 -2
- package/dist/clients/AnthropicGptClient.js +77 -77
- package/dist/clients/OpenAiGptClient.d.ts +14 -14
- package/dist/clients/OpenAiGptClient.js +183 -183
- package/dist/esm/apis/GptConfigsApi.d.ts +5 -5
- package/dist/esm/apis/GptConfigsApi.js +14 -14
- package/dist/esm/bindings/PageInteractionTracker.d.ts +1 -1
- package/dist/esm/bindings/PageInteractionTracker.js +3 -3
- package/dist/esm/bindings/SetDonobuAnnotations.d.ts +1 -1
- package/dist/esm/bindings/SetDonobuAnnotations.js +3 -3
- package/dist/esm/clients/AnthropicGptClient.d.ts +2 -2
- package/dist/esm/clients/AnthropicGptClient.js +77 -77
- package/dist/esm/clients/OpenAiGptClient.d.ts +14 -14
- package/dist/esm/clients/OpenAiGptClient.js +183 -183
- package/dist/esm/lib/ai/PageAi.js +2 -1
- package/dist/esm/lib/page/extendPage.js +2 -1
- package/dist/esm/lib/test/utils/TestFileUpdater.d.ts +9 -9
- package/dist/esm/lib/test/utils/TestFileUpdater.js +49 -49
- package/dist/esm/main.d.ts +2 -0
- package/dist/esm/managers/AdminApiController.d.ts +16 -16
- package/dist/esm/managers/AdminApiController.js +35 -35
- package/dist/esm/managers/DonobuFlow.d.ts +57 -36
- package/dist/esm/managers/DonobuFlow.js +489 -564
- package/dist/esm/managers/DonobuFlowsManager.js +13 -17
- package/dist/esm/managers/FlowDependencyAnalyzer.d.ts +12 -12
- package/dist/esm/managers/FlowDependencyAnalyzer.js +77 -77
- package/dist/esm/managers/PageInspector.d.ts +38 -38
- package/dist/esm/managers/PageInspector.js +745 -745
- package/dist/esm/managers/TargetInspector.d.ts +28 -33
- package/dist/esm/managers/TestsManager.d.ts +25 -25
- package/dist/esm/managers/TestsManager.js +74 -74
- package/dist/esm/managers/ToolManager.js +7 -5
- package/dist/esm/managers/ToolRegistry.d.ts +5 -1
- package/dist/esm/managers/WebTargetInspector.d.ts +9 -5
- package/dist/esm/managers/WebTargetInspector.js +45 -47
- package/dist/esm/models/AiQuery.d.ts +29 -15
- package/dist/esm/models/AiQuery.js +31 -0
- package/dist/esm/models/ControlPanel.d.ts +18 -13
- package/dist/esm/models/InteractableElement.d.ts +6 -0
- package/dist/esm/models/InteractableElement.js +7 -1
- package/dist/esm/models/Observation.d.ts +38 -0
- package/dist/esm/models/Observation.js +3 -0
- package/dist/esm/models/ToolCallContext.d.ts +3 -2
- package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.d.ts +2 -2
- package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.js +19 -18
- package/dist/esm/persistence/flows/FlowsPersistenceSqlite.js +2 -1
- package/dist/esm/targets/TargetProvider.d.ts +110 -0
- package/dist/esm/targets/TargetProvider.js +25 -0
- package/dist/esm/targets/TargetRuntime.d.ts +6 -3
- package/dist/esm/targets/WebDialogHandler.d.ts +14 -0
- package/dist/esm/targets/WebDialogHandler.js +198 -0
- package/dist/esm/targets/WebTargetProvider.d.ts +32 -0
- package/dist/esm/targets/WebTargetProvider.js +136 -0
- package/dist/esm/targets/WebTargetRuntime.d.ts +2 -2
- package/dist/esm/targets/WebTargetRuntime.js +2 -1
- package/dist/esm/tools/AcknowledgeUserInstruction.d.ts +6 -0
- package/dist/esm/tools/AcknowledgeUserInstruction.js +7 -0
- package/dist/esm/tools/AssertPageTool.d.ts +1 -1
- package/dist/esm/tools/AssertPageTool.js +3 -3
- package/dist/esm/tools/DetectBrokenLinksTool.d.ts +2 -2
- package/dist/esm/tools/DetectBrokenLinksTool.js +44 -44
- package/dist/esm/tools/InputFakerTool.d.ts +4 -4
- package/dist/esm/tools/InputFakerTool.js +10 -10
- package/dist/esm/tools/InputTextTool.d.ts +4 -4
- package/dist/esm/tools/InputTextTool.js +7 -7
- package/dist/esm/tools/ReplayableInteraction.d.ts +34 -34
- package/dist/esm/tools/ReplayableInteraction.js +245 -245
- package/dist/esm/tools/Tool.d.ts +6 -3
- package/dist/esm/tools/Tool.js +5 -2
- package/dist/esm/utils/BrowserUtils.d.ts +19 -19
- package/dist/esm/utils/BrowserUtils.js +57 -57
- package/dist/esm/utils/MiscUtils.d.ts +2 -2
- package/dist/esm/utils/MiscUtils.js +16 -16
- package/dist/esm/utils/PlaywrightUtils.d.ts +1 -1
- package/dist/esm/utils/TargetUtils.d.ts +1 -1
- package/dist/esm/utils/TargetUtils.js +15 -13
- package/dist/lib/ai/PageAi.js +2 -1
- package/dist/lib/page/extendPage.js +2 -1
- package/dist/lib/test/utils/TestFileUpdater.d.ts +9 -9
- package/dist/lib/test/utils/TestFileUpdater.js +49 -49
- package/dist/main.d.ts +2 -0
- package/dist/managers/AdminApiController.d.ts +16 -16
- package/dist/managers/AdminApiController.js +35 -35
- package/dist/managers/DonobuFlow.d.ts +57 -36
- package/dist/managers/DonobuFlow.js +489 -564
- package/dist/managers/DonobuFlowsManager.js +13 -17
- package/dist/managers/FlowDependencyAnalyzer.d.ts +12 -12
- package/dist/managers/FlowDependencyAnalyzer.js +77 -77
- package/dist/managers/PageInspector.d.ts +38 -38
- package/dist/managers/PageInspector.js +745 -745
- package/dist/managers/TargetInspector.d.ts +28 -33
- package/dist/managers/TestsManager.d.ts +25 -25
- package/dist/managers/TestsManager.js +74 -74
- package/dist/managers/ToolManager.js +7 -5
- package/dist/managers/ToolRegistry.d.ts +5 -1
- package/dist/managers/WebTargetInspector.d.ts +9 -5
- package/dist/managers/WebTargetInspector.js +45 -47
- package/dist/models/AiQuery.d.ts +29 -15
- package/dist/models/AiQuery.js +31 -0
- package/dist/models/ControlPanel.d.ts +18 -13
- package/dist/models/InteractableElement.d.ts +6 -0
- package/dist/models/InteractableElement.js +7 -1
- package/dist/models/Observation.d.ts +38 -0
- package/dist/models/Observation.js +3 -0
- package/dist/models/ToolCallContext.d.ts +3 -2
- package/dist/persistence/flows/FlowsPersistenceDonobuApi.d.ts +2 -2
- package/dist/persistence/flows/FlowsPersistenceDonobuApi.js +19 -18
- package/dist/persistence/flows/FlowsPersistenceSqlite.js +2 -1
- package/dist/targets/TargetProvider.d.ts +110 -0
- package/dist/targets/TargetProvider.js +25 -0
- package/dist/targets/TargetRuntime.d.ts +6 -3
- package/dist/targets/WebDialogHandler.d.ts +14 -0
- package/dist/targets/WebDialogHandler.js +198 -0
- package/dist/targets/WebTargetProvider.d.ts +32 -0
- package/dist/targets/WebTargetProvider.js +136 -0
- package/dist/targets/WebTargetRuntime.d.ts +2 -2
- package/dist/targets/WebTargetRuntime.js +2 -1
- package/dist/tools/AcknowledgeUserInstruction.d.ts +6 -0
- package/dist/tools/AcknowledgeUserInstruction.js +7 -0
- package/dist/tools/AssertPageTool.d.ts +1 -1
- package/dist/tools/AssertPageTool.js +3 -3
- package/dist/tools/DetectBrokenLinksTool.d.ts +2 -2
- package/dist/tools/DetectBrokenLinksTool.js +44 -44
- package/dist/tools/InputFakerTool.d.ts +4 -4
- package/dist/tools/InputFakerTool.js +10 -10
- package/dist/tools/InputTextTool.d.ts +4 -4
- package/dist/tools/InputTextTool.js +7 -7
- package/dist/tools/ReplayableInteraction.d.ts +34 -34
- package/dist/tools/ReplayableInteraction.js +245 -245
- package/dist/tools/Tool.d.ts +6 -3
- package/dist/tools/Tool.js +5 -2
- package/dist/utils/BrowserUtils.d.ts +19 -19
- package/dist/utils/BrowserUtils.js +57 -57
- package/dist/utils/MiscUtils.d.ts +2 -2
- package/dist/utils/MiscUtils.js +16 -16
- package/dist/utils/PlaywrightUtils.d.ts +1 -1
- package/dist/utils/TargetUtils.d.ts +1 -1
- package/dist/utils/TargetUtils.js +15 -13
- package/package.json +2 -1
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import type { InteractionTrackingHost } from '../bindings/PageInteractionTracker';
|
|
2
|
+
import type { ToolOption } from '../clients/GptClient';
|
|
2
3
|
import type { FlowMetadata } from '../models/FlowMetadata';
|
|
3
4
|
import type { FocusedTarget } from '../models/FocusedTarget';
|
|
5
|
+
import type { GptMessage, ProposedToolCallsMessage } from '../models/GptMessage';
|
|
4
6
|
import type { InteractableElement } from '../models/InteractableElement';
|
|
7
|
+
import type { ProposedToolCall } from '../models/ProposedToolCall';
|
|
8
|
+
import type { ToolCall } from '../models/ToolCall';
|
|
5
9
|
import type { FlowsPersistence } from '../persistence/flows/FlowsPersistence';
|
|
6
10
|
/**
|
|
7
11
|
* Abstraction over screen/page inspection for LLM-driven flows.
|
|
@@ -91,13 +95,6 @@ export interface TargetInspectorBase {
|
|
|
91
95
|
* Returns a fallback string if the target is disconnected.
|
|
92
96
|
*/
|
|
93
97
|
getCurrentLocation(): string;
|
|
94
|
-
/**
|
|
95
|
-
* Platform-specific text fragments used when constructing LLM prompts.
|
|
96
|
-
*
|
|
97
|
-
* Returning a single object avoids scattering many one-liner methods
|
|
98
|
-
* across every inspector implementation.
|
|
99
|
-
*/
|
|
100
|
-
getPlatformPromptInfo(): PlatformPromptInfo;
|
|
101
98
|
/**
|
|
102
99
|
* Context description for the LLM user message (e.g. browser tabs list
|
|
103
100
|
* for web, or a generic description for mobile).
|
|
@@ -121,25 +118,6 @@ export interface TargetInspectorBase {
|
|
|
121
118
|
*/
|
|
122
119
|
persistSessionState(persistence: FlowsPersistence, flowId: string): Promise<void>;
|
|
123
120
|
}
|
|
124
|
-
/**
|
|
125
|
-
* Platform-specific text fragments for LLM prompt construction.
|
|
126
|
-
*
|
|
127
|
-
* Each inspector implementation returns a static instance of this.
|
|
128
|
-
*/
|
|
129
|
-
export interface PlatformPromptInfo {
|
|
130
|
-
/** Opening paragraph(s) of the LLM system message. */
|
|
131
|
-
readonly systemPreamble: string;
|
|
132
|
-
/** e.g. "a webpage" or "a mobile app screen" — used after "two images of". */
|
|
133
|
-
readonly screenshotSubject: string;
|
|
134
|
-
/** e.g. "web page (i.e. the current viewport)" or "mobile app screen". */
|
|
135
|
-
readonly currentViewDescription: string;
|
|
136
|
-
/** e.g. "viewport of the web page" or "mobile app screen". */
|
|
137
|
-
readonly annotatedViewDescription: string;
|
|
138
|
-
/** e.g. "website" or "mobile app" — used in "interact with the {x}". */
|
|
139
|
-
readonly interactionTarget: string;
|
|
140
|
-
/** e.g. "webpage" or "app" — short noun for the target. */
|
|
141
|
-
readonly targetNoun: string;
|
|
142
|
-
}
|
|
143
121
|
/**
|
|
144
122
|
* Callbacks provided by the flow engine during {@link TargetInspectorBase.initialize}.
|
|
145
123
|
*
|
|
@@ -150,15 +128,15 @@ export interface TargetInitCallbacks {
|
|
|
150
128
|
/** Flow metadata (mutable reference — used to read `runMode` etc.). */
|
|
151
129
|
readonly metadata: FlowMetadata;
|
|
152
130
|
/**
|
|
153
|
-
*
|
|
131
|
+
* Flow-engine collaborators a provider needs to handle platform dialog
|
|
132
|
+
* events (e.g. browser alert/confirm/prompt). The provider owns the dialog
|
|
133
|
+
* handling logic and wires it to its own platform event; this shim gives it
|
|
134
|
+
* the narrow slice of engine state it needs, without depending on
|
|
135
|
+
* `DonobuFlow` directly (same inversion as {@link InteractionTrackingHost}).
|
|
154
136
|
*
|
|
155
|
-
*
|
|
156
|
-
* is the platform-specific dialog object (e.g. Playwright `Dialog`), typed
|
|
157
|
-
* as `unknown` to keep this interface platform-agnostic.
|
|
158
|
-
*
|
|
159
|
-
* Only used by web targets. Mobile targets ignore this.
|
|
137
|
+
* Only consumed by web targets. Other targets ignore it.
|
|
160
138
|
*/
|
|
161
|
-
readonly
|
|
139
|
+
readonly dialogHost?: DialogHost;
|
|
162
140
|
/**
|
|
163
141
|
* Flow-engine state needed for interaction tracking registration.
|
|
164
142
|
*
|
|
@@ -169,6 +147,23 @@ export interface TargetInitCallbacks {
|
|
|
169
147
|
*/
|
|
170
148
|
readonly interactionTrackingHost?: InteractionTrackingHost;
|
|
171
149
|
}
|
|
150
|
+
/**
|
|
151
|
+
* Narrow view of the flow engine that a provider's dialog handler needs.
|
|
152
|
+
* `DonobuFlow` structurally satisfies this. All array/metadata members are
|
|
153
|
+
* live references the handler may read and mutate (e.g. shifting a queued
|
|
154
|
+
* tool call, flipping `metadata.state`).
|
|
155
|
+
*/
|
|
156
|
+
export interface DialogHost {
|
|
157
|
+
readonly proposedToolCalls: ProposedToolCall[];
|
|
158
|
+
readonly invokedToolCalls: ToolCall[];
|
|
159
|
+
readonly gptMessages: GptMessage[];
|
|
160
|
+
readonly metadata: FlowMetadata;
|
|
161
|
+
readonly persistence: FlowsPersistence;
|
|
162
|
+
/** Query the LLM with a one-off tool set (used to decide a dialog response). */
|
|
163
|
+
queryGpt(messages: GptMessage[], tools: ToolOption[]): Promise<ProposedToolCallsMessage>;
|
|
164
|
+
/** Size-optimize the message history before a one-off query. */
|
|
165
|
+
optimizeHistory(history: GptMessage[]): GptMessage[];
|
|
166
|
+
}
|
|
172
167
|
/**
|
|
173
168
|
* Union of all target inspector implementations.
|
|
174
169
|
*
|
|
@@ -12,20 +12,6 @@ export declare class TestsManager {
|
|
|
12
12
|
constructor(testsPersistenceRegistry: TestsPersistenceRegistry, suitesPersistenceRegistry: SuitesPersistenceRegistry, flowsManager: DonobuFlowsManager);
|
|
13
13
|
createTest(params: CreateTest): Promise<TestMetadata>;
|
|
14
14
|
getTestById(testId: string): Promise<TestMetadata>;
|
|
15
|
-
/**
|
|
16
|
-
* Picks the tests persistence layer to use when creating a new test.
|
|
17
|
-
*
|
|
18
|
-
* - If `suiteId` is null/undefined: use the primary tests layer.
|
|
19
|
-
* - If `suiteId` is set: look up the suite's layer key and use the matching
|
|
20
|
-
* tests layer. If no tests layer matches the suite's key (rare — would
|
|
21
|
-
* require asymmetric registry config), fall back to the primary tests
|
|
22
|
-
* layer; the FK won't hold but at least the test is persisted somewhere.
|
|
23
|
-
* - If `suiteId` is set but the suite doesn't exist anywhere: fall back
|
|
24
|
-
* to the primary tests layer (the SQLite FK will reject if applicable;
|
|
25
|
-
* non-DB layers will accept the dangling reference).
|
|
26
|
-
*/
|
|
27
|
-
private resolveLayerForTestCreate;
|
|
28
|
-
private findSuiteLayerKey;
|
|
29
15
|
getTests(query: TestsQuery): Promise<PaginatedResult<TestListItem>>;
|
|
30
16
|
/**
|
|
31
17
|
* Update a test in the persistence layer where it exists.
|
|
@@ -41,17 +27,6 @@ export declare class TestsManager {
|
|
|
41
27
|
* layers (Volatile, S3, GCS).
|
|
42
28
|
*/
|
|
43
29
|
deleteTest(testId: string): Promise<void>;
|
|
44
|
-
/**
|
|
45
|
-
* Gets the list of tool calls to invoke when starting a new flow for the
|
|
46
|
-
* given test, based on the most recent successful flow for the test.
|
|
47
|
-
*
|
|
48
|
-
* @param test - The test to get the tool calls for
|
|
49
|
-
*
|
|
50
|
-
* @returns The tool calls to use when executing the test
|
|
51
|
-
*
|
|
52
|
-
* @throws {Error} if no previous successful flow is found
|
|
53
|
-
*/
|
|
54
|
-
private getTestToolCalls;
|
|
55
30
|
/**
|
|
56
31
|
* Creates a new flow (config) for the given test, which should be passed to
|
|
57
32
|
* `flowsManager.createFlow` to execute the test. The returned config's
|
|
@@ -63,5 +38,30 @@ export declare class TestsManager {
|
|
|
63
38
|
* @returns A new flow configuration
|
|
64
39
|
*/
|
|
65
40
|
getNewFlowFromTest(testId: string): Promise<CreateDonobuFlow>;
|
|
41
|
+
/**
|
|
42
|
+
* Picks the tests persistence layer to use when creating a new test.
|
|
43
|
+
*
|
|
44
|
+
* - If `suiteId` is null/undefined: use the primary tests layer.
|
|
45
|
+
* - If `suiteId` is set: look up the suite's layer key and use the matching
|
|
46
|
+
* tests layer. If no tests layer matches the suite's key (rare — would
|
|
47
|
+
* require asymmetric registry config), fall back to the primary tests
|
|
48
|
+
* layer; the FK won't hold but at least the test is persisted somewhere.
|
|
49
|
+
* - If `suiteId` is set but the suite doesn't exist anywhere: fall back
|
|
50
|
+
* to the primary tests layer (the SQLite FK will reject if applicable;
|
|
51
|
+
* non-DB layers will accept the dangling reference).
|
|
52
|
+
*/
|
|
53
|
+
private resolveLayerForTestCreate;
|
|
54
|
+
private findSuiteLayerKey;
|
|
55
|
+
/**
|
|
56
|
+
* Gets the list of tool calls to invoke when starting a new flow for the
|
|
57
|
+
* given test, based on the most recent successful flow for the test.
|
|
58
|
+
*
|
|
59
|
+
* @param test - The test to get the tool calls for
|
|
60
|
+
*
|
|
61
|
+
* @returns The tool calls to use when executing the test
|
|
62
|
+
*
|
|
63
|
+
* @throws {Error} if no previous successful flow is found
|
|
64
|
+
*/
|
|
65
|
+
private getTestToolCalls;
|
|
66
66
|
}
|
|
67
67
|
//# sourceMappingURL=TestsManager.d.ts.map
|
|
@@ -63,49 +63,6 @@ class TestsManager {
|
|
|
63
63
|
}
|
|
64
64
|
throw TestNotFoundException_1.TestNotFoundException.forId(testId);
|
|
65
65
|
}
|
|
66
|
-
/**
|
|
67
|
-
* Picks the tests persistence layer to use when creating a new test.
|
|
68
|
-
*
|
|
69
|
-
* - If `suiteId` is null/undefined: use the primary tests layer.
|
|
70
|
-
* - If `suiteId` is set: look up the suite's layer key and use the matching
|
|
71
|
-
* tests layer. If no tests layer matches the suite's key (rare — would
|
|
72
|
-
* require asymmetric registry config), fall back to the primary tests
|
|
73
|
-
* layer; the FK won't hold but at least the test is persisted somewhere.
|
|
74
|
-
* - If `suiteId` is set but the suite doesn't exist anywhere: fall back
|
|
75
|
-
* to the primary tests layer (the SQLite FK will reject if applicable;
|
|
76
|
-
* non-DB layers will accept the dangling reference).
|
|
77
|
-
*/
|
|
78
|
-
async resolveLayerForTestCreate(suiteId) {
|
|
79
|
-
if (!suiteId) {
|
|
80
|
-
return this.testsPersistenceRegistry.get();
|
|
81
|
-
}
|
|
82
|
-
let suiteLayerKey;
|
|
83
|
-
try {
|
|
84
|
-
suiteLayerKey = await this.findSuiteLayerKey(suiteId);
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
if (!(error instanceof SuiteNotFoundException_1.SuiteNotFoundException)) {
|
|
88
|
-
throw error;
|
|
89
|
-
}
|
|
90
|
-
return this.testsPersistenceRegistry.get();
|
|
91
|
-
}
|
|
92
|
-
const matched = await this.testsPersistenceRegistry.getByKey(suiteLayerKey);
|
|
93
|
-
return matched ?? (await this.testsPersistenceRegistry.get());
|
|
94
|
-
}
|
|
95
|
-
async findSuiteLayerKey(suiteId) {
|
|
96
|
-
for (const { key, persistence, } of await this.suitesPersistenceRegistry.getEntries()) {
|
|
97
|
-
try {
|
|
98
|
-
await persistence.getSuiteById(suiteId);
|
|
99
|
-
return key;
|
|
100
|
-
}
|
|
101
|
-
catch (error) {
|
|
102
|
-
if (!(error instanceof SuiteNotFoundException_1.SuiteNotFoundException)) {
|
|
103
|
-
throw error;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
throw SuiteNotFoundException_1.SuiteNotFoundException.forId(suiteId);
|
|
108
|
-
}
|
|
109
66
|
async getTests(query) {
|
|
110
67
|
const layers = (await this.testsPersistenceRegistry.getAll()).map((persistence) => ({
|
|
111
68
|
getItems: (q) => persistence.getTests(q),
|
|
@@ -210,37 +167,6 @@ class TestsManager {
|
|
|
210
167
|
console.warn(`Failed to delete ${flowsNotDeleted.length} flows belonging to test ${testId}: ${flowsNotDeleted.map((f) => f.id).join(', ')}`);
|
|
211
168
|
}
|
|
212
169
|
}
|
|
213
|
-
/**
|
|
214
|
-
* Gets the list of tool calls to invoke when starting a new flow for the
|
|
215
|
-
* given test, based on the most recent successful flow for the test.
|
|
216
|
-
*
|
|
217
|
-
* @param test - The test to get the tool calls for
|
|
218
|
-
*
|
|
219
|
-
* @returns The tool calls to use when executing the test
|
|
220
|
-
*
|
|
221
|
-
* @throws {Error} if no previous successful flow is found
|
|
222
|
-
*/
|
|
223
|
-
async getTestToolCalls(test) {
|
|
224
|
-
let previousSuccesfulFlow = undefined;
|
|
225
|
-
let pageToken = undefined;
|
|
226
|
-
while (!previousSuccesfulFlow) {
|
|
227
|
-
const { items: previousFlows, nextPageToken } = await this.flowsManager.getFlows({
|
|
228
|
-
testId: test.id,
|
|
229
|
-
sortBy: 'created_at',
|
|
230
|
-
sortOrder: 'desc',
|
|
231
|
-
pageToken,
|
|
232
|
-
});
|
|
233
|
-
previousSuccesfulFlow = previousFlows.find((f) => f.state === 'SUCCESS');
|
|
234
|
-
if (!previousSuccesfulFlow && !nextPageToken) {
|
|
235
|
-
throw new Error(`No previous successful flow found for test ${test.id}`);
|
|
236
|
-
}
|
|
237
|
-
pageToken = nextPageToken;
|
|
238
|
-
}
|
|
239
|
-
return this.flowsManager.getToolCallsForRerun(previousSuccesfulFlow, {
|
|
240
|
-
areElementIdsVolatile: false,
|
|
241
|
-
disableSelectorFailover: false,
|
|
242
|
-
});
|
|
243
|
-
}
|
|
244
170
|
/**
|
|
245
171
|
* Creates a new flow (config) for the given test, which should be passed to
|
|
246
172
|
* `flowsManager.createFlow` to execute the test. The returned config's
|
|
@@ -276,6 +202,80 @@ class TestsManager {
|
|
|
276
202
|
newFlowConfig.testId = test.id;
|
|
277
203
|
return newFlowConfig;
|
|
278
204
|
}
|
|
205
|
+
/**
|
|
206
|
+
* Picks the tests persistence layer to use when creating a new test.
|
|
207
|
+
*
|
|
208
|
+
* - If `suiteId` is null/undefined: use the primary tests layer.
|
|
209
|
+
* - If `suiteId` is set: look up the suite's layer key and use the matching
|
|
210
|
+
* tests layer. If no tests layer matches the suite's key (rare — would
|
|
211
|
+
* require asymmetric registry config), fall back to the primary tests
|
|
212
|
+
* layer; the FK won't hold but at least the test is persisted somewhere.
|
|
213
|
+
* - If `suiteId` is set but the suite doesn't exist anywhere: fall back
|
|
214
|
+
* to the primary tests layer (the SQLite FK will reject if applicable;
|
|
215
|
+
* non-DB layers will accept the dangling reference).
|
|
216
|
+
*/
|
|
217
|
+
async resolveLayerForTestCreate(suiteId) {
|
|
218
|
+
if (!suiteId) {
|
|
219
|
+
return this.testsPersistenceRegistry.get();
|
|
220
|
+
}
|
|
221
|
+
let suiteLayerKey;
|
|
222
|
+
try {
|
|
223
|
+
suiteLayerKey = await this.findSuiteLayerKey(suiteId);
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
if (!(error instanceof SuiteNotFoundException_1.SuiteNotFoundException)) {
|
|
227
|
+
throw error;
|
|
228
|
+
}
|
|
229
|
+
return this.testsPersistenceRegistry.get();
|
|
230
|
+
}
|
|
231
|
+
const matched = await this.testsPersistenceRegistry.getByKey(suiteLayerKey);
|
|
232
|
+
return matched ?? (await this.testsPersistenceRegistry.get());
|
|
233
|
+
}
|
|
234
|
+
async findSuiteLayerKey(suiteId) {
|
|
235
|
+
for (const { key, persistence, } of await this.suitesPersistenceRegistry.getEntries()) {
|
|
236
|
+
try {
|
|
237
|
+
await persistence.getSuiteById(suiteId);
|
|
238
|
+
return key;
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
if (!(error instanceof SuiteNotFoundException_1.SuiteNotFoundException)) {
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
throw SuiteNotFoundException_1.SuiteNotFoundException.forId(suiteId);
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Gets the list of tool calls to invoke when starting a new flow for the
|
|
250
|
+
* given test, based on the most recent successful flow for the test.
|
|
251
|
+
*
|
|
252
|
+
* @param test - The test to get the tool calls for
|
|
253
|
+
*
|
|
254
|
+
* @returns The tool calls to use when executing the test
|
|
255
|
+
*
|
|
256
|
+
* @throws {Error} if no previous successful flow is found
|
|
257
|
+
*/
|
|
258
|
+
async getTestToolCalls(test) {
|
|
259
|
+
let previousSuccesfulFlow = undefined;
|
|
260
|
+
let pageToken = undefined;
|
|
261
|
+
while (!previousSuccesfulFlow) {
|
|
262
|
+
const { items: previousFlows, nextPageToken } = await this.flowsManager.getFlows({
|
|
263
|
+
testId: test.id,
|
|
264
|
+
sortBy: 'created_at',
|
|
265
|
+
sortOrder: 'desc',
|
|
266
|
+
pageToken,
|
|
267
|
+
});
|
|
268
|
+
previousSuccesfulFlow = previousFlows.find((f) => f.state === 'SUCCESS');
|
|
269
|
+
if (!previousSuccesfulFlow && !nextPageToken) {
|
|
270
|
+
throw new Error(`No previous successful flow found for test ${test.id}`);
|
|
271
|
+
}
|
|
272
|
+
pageToken = nextPageToken;
|
|
273
|
+
}
|
|
274
|
+
return this.flowsManager.getToolCallsForRerun(previousSuccesfulFlow, {
|
|
275
|
+
areElementIdsVolatile: false,
|
|
276
|
+
disableSelectorFailover: false,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
279
|
}
|
|
280
280
|
exports.TestsManager = TestsManager;
|
|
281
281
|
//# sourceMappingURL=TestsManager.js.map
|
|
@@ -5,6 +5,7 @@ exports.buildToolInterpolationDataSource = buildToolInterpolationDataSource;
|
|
|
5
5
|
const v4_1 = require("zod/v4");
|
|
6
6
|
const PageClosedException_1 = require("../exceptions/PageClosedException");
|
|
7
7
|
const ToolRequiresGptException_1 = require("../exceptions/ToolRequiresGptException");
|
|
8
|
+
const TargetProvider_1 = require("../targets/TargetProvider");
|
|
8
9
|
const JsonUtils_1 = require("../utils/JsonUtils");
|
|
9
10
|
const Logger_1 = require("../utils/Logger");
|
|
10
11
|
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
|
|
@@ -39,9 +40,8 @@ class ToolManager {
|
|
|
39
40
|
* @returns A Promise resolving to a ToolCall object containing execution details and results
|
|
40
41
|
*/
|
|
41
42
|
async invokeTool(context, toolName, toolParameters, isFromGpt) {
|
|
42
|
-
const target = context.targetInspector.target;
|
|
43
43
|
const startedAt = new Date().getTime();
|
|
44
|
-
const initialPageUrl =
|
|
44
|
+
const initialPageUrl = (0, TargetProvider_1.currentLocation)(context.provider);
|
|
45
45
|
Logger_1.appLogger.info(`Taking action: ${toolName}`, {
|
|
46
46
|
toolName,
|
|
47
47
|
parameters: toolParameters,
|
|
@@ -83,7 +83,7 @@ class ToolManager {
|
|
|
83
83
|
parameters: toolParameters,
|
|
84
84
|
outcome: null,
|
|
85
85
|
postCallImageId: null,
|
|
86
|
-
page: initialPageUrl
|
|
86
|
+
page: initialPageUrl,
|
|
87
87
|
startedAt: startedAt,
|
|
88
88
|
completedAt: null,
|
|
89
89
|
});
|
|
@@ -127,8 +127,10 @@ class ToolManager {
|
|
|
127
127
|
}
|
|
128
128
|
if (toolCallResult.metadata?.exception !== new PageClosedException_1.PageClosedException().name) {
|
|
129
129
|
try {
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
// captureSnapshot returns null when nothing is capturable (no target,
|
|
131
|
+
// disconnected, or no visual snapshot), so no connectivity guard here.
|
|
132
|
+
const postCallImage = await (0, TargetProvider_1.captureSnapshot)(context.provider);
|
|
133
|
+
if (postCallImage) {
|
|
132
134
|
postCallImageId = await context.persistence.saveScreenShot(context.metadata.id, postCallImage);
|
|
133
135
|
}
|
|
134
136
|
}
|
|
@@ -18,7 +18,11 @@ export interface ToolRegistry {
|
|
|
18
18
|
defaultTools(): Tool<z.ZodObject, z.ZodObject>[];
|
|
19
19
|
/** Every available tool: prepackaged + plugins, with plugin overrides applied. */
|
|
20
20
|
allTools(): Tool<z.ZodObject, z.ZodObject>[];
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* All tools compatible with the given target type (e.g. `'web'`, `'mobile'`):
|
|
23
|
+
* a tool is included if it is universal (no declared targets) or supports the
|
|
24
|
+
* type.
|
|
25
|
+
*/
|
|
22
26
|
toolsForTarget(targetType: string): Tool<z.ZodObject, z.ZodObject>[];
|
|
23
27
|
}
|
|
24
28
|
/**
|
|
@@ -4,7 +4,7 @@ import type { InteractableElement } from '../models/InteractableElement';
|
|
|
4
4
|
import type { FlowsPersistence } from '../persistence/flows/FlowsPersistence';
|
|
5
5
|
import type { InteractionVisualizer } from './InteractionVisualizer';
|
|
6
6
|
import { PageInspector } from './PageInspector';
|
|
7
|
-
import type {
|
|
7
|
+
import type { TargetInitCallbacks, TargetInspectorBase } from './TargetInspector';
|
|
8
8
|
/**
|
|
9
9
|
* Web implementation of {@link TargetInspectorBase}.
|
|
10
10
|
*
|
|
@@ -27,12 +27,10 @@ export declare class WebTargetInspector implements TargetInspectorBase {
|
|
|
27
27
|
private readonly _interactionVisualizer;
|
|
28
28
|
readonly type: "web";
|
|
29
29
|
readonly pageInspector: PageInspector;
|
|
30
|
-
private static readonly PROMPT_INFO;
|
|
31
|
-
constructor(_target: WebTarget, _browserContext: BrowserContext, _interactionVisualizer: InteractionVisualizer);
|
|
32
30
|
get target(): WebTarget;
|
|
33
31
|
get connected(): boolean;
|
|
34
32
|
get interactableElementAttribute(): string;
|
|
35
|
-
|
|
33
|
+
constructor(_target: WebTarget, _browserContext: BrowserContext, _interactionVisualizer: InteractionVisualizer);
|
|
36
34
|
attributeInteractableElements(): Promise<void>;
|
|
37
35
|
getAttributedInteractableElements(): Promise<InteractableElement[]>;
|
|
38
36
|
annotateInteractableElements(): Promise<void>;
|
|
@@ -51,11 +49,17 @@ export declare class WebTargetInspector implements TargetInspectorBase {
|
|
|
51
49
|
}>;
|
|
52
50
|
showInteractionCursor(): Promise<void>;
|
|
53
51
|
hideInteractionCursor(): Promise<void>;
|
|
52
|
+
/**
|
|
53
|
+
* Close the browser context to interrupt any in-flight Playwright call so a
|
|
54
|
+
* cancelled flow's run loop unblocks at once. This only interrupts — it does
|
|
55
|
+
* not tear down the browser.
|
|
56
|
+
*/
|
|
57
|
+
interrupt(): Promise<void>;
|
|
54
58
|
getCurrentLocation(): string;
|
|
55
|
-
getPlatformPromptInfo(): PlatformPromptInfo;
|
|
56
59
|
getContextDescription(): string;
|
|
57
60
|
initialize(callbacks: TargetInitCallbacks): Promise<void>;
|
|
58
61
|
persistSessionState(persistence: FlowsPersistence, flowId: string): Promise<void>;
|
|
62
|
+
private requirePage;
|
|
59
63
|
private handleNewPage;
|
|
60
64
|
}
|
|
61
65
|
//# sourceMappingURL=WebTargetInspector.d.ts.map
|
|
@@ -4,6 +4,7 @@ exports.WebTargetInspector = void 0;
|
|
|
4
4
|
const PageInteractionTracker_1 = require("../bindings/PageInteractionTracker");
|
|
5
5
|
const SetDonobuAnnotations_1 = require("../bindings/SetDonobuAnnotations");
|
|
6
6
|
const PageClosedException_1 = require("../exceptions/PageClosedException");
|
|
7
|
+
const WebDialogHandler_1 = require("../targets/WebDialogHandler");
|
|
7
8
|
const BrowserUtils_1 = require("../utils/BrowserUtils");
|
|
8
9
|
const Logger_1 = require("../utils/Logger");
|
|
9
10
|
const PageLogListeners_1 = require("../utils/PageLogListeners");
|
|
@@ -26,37 +27,31 @@ const PageInspector_1 = require("./PageInspector");
|
|
|
26
27
|
* Page-specific helpers like `getHtmlSnippet`.
|
|
27
28
|
*/
|
|
28
29
|
class WebTargetInspector {
|
|
29
|
-
|
|
30
|
-
this._target
|
|
31
|
-
this._browserContext = _browserContext;
|
|
32
|
-
this._interactionVisualizer = _interactionVisualizer;
|
|
33
|
-
this.type = 'web';
|
|
34
|
-
this.pageInspector = new PageInspector_1.PageInspector();
|
|
30
|
+
get target() {
|
|
31
|
+
return this._target;
|
|
35
32
|
}
|
|
36
33
|
/* ------------------------------------------------------------------ */
|
|
37
34
|
/* Target accessors */
|
|
38
35
|
/* ------------------------------------------------------------------ */
|
|
39
|
-
get target() {
|
|
40
|
-
return this._target;
|
|
41
|
-
}
|
|
42
36
|
get connected() {
|
|
43
37
|
return this._target.current !== null;
|
|
44
38
|
}
|
|
45
39
|
get interactableElementAttribute() {
|
|
46
40
|
return this.pageInspector.interactableElementAttribute;
|
|
47
41
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
42
|
+
constructor(_target, _browserContext, _interactionVisualizer) {
|
|
43
|
+
this._target = _target;
|
|
44
|
+
this._browserContext = _browserContext;
|
|
45
|
+
this._interactionVisualizer = _interactionVisualizer;
|
|
46
|
+
this.type = 'web';
|
|
47
|
+
this.pageInspector = new PageInspector_1.PageInspector();
|
|
53
48
|
}
|
|
54
|
-
/* ------------------------------------------------------------------ */
|
|
55
|
-
/* Element inspection */
|
|
56
|
-
/* ------------------------------------------------------------------ */
|
|
57
49
|
async attributeInteractableElements() {
|
|
58
50
|
await this.pageInspector.attributeInteractableElements(this.requirePage());
|
|
59
51
|
}
|
|
52
|
+
/* ------------------------------------------------------------------ */
|
|
53
|
+
/* Element inspection */
|
|
54
|
+
/* ------------------------------------------------------------------ */
|
|
60
55
|
async getAttributedInteractableElements() {
|
|
61
56
|
return this.pageInspector.getAttributedInteractableElements(this.requirePage());
|
|
62
57
|
}
|
|
@@ -66,26 +61,26 @@ class WebTargetInspector {
|
|
|
66
61
|
async removeAnnotations() {
|
|
67
62
|
await this.pageInspector.removeDonobuAnnotations(this.requirePage());
|
|
68
63
|
}
|
|
69
|
-
/* ------------------------------------------------------------------ */
|
|
70
|
-
/* Screenshots */
|
|
71
|
-
/* ------------------------------------------------------------------ */
|
|
72
64
|
async takeCleanScreenshot() {
|
|
73
65
|
return PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(this.requirePage());
|
|
74
66
|
}
|
|
67
|
+
/* ------------------------------------------------------------------ */
|
|
68
|
+
/* Screenshots */
|
|
69
|
+
/* ------------------------------------------------------------------ */
|
|
75
70
|
async takeAnnotatedScreenshot() {
|
|
76
71
|
return PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(this.requirePage());
|
|
77
72
|
}
|
|
78
73
|
async captureScreenshot() {
|
|
79
74
|
return PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(this.requirePage());
|
|
80
75
|
}
|
|
81
|
-
/* ------------------------------------------------------------------ */
|
|
82
|
-
/* Connection lifecycle */
|
|
83
|
-
/* ------------------------------------------------------------------ */
|
|
84
76
|
checkConnectedOrThrow() {
|
|
85
77
|
if (!this._target.current) {
|
|
86
78
|
throw new PageClosedException_1.PageClosedException();
|
|
87
79
|
}
|
|
88
80
|
}
|
|
81
|
+
/* ------------------------------------------------------------------ */
|
|
82
|
+
/* Connection lifecycle */
|
|
83
|
+
/* ------------------------------------------------------------------ */
|
|
89
84
|
checkTargetAliveOrThrow() {
|
|
90
85
|
if (this._target.current?.isClosed()) {
|
|
91
86
|
throw new PageClosedException_1.PageClosedException();
|
|
@@ -106,28 +101,35 @@ class WebTargetInspector {
|
|
|
106
101
|
this._target.current = allPages[0];
|
|
107
102
|
return { recovered: true };
|
|
108
103
|
}
|
|
109
|
-
/* ------------------------------------------------------------------ */
|
|
110
|
-
/* Interaction cursor */
|
|
111
|
-
/* ------------------------------------------------------------------ */
|
|
112
104
|
async showInteractionCursor() {
|
|
113
105
|
if (this._target.current) {
|
|
114
106
|
await this._interactionVisualizer.showMouse(this._target.current);
|
|
115
107
|
}
|
|
116
108
|
}
|
|
109
|
+
/* ------------------------------------------------------------------ */
|
|
110
|
+
/* Interaction cursor */
|
|
111
|
+
/* ------------------------------------------------------------------ */
|
|
117
112
|
async hideInteractionCursor() {
|
|
118
113
|
if (this._target.current) {
|
|
119
114
|
await this._interactionVisualizer.hideMouse(this._target.current);
|
|
120
115
|
}
|
|
121
116
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
117
|
+
/**
|
|
118
|
+
* Close the browser context to interrupt any in-flight Playwright call so a
|
|
119
|
+
* cancelled flow's run loop unblocks at once. This only interrupts — it does
|
|
120
|
+
* not tear down the browser.
|
|
121
|
+
*/
|
|
122
|
+
async interrupt() {
|
|
123
|
+
if (this._target.current) {
|
|
124
|
+
await this._browserContext.close();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
125
127
|
getCurrentLocation() {
|
|
126
128
|
return this._target.current?.url() ?? 'about:blank';
|
|
127
129
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
130
|
+
/* ------------------------------------------------------------------ */
|
|
131
|
+
/* Location & platform identity */
|
|
132
|
+
/* ------------------------------------------------------------------ */
|
|
131
133
|
getContextDescription() {
|
|
132
134
|
if (!this._target.current) {
|
|
133
135
|
return 'The web browser has no open pages.';
|
|
@@ -141,9 +143,6 @@ class WebTargetInspector {
|
|
|
141
143
|
|
|
142
144
|
The active (i.e. in focus) tab is ${this._target.current.url()}`;
|
|
143
145
|
}
|
|
144
|
-
/* ------------------------------------------------------------------ */
|
|
145
|
-
/* Initialization & session state */
|
|
146
|
-
/* ------------------------------------------------------------------ */
|
|
147
146
|
async initialize(callbacks) {
|
|
148
147
|
const browserContext = this._browserContext;
|
|
149
148
|
// Register page handler — assigns new tabs/popups as the focused page
|
|
@@ -151,8 +150,8 @@ The active (i.e. in focus) tab is ${this._target.current.url()}`;
|
|
|
151
150
|
browserContext.on('page', (page) => this.handleNewPage(page, callbacks));
|
|
152
151
|
await PlaywrightUtils_1.PlaywrightUtils.setupBasicBrowserContext(browserContext);
|
|
153
152
|
// Wire flow-engine handlers onto the browser context.
|
|
154
|
-
if (callbacks.
|
|
155
|
-
browserContext.on('dialog', callbacks.
|
|
153
|
+
if (callbacks.dialogHost) {
|
|
154
|
+
browserContext.on('dialog', (0, WebDialogHandler_1.createWebDialogHandler)(callbacks.dialogHost));
|
|
156
155
|
}
|
|
157
156
|
if (callbacks.interactionTrackingHost) {
|
|
158
157
|
await PageInteractionTracker_1.PageInteractionTracker.register(callbacks.interactionTrackingHost, browserContext);
|
|
@@ -172,6 +171,9 @@ The active (i.e. in focus) tab is ${this._target.current.url()}`;
|
|
|
172
171
|
}
|
|
173
172
|
}
|
|
174
173
|
}
|
|
174
|
+
/* ------------------------------------------------------------------ */
|
|
175
|
+
/* Initialization & session state */
|
|
176
|
+
/* ------------------------------------------------------------------ */
|
|
175
177
|
async persistSessionState(persistence, flowId) {
|
|
176
178
|
try {
|
|
177
179
|
const browserState = await BrowserUtils_1.BrowserUtils.getBrowserStorageState(this._browserContext);
|
|
@@ -181,6 +183,12 @@ The active (i.e. in focus) tab is ${this._target.current.url()}`;
|
|
|
181
183
|
Logger_1.appLogger.error('Failed to persist browser state when completing flow', error);
|
|
182
184
|
}
|
|
183
185
|
}
|
|
186
|
+
requirePage() {
|
|
187
|
+
if (!this._target.current) {
|
|
188
|
+
throw new PageClosedException_1.PageClosedException();
|
|
189
|
+
}
|
|
190
|
+
return this._target.current;
|
|
191
|
+
}
|
|
184
192
|
/* ------------------------------------------------------------------ */
|
|
185
193
|
/* Internal helpers */
|
|
186
194
|
/* ------------------------------------------------------------------ */
|
|
@@ -195,14 +203,4 @@ The active (i.e. in focus) tab is ${this._target.current.url()}`;
|
|
|
195
203
|
}
|
|
196
204
|
}
|
|
197
205
|
exports.WebTargetInspector = WebTargetInspector;
|
|
198
|
-
WebTargetInspector.PROMPT_INFO = {
|
|
199
|
-
systemPreamble: `You are a web browser automation agent to help people navigate webpages to
|
|
200
|
-
accomplish an OVERALL OBJECTIVE. These webpages are being rendered in a web
|
|
201
|
-
browser and are powered using the Microsoft Playwright framework.`,
|
|
202
|
-
screenshotSubject: 'a webpage',
|
|
203
|
-
currentViewDescription: 'web page (i.e. the current viewport)',
|
|
204
|
-
annotatedViewDescription: 'viewport of the web page',
|
|
205
|
-
interactionTarget: 'website',
|
|
206
|
-
targetNoun: 'webpage',
|
|
207
|
-
};
|
|
208
206
|
//# sourceMappingURL=WebTargetInspector.js.map
|