@isoldex/sentinel 0.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +368 -62
- package/dist/__tests__/action-engine.test.d.ts +2 -0
- package/dist/__tests__/action-engine.test.d.ts.map +1 -0
- package/dist/__tests__/action-engine.test.js +144 -0
- package/dist/__tests__/action-engine.test.js.map +1 -0
- package/dist/__tests__/state-parser.test.d.ts +2 -0
- package/dist/__tests__/state-parser.test.d.ts.map +1 -0
- package/dist/__tests__/state-parser.test.js +176 -0
- package/dist/__tests__/state-parser.test.js.map +1 -0
- package/dist/__tests__/verifier.test.d.ts +2 -0
- package/dist/__tests__/verifier.test.d.ts.map +1 -0
- package/dist/__tests__/verifier.test.js +68 -0
- package/dist/__tests__/verifier.test.js.map +1 -0
- package/dist/__tests__/workflow-recorder.test.d.ts +2 -0
- package/dist/__tests__/workflow-recorder.test.d.ts.map +1 -0
- package/dist/__tests__/workflow-recorder.test.js +71 -0
- package/dist/__tests__/workflow-recorder.test.js.map +1 -0
- package/dist/agent/agent-loop.d.ts +36 -0
- package/dist/agent/agent-loop.d.ts.map +1 -0
- package/dist/agent/agent-loop.js +127 -0
- package/dist/agent/agent-loop.js.map +1 -0
- package/dist/agent/memory.d.ts +24 -0
- package/dist/agent/memory.d.ts.map +1 -0
- package/dist/agent/memory.js +34 -0
- package/dist/agent/memory.js.map +1 -0
- package/dist/agent/planner.d.ts +18 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +68 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/api/act.d.ts +4 -2
- package/dist/api/act.d.ts.map +1 -1
- package/dist/api/act.js +174 -30
- package/dist/api/act.js.map +1 -1
- package/dist/api/extract.d.ts +2 -3
- package/dist/api/extract.d.ts.map +1 -1
- package/dist/api/extract.js +0 -1
- package/dist/api/extract.js.map +1 -1
- package/dist/api/observe.d.ts +2 -2
- package/dist/api/observe.d.ts.map +1 -1
- package/dist/api/observe.js +0 -1
- package/dist/api/observe.js.map +1 -1
- package/dist/core/driver.d.ts +21 -1
- package/dist/core/driver.d.ts.map +1 -1
- package/dist/core/driver.js +109 -19
- package/dist/core/driver.js.map +1 -1
- package/dist/core/state-parser.d.ts +1 -0
- package/dist/core/state-parser.d.ts.map +1 -1
- package/dist/core/state-parser.js +60 -0
- package/dist/core/state-parser.js.map +1 -1
- package/dist/core/vision-grounding.d.ts +36 -0
- package/dist/core/vision-grounding.d.ts.map +1 -0
- package/dist/core/vision-grounding.js +118 -0
- package/dist/core/vision-grounding.js.map +1 -0
- package/dist/index.d.ts +111 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +168 -4
- package/dist/index.js.map +1 -1
- package/dist/recorder/workflow-recorder.d.ts +30 -0
- package/dist/recorder/workflow-recorder.d.ts.map +1 -0
- package/dist/recorder/workflow-recorder.js +83 -0
- package/dist/recorder/workflow-recorder.js.map +1 -0
- package/dist/reliability/verifier.d.ts +2 -2
- package/dist/reliability/verifier.d.ts.map +1 -1
- package/dist/reliability/verifier.js +0 -1
- package/dist/reliability/verifier.js.map +1 -1
- package/dist/types/errors.d.ts +45 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/errors.js +69 -0
- package/dist/types/errors.js.map +1 -0
- package/dist/utils/gemini.d.ts +3 -6
- package/dist/utils/gemini.d.ts.map +1 -1
- package/dist/utils/gemini.js +9 -94
- package/dist/utils/gemini.js.map +1 -1
- package/dist/utils/llm-provider.d.ts +28 -0
- package/dist/utils/llm-provider.d.ts.map +1 -0
- package/dist/utils/llm-provider.js +2 -0
- package/dist/utils/llm-provider.js.map +1 -0
- package/dist/utils/providers/claude-provider.d.ts +17 -0
- package/dist/utils/providers/claude-provider.d.ts.map +1 -0
- package/dist/utils/providers/claude-provider.js +49 -0
- package/dist/utils/providers/claude-provider.js.map +1 -0
- package/dist/utils/providers/gemini-provider.d.ts +14 -0
- package/dist/utils/providers/gemini-provider.d.ts.map +1 -0
- package/dist/utils/providers/gemini-provider.js +95 -0
- package/dist/utils/providers/gemini-provider.js.map +1 -0
- package/dist/utils/providers/ollama-provider.d.ts +18 -0
- package/dist/utils/providers/ollama-provider.d.ts.map +1 -0
- package/dist/utils/providers/ollama-provider.js +62 -0
- package/dist/utils/providers/ollama-provider.js.map +1 -0
- package/dist/utils/providers/openai-provider.d.ts +18 -0
- package/dist/utils/providers/openai-provider.d.ts.map +1 -0
- package/dist/utils/providers/openai-provider.js +51 -0
- package/dist/utils/providers/openai-provider.js.map +1 -0
- package/dist/utils/token-tracker.d.ts +25 -0
- package/dist/utils/token-tracker.d.ts.map +1 -0
- package/dist/utils/token-tracker.js +45 -0
- package/dist/utils/token-tracker.js.map +1 -0
- package/dist/whatsapp-test.js +58 -17
- package/dist/whatsapp-test.js.map +1 -1
- package/package.json +10 -4
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Uses Gemini to plan the next action step based on the current page state and history.
|
|
3
|
+
*/
|
|
4
|
+
export class Planner {
|
|
5
|
+
gemini;
|
|
6
|
+
constructor(gemini) {
|
|
7
|
+
this.gemini = gemini;
|
|
8
|
+
}
|
|
9
|
+
async planNextStep(goal, state, memory) {
|
|
10
|
+
const prompt = `
|
|
11
|
+
You are an autonomous browser agent. Your goal is: "${goal}"
|
|
12
|
+
|
|
13
|
+
Current page:
|
|
14
|
+
- URL: ${state.url}
|
|
15
|
+
- Title: ${state.title}
|
|
16
|
+
- Interactive elements: ${JSON.stringify(state.elements.slice(0, 40).map(e => ({ id: e.id, role: e.role, name: e.name })), null, 2)}
|
|
17
|
+
|
|
18
|
+
Steps taken so far:
|
|
19
|
+
${memory.getSummary()}
|
|
20
|
+
|
|
21
|
+
Based on the current page state and history, decide the SINGLE next action to take.
|
|
22
|
+
If the goal is already fully achieved based on the history and current page, set isGoalComplete to true.
|
|
23
|
+
If you are stuck (same action repeated 3+ times without progress), try a different approach.
|
|
24
|
+
|
|
25
|
+
Respond with:
|
|
26
|
+
- instruction: a clear natural language instruction for the next action (e.g. "Click the search button", "Fill 'laptop' into the search field", "Scroll down to see more results")
|
|
27
|
+
- reasoning: why this is the right next step
|
|
28
|
+
- isGoalComplete: true only if the goal has been fully achieved
|
|
29
|
+
`;
|
|
30
|
+
const schema = {
|
|
31
|
+
type: 'object',
|
|
32
|
+
properties: {
|
|
33
|
+
instruction: { type: 'string' },
|
|
34
|
+
reasoning: { type: 'string' },
|
|
35
|
+
isGoalComplete: { type: 'boolean' },
|
|
36
|
+
},
|
|
37
|
+
required: ['instruction', 'reasoning', 'isGoalComplete'],
|
|
38
|
+
};
|
|
39
|
+
return await this.gemini.generateStructuredData(prompt, schema);
|
|
40
|
+
}
|
|
41
|
+
async reflect(goal, memory, finalState) {
|
|
42
|
+
const prompt = `
|
|
43
|
+
You are an autonomous browser agent evaluating whether a goal has been achieved.
|
|
44
|
+
|
|
45
|
+
Goal: "${goal}"
|
|
46
|
+
|
|
47
|
+
Current page:
|
|
48
|
+
- URL: ${finalState.url}
|
|
49
|
+
- Title: ${finalState.title}
|
|
50
|
+
|
|
51
|
+
Steps taken:
|
|
52
|
+
${memory.getSummary()}
|
|
53
|
+
|
|
54
|
+
Has the goal been fully and successfully achieved? Answer with a single boolean.
|
|
55
|
+
`;
|
|
56
|
+
const schema = {
|
|
57
|
+
type: 'object',
|
|
58
|
+
properties: {
|
|
59
|
+
goalAchieved: { type: 'boolean' },
|
|
60
|
+
reason: { type: 'string' },
|
|
61
|
+
},
|
|
62
|
+
required: ['goalAchieved', 'reason'],
|
|
63
|
+
};
|
|
64
|
+
const result = await this.gemini.generateStructuredData(prompt, schema);
|
|
65
|
+
return result.goalAchieved;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=planner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"planner.js","sourceRoot":"","sources":["../../src/agent/planner.ts"],"names":[],"mappings":"AAUA;;GAEG;AACH,MAAM,OAAO,OAAO;IACE;IAApB,YAAoB,MAAmB;QAAnB,WAAM,GAAN,MAAM,CAAa;IAAG,CAAC;IAE3C,KAAK,CAAC,YAAY,CAChB,IAAY,EACZ,KAAsB,EACtB,MAAmB;QAEnB,MAAM,MAAM,GAAG;sDACmC,IAAI;;;SAGjD,KAAK,CAAC,GAAG;WACP,KAAK,CAAC,KAAK;0BACI,IAAI,CAAC,SAAS,CAClC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAChF,IAAI,EACJ,CAAC,CACF;;;EAGH,MAAM,CAAC,UAAU,EAAE;;;;;;;;;;KAUhB,CAAC;QAEF,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7B,cAAc,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;aACpC;YACD,QAAQ,EAAE,CAAC,aAAa,EAAE,WAAW,EAAE,gBAAgB,CAAC;SACzD,CAAC;QAEF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAc,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAY,EAAE,MAAmB,EAAE,UAA2B;QAC1E,MAAM,MAAM,GAAG;;;SAGV,IAAI;;;SAGJ,UAAU,CAAC,GAAG;WACZ,UAAU,CAAC,KAAK;;;EAGzB,MAAM,CAAC,UAAU,EAAE;;;KAGhB,CAAC;QAEF,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B;YACD,QAAQ,EAAE,CAAC,cAAc,EAAE,QAAQ,CAAC;SACrC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAGpD,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnB,OAAO,MAAM,CAAC,YAAY,CAAC;IAC7B,CAAC;CACF"}
|
package/dist/api/act.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Page } from 'playwright';
|
|
2
2
|
import { StateParser } from '../core/state-parser.js';
|
|
3
|
-
import {
|
|
3
|
+
import type { LLMProvider } from '../utils/llm-provider.js';
|
|
4
|
+
import type { VisionGrounding } from '../core/vision-grounding.js';
|
|
4
5
|
export interface ActOptions {
|
|
5
6
|
variables?: Record<string, string>;
|
|
6
7
|
retries?: number;
|
|
@@ -14,7 +15,8 @@ export declare class ActionEngine {
|
|
|
14
15
|
private page;
|
|
15
16
|
private stateParser;
|
|
16
17
|
private gemini;
|
|
17
|
-
|
|
18
|
+
private visionGrounding?;
|
|
19
|
+
constructor(page: Page, stateParser: StateParser, gemini: LLMProvider, visionGrounding?: VisionGrounding | undefined);
|
|
18
20
|
act(instruction: string, options?: ActOptions): Promise<ActionResult>;
|
|
19
21
|
private performAction;
|
|
20
22
|
private performSemanticFallback;
|
package/dist/api/act.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"act.d.ts","sourceRoot":"","sources":["../../src/api/act.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"act.d.ts","sourceRoot":"","sources":["../../src/api/act.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA+BD,qBAAa,YAAY;IAErB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe,CAAC;gBAHhB,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,WAAW,EACnB,eAAe,CAAC,EAAE,eAAe,YAAA;IAGrC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC;YAgI7D,aAAa;YAqGb,uBAAuB;CAkEtC"}
|
package/dist/api/act.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { StateParser } from '../core/state-parser.js';
|
|
2
|
-
import { GeminiService } from '../utils/gemini.js';
|
|
3
2
|
/**
|
|
4
3
|
* Replaces %variable% placeholders in an instruction string.
|
|
5
4
|
*/
|
|
@@ -20,10 +19,12 @@ export class ActionEngine {
|
|
|
20
19
|
page;
|
|
21
20
|
stateParser;
|
|
22
21
|
gemini;
|
|
23
|
-
|
|
22
|
+
visionGrounding;
|
|
23
|
+
constructor(page, stateParser, gemini, visionGrounding) {
|
|
24
24
|
this.page = page;
|
|
25
25
|
this.stateParser = stateParser;
|
|
26
26
|
this.gemini = gemini;
|
|
27
|
+
this.visionGrounding = visionGrounding;
|
|
27
28
|
}
|
|
28
29
|
async act(instruction, options) {
|
|
29
30
|
const resolvedInstruction = interpolateVariables(instruction, options?.variables);
|
|
@@ -32,30 +33,57 @@ export class ActionEngine {
|
|
|
32
33
|
Current Page URL: ${state.url}
|
|
33
34
|
Page Title: ${state.title}
|
|
34
35
|
Instruction: "${resolvedInstruction}"
|
|
35
|
-
|
|
36
|
+
|
|
36
37
|
Elements on page:
|
|
37
38
|
${JSON.stringify(state.elements.map(e => ({ id: e.id, role: e.role, name: e.name })), null, 2)}
|
|
38
|
-
|
|
39
|
-
Select the ID of the element to interact with and the action to perform
|
|
40
|
-
|
|
39
|
+
|
|
40
|
+
Select the ID of the element to interact with and the action to perform.
|
|
41
|
+
Available actions:
|
|
42
|
+
- "click": single click on an element
|
|
43
|
+
- "double-click": double click on an element
|
|
44
|
+
- "right-click": right-click (context menu) on an element
|
|
45
|
+
- "fill": type text into an input field (requires "value")
|
|
46
|
+
- "hover": move mouse over an element
|
|
47
|
+
- "press": press a keyboard key or shortcut (requires "value", e.g. "Enter", "Escape", "Tab", "Control+a")
|
|
48
|
+
- "select": select an option from a <select> dropdown (requires "value" = option text or value)
|
|
49
|
+
- "scroll-down": scroll the page down (elementId optional, use 0 if no specific element)
|
|
50
|
+
- "scroll-up": scroll the page up (elementId optional, use 0 if no specific element)
|
|
51
|
+
- "scroll-to": scroll to bring a specific element into view (requires elementId)
|
|
52
|
+
|
|
53
|
+
If the action is "fill", "press", or "select", provide the "value" field.
|
|
54
|
+
For scroll actions without a target element, set elementId to 0.
|
|
41
55
|
Provide clear reasoning for your choice.
|
|
42
56
|
`;
|
|
43
57
|
const schema = {
|
|
44
58
|
type: 'object',
|
|
45
59
|
properties: {
|
|
46
60
|
elementId: { type: 'number' },
|
|
47
|
-
action: {
|
|
61
|
+
action: {
|
|
62
|
+
type: 'string',
|
|
63
|
+
enum: [
|
|
64
|
+
'click', 'double-click', 'right-click',
|
|
65
|
+
'fill', 'hover', 'press', 'select',
|
|
66
|
+
'scroll-down', 'scroll-up', 'scroll-to',
|
|
67
|
+
],
|
|
68
|
+
},
|
|
48
69
|
value: { type: 'string' },
|
|
49
70
|
reasoning: { type: 'string' },
|
|
50
71
|
},
|
|
51
72
|
required: ['elementId', 'action', 'reasoning'],
|
|
52
73
|
};
|
|
53
74
|
const decision = await this.gemini.generateStructuredData(prompt, schema);
|
|
54
|
-
|
|
55
|
-
|
|
75
|
+
// Scroll actions without a target element are valid with elementId = 0
|
|
76
|
+
const isScrollWithoutTarget = (decision.action === 'scroll-down' || decision.action === 'scroll-up') &&
|
|
77
|
+
decision.elementId === 0;
|
|
78
|
+
const target = isScrollWithoutTarget
|
|
79
|
+
? null
|
|
80
|
+
: (state.elements.find(e => e.id === decision.elementId) ?? null);
|
|
81
|
+
if (!isScrollWithoutTarget && !target) {
|
|
56
82
|
return { success: false, message: `Could not find element with ID ${decision.elementId}` };
|
|
57
83
|
}
|
|
58
|
-
const actionLabel =
|
|
84
|
+
const actionLabel = target
|
|
85
|
+
? `${decision.action} on "${target.name}" (${target.role})`
|
|
86
|
+
: `${decision.action} (page)`;
|
|
59
87
|
console.log(`[Act] ${actionLabel} → ${decision.reasoning}`);
|
|
60
88
|
// Invalidate cache after action – state will change
|
|
61
89
|
this.stateParser.invalidateCache();
|
|
@@ -64,18 +92,40 @@ export class ActionEngine {
|
|
|
64
92
|
await waitForPageSettle(this.page);
|
|
65
93
|
return {
|
|
66
94
|
success: true,
|
|
67
|
-
message: `Successfully performed ${decision.action} on "${target.name}"`,
|
|
95
|
+
message: `Successfully performed ${decision.action}${target ? ` on "${target.name}"` : ''}`,
|
|
68
96
|
action: actionLabel,
|
|
69
97
|
};
|
|
70
98
|
}
|
|
71
99
|
catch (primaryError) {
|
|
72
|
-
|
|
100
|
+
// Vision-Grounding als zweite Stufe (nur wenn aktiviert)
|
|
101
|
+
if (this.visionGrounding && target) {
|
|
102
|
+
try {
|
|
103
|
+
const viewport = this.page.viewportSize() ?? { width: 1280, height: 720 };
|
|
104
|
+
const screenshot = await this.visionGrounding.takeScreenshot(this.page);
|
|
105
|
+
const bbox = await this.visionGrounding.findElement(`${decision.action} on "${target.name}"`, screenshot, viewport.width, viewport.height);
|
|
106
|
+
if (bbox) {
|
|
107
|
+
const cx = bbox.x + bbox.width / 2;
|
|
108
|
+
const cy = bbox.y + bbox.height / 2;
|
|
109
|
+
await this.page.mouse.click(cx, cy);
|
|
110
|
+
await waitForPageSettle(this.page);
|
|
111
|
+
return {
|
|
112
|
+
success: true,
|
|
113
|
+
message: `Successfully performed ${decision.action} on "${target.name}" (via Vision Grounding)`,
|
|
114
|
+
action: actionLabel,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
catch (visionError) {
|
|
119
|
+
console.warn(`[Act] Vision fallback failed: ${visionError.message}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
console.warn(`[Act] Primary action failed, trying semantic fallback... (${primaryError.message})`);
|
|
73
123
|
try {
|
|
74
124
|
await this.performSemanticFallback(decision.action, target, decision.value);
|
|
75
125
|
await waitForPageSettle(this.page);
|
|
76
126
|
return {
|
|
77
127
|
success: true,
|
|
78
|
-
message: `Successfully performed ${decision.action} on "${target.name}" (via fallback)`,
|
|
128
|
+
message: `Successfully performed ${decision.action}${target ? ` on "${target.name}"` : ''} (via fallback)`,
|
|
79
129
|
action: actionLabel,
|
|
80
130
|
};
|
|
81
131
|
}
|
|
@@ -85,31 +135,125 @@ export class ActionEngine {
|
|
|
85
135
|
}
|
|
86
136
|
}
|
|
87
137
|
async performAction(action, target, value) {
|
|
138
|
+
// Scroll actions that don't need a target element
|
|
139
|
+
if (action === 'scroll-down' && !target) {
|
|
140
|
+
await this.page.evaluate(() => window.scrollBy(0, 600));
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (action === 'scroll-up' && !target) {
|
|
144
|
+
await this.page.evaluate(() => window.scrollBy(0, -600));
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
if (!target)
|
|
148
|
+
throw new Error('No target element provided for action: ' + action);
|
|
88
149
|
const { x, y, width, height } = target.boundingClientRect;
|
|
89
150
|
const cx = x + width / 2;
|
|
90
151
|
const cy = y + height / 2;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
152
|
+
switch (action) {
|
|
153
|
+
case 'click':
|
|
154
|
+
await this.page.mouse.click(cx, cy);
|
|
155
|
+
break;
|
|
156
|
+
case 'double-click':
|
|
157
|
+
await this.page.mouse.dblclick(cx, cy);
|
|
158
|
+
break;
|
|
159
|
+
case 'right-click':
|
|
160
|
+
await this.page.mouse.click(cx, cy, { button: 'right' });
|
|
161
|
+
break;
|
|
162
|
+
case 'fill':
|
|
163
|
+
await this.page.mouse.click(cx, cy, { clickCount: 3 });
|
|
164
|
+
await this.page.keyboard.type(value || '');
|
|
165
|
+
break;
|
|
166
|
+
case 'hover':
|
|
167
|
+
await this.page.mouse.move(cx, cy);
|
|
168
|
+
break;
|
|
169
|
+
case 'press':
|
|
170
|
+
// Focus the element first, then press the key
|
|
171
|
+
await this.page.mouse.click(cx, cy);
|
|
172
|
+
await this.page.keyboard.press(value || 'Enter');
|
|
173
|
+
break;
|
|
174
|
+
case 'select':
|
|
175
|
+
// Use Playwright's selectOption via coordinate-based locator
|
|
176
|
+
await this.page.mouse.click(cx, cy);
|
|
177
|
+
await this.page.evaluate(({ x, y, val }) => {
|
|
178
|
+
const el = document.elementFromPoint(x, y);
|
|
179
|
+
if (el && el.tagName === 'SELECT') {
|
|
180
|
+
const opt = Array.from(el.options).find(o => o.text === val || o.value === val);
|
|
181
|
+
if (opt) {
|
|
182
|
+
el.value = opt.value;
|
|
183
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}, { x: cx, y: cy, val: value || '' });
|
|
187
|
+
break;
|
|
188
|
+
case 'scroll-down':
|
|
189
|
+
await this.page.evaluate(({ x, y }) => {
|
|
190
|
+
const el = document.elementFromPoint(x, y);
|
|
191
|
+
if (el)
|
|
192
|
+
el.scrollBy(0, 300);
|
|
193
|
+
}, { x: cx, y: cy });
|
|
194
|
+
break;
|
|
195
|
+
case 'scroll-up':
|
|
196
|
+
await this.page.evaluate(({ x, y }) => {
|
|
197
|
+
const el = document.elementFromPoint(x, y);
|
|
198
|
+
if (el)
|
|
199
|
+
el.scrollBy(0, -300);
|
|
200
|
+
}, { x: cx, y: cy });
|
|
201
|
+
break;
|
|
202
|
+
case 'scroll-to':
|
|
203
|
+
await this.page.evaluate(({ x, y }) => {
|
|
204
|
+
const el = document.elementFromPoint(x, y);
|
|
205
|
+
if (el)
|
|
206
|
+
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
|
207
|
+
}, { x: cx, y: cy });
|
|
208
|
+
break;
|
|
100
209
|
}
|
|
101
210
|
}
|
|
102
211
|
async performSemanticFallback(action, target, value) {
|
|
103
|
-
|
|
104
|
-
if (action === '
|
|
105
|
-
await
|
|
212
|
+
// Page-level scroll fallback
|
|
213
|
+
if (action === 'scroll-down' && !target) {
|
|
214
|
+
await this.page.keyboard.press('PageDown');
|
|
215
|
+
return;
|
|
106
216
|
}
|
|
107
|
-
|
|
108
|
-
await
|
|
109
|
-
|
|
217
|
+
if (action === 'scroll-up' && !target) {
|
|
218
|
+
await this.page.keyboard.press('PageUp');
|
|
219
|
+
return;
|
|
110
220
|
}
|
|
111
|
-
|
|
112
|
-
|
|
221
|
+
if (!target)
|
|
222
|
+
throw new Error('No target element for semantic fallback: ' + action);
|
|
223
|
+
const locator = this.page.locator(`[role="${target.role}"]`, target.name ? { hasText: target.name } : undefined).first();
|
|
224
|
+
switch (action) {
|
|
225
|
+
case 'click':
|
|
226
|
+
await locator.click({ timeout: 5000 });
|
|
227
|
+
break;
|
|
228
|
+
case 'double-click':
|
|
229
|
+
await locator.dblclick({ timeout: 5000 });
|
|
230
|
+
break;
|
|
231
|
+
case 'right-click':
|
|
232
|
+
await locator.click({ button: 'right', timeout: 5000 });
|
|
233
|
+
break;
|
|
234
|
+
case 'fill':
|
|
235
|
+
await locator.click({ clickCount: 3, timeout: 5000 });
|
|
236
|
+
await locator.pressSequentially(value || '', { delay: 30 });
|
|
237
|
+
break;
|
|
238
|
+
case 'hover':
|
|
239
|
+
await locator.hover({ timeout: 5000 });
|
|
240
|
+
break;
|
|
241
|
+
case 'press':
|
|
242
|
+
await locator.focus({ timeout: 5000 });
|
|
243
|
+
await locator.press(value || 'Enter');
|
|
244
|
+
break;
|
|
245
|
+
case 'select':
|
|
246
|
+
await locator.selectOption(value || '', { timeout: 5000 });
|
|
247
|
+
break;
|
|
248
|
+
case 'scroll-to':
|
|
249
|
+
await locator.scrollIntoViewIfNeeded({ timeout: 5000 });
|
|
250
|
+
break;
|
|
251
|
+
case 'scroll-down':
|
|
252
|
+
await locator.evaluate(el => el.scrollBy(0, 300));
|
|
253
|
+
break;
|
|
254
|
+
case 'scroll-up':
|
|
255
|
+
await locator.evaluate(el => el.scrollBy(0, -300));
|
|
256
|
+
break;
|
|
113
257
|
}
|
|
114
258
|
}
|
|
115
259
|
}
|
package/dist/api/act.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"act.js","sourceRoot":"","sources":["../../src/api/act.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"act.js","sourceRoot":"","sources":["../../src/api/act.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AA4BtD;;GAEG;AACH,SAAS,oBAAoB,CAAC,WAAmB,EAAE,SAAkC;IACnF,IAAI,CAAC,SAAS;QAAE,OAAO,WAAW,CAAC;IACnC,OAAO,WAAW,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC;AACnF,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,iBAAiB,CAAC,IAAU,EAAE,OAAO,GAAG,IAAI;IACzD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACjE,0DAA0D;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,YAAY;IAEb;IACA;IACA;IACA;IAJV,YACU,IAAU,EACV,WAAwB,EACxB,MAAmB,EACnB,eAAiC;QAHjC,SAAI,GAAJ,IAAI,CAAM;QACV,gBAAW,GAAX,WAAW,CAAa;QACxB,WAAM,GAAN,MAAM,CAAa;QACnB,oBAAe,GAAf,eAAe,CAAkB;IACxC,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,WAAmB,EAAE,OAAoB;QACjD,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE7C,MAAM,MAAM,GAAG;0BACO,KAAK,CAAC,GAAG;oBACf,KAAK,CAAC,KAAK;sBACT,mBAAmB;;;QAGjC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;;;;;;;;;;;;;;;;;;KAkB/F,CAAC;QAEF,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7B,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE;wBACJ,OAAO,EAAE,cAAc,EAAE,aAAa;wBACtC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;wBAClC,aAAa,EAAE,WAAW,EAAE,WAAW;qBACxC;iBACF;gBACD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC;SAC/C,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAKtD,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnB,uEAAuE;QACvE,MAAM,qBAAqB,GACzB,CAAC,QAAQ,CAAC,MAAM,KAAK,aAAa,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,CAAC;YACtE,QAAQ,CAAC,SAAS,KAAK,CAAC,CAAC;QAE3B,MAAM,MAAM,GAAG,qBAAqB;YAClC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC;QAEpE,IAAI,CAAC,qBAAqB,IAAI,CAAC,MAAM,EAAE,CAAC;YACtC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kCAAkC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAC7F,CAAC;QAED,MAAM,WAAW,GAAG,MAAM;YACxB,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,IAAI,GAAG;YAC3D,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,SAAS,CAAC;QAEhC,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QAE5D,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,0BAA0B,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC3F,MAAM,EAAE,WAAW;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,YAAiB,EAAE,CAAC;YAC3B,yDAAyD;YACzD,IAAI,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,CAAC;gBACnC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;oBAC1E,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACxE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CACjD,GAAG,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,IAAI,GAAG,EACxC,UAAU,EACV,QAAQ,CAAC,KAAK,EACd,QAAQ,CAAC,MAAM,CAChB,CAAC;oBACF,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;wBACnC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;wBACpC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;wBACpC,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBACnC,OAAO;4BACL,OAAO,EAAE,IAAI;4BACb,OAAO,EAAE,0BAA0B,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,IAAI,0BAA0B;4BAC/F,MAAM,EAAE,WAAW;yBACpB,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,OAAO,WAAgB,EAAE,CAAC;oBAC1B,OAAO,CAAC,IAAI,CAAC,iCAAiC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,6DAA6D,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;YACnG,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,0BAA0B,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,iBAAiB;oBAC1G,MAAM,EAAE,WAAW;iBACpB,CAAC;YACJ,CAAC;YAAC,OAAO,aAAkB,EAAE,CAAC;gBAC5B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,aAAa,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YACrG,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAkB,EAClB,MAAwB,EACxB,KAAc;QAEd,kDAAkD;QAClD,IAAI,MAAM,KAAK,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QACD,IAAI,MAAM,KAAK,WAAW,IAAI,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,MAAM,CAAC,CAAC;QAEjF,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,kBAAkB,CAAC;QAC1D,MAAM,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;QACzB,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;QAE1B,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,OAAO;gBACV,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACpC,MAAM;YAER,KAAK,cAAc;gBACjB,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACvC,MAAM;YAER,KAAK,aAAa;gBAChB,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzD,MAAM;YAER,KAAK,MAAM;gBACT,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;gBAC3C,MAAM;YAER,KAAK,OAAO;gBACV,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACnC,MAAM;YAER,KAAK,OAAO;gBACV,8CAA8C;gBAC9C,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACpC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;gBACjD,MAAM;YAER,KAAK,QAAQ;gBACX,6DAA6D;gBAC7D,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACpC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CACtB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAyC,EAAE,EAAE;oBACvD,MAAM,EAAE,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAA6B,CAAC;oBACvE,IAAI,EAAE,IAAI,EAAE,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;wBAClC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CACrC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,KAAK,GAAG,CACvC,CAAC;wBACF,IAAI,GAAG,EAAE,CAAC;4BACR,EAAE,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BACrB,EAAE,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;wBAC3D,CAAC;oBACH,CAAC;gBACH,CAAC,EACD,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,IAAI,EAAE,EAAE,CACnC,CAAC;gBACF,MAAM;YAER,KAAK,aAAa;gBAChB,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CACtB,CAAC,EAAE,CAAC,EAAE,CAAC,EAA4B,EAAE,EAAE;oBACrC,MAAM,EAAE,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC3C,IAAI,EAAE;wBAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC9B,CAAC,EACD,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACjB,CAAC;gBACF,MAAM;YAER,KAAK,WAAW;gBACd,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CACtB,CAAC,EAAE,CAAC,EAAE,CAAC,EAA4B,EAAE,EAAE;oBACrC,MAAM,EAAE,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC3C,IAAI,EAAE;wBAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC,EACD,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACjB,CAAC;gBACF,MAAM;YAER,KAAK,WAAW;gBACd,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CACtB,CAAC,EAAE,CAAC,EAAE,CAAC,EAA4B,EAAE,EAAE;oBACrC,MAAM,EAAE,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC3C,IAAI,EAAE;wBAAE,EAAE,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACrE,CAAC,EACD,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACjB,CAAC;gBACF,MAAM;QACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,MAAkB,EAClB,MAAwB,EACxB,KAAc;QAEd,6BAA6B;QAC7B,IAAI,MAAM,KAAK,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC3C,OAAO;QACT,CAAC;QACD,IAAI,MAAM,KAAK,WAAW,IAAI,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACzC,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,MAAM,CAAC,CAAC;QAEnF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAC/B,UAAU,MAAM,CAAC,IAAI,IAAI,EACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CACnD,CAAC,KAAK,EAAE,CAAC;QAEV,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,OAAO;gBACV,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvC,MAAM;YAER,KAAK,cAAc;gBACjB,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1C,MAAM;YAER,KAAK,aAAa;gBAChB,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM;YAER,KAAK,MAAM;gBACT,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtD,MAAM,OAAO,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC5D,MAAM;YAER,KAAK,OAAO;gBACV,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvC,MAAM;YAER,KAAK,OAAO;gBACV,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvC,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;gBACtC,MAAM;YAER,KAAK,QAAQ;gBACX,MAAM,OAAO,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3D,MAAM;YAER,KAAK,WAAW;gBACd,MAAM,OAAO,CAAC,sBAAsB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM;YAER,KAAK,aAAa;gBAChB,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBAClD,MAAM;YAER,KAAK,WAAW;gBACd,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBACnD,MAAM;QACV,CAAC;IACH,CAAC;CACF"}
|
package/dist/api/extract.d.ts
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import type { Page } from 'playwright';
|
|
2
2
|
import { StateParser } from '../core/state-parser.js';
|
|
3
|
-
import {
|
|
4
|
-
import type { SchemaInput } from '../utils/gemini.js';
|
|
3
|
+
import type { LLMProvider, SchemaInput } from '../utils/llm-provider.js';
|
|
5
4
|
export declare class ExtractionEngine {
|
|
6
5
|
private page;
|
|
7
6
|
private stateParser;
|
|
8
7
|
private gemini;
|
|
9
|
-
constructor(page: Page, stateParser: StateParser, gemini:
|
|
8
|
+
constructor(page: Page, stateParser: StateParser, gemini: LLMProvider);
|
|
10
9
|
extract<T>(instruction: string, schema: SchemaInput<T>): Promise<T>;
|
|
11
10
|
private getPageText;
|
|
12
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/api/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,
|
|
1
|
+
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/api/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAKzE,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;gBAFN,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,WAAW;IAGvB,OAAO,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YAyB3D,WAAW;CAc1B"}
|
package/dist/api/extract.js
CHANGED
package/dist/api/extract.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/api/extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/api/extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IAHV,YACU,IAAU,EACV,WAAwB,EACxB,MAAmB;QAFnB,SAAI,GAAJ,IAAI,CAAM;QACV,gBAAW,GAAX,WAAW,CAAa;QACxB,WAAM,GAAN,MAAM,CAAa;IAC1B,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAI,WAAmB,EAAE,MAAsB;QAC1D,kDAAkD;QAClD,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;YACxB,IAAI,CAAC,WAAW,EAAE;SACnB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG;+EAC4D,WAAW;;kBAExE,QAAQ,CAAC,GAAG;oBACV,QAAQ,CAAC,KAAK;;;QAG1B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;;;QAG/F,QAAQ;;;KAGX,CAAC;QAEF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAI,MAAM,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAgB,CAAC;gBAC3D,KAAK,CAAC,gBAAgB,CAAC,oDAAoD,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;gBACxG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAClF,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,MAAM,GAAG,mBAAmB;gBACtC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,GAAG,iBAAiB;gBACxD,CAAC,CAAC,IAAI,CAAC;QACX,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,+BAA+B,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}
|
package/dist/api/observe.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Page } from 'playwright';
|
|
2
2
|
import { StateParser } from '../core/state-parser.js';
|
|
3
|
-
import {
|
|
3
|
+
import type { LLMProvider } from '../utils/llm-provider.js';
|
|
4
4
|
export interface ObserveResult {
|
|
5
5
|
description: string;
|
|
6
6
|
selector?: string;
|
|
@@ -11,7 +11,7 @@ export declare class ObservationEngine {
|
|
|
11
11
|
private page;
|
|
12
12
|
private stateParser;
|
|
13
13
|
private gemini;
|
|
14
|
-
constructor(page: Page, stateParser: StateParser, gemini:
|
|
14
|
+
constructor(page: Page, stateParser: StateParser, gemini: LLMProvider);
|
|
15
15
|
observe(instruction?: string): Promise<ObserveResult[]>;
|
|
16
16
|
}
|
|
17
17
|
//# sourceMappingURL=observe.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/api/observe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/api/observe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;IACnE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;gBAFN,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,WAAW;IAGvB,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;CA+C9D"}
|
package/dist/api/observe.js
CHANGED
package/dist/api/observe.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"observe.js","sourceRoot":"","sources":["../../src/api/observe.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"observe.js","sourceRoot":"","sources":["../../src/api/observe.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAUtD,MAAM,OAAO,iBAAiB;IAElB;IACA;IACA;IAHV,YACU,IAAU,EACV,WAAwB,EACxB,MAAmB;QAFnB,SAAI,GAAJ,IAAI,CAAM;QACV,gBAAW,GAAX,WAAW,CAAa;QACxB,WAAM,GAAN,MAAM,CAAa;IAC1B,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,WAAoB;QAChC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,WAAW;YAC3B,CAAC,CAAC,2BAA2B,WAAW,GAAG;YAC3C,CAAC,CAAC,8DAA8D,CAAC;QAEnE,MAAM,MAAM,GAAG;;QAEX,SAAS;;aAEJ,SAAS,CAAC,GAAG;eACX,SAAS,CAAC,KAAK;;;QAGtB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;;;;;;;KAOnG,CAAC;QAEF,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BAC/B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BAC5B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;4BACxF,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;yBACxD;wBACD,QAAQ,EAAE,CAAC,aAAa,EAAE,QAAQ,CAAC;qBACpC;iBACF;aACF;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;SACtB,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAA+B,MAAM,EAAE,MAAM,CAAC,CAAC;QACtG,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;CACF"}
|
package/dist/core/driver.d.ts
CHANGED
|
@@ -1,23 +1,43 @@
|
|
|
1
1
|
import type { BrowserContext, Page, CDPSession } from 'playwright';
|
|
2
|
+
export type BrowserType = 'chromium' | 'firefox' | 'webkit';
|
|
3
|
+
export interface ProxyOptions {
|
|
4
|
+
server: string;
|
|
5
|
+
username?: string;
|
|
6
|
+
password?: string;
|
|
7
|
+
}
|
|
2
8
|
export interface DriverOptions {
|
|
3
9
|
headless?: boolean;
|
|
4
10
|
viewport?: {
|
|
5
11
|
width: number;
|
|
6
12
|
height: number;
|
|
7
13
|
};
|
|
14
|
+
browser?: BrowserType;
|
|
15
|
+
sessionPath?: string;
|
|
16
|
+
proxy?: ProxyOptions;
|
|
17
|
+
stealth?: boolean;
|
|
18
|
+
humanLike?: boolean;
|
|
8
19
|
}
|
|
9
20
|
export declare class SentinelDriver {
|
|
10
21
|
private options;
|
|
11
22
|
private browser;
|
|
12
23
|
private context;
|
|
13
|
-
private
|
|
24
|
+
private pages;
|
|
25
|
+
private activePageIndex;
|
|
14
26
|
private cdpSession;
|
|
15
27
|
constructor(options?: DriverOptions);
|
|
16
28
|
initialize(): Promise<void>;
|
|
29
|
+
newTab(url?: string): Promise<number>;
|
|
30
|
+
switchTab(index: number): Promise<void>;
|
|
31
|
+
closeTab(index: number): Promise<void>;
|
|
32
|
+
get tabCount(): number;
|
|
33
|
+
saveSession(filePath: string): Promise<void>;
|
|
34
|
+
hasLoginForm(): Promise<boolean>;
|
|
17
35
|
getPage(): Page;
|
|
18
36
|
getContext(): BrowserContext;
|
|
19
37
|
getCDPSession(): CDPSession;
|
|
20
38
|
close(): Promise<void>;
|
|
21
39
|
goto(url: string): Promise<void>;
|
|
40
|
+
private gotoPage;
|
|
41
|
+
private getRandomUserAgent;
|
|
22
42
|
}
|
|
23
43
|
//# sourceMappingURL=driver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/core/driver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAW,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/core/driver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAW,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAI5E,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5D,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,qBAAa,cAAc;IAOb,OAAO,CAAC,OAAO;IAN3B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,UAAU,CAA2B;gBAEzB,OAAO,GAAE,aAAmC;IAE1D,UAAU;IA4CV,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAYrC,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAavC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5C,IAAI,QAAQ,IAAI,MAAM,CAErB;IAIK,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS5C,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;IAetC,OAAO,IAAI,IAAI;IAKf,UAAU,IAAI,cAAc;IAK5B,aAAa,IAAI,UAAU;IAKrB,KAAK;IAML,IAAI,CAAC,GAAG,EAAE,MAAM;YAMR,QAAQ;IAWtB,OAAO,CAAC,kBAAkB;CAS3B"}
|