@isoldex/sentinel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +122 -0
- package/dist/api/act.d.ts +22 -0
- package/dist/api/act.d.ts.map +1 -0
- package/dist/api/act.js +116 -0
- package/dist/api/act.js.map +1 -0
- package/dist/api/extract.d.ts +13 -0
- package/dist/api/extract.d.ts.map +1 -0
- package/dist/api/extract.js +52 -0
- package/dist/api/extract.js.map +1 -0
- package/dist/api/observe.d.ts +17 -0
- package/dist/api/observe.d.ts.map +1 -0
- package/dist/api/observe.js +56 -0
- package/dist/api/observe.js.map +1 -0
- package/dist/complex-test.d.ts +2 -0
- package/dist/complex-test.d.ts.map +1 -0
- package/dist/complex-test.js +58 -0
- package/dist/complex-test.js.map +1 -0
- package/dist/core/driver.d.ts +23 -0
- package/dist/core/driver.d.ts.map +1 -0
- package/dist/core/driver.js +61 -0
- package/dist/core/driver.js.map +1 -0
- package/dist/core/state-parser.d.ts +38 -0
- package/dist/core/state-parser.d.ts.map +1 -0
- package/dist/core/state-parser.js +103 -0
- package/dist/core/state-parser.js.map +1 -0
- package/dist/debug-aom.d.ts +2 -0
- package/dist/debug-aom.d.ts.map +1 -0
- package/dist/debug-aom.js +22 -0
- package/dist/debug-aom.js.map +1 -0
- package/dist/demo.d.ts +2 -0
- package/dist/demo.d.ts.map +1 -0
- package/dist/demo.js +41 -0
- package/dist/demo.js.map +1 -0
- package/dist/index.d.ts +85 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +143 -0
- package/dist/index.js.map +1 -0
- package/dist/list-all.d.ts +2 -0
- package/dist/list-all.d.ts.map +1 -0
- package/dist/list-all.js +25 -0
- package/dist/list-all.js.map +1 -0
- package/dist/list-models.d.ts +2 -0
- package/dist/list-models.d.ts.map +1 -0
- package/dist/list-models.js +11 -0
- package/dist/list-models.js.map +1 -0
- package/dist/reliability/verifier.d.ts +18 -0
- package/dist/reliability/verifier.d.ts.map +1 -0
- package/dist/reliability/verifier.js +64 -0
- package/dist/reliability/verifier.js.map +1 -0
- package/dist/test-versions.d.ts +2 -0
- package/dist/test-versions.d.ts.map +1 -0
- package/dist/test-versions.js +33 -0
- package/dist/test-versions.js.map +1 -0
- package/dist/utils/gemini.d.ts +12 -0
- package/dist/utils/gemini.d.ts.map +1 -0
- package/dist/utils/gemini.js +103 -0
- package/dist/utils/gemini.js.map +1 -0
- package/dist/whatsapp-test.d.ts +2 -0
- package/dist/whatsapp-test.d.ts.map +1 -0
- package/dist/whatsapp-test.js +56 -0
- package/dist/whatsapp-test.js.map +1 -0
- package/package.json +53 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
ISC License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Sentinel Contributors
|
|
4
|
+
|
|
5
|
+
Permission to use, copy, modify, and/or distribute this software for any
|
|
6
|
+
purpose with or without fee is hereby granted, provided that the above
|
|
7
|
+
copyright notice and this permission notice appear in all copies.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
10
|
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
11
|
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
12
|
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
13
|
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
14
|
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
15
|
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# Sentinel 🛡️
|
|
2
|
+
|
|
3
|
+
Sentinel is a high-performance, AI-driven browser automation framework built on top of **Playwright** and powered by **Google Gemini**. It allows you to automate complex web tasks using natural language, extract structured data with ease, and perform robust interactions with self-healing capabilities.
|
|
4
|
+
|
|
5
|
+
Think of it as a **fast, lightweight, and cost-effective alternative to Stagehand**, specifically optimized for the Gemini ecosystem.
|
|
6
|
+
|
|
7
|
+
## ✨ Features
|
|
8
|
+
|
|
9
|
+
- **🗣️ Natural Language Interactions**: Perform actions like `act('Click the "Add to Cart" button')` without writing fragile CSS selectors.
|
|
10
|
+
- **📊 Structured Data Extraction**: Use Zod or JSON Schema to extract precisely formatted data from any page.
|
|
11
|
+
- **⚡ High Performance**: Optimized with parallel CDP (Chrome DevTools Protocol) requests and smart state caching.
|
|
12
|
+
- **🛡️ Robust & Reliable**: Includes a semantic verification loop that confirms every action and automatically retries with fallbacks on failure.
|
|
13
|
+
- **🔍 Deep Observation**: Understand page structure through the Accessibility Object Model (AOM) and raw text content.
|
|
14
|
+
- **🔧 Playwright Powered**: Direct access to the underlying Playwright `Page` and `BrowserContext`.
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## 🚀 Quickstart
|
|
19
|
+
|
|
20
|
+
### 1. Installation
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
npm install @isoldex/sentinel playwright
|
|
24
|
+
```
|
|
25
|
+
> [!NOTE]
|
|
26
|
+
> Playwright is a peer dependency. Make sure to install it alongside Sentinel.
|
|
27
|
+
|
|
28
|
+
### 2. Configuration
|
|
29
|
+
|
|
30
|
+
Set your Gemini API key in your environment or via a `.env` file:
|
|
31
|
+
|
|
32
|
+
```env
|
|
33
|
+
GEMINI_API_KEY=your_api_key_here
|
|
34
|
+
GEMINI_VERSION=gemini-3-flash-preview # recommended for speed
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### 3. Usage
|
|
38
|
+
|
|
39
|
+
```typescript
|
|
40
|
+
import { Sentinel, z } from '@isoldex/sentinel';
|
|
41
|
+
|
|
42
|
+
async function run() {
|
|
43
|
+
const sentinel = new Sentinel({
|
|
44
|
+
apiKey: process.env.GEMINI_API_KEY!,
|
|
45
|
+
headless: false,
|
|
46
|
+
verbose: 1
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
await sentinel.init();
|
|
50
|
+
|
|
51
|
+
// Navigate
|
|
52
|
+
await sentinel.goto('https://news.ycombinator.com');
|
|
53
|
+
|
|
54
|
+
// Extract structured data with Zod
|
|
55
|
+
const schema = z.object({
|
|
56
|
+
topStory: z.string(),
|
|
57
|
+
points: z.number()
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
const data = await sentinel.extract('Get the title and points of the #1 story', schema);
|
|
61
|
+
console.log('Top Story:', data);
|
|
62
|
+
|
|
63
|
+
// Perform natural language actions
|
|
64
|
+
await sentinel.act('Click on the "new" link in the header');
|
|
65
|
+
|
|
66
|
+
await sentinel.close();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
run();
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## 🛠️ API Reference
|
|
75
|
+
|
|
76
|
+
### `new Sentinel(options: SentinelOptions)`
|
|
77
|
+
- `apiKey`: Your Google AI API key.
|
|
78
|
+
- `headless`: Whether to run the browser in the background (default: `false`).
|
|
79
|
+
- `verbose`: Logging level (`0` = silent, `1` = actions, `2` = full debug).
|
|
80
|
+
- `enableCaching`: Enable state caching between calls (default: `true`).
|
|
81
|
+
|
|
82
|
+
### `sentinel.act(instruction: string, options?: ActOptions)`
|
|
83
|
+
Performs an action on the page.
|
|
84
|
+
- `instruction`: Natural language string (e.g., `"Click the login button"`).
|
|
85
|
+
- `options.variables`: Key-value pairs for string interpolation (e.g., `"%query%"`).
|
|
86
|
+
|
|
87
|
+
### `sentinel.extract<T>(instruction: string, schema: SchemaInput<T>)`
|
|
88
|
+
Returns structured data. Supports **Zod schemas** for full TypeScript type inference.
|
|
89
|
+
|
|
90
|
+
### `sentinel.observe(instruction?: string)`
|
|
91
|
+
Returns a list of possible interactive elements and their purposes.
|
|
92
|
+
|
|
93
|
+
### `sentinel.page` / `sentinel.context`
|
|
94
|
+
Direct access to the underlying **Playwright** `Page` and `BrowserContext`.
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## 🏗️ Development & Contributing
|
|
99
|
+
|
|
100
|
+
To develop Sentinel locally:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
git clone https://github.com/ArasHuseyin/sentinel.ai.git
|
|
104
|
+
cd sentinel.ai
|
|
105
|
+
npm install
|
|
106
|
+
npm run build
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Running Tests
|
|
110
|
+
A comprehensive WhatsApp Web test is included in the source:
|
|
111
|
+
```bash
|
|
112
|
+
npx tsc
|
|
113
|
+
node dist/whatsapp-test.js
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## 🧠 Why Sentinel?
|
|
119
|
+
|
|
120
|
+
Unlike standard automation tools that rely on brittle XPaths, Sentinel "sees" the page like a human through accessibility trees and visual text. It understands context, handles dynamic content automatically, and verifies its own success.
|
|
121
|
+
|
|
122
|
+
Licensed under [ISC](LICENSE).
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import { StateParser } from '../core/state-parser.js';
|
|
3
|
+
import { GeminiService } from '../utils/gemini.js';
|
|
4
|
+
export interface ActOptions {
|
|
5
|
+
variables?: Record<string, string>;
|
|
6
|
+
retries?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface ActionResult {
|
|
9
|
+
success: boolean;
|
|
10
|
+
message: string;
|
|
11
|
+
action?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare class ActionEngine {
|
|
14
|
+
private page;
|
|
15
|
+
private stateParser;
|
|
16
|
+
private gemini;
|
|
17
|
+
constructor(page: Page, stateParser: StateParser, gemini: GeminiService);
|
|
18
|
+
act(instruction: string, options?: ActOptions): Promise<ActionResult>;
|
|
19
|
+
private performAction;
|
|
20
|
+
private performSemanticFallback;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=act.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"act.d.ts","sourceRoot":"","sources":["../../src/api/act.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAmBD,qBAAa,YAAY;IAErB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;gBAFN,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,aAAa;IAGzB,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC;YAsE7D,aAAa;YAmBb,uBAAuB;CAmBtC"}
|
package/dist/api/act.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { StateParser } from '../core/state-parser.js';
|
|
2
|
+
import { GeminiService } from '../utils/gemini.js';
|
|
3
|
+
/**
|
|
4
|
+
* Replaces %variable% placeholders in an instruction string.
|
|
5
|
+
*/
|
|
6
|
+
function interpolateVariables(instruction, variables) {
|
|
7
|
+
if (!variables)
|
|
8
|
+
return instruction;
|
|
9
|
+
return instruction.replace(/%(\w+)%/g, (_, key) => variables[key] ?? `%${key}%`);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Waits for the page to settle after an action.
|
|
13
|
+
*/
|
|
14
|
+
async function waitForPageSettle(page, timeout = 3000) {
|
|
15
|
+
await page.waitForLoadState('networkidle', { timeout }).catch(() => {
|
|
16
|
+
// Page might be a SPA that never fully idles – that is OK
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
export class ActionEngine {
|
|
20
|
+
page;
|
|
21
|
+
stateParser;
|
|
22
|
+
gemini;
|
|
23
|
+
constructor(page, stateParser, gemini) {
|
|
24
|
+
this.page = page;
|
|
25
|
+
this.stateParser = stateParser;
|
|
26
|
+
this.gemini = gemini;
|
|
27
|
+
}
|
|
28
|
+
async act(instruction, options) {
|
|
29
|
+
const resolvedInstruction = interpolateVariables(instruction, options?.variables);
|
|
30
|
+
const state = await this.stateParser.parse();
|
|
31
|
+
const prompt = `
|
|
32
|
+
Current Page URL: ${state.url}
|
|
33
|
+
Page Title: ${state.title}
|
|
34
|
+
Instruction: "${resolvedInstruction}"
|
|
35
|
+
|
|
36
|
+
Elements on page:
|
|
37
|
+
${JSON.stringify(state.elements.map(e => ({ id: e.id, role: e.role, name: e.name })), null, 2)}
|
|
38
|
+
|
|
39
|
+
Select the ID of the element to interact with and the action to perform (click, fill, hover).
|
|
40
|
+
If the action is "fill", provide the value to fill.
|
|
41
|
+
Provide clear reasoning for your choice.
|
|
42
|
+
`;
|
|
43
|
+
const schema = {
|
|
44
|
+
type: 'object',
|
|
45
|
+
properties: {
|
|
46
|
+
elementId: { type: 'number' },
|
|
47
|
+
action: { type: 'string', enum: ['click', 'fill', 'hover'] },
|
|
48
|
+
value: { type: 'string' },
|
|
49
|
+
reasoning: { type: 'string' },
|
|
50
|
+
},
|
|
51
|
+
required: ['elementId', 'action', 'reasoning'],
|
|
52
|
+
};
|
|
53
|
+
const decision = await this.gemini.generateStructuredData(prompt, schema);
|
|
54
|
+
const target = state.elements.find(e => e.id === decision.elementId);
|
|
55
|
+
if (!target) {
|
|
56
|
+
return { success: false, message: `Could not find element with ID ${decision.elementId}` };
|
|
57
|
+
}
|
|
58
|
+
const actionLabel = `${decision.action} on "${target.name}" (${target.role})`;
|
|
59
|
+
console.log(`[Act] ${actionLabel} → ${decision.reasoning}`);
|
|
60
|
+
// Invalidate cache after action – state will change
|
|
61
|
+
this.stateParser.invalidateCache();
|
|
62
|
+
try {
|
|
63
|
+
await this.performAction(decision.action, target, decision.value);
|
|
64
|
+
await waitForPageSettle(this.page);
|
|
65
|
+
return {
|
|
66
|
+
success: true,
|
|
67
|
+
message: `Successfully performed ${decision.action} on "${target.name}"`,
|
|
68
|
+
action: actionLabel,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
catch (primaryError) {
|
|
72
|
+
console.warn(`[Act] Coordinate action failed, trying semantic fallback... (${primaryError.message})`);
|
|
73
|
+
try {
|
|
74
|
+
await this.performSemanticFallback(decision.action, target, decision.value);
|
|
75
|
+
await waitForPageSettle(this.page);
|
|
76
|
+
return {
|
|
77
|
+
success: true,
|
|
78
|
+
message: `Successfully performed ${decision.action} on "${target.name}" (via fallback)`,
|
|
79
|
+
action: actionLabel,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
catch (fallbackError) {
|
|
83
|
+
return { success: false, message: `Action failed: ${fallbackError.message}`, action: actionLabel };
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
async performAction(action, target, value) {
|
|
88
|
+
const { x, y, width, height } = target.boundingClientRect;
|
|
89
|
+
const cx = x + width / 2;
|
|
90
|
+
const cy = y + height / 2;
|
|
91
|
+
if (action === 'click') {
|
|
92
|
+
await this.page.mouse.click(cx, cy);
|
|
93
|
+
}
|
|
94
|
+
else if (action === 'fill') {
|
|
95
|
+
await this.page.mouse.click(cx, cy, { clickCount: 3 });
|
|
96
|
+
await this.page.keyboard.type(value || '');
|
|
97
|
+
}
|
|
98
|
+
else if (action === 'hover') {
|
|
99
|
+
await this.page.mouse.move(cx, cy);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
async performSemanticFallback(action, target, value) {
|
|
103
|
+
const locator = this.page.locator(`[role="${target.role}"]`, target.name ? { hasText: target.name } : undefined).first();
|
|
104
|
+
if (action === 'click') {
|
|
105
|
+
await locator.click({ timeout: 5000 });
|
|
106
|
+
}
|
|
107
|
+
else if (action === 'fill') {
|
|
108
|
+
await locator.click({ clickCount: 3, timeout: 5000 });
|
|
109
|
+
await locator.pressSequentially(value || '', { delay: 30 });
|
|
110
|
+
}
|
|
111
|
+
else if (action === 'hover') {
|
|
112
|
+
await locator.hover({ timeout: 5000 });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=act.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"act.js","sourceRoot":"","sources":["../../src/api/act.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAanD;;GAEG;AACH,SAAS,oBAAoB,CAAC,WAAmB,EAAE,SAAkC;IACnF,IAAI,CAAC,SAAS;QAAE,OAAO,WAAW,CAAC;IACnC,OAAO,WAAW,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC;AACnF,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,iBAAiB,CAAC,IAAU,EAAE,OAAO,GAAG,IAAI;IACzD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACjE,0DAA0D;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,YAAY;IAEb;IACA;IACA;IAHV,YACU,IAAU,EACV,WAAwB,EACxB,MAAqB;QAFrB,SAAI,GAAJ,IAAI,CAAM;QACV,gBAAW,GAAX,WAAW,CAAa;QACxB,WAAM,GAAN,MAAM,CAAe;IAC5B,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,WAAmB,EAAE,OAAoB;QACjD,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE7C,MAAM,MAAM,GAAG;0BACO,KAAK,CAAC,GAAG;oBACf,KAAK,CAAC,KAAK;sBACT,mBAAmB;;;QAGjC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;;;;;KAK/F,CAAC;QAEF,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;gBAC5D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC;SAC/C,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAKtD,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kCAAkC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAC7F,CAAC;QAED,MAAM,WAAW,GAAG,GAAG,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,IAAI,GAAG,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QAE5D,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,0BAA0B,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,IAAI,GAAG;gBACxE,MAAM,EAAE,WAAW;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,YAAiB,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,gEAAgE,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;YACtG,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,0BAA0B,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,IAAI,kBAAkB;oBACvF,MAAM,EAAE,WAAW;iBACpB,CAAC;YACJ,CAAC;YAAC,OAAO,aAAkB,EAAE,CAAC;gBAC5B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,aAAa,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YACrG,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAkC,EAClC,MAAiB,EACjB,KAAc;QAEd,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,kBAAkB,CAAC;QAC1D,MAAM,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;QACzB,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;QAE1B,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;YACvD,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;aAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,MAAkC,EAClC,MAAiB,EACjB,KAAc;QAEd,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAC/B,UAAU,MAAM,CAAC,IAAI,IAAI,EACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CACnD,CAAC,KAAK,EAAE,CAAC;QAEV,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;aAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,MAAM,OAAO,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import { StateParser } from '../core/state-parser.js';
|
|
3
|
+
import { GeminiService } from '../utils/gemini.js';
|
|
4
|
+
import type { SchemaInput } from '../utils/gemini.js';
|
|
5
|
+
export declare class ExtractionEngine {
|
|
6
|
+
private page;
|
|
7
|
+
private stateParser;
|
|
8
|
+
private gemini;
|
|
9
|
+
constructor(page: Page, stateParser: StateParser, gemini: GeminiService);
|
|
10
|
+
extract<T>(instruction: string, schema: SchemaInput<T>): Promise<T>;
|
|
11
|
+
private getPageText;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=extract.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/api/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtD,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;gBAFN,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,aAAa;IAGzB,OAAO,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YAyB3D,WAAW;CAc1B"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { StateParser } from '../core/state-parser.js';
|
|
2
|
+
import { GeminiService } from '../utils/gemini.js';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
const MAX_PAGE_TEXT_CHARS = 8000;
|
|
5
|
+
export class ExtractionEngine {
|
|
6
|
+
page;
|
|
7
|
+
stateParser;
|
|
8
|
+
gemini;
|
|
9
|
+
constructor(page, stateParser, gemini) {
|
|
10
|
+
this.page = page;
|
|
11
|
+
this.stateParser = stateParser;
|
|
12
|
+
this.gemini = gemini;
|
|
13
|
+
}
|
|
14
|
+
async extract(instruction, schema) {
|
|
15
|
+
// Run AOM parse and innerText capture in parallel
|
|
16
|
+
const [aomState, pageText] = await Promise.all([
|
|
17
|
+
this.stateParser.parse(),
|
|
18
|
+
this.getPageText(),
|
|
19
|
+
]);
|
|
20
|
+
const prompt = `
|
|
21
|
+
Extract structured data from a web page according to the instruction: "${instruction}"
|
|
22
|
+
|
|
23
|
+
Page URL: ${aomState.url}
|
|
24
|
+
Page Title: ${aomState.title}
|
|
25
|
+
|
|
26
|
+
--- INTERACTIVE ELEMENTS (AOM) ---
|
|
27
|
+
${JSON.stringify(aomState.elements.map(e => ({ id: e.id, role: e.role, name: e.name })), null, 2)}
|
|
28
|
+
|
|
29
|
+
--- VISIBLE PAGE TEXT ---
|
|
30
|
+
${pageText}
|
|
31
|
+
|
|
32
|
+
Return ONLY the requested JSON. Do not include explanations.
|
|
33
|
+
`;
|
|
34
|
+
return await this.gemini.generateStructuredData(prompt, schema);
|
|
35
|
+
}
|
|
36
|
+
async getPageText() {
|
|
37
|
+
try {
|
|
38
|
+
const text = await this.page.evaluate(() => {
|
|
39
|
+
const clone = document.body.cloneNode(true);
|
|
40
|
+
clone.querySelectorAll('script, style, noscript, svg, [aria-hidden="true"]').forEach(el => el.remove());
|
|
41
|
+
return (clone.innerText || clone.textContent || '').replace(/\s+/g, ' ').trim();
|
|
42
|
+
});
|
|
43
|
+
return text.length > MAX_PAGE_TEXT_CHARS
|
|
44
|
+
? text.slice(0, MAX_PAGE_TEXT_CHARS) + '... [truncated]'
|
|
45
|
+
: text;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return '[Could not extract page text]';
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=extract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/api/extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IAHV,YACU,IAAU,EACV,WAAwB,EACxB,MAAqB;QAFrB,SAAI,GAAJ,IAAI,CAAM;QACV,gBAAW,GAAX,WAAW,CAAa;QACxB,WAAM,GAAN,MAAM,CAAe;IAC5B,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAI,WAAmB,EAAE,MAAsB;QAC1D,kDAAkD;QAClD,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;YACxB,IAAI,CAAC,WAAW,EAAE;SACnB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG;+EAC4D,WAAW;;kBAExE,QAAQ,CAAC,GAAG;oBACV,QAAQ,CAAC,KAAK;;;QAG1B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;;;QAG/F,QAAQ;;;KAGX,CAAC;QAEF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAI,MAAM,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAgB,CAAC;gBAC3D,KAAK,CAAC,gBAAgB,CAAC,oDAAoD,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;gBACxG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAClF,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,MAAM,GAAG,mBAAmB;gBACtC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,GAAG,iBAAiB;gBACxD,CAAC,CAAC,IAAI,CAAC;QACX,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,+BAA+B,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import { StateParser } from '../core/state-parser.js';
|
|
3
|
+
import { GeminiService } from '../utils/gemini.js';
|
|
4
|
+
export interface ObserveResult {
|
|
5
|
+
description: string;
|
|
6
|
+
selector?: string;
|
|
7
|
+
method?: 'click' | 'fill' | 'hover' | 'select' | 'check' | 'press';
|
|
8
|
+
arguments?: string[];
|
|
9
|
+
}
|
|
10
|
+
export declare class ObservationEngine {
|
|
11
|
+
private page;
|
|
12
|
+
private stateParser;
|
|
13
|
+
private gemini;
|
|
14
|
+
constructor(page: Page, stateParser: StateParser, gemini: GeminiService);
|
|
15
|
+
observe(instruction?: string): Promise<ObserveResult[]>;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=observe.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/api/observe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;IACnE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;gBAFN,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,aAAa;IAGzB,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;CA+C9D"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { StateParser } from '../core/state-parser.js';
|
|
2
|
+
import { GeminiService } from '../utils/gemini.js';
|
|
3
|
+
export class ObservationEngine {
|
|
4
|
+
page;
|
|
5
|
+
stateParser;
|
|
6
|
+
gemini;
|
|
7
|
+
constructor(page, stateParser, gemini) {
|
|
8
|
+
this.page = page;
|
|
9
|
+
this.stateParser = stateParser;
|
|
10
|
+
this.gemini = gemini;
|
|
11
|
+
}
|
|
12
|
+
async observe(instruction) {
|
|
13
|
+
const fullState = await this.stateParser.parse();
|
|
14
|
+
const focusHint = instruction
|
|
15
|
+
? `Focus specifically on: "${instruction}"`
|
|
16
|
+
: 'List all possible interactions a user can take on this page.';
|
|
17
|
+
const prompt = `
|
|
18
|
+
You are a web observation agent analyzing interactive elements on a page.
|
|
19
|
+
${focusHint}
|
|
20
|
+
|
|
21
|
+
URL: ${fullState.url}
|
|
22
|
+
Title: ${fullState.title}
|
|
23
|
+
|
|
24
|
+
Interactive Elements (AOM):
|
|
25
|
+
${JSON.stringify(fullState.elements.map(e => ({ id: e.id, role: e.role, name: e.name })), null, 2)}
|
|
26
|
+
|
|
27
|
+
Return a list of possible actions. For each action provide:
|
|
28
|
+
- description: what the action does (human-readable)
|
|
29
|
+
- selector: CSS or role-based selector if determinable (optional)
|
|
30
|
+
- method: the interaction method (click, fill, hover, select, check, press)
|
|
31
|
+
- arguments: any arguments needed (e.g. text to fill) (optional)
|
|
32
|
+
`;
|
|
33
|
+
const schema = {
|
|
34
|
+
type: 'object',
|
|
35
|
+
properties: {
|
|
36
|
+
actions: {
|
|
37
|
+
type: 'array',
|
|
38
|
+
items: {
|
|
39
|
+
type: 'object',
|
|
40
|
+
properties: {
|
|
41
|
+
description: { type: 'string' },
|
|
42
|
+
selector: { type: 'string' },
|
|
43
|
+
method: { type: 'string', enum: ['click', 'fill', 'hover', 'select', 'check', 'press'] },
|
|
44
|
+
arguments: { type: 'array', items: { type: 'string' } },
|
|
45
|
+
},
|
|
46
|
+
required: ['description', 'method'],
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
required: ['actions'],
|
|
51
|
+
};
|
|
52
|
+
const result = await this.gemini.generateStructuredData(prompt, schema);
|
|
53
|
+
return result.actions;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=observe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"observe.js","sourceRoot":"","sources":["../../src/api/observe.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AASnD,MAAM,OAAO,iBAAiB;IAElB;IACA;IACA;IAHV,YACU,IAAU,EACV,WAAwB,EACxB,MAAqB;QAFrB,SAAI,GAAJ,IAAI,CAAM;QACV,gBAAW,GAAX,WAAW,CAAa;QACxB,WAAM,GAAN,MAAM,CAAe;IAC5B,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,WAAoB;QAChC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,WAAW;YAC3B,CAAC,CAAC,2BAA2B,WAAW,GAAG;YAC3C,CAAC,CAAC,8DAA8D,CAAC;QAEnE,MAAM,MAAM,GAAG;;QAEX,SAAS;;aAEJ,SAAS,CAAC,GAAG;eACX,SAAS,CAAC,KAAK;;;QAGtB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;;;;;;;KAOnG,CAAC;QAEF,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BAC/B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BAC5B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;4BACxF,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;yBACxD;wBACD,QAAQ,EAAE,CAAC,aAAa,EAAE,QAAQ,CAAC;qBACpC;iBACF;aACF;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;SACtB,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAA+B,MAAM,EAAE,MAAM,CAAC,CAAC;QACtG,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"complex-test.d.ts","sourceRoot":"","sources":["../src/complex-test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { Sentinel } from './index.js';
|
|
2
|
+
import dotenv from 'dotenv';
|
|
3
|
+
dotenv.config();
|
|
4
|
+
const API_KEY = process.env.GEMINI_API_KEY || "YOUR_API_KEY";
|
|
5
|
+
async function complexTest() {
|
|
6
|
+
const sentinel = new Sentinel({ apiKey: API_KEY, headless: false });
|
|
7
|
+
try {
|
|
8
|
+
await sentinel.init();
|
|
9
|
+
console.log("1. Navigating to Hacker News...");
|
|
10
|
+
await sentinel.goto("https://news.ycombinator.com");
|
|
11
|
+
console.log("2. Extracting top headlines...");
|
|
12
|
+
const headlines = await sentinel.extract("Extract the titles and ranks of the first 10 news items on the front page", {
|
|
13
|
+
type: "object",
|
|
14
|
+
properties: {
|
|
15
|
+
items: {
|
|
16
|
+
type: "array",
|
|
17
|
+
items: {
|
|
18
|
+
type: "object",
|
|
19
|
+
properties: {
|
|
20
|
+
title: { type: "string" },
|
|
21
|
+
rank: { type: "number" }
|
|
22
|
+
},
|
|
23
|
+
required: ["title", "rank"]
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
required: ["items"]
|
|
28
|
+
});
|
|
29
|
+
console.log("Top Headlines:", headlines.items.map(h => `${h.rank}. ${h.title}`));
|
|
30
|
+
// Find the first headline that contains 'AI' or 'model' or 'LLM'
|
|
31
|
+
const aiHeadline = headlines.items.find(h => /AI|model|LLM|GPT|intelligence/i.test(h.title)) || headlines.items[0];
|
|
32
|
+
if (aiHeadline) {
|
|
33
|
+
console.log(`3. Navigating to article: "${aiHeadline.title}"`);
|
|
34
|
+
await sentinel.act(`Click on the link with the text "${aiHeadline.title}"`);
|
|
35
|
+
console.log("4. Observing current page...");
|
|
36
|
+
const observation = await sentinel.observe();
|
|
37
|
+
console.log("Observation:", observation);
|
|
38
|
+
console.log("5. Extracting first paragraph from the article...");
|
|
39
|
+
const summary = await sentinel.extract("Extract the first meaningful paragraph or summary of this article", {
|
|
40
|
+
type: "object",
|
|
41
|
+
properties: {
|
|
42
|
+
paragraph: { type: "string" }
|
|
43
|
+
},
|
|
44
|
+
required: ["paragraph"]
|
|
45
|
+
});
|
|
46
|
+
console.log("Article Summary:", summary.paragraph);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
console.error("Complex Test Error:", error);
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
console.log("Test finished. Closing browser...");
|
|
54
|
+
await sentinel.close();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
complexTest();
|
|
58
|
+
//# sourceMappingURL=complex-test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"complex-test.js","sourceRoot":"","sources":["../src/complex-test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,cAAc,CAAC;AAE7D,KAAK,UAAU,WAAW;IACxB,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAEpD,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,OAAO,CACtC,2EAA2E,EAC3E;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;yBACzB;wBACD,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;qBAC5B;iBACF;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAEjF,iEAAiE;QACjE,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAC1C,gCAAgC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC/C,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAExB,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,8BAA8B,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;YAC/D,MAAM,QAAQ,CAAC,GAAG,CAAC,oCAAoC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;YAE5E,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YAC5C,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;YAEzC,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;YACjE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CACpC,mEAAmE,EACnE;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC9B;gBACD,QAAQ,EAAE,CAAC,WAAW,CAAC;aACxB,CACF,CAAC;YAEF,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACrD,CAAC;IAEH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED,WAAW,EAAE,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { BrowserContext, Page, CDPSession } from 'playwright';
|
|
2
|
+
export interface DriverOptions {
|
|
3
|
+
headless?: boolean;
|
|
4
|
+
viewport?: {
|
|
5
|
+
width: number;
|
|
6
|
+
height: number;
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
export declare class SentinelDriver {
|
|
10
|
+
private options;
|
|
11
|
+
private browser;
|
|
12
|
+
private context;
|
|
13
|
+
private page;
|
|
14
|
+
private cdpSession;
|
|
15
|
+
constructor(options?: DriverOptions);
|
|
16
|
+
initialize(): Promise<void>;
|
|
17
|
+
getPage(): Page;
|
|
18
|
+
getContext(): BrowserContext;
|
|
19
|
+
getCDPSession(): CDPSession;
|
|
20
|
+
close(): Promise<void>;
|
|
21
|
+
goto(url: string): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=driver.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/core/driver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAW,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE5E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9C;AAED,qBAAa,cAAc;IAMb,OAAO,CAAC,OAAO;IAL3B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,UAAU,CAA2B;gBAEzB,OAAO,GAAE,aAAmC;IAE1D,UAAU;IAyBhB,OAAO;IAKP,UAAU;IAKV,aAAa;IAKP,KAAK;IAML,IAAI,CAAC,GAAG,EAAE,MAAM;CAWvB"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { chromium } from 'playwright';
|
|
2
|
+
export class SentinelDriver {
|
|
3
|
+
options;
|
|
4
|
+
browser = null;
|
|
5
|
+
context = null;
|
|
6
|
+
page = null;
|
|
7
|
+
cdpSession = null;
|
|
8
|
+
constructor(options = { headless: false }) {
|
|
9
|
+
this.options = options;
|
|
10
|
+
}
|
|
11
|
+
async initialize() {
|
|
12
|
+
this.browser = await chromium.launch({
|
|
13
|
+
headless: this.options.headless ?? false,
|
|
14
|
+
args: [
|
|
15
|
+
'--disable-blink-features=AutomationControlled',
|
|
16
|
+
'--no-sandbox',
|
|
17
|
+
'--disable-dev-shm-usage',
|
|
18
|
+
],
|
|
19
|
+
});
|
|
20
|
+
this.context = await this.browser.newContext({
|
|
21
|
+
viewport: this.options.viewport || { width: 1280, height: 720 },
|
|
22
|
+
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
|
23
|
+
locale: 'de-AT',
|
|
24
|
+
timezoneId: 'Europe/Vienna',
|
|
25
|
+
});
|
|
26
|
+
this.page = await this.context.newPage();
|
|
27
|
+
this.cdpSession = await this.page.context().newCDPSession(this.page);
|
|
28
|
+
// Enable Accessibility tree access via CDP
|
|
29
|
+
await this.cdpSession.send('Accessibility.enable');
|
|
30
|
+
}
|
|
31
|
+
getPage() {
|
|
32
|
+
if (!this.page)
|
|
33
|
+
throw new Error('Driver not initialized');
|
|
34
|
+
return this.page;
|
|
35
|
+
}
|
|
36
|
+
getContext() {
|
|
37
|
+
if (!this.context)
|
|
38
|
+
throw new Error('Driver not initialized');
|
|
39
|
+
return this.context;
|
|
40
|
+
}
|
|
41
|
+
getCDPSession() {
|
|
42
|
+
if (!this.cdpSession)
|
|
43
|
+
throw new Error('Driver not initialized');
|
|
44
|
+
return this.cdpSession;
|
|
45
|
+
}
|
|
46
|
+
async close() {
|
|
47
|
+
if (this.browser) {
|
|
48
|
+
await this.browser.close();
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async goto(url) {
|
|
52
|
+
const page = this.getPage();
|
|
53
|
+
// Use domcontentloaded as primary – faster and sufficient for most pages
|
|
54
|
+
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
55
|
+
// Then wait for network to settle (SPA content), but don't crash if it times out
|
|
56
|
+
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {
|
|
57
|
+
console.warn(`[Driver] networkidle timeout for ${url} – proceeding anyway`);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=driver.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"driver.js","sourceRoot":"","sources":["../../src/core/driver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAQtC,MAAM,OAAO,cAAc;IAML;IALZ,OAAO,GAAmB,IAAI,CAAC;IAC/B,OAAO,GAA0B,IAAI,CAAC;IACtC,IAAI,GAAgB,IAAI,CAAC;IACzB,UAAU,GAAsB,IAAI,CAAC;IAE7C,YAAoB,UAAyB,EAAE,QAAQ,EAAE,KAAK,EAAE;QAA5C,YAAO,GAAP,OAAO,CAAqC;IAAG,CAAC;IAEpE,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACnC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK;YACxC,IAAI,EAAE;gBACJ,+CAA+C;gBAC/C,cAAc;gBACd,yBAAyB;aAC1B;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;YAC/D,SAAS,EACP,iHAAiH;YACnH,MAAM,EAAE,OAAO;YACf,UAAU,EAAE,eAAe;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAErE,2CAA2C;QAC3C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO;QACL,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC1D,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,UAAU;QACR,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,aAAa;QACX,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAE5B,yEAAyE;QACzE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAExE,iFAAiF;QACjF,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACvE,OAAO,CAAC,IAAI,CAAC,oCAAoC,GAAG,sBAAsB,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|