@askjo/camoufox-browser 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.bak +4 -0
- package/.github/workflows/deploy.yml +21 -0
- package/AGENTS.md +153 -0
- package/Dockerfile.camoufox +59 -0
- package/LICENSE +21 -0
- package/README.md +234 -0
- package/SKILL.md +165 -0
- package/experimental/chromium/Dockerfile +35 -0
- package/experimental/chromium/README.md +47 -0
- package/experimental/chromium/run.sh +24 -0
- package/experimental/chromium/server.js +812 -0
- package/fly.toml +29 -0
- package/jest.config.js +41 -0
- package/lib/macros.js +30 -0
- package/openclaw.plugin.json +31 -0
- package/package.json +30 -0
- package/plugin.ts +312 -0
- package/run-camoufox.sh +37 -0
- package/server-camoufox.js +946 -0
- package/tests/e2e/concurrency.test.js +103 -0
- package/tests/e2e/formSubmission.test.js +129 -0
- package/tests/e2e/macroNavigation.test.js +92 -0
- package/tests/e2e/navigation.test.js +128 -0
- package/tests/e2e/scroll.test.js +81 -0
- package/tests/e2e/snapshotLinks.test.js +141 -0
- package/tests/e2e/tabLifecycle.test.js +149 -0
- package/tests/e2e/typingEnter.test.js +147 -0
- package/tests/helpers/client.js +222 -0
- package/tests/helpers/startJoBrowser.js +95 -0
- package/tests/helpers/testSite.js +238 -0
- package/tests/live/googleSearch.test.js +93 -0
- package/tests/live/macroExpansion.test.js +132 -0
- package/tests/unit/macros.test.js +123 -0
|
@@ -0,0 +1,812 @@
|
|
|
1
|
+
const { chromium } = require('playwright-extra');
|
|
2
|
+
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
|
3
|
+
const express = require('express');
|
|
4
|
+
const crypto = require('crypto');
|
|
5
|
+
|
|
6
|
+
// Add stealth plugin to avoid bot detection
|
|
7
|
+
chromium.use(StealthPlugin());
|
|
8
|
+
|
|
9
|
+
const app = express();
|
|
10
|
+
app.use(express.json({ limit: '5mb' }));
|
|
11
|
+
|
|
12
|
+
let browser = null;
|
|
13
|
+
// userId -> { context, tabGroups: Map<listItemId, Map<tabId, TabState>>, lastAccess }
|
|
14
|
+
// TabState = { page, refs: Map<refId, {role, name, nth}>, visitedUrls: Set, toolCalls: number }
|
|
15
|
+
const sessions = new Map();
|
|
16
|
+
|
|
17
|
+
const SESSION_TIMEOUT_MS = 30 * 60 * 1000; // 30 min
|
|
18
|
+
const MAX_SNAPSHOT_NODES = 500;
|
|
19
|
+
|
|
20
|
+
async function ensureBrowser() {
|
|
21
|
+
if (!browser) {
|
|
22
|
+
const launchOptions = {
|
|
23
|
+
args: [
|
|
24
|
+
'--no-sandbox',
|
|
25
|
+
'--disable-dev-shm-usage',
|
|
26
|
+
'--disable-gpu',
|
|
27
|
+
'--disable-software-rasterizer',
|
|
28
|
+
'--disable-background-timer-throttling',
|
|
29
|
+
'--disable-backgrounding-occluded-windows',
|
|
30
|
+
'--disable-renderer-backgrounding'
|
|
31
|
+
]
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// Use CHROMIUM_PATH if set (for Docker/Fly.io), otherwise use Playwright's bundled browser
|
|
35
|
+
if (process.env.CHROMIUM_PATH) {
|
|
36
|
+
launchOptions.executablePath = process.env.CHROMIUM_PATH;
|
|
37
|
+
console.log(`Using custom Chromium path: ${process.env.CHROMIUM_PATH}`);
|
|
38
|
+
} else {
|
|
39
|
+
console.log('Using Playwright bundled Chromium');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
browser = await chromium.launch(launchOptions);
|
|
43
|
+
console.log('Browser launched');
|
|
44
|
+
}
|
|
45
|
+
return browser;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function getSession(userId) {
|
|
49
|
+
let session = sessions.get(userId);
|
|
50
|
+
if (!session) {
|
|
51
|
+
const b = await ensureBrowser();
|
|
52
|
+
const context = await b.newContext({
|
|
53
|
+
viewport: { width: 1280, height: 720 },
|
|
54
|
+
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
|
55
|
+
locale: 'en-US',
|
|
56
|
+
timezoneId: 'America/Los_Angeles',
|
|
57
|
+
// Add realistic browser permissions
|
|
58
|
+
permissions: ['geolocation'],
|
|
59
|
+
geolocation: { latitude: 37.7749, longitude: -122.4194 }, // San Francisco
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// Add anti-detection init script (similar to WebView.swift stealth approach)
|
|
63
|
+
await context.addInitScript(() => {
|
|
64
|
+
// Hide webdriver property
|
|
65
|
+
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
|
66
|
+
|
|
67
|
+
// Override plugins to look like real Chrome
|
|
68
|
+
Object.defineProperty(navigator, 'plugins', {
|
|
69
|
+
get: () => [
|
|
70
|
+
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer' },
|
|
71
|
+
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai' },
|
|
72
|
+
{ name: 'Native Client', filename: 'internal-nacl-plugin' }
|
|
73
|
+
]
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// Override languages
|
|
77
|
+
Object.defineProperty(navigator, 'languages', {
|
|
78
|
+
get: () => ['en-US', 'en']
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// Add chrome object (missing in headless)
|
|
82
|
+
window.chrome = {
|
|
83
|
+
runtime: {},
|
|
84
|
+
loadTimes: function() {},
|
|
85
|
+
csi: function() {},
|
|
86
|
+
app: {}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// Override permissions query to return 'prompt' for notifications
|
|
90
|
+
const originalQuery = window.Notification?.permission;
|
|
91
|
+
if (window.Notification) {
|
|
92
|
+
window.Notification.permission = 'default';
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
session = { context, tabGroups: new Map(), lastAccess: Date.now() };
|
|
97
|
+
sessions.set(userId, session);
|
|
98
|
+
console.log(`Session created for user ${userId}`);
|
|
99
|
+
}
|
|
100
|
+
session.lastAccess = Date.now();
|
|
101
|
+
return session;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function getTabGroup(session, listItemId) {
|
|
105
|
+
let group = session.tabGroups.get(listItemId);
|
|
106
|
+
if (!group) {
|
|
107
|
+
group = new Map();
|
|
108
|
+
session.tabGroups.set(listItemId, group);
|
|
109
|
+
}
|
|
110
|
+
return group;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function findTab(session, tabId) {
|
|
114
|
+
for (const [listItemId, group] of session.tabGroups) {
|
|
115
|
+
if (group.has(tabId)) {
|
|
116
|
+
const tabState = group.get(tabId);
|
|
117
|
+
return { tabState, listItemId, group };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function createTabState(page) {
|
|
124
|
+
return {
|
|
125
|
+
page,
|
|
126
|
+
refs: new Map(), // refId -> { role, name, nth }
|
|
127
|
+
visitedUrls: new Set(), // URLs visited in this tab
|
|
128
|
+
toolCalls: 0 // Track tool usage for validation
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Wait for page to be ready for accessibility snapshot
|
|
133
|
+
async function waitForPageReady(page, options = {}) {
|
|
134
|
+
const { timeout = 10000, waitForNetwork = true } = options;
|
|
135
|
+
|
|
136
|
+
try {
|
|
137
|
+
// Wait for DOM to be ready
|
|
138
|
+
await page.waitForLoadState('domcontentloaded', { timeout });
|
|
139
|
+
|
|
140
|
+
// Optionally wait for network to settle (useful for SPAs)
|
|
141
|
+
if (waitForNetwork) {
|
|
142
|
+
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {
|
|
143
|
+
// networkidle can timeout on busy pages, that's ok
|
|
144
|
+
console.log('waitForPageReady: networkidle timeout (continuing anyway)');
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Small delay for JS frameworks to finish rendering
|
|
149
|
+
await page.waitForTimeout(200);
|
|
150
|
+
|
|
151
|
+
return true;
|
|
152
|
+
} catch (err) {
|
|
153
|
+
console.log(`waitForPageReady: ${err.message}`);
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Build element refs from aria snapshot (Playwright 1.48+ uses locator.ariaSnapshot())
|
|
159
|
+
async function buildRefs(page) {
|
|
160
|
+
const refs = new Map();
|
|
161
|
+
|
|
162
|
+
if (!page || page.isClosed()) {
|
|
163
|
+
console.log('buildRefs: Page is closed or invalid');
|
|
164
|
+
return refs;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Wait for page to be ready before taking snapshot
|
|
168
|
+
await waitForPageReady(page, { waitForNetwork: false });
|
|
169
|
+
|
|
170
|
+
// Use the new ariaSnapshot API (Playwright 1.48+)
|
|
171
|
+
// This returns a YAML string representation of the accessibility tree
|
|
172
|
+
const ariaYaml = await page.locator('body').ariaSnapshot();
|
|
173
|
+
|
|
174
|
+
if (!ariaYaml) {
|
|
175
|
+
console.log('buildRefs: No aria snapshot available');
|
|
176
|
+
return refs;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Parse the YAML to extract interactive elements
|
|
180
|
+
// Format: "- role \"name\"" or "- role \"name\" [attr=value]"
|
|
181
|
+
const lines = ariaYaml.split('\n');
|
|
182
|
+
let refCounter = 1;
|
|
183
|
+
|
|
184
|
+
const interactiveRoles = [
|
|
185
|
+
'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox',
|
|
186
|
+
'menuitem', 'tab', 'searchbox', 'slider', 'spinbutton', 'switch'
|
|
187
|
+
];
|
|
188
|
+
|
|
189
|
+
for (const line of lines) {
|
|
190
|
+
if (refCounter > MAX_SNAPSHOT_NODES) break;
|
|
191
|
+
|
|
192
|
+
// Match patterns like "- button \"Click me\"" or "- link \"Home\""
|
|
193
|
+
const match = line.match(/^\s*-\s+(\w+)(?:\s+"([^"]*)")?/);
|
|
194
|
+
if (match) {
|
|
195
|
+
const [, role, name] = match;
|
|
196
|
+
if (interactiveRoles.includes(role.toLowerCase())) {
|
|
197
|
+
const refId = `e${refCounter++}`;
|
|
198
|
+
refs.set(refId, { role: role.toLowerCase(), name: name || '', nth: 0 });
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
return refs;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Get aria snapshot as YAML string (new Playwright API)
|
|
207
|
+
async function getAriaSnapshot(page) {
|
|
208
|
+
if (!page || page.isClosed()) {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
await waitForPageReady(page, { waitForNetwork: false });
|
|
212
|
+
return await page.locator('body').ariaSnapshot();
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Resolve ref to Playwright locator (like OpenClaw's refLocator)
|
|
216
|
+
function refToLocator(page, ref, refs) {
|
|
217
|
+
const info = refs.get(ref);
|
|
218
|
+
if (!info) return null;
|
|
219
|
+
|
|
220
|
+
const { role, name, nth } = info;
|
|
221
|
+
let locator = page.getByRole(role, name ? { name } : undefined);
|
|
222
|
+
|
|
223
|
+
if (nth > 0) {
|
|
224
|
+
locator = locator.nth(nth);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return locator;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// Format accessibility tree as compact text (like OpenClaw's AI snapshot format)
|
|
231
|
+
function formatSnapshotAsText(snapshot, refs) {
|
|
232
|
+
const lines = [];
|
|
233
|
+
|
|
234
|
+
function walk(node, indent = 0) {
|
|
235
|
+
if (!node) return;
|
|
236
|
+
|
|
237
|
+
const { role, name, value, children } = node;
|
|
238
|
+
const prefix = ' '.repeat(indent);
|
|
239
|
+
|
|
240
|
+
// Find if this node has a ref
|
|
241
|
+
let refLabel = '';
|
|
242
|
+
for (const [refId, info] of refs) {
|
|
243
|
+
if (info.role === (role || '').toLowerCase() && info.name === (name || '')) {
|
|
244
|
+
refLabel = `[${refId}] `;
|
|
245
|
+
break;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Build node description
|
|
250
|
+
let desc = role || 'unknown';
|
|
251
|
+
if (name) desc += ` "${name}"`;
|
|
252
|
+
if (value) desc += ` = ${value}`;
|
|
253
|
+
|
|
254
|
+
lines.push(`${prefix}${refLabel}${desc}`);
|
|
255
|
+
|
|
256
|
+
if (children) {
|
|
257
|
+
for (const child of children) {
|
|
258
|
+
walk(child, indent + 1);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
walk(snapshot);
|
|
264
|
+
return lines.join('\n');
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Health check
|
|
268
|
+
app.get('/health', (req, res) => {
|
|
269
|
+
res.json({ ok: true, sessions: sessions.size });
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
// List all tab groups for user
|
|
273
|
+
app.get('/tabs', async (req, res) => {
|
|
274
|
+
const userId = req.query.userId;
|
|
275
|
+
const session = sessions.get(userId);
|
|
276
|
+
if (!session) return res.json({ tabGroups: {} });
|
|
277
|
+
|
|
278
|
+
const tabGroups = {};
|
|
279
|
+
for (const [listItemId, group] of session.tabGroups) {
|
|
280
|
+
tabGroups[listItemId] = [];
|
|
281
|
+
for (const [tabId, tabState] of group) {
|
|
282
|
+
tabGroups[listItemId].push({
|
|
283
|
+
tabId,
|
|
284
|
+
url: tabState.page.url(),
|
|
285
|
+
toolCalls: tabState.toolCalls,
|
|
286
|
+
visitedCount: tabState.visitedUrls.size
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
res.json({ tabGroups });
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
// List tabs for specific list item
|
|
294
|
+
app.get('/tabs/group/:listItemId', async (req, res) => {
|
|
295
|
+
const userId = req.query.userId;
|
|
296
|
+
const session = sessions.get(userId);
|
|
297
|
+
const group = session?.tabGroups.get(req.params.listItemId);
|
|
298
|
+
if (!group) return res.json({ tabs: [] });
|
|
299
|
+
|
|
300
|
+
const tabs = [];
|
|
301
|
+
for (const [tabId, tabState] of group) {
|
|
302
|
+
tabs.push({
|
|
303
|
+
tabId,
|
|
304
|
+
url: tabState.page.url(),
|
|
305
|
+
toolCalls: tabState.toolCalls,
|
|
306
|
+
visitedCount: tabState.visitedUrls.size
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
res.json({ tabs });
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
// Create new tab in a tab group
|
|
313
|
+
app.post('/tabs', async (req, res) => {
|
|
314
|
+
try {
|
|
315
|
+
const { userId, listItemId, url } = req.body;
|
|
316
|
+
if (!listItemId) return res.status(400).json({ error: 'listItemId required' });
|
|
317
|
+
|
|
318
|
+
const session = await getSession(userId);
|
|
319
|
+
const group = getTabGroup(session, listItemId);
|
|
320
|
+
const tabId = crypto.randomUUID();
|
|
321
|
+
const page = await session.context.newPage();
|
|
322
|
+
const tabState = createTabState(page);
|
|
323
|
+
|
|
324
|
+
if (url) {
|
|
325
|
+
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
326
|
+
tabState.visitedUrls.add(url);
|
|
327
|
+
tabState.toolCalls++;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
group.set(tabId, tabState);
|
|
331
|
+
console.log(`Tab ${tabId} created for user ${userId} in group ${listItemId}`);
|
|
332
|
+
res.json({ tabId, listItemId, url: page.url() });
|
|
333
|
+
} catch (err) {
|
|
334
|
+
console.error('Create tab error:', err);
|
|
335
|
+
res.status(500).json({ error: err.message });
|
|
336
|
+
}
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
// URL macro expansion (like Jo's FnBrowserOpenMacroUrl)
|
|
340
|
+
const URL_MACROS = {
|
|
341
|
+
'@google_search': (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}`,
|
|
342
|
+
'@youtube_search': (q) => `https://www.youtube.com/results?search_query=${encodeURIComponent(q)}`,
|
|
343
|
+
'@amazon_search': (q) => `https://www.amazon.com/s?k=${encodeURIComponent(q)}`,
|
|
344
|
+
'@reddit_search': (q) => `https://www.reddit.com/search/?q=${encodeURIComponent(q)}`,
|
|
345
|
+
'@wikipedia_search': (q) => `https://en.wikipedia.org/wiki/Special:Search?search=${encodeURIComponent(q)}`,
|
|
346
|
+
'@twitter_search': (q) => `https://twitter.com/search?q=${encodeURIComponent(q)}`,
|
|
347
|
+
'@yelp_search': (q) => `https://www.yelp.com/search?find_desc=${encodeURIComponent(q)}`,
|
|
348
|
+
'@spotify_search': (q) => `https://open.spotify.com/search/${encodeURIComponent(q)}`,
|
|
349
|
+
'@netflix_search': (q) => `https://www.netflix.com/search?q=${encodeURIComponent(q)}`,
|
|
350
|
+
'@linkedin_search': (q) => `https://www.linkedin.com/search/results/all/?keywords=${encodeURIComponent(q)}`,
|
|
351
|
+
'@instagram_search': (q) => `https://www.instagram.com/explore/tags/${encodeURIComponent(q.replace(/\s+/g, ''))}`,
|
|
352
|
+
'@tiktok_search': (q) => `https://www.tiktok.com/search?q=${encodeURIComponent(q)}`,
|
|
353
|
+
'@twitch_search': (q) => `https://www.twitch.tv/search?term=${encodeURIComponent(q)}`,
|
|
354
|
+
};
|
|
355
|
+
|
|
356
|
+
// Navigate tab (supports URL or macro)
|
|
357
|
+
app.post('/tabs/:tabId/navigate', async (req, res) => {
|
|
358
|
+
try {
|
|
359
|
+
const { userId, url, macro, query } = req.body;
|
|
360
|
+
const session = sessions.get(userId);
|
|
361
|
+
const found = session && findTab(session, req.params.tabId);
|
|
362
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
363
|
+
|
|
364
|
+
const { tabState } = found;
|
|
365
|
+
let targetUrl = url;
|
|
366
|
+
|
|
367
|
+
// Handle macro expansion
|
|
368
|
+
if (macro) {
|
|
369
|
+
const expander = URL_MACROS[macro];
|
|
370
|
+
if (!expander) return res.status(400).json({ error: `Unknown macro: ${macro}` });
|
|
371
|
+
targetUrl = expander(query || '');
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
if (!targetUrl) return res.status(400).json({ error: 'url or macro required' });
|
|
375
|
+
|
|
376
|
+
await tabState.page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
377
|
+
tabState.visitedUrls.add(targetUrl);
|
|
378
|
+
tabState.toolCalls++;
|
|
379
|
+
tabState.refs.clear(); // Clear refs on navigation
|
|
380
|
+
|
|
381
|
+
res.json({ ok: true, url: tabState.page.url() });
|
|
382
|
+
} catch (err) {
|
|
383
|
+
console.error('Navigate error:', err);
|
|
384
|
+
res.status(500).json({ error: err.message });
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// Go back in history
|
|
389
|
+
app.post('/tabs/:tabId/back', async (req, res) => {
|
|
390
|
+
try {
|
|
391
|
+
const { userId } = req.body;
|
|
392
|
+
const session = sessions.get(userId);
|
|
393
|
+
const found = session && findTab(session, req.params.tabId);
|
|
394
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
395
|
+
|
|
396
|
+
const { tabState } = found;
|
|
397
|
+
tabState.toolCalls++;
|
|
398
|
+
await tabState.page.goBack({ timeout: 10000 }).catch(() => {});
|
|
399
|
+
tabState.refs.clear();
|
|
400
|
+
|
|
401
|
+
res.json({ ok: true, url: tabState.page.url() });
|
|
402
|
+
} catch (err) {
|
|
403
|
+
console.error('Back error:', err);
|
|
404
|
+
res.status(500).json({ error: err.message });
|
|
405
|
+
}
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
// Go forward in history
|
|
409
|
+
app.post('/tabs/:tabId/forward', async (req, res) => {
|
|
410
|
+
try {
|
|
411
|
+
const { userId } = req.body;
|
|
412
|
+
const session = sessions.get(userId);
|
|
413
|
+
const found = session && findTab(session, req.params.tabId);
|
|
414
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
415
|
+
|
|
416
|
+
const { tabState } = found;
|
|
417
|
+
tabState.toolCalls++;
|
|
418
|
+
await tabState.page.goForward({ timeout: 10000 }).catch(() => {});
|
|
419
|
+
tabState.refs.clear();
|
|
420
|
+
|
|
421
|
+
res.json({ ok: true, url: tabState.page.url() });
|
|
422
|
+
} catch (err) {
|
|
423
|
+
console.error('Forward error:', err);
|
|
424
|
+
res.status(500).json({ error: err.message });
|
|
425
|
+
}
|
|
426
|
+
});
|
|
427
|
+
|
|
428
|
+
// Refresh page
|
|
429
|
+
app.post('/tabs/:tabId/refresh', async (req, res) => {
|
|
430
|
+
try {
|
|
431
|
+
const { userId } = req.body;
|
|
432
|
+
const session = sessions.get(userId);
|
|
433
|
+
const found = session && findTab(session, req.params.tabId);
|
|
434
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
435
|
+
|
|
436
|
+
const { tabState } = found;
|
|
437
|
+
tabState.toolCalls++;
|
|
438
|
+
await tabState.page.reload({ waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
439
|
+
tabState.refs.clear();
|
|
440
|
+
|
|
441
|
+
res.json({ ok: true, url: tabState.page.url() });
|
|
442
|
+
} catch (err) {
|
|
443
|
+
console.error('Refresh error:', err);
|
|
444
|
+
res.status(500).json({ error: err.message });
|
|
445
|
+
}
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
// Wait for page to be ready (explicit wait endpoint)
|
|
449
|
+
app.post('/tabs/:tabId/wait', async (req, res) => {
|
|
450
|
+
try {
|
|
451
|
+
const { userId, timeout = 10000, waitForNetwork = true } = req.body;
|
|
452
|
+
const session = sessions.get(userId);
|
|
453
|
+
const found = session && findTab(session, req.params.tabId);
|
|
454
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
455
|
+
|
|
456
|
+
const { tabState } = found;
|
|
457
|
+
const ready = await waitForPageReady(tabState.page, { timeout, waitForNetwork });
|
|
458
|
+
|
|
459
|
+
res.json({
|
|
460
|
+
ok: ready,
|
|
461
|
+
url: tabState.page.url(),
|
|
462
|
+
message: ready ? 'Page is ready' : 'Page may still be loading'
|
|
463
|
+
});
|
|
464
|
+
} catch (err) {
|
|
465
|
+
console.error('Wait error:', err);
|
|
466
|
+
res.status(500).json({ error: err.message });
|
|
467
|
+
}
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
// Get accessibility snapshot with element refs
|
|
471
|
+
app.get('/tabs/:tabId/snapshot', async (req, res) => {
|
|
472
|
+
try {
|
|
473
|
+
const userId = req.query.userId;
|
|
474
|
+
const format = req.query.format || 'json'; // 'json' or 'text'
|
|
475
|
+
const waitForReady = req.query.wait !== 'false'; // Default: wait for page
|
|
476
|
+
const session = sessions.get(userId);
|
|
477
|
+
const found = session && findTab(session, req.params.tabId);
|
|
478
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
479
|
+
|
|
480
|
+
const { tabState, listItemId } = found;
|
|
481
|
+
const page = tabState.page;
|
|
482
|
+
|
|
483
|
+
// Validate page state
|
|
484
|
+
if (!page || page.isClosed()) {
|
|
485
|
+
return res.status(500).json({ error: 'Page is closed or invalid' });
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Wait for page to be ready before taking snapshot
|
|
489
|
+
if (waitForReady) {
|
|
490
|
+
await waitForPageReady(page, { waitForNetwork: true });
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// Build refs from aria snapshot (uses new Playwright API)
|
|
494
|
+
const refs = await buildRefs(page);
|
|
495
|
+
tabState.refs = refs;
|
|
496
|
+
tabState.toolCalls++;
|
|
497
|
+
|
|
498
|
+
// Get aria snapshot as YAML (new Playwright 1.48+ API)
|
|
499
|
+
let ariaSnapshot = await getAriaSnapshot(page);
|
|
500
|
+
|
|
501
|
+
// Retry once if snapshot is empty
|
|
502
|
+
if (!ariaSnapshot) {
|
|
503
|
+
console.log('Snapshot empty, retrying after short wait...');
|
|
504
|
+
await page.waitForTimeout(500);
|
|
505
|
+
ariaSnapshot = await getAriaSnapshot(page);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
if (!ariaSnapshot) {
|
|
509
|
+
return res.status(500).json({
|
|
510
|
+
error: 'Failed to get aria snapshot - page may not be ready',
|
|
511
|
+
url: page.url(),
|
|
512
|
+
hint: 'Try waiting longer or check if the page loaded correctly'
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// Convert refs Map to plain object for JSON response
|
|
517
|
+
const refsObj = {};
|
|
518
|
+
for (const [refId, info] of refs) {
|
|
519
|
+
refsObj[refId] = info;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// Both formats now return the YAML aria snapshot (text is more token efficient)
|
|
523
|
+
res.json({
|
|
524
|
+
snapshot: ariaSnapshot,
|
|
525
|
+
refs: refsObj,
|
|
526
|
+
url: page.url(),
|
|
527
|
+
title: await page.title(),
|
|
528
|
+
listItemId,
|
|
529
|
+
format: 'text'
|
|
530
|
+
});
|
|
531
|
+
} catch (err) {
|
|
532
|
+
console.error('Snapshot error:', err);
|
|
533
|
+
res.status(500).json({ error: err.message });
|
|
534
|
+
}
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
// Click element (by ref or selector)
|
|
538
|
+
app.post('/tabs/:tabId/click', async (req, res) => {
|
|
539
|
+
try {
|
|
540
|
+
const { userId, ref, selector } = req.body;
|
|
541
|
+
const session = sessions.get(userId);
|
|
542
|
+
const found = session && findTab(session, req.params.tabId);
|
|
543
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
544
|
+
|
|
545
|
+
const { tabState } = found;
|
|
546
|
+
tabState.toolCalls++;
|
|
547
|
+
|
|
548
|
+
if (ref) {
|
|
549
|
+
// Use ref-based clicking (like OpenClaw)
|
|
550
|
+
const locator = refToLocator(tabState.page, ref, tabState.refs);
|
|
551
|
+
if (!locator) return res.status(400).json({ error: `Unknown ref: ${ref}` });
|
|
552
|
+
await locator.click({ timeout: 10000 });
|
|
553
|
+
} else if (selector) {
|
|
554
|
+
await tabState.page.click(selector, { timeout: 10000 });
|
|
555
|
+
} else {
|
|
556
|
+
return res.status(400).json({ error: 'ref or selector required' });
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
await tabState.page.waitForLoadState('domcontentloaded', { timeout: 10000 }).catch(() => {});
|
|
560
|
+
|
|
561
|
+
// Track if URL changed
|
|
562
|
+
const newUrl = tabState.page.url();
|
|
563
|
+
tabState.visitedUrls.add(newUrl);
|
|
564
|
+
|
|
565
|
+
res.json({ ok: true, url: newUrl });
|
|
566
|
+
} catch (err) {
|
|
567
|
+
console.error('Click error:', err);
|
|
568
|
+
res.status(500).json({ error: err.message });
|
|
569
|
+
}
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
// Type into element (by ref or selector)
|
|
573
|
+
app.post('/tabs/:tabId/type', async (req, res) => {
|
|
574
|
+
try {
|
|
575
|
+
const { userId, ref, selector, text } = req.body;
|
|
576
|
+
const session = sessions.get(userId);
|
|
577
|
+
const found = session && findTab(session, req.params.tabId);
|
|
578
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
579
|
+
|
|
580
|
+
const { tabState } = found;
|
|
581
|
+
tabState.toolCalls++;
|
|
582
|
+
|
|
583
|
+
if (ref) {
|
|
584
|
+
const locator = refToLocator(tabState.page, ref, tabState.refs);
|
|
585
|
+
if (!locator) return res.status(400).json({ error: `Unknown ref: ${ref}` });
|
|
586
|
+
await locator.fill(text, { timeout: 10000 });
|
|
587
|
+
} else if (selector) {
|
|
588
|
+
await tabState.page.fill(selector, text, { timeout: 10000 });
|
|
589
|
+
} else {
|
|
590
|
+
return res.status(400).json({ error: 'ref or selector required' });
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
res.json({ ok: true });
|
|
594
|
+
} catch (err) {
|
|
595
|
+
console.error('Type error:', err);
|
|
596
|
+
res.status(500).json({ error: err.message });
|
|
597
|
+
}
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
// Press key
|
|
601
|
+
app.post('/tabs/:tabId/press', async (req, res) => {
|
|
602
|
+
try {
|
|
603
|
+
const { userId, key } = req.body;
|
|
604
|
+
const session = sessions.get(userId);
|
|
605
|
+
const found = session && findTab(session, req.params.tabId);
|
|
606
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
607
|
+
|
|
608
|
+
const { tabState } = found;
|
|
609
|
+
tabState.toolCalls++;
|
|
610
|
+
await tabState.page.keyboard.press(key);
|
|
611
|
+
res.json({ ok: true });
|
|
612
|
+
} catch (err) {
|
|
613
|
+
console.error('Press error:', err);
|
|
614
|
+
res.status(500).json({ error: err.message });
|
|
615
|
+
}
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
// Scroll down
|
|
619
|
+
app.post('/tabs/:tabId/scroll', async (req, res) => {
|
|
620
|
+
try {
|
|
621
|
+
const { userId, direction = 'down', amount = 500 } = req.body;
|
|
622
|
+
const session = sessions.get(userId);
|
|
623
|
+
const found = session && findTab(session, req.params.tabId);
|
|
624
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
625
|
+
|
|
626
|
+
const { tabState } = found;
|
|
627
|
+
tabState.toolCalls++;
|
|
628
|
+
|
|
629
|
+
const delta = direction === 'up' ? -amount : amount;
|
|
630
|
+
await tabState.page.mouse.wheel(0, delta);
|
|
631
|
+
await tabState.page.waitForTimeout(300); // Let content load
|
|
632
|
+
|
|
633
|
+
res.json({ ok: true });
|
|
634
|
+
} catch (err) {
|
|
635
|
+
console.error('Scroll error:', err);
|
|
636
|
+
res.status(500).json({ error: err.message });
|
|
637
|
+
}
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
// Get all links from page
|
|
641
|
+
app.get('/tabs/:tabId/links', async (req, res) => {
|
|
642
|
+
try {
|
|
643
|
+
const userId = req.query.userId;
|
|
644
|
+
const limit = parseInt(req.query.limit) || 50;
|
|
645
|
+
const offset = parseInt(req.query.offset) || 0;
|
|
646
|
+
const session = sessions.get(userId);
|
|
647
|
+
const found = session && findTab(session, req.params.tabId);
|
|
648
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
649
|
+
|
|
650
|
+
const { tabState } = found;
|
|
651
|
+
tabState.toolCalls++;
|
|
652
|
+
|
|
653
|
+
// Extract all links from page
|
|
654
|
+
const allLinks = await tabState.page.evaluate(() => {
|
|
655
|
+
const links = [];
|
|
656
|
+
document.querySelectorAll('a[href]').forEach(a => {
|
|
657
|
+
const href = a.href;
|
|
658
|
+
const text = a.textContent?.trim().slice(0, 100) || '';
|
|
659
|
+
if (href && href.startsWith('http')) {
|
|
660
|
+
links.push({ url: href, text });
|
|
661
|
+
}
|
|
662
|
+
});
|
|
663
|
+
return links;
|
|
664
|
+
});
|
|
665
|
+
|
|
666
|
+
const total = allLinks.length;
|
|
667
|
+
const paginated = allLinks.slice(offset, offset + limit);
|
|
668
|
+
|
|
669
|
+
res.json({
|
|
670
|
+
links: paginated,
|
|
671
|
+
pagination: {
|
|
672
|
+
total,
|
|
673
|
+
offset,
|
|
674
|
+
limit,
|
|
675
|
+
hasMore: offset + limit < total
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
} catch (err) {
|
|
679
|
+
console.error('Links error:', err);
|
|
680
|
+
res.status(500).json({ error: err.message });
|
|
681
|
+
}
|
|
682
|
+
});
|
|
683
|
+
|
|
684
|
+
// Screenshot (for debugging)
|
|
685
|
+
app.get('/tabs/:tabId/screenshot', async (req, res) => {
|
|
686
|
+
try {
|
|
687
|
+
const userId = req.query.userId;
|
|
688
|
+
const fullPage = req.query.fullPage === 'true';
|
|
689
|
+
const session = sessions.get(userId);
|
|
690
|
+
const found = session && findTab(session, req.params.tabId);
|
|
691
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
692
|
+
|
|
693
|
+
const { tabState } = found;
|
|
694
|
+
const buffer = await tabState.page.screenshot({
|
|
695
|
+
type: 'png',
|
|
696
|
+
fullPage
|
|
697
|
+
});
|
|
698
|
+
res.set('Content-Type', 'image/png');
|
|
699
|
+
res.send(buffer);
|
|
700
|
+
} catch (err) {
|
|
701
|
+
console.error('Screenshot error:', err);
|
|
702
|
+
res.status(500).json({ error: err.message });
|
|
703
|
+
}
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
// Get tab stats (visited URLs, tool calls)
|
|
707
|
+
app.get('/tabs/:tabId/stats', async (req, res) => {
|
|
708
|
+
try {
|
|
709
|
+
const userId = req.query.userId;
|
|
710
|
+
const session = sessions.get(userId);
|
|
711
|
+
const found = session && findTab(session, req.params.tabId);
|
|
712
|
+
if (!found) return res.status(404).json({ error: 'Tab not found' });
|
|
713
|
+
|
|
714
|
+
const { tabState, listItemId } = found;
|
|
715
|
+
res.json({
|
|
716
|
+
tabId: req.params.tabId,
|
|
717
|
+
listItemId,
|
|
718
|
+
url: tabState.page.url(),
|
|
719
|
+
visitedUrls: Array.from(tabState.visitedUrls),
|
|
720
|
+
toolCalls: tabState.toolCalls,
|
|
721
|
+
refsCount: tabState.refs.size
|
|
722
|
+
});
|
|
723
|
+
} catch (err) {
|
|
724
|
+
console.error('Stats error:', err);
|
|
725
|
+
res.status(500).json({ error: err.message });
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
|
|
729
|
+
// Close tab
|
|
730
|
+
app.delete('/tabs/:tabId', async (req, res) => {
|
|
731
|
+
try {
|
|
732
|
+
const { userId } = req.body;
|
|
733
|
+
const session = sessions.get(userId);
|
|
734
|
+
const found = session && findTab(session, req.params.tabId);
|
|
735
|
+
if (found) {
|
|
736
|
+
await found.tabState.page.close();
|
|
737
|
+
found.group.delete(req.params.tabId);
|
|
738
|
+
if (found.group.size === 0) {
|
|
739
|
+
session.tabGroups.delete(found.listItemId);
|
|
740
|
+
}
|
|
741
|
+
console.log(`Tab ${req.params.tabId} closed for user ${userId}`);
|
|
742
|
+
}
|
|
743
|
+
res.json({ ok: true });
|
|
744
|
+
} catch (err) {
|
|
745
|
+
console.error('Close tab error:', err);
|
|
746
|
+
res.status(500).json({ error: err.message });
|
|
747
|
+
}
|
|
748
|
+
});
|
|
749
|
+
|
|
750
|
+
// Close all tabs for a list item (conversation ended)
|
|
751
|
+
app.delete('/tabs/group/:listItemId', async (req, res) => {
|
|
752
|
+
try {
|
|
753
|
+
const { userId } = req.body;
|
|
754
|
+
const session = sessions.get(userId);
|
|
755
|
+
const group = session?.tabGroups.get(req.params.listItemId);
|
|
756
|
+
if (group) {
|
|
757
|
+
for (const [tabId, tabState] of group) {
|
|
758
|
+
await tabState.page.close().catch(() => {});
|
|
759
|
+
}
|
|
760
|
+
session.tabGroups.delete(req.params.listItemId);
|
|
761
|
+
console.log(`Tab group ${req.params.listItemId} closed for user ${userId}`);
|
|
762
|
+
}
|
|
763
|
+
res.json({ ok: true });
|
|
764
|
+
} catch (err) {
|
|
765
|
+
console.error('Close tab group error:', err);
|
|
766
|
+
res.status(500).json({ error: err.message });
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
// Close all tabs for user
|
|
771
|
+
app.delete('/sessions/:userId', async (req, res) => {
|
|
772
|
+
try {
|
|
773
|
+
const userId = req.params.userId;
|
|
774
|
+
const session = sessions.get(userId);
|
|
775
|
+
if (session) {
|
|
776
|
+
await session.context.close();
|
|
777
|
+
sessions.delete(userId);
|
|
778
|
+
console.log(`Session closed for user ${userId}`);
|
|
779
|
+
}
|
|
780
|
+
res.json({ ok: true });
|
|
781
|
+
} catch (err) {
|
|
782
|
+
console.error('Close session error:', err);
|
|
783
|
+
res.status(500).json({ error: err.message });
|
|
784
|
+
}
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
// Cleanup stale sessions
|
|
788
|
+
setInterval(() => {
|
|
789
|
+
const now = Date.now();
|
|
790
|
+
for (const [userId, session] of sessions) {
|
|
791
|
+
if (now - session.lastAccess > SESSION_TIMEOUT_MS) {
|
|
792
|
+
session.context.close().catch(() => {});
|
|
793
|
+
sessions.delete(userId);
|
|
794
|
+
console.log(`Session expired for user ${userId}`);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
}, 60_000);
|
|
798
|
+
|
|
799
|
+
// Graceful shutdown
|
|
800
|
+
process.on('SIGTERM', async () => {
|
|
801
|
+
console.log('Shutting down...');
|
|
802
|
+
for (const [userId, session] of sessions) {
|
|
803
|
+
await session.context.close().catch(() => {});
|
|
804
|
+
}
|
|
805
|
+
if (browser) await browser.close().catch(() => {});
|
|
806
|
+
process.exit(0);
|
|
807
|
+
});
|
|
808
|
+
|
|
809
|
+
const PORT = process.env.PORT || 3000;
|
|
810
|
+
app.listen(PORT, () => {
|
|
811
|
+
console.log(`🌐 jo-browser [Chrome/Playwright] listening on port ${PORT}`);
|
|
812
|
+
});
|