neoagent 3.0.1-beta.8 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -10
- package/README.md +12 -3
- package/docs/automation.md +37 -0
- package/docs/benchmarking.md +102 -0
- package/docs/billing.md +34 -8
- package/docs/configuration.md +11 -2
- package/docs/getting-started.md +10 -8
- package/docs/operations.md +1 -1
- package/flutter_app/lib/main.dart +3 -0
- package/flutter_app/lib/main_account_settings.dart +138 -0
- package/flutter_app/lib/main_app_shell.dart +38 -13
- package/flutter_app/lib/main_billing.dart +1465 -0
- package/flutter_app/lib/main_chat.dart +1612 -214
- package/flutter_app/lib/main_controller.dart +398 -26
- package/flutter_app/lib/main_devices.dart +293 -207
- package/flutter_app/lib/main_models.dart +142 -0
- package/flutter_app/lib/main_navigation.dart +19 -1
- package/flutter_app/lib/main_operations.dart +288 -9
- package/flutter_app/lib/main_settings.dart +510 -300
- package/flutter_app/lib/main_shared.dart +2 -0
- package/flutter_app/lib/main_timeline.dart +1378 -0
- package/flutter_app/lib/src/backend_client.dart +149 -19
- package/flutter_app/lib/src/desktop_companion_actions.dart +132 -21
- package/flutter_app/lib/src/desktop_companion_io.dart +65 -1
- package/flutter_app/lib/src/desktop_companion_stub.dart +12 -0
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +2 -0
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +125 -0
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +30 -0
- package/flutter_app/lib/src/desktop_passive_history.dart +332 -0
- package/flutter_app/lib/src/recording_bridge_io.dart +80 -72
- package/flutter_app/lib/src/recording_bridge_web.dart +127 -114
- package/flutter_app/lib/src/recording_chunk_queue.dart +149 -0
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +182 -0
- package/flutter_app/lib/src/recording_payloads.dart +9 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +25 -0
- package/flutter_app/windows/runner/flutter_window.cpp +75 -0
- package/landing/index.html +3 -3
- package/lib/manager.js +184 -66
- package/package.json +4 -1
- package/server/admin/access.js +12 -7
- package/server/admin/admin.css +78 -0
- package/server/admin/admin.js +436 -10
- package/server/admin/billing.js +165 -42
- package/server/admin/index.html +80 -5
- package/server/admin/users.js +15 -15
- package/server/db/database.js +125 -20
- package/server/http/routes.js +1 -0
- package/server/http/static.js +4 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/canvaskit/wimp.js.symbols +8475 -8467
- package/server/public/canvaskit/wimp.wasm +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/main.dart.js +91077 -87037
- package/server/routes/account.js +53 -0
- package/server/routes/admin.js +345 -64
- package/server/routes/agents.js +203 -21
- package/server/routes/billing.js +5 -0
- package/server/routes/browser.js +8 -1
- package/server/routes/recordings.js +96 -6
- package/server/routes/screenHistory.js +140 -2
- package/server/routes/timeline.js +43 -0
- package/server/services/account/erasure.js +263 -0
- package/server/services/ai/hooks.js +4 -1
- package/server/services/ai/loop/agent_engine_core.js +8 -1
- package/server/services/ai/loop/blank_recovery.js +36 -0
- package/server/services/ai/loop/conversation_loop.js +166 -33
- package/server/services/ai/messagingFallback.js +22 -0
- package/server/services/ai/rate_limits.js +28 -5
- package/server/services/ai/systemPrompt.js +6 -5
- package/server/services/ai/taskAnalysis.js +2 -0
- package/server/services/ai/toolEvidence.js +15 -0
- package/server/services/ai/toolResult.js +40 -0
- package/server/services/ai/tools.js +163 -4
- package/server/services/android/controller.js +6 -2
- package/server/services/billing/plans.js +2 -1
- package/server/services/browser/anti_detection.js +192 -0
- package/server/services/browser/controller.js +180 -54
- package/server/services/desktop/auth.js +3 -0
- package/server/services/desktop/registry.js +50 -2
- package/server/services/integrations/google/calendar.js +22 -14
- package/server/services/manager.js +12 -42
- package/server/services/memory/ingestion_chunking.js +268 -0
- package/server/services/messaging/telnyx.js +9 -8
- package/server/services/recordings/manager.js +60 -27
- package/server/services/runtime/backends/local-vm.js +40 -22
- package/server/services/runtime/docker-vm-manager.js +157 -266
- package/server/services/runtime/guest_bootstrap.js +17 -5
- package/server/services/runtime/guest_image.js +188 -0
- package/server/services/runtime/manager.js +0 -1
- package/server/services/runtime/validation.js +3 -8
- package/server/services/social_video/service.js +60 -10
- package/server/services/tasks/runtime.js +234 -9
- package/server/services/tasks/schedule_utils.js +5 -5
- package/server/services/tasks/task_repository.js +13 -0
- package/server/services/timeline/service.js +558 -0
- package/server/services/wearable/gateway.js +1 -1
- package/server/services/websocket.js +21 -3
- package/server/services/desktop/screenRecorder.js +0 -292
- package/server/services/desktop/screen_recorder_support.js +0 -46
|
@@ -2,6 +2,14 @@ const path = require('path');
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const { spawn } = require('child_process');
|
|
4
4
|
const { DATA_DIR } = require('../../../runtime/paths');
|
|
5
|
+
const {
|
|
6
|
+
chooseBrowserIdentity,
|
|
7
|
+
detectBotChallenge,
|
|
8
|
+
generateHumanMousePath,
|
|
9
|
+
normalizeChallengeRetry,
|
|
10
|
+
normalizeReferrerMode,
|
|
11
|
+
rand,
|
|
12
|
+
} = require('./anti_detection');
|
|
5
13
|
|
|
6
14
|
const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
|
|
7
15
|
if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
|
|
@@ -9,21 +17,6 @@ const BROWSER_PROFILE_ROOT = path.join(DATA_DIR, 'browser-profiles');
|
|
|
9
17
|
if (!fs.existsSync(BROWSER_PROFILE_ROOT)) fs.mkdirSync(BROWSER_PROFILE_ROOT, { recursive: true });
|
|
10
18
|
const BROWSER_READY_MARKER = '/var/lib/neoagent/browser-runtime-ready';
|
|
11
19
|
|
|
12
|
-
const USER_AGENTS = [
|
|
13
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
|
14
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
|
15
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
|
|
16
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
|
|
17
|
-
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
|
18
|
-
];
|
|
19
|
-
|
|
20
|
-
const VIEWPORTS = [
|
|
21
|
-
{ width: 1280, height: 800 },
|
|
22
|
-
{ width: 1366, height: 768 },
|
|
23
|
-
{ width: 1440, height: 900 },
|
|
24
|
-
{ width: 1920, height: 1080 },
|
|
25
|
-
];
|
|
26
|
-
|
|
27
20
|
// Injected into every page in the cloud VM browser context to deny local device access.
|
|
28
21
|
const DEVICE_DENY_SCRIPT = `(() => {
|
|
29
22
|
const denied = () => Promise.reject(Object.assign(
|
|
@@ -166,10 +159,6 @@ function installPlaywrightBrowserBinary(browserName) {
|
|
|
166
159
|
});
|
|
167
160
|
}
|
|
168
161
|
|
|
169
|
-
function rand(min, max) {
|
|
170
|
-
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
171
|
-
}
|
|
172
|
-
|
|
173
162
|
function sleep(ms) {
|
|
174
163
|
return new Promise(r => setTimeout(r, ms));
|
|
175
164
|
}
|
|
@@ -241,10 +230,15 @@ class BrowserController {
|
|
|
241
230
|
this.launchPromise = null;
|
|
242
231
|
this.browserBinaryInstallPromise = null;
|
|
243
232
|
this.headless = false;
|
|
244
|
-
this._viewport = VIEWPORTS[0];
|
|
245
|
-
this._userAgent = USER_AGENTS[0];
|
|
246
233
|
this.profileDir = path.join(BROWSER_PROFILE_ROOT, this.userId || 'default');
|
|
247
234
|
if (!fs.existsSync(this.profileDir)) fs.mkdirSync(this.profileDir, { recursive: true });
|
|
235
|
+
this._identity = chooseBrowserIdentity(this.userId || this.profileDir);
|
|
236
|
+
this._viewport = this._identity.viewport;
|
|
237
|
+
this._userAgent = this._identity.userAgent;
|
|
238
|
+
this._mousePosition = {
|
|
239
|
+
x: Math.round(this._viewport.width / 2),
|
|
240
|
+
y: Math.round(this._viewport.height / 2),
|
|
241
|
+
};
|
|
248
242
|
}
|
|
249
243
|
|
|
250
244
|
async setHeadless(val) {
|
|
@@ -260,6 +254,7 @@ class BrowserController {
|
|
|
260
254
|
async _applyStealthToPage(page) {
|
|
261
255
|
const ua = this._userAgent;
|
|
262
256
|
const vp = this._viewport;
|
|
257
|
+
const identity = this._identity || chooseBrowserIdentity(this.userId || this.profileDir);
|
|
263
258
|
|
|
264
259
|
if (typeof page.setUserAgent === 'function') {
|
|
265
260
|
await page.setUserAgent(ua);
|
|
@@ -281,9 +276,9 @@ class BrowserController {
|
|
|
281
276
|
|
|
282
277
|
// Realistic language/platform
|
|
283
278
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
|
284
|
-
Object.defineProperty(navigator, 'platform', { get: () =>
|
|
285
|
-
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => ${
|
|
286
|
-
Object.defineProperty(navigator, 'deviceMemory', { get: () => ${
|
|
279
|
+
Object.defineProperty(navigator, 'platform', { get: () => ${JSON.stringify(identity.platform)} });
|
|
280
|
+
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => ${Number(identity.hardwareConcurrency) || 8} });
|
|
281
|
+
Object.defineProperty(navigator, 'deviceMemory', { get: () => ${Number(identity.deviceMemory) || 8} });
|
|
287
282
|
|
|
288
283
|
// Make it look like a real Chrome install
|
|
289
284
|
window.chrome = {
|
|
@@ -323,9 +318,9 @@ class BrowserController {
|
|
|
323
318
|
apply: function(target, ctx, args) {
|
|
324
319
|
const param = args[0];
|
|
325
320
|
// UNMASKED_VENDOR_WEBGL
|
|
326
|
-
if (param === 37445) return
|
|
321
|
+
if (param === 37445) return ${JSON.stringify(identity.webglVendor)};
|
|
327
322
|
// UNMASKED_RENDERER_WEBGL
|
|
328
|
-
if (param === 37446) return
|
|
323
|
+
if (param === 37446) return ${JSON.stringify(identity.webglRenderer)};
|
|
329
324
|
return Reflect.apply(target, ctx, args);
|
|
330
325
|
}
|
|
331
326
|
};
|
|
@@ -388,8 +383,13 @@ class BrowserController {
|
|
|
388
383
|
}
|
|
389
384
|
await this.ensureVirtualDisplay();
|
|
390
385
|
|
|
391
|
-
this.
|
|
392
|
-
this.
|
|
386
|
+
this._identity = chooseBrowserIdentity(this.userId || this.profileDir);
|
|
387
|
+
this._userAgent = this._identity.userAgent;
|
|
388
|
+
this._viewport = this._identity.viewport;
|
|
389
|
+
this._mousePosition = {
|
|
390
|
+
x: Math.round(this._viewport.width / 2),
|
|
391
|
+
y: Math.round(this._viewport.height / 2),
|
|
392
|
+
};
|
|
393
393
|
|
|
394
394
|
let executablePath = resolveBrowserExecutablePath();
|
|
395
395
|
if (!executablePath) {
|
|
@@ -414,18 +414,28 @@ class BrowserController {
|
|
|
414
414
|
};
|
|
415
415
|
|
|
416
416
|
const launchArgs = [
|
|
417
|
+
'--start-maximized',
|
|
418
|
+
'--remote-allow-origins=*',
|
|
417
419
|
'--no-sandbox',
|
|
418
420
|
'--disable-setuid-sandbox',
|
|
419
421
|
'--disable-dev-shm-usage',
|
|
422
|
+
'--no-service-autorun',
|
|
420
423
|
'--disable-crash-reporter',
|
|
424
|
+
'--disable-breakpad',
|
|
421
425
|
'--disable-background-networking',
|
|
422
426
|
'--disable-component-update',
|
|
423
427
|
'--disable-blink-features=AutomationControlled',
|
|
424
428
|
'--disable-infobars',
|
|
429
|
+
'--disable-session-crashed-bubble',
|
|
430
|
+
'--disable-search-engine-choice-screen',
|
|
425
431
|
'--no-first-run',
|
|
426
432
|
'--no-default-browser-check',
|
|
433
|
+
'--homepage=about:blank',
|
|
434
|
+
'--no-pings',
|
|
435
|
+
'--password-store=basic',
|
|
427
436
|
'--disable-gpu',
|
|
428
437
|
'--lang=en-US,en',
|
|
438
|
+
`--user-agent=${this._userAgent}`,
|
|
429
439
|
`--window-size=${this._viewport.width},${this._viewport.height}`,
|
|
430
440
|
// Cloud security: disable hardware device APIs at the Chromium level
|
|
431
441
|
'--disable-features=WebBluetooth,WebUSB,WebSerial,WebOTP,DirectSockets',
|
|
@@ -440,6 +450,8 @@ class BrowserController {
|
|
|
440
450
|
env: launchEnv,
|
|
441
451
|
args: launchArgs,
|
|
442
452
|
viewport: this._viewport,
|
|
453
|
+
userAgent: this._userAgent,
|
|
454
|
+
locale: 'en-US',
|
|
443
455
|
ignoreHTTPSErrors: false,
|
|
444
456
|
timeout: 120000,
|
|
445
457
|
});
|
|
@@ -535,13 +547,48 @@ class BrowserController {
|
|
|
535
547
|
return page.screenshot(screenshotOptions);
|
|
536
548
|
}
|
|
537
549
|
|
|
550
|
+
async _navigatePage(page, url, options = {}) {
|
|
551
|
+
const referrerMode = normalizeReferrerMode(options.referrerMode);
|
|
552
|
+
const waitUntil = normalizeWaitUntil(options.waitUntil);
|
|
553
|
+
if (referrerMode === 'current' && page.url() && page.url() !== 'about:blank') {
|
|
554
|
+
const previousUrl = page.url();
|
|
555
|
+
await page.evaluate((nextUrl) => { window.location.href = nextUrl; }, url);
|
|
556
|
+
await page.waitForFunction((oldUrl) => window.location.href !== oldUrl, previousUrl, { timeout: 10000 }).catch(() => {});
|
|
557
|
+
await page.waitForLoadState(waitUntil, { timeout: 30000 }).catch(() => {});
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
const gotoOptions = {
|
|
562
|
+
waitUntil,
|
|
563
|
+
timeout: 30000,
|
|
564
|
+
};
|
|
565
|
+
if (referrerMode === 'google') {
|
|
566
|
+
gotoOptions.referer = 'https://www.google.com/';
|
|
567
|
+
} else if (referrerMode === 'current' && page.url() && page.url() !== 'about:blank') {
|
|
568
|
+
gotoOptions.referer = page.url();
|
|
569
|
+
}
|
|
570
|
+
return page.goto(url, gotoOptions);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
async _getBotDetection(page, rawHtml, pageContent) {
|
|
574
|
+
const title = await page.title().catch(() => '');
|
|
575
|
+
return detectBotChallenge({
|
|
576
|
+
title,
|
|
577
|
+
url: page.url(),
|
|
578
|
+
html: rawHtml,
|
|
579
|
+
pageContent,
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
|
|
538
583
|
async navigate(url, options = {}) {
|
|
539
584
|
const page = await this.ensurePage();
|
|
540
585
|
|
|
541
586
|
try {
|
|
542
|
-
const
|
|
543
|
-
|
|
544
|
-
|
|
587
|
+
const requestedReferrerMode = normalizeReferrerMode(options.referrerMode);
|
|
588
|
+
let activeReferrerMode = requestedReferrerMode;
|
|
589
|
+
let response = await this._navigatePage(page, url, {
|
|
590
|
+
...options,
|
|
591
|
+
referrerMode: activeReferrerMode,
|
|
545
592
|
});
|
|
546
593
|
|
|
547
594
|
if (options.waitFor) {
|
|
@@ -549,7 +596,34 @@ class BrowserController {
|
|
|
549
596
|
}
|
|
550
597
|
|
|
551
598
|
// Simulate human reading delay
|
|
552
|
-
await sleep(rand(
|
|
599
|
+
await sleep(rand(700, 1800));
|
|
600
|
+
|
|
601
|
+
let rawHtml = await page.content();
|
|
602
|
+
const { extractForLLM } = require('./contentExtractor');
|
|
603
|
+
let extraction = extractForLLM(rawHtml, { url: page.url() });
|
|
604
|
+
let botDetection = await this._getBotDetection(page, rawHtml, extraction.markdown);
|
|
605
|
+
let challengeRetried = false;
|
|
606
|
+
|
|
607
|
+
if (
|
|
608
|
+
botDetection.detected
|
|
609
|
+
&& normalizeChallengeRetry(options.challengeRetry)
|
|
610
|
+
&& requestedReferrerMode === 'direct'
|
|
611
|
+
) {
|
|
612
|
+
challengeRetried = true;
|
|
613
|
+
activeReferrerMode = 'google';
|
|
614
|
+
await sleep(rand(1200, 2600));
|
|
615
|
+
response = await this._navigatePage(page, url, {
|
|
616
|
+
...options,
|
|
617
|
+
referrerMode: activeReferrerMode,
|
|
618
|
+
});
|
|
619
|
+
if (options.waitFor) {
|
|
620
|
+
await page.waitForSelector(options.waitFor, { timeout: 10000 }).catch(() => { });
|
|
621
|
+
}
|
|
622
|
+
await sleep(rand(900, 2200));
|
|
623
|
+
rawHtml = await page.content();
|
|
624
|
+
extraction = extractForLLM(rawHtml, { url: page.url() });
|
|
625
|
+
botDetection = await this._getBotDetection(page, rawHtml, extraction.markdown);
|
|
626
|
+
}
|
|
553
627
|
|
|
554
628
|
const title = await page.title();
|
|
555
629
|
const currentUrl = page.url();
|
|
@@ -559,15 +633,14 @@ class BrowserController {
|
|
|
559
633
|
screenshot = await this.takeScreenshot({ fullPage: options.fullPage });
|
|
560
634
|
}
|
|
561
635
|
|
|
562
|
-
const rawHtml = await page.content();
|
|
563
|
-
const { extractForLLM } = require('./contentExtractor');
|
|
564
|
-
const extraction = extractForLLM(rawHtml, { url: currentUrl });
|
|
565
|
-
|
|
566
636
|
return {
|
|
567
637
|
title,
|
|
568
638
|
url: currentUrl,
|
|
569
639
|
status: response?.status() || 0,
|
|
570
640
|
pageContent: extraction.markdown,
|
|
641
|
+
botDetection,
|
|
642
|
+
referrerMode: activeReferrerMode,
|
|
643
|
+
challengeRetried,
|
|
571
644
|
screenshotPath: screenshot?.screenshotPath || null,
|
|
572
645
|
artifactId: screenshot?.artifactId || null,
|
|
573
646
|
fullPath: screenshot?.fullPath || null
|
|
@@ -578,6 +651,7 @@ class BrowserController {
|
|
|
578
651
|
return {
|
|
579
652
|
error: err.message,
|
|
580
653
|
url,
|
|
654
|
+
botDetection: { detected: false, provider: null },
|
|
581
655
|
screenshotPath: screenshot?.screenshotPath || null,
|
|
582
656
|
artifactId: screenshot?.artifactId || null,
|
|
583
657
|
fullPath: screenshot?.fullPath || null
|
|
@@ -585,6 +659,36 @@ class BrowserController {
|
|
|
585
659
|
}
|
|
586
660
|
}
|
|
587
661
|
|
|
662
|
+
async _moveMouseTo(page, x, y, options = {}) {
|
|
663
|
+
const target = {
|
|
664
|
+
x: Math.max(0, Math.min(this._viewport.width, Math.round(Number(x) || 0))),
|
|
665
|
+
y: Math.max(0, Math.min(this._viewport.height, Math.round(Number(y) || 0))),
|
|
666
|
+
};
|
|
667
|
+
const path = generateHumanMousePath(this._mousePosition, target, this._viewport);
|
|
668
|
+
for (const point of path) {
|
|
669
|
+
await page.mouse.move(point.x, point.y);
|
|
670
|
+
if (!options.fast) {
|
|
671
|
+
await sleep(rand(2, 10));
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
this._mousePosition = target;
|
|
675
|
+
return target;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
async _pointForElement(element) {
|
|
679
|
+
await element.scrollIntoViewIfNeeded?.().catch(() => {});
|
|
680
|
+
const box = await element.boundingBox();
|
|
681
|
+
if (!box || box.width <= 0 || box.height <= 0) {
|
|
682
|
+
return null;
|
|
683
|
+
}
|
|
684
|
+
const xRatio = rand(25, 75) / 100;
|
|
685
|
+
const yRatio = rand(30, 70) / 100;
|
|
686
|
+
return {
|
|
687
|
+
x: box.x + box.width * xRatio,
|
|
688
|
+
y: box.y + box.height * yRatio,
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
|
|
588
692
|
async click(selector, text, screenshot = true) {
|
|
589
693
|
const page = await this.ensurePage();
|
|
590
694
|
|
|
@@ -608,10 +712,14 @@ class BrowserController {
|
|
|
608
712
|
return { error: 'Either selector or text required' };
|
|
609
713
|
}
|
|
610
714
|
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
await
|
|
715
|
+
const point = await this._pointForElement(target);
|
|
716
|
+
if (!point) return { error: 'Element has no visible clickable area' };
|
|
717
|
+
|
|
718
|
+
await this._moveMouseTo(page, point.x, point.y);
|
|
719
|
+
await sleep(rand(170, 320));
|
|
720
|
+
await page.mouse.down();
|
|
721
|
+
await sleep(rand(80, 260));
|
|
722
|
+
await page.mouse.up();
|
|
615
723
|
|
|
616
724
|
await sleep(rand(800, 1800));
|
|
617
725
|
|
|
@@ -637,10 +745,10 @@ class BrowserController {
|
|
|
637
745
|
try {
|
|
638
746
|
const px = Math.max(0, Math.round(Number(x) || 0));
|
|
639
747
|
const py = Math.max(0, Math.round(Number(y) || 0));
|
|
640
|
-
await
|
|
641
|
-
await sleep(rand(
|
|
748
|
+
await this._moveMouseTo(page, px, py);
|
|
749
|
+
await sleep(rand(90, 220));
|
|
642
750
|
await page.mouse.down();
|
|
643
|
-
await sleep(rand(
|
|
751
|
+
await sleep(rand(70, 240));
|
|
644
752
|
await page.mouse.up();
|
|
645
753
|
await sleep(rand(500, 1200));
|
|
646
754
|
|
|
@@ -667,7 +775,7 @@ class BrowserController {
|
|
|
667
775
|
try {
|
|
668
776
|
const px = Math.max(0, Math.round(Number(x) || 0));
|
|
669
777
|
const py = Math.max(0, Math.round(Number(y) || 0));
|
|
670
|
-
await
|
|
778
|
+
await this._moveMouseTo(page, px, py, { fast: Number(options.steps) <= 1 });
|
|
671
779
|
return {
|
|
672
780
|
success: true,
|
|
673
781
|
x: px,
|
|
@@ -684,11 +792,19 @@ class BrowserController {
|
|
|
684
792
|
const page = await this.ensurePage();
|
|
685
793
|
|
|
686
794
|
try {
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
795
|
+
const x = Math.max(10, Math.min(this._viewport.width - 10, this._mousePosition.x + rand(-80, 80)));
|
|
796
|
+
const y = Math.max(10, Math.min(this._viewport.height - 10, this._mousePosition.y + rand(-80, 80)));
|
|
797
|
+
await this._moveMouseTo(page, x, y);
|
|
798
|
+
const totalX = Math.round(Number(deltaX) || 0);
|
|
799
|
+
const totalY = Math.round(Number(deltaY) || 0);
|
|
800
|
+
const chunks = Math.max(1, Math.min(6, Math.ceil(Math.max(Math.abs(totalX), Math.abs(totalY)) / 450)));
|
|
801
|
+
for (let i = 0; i < chunks; i += 1) {
|
|
802
|
+
await page.mouse.wheel({
|
|
803
|
+
deltaX: Math.round(totalX / chunks),
|
|
804
|
+
deltaY: Math.round(totalY / chunks),
|
|
805
|
+
});
|
|
806
|
+
await sleep(rand(120, 360));
|
|
807
|
+
}
|
|
692
808
|
|
|
693
809
|
let screenshotResult = null;
|
|
694
810
|
if (screenshot) screenshotResult = await this.takeScreenshot();
|
|
@@ -711,12 +827,20 @@ class BrowserController {
|
|
|
711
827
|
|
|
712
828
|
try {
|
|
713
829
|
if (options.clear !== false) {
|
|
714
|
-
await page
|
|
830
|
+
const element = await page.$(selector);
|
|
831
|
+
if (element) {
|
|
832
|
+
const point = await this._pointForElement(element);
|
|
833
|
+
if (point) {
|
|
834
|
+
await this._moveMouseTo(page, point.x, point.y);
|
|
835
|
+
await sleep(rand(120, 260));
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
await page.click(selector, { clickCount: 3, delay: rand(40, 120) });
|
|
715
839
|
await page.keyboard.press('Backspace');
|
|
716
840
|
}
|
|
717
841
|
|
|
718
842
|
for (const char of text) {
|
|
719
|
-
await page.type(selector, char, { delay: rand(
|
|
843
|
+
await page.type(selector, char, { delay: rand(45, 180) });
|
|
720
844
|
}
|
|
721
845
|
|
|
722
846
|
if (options.pressEnter) {
|
|
@@ -744,7 +868,7 @@ class BrowserController {
|
|
|
744
868
|
|
|
745
869
|
try {
|
|
746
870
|
for (const char of String(text || '')) {
|
|
747
|
-
await page.keyboard.type(char, { delay: rand(
|
|
871
|
+
await page.keyboard.type(char, { delay: rand(45, 160) });
|
|
748
872
|
}
|
|
749
873
|
|
|
750
874
|
if (options.pressEnter) {
|
|
@@ -797,6 +921,8 @@ class BrowserController {
|
|
|
797
921
|
const page = await this.ensurePage();
|
|
798
922
|
|
|
799
923
|
try {
|
|
924
|
+
const rawHtml = await page.content().catch(() => '');
|
|
925
|
+
const botDetection = await this._getBotDetection(page, rawHtml, '').catch(() => ({ detected: false, provider: null }));
|
|
800
926
|
if (all) {
|
|
801
927
|
const results = await page.$$eval(selector || 'body', (elements, attr) => {
|
|
802
928
|
return elements.map(el => {
|
|
@@ -806,7 +932,7 @@ class BrowserController {
|
|
|
806
932
|
return el.innerText || '';
|
|
807
933
|
});
|
|
808
934
|
}, attribute);
|
|
809
|
-
return { results: results.slice(0, 100) };
|
|
935
|
+
return { results: results.slice(0, 100), botDetection };
|
|
810
936
|
}
|
|
811
937
|
|
|
812
938
|
const result = await page.$eval(selector || 'body', (el, attr) => {
|
|
@@ -816,7 +942,7 @@ class BrowserController {
|
|
|
816
942
|
return el.innerText || '';
|
|
817
943
|
}, attribute);
|
|
818
944
|
|
|
819
|
-
return { result: typeof result === 'string' ? result.slice(0, 50000) : result };
|
|
945
|
+
return { result: typeof result === 'string' ? result.slice(0, 50000) : result, botDetection };
|
|
820
946
|
} catch (err) {
|
|
821
947
|
return { error: err.message };
|
|
822
948
|
}
|
|
@@ -29,6 +29,9 @@ function normalizeDesktopHello(message) {
|
|
|
29
29
|
: {},
|
|
30
30
|
displays: Array.isArray(device.displays) ? device.displays : [],
|
|
31
31
|
activeDisplayId: String(device.activeDisplayId || '').trim(),
|
|
32
|
+
passiveHistoryEnabled: device.passiveHistoryEnabled === true,
|
|
33
|
+
passiveHistoryLastUploadedAt: String(device.passiveHistoryLastUploadedAt || '').trim(),
|
|
34
|
+
passiveHistoryLastError: String(device.passiveHistoryLastError || '').trim(),
|
|
32
35
|
metadata: device.metadata && typeof device.metadata === 'object'
|
|
33
36
|
? device.metadata
|
|
34
37
|
: {},
|
|
@@ -101,8 +101,9 @@ class DesktopCompanionRegistry {
|
|
|
101
101
|
`INSERT INTO desktop_companion_devices (
|
|
102
102
|
id, user_id, device_id, activation_id, label, hostname, platform, platform_version, app_version,
|
|
103
103
|
companion_enabled, paused, status, display_count, active_display_id, permissions_json, capabilities_json,
|
|
104
|
-
metadata_json, session_id, last_connected_at, last_seen_at,
|
|
105
|
-
|
|
104
|
+
metadata_json, session_id, last_connected_at, last_seen_at, passive_history_enabled, passive_history_last_uploaded_at,
|
|
105
|
+
passive_history_last_error, revoked_at, created_at, updated_at
|
|
106
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'online', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
106
107
|
ON CONFLICT(user_id, device_id) DO UPDATE SET
|
|
107
108
|
activation_id = excluded.activation_id,
|
|
108
109
|
revoked_at = NULL,
|
|
@@ -122,6 +123,9 @@ class DesktopCompanionRegistry {
|
|
|
122
123
|
session_id = excluded.session_id,
|
|
123
124
|
last_connected_at = excluded.last_connected_at,
|
|
124
125
|
last_seen_at = excluded.last_seen_at,
|
|
126
|
+
passive_history_enabled = excluded.passive_history_enabled,
|
|
127
|
+
passive_history_last_uploaded_at = excluded.passive_history_last_uploaded_at,
|
|
128
|
+
passive_history_last_error = excluded.passive_history_last_error,
|
|
125
129
|
updated_at = excluded.updated_at`
|
|
126
130
|
).run(
|
|
127
131
|
rowId,
|
|
@@ -143,6 +147,9 @@ class DesktopCompanionRegistry {
|
|
|
143
147
|
sessionId,
|
|
144
148
|
now,
|
|
145
149
|
now,
|
|
150
|
+
hello.passiveHistoryEnabled ? 1 : 0,
|
|
151
|
+
hello.passiveHistoryLastUploadedAt || null,
|
|
152
|
+
hello.passiveHistoryLastError || null,
|
|
146
153
|
existing?.created_at || now,
|
|
147
154
|
now,
|
|
148
155
|
);
|
|
@@ -190,6 +197,9 @@ class DesktopCompanionRegistry {
|
|
|
190
197
|
sessionId: row.session_id || null,
|
|
191
198
|
lastConnectedAt: row.last_connected_at || null,
|
|
192
199
|
lastSeenAt: row.last_seen_at || null,
|
|
200
|
+
passiveHistoryEnabled: row.passive_history_enabled === 1,
|
|
201
|
+
passiveHistoryLastUploadedAt: row.passive_history_last_uploaded_at || null,
|
|
202
|
+
passiveHistoryLastError: row.passive_history_last_error || null,
|
|
193
203
|
revokedAt: row.revoked_at || null,
|
|
194
204
|
createdAt: row.created_at,
|
|
195
205
|
updatedAt: row.updated_at || row.created_at,
|
|
@@ -278,6 +288,9 @@ class DesktopCompanionRegistry {
|
|
|
278
288
|
permissions_json = ?,
|
|
279
289
|
capabilities_json = ?,
|
|
280
290
|
metadata_json = ?,
|
|
291
|
+
passive_history_enabled = ?,
|
|
292
|
+
passive_history_last_uploaded_at = ?,
|
|
293
|
+
passive_history_last_error = ?,
|
|
281
294
|
last_seen_at = datetime('now'),
|
|
282
295
|
updated_at = datetime('now')
|
|
283
296
|
WHERE user_id = ? AND device_id = ?`
|
|
@@ -290,6 +303,13 @@ class DesktopCompanionRegistry {
|
|
|
290
303
|
safeJson(patch.permissions || existing.permissions),
|
|
291
304
|
safeJson(patch.capabilities || existing.capabilities),
|
|
292
305
|
safeJson(metadata),
|
|
306
|
+
patch.passiveHistoryEnabled == null
|
|
307
|
+
? (existing.passiveHistoryEnabled ? 1 : 0)
|
|
308
|
+
: (patch.passiveHistoryEnabled === true ? 1 : 0),
|
|
309
|
+
patch.passiveHistoryLastUploadedAt || existing.passiveHistoryLastUploadedAt || null,
|
|
310
|
+
patch.passiveHistoryLastError == null
|
|
311
|
+
? (existing.passiveHistoryLastError || null)
|
|
312
|
+
: (patch.passiveHistoryLastError || null),
|
|
293
313
|
userId,
|
|
294
314
|
deviceId,
|
|
295
315
|
);
|
|
@@ -516,6 +536,34 @@ class DesktopCompanionRegistry {
|
|
|
516
536
|
};
|
|
517
537
|
}
|
|
518
538
|
|
|
539
|
+
updatePassiveHistoryState(userId, deviceId, patch = {}) {
|
|
540
|
+
const normalizedDeviceId = String(deviceId || '').trim();
|
|
541
|
+
if (!normalizedDeviceId) {
|
|
542
|
+
throw new DesktopCompanionUnavailableError();
|
|
543
|
+
}
|
|
544
|
+
const existing = this.db.prepare(
|
|
545
|
+
`SELECT device_id FROM desktop_companion_devices WHERE user_id = ? AND device_id = ?`
|
|
546
|
+
).get(userId, normalizedDeviceId);
|
|
547
|
+
if (!existing) {
|
|
548
|
+
throw new DesktopCompanionUnavailableError();
|
|
549
|
+
}
|
|
550
|
+
this.db.prepare(
|
|
551
|
+
`UPDATE desktop_companion_devices
|
|
552
|
+
SET passive_history_enabled = ?,
|
|
553
|
+
passive_history_last_uploaded_at = COALESCE(?, passive_history_last_uploaded_at),
|
|
554
|
+
passive_history_last_error = ?,
|
|
555
|
+
updated_at = datetime('now')
|
|
556
|
+
WHERE user_id = ? AND device_id = ?`
|
|
557
|
+
).run(
|
|
558
|
+
patch.enabled === true ? 1 : 0,
|
|
559
|
+
patch.lastUploadedAt || null,
|
|
560
|
+
patch.lastError || null,
|
|
561
|
+
userId,
|
|
562
|
+
normalizedDeviceId,
|
|
563
|
+
);
|
|
564
|
+
return this.getDeviceRecordByDeviceId(userId, normalizedDeviceId);
|
|
565
|
+
}
|
|
566
|
+
|
|
519
567
|
closeAll() {
|
|
520
568
|
for (const userMap of this.connectionsByUser.values()) {
|
|
521
569
|
for (const connection of userMap.values()) {
|
|
@@ -7,7 +7,7 @@ const calendarToolDefinitions = [
|
|
|
7
7
|
{
|
|
8
8
|
name: 'google_workspace_calendar_list_events',
|
|
9
9
|
access: 'read',
|
|
10
|
-
description: 'List or search Google Calendar events for the connected account. Results include all-day/multi-day events (birthdays, anniversaries, full-day markers) that overlap the window
|
|
10
|
+
description: 'List or search Google Calendar events for the connected account. Results include all-day/multi-day events (birthdays, anniversaries, full-day markers) that overlap the window. Each event has an allDay flag, timedCount/allDayCount summarize the mix, and timedEvents/nextTimedEvent expose the real timed appointments directly. For "what is scheduled / starting soon", decide from that single result instead of re-querying nearby windows to filter out all-day entries.',
|
|
11
11
|
parameters: {
|
|
12
12
|
type: 'object',
|
|
13
13
|
properties: {
|
|
@@ -167,6 +167,25 @@ function summarizeEvent(event) {
|
|
|
167
167
|
};
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
function summarizeListedEvents(items) {
|
|
171
|
+
const events = Array.isArray(items) ? items.map(summarizeEvent) : [];
|
|
172
|
+
const timedEvents = events.filter((event) => !event.allDay);
|
|
173
|
+
const allDayEvents = events.filter((event) => event.allDay);
|
|
174
|
+
return {
|
|
175
|
+
count: events.length,
|
|
176
|
+
timedCount: timedEvents.length,
|
|
177
|
+
allDayCount: allDayEvents.length,
|
|
178
|
+
hasTimedEvents: timedEvents.length > 0,
|
|
179
|
+
hasOnlyAllDayEvents: timedEvents.length === 0 && allDayEvents.length > 0,
|
|
180
|
+
nextTimedEvent: timedEvents[0] || null,
|
|
181
|
+
timedEvents,
|
|
182
|
+
allDayEvents,
|
|
183
|
+
// Keep the legacy merged list for existing callers, but put timed items first so
|
|
184
|
+
// the most actionable event is visible without extra filtering.
|
|
185
|
+
events: [...timedEvents, ...allDayEvents],
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
170
189
|
async function executeCalendarTool(toolName, args, auth) {
|
|
171
190
|
const calendar = google.calendar({ version: 'v3', auth });
|
|
172
191
|
const calendarId = String(args.calendar_id || 'primary').trim() || 'primary';
|
|
@@ -183,19 +202,7 @@ async function executeCalendarTool(toolName, args, auth) {
|
|
|
183
202
|
maxResults: Math.max(1, Math.min(Number(args.max_results) || 10, 50)),
|
|
184
203
|
});
|
|
185
204
|
const items = Array.isArray(response.data.items) ? response.data.items : [];
|
|
186
|
-
|
|
187
|
-
// Timed events first, then all-day markers, so the caller sees the entries it
|
|
188
|
-
// most likely cares about at the top. timedCount/allDayCount let a scheduled
|
|
189
|
-
// check conclude "nothing is actually scheduled" from one query instead of
|
|
190
|
-
// re-filtering the same flooded result set over and over.
|
|
191
|
-
const timed = events.filter((event) => !event.allDay);
|
|
192
|
-
const allDay = events.filter((event) => event.allDay);
|
|
193
|
-
return {
|
|
194
|
-
count: events.length,
|
|
195
|
-
timedCount: timed.length,
|
|
196
|
-
allDayCount: allDay.length,
|
|
197
|
-
events: [...timed, ...allDay],
|
|
198
|
-
};
|
|
205
|
+
return summarizeListedEvents(items);
|
|
199
206
|
}
|
|
200
207
|
|
|
201
208
|
case 'google_workspace_calendar_create_event': {
|
|
@@ -300,4 +307,5 @@ module.exports = {
|
|
|
300
307
|
calendarToolDefinitions,
|
|
301
308
|
executeCalendarTool,
|
|
302
309
|
summarizeEvent,
|
|
310
|
+
summarizeListedEvents,
|
|
303
311
|
};
|