neoagent 3.0.1-beta.8 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/.env.example +2 -10
  2. package/README.md +12 -3
  3. package/docs/automation.md +37 -0
  4. package/docs/benchmarking.md +102 -0
  5. package/docs/billing.md +34 -8
  6. package/docs/configuration.md +11 -2
  7. package/docs/getting-started.md +10 -8
  8. package/docs/operations.md +1 -1
  9. package/flutter_app/lib/main.dart +3 -0
  10. package/flutter_app/lib/main_account_settings.dart +138 -0
  11. package/flutter_app/lib/main_app_shell.dart +38 -13
  12. package/flutter_app/lib/main_billing.dart +1465 -0
  13. package/flutter_app/lib/main_chat.dart +1612 -214
  14. package/flutter_app/lib/main_controller.dart +398 -26
  15. package/flutter_app/lib/main_devices.dart +293 -207
  16. package/flutter_app/lib/main_models.dart +142 -0
  17. package/flutter_app/lib/main_navigation.dart +19 -1
  18. package/flutter_app/lib/main_operations.dart +288 -9
  19. package/flutter_app/lib/main_settings.dart +510 -300
  20. package/flutter_app/lib/main_shared.dart +2 -0
  21. package/flutter_app/lib/main_timeline.dart +1378 -0
  22. package/flutter_app/lib/src/backend_client.dart +149 -19
  23. package/flutter_app/lib/src/desktop_companion_actions.dart +132 -21
  24. package/flutter_app/lib/src/desktop_companion_io.dart +65 -1
  25. package/flutter_app/lib/src/desktop_companion_stub.dart +12 -0
  26. package/flutter_app/lib/src/desktop_ocr_bridge.dart +2 -0
  27. package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +125 -0
  28. package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +30 -0
  29. package/flutter_app/lib/src/desktop_passive_history.dart +332 -0
  30. package/flutter_app/lib/src/recording_bridge_io.dart +80 -72
  31. package/flutter_app/lib/src/recording_bridge_web.dart +127 -114
  32. package/flutter_app/lib/src/recording_chunk_queue.dart +149 -0
  33. package/flutter_app/lib/src/recording_chunk_queue_io.dart +182 -0
  34. package/flutter_app/lib/src/recording_payloads.dart +9 -0
  35. package/flutter_app/macos/Runner/AppDelegate.swift +25 -0
  36. package/flutter_app/windows/runner/flutter_window.cpp +75 -0
  37. package/landing/index.html +3 -3
  38. package/lib/manager.js +184 -66
  39. package/package.json +4 -1
  40. package/server/admin/access.js +12 -7
  41. package/server/admin/admin.css +78 -0
  42. package/server/admin/admin.js +436 -10
  43. package/server/admin/billing.js +165 -42
  44. package/server/admin/index.html +80 -5
  45. package/server/admin/users.js +15 -15
  46. package/server/db/database.js +125 -20
  47. package/server/http/routes.js +1 -0
  48. package/server/http/static.js +4 -2
  49. package/server/public/.last_build_id +1 -1
  50. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  51. package/server/public/canvaskit/wimp.js.symbols +8475 -8467
  52. package/server/public/canvaskit/wimp.wasm +0 -0
  53. package/server/public/flutter_bootstrap.js +2 -2
  54. package/server/public/main.dart.js +91077 -87037
  55. package/server/routes/account.js +53 -0
  56. package/server/routes/admin.js +345 -64
  57. package/server/routes/agents.js +203 -21
  58. package/server/routes/billing.js +5 -0
  59. package/server/routes/browser.js +8 -1
  60. package/server/routes/recordings.js +96 -6
  61. package/server/routes/screenHistory.js +140 -2
  62. package/server/routes/timeline.js +43 -0
  63. package/server/services/account/erasure.js +263 -0
  64. package/server/services/ai/hooks.js +4 -1
  65. package/server/services/ai/loop/agent_engine_core.js +8 -1
  66. package/server/services/ai/loop/blank_recovery.js +36 -0
  67. package/server/services/ai/loop/conversation_loop.js +166 -33
  68. package/server/services/ai/messagingFallback.js +22 -0
  69. package/server/services/ai/rate_limits.js +28 -5
  70. package/server/services/ai/systemPrompt.js +6 -5
  71. package/server/services/ai/taskAnalysis.js +2 -0
  72. package/server/services/ai/toolEvidence.js +15 -0
  73. package/server/services/ai/toolResult.js +40 -0
  74. package/server/services/ai/tools.js +163 -4
  75. package/server/services/android/controller.js +6 -2
  76. package/server/services/billing/plans.js +2 -1
  77. package/server/services/browser/anti_detection.js +192 -0
  78. package/server/services/browser/controller.js +180 -54
  79. package/server/services/desktop/auth.js +3 -0
  80. package/server/services/desktop/registry.js +50 -2
  81. package/server/services/integrations/google/calendar.js +22 -14
  82. package/server/services/manager.js +12 -42
  83. package/server/services/memory/ingestion_chunking.js +268 -0
  84. package/server/services/messaging/telnyx.js +9 -8
  85. package/server/services/recordings/manager.js +60 -27
  86. package/server/services/runtime/backends/local-vm.js +40 -22
  87. package/server/services/runtime/docker-vm-manager.js +157 -266
  88. package/server/services/runtime/guest_bootstrap.js +17 -5
  89. package/server/services/runtime/guest_image.js +188 -0
  90. package/server/services/runtime/manager.js +0 -1
  91. package/server/services/runtime/validation.js +3 -8
  92. package/server/services/social_video/service.js +60 -10
  93. package/server/services/tasks/runtime.js +234 -9
  94. package/server/services/tasks/schedule_utils.js +5 -5
  95. package/server/services/tasks/task_repository.js +13 -0
  96. package/server/services/timeline/service.js +558 -0
  97. package/server/services/wearable/gateway.js +1 -1
  98. package/server/services/websocket.js +21 -3
  99. package/server/services/desktop/screenRecorder.js +0 -292
  100. package/server/services/desktop/screen_recorder_support.js +0 -46
@@ -2,6 +2,14 @@ const path = require('path');
2
2
  const fs = require('fs');
3
3
  const { spawn } = require('child_process');
4
4
  const { DATA_DIR } = require('../../../runtime/paths');
5
+ const {
6
+ chooseBrowserIdentity,
7
+ detectBotChallenge,
8
+ generateHumanMousePath,
9
+ normalizeChallengeRetry,
10
+ normalizeReferrerMode,
11
+ rand,
12
+ } = require('./anti_detection');
5
13
 
6
14
  const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
7
15
  if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
@@ -9,21 +17,6 @@ const BROWSER_PROFILE_ROOT = path.join(DATA_DIR, 'browser-profiles');
9
17
  if (!fs.existsSync(BROWSER_PROFILE_ROOT)) fs.mkdirSync(BROWSER_PROFILE_ROOT, { recursive: true });
10
18
  const BROWSER_READY_MARKER = '/var/lib/neoagent/browser-runtime-ready';
11
19
 
12
- const USER_AGENTS = [
13
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
14
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
15
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
16
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
17
- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
18
- ];
19
-
20
- const VIEWPORTS = [
21
- { width: 1280, height: 800 },
22
- { width: 1366, height: 768 },
23
- { width: 1440, height: 900 },
24
- { width: 1920, height: 1080 },
25
- ];
26
-
27
20
  // Injected into every page in the cloud VM browser context to deny local device access.
28
21
  const DEVICE_DENY_SCRIPT = `(() => {
29
22
  const denied = () => Promise.reject(Object.assign(
@@ -166,10 +159,6 @@ function installPlaywrightBrowserBinary(browserName) {
166
159
  });
167
160
  }
168
161
 
169
- function rand(min, max) {
170
- return Math.floor(Math.random() * (max - min + 1)) + min;
171
- }
172
-
173
162
  function sleep(ms) {
174
163
  return new Promise(r => setTimeout(r, ms));
175
164
  }
@@ -241,10 +230,15 @@ class BrowserController {
241
230
  this.launchPromise = null;
242
231
  this.browserBinaryInstallPromise = null;
243
232
  this.headless = false;
244
- this._viewport = VIEWPORTS[0];
245
- this._userAgent = USER_AGENTS[0];
246
233
  this.profileDir = path.join(BROWSER_PROFILE_ROOT, this.userId || 'default');
247
234
  if (!fs.existsSync(this.profileDir)) fs.mkdirSync(this.profileDir, { recursive: true });
235
+ this._identity = chooseBrowserIdentity(this.userId || this.profileDir);
236
+ this._viewport = this._identity.viewport;
237
+ this._userAgent = this._identity.userAgent;
238
+ this._mousePosition = {
239
+ x: Math.round(this._viewport.width / 2),
240
+ y: Math.round(this._viewport.height / 2),
241
+ };
248
242
  }
249
243
 
250
244
  async setHeadless(val) {
@@ -260,6 +254,7 @@ class BrowserController {
260
254
  async _applyStealthToPage(page) {
261
255
  const ua = this._userAgent;
262
256
  const vp = this._viewport;
257
+ const identity = this._identity || chooseBrowserIdentity(this.userId || this.profileDir);
263
258
 
264
259
  if (typeof page.setUserAgent === 'function') {
265
260
  await page.setUserAgent(ua);
@@ -281,9 +276,9 @@ class BrowserController {
281
276
 
282
277
  // Realistic language/platform
283
278
  Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
284
- Object.defineProperty(navigator, 'platform', { get: () => 'MacIntel' });
285
- Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => ${rand(4, 16)} });
286
- Object.defineProperty(navigator, 'deviceMemory', { get: () => ${[4, 8, 16][rand(0, 2)]} });
279
+ Object.defineProperty(navigator, 'platform', { get: () => ${JSON.stringify(identity.platform)} });
280
+ Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => ${Number(identity.hardwareConcurrency) || 8} });
281
+ Object.defineProperty(navigator, 'deviceMemory', { get: () => ${Number(identity.deviceMemory) || 8} });
287
282
 
288
283
  // Make it look like a real Chrome install
289
284
  window.chrome = {
@@ -323,9 +318,9 @@ class BrowserController {
323
318
  apply: function(target, ctx, args) {
324
319
  const param = args[0];
325
320
  // UNMASKED_VENDOR_WEBGL
326
- if (param === 37445) return 'Google Inc. (Apple)';
321
+ if (param === 37445) return ${JSON.stringify(identity.webglVendor)};
327
322
  // UNMASKED_RENDERER_WEBGL
328
- if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
323
+ if (param === 37446) return ${JSON.stringify(identity.webglRenderer)};
329
324
  return Reflect.apply(target, ctx, args);
330
325
  }
331
326
  };
@@ -388,8 +383,13 @@ class BrowserController {
388
383
  }
389
384
  await this.ensureVirtualDisplay();
390
385
 
391
- this._userAgent = USER_AGENTS[rand(0, USER_AGENTS.length - 1)];
392
- this._viewport = VIEWPORTS[rand(0, VIEWPORTS.length - 1)];
386
+ this._identity = chooseBrowserIdentity(this.userId || this.profileDir);
387
+ this._userAgent = this._identity.userAgent;
388
+ this._viewport = this._identity.viewport;
389
+ this._mousePosition = {
390
+ x: Math.round(this._viewport.width / 2),
391
+ y: Math.round(this._viewport.height / 2),
392
+ };
393
393
 
394
394
  let executablePath = resolveBrowserExecutablePath();
395
395
  if (!executablePath) {
@@ -414,18 +414,28 @@ class BrowserController {
414
414
  };
415
415
 
416
416
  const launchArgs = [
417
+ '--start-maximized',
418
+ '--remote-allow-origins=*',
417
419
  '--no-sandbox',
418
420
  '--disable-setuid-sandbox',
419
421
  '--disable-dev-shm-usage',
422
+ '--no-service-autorun',
420
423
  '--disable-crash-reporter',
424
+ '--disable-breakpad',
421
425
  '--disable-background-networking',
422
426
  '--disable-component-update',
423
427
  '--disable-blink-features=AutomationControlled',
424
428
  '--disable-infobars',
429
+ '--disable-session-crashed-bubble',
430
+ '--disable-search-engine-choice-screen',
425
431
  '--no-first-run',
426
432
  '--no-default-browser-check',
433
+ '--homepage=about:blank',
434
+ '--no-pings',
435
+ '--password-store=basic',
427
436
  '--disable-gpu',
428
437
  '--lang=en-US,en',
438
+ `--user-agent=${this._userAgent}`,
429
439
  `--window-size=${this._viewport.width},${this._viewport.height}`,
430
440
  // Cloud security: disable hardware device APIs at the Chromium level
431
441
  '--disable-features=WebBluetooth,WebUSB,WebSerial,WebOTP,DirectSockets',
@@ -440,6 +450,8 @@ class BrowserController {
440
450
  env: launchEnv,
441
451
  args: launchArgs,
442
452
  viewport: this._viewport,
453
+ userAgent: this._userAgent,
454
+ locale: 'en-US',
443
455
  ignoreHTTPSErrors: false,
444
456
  timeout: 120000,
445
457
  });
@@ -535,13 +547,48 @@ class BrowserController {
535
547
  return page.screenshot(screenshotOptions);
536
548
  }
537
549
 
550
+ async _navigatePage(page, url, options = {}) {
551
+ const referrerMode = normalizeReferrerMode(options.referrerMode);
552
+ const waitUntil = normalizeWaitUntil(options.waitUntil);
553
+ if (referrerMode === 'current' && page.url() && page.url() !== 'about:blank') {
554
+ const previousUrl = page.url();
555
+ await page.evaluate((nextUrl) => { window.location.href = nextUrl; }, url);
556
+ await page.waitForFunction((oldUrl) => window.location.href !== oldUrl, previousUrl, { timeout: 10000 }).catch(() => {});
557
+ await page.waitForLoadState(waitUntil, { timeout: 30000 }).catch(() => {});
558
+ return null;
559
+ }
560
+
561
+ const gotoOptions = {
562
+ waitUntil,
563
+ timeout: 30000,
564
+ };
565
+ if (referrerMode === 'google') {
566
+ gotoOptions.referer = 'https://www.google.com/';
567
+ } else if (referrerMode === 'current' && page.url() && page.url() !== 'about:blank') {
568
+ gotoOptions.referer = page.url();
569
+ }
570
+ return page.goto(url, gotoOptions);
571
+ }
572
+
573
+ async _getBotDetection(page, rawHtml, pageContent) {
574
+ const title = await page.title().catch(() => '');
575
+ return detectBotChallenge({
576
+ title,
577
+ url: page.url(),
578
+ html: rawHtml,
579
+ pageContent,
580
+ });
581
+ }
582
+
538
583
  async navigate(url, options = {}) {
539
584
  const page = await this.ensurePage();
540
585
 
541
586
  try {
542
- const response = await page.goto(url, {
543
- waitUntil: normalizeWaitUntil(options.waitUntil),
544
- timeout: 30000
587
+ const requestedReferrerMode = normalizeReferrerMode(options.referrerMode);
588
+ let activeReferrerMode = requestedReferrerMode;
589
+ let response = await this._navigatePage(page, url, {
590
+ ...options,
591
+ referrerMode: activeReferrerMode,
545
592
  });
546
593
 
547
594
  if (options.waitFor) {
@@ -549,7 +596,34 @@ class BrowserController {
549
596
  }
550
597
 
551
598
  // Simulate human reading delay
552
- await sleep(rand(500, 1500));
599
+ await sleep(rand(700, 1800));
600
+
601
+ let rawHtml = await page.content();
602
+ const { extractForLLM } = require('./contentExtractor');
603
+ let extraction = extractForLLM(rawHtml, { url: page.url() });
604
+ let botDetection = await this._getBotDetection(page, rawHtml, extraction.markdown);
605
+ let challengeRetried = false;
606
+
607
+ if (
608
+ botDetection.detected
609
+ && normalizeChallengeRetry(options.challengeRetry)
610
+ && requestedReferrerMode === 'direct'
611
+ ) {
612
+ challengeRetried = true;
613
+ activeReferrerMode = 'google';
614
+ await sleep(rand(1200, 2600));
615
+ response = await this._navigatePage(page, url, {
616
+ ...options,
617
+ referrerMode: activeReferrerMode,
618
+ });
619
+ if (options.waitFor) {
620
+ await page.waitForSelector(options.waitFor, { timeout: 10000 }).catch(() => { });
621
+ }
622
+ await sleep(rand(900, 2200));
623
+ rawHtml = await page.content();
624
+ extraction = extractForLLM(rawHtml, { url: page.url() });
625
+ botDetection = await this._getBotDetection(page, rawHtml, extraction.markdown);
626
+ }
553
627
 
554
628
  const title = await page.title();
555
629
  const currentUrl = page.url();
@@ -559,15 +633,14 @@ class BrowserController {
559
633
  screenshot = await this.takeScreenshot({ fullPage: options.fullPage });
560
634
  }
561
635
 
562
- const rawHtml = await page.content();
563
- const { extractForLLM } = require('./contentExtractor');
564
- const extraction = extractForLLM(rawHtml, { url: currentUrl });
565
-
566
636
  return {
567
637
  title,
568
638
  url: currentUrl,
569
639
  status: response?.status() || 0,
570
640
  pageContent: extraction.markdown,
641
+ botDetection,
642
+ referrerMode: activeReferrerMode,
643
+ challengeRetried,
571
644
  screenshotPath: screenshot?.screenshotPath || null,
572
645
  artifactId: screenshot?.artifactId || null,
573
646
  fullPath: screenshot?.fullPath || null
@@ -578,6 +651,7 @@ class BrowserController {
578
651
  return {
579
652
  error: err.message,
580
653
  url,
654
+ botDetection: { detected: false, provider: null },
581
655
  screenshotPath: screenshot?.screenshotPath || null,
582
656
  artifactId: screenshot?.artifactId || null,
583
657
  fullPath: screenshot?.fullPath || null
@@ -585,6 +659,36 @@ class BrowserController {
585
659
  }
586
660
  }
587
661
 
662
+ async _moveMouseTo(page, x, y, options = {}) {
663
+ const target = {
664
+ x: Math.max(0, Math.min(this._viewport.width, Math.round(Number(x) || 0))),
665
+ y: Math.max(0, Math.min(this._viewport.height, Math.round(Number(y) || 0))),
666
+ };
667
+ const path = generateHumanMousePath(this._mousePosition, target, this._viewport);
668
+ for (const point of path) {
669
+ await page.mouse.move(point.x, point.y);
670
+ if (!options.fast) {
671
+ await sleep(rand(2, 10));
672
+ }
673
+ }
674
+ this._mousePosition = target;
675
+ return target;
676
+ }
677
+
678
+ async _pointForElement(element) {
679
+ await element.scrollIntoViewIfNeeded?.().catch(() => {});
680
+ const box = await element.boundingBox();
681
+ if (!box || box.width <= 0 || box.height <= 0) {
682
+ return null;
683
+ }
684
+ const xRatio = rand(25, 75) / 100;
685
+ const yRatio = rand(30, 70) / 100;
686
+ return {
687
+ x: box.x + box.width * xRatio,
688
+ y: box.y + box.height * yRatio,
689
+ };
690
+ }
691
+
588
692
  async click(selector, text, screenshot = true) {
589
693
  const page = await this.ensurePage();
590
694
 
@@ -608,10 +712,14 @@ class BrowserController {
608
712
  return { error: 'Either selector or text required' };
609
713
  }
610
714
 
611
- // Human-like: hover first, then click with a hold delay
612
- await target.hover();
613
- await sleep(rand(80, 250));
614
- await target.click({ delay: rand(50, 150) });
715
+ const point = await this._pointForElement(target);
716
+ if (!point) return { error: 'Element has no visible clickable area' };
717
+
718
+ await this._moveMouseTo(page, point.x, point.y);
719
+ await sleep(rand(170, 320));
720
+ await page.mouse.down();
721
+ await sleep(rand(80, 260));
722
+ await page.mouse.up();
615
723
 
616
724
  await sleep(rand(800, 1800));
617
725
 
@@ -637,10 +745,10 @@ class BrowserController {
637
745
  try {
638
746
  const px = Math.max(0, Math.round(Number(x) || 0));
639
747
  const py = Math.max(0, Math.round(Number(y) || 0));
640
- await page.mouse.move(px, py, { steps: rand(4, 10) });
641
- await sleep(rand(40, 140));
748
+ await this._moveMouseTo(page, px, py);
749
+ await sleep(rand(90, 220));
642
750
  await page.mouse.down();
643
- await sleep(rand(30, 110));
751
+ await sleep(rand(70, 240));
644
752
  await page.mouse.up();
645
753
  await sleep(rand(500, 1200));
646
754
 
@@ -667,7 +775,7 @@ class BrowserController {
667
775
  try {
668
776
  const px = Math.max(0, Math.round(Number(x) || 0));
669
777
  const py = Math.max(0, Math.round(Number(y) || 0));
670
- await page.mouse.move(px, py, { steps: options.steps || 1 });
778
+ await this._moveMouseTo(page, px, py, { fast: Number(options.steps) <= 1 });
671
779
  return {
672
780
  success: true,
673
781
  x: px,
@@ -684,11 +792,19 @@ class BrowserController {
684
792
  const page = await this.ensurePage();
685
793
 
686
794
  try {
687
- await page.mouse.wheel({
688
- deltaX: Math.round(Number(deltaX) || 0),
689
- deltaY: Math.round(Number(deltaY) || 0),
690
- });
691
- await sleep(rand(300, 900));
795
+ const x = Math.max(10, Math.min(this._viewport.width - 10, this._mousePosition.x + rand(-80, 80)));
796
+ const y = Math.max(10, Math.min(this._viewport.height - 10, this._mousePosition.y + rand(-80, 80)));
797
+ await this._moveMouseTo(page, x, y);
798
+ const totalX = Math.round(Number(deltaX) || 0);
799
+ const totalY = Math.round(Number(deltaY) || 0);
800
+ const chunks = Math.max(1, Math.min(6, Math.ceil(Math.max(Math.abs(totalX), Math.abs(totalY)) / 450)));
801
+ for (let i = 0; i < chunks; i += 1) {
802
+ await page.mouse.wheel({
803
+ deltaX: Math.round(totalX / chunks),
804
+ deltaY: Math.round(totalY / chunks),
805
+ });
806
+ await sleep(rand(120, 360));
807
+ }
692
808
 
693
809
  let screenshotResult = null;
694
810
  if (screenshot) screenshotResult = await this.takeScreenshot();
@@ -711,12 +827,20 @@ class BrowserController {
711
827
 
712
828
  try {
713
829
  if (options.clear !== false) {
714
- await page.click(selector, { clickCount: 3 });
830
+ const element = await page.$(selector);
831
+ if (element) {
832
+ const point = await this._pointForElement(element);
833
+ if (point) {
834
+ await this._moveMouseTo(page, point.x, point.y);
835
+ await sleep(rand(120, 260));
836
+ }
837
+ }
838
+ await page.click(selector, { clickCount: 3, delay: rand(40, 120) });
715
839
  await page.keyboard.press('Backspace');
716
840
  }
717
841
 
718
842
  for (const char of text) {
719
- await page.type(selector, char, { delay: rand(30, 150) });
843
+ await page.type(selector, char, { delay: rand(45, 180) });
720
844
  }
721
845
 
722
846
  if (options.pressEnter) {
@@ -744,7 +868,7 @@ class BrowserController {
744
868
 
745
869
  try {
746
870
  for (const char of String(text || '')) {
747
- await page.keyboard.type(char, { delay: rand(25, 110) });
871
+ await page.keyboard.type(char, { delay: rand(45, 160) });
748
872
  }
749
873
 
750
874
  if (options.pressEnter) {
@@ -797,6 +921,8 @@ class BrowserController {
797
921
  const page = await this.ensurePage();
798
922
 
799
923
  try {
924
+ const rawHtml = await page.content().catch(() => '');
925
+ const botDetection = await this._getBotDetection(page, rawHtml, '').catch(() => ({ detected: false, provider: null }));
800
926
  if (all) {
801
927
  const results = await page.$$eval(selector || 'body', (elements, attr) => {
802
928
  return elements.map(el => {
@@ -806,7 +932,7 @@ class BrowserController {
806
932
  return el.innerText || '';
807
933
  });
808
934
  }, attribute);
809
- return { results: results.slice(0, 100) };
935
+ return { results: results.slice(0, 100), botDetection };
810
936
  }
811
937
 
812
938
  const result = await page.$eval(selector || 'body', (el, attr) => {
@@ -816,7 +942,7 @@ class BrowserController {
816
942
  return el.innerText || '';
817
943
  }, attribute);
818
944
 
819
- return { result: typeof result === 'string' ? result.slice(0, 50000) : result };
945
+ return { result: typeof result === 'string' ? result.slice(0, 50000) : result, botDetection };
820
946
  } catch (err) {
821
947
  return { error: err.message };
822
948
  }
@@ -29,6 +29,9 @@ function normalizeDesktopHello(message) {
29
29
  : {},
30
30
  displays: Array.isArray(device.displays) ? device.displays : [],
31
31
  activeDisplayId: String(device.activeDisplayId || '').trim(),
32
+ passiveHistoryEnabled: device.passiveHistoryEnabled === true,
33
+ passiveHistoryLastUploadedAt: String(device.passiveHistoryLastUploadedAt || '').trim(),
34
+ passiveHistoryLastError: String(device.passiveHistoryLastError || '').trim(),
32
35
  metadata: device.metadata && typeof device.metadata === 'object'
33
36
  ? device.metadata
34
37
  : {},
@@ -101,8 +101,9 @@ class DesktopCompanionRegistry {
101
101
  `INSERT INTO desktop_companion_devices (
102
102
  id, user_id, device_id, activation_id, label, hostname, platform, platform_version, app_version,
103
103
  companion_enabled, paused, status, display_count, active_display_id, permissions_json, capabilities_json,
104
- metadata_json, session_id, last_connected_at, last_seen_at, revoked_at, created_at, updated_at
105
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'online', ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
104
+ metadata_json, session_id, last_connected_at, last_seen_at, passive_history_enabled, passive_history_last_uploaded_at,
105
+ passive_history_last_error, revoked_at, created_at, updated_at
106
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'online', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
106
107
  ON CONFLICT(user_id, device_id) DO UPDATE SET
107
108
  activation_id = excluded.activation_id,
108
109
  revoked_at = NULL,
@@ -122,6 +123,9 @@ class DesktopCompanionRegistry {
122
123
  session_id = excluded.session_id,
123
124
  last_connected_at = excluded.last_connected_at,
124
125
  last_seen_at = excluded.last_seen_at,
126
+ passive_history_enabled = excluded.passive_history_enabled,
127
+ passive_history_last_uploaded_at = excluded.passive_history_last_uploaded_at,
128
+ passive_history_last_error = excluded.passive_history_last_error,
125
129
  updated_at = excluded.updated_at`
126
130
  ).run(
127
131
  rowId,
@@ -143,6 +147,9 @@ class DesktopCompanionRegistry {
143
147
  sessionId,
144
148
  now,
145
149
  now,
150
+ hello.passiveHistoryEnabled ? 1 : 0,
151
+ hello.passiveHistoryLastUploadedAt || null,
152
+ hello.passiveHistoryLastError || null,
146
153
  existing?.created_at || now,
147
154
  now,
148
155
  );
@@ -190,6 +197,9 @@ class DesktopCompanionRegistry {
190
197
  sessionId: row.session_id || null,
191
198
  lastConnectedAt: row.last_connected_at || null,
192
199
  lastSeenAt: row.last_seen_at || null,
200
+ passiveHistoryEnabled: row.passive_history_enabled === 1,
201
+ passiveHistoryLastUploadedAt: row.passive_history_last_uploaded_at || null,
202
+ passiveHistoryLastError: row.passive_history_last_error || null,
193
203
  revokedAt: row.revoked_at || null,
194
204
  createdAt: row.created_at,
195
205
  updatedAt: row.updated_at || row.created_at,
@@ -278,6 +288,9 @@ class DesktopCompanionRegistry {
278
288
  permissions_json = ?,
279
289
  capabilities_json = ?,
280
290
  metadata_json = ?,
291
+ passive_history_enabled = ?,
292
+ passive_history_last_uploaded_at = ?,
293
+ passive_history_last_error = ?,
281
294
  last_seen_at = datetime('now'),
282
295
  updated_at = datetime('now')
283
296
  WHERE user_id = ? AND device_id = ?`
@@ -290,6 +303,13 @@ class DesktopCompanionRegistry {
290
303
  safeJson(patch.permissions || existing.permissions),
291
304
  safeJson(patch.capabilities || existing.capabilities),
292
305
  safeJson(metadata),
306
+ patch.passiveHistoryEnabled == null
307
+ ? (existing.passiveHistoryEnabled ? 1 : 0)
308
+ : (patch.passiveHistoryEnabled === true ? 1 : 0),
309
+ patch.passiveHistoryLastUploadedAt || existing.passiveHistoryLastUploadedAt || null,
310
+ patch.passiveHistoryLastError == null
311
+ ? (existing.passiveHistoryLastError || null)
312
+ : (patch.passiveHistoryLastError || null),
293
313
  userId,
294
314
  deviceId,
295
315
  );
@@ -516,6 +536,34 @@ class DesktopCompanionRegistry {
516
536
  };
517
537
  }
518
538
 
539
+ updatePassiveHistoryState(userId, deviceId, patch = {}) {
540
+ const normalizedDeviceId = String(deviceId || '').trim();
541
+ if (!normalizedDeviceId) {
542
+ throw new DesktopCompanionUnavailableError();
543
+ }
544
+ const existing = this.db.prepare(
545
+ `SELECT device_id FROM desktop_companion_devices WHERE user_id = ? AND device_id = ?`
546
+ ).get(userId, normalizedDeviceId);
547
+ if (!existing) {
548
+ throw new DesktopCompanionUnavailableError();
549
+ }
550
+ this.db.prepare(
551
+ `UPDATE desktop_companion_devices
552
+ SET passive_history_enabled = ?,
553
+ passive_history_last_uploaded_at = COALESCE(?, passive_history_last_uploaded_at),
554
+ passive_history_last_error = ?,
555
+ updated_at = datetime('now')
556
+ WHERE user_id = ? AND device_id = ?`
557
+ ).run(
558
+ patch.enabled === true ? 1 : 0,
559
+ patch.lastUploadedAt || null,
560
+ patch.lastError || null,
561
+ userId,
562
+ normalizedDeviceId,
563
+ );
564
+ return this.getDeviceRecordByDeviceId(userId, normalizedDeviceId);
565
+ }
566
+
519
567
  closeAll() {
520
568
  for (const userMap of this.connectionsByUser.values()) {
521
569
  for (const connection of userMap.values()) {
@@ -7,7 +7,7 @@ const calendarToolDefinitions = [
7
7
  {
8
8
  name: 'google_workspace_calendar_list_events',
9
9
  access: 'read',
10
- description: 'List or search Google Calendar events for the connected account. Results include all-day/multi-day events (birthdays, anniversaries, full-day markers) that overlap the window each event has an allDay flag, and the response reports timedCount and allDayCount. For "what is scheduled / starting soon", rely on timedCount and the allDay flag; do not re-query with different windows to filter out all-day entries.',
10
+ description: 'List or search Google Calendar events for the connected account. Results include all-day/multi-day events (birthdays, anniversaries, full-day markers) that overlap the window. Each event has an allDay flag, timedCount/allDayCount summarize the mix, and timedEvents/nextTimedEvent expose the real timed appointments directly. For "what is scheduled / starting soon", decide from that single result instead of re-querying nearby windows to filter out all-day entries.',
11
11
  parameters: {
12
12
  type: 'object',
13
13
  properties: {
@@ -167,6 +167,25 @@ function summarizeEvent(event) {
167
167
  };
168
168
  }
169
169
 
170
+ function summarizeListedEvents(items) {
171
+ const events = Array.isArray(items) ? items.map(summarizeEvent) : [];
172
+ const timedEvents = events.filter((event) => !event.allDay);
173
+ const allDayEvents = events.filter((event) => event.allDay);
174
+ return {
175
+ count: events.length,
176
+ timedCount: timedEvents.length,
177
+ allDayCount: allDayEvents.length,
178
+ hasTimedEvents: timedEvents.length > 0,
179
+ hasOnlyAllDayEvents: timedEvents.length === 0 && allDayEvents.length > 0,
180
+ nextTimedEvent: timedEvents[0] || null,
181
+ timedEvents,
182
+ allDayEvents,
183
+ // Keep the legacy merged list for existing callers, but put timed items first so
184
+ // the most actionable event is visible without extra filtering.
185
+ events: [...timedEvents, ...allDayEvents],
186
+ };
187
+ }
188
+
170
189
  async function executeCalendarTool(toolName, args, auth) {
171
190
  const calendar = google.calendar({ version: 'v3', auth });
172
191
  const calendarId = String(args.calendar_id || 'primary').trim() || 'primary';
@@ -183,19 +202,7 @@ async function executeCalendarTool(toolName, args, auth) {
183
202
  maxResults: Math.max(1, Math.min(Number(args.max_results) || 10, 50)),
184
203
  });
185
204
  const items = Array.isArray(response.data.items) ? response.data.items : [];
186
- const events = items.map(summarizeEvent);
187
- // Timed events first, then all-day markers, so the caller sees the entries it
188
- // most likely cares about at the top. timedCount/allDayCount let a scheduled
189
- // check conclude "nothing is actually scheduled" from one query instead of
190
- // re-filtering the same flooded result set over and over.
191
- const timed = events.filter((event) => !event.allDay);
192
- const allDay = events.filter((event) => event.allDay);
193
- return {
194
- count: events.length,
195
- timedCount: timed.length,
196
- allDayCount: allDay.length,
197
- events: [...timed, ...allDay],
198
- };
205
+ return summarizeListedEvents(items);
199
206
  }
200
207
 
201
208
  case 'google_workspace_calendar_create_event': {
@@ -300,4 +307,5 @@ module.exports = {
300
307
  calendarToolDefinitions,
301
308
  executeCalendarTool,
302
309
  summarizeEvent,
310
+ summarizeListedEvents,
303
311
  };