neoagent 3.0.1-beta.8 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/.env.example +2 -10
  2. package/README.md +12 -3
  3. package/docs/automation.md +37 -0
  4. package/docs/benchmarking.md +102 -0
  5. package/docs/billing.md +34 -8
  6. package/docs/configuration.md +11 -2
  7. package/docs/getting-started.md +10 -8
  8. package/docs/operations.md +1 -1
  9. package/flutter_app/lib/main.dart +3 -0
  10. package/flutter_app/lib/main_account_settings.dart +138 -0
  11. package/flutter_app/lib/main_app_shell.dart +38 -13
  12. package/flutter_app/lib/main_billing.dart +1465 -0
  13. package/flutter_app/lib/main_chat.dart +1612 -214
  14. package/flutter_app/lib/main_controller.dart +398 -26
  15. package/flutter_app/lib/main_devices.dart +293 -207
  16. package/flutter_app/lib/main_models.dart +142 -0
  17. package/flutter_app/lib/main_navigation.dart +19 -1
  18. package/flutter_app/lib/main_operations.dart +288 -9
  19. package/flutter_app/lib/main_settings.dart +510 -300
  20. package/flutter_app/lib/main_shared.dart +2 -0
  21. package/flutter_app/lib/main_timeline.dart +1378 -0
  22. package/flutter_app/lib/src/backend_client.dart +149 -19
  23. package/flutter_app/lib/src/desktop_companion_actions.dart +132 -21
  24. package/flutter_app/lib/src/desktop_companion_io.dart +65 -1
  25. package/flutter_app/lib/src/desktop_companion_stub.dart +12 -0
  26. package/flutter_app/lib/src/desktop_ocr_bridge.dart +2 -0
  27. package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +125 -0
  28. package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +30 -0
  29. package/flutter_app/lib/src/desktop_passive_history.dart +332 -0
  30. package/flutter_app/lib/src/recording_bridge_io.dart +80 -72
  31. package/flutter_app/lib/src/recording_bridge_web.dart +127 -114
  32. package/flutter_app/lib/src/recording_chunk_queue.dart +149 -0
  33. package/flutter_app/lib/src/recording_chunk_queue_io.dart +182 -0
  34. package/flutter_app/lib/src/recording_payloads.dart +9 -0
  35. package/flutter_app/macos/Runner/AppDelegate.swift +25 -0
  36. package/flutter_app/windows/runner/flutter_window.cpp +75 -0
  37. package/landing/index.html +3 -3
  38. package/lib/manager.js +184 -66
  39. package/package.json +4 -1
  40. package/server/admin/access.js +12 -7
  41. package/server/admin/admin.css +78 -0
  42. package/server/admin/admin.js +436 -10
  43. package/server/admin/billing.js +165 -42
  44. package/server/admin/index.html +80 -5
  45. package/server/admin/users.js +15 -15
  46. package/server/db/database.js +125 -20
  47. package/server/http/routes.js +1 -0
  48. package/server/http/static.js +4 -2
  49. package/server/public/.last_build_id +1 -1
  50. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  51. package/server/public/canvaskit/wimp.js.symbols +8475 -8467
  52. package/server/public/canvaskit/wimp.wasm +0 -0
  53. package/server/public/flutter_bootstrap.js +2 -2
  54. package/server/public/main.dart.js +91077 -87037
  55. package/server/routes/account.js +53 -0
  56. package/server/routes/admin.js +345 -64
  57. package/server/routes/agents.js +203 -21
  58. package/server/routes/billing.js +5 -0
  59. package/server/routes/browser.js +8 -1
  60. package/server/routes/recordings.js +96 -6
  61. package/server/routes/screenHistory.js +140 -2
  62. package/server/routes/timeline.js +43 -0
  63. package/server/services/account/erasure.js +263 -0
  64. package/server/services/ai/hooks.js +4 -1
  65. package/server/services/ai/loop/agent_engine_core.js +8 -1
  66. package/server/services/ai/loop/blank_recovery.js +36 -0
  67. package/server/services/ai/loop/conversation_loop.js +166 -33
  68. package/server/services/ai/messagingFallback.js +22 -0
  69. package/server/services/ai/rate_limits.js +28 -5
  70. package/server/services/ai/systemPrompt.js +6 -5
  71. package/server/services/ai/taskAnalysis.js +2 -0
  72. package/server/services/ai/toolEvidence.js +15 -0
  73. package/server/services/ai/toolResult.js +40 -0
  74. package/server/services/ai/tools.js +163 -4
  75. package/server/services/android/controller.js +6 -2
  76. package/server/services/billing/plans.js +2 -1
  77. package/server/services/browser/anti_detection.js +192 -0
  78. package/server/services/browser/controller.js +180 -54
  79. package/server/services/desktop/auth.js +3 -0
  80. package/server/services/desktop/registry.js +50 -2
  81. package/server/services/integrations/google/calendar.js +22 -14
  82. package/server/services/manager.js +12 -42
  83. package/server/services/memory/ingestion_chunking.js +268 -0
  84. package/server/services/messaging/telnyx.js +9 -8
  85. package/server/services/recordings/manager.js +60 -27
  86. package/server/services/runtime/backends/local-vm.js +40 -22
  87. package/server/services/runtime/docker-vm-manager.js +157 -266
  88. package/server/services/runtime/guest_bootstrap.js +17 -5
  89. package/server/services/runtime/guest_image.js +188 -0
  90. package/server/services/runtime/manager.js +0 -1
  91. package/server/services/runtime/validation.js +3 -8
  92. package/server/services/social_video/service.js +60 -10
  93. package/server/services/tasks/runtime.js +234 -9
  94. package/server/services/tasks/schedule_utils.js +5 -5
  95. package/server/services/tasks/task_repository.js +13 -0
  96. package/server/services/timeline/service.js +558 -0
  97. package/server/services/wearable/gateway.js +1 -1
  98. package/server/services/websocket.js +21 -3
  99. package/server/services/desktop/screenRecorder.js +0 -292
  100. package/server/services/desktop/screen_recorder_support.js +0 -46
@@ -213,7 +213,11 @@ class BackendClient {
213
213
  }
214
214
 
215
215
  Future<Map<String, dynamic>> completeOnboarding(String baseUrl) async {
216
- return postMap(baseUrl, '/api/auth/onboarding/complete', const <String, dynamic>{});
216
+ return postMap(
217
+ baseUrl,
218
+ '/api/auth/onboarding/complete',
219
+ const <String, dynamic>{},
220
+ );
217
221
  }
218
222
 
219
223
  Future<Map<String, dynamic>> fetchAgentProfiles(String baseUrl) async {
@@ -228,6 +232,23 @@ class BackendClient {
228
232
  return getMap(baseUrl, '/api/account/usage');
229
233
  }
230
234
 
235
+ /// GDPR data portability — returns a structured export of the signed-in
236
+ /// user's own data (credential fields are redacted server-side).
237
+ Future<Map<String, dynamic>> exportMyData(String baseUrl) async {
238
+ return getMap(baseUrl, '/api/account/export');
239
+ }
240
+
241
+ /// GDPR right to erasure — permanently deletes the signed-in user's account
242
+ /// and all associated data. [confirmUsername] must equal the account username.
243
+ Future<Map<String, dynamic>> deleteMyAccount({
244
+ required String baseUrl,
245
+ required String confirmUsername,
246
+ }) async {
247
+ return postMap(baseUrl, '/api/account/delete', <String, dynamic>{
248
+ 'confirmUsername': confirmUsername,
249
+ });
250
+ }
251
+
231
252
  Future<Map<String, dynamic>> updateAccountEmail({
232
253
  required String baseUrl,
233
254
  required String email,
@@ -342,6 +363,59 @@ class BackendClient {
342
363
  return deleteMap(baseUrl, '/api/account/sessions/$sessionId');
343
364
  }
344
365
 
366
+ // ── Billing ──────────────────────────────────────────────────────────────
367
+
368
+ Future<Map<String, dynamic>> getBillingPlans(String baseUrl) async {
369
+ return getMap(baseUrl, '/api/billing/plans', allowUnauthorized: true);
370
+ }
371
+
372
+ Future<Map<String, dynamic>> getBillingInfo(String baseUrl) async {
373
+ return getMap(baseUrl, '/api/billing/');
374
+ }
375
+
376
+ Future<Map<String, dynamic>> getBillingInvoices(String baseUrl) async {
377
+ return getMap(baseUrl, '/api/billing/invoices');
378
+ }
379
+
380
+ Future<Map<String, dynamic>> createCheckoutSession({
381
+ required String baseUrl,
382
+ required String planId,
383
+ required String successUrl,
384
+ required String cancelUrl,
385
+ }) async {
386
+ return postMap(baseUrl, '/api/billing/checkout', <String, dynamic>{
387
+ 'planId': planId,
388
+ 'successUrl': successUrl,
389
+ 'cancelUrl': cancelUrl,
390
+ });
391
+ }
392
+
393
+ Future<Map<String, dynamic>> createPortalSession({
394
+ required String baseUrl,
395
+ required String returnUrl,
396
+ }) async {
397
+ return postMap(baseUrl, '/api/billing/portal', <String, dynamic>{
398
+ 'returnUrl': returnUrl,
399
+ });
400
+ }
401
+
402
+ Future<Map<String, dynamic>> cancelBillingSubscription(String baseUrl) async {
403
+ return postMap(baseUrl, '/api/billing/cancel', const <String, dynamic>{});
404
+ }
405
+
406
+ Future<Map<String, dynamic>> startBillingTrial({
407
+ required String baseUrl,
408
+ required String planId,
409
+ String? deviceFingerprint,
410
+ }) async {
411
+ return postMap(baseUrl, '/api/billing/trial', <String, dynamic>{
412
+ 'planId': planId,
413
+ if (deviceFingerprint != null) 'deviceFingerprint': deviceFingerprint,
414
+ });
415
+ }
416
+
417
+ // ── Agent profiles ────────────────────────────────────────────────────────
418
+
345
419
  Future<Map<String, dynamic>> createAgentProfile(
346
420
  String baseUrl,
347
421
  Map<String, dynamic> payload,
@@ -371,10 +445,23 @@ class BackendClient {
371
445
  Future<Map<String, dynamic>> fetchChatHistory(
372
446
  String baseUrl, {
373
447
  String? agentId,
448
+ int limit = 40,
449
+ String? beforeCreatedAt,
450
+ String? beforeSource,
451
+ String? beforeId,
374
452
  }) async {
453
+ final params = <String>[
454
+ 'limit=$limit',
455
+ if (beforeCreatedAt != null && beforeCreatedAt.trim().isNotEmpty)
456
+ 'beforeCreatedAt=${Uri.encodeQueryComponent(beforeCreatedAt.trim())}',
457
+ if (beforeSource != null && beforeSource.trim().isNotEmpty)
458
+ 'beforeSource=${Uri.encodeQueryComponent(beforeSource.trim())}',
459
+ if (beforeId != null && beforeId.trim().isNotEmpty)
460
+ 'beforeId=${Uri.encodeQueryComponent(beforeId.trim())}',
461
+ ];
375
462
  return getMap(
376
463
  baseUrl,
377
- _withAgentQuery('/api/agents/chat-history?limit=120', agentId),
464
+ _withAgentQuery('/api/agents/chat-history?${params.join('&')}', agentId),
378
465
  );
379
466
  }
380
467
 
@@ -461,6 +548,32 @@ class BackendClient {
461
548
  return getMap(baseUrl, _withAgentQuery('/api/agents?limit=20', agentId));
462
549
  }
463
550
 
551
+ Future<Map<String, dynamic>> fetchTimeline(
552
+ String baseUrl, {
553
+ Iterable<String>? sources,
554
+ String? agentId,
555
+ int limit = 50,
556
+ String? beforeOccurredAt,
557
+ int? beforeId,
558
+ }) async {
559
+ final query = <String>[
560
+ 'limit=${limit.clamp(1, 200)}',
561
+ if (agentId?.trim().isNotEmpty == true)
562
+ 'agentId=${Uri.encodeQueryComponent(agentId!.trim())}',
563
+ if (beforeOccurredAt?.trim().isNotEmpty == true)
564
+ 'beforeOccurredAt=${Uri.encodeQueryComponent(beforeOccurredAt!.trim())}',
565
+ if (beforeId != null && beforeId > 0) 'beforeId=$beforeId',
566
+ ];
567
+ for (final source in (sources ?? const <String>[])) {
568
+ final normalized = source.trim().toLowerCase();
569
+ if (normalized.isEmpty) {
570
+ continue;
571
+ }
572
+ query.add('source=${Uri.encodeQueryComponent(normalized)}');
573
+ }
574
+ return getMap(baseUrl, '/api/timeline?${query.join('&')}');
575
+ }
576
+
464
577
  Future<Map<String, dynamic>> fetchRunSteps(
465
578
  String baseUrl,
466
579
  String runId,
@@ -506,9 +619,11 @@ class BackendClient {
506
619
  String baseUrl, {
507
620
  required String tokenId,
508
621
  }) async {
509
- return postMap(baseUrl, '/api/browser-extension/select-token', <String, dynamic>{
510
- 'tokenId': tokenId,
511
- });
622
+ return postMap(
623
+ baseUrl,
624
+ '/api/browser-extension/select-token',
625
+ <String, dynamic>{'tokenId': tokenId},
626
+ );
512
627
  }
513
628
 
514
629
  Future<Map<String, dynamic>> launchBrowser(
@@ -1204,7 +1319,10 @@ class BackendClient {
1204
1319
  String? agentId,
1205
1320
  }) async {
1206
1321
  final response = await _httpClient.delete(
1207
- _resolveUri(baseUrl, _withAgentQuery('/api/integrations/$providerId/config', agentId)),
1322
+ _resolveUri(
1323
+ baseUrl,
1324
+ _withAgentQuery('/api/integrations/$providerId/config', agentId),
1325
+ ),
1208
1326
  headers: const <String, String>{'Accept': 'application/json'},
1209
1327
  );
1210
1328
  _throwIfError(response);
@@ -1549,6 +1667,7 @@ class BackendClient {
1549
1667
  required String triggerType,
1550
1668
  required Map<String, dynamic> triggerConfig,
1551
1669
  required String prompt,
1670
+ Map<String, dynamic>? taskConfig,
1552
1671
  String? model,
1553
1672
  bool enabled = true,
1554
1673
  String? agentId,
@@ -1557,6 +1676,7 @@ class BackendClient {
1557
1676
  'name': name,
1558
1677
  'triggerType': triggerType,
1559
1678
  'triggerConfig': triggerConfig,
1679
+ if (taskConfig != null) 'taskConfig': taskConfig,
1560
1680
  'prompt': prompt,
1561
1681
  'model': model,
1562
1682
  'enabled': enabled,
@@ -1710,10 +1830,11 @@ class BackendClient {
1710
1830
  required String audioBase64,
1711
1831
  String mimeType = 'audio/pcm;rate=16000;channels=1',
1712
1832
  }) {
1713
- return postMap(baseUrl, '/api/voice-assistant/transcribe', <String, dynamic>{
1714
- 'audioBase64': audioBase64,
1715
- 'mimeType': mimeType,
1716
- });
1833
+ return postMap(
1834
+ baseUrl,
1835
+ '/api/voice-assistant/transcribe',
1836
+ <String, dynamic>{'audioBase64': audioBase64, 'mimeType': mimeType},
1837
+ );
1717
1838
  }
1718
1839
 
1719
1840
  Future<Map<String, dynamic>> runVoiceAssistantTurn(
@@ -2029,8 +2150,13 @@ class BackendClient {
2029
2150
  return getMap(baseUrl, '/api/security/mode');
2030
2151
  }
2031
2152
 
2032
- Future<Map<String, dynamic>> saveSecurityMode(String baseUrl, String mode) async {
2033
- return putMap(baseUrl, '/api/security/mode', <String, dynamic>{'mode': mode});
2153
+ Future<Map<String, dynamic>> saveSecurityMode(
2154
+ String baseUrl,
2155
+ String mode,
2156
+ ) async {
2157
+ return putMap(baseUrl, '/api/security/mode', <String, dynamic>{
2158
+ 'mode': mode,
2159
+ });
2034
2160
  }
2035
2161
 
2036
2162
  Future<Map<String, dynamic>> saveSecurityPolicy(
@@ -2053,13 +2179,17 @@ class BackendClient {
2053
2179
  String? toolName,
2054
2180
  Map<String, dynamic>? toolArgs,
2055
2181
  }) async {
2056
- return postMap(baseUrl, '/api/security/approvals/$approvalId', <String, dynamic>{
2057
- 'decision': decision,
2058
- 'scope': scope,
2059
- if (runId != null) 'runId': runId,
2060
- if (toolName != null) 'toolName': toolName,
2061
- if (toolArgs != null) 'toolArgs': toolArgs,
2062
- });
2182
+ return postMap(
2183
+ baseUrl,
2184
+ '/api/security/approvals/$approvalId',
2185
+ <String, dynamic>{
2186
+ 'decision': decision,
2187
+ 'scope': scope,
2188
+ if (runId != null) 'runId': runId,
2189
+ if (toolName != null) 'toolName': toolName,
2190
+ if (toolArgs != null) 'toolArgs': toolArgs,
2191
+ },
2192
+ );
2063
2193
  }
2064
2194
  }
2065
2195
 
@@ -86,6 +86,12 @@ class DesktopCompanionActions {
86
86
  'metadata': <String, Object?>{
87
87
  'captureSupported': _screenCapture.isSupported,
88
88
  },
89
+ if (platformStatus['sessionLocked'] != null)
90
+ 'sessionLocked': platformStatus['sessionLocked'],
91
+ if (platformStatus['idleSeconds'] != null)
92
+ 'idleSeconds': platformStatus['idleSeconds'],
93
+ if (platformStatus['userIdle'] != null)
94
+ 'userIdle': platformStatus['userIdle'],
89
95
  };
90
96
  }
91
97
 
@@ -178,6 +184,12 @@ class DesktopCompanionActions {
178
184
  'frontmostApp': platformStatus['frontmostApp'],
179
185
  if (platformStatus['frontmostWindowTitle'] != null)
180
186
  'frontmostWindowTitle': platformStatus['frontmostWindowTitle'],
187
+ if (platformStatus['sessionLocked'] != null)
188
+ 'sessionLocked': platformStatus['sessionLocked'],
189
+ if (platformStatus['idleSeconds'] != null)
190
+ 'idleSeconds': platformStatus['idleSeconds'],
191
+ if (platformStatus['userIdle'] != null)
192
+ 'userIdle': platformStatus['userIdle'],
181
193
  };
182
194
  }
183
195
 
@@ -206,10 +218,10 @@ class DesktopCompanionActions {
206
218
  if (_looksLikeJpeg(raw)) return raw;
207
219
  // Run the pure-Dart PNG decode + JPEG encode in a background isolate so the
208
220
  // main isolate's event loop stays responsive for incoming commands.
209
- return compute(
210
- _compressJpegInIsolate,
211
- (bytes: raw, quality: quality.clamp(30, 95)),
212
- );
221
+ return compute(_compressJpegInIsolate, (
222
+ bytes: raw,
223
+ quality: quality.clamp(30, 95),
224
+ ));
213
225
  }
214
226
 
215
227
  Future<Map<String, Object?>> observe({
@@ -265,19 +277,9 @@ class DesktopCompanionActions {
265
277
  }) async {
266
278
  await _assertInputSupported('mouseMove');
267
279
  if (_usesNativeDesktopBridge) {
268
- await _nativeBridge.mouseMove(
269
- x: x,
270
- y: y,
271
- displayId: displayId,
272
- );
280
+ await _nativeBridge.mouseMove(x: x, y: y, displayId: displayId);
273
281
  } else if (defaultTargetPlatform == TargetPlatform.linux) {
274
- await _run(
275
- _ShellCommand('xdotool', <String>[
276
- 'mousemove',
277
- '$x',
278
- '$y',
279
- ]),
280
- );
282
+ await _run(_ShellCommand('xdotool', <String>['mousemove', '$x', '$y']));
281
283
  } else {
282
284
  throw Exception('mouseMove is not supported on this platform.');
283
285
  }
@@ -464,9 +466,15 @@ class DesktopCompanionActions {
464
466
  int? timeoutMs,
465
467
  String? stdinInput,
466
468
  }) async {
467
- final shell = Platform.isWindows ? 'cmd.exe' : (Platform.environment['SHELL'] ?? '/bin/sh');
468
- final args = Platform.isWindows ? <String>['/c', command] : <String>['-lc', command];
469
- final workingDir = cwd?.trim().isNotEmpty == true ? cwd : Platform.environment['HOME'];
469
+ final shell = Platform.isWindows
470
+ ? 'cmd.exe'
471
+ : (Platform.environment['SHELL'] ?? '/bin/sh');
472
+ final args = Platform.isWindows
473
+ ? <String>['/c', command]
474
+ : <String>['-lc', command];
475
+ final workingDir = cwd?.trim().isNotEmpty == true
476
+ ? cwd
477
+ : Platform.environment['HOME'];
470
478
  final startedAt = DateTime.now();
471
479
 
472
480
  final process = await Process.start(
@@ -495,7 +503,9 @@ class DesktopCompanionActions {
495
503
  });
496
504
 
497
505
  final effectiveTimeout = Duration(
498
- milliseconds: (timeoutMs != null && timeoutMs > 0) ? timeoutMs : 15 * 60 * 1000,
506
+ milliseconds: (timeoutMs != null && timeoutMs > 0)
507
+ ? timeoutMs
508
+ : 15 * 60 * 1000,
499
509
  );
500
510
 
501
511
  bool timedOut = false;
@@ -513,7 +523,9 @@ class DesktopCompanionActions {
513
523
 
514
524
  String trimOutput(StringBuffer buf) {
515
525
  final s = buf.toString().trim();
516
- return s.length > maxChars ? '${s.substring(0, maxChars)}\n...[truncated, ${s.length} total chars]' : s;
526
+ return s.length > maxChars
527
+ ? '${s.substring(0, maxChars)}\n...[truncated, ${s.length} total chars]'
528
+ : s;
517
529
  }
518
530
 
519
531
  return <String, Object?>{
@@ -607,9 +619,108 @@ class DesktopCompanionActions {
607
619
  if (_usesNativeDesktopBridge) {
608
620
  return await _nativeBridge.getStatus();
609
621
  }
622
+ if (defaultTargetPlatform == TargetPlatform.linux) {
623
+ return _linuxPlatformStatus();
624
+ }
610
625
  return const <String, Object?>{};
611
626
  }
612
627
 
628
+ Future<Map<String, Object?>> _linuxPlatformStatus() async {
629
+ try {
630
+ final sessionState = await _linuxSessionState();
631
+ final windowIdResult = await Process.run('xdotool', <String>[
632
+ 'getactivewindow',
633
+ ]);
634
+ if (windowIdResult.exitCode != 0) {
635
+ return sessionState;
636
+ }
637
+ final windowId = windowIdResult.stdout?.toString().trim() ?? '';
638
+ if (windowId.isEmpty) {
639
+ return sessionState;
640
+ }
641
+ final titleResult = await Process.run('xdotool', <String>[
642
+ 'getwindowname',
643
+ windowId,
644
+ ]);
645
+ final classResult = await Process.run('xprop', <String>[
646
+ '-id',
647
+ windowId,
648
+ 'WM_CLASS',
649
+ ]);
650
+ final windowTitle = titleResult.exitCode == 0
651
+ ? titleResult.stdout?.toString().trim() ?? ''
652
+ : '';
653
+ final appName = _parseLinuxWmClass(classResult.stdout?.toString() ?? '');
654
+ return <String, Object?>{
655
+ ...sessionState,
656
+ if (appName.isNotEmpty) 'frontmostApp': appName,
657
+ if (windowTitle.isNotEmpty) 'frontmostWindowTitle': windowTitle,
658
+ };
659
+ } catch (_) {
660
+ return const <String, Object?>{};
661
+ }
662
+ }
663
+
664
+ String _parseLinuxWmClass(String raw) {
665
+ final match = RegExp(r'"([^"]+)"\s*,\s*"([^"]+)"').firstMatch(raw);
666
+ if (match == null) {
667
+ return '';
668
+ }
669
+ final app = match.group(2)?.trim() ?? '';
670
+ if (app.isNotEmpty) {
671
+ return app;
672
+ }
673
+ return match.group(1)?.trim() ?? '';
674
+ }
675
+
676
+ Future<Map<String, Object?>> _linuxSessionState() async {
677
+ final state = <String, Object?>{};
678
+ final sessionId = Platform.environment['XDG_SESSION_ID']?.trim() ?? '';
679
+ if (sessionId.isNotEmpty) {
680
+ try {
681
+ final result = await Process.run('loginctl', <String>[
682
+ 'show-session',
683
+ sessionId,
684
+ '-p',
685
+ 'LockedHint',
686
+ '-p',
687
+ 'IdleHint',
688
+ ]);
689
+ if (result.exitCode == 0) {
690
+ final lines = result.stdout
691
+ ?.toString()
692
+ .split(RegExp(r'\r?\n'))
693
+ .map((line) => line.trim())
694
+ .where((line) => line.isNotEmpty)
695
+ .toList(growable: false) ??
696
+ const <String>[];
697
+ for (final line in lines) {
698
+ if (line.startsWith('LockedHint=')) {
699
+ state['sessionLocked'] =
700
+ line.substring('LockedHint='.length).trim() == 'yes';
701
+ } else if (line.startsWith('IdleHint=')) {
702
+ state['userIdle'] =
703
+ line.substring('IdleHint='.length).trim() == 'yes';
704
+ }
705
+ }
706
+ }
707
+ } catch (_) {}
708
+ }
709
+ try {
710
+ final result = await Process.run('xprintidle', const <String>[]);
711
+ if (result.exitCode == 0) {
712
+ final idleMs = num.tryParse(result.stdout?.toString().trim() ?? '');
713
+ if (idleMs != null) {
714
+ final idleSeconds = idleMs / 1000;
715
+ state['idleSeconds'] = idleSeconds;
716
+ state['userIdle'] =
717
+ (state['userIdle'] == true) || idleSeconds >= 300;
718
+ }
719
+ }
720
+ } catch (_) {}
721
+ return state;
722
+ }
723
+
613
724
  bool get _usesNativeDesktopBridge =>
614
725
  defaultTargetPlatform == TargetPlatform.macOS ||
615
726
  defaultTargetPlatform == TargetPlatform.windows;
@@ -7,6 +7,8 @@ import 'package:flutter/foundation.dart';
7
7
  import 'package:shared_preferences/shared_preferences.dart';
8
8
 
9
9
  import 'desktop_companion_actions.dart';
10
+ import 'desktop_ocr_bridge.dart';
11
+ import 'desktop_passive_history.dart';
10
12
  import 'desktop_screen_capture.dart';
11
13
 
12
14
  const String desktopCompanionEnabledPrefsKey = 'desktop.companion.enabled';
@@ -20,9 +22,16 @@ const String desktopCompanionActiveDisplayPrefsKey =
20
22
 
21
23
  class DesktopCompanionManager extends ChangeNotifier {
22
24
  DesktopCompanionManager({required DesktopScreenCapture screenCapture})
23
- : _actions = DesktopCompanionActions(screenCapture: screenCapture);
25
+ : _actions = DesktopCompanionActions(screenCapture: screenCapture),
26
+ _passiveHistory = DesktopPassiveHistoryManager(
27
+ actions: DesktopCompanionActions(screenCapture: screenCapture),
28
+ ocrBridge: createDesktopOcrBridge(),
29
+ ) {
30
+ _passiveHistory.addListener(_handlePassiveHistoryChanged);
31
+ }
24
32
 
25
33
  final DesktopCompanionActions _actions;
34
+ final DesktopPassiveHistoryManager _passiveHistory;
26
35
  WebSocket? _socket;
27
36
  Timer? _reconnectTimer;
28
37
  Timer? _connectionWatchdogTimer;
@@ -54,6 +63,7 @@ class DesktopCompanionManager extends ChangeNotifier {
54
63
  Map<String, Object?> _status = const <String, Object?>{};
55
64
 
56
65
  bool get enabled => _enabled;
66
+ bool get passiveHistoryEnabled => _passiveHistory.enabled;
57
67
  bool get paused => _paused;
58
68
  bool get connecting => _connecting;
59
69
  bool get connected => _connected;
@@ -61,6 +71,8 @@ class DesktopCompanionManager extends ChangeNotifier {
61
71
  String get label => _label;
62
72
  String get deviceId => _deviceId;
63
73
  String get activationId => _activationId;
74
+ String? get passiveHistoryLastUploadedAt => _passiveHistory.lastUploadedAt;
75
+ String? get passiveHistoryLastError => _passiveHistory.lastError;
64
76
  Map<String, Object?> get status => _status;
65
77
 
66
78
  Future<void> bootstrap(SharedPreferences prefs) async {
@@ -81,6 +93,7 @@ class DesktopCompanionManager extends ChangeNotifier {
81
93
  'primary';
82
94
  await prefs.setString(desktopCompanionDeviceIdPrefsKey, _deviceId);
83
95
  await prefs.setString(desktopCompanionActivationIdPrefsKey, _activationId);
96
+ await _passiveHistory.bootstrap(prefs);
84
97
  }
85
98
 
86
99
  Future<void> updateSession({
@@ -91,6 +104,7 @@ class DesktopCompanionManager extends ChangeNotifier {
91
104
  _backendUrl = backendUrl.trim();
92
105
  _sessionCookie = sessionCookie.trim();
93
106
  _authenticated = authenticated;
107
+ _syncPassiveHistoryRuntimeState();
94
108
  if (!_authenticated || !_enabled || _sessionCookie.isEmpty) {
95
109
  await disconnect();
96
110
  return;
@@ -115,6 +129,7 @@ class DesktopCompanionManager extends ChangeNotifier {
115
129
  await disconnect();
116
130
  return;
117
131
  }
132
+ _syncPassiveHistoryRuntimeState();
118
133
  await _ensureConnected();
119
134
  }
120
135
 
@@ -123,6 +138,7 @@ class DesktopCompanionManager extends ChangeNotifier {
123
138
  _label = normalized;
124
139
  await prefs.setString(desktopCompanionLabelPrefsKey, normalized);
125
140
  notifyListeners();
141
+ _syncPassiveHistoryRuntimeState();
126
142
  if (_connected) {
127
143
  _status = {..._status, 'label': normalized};
128
144
  await _sendEvent('statusChanged', <String, Object?>{'label': normalized});
@@ -132,11 +148,24 @@ class DesktopCompanionManager extends ChangeNotifier {
132
148
  Future<void> setPaused(bool value, SharedPreferences prefs) async {
133
149
  _paused = value;
134
150
  notifyListeners();
151
+ _syncPassiveHistoryRuntimeState();
135
152
  if (_connected) {
136
153
  await _sendEvent('statusChanged', <String, Object?>{'paused': value});
137
154
  }
138
155
  }
139
156
 
157
+ Future<void> setPassiveHistoryEnabled(
158
+ bool value,
159
+ SharedPreferences prefs,
160
+ ) async {
161
+ await _passiveHistory.setEnabled(value, prefs);
162
+ _syncPassiveHistoryRuntimeState();
163
+ _mergePassiveHistoryStatus();
164
+ if (_connected) {
165
+ await _sendEvent('statusChanged', _passiveHistory.statusPayload());
166
+ }
167
+ }
168
+
140
169
  Future<void> disconnect() async {
141
170
  _reconnectTimer?.cancel();
142
171
  _reconnectTimer = null;
@@ -152,6 +181,7 @@ class DesktopCompanionManager extends ChangeNotifier {
152
181
  await socket.close();
153
182
  } catch (_) {}
154
183
  }
184
+ _syncPassiveHistoryRuntimeState();
155
185
  notifyListeners();
156
186
  }
157
187
 
@@ -180,7 +210,9 @@ class DesktopCompanionManager extends ChangeNotifier {
180
210
  'hostname': _localHostname(),
181
211
  'companionEnabled': _enabled,
182
212
  'paused': _paused,
213
+ ..._passiveHistory.statusPayload(),
183
214
  };
215
+ _syncPassiveHistoryRuntimeState();
184
216
  notifyListeners();
185
217
  if (_connected) {
186
218
  await _sendEvent('statusChanged', <String, Object?>{
@@ -188,6 +220,7 @@ class DesktopCompanionManager extends ChangeNotifier {
188
220
  'capabilities': _status['capabilities'],
189
221
  'displays': _status['displays'],
190
222
  'activeDisplayId': _status['activeDisplayId'],
223
+ ..._passiveHistory.statusPayload(),
191
224
  });
192
225
  }
193
226
  return _status;
@@ -247,6 +280,7 @@ class DesktopCompanionManager extends ChangeNotifier {
247
280
  paused: _paused,
248
281
  activeDisplayId: _activeDisplayId,
249
282
  );
283
+ hello.addAll(_passiveHistory.statusPayload());
250
284
  socket.add(
251
285
  jsonEncode(<String, Object?>{'type': 'hello', 'device': hello}),
252
286
  );
@@ -281,6 +315,8 @@ class DesktopCompanionManager extends ChangeNotifier {
281
315
  _status = device is Map
282
316
  ? device.map((key, value) => MapEntry(key.toString(), value))
283
317
  : const <String, Object?>{};
318
+ _mergePassiveHistoryStatus();
319
+ _syncPassiveHistoryRuntimeState();
284
320
  _activeDisplayId =
285
321
  _status['activeDisplayId']?.toString() ?? _activeDisplayId;
286
322
  notifyListeners();
@@ -467,6 +503,7 @@ class DesktopCompanionManager extends ChangeNotifier {
467
503
  _socket = null;
468
504
  _connecting = false;
469
505
  _connected = false;
506
+ _syncPassiveHistoryRuntimeState();
470
507
  notifyListeners();
471
508
  _scheduleReconnect();
472
509
  }
@@ -481,6 +518,8 @@ class DesktopCompanionManager extends ChangeNotifier {
481
518
  _connecting = false;
482
519
  _connected = false;
483
520
  _enabled = false;
521
+ _passiveHistory.removeListener(_handlePassiveHistoryChanged);
522
+ _passiveHistory.dispose();
484
523
  final socket = _socket;
485
524
  _socket = null;
486
525
  if (socket != null) {
@@ -527,6 +566,31 @@ class DesktopCompanionManager extends ChangeNotifier {
527
566
  );
528
567
  }
529
568
 
569
+ void _syncPassiveHistoryRuntimeState() {
570
+ _passiveHistory.updateRuntimeState(
571
+ backendUrl: _backendUrl,
572
+ sessionCookie: _sessionCookie,
573
+ authenticated: _authenticated,
574
+ connected: _connected,
575
+ paused: _paused,
576
+ deviceId: _deviceId,
577
+ activationId: _activationId,
578
+ label: _label,
579
+ );
580
+ }
581
+
582
+ void _mergePassiveHistoryStatus() {
583
+ _status = <String, Object?>{..._status, ..._passiveHistory.statusPayload()};
584
+ }
585
+
586
+ void _handlePassiveHistoryChanged() {
587
+ _mergePassiveHistoryStatus();
588
+ if (_connected) {
589
+ unawaited(_sendEvent('statusChanged', _passiveHistory.statusPayload()));
590
+ }
591
+ notifyListeners();
592
+ }
593
+
530
594
  Future<Map<String, Object?>> _startStreaming(
531
595
  Map<String, Object?> payload,
532
596
  ) async {
@@ -9,11 +9,14 @@ const String desktopCompanionDeviceIdPrefsKey = 'desktop.companion.deviceId';
9
9
  const String desktopCompanionActivationIdPrefsKey =
10
10
  'desktop.companion.activationId';
11
11
  const String desktopCompanionPausedPrefsKey = 'desktop.companion.paused';
12
+ const String desktopPassiveHistoryEnabledPrefsKey =
13
+ 'desktop.passiveHistory.enabled';
12
14
 
13
15
  class DesktopCompanionManager extends ChangeNotifier {
14
16
  DesktopCompanionManager({required DesktopScreenCapture screenCapture});
15
17
 
16
18
  bool get enabled => false;
19
+ bool get passiveHistoryEnabled => false;
17
20
  bool get paused => false;
18
21
  bool get connecting => false;
19
22
  bool get connected => false;
@@ -21,6 +24,8 @@ class DesktopCompanionManager extends ChangeNotifier {
21
24
  String get label => 'Desktop';
22
25
  String get deviceId => '';
23
26
  String get activationId => '';
27
+ String? get passiveHistoryLastUploadedAt => null;
28
+ String? get passiveHistoryLastError => null;
24
29
  Map<String, Object?> get status => const <String, Object?>{};
25
30
 
26
31
  Future<void> bootstrap(SharedPreferences prefs) async {}
@@ -43,6 +48,13 @@ class DesktopCompanionManager extends ChangeNotifier {
43
48
  throw UnsupportedError('Desktop companion is not available here.');
44
49
  }
45
50
 
51
+ Future<void> setPassiveHistoryEnabled(
52
+ bool value,
53
+ SharedPreferences prefs,
54
+ ) async {
55
+ throw UnsupportedError('Desktop companion is not available here.');
56
+ }
57
+
46
58
  Future<void> disconnect() async {}
47
59
 
48
60
  Future<void> rotateIdentity(SharedPreferences prefs) async {
@@ -0,0 +1,2 @@
1
+ export 'desktop_ocr_bridge_stub.dart'
2
+ if (dart.library.io) 'desktop_ocr_bridge_io.dart';