neoagent 3.0.1-beta.8 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -10
- package/README.md +12 -3
- package/docs/automation.md +37 -0
- package/docs/benchmarking.md +102 -0
- package/docs/billing.md +34 -8
- package/docs/configuration.md +11 -2
- package/docs/getting-started.md +10 -8
- package/docs/operations.md +1 -1
- package/flutter_app/lib/main.dart +3 -0
- package/flutter_app/lib/main_account_settings.dart +138 -0
- package/flutter_app/lib/main_app_shell.dart +38 -13
- package/flutter_app/lib/main_billing.dart +1465 -0
- package/flutter_app/lib/main_chat.dart +1612 -214
- package/flutter_app/lib/main_controller.dart +398 -26
- package/flutter_app/lib/main_devices.dart +293 -207
- package/flutter_app/lib/main_models.dart +142 -0
- package/flutter_app/lib/main_navigation.dart +19 -1
- package/flutter_app/lib/main_operations.dart +288 -9
- package/flutter_app/lib/main_settings.dart +510 -300
- package/flutter_app/lib/main_shared.dart +2 -0
- package/flutter_app/lib/main_timeline.dart +1378 -0
- package/flutter_app/lib/src/backend_client.dart +149 -19
- package/flutter_app/lib/src/desktop_companion_actions.dart +132 -21
- package/flutter_app/lib/src/desktop_companion_io.dart +65 -1
- package/flutter_app/lib/src/desktop_companion_stub.dart +12 -0
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +2 -0
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +125 -0
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +30 -0
- package/flutter_app/lib/src/desktop_passive_history.dart +332 -0
- package/flutter_app/lib/src/recording_bridge_io.dart +80 -72
- package/flutter_app/lib/src/recording_bridge_web.dart +127 -114
- package/flutter_app/lib/src/recording_chunk_queue.dart +149 -0
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +182 -0
- package/flutter_app/lib/src/recording_payloads.dart +9 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +25 -0
- package/flutter_app/windows/runner/flutter_window.cpp +75 -0
- package/landing/index.html +3 -3
- package/lib/manager.js +184 -66
- package/package.json +4 -1
- package/server/admin/access.js +12 -7
- package/server/admin/admin.css +78 -0
- package/server/admin/admin.js +436 -10
- package/server/admin/billing.js +165 -42
- package/server/admin/index.html +80 -5
- package/server/admin/users.js +15 -15
- package/server/db/database.js +125 -20
- package/server/http/routes.js +1 -0
- package/server/http/static.js +4 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/canvaskit/wimp.js.symbols +8475 -8467
- package/server/public/canvaskit/wimp.wasm +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/main.dart.js +91077 -87037
- package/server/routes/account.js +53 -0
- package/server/routes/admin.js +345 -64
- package/server/routes/agents.js +203 -21
- package/server/routes/billing.js +5 -0
- package/server/routes/browser.js +8 -1
- package/server/routes/recordings.js +96 -6
- package/server/routes/screenHistory.js +140 -2
- package/server/routes/timeline.js +43 -0
- package/server/services/account/erasure.js +263 -0
- package/server/services/ai/hooks.js +4 -1
- package/server/services/ai/loop/agent_engine_core.js +8 -1
- package/server/services/ai/loop/blank_recovery.js +36 -0
- package/server/services/ai/loop/conversation_loop.js +166 -33
- package/server/services/ai/messagingFallback.js +22 -0
- package/server/services/ai/rate_limits.js +28 -5
- package/server/services/ai/systemPrompt.js +6 -5
- package/server/services/ai/taskAnalysis.js +2 -0
- package/server/services/ai/toolEvidence.js +15 -0
- package/server/services/ai/toolResult.js +40 -0
- package/server/services/ai/tools.js +163 -4
- package/server/services/android/controller.js +6 -2
- package/server/services/billing/plans.js +2 -1
- package/server/services/browser/anti_detection.js +192 -0
- package/server/services/browser/controller.js +180 -54
- package/server/services/desktop/auth.js +3 -0
- package/server/services/desktop/registry.js +50 -2
- package/server/services/integrations/google/calendar.js +22 -14
- package/server/services/manager.js +12 -42
- package/server/services/memory/ingestion_chunking.js +268 -0
- package/server/services/messaging/telnyx.js +9 -8
- package/server/services/recordings/manager.js +60 -27
- package/server/services/runtime/backends/local-vm.js +40 -22
- package/server/services/runtime/docker-vm-manager.js +157 -266
- package/server/services/runtime/guest_bootstrap.js +17 -5
- package/server/services/runtime/guest_image.js +188 -0
- package/server/services/runtime/manager.js +0 -1
- package/server/services/runtime/validation.js +3 -8
- package/server/services/social_video/service.js +60 -10
- package/server/services/tasks/runtime.js +234 -9
- package/server/services/tasks/schedule_utils.js +5 -5
- package/server/services/tasks/task_repository.js +13 -0
- package/server/services/timeline/service.js +558 -0
- package/server/services/wearable/gateway.js +1 -1
- package/server/services/websocket.js +21 -3
- package/server/services/desktop/screenRecorder.js +0 -292
- package/server/services/desktop/screen_recorder_support.js +0 -46
|
@@ -213,7 +213,11 @@ class BackendClient {
|
|
|
213
213
|
}
|
|
214
214
|
|
|
215
215
|
Future<Map<String, dynamic>> completeOnboarding(String baseUrl) async {
|
|
216
|
-
return postMap(
|
|
216
|
+
return postMap(
|
|
217
|
+
baseUrl,
|
|
218
|
+
'/api/auth/onboarding/complete',
|
|
219
|
+
const <String, dynamic>{},
|
|
220
|
+
);
|
|
217
221
|
}
|
|
218
222
|
|
|
219
223
|
Future<Map<String, dynamic>> fetchAgentProfiles(String baseUrl) async {
|
|
@@ -228,6 +232,23 @@ class BackendClient {
|
|
|
228
232
|
return getMap(baseUrl, '/api/account/usage');
|
|
229
233
|
}
|
|
230
234
|
|
|
235
|
+
/// GDPR data portability — returns a structured export of the signed-in
|
|
236
|
+
/// user's own data (credential fields are redacted server-side).
|
|
237
|
+
Future<Map<String, dynamic>> exportMyData(String baseUrl) async {
|
|
238
|
+
return getMap(baseUrl, '/api/account/export');
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/// GDPR right to erasure — permanently deletes the signed-in user's account
|
|
242
|
+
/// and all associated data. [confirmUsername] must equal the account username.
|
|
243
|
+
Future<Map<String, dynamic>> deleteMyAccount({
|
|
244
|
+
required String baseUrl,
|
|
245
|
+
required String confirmUsername,
|
|
246
|
+
}) async {
|
|
247
|
+
return postMap(baseUrl, '/api/account/delete', <String, dynamic>{
|
|
248
|
+
'confirmUsername': confirmUsername,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
|
|
231
252
|
Future<Map<String, dynamic>> updateAccountEmail({
|
|
232
253
|
required String baseUrl,
|
|
233
254
|
required String email,
|
|
@@ -342,6 +363,59 @@ class BackendClient {
|
|
|
342
363
|
return deleteMap(baseUrl, '/api/account/sessions/$sessionId');
|
|
343
364
|
}
|
|
344
365
|
|
|
366
|
+
// ── Billing ──────────────────────────────────────────────────────────────
|
|
367
|
+
|
|
368
|
+
Future<Map<String, dynamic>> getBillingPlans(String baseUrl) async {
|
|
369
|
+
return getMap(baseUrl, '/api/billing/plans', allowUnauthorized: true);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
Future<Map<String, dynamic>> getBillingInfo(String baseUrl) async {
|
|
373
|
+
return getMap(baseUrl, '/api/billing/');
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
Future<Map<String, dynamic>> getBillingInvoices(String baseUrl) async {
|
|
377
|
+
return getMap(baseUrl, '/api/billing/invoices');
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
Future<Map<String, dynamic>> createCheckoutSession({
|
|
381
|
+
required String baseUrl,
|
|
382
|
+
required String planId,
|
|
383
|
+
required String successUrl,
|
|
384
|
+
required String cancelUrl,
|
|
385
|
+
}) async {
|
|
386
|
+
return postMap(baseUrl, '/api/billing/checkout', <String, dynamic>{
|
|
387
|
+
'planId': planId,
|
|
388
|
+
'successUrl': successUrl,
|
|
389
|
+
'cancelUrl': cancelUrl,
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
Future<Map<String, dynamic>> createPortalSession({
|
|
394
|
+
required String baseUrl,
|
|
395
|
+
required String returnUrl,
|
|
396
|
+
}) async {
|
|
397
|
+
return postMap(baseUrl, '/api/billing/portal', <String, dynamic>{
|
|
398
|
+
'returnUrl': returnUrl,
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
Future<Map<String, dynamic>> cancelBillingSubscription(String baseUrl) async {
|
|
403
|
+
return postMap(baseUrl, '/api/billing/cancel', const <String, dynamic>{});
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
Future<Map<String, dynamic>> startBillingTrial({
|
|
407
|
+
required String baseUrl,
|
|
408
|
+
required String planId,
|
|
409
|
+
String? deviceFingerprint,
|
|
410
|
+
}) async {
|
|
411
|
+
return postMap(baseUrl, '/api/billing/trial', <String, dynamic>{
|
|
412
|
+
'planId': planId,
|
|
413
|
+
if (deviceFingerprint != null) 'deviceFingerprint': deviceFingerprint,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// ── Agent profiles ────────────────────────────────────────────────────────
|
|
418
|
+
|
|
345
419
|
Future<Map<String, dynamic>> createAgentProfile(
|
|
346
420
|
String baseUrl,
|
|
347
421
|
Map<String, dynamic> payload,
|
|
@@ -371,10 +445,23 @@ class BackendClient {
|
|
|
371
445
|
Future<Map<String, dynamic>> fetchChatHistory(
|
|
372
446
|
String baseUrl, {
|
|
373
447
|
String? agentId,
|
|
448
|
+
int limit = 40,
|
|
449
|
+
String? beforeCreatedAt,
|
|
450
|
+
String? beforeSource,
|
|
451
|
+
String? beforeId,
|
|
374
452
|
}) async {
|
|
453
|
+
final params = <String>[
|
|
454
|
+
'limit=$limit',
|
|
455
|
+
if (beforeCreatedAt != null && beforeCreatedAt.trim().isNotEmpty)
|
|
456
|
+
'beforeCreatedAt=${Uri.encodeQueryComponent(beforeCreatedAt.trim())}',
|
|
457
|
+
if (beforeSource != null && beforeSource.trim().isNotEmpty)
|
|
458
|
+
'beforeSource=${Uri.encodeQueryComponent(beforeSource.trim())}',
|
|
459
|
+
if (beforeId != null && beforeId.trim().isNotEmpty)
|
|
460
|
+
'beforeId=${Uri.encodeQueryComponent(beforeId.trim())}',
|
|
461
|
+
];
|
|
375
462
|
return getMap(
|
|
376
463
|
baseUrl,
|
|
377
|
-
_withAgentQuery('/api/agents/chat-history
|
|
464
|
+
_withAgentQuery('/api/agents/chat-history?${params.join('&')}', agentId),
|
|
378
465
|
);
|
|
379
466
|
}
|
|
380
467
|
|
|
@@ -461,6 +548,32 @@ class BackendClient {
|
|
|
461
548
|
return getMap(baseUrl, _withAgentQuery('/api/agents?limit=20', agentId));
|
|
462
549
|
}
|
|
463
550
|
|
|
551
|
+
Future<Map<String, dynamic>> fetchTimeline(
|
|
552
|
+
String baseUrl, {
|
|
553
|
+
Iterable<String>? sources,
|
|
554
|
+
String? agentId,
|
|
555
|
+
int limit = 50,
|
|
556
|
+
String? beforeOccurredAt,
|
|
557
|
+
int? beforeId,
|
|
558
|
+
}) async {
|
|
559
|
+
final query = <String>[
|
|
560
|
+
'limit=${limit.clamp(1, 200)}',
|
|
561
|
+
if (agentId?.trim().isNotEmpty == true)
|
|
562
|
+
'agentId=${Uri.encodeQueryComponent(agentId!.trim())}',
|
|
563
|
+
if (beforeOccurredAt?.trim().isNotEmpty == true)
|
|
564
|
+
'beforeOccurredAt=${Uri.encodeQueryComponent(beforeOccurredAt!.trim())}',
|
|
565
|
+
if (beforeId != null && beforeId > 0) 'beforeId=$beforeId',
|
|
566
|
+
];
|
|
567
|
+
for (final source in (sources ?? const <String>[])) {
|
|
568
|
+
final normalized = source.trim().toLowerCase();
|
|
569
|
+
if (normalized.isEmpty) {
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
query.add('source=${Uri.encodeQueryComponent(normalized)}');
|
|
573
|
+
}
|
|
574
|
+
return getMap(baseUrl, '/api/timeline?${query.join('&')}');
|
|
575
|
+
}
|
|
576
|
+
|
|
464
577
|
Future<Map<String, dynamic>> fetchRunSteps(
|
|
465
578
|
String baseUrl,
|
|
466
579
|
String runId,
|
|
@@ -506,9 +619,11 @@ class BackendClient {
|
|
|
506
619
|
String baseUrl, {
|
|
507
620
|
required String tokenId,
|
|
508
621
|
}) async {
|
|
509
|
-
return postMap(
|
|
510
|
-
|
|
511
|
-
|
|
622
|
+
return postMap(
|
|
623
|
+
baseUrl,
|
|
624
|
+
'/api/browser-extension/select-token',
|
|
625
|
+
<String, dynamic>{'tokenId': tokenId},
|
|
626
|
+
);
|
|
512
627
|
}
|
|
513
628
|
|
|
514
629
|
Future<Map<String, dynamic>> launchBrowser(
|
|
@@ -1204,7 +1319,10 @@ class BackendClient {
|
|
|
1204
1319
|
String? agentId,
|
|
1205
1320
|
}) async {
|
|
1206
1321
|
final response = await _httpClient.delete(
|
|
1207
|
-
_resolveUri(
|
|
1322
|
+
_resolveUri(
|
|
1323
|
+
baseUrl,
|
|
1324
|
+
_withAgentQuery('/api/integrations/$providerId/config', agentId),
|
|
1325
|
+
),
|
|
1208
1326
|
headers: const <String, String>{'Accept': 'application/json'},
|
|
1209
1327
|
);
|
|
1210
1328
|
_throwIfError(response);
|
|
@@ -1549,6 +1667,7 @@ class BackendClient {
|
|
|
1549
1667
|
required String triggerType,
|
|
1550
1668
|
required Map<String, dynamic> triggerConfig,
|
|
1551
1669
|
required String prompt,
|
|
1670
|
+
Map<String, dynamic>? taskConfig,
|
|
1552
1671
|
String? model,
|
|
1553
1672
|
bool enabled = true,
|
|
1554
1673
|
String? agentId,
|
|
@@ -1557,6 +1676,7 @@ class BackendClient {
|
|
|
1557
1676
|
'name': name,
|
|
1558
1677
|
'triggerType': triggerType,
|
|
1559
1678
|
'triggerConfig': triggerConfig,
|
|
1679
|
+
if (taskConfig != null) 'taskConfig': taskConfig,
|
|
1560
1680
|
'prompt': prompt,
|
|
1561
1681
|
'model': model,
|
|
1562
1682
|
'enabled': enabled,
|
|
@@ -1710,10 +1830,11 @@ class BackendClient {
|
|
|
1710
1830
|
required String audioBase64,
|
|
1711
1831
|
String mimeType = 'audio/pcm;rate=16000;channels=1',
|
|
1712
1832
|
}) {
|
|
1713
|
-
return postMap(
|
|
1714
|
-
|
|
1715
|
-
'
|
|
1716
|
-
|
|
1833
|
+
return postMap(
|
|
1834
|
+
baseUrl,
|
|
1835
|
+
'/api/voice-assistant/transcribe',
|
|
1836
|
+
<String, dynamic>{'audioBase64': audioBase64, 'mimeType': mimeType},
|
|
1837
|
+
);
|
|
1717
1838
|
}
|
|
1718
1839
|
|
|
1719
1840
|
Future<Map<String, dynamic>> runVoiceAssistantTurn(
|
|
@@ -2029,8 +2150,13 @@ class BackendClient {
|
|
|
2029
2150
|
return getMap(baseUrl, '/api/security/mode');
|
|
2030
2151
|
}
|
|
2031
2152
|
|
|
2032
|
-
Future<Map<String, dynamic>> saveSecurityMode(
|
|
2033
|
-
|
|
2153
|
+
Future<Map<String, dynamic>> saveSecurityMode(
|
|
2154
|
+
String baseUrl,
|
|
2155
|
+
String mode,
|
|
2156
|
+
) async {
|
|
2157
|
+
return putMap(baseUrl, '/api/security/mode', <String, dynamic>{
|
|
2158
|
+
'mode': mode,
|
|
2159
|
+
});
|
|
2034
2160
|
}
|
|
2035
2161
|
|
|
2036
2162
|
Future<Map<String, dynamic>> saveSecurityPolicy(
|
|
@@ -2053,13 +2179,17 @@ class BackendClient {
|
|
|
2053
2179
|
String? toolName,
|
|
2054
2180
|
Map<String, dynamic>? toolArgs,
|
|
2055
2181
|
}) async {
|
|
2056
|
-
return postMap(
|
|
2057
|
-
|
|
2058
|
-
'
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2182
|
+
return postMap(
|
|
2183
|
+
baseUrl,
|
|
2184
|
+
'/api/security/approvals/$approvalId',
|
|
2185
|
+
<String, dynamic>{
|
|
2186
|
+
'decision': decision,
|
|
2187
|
+
'scope': scope,
|
|
2188
|
+
if (runId != null) 'runId': runId,
|
|
2189
|
+
if (toolName != null) 'toolName': toolName,
|
|
2190
|
+
if (toolArgs != null) 'toolArgs': toolArgs,
|
|
2191
|
+
},
|
|
2192
|
+
);
|
|
2063
2193
|
}
|
|
2064
2194
|
}
|
|
2065
2195
|
|
|
@@ -86,6 +86,12 @@ class DesktopCompanionActions {
|
|
|
86
86
|
'metadata': <String, Object?>{
|
|
87
87
|
'captureSupported': _screenCapture.isSupported,
|
|
88
88
|
},
|
|
89
|
+
if (platformStatus['sessionLocked'] != null)
|
|
90
|
+
'sessionLocked': platformStatus['sessionLocked'],
|
|
91
|
+
if (platformStatus['idleSeconds'] != null)
|
|
92
|
+
'idleSeconds': platformStatus['idleSeconds'],
|
|
93
|
+
if (platformStatus['userIdle'] != null)
|
|
94
|
+
'userIdle': platformStatus['userIdle'],
|
|
89
95
|
};
|
|
90
96
|
}
|
|
91
97
|
|
|
@@ -178,6 +184,12 @@ class DesktopCompanionActions {
|
|
|
178
184
|
'frontmostApp': platformStatus['frontmostApp'],
|
|
179
185
|
if (platformStatus['frontmostWindowTitle'] != null)
|
|
180
186
|
'frontmostWindowTitle': platformStatus['frontmostWindowTitle'],
|
|
187
|
+
if (platformStatus['sessionLocked'] != null)
|
|
188
|
+
'sessionLocked': platformStatus['sessionLocked'],
|
|
189
|
+
if (platformStatus['idleSeconds'] != null)
|
|
190
|
+
'idleSeconds': platformStatus['idleSeconds'],
|
|
191
|
+
if (platformStatus['userIdle'] != null)
|
|
192
|
+
'userIdle': platformStatus['userIdle'],
|
|
181
193
|
};
|
|
182
194
|
}
|
|
183
195
|
|
|
@@ -206,10 +218,10 @@ class DesktopCompanionActions {
|
|
|
206
218
|
if (_looksLikeJpeg(raw)) return raw;
|
|
207
219
|
// Run the pure-Dart PNG decode + JPEG encode in a background isolate so the
|
|
208
220
|
// main isolate's event loop stays responsive for incoming commands.
|
|
209
|
-
return compute(
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
);
|
|
221
|
+
return compute(_compressJpegInIsolate, (
|
|
222
|
+
bytes: raw,
|
|
223
|
+
quality: quality.clamp(30, 95),
|
|
224
|
+
));
|
|
213
225
|
}
|
|
214
226
|
|
|
215
227
|
Future<Map<String, Object?>> observe({
|
|
@@ -265,19 +277,9 @@ class DesktopCompanionActions {
|
|
|
265
277
|
}) async {
|
|
266
278
|
await _assertInputSupported('mouseMove');
|
|
267
279
|
if (_usesNativeDesktopBridge) {
|
|
268
|
-
await _nativeBridge.mouseMove(
|
|
269
|
-
x: x,
|
|
270
|
-
y: y,
|
|
271
|
-
displayId: displayId,
|
|
272
|
-
);
|
|
280
|
+
await _nativeBridge.mouseMove(x: x, y: y, displayId: displayId);
|
|
273
281
|
} else if (defaultTargetPlatform == TargetPlatform.linux) {
|
|
274
|
-
await _run(
|
|
275
|
-
_ShellCommand('xdotool', <String>[
|
|
276
|
-
'mousemove',
|
|
277
|
-
'$x',
|
|
278
|
-
'$y',
|
|
279
|
-
]),
|
|
280
|
-
);
|
|
282
|
+
await _run(_ShellCommand('xdotool', <String>['mousemove', '$x', '$y']));
|
|
281
283
|
} else {
|
|
282
284
|
throw Exception('mouseMove is not supported on this platform.');
|
|
283
285
|
}
|
|
@@ -464,9 +466,15 @@ class DesktopCompanionActions {
|
|
|
464
466
|
int? timeoutMs,
|
|
465
467
|
String? stdinInput,
|
|
466
468
|
}) async {
|
|
467
|
-
final shell = Platform.isWindows
|
|
468
|
-
|
|
469
|
-
|
|
469
|
+
final shell = Platform.isWindows
|
|
470
|
+
? 'cmd.exe'
|
|
471
|
+
: (Platform.environment['SHELL'] ?? '/bin/sh');
|
|
472
|
+
final args = Platform.isWindows
|
|
473
|
+
? <String>['/c', command]
|
|
474
|
+
: <String>['-lc', command];
|
|
475
|
+
final workingDir = cwd?.trim().isNotEmpty == true
|
|
476
|
+
? cwd
|
|
477
|
+
: Platform.environment['HOME'];
|
|
470
478
|
final startedAt = DateTime.now();
|
|
471
479
|
|
|
472
480
|
final process = await Process.start(
|
|
@@ -495,7 +503,9 @@ class DesktopCompanionActions {
|
|
|
495
503
|
});
|
|
496
504
|
|
|
497
505
|
final effectiveTimeout = Duration(
|
|
498
|
-
milliseconds: (timeoutMs != null && timeoutMs > 0)
|
|
506
|
+
milliseconds: (timeoutMs != null && timeoutMs > 0)
|
|
507
|
+
? timeoutMs
|
|
508
|
+
: 15 * 60 * 1000,
|
|
499
509
|
);
|
|
500
510
|
|
|
501
511
|
bool timedOut = false;
|
|
@@ -513,7 +523,9 @@ class DesktopCompanionActions {
|
|
|
513
523
|
|
|
514
524
|
String trimOutput(StringBuffer buf) {
|
|
515
525
|
final s = buf.toString().trim();
|
|
516
|
-
return s.length > maxChars
|
|
526
|
+
return s.length > maxChars
|
|
527
|
+
? '${s.substring(0, maxChars)}\n...[truncated, ${s.length} total chars]'
|
|
528
|
+
: s;
|
|
517
529
|
}
|
|
518
530
|
|
|
519
531
|
return <String, Object?>{
|
|
@@ -607,9 +619,108 @@ class DesktopCompanionActions {
|
|
|
607
619
|
if (_usesNativeDesktopBridge) {
|
|
608
620
|
return await _nativeBridge.getStatus();
|
|
609
621
|
}
|
|
622
|
+
if (defaultTargetPlatform == TargetPlatform.linux) {
|
|
623
|
+
return _linuxPlatformStatus();
|
|
624
|
+
}
|
|
610
625
|
return const <String, Object?>{};
|
|
611
626
|
}
|
|
612
627
|
|
|
628
|
+
Future<Map<String, Object?>> _linuxPlatformStatus() async {
|
|
629
|
+
try {
|
|
630
|
+
final sessionState = await _linuxSessionState();
|
|
631
|
+
final windowIdResult = await Process.run('xdotool', <String>[
|
|
632
|
+
'getactivewindow',
|
|
633
|
+
]);
|
|
634
|
+
if (windowIdResult.exitCode != 0) {
|
|
635
|
+
return sessionState;
|
|
636
|
+
}
|
|
637
|
+
final windowId = windowIdResult.stdout?.toString().trim() ?? '';
|
|
638
|
+
if (windowId.isEmpty) {
|
|
639
|
+
return sessionState;
|
|
640
|
+
}
|
|
641
|
+
final titleResult = await Process.run('xdotool', <String>[
|
|
642
|
+
'getwindowname',
|
|
643
|
+
windowId,
|
|
644
|
+
]);
|
|
645
|
+
final classResult = await Process.run('xprop', <String>[
|
|
646
|
+
'-id',
|
|
647
|
+
windowId,
|
|
648
|
+
'WM_CLASS',
|
|
649
|
+
]);
|
|
650
|
+
final windowTitle = titleResult.exitCode == 0
|
|
651
|
+
? titleResult.stdout?.toString().trim() ?? ''
|
|
652
|
+
: '';
|
|
653
|
+
final appName = _parseLinuxWmClass(classResult.stdout?.toString() ?? '');
|
|
654
|
+
return <String, Object?>{
|
|
655
|
+
...sessionState,
|
|
656
|
+
if (appName.isNotEmpty) 'frontmostApp': appName,
|
|
657
|
+
if (windowTitle.isNotEmpty) 'frontmostWindowTitle': windowTitle,
|
|
658
|
+
};
|
|
659
|
+
} catch (_) {
|
|
660
|
+
return const <String, Object?>{};
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
String _parseLinuxWmClass(String raw) {
|
|
665
|
+
final match = RegExp(r'"([^"]+)"\s*,\s*"([^"]+)"').firstMatch(raw);
|
|
666
|
+
if (match == null) {
|
|
667
|
+
return '';
|
|
668
|
+
}
|
|
669
|
+
final app = match.group(2)?.trim() ?? '';
|
|
670
|
+
if (app.isNotEmpty) {
|
|
671
|
+
return app;
|
|
672
|
+
}
|
|
673
|
+
return match.group(1)?.trim() ?? '';
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
Future<Map<String, Object?>> _linuxSessionState() async {
|
|
677
|
+
final state = <String, Object?>{};
|
|
678
|
+
final sessionId = Platform.environment['XDG_SESSION_ID']?.trim() ?? '';
|
|
679
|
+
if (sessionId.isNotEmpty) {
|
|
680
|
+
try {
|
|
681
|
+
final result = await Process.run('loginctl', <String>[
|
|
682
|
+
'show-session',
|
|
683
|
+
sessionId,
|
|
684
|
+
'-p',
|
|
685
|
+
'LockedHint',
|
|
686
|
+
'-p',
|
|
687
|
+
'IdleHint',
|
|
688
|
+
]);
|
|
689
|
+
if (result.exitCode == 0) {
|
|
690
|
+
final lines = result.stdout
|
|
691
|
+
?.toString()
|
|
692
|
+
.split(RegExp(r'\r?\n'))
|
|
693
|
+
.map((line) => line.trim())
|
|
694
|
+
.where((line) => line.isNotEmpty)
|
|
695
|
+
.toList(growable: false) ??
|
|
696
|
+
const <String>[];
|
|
697
|
+
for (final line in lines) {
|
|
698
|
+
if (line.startsWith('LockedHint=')) {
|
|
699
|
+
state['sessionLocked'] =
|
|
700
|
+
line.substring('LockedHint='.length).trim() == 'yes';
|
|
701
|
+
} else if (line.startsWith('IdleHint=')) {
|
|
702
|
+
state['userIdle'] =
|
|
703
|
+
line.substring('IdleHint='.length).trim() == 'yes';
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
} catch (_) {}
|
|
708
|
+
}
|
|
709
|
+
try {
|
|
710
|
+
final result = await Process.run('xprintidle', const <String>[]);
|
|
711
|
+
if (result.exitCode == 0) {
|
|
712
|
+
final idleMs = num.tryParse(result.stdout?.toString().trim() ?? '');
|
|
713
|
+
if (idleMs != null) {
|
|
714
|
+
final idleSeconds = idleMs / 1000;
|
|
715
|
+
state['idleSeconds'] = idleSeconds;
|
|
716
|
+
state['userIdle'] =
|
|
717
|
+
(state['userIdle'] == true) || idleSeconds >= 300;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
} catch (_) {}
|
|
721
|
+
return state;
|
|
722
|
+
}
|
|
723
|
+
|
|
613
724
|
bool get _usesNativeDesktopBridge =>
|
|
614
725
|
defaultTargetPlatform == TargetPlatform.macOS ||
|
|
615
726
|
defaultTargetPlatform == TargetPlatform.windows;
|
|
@@ -7,6 +7,8 @@ import 'package:flutter/foundation.dart';
|
|
|
7
7
|
import 'package:shared_preferences/shared_preferences.dart';
|
|
8
8
|
|
|
9
9
|
import 'desktop_companion_actions.dart';
|
|
10
|
+
import 'desktop_ocr_bridge.dart';
|
|
11
|
+
import 'desktop_passive_history.dart';
|
|
10
12
|
import 'desktop_screen_capture.dart';
|
|
11
13
|
|
|
12
14
|
const String desktopCompanionEnabledPrefsKey = 'desktop.companion.enabled';
|
|
@@ -20,9 +22,16 @@ const String desktopCompanionActiveDisplayPrefsKey =
|
|
|
20
22
|
|
|
21
23
|
class DesktopCompanionManager extends ChangeNotifier {
|
|
22
24
|
DesktopCompanionManager({required DesktopScreenCapture screenCapture})
|
|
23
|
-
: _actions = DesktopCompanionActions(screenCapture: screenCapture)
|
|
25
|
+
: _actions = DesktopCompanionActions(screenCapture: screenCapture),
|
|
26
|
+
_passiveHistory = DesktopPassiveHistoryManager(
|
|
27
|
+
actions: DesktopCompanionActions(screenCapture: screenCapture),
|
|
28
|
+
ocrBridge: createDesktopOcrBridge(),
|
|
29
|
+
) {
|
|
30
|
+
_passiveHistory.addListener(_handlePassiveHistoryChanged);
|
|
31
|
+
}
|
|
24
32
|
|
|
25
33
|
final DesktopCompanionActions _actions;
|
|
34
|
+
final DesktopPassiveHistoryManager _passiveHistory;
|
|
26
35
|
WebSocket? _socket;
|
|
27
36
|
Timer? _reconnectTimer;
|
|
28
37
|
Timer? _connectionWatchdogTimer;
|
|
@@ -54,6 +63,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
54
63
|
Map<String, Object?> _status = const <String, Object?>{};
|
|
55
64
|
|
|
56
65
|
bool get enabled => _enabled;
|
|
66
|
+
bool get passiveHistoryEnabled => _passiveHistory.enabled;
|
|
57
67
|
bool get paused => _paused;
|
|
58
68
|
bool get connecting => _connecting;
|
|
59
69
|
bool get connected => _connected;
|
|
@@ -61,6 +71,8 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
61
71
|
String get label => _label;
|
|
62
72
|
String get deviceId => _deviceId;
|
|
63
73
|
String get activationId => _activationId;
|
|
74
|
+
String? get passiveHistoryLastUploadedAt => _passiveHistory.lastUploadedAt;
|
|
75
|
+
String? get passiveHistoryLastError => _passiveHistory.lastError;
|
|
64
76
|
Map<String, Object?> get status => _status;
|
|
65
77
|
|
|
66
78
|
Future<void> bootstrap(SharedPreferences prefs) async {
|
|
@@ -81,6 +93,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
81
93
|
'primary';
|
|
82
94
|
await prefs.setString(desktopCompanionDeviceIdPrefsKey, _deviceId);
|
|
83
95
|
await prefs.setString(desktopCompanionActivationIdPrefsKey, _activationId);
|
|
96
|
+
await _passiveHistory.bootstrap(prefs);
|
|
84
97
|
}
|
|
85
98
|
|
|
86
99
|
Future<void> updateSession({
|
|
@@ -91,6 +104,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
91
104
|
_backendUrl = backendUrl.trim();
|
|
92
105
|
_sessionCookie = sessionCookie.trim();
|
|
93
106
|
_authenticated = authenticated;
|
|
107
|
+
_syncPassiveHistoryRuntimeState();
|
|
94
108
|
if (!_authenticated || !_enabled || _sessionCookie.isEmpty) {
|
|
95
109
|
await disconnect();
|
|
96
110
|
return;
|
|
@@ -115,6 +129,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
115
129
|
await disconnect();
|
|
116
130
|
return;
|
|
117
131
|
}
|
|
132
|
+
_syncPassiveHistoryRuntimeState();
|
|
118
133
|
await _ensureConnected();
|
|
119
134
|
}
|
|
120
135
|
|
|
@@ -123,6 +138,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
123
138
|
_label = normalized;
|
|
124
139
|
await prefs.setString(desktopCompanionLabelPrefsKey, normalized);
|
|
125
140
|
notifyListeners();
|
|
141
|
+
_syncPassiveHistoryRuntimeState();
|
|
126
142
|
if (_connected) {
|
|
127
143
|
_status = {..._status, 'label': normalized};
|
|
128
144
|
await _sendEvent('statusChanged', <String, Object?>{'label': normalized});
|
|
@@ -132,11 +148,24 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
132
148
|
Future<void> setPaused(bool value, SharedPreferences prefs) async {
|
|
133
149
|
_paused = value;
|
|
134
150
|
notifyListeners();
|
|
151
|
+
_syncPassiveHistoryRuntimeState();
|
|
135
152
|
if (_connected) {
|
|
136
153
|
await _sendEvent('statusChanged', <String, Object?>{'paused': value});
|
|
137
154
|
}
|
|
138
155
|
}
|
|
139
156
|
|
|
157
|
+
Future<void> setPassiveHistoryEnabled(
|
|
158
|
+
bool value,
|
|
159
|
+
SharedPreferences prefs,
|
|
160
|
+
) async {
|
|
161
|
+
await _passiveHistory.setEnabled(value, prefs);
|
|
162
|
+
_syncPassiveHistoryRuntimeState();
|
|
163
|
+
_mergePassiveHistoryStatus();
|
|
164
|
+
if (_connected) {
|
|
165
|
+
await _sendEvent('statusChanged', _passiveHistory.statusPayload());
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
140
169
|
Future<void> disconnect() async {
|
|
141
170
|
_reconnectTimer?.cancel();
|
|
142
171
|
_reconnectTimer = null;
|
|
@@ -152,6 +181,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
152
181
|
await socket.close();
|
|
153
182
|
} catch (_) {}
|
|
154
183
|
}
|
|
184
|
+
_syncPassiveHistoryRuntimeState();
|
|
155
185
|
notifyListeners();
|
|
156
186
|
}
|
|
157
187
|
|
|
@@ -180,7 +210,9 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
180
210
|
'hostname': _localHostname(),
|
|
181
211
|
'companionEnabled': _enabled,
|
|
182
212
|
'paused': _paused,
|
|
213
|
+
..._passiveHistory.statusPayload(),
|
|
183
214
|
};
|
|
215
|
+
_syncPassiveHistoryRuntimeState();
|
|
184
216
|
notifyListeners();
|
|
185
217
|
if (_connected) {
|
|
186
218
|
await _sendEvent('statusChanged', <String, Object?>{
|
|
@@ -188,6 +220,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
188
220
|
'capabilities': _status['capabilities'],
|
|
189
221
|
'displays': _status['displays'],
|
|
190
222
|
'activeDisplayId': _status['activeDisplayId'],
|
|
223
|
+
..._passiveHistory.statusPayload(),
|
|
191
224
|
});
|
|
192
225
|
}
|
|
193
226
|
return _status;
|
|
@@ -247,6 +280,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
247
280
|
paused: _paused,
|
|
248
281
|
activeDisplayId: _activeDisplayId,
|
|
249
282
|
);
|
|
283
|
+
hello.addAll(_passiveHistory.statusPayload());
|
|
250
284
|
socket.add(
|
|
251
285
|
jsonEncode(<String, Object?>{'type': 'hello', 'device': hello}),
|
|
252
286
|
);
|
|
@@ -281,6 +315,8 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
281
315
|
_status = device is Map
|
|
282
316
|
? device.map((key, value) => MapEntry(key.toString(), value))
|
|
283
317
|
: const <String, Object?>{};
|
|
318
|
+
_mergePassiveHistoryStatus();
|
|
319
|
+
_syncPassiveHistoryRuntimeState();
|
|
284
320
|
_activeDisplayId =
|
|
285
321
|
_status['activeDisplayId']?.toString() ?? _activeDisplayId;
|
|
286
322
|
notifyListeners();
|
|
@@ -467,6 +503,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
467
503
|
_socket = null;
|
|
468
504
|
_connecting = false;
|
|
469
505
|
_connected = false;
|
|
506
|
+
_syncPassiveHistoryRuntimeState();
|
|
470
507
|
notifyListeners();
|
|
471
508
|
_scheduleReconnect();
|
|
472
509
|
}
|
|
@@ -481,6 +518,8 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
481
518
|
_connecting = false;
|
|
482
519
|
_connected = false;
|
|
483
520
|
_enabled = false;
|
|
521
|
+
_passiveHistory.removeListener(_handlePassiveHistoryChanged);
|
|
522
|
+
_passiveHistory.dispose();
|
|
484
523
|
final socket = _socket;
|
|
485
524
|
_socket = null;
|
|
486
525
|
if (socket != null) {
|
|
@@ -527,6 +566,31 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
527
566
|
);
|
|
528
567
|
}
|
|
529
568
|
|
|
569
|
+
void _syncPassiveHistoryRuntimeState() {
|
|
570
|
+
_passiveHistory.updateRuntimeState(
|
|
571
|
+
backendUrl: _backendUrl,
|
|
572
|
+
sessionCookie: _sessionCookie,
|
|
573
|
+
authenticated: _authenticated,
|
|
574
|
+
connected: _connected,
|
|
575
|
+
paused: _paused,
|
|
576
|
+
deviceId: _deviceId,
|
|
577
|
+
activationId: _activationId,
|
|
578
|
+
label: _label,
|
|
579
|
+
);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
void _mergePassiveHistoryStatus() {
|
|
583
|
+
_status = <String, Object?>{..._status, ..._passiveHistory.statusPayload()};
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
void _handlePassiveHistoryChanged() {
|
|
587
|
+
_mergePassiveHistoryStatus();
|
|
588
|
+
if (_connected) {
|
|
589
|
+
unawaited(_sendEvent('statusChanged', _passiveHistory.statusPayload()));
|
|
590
|
+
}
|
|
591
|
+
notifyListeners();
|
|
592
|
+
}
|
|
593
|
+
|
|
530
594
|
Future<Map<String, Object?>> _startStreaming(
|
|
531
595
|
Map<String, Object?> payload,
|
|
532
596
|
) async {
|
|
@@ -9,11 +9,14 @@ const String desktopCompanionDeviceIdPrefsKey = 'desktop.companion.deviceId';
|
|
|
9
9
|
const String desktopCompanionActivationIdPrefsKey =
|
|
10
10
|
'desktop.companion.activationId';
|
|
11
11
|
const String desktopCompanionPausedPrefsKey = 'desktop.companion.paused';
|
|
12
|
+
const String desktopPassiveHistoryEnabledPrefsKey =
|
|
13
|
+
'desktop.passiveHistory.enabled';
|
|
12
14
|
|
|
13
15
|
class DesktopCompanionManager extends ChangeNotifier {
|
|
14
16
|
DesktopCompanionManager({required DesktopScreenCapture screenCapture});
|
|
15
17
|
|
|
16
18
|
bool get enabled => false;
|
|
19
|
+
bool get passiveHistoryEnabled => false;
|
|
17
20
|
bool get paused => false;
|
|
18
21
|
bool get connecting => false;
|
|
19
22
|
bool get connected => false;
|
|
@@ -21,6 +24,8 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
21
24
|
String get label => 'Desktop';
|
|
22
25
|
String get deviceId => '';
|
|
23
26
|
String get activationId => '';
|
|
27
|
+
String? get passiveHistoryLastUploadedAt => null;
|
|
28
|
+
String? get passiveHistoryLastError => null;
|
|
24
29
|
Map<String, Object?> get status => const <String, Object?>{};
|
|
25
30
|
|
|
26
31
|
Future<void> bootstrap(SharedPreferences prefs) async {}
|
|
@@ -43,6 +48,13 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
43
48
|
throw UnsupportedError('Desktop companion is not available here.');
|
|
44
49
|
}
|
|
45
50
|
|
|
51
|
+
Future<void> setPassiveHistoryEnabled(
|
|
52
|
+
bool value,
|
|
53
|
+
SharedPreferences prefs,
|
|
54
|
+
) async {
|
|
55
|
+
throw UnsupportedError('Desktop companion is not available here.');
|
|
56
|
+
}
|
|
57
|
+
|
|
46
58
|
Future<void> disconnect() async {}
|
|
47
59
|
|
|
48
60
|
Future<void> rotateIdentity(SharedPreferences prefs) async {
|