neoagent 2.4.1-beta.13 → 2.4.1-beta.15

package/extensions/chrome-browser/background.mjs

@@ -8,6 +8,8 @@ let reconnectTimer = null;
 let suppressSocketClose = false;
 const DEFAULT_FETCH_TIMEOUT_MS = 10000;
 const DEFAULT_WS_CONNECT_TIMEOUT_MS = 10000;
+const KEEPALIVE_ALARM_NAME = 'neoagent-extension-keepalive';
+const KEEPALIVE_ALARM_MINUTES = 1;
 const EXTENSION_PROTOCOL_VERSION = 1;
 
 function getStorage(keys = STORAGE_KEYS) {
@@ -81,6 +83,12 @@ function clearReconnectTimer() {
   reconnectTimer = null;
 }
 
+function ensureKeepaliveAlarm() {
+  chrome.alarms?.create?.(KEEPALIVE_ALARM_NAME, {
+    periodInMinutes: KEEPALIVE_ALARM_MINUTES,
+  });
+}
+
 async function handleSocketDisconnected(ws) {
   if (socket !== ws) {
     return;
@@ -308,4 +316,22 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
   return true;
 });
 
+chrome.alarms?.onAlarm?.addListener((alarm) => {
+  if (alarm?.name !== KEEPALIVE_ALARM_NAME) return;
+  connect().catch((error) => {
+    console.error('NeoAgent keepalive reconnect failed', error);
+  });
+});
+
+chrome.runtime.onStartup?.addListener(() => {
+  ensureKeepaliveAlarm();
+  connect().catch(() => {});
+});
+
+chrome.runtime.onInstalled?.addListener(() => {
+  ensureKeepaliveAlarm();
+  connect().catch(() => {});
+});
+
+ensureKeepaliveAlarm();
 connect().catch(() => {});

package/extensions/chrome-browser/manifest.json

@@ -1,11 +1,11 @@
 {
   "manifest_version": 3,
   "name": "NeoAgent Browser",
-  "version": "2.4.1.7",
-  "version_name": "2.4.1-beta.7",
+  "version": "2.4.1.8",
+  "version_name": "2.4.1-beta.8",
   "description": "Connect this Chrome browser to NeoAgent for browser automation.",
   "minimum_chrome_version": "118",
-  "permissions": ["debugger", "storage", "tabs"],
+  "permissions": ["alarms", "debugger", "storage", "tabs"],
   "host_permissions": ["http://*/*", "https://*/*"],
   "icons": {
     "16": "icons/icon16.png",

package/flutter_app/android/app/src/main/AndroidManifest.xml

@@ -124,7 +124,8 @@
         
         <service
             android:name=".auto.NeoAgentCarAppService"
-            android:exported="true">
+            android:exported="true"
+            android:permission="android.car.permission.BIND_CAR_APP_SERVICE">
             <intent-filter>
                 <action android:name="androidx.car.app.CarAppService" />
                 <category android:name="androidx.car.app.category.IOT" />

package/flutter_app/lib/main_chat.dart

@@ -9,23 +9,35 @@ class ChatPanel extends StatefulWidget {
   State<ChatPanel> createState() => _ChatPanelState();
 }
 
-class _ChatPanelState extends State<ChatPanel> {
+class _ChatPanelState extends State<ChatPanel> with WidgetsBindingObserver {
+  static const double _autoScrollBottomThreshold = 120;
+  static const int _autoScrollSettlePasses = 4;
+
   late final TextEditingController _composerController;
   final ScrollController _scrollController = ScrollController();
   List<SharedChatAttachment> _pendingSharedAttachments =
       const <SharedChatAttachment>[];
   String? _appliedSharedPayloadSignature;
-  int _lastMessageCount = 0;
-  int _lastToolCount = 0;
-  String _lastStream = '';
+  String _lastScrollContentSignature = '';
+  bool _stickToBottom = true;
+  bool _ignoreScrollUpdates = false;
+  int _scrollGeneration = 0;
   bool _isSendingChatMessage = false;
+  bool _isDictating = false;
+  bool _isTranscribing = false;
+  LiveVoiceCapture? _dictationCapture;
+  final List<Uint8List> _dictationChunks = [];
 
   @override
   void initState() {
     super.initState();
+    WidgetsBinding.instance.addObserver(this);
     _composerController = TextEditingController();
+    _composerController.addListener(_handleComposerLayoutChanged);
+    _scrollController.addListener(_handleScrollPositionChanged);
     widget.controller.addListener(_consumeQueuedDraft);
     _consumeQueuedDraft();
+    _scheduleScrollToBottom(force: true);
   }
 
   @override
@@ -35,18 +47,98 @@ class _ChatPanelState extends State<ChatPanel> {
       oldWidget.controller.removeListener(_consumeQueuedDraft);
       widget.controller.addListener(_consumeQueuedDraft);
       _appliedSharedPayloadSignature = null;
+      _lastScrollContentSignature = '';
+      _stickToBottom = true;
       _consumeQueuedDraft();
+      _scheduleScrollToBottom(force: true);
     }
   }
 
   @override
   void dispose() {
+    WidgetsBinding.instance.removeObserver(this);
     widget.controller.removeListener(_consumeQueuedDraft);
+    _composerController.removeListener(_handleComposerLayoutChanged);
+    _scrollController.removeListener(_handleScrollPositionChanged);
     _composerController.dispose();
     _scrollController.dispose();
+    _dictationCapture?.dispose();
     super.dispose();
   }
 
+  @override
+  void didChangeMetrics() {
+    super.didChangeMetrics();
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
+  }
+
+  Future<void> _startDictation() async {
+    if (_isDictating || _isTranscribing) return;
+    final capture = LiveVoiceCapture();
+    _dictationCapture = capture;
+    _dictationChunks.clear();
+    try {
+      await capture.start(
+        onChunk: (chunk) => _dictationChunks.add(chunk),
+        sampleRate: 16000,
+        channels: 1,
+      );
+      if (mounted) setState(() => _isDictating = true);
+    } catch (e) {
+      await capture.dispose();
+      _dictationCapture = null;
+      if (mounted) {
+        ScaffoldMessenger.of(
+          context,
+        ).showSnackBar(SnackBar(content: Text('Microphone error: $e')));
+      }
+    }
+  }
+
+  Future<void> _stopAndTranscribe() async {
+    if (!_isDictating) return;
+    final capture = _dictationCapture;
+    _dictationCapture = null;
+    setState(() {
+      _isDictating = false;
+      _isTranscribing = true;
+    });
+    try {
+      await capture?.stop();
+      if (_dictationChunks.isEmpty) return;
+      final allBytes = _dictationChunks.fold<List<int>>(
+        <int>[],
+        (acc, chunk) => acc..addAll(chunk),
+      );
+      final audioBase64 = base64Encode(Uint8List.fromList(allBytes));
+      final transcript = await widget.controller.transcribeDictationAudio(
+        audioBase64: audioBase64,
+      );
+      if (mounted && transcript.isNotEmpty) {
+        final current = _composerController.text;
+        final separator = current.isNotEmpty && !current.endsWith(' ')
+            ? ' '
+            : '';
+        _composerController.text = '$current$separator$transcript';
+        _composerController.selection = TextSelection.collapsed(
+          offset: _composerController.text.length,
+        );
+      }
+    } catch (e) {
+      if (mounted) {
+        ScaffoldMessenger.of(
+          context,
+        ).showSnackBar(SnackBar(content: Text('Transcription failed: $e')));
+      }
+    } finally {
+      await capture?.dispose();
+      _dictationChunks.clear();
+      if (mounted) setState(() => _isTranscribing = false);
+    }
+  }
+
   void _consumeQueuedDraft() {
     final draft = widget.controller.peekPendingChatDraft();
     final attachments = widget.controller.peekPendingSharedChatAttachments();
@@ -90,6 +182,9 @@ class _ChatPanelState extends State<ChatPanel> {
     setState(() {
       _pendingSharedAttachments = const <SharedChatAttachment>[];
     });
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
   }
 
   String _mimeTypeForFileName(String fileName) {
@@ -136,10 +231,9 @@ class _ChatPanelState extends State<ChatPanel> {
     }
     final attachments = result.files
         .where(
-          (file) =>
-              kIsWeb
-                  ? file.bytes != null
-                  : file.path?.trim().isNotEmpty == true,
+          (file) => kIsWeb
+              ? file.bytes != null
+              : file.path?.trim().isNotEmpty == true,
         )
         .map(
           (file) => SharedChatAttachment(
@@ -161,108 +255,178 @@ class _ChatPanelState extends State<ChatPanel> {
         ...attachments,
       ];
     });
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
   }
 
   bool get _isNearBottom {
     if (!_scrollController.hasClients) return true;
     final pos = _scrollController.position;
     if (!pos.hasContentDimensions) return true;
-    return pos.pixels >= pos.maxScrollExtent - 80;
+    return pos.pixels >= pos.maxScrollExtent - _autoScrollBottomThreshold;
   }
 
-  void _scrollToBottom() {
-    WidgetsBinding.instance.addPostFrameCallback((_) {
-      if (!mounted || !_scrollController.hasClients) return;
-      final pos = _scrollController.position;
-      if (pos.hasContentDimensions) {
-        _scrollController.jumpTo(pos.maxScrollExtent);
-      }
-    });
+  void _handleScrollPositionChanged() {
+    if (_ignoreScrollUpdates || !_scrollController.hasClients) return;
+    final nearBottom = _isNearBottom;
+    if (_stickToBottom && !nearBottom) {
+      _scrollGeneration++;
+    }
+    _stickToBottom = nearBottom;
+  }
+
+  void _handleComposerLayoutChanged() {
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
+  }
+
+  String _scrollContentSignature(
+    List<ChatEntry> messages,
+    NeoAgentController controller,
+  ) {
+    final last = messages.isEmpty ? null : messages.last;
+    final activeRun = controller.activeRun;
+    return <Object?>[
+      messages.length,
+      last?.id,
+      last?.role,
+      last?.content.length,
+      last?.typing,
+      controller.toolEvents.length,
+      activeRun?.runId,
+      activeRun?.phase,
+      activeRun?.iteration,
+      controller.streamingAssistant.length,
+      controller.isSendingMessage,
+    ].join('|');
+  }
+
+  void _scheduleScrollToBottom({bool force = false}) {
+    if (!force && !_stickToBottom) return;
+    final generation = ++_scrollGeneration;
+
+    void settle(int remainingPasses) {
+      WidgetsBinding.instance.addPostFrameCallback((_) {
+        if (!mounted ||
+            generation != _scrollGeneration ||
+            !_scrollController.hasClients) {
+          return;
+        }
+        final pos = _scrollController.position;
+        if (pos.hasContentDimensions) {
+          final target = pos.maxScrollExtent;
+          if ((pos.pixels - target).abs() > 0.5) {
+            _ignoreScrollUpdates = true;
+            _scrollController.jumpTo(target);
+            _ignoreScrollUpdates = false;
+          }
+        }
+        _stickToBottom = true;
+        if (remainingPasses > 0) {
+          Timer(const Duration(milliseconds: 24), () {
+            if (mounted && generation == _scrollGeneration) {
+              settle(remainingPasses - 1);
+            }
+          });
+        }
+      });
+    }
+
+    settle(_autoScrollSettlePasses);
+  }
+
+  void _maybeFollowChatContent(
+    List<ChatEntry> messages,
+    NeoAgentController controller,
+  ) {
+    final signature = _scrollContentSignature(messages, controller);
+    if (_lastScrollContentSignature == signature) return;
+    final isInitialContent = _lastScrollContentSignature.isEmpty;
+    final shouldFollow = _stickToBottom || isInitialContent;
+    _lastScrollContentSignature = signature;
+    if (shouldFollow) {
+      _scheduleScrollToBottom(force: isInitialContent);
+    }
   }
 
   @override
   Widget build(BuildContext context) {
     final controller = widget.controller;
     final messages = controller.visibleChatMessages;
-    if (_lastMessageCount != messages.length ||
-        _lastToolCount != controller.toolEvents.length ||
-        _lastStream != controller.streamingAssistant) {
-      _lastMessageCount = messages.length;
-      _lastToolCount = controller.toolEvents.length;
-      _lastStream = controller.streamingAssistant;
-      if (_isNearBottom) _scrollToBottom();
-    }
+    _maybeFollowChatContent(messages, controller);
 
     return Column(
       children: <Widget>[
         Expanded(
           child: SelectionArea(
             child: ListView(
-            controller: _scrollController,
-            padding: _pagePadding(context),
-            children: <Widget>[
-              _PageTitle(
-                title: 'Chat',
-                subtitle: 'Live agent chat with tool and stream status.',
-                trailing: Wrap(
-                  spacing: 10,
-                  runSpacing: 10,
-                  crossAxisAlignment: WrapCrossAlignment.center,
-                  children: <Widget>[
-                    FilledButton.icon(
-                      onPressed: () => controller.setSelectedSection(
-                        AppSection.voiceAssistant,
+              controller: _scrollController,
+              padding: _pagePadding(context),
+              children: <Widget>[
+                _PageTitle(
+                  title: 'Chat',
+                  subtitle: 'Live agent chat with tool and stream status.',
+                  trailing: Wrap(
+                    spacing: 10,
+                    runSpacing: 10,
+                    crossAxisAlignment: WrapCrossAlignment.center,
+                    children: <Widget>[
+                      FilledButton.icon(
+                        onPressed: () => controller.setSelectedSection(
+                          AppSection.voiceAssistant,
+                        ),
+                        icon: Icon(Icons.call),
+                        label: Text('Call'),
                       ),
-                      icon: Icon(Icons.call),
-                      label: Text('Call'),
-                    ),
-                    _MetaPill(
-                      label: controller.modelIndicator,
-                      icon: Icons.memory_outlined,
-                    ),
-                    _MetaPill(
-                      label: 'Agent: ${controller.activeAgentLabel}',
-                      icon: Icons.smart_toy_outlined,
-                    ),
-                  ],
-                ),
-              ),
-              if (controller.errorMessage != null) ...<Widget>[
-                _InlineError(message: controller.errorMessage!),
-                const SizedBox(height: 16),
-              ],
-              if (controller.activeRun != null ||
-                  controller.toolEvents.isNotEmpty)
-                Padding(
-                  padding: const EdgeInsets.only(bottom: 16),
-                  child: _RunStatusPanel(
-                    run: controller.activeRun,
-                    tools: controller.toolEvents,
+                      _MetaPill(
+                        label: controller.modelIndicator,
+                        icon: Icons.memory_outlined,
+                      ),
+                      _MetaPill(
+                        label: 'Agent: ${controller.activeAgentLabel}',
+                        icon: Icons.smart_toy_outlined,
+                      ),
+                    ],
                   ),
                 ),
-              if (messages.isEmpty)
-                Padding(
-                  padding: EdgeInsets.only(top: 64),
-                  child: Center(
-                    child: _EmptyState(
-                      title: 'How can I help?',
-                      subtitle:
-                          'Runs, tools, memory, scheduling, skills, and MCP are all available here.',
+                if (controller.errorMessage != null) ...<Widget>[
+                  _InlineError(message: controller.errorMessage!),
+                  const SizedBox(height: 16),
+                ],
+                if (controller.activeRun != null ||
+                    controller.toolEvents.isNotEmpty)
+                  Padding(
+                    padding: const EdgeInsets.only(bottom: 16),
+                    child: _RunStatusPanel(
+                      run: controller.activeRun,
+                      tools: controller.toolEvents,
                     ),
                   ),
-                )
-              else
-                ...messages.map(
-                  (entry) => Padding(
-                    padding: const EdgeInsets.only(bottom: 18),
-                    child: _ChatBubble(
-                      entry: entry,
-                      onLoadRunDetail: controller.fetchRunDetail,
+                if (messages.isEmpty)
+                  Padding(
+                    padding: EdgeInsets.only(top: 64),
+                    child: Center(
+                      child: _EmptyState(
+                        title: 'How can I help?',
+                        subtitle:
+                            'Runs, tools, memory, scheduling, skills, and MCP are all available here.',
+                      ),
+                    ),
+                  )
+                else
+                  ...messages.map(
+                    (entry) => Padding(
+                      padding: const EdgeInsets.only(bottom: 18),
+                      child: _ChatBubble(
+                        entry: entry,
+                        onLoadRunDetail: controller.fetchRunDetail,
+                      ),
                     ),
                   ),
-                ),
-            ],
-          ),
+              ],
+            ),
           ),
         ),
         Container(
@@ -287,6 +451,9 @@ class _ChatPanelState extends State<ChatPanel> {
                             .map((entry) => entry.value)
                             .toList(growable: false);
                       });
+                      if (_stickToBottom) {
+                        _scheduleScrollToBottom();
+                      }
                       if (_pendingSharedAttachments.isEmpty) {
                         _clearSharedPayload();
                       }
@@ -331,6 +498,32 @@ class _ChatPanelState extends State<ChatPanel> {
                       color: _textSecondary,
                     ),
                     const SizedBox(width: 2),
+                    _isTranscribing
+                        ? const SizedBox(
+                            width: 40,
+                            height: 40,
+                            child: Padding(
+                              padding: EdgeInsets.all(10),
+                              child: CircularProgressIndicator(strokeWidth: 2),
+                            ),
+                          )
+                        : IconButton(
+                            tooltip: _isDictating
+                                ? 'Stop & transcribe'
+                                : 'Dictate',
+                            onPressed: _isDictating
+                                ? _stopAndTranscribe
+                                : _startDictation,
+                            icon: Icon(
+                              _isDictating
+                                  ? Icons.stop_circle_outlined
+                                  : Icons.mic_none_rounded,
+                            ),
+                            color: _isDictating
+                                ? Theme.of(context).colorScheme.error
+                                : _textSecondary,
+                          ),
+                    const SizedBox(width: 2),
                     FilledButton(
                       onPressed: () => controller.setSelectedSection(
                         AppSection.voiceAssistant,

package/flutter_app/lib/main_controller.dart

@@ -4602,6 +4602,18 @@ class NeoAgentController extends ChangeNotifier {
     }
   }
 
+  Future<String> transcribeDictationAudio({
+    required String audioBase64,
+    String mimeType = 'audio/pcm;rate=16000;channels=1',
+  }) async {
+    final result = await _backendClient.transcribeAudio(
+      backendUrl,
+      audioBase64: audioBase64,
+      mimeType: mimeType,
+    );
+    return result['transcript']?.toString() ?? '';
+  }
+
   Future<void> sendMessage(
     String task, {
     List<SharedChatAttachment> sharedAttachments =

package/flutter_app/lib/main_devices.dart

@@ -1537,6 +1537,40 @@ class _InteractiveSurfacePreviewState
     } catch (_) {}
   }
 
+  void _handleStreamFirstFrame(String streamKey) {
+    if (!mounted || _requestedStreamKey != streamKey) {
+      return;
+    }
+    if (_streamFailedKey == streamKey) {
+      setState(() => _streamFailedKey = null);
+    }
+  }
+
+  void _handleStreamFrameTimeout(String streamKey) {
+    if (!mounted || _requestedStreamKey != streamKey) {
+      return;
+    }
+    setState(() => _streamFailedKey = streamKey);
+    unawaited(_stopActiveStream());
+    if ((widget.screenshotPath ?? '').isEmpty) {
+      unawaited(_refreshStaticFrame());
+    }
+  }
+
+  Future<void> _refreshStaticFrame() async {
+    switch (widget.surface) {
+      case _DeviceSurface.browser:
+        await widget.controller.screenshotBrowserRuntime();
+        break;
+      case _DeviceSurface.android:
+        await widget.controller.screenshotAndroidRuntime();
+        break;
+      case _DeviceSurface.desktop:
+        await widget.controller.screenshotDesktopRuntime();
+        break;
+    }
+  }
+
   void _detachImageListener() {
     if (_imageStream != null && _imageListener != null) {
       _imageStream!.removeListener(_imageListener!);
@@ -1681,6 +1715,7 @@ class _InteractiveSurfacePreviewState
                         streamDeviceId != null &&
                         streamDeviceId.isNotEmpty &&
                         streamKey != _streamFailedKey) {
+                      final activeStreamKey = streamKey!;
                       return Stack(
                         fit: StackFit.expand,
                         children: <Widget>[
@@ -1690,6 +1725,10 @@ class _InteractiveSurfacePreviewState
                             deviceId: streamDeviceId,
                             platform: streamPlatform,
                             remoteResolution: _pixelSize,
+                            onFirstFrame: () =>
+                                _handleStreamFirstFrame(activeStreamKey),
+                            onFrameTimeout: () =>
+                                _handleStreamFrameTimeout(activeStreamKey),
                             onTap: widget.busy
                                 ? null
                                 : (x, y) => unawaited(

package/flutter_app/lib/main_integrations.dart

@@ -188,7 +188,9 @@ class OfficialIntegrationsTab extends StatelessWidget {
                       !item.env.configured
                           ? item.env.summary
                           : item.hasExpiredAccounts
-                          ? 'One or more accounts expired. Reconnect the affected account to restore tool access.'
+                          ? item.id == 'google_workspace'
+                              ? 'One or more accounts expired. Reconnect to restore access. If this keeps happening, your Google Cloud OAuth app may be in Testing mode — publish it to Production in Google Cloud Console to get long-lived tokens.'
+                              : 'One or more accounts expired. Reconnect the affected account to restore tool access.'
                           : !item.supportsMultipleAccounts && item.isConnected
                           ? 'This integration currently supports one connected account per agent. Re-open setup to replace it.'
                           : item.isConnected