neoagent 2.4.1-beta.14 → 2.4.1-beta.15

package/extensions/chrome-browser/background.mjs

@@ -8,6 +8,8 @@ let reconnectTimer = null;
 let suppressSocketClose = false;
 const DEFAULT_FETCH_TIMEOUT_MS = 10000;
 const DEFAULT_WS_CONNECT_TIMEOUT_MS = 10000;
+const KEEPALIVE_ALARM_NAME = 'neoagent-extension-keepalive';
+const KEEPALIVE_ALARM_MINUTES = 1;
 const EXTENSION_PROTOCOL_VERSION = 1;
 
 function getStorage(keys = STORAGE_KEYS) {
@@ -81,6 +83,12 @@ function clearReconnectTimer() {
   reconnectTimer = null;
 }
 
+function ensureKeepaliveAlarm() {
+  chrome.alarms?.create?.(KEEPALIVE_ALARM_NAME, {
+    periodInMinutes: KEEPALIVE_ALARM_MINUTES,
+  });
+}
+
 async function handleSocketDisconnected(ws) {
   if (socket !== ws) {
     return;
@@ -308,4 +316,22 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
   return true;
 });
 
+chrome.alarms?.onAlarm?.addListener((alarm) => {
+  if (alarm?.name !== KEEPALIVE_ALARM_NAME) return;
+  connect().catch((error) => {
+    console.error('NeoAgent keepalive reconnect failed', error);
+  });
+});
+
+chrome.runtime.onStartup?.addListener(() => {
+  ensureKeepaliveAlarm();
+  connect().catch(() => {});
+});
+
+chrome.runtime.onInstalled?.addListener(() => {
+  ensureKeepaliveAlarm();
+  connect().catch(() => {});
+});
+
+ensureKeepaliveAlarm();
 connect().catch(() => {});

package/extensions/chrome-browser/manifest.json

@@ -1,11 +1,11 @@
 {
   "manifest_version": 3,
   "name": "NeoAgent Browser",
-  "version": "2.4.1.7",
-  "version_name": "2.4.1-beta.7",
+  "version": "2.4.1.8",
+  "version_name": "2.4.1-beta.8",
   "description": "Connect this Chrome browser to NeoAgent for browser automation.",
   "minimum_chrome_version": "118",
-  "permissions": ["debugger", "storage", "tabs"],
+  "permissions": ["alarms", "debugger", "storage", "tabs"],
   "host_permissions": ["http://*/*", "https://*/*"],
   "icons": {
     "16": "icons/icon16.png",

package/flutter_app/lib/main_chat.dart

@@ -9,15 +9,19 @@ class ChatPanel extends StatefulWidget {
   State<ChatPanel> createState() => _ChatPanelState();
 }
 
-class _ChatPanelState extends State<ChatPanel> {
+class _ChatPanelState extends State<ChatPanel> with WidgetsBindingObserver {
+  static const double _autoScrollBottomThreshold = 120;
+  static const int _autoScrollSettlePasses = 4;
+
   late final TextEditingController _composerController;
   final ScrollController _scrollController = ScrollController();
   List<SharedChatAttachment> _pendingSharedAttachments =
       const <SharedChatAttachment>[];
   String? _appliedSharedPayloadSignature;
-  int _lastMessageCount = 0;
-  int _lastToolCount = 0;
-  String _lastStream = '';
+  String _lastScrollContentSignature = '';
+  bool _stickToBottom = true;
+  bool _ignoreScrollUpdates = false;
+  int _scrollGeneration = 0;
   bool _isSendingChatMessage = false;
   bool _isDictating = false;
   bool _isTranscribing = false;
@@ -27,9 +31,13 @@ class _ChatPanelState extends State<ChatPanel> {
   @override
   void initState() {
     super.initState();
+    WidgetsBinding.instance.addObserver(this);
     _composerController = TextEditingController();
+    _composerController.addListener(_handleComposerLayoutChanged);
+    _scrollController.addListener(_handleScrollPositionChanged);
     widget.controller.addListener(_consumeQueuedDraft);
     _consumeQueuedDraft();
+    _scheduleScrollToBottom(force: true);
   }
 
   @override
@@ -39,19 +47,33 @@ class _ChatPanelState extends State<ChatPanel> {
       oldWidget.controller.removeListener(_consumeQueuedDraft);
       widget.controller.addListener(_consumeQueuedDraft);
       _appliedSharedPayloadSignature = null;
+      _lastScrollContentSignature = '';
+      _stickToBottom = true;
       _consumeQueuedDraft();
+      _scheduleScrollToBottom(force: true);
     }
   }
 
   @override
   void dispose() {
+    WidgetsBinding.instance.removeObserver(this);
     widget.controller.removeListener(_consumeQueuedDraft);
+    _composerController.removeListener(_handleComposerLayoutChanged);
+    _scrollController.removeListener(_handleScrollPositionChanged);
     _composerController.dispose();
     _scrollController.dispose();
     _dictationCapture?.dispose();
     super.dispose();
   }
 
+  @override
+  void didChangeMetrics() {
+    super.didChangeMetrics();
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
+  }
+
   Future<void> _startDictation() async {
     if (_isDictating || _isTranscribing) return;
     final capture = LiveVoiceCapture();
@@ -68,9 +90,9 @@ class _ChatPanelState extends State<ChatPanel> {
       await capture.dispose();
       _dictationCapture = null;
       if (mounted) {
-        ScaffoldMessenger.of(context).showSnackBar(
-          SnackBar(content: Text('Microphone error: $e')),
-        );
+        ScaffoldMessenger.of(
+          context,
+        ).showSnackBar(SnackBar(content: Text('Microphone error: $e')));
       }
     }
   }
@@ -96,7 +118,9 @@ class _ChatPanelState extends State<ChatPanel> {
       );
       if (mounted && transcript.isNotEmpty) {
         final current = _composerController.text;
-        final separator = current.isNotEmpty && !current.endsWith(' ') ? ' ' : '';
+        final separator = current.isNotEmpty && !current.endsWith(' ')
+            ? ' '
+            : '';
         _composerController.text = '$current$separator$transcript';
         _composerController.selection = TextSelection.collapsed(
           offset: _composerController.text.length,
@@ -104,9 +128,9 @@ class _ChatPanelState extends State<ChatPanel> {
       }
     } catch (e) {
       if (mounted) {
-        ScaffoldMessenger.of(context).showSnackBar(
-          SnackBar(content: Text('Transcription failed: $e')),
-        );
+        ScaffoldMessenger.of(
+          context,
+        ).showSnackBar(SnackBar(content: Text('Transcription failed: $e')));
       }
     } finally {
       await capture?.dispose();
@@ -158,6 +182,9 @@ class _ChatPanelState extends State<ChatPanel> {
     setState(() {
       _pendingSharedAttachments = const <SharedChatAttachment>[];
     });
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
   }
 
   String _mimeTypeForFileName(String fileName) {
@@ -204,10 +231,9 @@ class _ChatPanelState extends State<ChatPanel> {
     }
     final attachments = result.files
         .where(
-          (file) =>
-              kIsWeb
-                  ? file.bytes != null
-                  : file.path?.trim().isNotEmpty == true,
+          (file) => kIsWeb
+              ? file.bytes != null
+              : file.path?.trim().isNotEmpty == true,
         )
         .map(
           (file) => SharedChatAttachment(
@@ -229,108 +255,178 @@ class _ChatPanelState extends State<ChatPanel> {
         ...attachments,
       ];
     });
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
   }
 
   bool get _isNearBottom {
     if (!_scrollController.hasClients) return true;
     final pos = _scrollController.position;
     if (!pos.hasContentDimensions) return true;
-    return pos.pixels >= pos.maxScrollExtent - 80;
+    return pos.pixels >= pos.maxScrollExtent - _autoScrollBottomThreshold;
   }
 
-  void _scrollToBottom() {
-    WidgetsBinding.instance.addPostFrameCallback((_) {
-      if (!mounted || !_scrollController.hasClients) return;
-      final pos = _scrollController.position;
-      if (pos.hasContentDimensions) {
-        _scrollController.jumpTo(pos.maxScrollExtent);
-      }
-    });
+  void _handleScrollPositionChanged() {
+    if (_ignoreScrollUpdates || !_scrollController.hasClients) return;
+    final nearBottom = _isNearBottom;
+    if (_stickToBottom && !nearBottom) {
+      _scrollGeneration++;
+    }
+    _stickToBottom = nearBottom;
+  }
+
+  void _handleComposerLayoutChanged() {
+    if (_stickToBottom) {
+      _scheduleScrollToBottom();
+    }
+  }
+
+  String _scrollContentSignature(
+    List<ChatEntry> messages,
+    NeoAgentController controller,
+  ) {
+    final last = messages.isEmpty ? null : messages.last;
+    final activeRun = controller.activeRun;
+    return <Object?>[
+      messages.length,
+      last?.id,
+      last?.role,
+      last?.content.length,
+      last?.typing,
+      controller.toolEvents.length,
+      activeRun?.runId,
+      activeRun?.phase,
+      activeRun?.iteration,
+      controller.streamingAssistant.length,
+      controller.isSendingMessage,
+    ].join('|');
+  }
+
+  void _scheduleScrollToBottom({bool force = false}) {
+    if (!force && !_stickToBottom) return;
+    final generation = ++_scrollGeneration;
+
+    void settle(int remainingPasses) {
+      WidgetsBinding.instance.addPostFrameCallback((_) {
+        if (!mounted ||
+            generation != _scrollGeneration ||
+            !_scrollController.hasClients) {
+          return;
+        }
+        final pos = _scrollController.position;
+        if (pos.hasContentDimensions) {
+          final target = pos.maxScrollExtent;
+          if ((pos.pixels - target).abs() > 0.5) {
+            _ignoreScrollUpdates = true;
+            _scrollController.jumpTo(target);
+            _ignoreScrollUpdates = false;
+          }
+        }
+        _stickToBottom = true;
+        if (remainingPasses > 0) {
+          Timer(const Duration(milliseconds: 24), () {
+            if (mounted && generation == _scrollGeneration) {
+              settle(remainingPasses - 1);
+            }
+          });
+        }
+      });
+    }
+
+    settle(_autoScrollSettlePasses);
+  }
+
+  void _maybeFollowChatContent(
+    List<ChatEntry> messages,
+    NeoAgentController controller,
+  ) {
+    final signature = _scrollContentSignature(messages, controller);
+    if (_lastScrollContentSignature == signature) return;
+    final isInitialContent = _lastScrollContentSignature.isEmpty;
+    final shouldFollow = _stickToBottom || isInitialContent;
+    _lastScrollContentSignature = signature;
+    if (shouldFollow) {
+      _scheduleScrollToBottom(force: isInitialContent);
+    }
   }
 
   @override
   Widget build(BuildContext context) {
     final controller = widget.controller;
     final messages = controller.visibleChatMessages;
-    if (_lastMessageCount != messages.length ||
-        _lastToolCount != controller.toolEvents.length ||
-        _lastStream != controller.streamingAssistant) {
-      _lastMessageCount = messages.length;
-      _lastToolCount = controller.toolEvents.length;
-      _lastStream = controller.streamingAssistant;
-      if (_isNearBottom) _scrollToBottom();
-    }
+    _maybeFollowChatContent(messages, controller);
 
     return Column(
       children: <Widget>[
         Expanded(
           child: SelectionArea(
             child: ListView(
-            controller: _scrollController,
-            padding: _pagePadding(context),
-            children: <Widget>[
-              _PageTitle(
-                title: 'Chat',
-                subtitle: 'Live agent chat with tool and stream status.',
-                trailing: Wrap(
-                  spacing: 10,
-                  runSpacing: 10,
-                  crossAxisAlignment: WrapCrossAlignment.center,
-                  children: <Widget>[
-                    FilledButton.icon(
-                      onPressed: () => controller.setSelectedSection(
-                        AppSection.voiceAssistant,
+              controller: _scrollController,
+              padding: _pagePadding(context),
+              children: <Widget>[
+                _PageTitle(
+                  title: 'Chat',
+                  subtitle: 'Live agent chat with tool and stream status.',
+                  trailing: Wrap(
+                    spacing: 10,
+                    runSpacing: 10,
+                    crossAxisAlignment: WrapCrossAlignment.center,
+                    children: <Widget>[
+                      FilledButton.icon(
+                        onPressed: () => controller.setSelectedSection(
+                          AppSection.voiceAssistant,
+                        ),
+                        icon: Icon(Icons.call),
+                        label: Text('Call'),
                       ),
-                      icon: Icon(Icons.call),
-                      label: Text('Call'),
-                    ),
-                    _MetaPill(
-                      label: controller.modelIndicator,
-                      icon: Icons.memory_outlined,
-                    ),
-                    _MetaPill(
-                      label: 'Agent: ${controller.activeAgentLabel}',
-                      icon: Icons.smart_toy_outlined,
-                    ),
-                  ],
-                ),
-              ),
-              if (controller.errorMessage != null) ...<Widget>[
-                _InlineError(message: controller.errorMessage!),
-                const SizedBox(height: 16),
-              ],
-              if (controller.activeRun != null ||
-                  controller.toolEvents.isNotEmpty)
-                Padding(
-                  padding: const EdgeInsets.only(bottom: 16),
-                  child: _RunStatusPanel(
-                    run: controller.activeRun,
-                    tools: controller.toolEvents,
+                      _MetaPill(
+                        label: controller.modelIndicator,
+                        icon: Icons.memory_outlined,
+                      ),
+                      _MetaPill(
+                        label: 'Agent: ${controller.activeAgentLabel}',
+                        icon: Icons.smart_toy_outlined,
+                      ),
+                    ],
                   ),
                 ),
-              if (messages.isEmpty)
-                Padding(
-                  padding: EdgeInsets.only(top: 64),
-                  child: Center(
-                    child: _EmptyState(
-                      title: 'How can I help?',
-                      subtitle:
-                          'Runs, tools, memory, scheduling, skills, and MCP are all available here.',
+                if (controller.errorMessage != null) ...<Widget>[
+                  _InlineError(message: controller.errorMessage!),
+                  const SizedBox(height: 16),
+                ],
+                if (controller.activeRun != null ||
+                    controller.toolEvents.isNotEmpty)
+                  Padding(
+                    padding: const EdgeInsets.only(bottom: 16),
+                    child: _RunStatusPanel(
+                      run: controller.activeRun,
+                      tools: controller.toolEvents,
                     ),
                   ),
-                )
-              else
-                ...messages.map(
-                  (entry) => Padding(
-                    padding: const EdgeInsets.only(bottom: 18),
-                    child: _ChatBubble(
-                      entry: entry,
-                      onLoadRunDetail: controller.fetchRunDetail,
+                if (messages.isEmpty)
+                  Padding(
+                    padding: EdgeInsets.only(top: 64),
+                    child: Center(
+                      child: _EmptyState(
+                        title: 'How can I help?',
+                        subtitle:
+                            'Runs, tools, memory, scheduling, skills, and MCP are all available here.',
+                      ),
+                    ),
+                  )
+                else
+                  ...messages.map(
+                    (entry) => Padding(
+                      padding: const EdgeInsets.only(bottom: 18),
+                      child: _ChatBubble(
+                        entry: entry,
+                        onLoadRunDetail: controller.fetchRunDetail,
+                      ),
                     ),
                   ),
-                ),
-            ],
-          ),
+              ],
+            ),
           ),
         ),
         Container(
@@ -355,6 +451,9 @@ class _ChatPanelState extends State<ChatPanel> {
                             .map((entry) => entry.value)
                             .toList(growable: false);
                       });
+                      if (_stickToBottom) {
+                        _scheduleScrollToBottom();
+                      }
                       if (_pendingSharedAttachments.isEmpty) {
                         _clearSharedPayload();
                       }
@@ -409,14 +508,20 @@ class _ChatPanelState extends State<ChatPanel> {
                             ),
                           )
                         : IconButton(
-                            tooltip: _isDictating ? 'Stop & transcribe' : 'Dictate',
-                            onPressed: _isDictating ? _stopAndTranscribe : _startDictation,
+                            tooltip: _isDictating
+                                ? 'Stop & transcribe'
+                                : 'Dictate',
+                            onPressed: _isDictating
+                                ? _stopAndTranscribe
+                                : _startDictation,
                             icon: Icon(
                               _isDictating
                                   ? Icons.stop_circle_outlined
                                   : Icons.mic_none_rounded,
                             ),
-                            color: _isDictating ? Theme.of(context).colorScheme.error : _textSecondary,
+                            color: _isDictating
+                                ? Theme.of(context).colorScheme.error
+                                : _textSecondary,
                           ),
                     const SizedBox(width: 2),
                     FilledButton(

package/flutter_app/lib/main_devices.dart

@@ -1537,6 +1537,40 @@ class _InteractiveSurfacePreviewState
     } catch (_) {}
   }
 
+  void _handleStreamFirstFrame(String streamKey) {
+    if (!mounted || _requestedStreamKey != streamKey) {
+      return;
+    }
+    if (_streamFailedKey == streamKey) {
+      setState(() => _streamFailedKey = null);
+    }
+  }
+
+  void _handleStreamFrameTimeout(String streamKey) {
+    if (!mounted || _requestedStreamKey != streamKey) {
+      return;
+    }
+    setState(() => _streamFailedKey = streamKey);
+    unawaited(_stopActiveStream());
+    if ((widget.screenshotPath ?? '').isEmpty) {
+      unawaited(_refreshStaticFrame());
+    }
+  }
+
+  Future<void> _refreshStaticFrame() async {
+    switch (widget.surface) {
+      case _DeviceSurface.browser:
+        await widget.controller.screenshotBrowserRuntime();
+        break;
+      case _DeviceSurface.android:
+        await widget.controller.screenshotAndroidRuntime();
+        break;
+      case _DeviceSurface.desktop:
+        await widget.controller.screenshotDesktopRuntime();
+        break;
+    }
+  }
+
   void _detachImageListener() {
     if (_imageStream != null && _imageListener != null) {
       _imageStream!.removeListener(_imageListener!);
@@ -1681,6 +1715,7 @@ class _InteractiveSurfacePreviewState
                         streamDeviceId != null &&
                         streamDeviceId.isNotEmpty &&
                         streamKey != _streamFailedKey) {
+                      final activeStreamKey = streamKey!;
                       return Stack(
                         fit: StackFit.expand,
                         children: <Widget>[
@@ -1690,6 +1725,10 @@ class _InteractiveSurfacePreviewState
                             deviceId: streamDeviceId,
                             platform: streamPlatform,
                             remoteResolution: _pixelSize,
+                            onFirstFrame: () =>
+                                _handleStreamFirstFrame(activeStreamKey),
+                            onFrameTimeout: () =>
+                                _handleStreamFrameTimeout(activeStreamKey),
                             onTap: widget.busy
                                 ? null
                                 : (x, y) => unawaited(

package/flutter_app/lib/main_unified.dart

@@ -129,6 +129,7 @@ class RunsAndLogsPanel extends StatefulWidget {
 class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
     with SingleTickerProviderStateMixin {
   late final TabController _tabController;
+  bool _syncingFromController = false;
 
   @override
   void initState() {
@@ -137,7 +138,7 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
       length: _RunsPageTab.values.length,
       vsync: this,
       initialIndex: _tabForSection(widget.controller.selectedSection).index,
-    );
+    )..addListener(_handleTabChanged);
   }
 
   @override
@@ -149,17 +150,34 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
             selectedSection == AppSection.logs)) {
       final targetIndex = _tabForSection(selectedSection).index;
       if (_tabController.index != targetIndex) {
+        _syncingFromController = true;
         _tabController.index = targetIndex;
+        _syncingFromController = false;
       }
     }
   }
 
   @override
   void dispose() {
+    _tabController.removeListener(_handleTabChanged);
     _tabController.dispose();
     super.dispose();
   }
 
+  void _handleTabChanged() {
+    if (_syncingFromController || _tabController.indexIsChanging) {
+      return;
+    }
+    _selectSectionForTabIndex(_tabController.index);
+  }
+
+  void _selectSectionForTabIndex(int index) {
+    final section = _sectionForTab(_RunsPageTab.values[index]);
+    if (widget.controller.selectedSection != section) {
+      widget.controller.setSelectedSection(section);
+    }
+  }
+
   _RunsPageTab _tabForSection(AppSection section) {
     switch (section) {
       case AppSection.logs:
@@ -169,6 +187,15 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
     }
   }
 
+  AppSection _sectionForTab(_RunsPageTab tab) {
+    switch (tab) {
+      case _RunsPageTab.logs:
+        return AppSection.logs;
+      case _RunsPageTab.runs:
+        return AppSection.runs;
+    }
+  }
+
   @override
   Widget build(BuildContext context) {
     final controller = widget.controller;
@@ -193,6 +220,7 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
               dividerColor: _border,
               indicatorSize: TabBarIndicatorSize.tab,
               labelStyle: const TextStyle(fontWeight: FontWeight.w700),
+              onTap: _selectSectionForTabIndex,
               tabs: <Widget>[
                 Tab(text: 'Runs (${controller.recentRuns.length})'),
                 Tab(text: 'Logs (${controller.logs.length})'),

package/flutter_app/lib/src/desktop_companion_io.dart

@@ -25,6 +25,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   final DesktopCompanionActions _actions;
   WebSocket? _socket;
   Timer? _reconnectTimer;
+  Timer? _connectionWatchdogTimer;
   Timer? _streamTimer;
   bool _streamCaptureInFlight = false;
   // Set true while a click / drag / scroll / typeText / pressKey command is
@@ -94,6 +95,7 @@ class DesktopCompanionManager extends ChangeNotifier {
       await disconnect();
       return;
     }
+    _ensureConnectionWatchdog();
     await _ensureConnected();
   }
 
@@ -138,6 +140,8 @@ class DesktopCompanionManager extends ChangeNotifier {
   Future<void> disconnect() async {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _connectionWatchdogTimer?.cancel();
+    _connectionWatchdogTimer = null;
     _stopStreaming();
     _connecting = false;
     _connected = false;
@@ -224,6 +228,7 @@ class DesktopCompanionManager extends ChangeNotifier {
         uri.toString(),
         headers: <String, Object>{'Cookie': _sessionCookie},
       );
+      socket.pingInterval = const Duration(seconds: 25);
       _socket = socket;
       socket.listen(
         _handleMessage,
@@ -470,6 +475,8 @@ class DesktopCompanionManager extends ChangeNotifier {
   void dispose() {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _connectionWatchdogTimer?.cancel();
+    _connectionWatchdogTimer = null;
     _stopStreaming();
     _connecting = false;
     _connected = false;
@@ -486,12 +493,28 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   void _scheduleReconnect() {
     if (!_enabled || !_authenticated || _sessionCookie.isEmpty) return;
+    _ensureConnectionWatchdog();
     _reconnectTimer?.cancel();
     _reconnectTimer = Timer(const Duration(seconds: 5), () {
       unawaited(_ensureConnected());
     });
   }
 
+  void _ensureConnectionWatchdog() {
+    if (!_enabled || !_authenticated || _sessionCookie.isEmpty) return;
+    if (_connectionWatchdogTimer != null) return;
+    _connectionWatchdogTimer = Timer.periodic(const Duration(seconds: 30), (_) {
+      if (!_enabled || !_authenticated || _sessionCookie.isEmpty) {
+        _connectionWatchdogTimer?.cancel();
+        _connectionWatchdogTimer = null;
+        return;
+      }
+      if (!_connected && !_connecting) {
+        unawaited(_ensureConnected());
+      }
+    });
+  }
+
   Future<void> _sendEvent(String event, Map<String, Object?> payload) async {
     final socket = _socket;
     if (socket == null || !_connected) return;