neoagent 2.4.1-beta.13 → 2.4.1-beta.15

package/flutter_app/lib/main_unified.dart

@@ -129,6 +129,7 @@ class RunsAndLogsPanel extends StatefulWidget {
 class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
     with SingleTickerProviderStateMixin {
   late final TabController _tabController;
+  bool _syncingFromController = false;
 
   @override
   void initState() {
@@ -137,7 +138,7 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
       length: _RunsPageTab.values.length,
       vsync: this,
       initialIndex: _tabForSection(widget.controller.selectedSection).index,
-    );
+    )..addListener(_handleTabChanged);
   }
 
   @override
@@ -149,17 +150,34 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
             selectedSection == AppSection.logs)) {
       final targetIndex = _tabForSection(selectedSection).index;
       if (_tabController.index != targetIndex) {
+        _syncingFromController = true;
         _tabController.index = targetIndex;
+        _syncingFromController = false;
       }
     }
   }
 
   @override
   void dispose() {
+    _tabController.removeListener(_handleTabChanged);
     _tabController.dispose();
     super.dispose();
   }
 
+  void _handleTabChanged() {
+    if (_syncingFromController || _tabController.indexIsChanging) {
+      return;
+    }
+    _selectSectionForTabIndex(_tabController.index);
+  }
+
+  void _selectSectionForTabIndex(int index) {
+    final section = _sectionForTab(_RunsPageTab.values[index]);
+    if (widget.controller.selectedSection != section) {
+      widget.controller.setSelectedSection(section);
+    }
+  }
+
   _RunsPageTab _tabForSection(AppSection section) {
     switch (section) {
       case AppSection.logs:
@@ -169,6 +187,15 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
     }
   }
 
+  AppSection _sectionForTab(_RunsPageTab tab) {
+    switch (tab) {
+      case _RunsPageTab.logs:
+        return AppSection.logs;
+      case _RunsPageTab.runs:
+        return AppSection.runs;
+    }
+  }
+
   @override
   Widget build(BuildContext context) {
     final controller = widget.controller;
@@ -193,6 +220,7 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
               dividerColor: _border,
               indicatorSize: TabBarIndicatorSize.tab,
               labelStyle: const TextStyle(fontWeight: FontWeight.w700),
+              onTap: _selectSectionForTabIndex,
               tabs: <Widget>[
                 Tab(text: 'Runs (${controller.recentRuns.length})'),
                 Tab(text: 'Logs (${controller.logs.length})'),

package/flutter_app/lib/src/backend_client.dart

@@ -1644,6 +1644,17 @@ class BackendClient {
     await deleteMap(baseUrl, '/api/recordings/$sessionId');
   }
 
+  Future<Map<String, dynamic>> transcribeAudio(
+    String baseUrl, {
+    required String audioBase64,
+    String mimeType = 'audio/pcm;rate=16000;channels=1',
+  }) {
+    return postMap(baseUrl, '/api/voice-assistant/transcribe', <String, dynamic>{
+      'audioBase64': audioBase64,
+      'mimeType': mimeType,
+    });
+  }
+
   Future<Map<String, dynamic>> runVoiceAssistantTurn(
     String baseUrl, {
     required String sessionId,

package/flutter_app/lib/src/desktop_companion_io.dart

@@ -25,6 +25,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   final DesktopCompanionActions _actions;
   WebSocket? _socket;
   Timer? _reconnectTimer;
+  Timer? _connectionWatchdogTimer;
   Timer? _streamTimer;
   bool _streamCaptureInFlight = false;
   // Set true while a click / drag / scroll / typeText / pressKey command is
@@ -94,6 +95,7 @@ class DesktopCompanionManager extends ChangeNotifier {
       await disconnect();
       return;
     }
+    _ensureConnectionWatchdog();
     await _ensureConnected();
   }
 
@@ -138,6 +140,8 @@ class DesktopCompanionManager extends ChangeNotifier {
   Future<void> disconnect() async {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _connectionWatchdogTimer?.cancel();
+    _connectionWatchdogTimer = null;
     _stopStreaming();
     _connecting = false;
     _connected = false;
@@ -224,6 +228,7 @@ class DesktopCompanionManager extends ChangeNotifier {
         uri.toString(),
         headers: <String, Object>{'Cookie': _sessionCookie},
       );
+      socket.pingInterval = const Duration(seconds: 25);
       _socket = socket;
       socket.listen(
         _handleMessage,
@@ -470,6 +475,8 @@ class DesktopCompanionManager extends ChangeNotifier {
   void dispose() {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _connectionWatchdogTimer?.cancel();
+    _connectionWatchdogTimer = null;
     _stopStreaming();
     _connecting = false;
     _connected = false;
@@ -486,12 +493,28 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   void _scheduleReconnect() {
     if (!_enabled || !_authenticated || _sessionCookie.isEmpty) return;
+    _ensureConnectionWatchdog();
     _reconnectTimer?.cancel();
     _reconnectTimer = Timer(const Duration(seconds: 5), () {
       unawaited(_ensureConnected());
     });
   }
 
+  void _ensureConnectionWatchdog() {
+    if (!_enabled || !_authenticated || _sessionCookie.isEmpty) return;
+    if (_connectionWatchdogTimer != null) return;
+    _connectionWatchdogTimer = Timer.periodic(const Duration(seconds: 30), (_) {
+      if (!_enabled || !_authenticated || _sessionCookie.isEmpty) {
+        _connectionWatchdogTimer?.cancel();
+        _connectionWatchdogTimer = null;
+        return;
+      }
+      if (!_connected && !_connecting) {
+        unawaited(_ensureConnected());
+      }
+    });
+  }
+
   Future<void> _sendEvent(String event, Map<String, Object?> payload) async {
     final socket = _socket;
     if (socket == null || !_connected) return;

package/flutter_app/lib/src/stream_renderer.dart

@@ -15,6 +15,9 @@ class StreamRenderer extends StatefulWidget {
     this.onSwipe,
     this.onType,
     this.onHover,
+    this.onFirstFrame,
+    this.onFrameTimeout,
+    this.firstFrameTimeout = const Duration(seconds: 8),
     this.fit = BoxFit.contain,
     this.alignment = Alignment.center,
   });
@@ -27,6 +30,9 @@ class StreamRenderer extends StatefulWidget {
   final void Function(double x1, double y1, double x2, double y2)? onSwipe;
   final void Function(String text)? onType;
   final void Function(double x, double y)? onHover;
+  final VoidCallback? onFirstFrame;
+  final VoidCallback? onFrameTimeout;
+  final Duration firstFrameTimeout;
   final BoxFit fit;
   final Alignment alignment;
 
@@ -43,6 +49,8 @@ class _StreamRendererState extends State<StreamRenderer> {
   Offset? _dragEnd;
 
   Timer? _hoverThrottleTimer;
+  Timer? _firstFrameTimer;
+  bool _firstFrameTimedOut = false;
   Offset? _pendingHoverOffset;
   DateTime _lastHoverTime = DateTime.fromMillisecondsSinceEpoch(0);
 
@@ -54,6 +62,7 @@ class _StreamRendererState extends State<StreamRenderer> {
       'deviceId': widget.deviceId,
       'platform': widget.platform,
     });
+    _startFirstFrameTimer();
   }
 
   @override
@@ -65,7 +74,9 @@ class _StreamRendererState extends State<StreamRenderer> {
       return;
     }
     _frameSize = null;
+    _frame = null;
     _detachImageListener();
+    _firstFrameTimer?.cancel();
     oldWidget.socket.off('stream:frame', _onFrame);
     oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
       'deviceId': oldWidget.deviceId,
@@ -76,6 +87,24 @@ class _StreamRendererState extends State<StreamRenderer> {
       'deviceId': widget.deviceId,
       'platform': widget.platform,
     });
+    _startFirstFrameTimer();
+  }
+
+  void _startFirstFrameTimer() {
+    _firstFrameTimer?.cancel();
+    _firstFrameTimedOut = false;
+    if (widget.onFrameTimeout == null ||
+        widget.firstFrameTimeout <= Duration.zero ||
+        _frame != null) {
+      return;
+    }
+    _firstFrameTimer = Timer(widget.firstFrameTimeout, () {
+      if (!mounted || _frame != null) {
+        return;
+      }
+      _firstFrameTimedOut = true;
+      widget.onFrameTimeout?.call();
+    });
   }
 
   void _onFrame(dynamic data) {
@@ -102,6 +131,14 @@ class _StreamRendererState extends State<StreamRenderer> {
       _ => null,
     };
     if (frame == null || frame.isEmpty || !mounted) return;
+    final hadFrame = _frame != null;
+    if (!hadFrame) {
+      _firstFrameTimer?.cancel();
+      _firstFrameTimer = null;
+      if (!_firstFrameTimedOut) {
+        widget.onFirstFrame?.call();
+      }
+    }
     if (_frameSize == null) {
       _resolveFrameSize(frame);
     }
@@ -146,7 +183,8 @@ class _StreamRendererState extends State<StreamRenderer> {
         return MouseRegion(
           onHover: widget.onHover == null
               ? null
-              : (event) => _handleHoverEvent(event.localPosition, constraints.biggest),
+              : (event) =>
+                    _handleHoverEvent(event.localPosition, constraints.biggest),
           child: GestureDetector(
             behavior: HitTestBehavior.opaque,
             onTapDown: widget.onTap == null
@@ -197,7 +235,7 @@ class _StreamRendererState extends State<StreamRenderer> {
     final now = DateTime.now();
     final elapsed = now.difference(_lastHoverTime);
     const throttleDuration = Duration(milliseconds: 70);
-    
+
     if (elapsed >= throttleDuration) {
       _sendPendingHover(boxSize);
     } else {
@@ -213,7 +251,7 @@ class _StreamRendererState extends State<StreamRenderer> {
     if (offset == null) return;
     _pendingHoverOffset = null;
     _lastHoverTime = DateTime.now();
-    
+
     final point = _mapToRemote(offset, boxSize);
     if (point != null) {
       widget.onHover?.call(point.dx, point.dy);
@@ -275,6 +313,7 @@ class _StreamRendererState extends State<StreamRenderer> {
   @override
   void dispose() {
     _hoverThrottleTimer?.cancel();
+    _firstFrameTimer?.cancel();
     widget.socket.emit('stream:unsubscribe', <String, Object?>{
       'deviceId': widget.deviceId,
       'platform': widget.platform,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neoagent",
-  "version": "2.4.1-beta.13",
+  "version": "2.4.1-beta.15",
   "description": "Proactive personal AI agent with no limits",
   "license": "AGPL-3.0-only",
   "main": "server/index.js",

package/server/public/.last_build_id

@@ -1 +1 @@
-da05ba4590c63eb13c0254be7a8b0840
+8dab82a7512518b6f2bb17abf16b00fe

package/server/public/flutter_bootstrap.js

@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"4c525dac5ebe5971c5708ef73558ed8edcf4a3
 
 _flutter.loader.load({
   serviceWorkerSettings: {
-    serviceWorkerVersion: "2422850760" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
+    serviceWorkerVersion: "617137012" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
   }
 });