neoagent 2.3.1-beta.99 → 2.4.1-beta.10

package/flutter_app/lib/src/stream_renderer.dart

@@ -0,0 +1,286 @@
+import 'dart:async';
+import 'dart:typed_data';
+
+import 'package:flutter/material.dart';
+import 'package:socket_io_client/socket_io_client.dart' as io;
+
+class StreamRenderer extends StatefulWidget {
+  const StreamRenderer({
+    super.key,
+    required this.socket,
+    required this.deviceId,
+    required this.platform,
+    this.remoteResolution,
+    this.onTap,
+    this.onSwipe,
+    this.onType,
+    this.onHover,
+    this.fit = BoxFit.contain,
+    this.alignment = Alignment.center,
+  });
+
+  final io.Socket socket;
+  final String deviceId;
+  final String platform;
+  final Size? remoteResolution;
+  final void Function(double x, double y)? onTap;
+  final void Function(double x1, double y1, double x2, double y2)? onSwipe;
+  final void Function(String text)? onType;
+  final void Function(double x, double y)? onHover;
+  final BoxFit fit;
+  final Alignment alignment;
+
+  @override
+  State<StreamRenderer> createState() => _StreamRendererState();
+}
+
+class _StreamRendererState extends State<StreamRenderer> {
+  Uint8List? _frame;
+  Size? _frameSize;
+  ImageStream? _imageStream;
+  ImageStreamListener? _imageListener;
+  Offset? _dragStart;
+  Offset? _dragEnd;
+
+  Timer? _hoverThrottleTimer;
+  Offset? _pendingHoverOffset;
+  DateTime _lastHoverTime = DateTime.fromMillisecondsSinceEpoch(0);
+
+  @override
+  void initState() {
+    super.initState();
+    widget.socket.on('stream:frame', _onFrame);
+    widget.socket.emit('stream:subscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+  }
+
+  @override
+  void didUpdateWidget(StreamRenderer oldWidget) {
+    super.didUpdateWidget(oldWidget);
+    if (oldWidget.socket == widget.socket &&
+        oldWidget.deviceId == widget.deviceId &&
+        oldWidget.platform == widget.platform) {
+      return;
+    }
+    _frameSize = null;
+    _detachImageListener();
+    oldWidget.socket.off('stream:frame', _onFrame);
+    oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
+      'deviceId': oldWidget.deviceId,
+      'platform': oldWidget.platform,
+    });
+    widget.socket.on('stream:frame', _onFrame);
+    widget.socket.emit('stream:subscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+  }
+
+  void _onFrame(dynamic data) {
+    Object? meta;
+    Object? bytes;
+    if (data is List && data.length >= 2) {
+      meta = data[0];
+      bytes = data[1];
+    }
+    if (meta is Map) {
+      final frameDeviceId = meta['deviceId']?.toString() ?? '';
+      final framePlatform = meta['platform']?.toString() ?? '';
+      if (frameDeviceId.isNotEmpty && frameDeviceId != widget.deviceId) {
+        return;
+      }
+      if (framePlatform.isNotEmpty && framePlatform != widget.platform) {
+        return;
+      }
+    }
+    final frame = switch (bytes) {
+      Uint8List value => value,
+      List<int> value => Uint8List.fromList(value),
+      ByteBuffer value => Uint8List.view(value),
+      _ => null,
+    };
+    if (frame == null || frame.isEmpty || !mounted) return;
+    if (_frameSize == null) {
+      _resolveFrameSize(frame);
+    }
+    setState(() => _frame = frame);
+  }
+
+  void _resolveFrameSize(Uint8List frame) {
+    _detachImageListener();
+    final provider = MemoryImage(frame);
+    final stream = provider.resolve(const ImageConfiguration());
+    final listener = ImageStreamListener((image, _) {
+      if (!mounted || _frameSize != null) return;
+      setState(() {
+        _frameSize = Size(
+          image.image.width.toDouble(),
+          image.image.height.toDouble(),
+        );
+      });
+      _detachImageListener();
+    });
+    _imageStream = stream;
+    _imageListener = listener;
+    stream.addListener(listener);
+  }
+
+  void _detachImageListener() {
+    if (_imageStream != null && _imageListener != null) {
+      _imageStream!.removeListener(_imageListener!);
+    }
+    _imageStream = null;
+    _imageListener = null;
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final frame = _frame;
+    if (frame == null) {
+      return const Center(child: CircularProgressIndicator());
+    }
+    return LayoutBuilder(
+      builder: (context, constraints) {
+        return MouseRegion(
+          onHover: widget.onHover == null
+              ? null
+              : (event) => _handleHoverEvent(event.localPosition, constraints.biggest),
+          child: GestureDetector(
+            behavior: HitTestBehavior.opaque,
+            onTapDown: widget.onTap == null
+                ? null
+                : (details) => _handleTap(details, constraints.biggest),
+            onPanStart: widget.onSwipe == null
+                ? null
+                : (details) {
+                    _dragStart = details.localPosition;
+                    _dragEnd = details.localPosition;
+                  },
+            onPanUpdate: widget.onSwipe == null
+                ? null
+                : (details) {
+                    _dragEnd = details.localPosition;
+                  },
+            onPanEnd: widget.onSwipe == null
+                ? null
+                : (_) {
+                    final start = _dragStart;
+                    final end = _dragEnd;
+                    _dragStart = null;
+                    _dragEnd = null;
+                    if (start == null ||
+                        end == null ||
+                        (start - end).distance < 12) {
+                      return;
+                    }
+                    _handleSwipe(start, end, constraints.biggest);
+                  },
+            child: Image.memory(
+              frame,
+              gaplessPlayback: true,
+              fit: widget.fit,
+              alignment: widget.alignment,
+              width: constraints.maxWidth,
+              height: constraints.maxHeight,
+            ),
+          ),
+        );
+      },
+    );
+  }
+
+  void _handleHoverEvent(Offset localPosition, Size boxSize) {
+    if (widget.onHover == null) return;
+    _pendingHoverOffset = localPosition;
+    final now = DateTime.now();
+    final elapsed = now.difference(_lastHoverTime);
+    const throttleDuration = Duration(milliseconds: 70);
+    
+    if (elapsed >= throttleDuration) {
+      _sendPendingHover(boxSize);
+    } else {
+      _hoverThrottleTimer ??= Timer(throttleDuration - elapsed, () {
+        _hoverThrottleTimer = null;
+        _sendPendingHover(boxSize);
+      });
+    }
+  }
+
+  void _sendPendingHover(Size boxSize) {
+    final offset = _pendingHoverOffset;
+    if (offset == null) return;
+    _pendingHoverOffset = null;
+    _lastHoverTime = DateTime.now();
+    
+    final point = _mapToRemote(offset, boxSize);
+    if (point != null) {
+      widget.onHover?.call(point.dx, point.dy);
+    }
+  }
+
+  void _handleTap(TapDownDetails details, Size boxSize) {
+    final point = _mapToRemote(details.localPosition, boxSize);
+    if (point == null) {
+      return;
+    }
+    widget.onTap?.call(point.dx, point.dy);
+  }
+
+  void _handleSwipe(Offset start, Offset end, Size boxSize) {
+    final mappedStart = _mapToRemote(start, boxSize);
+    final mappedEnd = _mapToRemote(end, boxSize);
+    if (mappedStart == null || mappedEnd == null) {
+      return;
+    }
+    widget.onSwipe?.call(
+      mappedStart.dx,
+      mappedStart.dy,
+      mappedEnd.dx,
+      mappedEnd.dy,
+    );
+  }
+
+  Offset? _mapToRemote(Offset localPosition, Size boxSize) {
+    final remote = widget.remoteResolution ?? _frameSize;
+    if (remote == null ||
+        remote.width <= 0 ||
+        remote.height <= 0 ||
+        boxSize.width <= 0 ||
+        boxSize.height <= 0) {
+      return null;
+    }
+    final remoteSize = Size(remote.width, remote.height);
+    final fitted = applyBoxFit(widget.fit, remoteSize, boxSize);
+    final sourceRect = widget.alignment.inscribe(
+      fitted.source,
+      Offset.zero & remoteSize,
+    );
+    final destRect = widget.alignment.inscribe(
+      fitted.destination,
+      Offset.zero & boxSize,
+    );
+    if (!destRect.contains(localPosition)) {
+      return null;
+    }
+    final localX = localPosition.dx - destRect.left;
+    final localY = localPosition.dy - destRect.top;
+    return Offset(
+      sourceRect.left + localX * fitted.source.width / destRect.width,
+      sourceRect.top + localY * fitted.source.height / destRect.height,
+    );
+  }
+
+  @override
+  void dispose() {
+    _hoverThrottleTimer?.cancel();
+    widget.socket.emit('stream:unsubscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+    widget.socket.off('stream:frame', _onFrame);
+    _detachImageListener();
+    super.dispose();
+  }
+}

package/flutter_app/macos/Runner/AppDelegate.swift

@@ -106,6 +106,36 @@ final class DesktopCompanionNativePlugin: NSObject {
         displayId: displayId
       )
       result(nil)
+    case "mouseMove":
+      guard isAccessibilityTrusted() else {
+        result(
+          FlutterError(
+            code: "accessibility_permission_denied",
+            message: "Accessibility permission is required for input control.",
+            details: nil
+          )
+        )
+        return
+      }
+      guard let arguments = call.arguments as? [String: Any],
+            let x = arguments["x"] as? NSNumber,
+            let y = arguments["y"] as? NSNumber else {
+        result(
+          FlutterError(
+            code: "invalid_arguments",
+            message: "Missing mouseMove coordinates.",
+            details: nil
+          )
+        )
+        return
+      }
+      let displayId = arguments["displayId"] as? String
+      performMouseMove(
+        x: x.doubleValue,
+        y: y.doubleValue,
+        displayId: displayId
+      )
+      result(nil)
     case "drag":
       guard isAccessibilityTrusted() else {
         result(
@@ -302,11 +332,32 @@ final class DesktopCompanionNativePlugin: NSObject {
   }
 
   private func preflightScreenCapturePermission() -> Bool {
-    CGPreflightScreenCaptureAccess()
+    if CGPreflightScreenCaptureAccess() { return true }
+    // CGPreflightScreenCaptureAccess() caches the result per-process on macOS 14+
+    // and won't reflect a System Settings grant until the app restarts. Fall back to
+    // a live 1×1 capture probe which returns nil when recording is actually blocked.
+    let probe = CGWindowListCreateImage(
+      CGRect(x: 0, y: 0, width: 1, height: 1),
+      .optionOnScreenOnly,
+      kCGNullWindowID,
+      .bestResolution
+    )
+    return probe != nil
   }
 
   private func isAccessibilityTrusted() -> Bool {
-    AXIsProcessTrusted()
+    if AXIsProcessTrusted() { return true }
+    // AXIsProcessTrusted() may cache false on macOS 14+ after a System Settings grant.
+    // Probe with a live AX read: .apiDisabled is the error returned when the process
+    // lacks accessibility permission (AXError has no .notTrusted case in the macOS SDK).
+    let sysElement = AXUIElementCreateSystemWide()
+    var value: CFTypeRef?
+    let status = AXUIElementCopyAttributeValue(
+      sysElement,
+      kAXFocusedApplicationAttribute as CFString,
+      &value
+    )
+    return status != .apiDisabled
   }
 
   private func resolveDisplayId(_ raw: String?) -> CGDirectDisplayID {
@@ -396,6 +447,20 @@ final class DesktopCompanionNativePlugin: NSObject {
     up.post(tap: .cghidEventTap)
   }
 
+  private func performMouseMove(x: Double, y: Double, displayId: String?) {
+    let point = nativePointForCapturedPixel(x: x, y: y, displayId: displayId)
+    CGWarpMouseCursorPosition(point)
+    guard let moveEvent = CGEvent(
+      mouseEventSource: nil,
+      mouseType: .mouseMoved,
+      mouseCursorPosition: point,
+      mouseButton: .left
+    ) else {
+      return
+    }
+    moveEvent.post(tap: .cghidEventTap)
+  }
+
   private func performDrag(
     x1: Double,
     y1: Double,

package/flutter_app/macos/Runner/DebugProfile.entitlements

@@ -12,5 +12,9 @@
 	<true/>
 	<key>com.apple.security.network.server</key>
 	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$(AppIdentifierPrefix)$(CFBundleIdentifier)</string>
+	</array>
 </dict>
 </plist>

package/flutter_app/macos/Runner/Release.entitlements

@@ -8,5 +8,9 @@
 	<true/>
 	<key>com.apple.security.network.client</key>
 	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$(AppIdentifierPrefix)$(CFBundleIdentifier)</string>
+	</array>
 </dict>
 </plist>

package/flutter_app/pubspec.lock

@@ -594,10 +594,10 @@ packages:
     dependency: transitive
     description:
       name: meta
-      sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
+      sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349"
       url: "https://pub.dev"
     source: hosted
-    version: "1.17.0"
+    version: "1.18.0"
   mixpanel_flutter:
     dependency: "direct main"
     description:
@@ -1039,10 +1039,10 @@ packages:
     dependency: transitive
     description:
       name: test_api
-      sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a"
+      sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.10"
+    version: "0.7.11"
   tray_manager:
     dependency: "direct main"
     description:
@@ -1252,5 +1252,5 @@ packages:
     source: hosted
     version: "6.6.1"
 sdks:
-  dart: ">=3.9.2 <4.0.0"
+  dart: ">=3.10.0-0 <4.0.0"
   flutter: ">=3.35.0"

package/lib/manager.js

@@ -5,6 +5,7 @@ const net = require('net');
 const crypto = require('crypto');
 const readline = require('readline');
 const { spawn, spawnSync } = require('child_process');
+const { CLAUDE_CODE_SCOPES } = require('../server/services/ai/providers/claudeCode');
 const {
   buildBundledWebClientIfPossible: buildWebClient,
   commandExists: sharedCommandExists,
@@ -786,10 +787,168 @@ async function pollDeviceCode({ pollUrl, pollBody, pollHeaders = {}, intervalMs,
   throw new Error('Authentication timed out after 15 minutes.');
 }
 
+async function cmdLoginClaudeCode() {
+  heading('Claude Code Login');
+
+  // Check for Claude CLI credential file first (set by `claude login`)
+  const cliCredsPath = path.join(os.homedir(), '.claude', '.credentials.json');
+  if (fs.existsSync(cliCredsPath)) {
+    try {
+      const raw = fs.readFileSync(cliCredsPath, 'utf8');
+      const data = JSON.parse(raw);
+      const token = data?.claudeAiOauthTokens?.accessToken;
+      if (token) {
+        upsertEnvValue('CLAUDE_CODE_OAUTH_TOKEN', token);
+        logOk('Imported access token from Claude CLI credentials store');
+        logInfo('Restarting NeoAgent to apply credentials...');
+        cmdRestart();
+        return;
+      }
+    } catch { }
+  }
+
+  // Browser-based PKCE OAuth flow.
+  // client_id is the metadata URL per claude.ai's dynamic client registration.
+  // Redirect URIs registered: http://localhost/callback and http://127.0.0.1/callback (port 80).
+  // Per RFC 8252 §7.3, servers SHOULD allow any loopback port — we try high ports first
+  // and fall back to 80 if everything else is occupied.
+  const http = require('http');
+  const { URL: NodeURL } = require('url');
+
+  const clientId = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
+  const SCOPES = CLAUDE_CODE_SCOPES;
+
+  // The registered redirect URIs are http://localhost/callback and http://127.0.0.1/callback
+  // (port 80). The OAuth server validates the URI exactly, so we must use port 80.
+  // Dynamic high port — the server accepts http://localhost:{any-port}/callback per RFC 8252.
+  const redirectPort = Math.floor(Math.random() * 10000) + 49152;
+  const redirectUri = `http://localhost:${redirectPort}/callback`;
+
+  // Generate PKCE verifier and challenge
+  const codeVerifier = crypto.randomBytes(48).toString('base64url');
+  const codeChallenge = crypto
+    .createHash('sha256')
+    .update(codeVerifier)
+    .digest('base64url');
+
+  const state = crypto.randomBytes(16).toString('hex');
+
+  const authUrl = new URL('https://platform.claude.com/oauth/authorize');
+  authUrl.searchParams.set('response_type', 'code');
+  authUrl.searchParams.set('client_id', clientId);
+  authUrl.searchParams.set('redirect_uri', redirectUri);
+  authUrl.searchParams.set('scope', SCOPES);
+  authUrl.searchParams.set('code_challenge', codeChallenge);
+  authUrl.searchParams.set('code_challenge_method', 'S256');
+  authUrl.searchParams.set('state', state);
+
+  console.log(`\n  ${COLORS.cyan}Opening browser for Claude Code authorization...${COLORS.reset}`);
+  console.log(`  ${COLORS.dim}If the browser doesn't open, visit:${COLORS.reset}`);
+  console.log(`  ${authUrl.toString()}\n`);
+
+  // Open browser
+  const openCmd = process.platform === 'darwin' ? 'open'
+    : process.platform === 'win32' ? 'start'
+    : 'xdg-open';
+  spawnSync(openCmd, [authUrl.toString()], { stdio: 'ignore' });
+
+  // Start local redirect server to capture authorization code
+  const authCode = await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      server.close();
+      reject(new Error('Claude Code authorization timed out after 5 minutes.'));
+    }, 5 * 60 * 1000);
+
+    const server = http.createServer((req, res) => {
+      try {
+        const reqUrl = new NodeURL(req.url, redirectUri);
+        const code = reqUrl.searchParams.get('code');
+        const returnedState = reqUrl.searchParams.get('state');
+        const error = reqUrl.searchParams.get('error');
+
+        if (error) {
+          res.writeHead(200, { 'Content-Type': 'text/html' });
+          res.end('<html><body><h2>Authorization failed.</h2><p>You can close this tab.</p></body></html>');
+          clearTimeout(timeout);
+          server.close();
+          reject(new Error(`OAuth error: ${error}`));
+          return;
+        }
+
+        if (returnedState && returnedState !== state) {
+          res.writeHead(200, { 'Content-Type': 'text/html' });
+          res.end('<html><body><h2>Authorization failed.</h2><p>State mismatch. You can close this tab.</p></body></html>');
+          clearTimeout(timeout);
+          server.close();
+          reject(new Error('OAuth state mismatch — possible CSRF attempt.'));
+          return;
+        }
+
+        if (code) {
+          res.writeHead(200, { 'Content-Type': 'text/html' });
+          res.end('<html><body><h2>Authorization successful!</h2><p>You can close this tab and return to the terminal.</p></body></html>');
+          clearTimeout(timeout);
+          server.close();
+          resolve(code);
+        }
+      } catch (err) {
+        res.writeHead(500);
+        res.end('Internal error');
+      }
+    });
+
+    server.listen(redirectPort, 'localhost', () => {
+      logInfo(`Waiting for OAuth callback on ${redirectUri} ...`);
+    });
+    server.on('error', (err) => {
+      clearTimeout(timeout);
+      reject(new Error(`Could not start OAuth callback server: ${err.message}`));
+    });
+  });
+
+  logInfo('Exchanging authorization code for access token...');
+  const tokenRes = await fetch('https://platform.claude.com/v1/oauth/token', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': 'application/json',
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify({
+      grant_type: 'authorization_code',
+      code: authCode,
+      redirect_uri: redirectUri,
+      client_id: clientId,
+      code_verifier: codeVerifier,
+      scope: SCOPES,
+      state,
+    }),
+  });
+
+  if (!tokenRes.ok) {
+    const text = await tokenRes.text().catch(() => 'Unknown error');
+    throw new Error(`Token exchange failed: HTTP ${tokenRes.status} — ${text}`);
+  }
+
+  const tokenData = await tokenRes.json();
+  const accessToken = tokenData.access_token;
+  if (!accessToken) {
+    throw new Error('Token exchange succeeded but no access_token was returned.');
+  }
+
+  upsertEnvValue('CLAUDE_CODE_OAUTH_TOKEN', accessToken);
+  if (tokenData.refresh_token) {
+    upsertEnvValue('CLAUDE_CODE_REFRESH_TOKEN', tokenData.refresh_token);
+  }
+  logOk('Saved Claude Code OAuth token to .env');
+  logInfo('Restarting NeoAgent to apply credentials...');
+  cmdRestart();
+}
+
 async function cmdLogin(args = []) {
   const provider = args[0];
-  if (provider !== 'github-copilot' && provider !== 'openai-codex') {
-    throw new Error(`Unsupported login provider: ${provider || 'none'}. Available: github-copilot, openai-codex`);
+  if (provider !== 'github-copilot' && provider !== 'openai-codex' && provider !== 'claude-code') {
+    throw new Error(`Unsupported login provider: ${provider || 'none'}. Available: github-copilot, openai-codex, claude-code`);
   }
 
   if (provider === 'github-copilot') {
@@ -893,6 +1052,8 @@ async function cmdLogin(args = []) {
     logOk('Saved OpenAI Codex tokens to .env');
     logInfo('Restarting NeoAgent to apply credentials...');
     cmdRestart();
+  } else if (provider === 'claude-code') {
+    await cmdLoginClaudeCode();
   }
 }
 
@@ -1471,6 +1632,7 @@ function printHelp() {
   row('update stable|beta',  'Update and switch channel');
   row('login github-copilot','Authenticate GitHub Copilot');
   row('login openai-codex',  'Authenticate OpenAI Codex');
+  row('login claude-code',   'Authenticate Claude Code');
   console.log('');
 
   console.log(`${c.bold}Maintenance${c.reset}`);

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "neoagent",
-  "version": "2.3.1-beta.99",
+  "version": "2.4.1-beta.10",
   "description": "Proactive personal AI agent with no limits",
-  "license": "MIT",
+  "license": "AGPL-3.0-only",
   "main": "server/index.js",
   "engines": {
     "node": ">=20"