neoagent 3.0.1-beta.8 → 3.1.0

package/flutter_app/lib/src/recording_chunk_queue.dart

@@ -0,0 +1,149 @@
+import 'dart:async';
+import 'dart:typed_data';
+
+/// Metadata plus bytes for one recording chunk awaiting upload.
+class PendingChunk {
+  const PendingChunk({
+    required this.sourceKey,
+    required this.sequence,
+    required this.startMs,
+    required this.endMs,
+    required this.mimeType,
+    required this.bytes,
+  });
+
+  final String sourceKey;
+  final int sequence;
+  final int startMs;
+  final int endMs;
+  final String mimeType;
+  final Uint8List bytes;
+
+  Map<String, Object?> toMeta() => <String, Object?>{
+    'sourceKey': sourceKey,
+    'sequence': sequence,
+    'startMs': startMs,
+    'endMs': endMs,
+    'mimeType': mimeType,
+  };
+
+  static PendingChunk fromMeta(Map<String, Object?> meta, Uint8List bytes) {
+    return PendingChunk(
+      sourceKey: '${meta['sourceKey']}',
+      sequence: (meta['sequence'] as num?)?.toInt() ?? 0,
+      startMs: (meta['startMs'] as num?)?.toInt() ?? 0,
+      endMs: (meta['endMs'] as num?)?.toInt() ?? 0,
+      mimeType: meta['mimeType'] as String? ?? 'application/octet-stream',
+      bytes: bytes,
+    );
+  }
+}
+
+/// Uploads a single chunk. Must throw on any failure so the queue retries.
+typedef ChunkUploader = Future<void> Function(PendingChunk chunk);
+
+typedef QueueLogger =
+    void Function(String event, {Map<String, Object?> data, Object? error});
+
+/// In-memory, ordered, retry-until-success upload queue.
+///
+/// Chunks upload one at a time in enqueue order. A chunk that fails is kept and
+/// retried (with backoff) until it succeeds or [dispose] is called, so a
+/// transient network failure never silently drops audio while new chunks keep
+/// buffering. Used on web, where no filesystem is available; the queue lives
+/// only for the lifetime of the tab.
+class MemoryChunkUploadQueue {
+  MemoryChunkUploadQueue({
+    required ChunkUploader uploader,
+    this.maxAttemptsPerPass = 3,
+    this.retryBackoff = const Duration(seconds: 2),
+    QueueLogger? logger,
+  }) : _uploader = uploader,
+       _logger = logger;
+
+  final ChunkUploader _uploader;
+  final int maxAttemptsPerPass;
+  final Duration retryBackoff;
+  final QueueLogger? _logger;
+
+  final List<PendingChunk> _pending = <PendingChunk>[];
+  bool _draining = false;
+  bool _disposed = false;
+  Timer? _retryTimer;
+
+  int get pendingCount => _pending.length;
+
+  void enqueue(PendingChunk chunk) {
+    if (_disposed) {
+      return;
+    }
+    _pending.add(chunk);
+    unawaited(drain());
+  }
+
+  Future<void> drain() async {
+    if (_draining || _disposed) {
+      return;
+    }
+    _draining = true;
+    try {
+      while (_pending.isNotEmpty && !_disposed) {
+        final chunk = _pending.first;
+        final uploaded = await _attemptUpload(chunk);
+        if (!uploaded) {
+          _scheduleRetry();
+          break;
+        }
+        if (_pending.isNotEmpty && identical(_pending.first, chunk)) {
+          _pending.removeAt(0);
+        }
+      }
+    } finally {
+      _draining = false;
+    }
+  }
+
+  Future<bool> _attemptUpload(PendingChunk chunk) async {
+    for (var attempt = 0; attempt < maxAttemptsPerPass && !_disposed; attempt += 1) {
+      try {
+        await _uploader(chunk);
+        return true;
+      } catch (error) {
+        _logger?.call(
+          'chunk.upload.failed',
+          data: <String, Object?>{
+            'sourceKey': chunk.sourceKey,
+            'sequence': chunk.sequence,
+            'attempt': attempt + 1,
+          },
+          error: error,
+        );
+        await Future<void>.delayed(retryBackoff * (attempt + 1));
+      }
+    }
+    return false;
+  }
+
+  void _scheduleRetry() {
+    _retryTimer?.cancel();
+    _retryTimer = Timer(retryBackoff, () => unawaited(drain()));
+  }
+
+  /// Wait until every queued chunk has uploaded, or [timeout] elapses.
+  Future<void> flush({Duration timeout = const Duration(seconds: 30)}) async {
+    final deadline = DateTime.now().add(timeout);
+    while (_pending.isNotEmpty && !_disposed && DateTime.now().isBefore(deadline)) {
+      await drain();
+      if (_pending.isNotEmpty) {
+        await Future<void>.delayed(const Duration(milliseconds: 200));
+      }
+    }
+  }
+
+  Future<void> dispose() async {
+    _disposed = true;
+    _retryTimer?.cancel();
+    _retryTimer = null;
+    _pending.clear();
+  }
+}

package/flutter_app/lib/src/recording_chunk_queue_io.dart

@@ -0,0 +1,182 @@
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'recording_chunk_queue.dart';
+
+/// Disk-backed, ordered, retry-until-success upload queue for desktop.
+///
+/// Each chunk is persisted atomically before upload, so a network outage or
+/// backend hiccup mid-meeting never loses audio: the chunk stays on disk and is
+/// re-sent on the next drain. Files are deleted only after a confirmed upload.
+class DiskChunkUploadQueue {
+  DiskChunkUploadQueue({
+    required Directory directory,
+    required ChunkUploader uploader,
+    this.maxAttemptsPerPass = 3,
+    this.retryBackoff = const Duration(seconds: 2),
+    QueueLogger? logger,
+  }) : _dir = directory,
+       _uploader = uploader,
+       _logger = logger;
+
+  final Directory _dir;
+  final ChunkUploader _uploader;
+  final int maxAttemptsPerPass;
+  final Duration retryBackoff;
+  final QueueLogger? _logger;
+
+  bool _draining = false;
+  bool _disposed = false;
+  Timer? _retryTimer;
+  final Set<Future<void>> _pendingEnqueues = <Future<void>>{};
+
+  Future<void> enqueue(PendingChunk chunk) async {
+    if (_disposed) {
+      return;
+    }
+    final write = _persistChunk(chunk);
+    _pendingEnqueues.add(write);
+    try {
+      await write;
+    } finally {
+      _pendingEnqueues.remove(write);
+    }
+    unawaited(drain());
+  }
+
+  Future<void> drain() async {
+    if (_draining || _disposed) {
+      return;
+    }
+    _draining = true;
+    try {
+      while (!_disposed) {
+        final metaFiles = await _listMetaFiles();
+        if (metaFiles.isEmpty) {
+          break;
+        }
+        final metaFile = metaFiles.first;
+        final audioFile = File(
+          metaFile.path.replaceFirst(RegExp(r'\.json$'), '.bin'),
+        );
+        if (!await audioFile.exists()) {
+          await _deleteQuietly(metaFile);
+          continue;
+        }
+
+        final PendingChunk chunk;
+        try {
+          final meta =
+              jsonDecode(await metaFile.readAsString()) as Map<String, dynamic>;
+          chunk = PendingChunk.fromMeta(meta, await audioFile.readAsBytes());
+        } catch (error) {
+          // Corrupt metadata: drop the pair rather than blocking the queue.
+          _logger?.call('chunk.read.failed', error: error);
+          await _deleteQuietly(metaFile);
+          await _deleteQuietly(audioFile);
+          continue;
+        }
+
+        final uploaded = await _attemptUpload(chunk);
+        if (!uploaded) {
+          _scheduleRetry();
+          break;
+        }
+        await _deleteQuietly(audioFile);
+        await _deleteQuietly(metaFile);
+      }
+    } finally {
+      _draining = false;
+    }
+  }
+
+  Future<bool> _attemptUpload(PendingChunk chunk) async {
+    for (var attempt = 0; attempt < maxAttemptsPerPass && !_disposed; attempt += 1) {
+      try {
+        await _uploader(chunk);
+        return true;
+      } catch (error) {
+        _logger?.call(
+          'chunk.upload.failed',
+          data: <String, Object?>{
+            'sourceKey': chunk.sourceKey,
+            'sequence': chunk.sequence,
+            'attempt': attempt + 1,
+          },
+          error: error,
+        );
+        await Future<void>.delayed(retryBackoff * (attempt + 1));
+      }
+    }
+    return false;
+  }
+
+  Future<List<File>> _listMetaFiles() async {
+    if (!await _dir.exists()) {
+      return const <File>[];
+    }
+    final entries = await _dir.list().toList();
+    final metaFiles = entries
+        .whereType<File>()
+        .where((file) => file.path.endsWith('.json'))
+        .toList()
+      ..sort((a, b) => a.path.compareTo(b.path));
+    return metaFiles;
+  }
+
+  Future<int> pendingCount() async => (await _listMetaFiles()).length;
+
+  Future<void> _persistChunk(PendingChunk chunk) async {
+    await _dir.create(recursive: true);
+    final base = '${chunk.sourceKey}-${chunk.sequence.toString().padLeft(6, '0')}';
+    final audioFile = File('${_dir.path}/$base.bin');
+    final metaFile = File('${_dir.path}/$base.json');
+    final audioTmp = File('${audioFile.path}.tmp');
+    final metaTmp = File('${metaFile.path}.tmp');
+    await audioTmp.writeAsBytes(chunk.bytes, flush: true);
+    await metaTmp.writeAsString(jsonEncode(chunk.toMeta()), flush: true);
+    await audioTmp.rename(audioFile.path);
+    await metaTmp.rename(metaFile.path);
+  }
+
+  Future<void> _waitForPendingEnqueues() async {
+    while (_pendingEnqueues.isNotEmpty && !_disposed) {
+      await Future.wait<void>(List<Future<void>>.from(_pendingEnqueues));
+    }
+  }
+
+  void _scheduleRetry() {
+    _retryTimer?.cancel();
+    _retryTimer = Timer(retryBackoff, () => unawaited(drain()));
+  }
+
+  Future<void> _deleteQuietly(File file) async {
+    try {
+      if (await file.exists()) {
+        await file.delete();
+      }
+    } catch (error) {
+      _logger?.call('chunk.delete.failed', error: error);
+    }
+  }
+
+  /// Wait until every persisted chunk has uploaded, or [timeout] elapses.
+  Future<void> flush({Duration timeout = const Duration(seconds: 30)}) async {
+    final deadline = DateTime.now().add(timeout);
+    while (!_disposed && DateTime.now().isBefore(deadline)) {
+      await _waitForPendingEnqueues();
+      await drain();
+      if (_pendingEnqueues.isEmpty && await pendingCount() == 0) {
+        break;
+      }
+      await Future<void>.delayed(const Duration(milliseconds: 200));
+    }
+  }
+
+  Future<void> dispose() async {
+    _disposed = true;
+    _retryTimer?.cancel();
+    _retryTimer = null;
+  }
+}

package/flutter_app/lib/src/recording_payloads.dart

@@ -13,6 +13,15 @@ Map<String, dynamic> buildWebScreenAndMicRecordingPayload() {
           'transcribe': false,
         },
       },
+      <String, dynamic>{
+        'sourceKey': 'system',
+        'sourceKind': 'system-audio',
+        'mediaKind': 'audio',
+        'mimeType': 'audio/webm',
+        'metadata': <String, dynamic>{
+          'transcribe': true,
+        },
+      },
       <String, dynamic>{
         'sourceKey': 'microphone',
         'sourceKind': 'microphone',

package/flutter_app/macos/Runner/AppDelegate.swift

@@ -16,6 +16,7 @@ class AppDelegate: FlutterAppDelegate {
 
 final class DesktopCompanionNativePlugin: NSObject {
   private static let channelName = "neoagent/desktop_companion_native"
+  private static let idleThresholdSeconds: Double = 300
   private static var channelAssociationKey: UInt8 = 0
   private static var instanceAssociationKey: UInt8 = 0
 
@@ -268,6 +269,7 @@ final class DesktopCompanionNativePlugin: NSObject {
 
   private func buildDesktopStatus() -> [String: Any] {
     let accessibilityTrusted = isAccessibilityTrusted()
+    let idleSeconds = userIdleSeconds()
     var status: [String: Any] = [
       "permissions": [
         "screenCapture": preflightScreenCapturePermission() ? "available" : "required",
@@ -276,6 +278,9 @@ final class DesktopCompanionNativePlugin: NSObject {
       ],
       "displays": desktopDisplays(),
       "activeDisplayId": defaultDisplayIdentifier(),
+      "sessionLocked": isSessionLocked(),
+      "idleSeconds": idleSeconds,
+      "userIdle": idleSeconds >= Self.idleThresholdSeconds,
     ]
 
     if let appName = NSWorkspace.shared.frontmostApplication?.localizedName {
@@ -424,6 +429,26 @@ final class DesktopCompanionNativePlugin: NSObject {
     return title
   }
 
+  private func isSessionLocked() -> Bool {
+    guard let session = CGSessionCopyCurrentDictionary() as? [String: Any] else {
+      return false
+    }
+    if let locked = session["CGSSessionScreenIsLocked"] as? Bool {
+      return locked
+    }
+    if let locked = session["CGSSessionScreenIsLocked"] as? NSNumber {
+      return locked.boolValue
+    }
+    return false
+  }
+
+  private func userIdleSeconds() -> Double {
+    CGEventSource.secondsSinceLastEventType(
+      .combinedSessionState,
+      eventType: .null
+    )
+  }
+
   private func performClick(x: Double, y: Double, button: String, displayId: String?) {
     let point = nativePointForCapturedPixel(x: x, y: y, displayId: displayId)
     let mouseButton = cgMouseButton(button)

package/flutter_app/windows/runner/flutter_window.cpp

@@ -241,6 +241,70 @@ std::string ForegroundWindowTitle() {
   return WideToUtf8(std::wstring(buffer, buffer + length));
 }
 
+std::string BasenameFromPath(const std::wstring& path) {
+  if (path.empty()) {
+    return std::string();
+  }
+  const size_t slash = path.find_last_of(L"\\/");
+  const std::wstring name =
+      slash == std::wstring::npos ? path : path.substr(slash + 1);
+  return WideToUtf8(name);
+}
+
+std::string ForegroundWindowApp() {
+  HWND window = GetForegroundWindow();
+  if (window == nullptr) {
+    return std::string();
+  }
+  DWORD process_id = 0;
+  GetWindowThreadProcessId(window, &process_id);
+  if (process_id == 0) {
+    return std::string();
+  }
+  HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, process_id);
+  if (process == nullptr) {
+    return std::string();
+  }
+  wchar_t buffer[MAX_PATH];
+  DWORD size = MAX_PATH;
+  std::string app_name;
+  if (QueryFullProcessImageNameW(process, 0, buffer, &size)) {
+    app_name = BasenameFromPath(std::wstring(buffer, size));
+  }
+  CloseHandle(process);
+  return app_name;
+}
+
+double UserIdleSeconds() {
+  LASTINPUTINFO info{};
+  info.cbSize = sizeof(LASTINPUTINFO);
+  if (!GetLastInputInfo(&info)) {
+    return 0;
+  }
+  const ULONGLONG now = GetTickCount64();
+  const ULONGLONG then = static_cast<ULONGLONG>(info.dwTime);
+  if (now < then) {
+    return 0;
+  }
+  return static_cast<double>(now - then) / 1000.0;
+}
+
+bool IsWorkstationLocked() {
+  HDESK input_desktop = OpenInputDesktop(0, FALSE, GENERIC_READ);
+  if (input_desktop == nullptr) {
+    return false;
+  }
+  wchar_t name[256];
+  DWORD needed = 0;
+  const bool ok = GetUserObjectInformationW(
+      input_desktop, UOI_NAME, name, sizeof(name), &needed);
+  CloseDesktop(input_desktop);
+  if (!ok) {
+    return false;
+  }
+  return _wcsicmp(name, L"Default") != 0;
+}
+
 void SendMouseButton(DWORD flag) {
   INPUT input{};
   input.type = INPUT_MOUSE;
@@ -329,6 +393,8 @@ bool FlutterWindow::OnCreate() {
         const auto* arguments = std::get_if<EncodableMap>(call.arguments());
 
         if (call.method_name() == "getStatus") {
+          const std::string frontmost_app = ForegroundWindowApp();
+          const double idle_seconds = UserIdleSeconds();
           EncodableMap permissions;
           permissions[EncodableValue("screenCapture")] = EncodableValue("available");
           permissions[EncodableValue("inputControl")] = EncodableValue("available");
@@ -339,6 +405,15 @@ bool FlutterWindow::OnCreate() {
           status[EncodableValue("displays")] = EncodableValue(DisplaysToEncodable());
           status[EncodableValue("activeDisplayId")] =
               EncodableValue(WideToUtf8(ResolveDisplay("").id));
+          status[EncodableValue("sessionLocked")] =
+              EncodableValue(IsWorkstationLocked());
+          status[EncodableValue("idleSeconds")] = EncodableValue(idle_seconds);
+          status[EncodableValue("userIdle")] =
+              EncodableValue(idle_seconds >= 300.0);
+          if (!frontmost_app.empty()) {
+            status[EncodableValue("frontmostApp")] =
+                EncodableValue(frontmost_app);
+          }
           const std::string window_title = ForegroundWindowTitle();
           if (!window_title.empty()) {
             status[EncodableValue("frontmostWindowTitle")] =

package/landing/index.html

@@ -640,7 +640,7 @@ p { margin: 0; text-wrap: pretty; }
 .hero-note { margin-top: 16px; font-family: var(--mono); font-size: 12.5px; color: var(--ink-3); letter-spacing: 0.02em; }
 
 /* ---------- orbit system ---------- */
-.orbit-stage { position: relative; width: min(460px, 86vw); aspect-ratio: 1; margin-inline: auto; }
+.orbit-stage { position: relative; width: min(460px, 86vw, 100%); aspect-ratio: 1; margin-inline: auto; }
 .orbit-glow { position: absolute; inset: 16%; border-radius: 50%; background: radial-gradient(circle, rgba(199,154,82,0.28), transparent 65%); filter: blur(22px); animation: corepulse 7s ease-in-out infinite; }
 @media (prefers-color-scheme: dark) { .orbit-glow { background: radial-gradient(circle, rgba(228,197,141,0.22), transparent 65%); } }
 @keyframes corepulse { 0%,100% { opacity: .7; transform: scale(1); } 50% { opacity: 1; transform: scale(1.07); } }
@@ -830,7 +830,7 @@ p { margin: 0; text-wrap: pretty; }
   .hero-actions { flex-direction: column; width: 100%; }
   .hero-actions .btn { width: 100%; justify-content: center; }
   .final .hero-actions .btn { width: auto; }
-  .orbit-stage { width: min(320px, 84vw); }
+  .orbit-stage { width: min(320px, 78vw); }
   .frame-bar .url { max-width: 70%; }
   .footer-inner { flex-direction: column; gap: 22px; }
   .footer-bar { flex-direction: column; align-items: flex-start; gap: 8px; }
@@ -848,7 +848,7 @@ p { margin: 0; text-wrap: pretty; }
   .logo .ri { transform-box: fill-box; transform-origin: center; }
   .logo .ri-a { animation: spinA 18s linear infinite; }
   .logo .ri-b { animation: spinB 26s linear infinite; }
-  .logo .nd  { animation: nd 14s linear infinite; transform-box: fill-box; transform-origin: 50px 50px; }
+  .logo .nd  { animation: nd 14s linear infinite; transform-box: view-box; transform-origin: 50px 50px; }
   @keyframes spinA { to { transform: rotate(360deg); } }
   @keyframes spinB { to { transform: rotate(-360deg); } }
   @keyframes nd    { to { transform: rotate(360deg); } }