neoagent 3.1.1-beta.9 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/.env.example +2 -2
  2. package/README.md +5 -1
  3. package/docs/architecture.md +1 -1
  4. package/docs/benchmarking.md +68 -87
  5. package/docs/capabilities.md +1 -1
  6. package/docs/clients-and-devices.md +3 -4
  7. package/docs/configuration.md +1 -1
  8. package/docs/devices.md +1 -2
  9. package/docs/health.md +24 -0
  10. package/docs/index.md +2 -2
  11. package/docs/integrations-architecture.md +6 -0
  12. package/docs/integrations.md +15 -1
  13. package/docs/models.md +1 -1
  14. package/docs/persistence.md +1 -1
  15. package/docs/why-neoagent.md +1 -1
  16. package/flutter_app/README.md +0 -5
  17. package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
  19. package/flutter_app/lib/main.dart +0 -6
  20. package/flutter_app/lib/main_app_shell.dart +0 -2
  21. package/flutter_app/lib/main_controller.dart +2 -867
  22. package/flutter_app/lib/main_devices.dart +0 -73
  23. package/flutter_app/lib/main_integrations.dart +339 -187
  24. package/flutter_app/lib/main_launcher.dart +2 -212
  25. package/flutter_app/lib/main_models.dart +0 -259
  26. package/flutter_app/lib/main_navigation.dart +2 -14
  27. package/flutter_app/lib/main_operations.dart +322 -364
  28. package/flutter_app/lib/main_runtime.dart +9 -208
  29. package/flutter_app/lib/main_settings.dart +17 -199
  30. package/flutter_app/lib/main_shared.dart +20 -591
  31. package/flutter_app/lib/main_timeline.dart +26 -122
  32. package/flutter_app/lib/main_voice_assistant.dart +0 -5
  33. package/flutter_app/lib/src/backend_client.dart +0 -102
  34. package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
  35. package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
  36. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
  37. package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
  38. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
  39. package/flutter_app/macos/Podfile.lock +56 -5
  40. package/flutter_app/pubspec.lock +0 -7
  41. package/flutter_app/pubspec.yaml +0 -2
  42. package/flutter_app/web/index.html +174 -0
  43. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
  44. package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
  45. package/landing/index.html +0 -2
  46. package/lib/manager.js +1 -1
  47. package/lib/schema_migrations.js +75 -0
  48. package/package.json +1 -4
  49. package/server/db/database.js +1 -133
  50. package/server/http/middleware.js +2 -16
  51. package/server/http/routes.js +0 -2
  52. package/server/public/.last_build_id +1 -1
  53. package/server/public/assets/NOTICES +0 -25
  54. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  55. package/server/public/flutter_bootstrap.js +1 -1
  56. package/server/public/index.html +174 -0
  57. package/server/public/main.dart.js +88081 -90036
  58. package/server/routes/auth.js +11 -2
  59. package/server/routes/settings.js +0 -5
  60. package/server/routes/voice_assistant.js +1 -128
  61. package/server/services/account/erasure.js +0 -3
  62. package/server/services/ai/deliverables/artifact_helpers.js +1 -1
  63. package/server/services/ai/loop/conversation_loop.js +6 -2
  64. package/server/services/ai/loop/tool_dispatch.js +0 -3
  65. package/server/services/ai/messagingFallback.js +1 -1
  66. package/server/services/ai/settings.js +0 -6
  67. package/server/services/ai/toolEvidence.js +1 -4
  68. package/server/services/ai/tools.js +0 -234
  69. package/server/services/desktop/auth.js +0 -3
  70. package/server/services/desktop/registry.js +2 -50
  71. package/server/services/integrations/neoarchive/provider.js +616 -0
  72. package/server/services/integrations/neorecall/client.js +140 -0
  73. package/server/services/integrations/neorecall/constants.js +87 -0
  74. package/server/services/integrations/neorecall/provider.js +163 -0
  75. package/server/services/integrations/registry.js +4 -2
  76. package/server/services/manager.js +0 -22
  77. package/server/services/memory/manager.js +44 -19
  78. package/server/services/memory/retrieval_reasoning.js +36 -14
  79. package/server/services/messaging/whatsapp.js +18 -0
  80. package/server/services/security/tool_categories.js +0 -5
  81. package/server/services/social_video/service.js +2 -20
  82. package/server/services/tasks/adapters/index.js +1 -1
  83. package/server/services/tasks/integration_runtime.js +3 -67
  84. package/server/services/tasks/runtime.js +1 -6
  85. package/server/services/tasks/task_repository.js +6 -13
  86. package/server/services/timeline/service.js +0 -224
  87. package/server/services/voice/message.js +0 -24
  88. package/server/services/voice/providers.js +42 -1
  89. package/server/services/voice/runtimeManager.js +9 -5
  90. package/server/services/voice/turnRunner.js +0 -22
  91. package/server/services/wearable/service.js +0 -5
  92. package/docs/recordings-and-health.md +0 -41
  93. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
  94. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
  95. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
  96. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
  97. package/flutter_app/lib/main_recordings.dart +0 -920
  98. package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
  99. package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
  100. package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
  101. package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
  102. package/flutter_app/lib/src/recording_bridge.dart +0 -232
  103. package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
  104. package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
  105. package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
  106. package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
  107. package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
  108. package/flutter_app/lib/src/recording_payloads.dart +0 -95
  109. package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
  110. package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
  111. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
  112. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
  113. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
  114. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
  115. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
  116. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
  117. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
  118. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
  119. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
  120. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
  121. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
  122. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
  123. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
  124. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
  125. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
  126. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
  127. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
  128. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
  129. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
  130. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
  131. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
  132. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
  133. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
  134. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
  135. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
  136. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
  137. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
  138. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
  139. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
  140. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
  141. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
  142. package/server/routes/recordings.js +0 -335
  143. package/server/routes/screenHistory.js +0 -190
  144. package/server/services/ai/recordingInsights.js +0 -105
  145. package/server/services/integrations/neomail/provider.js +0 -993
  146. package/server/services/recordings/deepgram.js +0 -58
  147. package/server/services/recordings/manager.js +0 -1273
  148. package/server/services/tasks/adapters/neomail_email_received.js +0 -46
  149. package/server/services/voice/screenshotContext.js +0 -73
  150. package/server/services/voice/turnCoordinator.js +0 -51
package/.env.example CHANGED
@@ -236,8 +236,8 @@ TRELLO_API_KEY=your-trello-api-key
236
236
  BRAVE_SEARCH_API_KEY=your-brave-search-api-key-here
237
237
 
238
238
  # Deepgram API key — used for:
239
- # • Recordings transcription (Deepgram Nova-3 multilingual via /api/recordings)
240
- # Without this key: recordings can still be captured and stored, but transcription will fail until retried with a valid key.
239
+ # • Voice transcription (Deepgram Nova-3 multilingual)
240
+ # Without this key: Deepgram-backed voice transcription is unavailable.
241
241
  # Get your key at: https://console.deepgram.com/
242
242
  DEEPGRAM_API_KEY=your-deepgram-api-key-here
243
243
  DEEPGRAM_BASE_URL=https://api.deepgram.com
package/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  <p align="center">
2
- <img src="flutter_app/assets/branding/app_icon_128.png" width="80" alt="NeoAgent">
2
+ <img src="marketing/final/github-banner.png" alt="NeoAgent — an AI assistant that keeps working" width="100%">
3
3
  </p>
4
4
 
5
5
  <h1 align="center">NeoAgent</h1>
@@ -28,6 +28,10 @@
28
28
  <a href="https://www.producthunt.com/products/neoagent-2?embed=true&utm_source=badge-featured&utm_medium=badge&utm_campaign=badge-neoagent-2" target="_blank" rel="noopener noreferrer"><img alt="NeoAgent - The next-gen self-hosted AI agent that works beyond chat | Product Hunt" width="250" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=1171573&theme=light&t=1781447653039"></a>
29
29
  </p>
30
30
 
31
+ <p align="center">
32
+ <img src="marketing/final/trailer.gif" alt="NeoAgent trailer" width="100%">
33
+ </p>
34
+
31
35
  <p align="center">
32
36
  <img src="demo.gif" alt="NeoAgent demo" width="100%">
33
37
  </p>
@@ -36,7 +36,7 @@ bridge files and native Android code where required.
36
36
 
37
37
  NeoAgent uses one `better-sqlite3` database instance. It stores users, agents,
38
38
  settings, conversations, runs, tasks, integration connections, memory,
39
- recordings, health data, permissions, and operational state.
39
+ health data, permissions, and operational state.
40
40
 
41
41
  Runtime files live outside the package source under `NEOAGENT_HOME`. Schema
42
42
  changes are applied through the migration layer rather than ad hoc service SQL.
@@ -1,102 +1,83 @@
1
1
  # Benchmarking
2
2
 
3
- NeoAgent includes an end-to-end benchmark harness that drives the app through
4
- its authenticated HTTP surface instead of calling internal engine methods
5
- directly. The harness configures OpenRouter models, submits benchmark tasks,
6
- collects run events and usage data, and writes normalized result artifacts.
3
+ NeoAgent's benchmark harness measures the actual quality of its memory system on
4
+ **LoCoMo** ([Snap Research](https://github.com/snap-research/locomo), CC BY-NC 4.0), the
5
+ standard long-term conversational memory benchmark. It follows the same protocol
6
+ mem0/Zep/Letta/[Omi](https://github.com/BasedHardware/omi-memory-benchmarks) use to publish
7
+ their numbers, so results are comparable: a thin QA head over the memory system's real
8
+ retrieval API, not the consumer chat agent.
7
9
 
8
10
  ## Latest dashboard
9
11
 
10
12
  ![Latest benchmark dashboard](/benchmarks/latest-dashboard.png)
11
13
 
12
- The generated image is backed by these local outputs:
14
+ Backed by these local outputs:
13
15
 
14
- - `benchmark/results/latest-results.json`
15
- - `benchmark/results/latest-summary.json`
16
- - `benchmark/results/latest-summary.md`
16
+ - `benchmark/results/latest-results.json` — every question, gold answer, generated
17
+ answer, and CORRECT/WRONG verdict.
18
+ - `benchmark/results/latest-summary.json` / `latest-summary.md` — overall and
19
+ per-category accuracy.
17
20
  - `static/benchmarks/latest-dashboard.png`
18
21
 
19
22
  ## Commands
20
23
 
21
24
  ```bash
22
- npm run benchmark:setup
23
- npm run benchmark:run
24
- npm run benchmark:report
25
+ npm run benchmark:setup # downloads and validates the LoCoMo dataset
26
+ npm run benchmark:run # ingests, retrieves, answers, judges, scores
27
+ npm run benchmark:report # re-renders the dashboard from the last run's results.json
25
28
  ```
26
29
 
27
- ## Model selection
28
-
29
- The benchmark harness is OpenRouter-first. Edit the `OPENROUTER_MODEL_IDS`
30
- variable in `benchmark/config.js` to pin exact model IDs, or leave it empty to
31
- use the current cheap-tier OpenRouter models discovered at runtime.
32
-
33
- ## Benchmark coverage
34
-
35
- - `GAIA`, `BrowseComp`, `WebArena`, `VisualWebArena`, and `SWE-bench` are wired
36
- as exact public-suite adapters. When their official datasets or external
37
- runners are not installed, they are reported as blocked instead of being
38
- approximated.
39
- - `NeoAgent Representative Tasks` runs first-party agent tasks through
40
- `/api/agents` and scores them from the persisted run details.
41
- - `NeoAgent Memory Retrieval` seeds memories over `/api/memory`, recalls them
42
- through the app routes, and scores retrieval quality with the same metric
43
- code used by the server evaluation module.
44
-
45
- ## Reproducibility
46
-
47
- - Public-suite setup manifests are written under `benchmark/workdir/<suite>/`.
48
- - The harness uses one benchmark account and persists per-run usage summaries
49
- through the existing NeoAgent run detail endpoints.
50
- - Blocked suites are intentional and explicit. The harness never substitutes a
51
- simplified local task and labels it as a public benchmark result.
52
-
53
- ## Public benchmark prerequisites
54
-
55
- ### GAIA
56
-
57
- - Access to the gated [GAIA dataset](https://huggingface.co/datasets/gaia-benchmark/GAIA).
58
- - `HF_TOKEN` in the environment.
59
- - A Python environment with `datasets` installed if you want `benchmark:setup`
60
- to export a normalized local cache automatically.
61
- - Either an exact evaluator wired through
62
- `NEOAGENT_BENCHMARK_GAIA_RUNNER` or a repository-local exact runner added to
63
- the harness. The current adapter intentionally reports `blocked` until an
64
- official evaluator path is configured.
65
-
66
- ### BrowseComp
67
-
68
- - A checkout of [openai/simple-evals](https://github.com/openai/simple-evals),
69
- which `benchmark:setup` can clone into `benchmark/workdir/browsecomp/`.
70
- - Exact BrowseComp case extraction from that repo.
71
- - An official evaluator command exposed through
72
- `NEOAGENT_BENCHMARK_BROWSECOMP_RUNNER`.
73
-
74
- ### WebArena
75
-
76
- - Docker installed and running.
77
- - The official [WebArena](https://webarena.dev/) environment and task sites.
78
- - Browser automation dependencies required by the official runner.
79
- - An exact runner command exposed through
80
- `NEOAGENT_BENCHMARK_WEBARENA_RUNNER`.
81
-
82
- ### VisualWebArena
83
-
84
- - Docker installed and running.
85
- - The official VisualWebArena environment.
86
- - A vision-capable configured benchmark model.
87
- - An exact runner command exposed through
88
- `NEOAGENT_BENCHMARK_VISUAL_WEBARENA_RUNNER`.
89
-
90
- ### SWE-bench
91
-
92
- - Docker installed and running.
93
- - A checkout of [SWE-bench](https://github.com/SWE-bench/SWE-bench), which
94
- `benchmark:setup` can clone into `benchmark/workdir/swebench/`.
95
- - The official dataset/runtime and enough disk/network budget for repo/image
96
- setup.
97
- - An exact runner command exposed through
98
- `NEOAGENT_BENCHMARK_SWEBENCH_RUNNER`.
99
-
100
- On this machine, `docker` is currently missing, so exact `WebArena`,
101
- `VisualWebArena`, and `SWE-bench` execution remain blocked until Docker and the
102
- official runners are installed.
30
+ `benchmark:run` requires:
31
+ - A running NeoAgent server (`NEOAGENT_BENCHMARK_BASE_URL`, default `http://127.0.0.1:3333`).
32
+ - `OPENROUTER_API_KEY` in the environment it configures each benchmark account's own
33
+ memory-extraction provider, and drives the answerer/judge models.
34
+
35
+ ## How it works
36
+
37
+ 1. **Ingestion.** For each LoCoMo conversation, the harness registers an isolated
38
+ benchmark account and pushes every session through NeoAgent's real document-ingestion
39
+ pipeline (`POST /api/memory/ingestion/documents`) one document per session, dated and
40
+ speaker-labeled, chunked exactly like any other real ingested source.
41
+ 2. **Extraction draining.** NeoAgent promotes raw ingested chunks into structured
42
+ memories/facts as a side effect of the agent loop's memory-recall step, not the raw
43
+ ingestion call itself. The harness issues a handful of trivial agent runs per account to
44
+ settle this before measuring retrieval using the app's real settling mechanism rather
45
+ than reaching into internals.
46
+ 3. **Retrieval.** `POST /api/memory/memories/recall` NeoAgent's real fused
47
+ lexical+entity+vector retrieval surface.
48
+ 4. **Answering.** One OpenRouter chat completion per question, using mem0's verbatim
49
+ `ANSWER_PROMPT` (≤5-6-word answers, relative→absolute date conversion).
50
+ 5. **Judging.** One OpenRouter chat completion using mem0's verbatim `ACCURACY_PROMPT`
51
+ (lenient CORRECT/WRONG) the same judge all of the above vendors report against.
52
+ LoCoMo category 5 (adversarial) is excluded, exactly as mem0's own eval code does.
53
+
54
+ Prompts live verbatim in [`benchmark/locomo/prompts.js`](../benchmark/locomo/prompts.js).
55
+
56
+ ## Configuration
57
+
58
+ | Env var | Default | Purpose |
59
+ | --- | --- | --- |
60
+ | `NEOAGENT_BENCHMARK_BASE_URL` | `http://127.0.0.1:3333` | Server to benchmark |
61
+ | `OPENROUTER_API_KEY` | (required) | Answerer/judge models + per-account memory extraction |
62
+ | `BENCHMARK_ANSWER_MODEL` / `BENCHMARK_JUDGE_MODEL` | cheapest priced OpenRouter model | Pin exact model IDs |
63
+ | `BENCHMARK_LOCOMO_SAMPLES` / `BENCHMARK_LOCOMO_OFFSET` | `10` / `0` | Which conversations to run |
64
+ | `BENCHMARK_LOCOMO_QA_PER_SAMPLE` | `0` (all) | Cap questions per conversation |
65
+ | `BENCHMARK_LOCOMO_MEM_K` | `20` | Recall depth per question |
66
+ | `BENCHMARK_CONCURRENCY` | `1` | Parallel QA-head requests |
67
+ | `BENCHMARK_CASE_LIMIT` | `0` (no cap) | Global question cap, useful for smoke tests |
68
+
69
+ For a cheap smoke test: `BENCHMARK_LOCOMO_SAMPLES=1 BENCHMARK_LOCOMO_QA_PER_SAMPLE=5 npm run benchmark:run`.
70
+
71
+ ## Honest caveats
72
+
73
+ - **Single run, temperature 0.** No averaging across repeats.
74
+ - **Category 5 (adversarial) excluded**, matching the standard mem0 LoCoMo protocol.
75
+ - **Ingestion granularity.** LoCoMo sessions are ingested as NeoAgent's native document
76
+ unit (one per session); this differs from message-pair-level ingestion some vendors use,
77
+ which can make retrieval comparatively easier or harder depending on the question.
78
+ - **Judge leniency** varies significantly across published papers; this harness uses mem0's
79
+ lenient judge verbatim so numbers line up with mem0/Zep/Letta/Omi's published columns.
80
+ - The LoCoMo dataset (CC BY-NC 4.0) is downloaded on demand into `benchmark/workdir/`
81
+ (gitignored) never vendored into the repository. Result files may quote benchmark
82
+ questions/answers for verifiability; no NeoAgent user data is involved (every run uses
83
+ synthetic benchmark accounts).
@@ -14,7 +14,7 @@ capabilities by workflow instead of maintaining one long inventory.
14
14
 
15
15
  - [Integrations and messaging](integrations.md)
16
16
  - [Devices and interfaces](devices.md)
17
- - [Recordings and health](recordings-and-health.md)
17
+ - [Health](health.md)
18
18
 
19
19
  ## Run unattended work
20
20
 
@@ -21,10 +21,9 @@ updates into `MainController`, which notifies the relevant UI.
21
21
 
22
22
  ## Android bridges
23
23
 
24
- Native Kotlin code handles Health Connect, background recording,
25
- notifications, launcher functions, telecom integration, and Android home
26
- widgets. Flutter bridges normalize permission and lifecycle behavior before
27
- updating application state.
24
+ Native Kotlin code handles Health Connect, notifications, launcher functions,
25
+ telecom integration, and Android home widgets. Flutter bridges normalize
26
+ permission and lifecycle behavior before updating application state.
28
27
 
29
28
  ## Desktop and Chrome pairing
30
29
 
@@ -49,7 +49,7 @@ neoagent env set SESSION_SECRET "$(openssl rand -hex 32)"
49
49
  | `ANTHROPIC_BASE_URL` | Anthropic-compatible base URL override |
50
50
  | `OLLAMA_URL` | Ollama server URL |
51
51
  | `BRAVE_SEARCH_API_KEY` | Web search |
52
- | `DEEPGRAM_API_KEY` | Recording transcription |
52
+ | `DEEPGRAM_API_KEY` | Voice transcription |
53
53
  | `DEEPGRAM_BASE_URL` | Deepgram-compatible base URL override |
54
54
  | `DEEPGRAM_MODEL` | Speech model |
55
55
  | `DEEPGRAM_LANGUAGE` | Speech language mode |
package/docs/devices.md CHANGED
@@ -7,7 +7,7 @@ explicitly paired as a runtime backend.
7
7
  ## Web interface
8
8
 
9
9
  The web interface provides chat, runs, tasks, agents, integrations, memory,
10
- devices, recordings, health, settings, permissions, and logs. It is served by
10
+ devices, health, settings, permissions, and logs. It is served by
11
11
  the NeoAgent backend.
12
12
 
13
13
  ## Android app
@@ -15,7 +15,6 @@ the NeoAgent backend.
15
15
  The Android app connects to the same backend and adds:
16
16
 
17
17
  - Health Connect synchronization
18
- - Background microphone recording
19
18
  - Android notification forwarding
20
19
  - Native notifications and device pairing
21
20
 
package/docs/health.md ADDED
@@ -0,0 +1,24 @@
1
+ # Health
2
+
3
+ Health data is an optional feature of the NeoAgent clients. It stores
4
+ processed data on the NeoAgent backend.
5
+
6
+ ## Health Connect
7
+
8
+ The Android app can read Health Connect records after the user grants Android
9
+ permissions. Supported data includes steps, heart rate, sleep, exercise, and
10
+ weight.
11
+
12
+ The app uploads granted records to the user's NeoAgent backend. Stored metrics
13
+ appear in **Health** and are available to the owning agent through the health
14
+ tool.
15
+
16
+ Health Connect access can be revoked in Android settings. Revoking access stops
17
+ future reads but does not remove records already uploaded to NeoAgent; delete
18
+ stored server data separately when required.
19
+
20
+ ## Privacy
21
+
22
+ Health records are sensitive personal data. Protect the server account, use
23
+ HTTPS for remote clients, restrict agents and integrations, and include this
24
+ data store in backup and deletion procedures.
package/docs/index.md CHANGED
@@ -8,7 +8,7 @@ sidebar_label: Overview
8
8
 
9
9
  NeoAgent is a self-hosted AI agent that runs continuously on a macOS or Linux
10
10
  server. It provides an operator interface for chat, automation, integrations,
11
- memory, recordings, and connected devices.
11
+ memory, and connected devices.
12
12
 
13
13
  NeoAgent is beta software. Install it on a machine you administer, start with
14
14
  restricted tool permissions, and read the security guide before connecting
@@ -42,7 +42,7 @@ Continue with:
42
42
  | [Memory](memory.md) | What NeoAgent remembers and how to manage it |
43
43
  | [Integrations](integrations.md) | Connected app accounts and messaging channels |
44
44
  | [Devices and interfaces](devices.md) | Android, desktop, Chrome, launcher, and wearable surfaces |
45
- | [Recordings and health](recordings-and-health.md) | Audio transcription and Health Connect |
45
+ | [Health](health.md) | Health Connect |
46
46
  | [Skills and MCP](skills.md) | Reusable instructions and external tool servers |
47
47
  | [Operations](operations.md) | Updates, backups, logs, and recovery |
48
48
  | [Billing](billing.md) | Stripe subscriptions, plans, trials, and webhooks |
@@ -35,6 +35,12 @@ chunks.
35
35
  Provider snapshots shown in the client are decorated with ingestion coverage
36
36
  and status.
37
37
 
38
+ NeoRecall is intentionally queried on demand instead of being copied into
39
+ NeoAgent memory. Its integration uses a dynamic public OAuth client with PKCE,
40
+ rotating refresh tokens, and read-only scopes. The available tools call
41
+ NeoRecall's local hybrid search and evidence APIs; they cannot access audio or
42
+ trigger NeoRecall's Ask endpoint.
43
+
38
44
  ## Messaging
39
45
 
40
46
  Messaging providers are long-running channel adapters managed separately from
@@ -19,6 +19,8 @@ Connect accounts in **Integrations**.
19
19
  | Notion | Pages, databases, blocks, search |
20
20
  | Slack | Conversations, history, search, messages |
21
21
  | Figma | Files, nodes, rendered images, comments |
22
+ | NeoArchive | Documents, extracted text, metadata, search, upload, archive, reprocessing |
23
+ | NeoRecall | Local hybrid recall search, memories, mini-memories, daily summaries, conversations, transcript evidence |
22
24
  | Home Assistant | Entity state and service calls |
23
25
  | Trello | Boards, lists, cards, comments, search |
24
26
  | Spotify | Playback, history, search, queue |
@@ -48,7 +50,19 @@ Google, Microsoft, GitHub, Notion, Slack, Figma, and Spotify require OAuth appli
48
50
  - `GITHUB_CLIENT_ID=your_id_here`
49
51
  - `GITHUB_CLIENT_SECRET=your_secret_here`
50
52
 
51
- Home Assistant and Trello can be configured for a user from the application. See Configuration.
53
+ NeoArchive, NeoRecall, Home Assistant, and Trello can be configured for a user from the application. See Configuration.
54
+
55
+ ### NeoRecall
56
+
57
+ Open **Integrations**, choose **NeoRecall**, and enter the base URL of the
58
+ NeoRecall server. NeoAgent validates the server and opens NeoRecall's own
59
+ authorization page. Sign in there and approve the three read-only scopes.
60
+
61
+ NeoRecall contributes seven on-demand tools for local hybrid search and for
62
+ reading memories, mini-memories, daily summaries, conversations, and transcript
63
+ evidence. It does not expose audio, recording, memory mutation, consolidation,
64
+ or NeoRecall Ask. This keeps the products isolated and avoids a second LLM call
65
+ when NeoAgent only needs retrieval context.
52
66
 
53
67
  ## Messaging channels
54
68
 
package/docs/models.md CHANGED
@@ -44,7 +44,7 @@ Some features use separate providers:
44
44
 
45
45
  - Embeddings for memory search use a configured supported embedding provider
46
46
  and fall back to lexical retrieval when embeddings are unavailable.
47
- - Recording transcription uses Deepgram when enabled.
47
+ - Voice transcription can use Deepgram when enabled.
48
48
  - Image generation and analysis depend on the selected provider and model.
49
49
 
50
50
  ## Credential handling
@@ -13,7 +13,7 @@ The database owns:
13
13
  - Tasks, triggers, webhook deliveries, and widgets
14
14
  - Integration connections and encrypted credentials
15
15
  - Memory, facts, entities, source documents, and retrieval telemetry
16
- - Recordings, transcripts, health records, and device registrations
16
+ - Health records and device registrations
17
17
  - Tool policies and approval history
18
18
 
19
19
  Larger artifacts and editable agent data use runtime directories under
@@ -28,7 +28,7 @@ the current NeoAgent source.
28
28
  - You want structured memory and integration ingestion without a separate
29
29
  memory service.
30
30
  - You need multiple users or specialist agents with separate state.
31
- - Launcher mode, Health Connect, recordings, or the bundled device interfaces
31
+ - Launcher mode, Health Connect, or the bundled device interfaces
32
32
  are part of the use case.
33
33
 
34
34
  ## Consider OpenClaw when
@@ -14,8 +14,3 @@ A few resources to get you started if this is your first Flutter project:
14
14
  For help getting started with Flutter development, view the
15
15
  [online documentation](https://docs.flutter.dev/), which offers tutorials,
16
16
  samples, guidance on mobile development, and a full API reference.
17
-
18
- ## Desktop audio capture note
19
-
20
- Desktop recording currently depends on `desktop_audio_capture` and is pinned to
21
- `0.0.2` to avoid unexpected API shifts from automatic minor updates.
@@ -4,10 +4,8 @@
4
4
  <uses-permission android:name="android.permission.MANAGE_OWN_CALLS" />
5
5
  <uses-permission android:name="android.permission.CAMERA" />
6
6
  <uses-permission android:name="android.permission.RECORD_AUDIO" />
7
- <uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
8
7
  <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
9
8
  <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
10
- <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
11
9
  <uses-permission android:name="android.permission.health.READ_STEPS" />
12
10
  <uses-permission android:name="android.permission.health.READ_HEART_RATE" />
13
11
  <uses-permission android:name="android.permission.health.READ_SLEEP" />
@@ -134,18 +132,6 @@
134
132
  android:name="androidx.car.app.minCarApiLevel"
135
133
  android:value="1" />
136
134
  </service>
137
- <service
138
- android:name=".recording.RecordingForegroundService"
139
- android:exported="false"
140
- android:foregroundServiceType="microphone|connectedDevice" />
141
- <receiver
142
- android:name=".recording.RecordingBootReceiver"
143
- android:enabled="true"
144
- android:exported="false">
145
- <intent-filter>
146
- <action android:name="android.intent.action.BOOT_COMPLETED" />
147
- </intent-filter>
148
- </receiver>
149
135
  <receiver
150
136
  android:name=".widgets.AiHomeWidgetProvider"
151
137
  android:exported="true">
@@ -11,15 +11,11 @@ import android.provider.Settings
11
11
  import android.provider.OpenableColumns
12
12
  import android.view.KeyEvent
13
13
  import androidx.activity.result.ActivityResultLauncher
14
- import androidx.activity.result.contract.ActivityResultContracts
15
- import androidx.core.content.ContextCompat
16
14
  import androidx.core.content.FileProvider
17
15
  import androidx.health.connect.client.PermissionController
18
16
  import androidx.lifecycle.lifecycleScope
19
17
  import com.neoagent.flutter_app.health.HealthConnectGateway
20
18
  import com.neoagent.flutter_app.health.HealthSyncScheduler
21
- import com.neoagent.flutter_app.recording.RecordingForegroundService
22
- import com.neoagent.flutter_app.recording.RecordingStateStore
23
19
  import com.neoagent.flutter_app.widgets.AiHomeWidgetProvider
24
20
  import com.neoagent.flutter_app.widgets.AiWidgetStore
25
21
  import com.neoagent.flutter_app.widgets.VoiceLaunchWidgetProvider
@@ -46,12 +42,8 @@ class MainActivity : FlutterFragmentActivity() {
46
42
  private lateinit var healthGateway: HealthConnectGateway
47
43
  private lateinit var healthSyncScheduler: HealthSyncScheduler
48
44
  private lateinit var widgetSyncScheduler: WidgetSyncScheduler
49
- private lateinit var recordingStateStore: RecordingStateStore
50
45
  private lateinit var permissionLauncher: ActivityResultLauncher<Set<String>>
51
- private lateinit var microphonePermissionLauncher: ActivityResultLauncher<String>
52
46
  private var pendingPermissionResult: MethodChannel.Result? = null
53
- private var pendingRecordingResult: MethodChannel.Result? = null
54
- private var pendingRecordingArgs: Map<*, *>? = null
55
47
  private var launcherButtonSink: EventChannel.EventSink? = null
56
48
  private var widgetEventSink: EventChannel.EventSink? = null
57
49
  private var appLaunchEventSink: EventChannel.EventSink? = null
@@ -76,7 +68,6 @@ class MainActivity : FlutterFragmentActivity() {
76
68
  .build()
77
69
  telecomManager.registerPhoneAccount(phoneAccount)
78
70
  }
79
- recordingStateStore = RecordingStateStore(this)
80
71
  permissionLauncher = registerForActivityResult(
81
72
  PermissionController.createRequestPermissionResultContract(),
82
73
  ) {
@@ -86,32 +77,6 @@ class MainActivity : FlutterFragmentActivity() {
86
77
  pending?.success(buildStatusMap())
87
78
  }
88
79
  }
89
- microphonePermissionLauncher = registerForActivityResult(
90
- ActivityResultContracts.RequestPermission(),
91
- ) { granted ->
92
- val pending = pendingRecordingResult
93
- val args = pendingRecordingArgs
94
- pendingRecordingResult = null
95
- pendingRecordingArgs = null
96
- if (!granted) {
97
- pending?.error(
98
- "recording_permission_denied",
99
- "Microphone permission is required for background recording.",
100
- null,
101
- )
102
- return@registerForActivityResult
103
- }
104
- try {
105
- startRecordingService(args)
106
- pending?.success(recordingStateStore.statusMap())
107
- } catch (err: Exception) {
108
- pending?.error(
109
- "recording_start_failed",
110
- err.message ?: err.javaClass.simpleName,
111
- null,
112
- )
113
- }
114
- }
115
80
  MethodChannel(
116
81
  flutterEngine.dartExecutor.binaryMessenger,
117
82
  "neoagent/telecom",
@@ -258,56 +223,6 @@ class MainActivity : FlutterFragmentActivity() {
258
223
  }
259
224
  }
260
225
 
261
- MethodChannel(
262
- flutterEngine.dartExecutor.binaryMessenger,
263
- "neoagent/recordings",
264
- ).setMethodCallHandler { call, result ->
265
- when (call.method) {
266
- "status" -> result.success(recordingStateStore.statusMap())
267
-
268
- "startBackgroundRecording" -> {
269
- try {
270
- val args = call.arguments as? Map<*, *>
271
- if (ContextCompat.checkSelfPermission(
272
- this,
273
- android.Manifest.permission.RECORD_AUDIO,
274
- ) == android.content.pm.PackageManager.PERMISSION_GRANTED
275
- ) {
276
- startRecordingService(args)
277
- result.success(recordingStateStore.statusMap())
278
- } else {
279
- pendingRecordingResult = result
280
- pendingRecordingArgs = args
281
- microphonePermissionLauncher.launch(android.Manifest.permission.RECORD_AUDIO)
282
- }
283
- } catch (err: Exception) {
284
- result.error(
285
- "recording_start_failed",
286
- err.message ?: err.javaClass.simpleName,
287
- null,
288
- )
289
- }
290
- }
291
-
292
- "pauseBackgroundRecording" -> {
293
- startService(RecordingForegroundService.buildPauseIntent(this))
294
- result.success(recordingStateStore.statusMap())
295
- }
296
-
297
- "resumeBackgroundRecording" -> {
298
- startService(RecordingForegroundService.buildResumeIntent(this))
299
- result.success(recordingStateStore.statusMap())
300
- }
301
-
302
- "stopBackgroundRecording" -> {
303
- startService(RecordingForegroundService.buildStopIntent(this))
304
- result.success(recordingStateStore.statusMap())
305
- }
306
-
307
- else -> result.notImplemented()
308
- }
309
- }
310
-
311
226
  MethodChannel(
312
227
  flutterEngine.dartExecutor.binaryMessenger,
313
228
  "neoagent/app_update",
@@ -613,19 +528,6 @@ class MainActivity : FlutterFragmentActivity() {
613
528
  return super.onKeyUp(keyCode, event)
614
529
  }
615
530
 
616
- private fun startRecordingService(args: Map<*, *>?) {
617
- val backendUrl = args?.get("backendUrl")?.toString().orEmpty()
618
- val sessionCookie = args?.get("sessionCookie")?.toString().orEmpty()
619
- val sessionId = args?.get("sessionId")?.toString().orEmpty()
620
- val intent = RecordingForegroundService.buildStartIntent(
621
- this,
622
- backendUrl = backendUrl,
623
- sessionCookie = sessionCookie,
624
- sessionId = sessionId,
625
- )
626
- ContextCompat.startForegroundService(this, intent)
627
- }
628
-
629
531
  private fun currentAppMode(): String {
630
532
  return try {
631
533
  val appInfo = packageManager.getApplicationInfo(
@@ -17,7 +17,6 @@ import 'package:flutter_svg/flutter_svg.dart';
17
17
  import 'package:file_picker/file_picker.dart';
18
18
  import 'package:flutter_secure_storage/flutter_secure_storage.dart';
19
19
  import 'package:google_fonts/google_fonts.dart';
20
- import 'package:image/image.dart' as img;
21
20
  import 'package:mobile_scanner/mobile_scanner.dart';
22
21
  import 'package:qr_flutter/qr_flutter.dart';
23
22
  import 'package:shared_preferences/shared_preferences.dart';
@@ -39,8 +38,6 @@ import 'src/health_bridge.dart';
39
38
  import 'src/live_voice_capture.dart';
40
39
  import 'src/messaging_access_summary.dart';
41
40
  import 'src/oauth_launcher.dart';
42
- import 'src/recording_bridge.dart';
43
- import 'src/recording_payloads.dart';
44
41
  import 'src/security/password_strength.dart';
45
42
  import 'src/stream_renderer.dart';
46
43
  import 'src/theme/palette.dart';
@@ -66,7 +63,6 @@ part 'main_timeline.dart';
66
63
  part 'main_runtime.dart';
67
64
  part 'main_controller.dart';
68
65
  part 'main_devices.dart';
69
- part 'main_recordings.dart';
70
66
  part 'main_chat.dart';
71
67
  part 'main_account_settings.dart';
72
68
  part 'main_settings.dart';
@@ -104,8 +100,6 @@ const String _neoMailLogoAsset = 'assets/branding/neomail_logo.svg';
104
100
  const String _sessionCookiePrefsKey = 'auth.sessionCookie';
105
101
  const String _sessionCookieBackendPrefsKey = 'auth.sessionCookieBackend';
106
102
  const String _sessionCookieSecureStorageKey = 'auth.sessionCookie.secure';
107
- const int _voiceAssistantScreenshotMaxDimension = 1600;
108
- const int _voiceAssistantScreenshotMaxBytes = 900 * 1024;
109
103
 
110
104
  String get _desktopTrayIconAsset =>
111
105
  defaultTargetPlatform == TargetPlatform.macOS
@@ -2521,8 +2521,6 @@ class _SectionBody extends StatelessWidget {
2521
2521
  return VoiceAssistantPanel(controller: controller);
2522
2522
  case AppSection.devices:
2523
2523
  return DevicesPanel(key: devicesPanelKey, controller: controller);
2524
- case AppSection.recordings:
2525
- return RecordingsPanel(controller: controller);
2526
2524
  case AppSection.messaging:
2527
2525
  return MessagingPanel(controller: controller);
2528
2526
  case AppSection.runs: