neoagent 3.1.1-beta.9 → 3.2.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -2
- package/README.md +5 -1
- package/docs/architecture.md +1 -1
- package/docs/benchmarking.md +68 -87
- package/docs/capabilities.md +1 -1
- package/docs/clients-and-devices.md +3 -4
- package/docs/configuration.md +1 -1
- package/docs/devices.md +1 -2
- package/docs/health.md +24 -0
- package/docs/index.md +2 -2
- package/docs/integrations-architecture.md +6 -0
- package/docs/integrations.md +15 -1
- package/docs/models.md +1 -1
- package/docs/persistence.md +1 -1
- package/docs/why-neoagent.md +1 -1
- package/flutter_app/README.md +0 -5
- package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
- package/flutter_app/lib/main.dart +0 -6
- package/flutter_app/lib/main_app_shell.dart +0 -2
- package/flutter_app/lib/main_controller.dart +2 -867
- package/flutter_app/lib/main_devices.dart +0 -73
- package/flutter_app/lib/main_integrations.dart +339 -187
- package/flutter_app/lib/main_launcher.dart +2 -212
- package/flutter_app/lib/main_models.dart +0 -259
- package/flutter_app/lib/main_navigation.dart +2 -14
- package/flutter_app/lib/main_operations.dart +875 -383
- package/flutter_app/lib/main_runtime.dart +9 -208
- package/flutter_app/lib/main_settings.dart +17 -199
- package/flutter_app/lib/main_shared.dart +20 -591
- package/flutter_app/lib/main_timeline.dart +26 -122
- package/flutter_app/lib/main_voice_assistant.dart +0 -5
- package/flutter_app/lib/src/backend_client.dart +0 -102
- package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
- package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
- package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
- package/flutter_app/macos/Podfile.lock +56 -5
- package/flutter_app/pubspec.lock +0 -7
- package/flutter_app/pubspec.yaml +0 -2
- package/flutter_app/web/index.html +174 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
- package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
- package/landing/index.html +0 -2
- package/lib/manager.js +1 -1
- package/lib/schema_migrations.js +75 -0
- package/package.json +1 -4
- package/server/db/database.js +1 -133
- package/server/http/middleware.js +2 -16
- package/server/http/routes.js +0 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +0 -25
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/index.html +174 -0
- package/server/public/main.dart.js +92036 -91145
- package/server/routes/auth.js +11 -2
- package/server/routes/settings.js +0 -5
- package/server/routes/voice_assistant.js +1 -128
- package/server/services/account/erasure.js +0 -3
- package/server/services/ai/deliverables/artifact_helpers.js +1 -1
- package/server/services/ai/loop/conversation_loop.js +6 -2
- package/server/services/ai/loop/tool_dispatch.js +0 -3
- package/server/services/ai/messagingFallback.js +1 -1
- package/server/services/ai/settings.js +0 -6
- package/server/services/ai/toolEvidence.js +1 -4
- package/server/services/ai/tools.js +0 -234
- package/server/services/desktop/auth.js +0 -3
- package/server/services/desktop/registry.js +2 -50
- package/server/services/integrations/neoarchive/provider.js +616 -0
- package/server/services/integrations/neorecall/client.js +140 -0
- package/server/services/integrations/neorecall/constants.js +87 -0
- package/server/services/integrations/neorecall/provider.js +163 -0
- package/server/services/integrations/registry.js +4 -2
- package/server/services/manager.js +0 -22
- package/server/services/memory/manager.js +44 -19
- package/server/services/memory/retrieval_reasoning.js +36 -14
- package/server/services/messaging/formatting_guides.js +1 -0
- package/server/services/messaging/whatsapp.js +18 -0
- package/server/services/security/tool_categories.js +0 -5
- package/server/services/social_video/service.js +2 -20
- package/server/services/tasks/adapters/index.js +1 -1
- package/server/services/tasks/integration_runtime.js +3 -67
- package/server/services/tasks/runtime.js +1 -6
- package/server/services/tasks/task_repository.js +6 -13
- package/server/services/timeline/service.js +0 -224
- package/server/services/voice/message.js +0 -24
- package/server/services/voice/providers.js +42 -1
- package/server/services/voice/runtimeManager.js +9 -5
- package/server/services/voice/turnRunner.js +0 -22
- package/server/services/wearable/service.js +0 -5
- package/docs/recordings-and-health.md +0 -41
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
- package/flutter_app/lib/main_recordings.dart +0 -920
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
- package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
- package/flutter_app/lib/src/recording_bridge.dart +0 -232
- package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
- package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
- package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
- package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
- package/flutter_app/lib/src/recording_payloads.dart +0 -95
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
- package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
- package/server/routes/recordings.js +0 -335
- package/server/routes/screenHistory.js +0 -190
- package/server/services/ai/recordingInsights.js +0 -105
- package/server/services/integrations/neomail/provider.js +0 -993
- package/server/services/recordings/deepgram.js +0 -58
- package/server/services/recordings/manager.js +0 -1273
- package/server/services/tasks/adapters/neomail_email_received.js +0 -46
- package/server/services/voice/screenshotContext.js +0 -73
- package/server/services/voice/turnCoordinator.js +0 -51
package/.env.example
CHANGED
|
@@ -236,8 +236,8 @@ TRELLO_API_KEY=your-trello-api-key
|
|
|
236
236
|
BRAVE_SEARCH_API_KEY=your-brave-search-api-key-here
|
|
237
237
|
|
|
238
238
|
# Deepgram API key — used for:
|
|
239
|
-
# •
|
|
240
|
-
# Without this key:
|
|
239
|
+
# • Voice transcription (Deepgram Nova-3 multilingual)
|
|
240
|
+
# Without this key: Deepgram-backed voice transcription is unavailable.
|
|
241
241
|
# Get your key at: https://console.deepgram.com/
|
|
242
242
|
DEEPGRAM_API_KEY=your-deepgram-api-key-here
|
|
243
243
|
DEEPGRAM_BASE_URL=https://api.deepgram.com
|
package/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="
|
|
2
|
+
<img src="marketing/final/github-banner.png" alt="NeoAgent — an AI assistant that keeps working" width="100%">
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<h1 align="center">NeoAgent</h1>
|
|
@@ -28,6 +28,10 @@
|
|
|
28
28
|
<a href="https://www.producthunt.com/products/neoagent-2?embed=true&utm_source=badge-featured&utm_medium=badge&utm_campaign=badge-neoagent-2" target="_blank" rel="noopener noreferrer"><img alt="NeoAgent - The next-gen self-hosted AI agent that works beyond chat | Product Hunt" width="250" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=1171573&theme=light&t=1781447653039"></a>
|
|
29
29
|
</p>
|
|
30
30
|
|
|
31
|
+
<p align="center">
|
|
32
|
+
<img src="marketing/final/trailer.gif" alt="NeoAgent trailer" width="100%">
|
|
33
|
+
</p>
|
|
34
|
+
|
|
31
35
|
<p align="center">
|
|
32
36
|
<img src="demo.gif" alt="NeoAgent demo" width="100%">
|
|
33
37
|
</p>
|
package/docs/architecture.md
CHANGED
|
@@ -36,7 +36,7 @@ bridge files and native Android code where required.
|
|
|
36
36
|
|
|
37
37
|
NeoAgent uses one `better-sqlite3` database instance. It stores users, agents,
|
|
38
38
|
settings, conversations, runs, tasks, integration connections, memory,
|
|
39
|
-
|
|
39
|
+
health data, permissions, and operational state.
|
|
40
40
|
|
|
41
41
|
Runtime files live outside the package source under `NEOAGENT_HOME`. Schema
|
|
42
42
|
changes are applied through the migration layer rather than ad hoc service SQL.
|
package/docs/benchmarking.md
CHANGED
|
@@ -1,102 +1,83 @@
|
|
|
1
1
|
# Benchmarking
|
|
2
2
|
|
|
3
|
-
NeoAgent
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
NeoAgent's benchmark harness measures the actual quality of its memory system on
|
|
4
|
+
**LoCoMo** ([Snap Research](https://github.com/snap-research/locomo), CC BY-NC 4.0), the
|
|
5
|
+
standard long-term conversational memory benchmark. It follows the same protocol
|
|
6
|
+
mem0/Zep/Letta/[Omi](https://github.com/BasedHardware/omi-memory-benchmarks) use to publish
|
|
7
|
+
their numbers, so results are comparable: a thin QA head over the memory system's real
|
|
8
|
+
retrieval API, not the consumer chat agent.
|
|
7
9
|
|
|
8
10
|
## Latest dashboard
|
|
9
11
|
|
|
10
12
|

|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
Backed by these local outputs:
|
|
13
15
|
|
|
14
|
-
- `benchmark/results/latest-results.json`
|
|
15
|
-
|
|
16
|
-
- `benchmark/results/latest-summary.md`
|
|
16
|
+
- `benchmark/results/latest-results.json` — every question, gold answer, generated
|
|
17
|
+
answer, and CORRECT/WRONG verdict.
|
|
18
|
+
- `benchmark/results/latest-summary.json` / `latest-summary.md` — overall and
|
|
19
|
+
per-category accuracy.
|
|
17
20
|
- `static/benchmarks/latest-dashboard.png`
|
|
18
21
|
|
|
19
22
|
## Commands
|
|
20
23
|
|
|
21
24
|
```bash
|
|
22
|
-
npm run benchmark:setup
|
|
23
|
-
npm run benchmark:run
|
|
24
|
-
npm run benchmark:report
|
|
25
|
+
npm run benchmark:setup # downloads and validates the LoCoMo dataset
|
|
26
|
+
npm run benchmark:run # ingests, retrieves, answers, judges, scores
|
|
27
|
+
npm run benchmark:report # re-renders the dashboard from the last run's results.json
|
|
25
28
|
```
|
|
26
29
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
##
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
-
|
|
71
|
-
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
- The
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
### VisualWebArena
|
|
83
|
-
|
|
84
|
-
- Docker installed and running.
|
|
85
|
-
- The official VisualWebArena environment.
|
|
86
|
-
- A vision-capable configured benchmark model.
|
|
87
|
-
- An exact runner command exposed through
|
|
88
|
-
`NEOAGENT_BENCHMARK_VISUAL_WEBARENA_RUNNER`.
|
|
89
|
-
|
|
90
|
-
### SWE-bench
|
|
91
|
-
|
|
92
|
-
- Docker installed and running.
|
|
93
|
-
- A checkout of [SWE-bench](https://github.com/SWE-bench/SWE-bench), which
|
|
94
|
-
`benchmark:setup` can clone into `benchmark/workdir/swebench/`.
|
|
95
|
-
- The official dataset/runtime and enough disk/network budget for repo/image
|
|
96
|
-
setup.
|
|
97
|
-
- An exact runner command exposed through
|
|
98
|
-
`NEOAGENT_BENCHMARK_SWEBENCH_RUNNER`.
|
|
99
|
-
|
|
100
|
-
On this machine, `docker` is currently missing, so exact `WebArena`,
|
|
101
|
-
`VisualWebArena`, and `SWE-bench` execution remain blocked until Docker and the
|
|
102
|
-
official runners are installed.
|
|
30
|
+
`benchmark:run` requires:
|
|
31
|
+
- A running NeoAgent server (`NEOAGENT_BENCHMARK_BASE_URL`, default `http://127.0.0.1:3333`).
|
|
32
|
+
- `OPENROUTER_API_KEY` in the environment — it configures each benchmark account's own
|
|
33
|
+
memory-extraction provider, and drives the answerer/judge models.
|
|
34
|
+
|
|
35
|
+
## How it works
|
|
36
|
+
|
|
37
|
+
1. **Ingestion.** For each LoCoMo conversation, the harness registers an isolated
|
|
38
|
+
benchmark account and pushes every session through NeoAgent's real document-ingestion
|
|
39
|
+
pipeline (`POST /api/memory/ingestion/documents`) — one document per session, dated and
|
|
40
|
+
speaker-labeled, chunked exactly like any other real ingested source.
|
|
41
|
+
2. **Extraction draining.** NeoAgent promotes raw ingested chunks into structured
|
|
42
|
+
memories/facts as a side effect of the agent loop's memory-recall step, not the raw
|
|
43
|
+
ingestion call itself. The harness issues a handful of trivial agent runs per account to
|
|
44
|
+
settle this before measuring retrieval — using the app's real settling mechanism rather
|
|
45
|
+
than reaching into internals.
|
|
46
|
+
3. **Retrieval.** `POST /api/memory/memories/recall` — NeoAgent's real fused
|
|
47
|
+
lexical+entity+vector retrieval surface.
|
|
48
|
+
4. **Answering.** One OpenRouter chat completion per question, using mem0's verbatim
|
|
49
|
+
`ANSWER_PROMPT` (≤5-6-word answers, relative→absolute date conversion).
|
|
50
|
+
5. **Judging.** One OpenRouter chat completion using mem0's verbatim `ACCURACY_PROMPT`
|
|
51
|
+
(lenient CORRECT/WRONG) — the same judge all of the above vendors report against.
|
|
52
|
+
LoCoMo category 5 (adversarial) is excluded, exactly as mem0's own eval code does.
|
|
53
|
+
|
|
54
|
+
Prompts live verbatim in [`benchmark/locomo/prompts.js`](../benchmark/locomo/prompts.js).
|
|
55
|
+
|
|
56
|
+
## Configuration
|
|
57
|
+
|
|
58
|
+
| Env var | Default | Purpose |
|
|
59
|
+
| --- | --- | --- |
|
|
60
|
+
| `NEOAGENT_BENCHMARK_BASE_URL` | `http://127.0.0.1:3333` | Server to benchmark |
|
|
61
|
+
| `OPENROUTER_API_KEY` | — (required) | Answerer/judge models + per-account memory extraction |
|
|
62
|
+
| `BENCHMARK_ANSWER_MODEL` / `BENCHMARK_JUDGE_MODEL` | cheapest priced OpenRouter model | Pin exact model IDs |
|
|
63
|
+
| `BENCHMARK_LOCOMO_SAMPLES` / `BENCHMARK_LOCOMO_OFFSET` | `10` / `0` | Which conversations to run |
|
|
64
|
+
| `BENCHMARK_LOCOMO_QA_PER_SAMPLE` | `0` (all) | Cap questions per conversation |
|
|
65
|
+
| `BENCHMARK_LOCOMO_MEM_K` | `20` | Recall depth per question |
|
|
66
|
+
| `BENCHMARK_CONCURRENCY` | `1` | Parallel QA-head requests |
|
|
67
|
+
| `BENCHMARK_CASE_LIMIT` | `0` (no cap) | Global question cap, useful for smoke tests |
|
|
68
|
+
|
|
69
|
+
For a cheap smoke test: `BENCHMARK_LOCOMO_SAMPLES=1 BENCHMARK_LOCOMO_QA_PER_SAMPLE=5 npm run benchmark:run`.
|
|
70
|
+
|
|
71
|
+
## Honest caveats
|
|
72
|
+
|
|
73
|
+
- **Single run, temperature 0.** No averaging across repeats.
|
|
74
|
+
- **Category 5 (adversarial) excluded**, matching the standard mem0 LoCoMo protocol.
|
|
75
|
+
- **Ingestion granularity.** LoCoMo sessions are ingested as NeoAgent's native document
|
|
76
|
+
unit (one per session); this differs from message-pair-level ingestion some vendors use,
|
|
77
|
+
which can make retrieval comparatively easier or harder depending on the question.
|
|
78
|
+
- **Judge leniency** varies significantly across published papers; this harness uses mem0's
|
|
79
|
+
lenient judge verbatim so numbers line up with mem0/Zep/Letta/Omi's published columns.
|
|
80
|
+
- The LoCoMo dataset (CC BY-NC 4.0) is downloaded on demand into `benchmark/workdir/`
|
|
81
|
+
(gitignored) — never vendored into the repository. Result files may quote benchmark
|
|
82
|
+
questions/answers for verifiability; no NeoAgent user data is involved (every run uses
|
|
83
|
+
synthetic benchmark accounts).
|
package/docs/capabilities.md
CHANGED
|
@@ -14,7 +14,7 @@ capabilities by workflow instead of maintaining one long inventory.
|
|
|
14
14
|
|
|
15
15
|
- [Integrations and messaging](integrations.md)
|
|
16
16
|
- [Devices and interfaces](devices.md)
|
|
17
|
-
- [
|
|
17
|
+
- [Health](health.md)
|
|
18
18
|
|
|
19
19
|
## Run unattended work
|
|
20
20
|
|
|
@@ -21,10 +21,9 @@ updates into `MainController`, which notifies the relevant UI.
|
|
|
21
21
|
|
|
22
22
|
## Android bridges
|
|
23
23
|
|
|
24
|
-
Native Kotlin code handles Health Connect,
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
updating application state.
|
|
24
|
+
Native Kotlin code handles Health Connect, notifications, launcher functions,
|
|
25
|
+
telecom integration, and Android home widgets. Flutter bridges normalize
|
|
26
|
+
permission and lifecycle behavior before updating application state.
|
|
28
27
|
|
|
29
28
|
## Desktop and Chrome pairing
|
|
30
29
|
|
package/docs/configuration.md
CHANGED
|
@@ -49,7 +49,7 @@ neoagent env set SESSION_SECRET "$(openssl rand -hex 32)"
|
|
|
49
49
|
| `ANTHROPIC_BASE_URL` | Anthropic-compatible base URL override |
|
|
50
50
|
| `OLLAMA_URL` | Ollama server URL |
|
|
51
51
|
| `BRAVE_SEARCH_API_KEY` | Web search |
|
|
52
|
-
| `DEEPGRAM_API_KEY` |
|
|
52
|
+
| `DEEPGRAM_API_KEY` | Voice transcription |
|
|
53
53
|
| `DEEPGRAM_BASE_URL` | Deepgram-compatible base URL override |
|
|
54
54
|
| `DEEPGRAM_MODEL` | Speech model |
|
|
55
55
|
| `DEEPGRAM_LANGUAGE` | Speech language mode |
|
package/docs/devices.md
CHANGED
|
@@ -7,7 +7,7 @@ explicitly paired as a runtime backend.
|
|
|
7
7
|
## Web interface
|
|
8
8
|
|
|
9
9
|
The web interface provides chat, runs, tasks, agents, integrations, memory,
|
|
10
|
-
devices,
|
|
10
|
+
devices, health, settings, permissions, and logs. It is served by
|
|
11
11
|
the NeoAgent backend.
|
|
12
12
|
|
|
13
13
|
## Android app
|
|
@@ -15,7 +15,6 @@ the NeoAgent backend.
|
|
|
15
15
|
The Android app connects to the same backend and adds:
|
|
16
16
|
|
|
17
17
|
- Health Connect synchronization
|
|
18
|
-
- Background microphone recording
|
|
19
18
|
- Android notification forwarding
|
|
20
19
|
- Native notifications and device pairing
|
|
21
20
|
|
package/docs/health.md
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Health
|
|
2
|
+
|
|
3
|
+
Health data is an optional feature of the NeoAgent clients. It stores
|
|
4
|
+
processed data on the NeoAgent backend.
|
|
5
|
+
|
|
6
|
+
## Health Connect
|
|
7
|
+
|
|
8
|
+
The Android app can read Health Connect records after the user grants Android
|
|
9
|
+
permissions. Supported data includes steps, heart rate, sleep, exercise, and
|
|
10
|
+
weight.
|
|
11
|
+
|
|
12
|
+
The app uploads granted records to the user's NeoAgent backend. Stored metrics
|
|
13
|
+
appear in **Health** and are available to the owning agent through the health
|
|
14
|
+
tool.
|
|
15
|
+
|
|
16
|
+
Health Connect access can be revoked in Android settings. Revoking access stops
|
|
17
|
+
future reads but does not remove records already uploaded to NeoAgent; delete
|
|
18
|
+
stored server data separately when required.
|
|
19
|
+
|
|
20
|
+
## Privacy
|
|
21
|
+
|
|
22
|
+
Health records are sensitive personal data. Protect the server account, use
|
|
23
|
+
HTTPS for remote clients, restrict agents and integrations, and include this
|
|
24
|
+
data store in backup and deletion procedures.
|
package/docs/index.md
CHANGED
|
@@ -8,7 +8,7 @@ sidebar_label: Overview
|
|
|
8
8
|
|
|
9
9
|
NeoAgent is a self-hosted AI agent that runs continuously on a macOS or Linux
|
|
10
10
|
server. It provides an operator interface for chat, automation, integrations,
|
|
11
|
-
memory,
|
|
11
|
+
memory, and connected devices.
|
|
12
12
|
|
|
13
13
|
NeoAgent is beta software. Install it on a machine you administer, start with
|
|
14
14
|
restricted tool permissions, and read the security guide before connecting
|
|
@@ -42,7 +42,7 @@ Continue with:
|
|
|
42
42
|
| [Memory](memory.md) | What NeoAgent remembers and how to manage it |
|
|
43
43
|
| [Integrations](integrations.md) | Connected app accounts and messaging channels |
|
|
44
44
|
| [Devices and interfaces](devices.md) | Android, desktop, Chrome, launcher, and wearable surfaces |
|
|
45
|
-
| [
|
|
45
|
+
| [Health](health.md) | Health Connect |
|
|
46
46
|
| [Skills and MCP](skills.md) | Reusable instructions and external tool servers |
|
|
47
47
|
| [Operations](operations.md) | Updates, backups, logs, and recovery |
|
|
48
48
|
| [Billing](billing.md) | Stripe subscriptions, plans, trials, and webhooks |
|
|
@@ -35,6 +35,12 @@ chunks.
|
|
|
35
35
|
Provider snapshots shown in the client are decorated with ingestion coverage
|
|
36
36
|
and status.
|
|
37
37
|
|
|
38
|
+
NeoRecall is intentionally queried on demand instead of being copied into
|
|
39
|
+
NeoAgent memory. Its integration uses a dynamic public OAuth client with PKCE,
|
|
40
|
+
rotating refresh tokens, and read-only scopes. The available tools call
|
|
41
|
+
NeoRecall's local hybrid search and evidence APIs; they cannot access audio or
|
|
42
|
+
trigger NeoRecall's Ask endpoint.
|
|
43
|
+
|
|
38
44
|
## Messaging
|
|
39
45
|
|
|
40
46
|
Messaging providers are long-running channel adapters managed separately from
|
package/docs/integrations.md
CHANGED
|
@@ -19,6 +19,8 @@ Connect accounts in **Integrations**.
|
|
|
19
19
|
| Notion | Pages, databases, blocks, search |
|
|
20
20
|
| Slack | Conversations, history, search, messages |
|
|
21
21
|
| Figma | Files, nodes, rendered images, comments |
|
|
22
|
+
| NeoArchive | Documents, extracted text, metadata, search, upload, archive, reprocessing |
|
|
23
|
+
| NeoRecall | Local hybrid recall search, memories, mini-memories, daily summaries, conversations, transcript evidence |
|
|
22
24
|
| Home Assistant | Entity state and service calls |
|
|
23
25
|
| Trello | Boards, lists, cards, comments, search |
|
|
24
26
|
| Spotify | Playback, history, search, queue |
|
|
@@ -48,7 +50,19 @@ Google, Microsoft, GitHub, Notion, Slack, Figma, and Spotify require OAuth appli
|
|
|
48
50
|
- `GITHUB_CLIENT_ID=your_id_here`
|
|
49
51
|
- `GITHUB_CLIENT_SECRET=your_secret_here`
|
|
50
52
|
|
|
51
|
-
Home Assistant and Trello can be configured for a user from the application. See Configuration.
|
|
53
|
+
NeoArchive, NeoRecall, Home Assistant, and Trello can be configured for a user from the application. See Configuration.
|
|
54
|
+
|
|
55
|
+
### NeoRecall
|
|
56
|
+
|
|
57
|
+
Open **Integrations**, choose **NeoRecall**, and enter the base URL of the
|
|
58
|
+
NeoRecall server. NeoAgent validates the server and opens NeoRecall's own
|
|
59
|
+
authorization page. Sign in there and approve the three read-only scopes.
|
|
60
|
+
|
|
61
|
+
NeoRecall contributes seven on-demand tools for local hybrid search and for
|
|
62
|
+
reading memories, mini-memories, daily summaries, conversations, and transcript
|
|
63
|
+
evidence. It does not expose audio, recording, memory mutation, consolidation,
|
|
64
|
+
or NeoRecall Ask. This keeps the products isolated and avoids a second LLM call
|
|
65
|
+
when NeoAgent only needs retrieval context.
|
|
52
66
|
|
|
53
67
|
## Messaging channels
|
|
54
68
|
|
package/docs/models.md
CHANGED
|
@@ -44,7 +44,7 @@ Some features use separate providers:
|
|
|
44
44
|
|
|
45
45
|
- Embeddings for memory search use a configured supported embedding provider
|
|
46
46
|
and fall back to lexical retrieval when embeddings are unavailable.
|
|
47
|
-
-
|
|
47
|
+
- Voice transcription can use Deepgram when enabled.
|
|
48
48
|
- Image generation and analysis depend on the selected provider and model.
|
|
49
49
|
|
|
50
50
|
## Credential handling
|
package/docs/persistence.md
CHANGED
|
@@ -13,7 +13,7 @@ The database owns:
|
|
|
13
13
|
- Tasks, triggers, webhook deliveries, and widgets
|
|
14
14
|
- Integration connections and encrypted credentials
|
|
15
15
|
- Memory, facts, entities, source documents, and retrieval telemetry
|
|
16
|
-
-
|
|
16
|
+
- Health records and device registrations
|
|
17
17
|
- Tool policies and approval history
|
|
18
18
|
|
|
19
19
|
Larger artifacts and editable agent data use runtime directories under
|
package/docs/why-neoagent.md
CHANGED
|
@@ -28,7 +28,7 @@ the current NeoAgent source.
|
|
|
28
28
|
- You want structured memory and integration ingestion without a separate
|
|
29
29
|
memory service.
|
|
30
30
|
- You need multiple users or specialist agents with separate state.
|
|
31
|
-
- Launcher mode, Health Connect,
|
|
31
|
+
- Launcher mode, Health Connect, or the bundled device interfaces
|
|
32
32
|
are part of the use case.
|
|
33
33
|
|
|
34
34
|
## Consider OpenClaw when
|
package/flutter_app/README.md
CHANGED
|
@@ -14,8 +14,3 @@ A few resources to get you started if this is your first Flutter project:
|
|
|
14
14
|
For help getting started with Flutter development, view the
|
|
15
15
|
[online documentation](https://docs.flutter.dev/), which offers tutorials,
|
|
16
16
|
samples, guidance on mobile development, and a full API reference.
|
|
17
|
-
|
|
18
|
-
## Desktop audio capture note
|
|
19
|
-
|
|
20
|
-
Desktop recording currently depends on `desktop_audio_capture` and is pinned to
|
|
21
|
-
`0.0.2` to avoid unexpected API shifts from automatic minor updates.
|
|
@@ -4,10 +4,8 @@
|
|
|
4
4
|
<uses-permission android:name="android.permission.MANAGE_OWN_CALLS" />
|
|
5
5
|
<uses-permission android:name="android.permission.CAMERA" />
|
|
6
6
|
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
|
7
|
-
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
|
|
8
7
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
|
9
8
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
|
|
10
|
-
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
|
11
9
|
<uses-permission android:name="android.permission.health.READ_STEPS" />
|
|
12
10
|
<uses-permission android:name="android.permission.health.READ_HEART_RATE" />
|
|
13
11
|
<uses-permission android:name="android.permission.health.READ_SLEEP" />
|
|
@@ -134,18 +132,6 @@
|
|
|
134
132
|
android:name="androidx.car.app.minCarApiLevel"
|
|
135
133
|
android:value="1" />
|
|
136
134
|
</service>
|
|
137
|
-
<service
|
|
138
|
-
android:name=".recording.RecordingForegroundService"
|
|
139
|
-
android:exported="false"
|
|
140
|
-
android:foregroundServiceType="microphone|connectedDevice" />
|
|
141
|
-
<receiver
|
|
142
|
-
android:name=".recording.RecordingBootReceiver"
|
|
143
|
-
android:enabled="true"
|
|
144
|
-
android:exported="false">
|
|
145
|
-
<intent-filter>
|
|
146
|
-
<action android:name="android.intent.action.BOOT_COMPLETED" />
|
|
147
|
-
</intent-filter>
|
|
148
|
-
</receiver>
|
|
149
135
|
<receiver
|
|
150
136
|
android:name=".widgets.AiHomeWidgetProvider"
|
|
151
137
|
android:exported="true">
|
|
@@ -11,15 +11,11 @@ import android.provider.Settings
|
|
|
11
11
|
import android.provider.OpenableColumns
|
|
12
12
|
import android.view.KeyEvent
|
|
13
13
|
import androidx.activity.result.ActivityResultLauncher
|
|
14
|
-
import androidx.activity.result.contract.ActivityResultContracts
|
|
15
|
-
import androidx.core.content.ContextCompat
|
|
16
14
|
import androidx.core.content.FileProvider
|
|
17
15
|
import androidx.health.connect.client.PermissionController
|
|
18
16
|
import androidx.lifecycle.lifecycleScope
|
|
19
17
|
import com.neoagent.flutter_app.health.HealthConnectGateway
|
|
20
18
|
import com.neoagent.flutter_app.health.HealthSyncScheduler
|
|
21
|
-
import com.neoagent.flutter_app.recording.RecordingForegroundService
|
|
22
|
-
import com.neoagent.flutter_app.recording.RecordingStateStore
|
|
23
19
|
import com.neoagent.flutter_app.widgets.AiHomeWidgetProvider
|
|
24
20
|
import com.neoagent.flutter_app.widgets.AiWidgetStore
|
|
25
21
|
import com.neoagent.flutter_app.widgets.VoiceLaunchWidgetProvider
|
|
@@ -46,12 +42,8 @@ class MainActivity : FlutterFragmentActivity() {
|
|
|
46
42
|
private lateinit var healthGateway: HealthConnectGateway
|
|
47
43
|
private lateinit var healthSyncScheduler: HealthSyncScheduler
|
|
48
44
|
private lateinit var widgetSyncScheduler: WidgetSyncScheduler
|
|
49
|
-
private lateinit var recordingStateStore: RecordingStateStore
|
|
50
45
|
private lateinit var permissionLauncher: ActivityResultLauncher<Set<String>>
|
|
51
|
-
private lateinit var microphonePermissionLauncher: ActivityResultLauncher<String>
|
|
52
46
|
private var pendingPermissionResult: MethodChannel.Result? = null
|
|
53
|
-
private var pendingRecordingResult: MethodChannel.Result? = null
|
|
54
|
-
private var pendingRecordingArgs: Map<*, *>? = null
|
|
55
47
|
private var launcherButtonSink: EventChannel.EventSink? = null
|
|
56
48
|
private var widgetEventSink: EventChannel.EventSink? = null
|
|
57
49
|
private var appLaunchEventSink: EventChannel.EventSink? = null
|
|
@@ -76,7 +68,6 @@ class MainActivity : FlutterFragmentActivity() {
|
|
|
76
68
|
.build()
|
|
77
69
|
telecomManager.registerPhoneAccount(phoneAccount)
|
|
78
70
|
}
|
|
79
|
-
recordingStateStore = RecordingStateStore(this)
|
|
80
71
|
permissionLauncher = registerForActivityResult(
|
|
81
72
|
PermissionController.createRequestPermissionResultContract(),
|
|
82
73
|
) {
|
|
@@ -86,32 +77,6 @@ class MainActivity : FlutterFragmentActivity() {
|
|
|
86
77
|
pending?.success(buildStatusMap())
|
|
87
78
|
}
|
|
88
79
|
}
|
|
89
|
-
microphonePermissionLauncher = registerForActivityResult(
|
|
90
|
-
ActivityResultContracts.RequestPermission(),
|
|
91
|
-
) { granted ->
|
|
92
|
-
val pending = pendingRecordingResult
|
|
93
|
-
val args = pendingRecordingArgs
|
|
94
|
-
pendingRecordingResult = null
|
|
95
|
-
pendingRecordingArgs = null
|
|
96
|
-
if (!granted) {
|
|
97
|
-
pending?.error(
|
|
98
|
-
"recording_permission_denied",
|
|
99
|
-
"Microphone permission is required for background recording.",
|
|
100
|
-
null,
|
|
101
|
-
)
|
|
102
|
-
return@registerForActivityResult
|
|
103
|
-
}
|
|
104
|
-
try {
|
|
105
|
-
startRecordingService(args)
|
|
106
|
-
pending?.success(recordingStateStore.statusMap())
|
|
107
|
-
} catch (err: Exception) {
|
|
108
|
-
pending?.error(
|
|
109
|
-
"recording_start_failed",
|
|
110
|
-
err.message ?: err.javaClass.simpleName,
|
|
111
|
-
null,
|
|
112
|
-
)
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
80
|
MethodChannel(
|
|
116
81
|
flutterEngine.dartExecutor.binaryMessenger,
|
|
117
82
|
"neoagent/telecom",
|
|
@@ -258,56 +223,6 @@ class MainActivity : FlutterFragmentActivity() {
|
|
|
258
223
|
}
|
|
259
224
|
}
|
|
260
225
|
|
|
261
|
-
MethodChannel(
|
|
262
|
-
flutterEngine.dartExecutor.binaryMessenger,
|
|
263
|
-
"neoagent/recordings",
|
|
264
|
-
).setMethodCallHandler { call, result ->
|
|
265
|
-
when (call.method) {
|
|
266
|
-
"status" -> result.success(recordingStateStore.statusMap())
|
|
267
|
-
|
|
268
|
-
"startBackgroundRecording" -> {
|
|
269
|
-
try {
|
|
270
|
-
val args = call.arguments as? Map<*, *>
|
|
271
|
-
if (ContextCompat.checkSelfPermission(
|
|
272
|
-
this,
|
|
273
|
-
android.Manifest.permission.RECORD_AUDIO,
|
|
274
|
-
) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
|
275
|
-
) {
|
|
276
|
-
startRecordingService(args)
|
|
277
|
-
result.success(recordingStateStore.statusMap())
|
|
278
|
-
} else {
|
|
279
|
-
pendingRecordingResult = result
|
|
280
|
-
pendingRecordingArgs = args
|
|
281
|
-
microphonePermissionLauncher.launch(android.Manifest.permission.RECORD_AUDIO)
|
|
282
|
-
}
|
|
283
|
-
} catch (err: Exception) {
|
|
284
|
-
result.error(
|
|
285
|
-
"recording_start_failed",
|
|
286
|
-
err.message ?: err.javaClass.simpleName,
|
|
287
|
-
null,
|
|
288
|
-
)
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
"pauseBackgroundRecording" -> {
|
|
293
|
-
startService(RecordingForegroundService.buildPauseIntent(this))
|
|
294
|
-
result.success(recordingStateStore.statusMap())
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
"resumeBackgroundRecording" -> {
|
|
298
|
-
startService(RecordingForegroundService.buildResumeIntent(this))
|
|
299
|
-
result.success(recordingStateStore.statusMap())
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
"stopBackgroundRecording" -> {
|
|
303
|
-
startService(RecordingForegroundService.buildStopIntent(this))
|
|
304
|
-
result.success(recordingStateStore.statusMap())
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
else -> result.notImplemented()
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
|
|
311
226
|
MethodChannel(
|
|
312
227
|
flutterEngine.dartExecutor.binaryMessenger,
|
|
313
228
|
"neoagent/app_update",
|
|
@@ -613,19 +528,6 @@ class MainActivity : FlutterFragmentActivity() {
|
|
|
613
528
|
return super.onKeyUp(keyCode, event)
|
|
614
529
|
}
|
|
615
530
|
|
|
616
|
-
private fun startRecordingService(args: Map<*, *>?) {
|
|
617
|
-
val backendUrl = args?.get("backendUrl")?.toString().orEmpty()
|
|
618
|
-
val sessionCookie = args?.get("sessionCookie")?.toString().orEmpty()
|
|
619
|
-
val sessionId = args?.get("sessionId")?.toString().orEmpty()
|
|
620
|
-
val intent = RecordingForegroundService.buildStartIntent(
|
|
621
|
-
this,
|
|
622
|
-
backendUrl = backendUrl,
|
|
623
|
-
sessionCookie = sessionCookie,
|
|
624
|
-
sessionId = sessionId,
|
|
625
|
-
)
|
|
626
|
-
ContextCompat.startForegroundService(this, intent)
|
|
627
|
-
}
|
|
628
|
-
|
|
629
531
|
private fun currentAppMode(): String {
|
|
630
532
|
return try {
|
|
631
533
|
val appInfo = packageManager.getApplicationInfo(
|
|
@@ -17,7 +17,6 @@ import 'package:flutter_svg/flutter_svg.dart';
|
|
|
17
17
|
import 'package:file_picker/file_picker.dart';
|
|
18
18
|
import 'package:flutter_secure_storage/flutter_secure_storage.dart';
|
|
19
19
|
import 'package:google_fonts/google_fonts.dart';
|
|
20
|
-
import 'package:image/image.dart' as img;
|
|
21
20
|
import 'package:mobile_scanner/mobile_scanner.dart';
|
|
22
21
|
import 'package:qr_flutter/qr_flutter.dart';
|
|
23
22
|
import 'package:shared_preferences/shared_preferences.dart';
|
|
@@ -39,8 +38,6 @@ import 'src/health_bridge.dart';
|
|
|
39
38
|
import 'src/live_voice_capture.dart';
|
|
40
39
|
import 'src/messaging_access_summary.dart';
|
|
41
40
|
import 'src/oauth_launcher.dart';
|
|
42
|
-
import 'src/recording_bridge.dart';
|
|
43
|
-
import 'src/recording_payloads.dart';
|
|
44
41
|
import 'src/security/password_strength.dart';
|
|
45
42
|
import 'src/stream_renderer.dart';
|
|
46
43
|
import 'src/theme/palette.dart';
|
|
@@ -66,7 +63,6 @@ part 'main_timeline.dart';
|
|
|
66
63
|
part 'main_runtime.dart';
|
|
67
64
|
part 'main_controller.dart';
|
|
68
65
|
part 'main_devices.dart';
|
|
69
|
-
part 'main_recordings.dart';
|
|
70
66
|
part 'main_chat.dart';
|
|
71
67
|
part 'main_account_settings.dart';
|
|
72
68
|
part 'main_settings.dart';
|
|
@@ -104,8 +100,6 @@ const String _neoMailLogoAsset = 'assets/branding/neomail_logo.svg';
|
|
|
104
100
|
const String _sessionCookiePrefsKey = 'auth.sessionCookie';
|
|
105
101
|
const String _sessionCookieBackendPrefsKey = 'auth.sessionCookieBackend';
|
|
106
102
|
const String _sessionCookieSecureStorageKey = 'auth.sessionCookie.secure';
|
|
107
|
-
const int _voiceAssistantScreenshotMaxDimension = 1600;
|
|
108
|
-
const int _voiceAssistantScreenshotMaxBytes = 900 * 1024;
|
|
109
103
|
|
|
110
104
|
String get _desktopTrayIconAsset =>
|
|
111
105
|
defaultTargetPlatform == TargetPlatform.macOS
|
|
@@ -2521,8 +2521,6 @@ class _SectionBody extends StatelessWidget {
|
|
|
2521
2521
|
return VoiceAssistantPanel(controller: controller);
|
|
2522
2522
|
case AppSection.devices:
|
|
2523
2523
|
return DevicesPanel(key: devicesPanelKey, controller: controller);
|
|
2524
|
-
case AppSection.recordings:
|
|
2525
|
-
return RecordingsPanel(controller: controller);
|
|
2526
2524
|
case AppSection.messaging:
|
|
2527
2525
|
return MessagingPanel(controller: controller);
|
|
2528
2526
|
case AppSection.runs:
|