neoagent 2.1.18-beta.2 → 2.1.18-beta.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/.env.example +121 -12
  2. package/README.md +5 -10
  3. package/docs/.vitepress/config.mts +102 -0
  4. package/docs/automation.md +53 -0
  5. package/docs/capabilities.md +124 -0
  6. package/docs/configuration.md +80 -30
  7. package/docs/getting-started.md +64 -0
  8. package/docs/index.md +68 -0
  9. package/docs/integrations.md +68 -0
  10. package/docs/operations.md +58 -0
  11. package/docs/skills.md +21 -11
  12. package/docs/why-neoagent.md +19 -0
  13. package/lib/manager.js +152 -19
  14. package/package.json +8 -1
  15. package/server/db/database.js +509 -31
  16. package/server/guest_agent.js +233 -0
  17. package/server/http/middleware.js +6 -1
  18. package/server/http/routes.js +21 -1
  19. package/server/index.js +18 -1
  20. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  21. package/server/public/flutter_bootstrap.js +1 -1
  22. package/server/public/main.dart.js +70200 -68621
  23. package/server/routes/agent_profiles.js +64 -0
  24. package/server/routes/agents.js +25 -16
  25. package/server/routes/android.js +4 -0
  26. package/server/routes/artifacts.js +47 -0
  27. package/server/routes/auth.js +24 -6
  28. package/server/routes/browser.js +9 -2
  29. package/server/routes/integrations.js +41 -6
  30. package/server/routes/mcp.js +34 -14
  31. package/server/routes/memory.js +34 -17
  32. package/server/routes/messaging.js +73 -20
  33. package/server/routes/scheduler.js +3 -1
  34. package/server/routes/settings.js +171 -22
  35. package/server/services/agents/manager.js +345 -0
  36. package/server/services/ai/capabilityHealth.js +114 -36
  37. package/server/services/ai/engine.js +188 -73
  38. package/server/services/ai/history.js +23 -18
  39. package/server/services/ai/models.js +33 -39
  40. package/server/services/ai/multiStep.js +1 -0
  41. package/server/services/ai/settings.js +102 -15
  42. package/server/services/ai/systemPrompt.js +72 -2
  43. package/server/services/ai/taskAnalysis.js +20 -1
  44. package/server/services/ai/toolResult.js +30 -1
  45. package/server/services/ai/toolRunner.js +15 -3
  46. package/server/services/ai/tools.js +180 -57
  47. package/server/services/android/controller.js +46 -6
  48. package/server/services/artifacts/store.js +147 -0
  49. package/server/services/browser/controller.js +33 -5
  50. package/server/services/commands/router.js +53 -38
  51. package/server/services/integrations/env.js +44 -10
  52. package/server/services/integrations/figma/provider.js +321 -0
  53. package/server/services/integrations/google/calendar.js +36 -3
  54. package/server/services/integrations/google/common.js +39 -0
  55. package/server/services/integrations/google/docs.js +71 -21
  56. package/server/services/integrations/google/drive.js +93 -15
  57. package/server/services/integrations/google/gmail.js +43 -5
  58. package/server/services/integrations/google/provider.js +60 -10
  59. package/server/services/integrations/google/sheets.js +97 -63
  60. package/server/services/integrations/manager.js +68 -37
  61. package/server/services/integrations/microsoft/provider.js +422 -0
  62. package/server/services/integrations/notion/provider.js +385 -0
  63. package/server/services/integrations/oauth_provider.js +392 -0
  64. package/server/services/integrations/registry.js +11 -1
  65. package/server/services/integrations/secrets.js +57 -0
  66. package/server/services/integrations/slack/provider.js +300 -0
  67. package/server/services/manager.js +78 -11
  68. package/server/services/mcp/client.js +15 -8
  69. package/server/services/memory/manager.js +120 -84
  70. package/server/services/memory/policy.js +67 -0
  71. package/server/services/messaging/automation.js +73 -21
  72. package/server/services/messaging/http_platforms.js +794 -0
  73. package/server/services/messaging/manager.js +268 -66
  74. package/server/services/messaging/telegram.js +34 -14
  75. package/server/services/runtime/backends/host.js +37 -0
  76. package/server/services/runtime/backends/local-vm.js +359 -0
  77. package/server/services/runtime/backends/remote.js +53 -0
  78. package/server/services/runtime/manager.js +98 -0
  79. package/server/services/runtime/mcp.js +18 -0
  80. package/server/services/runtime/qemu.js +391 -0
  81. package/server/services/runtime/settings.js +213 -0
  82. package/server/services/runtime/validation.js +63 -0
  83. package/server/services/scheduler/cron.js +62 -38
  84. package/server/services/websocket.js +72 -33
  85. package/server/utils/deployment.js +97 -0
  86. package/server/utils/version.js +8 -0
package/.env.example CHANGED
@@ -1,17 +1,55 @@
1
+ ########################################
2
+ # Core runtime
3
+ ########################################
4
+
1
5
  PORT=3333
2
- NODE_ENV=development
6
+ NODE_ENV=production
3
7
  SESSION_SECRET=change-this-to-a-random-secret-in-production
4
8
 
5
- # Comma-separated list of allowed CORS origins (leave empty to block all cross-origin requests)
6
- # Example: ALLOWED_ORIGINS=http://localhost:3333,https://yourdomain.com
9
+ # Public base URL used for OAuth callbacks and external links.
10
+ # Example: https://agent.example.com
11
+ PUBLIC_URL=
12
+
13
+ # Set true when running behind HTTPS or a TLS-terminating proxy.
14
+ SECURE_COOKIES=false
15
+
16
+ # Use `managed` for SaaS-style deployments to hide self-update controls in the UI.
17
+ NEOAGENT_DEPLOYMENT_MODE=self_hosted
18
+
19
+ # Deployment profile:
20
+ # private -> single-user / trusted host runtime
21
+ # prod -> multi-user / isolated per-user VM runtime
22
+ NEOAGENT_PROFILE=private
23
+
24
+ # VM runtime settings used by `prod`.
25
+ # Set these before switching NEOAGENT_PROFILE=prod.
26
+ # The app can cache a shared base image automatically from a URL, then create
27
+ # lightweight per-user qcow2 overlays on top of it.
28
+ # For Apple Silicon / arm64 hosts, switch this to:
29
+ # https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img
30
+ NEOAGENT_VM_BASE_IMAGE_URL=https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img
31
+
32
+ # Optional local override. If set, this takes precedence over the download URL.
33
+ NEOAGENT_VM_BASE_IMAGE=
34
+
35
+ # The guest image must boot a Linux guest that runs `npm run start:guest-agent`
36
+ # and exposes the guest agent on port 8421.
37
+ # Replace this placeholder before using NEOAGENT_PROFILE=prod.
38
+ # Example: openssl rand -hex 32
39
+ NEOAGENT_VM_GUEST_TOKEN=change-this-guest-token-before-prod
40
+ NEOAGENT_VM_MEMORY_MB=4096
41
+ NEOAGENT_VM_CPUS=2
42
+
43
+ # Self-hosted release track used by `neoagent update`.
44
+ NEOAGENT_RELEASE_CHANNEL=stable
45
+
46
+ # Comma-separated list of allowed CORS origins.
47
+ # Example: http://localhost:3333,https://yourdomain.com
7
48
  ALLOWED_ORIGINS=
8
49
 
9
- # xAI API key — used for:
10
- # Main AI reasoning (grok-4-1-fast-reasoning — all agent tasks and conversations)
11
- # • Image generation (grok-imagine-image — generate_image tool)
12
- # • Image vision / analysis (grok-4-1-fast-reasoning has native image input — analyze_image tool, incoming WhatsApp photos)
13
- # Get your key at: https://console.x.ai
14
- XAI_API_KEY=your-xai-api-key-here
50
+ ########################################
51
+ # Hosted AI providers
52
+ ########################################
15
53
 
16
54
  # OpenAI API key — used for:
17
55
  # • Semantic memory embeddings (text-embedding-3-small, 1536 dims)
@@ -22,16 +60,74 @@ XAI_API_KEY=your-xai-api-key-here
22
60
  # Get your key at: https://platform.openai.com/api-keys
23
61
  OPENAI_API_KEY=your-openai-api-key-here
24
62
 
63
+ # Anthropic API key — used for:
64
+ # • Claude models for long-context drafting and analysis
65
+ ANTHROPIC_API_KEY=your-anthropic-api-key-here
66
+
67
+ # xAI API key — used for:
68
+ # • Main AI reasoning (grok-4-1-fast-reasoning — all agent tasks and conversations)
69
+ # • Image generation (grok-imagine-image — generate_image tool)
70
+ # • Image vision / analysis (grok-4-1-fast-reasoning has native image input — analyze_image tool, incoming WhatsApp photos)
71
+ # Get your key at: https://console.x.ai
72
+ XAI_API_KEY=your-xai-api-key-here
73
+
25
74
  # Google AI Studio API key — used for:
26
75
  # • Gemini models (e.g. gemini-3.1-flash-lite-preview)
27
76
  # Get your key at: https://aistudio.google.com/app/apikey
28
77
  GOOGLE_AI_KEY=your-google-ai-key-here
29
78
 
30
- # Google Workspace official integration OAuth client
31
- # Redirect URI should match ${PUBLIC_URL:-http://localhost:3333}/api/integrations/oauth/callback
79
+ # MiniMax Coding Plan API key — used for:
80
+ # MiniMax-M2.7 via the Anthropic-compatible MiniMax endpoint
81
+ MINIMAX_API_KEY=your-minimax-api-key-here
82
+
83
+ ########################################
84
+ # Provider endpoint overrides
85
+ ########################################
86
+
87
+ # OPENAI_BASE_URL=https://your-openai-compatible-endpoint/v1
88
+ # ANTHROPIC_BASE_URL=https://your-anthropic-compatible-endpoint
89
+ # XAI_BASE_URL=https://api.x.ai/v1
90
+
91
+ ########################################
92
+ # Official integrations
93
+ ########################################
94
+
95
+ # Google Workspace official integration OAuth client.
96
+ # Redirect URI should match <PUBLIC_URL>/api/integrations/oauth/callback.
97
+ # Set PUBLIC_URL to your app's public base URL. Local default is http://localhost:3333.
32
98
  GOOGLE_OAUTH_CLIENT_ID=your-google-oauth-client-id
33
99
  GOOGLE_OAUTH_CLIENT_SECRET=your-google-oauth-client-secret
34
- # GOOGLE_OAUTH_REDIRECT_URI=http://localhost:3333/api/integrations/oauth/callback
100
+ GOOGLE_OAUTH_REDIRECT_URI=
101
+
102
+ # Notion official integration OAuth client.
103
+ # Redirect URI should match <PUBLIC_URL>/api/integrations/oauth/callback.
104
+ NOTION_OAUTH_CLIENT_ID=your-notion-oauth-client-id
105
+ NOTION_OAUTH_CLIENT_SECRET=your-notion-oauth-client-secret
106
+ NOTION_OAUTH_REDIRECT_URI=
107
+
108
+ # Microsoft 365 official integration OAuth client.
109
+ # Redirect URI should match <PUBLIC_URL>/api/integrations/oauth/callback.
110
+ # Leave tenant ID as `common` for a multi-tenant app, or set a specific Entra tenant ID.
111
+ MICROSOFT_OAUTH_CLIENT_ID=your-microsoft-oauth-client-id
112
+ MICROSOFT_OAUTH_CLIENT_SECRET=your-microsoft-oauth-client-secret
113
+ MICROSOFT_OAUTH_REDIRECT_URI=
114
+ MICROSOFT_OAUTH_TENANT_ID=common
115
+
116
+ # Slack official integration OAuth client.
117
+ # Redirect URI should match <PUBLIC_URL>/api/integrations/oauth/callback.
118
+ SLACK_OAUTH_CLIENT_ID=your-slack-oauth-client-id
119
+ SLACK_OAUTH_CLIENT_SECRET=your-slack-oauth-client-secret
120
+ SLACK_OAUTH_REDIRECT_URI=
121
+
122
+ # Figma official integration OAuth client.
123
+ # Redirect URI should match <PUBLIC_URL>/api/integrations/oauth/callback.
124
+ FIGMA_OAUTH_CLIENT_ID=your-figma-oauth-client-id
125
+ FIGMA_OAUTH_CLIENT_SECRET=your-figma-oauth-client-secret
126
+ FIGMA_OAUTH_REDIRECT_URI=
127
+
128
+ ########################################
129
+ # Tools and media services
130
+ ########################################
35
131
 
36
132
  # Brave Search API key — used for:
37
133
  # • Native web_search tool (search the web without driving the browser)
@@ -43,5 +139,18 @@ BRAVE_SEARCH_API_KEY=your-brave-search-api-key-here
43
139
  # Without this key: recordings can still be captured and stored, but transcription will fail until retried with a valid key.
44
140
  # Get your key at: https://console.deepgram.com/
45
141
  DEEPGRAM_API_KEY=your-deepgram-api-key-here
142
+ DEEPGRAM_BASE_URL=https://api.deepgram.com
46
143
  DEEPGRAM_MODEL=nova-3
47
144
  DEEPGRAM_LANGUAGE=multi
145
+
146
+ ########################################
147
+ # Local model runtime
148
+ ########################################
149
+
150
+ OLLAMA_URL=http://localhost:11434
151
+
152
+ ########################################
153
+ # Messaging and voice integrations
154
+ ########################################
155
+
156
+ TELNYX_WEBHOOK_TOKEN=
package/README.md CHANGED
@@ -9,15 +9,12 @@
9
9
  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-a855f7?style=flat-square" alt="License"></a>
10
10
  </p>
11
11
 
12
- <p align="center">
13
- A self-hosted, proactive AI agent with a Flutter client for web and Android.<br>
14
- Connects to OpenAI, xAI, Google, MiniMax Code, and local Ollama.<br>
15
- Runs tasks on a schedule, controls a browser, manages files, and talks to you over Telegram, Discord, or WhatsApp.
16
- </p>
12
+ <p align="center">A self-hosted, proactive AI agent with a Flutter client for web and Android.</p>
17
13
 
18
14
  | | |
19
15
  | --- | --- |
20
- | <img alt="WebUI" src="https://github.com/user-attachments/assets/3c76d59a-b6e3-4698-929b-9c94741ccf1e" height="420"> | <img alt="Mobile Telegram" src="https://github.com/user-attachments/assets/1fd41a9b-5452-4aa4-9478-888c8ad7363a" height="420"> |
16
+ | <img alt="WebUI" src="https://github.com/user-attachments/assets/3c76d59a-b6e3-4698-929b-9c94741ccf1e" height="420"> | <img height="494" alt="Android" src="https://github.com/user-attachments/assets/e8a0af7a-6881-485d-ad52-f3bc6f2023ca"> | <img alt="Mobile Telegram" src="https://github.com/user-attachments/assets/1fd41a9b-5452-4aa4-9478-888c8ad7363a" height="420"> |
17
+
21
18
 
22
19
  ## Install
23
20
 
@@ -36,14 +33,12 @@ neoagent fix
36
33
  neoagent logs
37
34
  ```
38
35
 
39
- Use `neoagent fix` if a self-edit or broken local install leaves NeoAgent in a bad state. On git installs it backs up runtime data, saves local tracked changes, resets tracked source files, reinstalls dependencies, and restarts the service.
40
-
41
36
  ## Links
42
37
 
43
- [⚙️ Configuration](docs/configuration.md) · [🧰 Skills](docs/skills.md) · [🐛 Issues](https://github.com/NeoLabs-Systems/NeoAgent/issues)
38
+ [Docs](https://neolabs-systems.github.io/NeoAgent/) | [Issues](https://github.com/NeoLabs-Systems/NeoAgent/issues)
44
39
 
45
40
  ---
46
41
 
47
42
  <p align="center">
48
- Made with ❤️ by <a href="https://github.com/neooriginal">Neo</a> · <a href="https://github.com/NeoLabs-Systems">NeoLabs Systems</a>
43
+ Made by <a href="https://github.com/neooriginal">Neo</a> | <a href="https://github.com/NeoLabs-Systems">NeoLabs Systems</a>
49
44
  </p>
@@ -0,0 +1,102 @@
1
+ import { defineConfig } from 'vitepress';
2
+
3
+ export default defineConfig({
4
+ title: 'NeoAgent',
5
+ description: 'Self-hosted proactive AI agent docs',
6
+ base: '/NeoAgent/',
7
+ cleanUrls: true,
8
+ lastUpdated: true,
9
+ themeConfig: {
10
+ nav: [
11
+ {
12
+ text: 'Start',
13
+ activeMatch: '^/(getting-started|why-neoagent)?$',
14
+ items: [
15
+ { text: 'Overview', link: '/' },
16
+ { text: 'Getting Started', link: '/getting-started' },
17
+ { text: 'Why NeoAgent', link: '/why-neoagent' },
18
+ ],
19
+ },
20
+ {
21
+ text: 'Product',
22
+ activeMatch: '^/(capabilities|automation|integrations|skills)',
23
+ items: [
24
+ { text: 'Capabilities', link: '/capabilities' },
25
+ { text: 'Android Control', link: '/capabilities#android-control' },
26
+ { text: 'Recordings', link: '/capabilities#recordings' },
27
+ { text: 'Integrations', link: '/integrations' },
28
+ { text: 'Automation', link: '/automation' },
29
+ ],
30
+ },
31
+ {
32
+ text: 'Operate',
33
+ activeMatch: '^/(configuration|operations)',
34
+ items: [
35
+ { text: 'Configuration', link: '/configuration' },
36
+ { text: 'Skills', link: '/skills' },
37
+ { text: 'Operations', link: '/operations' },
38
+ ],
39
+ },
40
+ { text: 'Why NeoAgent', link: '/why-neoagent' },
41
+ { text: 'GitHub', link: 'https://github.com/NeoLabs-Systems/NeoAgent' },
42
+ ],
43
+ sidebar: [
44
+ {
45
+ text: 'Start',
46
+ items: [
47
+ { text: 'Overview', link: '/' },
48
+ { text: 'Getting Started', link: '/getting-started' },
49
+ { text: 'Why NeoAgent', link: '/why-neoagent' },
50
+ ],
51
+ },
52
+ {
53
+ text: 'Product Surface',
54
+ items: [
55
+ {
56
+ text: 'Capabilities',
57
+ link: '/capabilities',
58
+ items: [
59
+ { text: 'Android Control', link: '/capabilities#android-control' },
60
+ { text: 'Recordings', link: '/capabilities#recordings' },
61
+ { text: 'Health Data', link: '/capabilities#health-data' },
62
+ { text: 'Agent Tools', link: '/capabilities#agent-tools' },
63
+ { text: 'Runtime Modes', link: '/capabilities#runtime-modes' },
64
+ ],
65
+ },
66
+ { text: 'Automation', link: '/automation' },
67
+ { text: 'Integrations', link: '/integrations' },
68
+ { text: 'Skills', link: '/skills' },
69
+ ],
70
+ },
71
+ {
72
+ text: 'Operate',
73
+ items: [
74
+ { text: 'Configuration', link: '/configuration' },
75
+ { text: 'Operations', link: '/operations' },
76
+ ],
77
+ },
78
+ ],
79
+ socialLinks: [
80
+ { icon: 'github', link: 'https://github.com/NeoLabs-Systems/NeoAgent' },
81
+ ],
82
+ search: {
83
+ provider: 'local',
84
+ },
85
+ outline: {
86
+ level: [2, 3],
87
+ label: 'On This Page',
88
+ },
89
+ editLink: {
90
+ pattern: 'https://github.com/NeoLabs-Systems/NeoAgent/edit/main/docs/:path',
91
+ text: 'Edit this page on GitHub',
92
+ },
93
+ docFooter: {
94
+ prev: 'Previous',
95
+ next: 'Next',
96
+ },
97
+ footer: {
98
+ message: 'Released under the MIT License.',
99
+ copyright: 'Copyright NeoLabs Systems',
100
+ },
101
+ },
102
+ });
@@ -0,0 +1,53 @@
1
+ # Automation
2
+
3
+ NeoAgent is built for proactive work: tasks that run later, repeat on a schedule, use tools, and notify you when there is something useful to report.
4
+
5
+ ## Scheduler
6
+
7
+ Use the **Scheduler** section in the UI to create recurring tasks. A scheduled task has:
8
+
9
+ | Field | Purpose |
10
+ |---|---|
11
+ | Name | Human-readable task label |
12
+ | Cron expression | Five-field cron schedule |
13
+ | Prompt | Self-contained instruction for the future run |
14
+ | Enabled state | Active or paused |
15
+ | Model override | Optional model for this task |
16
+
17
+ Scheduled prompts should include the condition for notifying you. For example, ask NeoAgent to message you only when a monitored thing changes, fails, or needs attention.
18
+
19
+ ## Tool Capabilities
20
+
21
+ Automation can use the same tool surface as normal chat runs:
22
+
23
+ | Capability | Examples |
24
+ |---|---|
25
+ | Browser | Navigate, click, type, extract page content, take screenshots, evaluate page JavaScript |
26
+ | Files | Read, write, search, and summarize host files through skills |
27
+ | CLI | Run shell commands in a persistent terminal through skills |
28
+ | Memory | Store durable facts and retrieve useful context |
29
+ | Messaging | Send a proactive result through a connected platform |
30
+ | MCP | Use tools exposed by configured remote MCP servers |
31
+ | Official integrations | Use structured OAuth-backed app tools where available |
32
+ | Recordings | List, open, and search recording transcripts |
33
+ | Health | Read synced Android Health Connect metrics as summaries |
34
+ | Android | Drive a server-attached emulator or device through UI and ADB tools |
35
+ | Subagents | Spawn async helper agents inside a longer run |
36
+ | Outputs | Generate artifacts, Grok images, Mermaid graphs, and markdown tables |
37
+
38
+ Prefer official integrations and structured MCP tools over browser automation when both can answer the task. They are usually less brittle and easier to audit.
39
+
40
+ See [Capabilities](capabilities.md) for the broader tool inventory.
41
+
42
+ ## Safety Expectations
43
+
44
+ NeoAgent runs on your server and can touch real files, messaging surfaces, browser sessions, and connected services. Keep scheduled prompts narrow and self-contained.
45
+
46
+ For sensitive automations:
47
+
48
+ - Use allowlists for messaging platforms.
49
+ - Keep secrets in server config, not prompts or skills.
50
+ - Prefer read-only checks unless the task explicitly needs to mutate data.
51
+ - Ask for notification only when a condition is met to avoid noisy repeated messages.
52
+ - Review run history in **Runs** and service logs in **Logs** when behavior is surprising.
53
+ - Remember that browser, CLI, Android runtime, and local file tools run on the NeoAgent server or configured worker, not necessarily on your current laptop.
@@ -0,0 +1,124 @@
1
+ # Capabilities
2
+
3
+ This page lists the product surfaces that are easy to miss from the short README. It is based on the current server routes, agent tool registry, Flutter sections, Android bridge code, and integration providers in this repository.
4
+
5
+ ## Operator UI
6
+
7
+ The Flutter client exposes the main operator surfaces:
8
+
9
+ | Section | What it is for |
10
+ |---|---|
11
+ | Chat | Normal agent runs with tools, memory, integrations, and messaging |
12
+ | Runs | Live and historical run steps, including browser, Android, CLI, messaging, scheduler, MCP, and subagent work |
13
+ | Logs | Service logs and diagnostics from the server you are connected to |
14
+ | Scheduler | Recurring cron tasks and one-time future runs |
15
+ | Skills | Built-in and custom reusable workflows |
16
+ | Integrations | OAuth account connections for structured app tools |
17
+ | MCP | Remote MCP server registration and tool discovery |
18
+ | Memory | Long-term memory, core memory, and session search |
19
+ | Devices | Server-side browser and Android runtime controls |
20
+ | Recordings | Recording sessions, transcripts, segments, and playback |
21
+ | Health | Android Health Connect sync status and synced metrics |
22
+ | Wearables | Supported Bluetooth recording devices and audio upload status |
23
+ | Settings | AI providers, model routing, runtime settings, messaging, and service controls |
24
+
25
+ ## Recordings
26
+
27
+ NeoAgent records audio as server-side sessions with one or more sources. The web client can record browser microphone and screen audio, the Android app can record phone microphone audio through a foreground service, and the wearable bridge can upload audio chunks from supported Bluetooth devices.
28
+
29
+ Recording sessions support:
30
+
31
+ - Chunked uploads with per-source sequence checks.
32
+ - Sources, chunks, transcript segments, session status, and playback URLs.
33
+ - Statuses for recording, processing, completed, failed, and cancelled sessions.
34
+ - Retry transcription and delete transcript segment actions.
35
+ - Full session deletion with storage cleanup.
36
+ - Agent tools for listing, opening, and searching transcripts: `recordings_list`, `recordings_get`, and `recordings_search`.
37
+
38
+ Transcription uses Deepgram when `DEEPGRAM_API_KEY` is configured. The default speech model is `nova-3`, and the default language mode is `multi`. When `auto_recording_insights` is enabled in AI settings, NeoAgent can generate structured recording insights such as a summary, action items, and events.
39
+
40
+ ## Android Control
41
+
42
+ NeoAgent can let the AI control an Android emulator or device attached to the NeoAgent server or configured worker. This is the Android capability in the comparison: the agent can observe and operate Android, not only run an Android companion app.
43
+
44
+ Android control supports:
45
+
46
+ - Starting and stopping the managed Android emulator.
47
+ - Listing ADB-connected devices and installed apps.
48
+ - Taking screenshots and UIAutomator XML dumps.
49
+ - Observing visible UI nodes.
50
+ - Opening apps and Android intents.
51
+ - Tapping, long pressing, typing, swiping, and pressing Android navigation keys.
52
+ - Waiting for text, resource IDs, descriptions, or classes to appear.
53
+ - Installing `.apk` and universal `.apks` bundles.
54
+ - Running `adb shell` commands when higher-level tools are not enough.
55
+
56
+ These actions run where the NeoAgent backend or runtime worker is running. If NeoAgent is deployed on a remote server, the AI controls the Android runtime attached to that server, not the laptop where you are reading the docs.
57
+
58
+ ## Android App, Health, And Wearables
59
+
60
+ The Flutter Android app is still useful as a client. It can sign in to the same self-hosted backend, run chat and operator UI flows, sync Health Connect data, record audio locally, and bridge supported wearables.
61
+
62
+ Android app capabilities include:
63
+
64
+ - `NEOAGENT_BACKEND_URL` build/run configuration for real devices.
65
+ - Health Connect permission flow and background sync.
66
+ - Microphone recording through an Android foreground service.
67
+ - Boot restore hooks for recording and wearable services when Android allows them.
68
+ - Bluetooth wearable bridge support for HeyPocket-style devices.
69
+ - Upload of wearable chunks and synchronization state to the backend.
70
+
71
+ ## Health Data
72
+
73
+ Health data comes from the Android app through `/api/mobile/health`. NeoAgent stores sync runs and normalized metric samples. The built-in metric aliases include steps, heart rate, sleep sessions, exercise sessions, and weight.
74
+
75
+ The agent tool `read_health_data` returns summaries and recent samples. It is designed to answer questions such as recent step totals or available health metrics without dumping every raw record.
76
+
77
+ ## Integrations And Messaging
78
+
79
+ NeoAgent has two separate integration layers:
80
+
81
+ - Official OAuth integrations expose structured tools for Google Workspace, Microsoft 365, Notion, Slack, and Figma.
82
+ - Messaging platforms let the agent talk through WhatsApp, Telegram, Discord, Slack, Google Chat, Teams, Matrix, Signal, iMessage/BlueBubbles, IRC, Twitch, LINE, Mattermost, configurable webhook bridges, and Telnyx Voice.
83
+
84
+ Official integration examples include Gmail thread search and send mail, Google Calendar events, Drive upload/download/export/share links, Docs create/append/replace, Sheets read/update/append/create, Microsoft Outlook/Calendar/OneDrive/Teams tools, Notion search/page/block/database tools, Slack conversation/message tools, and Figma file/node/comment/image tools.
85
+
86
+ Messaging examples include Telegram and Discord messages, Slack channel replies, Matrix room messages, Google Chat and Teams webhook delivery, Signal bridge delivery, iMessage/BlueBubbles sends, WhatsApp text and media sends, Telnyx inbound voice, Telnyx outbound calls, and scheduled-task call delivery.
87
+
88
+ ## Agent Tools
89
+
90
+ NeoAgent's agent tool surface includes more than basic chat:
91
+
92
+ | Area | Examples |
93
+ |---|---|
94
+ | CLI | PTY-capable `execute_command` with stdin, timeout, stdout, stderr, exit code, and duration |
95
+ | Browser | Navigate, click, type, extract, screenshot, and evaluate page JavaScript |
96
+ | Android control | UI observation, input, screenshots, app launch, intent launch, APK install, and shell commands |
97
+ | Web search | Brave Search API through `web_search` |
98
+ | Files | Read, write, edit, list, and search files |
99
+ | HTTP | Direct HTTP requests |
100
+ | Memory | Semantic memory, session search, daily logs, API key name reads, and core memory |
101
+ | Skills | Create, list, update, and delete persistent skills |
102
+ | Scheduler | Recurring tasks, one-time runs, model overrides, and optional Telnyx call delivery |
103
+ | MCP | Add, list, and remove MCP servers, plus dynamic MCP tool use |
104
+ | Subagents | Spawn, list, wait for, and cancel async subagents inside a run |
105
+ | Output | Generate markdown tables and Mermaid graphs |
106
+ | Images | Generate images with Grok and analyze local image files with a vision-capable model |
107
+ | Recordings | List, inspect, and search recording transcripts |
108
+ | Health | Read synced mobile health metrics |
109
+
110
+ Generated binary or text artifacts can be promoted into user-scoped artifact storage under `~/.neoagent/data/artifacts` and served through authenticated `/api/artifacts/:id/content` URLs.
111
+
112
+ ## Runtime Modes
113
+
114
+ Runtime settings let operators choose where higher-risk work runs:
115
+
116
+ | Profile | Runtime shape |
117
+ |---|---|
118
+ | `trusted-host` | CLI, browser, and Android tools run on the host |
119
+ | `secure-vm` | CLI, browser, and Android tools run through the local VM backend |
120
+ | `hybrid` | CLI, browser, and Android tools use a configured remote worker |
121
+
122
+ Remote execution uses `remote_worker_base_url` and an encrypted `remote_worker_token`. Production policy can require the secure VM profile and a strong VM guest token.
123
+
124
+ These controls matter operationally: the browser, Android emulator, local files, and shell commands run wherever the NeoAgent backend or configured worker is running, not necessarily on the computer where you are reading the docs.
@@ -1,57 +1,107 @@
1
1
  # Configuration
2
2
 
3
- All settings live in `~/.neoagent/.env` by default. Run `neoagent setup` to regenerate interactively. If a self-edit or local install issue leaves NeoAgent broken, `neoagent fix` will back up `~/.neoagent`, repair the installation, and restart the service.
4
- You can override the runtime root with `NEOAGENT_HOME`.
3
+ NeoAgent keeps deployment secrets on the server. The default config file is `~/.neoagent/.env`; run `neoagent setup` to regenerate it interactively. You can move the runtime root by setting `NEOAGENT_HOME`.
5
4
 
6
- ## Variables
5
+ AI provider credentials, OAuth client secrets, and deployment controls are not configured through the public web client. The Flutter UI can select providers and models, but the secrets stay in server-side environment variables or in the local NeoAgent database where the app explicitly stores channel settings.
7
6
 
8
- ### Core
7
+ ## Core Variables
9
8
 
10
9
  | Variable | Default | Description |
11
- |---|---|---|
12
- | `PORT` | `3333` | HTTP port |
13
- | `SESSION_SECRET` | *(required)* | Random string for session signing generate with `openssl rand -hex 32` |
14
- | `NODE_ENV` | `production` | Set to `development` to enable verbose logs |
15
- | `SECURE_COOKIES` | `false` | Set `true` when behind a TLS-terminating proxy |
16
- | `ALLOWED_ORIGINS` | *(none)* | Comma-separated CORS origins, e.g. `https://example.com` |
10
+ |---|---:|---|
11
+ | `PORT` | `3333` | HTTP port for the NeoAgent server. |
12
+ | `PUBLIC_URL` | optional | Public base URL used for OAuth callbacks and external links. |
13
+ | `SESSION_SECRET` | required | Random string for session signing. Generate one with `openssl rand -hex 32`. |
14
+ | `NODE_ENV` | `production` | Set to `development` to enable verbose logs. |
15
+ | `SECURE_COOKIES` | `false` | Set `true` when NeoAgent is behind a TLS-terminating proxy. |
16
+ | `ALLOWED_ORIGINS` | none | Comma-separated CORS origins, for example `https://example.com`. |
17
+ | `NEOAGENT_DEPLOYMENT_MODE` | `self_hosted` | `self_hosted` enables in-app update controls; `managed` hides operator-only controls for SaaS deployments. |
18
+ | `NEOAGENT_RELEASE_CHANNEL` | `stable` | Release track used by the self-hosted updater. |
17
19
 
18
- ### AI Providers
20
+ ## AI Providers
19
21
 
20
- At least one API key is required. The active provider and model are configured in the Flutter app.
22
+ At least one hosted-provider API key is required unless you only use local Ollama. The active provider and model routing are selected in the app, but credentials are read from server-side config.
21
23
 
22
24
  | Variable | Provider |
23
25
  |---|---|
24
26
  | `ANTHROPIC_API_KEY` | Claude (Anthropic) |
25
- | `OPENAI_API_KEY` | GPT-4o / Whisper (OpenAI) |
27
+ | `OPENAI_API_KEY` | GPT and Whisper (OpenAI) |
26
28
  | `XAI_API_KEY` | Grok (xAI) |
29
+ | `XAI_BASE_URL` | Optional xAI-compatible base URL override |
27
30
  | `GOOGLE_AI_KEY` | Gemini (Google) |
28
- | `GOOGLE_OAUTH_CLIENT_ID` | Google Workspace official integrations OAuth client ID |
29
- | `GOOGLE_OAUTH_CLIENT_SECRET` | Google Workspace official integrations OAuth client secret |
30
- | `GOOGLE_OAUTH_REDIRECT_URI` | Optional override for the Google Workspace OAuth callback URL |
31
- | `MINIMAX_API_KEY` | MiniMax Code (Coding Plan / Token Plan for `MiniMax-M2.7`) |
31
+ | `MINIMAX_API_KEY` | MiniMax Code, including `MiniMax-M2.7` |
32
32
  | `BRAVE_SEARCH_API_KEY` | Brave Search API for the native `web_search` tool |
33
- | `DEEPGRAM_API_KEY` | Recordings transcription with Deepgram Nova-3 multilingual |
34
- | `DEEPGRAM_MODEL` | Deepgram speech model override (defaults to `nova-3`) |
35
- | `DEEPGRAM_LANGUAGE` | Deepgram language override (defaults to `multi`) |
36
- | `OLLAMA_URL` | Local Ollama (`http://localhost:11434`) |
33
+ | `OPENAI_BASE_URL` | Optional OpenAI-compatible base URL override |
34
+ | `ANTHROPIC_BASE_URL` | Optional Anthropic-compatible base URL override |
35
+ | `DEEPGRAM_API_KEY` | Recordings transcription with Deepgram |
36
+ | `DEEPGRAM_BASE_URL` | Optional Deepgram API base URL override |
37
+ | `DEEPGRAM_MODEL` | Deepgram speech model override, defaults to `nova-3` |
38
+ | `DEEPGRAM_LANGUAGE` | Deepgram language override, defaults to `multi` |
39
+ | `OLLAMA_URL` | Local Ollama server, usually `http://localhost:11434` |
37
40
 
38
- ### Messaging
41
+ Recording insight generation is controlled in app AI settings with `auto_recording_insights`. It uses the configured AI providers after Deepgram transcription has produced transcript text.
42
+
43
+ ## Official Integrations
44
+
45
+ Official integrations use OAuth and expose structured tools to the agent. The built-in registry currently covers Google Workspace, Notion, Microsoft 365, Slack, and Figma.
46
+
47
+ All OAuth callbacks default to `PUBLIC_URL + /api/integrations/oauth/callback` unless you set a provider-specific redirect URI.
39
48
 
40
49
  | Variable | Description |
41
50
  |---|---|
42
- | `TELNYX_WEBHOOK_TOKEN` | Telnyx webhook signature verification |
51
+ | `GOOGLE_OAUTH_CLIENT_ID` | Google Workspace OAuth client ID |
52
+ | `GOOGLE_OAUTH_CLIENT_SECRET` | Google Workspace OAuth client secret |
53
+ | `GOOGLE_OAUTH_REDIRECT_URI` | Optional Google Workspace OAuth callback URL |
54
+ | `NOTION_OAUTH_CLIENT_ID` | Notion OAuth client ID |
55
+ | `NOTION_OAUTH_CLIENT_SECRET` | Notion OAuth client secret |
56
+ | `NOTION_OAUTH_REDIRECT_URI` | Optional Notion OAuth callback URL |
57
+ | `MICROSOFT_OAUTH_CLIENT_ID` | Microsoft 365 OAuth client ID |
58
+ | `MICROSOFT_OAUTH_CLIENT_SECRET` | Microsoft 365 OAuth client secret |
59
+ | `MICROSOFT_OAUTH_REDIRECT_URI` | Optional Microsoft 365 OAuth callback URL |
60
+ | `MICROSOFT_OAUTH_TENANT_ID` | Optional Entra tenant selector, defaults to `common` |
61
+ | `SLACK_OAUTH_CLIENT_ID` | Slack OAuth client ID |
62
+ | `SLACK_OAUTH_CLIENT_SECRET` | Slack OAuth client secret |
63
+ | `SLACK_OAUTH_REDIRECT_URI` | Optional Slack OAuth callback URL |
64
+ | `FIGMA_OAUTH_CLIENT_ID` | Figma OAuth client ID |
65
+ | `FIGMA_OAUTH_CLIENT_SECRET` | Figma OAuth client secret |
66
+ | `FIGMA_OAUTH_REDIRECT_URI` | Optional Figma OAuth callback URL |
43
67
 
44
- Telegram, Discord, and WhatsApp tokens are stored in the database via the Flutter app settings page — not in `.env`.
68
+ ## Messaging
45
69
 
46
- ## Runtime data paths
70
+ Messaging platform credentials are stored through the Flutter app messaging tab, not in `.env`. This includes Telegram, Discord, Slack, Google Chat, Microsoft Teams, Matrix, Signal, iMessage/BlueBubbles, IRC, Twitch, LINE, Mattermost, and the configurable webhook bridges. Use the app to set platform tokens, webhook URLs, inbound secrets, polling options, and access lists.
47
71
 
48
- - Config: `~/.neoagent/.env`
49
- - Database/session/logs: `~/.neoagent/data/`
50
- - Skills/memory/daily data files: `~/.neoagent/agent-data/`
72
+ Generic inbound messaging callbacks use:
73
+
74
+ ```text
75
+ PUBLIC_URL + /api/messaging/webhook/:platform
76
+ ```
77
+
78
+ Telnyx webhook verification is configured through the environment.
79
+
80
+ | Variable | Description |
81
+ |---|---|
82
+ | `TELNYX_WEBHOOK_TOKEN` | Telnyx webhook signature verification token |
51
83
 
52
- ---
84
+ ## Runtime Isolation
85
+
86
+ Runtime profile and backend selection are stored in user settings, not normally in `.env`. The available profiles are `trusted-host`, `secure-vm`, and `hybrid`. They control where CLI, browser, and Android tools run: on the host, through a local VM, or through a configured remote worker.
87
+
88
+ Production policy can require the VM backend. In that case, set a strong `NEOAGENT_VM_GUEST_TOKEN` of at least 32 characters and avoid placeholder values.
89
+
90
+ Remote worker settings are stored through the app as `remote_worker_base_url` and encrypted `remote_worker_token` values. Use an `http` or `https` worker URL only.
91
+
92
+ ## Secrets Guidance
93
+
94
+ Treat `SESSION_SECRET`, provider API keys, OAuth client secrets, messaging credentials, and Telnyx tokens as sensitive. Do not commit them, print them in logs, or expose them in client-side code. Store them in server-side environment variables or a secrets manager, restrict access to operators who need them, and rotate them immediately if you suspect exposure.
95
+
96
+ ## Runtime Paths
97
+
98
+ | Path | Purpose |
99
+ |---|---|
100
+ | `~/.neoagent/.env` | Server config and deployment secrets |
101
+ | `~/.neoagent/data/` | Database, sessions, update status, and logs |
102
+ | `~/.neoagent/agent-data/` | Skills, memory, and daily data files |
53
103
 
54
- ## Minimal `.env` example
104
+ ## Minimal `.env` Example
55
105
 
56
106
  ```dotenv
57
107
  PORT=3333