@geravant/sinain 1.24.0 → 1.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/README.md +236 -141
  2. package/config-shared.js +176 -25
  3. package/config.js +21 -3
  4. package/launcher.js +18 -3
  5. package/onboard.js +144 -38
  6. package/package.json +1 -1
  7. package/sense_client/ollama_vision.py +1 -1
  8. package/sinain-core/src/agent/analyzer.ts +2 -1
  9. package/sinain-core/src/config.ts +19 -0
package/README.md CHANGED
@@ -1,189 +1,284 @@
1
- # @geravant/sinain
1
+ # Sinain <img src="https://raw.githubusercontent.com/anthillnet/sinain-hud/main/media/screen-recording-2026-03-26.gif" alt="Sinain HUD" width="120" align="right">
2
2
 
3
- This package serves two roles:
3
+ [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/anthillnet/sinain-hud/blob/main/LICENSE)
4
+ [![CI](https://github.com/anthillnet/sinain-hud/actions/workflows/ci.yml/badge.svg)](https://github.com/anthillnet/sinain-hud/actions/workflows/ci.yml)
5
+ [![npm](https://img.shields.io/npm/v/@geravant/sinain)](https://www.npmjs.com/package/@geravant/sinain)
6
+ [![macOS 12.3+](https://img.shields.io/badge/macOS-12.3%2B-black?logo=apple)](https://support.apple.com/macos)
7
+ [![LongMemEval IPR 82.8%](https://img.shields.io/badge/LongMemEval%20IPR-82.8%25-success)](https://github.com/anthillnet/sinain-hud/blob/main/docs/LONGMEMEVAL-AUDIT.md)
4
8
 
5
- 1. **Standalone launcher** — run `npx @geravant/sinain start` to launch the full sinain stack (core, sense, overlay, agent) on your Mac. See the [main README](../README.md#quick-start) for usage.
6
- 2. **OpenClaw plugin** — when installed on an OpenClaw gateway server (`npx @geravant/sinain install`), it manages the sinain-hud agent lifecycle.
9
+ **Context OS** — ambient intelligence for builders. Captures what you see and hear, distills it into a private knowledge graph, accessible from MCP, a web UI, and a screen-recording-invisible HUD overlay.
7
10
 
8
- ---
11
+ <p align="center">
12
+ <img src="https://raw.githubusercontent.com/anthillnet/sinain-hud/main/media/sinain-demo-full.gif" alt="Sinain demo" width="720">
13
+ </p>
9
14
 
10
- ## OpenClaw Plugin
15
+ **[Quick Start](#quick-start)** · **[Docs](https://github.com/anthillnet/sinain-hud/tree/main/docs)** · **[Privacy](https://github.com/anthillnet/sinain-hud/blob/main/docs/privacy-protection-design.md)** · **[Configuration](https://github.com/anthillnet/sinain-hud/blob/main/docs/CONFIGURATION.md)** · **[Contributing](https://github.com/anthillnet/sinain-hud/blob/main/CONTRIBUTING.md)**
11
16
 
12
- Plugin for the [anthillnet fork of OpenClaw](https://github.com/anthillnet/openclaw) that manages the sinain-hud agent lifecycle on the server.
17
+ ---
13
18
 
14
- ## What It Does
19
+ ### You, Augmented
15
20
 
16
- Five lifecycle hooks, one tool, four commands, and a background service:
21
+ Sinain captures your screen and audio continuously, distills the stream into a **structured live knowledge graph** of facts, entities, and decisions, and exposes that graph through every interface where you might need it.
17
22
 
18
- ### Hooks
23
+ - **Capture** — screen frames + system audio, with `<private>` tag stripping and auto-redaction of credentials and tokens *before* anything leaves your machine.
24
+ - **Distill** — facts (atomic claims), entities (people / projects / topics), decisions (what was chosen and why) — extracted by an LLM, integrated by deterministic graph operations (no LLM in the integration step → **[82.8% Information Preservation Rate](https://github.com/anthillnet/sinain-hud/blob/main/docs/LONGMEMEVAL-AUDIT.md)** on the [LongMemEval benchmark](https://github.com/xiaowu0162/LongMemEval) (ICLR 2025, 500 questions), measured against full-context oracle).
25
+ - **Access from anywhere** — **MCP server** for agents (Claude Code, Codex, Goose, OpenClaude, Junie), **web UI** at `localhost:9500/knowledge/ui/` for browsing, and a **HUD overlay** for in-the-moment recall.
19
26
 
20
- | Hook | Purpose |
21
- |---|---|
22
- | `session_start` | Initializes per-session tool usage and compliance tracking |
23
- | `before_agent_start` | Syncs HEARTBEAT.md, SKILL.md, sinain-memory/ (recursively, including eval/), and modules/ from `sinain-sources/` to workspace; generates effective playbook; creates memory directories |
24
- | `tool_result_persist` | Strips `<private>` tags from tool results; tracks `sinain_heartbeat_tick` calls for compliance validation |
25
- | `agent_end` | Writes structured session summary; validates heartbeat compliance (warns on skip, escalates after 3 consecutive skips) |
26
- | `session_end` | Cleans up orphaned session state |
27
+ The HUD overlay is invisible to screen capture — never appears in screenshots, recordings, or screen shares.
27
28
 
28
- ### Tool
29
+ ### Agent-Agnostic
29
30
 
30
- | Tool | Purpose |
31
- |---|---|
32
- | `sinain_heartbeat_tick` | Executes all heartbeat mechanical work (signal analysis, insight synthesis, log writing). Returns structured JSON with results, recommended actions, and Telegram output. |
31
+ Sinain feeds the same screen and audio context to any MCP-compatible agent. Switch agents on the fly — no restart, no context loss.
33
32
 
34
- The heartbeat tool accepts `{ sessionSummary: string, idle: boolean }` and runs:
35
- 1. `uv run python3 sinain-memory/signal_analyzer.py` (60s timeout)
36
- 2. `uv run python3 sinain-memory/insight_synthesizer.py` (60s timeout)
37
- 3. Writes log entry to `memory/playbook-logs/YYYY-MM-DD.jsonl`
33
+ - Tested with Claude Code, OpenClaude, Codex, Goose, and Junie. Any MCP-compatible agent works.
34
+ - Pick agents in the overlay's flash-icon selector — spawn tasks can route to any profile in your roster.
35
+ - Add custom profiles (personal Claude config, alternate models) by editing [`agents.json`](https://github.com/anthillnet/sinain-hud/blob/main/docs/AGENT-ROSTER.md). The roster is the source of truth.
36
+ - Knowledge modules travel with you — export from one machine, import on another.
38
37
 
39
- ### Commands
38
+ ### Privacy Controls
40
39
 
41
- | Command | Purpose |
42
- |---|---|
43
- | `/sinain_status` | Shows persistent session data from `sessions.json` (update time, tokens, compactions, transcript size) and resilience metrics |
44
- | `/sinain_modules` | Shows active knowledge module stack, suspended and disabled modules |
45
- | `/sinain_eval` | Shows latest evaluation report and recent tick evaluation metrics |
46
- | `/sinain_eval_level` | Sets evaluation level: `mechanical`, `sampled`, or `full` |
47
-
48
- ### Service
49
-
50
- **Curation pipeline** — runs every 30 minutes in the background:
51
- 1. Feedback analysis (`feedback_analyzer.py`) → extracts `curateDirective` + effectiveness metrics
52
- 2. Memory mining (`memory_miner.py`) → reads unread daily memory files
53
- 3. Playbook curation (`playbook_curator.py`) → archives, applies changes
54
- 4. Effectiveness footer update → writes metrics into playbook
55
- 5. Effective playbook regeneration → merges base playbook + active module patterns
56
- 6. Tick evaluation (`tick_evaluator.py`) → runs mechanical + sampled judges (120s timeout)
57
- 7. Daily eval report (`eval_reporter.py`) → generates report once per day after 03:00 UTC
40
+ By default, sinain uses cloud APIs (OpenRouter) for transcription and analysis. When you need tighter control, switch privacy modes — no code changes, one env var.
58
41
 
59
- ## Configuration
42
+ - `off` → `standard` → `strict` → `paranoid` — four modes in `~/.sinain/.env`.
43
+ - `paranoid` mode: Ollama + whisper.cpp. **Zero network calls at runtime** — the embedding model is pre-cached during the setup wizard (`sinain setup-embedding`), so subsequent runs are fully offline.
44
+ - HUD overlay is invisible to screen capture (`NSWindow.sharingType = .none` — see [`overlay/macos/Runner/AppDelegate.swift:70`](https://github.com/anthillnet/sinain-hud/blob/main/overlay/macos/Runner/AppDelegate.swift#L70)). Verifiable via split-screen recording with QuickTime + OBS + Loom — the HUD is absent from all three.
60
45
 
61
- Configured in `openclaw.json` under `plugins.entries.sinain-hud`:
62
-
63
- ```json
64
- {
65
- "plugins": {
66
- "entries": {
67
- "sinain-hud": {
68
- "enabled": true,
69
- "config": {
70
- "heartbeatPath": "/home/node/.openclaw/sinain-sources/HEARTBEAT.md",
71
- "skillPath": "/home/node/.openclaw/sinain-sources/SKILL.md",
72
- "memoryPath": "/home/node/.openclaw/sinain-sources/sinain-memory",
73
- "modulesPath": "/home/node/.openclaw/sinain-sources/modules",
74
- "sessionKey": "agent:main:sinain"
75
- }
76
- }
77
- }
78
- }
79
- }
46
+ ## Quick Start
47
+
48
+ ```bash
49
+ npx @geravant/sinain@latest start
80
50
  ```
81
51
 
82
- | Field | Type | Description |
83
- |---|---|---|
84
- | `heartbeatPath` | string | Path to HEARTBEAT.md source (resolved relative to state dir) |
85
- | `skillPath` | string | Path to SKILL.md source |
86
- | `memoryPath` | string | Path to sinain-memory/ scripts directory |
87
- | `modulesPath` | string | Path to modules/ directory for knowledge module system |
88
- | `sessionKey` | string | Session key for the sinain agent |
52
+ That's it. On first run, sinain will:
53
+ 1. Run an **interactive setup wizard** — transcription backend, API key, agent, privacy mode
54
+ 2. **Auto-download** the overlay app (~17MB), sck-capture binary (~5MB), embedding model (~90MB), and Python dependencies
55
+ 3. **Start all services** — sinain-core, sense_client, overlay, and agent
56
+
57
+ All assets are cached locally after the first install. In `paranoid` mode, subsequent runs are fully offline — no network calls at runtime.
58
+
59
+ > **Pin `@latest`** on every invocation. `npx @geravant/sinain` (without `@latest`) caches *forever* against the unversioned spec — you'd silently keep running an old version for months. Sinain self-updates automatically when stale, but pinning `@latest` makes it explicit and saves a redundant relaunch.
60
+
61
+ > **Re-run the wizard** anytime: `npx @geravant/sinain@latest start --setup`
62
+
63
+ ### Prerequisites
64
+
65
+ - **macOS 12.3+** — Sinain uses ScreenCaptureKit (introduced in 12.3). Earlier versions are not supported in this release. Apple Silicon and Intel both work.
66
+ - **Node.js 18+** — [nodejs.org](https://nodejs.org/) (LTS recommended)
67
+ - **Python 3.10+** — `brew install python3` (macOS) or [python.org](https://www.python.org/downloads/)
68
+ - **OpenRouter API key** (optional for local-only mode) — [openrouter.ai](https://openrouter.ai)
69
+ - **Network access during first install** — the wizard downloads ~112MB total (overlay app, sck-capture binary, sentence-transformer embedding model). All cached locally; subsequent runs need network only for cloud LLM API calls (or zero network in `paranoid` mode).
70
+
71
+ > **Fully local?** No API key needed. Ollama + whisper-cli = zero cloud at runtime. See [Running Fully Local](#running-fully-local).
72
+
73
+ > **First install reproducibility?** See [docs/cold-install-verification.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/cold-install-verification.md) for a step-by-step verified-on-fresh-user-account guide, including the timing measurement and the failure modes the audit caught + fixed.
89
74
 
90
- ## File Auto-Deploy
75
+ ### macOS Permissions
91
76
 
92
- The `before_agent_start` hook copies files from the persistent source directory to the agent workspace:
77
+ 1. **System Settings → Privacy & Security → Screen Recording** — add your Terminal, then **quit and reopen Terminal** (macOS TCC entitlements only apply to processes started after the grant)
78
+ 2. **System Settings → Privacy & Security → Microphone** — same: add Terminal, then restart Terminal
79
+
80
+ > Sinain detects when these permissions are missing and surfaces a clear restart-instruction banner; you'll never get a silent degraded mode.
81
+
82
+ ### Managing sinain
83
+
84
+ ```bash
85
+ npx @geravant/sinain@latest stop # stop all services
86
+ npx @geravant/sinain@latest status # check what's running
87
+ npx @geravant/sinain@latest start --setup # re-run setup wizard
88
+ npx @geravant/sinain@latest start --no-sense # skip screen capture
89
+ npx @geravant/sinain@latest start --no-overlay # headless mode
90
+ ```
91
+
92
+ > Always pin `@latest` — see the note in [Quick Start](#quick-start) above.
93
+
94
+ ## Architecture
93
95
 
94
96
  ```
95
- /mnt/openclaw-state/sinain-sources/ → /home/node/.openclaw/workspace/
96
- HEARTBEAT.md HEARTBEAT.md
97
- SKILL.md SKILL.md
98
- sinain-memory/ sinain-memory/
99
- *.json, *.sh, *.txt (always overwritten)
100
- *.py (deploy-once — skip if exists)
101
- modules/ modules/
102
- manifest.json (always overwritten)
103
- module-registry.json (deploy-once)
104
- */patterns.md (deploy-once)
105
- sinain-memory/eval/ sinain-memory/eval/ (recursive)
106
- *.py (deploy-once)
107
- *.json, *.jsonl (always overwritten)
97
+ ┌─── Your Device ─────────────────────────────────────────────────────┐
98
+ │ │
99
+ │ sck-capture (Swift) │
100
+ │ ├─ system audio (PCM) ──► sinain-core :9500 │
101
+ │ └─ screen frames (JPEG) ──► sense_client ─── POST /sense ──► │
102
+ │ │ │
103
+ │ ┌────────────────────────┘ │
104
+ │ │ │
105
+ │ sinain-core │
106
+ │ ├─ audio pipeline → transcription │
107
+ │ ├─ agent loop → digest + HUD text │
108
+ │ ├─ knowledge graph (private, on-device) │
109
+ │ └─ WebSocket feed │
110
+ │ │ │
111
+ │ ▼ │
112
+ │ overlay (Flutter) │
113
+ │ private, invisible to screen capture │
114
+ │ │
115
+ └─────────────────────────────────────────────────────────────────────┘
108
116
  ```
109
117
 
110
- Only writes if content has actually changed (avoids unnecessary git diffs).
118
+ ## Components
119
+
120
+ | Component | Language | What it does | Docs |
121
+ |---|---|---|---|
122
+ | **sinain-core** | TypeScript | Central hub: audio pipeline, agent loop, knowledge graph, WS feed | [README](https://github.com/anthillnet/sinain-hud/blob/main/sinain-core/README.md) |
123
+ | **overlay** | Dart / Swift | Private HUD (macOS), display modes, hotkeys | [Hotkeys](https://github.com/anthillnet/sinain-hud/blob/main/docs/HOTKEYS.md) |
124
+ | **sense_client** | Python | Screen capture, SSIM diff, OCR, privacy filter | [sense_client/](https://github.com/anthillnet/sinain-hud/tree/main/sense_client) |
125
+ | **sck-capture** | Swift | ScreenCaptureKit: system audio + screen frames | [tools/sck-capture/](https://github.com/anthillnet/sinain-hud/tree/main/tools/sck-capture) |
126
+ | **sinain-agent** | Bash | Shell harness that connects any agent to sinain-core | [sinain-agent/](https://github.com/anthillnet/sinain-hud/tree/main/sinain-agent) |
127
+ | **sinain-knowledge** | TypeScript | Curation, playbook, eval, portable knowledge modules | [Knowledge System](https://github.com/anthillnet/sinain-hud/blob/main/docs/knowledge-system.md) |
128
+ | **sinain-mcp-server** | TypeScript | MCP server exposing sinain tools to agents | [sinain-mcp-server/](https://github.com/anthillnet/sinain-hud/tree/main/sinain-mcp-server) |
129
+
130
+ ## Configuration
131
+
132
+ Sinain splits config across two files in `~/.sinain/`:
133
+
134
+ - **`.env`** — secrets (API keys) and infrastructure (ports, audio device, privacy mode, analyzer LLM).
135
+ - **`agents.json`** — agent roster (default agent, allowed-tools whitelists, analyzer pacing).
136
+
137
+ Both are created by the setup wizard. To re-run: `npx @geravant/sinain start --setup`.
138
+
139
+ ### Agents & profiles → `agents.json`
140
+
141
+ The agent roster lives in `~/.sinain/agents.json`. Each entry is a profile mapping a name to a binary + behavior type + optional env, settings, and model overrides. The overlay's flash-icon selector lets you pick which profile handles spawn tasks at runtime. Custom profiles like `pclaude` (personal claude with its own config dir) are first-class — the dispatch decision keys off `profile.type`, not the profile name.
142
+
143
+ See **[Agent Roster & Profiles](https://github.com/anthillnet/sinain-hud/blob/main/docs/AGENT-ROSTER.md)** for the complete schema, recipes, and routing model.
111
144
 
112
- Also ensures these directories exist:
113
- - `memory/`, `memory/playbook-archive/`, `memory/playbook-logs/`
114
- - `memory/eval-logs/`, `memory/eval-reports/`
145
+ ### Context Analysis (HUD summarizer) → `.env`
115
146
 
116
- After syncing modules, the plugin generates `memory/sinain-playbook-effective.md` — a merged view of active module patterns (sorted by priority) plus the base playbook.
147
+ The context analysis loop runs every 3–30 seconds, sending recent audio/screen context to an LLM. It produces the short HUD text shown on the overlay plus a richer digest stored in the feed buffer for the knowledge graph.
117
148
 
118
- ## Heartbeat Compliance Validation
149
+ | Variable | Default | Description |
150
+ |---|---|---|
151
+ | `ANALYSIS_PROVIDER` | `openrouter` | `openrouter` (cloud) or `ollama` (local, free) |
152
+ | `ANALYSIS_MODEL` | `google/gemini-2.5-flash-lite` | Primary model for text analysis |
153
+ | `ANALYSIS_VISION_MODEL` | `google/gemini-2.5-flash` | Auto-selected when screen images are present |
154
+ | `ANALYSIS_ENDPOINT` | *(auto per provider)* | Override for custom OpenAI-compatible endpoints |
155
+ | `ANALYSIS_API_KEY` | *(from OPENROUTER_API_KEY)* | API key; not needed for ollama |
156
+ | `ANALYSIS_FALLBACK_MODELS` | `gemini-2.5-flash,...` | Comma-separated fallback chain |
157
+
158
+ ### Other Key Settings → `.env`
159
+
160
+ | Variable | Default | Description |
161
+ |---|---|---|
162
+ | `OPENROUTER_API_KEY` | — | Required (unless `ANALYSIS_PROVIDER=ollama` + local transcription) |
163
+ | `PRIVACY_MODE` | `off` | `off` / `standard` / `strict` / `paranoid` |
164
+
165
+ See [docs/CONFIGURATION.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/CONFIGURATION.md) for the full reference.
166
+
167
+ ## Privacy Modes
168
+
169
+ | Mode | What it does |
170
+ |---|---|
171
+ | `off` | All data flows freely — maximum insight quality |
172
+ | `standard` | Auto-redacts credentials before cloud APIs (wizard default) |
173
+ | `strict` | Only summaries leave your machine — no raw text sent to cloud |
174
+ | `paranoid` | Fully local: Ollama + whisper.cpp. Zero network calls at runtime (embedding model pre-cached at install). |
119
175
 
120
- The plugin enforces that the agent actually calls `sinain_heartbeat_tick` during heartbeat runs:
176
+ See [Privacy Threat Model](https://github.com/anthillnet/sinain-hud/blob/main/docs/privacy-protection-design.md) for the full design.
121
177
 
122
- 1. `tool_result_persist` sets `heartbeatToolCalled = true` when `sinain_heartbeat_tick` is invoked
123
- 2. `agent_end` checks if the run was a heartbeat (`messageProvider === "heartbeat"`)
124
- 3. If tool wasn't called: logs warning, increments `consecutiveHeartbeatSkips` counter
125
- 4. After 3 consecutive skips: logs ESCALATION warning
126
- 5. A successful tool call resets the counter to 0
178
+ ## Hotkeys
127
179
 
128
- ## Privacy Tag Stripping
180
+ Global hotkeys use **Cmd+Shift**:
129
181
 
130
- The `tool_result_persist` hook intercepts tool results before they're saved to session history. Any `<private>...</private>` blocks are removed from:
131
- - String content (simple tool results)
132
- - Text blocks in array content (structured tool results)
182
+ | Shortcut | Action |
183
+ |---|---|
184
+ | `Cmd+Shift+Space` | Toggle overlay visibility |
185
+ | `Cmd+Shift+M` | Cycle display mode |
186
+ | `Cmd+Shift+/` | Open command input |
187
+ | `Cmd+Shift+H` | Quit overlay |
188
+
189
+ See [docs/HOTKEYS.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/HOTKEYS.md) for all 15 shortcuts.
133
190
 
134
- This is the server-side complement to sense_client's client-side `apply_privacy()` filter.
191
+ ## Running Fully Local
135
192
 
136
- ## Session Summaries
193
+ No cloud APIs needed. Local models handle everything:
194
+
195
+ ```bash
196
+ # 1. Install local transcription
197
+ ./setup-local-stt.sh
137
198
 
138
- On `agent_end`, the plugin appends a JSON line to `memory/session-summaries.jsonl`:
199
+ # 2. Install Ollama + vision model
200
+ brew install ollama && ollama pull llava
139
201
 
140
- ```json
141
- {
142
- "ts": "2026-02-18T12:00:00.000Z",
143
- "sessionKey": "agent:main:sinain",
144
- "agentId": "...",
145
- "durationMs": 45000,
146
- "success": true,
147
- "error": null,
148
- "toolCallCount": 12,
149
- "toolBreakdown": { "sessions_history": 3, "sinain_heartbeat_tick": 1, "Write": 5 },
150
- "messageCount": 8
151
- }
202
+ # 3. Start in local mode
203
+ ./start-local.sh
152
204
  ```
153
205
 
154
- ## Context Overflow Watchdog
206
+ | Model | Size | Speed | Best for |
207
+ |---|---|---|---|
208
+ | `llava` | 4.7 GB | ~2s/frame | General use (recommended) |
209
+ | `llama3.2-vision` | 7.9 GB | ~4s/frame | Best accuracy |
210
+ | `moondream` | 1.7 GB | ~1s/frame | Fastest, lower quality |
211
+
212
+ ## Setup Guides
213
+
214
+ | Setup | Guide |
215
+ |---|---|
216
+ | **Agent Roster & Profiles** | [docs/AGENT-ROSTER.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/AGENT-ROSTER.md) — pick agents, add custom profiles |
217
+ | **Bare Agent** | [docs/INSTALL-BARE-AGENT.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/INSTALL-BARE-AGENT.md) — the default install path |
218
+ | From Source | `git clone`, `cp .env.example ~/.sinain/.env`, `./start.sh` |
219
+
220
+ ## Knowledge System
155
221
 
156
- Automatically recovers from runaway context growth that causes repeated agent failures.
222
+ Sinain builds a persistent knowledge graph from everything it captures — audio transcriptions, screen OCR, and agent interactions. Facts are distilled incrementally (on buffer full and session end), stored in an EAV triplestore with graph relationships, and retrieved via hybrid search (FTS5 + tag-based + entity graph backrefs with RRF fusion).
157
223
 
158
- - **Detection:** Tracks consecutive errors matching `/overloaded|context.*too.*long|token.*limit/i` on `cfg.sessionKey`
159
- - **Trigger:** 5 consecutive errors + transcript ≥ 1 MB
160
- - **Action:** Archives transcript via `copyFileSync`, truncates to empty, resets `contextTokens` in `sessions.json`
161
- - **Resets:** Counter clears on any successful session completion and on `gateway_start`
224
+ The integration step is fully deterministic — no LLM decides what to store. Every extracted fact is preserved.
162
225
 
163
- The 1 MB minimum guard prevents resets from transient API outages when the transcript is small.
226
+ ```bash
227
+ npx @geravant/sinain export-knowledge # export playbook, modules, graph
228
+ npx @geravant/sinain import-knowledge ~/sinain-knowledge-export.tar.gz
229
+ ```
230
+
231
+ See [Knowledge System docs](https://github.com/anthillnet/sinain-hud/blob/main/docs/knowledge-system.md) for architecture details.
232
+
233
+ ### Querying knowledge from any MCP agent
164
234
 
165
- ## Deployment
235
+ Sinain's knowledge graph is exposed to any MCP-aware agent via the bundled MCP server. See **[Connect Your Coding Agent (MCP)](#connect-your-coding-agent-mcp)** below for setup.
166
236
 
167
- **IMPORTANT:** Use `docker-compose.openclaw.yml` — the default compose file uses unset env vars and will fail.
237
+ ## Connect Your Coding Agent (MCP)
238
+
239
+ Sinain ships an MCP server that exposes 15 `sinain_*` tools — including `sinain_knowledge_query`, `sinain_get_knowledge`, `sinain_distill_session`, `sinain_get_context`, and `sinain_respond` — to any MCP-aware agent. Register it once and the agent can read your knowledge graph and surface text on the HUD from any project.
168
240
 
169
241
  ```bash
170
- # Upload plugin files to the server
171
- scp -i ~/.ssh/<your-key> \
172
- sinain-hud-plugin/index.ts sinain-hud-plugin/openclaw.plugin.json \
173
- root@<your-server-ip>:/mnt/openclaw-state/extensions/sinain-hud/
174
-
175
- # Restart gateway to load updated plugin
176
- ssh -i ~/.ssh/<your-key> root@<your-server-ip> \
177
- 'cd /opt/openclaw && docker compose -f docker-compose.openclaw.yml restart'
178
-
179
- # Verify plugin loaded
180
- ssh -i ~/.ssh/<your-key> root@<your-server-ip> \
181
- 'cd /opt/openclaw && docker compose -f docker-compose.openclaw.yml logs --tail=30 openclaw-gateway 2>&1 | grep sinain'
242
+ npx @geravant/sinain@latest mcp install
182
243
  ```
183
244
 
184
- ## Files
245
+ The wizard detects which MCP agents you have installed and registers sinain for the ones you select. Re-runnable any time; idempotent.
246
+
247
+ | Agent | Setup | Config it touches |
248
+ |---|---|---|
249
+ | **Claude Code** | `mcp install` (auto via wizard) | `~/.claude.json` (`claude mcp add`) |
250
+ | **Claude Desktop** | `mcp install` (auto via wizard) | `~/Library/Application Support/Claude/claude_desktop_config.json` (mac) |
251
+ | **Cursor** | `mcp install` (auto via wizard) | `~/.cursor/mcp.json` |
252
+ | **Codex** | `mcp install` (auto via wizard) | `~/.codex/config.toml` (`codex mcp add`) |
253
+ | **Goose** | `mcp install` (auto via wizard) | `~/.config/goose/config.yaml` |
254
+ | **Junie** | `mcp install` (auto via wizard) | `~/.junie/mcp/mcp.json` |
255
+
256
+ > **Already in `sinain onboard`** — step 6 of the advanced flow runs the same registration. Quickstart asks once if any MCP agent is detected.
185
257
 
186
- | File | Purpose |
258
+ - See [docs/MCP-INTEGRATION.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/MCP-INTEGRATION.md) for setup details, troubleshooting, and the manual `pclaude` / alternate `CLAUDE_CONFIG_DIR` recipe.
259
+ - See [docs/MCP-CAPABILITIES.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/MCP-CAPABILITIES.md) for what each tool enables, with example prompts and end-to-end recipes.
260
+
261
+ ## Deep Dives
262
+
263
+ | Topic | Doc |
187
264
  |---|---|
188
- | `index.ts` | Plugin implementation (hooks, tool, commands, curation service) |
189
- | `openclaw.plugin.json` | Plugin manifest (metadata, config schema, UI hints) |
265
+ | Knowledge System | [docs/knowledge-system.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/knowledge-system.md) |
266
+ | Knowledge API (HTTP) | [docs/KNOWLEDGE-API.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/KNOWLEDGE-API.md) |
267
+ | MCP Integration (setup) | [docs/MCP-INTEGRATION.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/MCP-INTEGRATION.md) |
268
+ | MCP Capabilities (tools + recipes) | [docs/MCP-CAPABILITIES.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/MCP-CAPABILITIES.md) |
269
+ | LongMemEval Audit | [docs/LONGMEMEVAL-AUDIT.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/LONGMEMEVAL-AUDIT.md) |
270
+ | Privacy Threat Model | [docs/privacy-protection-design.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/privacy-protection-design.md) |
271
+ | Full Configuration | [docs/CONFIGURATION.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/CONFIGURATION.md) |
272
+ | All Hotkeys | [docs/HOTKEYS.md](https://github.com/anthillnet/sinain-hud/blob/main/docs/HOTKEYS.md) |
273
+
274
+ ## Source
275
+
276
+ Full source, issues, and contributing guide: **[github.com/anthillnet/sinain-hud](https://github.com/anthillnet/sinain-hud)**
277
+
278
+ ## Contributing
279
+
280
+ See [CONTRIBUTING.md](https://github.com/anthillnet/sinain-hud/blob/main/CONTRIBUTING.md).
281
+
282
+ ## License
283
+
284
+ MIT
package/config-shared.js CHANGED
@@ -531,34 +531,185 @@ async function setupLocalGateway(existing) {
531
531
  };
532
532
  }
533
533
 
534
- export async function stepPrivacy(existing, label = "Privacy mode") {
534
+ /**
535
+ * Local mode: run everything on-device with Ollama + whisper.cpp.
536
+ *
537
+ * Returns null (skip) or { llm, vision } model names.
538
+ * When enabled, also checks Ollama is reachable and offers to pull models.
539
+ */
540
+ export async function stepLocalMode(existing, label = "Local mode (Ollama)") {
541
+ const currentEnabled = existing.SINAIN_LOCAL_MODE === "true";
542
+ const enable = guard(await p.confirm({
543
+ message: `${label} — run analysis + OCR on your machine, no cloud?`,
544
+ initialValue: currentEnabled,
545
+ }));
546
+
547
+ if (!enable) return null;
548
+
549
+ // Check Ollama
550
+ let ollamaOk = false;
551
+ let availableModels = [];
552
+ const s = p.spinner();
553
+ s.start("Checking Ollama...");
554
+ try {
555
+ const res = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(3000) });
556
+ if (res.ok) {
557
+ const data = await res.json();
558
+ availableModels = (data.models || []).map((m) => m.name);
559
+ ollamaOk = true;
560
+ s.stop(c.green(`Ollama running (${availableModels.length} models).`));
561
+ } else {
562
+ s.stop(c.yellow("Ollama responded but returned an error."));
563
+ }
564
+ } catch {
565
+ s.stop(c.yellow("Ollama not reachable at localhost:11434."));
566
+ }
567
+
568
+ if (!ollamaOk) {
569
+ p.note(
570
+ "Install and start Ollama first:\n" +
571
+ " brew install ollama && ollama serve\n" +
572
+ "Then re-run setup.",
573
+ "Ollama required",
574
+ );
575
+ const proceed = guard(await p.confirm({
576
+ message: "Continue anyway? (config will be saved, but won't work until Ollama runs)",
577
+ initialValue: false,
578
+ }));
579
+ if (!proceed) return null;
580
+ }
581
+
582
+ // LLM model (analysis + distillation)
583
+ const currentLlm = existing.SINAIN_LOCAL_LLM || "phi4-mini";
584
+ const llmOptions = [
585
+ { value: "phi4-mini", label: "phi4-mini", hint: "2.5 GB — fast, good quality (recommended)" },
586
+ { value: "gemma3:4b", label: "gemma3:4b", hint: "2.5 GB — Google, competitive quality" },
587
+ { value: "llama3.2:3b", label: "llama3.2:3b", hint: "2.0 GB — Meta, smallest" },
588
+ ];
589
+ // Add current model if it's custom and not in the list
590
+ if (!llmOptions.some((o) => o.value === currentLlm)) {
591
+ llmOptions.push({ value: currentLlm, label: currentLlm, hint: "currently configured" });
592
+ }
593
+ llmOptions.push({ value: "custom", label: "Custom", hint: "Enter any Ollama model name" });
594
+
595
+ let llm = guard(await p.select({
596
+ message: "LLM model (analysis + knowledge distillation)",
597
+ options: llmOptions,
598
+ initialValue: llmOptions.some((o) => o.value === currentLlm) ? currentLlm : "custom",
599
+ }));
600
+
601
+ if (llm === "custom") {
602
+ llm = guard(await p.text({
603
+ message: "Ollama model name for LLM",
604
+ placeholder: "model-name or model-name:tag",
605
+ validate: (val) => { if (!val) return "Model name required"; },
606
+ }));
607
+ }
608
+
609
+ // Vision model (screen OCR)
610
+ const currentVision = existing.SINAIN_LOCAL_VISION || "qwen2.5vl:7b";
611
+ const visionOptions = [
612
+ { value: "qwen2.5vl:7b", label: "qwen2.5vl:7b", hint: "4.7 GB — best OCR quality (recommended)" },
613
+ { value: "gemma4:e2b", label: "gemma4:e2b", hint: "5.2 GB — Google multimodal, new" },
614
+ { value: "llava:7b", label: "llava:7b", hint: "4.7 GB — general purpose vision" },
615
+ { value: "moondream", label: "moondream", hint: "1.7 GB — fastest, lower quality" },
616
+ ];
617
+ if (!visionOptions.some((o) => o.value === currentVision)) {
618
+ visionOptions.push({ value: currentVision, label: currentVision, hint: "currently configured" });
619
+ }
620
+ visionOptions.push({ value: "custom", label: "Custom", hint: "Enter any Ollama vision model" });
621
+
622
+ let vision = guard(await p.select({
623
+ message: "Vision model (screen OCR)",
624
+ options: visionOptions,
625
+ initialValue: visionOptions.some((o) => o.value === currentVision) ? currentVision : "custom",
626
+ }));
627
+
628
+ if (vision === "custom") {
629
+ vision = guard(await p.text({
630
+ message: "Ollama model name for vision",
631
+ placeholder: "model-name:tag",
632
+ validate: (val) => { if (!val) return "Model name required"; },
633
+ }));
634
+ }
635
+
636
+ // Offer to pull missing models
637
+ if (ollamaOk) {
638
+ const missing = [llm, vision].filter((m) => !availableModels.some((a) => a.startsWith(m)));
639
+ if (missing.length > 0) {
640
+ const pull = guard(await p.confirm({
641
+ message: `Pull missing models? (${missing.join(", ")})`,
642
+ initialValue: true,
643
+ }));
644
+ if (pull) {
645
+ for (const model of missing) {
646
+ const sp = p.spinner();
647
+ sp.start(`Pulling ${model}...`);
648
+ try {
649
+ execFileSync("ollama", ["pull", model], { stdio: "pipe", timeout: 600_000 });
650
+ sp.stop(c.green(`${model} pulled.`));
651
+ } catch {
652
+ sp.stop(c.yellow(`Failed to pull ${model} — pull manually: ollama pull ${model}`));
653
+ }
654
+ }
655
+ }
656
+ }
657
+ }
658
+
659
+ return { llm, vision };
660
+ }
661
+
662
+ export async function stepPrivacy(existing, label = "Privacy mode", { localModeEnabled = false } = {}) {
535
663
  const current = existing.PRIVACY_MODE || "standard";
536
- return guard(await p.select({
664
+
665
+ const options = [
666
+ {
667
+ value: "off",
668
+ label: "Off",
669
+ hint: "No filtering — screen text, credentials, everything sent to cloud",
670
+ },
671
+ {
672
+ value: "standard",
673
+ label: "Standard",
674
+ hint: "Auto-redacts cards, API keys, tokens before sending to cloud",
675
+ },
676
+ {
677
+ value: "strict",
678
+ label: "Strict",
679
+ hint: "Only summaries leave your machine, no raw screen text or audio",
680
+ },
681
+ ];
682
+
683
+ if (localModeEnabled) {
684
+ options.push({
685
+ value: "paranoid",
686
+ label: "Paranoid",
687
+ hint: "Zero cloud calls — all processing stays on-device via Ollama + Whisper",
688
+ });
689
+ } else {
690
+ options.push({
691
+ value: "paranoid",
692
+ label: "Paranoid",
693
+ hint: c.dim("Requires local mode — enable it first"),
694
+ });
695
+ }
696
+
697
+ const choice = guard(await p.select({
537
698
  message: label,
538
- options: [
539
- {
540
- value: "off",
541
- label: "Off",
542
- hint: "No filtering — screen text, credentials, everything sent to cloud",
543
- },
544
- {
545
- value: "standard",
546
- label: "Standard",
547
- hint: "Auto-redacts cards, API keys, tokens before sending to cloud",
548
- },
549
- {
550
- value: "strict",
551
- label: "Strict",
552
- hint: "Only summaries leave your machine, no raw screen text or audio",
553
- },
554
- {
555
- value: "paranoid",
556
- label: "Paranoid",
557
- hint: "Zero cloud calls — needs Whisper + Ollama installed or nothing works",
558
- },
559
- ],
560
- initialValue: current,
699
+ options,
700
+ initialValue: current === "paranoid" && !localModeEnabled ? "standard" : current,
561
701
  }));
702
+
703
+ if (choice === "paranoid" && !localModeEnabled) {
704
+ p.log.warn("Paranoid mode requires local mode (Ollama + Whisper). Enable local mode first.");
705
+ return guard(await p.select({
706
+ message: `${label} (local mode not enabled)`,
707
+ options: options.slice(0, 3),
708
+ initialValue: "standard",
709
+ }));
710
+ }
711
+
712
+ return choice;
562
713
  }
563
714
 
564
715
  export async function stepModel(existing, label = "AI model for HUD analysis") {
package/config.js CHANGED
@@ -6,7 +6,7 @@
6
6
  import * as p from "@clack/prompts";
7
7
  import {
8
8
  c, guard, readEnv, writeEnv, summarizeConfig, runHealthCheck,
9
- stepApiKey, stepTranscription, stepGateway, stepPrivacy, stepModel, stepAgent,
9
+ stepApiKey, stepTranscription, stepGateway, stepPrivacy, stepModel, stepAgent, stepLocalMode,
10
10
  ENV_PATH, IS_WINDOWS, HOME, PKG_DIR,
11
11
  } from "./config-shared.js";
12
12
  import fs from "fs";
@@ -16,6 +16,7 @@ import path from "path";
16
16
 
17
17
  const SECTIONS = [
18
18
  { value: "apikey", label: "API Key", hint: "OpenRouter API key" },
19
+ { value: "localmode", label: "Local Mode", hint: "Ollama + Whisper, zero cloud" },
19
20
  { value: "transcription", label: "Transcription", hint: "Cloud or local whisper" },
20
21
  { value: "model", label: "Model", hint: "AI model for analysis" },
21
22
  { value: "privacy", label: "Privacy", hint: "Standard / strict / paranoid" },
@@ -48,9 +49,26 @@ async function runSection(section, existing) {
48
49
  const model = await stepModel(existing);
49
50
  return { AGENT_MODEL: model };
50
51
  }
52
+ case "localmode": {
53
+ const result = await stepLocalMode(existing);
54
+ if (result) {
55
+ return {
56
+ SINAIN_LOCAL_MODE: "true",
57
+ SINAIN_LOCAL_LLM: result.llm,
58
+ SINAIN_LOCAL_VISION: result.vision,
59
+ };
60
+ }
61
+ return { SINAIN_LOCAL_MODE: "" };
62
+ }
51
63
  case "privacy": {
52
- const mode = await stepPrivacy(existing);
53
- return { PRIVACY_MODE: mode };
64
+ const localModeEnabled = existing.SINAIN_LOCAL_MODE === "true";
65
+ const mode = await stepPrivacy(existing, "Privacy mode", { localModeEnabled });
66
+ const vars = { PRIVACY_MODE: mode };
67
+ if (mode === "paranoid" && localModeEnabled) {
68
+ vars.PRIVACY_OCR_AGENT_GATEWAY = "redacted";
69
+ vars.PRIVACY_AUDIO_AGENT_GATEWAY = "redacted";
70
+ }
71
+ return vars;
54
72
  }
55
73
  case "gateway": {
56
74
  return await stepGateway(existing);
package/launcher.js CHANGED
@@ -106,6 +106,20 @@ async function main() {
106
106
  // Load user config
107
107
  loadUserEnv();
108
108
 
109
+ // Propagate unified local mode config to component-level vars
110
+ if (process.env.SINAIN_LOCAL_MODE === "true") {
111
+ const llm = process.env.SINAIN_LOCAL_LLM || "phi4-mini";
112
+ const vision = process.env.SINAIN_LOCAL_VISION || "qwen2.5vl:7b";
113
+ if (!process.env.LOCAL_VISION_ENABLED) process.env.LOCAL_VISION_ENABLED = "true";
114
+ if (!process.env.LOCAL_VISION_MODEL) process.env.LOCAL_VISION_MODEL = vision;
115
+ if (!process.env.ANALYSIS_PROVIDER) process.env.ANALYSIS_PROVIDER = "ollama";
116
+ if (!process.env.ANALYSIS_MODEL) process.env.ANALYSIS_MODEL = llm;
117
+ if (!process.env.TRANSCRIPTION_BACKEND) process.env.TRANSCRIPTION_BACKEND = "local";
118
+ if (!process.env.SINAIN_FAST_MODEL) process.env.SINAIN_FAST_MODEL = `ollama/${llm}`;
119
+ if (!process.env.SINAIN_SMART_MODEL) process.env.SINAIN_SMART_MODEL = `ollama/${llm}`;
120
+ log(`${MAGENTA}LOCAL MODE${RESET} — LLM: ${llm}, Vision: ${vision}`);
121
+ }
122
+
109
123
  // Ensure Ollama is running (if local vision enabled)
110
124
  if (process.env.LOCAL_VISION_ENABLED === "true") {
111
125
  await ensureOllama();
@@ -162,10 +176,11 @@ async function main() {
162
176
  color: CYAN,
163
177
  });
164
178
 
165
- // Health check
166
- const healthy = await healthCheck("http://localhost:9500/health", 20);
179
+ // Health check (local mode needs longer — cold model load + startup distillation)
180
+ const healthTimeout = process.env.SINAIN_LOCAL_MODE === "true" ? 45 : 20;
181
+ const healthy = await healthCheck("http://localhost:9500/health", healthTimeout);
167
182
  if (!healthy) {
168
- fail("sinain-core did not become healthy after 20s");
183
+ fail(`sinain-core did not become healthy after ${healthTimeout}s`);
169
184
  }
170
185
  ok("sinain-core healthy on :9500");
171
186
 
package/onboard.js CHANGED
@@ -8,8 +8,8 @@ import fs from "fs";
8
8
  import path from "path";
9
9
  import { execFileSync } from "child_process";
10
10
  import {
11
- c, guard, maskKey, readEnv, writeEnv, writeAgentsConfig, summarizeConfig, runHealthCheck,
12
- stepApiKey, stepTranscription, stepGateway, stepPrivacy, stepModel,
11
+ c, guard, cmdExists, maskKey, readEnv, writeEnv, writeAgentsConfig, summarizeConfig, runHealthCheck,
12
+ stepApiKey, stepTranscription, stepGateway, stepPrivacy, stepModel, stepLocalMode,
13
13
  HOME, SINAIN_DIR, ENV_PATH, PKG_DIR, IS_WINDOWS, IS_MAC,
14
14
  } from "./config-shared.js";
15
15
  import { stepMcpInstall, detectMcpAgents } from "./mcp-register.js";
@@ -130,6 +130,11 @@ export async function runOnboard(args = {}) {
130
130
  label: "QuickStart",
131
131
  hint: "Get running in 2 minutes. Configure details later.",
132
132
  },
133
+ {
134
+ value: "local",
135
+ label: "Local / Paranoid",
136
+ hint: "Fully offline — Ollama + Whisper, zero cloud calls.",
137
+ },
133
138
  {
134
139
  value: "advanced",
135
140
  label: "Advanced",
@@ -139,7 +144,7 @@ export async function runOnboard(args = {}) {
139
144
  initialValue: "quickstart",
140
145
  }));
141
146
 
142
- const totalSteps = flow === "quickstart" ? 2 : 6;
147
+ const totalSteps = flow === "quickstart" ? 2 : flow === "local" ? 4 : 6;
143
148
 
144
149
  // ── Collect vars ────────────────────────────────────────────────────────
145
150
 
@@ -149,12 +154,130 @@ export async function runOnboard(args = {}) {
149
154
  // complete so we don't churn ~/.sinain/agents.json on every prompt.
150
155
  let agentsPatch = {};
151
156
 
152
- // Step 1: API key (both flows)
153
- const apiKey = await stepApiKey(base, `[1/${totalSteps}] OpenRouter API key`);
154
- vars.OPENROUTER_API_KEY = apiKey;
155
- p.log.success("API key saved.");
157
+ // Step 1: API key (quickstart + advanced only — local mode skips cloud)
158
+ if (flow !== "local") {
159
+ const apiKey = await stepApiKey(base, `[1/${totalSteps}] OpenRouter API key`);
160
+ vars.OPENROUTER_API_KEY = apiKey;
161
+ p.log.success("API key saved.");
162
+ }
163
+
164
+ if (flow === "local") {
165
+ // ── Local / Paranoid flow ─────────────────────────────────────────────
166
+ // Step 1: Local models (Ollama)
167
+ const localResult = await stepLocalMode(base, `[1/${totalSteps}] Local models`);
168
+ if (localResult) {
169
+ vars.SINAIN_LOCAL_MODE = "true";
170
+ vars.SINAIN_LOCAL_LLM = localResult.llm;
171
+ vars.SINAIN_LOCAL_VISION = localResult.vision;
172
+ p.log.success(`LLM: ${localResult.llm}, Vision: ${localResult.vision}`);
173
+ } else {
174
+ p.log.warn("Local mode cancelled — switching to QuickStart defaults.");
175
+ vars.TRANSCRIPTION_BACKEND = "openrouter";
176
+ vars.PRIVACY_MODE = "standard";
177
+ vars.AGENT_MODEL = "google/gemini-2.5-flash-lite";
178
+ }
179
+
180
+ // Step 2: Whisper setup (if local mode enabled)
181
+ if (vars.SINAIN_LOCAL_MODE === "true") {
182
+ vars.TRANSCRIPTION_BACKEND = "local";
183
+ const hasWhisper = !IS_WINDOWS && cmdExists("whisper-cli");
184
+ if (hasWhisper) {
185
+ p.log.success(`[2/${totalSteps}] whisper-cli found — local transcription enabled.`);
186
+ } else if (IS_MAC) {
187
+ const install = guard(await p.confirm({
188
+ message: `[2/${totalSteps}] whisper-cli not found. Install via Homebrew?`,
189
+ initialValue: true,
190
+ }));
191
+ if (install) {
192
+ const s = p.spinner();
193
+ s.start("Installing whisper-cpp...");
194
+ try {
195
+ execFileSync("brew", ["install", "whisper-cpp"], { stdio: "pipe" });
196
+ s.stop(c.green("whisper-cpp installed."));
197
+ } catch {
198
+ s.stop(c.yellow("Install failed — audio transcription won't work offline."));
199
+ }
200
+ }
201
+ }
202
+ // Check whisper model
203
+ const modelDir = path.join(HOME, "models");
204
+ const modelPath = path.join(modelDir, "ggml-large-v3-turbo.bin");
205
+ if (fs.existsSync(modelPath)) {
206
+ vars.LOCAL_WHISPER_MODEL = modelPath;
207
+ p.log.info(`Whisper model: ${c.dim(modelPath)}`);
208
+ } else {
209
+ const download = guard(await p.confirm({
210
+ message: "Download Whisper model (~1.5 GB)?",
211
+ initialValue: true,
212
+ }));
213
+ if (download) {
214
+ const s = p.spinner();
215
+ s.start("Downloading Whisper model...");
216
+ try {
217
+ fs.mkdirSync(modelDir, { recursive: true });
218
+ execFileSync("curl", [
219
+ "-L", "--progress-bar",
220
+ "-o", modelPath,
221
+ "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin",
222
+ ], { stdio: "inherit" });
223
+ s.stop(c.green("Model downloaded."));
224
+ vars.LOCAL_WHISPER_MODEL = modelPath;
225
+ } catch {
226
+ s.stop(c.yellow("Download failed. Run manually later."));
227
+ }
228
+ }
229
+ }
230
+
231
+ // Step 3: Privacy — default to paranoid since user chose local mode
232
+ vars.PRIVACY_MODE = "paranoid";
233
+ const privacy = await stepPrivacy(base, `[3/${totalSteps}] Privacy mode`, { localModeEnabled: true });
234
+ vars.PRIVACY_MODE = privacy;
235
+ p.log.success(`Privacy: ${privacy}.`);
236
+
237
+ // Privacy overrides for escalation (redacted OCR+audio in escalation)
238
+ if (privacy === "paranoid") {
239
+ vars.PRIVACY_OCR_AGENT_GATEWAY = "redacted";
240
+ vars.PRIVACY_AUDIO_AGENT_GATEWAY = "redacted";
241
+ }
242
+ }
243
+
244
+ // Step 4: Gateway (optional — works with local mode too)
245
+ const hasExistingGateway = (() => {
246
+ try {
247
+ const agentsPath = path.join(SINAIN_DIR, "agents.json");
248
+ if (!fs.existsSync(agentsPath)) return false;
249
+ const cfg = JSON.parse(fs.readFileSync(agentsPath, "utf-8"));
250
+ return !!cfg?.profiles?.openclaw;
251
+ } catch { return false; }
252
+ })();
253
+ const enableGateway = guard(await p.confirm({
254
+ message: `[4/${totalSteps}] Enable OpenClaw gateway? (escalation agent for deeper analysis)`,
255
+ initialValue: hasExistingGateway,
256
+ }));
257
+ if (enableGateway) {
258
+ const gatewayResult = await stepGateway(base, "OpenClaw gateway");
259
+ Object.assign(vars, gatewayResult.envVars);
260
+ Object.assign(agentsPatch, gatewayResult.agentsPatch);
261
+ } else {
262
+ agentsPatch.openclawProfile = null;
263
+ }
264
+ agentsPatch.default = base.SINAIN_AGENT || "claude";
156
265
 
157
- if (flow === "quickstart") {
266
+ p.note(
267
+ [
268
+ `Local mode: ${vars.SINAIN_LOCAL_MODE === "true" ? c.green("enabled") : "disabled"}`,
269
+ vars.SINAIN_LOCAL_LLM ? ` LLM: ${vars.SINAIN_LOCAL_LLM}` : null,
270
+ vars.SINAIN_LOCAL_VISION ? ` Vision: ${vars.SINAIN_LOCAL_VISION}` : null,
271
+ `Transcription: ${vars.TRANSCRIPTION_BACKEND}`,
272
+ `Privacy: ${vars.PRIVACY_MODE}`,
273
+ `OpenClaw gateway: ${enableGateway ? "enabled" : "disabled"}`,
274
+ "",
275
+ `Start with: ./start.sh --paranoid`,
276
+ `Change later: sinain config`,
277
+ ].filter(Boolean).join("\n"),
278
+ "Local mode summary",
279
+ );
280
+ } else if (flow === "quickstart") {
158
281
  // QuickStart: sensible defaults + a single opt-in question for OpenClaw.
159
282
  // Gateway integration is off by default; users who want it run Advanced
160
283
  // (or answer Yes here, which then walks them through stepGateway).
@@ -267,35 +390,14 @@ export async function runOnboard(args = {}) {
267
390
  }
268
391
  }
269
392
 
270
- // If Ollama is installed, offer to pull a local LLM for paranoid-mode
271
- // analysis. Mirrors the whisper download pattern — auto-acquire optional,
272
- // user can `ollama pull <model>` manually later if they skip here.
273
- let ollamaInstalled = false;
274
- try {
275
- execFileSync("ollama", ["--version"], { stdio: "ignore" });
276
- ollamaInstalled = true;
277
- } catch { /* ollama not on PATH */ }
278
-
279
- if (ollamaInstalled) {
280
- const pullOllama = guard(await p.confirm({
281
- message: "Pull an Ollama model for paranoid-mode analysis (~4.7 GB for llava)?",
282
- initialValue: true,
283
- }));
284
- if (pullOllama) {
285
- const modelName = guard(await p.text({
286
- message: "Ollama model to pull",
287
- placeholder: "llava",
288
- defaultValue: "llava",
289
- }));
290
- const s = p.spinner();
291
- s.start(`Pulling ${modelName} via Ollama (this can take several minutes)...`);
292
- try {
293
- execFileSync("ollama", ["pull", modelName], { stdio: "inherit" });
294
- s.stop(c.green(`Pulled ${modelName}.`));
295
- } catch {
296
- s.stop(c.yellow(`Pull failed. Run \`ollama pull ${modelName}\` manually later.`));
297
- }
298
- }
393
+ // Offer local mode (Ollama) — enables paranoid privacy
394
+ const localResult = await stepLocalMode(base, "Local mode (Ollama)");
395
+ const localModeEnabled = !!localResult;
396
+ if (localResult) {
397
+ vars.SINAIN_LOCAL_MODE = "true";
398
+ vars.SINAIN_LOCAL_LLM = localResult.llm;
399
+ vars.SINAIN_LOCAL_VISION = localResult.vision;
400
+ p.log.success(`Local mode: LLM=${localResult.llm}, Vision=${localResult.vision}`);
299
401
  }
300
402
 
301
403
  // OpenClaw gateway is opt-in: most users run sinain in standalone mode
@@ -333,8 +435,12 @@ export async function runOnboard(args = {}) {
333
435
  p.log.info("Standalone mode (no gateway).");
334
436
  }
335
437
 
336
- const privacy = await stepPrivacy(base, "[4/6] Privacy mode");
438
+ const privacy = await stepPrivacy(base, "[4/6] Privacy mode", { localModeEnabled });
337
439
  vars.PRIVACY_MODE = privacy;
440
+ if (privacy === "paranoid" && localModeEnabled) {
441
+ vars.PRIVACY_OCR_AGENT_GATEWAY = "redacted";
442
+ vars.PRIVACY_AUDIO_AGENT_GATEWAY = "redacted";
443
+ }
338
444
  p.log.success(`Privacy: ${privacy}.`);
339
445
 
340
446
  const model = await stepModel(base, "[5/6] AI model for HUD analysis");
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@geravant/sinain",
3
- "version": "1.24.0",
3
+ "version": "1.25.0",
4
4
  "description": "Context OS — ambient intelligence for builders. Captures screen + audio, distills into a private knowledge graph, accessible from MCP, web UI, and HUD overlay.",
5
5
  "type": "module",
6
6
  "bin": {
package/sense_client/ollama_vision.py CHANGED
@@ -39,7 +39,7 @@ class OllamaVision:
39
39
  self,
40
40
  model: str = "llava",
41
41
  base_url: str = "http://localhost:11434",
42
- timeout: float = 10.0,
42
+ timeout: float = 30.0,
43
43
  max_tokens: int = 200,
44
44
  ):
45
45
  self.model = model
package/sinain-core/src/agent/analyzer.ts CHANGED
@@ -366,7 +366,8 @@ async function callOllama(
366
366
  ): Promise<AgentResult> {
367
367
  const start = Date.now();
368
368
  const controller = new AbortController();
369
- const timeout = setTimeout(() => controller.abort(), config.timeout);
369
+ // Local Ollama models need more time than cloud APIs (cold start + generation)
370
+ const timeout = setTimeout(() => controller.abort(), Math.max(config.timeout, 45_000));
370
371
 
371
372
  try {
372
373
  const imageB64List = (images || []).map((img) => img.data);
package/sinain-core/src/config.ts CHANGED
@@ -186,6 +186,25 @@ export function loadConfig(): CoreConfig {
186
186
  gainDb: intEnv("MIC_GAIN_DB", 0),
187
187
  };
188
188
 
189
+ // ── Local mode: unified config ──────────────────────────────────────────
190
+ // SINAIN_LOCAL_MODE=true auto-derives all component config from two vars:
191
+ // SINAIN_LOCAL_LLM=phi4-mini → analyzer + distiller
192
+ // SINAIN_LOCAL_VISION=qwen2.5vl:7b → sense_client (propagated via start.sh)
193
+ // Must run BEFORE transcriptionConfig / analysisConfig are read.
194
+ const localMode = boolEnv("SINAIN_LOCAL_MODE", false);
195
+ if (localMode) {
196
+ const localLlm = env("SINAIN_LOCAL_LLM", "phi4-mini");
197
+ const localVision = env("SINAIN_LOCAL_VISION", "qwen2.5vl:7b");
198
+ if (!process.env.ANALYSIS_PROVIDER) process.env.ANALYSIS_PROVIDER = "ollama";
199
+ if (!process.env.ANALYSIS_MODEL) process.env.ANALYSIS_MODEL = localLlm;
200
+ if (!process.env.ANALYSIS_VISION_MODEL) process.env.ANALYSIS_VISION_MODEL = localLlm;
201
+ if (!process.env.TRANSCRIPTION_BACKEND) process.env.TRANSCRIPTION_BACKEND = "local";
202
+ if (!process.env.LOCAL_VISION_ENABLED) process.env.LOCAL_VISION_ENABLED = "true";
203
+ if (!process.env.LOCAL_VISION_MODEL) process.env.LOCAL_VISION_MODEL = localVision;
204
+ if (!process.env.SINAIN_FAST_MODEL) process.env.SINAIN_FAST_MODEL = `ollama/${localLlm}`;
205
+ if (!process.env.SINAIN_SMART_MODEL) process.env.SINAIN_SMART_MODEL = `ollama/${localLlm}`;
206
+ }
207
+
189
208
  const transcriptionConfig: TranscriptionConfig = {
190
209
  backend: env("TRANSCRIPTION_BACKEND", "openrouter") as TranscriptionConfig["backend"],
191
210
  openrouterApiKey: env("OPENROUTER_API_KEY", ""),