@geravant/sinain 1.9.0 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +17 -14
- package/HEARTBEAT.md +1 -1
- package/README.md +4 -7
- package/index.ts +1 -3
- package/package.json +1 -1
- package/sense_client/ocr.py +6 -3
- package/sinain-agent/CLAUDE.md +50 -1
- package/sinain-agent/run.sh +18 -8
- package/sinain-core/src/agent/analyzer.ts +31 -56
- package/sinain-core/src/agent/loop.ts +11 -10
- package/sinain-core/src/config.ts +17 -14
- package/sinain-core/src/index.ts +297 -26
- package/sinain-core/src/learning/local-curation.ts +373 -0
- package/sinain-core/src/overlay/commands.ts +9 -0
- package/sinain-core/src/overlay/ws-handler.ts +3 -0
- package/sinain-core/src/server.ts +197 -0
- package/sinain-core/src/types.ts +13 -10
- package/sinain-knowledge/curation/engine.ts +0 -17
- package/sinain-knowledge/protocol/heartbeat.md +1 -1
- package/sinain-mcp-server/index.ts +38 -24
- package/sinain-memory/__pycache__/common.cpython-312.pyc +0 -0
- package/sinain-memory/__pycache__/knowledge_integrator.cpython-312.pyc +0 -0
- package/sinain-memory/__pycache__/session_distiller.cpython-312.pyc +0 -0
- package/sinain-memory/__pycache__/triplestore.cpython-312.pyc +0 -0
- package/sinain-memory/eval/retrieval_benchmark.jsonl +12 -0
- package/sinain-memory/eval/retrieval_evaluator.py +186 -0
- package/sinain-memory/graph_query.py +34 -1
- package/sinain-memory/knowledge_integrator.py +54 -0
- package/sinain-memory/triplestore.py +76 -5
- package/sinain-agent/.env.example +0 -23
- package/sinain-memory/git_backup.sh +0 -19
package/.env.example
CHANGED
|
@@ -3,8 +3,18 @@
|
|
|
3
3
|
# The launcher reads this file on every start. sinain-core and sinain-agent
|
|
4
4
|
# inherit all vars via the launcher's process environment.
|
|
5
5
|
|
|
6
|
-
# ──
|
|
6
|
+
# ── Context Analysis (HUD summarizer) ────────────────────────────────────────
|
|
7
|
+
# Provider: openrouter (cloud, needs API key) or ollama (local, free)
|
|
8
|
+
ANALYSIS_PROVIDER=openrouter
|
|
9
|
+
ANALYSIS_MODEL=google/gemini-2.5-flash-lite
|
|
10
|
+
# ANALYSIS_VISION_MODEL=google/gemini-2.5-flash # auto-upgrade for image ticks
|
|
11
|
+
# ANALYSIS_ENDPOINT= # default per provider; override for custom
|
|
12
|
+
# ANALYSIS_API_KEY= # uses OPENROUTER_API_KEY if not set
|
|
13
|
+
# ANALYSIS_FALLBACK_MODELS=google/gemini-2.5-flash,anthropic/claude-3.5-haiku
|
|
14
|
+
|
|
15
|
+
# ── API Keys ─────────────────────────────────────────────────────────────────
|
|
7
16
|
OPENROUTER_API_KEY= # get one free at https://openrouter.ai
|
|
17
|
+
# used by context analysis + transcription
|
|
8
18
|
|
|
9
19
|
# ── Privacy ──────────────────────────────────────────────────────────────────
|
|
10
20
|
PRIVACY_MODE=standard # off | standard | strict | paranoid
|
|
@@ -20,6 +30,11 @@ SINAIN_CORE_URL=http://localhost:9500
|
|
|
20
30
|
SINAIN_POLL_INTERVAL=5 # seconds between escalation polls
|
|
21
31
|
SINAIN_HEARTBEAT_INTERVAL=900 # seconds between heartbeat ticks (15 min)
|
|
22
32
|
SINAIN_WORKSPACE=~/.openclaw/workspace # knowledge files, curation scripts, playbook
|
|
33
|
+
# SINAIN_ALLOWED_TOOLS=mcp__sinain # MCP tools auto-approved for bare agent
|
|
34
|
+
SINAIN_AGENT_MAX_TURNS=5 # max tool-use turns for escalation responses
|
|
35
|
+
SINAIN_SPAWN_MAX_TURNS=25 # max tool-use turns for spawn tasks (Shift+Enter)
|
|
36
|
+
# auto-derived from mcp-config.json if unset
|
|
37
|
+
# format: mcp__<server> (all) | mcp__<server>__<tool> (specific)
|
|
23
38
|
|
|
24
39
|
# ── Escalation ───────────────────────────────────────────────────────────────
|
|
25
40
|
ESCALATION_MODE=rich # off | selective | focus | rich
|
|
@@ -27,7 +42,7 @@ ESCALATION_MODE=rich # off | selective | focus | rich
|
|
|
27
42
|
# selective: score-based (errors, questions trigger it)
|
|
28
43
|
# focus: always escalate every tick
|
|
29
44
|
# rich: always escalate with maximum context
|
|
30
|
-
ESCALATION_COOLDOWN_MS=
|
|
45
|
+
# ESCALATION_COOLDOWN_MS=10000
|
|
31
46
|
# ESCALATION_TRANSPORT=auto # ws | http | auto
|
|
32
47
|
# auto = WS when gateway connected, HTTP fallback
|
|
33
48
|
# http = bare agent only (no gateway)
|
|
@@ -70,18 +85,6 @@ TRANSCRIPTION_LANGUAGE=en-US
|
|
|
70
85
|
# LOCAL_WHISPER_MODEL=~/models/ggml-large-v3-turbo.bin
|
|
71
86
|
# LOCAL_WHISPER_TIMEOUT_MS=15000
|
|
72
87
|
|
|
73
|
-
# ── Local Agent Loop ─────────────────────────────────────────────────────────
|
|
74
|
-
AGENT_ENABLED=true
|
|
75
|
-
AGENT_MODEL=google/gemini-2.5-flash-lite
|
|
76
|
-
# AGENT_FALLBACK_MODELS=google/gemini-2.5-flash,anthropic/claude-3.5-haiku
|
|
77
|
-
AGENT_MAX_TOKENS=300
|
|
78
|
-
AGENT_TEMPERATURE=0.3
|
|
79
|
-
AGENT_PUSH_TO_FEED=true
|
|
80
|
-
AGENT_DEBOUNCE_MS=3000
|
|
81
|
-
AGENT_MAX_INTERVAL_MS=30000
|
|
82
|
-
AGENT_COOLDOWN_MS=10000
|
|
83
|
-
AGENT_MAX_AGE_MS=120000 # context window lookback (2 min)
|
|
84
|
-
|
|
85
88
|
# ── OpenClaw / NemoClaw Gateway ──────────────────────────────────────────────
|
|
86
89
|
# Leave blank to run without a gateway (bare agent mode).
|
|
87
90
|
# The setup wizard fills these in if you have an OpenClaw gateway.
|
package/HEARTBEAT.md
CHANGED
|
@@ -59,4 +59,4 @@ SINAIN_BACKUP_REPO=<git-url> npx sinain
|
|
|
59
59
|
- Token printed at end (or visible in Brev dashboard → Gateway Token)
|
|
60
60
|
- Mac side: `./setup-nemoclaw.sh` → 5 prompts → overlay starts
|
|
61
61
|
|
|
62
|
-
Memory is
|
|
62
|
+
Memory is backed up via knowledge snapshots to `~/.sinain/knowledge-snapshots/`. New instances restore instantly via `SINAIN_BACKUP_REPO`.
|
package/README.md
CHANGED
|
@@ -29,13 +29,12 @@ Five lifecycle hooks, one tool, four commands, and a background service:
|
|
|
29
29
|
|
|
30
30
|
| Tool | Purpose |
|
|
31
31
|
|---|---|
|
|
32
|
-
| `sinain_heartbeat_tick` | Executes all heartbeat mechanical work (
|
|
32
|
+
| `sinain_heartbeat_tick` | Executes all heartbeat mechanical work (signal analysis, insight synthesis, log writing). Returns structured JSON with results, recommended actions, and Telegram output. |
|
|
33
33
|
|
|
34
34
|
The heartbeat tool accepts `{ sessionSummary: string, idle: boolean }` and runs:
|
|
35
|
-
1. `
|
|
36
|
-
2. `uv run python3 sinain-memory/
|
|
37
|
-
3.
|
|
38
|
-
4. Writes log entry to `memory/playbook-logs/YYYY-MM-DD.jsonl`
|
|
35
|
+
1. `uv run python3 sinain-memory/signal_analyzer.py` (60s timeout)
|
|
36
|
+
2. `uv run python3 sinain-memory/insight_synthesizer.py` (60s timeout)
|
|
37
|
+
3. Writes log entry to `memory/playbook-logs/YYYY-MM-DD.jsonl`
|
|
39
38
|
|
|
40
39
|
### Commands
|
|
41
40
|
|
|
@@ -114,8 +113,6 @@ Also ensures these directories exist:
|
|
|
114
113
|
- `memory/`, `memory/playbook-archive/`, `memory/playbook-logs/`
|
|
115
114
|
- `memory/eval-logs/`, `memory/eval-reports/`
|
|
116
115
|
|
|
117
|
-
The `git_backup.sh` script is automatically made executable (chmod 755) after sync.
|
|
118
|
-
|
|
119
116
|
After syncing modules, the plugin generates `memory/sinain-playbook-effective.md` — a merged view of active module patterns (sorted by priority) plus the base playbook.
|
|
120
117
|
|
|
121
118
|
## Heartbeat Compliance Validation
|
package/index.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* - Strips <private> tags from tool results before persistence
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync, statSync,
|
|
11
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync, statSync, copyFileSync } from "node:fs";
|
|
12
12
|
import { join, dirname } from "node:path";
|
|
13
13
|
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
|
14
14
|
|
|
@@ -358,8 +358,6 @@ export default function sinainHudPlugin(api: OpenClawPluginApi): void {
|
|
|
358
358
|
const memorySource = cfg.memoryPath ? api.resolvePath(cfg.memoryPath) : undefined;
|
|
359
359
|
if (memorySource) {
|
|
360
360
|
store.deployDir(memorySource, "sinain-memory");
|
|
361
|
-
const gbPath = join(workspaceDir, "sinain-memory", "git_backup.sh");
|
|
362
|
-
if (existsSync(gbPath)) try { chmodSync(gbPath, 0o755); } catch {}
|
|
363
361
|
}
|
|
364
362
|
|
|
365
363
|
const modulesSource = cfg.modulesPath ? api.resolvePath(cfg.modulesPath) : undefined;
|
package/package.json
CHANGED
package/sense_client/ocr.py
CHANGED
|
@@ -158,8 +158,10 @@ class VisionOCR:
|
|
|
158
158
|
|
|
159
159
|
# Execute
|
|
160
160
|
handler = VNImageRequestHandler.alloc().initWithCGImage_options_(cg_image, None)
|
|
161
|
-
|
|
162
|
-
|
|
161
|
+
result = handler.performRequests_error_([request], objc.nil)
|
|
162
|
+
# PyObjC may return (bool, error) tuple or just bool depending on version
|
|
163
|
+
success = result[0] if isinstance(result, tuple) else result
|
|
164
|
+
if not success:
|
|
163
165
|
return OCRResult(text="", confidence=0, word_count=0)
|
|
164
166
|
|
|
165
167
|
results = request.results()
|
|
@@ -172,7 +174,8 @@ class VisionOCR:
|
|
|
172
174
|
|
|
173
175
|
for observation in results:
|
|
174
176
|
candidate = observation.topCandidates_(1)
|
|
175
|
-
|
|
177
|
+
# PyObjC may return bool instead of list depending on version
|
|
178
|
+
if not candidate or isinstance(candidate, bool):
|
|
176
179
|
continue
|
|
177
180
|
text = candidate[0].string()
|
|
178
181
|
conf = candidate[0].confidence()
|
package/sinain-agent/CLAUDE.md
CHANGED
|
@@ -47,7 +47,6 @@ When responding to escalations:
|
|
|
47
47
|
|
|
48
48
|
1. Call `sinain_heartbeat_tick` with a brief session summary
|
|
49
49
|
2. The tool runs the full pipeline automatically:
|
|
50
|
-
- Git backup of memory directory
|
|
51
50
|
- Signal analysis (detects opportunities from session patterns)
|
|
52
51
|
- **Session distillation** — fetches new feed items from sinain-core, distills patterns/learnings
|
|
53
52
|
- **Knowledge integration** — updates playbook (working memory) and knowledge graph (long-term memory)
|
|
@@ -56,6 +55,56 @@ When responding to escalations:
|
|
|
56
55
|
4. Optionally call `sinain_get_knowledge` to review the portable knowledge document
|
|
57
56
|
5. Optionally call `sinain_get_feedback` to review recent escalation scores
|
|
58
57
|
|
|
58
|
+
## Knowledge System
|
|
59
|
+
|
|
60
|
+
Knowledge is stored in a **dual-database** architecture with two SQLite triplestore databases:
|
|
61
|
+
|
|
62
|
+
| Database | Path | Written by |
|
|
63
|
+
|----------|------|------------|
|
|
64
|
+
| **Local** | `~/.sinain/memory/knowledge-graph.db` | `LocalCurationService` (session distillation on shutdown, periodic curation every 30 min) |
|
|
65
|
+
| **Workspace** | `~/.openclaw/workspace/memory/knowledge-graph.db` | Heartbeat curation scripts (`sinain_heartbeat_tick`) |
|
|
66
|
+
|
|
67
|
+
### Knowledge Tools
|
|
68
|
+
|
|
69
|
+
| Tool | What it does |
|
|
70
|
+
|------|-------------|
|
|
71
|
+
| `sinain_get_knowledge` | Read the portable knowledge document (playbook + top facts) from workspace |
|
|
72
|
+
| `sinain_knowledge_query` | Query facts by entity/keyword — queries **both** DBs via sinain-core API |
|
|
73
|
+
| `sinain_distill_session` | Explicitly distill current session into knowledge updates |
|
|
74
|
+
|
|
75
|
+
### HTTP Knowledge API (sinain-core, port 9500)
|
|
76
|
+
|
|
77
|
+
These endpoints query **both** databases and merge results:
|
|
78
|
+
|
|
79
|
+
| Endpoint | Purpose |
|
|
80
|
+
|----------|---------|
|
|
81
|
+
| `GET /knowledge` | Portable knowledge document |
|
|
82
|
+
| `GET /knowledge/facts?entities=X&max=N` | Query facts by keyword tags |
|
|
83
|
+
| `GET /knowledge/entities?max=N` | List all entities with attributes |
|
|
84
|
+
| `GET /knowledge/export?domain=X&max=N` | Export facts as portable JSON |
|
|
85
|
+
| `POST /knowledge/import` | Import facts (deduplicates automatically) |
|
|
86
|
+
| `GET /knowledge/ui` | Web UI for browsing/managing knowledge |
|
|
87
|
+
|
|
88
|
+
### How Knowledge Flows
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
Session (screen + audio) → LocalCurationService → Local DB
|
|
92
|
+
↓ (queried together)
|
|
93
|
+
Heartbeat tick → curation scripts ──────────→ Workspace DB
|
|
94
|
+
↓
|
|
95
|
+
Knowledge API (localhost:9500) ← merges both DBs ← queries
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
- **Local DB** gets real-time session knowledge (audio transcripts, screen patterns, German lessons, etc.)
|
|
99
|
+
- **Workspace DB** gets heartbeat-curated knowledge (playbook patterns, feedback analysis)
|
|
100
|
+
- The Knowledge API merges both — use `sinain_knowledge_query` for combined results
|
|
101
|
+
- Facts have confidence decay (60-day half-life) — reinforcement resets the clock
|
|
102
|
+
- Export/import via `/knowledge/export` → `/knowledge/import` enables cross-instance transfer
|
|
103
|
+
|
|
104
|
+
### Using Knowledge in Escalation Responses
|
|
105
|
+
|
|
106
|
+
When responding to escalations, call `sinain_knowledge_query` with relevant entities to enrich your response with long-term knowledge. Example: if the user is working on German grammar, query `sinain_knowledge_query({ entities: ["german", "grammar"] })` to retrieve previously learned patterns.
|
|
107
|
+
|
|
59
108
|
## Spawning Background Tasks
|
|
60
109
|
|
|
61
110
|
When an escalation suggests deeper research would help:
|
package/sinain-agent/run.sh
CHANGED
|
@@ -5,7 +5,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
5
5
|
|
|
6
6
|
# Load .env as fallback — does NOT override vars already in the environment
|
|
7
7
|
# (e.g. vars set by the launcher from ~/.sinain/.env)
|
|
8
|
-
|
|
8
|
+
# Load project root .env (single config for all subsystems)
|
|
9
|
+
ENV_FILE="$SCRIPT_DIR/../.env"
|
|
10
|
+
if [ -f "$ENV_FILE" ]; then
|
|
9
11
|
while IFS='=' read -r key val; do
|
|
10
12
|
# Skip comments and blank lines
|
|
11
13
|
[[ -z "$key" || "$key" =~ ^[[:space:]]*# ]] && continue
|
|
@@ -19,7 +21,7 @@ if [ -f "$SCRIPT_DIR/.env" ]; then
|
|
|
19
21
|
if [ -z "${!key+x}" ]; then
|
|
20
22
|
export "$key=$val"
|
|
21
23
|
fi
|
|
22
|
-
done < "$
|
|
24
|
+
done < "$ENV_FILE"
|
|
23
25
|
fi
|
|
24
26
|
|
|
25
27
|
MCP_CONFIG="${MCP_CONFIG:-$SCRIPT_DIR/mcp-config.json}"
|
|
@@ -28,6 +30,8 @@ POLL_INTERVAL="${SINAIN_POLL_INTERVAL:-2}"
|
|
|
28
30
|
HEARTBEAT_INTERVAL="${SINAIN_HEARTBEAT_INTERVAL:-900}" # 15 minutes
|
|
29
31
|
AGENT="${SINAIN_AGENT:-claude}"
|
|
30
32
|
WORKSPACE="${SINAIN_WORKSPACE:-$HOME/.openclaw/workspace}"
|
|
33
|
+
AGENT_MAX_TURNS="${SINAIN_AGENT_MAX_TURNS:-5}"
|
|
34
|
+
SPAWN_MAX_TURNS="${SINAIN_SPAWN_MAX_TURNS:-25}"
|
|
31
35
|
|
|
32
36
|
# Build allowed tools list for Claude's --allowedTools flag.
|
|
33
37
|
# SINAIN_ALLOWED_TOOLS in .env overrides; otherwise auto-derive from MCP config.
|
|
@@ -64,10 +68,11 @@ invoke_agent() {
|
|
|
64
68
|
local prompt="$1"
|
|
65
69
|
case "$AGENT" in
|
|
66
70
|
claude)
|
|
71
|
+
local turns="${2:-$AGENT_MAX_TURNS}"
|
|
67
72
|
claude --enable-auto-mode \
|
|
68
73
|
--mcp-config "$MCP_CONFIG" \
|
|
69
74
|
${ALLOWED_TOOLS:+--allowedTools $ALLOWED_TOOLS} \
|
|
70
|
-
--max-turns
|
|
75
|
+
--max-turns "$turns" --output-format text \
|
|
71
76
|
-p "$prompt"
|
|
72
77
|
;;
|
|
73
78
|
codex)
|
|
@@ -89,9 +94,10 @@ invoke_agent() {
|
|
|
89
94
|
fi
|
|
90
95
|
;;
|
|
91
96
|
goose)
|
|
97
|
+
local turns="${2:-$AGENT_MAX_TURNS}"
|
|
92
98
|
GOOSE_MODE=auto goose run --text "$prompt" \
|
|
93
99
|
--output-format text \
|
|
94
|
-
--max-turns
|
|
100
|
+
--max-turns "$turns"
|
|
95
101
|
;;
|
|
96
102
|
aider)
|
|
97
103
|
# No MCP support — signal pipe mode
|
|
@@ -215,9 +221,13 @@ Call sinain_get_escalation to see the full context, then call sinain_respond wit
|
|
|
215
221
|
Response guidelines: 5-10 sentences, address errors first, reference specific screen/audio context, never NO_REPLY. Max 4000 chars for coding context, 3000 otherwise.'
|
|
216
222
|
|
|
217
223
|
HEARTBEAT_PROMPT='You are the sinain HUD agent. Run the heartbeat cycle:
|
|
218
|
-
1. Call sinain_heartbeat_tick with a brief session summary
|
|
219
|
-
2. If the result contains a suggestion, post it to HUD via sinain_post_feed
|
|
220
|
-
3. Call
|
|
224
|
+
1. Call sinain_heartbeat_tick with a brief session summary (runs signal analysis, session distillation, knowledge integration, insight synthesis)
|
|
225
|
+
2. If the result contains a suggestion or insight, post it to HUD via sinain_post_feed
|
|
226
|
+
3. Call sinain_get_knowledge to review the merged knowledge document (draws from both local and workspace databases)
|
|
227
|
+
4. Optionally call sinain_knowledge_query with relevant entities to check long-term knowledge state
|
|
228
|
+
5. Call sinain_get_feedback to review recent escalation scores
|
|
229
|
+
|
|
230
|
+
Knowledge context: sinain-core maintains two knowledge databases — local (session distillation) and workspace (heartbeat curation). The knowledge tools query both via the sinain-core API. Facts have confidence decay (60-day half-life).'
|
|
221
231
|
|
|
222
232
|
# --- Main loop ---
|
|
223
233
|
|
|
@@ -267,7 +277,7 @@ while true; do
|
|
|
267
277
|
SPAWN_PROMPT="You have a background task to complete. Task: $SPAWN_TASK
|
|
268
278
|
|
|
269
279
|
Complete this task thoroughly. Use sinain_get_knowledge and sinain_knowledge_query if you need context from past sessions. Summarize your findings concisely."
|
|
270
|
-
SPAWN_RESULT=$(invoke_agent "$SPAWN_PROMPT" || echo "ERROR: agent invocation failed")
|
|
280
|
+
SPAWN_RESULT=$(invoke_agent "$SPAWN_PROMPT" "$SPAWN_MAX_TURNS" || echo "ERROR: agent invocation failed")
|
|
271
281
|
else
|
|
272
282
|
# Pipe path: agent gets task text directly
|
|
273
283
|
SPAWN_RESULT=$(invoke_pipe "Background task: $SPAWN_TASK" || echo "No output")
|
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AnalysisConfig, AgentResult, ContextWindow, RecorderStatus, RecordCommand } from "../types.js";
|
|
2
2
|
import { normalizeAppName } from "./context-window.js";
|
|
3
3
|
import { log, error } from "../log.js";
|
|
4
4
|
import { levelFor, applyLevel } from "../privacy/index.js";
|
|
5
5
|
|
|
6
6
|
const TAG = "agent";
|
|
7
7
|
|
|
8
|
-
/** Guard: only one Ollama vision call at a time (latest-wins, skip if busy). */
|
|
9
|
-
let ollamaInFlight = false;
|
|
10
|
-
|
|
11
8
|
/**
|
|
12
9
|
* Model-specific timeouts in milliseconds.
|
|
13
10
|
* Only increases timeouts for slow models to avoid false timeouts.
|
|
@@ -211,75 +208,54 @@ function parseTask(parsed: any): string | undefined {
|
|
|
211
208
|
*/
|
|
212
209
|
export async function analyzeContext(
|
|
213
210
|
contextWindow: ContextWindow,
|
|
214
|
-
config:
|
|
211
|
+
config: AnalysisConfig,
|
|
215
212
|
recorderStatus: RecorderStatus | null = null,
|
|
216
213
|
traitSystemPrompt?: string,
|
|
217
214
|
): Promise<AgentResult> {
|
|
218
215
|
const userPrompt = buildUserPrompt(contextWindow, recorderStatus);
|
|
219
|
-
|
|
216
|
+
|
|
217
|
+
// Apply privacy gating for images based on provider
|
|
220
218
|
let images = contextWindow.images || [];
|
|
219
|
+
const privacyDest = config.provider === "ollama" ? "local_llm" : "openrouter";
|
|
221
220
|
try {
|
|
222
|
-
|
|
223
|
-
if (imgLevel === "none") {
|
|
224
|
-
images = [];
|
|
225
|
-
}
|
|
221
|
+
if (levelFor("screen_images", privacyDest) === "none") images = [];
|
|
226
222
|
} catch { /* privacy not initialized, keep images */ }
|
|
223
|
+
|
|
227
224
|
const systemPrompt = traitSystemPrompt ?? SYSTEM_PROMPT;
|
|
228
225
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
if (config.localVisionEnabled && !ollamaInFlight) {
|
|
232
|
-
ollamaInFlight = true;
|
|
233
|
-
try {
|
|
234
|
-
const result = await callOllamaVision(systemPrompt, userPrompt, images, config);
|
|
235
|
-
const mode = images.length > 0 ? "vision" : "text";
|
|
236
|
-
log(TAG, `local ollama (${config.localVisionModel}, ${mode}): success`);
|
|
237
|
-
return result;
|
|
238
|
-
} catch (err: any) {
|
|
239
|
-
log(TAG, `local ollama failed: ${err.message || err}, falling back to OpenRouter`);
|
|
240
|
-
} finally {
|
|
241
|
-
ollamaInFlight = false;
|
|
242
|
-
}
|
|
226
|
+
if (config.provider === "ollama") {
|
|
227
|
+
return await callOllama(systemPrompt, userPrompt, images, config);
|
|
243
228
|
}
|
|
244
229
|
|
|
245
|
-
//
|
|
246
|
-
if (!config.
|
|
247
|
-
|
|
248
|
-
throw new Error("local ollama failed and no OpenRouter API key — cannot analyze");
|
|
249
|
-
}
|
|
250
|
-
throw new Error("no OpenRouter API key configured");
|
|
230
|
+
// OpenRouter path: model chain with fallbacks
|
|
231
|
+
if (!config.apiKey) {
|
|
232
|
+
throw new Error("ANALYSIS_API_KEY / OPENROUTER_API_KEY not set");
|
|
251
233
|
}
|
|
252
234
|
|
|
253
235
|
const models = [config.model, ...config.fallbackModels];
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
// Insert vision model at the front if not already there
|
|
258
|
-
if (!models.includes(config.visionModel)) {
|
|
259
|
-
models.unshift(config.visionModel);
|
|
260
|
-
}
|
|
236
|
+
// Auto-upgrade to vision model when images are present
|
|
237
|
+
if (images.length > 0 && config.visionModel && !models.includes(config.visionModel)) {
|
|
238
|
+
models.unshift(config.visionModel);
|
|
261
239
|
}
|
|
262
240
|
|
|
263
241
|
let lastError: Error | null = null;
|
|
264
|
-
|
|
265
242
|
for (const model of models) {
|
|
266
243
|
try {
|
|
267
|
-
return await
|
|
244
|
+
return await callOpenRouter(systemPrompt, userPrompt, images, model, config);
|
|
268
245
|
} catch (err: any) {
|
|
269
246
|
lastError = err;
|
|
270
247
|
log(TAG, `model ${model} failed: ${err.message || err}, trying next...`);
|
|
271
248
|
}
|
|
272
249
|
}
|
|
273
|
-
|
|
274
250
|
throw lastError || new Error("all models failed");
|
|
275
251
|
}
|
|
276
252
|
|
|
277
|
-
async function
|
|
253
|
+
async function callOpenRouter(
|
|
278
254
|
systemPrompt: string,
|
|
279
255
|
userPrompt: string,
|
|
280
256
|
images: ContextWindow["images"],
|
|
281
257
|
model: string,
|
|
282
|
-
config:
|
|
258
|
+
config: AnalysisConfig,
|
|
283
259
|
): Promise<AgentResult> {
|
|
284
260
|
const start = Date.now();
|
|
285
261
|
const controller = new AbortController();
|
|
@@ -307,10 +283,10 @@ async function callModel(
|
|
|
307
283
|
|
|
308
284
|
const imageCount = images?.length || 0;
|
|
309
285
|
|
|
310
|
-
const response = await fetch(
|
|
286
|
+
const response = await fetch(config.endpoint, {
|
|
311
287
|
method: "POST",
|
|
312
288
|
headers: {
|
|
313
|
-
"Authorization": `Bearer ${config.
|
|
289
|
+
"Authorization": `Bearer ${config.apiKey}`,
|
|
314
290
|
"Content-Type": "application/json",
|
|
315
291
|
},
|
|
316
292
|
body: JSON.stringify({
|
|
@@ -398,28 +374,27 @@ async function callModel(
|
|
|
398
374
|
}
|
|
399
375
|
|
|
400
376
|
/**
|
|
401
|
-
* Call Ollama local
|
|
402
|
-
* Uses the /api/chat endpoint with base64 images.
|
|
403
|
-
* Falls back to OpenRouter on any failure.
|
|
377
|
+
* Call Ollama local model for context analysis.
|
|
378
|
+
* Uses the /api/chat endpoint with optional base64 images.
|
|
404
379
|
*/
|
|
405
|
-
async function
|
|
380
|
+
async function callOllama(
|
|
406
381
|
systemPrompt: string,
|
|
407
382
|
userPrompt: string,
|
|
408
383
|
images: ContextWindow["images"],
|
|
409
|
-
config:
|
|
384
|
+
config: AnalysisConfig,
|
|
410
385
|
): Promise<AgentResult> {
|
|
411
386
|
const start = Date.now();
|
|
412
387
|
const controller = new AbortController();
|
|
413
|
-
const timeout = setTimeout(() => controller.abort(), config.
|
|
388
|
+
const timeout = setTimeout(() => controller.abort(), config.timeout);
|
|
414
389
|
|
|
415
390
|
try {
|
|
416
391
|
const imageB64List = (images || []).map((img) => img.data);
|
|
417
392
|
|
|
418
|
-
const response = await fetch(`${config.
|
|
393
|
+
const response = await fetch(`${config.endpoint}/api/chat`, {
|
|
419
394
|
method: "POST",
|
|
420
395
|
headers: { "Content-Type": "application/json" },
|
|
421
396
|
body: JSON.stringify({
|
|
422
|
-
model: config.
|
|
397
|
+
model: config.model,
|
|
423
398
|
messages: [
|
|
424
399
|
{ role: "system", content: systemPrompt },
|
|
425
400
|
{ role: "user", content: userPrompt, images: imageB64List },
|
|
@@ -445,7 +420,7 @@ async function callOllamaVision(
|
|
|
445
420
|
const tokensIn = data.prompt_eval_count || 0;
|
|
446
421
|
const tokensOut = data.eval_count || 0;
|
|
447
422
|
|
|
448
|
-
log(TAG, `ollama vision: model=${config.
|
|
423
|
+
log(TAG, `ollama vision: model=${config.model} latency=${latencyMs}ms tokens=${tokensIn}+${tokensOut}`);
|
|
449
424
|
|
|
450
425
|
// Parse the response (same format as OpenRouter)
|
|
451
426
|
// Parse JSON response (same logic as callModel)
|
|
@@ -459,7 +434,7 @@ async function callOllamaVision(
|
|
|
459
434
|
task: parseTask(parsed),
|
|
460
435
|
latencyMs,
|
|
461
436
|
tokensIn, tokensOut,
|
|
462
|
-
model: config.
|
|
437
|
+
model: config.model,
|
|
463
438
|
parsedOk: true,
|
|
464
439
|
};
|
|
465
440
|
} catch {
|
|
@@ -475,7 +450,7 @@ async function callOllamaVision(
|
|
|
475
450
|
task: parseTask(parsed),
|
|
476
451
|
latencyMs,
|
|
477
452
|
tokensIn, tokensOut,
|
|
478
|
-
model: config.
|
|
453
|
+
model: config.model,
|
|
479
454
|
parsedOk: true,
|
|
480
455
|
};
|
|
481
456
|
}
|
|
@@ -486,7 +461,7 @@ async function callOllamaVision(
|
|
|
486
461
|
digest: content || "\u2014",
|
|
487
462
|
latencyMs,
|
|
488
463
|
tokensIn, tokensOut,
|
|
489
|
-
model: config.
|
|
464
|
+
model: config.model,
|
|
490
465
|
parsedOk: false,
|
|
491
466
|
};
|
|
492
467
|
}
|
|
@@ -2,7 +2,7 @@ import { EventEmitter } from "node:events";
|
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import type { FeedBuffer } from "../buffers/feed-buffer.js";
|
|
4
4
|
import type { SenseBuffer } from "../buffers/sense-buffer.js";
|
|
5
|
-
import type {
|
|
5
|
+
import type { AnalysisConfig, AgentEntry, ContextWindow, EscalationMode, ContextRichness, RecorderStatus, SenseEvent, FeedbackRecord } from "../types.js";
|
|
6
6
|
import type { Profiler } from "../profiler.js";
|
|
7
7
|
import type { CostTracker } from "../cost/tracker.js";
|
|
8
8
|
import { buildContextWindow, RICHNESS_PRESETS } from "./context-window.js";
|
|
@@ -18,7 +18,7 @@ const TAG = "agent";
|
|
|
18
18
|
export interface AgentLoopDeps {
|
|
19
19
|
feedBuffer: FeedBuffer;
|
|
20
20
|
senseBuffer: SenseBuffer;
|
|
21
|
-
agentConfig:
|
|
21
|
+
agentConfig: AnalysisConfig;
|
|
22
22
|
escalationMode: EscalationMode;
|
|
23
23
|
situationMdPath: string;
|
|
24
24
|
/** Called after analysis with digest + context for escalation check. */
|
|
@@ -106,9 +106,10 @@ export class AgentLoop extends EventEmitter {
|
|
|
106
106
|
/** Start the agent loop. */
|
|
107
107
|
start(): void {
|
|
108
108
|
if (this.started) return;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
const ac = this.deps.agentConfig;
|
|
110
|
+
if (!ac.enabled || (ac.provider !== "ollama" && !ac.apiKey)) {
|
|
111
|
+
if (ac.enabled) {
|
|
112
|
+
warn(TAG, "AGENT_ENABLED=true but no API key and provider is not ollama \u2014 analysis disabled");
|
|
112
113
|
}
|
|
113
114
|
return;
|
|
114
115
|
}
|
|
@@ -177,8 +178,8 @@ export class AgentLoop extends EventEmitter {
|
|
|
177
178
|
|
|
178
179
|
/** Get config (safe — no API key). */
|
|
179
180
|
getConfig(): Record<string, unknown> {
|
|
180
|
-
const {
|
|
181
|
-
return { ...safe, hasApiKey: !!
|
|
181
|
+
const { apiKey, ...safe } = this.deps.agentConfig;
|
|
182
|
+
return { ...safe, hasApiKey: !!apiKey, escalationMode: this.deps.escalationMode };
|
|
182
183
|
}
|
|
183
184
|
|
|
184
185
|
/** Get stats for /health. */
|
|
@@ -219,10 +220,10 @@ export class AgentLoop extends EventEmitter {
|
|
|
219
220
|
if (updates.maxIntervalMs !== undefined) c.maxIntervalMs = Math.max(5000, parseInt(String(updates.maxIntervalMs)));
|
|
220
221
|
if (updates.cooldownMs !== undefined) c.cooldownMs = Math.max(3000, parseInt(String(updates.cooldownMs)));
|
|
221
222
|
if (updates.fallbackModels !== undefined) c.fallbackModels = Array.isArray(updates.fallbackModels) ? updates.fallbackModels : [];
|
|
222
|
-
if (updates.
|
|
223
|
+
if (updates.apiKey !== undefined) c.apiKey = String(updates.apiKey);
|
|
223
224
|
|
|
224
225
|
// Restart loop if needed
|
|
225
|
-
if (c.enabled && c.
|
|
226
|
+
if (c.enabled && (c.provider === "ollama" || c.apiKey)) {
|
|
226
227
|
if (!this.started) this.start();
|
|
227
228
|
else {
|
|
228
229
|
// Reset max interval timer with new config
|
|
@@ -238,7 +239,7 @@ export class AgentLoop extends EventEmitter {
|
|
|
238
239
|
|
|
239
240
|
private async run(): Promise<void> {
|
|
240
241
|
if (this.running) return;
|
|
241
|
-
if (!this.deps.agentConfig.
|
|
242
|
+
if (this.deps.agentConfig.provider !== "ollama" && !this.deps.agentConfig.apiKey) return;
|
|
242
243
|
|
|
243
244
|
// Cooldown: don't re-analyze within cooldownMs of last run (unless urgent)
|
|
244
245
|
const isUrgent = this.urgentPending;
|
|
@@ -2,7 +2,7 @@ import { readFileSync, existsSync } from "node:fs";
|
|
|
2
2
|
import { resolve, dirname } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import os from "node:os";
|
|
5
|
-
import type { CoreConfig, AudioPipelineConfig, TranscriptionConfig,
|
|
5
|
+
import type { CoreConfig, AudioPipelineConfig, TranscriptionConfig, AnalysisConfig, EscalationConfig, OpenClawConfig, EscalationMode, EscalationTransport, LearningConfig, TraitConfig, PrivacyConfig, PrivacyMatrix, PrivacyLevel, PrivacyRow } from "./types.js";
|
|
6
6
|
import { PRESETS } from "./privacy/presets.js";
|
|
7
7
|
|
|
8
8
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -178,25 +178,28 @@ export function loadConfig(): CoreConfig {
|
|
|
178
178
|
},
|
|
179
179
|
};
|
|
180
180
|
|
|
181
|
-
const
|
|
181
|
+
const analysisProvider = env("ANALYSIS_PROVIDER", "openrouter") as import("./types.js").AnalysisProvider;
|
|
182
|
+
const defaultEndpoint = analysisProvider === "ollama"
|
|
183
|
+
? "http://localhost:11434"
|
|
184
|
+
: "https://openrouter.ai/api/v1/chat/completions";
|
|
185
|
+
|
|
186
|
+
const agentConfig: import("./types.js").AnalysisConfig = {
|
|
182
187
|
enabled: boolEnv("AGENT_ENABLED", true),
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
188
|
+
provider: analysisProvider,
|
|
189
|
+
model: env("ANALYSIS_MODEL", "google/gemini-2.5-flash-lite"),
|
|
190
|
+
visionModel: env("ANALYSIS_VISION_MODEL", "google/gemini-2.5-flash"),
|
|
191
|
+
endpoint: env("ANALYSIS_ENDPOINT", defaultEndpoint),
|
|
192
|
+
apiKey: env("ANALYSIS_API_KEY", env("OPENROUTER_API_KEY", "")),
|
|
193
|
+
maxTokens: intEnv("ANALYSIS_MAX_TOKENS", 800),
|
|
194
|
+
temperature: floatEnv("ANALYSIS_TEMPERATURE", 0.3),
|
|
195
|
+
fallbackModels: env("ANALYSIS_FALLBACK_MODELS", "google/gemini-2.5-flash,anthropic/claude-3.5-haiku")
|
|
196
|
+
.split(",").map(s => s.trim()).filter(Boolean),
|
|
197
|
+
timeout: intEnv("ANALYSIS_TIMEOUT", 15000),
|
|
193
198
|
pushToFeed: boolEnv("AGENT_PUSH_TO_FEED", true),
|
|
194
199
|
debounceMs: intEnv("AGENT_DEBOUNCE_MS", 3000),
|
|
195
200
|
maxIntervalMs: intEnv("AGENT_MAX_INTERVAL_MS", 30000),
|
|
196
201
|
cooldownMs: intEnv("AGENT_COOLDOWN_MS", 10000),
|
|
197
202
|
maxAgeMs: intEnv("AGENT_MAX_AGE_MS", 120000),
|
|
198
|
-
fallbackModels: env("AGENT_FALLBACK_MODELS", "google/gemini-2.5-flash,anthropic/claude-3.5-haiku")
|
|
199
|
-
.split(",").map(s => s.trim()).filter(Boolean),
|
|
200
203
|
historyLimit: intEnv("AGENT_HISTORY_LIMIT", 50),
|
|
201
204
|
};
|
|
202
205
|
|