@geravant/sinain 1.0.18 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +10 -1
  2. package/cli.js +176 -0
  3. package/index.ts +163 -1257
  4. package/install.js +12 -2
  5. package/launcher.js +622 -0
  6. package/openclaw.plugin.json +4 -0
  7. package/pack-prepare.js +48 -0
  8. package/package.json +26 -5
  9. package/sense_client/README.md +82 -0
  10. package/sense_client/__init__.py +1 -0
  11. package/sense_client/__main__.py +462 -0
  12. package/sense_client/app_detector.py +54 -0
  13. package/sense_client/app_detector_win.py +83 -0
  14. package/sense_client/capture.py +215 -0
  15. package/sense_client/capture_win.py +88 -0
  16. package/sense_client/change_detector.py +86 -0
  17. package/sense_client/config.py +64 -0
  18. package/sense_client/gate.py +145 -0
  19. package/sense_client/ocr.py +347 -0
  20. package/sense_client/privacy.py +65 -0
  21. package/sense_client/requirements.txt +13 -0
  22. package/sense_client/roi_extractor.py +84 -0
  23. package/sense_client/sender.py +173 -0
  24. package/sense_client/tests/__init__.py +0 -0
  25. package/sense_client/tests/test_stream1_optimizations.py +234 -0
  26. package/setup-overlay.js +82 -0
  27. package/sinain-agent/.env.example +17 -0
  28. package/sinain-agent/CLAUDE.md +80 -0
  29. package/sinain-agent/mcp-config.json +12 -0
  30. package/sinain-agent/run.sh +248 -0
  31. package/sinain-core/.env.example +93 -0
  32. package/sinain-core/package-lock.json +552 -0
  33. package/sinain-core/package.json +21 -0
  34. package/sinain-core/src/agent/analyzer.ts +366 -0
  35. package/sinain-core/src/agent/context-window.ts +172 -0
  36. package/sinain-core/src/agent/loop.ts +404 -0
  37. package/sinain-core/src/agent/situation-writer.ts +187 -0
  38. package/sinain-core/src/agent/traits.ts +520 -0
  39. package/sinain-core/src/audio/capture-spawner-macos.ts +44 -0
  40. package/sinain-core/src/audio/capture-spawner-win.ts +37 -0
  41. package/sinain-core/src/audio/capture-spawner.ts +14 -0
  42. package/sinain-core/src/audio/pipeline.ts +335 -0
  43. package/sinain-core/src/audio/transcription-local.ts +141 -0
  44. package/sinain-core/src/audio/transcription.ts +278 -0
  45. package/sinain-core/src/buffers/feed-buffer.ts +71 -0
  46. package/sinain-core/src/buffers/sense-buffer.ts +425 -0
  47. package/sinain-core/src/config.ts +245 -0
  48. package/sinain-core/src/escalation/escalation-slot.ts +136 -0
  49. package/sinain-core/src/escalation/escalator.ts +812 -0
  50. package/sinain-core/src/escalation/message-builder.ts +323 -0
  51. package/sinain-core/src/escalation/openclaw-ws.ts +726 -0
  52. package/sinain-core/src/escalation/scorer.ts +166 -0
  53. package/sinain-core/src/index.ts +507 -0
  54. package/sinain-core/src/learning/feedback-store.ts +253 -0
  55. package/sinain-core/src/learning/signal-collector.ts +218 -0
  56. package/sinain-core/src/log.ts +24 -0
  57. package/sinain-core/src/overlay/commands.ts +126 -0
  58. package/sinain-core/src/overlay/ws-handler.ts +267 -0
  59. package/sinain-core/src/privacy/index.ts +18 -0
  60. package/sinain-core/src/privacy/presets.ts +40 -0
  61. package/sinain-core/src/privacy/redact.ts +92 -0
  62. package/sinain-core/src/profiler.ts +181 -0
  63. package/sinain-core/src/recorder.ts +186 -0
  64. package/sinain-core/src/server.ts +417 -0
  65. package/sinain-core/src/trace/trace-store.ts +73 -0
  66. package/sinain-core/src/trace/tracer.ts +94 -0
  67. package/sinain-core/src/types.ts +427 -0
  68. package/sinain-core/src/util/dedup.ts +48 -0
  69. package/sinain-core/src/util/task-store.ts +84 -0
  70. package/sinain-core/tsconfig.json +18 -0
  71. package/sinain-knowledge/adapters/generic/adapter.ts +103 -0
  72. package/sinain-knowledge/adapters/interface.ts +72 -0
  73. package/sinain-knowledge/adapters/openclaw/adapter.ts +223 -0
  74. package/sinain-knowledge/curation/engine.ts +493 -0
  75. package/sinain-knowledge/curation/resilience.ts +336 -0
  76. package/sinain-knowledge/data/git-store.ts +312 -0
  77. package/sinain-knowledge/data/schema.ts +89 -0
  78. package/sinain-knowledge/data/snapshot.ts +226 -0
  79. package/sinain-knowledge/data/store.ts +488 -0
  80. package/sinain-knowledge/deploy/cli.ts +214 -0
  81. package/sinain-knowledge/deploy/manifest.ts +80 -0
  82. package/sinain-knowledge/protocol/bindings/generic.md +5 -0
  83. package/sinain-knowledge/protocol/bindings/openclaw.md +5 -0
  84. package/sinain-knowledge/protocol/heartbeat.md +62 -0
  85. package/sinain-knowledge/protocol/renderer.ts +56 -0
  86. package/sinain-knowledge/protocol/skill.md +335 -0
  87. package/sinain-mcp-server/index.ts +337 -0
  88. package/sinain-mcp-server/package.json +19 -0
  89. package/sinain-mcp-server/tsconfig.json +15 -0
@@ -0,0 +1,173 @@
1
+ """POST sense events to the relay server."""
2
+
3
+ import base64
4
+ import io
5
+ import time
6
+
7
+ import requests
8
+ from PIL import Image
9
+
10
+ from .gate import SenseEvent
11
+
12
+ # Retry config for /sense POST
13
+ _MAX_RETRIES = 3
14
+ _RETRY_BASE_DELAY_S = 1.0 # 1s, 2s, 4s (exponential)
15
+
16
+
17
+ class SenseSender:
18
+ """POSTs sense events to the relay server with retry and backoff."""
19
+
20
+ def __init__(self, url: str = "http://localhost:9500",
21
+ max_image_kb: int = 500, send_thumbnails: bool = True):
22
+ self.url = url.rstrip("/")
23
+ self.max_image_kb = max_image_kb
24
+ self.send_thumbnails = send_thumbnails
25
+ self._latencies: list[float] = []
26
+ self._last_stats_ts: float = time.time()
27
+ self._consecutive_failures: int = 0
28
+
29
+ def send(self, event: SenseEvent) -> bool:
30
+ """POST /sense with JSON payload. Returns True on success."""
31
+ payload = {
32
+ "type": event.type,
33
+ "ts": event.ts,
34
+ "ocr": event.ocr,
35
+ "meta": {
36
+ "ssim": event.meta.ssim,
37
+ "app": event.meta.app,
38
+ "windowTitle": event.meta.window_title,
39
+ "screen": event.meta.screen,
40
+ },
41
+ }
42
+ if event.roi:
43
+ payload["roi"] = event.roi
44
+ if event.diff:
45
+ payload["diff"] = event.diff
46
+ if event.observation and event.observation.title:
47
+ payload["observation"] = {
48
+ "title": event.observation.title,
49
+ "subtitle": event.observation.subtitle,
50
+ "facts": event.observation.facts,
51
+ "narrative": event.observation.narrative,
52
+ "concepts": event.observation.concepts,
53
+ }
54
+
55
+ for attempt in range(_MAX_RETRIES):
56
+ try:
57
+ start = time.time()
58
+ resp = requests.post(
59
+ f"{self.url}/sense",
60
+ json=payload,
61
+ timeout=5,
62
+ )
63
+ elapsed_ms = (time.time() - start) * 1000
64
+ self._latencies.append(elapsed_ms)
65
+ self._maybe_log_stats()
66
+
67
+ if resp.status_code == 200:
68
+ if self._consecutive_failures > 0:
69
+ print(f"[sender] reconnected after {self._consecutive_failures} failure(s)", flush=True)
70
+ self._consecutive_failures = 0
71
+ return True
72
+
73
+ # Non-200 but not an exception — log and retry
74
+ print(f"[sender] HTTP {resp.status_code} on attempt {attempt + 1}/{_MAX_RETRIES}", flush=True)
75
+
76
+ except requests.exceptions.ConnectionError as e:
77
+ print(f"[sender] connection error (attempt {attempt + 1}/{_MAX_RETRIES}): {e}", flush=True)
78
+ except requests.exceptions.Timeout:
79
+ print(f"[sender] timeout (attempt {attempt + 1}/{_MAX_RETRIES})", flush=True)
80
+ except Exception as e:
81
+ print(f"[sender] unexpected error (attempt {attempt + 1}/{_MAX_RETRIES}): {e}", flush=True)
82
+
83
+ # Don't sleep after the last attempt
84
+ if attempt < _MAX_RETRIES - 1:
85
+ delay = _RETRY_BASE_DELAY_S * (2 ** attempt)
86
+ print(f"[sender] retrying in {delay:.1f}s...", flush=True)
87
+ time.sleep(delay)
88
+
89
+ self._consecutive_failures += 1
90
+ if self._consecutive_failures == 1 or self._consecutive_failures % 10 == 0:
91
+ print(f"[sender] all {_MAX_RETRIES} attempts failed (consecutive failures: {self._consecutive_failures})", flush=True)
92
+ return False
93
+
94
+ def _maybe_log_stats(self):
95
+ """Log P50/P95 send latencies every 60s."""
96
+ now = time.time()
97
+ if now - self._last_stats_ts < 60:
98
+ return
99
+ if not self._latencies:
100
+ return
101
+ sorted_lat = sorted(self._latencies)
102
+ p50 = sorted_lat[len(sorted_lat) // 2]
103
+ p95 = sorted_lat[int(len(sorted_lat) * 0.95)]
104
+ print(f"[sender] relay latency: p50={p50:.0f}ms p95={p95:.0f}ms (n={len(sorted_lat)})", flush=True)
105
+ self._latencies.clear()
106
+ self._last_stats_ts = now
107
+
108
+
109
+ def encode_image(img: Image.Image, max_kb: int, max_px: int = 0) -> str:
110
+ """Encode PIL Image to base64 JPEG, reducing quality until under max_kb."""
111
+ if max_px:
112
+ ratio = max_px / max(img.size)
113
+ if ratio < 1:
114
+ img = img.resize(
115
+ (int(img.width * ratio), int(img.height * ratio)),
116
+ Image.LANCZOS,
117
+ )
118
+
119
+ if img.mode == "RGBA":
120
+ img = img.convert("RGB")
121
+
122
+ # Try high quality first — often fits
123
+ max_bytes = max_kb * 1024
124
+ buf = io.BytesIO()
125
+ img.save(buf, format="JPEG", quality=85)
126
+ if buf.tell() <= max_bytes:
127
+ return base64.b64encode(buf.getvalue()).decode()
128
+
129
+ # Binary search for the highest quality that fits
130
+ lo, hi = 20, 80
131
+ best_buf = None
132
+ while lo <= hi:
133
+ mid = (lo + hi) // 2
134
+ buf = io.BytesIO()
135
+ img.save(buf, format="JPEG", quality=mid)
136
+ if buf.tell() <= max_bytes:
137
+ best_buf = buf
138
+ lo = mid + 1
139
+ else:
140
+ hi = mid - 1
141
+
142
+ if best_buf is not None:
143
+ return base64.b64encode(best_buf.getvalue()).decode()
144
+
145
+ # Last resort: return at lowest quality
146
+ buf = io.BytesIO()
147
+ img.save(buf, format="JPEG", quality=20)
148
+ return base64.b64encode(buf.getvalue()).decode()
149
+
150
+
151
+ def package_full_frame(frame: Image.Image, max_px: int = 384) -> dict:
152
+ """Package a full frame as a small thumbnail for context events."""
153
+ return {
154
+ "data": encode_image(frame, max_kb=200, max_px=max_px),
155
+ "bbox": [0, 0, frame.width, frame.height],
156
+ "thumb": True,
157
+ }
158
+
159
+
160
+ def package_roi(roi, thumb: bool = True) -> dict:
161
+ """Package an ROI as a small thumbnail for text/visual events."""
162
+ return {
163
+ "data": encode_image(roi.image, max_kb=60, max_px=384),
164
+ "bbox": list(roi.bbox),
165
+ "thumb": True,
166
+ }
167
+
168
+
169
+ def package_diff(diff_image: Image.Image) -> dict:
170
+ """Package a diff image."""
171
+ return {
172
+ "data": encode_image(diff_image, max_kb=200),
173
+ }
File without changes
@@ -0,0 +1,234 @@
1
+ """Tests for Stream 1 optimizations: adaptive cooldown, adaptive SSIM,
2
+ parallel OCR, skip image for text events, latency instrumentation."""
3
+
4
+ import time
5
+ import unittest
6
+ from unittest.mock import MagicMock, patch
7
+
8
+ from sense_client.change_detector import ChangeDetector, ChangeResult
9
+ from sense_client.config import load_config, DEFAULTS
10
+ from sense_client.gate import DecisionGate, SenseEvent, SenseMeta
11
+ from sense_client.ocr import OCRResult
12
+ from sense_client.sender import SenseSender
13
+
14
+
15
+ class TestConfig(unittest.TestCase):
16
+ """Test that adaptiveCooldownMs is present in defaults."""
17
+
18
+ def test_defaults_has_adaptive_cooldown(self):
19
+ config = load_config(None)
20
+ self.assertIn("adaptiveCooldownMs", config["gate"])
21
+ self.assertEqual(config["gate"]["adaptiveCooldownMs"], 2000)
22
+
23
+ def test_defaults_cooldown_unchanged(self):
24
+ config = load_config(None)
25
+ self.assertEqual(config["gate"]["cooldownMs"], 5000)
26
+
27
+
28
+ class TestChangeDetectorSetThreshold(unittest.TestCase):
29
+ """Test dynamic SSIM threshold adjustment."""
30
+
31
+ def test_set_threshold(self):
32
+ det = ChangeDetector(threshold=0.92)
33
+ self.assertEqual(det.threshold, 0.92)
34
+ det.set_threshold(0.85)
35
+ self.assertEqual(det.threshold, 0.85)
36
+ det.set_threshold(0.95)
37
+ self.assertEqual(det.threshold, 0.95)
38
+
39
+
40
+ class TestAdaptiveCooldown(unittest.TestCase):
41
+ """Test that DecisionGate uses shorter cooldown after app switch."""
42
+
43
+ def _make_change(self, ssim=0.80):
44
+ return ChangeResult(
45
+ ssim_score=ssim,
46
+ diff_image=MagicMock(),
47
+ contours=[],
48
+ bbox=(0, 0, 100, 100),
49
+ )
50
+
51
+ def _make_ocr(self, text=""):
52
+ return OCRResult(text=text, confidence=90, word_count=len(text.split()))
53
+
54
+ def test_init_has_adaptive_params(self):
55
+ gate = DecisionGate(adaptive_cooldown_ms=2000)
56
+ self.assertEqual(gate.adaptive_cooldown_ms, 2000)
57
+ self.assertEqual(gate.last_app_change_ts, 0)
58
+
59
+ def test_app_change_sets_timestamp(self):
60
+ gate = DecisionGate(cooldown_ms=5000, adaptive_cooldown_ms=2000,
61
+ context_cooldown_ms=0)
62
+ change = self._make_change()
63
+ ocr = self._make_ocr()
64
+
65
+ # First call with app_changed — should set last_app_change_ts
66
+ event = gate.classify(change, ocr, app_changed=True)
67
+ self.assertIsNotNone(event)
68
+ self.assertEqual(event.type, "context")
69
+ self.assertGreater(gate.last_app_change_ts, 0)
70
+
71
+ def test_adaptive_cooldown_shorter_after_app_switch(self):
72
+ gate = DecisionGate(cooldown_ms=5000, adaptive_cooldown_ms=2000,
73
+ context_cooldown_ms=0)
74
+ long_text = "This is a sufficiently long OCR text for testing purposes here"
75
+ change = self._make_change(ssim=0.80)
76
+ ocr = self._make_ocr(long_text)
77
+
78
+ # Trigger app change to set last_app_change_ts
79
+ gate.classify(change, ocr, app_changed=True)
80
+
81
+ # Immediately after: should use 2s adaptive cooldown
82
+ # Within 2s window, should be gated
83
+ event = gate.classify(change, ocr, app_changed=False)
84
+ self.assertIsNone(event) # within 2s cooldown
85
+
86
+ # Simulate 2.1s passing by adjusting timestamps
87
+ gate.last_send_ts -= 2100 # pretend 2.1s have passed
88
+ gate.last_app_change_ts = time.time() * 1000 - 1000 # app change 1s ago (within 10s)
89
+
90
+ event = gate.classify(change, self._make_ocr(long_text + " extra"), app_changed=False)
91
+ self.assertIsNotNone(event)
92
+ self.assertEqual(event.type, "text")
93
+
94
+ def test_normal_cooldown_when_no_recent_app_change(self):
95
+ gate = DecisionGate(cooldown_ms=5000, adaptive_cooldown_ms=2000,
96
+ context_cooldown_ms=10000)
97
+ long_text = "This is a sufficiently long OCR text for testing purposes here"
98
+ change = self._make_change(ssim=0.80)
99
+ ocr = self._make_ocr(long_text)
100
+
101
+ # No app change ever — last_app_change_ts is 0
102
+ # Send an initial event
103
+ event = gate.classify(change, ocr, app_changed=False)
104
+ self.assertIsNotNone(event)
105
+
106
+ # Try again after 3s (> adaptive but < normal cooldown)
107
+ gate.last_send_ts -= 3000
108
+ event = gate.classify(change, self._make_ocr(long_text + " more"), app_changed=False)
109
+ # Should be gated because no recent app change → uses 5s cooldown
110
+ self.assertIsNone(event)
111
+
112
+
113
+ class TestSkipImageForTextEvents(unittest.TestCase):
114
+ """Verify that text events don't get image data packaged."""
115
+
116
+ def test_text_event_has_no_roi(self):
117
+ """Simulate what __main__.py does: text events skip package_roi."""
118
+ event = SenseEvent(type="text", ts=time.time() * 1000, ocr="some text")
119
+ # The logic in __main__.py:
120
+ # if event.type == "context": package_full_frame
121
+ # elif event.type == "visual" and rois: package_roi
122
+ # text events: nothing
123
+ if event.type == "context":
124
+ event.roi = {"data": "base64..."}
125
+ elif event.type == "visual":
126
+ event.roi = {"data": "base64..."}
127
+ # text: no roi set
128
+ self.assertIsNone(event.roi)
129
+
130
+ def test_visual_event_gets_roi(self):
131
+ event = SenseEvent(type="visual", ts=time.time() * 1000, ocr="")
132
+ rois = [MagicMock()]
133
+ if event.type == "visual" and rois:
134
+ event.roi = {"data": "base64...", "bbox": [0, 0, 100, 100]}
135
+ self.assertIsNotNone(event.roi)
136
+
137
+
138
+ class TestSenderLatencyTracking(unittest.TestCase):
139
+ """Test that SenseSender tracks send latencies."""
140
+
141
+ def test_init_has_latency_fields(self):
142
+ sender = SenseSender()
143
+ self.assertIsInstance(sender._latencies, list)
144
+ self.assertEqual(len(sender._latencies), 0)
145
+ self.assertIsInstance(sender._last_stats_ts, float)
146
+
147
+ @patch("sense_client.sender.requests.post")
148
+ def test_send_tracks_latency(self, mock_post):
149
+ mock_resp = MagicMock()
150
+ mock_resp.status_code = 200
151
+ mock_post.return_value = mock_resp
152
+
153
+ sender = SenseSender(url="http://localhost:18791")
154
+ event = SenseEvent(type="text", ts=time.time() * 1000, ocr="test",
155
+ meta=SenseMeta(ssim=0.9, app="test"))
156
+
157
+ ok = sender.send(event)
158
+ self.assertTrue(ok)
159
+ self.assertEqual(len(sender._latencies), 1)
160
+ self.assertGreater(sender._latencies[0], 0)
161
+
162
+ @patch("sense_client.sender.requests.post")
163
+ def test_stats_logged_after_interval(self, mock_post):
164
+ mock_resp = MagicMock()
165
+ mock_resp.status_code = 200
166
+ mock_post.return_value = mock_resp
167
+
168
+ sender = SenseSender(url="http://localhost:18791")
169
+ sender._last_stats_ts = time.time() - 61 # pretend 61s ago
170
+
171
+ event = SenseEvent(type="text", ts=time.time() * 1000, ocr="test",
172
+ meta=SenseMeta(ssim=0.9, app="test"))
173
+
174
+ with patch("builtins.print") as mock_print:
175
+ sender.send(event)
176
+ # Should have logged stats
177
+ calls = [str(c) for c in mock_print.call_args_list]
178
+ stats_logged = any("[sender] relay latency" in c for c in calls)
179
+ self.assertTrue(stats_logged, f"Expected latency stats log, got: {calls}")
180
+
181
+ @patch("sense_client.sender.requests.post")
182
+ def test_stats_not_logged_before_interval(self, mock_post):
183
+ mock_resp = MagicMock()
184
+ mock_resp.status_code = 200
185
+ mock_post.return_value = mock_resp
186
+
187
+ sender = SenseSender(url="http://localhost:18791")
188
+ # _last_stats_ts defaults to now, so interval not elapsed
189
+
190
+ event = SenseEvent(type="text", ts=time.time() * 1000, ocr="test",
191
+ meta=SenseMeta(ssim=0.9, app="test"))
192
+
193
+ with patch("builtins.print") as mock_print:
194
+ sender.send(event)
195
+ calls = [str(c) for c in mock_print.call_args_list]
196
+ stats_logged = any("[sender] relay latency" in c for c in calls)
197
+ self.assertFalse(stats_logged)
198
+
199
+
200
+ class TestParallelOCR(unittest.TestCase):
201
+ """Test parallel OCR logic from __main__.py."""
202
+
203
+ def test_single_roi_no_threadpool(self):
204
+ """With 1 ROI, OCR is called directly (no ThreadPoolExecutor)."""
205
+ mock_ocr = MagicMock()
206
+ mock_ocr.extract.return_value = OCRResult(text="hello world test text here", confidence=90, word_count=5)
207
+
208
+ roi = MagicMock()
209
+ rois = [roi]
210
+
211
+ # Simulate single-ROI path
212
+ if len(rois) == 1:
213
+ result = mock_ocr.extract(rois[0].image)
214
+ else:
215
+ result = None
216
+
217
+ self.assertIsNotNone(result)
218
+ self.assertEqual(result.text, "hello world test text here")
219
+ mock_ocr.extract.assert_called_once()
220
+
221
+ def test_multiple_rois_picks_best(self):
222
+ """With multiple ROIs, pick the result with most text."""
223
+ results = [
224
+ OCRResult(text="short", confidence=90, word_count=1),
225
+ OCRResult(text="this is the longest text result here", confidence=85, word_count=7),
226
+ OCRResult(text="medium text", confidence=88, word_count=2),
227
+ ]
228
+
229
+ best = max(results, key=lambda r: len(r.text))
230
+ self.assertEqual(best.text, "this is the longest text result here")
231
+
232
+
233
+ if __name__ == "__main__":
234
+ unittest.main()
@@ -0,0 +1,82 @@
1
+ #!/usr/bin/env node
2
+ // sinain setup-overlay — clone and build the Flutter overlay app
3
+
4
+ import { execSync } from "child_process";
5
+ import fs from "fs";
6
+ import path from "path";
7
+ import os from "os";
8
+
9
+ const HOME = os.homedir();
10
+ const SINAIN_DIR = path.join(HOME, ".sinain");
11
+ const REPO_DIR = path.join(SINAIN_DIR, "overlay-repo");
12
+ const OVERLAY_LINK = path.join(SINAIN_DIR, "overlay");
13
+
14
+ const BOLD = "\x1b[1m";
15
+ const GREEN = "\x1b[32m";
16
+ const RED = "\x1b[31m";
17
+ const RESET = "\x1b[0m";
18
+
19
+ function log(msg) { console.log(`${BOLD}[setup-overlay]${RESET} ${msg}`); }
20
+ function ok(msg) { console.log(`${BOLD}[setup-overlay]${RESET} ${GREEN}✓${RESET} ${msg}`); }
21
+ function fail(msg) { console.error(`${BOLD}[setup-overlay]${RESET} ${RED}✗${RESET} ${msg}`); process.exit(1); }
22
+
23
+ // Check flutter
24
+ try {
25
+ execSync("which flutter", { stdio: "pipe" });
26
+ } catch {
27
+ fail("flutter not found. Install it: https://docs.flutter.dev/get-started/install");
28
+ }
29
+
30
+ const flutterVer = execSync("flutter --version 2>&1", { encoding: "utf-8" }).split("\n")[0];
31
+ ok(`flutter: ${flutterVer}`);
32
+
33
+ fs.mkdirSync(SINAIN_DIR, { recursive: true });
34
+
35
+ // Clone or update
36
+ if (fs.existsSync(path.join(REPO_DIR, ".git"))) {
37
+ log("Updating existing overlay repo...");
38
+ execSync("git pull --ff-only", { cwd: REPO_DIR, stdio: "inherit" });
39
+ ok("Repository updated");
40
+ } else {
41
+ log("Cloning overlay (sparse checkout — only overlay/ directory)...");
42
+ if (fs.existsSync(REPO_DIR)) {
43
+ fs.rmSync(REPO_DIR, { recursive: true, force: true });
44
+ }
45
+ execSync(
46
+ `git clone --depth 1 --filter=blob:none --sparse https://github.com/anthillnet/sinain-hud.git "${REPO_DIR}"`,
47
+ { stdio: "inherit" }
48
+ );
49
+ execSync("git sparse-checkout set overlay", { cwd: REPO_DIR, stdio: "inherit" });
50
+ ok("Repository cloned");
51
+ }
52
+
53
+ // Build
54
+ const overlayDir = path.join(REPO_DIR, "overlay");
55
+ if (!fs.existsSync(path.join(overlayDir, "pubspec.yaml"))) {
56
+ fail("overlay/pubspec.yaml not found — sparse checkout may have failed");
57
+ }
58
+
59
+ log("Installing Flutter dependencies...");
60
+ execSync("flutter pub get", { cwd: overlayDir, stdio: "inherit" });
61
+
62
+ log("Building overlay (this may take a few minutes)...");
63
+ execSync("flutter build macos", { cwd: overlayDir, stdio: "inherit" });
64
+ ok("Overlay built successfully");
65
+
66
+ // Symlink ~/.sinain/overlay → the overlay source dir
67
+ try {
68
+ if (fs.existsSync(OVERLAY_LINK)) {
69
+ fs.unlinkSync(OVERLAY_LINK);
70
+ }
71
+ fs.symlinkSync(overlayDir, OVERLAY_LINK);
72
+ ok(`Symlinked: ${OVERLAY_LINK} → ${overlayDir}`);
73
+ } catch (e) {
74
+ // Symlink may fail on some systems — fall back to just noting the path
75
+ log(`Overlay built at: ${overlayDir}`);
76
+ }
77
+
78
+ console.log(`
79
+ ${GREEN}✓${RESET} Overlay setup complete!
80
+ The overlay will auto-start with: ${BOLD}sinain start${RESET}
81
+ Or run manually: cd ${overlayDir} && flutter run -d macos
82
+ `);
@@ -0,0 +1,17 @@
1
+ # sinain-agent configuration
2
+ # Copy to .env and customize: cp .env.example .env
3
+
4
+ # ── Agent ──
5
+ SINAIN_AGENT=claude # claude | codex | junie | goose | aider | <custom command>
6
+ # MCP agents (claude, codex, junie, goose) call sinain tools directly
7
+ # Pipe agents (aider, custom) receive escalation text on stdin
8
+
9
+ # ── Core connection ──
10
+ SINAIN_CORE_URL=http://localhost:9500
11
+
12
+ # ── Timing ──
13
+ SINAIN_POLL_INTERVAL=5 # seconds between escalation polls
14
+ SINAIN_HEARTBEAT_INTERVAL=900 # seconds between heartbeat ticks (15 min)
15
+
16
+ # ── Workspace ──
17
+ SINAIN_WORKSPACE=~/.openclaw/workspace # knowledge files, curation scripts, playbook
@@ -0,0 +1,80 @@
1
+ # Sinain HUD Agent
2
+
3
+ You are a coding assistant connected to sinain-hud, a privacy-first AI overlay for macOS. You observe the user's screen and audio context via the HUD system and provide real-time advice displayed on an invisible overlay.
4
+
5
+ ## Your Tools
6
+
7
+ You have MCP tools from `sinain-mcp-server`:
8
+ - `sinain_get_escalation` — poll for pending escalation (call every 3-5 seconds in your main loop)
9
+ - `sinain_respond` — submit your response to an escalation (appears on user's HUD)
10
+ - `sinain_get_context` — get the full context window (screen OCR, audio transcripts, app history)
11
+ - `sinain_get_digest` — get the current agent analysis summary
12
+ - `sinain_get_feedback` — get feedback signals from recent escalations
13
+ - `sinain_post_feed` — push an arbitrary message to the HUD
14
+ - `sinain_health` — check system health
15
+ - `sinain_knowledge_query` — query the knowledge graph for relevant context
16
+ - `sinain_heartbeat_tick` — run the full curation pipeline (git backup, signals, insights, playbook)
17
+ - `sinain_module_guidance` — get active module guidance
18
+
19
+ ## Main Loop
20
+
21
+ Your primary job is an escalation response loop:
22
+
23
+ 1. Call `sinain_get_escalation` to check for pending escalations
24
+ 2. If an escalation is present:
25
+ a. Read the escalation message carefully — it contains screen OCR, audio transcripts, app context, and the local agent's digest
26
+ b. Optionally call `sinain_knowledge_query` with key context from the escalation to enrich your response
27
+ c. Optionally call `sinain_module_guidance` to get active module instructions
28
+ d. Craft a response and call `sinain_respond` with the escalation ID and your response
29
+ 3. If no escalation is pending, wait a few seconds and poll again
30
+ 4. Every 15 minutes, run `sinain_heartbeat_tick` for curation maintenance
31
+
32
+ ## Response Guidelines
33
+
34
+ When responding to escalations:
35
+
36
+ - **5-10 sentences** — concise but substantive
37
+ - **Address errors first** — if the context shows errors, stack traces, or failures, diagnose and suggest fixes
38
+ - **Reference specific context** — quote screen text or audio when relevant ("I see you have a TypeError on line 42...")
39
+ - **Coding context** — if the user is in an IDE or coding platform, focus on code-level help (fixes, patterns, suggestions). Max 4000 chars.
40
+ - **Non-coding context** — share insights, connections, tips relevant to what's on screen. Max 3000 chars.
41
+ - **Never NO_REPLY** — always provide value. If context is minimal, share a relevant insight or tech joke.
42
+ - **Never describe what the user is doing** — they can see their own screen. Add value, don't narrate.
43
+
44
+ ## Heartbeat Cycle (every 15 minutes)
45
+
46
+ 1. Call `sinain_heartbeat_tick` with a brief session summary
47
+ 2. The tool runs the full pipeline automatically:
48
+ - Git backup of memory directory
49
+ - Signal analysis (detects opportunities from session patterns)
50
+ - Insight synthesis (generates suggestions from accumulated patterns)
51
+ - Playbook curation (updates effective playbook based on feedback)
52
+ 3. If the result contains a suggestion or insight, post it to the HUD via `sinain_post_feed`
53
+ 4. Optionally call `sinain_get_feedback` to review recent escalation scores
54
+
55
+ ## Spawning Background Tasks
56
+
57
+ When an escalation suggests deeper research would help:
58
+
59
+ 1. **Respond first** — the user sees your immediate HUD response
60
+ 2. Spawn a background research task (use your native subprocess/agent capabilities)
61
+ 3. On next escalation or heartbeat, check for completed task results
62
+ 4. Integrate findings into your response or playbook
63
+
64
+ Rules:
65
+ - Max 2 spawns per hour
66
+ - Never duplicate a recent task
67
+ - Keep spawned tasks focused and time-bounded
68
+
69
+ ## Files You Manage
70
+
71
+ Your working memory lives at `~/.openclaw/workspace/memory/`:
72
+ - `playbook.md` — your effective playbook (updated by curation pipeline)
73
+ - `triplestore.db` — knowledge graph (SQLite, managed by Python scripts)
74
+ - `playbook-logs/YYYY-MM-DD.jsonl` — decision logs
75
+
76
+ ## Privacy
77
+
78
+ The HUD overlay is invisible to screen capture. All content you receive has already been privacy-stripped by sinain-core. Your responses appear only on the ghost overlay — they are never captured in screenshots or recordings.
79
+
80
+ Never include `<private>` tagged content in your responses — it will be stripped automatically, but avoid echoing it.
@@ -0,0 +1,12 @@
1
+ {
2
+ "mcpServers": {
3
+ "sinain": {
4
+ "command": "../sinain-core/node_modules/.bin/tsx",
5
+ "args": ["../sinain-mcp-server/index.ts"],
6
+ "env": {
7
+ "SINAIN_CORE_URL": "http://localhost:9500",
8
+ "SINAIN_WORKSPACE": "~/.openclaw/workspace"
9
+ }
10
+ }
11
+ }
12
+ }