@geravant/sinain 1.0.18 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/cli.js +176 -0
- package/index.ts +163 -1257
- package/install.js +12 -2
- package/launcher.js +622 -0
- package/openclaw.plugin.json +4 -0
- package/pack-prepare.js +48 -0
- package/package.json +26 -5
- package/sense_client/README.md +82 -0
- package/sense_client/__init__.py +1 -0
- package/sense_client/__main__.py +462 -0
- package/sense_client/app_detector.py +54 -0
- package/sense_client/app_detector_win.py +83 -0
- package/sense_client/capture.py +215 -0
- package/sense_client/capture_win.py +88 -0
- package/sense_client/change_detector.py +86 -0
- package/sense_client/config.py +64 -0
- package/sense_client/gate.py +145 -0
- package/sense_client/ocr.py +347 -0
- package/sense_client/privacy.py +65 -0
- package/sense_client/requirements.txt +13 -0
- package/sense_client/roi_extractor.py +84 -0
- package/sense_client/sender.py +173 -0
- package/sense_client/tests/__init__.py +0 -0
- package/sense_client/tests/test_stream1_optimizations.py +234 -0
- package/setup-overlay.js +82 -0
- package/sinain-agent/.env.example +17 -0
- package/sinain-agent/CLAUDE.md +80 -0
- package/sinain-agent/mcp-config.json +12 -0
- package/sinain-agent/run.sh +248 -0
- package/sinain-core/.env.example +93 -0
- package/sinain-core/package-lock.json +552 -0
- package/sinain-core/package.json +21 -0
- package/sinain-core/src/agent/analyzer.ts +366 -0
- package/sinain-core/src/agent/context-window.ts +172 -0
- package/sinain-core/src/agent/loop.ts +404 -0
- package/sinain-core/src/agent/situation-writer.ts +187 -0
- package/sinain-core/src/agent/traits.ts +520 -0
- package/sinain-core/src/audio/capture-spawner-macos.ts +44 -0
- package/sinain-core/src/audio/capture-spawner-win.ts +37 -0
- package/sinain-core/src/audio/capture-spawner.ts +14 -0
- package/sinain-core/src/audio/pipeline.ts +335 -0
- package/sinain-core/src/audio/transcription-local.ts +141 -0
- package/sinain-core/src/audio/transcription.ts +278 -0
- package/sinain-core/src/buffers/feed-buffer.ts +71 -0
- package/sinain-core/src/buffers/sense-buffer.ts +425 -0
- package/sinain-core/src/config.ts +245 -0
- package/sinain-core/src/escalation/escalation-slot.ts +136 -0
- package/sinain-core/src/escalation/escalator.ts +812 -0
- package/sinain-core/src/escalation/message-builder.ts +323 -0
- package/sinain-core/src/escalation/openclaw-ws.ts +726 -0
- package/sinain-core/src/escalation/scorer.ts +166 -0
- package/sinain-core/src/index.ts +507 -0
- package/sinain-core/src/learning/feedback-store.ts +253 -0
- package/sinain-core/src/learning/signal-collector.ts +218 -0
- package/sinain-core/src/log.ts +24 -0
- package/sinain-core/src/overlay/commands.ts +126 -0
- package/sinain-core/src/overlay/ws-handler.ts +267 -0
- package/sinain-core/src/privacy/index.ts +18 -0
- package/sinain-core/src/privacy/presets.ts +40 -0
- package/sinain-core/src/privacy/redact.ts +92 -0
- package/sinain-core/src/profiler.ts +181 -0
- package/sinain-core/src/recorder.ts +186 -0
- package/sinain-core/src/server.ts +417 -0
- package/sinain-core/src/trace/trace-store.ts +73 -0
- package/sinain-core/src/trace/tracer.ts +94 -0
- package/sinain-core/src/types.ts +427 -0
- package/sinain-core/src/util/dedup.ts +48 -0
- package/sinain-core/src/util/task-store.ts +84 -0
- package/sinain-core/tsconfig.json +18 -0
- package/sinain-knowledge/adapters/generic/adapter.ts +103 -0
- package/sinain-knowledge/adapters/interface.ts +72 -0
- package/sinain-knowledge/adapters/openclaw/adapter.ts +223 -0
- package/sinain-knowledge/curation/engine.ts +493 -0
- package/sinain-knowledge/curation/resilience.ts +336 -0
- package/sinain-knowledge/data/git-store.ts +312 -0
- package/sinain-knowledge/data/schema.ts +89 -0
- package/sinain-knowledge/data/snapshot.ts +226 -0
- package/sinain-knowledge/data/store.ts +488 -0
- package/sinain-knowledge/deploy/cli.ts +214 -0
- package/sinain-knowledge/deploy/manifest.ts +80 -0
- package/sinain-knowledge/protocol/bindings/generic.md +5 -0
- package/sinain-knowledge/protocol/bindings/openclaw.md +5 -0
- package/sinain-knowledge/protocol/heartbeat.md +62 -0
- package/sinain-knowledge/protocol/renderer.ts +56 -0
- package/sinain-knowledge/protocol/skill.md +335 -0
- package/sinain-mcp-server/index.ts +337 -0
- package/sinain-mcp-server/package.json +19 -0
- package/sinain-mcp-server/tsconfig.json +15 -0
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""POST sense events to the relay server."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import io
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
from PIL import Image
|
|
9
|
+
|
|
10
|
+
from .gate import SenseEvent
|
|
11
|
+
|
|
12
|
+
# Retry config for /sense POST
|
|
13
|
+
_MAX_RETRIES = 3
|
|
14
|
+
_RETRY_BASE_DELAY_S = 1.0 # 1s, 2s, 4s (exponential)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SenseSender:
|
|
18
|
+
"""POSTs sense events to the relay server with retry and backoff."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, url: str = "http://localhost:9500",
|
|
21
|
+
max_image_kb: int = 500, send_thumbnails: bool = True):
|
|
22
|
+
self.url = url.rstrip("/")
|
|
23
|
+
self.max_image_kb = max_image_kb
|
|
24
|
+
self.send_thumbnails = send_thumbnails
|
|
25
|
+
self._latencies: list[float] = []
|
|
26
|
+
self._last_stats_ts: float = time.time()
|
|
27
|
+
self._consecutive_failures: int = 0
|
|
28
|
+
|
|
29
|
+
def send(self, event: SenseEvent) -> bool:
|
|
30
|
+
"""POST /sense with JSON payload. Returns True on success."""
|
|
31
|
+
payload = {
|
|
32
|
+
"type": event.type,
|
|
33
|
+
"ts": event.ts,
|
|
34
|
+
"ocr": event.ocr,
|
|
35
|
+
"meta": {
|
|
36
|
+
"ssim": event.meta.ssim,
|
|
37
|
+
"app": event.meta.app,
|
|
38
|
+
"windowTitle": event.meta.window_title,
|
|
39
|
+
"screen": event.meta.screen,
|
|
40
|
+
},
|
|
41
|
+
}
|
|
42
|
+
if event.roi:
|
|
43
|
+
payload["roi"] = event.roi
|
|
44
|
+
if event.diff:
|
|
45
|
+
payload["diff"] = event.diff
|
|
46
|
+
if event.observation and event.observation.title:
|
|
47
|
+
payload["observation"] = {
|
|
48
|
+
"title": event.observation.title,
|
|
49
|
+
"subtitle": event.observation.subtitle,
|
|
50
|
+
"facts": event.observation.facts,
|
|
51
|
+
"narrative": event.observation.narrative,
|
|
52
|
+
"concepts": event.observation.concepts,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
for attempt in range(_MAX_RETRIES):
|
|
56
|
+
try:
|
|
57
|
+
start = time.time()
|
|
58
|
+
resp = requests.post(
|
|
59
|
+
f"{self.url}/sense",
|
|
60
|
+
json=payload,
|
|
61
|
+
timeout=5,
|
|
62
|
+
)
|
|
63
|
+
elapsed_ms = (time.time() - start) * 1000
|
|
64
|
+
self._latencies.append(elapsed_ms)
|
|
65
|
+
self._maybe_log_stats()
|
|
66
|
+
|
|
67
|
+
if resp.status_code == 200:
|
|
68
|
+
if self._consecutive_failures > 0:
|
|
69
|
+
print(f"[sender] reconnected after {self._consecutive_failures} failure(s)", flush=True)
|
|
70
|
+
self._consecutive_failures = 0
|
|
71
|
+
return True
|
|
72
|
+
|
|
73
|
+
# Non-200 but not an exception — log and retry
|
|
74
|
+
print(f"[sender] HTTP {resp.status_code} on attempt {attempt + 1}/{_MAX_RETRIES}", flush=True)
|
|
75
|
+
|
|
76
|
+
except requests.exceptions.ConnectionError as e:
|
|
77
|
+
print(f"[sender] connection error (attempt {attempt + 1}/{_MAX_RETRIES}): {e}", flush=True)
|
|
78
|
+
except requests.exceptions.Timeout:
|
|
79
|
+
print(f"[sender] timeout (attempt {attempt + 1}/{_MAX_RETRIES})", flush=True)
|
|
80
|
+
except Exception as e:
|
|
81
|
+
print(f"[sender] unexpected error (attempt {attempt + 1}/{_MAX_RETRIES}): {e}", flush=True)
|
|
82
|
+
|
|
83
|
+
# Don't sleep after the last attempt
|
|
84
|
+
if attempt < _MAX_RETRIES - 1:
|
|
85
|
+
delay = _RETRY_BASE_DELAY_S * (2 ** attempt)
|
|
86
|
+
print(f"[sender] retrying in {delay:.1f}s...", flush=True)
|
|
87
|
+
time.sleep(delay)
|
|
88
|
+
|
|
89
|
+
self._consecutive_failures += 1
|
|
90
|
+
if self._consecutive_failures == 1 or self._consecutive_failures % 10 == 0:
|
|
91
|
+
print(f"[sender] all {_MAX_RETRIES} attempts failed (consecutive failures: {self._consecutive_failures})", flush=True)
|
|
92
|
+
return False
|
|
93
|
+
|
|
94
|
+
def _maybe_log_stats(self):
|
|
95
|
+
"""Log P50/P95 send latencies every 60s."""
|
|
96
|
+
now = time.time()
|
|
97
|
+
if now - self._last_stats_ts < 60:
|
|
98
|
+
return
|
|
99
|
+
if not self._latencies:
|
|
100
|
+
return
|
|
101
|
+
sorted_lat = sorted(self._latencies)
|
|
102
|
+
p50 = sorted_lat[len(sorted_lat) // 2]
|
|
103
|
+
p95 = sorted_lat[int(len(sorted_lat) * 0.95)]
|
|
104
|
+
print(f"[sender] relay latency: p50={p50:.0f}ms p95={p95:.0f}ms (n={len(sorted_lat)})", flush=True)
|
|
105
|
+
self._latencies.clear()
|
|
106
|
+
self._last_stats_ts = now
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def encode_image(img: Image.Image, max_kb: int, max_px: int = 0) -> str:
|
|
110
|
+
"""Encode PIL Image to base64 JPEG, reducing quality until under max_kb."""
|
|
111
|
+
if max_px:
|
|
112
|
+
ratio = max_px / max(img.size)
|
|
113
|
+
if ratio < 1:
|
|
114
|
+
img = img.resize(
|
|
115
|
+
(int(img.width * ratio), int(img.height * ratio)),
|
|
116
|
+
Image.LANCZOS,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
if img.mode == "RGBA":
|
|
120
|
+
img = img.convert("RGB")
|
|
121
|
+
|
|
122
|
+
# Try high quality first — often fits
|
|
123
|
+
max_bytes = max_kb * 1024
|
|
124
|
+
buf = io.BytesIO()
|
|
125
|
+
img.save(buf, format="JPEG", quality=85)
|
|
126
|
+
if buf.tell() <= max_bytes:
|
|
127
|
+
return base64.b64encode(buf.getvalue()).decode()
|
|
128
|
+
|
|
129
|
+
# Binary search for the highest quality that fits
|
|
130
|
+
lo, hi = 20, 80
|
|
131
|
+
best_buf = None
|
|
132
|
+
while lo <= hi:
|
|
133
|
+
mid = (lo + hi) // 2
|
|
134
|
+
buf = io.BytesIO()
|
|
135
|
+
img.save(buf, format="JPEG", quality=mid)
|
|
136
|
+
if buf.tell() <= max_bytes:
|
|
137
|
+
best_buf = buf
|
|
138
|
+
lo = mid + 1
|
|
139
|
+
else:
|
|
140
|
+
hi = mid - 1
|
|
141
|
+
|
|
142
|
+
if best_buf is not None:
|
|
143
|
+
return base64.b64encode(best_buf.getvalue()).decode()
|
|
144
|
+
|
|
145
|
+
# Last resort: return at lowest quality
|
|
146
|
+
buf = io.BytesIO()
|
|
147
|
+
img.save(buf, format="JPEG", quality=20)
|
|
148
|
+
return base64.b64encode(buf.getvalue()).decode()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def package_full_frame(frame: Image.Image, max_px: int = 384) -> dict:
|
|
152
|
+
"""Package a full frame as a small thumbnail for context events."""
|
|
153
|
+
return {
|
|
154
|
+
"data": encode_image(frame, max_kb=200, max_px=max_px),
|
|
155
|
+
"bbox": [0, 0, frame.width, frame.height],
|
|
156
|
+
"thumb": True,
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def package_roi(roi, thumb: bool = True) -> dict:
|
|
161
|
+
"""Package an ROI as a small thumbnail for text/visual events."""
|
|
162
|
+
return {
|
|
163
|
+
"data": encode_image(roi.image, max_kb=60, max_px=384),
|
|
164
|
+
"bbox": list(roi.bbox),
|
|
165
|
+
"thumb": True,
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def package_diff(diff_image: Image.Image) -> dict:
|
|
170
|
+
"""Package a diff image."""
|
|
171
|
+
return {
|
|
172
|
+
"data": encode_image(diff_image, max_kb=200),
|
|
173
|
+
}
|
|
File without changes
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Tests for Stream 1 optimizations: adaptive cooldown, adaptive SSIM,
|
|
2
|
+
parallel OCR, skip image for text events, latency instrumentation."""
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
import unittest
|
|
6
|
+
from unittest.mock import MagicMock, patch
|
|
7
|
+
|
|
8
|
+
from sense_client.change_detector import ChangeDetector, ChangeResult
|
|
9
|
+
from sense_client.config import load_config, DEFAULTS
|
|
10
|
+
from sense_client.gate import DecisionGate, SenseEvent, SenseMeta
|
|
11
|
+
from sense_client.ocr import OCRResult
|
|
12
|
+
from sense_client.sender import SenseSender
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TestConfig(unittest.TestCase):
|
|
16
|
+
"""Test that adaptiveCooldownMs is present in defaults."""
|
|
17
|
+
|
|
18
|
+
def test_defaults_has_adaptive_cooldown(self):
|
|
19
|
+
config = load_config(None)
|
|
20
|
+
self.assertIn("adaptiveCooldownMs", config["gate"])
|
|
21
|
+
self.assertEqual(config["gate"]["adaptiveCooldownMs"], 2000)
|
|
22
|
+
|
|
23
|
+
def test_defaults_cooldown_unchanged(self):
|
|
24
|
+
config = load_config(None)
|
|
25
|
+
self.assertEqual(config["gate"]["cooldownMs"], 5000)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TestChangeDetectorSetThreshold(unittest.TestCase):
|
|
29
|
+
"""Test dynamic SSIM threshold adjustment."""
|
|
30
|
+
|
|
31
|
+
def test_set_threshold(self):
|
|
32
|
+
det = ChangeDetector(threshold=0.92)
|
|
33
|
+
self.assertEqual(det.threshold, 0.92)
|
|
34
|
+
det.set_threshold(0.85)
|
|
35
|
+
self.assertEqual(det.threshold, 0.85)
|
|
36
|
+
det.set_threshold(0.95)
|
|
37
|
+
self.assertEqual(det.threshold, 0.95)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TestAdaptiveCooldown(unittest.TestCase):
|
|
41
|
+
"""Test that DecisionGate uses shorter cooldown after app switch."""
|
|
42
|
+
|
|
43
|
+
def _make_change(self, ssim=0.80):
|
|
44
|
+
return ChangeResult(
|
|
45
|
+
ssim_score=ssim,
|
|
46
|
+
diff_image=MagicMock(),
|
|
47
|
+
contours=[],
|
|
48
|
+
bbox=(0, 0, 100, 100),
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def _make_ocr(self, text=""):
|
|
52
|
+
return OCRResult(text=text, confidence=90, word_count=len(text.split()))
|
|
53
|
+
|
|
54
|
+
def test_init_has_adaptive_params(self):
|
|
55
|
+
gate = DecisionGate(adaptive_cooldown_ms=2000)
|
|
56
|
+
self.assertEqual(gate.adaptive_cooldown_ms, 2000)
|
|
57
|
+
self.assertEqual(gate.last_app_change_ts, 0)
|
|
58
|
+
|
|
59
|
+
def test_app_change_sets_timestamp(self):
|
|
60
|
+
gate = DecisionGate(cooldown_ms=5000, adaptive_cooldown_ms=2000,
|
|
61
|
+
context_cooldown_ms=0)
|
|
62
|
+
change = self._make_change()
|
|
63
|
+
ocr = self._make_ocr()
|
|
64
|
+
|
|
65
|
+
# First call with app_changed — should set last_app_change_ts
|
|
66
|
+
event = gate.classify(change, ocr, app_changed=True)
|
|
67
|
+
self.assertIsNotNone(event)
|
|
68
|
+
self.assertEqual(event.type, "context")
|
|
69
|
+
self.assertGreater(gate.last_app_change_ts, 0)
|
|
70
|
+
|
|
71
|
+
def test_adaptive_cooldown_shorter_after_app_switch(self):
|
|
72
|
+
gate = DecisionGate(cooldown_ms=5000, adaptive_cooldown_ms=2000,
|
|
73
|
+
context_cooldown_ms=0)
|
|
74
|
+
long_text = "This is a sufficiently long OCR text for testing purposes here"
|
|
75
|
+
change = self._make_change(ssim=0.80)
|
|
76
|
+
ocr = self._make_ocr(long_text)
|
|
77
|
+
|
|
78
|
+
# Trigger app change to set last_app_change_ts
|
|
79
|
+
gate.classify(change, ocr, app_changed=True)
|
|
80
|
+
|
|
81
|
+
# Immediately after: should use 2s adaptive cooldown
|
|
82
|
+
# Within 2s window, should be gated
|
|
83
|
+
event = gate.classify(change, ocr, app_changed=False)
|
|
84
|
+
self.assertIsNone(event) # within 2s cooldown
|
|
85
|
+
|
|
86
|
+
# Simulate 2.1s passing by adjusting timestamps
|
|
87
|
+
gate.last_send_ts -= 2100 # pretend 2.1s have passed
|
|
88
|
+
gate.last_app_change_ts = time.time() * 1000 - 1000 # app change 1s ago (within 10s)
|
|
89
|
+
|
|
90
|
+
event = gate.classify(change, self._make_ocr(long_text + " extra"), app_changed=False)
|
|
91
|
+
self.assertIsNotNone(event)
|
|
92
|
+
self.assertEqual(event.type, "text")
|
|
93
|
+
|
|
94
|
+
def test_normal_cooldown_when_no_recent_app_change(self):
|
|
95
|
+
gate = DecisionGate(cooldown_ms=5000, adaptive_cooldown_ms=2000,
|
|
96
|
+
context_cooldown_ms=10000)
|
|
97
|
+
long_text = "This is a sufficiently long OCR text for testing purposes here"
|
|
98
|
+
change = self._make_change(ssim=0.80)
|
|
99
|
+
ocr = self._make_ocr(long_text)
|
|
100
|
+
|
|
101
|
+
# No app change ever — last_app_change_ts is 0
|
|
102
|
+
# Send an initial event
|
|
103
|
+
event = gate.classify(change, ocr, app_changed=False)
|
|
104
|
+
self.assertIsNotNone(event)
|
|
105
|
+
|
|
106
|
+
# Try again after 3s (> adaptive but < normal cooldown)
|
|
107
|
+
gate.last_send_ts -= 3000
|
|
108
|
+
event = gate.classify(change, self._make_ocr(long_text + " more"), app_changed=False)
|
|
109
|
+
# Should be gated because no recent app change → uses 5s cooldown
|
|
110
|
+
self.assertIsNone(event)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class TestSkipImageForTextEvents(unittest.TestCase):
|
|
114
|
+
"""Verify that text events don't get image data packaged."""
|
|
115
|
+
|
|
116
|
+
def test_text_event_has_no_roi(self):
|
|
117
|
+
"""Simulate what __main__.py does: text events skip package_roi."""
|
|
118
|
+
event = SenseEvent(type="text", ts=time.time() * 1000, ocr="some text")
|
|
119
|
+
# The logic in __main__.py:
|
|
120
|
+
# if event.type == "context": package_full_frame
|
|
121
|
+
# elif event.type == "visual" and rois: package_roi
|
|
122
|
+
# text events: nothing
|
|
123
|
+
if event.type == "context":
|
|
124
|
+
event.roi = {"data": "base64..."}
|
|
125
|
+
elif event.type == "visual":
|
|
126
|
+
event.roi = {"data": "base64..."}
|
|
127
|
+
# text: no roi set
|
|
128
|
+
self.assertIsNone(event.roi)
|
|
129
|
+
|
|
130
|
+
def test_visual_event_gets_roi(self):
|
|
131
|
+
event = SenseEvent(type="visual", ts=time.time() * 1000, ocr="")
|
|
132
|
+
rois = [MagicMock()]
|
|
133
|
+
if event.type == "visual" and rois:
|
|
134
|
+
event.roi = {"data": "base64...", "bbox": [0, 0, 100, 100]}
|
|
135
|
+
self.assertIsNotNone(event.roi)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class TestSenderLatencyTracking(unittest.TestCase):
|
|
139
|
+
"""Test that SenseSender tracks send latencies."""
|
|
140
|
+
|
|
141
|
+
def test_init_has_latency_fields(self):
|
|
142
|
+
sender = SenseSender()
|
|
143
|
+
self.assertIsInstance(sender._latencies, list)
|
|
144
|
+
self.assertEqual(len(sender._latencies), 0)
|
|
145
|
+
self.assertIsInstance(sender._last_stats_ts, float)
|
|
146
|
+
|
|
147
|
+
@patch("sense_client.sender.requests.post")
|
|
148
|
+
def test_send_tracks_latency(self, mock_post):
|
|
149
|
+
mock_resp = MagicMock()
|
|
150
|
+
mock_resp.status_code = 200
|
|
151
|
+
mock_post.return_value = mock_resp
|
|
152
|
+
|
|
153
|
+
sender = SenseSender(url="http://localhost:18791")
|
|
154
|
+
event = SenseEvent(type="text", ts=time.time() * 1000, ocr="test",
|
|
155
|
+
meta=SenseMeta(ssim=0.9, app="test"))
|
|
156
|
+
|
|
157
|
+
ok = sender.send(event)
|
|
158
|
+
self.assertTrue(ok)
|
|
159
|
+
self.assertEqual(len(sender._latencies), 1)
|
|
160
|
+
self.assertGreater(sender._latencies[0], 0)
|
|
161
|
+
|
|
162
|
+
@patch("sense_client.sender.requests.post")
|
|
163
|
+
def test_stats_logged_after_interval(self, mock_post):
|
|
164
|
+
mock_resp = MagicMock()
|
|
165
|
+
mock_resp.status_code = 200
|
|
166
|
+
mock_post.return_value = mock_resp
|
|
167
|
+
|
|
168
|
+
sender = SenseSender(url="http://localhost:18791")
|
|
169
|
+
sender._last_stats_ts = time.time() - 61 # pretend 61s ago
|
|
170
|
+
|
|
171
|
+
event = SenseEvent(type="text", ts=time.time() * 1000, ocr="test",
|
|
172
|
+
meta=SenseMeta(ssim=0.9, app="test"))
|
|
173
|
+
|
|
174
|
+
with patch("builtins.print") as mock_print:
|
|
175
|
+
sender.send(event)
|
|
176
|
+
# Should have logged stats
|
|
177
|
+
calls = [str(c) for c in mock_print.call_args_list]
|
|
178
|
+
stats_logged = any("[sender] relay latency" in c for c in calls)
|
|
179
|
+
self.assertTrue(stats_logged, f"Expected latency stats log, got: {calls}")
|
|
180
|
+
|
|
181
|
+
@patch("sense_client.sender.requests.post")
|
|
182
|
+
def test_stats_not_logged_before_interval(self, mock_post):
|
|
183
|
+
mock_resp = MagicMock()
|
|
184
|
+
mock_resp.status_code = 200
|
|
185
|
+
mock_post.return_value = mock_resp
|
|
186
|
+
|
|
187
|
+
sender = SenseSender(url="http://localhost:18791")
|
|
188
|
+
# _last_stats_ts defaults to now, so interval not elapsed
|
|
189
|
+
|
|
190
|
+
event = SenseEvent(type="text", ts=time.time() * 1000, ocr="test",
|
|
191
|
+
meta=SenseMeta(ssim=0.9, app="test"))
|
|
192
|
+
|
|
193
|
+
with patch("builtins.print") as mock_print:
|
|
194
|
+
sender.send(event)
|
|
195
|
+
calls = [str(c) for c in mock_print.call_args_list]
|
|
196
|
+
stats_logged = any("[sender] relay latency" in c for c in calls)
|
|
197
|
+
self.assertFalse(stats_logged)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class TestParallelOCR(unittest.TestCase):
|
|
201
|
+
"""Test parallel OCR logic from __main__.py."""
|
|
202
|
+
|
|
203
|
+
def test_single_roi_no_threadpool(self):
|
|
204
|
+
"""With 1 ROI, OCR is called directly (no ThreadPoolExecutor)."""
|
|
205
|
+
mock_ocr = MagicMock()
|
|
206
|
+
mock_ocr.extract.return_value = OCRResult(text="hello world test text here", confidence=90, word_count=5)
|
|
207
|
+
|
|
208
|
+
roi = MagicMock()
|
|
209
|
+
rois = [roi]
|
|
210
|
+
|
|
211
|
+
# Simulate single-ROI path
|
|
212
|
+
if len(rois) == 1:
|
|
213
|
+
result = mock_ocr.extract(rois[0].image)
|
|
214
|
+
else:
|
|
215
|
+
result = None
|
|
216
|
+
|
|
217
|
+
self.assertIsNotNone(result)
|
|
218
|
+
self.assertEqual(result.text, "hello world test text here")
|
|
219
|
+
mock_ocr.extract.assert_called_once()
|
|
220
|
+
|
|
221
|
+
def test_multiple_rois_picks_best(self):
|
|
222
|
+
"""With multiple ROIs, pick the result with most text."""
|
|
223
|
+
results = [
|
|
224
|
+
OCRResult(text="short", confidence=90, word_count=1),
|
|
225
|
+
OCRResult(text="this is the longest text result here", confidence=85, word_count=7),
|
|
226
|
+
OCRResult(text="medium text", confidence=88, word_count=2),
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
best = max(results, key=lambda r: len(r.text))
|
|
230
|
+
self.assertEqual(best.text, "this is the longest text result here")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
if __name__ == "__main__":
|
|
234
|
+
unittest.main()
|
package/setup-overlay.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// sinain setup-overlay — clone and build the Flutter overlay app
|
|
3
|
+
|
|
4
|
+
import { execSync } from "child_process";
|
|
5
|
+
import fs from "fs";
|
|
6
|
+
import path from "path";
|
|
7
|
+
import os from "os";
|
|
8
|
+
|
|
9
|
+
const HOME = os.homedir();
|
|
10
|
+
const SINAIN_DIR = path.join(HOME, ".sinain");
|
|
11
|
+
const REPO_DIR = path.join(SINAIN_DIR, "overlay-repo");
|
|
12
|
+
const OVERLAY_LINK = path.join(SINAIN_DIR, "overlay");
|
|
13
|
+
|
|
14
|
+
const BOLD = "\x1b[1m";
|
|
15
|
+
const GREEN = "\x1b[32m";
|
|
16
|
+
const RED = "\x1b[31m";
|
|
17
|
+
const RESET = "\x1b[0m";
|
|
18
|
+
|
|
19
|
+
function log(msg) { console.log(`${BOLD}[setup-overlay]${RESET} ${msg}`); }
|
|
20
|
+
function ok(msg) { console.log(`${BOLD}[setup-overlay]${RESET} ${GREEN}✓${RESET} ${msg}`); }
|
|
21
|
+
function fail(msg) { console.error(`${BOLD}[setup-overlay]${RESET} ${RED}✗${RESET} ${msg}`); process.exit(1); }
|
|
22
|
+
|
|
23
|
+
// Check flutter
|
|
24
|
+
try {
|
|
25
|
+
execSync("which flutter", { stdio: "pipe" });
|
|
26
|
+
} catch {
|
|
27
|
+
fail("flutter not found. Install it: https://docs.flutter.dev/get-started/install");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const flutterVer = execSync("flutter --version 2>&1", { encoding: "utf-8" }).split("\n")[0];
|
|
31
|
+
ok(`flutter: ${flutterVer}`);
|
|
32
|
+
|
|
33
|
+
fs.mkdirSync(SINAIN_DIR, { recursive: true });
|
|
34
|
+
|
|
35
|
+
// Clone or update
|
|
36
|
+
if (fs.existsSync(path.join(REPO_DIR, ".git"))) {
|
|
37
|
+
log("Updating existing overlay repo...");
|
|
38
|
+
execSync("git pull --ff-only", { cwd: REPO_DIR, stdio: "inherit" });
|
|
39
|
+
ok("Repository updated");
|
|
40
|
+
} else {
|
|
41
|
+
log("Cloning overlay (sparse checkout — only overlay/ directory)...");
|
|
42
|
+
if (fs.existsSync(REPO_DIR)) {
|
|
43
|
+
fs.rmSync(REPO_DIR, { recursive: true, force: true });
|
|
44
|
+
}
|
|
45
|
+
execSync(
|
|
46
|
+
`git clone --depth 1 --filter=blob:none --sparse https://github.com/anthillnet/sinain-hud.git "${REPO_DIR}"`,
|
|
47
|
+
{ stdio: "inherit" }
|
|
48
|
+
);
|
|
49
|
+
execSync("git sparse-checkout set overlay", { cwd: REPO_DIR, stdio: "inherit" });
|
|
50
|
+
ok("Repository cloned");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Build
|
|
54
|
+
const overlayDir = path.join(REPO_DIR, "overlay");
|
|
55
|
+
if (!fs.existsSync(path.join(overlayDir, "pubspec.yaml"))) {
|
|
56
|
+
fail("overlay/pubspec.yaml not found — sparse checkout may have failed");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
log("Installing Flutter dependencies...");
|
|
60
|
+
execSync("flutter pub get", { cwd: overlayDir, stdio: "inherit" });
|
|
61
|
+
|
|
62
|
+
log("Building overlay (this may take a few minutes)...");
|
|
63
|
+
execSync("flutter build macos", { cwd: overlayDir, stdio: "inherit" });
|
|
64
|
+
ok("Overlay built successfully");
|
|
65
|
+
|
|
66
|
+
// Symlink ~/.sinain/overlay → the overlay source dir
|
|
67
|
+
try {
|
|
68
|
+
if (fs.existsSync(OVERLAY_LINK)) {
|
|
69
|
+
fs.unlinkSync(OVERLAY_LINK);
|
|
70
|
+
}
|
|
71
|
+
fs.symlinkSync(overlayDir, OVERLAY_LINK);
|
|
72
|
+
ok(`Symlinked: ${OVERLAY_LINK} → ${overlayDir}`);
|
|
73
|
+
} catch (e) {
|
|
74
|
+
// Symlink may fail on some systems — fall back to just noting the path
|
|
75
|
+
log(`Overlay built at: ${overlayDir}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
console.log(`
|
|
79
|
+
${GREEN}✓${RESET} Overlay setup complete!
|
|
80
|
+
The overlay will auto-start with: ${BOLD}sinain start${RESET}
|
|
81
|
+
Or run manually: cd ${overlayDir} && flutter run -d macos
|
|
82
|
+
`);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# sinain-agent configuration
|
|
2
|
+
# Copy to .env and customize: cp .env.example .env
|
|
3
|
+
|
|
4
|
+
# ── Agent ──
|
|
5
|
+
SINAIN_AGENT=claude # claude | codex | junie | goose | aider | <custom command>
|
|
6
|
+
# MCP agents (claude, codex, junie, goose) call sinain tools directly
|
|
7
|
+
# Pipe agents (aider, custom) receive escalation text on stdin
|
|
8
|
+
|
|
9
|
+
# ── Core connection ──
|
|
10
|
+
SINAIN_CORE_URL=http://localhost:9500
|
|
11
|
+
|
|
12
|
+
# ── Timing ──
|
|
13
|
+
SINAIN_POLL_INTERVAL=5 # seconds between escalation polls
|
|
14
|
+
SINAIN_HEARTBEAT_INTERVAL=900 # seconds between heartbeat ticks (15 min)
|
|
15
|
+
|
|
16
|
+
# ── Workspace ──
|
|
17
|
+
SINAIN_WORKSPACE=~/.openclaw/workspace # knowledge files, curation scripts, playbook
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Sinain HUD Agent
|
|
2
|
+
|
|
3
|
+
You are a coding assistant connected to sinain-hud, a privacy-first AI overlay for macOS. You observe the user's screen and audio context via the HUD system and provide real-time advice displayed on an invisible overlay.
|
|
4
|
+
|
|
5
|
+
## Your Tools
|
|
6
|
+
|
|
7
|
+
You have MCP tools from `sinain-mcp-server`:
|
|
8
|
+
- `sinain_get_escalation` — poll for pending escalation (call every 3-5 seconds in your main loop)
|
|
9
|
+
- `sinain_respond` — submit your response to an escalation (appears on user's HUD)
|
|
10
|
+
- `sinain_get_context` — get the full context window (screen OCR, audio transcripts, app history)
|
|
11
|
+
- `sinain_get_digest` — get the current agent analysis summary
|
|
12
|
+
- `sinain_get_feedback` — get feedback signals from recent escalations
|
|
13
|
+
- `sinain_post_feed` — push an arbitrary message to the HUD
|
|
14
|
+
- `sinain_health` — check system health
|
|
15
|
+
- `sinain_knowledge_query` — query the knowledge graph for relevant context
|
|
16
|
+
- `sinain_heartbeat_tick` — run the full curation pipeline (git backup, signals, insights, playbook)
|
|
17
|
+
- `sinain_module_guidance` — get active module guidance
|
|
18
|
+
|
|
19
|
+
## Main Loop
|
|
20
|
+
|
|
21
|
+
Your primary job is an escalation response loop:
|
|
22
|
+
|
|
23
|
+
1. Call `sinain_get_escalation` to check for pending escalations
|
|
24
|
+
2. If an escalation is present:
|
|
25
|
+
a. Read the escalation message carefully — it contains screen OCR, audio transcripts, app context, and the local agent's digest
|
|
26
|
+
b. Optionally call `sinain_knowledge_query` with key context from the escalation to enrich your response
|
|
27
|
+
c. Optionally call `sinain_module_guidance` to get active module instructions
|
|
28
|
+
d. Craft a response and call `sinain_respond` with the escalation ID and your response
|
|
29
|
+
3. If no escalation is pending, wait a few seconds and poll again
|
|
30
|
+
4. Every 15 minutes, run `sinain_heartbeat_tick` for curation maintenance
|
|
31
|
+
|
|
32
|
+
## Response Guidelines
|
|
33
|
+
|
|
34
|
+
When responding to escalations:
|
|
35
|
+
|
|
36
|
+
- **5-10 sentences** — concise but substantive
|
|
37
|
+
- **Address errors first** — if the context shows errors, stack traces, or failures, diagnose and suggest fixes
|
|
38
|
+
- **Reference specific context** — quote screen text or audio when relevant ("I see you have a TypeError on line 42...")
|
|
39
|
+
- **Coding context** — if the user is in an IDE or coding platform, focus on code-level help (fixes, patterns, suggestions). Max 4000 chars.
|
|
40
|
+
- **Non-coding context** — share insights, connections, tips relevant to what's on screen. Max 3000 chars.
|
|
41
|
+
- **Never NO_REPLY** — always provide value. If context is minimal, share a relevant insight or tech joke.
|
|
42
|
+
- **Never describe what the user is doing** — they can see their own screen. Add value, don't narrate.
|
|
43
|
+
|
|
44
|
+
## Heartbeat Cycle (every 15 minutes)
|
|
45
|
+
|
|
46
|
+
1. Call `sinain_heartbeat_tick` with a brief session summary
|
|
47
|
+
2. The tool runs the full pipeline automatically:
|
|
48
|
+
- Git backup of memory directory
|
|
49
|
+
- Signal analysis (detects opportunities from session patterns)
|
|
50
|
+
- Insight synthesis (generates suggestions from accumulated patterns)
|
|
51
|
+
- Playbook curation (updates effective playbook based on feedback)
|
|
52
|
+
3. If the result contains a suggestion or insight, post it to the HUD via `sinain_post_feed`
|
|
53
|
+
4. Optionally call `sinain_get_feedback` to review recent escalation scores
|
|
54
|
+
|
|
55
|
+
## Spawning Background Tasks
|
|
56
|
+
|
|
57
|
+
When an escalation suggests deeper research would help:
|
|
58
|
+
|
|
59
|
+
1. **Respond first** — the user sees your immediate HUD response
|
|
60
|
+
2. Spawn a background research task (use your native subprocess/agent capabilities)
|
|
61
|
+
3. On next escalation or heartbeat, check for completed task results
|
|
62
|
+
4. Integrate findings into your response or playbook
|
|
63
|
+
|
|
64
|
+
Rules:
|
|
65
|
+
- Max 2 spawns per hour
|
|
66
|
+
- Never duplicate a recent task
|
|
67
|
+
- Keep spawned tasks focused and time-bounded
|
|
68
|
+
|
|
69
|
+
## Files You Manage
|
|
70
|
+
|
|
71
|
+
Your working memory lives at `~/.openclaw/workspace/memory/`:
|
|
72
|
+
- `playbook.md` — your effective playbook (updated by curation pipeline)
|
|
73
|
+
- `triplestore.db` — knowledge graph (SQLite, managed by Python scripts)
|
|
74
|
+
- `playbook-logs/YYYY-MM-DD.jsonl` — decision logs
|
|
75
|
+
|
|
76
|
+
## Privacy
|
|
77
|
+
|
|
78
|
+
The HUD overlay is invisible to screen capture. All content you receive has already been privacy-stripped by sinain-core. Your responses appear only on the ghost overlay — they are never captured in screenshots or recordings.
|
|
79
|
+
|
|
80
|
+
Never include `<private>` tagged content in your responses — it will be stripped automatically, but avoid echoing it.
|