@simbimbo/memory-ocmemog 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +83 -18
- package/brain/runtime/__init__.py +2 -12
- package/brain/runtime/config.py +1 -24
- package/brain/runtime/inference.py +1 -151
- package/brain/runtime/instrumentation.py +1 -15
- package/brain/runtime/memory/__init__.py +3 -13
- package/brain/runtime/memory/api.py +1 -1219
- package/brain/runtime/memory/candidate.py +1 -185
- package/brain/runtime/memory/conversation_state.py +1 -1823
- package/brain/runtime/memory/distill.py +1 -344
- package/brain/runtime/memory/embedding_engine.py +1 -92
- package/brain/runtime/memory/freshness.py +1 -112
- package/brain/runtime/memory/health.py +1 -40
- package/brain/runtime/memory/integrity.py +1 -186
- package/brain/runtime/memory/memory_consolidation.py +1 -58
- package/brain/runtime/memory/memory_links.py +1 -107
- package/brain/runtime/memory/memory_salience.py +1 -233
- package/brain/runtime/memory/memory_synthesis.py +1 -31
- package/brain/runtime/memory/memory_taxonomy.py +1 -33
- package/brain/runtime/memory/pondering_engine.py +1 -654
- package/brain/runtime/memory/promote.py +1 -277
- package/brain/runtime/memory/provenance.py +1 -406
- package/brain/runtime/memory/reinforcement.py +1 -71
- package/brain/runtime/memory/retrieval.py +1 -210
- package/brain/runtime/memory/semantic_search.py +1 -64
- package/brain/runtime/memory/store.py +1 -429
- package/brain/runtime/memory/unresolved_state.py +1 -91
- package/brain/runtime/memory/vector_index.py +1 -323
- package/brain/runtime/model_roles.py +1 -9
- package/brain/runtime/model_router.py +1 -22
- package/brain/runtime/providers.py +1 -66
- package/brain/runtime/security/redaction.py +1 -12
- package/brain/runtime/state_store.py +1 -23
- package/brain/runtime/storage_paths.py +1 -39
- package/docs/architecture/memory.md +20 -24
- package/docs/release-checklist.md +19 -6
- package/docs/usage.md +33 -17
- package/index.ts +8 -1
- package/ocmemog/__init__.py +11 -0
- package/ocmemog/doctor.py +1255 -0
- package/ocmemog/runtime/__init__.py +18 -0
- package/ocmemog/runtime/_compat_bridge.py +28 -0
- package/ocmemog/runtime/config.py +34 -0
- package/ocmemog/runtime/identity.py +115 -0
- package/ocmemog/runtime/inference.py +163 -0
- package/ocmemog/runtime/instrumentation.py +20 -0
- package/ocmemog/runtime/memory/__init__.py +91 -0
- package/ocmemog/runtime/memory/api.py +1594 -0
- package/ocmemog/runtime/memory/candidate.py +192 -0
- package/ocmemog/runtime/memory/conversation_state.py +1831 -0
- package/ocmemog/runtime/memory/distill.py +282 -0
- package/ocmemog/runtime/memory/embedding_engine.py +151 -0
- package/ocmemog/runtime/memory/freshness.py +114 -0
- package/ocmemog/runtime/memory/health.py +93 -0
- package/ocmemog/runtime/memory/integrity.py +208 -0
- package/ocmemog/runtime/memory/memory_consolidation.py +60 -0
- package/ocmemog/runtime/memory/memory_links.py +109 -0
- package/ocmemog/runtime/memory/memory_salience.py +235 -0
- package/ocmemog/runtime/memory/memory_synthesis.py +33 -0
- package/ocmemog/runtime/memory/memory_taxonomy.py +35 -0
- package/ocmemog/runtime/memory/pondering_engine.py +681 -0
- package/ocmemog/runtime/memory/promote.py +279 -0
- package/ocmemog/runtime/memory/provenance.py +408 -0
- package/ocmemog/runtime/memory/reinforcement.py +73 -0
- package/ocmemog/runtime/memory/retrieval.py +224 -0
- package/ocmemog/runtime/memory/semantic_search.py +66 -0
- package/ocmemog/runtime/memory/store.py +433 -0
- package/ocmemog/runtime/memory/unresolved_state.py +93 -0
- package/ocmemog/runtime/memory/vector_index.py +411 -0
- package/ocmemog/runtime/model_roles.py +15 -0
- package/ocmemog/runtime/model_router.py +28 -0
- package/ocmemog/runtime/providers.py +78 -0
- package/ocmemog/runtime/roles.py +92 -0
- package/ocmemog/runtime/security/__init__.py +8 -0
- package/ocmemog/runtime/security/redaction.py +17 -0
- package/ocmemog/runtime/state_store.py +32 -0
- package/ocmemog/runtime/storage_paths.py +70 -0
- package/ocmemog/sidecar/app.py +421 -60
- package/ocmemog/sidecar/compat.py +50 -13
- package/ocmemog/sidecar/transcript_watcher.py +327 -242
- package/openclaw.plugin.json +4 -0
- package/package.json +1 -1
- package/scripts/ocmemog-backfill-vectors.py +5 -3
- package/scripts/ocmemog-continuity-benchmark.py +1 -1
- package/scripts/ocmemog-demo.py +1 -1
- package/scripts/ocmemog-doctor.py +15 -0
- package/scripts/ocmemog-install.sh +29 -7
- package/scripts/ocmemog-integrated-proof.py +374 -0
- package/scripts/ocmemog-reindex-vectors.py +5 -3
- package/scripts/ocmemog-release-check.sh +330 -0
- package/scripts/ocmemog-sidecar.sh +4 -2
- package/scripts/ocmemog-test-rig.py +5 -3
- package/brain/runtime/memory/artifacts.py +0 -33
- package/brain/runtime/memory/context_builder.py +0 -112
- package/brain/runtime/memory/interaction_memory.py +0 -57
- package/brain/runtime/memory/memory_gate.py +0 -38
- package/brain/runtime/memory/memory_graph.py +0 -54
- package/brain/runtime/memory/person_identity.py +0 -83
- package/brain/runtime/memory/person_memory.py +0 -138
- package/brain/runtime/memory/sentiment_memory.py +0 -67
- package/brain/runtime/memory/tool_catalog.py +0 -68
|
@@ -1,656 +1,3 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import json
|
|
5
|
-
import re
|
|
6
|
-
import threading
|
|
7
|
-
from queue import Queue
|
|
8
|
-
from typing import Any, Callable, Dict, List, Optional
|
|
9
|
-
|
|
10
|
-
from brain.runtime import config, inference, state_store
|
|
11
|
-
from brain.runtime.instrumentation import emit_event
|
|
12
|
-
from brain.runtime.memory import api, integrity, memory_consolidation, memory_links, provenance, store, unresolved_state, vector_index
|
|
13
|
-
|
|
14
|
-
LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
|
|
15
|
-
_WRITABLE_MEMORY_TABLES = set(store.MEMORY_TABLES)
|
|
16
|
-
_SUMMARY_PREFIX_RE = re.compile(r"^(?:insight|recommendation|lesson)\s*:\s*", re.IGNORECASE)
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def _run_with_timeout(name: str, fn: Callable[[], Any], timeout_s: float, default: Any) -> Any:
|
|
20
|
-
emit_event(LOGFILE, f"brain_ponder_{name}_start", status="ok")
|
|
21
|
-
result_queue: Queue[tuple[str, Any]] = Queue(maxsize=1)
|
|
22
|
-
|
|
23
|
-
def _target() -> None:
|
|
24
|
-
try:
|
|
25
|
-
result_queue.put(("ok", fn()))
|
|
26
|
-
except Exception as exc: # pragma: no cover
|
|
27
|
-
result_queue.put(("error", exc))
|
|
28
|
-
|
|
29
|
-
worker = threading.Thread(target=_target, name=f"ocmemog-ponder-{name}", daemon=True)
|
|
30
|
-
worker.start()
|
|
31
|
-
worker.join(timeout_s)
|
|
32
|
-
if worker.is_alive():
|
|
33
|
-
emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="timeout")
|
|
34
|
-
return default
|
|
35
|
-
if result_queue.empty():
|
|
36
|
-
emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="error", error="missing_result")
|
|
37
|
-
return default
|
|
38
|
-
status, payload = result_queue.get_nowait()
|
|
39
|
-
if status == "error":
|
|
40
|
-
emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="error", error=str(payload))
|
|
41
|
-
return default
|
|
42
|
-
emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="ok")
|
|
43
|
-
return payload
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def _infer_with_timeout(prompt: str, timeout_s: float = 20.0) -> Dict[str, str]:
|
|
47
|
-
return _run_with_timeout(
|
|
48
|
-
"infer",
|
|
49
|
-
lambda: inference.infer(prompt, provider_name=config.OCMEMOG_PONDER_MODEL),
|
|
50
|
-
timeout_s,
|
|
51
|
-
{"status": "timeout", "output": ""},
|
|
52
|
-
)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def _load_recent(table: str, limit: int) -> List[Dict[str, object]]:
|
|
56
|
-
if table not in _WRITABLE_MEMORY_TABLES:
|
|
57
|
-
return []
|
|
58
|
-
conn = store.connect(ensure_schema=False)
|
|
59
|
-
try:
|
|
60
|
-
rows = conn.execute(
|
|
61
|
-
f"SELECT id, content, confidence, timestamp, source, metadata_json FROM {table} ORDER BY id DESC LIMIT ?",
|
|
62
|
-
(limit,),
|
|
63
|
-
).fetchall()
|
|
64
|
-
except Exception:
|
|
65
|
-
rows = []
|
|
66
|
-
finally:
|
|
67
|
-
conn.close()
|
|
68
|
-
items: List[Dict[str, object]] = []
|
|
69
|
-
for row in rows:
|
|
70
|
-
try:
|
|
71
|
-
metadata = json.loads(row["metadata_json"] or "{}")
|
|
72
|
-
except Exception:
|
|
73
|
-
metadata = {}
|
|
74
|
-
items.append(
|
|
75
|
-
{
|
|
76
|
-
"reference": f"{table}:{row['id']}",
|
|
77
|
-
"content": str(row["content"] or ""),
|
|
78
|
-
"confidence": float(row["confidence"] or 0.0),
|
|
79
|
-
"timestamp": row["timestamp"],
|
|
80
|
-
"source": row["source"],
|
|
81
|
-
"metadata": metadata,
|
|
82
|
-
"candidate_kind": "memory",
|
|
83
|
-
"memory_type": table,
|
|
84
|
-
}
|
|
85
|
-
)
|
|
86
|
-
return items
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _load_continuity_candidates(limit: int) -> List[Dict[str, object]]:
|
|
90
|
-
conn = store.connect(ensure_schema=False)
|
|
91
|
-
items: List[Dict[str, object]] = []
|
|
92
|
-
try:
|
|
93
|
-
checkpoint_rows = conn.execute(
|
|
94
|
-
"""
|
|
95
|
-
SELECT id, session_id, thread_id, conversation_id, summary, latest_user_ask,
|
|
96
|
-
last_assistant_commitment, metadata_json, timestamp
|
|
97
|
-
FROM conversation_checkpoints
|
|
98
|
-
ORDER BY id DESC LIMIT ?
|
|
99
|
-
""",
|
|
100
|
-
(limit,),
|
|
101
|
-
).fetchall()
|
|
102
|
-
for row in checkpoint_rows:
|
|
103
|
-
try:
|
|
104
|
-
metadata = json.loads(row["metadata_json"] or "{}")
|
|
105
|
-
except Exception:
|
|
106
|
-
metadata = {}
|
|
107
|
-
content_parts = [str(row["summary"] or "").strip()]
|
|
108
|
-
latest_user_ask = str(row["latest_user_ask"] or "").strip()
|
|
109
|
-
if latest_user_ask:
|
|
110
|
-
content_parts.append(f"User ask: {latest_user_ask}")
|
|
111
|
-
last_commitment = str(row["last_assistant_commitment"] or "").strip()
|
|
112
|
-
if last_commitment:
|
|
113
|
-
content_parts.append(f"Assistant commitment: {last_commitment}")
|
|
114
|
-
items.append(
|
|
115
|
-
{
|
|
116
|
-
"reference": f"conversation_checkpoints:{row['id']}",
|
|
117
|
-
"content": " | ".join(part for part in content_parts if part),
|
|
118
|
-
"timestamp": row["timestamp"],
|
|
119
|
-
"source": "continuity",
|
|
120
|
-
"metadata": {
|
|
121
|
-
**metadata,
|
|
122
|
-
"conversation_id": row["conversation_id"],
|
|
123
|
-
"session_id": row["session_id"],
|
|
124
|
-
"thread_id": row["thread_id"],
|
|
125
|
-
},
|
|
126
|
-
"candidate_kind": "checkpoint",
|
|
127
|
-
"memory_type": "runbooks",
|
|
128
|
-
}
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
state_rows = conn.execute(
|
|
132
|
-
"""
|
|
133
|
-
SELECT id, scope_type, scope_id, latest_user_ask, last_assistant_commitment,
|
|
134
|
-
open_loops_json, pending_actions_json, unresolved_state_json, metadata_json, updated_at
|
|
135
|
-
FROM conversation_state
|
|
136
|
-
ORDER BY updated_at DESC, id DESC LIMIT ?
|
|
137
|
-
""",
|
|
138
|
-
(limit,),
|
|
139
|
-
).fetchall()
|
|
140
|
-
for row in state_rows:
|
|
141
|
-
try:
|
|
142
|
-
open_loops = json.loads(row["open_loops_json"] or "[]")
|
|
143
|
-
except Exception:
|
|
144
|
-
open_loops = []
|
|
145
|
-
try:
|
|
146
|
-
pending_actions = json.loads(row["pending_actions_json"] or "[]")
|
|
147
|
-
except Exception:
|
|
148
|
-
pending_actions = []
|
|
149
|
-
try:
|
|
150
|
-
unresolved_items = json.loads(row["unresolved_state_json"] or "[]")
|
|
151
|
-
except Exception:
|
|
152
|
-
unresolved_items = []
|
|
153
|
-
try:
|
|
154
|
-
metadata = json.loads(row["metadata_json"] or "{}")
|
|
155
|
-
except Exception:
|
|
156
|
-
metadata = {}
|
|
157
|
-
content_parts = [f"Continuity scope {row['scope_type']}:{row['scope_id']}"]
|
|
158
|
-
latest_user_ask = str(row["latest_user_ask"] or "").strip()
|
|
159
|
-
if latest_user_ask:
|
|
160
|
-
content_parts.append(f"Latest user ask: {latest_user_ask}")
|
|
161
|
-
last_commitment = str(row["last_assistant_commitment"] or "").strip()
|
|
162
|
-
if last_commitment:
|
|
163
|
-
content_parts.append(f"Assistant commitment: {last_commitment}")
|
|
164
|
-
for label, payload in (("Open loop", open_loops), ("Pending action", pending_actions), ("Unresolved", unresolved_items)):
|
|
165
|
-
for item in payload[:2]:
|
|
166
|
-
summary = str((item or {}).get("summary") or "").strip()
|
|
167
|
-
if summary:
|
|
168
|
-
content_parts.append(f"{label}: {summary}")
|
|
169
|
-
items.append(
|
|
170
|
-
{
|
|
171
|
-
"reference": f"conversation_state:{row['id']}",
|
|
172
|
-
"content": " | ".join(part for part in content_parts if part),
|
|
173
|
-
"timestamp": row["updated_at"],
|
|
174
|
-
"source": "continuity",
|
|
175
|
-
"metadata": metadata,
|
|
176
|
-
"candidate_kind": "continuity_state",
|
|
177
|
-
"memory_type": "runbooks",
|
|
178
|
-
}
|
|
179
|
-
)
|
|
180
|
-
|
|
181
|
-
turn_rows = conn.execute(
|
|
182
|
-
"""
|
|
183
|
-
SELECT id, role, content, session_id, thread_id, conversation_id, message_id, metadata_json, timestamp
|
|
184
|
-
FROM conversation_turns
|
|
185
|
-
ORDER BY id DESC LIMIT ?
|
|
186
|
-
""",
|
|
187
|
-
(limit,),
|
|
188
|
-
).fetchall()
|
|
189
|
-
for row in turn_rows:
|
|
190
|
-
try:
|
|
191
|
-
metadata = json.loads(row["metadata_json"] or "{}")
|
|
192
|
-
except Exception:
|
|
193
|
-
metadata = {}
|
|
194
|
-
items.append(
|
|
195
|
-
{
|
|
196
|
-
"reference": f"conversation_turns:{row['id']}",
|
|
197
|
-
"content": f"{row['role']}: {str(row['content'] or '').strip()}",
|
|
198
|
-
"timestamp": row["timestamp"],
|
|
199
|
-
"source": "continuity",
|
|
200
|
-
"metadata": {
|
|
201
|
-
**metadata,
|
|
202
|
-
"conversation_id": row["conversation_id"],
|
|
203
|
-
"session_id": row["session_id"],
|
|
204
|
-
"thread_id": row["thread_id"],
|
|
205
|
-
"message_id": row["message_id"],
|
|
206
|
-
},
|
|
207
|
-
"candidate_kind": "turn",
|
|
208
|
-
"memory_type": "reflections",
|
|
209
|
-
}
|
|
210
|
-
)
|
|
211
|
-
except Exception as exc:
|
|
212
|
-
emit_event(LOGFILE, "brain_ponder_continuity_candidates_failed", status="error", error=str(exc))
|
|
213
|
-
finally:
|
|
214
|
-
conn.close()
|
|
215
|
-
return items[:limit]
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
def _low_value_candidate(record: Dict[str, object]) -> bool:
|
|
219
|
-
content = str(record.get("content") or "").strip()
|
|
220
|
-
if not content:
|
|
221
|
-
return True
|
|
222
|
-
normalized = re.sub(r"\s+", " ", content.lower())
|
|
223
|
-
if normalized.startswith("202") and "[assistant]" in normalized and "[[reply_to_current]]" in normalized:
|
|
224
|
-
return True
|
|
225
|
-
if "**current target**" in normalized and "validation performed" in normalized:
|
|
226
|
-
return True
|
|
227
|
-
if normalized.startswith("recent memory worth reinforcing:"):
|
|
228
|
-
return True
|
|
229
|
-
if normalized.startswith("consolidated pattern:"):
|
|
230
|
-
return True
|
|
231
|
-
return False
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
def _dedupe_candidates(items: List[Dict[str, object]], limit: int) -> List[Dict[str, object]]:
|
|
235
|
-
deduped: List[Dict[str, object]] = []
|
|
236
|
-
seen: set[str] = set()
|
|
237
|
-
for item in items:
|
|
238
|
-
reference = str(item.get("reference") or "")
|
|
239
|
-
content = str(item.get("content") or "").strip()
|
|
240
|
-
if _low_value_candidate(item):
|
|
241
|
-
continue
|
|
242
|
-
normalized = re.sub(r"\s+", " ", content.lower())[:1200]
|
|
243
|
-
content_key = hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest() if normalized else ""
|
|
244
|
-
key = content_key or reference
|
|
245
|
-
if not key or key in seen or not content:
|
|
246
|
-
continue
|
|
247
|
-
seen.add(key)
|
|
248
|
-
deduped.append(item)
|
|
249
|
-
if len(deduped) >= limit:
|
|
250
|
-
break
|
|
251
|
-
return deduped
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
def _heuristic_summary(text: str, limit: int = 220) -> str:
|
|
255
|
-
collapsed = re.sub(r"\s+", " ", text or "").strip()
|
|
256
|
-
collapsed = re.sub(r"^\d{4}-\d{2}-\d{2}T[^ ]+\s+\[[^\]]+\]\s*", "", collapsed)
|
|
257
|
-
collapsed = re.sub(r"^\d{4}-\d{2}-\d{2}t[^ ]+\s+\[[^\]]+\]\s*", "", collapsed, flags=re.IGNORECASE)
|
|
258
|
-
collapsed = re.sub(r"^\[\[reply_to_current\]\]\s*", "", collapsed)
|
|
259
|
-
if len(collapsed) <= limit:
|
|
260
|
-
return collapsed
|
|
261
|
-
return f"{collapsed[: limit - 1].rstrip()}…"
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
def _needs_unresolved_refine(summary: str) -> bool:
|
|
265
|
-
text = (summary or "").strip().lower()
|
|
266
|
-
if not text:
|
|
267
|
-
return True
|
|
268
|
-
if text.startswith(("## ", "### ", "1)", "2)", "- ", "* ")):
|
|
269
|
-
return True
|
|
270
|
-
trigger_phrases = (
|
|
271
|
-
"next steps",
|
|
272
|
-
"open questions",
|
|
273
|
-
"recommended next action",
|
|
274
|
-
"current status",
|
|
275
|
-
"quick recap",
|
|
276
|
-
"paused",
|
|
277
|
-
"todo:",
|
|
278
|
-
)
|
|
279
|
-
return any(phrase in text for phrase in trigger_phrases)
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
def _heuristic_unresolved_rewrite(raw: str) -> str:
|
|
283
|
-
text = _heuristic_summary(raw, limit=500).strip()
|
|
284
|
-
lowered = text.lower()
|
|
285
|
-
text = re.sub(r"^(##+\s*|\*\*|\d+\)\s*)", "", text).strip("* ")
|
|
286
|
-
if lowered.startswith("todo:"):
|
|
287
|
-
body = text.split(":", 1)[1].strip() if ":" in text else text[5:].strip()
|
|
288
|
-
return _heuristic_summary(f"Outstanding task: {body}", limit=180)
|
|
289
|
-
if "next steps / open questions" in lowered or "current status / next steps" in lowered or "recommended next action" in lowered:
|
|
290
|
-
return "Review the linked note and extract the concrete pending decision or next action."
|
|
291
|
-
if lowered.startswith("paused"):
|
|
292
|
-
return "Resume the paused work from its saved checkpoint and confirm the next concrete action."
|
|
293
|
-
return _heuristic_summary(text, limit=180)
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
def _refine_unresolved_summary(summary: str, reference: str = "") -> str:
|
|
297
|
-
raw = _heuristic_summary(summary, limit=500)
|
|
298
|
-
if not _needs_unresolved_refine(raw):
|
|
299
|
-
return _heuristic_summary(raw)
|
|
300
|
-
if raw and not raw.startswith(("#", "*", "1)", "2)", "TODO:")) and len(raw.split()) >= 5:
|
|
301
|
-
return _heuristic_summary(raw, limit=180)
|
|
302
|
-
prompt = (
|
|
303
|
-
"Rewrite this unresolved item as one concise actionable unresolved summary. "
|
|
304
|
-
"Keep it under 180 characters. Focus on the decision, blocker, or next action. "
|
|
305
|
-
"Do not use markdown headings or numbering.\n\n"
|
|
306
|
-
f"Reference: {reference}\n"
|
|
307
|
-
f"Unresolved item: {raw}\n\n"
|
|
308
|
-
"Summary:"
|
|
309
|
-
)
|
|
310
|
-
result = _infer_with_timeout(prompt)
|
|
311
|
-
output = str(result.get("output") or "").strip()
|
|
312
|
-
cleaned = _SUMMARY_PREFIX_RE.sub("", output).strip()
|
|
313
|
-
if cleaned and len(cleaned) >= 12 and cleaned.lower() != raw.lower() and not _needs_unresolved_refine(cleaned):
|
|
314
|
-
return _heuristic_summary(cleaned, limit=180)
|
|
315
|
-
return _heuristic_unresolved_rewrite(raw)
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
def _heuristic_ponder(record: Dict[str, object]) -> Dict[str, str]:
|
|
319
|
-
text = str(record.get("content") or "").strip()
|
|
320
|
-
reference = str(record.get("reference") or "")
|
|
321
|
-
kind = str(record.get("candidate_kind") or "memory")
|
|
322
|
-
metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
|
|
323
|
-
summary = _heuristic_summary(text)
|
|
324
|
-
if kind == "checkpoint":
|
|
325
|
-
return {
|
|
326
|
-
"insight": f"Checkpoint captured active continuity: {summary}",
|
|
327
|
-
"recommendation": "Promote the checkpoint summary into durable reflections and keep linked open loops hydrated at answer time.",
|
|
328
|
-
}
|
|
329
|
-
if kind == "continuity_state":
|
|
330
|
-
return {
|
|
331
|
-
"insight": f"Conversation continuity still carries unresolved context: {summary}",
|
|
332
|
-
"recommendation": "Hydrate this scope before answering so pending actions and open loops stay visible after restarts.",
|
|
333
|
-
}
|
|
334
|
-
if kind == "turn":
|
|
335
|
-
role = str(metadata.get("role") or "conversation")
|
|
336
|
-
return {
|
|
337
|
-
"insight": f"Recent {role} turn changed active context: {summary}",
|
|
338
|
-
"recommendation": "Preserve only the decision, lesson, or next action from this turn instead of the full transcript wording.",
|
|
339
|
-
}
|
|
340
|
-
return {
|
|
341
|
-
"insight": f"Potential durable learning: {summary}",
|
|
342
|
-
"recommendation": "Capture the concrete lesson, decision, or next action so this memory is reusable instead of just retrievable.",
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
def _parse_structured_output(output: str) -> Dict[str, str]:
|
|
347
|
-
insight = ""
|
|
348
|
-
recommendation = ""
|
|
349
|
-
for line in output.splitlines():
|
|
350
|
-
if line.lower().startswith("insight:"):
|
|
351
|
-
insight = line.split(":", 1)[-1].strip()
|
|
352
|
-
elif line.lower().startswith("recommendation:"):
|
|
353
|
-
recommendation = line.split(":", 1)[-1].strip()
|
|
354
|
-
cleaned = [
|
|
355
|
-
_SUMMARY_PREFIX_RE.sub("", line).strip()
|
|
356
|
-
for line in output.splitlines()
|
|
357
|
-
if _SUMMARY_PREFIX_RE.sub("", line).strip()
|
|
358
|
-
]
|
|
359
|
-
if not insight and cleaned:
|
|
360
|
-
insight = cleaned[0]
|
|
361
|
-
if not recommendation and len(cleaned) > 1:
|
|
362
|
-
recommendation = cleaned[1]
|
|
363
|
-
return {"insight": insight[:280], "recommendation": recommendation[:280]}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
def _ponder_with_model(record: Dict[str, object]) -> Dict[str, str]:
|
|
367
|
-
text = str(record.get("content") or "").strip()
|
|
368
|
-
if not text:
|
|
369
|
-
return {"insight": "", "recommendation": ""}
|
|
370
|
-
prompt = (
|
|
371
|
-
"You are the memory pondering engine.\n"
|
|
372
|
-
"Given this memory/context item, return: (1) a concise insight, (2) a concrete recommendation.\n"
|
|
373
|
-
"Keep both actionable and under 220 characters each.\n\n"
|
|
374
|
-
f"Reference: {record.get('reference')}\n"
|
|
375
|
-
f"Kind: {record.get('candidate_kind') or 'memory'}\n"
|
|
376
|
-
f"Memory: {text}\n\n"
|
|
377
|
-
"Format:\nInsight: ...\nRecommendation: ..."
|
|
378
|
-
)
|
|
379
|
-
result = _infer_with_timeout(prompt)
|
|
380
|
-
output = str(result.get("output") or "").strip()
|
|
381
|
-
parsed = _parse_structured_output(output)
|
|
382
|
-
if parsed.get("insight") and parsed.get("recommendation"):
|
|
383
|
-
return parsed
|
|
384
|
-
heuristic = _heuristic_ponder(record)
|
|
385
|
-
return {
|
|
386
|
-
"insight": parsed.get("insight") or heuristic["insight"],
|
|
387
|
-
"recommendation": parsed.get("recommendation") or heuristic["recommendation"],
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
def _extract_lesson(record: Dict[str, object]) -> str | None:
|
|
392
|
-
text = str(record.get("content") or "").strip()
|
|
393
|
-
if not text:
|
|
394
|
-
return None
|
|
395
|
-
prompt = (
|
|
396
|
-
"Extract a single actionable lesson learned from this memory/context item.\n"
|
|
397
|
-
"If there is no clear lesson, reply with NONE. Keep it under 220 characters.\n\n"
|
|
398
|
-
f"Reference: {record.get('reference')}\n"
|
|
399
|
-
f"Memory: {text}\n\n"
|
|
400
|
-
"Lesson:"
|
|
401
|
-
)
|
|
402
|
-
result = _infer_with_timeout(prompt)
|
|
403
|
-
output = str(result.get("output") or "").strip()
|
|
404
|
-
if not output or output.upper().startswith("NONE"):
|
|
405
|
-
return None
|
|
406
|
-
output = _SUMMARY_PREFIX_RE.sub("", output).strip()
|
|
407
|
-
return output[:240] if output else None
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
def _memory_exists(memory_type: str, content: str, metadata: Optional[Dict[str, object]] = None) -> Optional[int]:
|
|
411
|
-
if memory_type not in _WRITABLE_MEMORY_TABLES:
|
|
412
|
-
return None
|
|
413
|
-
conn = store.connect(ensure_schema=False)
|
|
414
|
-
try:
|
|
415
|
-
rows = conn.execute(
|
|
416
|
-
f"SELECT id, metadata_json FROM {memory_type} WHERE content = ? ORDER BY id DESC LIMIT 25",
|
|
417
|
-
(content,),
|
|
418
|
-
).fetchall()
|
|
419
|
-
except Exception:
|
|
420
|
-
rows = []
|
|
421
|
-
finally:
|
|
422
|
-
conn.close()
|
|
423
|
-
if not rows:
|
|
424
|
-
return None
|
|
425
|
-
wanted_ref = str((metadata or {}).get("source_reference") or "")
|
|
426
|
-
for row in rows:
|
|
427
|
-
if not wanted_ref:
|
|
428
|
-
return int(row["id"])
|
|
429
|
-
try:
|
|
430
|
-
row_meta = json.loads(row["metadata_json"] or "{}")
|
|
431
|
-
except Exception:
|
|
432
|
-
row_meta = {}
|
|
433
|
-
if str(row_meta.get("source_reference") or "") == wanted_ref:
|
|
434
|
-
return int(row["id"])
|
|
435
|
-
return None
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
def _link_once(source_reference: str, link_type: str, target_reference: str) -> None:
|
|
439
|
-
if not source_reference or not target_reference:
|
|
440
|
-
return
|
|
441
|
-
existing = memory_links.get_memory_links(source_reference)
|
|
442
|
-
if any(item.get("link_type") == link_type and item.get("target_reference") == target_reference for item in existing):
|
|
443
|
-
return
|
|
444
|
-
memory_links.add_memory_link(source_reference, link_type, target_reference)
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
def _store_reflection(summary: str, *, source_reference: str, recommendation: str = "", metadata: Optional[Dict[str, object]] = None) -> str:
|
|
448
|
-
content = summary.strip()
|
|
449
|
-
if recommendation.strip():
|
|
450
|
-
content = f"{content}\nRecommendation: {recommendation.strip()}"
|
|
451
|
-
content = content.strip()
|
|
452
|
-
inherited_refs = provenance.collect_source_references(source_reference, depth=2) if source_reference else []
|
|
453
|
-
source_refs = [ref for ref in inherited_refs if ref]
|
|
454
|
-
if source_reference and source_reference not in source_refs:
|
|
455
|
-
source_refs.insert(0, source_reference)
|
|
456
|
-
reflection_metadata = {
|
|
457
|
-
**(metadata or {}),
|
|
458
|
-
"source_reference": source_reference,
|
|
459
|
-
"source_references": source_refs,
|
|
460
|
-
"kind": "ponder_reflection",
|
|
461
|
-
"derived_via": "ponder",
|
|
462
|
-
}
|
|
463
|
-
existing_id = _memory_exists("reflections", content, reflection_metadata)
|
|
464
|
-
if existing_id:
|
|
465
|
-
return f"reflections:{existing_id}"
|
|
466
|
-
reflection_id = api.store_memory("reflections", content, source="ponder", metadata=reflection_metadata)
|
|
467
|
-
reflection_ref = f"reflections:{reflection_id}"
|
|
468
|
-
_link_once(reflection_ref, "derived_from", source_reference)
|
|
469
|
-
return reflection_ref
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
def _store_lesson_once(lesson: str, *, source_reference: str) -> Optional[str]:
|
|
473
|
-
normalized = lesson.strip()
|
|
474
|
-
if not normalized:
|
|
475
|
-
return None
|
|
476
|
-
inherited_refs = provenance.collect_source_references(source_reference, depth=2) if source_reference else []
|
|
477
|
-
metadata = {
|
|
478
|
-
"reference": source_reference,
|
|
479
|
-
"source_reference": source_reference,
|
|
480
|
-
"source_references": inherited_refs or ([source_reference] if source_reference else []),
|
|
481
|
-
"kind": "ponder_lesson",
|
|
482
|
-
"derived_via": "ponder",
|
|
483
|
-
}
|
|
484
|
-
existing_id = _memory_exists("lessons", normalized, metadata)
|
|
485
|
-
if existing_id:
|
|
486
|
-
return f"lessons:{existing_id}"
|
|
487
|
-
lesson_id = api.store_memory("lessons", normalized, source="ponder", metadata=metadata)
|
|
488
|
-
lesson_ref = f"lessons:{lesson_id}"
|
|
489
|
-
_link_once(lesson_ref, "derived_from", source_reference)
|
|
490
|
-
return lesson_ref
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
def _candidate_memories(max_items: int) -> List[Dict[str, object]]:
|
|
494
|
-
base_candidates: List[Dict[str, object]] = []
|
|
495
|
-
for table in ("knowledge", "tasks", "runbooks", "lessons"):
|
|
496
|
-
base_candidates.extend(_load_recent(table, max_items))
|
|
497
|
-
base_candidates.extend(_load_continuity_candidates(max_items))
|
|
498
|
-
return _dedupe_candidates(base_candidates, max_items)
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
def run_ponder_cycle(max_items: int = 5) -> Dict[str, object]:
|
|
502
|
-
emit_event(LOGFILE, "brain_ponder_cycle_start", status="ok")
|
|
503
|
-
|
|
504
|
-
unresolved = _run_with_timeout(
|
|
505
|
-
"unresolved",
|
|
506
|
-
lambda: unresolved_state.list_unresolved_state(limit=max_items),
|
|
507
|
-
5.0,
|
|
508
|
-
[],
|
|
509
|
-
)
|
|
510
|
-
candidates = _candidate_memories(max_items)
|
|
511
|
-
consolidation = _run_with_timeout(
|
|
512
|
-
"consolidation",
|
|
513
|
-
lambda: memory_consolidation.consolidate_memories(candidates, max_clusters=max_items),
|
|
514
|
-
15.0,
|
|
515
|
-
{"consolidated": [], "reinforcement": []},
|
|
516
|
-
)
|
|
517
|
-
|
|
518
|
-
insights: List[Dict[str, object]] = []
|
|
519
|
-
for item in unresolved[:max_items]:
|
|
520
|
-
raw_summary = str(item.get("summary") or "").strip()
|
|
521
|
-
if not raw_summary:
|
|
522
|
-
continue
|
|
523
|
-
source_reference = str(item.get("reference") or "") or str(item.get("target_reference") or "")
|
|
524
|
-
summary = _refine_unresolved_summary(raw_summary, source_reference)
|
|
525
|
-
reflection_ref = _store_reflection(
|
|
526
|
-
f"Unresolved state remains active: {summary}",
|
|
527
|
-
source_reference=source_reference or "unresolved_state",
|
|
528
|
-
recommendation="Resolve or checkpoint this item so it stays visible during future hydration.",
|
|
529
|
-
metadata={"state_type": item.get("state_type"), "kind": "unresolved"},
|
|
530
|
-
)
|
|
531
|
-
insights.append(
|
|
532
|
-
{
|
|
533
|
-
"type": "unresolved",
|
|
534
|
-
"summary": summary,
|
|
535
|
-
"reference": source_reference,
|
|
536
|
-
"reflection_reference": reflection_ref,
|
|
537
|
-
}
|
|
538
|
-
)
|
|
539
|
-
emit_event(LOGFILE, "brain_ponder_insight_generated", status="ok", kind="unresolved")
|
|
540
|
-
|
|
541
|
-
if str(config.OCMEMOG_PONDER_ENABLED).lower() in {"1", "true", "yes"}:
|
|
542
|
-
for item in candidates:
|
|
543
|
-
content = str(item.get("content") or "").strip()
|
|
544
|
-
if not content:
|
|
545
|
-
continue
|
|
546
|
-
model_result = _ponder_with_model(item)
|
|
547
|
-
insight = str(model_result.get("insight") or "").strip()
|
|
548
|
-
recommendation = str(model_result.get("recommendation") or "").strip()
|
|
549
|
-
if not insight:
|
|
550
|
-
continue
|
|
551
|
-
reference = str(item.get("reference") or "")
|
|
552
|
-
reflection_ref = _store_reflection(
|
|
553
|
-
insight,
|
|
554
|
-
source_reference=reference or "ponder",
|
|
555
|
-
recommendation=recommendation,
|
|
556
|
-
metadata={
|
|
557
|
-
"candidate_kind": item.get("candidate_kind"),
|
|
558
|
-
"memory_type": item.get("memory_type"),
|
|
559
|
-
},
|
|
560
|
-
)
|
|
561
|
-
insights.append(
|
|
562
|
-
{
|
|
563
|
-
"type": str(item.get("candidate_kind") or "memory"),
|
|
564
|
-
"reference": reference,
|
|
565
|
-
"summary": insight,
|
|
566
|
-
"recommendation": recommendation,
|
|
567
|
-
"reflection_reference": reflection_ref,
|
|
568
|
-
}
|
|
569
|
-
)
|
|
570
|
-
emit_event(LOGFILE, "brain_ponder_insight_generated", status="ok", kind=str(item.get("candidate_kind") or "memory"))
|
|
571
|
-
|
|
572
|
-
lessons: List[Dict[str, object]] = []
|
|
573
|
-
if str(config.OCMEMOG_LESSON_MINING_ENABLED).lower() in {"1", "true", "yes"}:
|
|
574
|
-
for item in candidates:
|
|
575
|
-
reference = str(item.get("reference") or "")
|
|
576
|
-
if not reference:
|
|
577
|
-
continue
|
|
578
|
-
if not (reference.startswith("reflections:") or reference.startswith("conversation_checkpoints:")):
|
|
579
|
-
continue
|
|
580
|
-
lesson = _extract_lesson(item)
|
|
581
|
-
if not lesson:
|
|
582
|
-
continue
|
|
583
|
-
lesson_ref = _store_lesson_once(lesson, source_reference=reference)
|
|
584
|
-
lessons.append({"reference": reference, "lesson": lesson, "lesson_reference": lesson_ref})
|
|
585
|
-
emit_event(LOGFILE, "brain_ponder_lesson_generated", status="ok")
|
|
586
|
-
|
|
587
|
-
links: List[Dict[str, object]] = []
|
|
588
|
-
for cluster in consolidation.get("consolidated", []):
|
|
589
|
-
summary = str(cluster.get("summary") or "").strip()
|
|
590
|
-
if not summary:
|
|
591
|
-
continue
|
|
592
|
-
reflection_ref = _store_reflection(
|
|
593
|
-
f"Consolidated pattern: {summary}",
|
|
594
|
-
source_reference=str(cluster.get("references", ["ponder"])[0]),
|
|
595
|
-
recommendation=f"Review grouped references together ({int(cluster.get('count') or 0)} items).",
|
|
596
|
-
metadata={"kind": "cluster", "cluster_kind": cluster.get("memory_type")},
|
|
597
|
-
)
|
|
598
|
-
for target_reference in cluster.get("references", []) or []:
|
|
599
|
-
if isinstance(target_reference, str) and target_reference:
|
|
600
|
-
_link_once(reflection_ref, "conceptual", target_reference)
|
|
601
|
-
links.append(
|
|
602
|
-
{
|
|
603
|
-
"type": "cluster",
|
|
604
|
-
"summary": summary,
|
|
605
|
-
"count": int(cluster.get("count") or 0),
|
|
606
|
-
"references": cluster.get("references") or [],
|
|
607
|
-
"reflection_reference": reflection_ref,
|
|
608
|
-
}
|
|
609
|
-
)
|
|
610
|
-
|
|
611
|
-
maintenance = _run_with_timeout(
|
|
612
|
-
"integrity",
|
|
613
|
-
integrity.run_integrity_check,
|
|
614
|
-
10.0,
|
|
615
|
-
{"issues": []},
|
|
616
|
-
)
|
|
617
|
-
if "vector_orphan" in set(maintenance.get("repairable_issues") or []):
|
|
618
|
-
maintenance["repair"] = _run_with_timeout(
|
|
619
|
-
"integrity_repair",
|
|
620
|
-
integrity.repair_integrity,
|
|
621
|
-
10.0,
|
|
622
|
-
{"ok": False, "repaired": []},
|
|
623
|
-
)
|
|
624
|
-
maintenance = _run_with_timeout(
|
|
625
|
-
"integrity_post_repair",
|
|
626
|
-
integrity.run_integrity_check,
|
|
627
|
-
10.0,
|
|
628
|
-
maintenance,
|
|
629
|
-
)
|
|
630
|
-
if any(item.startswith("vector_missing") or item.startswith("vector_orphan") for item in maintenance.get("issues", [])):
|
|
631
|
-
rebuild_count = _run_with_timeout(
|
|
632
|
-
"vector_rebuild",
|
|
633
|
-
vector_index.rebuild_vector_index,
|
|
634
|
-
30.0,
|
|
635
|
-
0,
|
|
636
|
-
)
|
|
637
|
-
maintenance["vector_rebuild"] = rebuild_count
|
|
638
|
-
|
|
639
|
-
emit_event(
|
|
640
|
-
LOGFILE,
|
|
641
|
-
"brain_ponder_cycle_complete",
|
|
642
|
-
status="ok",
|
|
643
|
-
candidates=len(candidates),
|
|
644
|
-
insights=len(insights),
|
|
645
|
-
lessons=len(lessons),
|
|
646
|
-
links=len(links),
|
|
647
|
-
)
|
|
648
|
-
return {
|
|
649
|
-
"unresolved": unresolved,
|
|
650
|
-
"candidates": candidates,
|
|
651
|
-
"insights": insights,
|
|
652
|
-
"lessons": lessons,
|
|
653
|
-
"links": links,
|
|
654
|
-
"maintenance": maintenance,
|
|
655
|
-
"consolidation": consolidation,
|
|
656
|
-
}
|
|
3
|
+
from ocmemog.runtime.memory.pondering_engine import * # noqa: F401,F403
|