@simbimbo/memory-ocmemog 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/README.md +83 -18
  3. package/brain/runtime/__init__.py +2 -12
  4. package/brain/runtime/config.py +1 -24
  5. package/brain/runtime/inference.py +1 -151
  6. package/brain/runtime/instrumentation.py +1 -15
  7. package/brain/runtime/memory/__init__.py +3 -13
  8. package/brain/runtime/memory/api.py +1 -1219
  9. package/brain/runtime/memory/candidate.py +1 -185
  10. package/brain/runtime/memory/conversation_state.py +1 -1823
  11. package/brain/runtime/memory/distill.py +1 -344
  12. package/brain/runtime/memory/embedding_engine.py +1 -92
  13. package/brain/runtime/memory/freshness.py +1 -112
  14. package/brain/runtime/memory/health.py +1 -40
  15. package/brain/runtime/memory/integrity.py +1 -186
  16. package/brain/runtime/memory/memory_consolidation.py +1 -58
  17. package/brain/runtime/memory/memory_links.py +1 -107
  18. package/brain/runtime/memory/memory_salience.py +1 -233
  19. package/brain/runtime/memory/memory_synthesis.py +1 -31
  20. package/brain/runtime/memory/memory_taxonomy.py +1 -33
  21. package/brain/runtime/memory/pondering_engine.py +1 -654
  22. package/brain/runtime/memory/promote.py +1 -277
  23. package/brain/runtime/memory/provenance.py +1 -406
  24. package/brain/runtime/memory/reinforcement.py +1 -71
  25. package/brain/runtime/memory/retrieval.py +1 -210
  26. package/brain/runtime/memory/semantic_search.py +1 -64
  27. package/brain/runtime/memory/store.py +1 -429
  28. package/brain/runtime/memory/unresolved_state.py +1 -91
  29. package/brain/runtime/memory/vector_index.py +1 -323
  30. package/brain/runtime/model_roles.py +1 -9
  31. package/brain/runtime/model_router.py +1 -22
  32. package/brain/runtime/providers.py +1 -66
  33. package/brain/runtime/security/redaction.py +1 -12
  34. package/brain/runtime/state_store.py +1 -23
  35. package/brain/runtime/storage_paths.py +1 -39
  36. package/docs/architecture/memory.md +20 -24
  37. package/docs/release-checklist.md +19 -6
  38. package/docs/usage.md +33 -17
  39. package/index.ts +8 -1
  40. package/ocmemog/__init__.py +11 -0
  41. package/ocmemog/doctor.py +1255 -0
  42. package/ocmemog/runtime/__init__.py +18 -0
  43. package/ocmemog/runtime/_compat_bridge.py +28 -0
  44. package/ocmemog/runtime/config.py +34 -0
  45. package/ocmemog/runtime/identity.py +115 -0
  46. package/ocmemog/runtime/inference.py +163 -0
  47. package/ocmemog/runtime/instrumentation.py +20 -0
  48. package/ocmemog/runtime/memory/__init__.py +91 -0
  49. package/ocmemog/runtime/memory/api.py +1594 -0
  50. package/ocmemog/runtime/memory/candidate.py +192 -0
  51. package/ocmemog/runtime/memory/conversation_state.py +1831 -0
  52. package/ocmemog/runtime/memory/distill.py +282 -0
  53. package/ocmemog/runtime/memory/embedding_engine.py +151 -0
  54. package/ocmemog/runtime/memory/freshness.py +114 -0
  55. package/ocmemog/runtime/memory/health.py +93 -0
  56. package/ocmemog/runtime/memory/integrity.py +208 -0
  57. package/ocmemog/runtime/memory/memory_consolidation.py +60 -0
  58. package/ocmemog/runtime/memory/memory_links.py +109 -0
  59. package/ocmemog/runtime/memory/memory_salience.py +235 -0
  60. package/ocmemog/runtime/memory/memory_synthesis.py +33 -0
  61. package/ocmemog/runtime/memory/memory_taxonomy.py +35 -0
  62. package/ocmemog/runtime/memory/pondering_engine.py +681 -0
  63. package/ocmemog/runtime/memory/promote.py +279 -0
  64. package/ocmemog/runtime/memory/provenance.py +408 -0
  65. package/ocmemog/runtime/memory/reinforcement.py +73 -0
  66. package/ocmemog/runtime/memory/retrieval.py +224 -0
  67. package/ocmemog/runtime/memory/semantic_search.py +66 -0
  68. package/ocmemog/runtime/memory/store.py +433 -0
  69. package/ocmemog/runtime/memory/unresolved_state.py +93 -0
  70. package/ocmemog/runtime/memory/vector_index.py +411 -0
  71. package/ocmemog/runtime/model_roles.py +15 -0
  72. package/ocmemog/runtime/model_router.py +28 -0
  73. package/ocmemog/runtime/providers.py +78 -0
  74. package/ocmemog/runtime/roles.py +92 -0
  75. package/ocmemog/runtime/security/__init__.py +8 -0
  76. package/ocmemog/runtime/security/redaction.py +17 -0
  77. package/ocmemog/runtime/state_store.py +32 -0
  78. package/ocmemog/runtime/storage_paths.py +70 -0
  79. package/ocmemog/sidecar/app.py +421 -60
  80. package/ocmemog/sidecar/compat.py +50 -13
  81. package/ocmemog/sidecar/transcript_watcher.py +327 -242
  82. package/openclaw.plugin.json +4 -0
  83. package/package.json +1 -1
  84. package/scripts/ocmemog-backfill-vectors.py +5 -3
  85. package/scripts/ocmemog-continuity-benchmark.py +1 -1
  86. package/scripts/ocmemog-demo.py +1 -1
  87. package/scripts/ocmemog-doctor.py +15 -0
  88. package/scripts/ocmemog-install.sh +29 -7
  89. package/scripts/ocmemog-integrated-proof.py +374 -0
  90. package/scripts/ocmemog-reindex-vectors.py +5 -3
  91. package/scripts/ocmemog-release-check.sh +330 -0
  92. package/scripts/ocmemog-sidecar.sh +4 -2
  93. package/scripts/ocmemog-test-rig.py +5 -3
  94. package/brain/runtime/memory/artifacts.py +0 -33
  95. package/brain/runtime/memory/context_builder.py +0 -112
  96. package/brain/runtime/memory/interaction_memory.py +0 -57
  97. package/brain/runtime/memory/memory_gate.py +0 -38
  98. package/brain/runtime/memory/memory_graph.py +0 -54
  99. package/brain/runtime/memory/person_identity.py +0 -83
  100. package/brain/runtime/memory/person_memory.py +0 -138
  101. package/brain/runtime/memory/sentiment_memory.py +0 -67
  102. package/brain/runtime/memory/tool_catalog.py +0 -68
@@ -1,656 +1,3 @@
1
1
  from __future__ import annotations
2
2
 
3
- import hashlib
4
- import json
5
- import re
6
- import threading
7
- from queue import Queue
8
- from typing import Any, Callable, Dict, List, Optional
9
-
10
- from brain.runtime import config, inference, state_store
11
- from brain.runtime.instrumentation import emit_event
12
- from brain.runtime.memory import api, integrity, memory_consolidation, memory_links, provenance, store, unresolved_state, vector_index
13
-
14
- LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
15
- _WRITABLE_MEMORY_TABLES = set(store.MEMORY_TABLES)
16
- _SUMMARY_PREFIX_RE = re.compile(r"^(?:insight|recommendation|lesson)\s*:\s*", re.IGNORECASE)
17
-
18
-
19
- def _run_with_timeout(name: str, fn: Callable[[], Any], timeout_s: float, default: Any) -> Any:
20
- emit_event(LOGFILE, f"brain_ponder_{name}_start", status="ok")
21
- result_queue: Queue[tuple[str, Any]] = Queue(maxsize=1)
22
-
23
- def _target() -> None:
24
- try:
25
- result_queue.put(("ok", fn()))
26
- except Exception as exc: # pragma: no cover
27
- result_queue.put(("error", exc))
28
-
29
- worker = threading.Thread(target=_target, name=f"ocmemog-ponder-{name}", daemon=True)
30
- worker.start()
31
- worker.join(timeout_s)
32
- if worker.is_alive():
33
- emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="timeout")
34
- return default
35
- if result_queue.empty():
36
- emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="error", error="missing_result")
37
- return default
38
- status, payload = result_queue.get_nowait()
39
- if status == "error":
40
- emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="error", error=str(payload))
41
- return default
42
- emit_event(LOGFILE, f"brain_ponder_{name}_complete", status="ok")
43
- return payload
44
-
45
-
46
- def _infer_with_timeout(prompt: str, timeout_s: float = 20.0) -> Dict[str, str]:
47
- return _run_with_timeout(
48
- "infer",
49
- lambda: inference.infer(prompt, provider_name=config.OCMEMOG_PONDER_MODEL),
50
- timeout_s,
51
- {"status": "timeout", "output": ""},
52
- )
53
-
54
-
55
- def _load_recent(table: str, limit: int) -> List[Dict[str, object]]:
56
- if table not in _WRITABLE_MEMORY_TABLES:
57
- return []
58
- conn = store.connect(ensure_schema=False)
59
- try:
60
- rows = conn.execute(
61
- f"SELECT id, content, confidence, timestamp, source, metadata_json FROM {table} ORDER BY id DESC LIMIT ?",
62
- (limit,),
63
- ).fetchall()
64
- except Exception:
65
- rows = []
66
- finally:
67
- conn.close()
68
- items: List[Dict[str, object]] = []
69
- for row in rows:
70
- try:
71
- metadata = json.loads(row["metadata_json"] or "{}")
72
- except Exception:
73
- metadata = {}
74
- items.append(
75
- {
76
- "reference": f"{table}:{row['id']}",
77
- "content": str(row["content"] or ""),
78
- "confidence": float(row["confidence"] or 0.0),
79
- "timestamp": row["timestamp"],
80
- "source": row["source"],
81
- "metadata": metadata,
82
- "candidate_kind": "memory",
83
- "memory_type": table,
84
- }
85
- )
86
- return items
87
-
88
-
89
- def _load_continuity_candidates(limit: int) -> List[Dict[str, object]]:
90
- conn = store.connect(ensure_schema=False)
91
- items: List[Dict[str, object]] = []
92
- try:
93
- checkpoint_rows = conn.execute(
94
- """
95
- SELECT id, session_id, thread_id, conversation_id, summary, latest_user_ask,
96
- last_assistant_commitment, metadata_json, timestamp
97
- FROM conversation_checkpoints
98
- ORDER BY id DESC LIMIT ?
99
- """,
100
- (limit,),
101
- ).fetchall()
102
- for row in checkpoint_rows:
103
- try:
104
- metadata = json.loads(row["metadata_json"] or "{}")
105
- except Exception:
106
- metadata = {}
107
- content_parts = [str(row["summary"] or "").strip()]
108
- latest_user_ask = str(row["latest_user_ask"] or "").strip()
109
- if latest_user_ask:
110
- content_parts.append(f"User ask: {latest_user_ask}")
111
- last_commitment = str(row["last_assistant_commitment"] or "").strip()
112
- if last_commitment:
113
- content_parts.append(f"Assistant commitment: {last_commitment}")
114
- items.append(
115
- {
116
- "reference": f"conversation_checkpoints:{row['id']}",
117
- "content": " | ".join(part for part in content_parts if part),
118
- "timestamp": row["timestamp"],
119
- "source": "continuity",
120
- "metadata": {
121
- **metadata,
122
- "conversation_id": row["conversation_id"],
123
- "session_id": row["session_id"],
124
- "thread_id": row["thread_id"],
125
- },
126
- "candidate_kind": "checkpoint",
127
- "memory_type": "runbooks",
128
- }
129
- )
130
-
131
- state_rows = conn.execute(
132
- """
133
- SELECT id, scope_type, scope_id, latest_user_ask, last_assistant_commitment,
134
- open_loops_json, pending_actions_json, unresolved_state_json, metadata_json, updated_at
135
- FROM conversation_state
136
- ORDER BY updated_at DESC, id DESC LIMIT ?
137
- """,
138
- (limit,),
139
- ).fetchall()
140
- for row in state_rows:
141
- try:
142
- open_loops = json.loads(row["open_loops_json"] or "[]")
143
- except Exception:
144
- open_loops = []
145
- try:
146
- pending_actions = json.loads(row["pending_actions_json"] or "[]")
147
- except Exception:
148
- pending_actions = []
149
- try:
150
- unresolved_items = json.loads(row["unresolved_state_json"] or "[]")
151
- except Exception:
152
- unresolved_items = []
153
- try:
154
- metadata = json.loads(row["metadata_json"] or "{}")
155
- except Exception:
156
- metadata = {}
157
- content_parts = [f"Continuity scope {row['scope_type']}:{row['scope_id']}"]
158
- latest_user_ask = str(row["latest_user_ask"] or "").strip()
159
- if latest_user_ask:
160
- content_parts.append(f"Latest user ask: {latest_user_ask}")
161
- last_commitment = str(row["last_assistant_commitment"] or "").strip()
162
- if last_commitment:
163
- content_parts.append(f"Assistant commitment: {last_commitment}")
164
- for label, payload in (("Open loop", open_loops), ("Pending action", pending_actions), ("Unresolved", unresolved_items)):
165
- for item in payload[:2]:
166
- summary = str((item or {}).get("summary") or "").strip()
167
- if summary:
168
- content_parts.append(f"{label}: {summary}")
169
- items.append(
170
- {
171
- "reference": f"conversation_state:{row['id']}",
172
- "content": " | ".join(part for part in content_parts if part),
173
- "timestamp": row["updated_at"],
174
- "source": "continuity",
175
- "metadata": metadata,
176
- "candidate_kind": "continuity_state",
177
- "memory_type": "runbooks",
178
- }
179
- )
180
-
181
- turn_rows = conn.execute(
182
- """
183
- SELECT id, role, content, session_id, thread_id, conversation_id, message_id, metadata_json, timestamp
184
- FROM conversation_turns
185
- ORDER BY id DESC LIMIT ?
186
- """,
187
- (limit,),
188
- ).fetchall()
189
- for row in turn_rows:
190
- try:
191
- metadata = json.loads(row["metadata_json"] or "{}")
192
- except Exception:
193
- metadata = {}
194
- items.append(
195
- {
196
- "reference": f"conversation_turns:{row['id']}",
197
- "content": f"{row['role']}: {str(row['content'] or '').strip()}",
198
- "timestamp": row["timestamp"],
199
- "source": "continuity",
200
- "metadata": {
201
- **metadata,
202
- "conversation_id": row["conversation_id"],
203
- "session_id": row["session_id"],
204
- "thread_id": row["thread_id"],
205
- "message_id": row["message_id"],
206
- },
207
- "candidate_kind": "turn",
208
- "memory_type": "reflections",
209
- }
210
- )
211
- except Exception as exc:
212
- emit_event(LOGFILE, "brain_ponder_continuity_candidates_failed", status="error", error=str(exc))
213
- finally:
214
- conn.close()
215
- return items[:limit]
216
-
217
-
218
- def _low_value_candidate(record: Dict[str, object]) -> bool:
219
- content = str(record.get("content") or "").strip()
220
- if not content:
221
- return True
222
- normalized = re.sub(r"\s+", " ", content.lower())
223
- if normalized.startswith("202") and "[assistant]" in normalized and "[[reply_to_current]]" in normalized:
224
- return True
225
- if "**current target**" in normalized and "validation performed" in normalized:
226
- return True
227
- if normalized.startswith("recent memory worth reinforcing:"):
228
- return True
229
- if normalized.startswith("consolidated pattern:"):
230
- return True
231
- return False
232
-
233
-
234
- def _dedupe_candidates(items: List[Dict[str, object]], limit: int) -> List[Dict[str, object]]:
235
- deduped: List[Dict[str, object]] = []
236
- seen: set[str] = set()
237
- for item in items:
238
- reference = str(item.get("reference") or "")
239
- content = str(item.get("content") or "").strip()
240
- if _low_value_candidate(item):
241
- continue
242
- normalized = re.sub(r"\s+", " ", content.lower())[:1200]
243
- content_key = hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest() if normalized else ""
244
- key = content_key or reference
245
- if not key or key in seen or not content:
246
- continue
247
- seen.add(key)
248
- deduped.append(item)
249
- if len(deduped) >= limit:
250
- break
251
- return deduped
252
-
253
-
254
- def _heuristic_summary(text: str, limit: int = 220) -> str:
255
- collapsed = re.sub(r"\s+", " ", text or "").strip()
256
- collapsed = re.sub(r"^\d{4}-\d{2}-\d{2}T[^ ]+\s+\[[^\]]+\]\s*", "", collapsed)
257
- collapsed = re.sub(r"^\d{4}-\d{2}-\d{2}t[^ ]+\s+\[[^\]]+\]\s*", "", collapsed, flags=re.IGNORECASE)
258
- collapsed = re.sub(r"^\[\[reply_to_current\]\]\s*", "", collapsed)
259
- if len(collapsed) <= limit:
260
- return collapsed
261
- return f"{collapsed[: limit - 1].rstrip()}…"
262
-
263
-
264
- def _needs_unresolved_refine(summary: str) -> bool:
265
- text = (summary or "").strip().lower()
266
- if not text:
267
- return True
268
- if text.startswith(("## ", "### ", "1)", "2)", "- ", "* ")):
269
- return True
270
- trigger_phrases = (
271
- "next steps",
272
- "open questions",
273
- "recommended next action",
274
- "current status",
275
- "quick recap",
276
- "paused",
277
- "todo:",
278
- )
279
- return any(phrase in text for phrase in trigger_phrases)
280
-
281
-
282
- def _heuristic_unresolved_rewrite(raw: str) -> str:
283
- text = _heuristic_summary(raw, limit=500).strip()
284
- lowered = text.lower()
285
- text = re.sub(r"^(##+\s*|\*\*|\d+\)\s*)", "", text).strip("* ")
286
- if lowered.startswith("todo:"):
287
- body = text.split(":", 1)[1].strip() if ":" in text else text[5:].strip()
288
- return _heuristic_summary(f"Outstanding task: {body}", limit=180)
289
- if "next steps / open questions" in lowered or "current status / next steps" in lowered or "recommended next action" in lowered:
290
- return "Review the linked note and extract the concrete pending decision or next action."
291
- if lowered.startswith("paused"):
292
- return "Resume the paused work from its saved checkpoint and confirm the next concrete action."
293
- return _heuristic_summary(text, limit=180)
294
-
295
-
296
- def _refine_unresolved_summary(summary: str, reference: str = "") -> str:
297
- raw = _heuristic_summary(summary, limit=500)
298
- if not _needs_unresolved_refine(raw):
299
- return _heuristic_summary(raw)
300
- if raw and not raw.startswith(("#", "*", "1)", "2)", "TODO:")) and len(raw.split()) >= 5:
301
- return _heuristic_summary(raw, limit=180)
302
- prompt = (
303
- "Rewrite this unresolved item as one concise actionable unresolved summary. "
304
- "Keep it under 180 characters. Focus on the decision, blocker, or next action. "
305
- "Do not use markdown headings or numbering.\n\n"
306
- f"Reference: {reference}\n"
307
- f"Unresolved item: {raw}\n\n"
308
- "Summary:"
309
- )
310
- result = _infer_with_timeout(prompt)
311
- output = str(result.get("output") or "").strip()
312
- cleaned = _SUMMARY_PREFIX_RE.sub("", output).strip()
313
- if cleaned and len(cleaned) >= 12 and cleaned.lower() != raw.lower() and not _needs_unresolved_refine(cleaned):
314
- return _heuristic_summary(cleaned, limit=180)
315
- return _heuristic_unresolved_rewrite(raw)
316
-
317
-
318
- def _heuristic_ponder(record: Dict[str, object]) -> Dict[str, str]:
319
- text = str(record.get("content") or "").strip()
320
- reference = str(record.get("reference") or "")
321
- kind = str(record.get("candidate_kind") or "memory")
322
- metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
323
- summary = _heuristic_summary(text)
324
- if kind == "checkpoint":
325
- return {
326
- "insight": f"Checkpoint captured active continuity: {summary}",
327
- "recommendation": "Promote the checkpoint summary into durable reflections and keep linked open loops hydrated at answer time.",
328
- }
329
- if kind == "continuity_state":
330
- return {
331
- "insight": f"Conversation continuity still carries unresolved context: {summary}",
332
- "recommendation": "Hydrate this scope before answering so pending actions and open loops stay visible after restarts.",
333
- }
334
- if kind == "turn":
335
- role = str(metadata.get("role") or "conversation")
336
- return {
337
- "insight": f"Recent {role} turn changed active context: {summary}",
338
- "recommendation": "Preserve only the decision, lesson, or next action from this turn instead of the full transcript wording.",
339
- }
340
- return {
341
- "insight": f"Potential durable learning: {summary}",
342
- "recommendation": "Capture the concrete lesson, decision, or next action so this memory is reusable instead of just retrievable.",
343
- }
344
-
345
-
346
- def _parse_structured_output(output: str) -> Dict[str, str]:
347
- insight = ""
348
- recommendation = ""
349
- for line in output.splitlines():
350
- if line.lower().startswith("insight:"):
351
- insight = line.split(":", 1)[-1].strip()
352
- elif line.lower().startswith("recommendation:"):
353
- recommendation = line.split(":", 1)[-1].strip()
354
- cleaned = [
355
- _SUMMARY_PREFIX_RE.sub("", line).strip()
356
- for line in output.splitlines()
357
- if _SUMMARY_PREFIX_RE.sub("", line).strip()
358
- ]
359
- if not insight and cleaned:
360
- insight = cleaned[0]
361
- if not recommendation and len(cleaned) > 1:
362
- recommendation = cleaned[1]
363
- return {"insight": insight[:280], "recommendation": recommendation[:280]}
364
-
365
-
366
- def _ponder_with_model(record: Dict[str, object]) -> Dict[str, str]:
367
- text = str(record.get("content") or "").strip()
368
- if not text:
369
- return {"insight": "", "recommendation": ""}
370
- prompt = (
371
- "You are the memory pondering engine.\n"
372
- "Given this memory/context item, return: (1) a concise insight, (2) a concrete recommendation.\n"
373
- "Keep both actionable and under 220 characters each.\n\n"
374
- f"Reference: {record.get('reference')}\n"
375
- f"Kind: {record.get('candidate_kind') or 'memory'}\n"
376
- f"Memory: {text}\n\n"
377
- "Format:\nInsight: ...\nRecommendation: ..."
378
- )
379
- result = _infer_with_timeout(prompt)
380
- output = str(result.get("output") or "").strip()
381
- parsed = _parse_structured_output(output)
382
- if parsed.get("insight") and parsed.get("recommendation"):
383
- return parsed
384
- heuristic = _heuristic_ponder(record)
385
- return {
386
- "insight": parsed.get("insight") or heuristic["insight"],
387
- "recommendation": parsed.get("recommendation") or heuristic["recommendation"],
388
- }
389
-
390
-
391
- def _extract_lesson(record: Dict[str, object]) -> str | None:
392
- text = str(record.get("content") or "").strip()
393
- if not text:
394
- return None
395
- prompt = (
396
- "Extract a single actionable lesson learned from this memory/context item.\n"
397
- "If there is no clear lesson, reply with NONE. Keep it under 220 characters.\n\n"
398
- f"Reference: {record.get('reference')}\n"
399
- f"Memory: {text}\n\n"
400
- "Lesson:"
401
- )
402
- result = _infer_with_timeout(prompt)
403
- output = str(result.get("output") or "").strip()
404
- if not output or output.upper().startswith("NONE"):
405
- return None
406
- output = _SUMMARY_PREFIX_RE.sub("", output).strip()
407
- return output[:240] if output else None
408
-
409
-
410
- def _memory_exists(memory_type: str, content: str, metadata: Optional[Dict[str, object]] = None) -> Optional[int]:
411
- if memory_type not in _WRITABLE_MEMORY_TABLES:
412
- return None
413
- conn = store.connect(ensure_schema=False)
414
- try:
415
- rows = conn.execute(
416
- f"SELECT id, metadata_json FROM {memory_type} WHERE content = ? ORDER BY id DESC LIMIT 25",
417
- (content,),
418
- ).fetchall()
419
- except Exception:
420
- rows = []
421
- finally:
422
- conn.close()
423
- if not rows:
424
- return None
425
- wanted_ref = str((metadata or {}).get("source_reference") or "")
426
- for row in rows:
427
- if not wanted_ref:
428
- return int(row["id"])
429
- try:
430
- row_meta = json.loads(row["metadata_json"] or "{}")
431
- except Exception:
432
- row_meta = {}
433
- if str(row_meta.get("source_reference") or "") == wanted_ref:
434
- return int(row["id"])
435
- return None
436
-
437
-
438
- def _link_once(source_reference: str, link_type: str, target_reference: str) -> None:
439
- if not source_reference or not target_reference:
440
- return
441
- existing = memory_links.get_memory_links(source_reference)
442
- if any(item.get("link_type") == link_type and item.get("target_reference") == target_reference for item in existing):
443
- return
444
- memory_links.add_memory_link(source_reference, link_type, target_reference)
445
-
446
-
447
- def _store_reflection(summary: str, *, source_reference: str, recommendation: str = "", metadata: Optional[Dict[str, object]] = None) -> str:
448
- content = summary.strip()
449
- if recommendation.strip():
450
- content = f"{content}\nRecommendation: {recommendation.strip()}"
451
- content = content.strip()
452
- inherited_refs = provenance.collect_source_references(source_reference, depth=2) if source_reference else []
453
- source_refs = [ref for ref in inherited_refs if ref]
454
- if source_reference and source_reference not in source_refs:
455
- source_refs.insert(0, source_reference)
456
- reflection_metadata = {
457
- **(metadata or {}),
458
- "source_reference": source_reference,
459
- "source_references": source_refs,
460
- "kind": "ponder_reflection",
461
- "derived_via": "ponder",
462
- }
463
- existing_id = _memory_exists("reflections", content, reflection_metadata)
464
- if existing_id:
465
- return f"reflections:{existing_id}"
466
- reflection_id = api.store_memory("reflections", content, source="ponder", metadata=reflection_metadata)
467
- reflection_ref = f"reflections:{reflection_id}"
468
- _link_once(reflection_ref, "derived_from", source_reference)
469
- return reflection_ref
470
-
471
-
472
- def _store_lesson_once(lesson: str, *, source_reference: str) -> Optional[str]:
473
- normalized = lesson.strip()
474
- if not normalized:
475
- return None
476
- inherited_refs = provenance.collect_source_references(source_reference, depth=2) if source_reference else []
477
- metadata = {
478
- "reference": source_reference,
479
- "source_reference": source_reference,
480
- "source_references": inherited_refs or ([source_reference] if source_reference else []),
481
- "kind": "ponder_lesson",
482
- "derived_via": "ponder",
483
- }
484
- existing_id = _memory_exists("lessons", normalized, metadata)
485
- if existing_id:
486
- return f"lessons:{existing_id}"
487
- lesson_id = api.store_memory("lessons", normalized, source="ponder", metadata=metadata)
488
- lesson_ref = f"lessons:{lesson_id}"
489
- _link_once(lesson_ref, "derived_from", source_reference)
490
- return lesson_ref
491
-
492
-
493
- def _candidate_memories(max_items: int) -> List[Dict[str, object]]:
494
- base_candidates: List[Dict[str, object]] = []
495
- for table in ("knowledge", "tasks", "runbooks", "lessons"):
496
- base_candidates.extend(_load_recent(table, max_items))
497
- base_candidates.extend(_load_continuity_candidates(max_items))
498
- return _dedupe_candidates(base_candidates, max_items)
499
-
500
-
501
- def run_ponder_cycle(max_items: int = 5) -> Dict[str, object]:
502
- emit_event(LOGFILE, "brain_ponder_cycle_start", status="ok")
503
-
504
- unresolved = _run_with_timeout(
505
- "unresolved",
506
- lambda: unresolved_state.list_unresolved_state(limit=max_items),
507
- 5.0,
508
- [],
509
- )
510
- candidates = _candidate_memories(max_items)
511
- consolidation = _run_with_timeout(
512
- "consolidation",
513
- lambda: memory_consolidation.consolidate_memories(candidates, max_clusters=max_items),
514
- 15.0,
515
- {"consolidated": [], "reinforcement": []},
516
- )
517
-
518
- insights: List[Dict[str, object]] = []
519
- for item in unresolved[:max_items]:
520
- raw_summary = str(item.get("summary") or "").strip()
521
- if not raw_summary:
522
- continue
523
- source_reference = str(item.get("reference") or "") or str(item.get("target_reference") or "")
524
- summary = _refine_unresolved_summary(raw_summary, source_reference)
525
- reflection_ref = _store_reflection(
526
- f"Unresolved state remains active: {summary}",
527
- source_reference=source_reference or "unresolved_state",
528
- recommendation="Resolve or checkpoint this item so it stays visible during future hydration.",
529
- metadata={"state_type": item.get("state_type"), "kind": "unresolved"},
530
- )
531
- insights.append(
532
- {
533
- "type": "unresolved",
534
- "summary": summary,
535
- "reference": source_reference,
536
- "reflection_reference": reflection_ref,
537
- }
538
- )
539
- emit_event(LOGFILE, "brain_ponder_insight_generated", status="ok", kind="unresolved")
540
-
541
- if str(config.OCMEMOG_PONDER_ENABLED).lower() in {"1", "true", "yes"}:
542
- for item in candidates:
543
- content = str(item.get("content") or "").strip()
544
- if not content:
545
- continue
546
- model_result = _ponder_with_model(item)
547
- insight = str(model_result.get("insight") or "").strip()
548
- recommendation = str(model_result.get("recommendation") or "").strip()
549
- if not insight:
550
- continue
551
- reference = str(item.get("reference") or "")
552
- reflection_ref = _store_reflection(
553
- insight,
554
- source_reference=reference or "ponder",
555
- recommendation=recommendation,
556
- metadata={
557
- "candidate_kind": item.get("candidate_kind"),
558
- "memory_type": item.get("memory_type"),
559
- },
560
- )
561
- insights.append(
562
- {
563
- "type": str(item.get("candidate_kind") or "memory"),
564
- "reference": reference,
565
- "summary": insight,
566
- "recommendation": recommendation,
567
- "reflection_reference": reflection_ref,
568
- }
569
- )
570
- emit_event(LOGFILE, "brain_ponder_insight_generated", status="ok", kind=str(item.get("candidate_kind") or "memory"))
571
-
572
- lessons: List[Dict[str, object]] = []
573
- if str(config.OCMEMOG_LESSON_MINING_ENABLED).lower() in {"1", "true", "yes"}:
574
- for item in candidates:
575
- reference = str(item.get("reference") or "")
576
- if not reference:
577
- continue
578
- if not (reference.startswith("reflections:") or reference.startswith("conversation_checkpoints:")):
579
- continue
580
- lesson = _extract_lesson(item)
581
- if not lesson:
582
- continue
583
- lesson_ref = _store_lesson_once(lesson, source_reference=reference)
584
- lessons.append({"reference": reference, "lesson": lesson, "lesson_reference": lesson_ref})
585
- emit_event(LOGFILE, "brain_ponder_lesson_generated", status="ok")
586
-
587
- links: List[Dict[str, object]] = []
588
- for cluster in consolidation.get("consolidated", []):
589
- summary = str(cluster.get("summary") or "").strip()
590
- if not summary:
591
- continue
592
- reflection_ref = _store_reflection(
593
- f"Consolidated pattern: {summary}",
594
- source_reference=str(cluster.get("references", ["ponder"])[0]),
595
- recommendation=f"Review grouped references together ({int(cluster.get('count') or 0)} items).",
596
- metadata={"kind": "cluster", "cluster_kind": cluster.get("memory_type")},
597
- )
598
- for target_reference in cluster.get("references", []) or []:
599
- if isinstance(target_reference, str) and target_reference:
600
- _link_once(reflection_ref, "conceptual", target_reference)
601
- links.append(
602
- {
603
- "type": "cluster",
604
- "summary": summary,
605
- "count": int(cluster.get("count") or 0),
606
- "references": cluster.get("references") or [],
607
- "reflection_reference": reflection_ref,
608
- }
609
- )
610
-
611
- maintenance = _run_with_timeout(
612
- "integrity",
613
- integrity.run_integrity_check,
614
- 10.0,
615
- {"issues": []},
616
- )
617
- if "vector_orphan" in set(maintenance.get("repairable_issues") or []):
618
- maintenance["repair"] = _run_with_timeout(
619
- "integrity_repair",
620
- integrity.repair_integrity,
621
- 10.0,
622
- {"ok": False, "repaired": []},
623
- )
624
- maintenance = _run_with_timeout(
625
- "integrity_post_repair",
626
- integrity.run_integrity_check,
627
- 10.0,
628
- maintenance,
629
- )
630
- if any(item.startswith("vector_missing") or item.startswith("vector_orphan") for item in maintenance.get("issues", [])):
631
- rebuild_count = _run_with_timeout(
632
- "vector_rebuild",
633
- vector_index.rebuild_vector_index,
634
- 30.0,
635
- 0,
636
- )
637
- maintenance["vector_rebuild"] = rebuild_count
638
-
639
- emit_event(
640
- LOGFILE,
641
- "brain_ponder_cycle_complete",
642
- status="ok",
643
- candidates=len(candidates),
644
- insights=len(insights),
645
- lessons=len(lessons),
646
- links=len(links),
647
- )
648
- return {
649
- "unresolved": unresolved,
650
- "candidates": candidates,
651
- "insights": insights,
652
- "lessons": lessons,
653
- "links": links,
654
- "maintenance": maintenance,
655
- "consolidation": consolidation,
656
- }
3
+ from ocmemog.runtime.memory.pondering_engine import * # noqa: F401,F403