@simbimbo/memory-ocmemog 0.1.4 → 0.1.5

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.1.5 — 2026-03-18
+
+Repair and hardening follow-up after the 0.1.4 publish.
+
+### Highlights
+- Fixed vector reindex defaults so repair scripts use provider-backed Ollama embeddings instead of silently rebuilding weak local/hash vectors
+- Added battery-aware sidecar defaults for macOS laptops (`OCMEMOG_LAPTOP_MODE=auto|ac|battery`)
+- Fixed `record_reinforcement()` so new experiences preserve `memory_reference`, and added integrity repair to backfill legacy missing references
+- Added incremental vector backfill tooling (`scripts/ocmemog-backfill-vectors.py`) for non-destructive backlog repair
+- Cleaned freshness summaries so junk placeholders (`promoted`, `summary`, `No local memory summary available`) do not pollute advisories
+- Improved integrity reporting to count duplicate promotion groups accurately
+
+### Notes
+- Historical vector backlog still exists and should be burned down in staged backfills, especially for `knowledge`
+- Detailed repair notes: `docs/notes/2026-03-18-memory-repair-and-backfill.md`
+
 ## 0.1.4 — 2026-03-18
 
 Package ownership + runtime safety release.

package/README.md

@@ -83,14 +83,15 @@ Optional environment variables:
 - `OCMEMOG_TRANSCRIPT_ROOTS` (comma-separated allowed roots for transcript context retrieval; default: `~/.openclaw/workspace/memory`)
 - `OCMEMOG_API_TOKEN` (optional; if set, requests must include `x-ocmemog-token` or `Authorization: Bearer ...`)
 - `OCMEMOG_AUTO_HYDRATION` (`true` to re-enable prompt-time continuity prepending; defaults to `false` as a safety guard until the host runtime is verified not to persist prepended context into session history)
+- `OCMEMOG_LAPTOP_MODE` (`auto` by default; on macOS battery power this slows watcher polling, reduces ingest batch size, and disables sentiment reinforcement unless explicitly overridden)
 - `OCMEMOG_USE_OLLAMA` (`true` to use Ollama for distill/inference)
 - `OCMEMOG_OLLAMA_HOST` (default: `http://127.0.0.1:11434`)
-- `OCMEMOG_OLLAMA_MODEL` (default: `phi3:latest`)
+- `OCMEMOG_OLLAMA_MODEL` (default: `phi3:latest`; lightweight local fallback / cheap cognition)
 - `OCMEMOG_OLLAMA_EMBED_MODEL` (default: `nomic-embed-text:latest`)
 - `OCMEMOG_PROMOTION_THRESHOLD` (default: `0.5`)
 - `OCMEMOG_DEMOTION_THRESHOLD` (default: `0.2`)
 - `OCMEMOG_PONDER_ENABLED` (default: `true`)
-- `OCMEMOG_PONDER_MODEL` (default: `OCMEMOG_MEMORY_MODEL`)
+- `OCMEMOG_PONDER_MODEL` (default via launcher: `qwen2.5:7b`; recommended for structured local memory refinement)
 - `OCMEMOG_LESSON_MINING_ENABLED` (default: `true`)
 
 ## Security
@@ -153,7 +154,7 @@ launchctl bootstrap gui/$UID scripts/launchagents/com.openclaw.ocmemog.guard.pli
 
 ## Recent changes
 
-### 0.1.4 (unreleased / current main)
+### 0.1.5 (current main)
 
 Package ownership + runtime safety release:
 - Publish package under `@simbimbo/memory-ocmemog` instead of the unauthorized `@openclaw` scope
@@ -221,3 +222,4 @@ Current limitations before broader public rollout:
 - Public release/distribution metadata is still being tightened up
 
 When a richer path is unavailable, the sidecar is designed to fail soft with explicit warnings rather than crash.
+ soft with explicit warnings rather than crash.

package/brain/runtime/memory/api.py

@@ -102,17 +102,18 @@ def store_memory(
 def record_reinforcement(task_id: str, outcome: str, note: str, *, source_module: str | None = None) -> None:
     outcome = _sanitize(outcome)
     note = _sanitize(note)
+    memory_reference = f"reinforcement:{task_id or 'unknown'}:{source_module or 'unspecified'}"
     def _write() -> None:
         conn = store.connect()
         try:
             conn.execute(
-                "INSERT INTO experiences (task_id, outcome, reward_score, confidence, experience_type, source_module, schema_version) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?)",
-                (task_id, outcome, None, 1.0, "reinforcement", source_module, store.SCHEMA_VERSION),
+                "INSERT INTO experiences (task_id, outcome, reward_score, confidence, memory_reference, experience_type, source_module, schema_version) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                (task_id, outcome, None, 1.0, memory_reference, "reinforcement", source_module, store.SCHEMA_VERSION),
             )
             conn.execute(
                 "INSERT INTO memory_events (event_type, source, details_json, schema_version) VALUES (?, ?, ?, ?)",
-                ("reinforcement_note", source_module, json.dumps({"task_id": task_id, "note": note}), store.SCHEMA_VERSION),
+                ("reinforcement_note", source_module, json.dumps({"task_id": task_id, "note": note, "memory_reference": memory_reference}), store.SCHEMA_VERSION),
             )
             conn.commit()
         finally:

package/brain/runtime/memory/freshness.py

@@ -12,6 +12,29 @@ DEFAULT_CONFIDENCE_THRESHOLD = 0.6
 DEFAULT_LIMIT = 25
 
 
+_BAD_SUMMARY_PREFIXES = (
+    "promoted",
+    "candidate_promoted",
+    "no local memory summary available",
+    "summary",
+)
+
+
+def _summary_from_content(content: Any, limit: int = 120) -> str:
+    text = str(content or "").strip()
+    if not text:
+        return "(empty memory content)"
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    for line in lines:
+        lowered = line.lower()
+        if lowered in _BAD_SUMMARY_PREFIXES:
+            continue
+        if any(lowered.startswith(prefix + ":") for prefix in _BAD_SUMMARY_PREFIXES):
+            continue
+        return line[:limit]
+    return "(needs summary cleanup)"
+
+
 def scan_freshness(
     stale_days: int = DEFAULT_STALE_DAYS,
     confidence_threshold: float = DEFAULT_CONFIDENCE_THRESHOLD,
@@ -66,7 +89,7 @@ def scan_freshness(
                 "memory_id": row["id"],
                 "timestamp": row["timestamp"],
                 "confidence": confidence,
-                "summary": str(row["content"])[:120],
+                "summary": _summary_from_content(row["content"]),
                 "freshness_score": round(freshness_score, 3),
                 "refresh_recommended": refresh_recommended,
             }

package/brain/runtime/memory/integrity.py

@@ -50,11 +50,11 @@ def run_integrity_check() -> Dict[str, Any]:
 
     # duplicate promotions
     try:
-        dup = conn.execute(
-            "SELECT COUNT(*) FROM promotions GROUP BY source, content HAVING COUNT(*) > 1",
-        ).fetchone()
-        if dup:
-            issues.append("duplicate_promotions")
+        dup_groups = conn.execute(
+            "SELECT COUNT(*) FROM (SELECT 1 FROM promotions GROUP BY source, content HAVING COUNT(*) > 1)",
+        ).fetchone()[0]
+        if dup_groups:
+            issues.append(f"duplicate_promotions:{dup_groups}")
             emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
     except Exception:
         pass
@@ -66,6 +66,7 @@ def run_integrity_check() -> Dict[str, Any]:
         ).fetchone()[0]
         if missing_ref:
             issues.append(f"missing_memory_reference:{missing_ref}")
+            repairable.append("missing_memory_reference")
             emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
     except Exception:
         pass
@@ -141,6 +142,7 @@ def repair_integrity() -> Dict[str, Any]:
     def _write() -> Dict[str, Any]:
         conn = store.connect()
         removed_orphans = 0
+        repaired_missing_refs = 0
         try:
             tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
             if "vector_embeddings" in tables:
@@ -158,8 +160,19 @@ def repair_integrity() -> Dict[str, Any]:
                         """,
                         (table,),
                     ).rowcount
+            if "experiences" in tables:
+                repaired_missing_refs += conn.execute(
+                    """
+                    UPDATE experiences
+                    SET memory_reference = 'legacy:' || COALESCE(experience_type, 'unknown') || ':' || id
+                    WHERE memory_reference IS NULL OR memory_reference = ''
+                    """
+                ).rowcount
             conn.commit()
-            return {"removed_orphan_vectors": int(removed_orphans)}
+            return {
+                "removed_orphan_vectors": int(removed_orphans),
+                "repaired_missing_memory_references": int(repaired_missing_refs),
+            }
         finally:
             conn.close()
 
@@ -167,4 +180,7 @@ def repair_integrity() -> Dict[str, Any]:
     if int(result.get("removed_orphan_vectors") or 0) > 0:
         repaired.append(f"vector_orphan:{int(result['removed_orphan_vectors'])}")
         emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_repair", status="ok", repaired="vector_orphan", count=int(result["removed_orphan_vectors"]))
+    if int(result.get("repaired_missing_memory_references") or 0) > 0:
+        repaired.append(f"missing_memory_reference:{int(result['repaired_missing_memory_references'])}")
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_repair", status="ok", repaired="missing_memory_reference", count=int(result["repaired_missing_memory_references"]))
     return {"ok": True, "repaired": repaired, **result}

package/brain/runtime/memory/pondering_engine.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import re
 import threading
@@ -214,13 +215,33 @@ def _load_continuity_candidates(limit: int) -> List[Dict[str, object]]:
     return items[:limit]
 
 
+def _low_value_candidate(record: Dict[str, object]) -> bool:
+    content = str(record.get("content") or "").strip()
+    if not content:
+        return True
+    normalized = re.sub(r"\s+", " ", content.lower())
+    if normalized.startswith("202") and "[assistant]" in normalized and "[[reply_to_current]]" in normalized:
+        return True
+    if "**current target**" in normalized and "validation performed" in normalized:
+        return True
+    if normalized.startswith("recent memory worth reinforcing:"):
+        return True
+    if normalized.startswith("consolidated pattern:"):
+        return True
+    return False
+
+
 def _dedupe_candidates(items: List[Dict[str, object]], limit: int) -> List[Dict[str, object]]:
     deduped: List[Dict[str, object]] = []
     seen: set[str] = set()
     for item in items:
         reference = str(item.get("reference") or "")
         content = str(item.get("content") or "").strip()
-        key = reference or content.lower()
+        if _low_value_candidate(item):
+            continue
+        normalized = re.sub(r"\s+", " ", content.lower())[:1200]
+        content_key = hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest() if normalized else ""
+        key = content_key or reference
         if not key or key in seen or not content:
             continue
         seen.add(key)
@@ -232,11 +253,66 @@ def _dedupe_candidates(items: List[Dict[str, object]], limit: int) -> List[Dict[
 
 def _heuristic_summary(text: str, limit: int = 220) -> str:
     collapsed = re.sub(r"\s+", " ", text or "").strip()
+    collapsed = re.sub(r"^\d{4}-\d{2}-\d{2}T[^ ]+\s+\[[^\]]+\]\s*", "", collapsed)
+    collapsed = re.sub(r"^\d{4}-\d{2}-\d{2}t[^ ]+\s+\[[^\]]+\]\s*", "", collapsed, flags=re.IGNORECASE)
+    collapsed = re.sub(r"^\[\[reply_to_current\]\]\s*", "", collapsed)
     if len(collapsed) <= limit:
         return collapsed
     return f"{collapsed[: limit - 1].rstrip()}…"
 
 
+def _needs_unresolved_refine(summary: str) -> bool:
+    text = (summary or "").strip().lower()
+    if not text:
+        return True
+    if text.startswith(("## ", "### ", "1)", "2)", "- ", "* ")):
+        return True
+    trigger_phrases = (
+        "next steps",
+        "open questions",
+        "recommended next action",
+        "current status",
+        "quick recap",
+        "paused",
+        "todo:",
+    )
+    return any(phrase in text for phrase in trigger_phrases)
+
+
+def _heuristic_unresolved_rewrite(raw: str) -> str:
+    text = _heuristic_summary(raw, limit=500).strip()
+    lowered = text.lower()
+    text = re.sub(r"^(##+\s*|\*\*|\d+\)\s*)", "", text).strip("* ")
+    if lowered.startswith("todo:"):
+        body = text.split(":", 1)[1].strip() if ":" in text else text[5:].strip()
+        return _heuristic_summary(f"Outstanding task: {body}", limit=180)
+    if "next steps / open questions" in lowered or "current status / next steps" in lowered or "recommended next action" in lowered:
+        return "Review the linked note and extract the concrete pending decision or next action."
+    if lowered.startswith("paused"):
+        return "Resume the paused work from its saved checkpoint and confirm the next concrete action."
+    return _heuristic_summary(text, limit=180)
+
+
+def _refine_unresolved_summary(summary: str, reference: str = "") -> str:
+    raw = _heuristic_summary(summary, limit=500)
+    if not _needs_unresolved_refine(raw):
+        return _heuristic_summary(raw)
+    prompt = (
+        "Rewrite this unresolved item as one concise actionable unresolved summary. "
+        "Keep it under 180 characters. Focus on the decision, blocker, or next action. "
+        "Do not use markdown headings or numbering.\n\n"
+        f"Reference: {reference}\n"
+        f"Unresolved item: {raw}\n\n"
+        "Summary:"
+    )
+    result = _infer_with_timeout(prompt)
+    output = str(result.get("output") or "").strip()
+    cleaned = _SUMMARY_PREFIX_RE.sub("", output).strip()
+    if cleaned and len(cleaned) >= 12 and cleaned.lower() != raw.lower() and not _needs_unresolved_refine(cleaned):
+        return _heuristic_summary(cleaned, limit=180)
+    return _heuristic_unresolved_rewrite(raw)
+
+
 def _heuristic_ponder(record: Dict[str, object]) -> Dict[str, str]:
     text = str(record.get("content") or "").strip()
     reference = str(record.get("reference") or "")
@@ -256,12 +332,12 @@ def _heuristic_ponder(record: Dict[str, object]) -> Dict[str, str]:
     if kind == "turn":
         role = str(metadata.get("role") or "conversation")
         return {
-            "insight": f"Recent {role} turn may shape near-term continuity: {summary}",
-            "recommendation": "Retain the turn in short-horizon context and checkpoint if it changes the active branch or next action.",
+            "insight": f"Recent {role} turn changed active context: {summary}",
+            "recommendation": "Preserve only the decision, lesson, or next action from this turn instead of the full transcript wording.",
         }
     return {
-        "insight": f"Recent memory worth reinforcing: {summary}",
-        "recommendation": "Link the reflection back to its source memory so future retrieval can hydrate it with provenance.",
+        "insight": f"Potential durable learning: {summary}",
+        "recommendation": "Capture the concrete lesson, decision, or next action so this memory is reusable instead of just retrievable.",
     }
 
 
@@ -414,7 +490,7 @@ def _store_lesson_once(lesson: str, *, source_reference: str) -> Optional[str]:
 
 def _candidate_memories(max_items: int) -> List[Dict[str, object]]:
     base_candidates: List[Dict[str, object]] = []
-    for table in ("reflections", "knowledge", "tasks", "runbooks"):
+    for table in ("knowledge", "tasks", "runbooks", "lessons"):
         base_candidates.extend(_load_recent(table, max_items))
     base_candidates.extend(_load_continuity_candidates(max_items))
     return _dedupe_candidates(base_candidates, max_items)
@@ -439,10 +515,11 @@ def run_ponder_cycle(max_items: int = 5) -> Dict[str, object]:
 
     insights: List[Dict[str, object]] = []
     for item in unresolved[:max_items]:
-        summary = str(item.get("summary") or "").strip()
-        if not summary:
+        raw_summary = str(item.get("summary") or "").strip()
+        if not raw_summary:
             continue
         source_reference = str(item.get("reference") or "") or str(item.get("target_reference") or "")
+        summary = _refine_unresolved_summary(raw_summary, source_reference)
         reflection_ref = _store_reflection(
             f"Unresolved state remains active: {summary}",
             source_reference=source_reference or "unresolved_state",

package/brain/runtime/memory/vector_index.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import math
+import re
 import threading
 from typing import Any, Dict, List, Iterable
 
@@ -22,6 +24,9 @@ EMBEDDING_TABLES: tuple[str, ...] = (
 )
 _REBUILD_LOCK = threading.Lock()
 _WRITE_CHUNK_SIZE = 64
+_EMBEDDING_TEXT_LIMIT = 8000
+_HTML_TAG_RE = re.compile(r"<[^>]+>")
+_WHITESPACE_RE = re.compile(r"\s+")
 
 
 def _ensure_vector_table(conn) -> None:
@@ -94,30 +99,66 @@ def insert_memory(memory_id: int, content: str, confidence: float, *, source_typ
     store.submit_write(_write, timeout=30.0)
 
 
-def _load_table_rows(table: str, *, limit: int | None = None, descending: bool = False) -> List[Dict[str, Any]]:
+def _load_table_rows(table: str, *, limit: int | None = None, descending: bool = False, missing_only: bool = False) -> List[Dict[str, Any]]:
     conn = store.connect()
     try:
         order = "DESC" if descending else "ASC"
+        where = ""
+        params: list[Any] = []
+        if missing_only:
+            where = " WHERE CAST(id AS TEXT) NOT IN (SELECT source_id FROM vector_embeddings WHERE source_type = ?)"
+            params.append(table)
         if limit is None:
             rows = conn.execute(
-                f"SELECT id, content, confidence, metadata_json FROM {table} ORDER BY id {order}",
+                f"SELECT id, content, confidence, metadata_json FROM {table}{where} ORDER BY id {order}",
+                tuple(params),
             ).fetchall()
         else:
             rows = conn.execute(
-                f"SELECT id, content, confidence, metadata_json FROM {table} ORDER BY id {order} LIMIT ?",
-                (limit,),
+                f"SELECT id, content, confidence, metadata_json FROM {table}{where} ORDER BY id {order} LIMIT ?",
+                tuple(params + [limit]),
             ).fetchall()
     finally:
         conn.close()
     return [dict(row) for row in rows]
 
 
+def _embedding_input(text: str, *, table: str = "knowledge") -> str:
+    cleaned = _HTML_TAG_RE.sub(" ", text)
+    cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip()
+    lowered = cleaned.lower()
+    artifactish = (
+        "| chunk " in lowered
+        or ".sql" in lowered
+        or "topology/" in lowered
+        or cleaned.count("),(") >= 8
+    )
+    if table == "knowledge" and artifactish:
+        return cleaned[:500]
+    if table == "knowledge" and len(cleaned) > 9000:
+        return cleaned[:1000]
+    if table == "reflections" and len(cleaned) > 8000:
+        return cleaned[:1200]
+    if len(cleaned) > 20000:
+        return cleaned[:2000]
+    if len(cleaned) > 12000:
+        return cleaned[:4000]
+    return cleaned[:_EMBEDDING_TEXT_LIMIT]
+
+
 def _prepare_embedding_rows(rows: Iterable[Dict[str, Any]], *, table: str) -> List[Dict[str, Any]]:
     prepared: List[Dict[str, Any]] = []
+    embedding_cache: Dict[str, List[float] | None] = {}
     for row in rows:
         content = str(row.get("content") or "")
         redacted_content, changed = redaction.redact_text(content)
-        embedding = embedding_engine.generate_embedding(redacted_content)
+        embedding_input = _embedding_input(redacted_content, table=table)
+        cache_key = hashlib.sha256(embedding_input.encode("utf-8", errors="ignore")).hexdigest()
+        if cache_key in embedding_cache:
+            embedding = embedding_cache[cache_key]
+        else:
+            embedding = embedding_engine.generate_embedding(embedding_input)
+            embedding_cache[cache_key] = embedding
         if not embedding:
             continue
         try:
@@ -213,6 +254,27 @@ def rebuild_vector_index(*, tables: Iterable[str] | None = None) -> int:
     return count
 
 
+def backfill_missing_vectors(*, tables: Iterable[str] | None = None, limit_per_table: int | None = None) -> int:
+    emit_event(LOGFILE, "brain_memory_vector_backfill_start", status="ok")
+    if not _REBUILD_LOCK.acquire(blocking=False):
+        emit_event(LOGFILE, "brain_memory_vector_backfill_complete", status="skipped", reason="already_running")
+        return 0
+    count = 0
+    try:
+        requested_tables = [table for table in (tables or EMBEDDING_TABLES) if table in EMBEDDING_TABLES]
+        for table in requested_tables:
+            prepared = _prepare_embedding_rows(
+                _load_table_rows(table, limit=limit_per_table, missing_only=True),
+                table=table,
+            )
+            for offset in range(0, len(prepared), _WRITE_CHUNK_SIZE):
+                count += _write_embedding_chunk(table, prepared[offset: offset + _WRITE_CHUNK_SIZE])
+    finally:
+        _REBUILD_LOCK.release()
+    emit_event(LOGFILE, "brain_memory_vector_backfill_complete", status="ok", indexed=count)
+    return count
+
+
 def search_memory(query: str, limit: int = 5) -> List[Dict[str, Any]]:
     emit_event(LOGFILE, "brain_memory_vector_search_start", status="ok")
     conn = store.connect()

package/docs/notes/2026-03-18-memory-repair-and-backfill.md

@@ -0,0 +1,70 @@
+# 2026-03-18 — Memory repair, integrity cleanup, and backfill tooling
+
+## Summary
+This pass focused on turning `ocmemog` from a noisy/fragile memory stack into a more repairable and laptop-safe system. The work addressed:
+- bad default vector rebuild behavior
+- misleading health/compat signals
+- missing `memory_reference` writer debt
+- poor freshness summaries
+- lack of an incremental vector backfill path
+- battery-unfriendly defaults in the sidecar launcher
+
+## Changes landed
+
+### Embedding and rebuild behavior
+- Fixed the vector reindex entrypoint so it defaults to provider-backed Ollama embeddings instead of silently rebuilding weak hash/simple vectors.
+- Confirmed local Ollama embeddings (`nomic-embed-text:latest`) are available and produce 768-dim vectors.
+- Added a new incremental repair path:
+  - `backfill_missing_vectors()` in `brain/runtime/memory/vector_index.py`
+  - `scripts/ocmemog-backfill-vectors.py`
+- This gives a non-destructive, table-by-table, chunkable way to backfill missing vectors without requiring a full destructive rebuild.
+
+### Integrity and writer correctness
+- Fixed `record_reinforcement()` so new `experiences` rows preserve a deterministic `memory_reference`.
+- Added repair support for legacy rows missing `memory_reference`.
+- Ran integrity repair and backfilled `1807` missing references.
+- Fixed duplicate promotion integrity reporting so grouped duplicate counts are reported accurately.
+
+### Health and output quality
+- Fixed sidecar compat/health reporting so provider-backed embeddings do not falsely report local hash fallback warnings.
+- Cleaned freshness summaries so placeholder content like `promoted`, `candidate_promoted`, `summary`, and `No local memory summary available` do not pollute advisories.
+- Junk-only rows now surface as `(needs summary cleanup)` instead of pretending they contain a meaningful summary.
+
+### Laptop/battery-aware behavior
+- Added battery-aware defaults to `scripts/ocmemog-sidecar.sh`.
+- `OCMEMOG_LAPTOP_MODE=auto|ac|battery` now controls watcher/ingest aggressiveness.
+- On battery the sidecar uses slower polling, smaller batches, and disables sentiment reinforcement by default.
+
+## Current integrity state
+After writer/reference repair:
+- `missing_memory_reference` debt is cleared
+- remaining integrity issue is primarily vector backlog:
+  - `vector_missing:19935`
+
+Observed coverage snapshot during staged backfill work:
+- `knowledge`: 15999 rows, 0 vectors
+- `runbooks`: 179 rows, 152 vectors
+- `lessons`: 76 rows, 76 vectors
+- `directives`: 233 rows, 206 vectors
+- `reflections`: 3460 rows, 83 vectors
+- `tasks`: 505 rows, 0 vectors
+
+## Why backlog remains
+The remaining `vector_missing` debt is mostly historical backlog rather than an active write-path failure. Existing new writes can index correctly; the old corpus simply was never fully rebuilt under the corrected provider-backed embedding path.
+
+## Recommended staged follow-up
+For laptop-friendly backlog burn-down, use staged backfills in roughly this order:
+1. directives
+2. tasks
+3. runbooks
+4. lessons
+5. reflections
+6. knowledge last
+
+## Commits from this sweep
+- `f3d3dd9` — fix: default vector reindex to ollama embeddings
+- `759d23d` — feat: add battery-aware sidecar defaults
+- `4a102eb` — fix: clean memory freshness summaries
+- `9ee7966` — fix: report duplicate promotion counts accurately
+- `8704db9` — fix: preserve and repair experience memory references
+- `5dc3cb9` — feat: add incremental vector backfill tooling

package/docs/notes/local-model-role-matrix-2026-03-18.md

@@ -0,0 +1,50 @@
+# Local model role matrix — 2026-03-18
+
+Purpose: document which installed local model is best suited for which `ocmemog` task so background cognition can be smarter without putting heavy/slow models on every path.
+
+Installed local models observed:
+- `phi3:latest`
+- `qwen2.5:7b`
+- `llama3.1:8b`
+- embeddings: `nomic-embed-text:latest`
+
+## Intended decision areas
+- unresolved-state rewrite
+- lesson extraction
+- ponder/reflection shaping
+- cluster recommendation wording
+- fallback/speed path
+
+## Bakeoff results
+
+### Unresolved-state rewrite
+- **Winner:** `qwen2.5:7b`
+- Why: cleanest concise rewrite, best instruction-following, least rambling.
+- Notes:
+  - `phi3:latest` tended to be verbose and occasionally hallucination-prone.
+  - `llama3.1:8b` produced one outright unusable response ("None found...").
+
+### Lesson extraction
+- **Winner:** `qwen2.5:7b`
+- Strong alternate: `llama3.1:8b`
+- Why: `qwen2.5:7b` produced the clearest operational lesson with good cause/effect preservation.
+- Notes:
+  - `phi3:latest` was weaker and more generic.
+
+### Cluster insight / recommendation shaping
+- **Winner:** `qwen2.5:7b`
+- Why: best structured output, least fluff, most concrete recommendation wording.
+- Notes:
+  - `llama3.1:8b` was decent but more wordy/stylized.
+  - `phi3:latest` timed out or underperformed on this task.
+
+## Recommended model-role split
+- embeddings: `nomic-embed-text:latest`
+- fast fallback cognition: `phi3:latest`
+- default structured memory refinement / ponder model: `qwen2.5:7b`
+- richer optional background cognition: `llama3.1:8b`
+
+## Operational recommendation
+- Keep `OCMEMOG_OLLAMA_MODEL=phi3:latest` for lightweight local fallback behavior.
+- Set `OCMEMOG_PONDER_MODEL=qwen2.5:7b` for unresolved-state rewrite, lesson extraction, and cluster recommendation shaping.
+- Consider `llama3.1:8b` for optional deeper background cognition passes where latency is acceptable.

package/docs/usage.md

@@ -34,6 +34,8 @@ export OCMEMOG_TRANSCRIPT_WATCHER=true
 ./scripts/ocmemog-sidecar.sh
 ```
 
+On macOS laptops, the launcher defaults to `OCMEMOG_LAPTOP_MODE=auto`, which detects battery power and uses lower-impact watcher settings automatically. Override with `OCMEMOG_LAPTOP_MODE=ac` for wall-power behavior or `OCMEMOG_LAPTOP_MODE=battery` to force conservative mode.
+
 Useful environment variables:
 
 ```bash

package/ocmemog/sidecar/app.py

@@ -34,6 +34,32 @@ QUEUE_STATS = {
 }
 
 
+def _queue_stats_path() -> Path:
+    path = state_store.data_dir() / "queue_stats.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _load_queue_stats() -> None:
+    path = _queue_stats_path()
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return
+    if not isinstance(data, dict):
+        return
+    for key in list(QUEUE_STATS.keys()):
+        if key in data:
+            QUEUE_STATS[key] = data[key]
+
+
+def _save_queue_stats() -> None:
+    path = _queue_stats_path()
+    tmp = path.with_suffix('.tmp')
+    tmp.write_text(json.dumps(QUEUE_STATS, indent=2, sort_keys=True), encoding='utf-8')
+    tmp.replace(path)
+
+
 @app.middleware("http")
 async def _auth_middleware(request: Request, call_next):
     if API_TOKEN:
@@ -177,6 +203,7 @@ def _process_queue(limit: Optional[int] = None) -> Dict[str, Any]:
         QUEUE_STATS["errors"] += errors
     if last_error:
         QUEUE_STATS["last_error"] = last_error
+    _save_queue_stats()
     return {"processed": processed, "errors": errors, "last_error": last_error}
 
 
@@ -388,7 +415,38 @@ def _fallback_search(query: str, limit: int, categories: List[str]) -> List[Dict
         conn.close()
 
 
+_ALLOWED_MEMORY_REFERENCE_TYPES = {
+    "knowledge",
+    "reflections",
+    "directives",
+    "tasks",
+    "runbooks",
+    "lessons",
+    "conversation_turns",
+    "conversation_checkpoints",
+}
+
+
+def _parse_reference(reference: str) -> tuple[str, str] | None:
+    if not isinstance(reference, str) or ":" not in reference:
+        return None
+    prefix, identifier = reference.split(":", 1)
+    prefix = prefix.strip()
+    identifier = identifier.strip()
+    if not prefix or not identifier:
+        return None
+    return prefix, identifier
+
+
 def _get_row(reference: str) -> Optional[Dict[str, Any]]:
+    parsed = _parse_reference(reference)
+    if not parsed:
+        return None
+    prefix, identifier = parsed
+    if prefix not in _ALLOWED_MEMORY_REFERENCE_TYPES:
+        return None
+    if prefix in {"knowledge", "reflections", "directives", "tasks", "runbooks", "lessons", "conversation_turns", "conversation_checkpoints"} and not identifier.isdigit():
+        return None
     return provenance.hydrate_reference(reference, depth=2)
 
 
@@ -541,11 +599,38 @@ def memory_search(request: SearchRequest) -> dict[str, Any]:
 @app.post("/memory/get")
 def memory_get(request: GetRequest) -> dict[str, Any]:
     runtime = _runtime_payload()
-    row = _get_row(request.reference)
+    parsed = _parse_reference(request.reference)
+    if not parsed:
+        return {
+            "ok": False,
+            "error": "invalid_reference",
+            "message": "Reference must be in the form type:id",
+            "reference": request.reference,
+            **runtime,
+        }
+    prefix, identifier = parsed
+    if prefix not in _ALLOWED_MEMORY_REFERENCE_TYPES:
+        return {
+            "ok": False,
+            "error": "unsupported_reference_type",
+            "message": f"Unsupported memory reference type: {prefix}",
+            "reference": request.reference,
+            **runtime,
+        }
+    if prefix in {"knowledge", "reflections", "directives", "tasks", "runbooks", "lessons", "conversation_turns", "conversation_checkpoints"} and not identifier.isdigit():
+        return {
+            "ok": False,
+            "error": "invalid_reference_id",
+            "message": f"Reference id for {prefix} must be numeric",
+            "reference": request.reference,
+            **runtime,
+        }
+    row = provenance.hydrate_reference(request.reference, depth=2)
     if row is None:
         return {
             "ok": False,
-            "error": "TODO: memory reference was not found or is not yet supported by the sidecar.",
+            "error": "reference_not_found",
+            "message": "Reference was well-formed but no matching memory was found",
             "reference": request.reference,
             **runtime,
         }
@@ -761,11 +846,19 @@ def memory_ponder_latest(limit: int = 5) -> dict[str, Any]:
             meta = json.loads(row["metadata_json"] or "{}")
         except Exception:
             meta = {}
+        content = str(row["content"] or "")
+        summary = content
+        recommendation = meta.get("recommendation")
+        if "\nRecommendation:" in content:
+            summary, _, tail = content.partition("\nRecommendation:")
+            summary = summary.strip()
+            if not recommendation:
+                recommendation = tail.strip()
         items.append({
             "reference": f"reflections:{row['id']}",
             "timestamp": row["timestamp"],
-            "summary": row["content"],
-            "recommendation": meta.get("recommendation"),
+            "summary": summary,
+            "recommendation": recommendation,
             "source_reference": meta.get("source_reference") or ((meta.get("provenance") or {}).get("source_reference") if isinstance(meta.get("provenance"), dict) else None),
             "provenance": provenance.preview_from_metadata(meta),
         })
@@ -958,6 +1051,20 @@ def metrics() -> dict[str, Any]:
     counts["queue_processed"] = QUEUE_STATS.get("processed", 0)
     counts["queue_errors"] = QUEUE_STATS.get("errors", 0)
     payload["counts"] = counts
+    coverage_tables = ["knowledge", "runbooks", "lessons", "directives", "reflections", "tasks"]
+    conn = store.connect()
+    try:
+        payload["coverage"] = [
+            {
+                "table": table,
+                "rows": int(counts.get(table, 0) or 0),
+                "vectors": int(conn.execute("SELECT COUNT(*) FROM vector_embeddings WHERE source_type=?", (table,)).fetchone()[0] or 0),
+                "missing": max(int(counts.get(table, 0) or 0) - int(conn.execute("SELECT COUNT(*) FROM vector_embeddings WHERE source_type=?", (table,)).fetchone()[0] or 0), 0),
+            }
+            for table in coverage_tables
+        ]
+    finally:
+        conn.close()
     payload["queue"] = QUEUE_STATS
     return {"ok": True, "metrics": payload, **runtime}
 
@@ -997,8 +1104,37 @@ def _tail_events(limit: int = 50) -> str:
 def dashboard() -> HTMLResponse:
     metrics_payload = health.get_memory_health()
     counts = metrics_payload.get("counts", {})
+    coverage_tables = ["knowledge", "runbooks", "lessons", "directives", "reflections", "tasks"]
+    conn = store.connect()
+    try:
+        coverage_rows = []
+        for table in coverage_tables:
+            total = int(counts.get(table, 0) or 0)
+            vectors = int(
+                conn.execute(
+                    "SELECT COUNT(*) FROM vector_embeddings WHERE source_type=?",
+                    (table,),
+                ).fetchone()[0]
+                or 0
+            )
+            missing = max(total - vectors, 0)
+            coverage_rows.append({"table": table, "rows": total, "vectors": vectors, "missing": missing})
+    finally:
+        conn.close()
+
+    metrics_cards = [{"label": key, "value": value} for key, value in counts.items()]
+    metrics_cards.extend(
+        [
+            {"label": "vector_index_count", "value": metrics_payload.get("vector_index_count", 0)},
+            {"label": "vector_index_coverage", "value": metrics_payload.get("vector_index_coverage", 0)},
+        ]
+    )
     metrics_html = "".join(
-        f"<div class='card'><strong>{key}</strong><br/>{value}</div>" for key, value in counts.items()
+        f"<div class='card'><strong>{card['label']}</strong><br/>{card['value']}</div>" for card in metrics_cards
+    )
+    coverage_html = "".join(
+        f"<div class='card'><strong>{row['table']}</strong><br/>rows: {row['rows']}<br/>vectors: {row['vectors']}<br/>missing: {row['missing']}</div>"
+        for row in coverage_rows
     )
     events_html = _tail_events()
 
@@ -1018,6 +1154,8 @@ def dashboard() -> HTMLResponse:
     <body>
       <h2>ocmemog realtime</h2>
       <div class="metrics" id="metrics">{metrics_html}</div>
+      <h3>Vector coverage</h3>
+      <div class="metrics" id="coverage">{coverage_html}</div>
       <h3>Ponder recommendations</h3>
       <div id="ponder-meta" style="margin-bottom:8px; color:#666;"></div>
       <div id="ponder"></div>
@@ -1025,6 +1163,7 @@ def dashboard() -> HTMLResponse:
       <pre id="events">{events_html}</pre>
       <script>
         const metricsEl = document.getElementById('metrics');
+        const coverageEl = document.getElementById('coverage');
         const ponderEl = document.getElementById('ponder');
         const ponderMetaEl = document.getElementById('ponder-meta');
         const eventsEl = document.getElementById('events');
@@ -1033,8 +1172,17 @@ def dashboard() -> HTMLResponse:
           const res = await fetch('/metrics');
           const data = await res.json();
           const counts = data.metrics?.counts || {{}};
-          metricsEl.innerHTML = Object.entries(counts).map(([k,v]) =>
-            `<div class=\"card\"><strong>${{k}}</strong><br/>${{v}}</div>`
+          const cards = [
+            ...Object.entries(counts).map(([k, v]) => ({{ label: k, value: v }})),
+            {{ label: 'vector_index_count', value: data.metrics?.vector_index_count ?? 0 }},
+            {{ label: 'vector_index_coverage', value: data.metrics?.vector_index_coverage ?? 0 }},
+          ];
+          metricsEl.innerHTML = cards.map((card) =>
+            `<div class="card"><strong>${{card.label}}</strong><br/>${{card.value}}</div>`
+          ).join('');
+          const coverage = data.metrics?.coverage || [];
+          coverageEl.innerHTML = coverage.map((row) =>
+            `<div class="card"><strong>${{row.table}}</strong><br/>rows: ${{row.rows}}<br/>vectors: ${{row.vectors}}<br/>missing: ${{row.missing}}</div>`
           ).join('');
         }}
 
@@ -1047,7 +1195,7 @@ def dashboard() -> HTMLResponse:
           const mode = data.mode || 'n/a';
           ponderMetaEl.textContent = `Last update: ${{lastTs}} • Mode: ${{mode}}${{warnings ? ' • ' + warnings : ''}}`;
           ponderEl.innerHTML = items.map((item) =>
-            `<div class=\"card\"><strong>${{item.summary}}</strong><br/><em>${{item.recommendation || ''}}</em><br/><small>${{item.timestamp || ''}} • ${{item.reference || ''}}</small></div>`
+            `<div class="card"><strong>${{item.summary}}</strong><br/><em>${{item.recommendation || ''}}</em><br/><small>${{item.timestamp || ''}} • ${{item.reference || ''}}</small></div>`
           ).join('');
         }}
 

package/ocmemog/sidecar/compat.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import importlib
 import importlib.util
+import os
 from dataclasses import dataclass
 from typing import Any
 
@@ -35,7 +36,8 @@ def probe_runtime() -> RuntimeStatus:
         except Exception as exc:
             missing_deps.append(f"{module_name}: {exc}")
 
-    if importlib.util.find_spec("sentence_transformers") is None:
+    provider = os.environ.get("BRAIN_EMBED_MODEL_PROVIDER", "").strip().lower()
+    if importlib.util.find_spec("sentence_transformers") is None and provider not in {"ollama", "openai", "openai_compatible", "openai-compatible", "local-ollama"}:
         warnings.append("Optional dependency missing: sentence-transformers; using local hash embeddings.")
 
     try:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@simbimbo/memory-ocmemog",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Advanced OpenClaw memory plugin with durable recall, transcript-backed continuity, and sidecar APIs",
   "license": "MIT",
   "repository": {

package/scripts/ocmemog-backfill-vectors.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT))
+
+os.environ.setdefault("OCMEMOG_USE_OLLAMA", "true")
+os.environ.setdefault("OCMEMOG_OLLAMA_MODEL", "phi3:latest")
+os.environ.setdefault("OCMEMOG_OLLAMA_EMBED_MODEL", "nomic-embed-text:latest")
+os.environ.setdefault("BRAIN_EMBED_MODEL_PROVIDER", "ollama")
+os.environ.setdefault("BRAIN_EMBED_MODEL_LOCAL", "")
+os.environ.setdefault("OCMEMOG_STATE_DIR", str(REPO_ROOT / ".ocmemog-state"))
+
+from brain.runtime.memory import vector_index
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Backfill missing vector embeddings without clearing existing ones")
+    parser.add_argument("--table", dest="tables", action="append", help="Table to backfill (repeatable)")
+    parser.add_argument("--limit-per-table", type=int, default=None, help="Optional max missing rows per table")
+    args = parser.parse_args()
+    count = vector_index.backfill_missing_vectors(tables=args.tables, limit_per_table=args.limit_per_table)
+    print(f"backfilled: {count}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/ocmemog-reindex-vectors.py

@@ -1,12 +1,20 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 
+import os
 import sys
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(REPO_ROOT))
 
+os.environ.setdefault("OCMEMOG_USE_OLLAMA", "true")
+os.environ.setdefault("OCMEMOG_OLLAMA_MODEL", "phi3:latest")
+os.environ.setdefault("OCMEMOG_OLLAMA_EMBED_MODEL", "nomic-embed-text:latest")
+os.environ.setdefault("BRAIN_EMBED_MODEL_PROVIDER", "ollama")
+os.environ.setdefault("BRAIN_EMBED_MODEL_LOCAL", "")
+os.environ.setdefault("OCMEMOG_STATE_DIR", str(REPO_ROOT / ".ocmemog-state"))
+
 from brain.runtime.memory import vector_index
 
 if __name__ == "__main__":

package/scripts/ocmemog-sidecar.sh

@@ -13,21 +13,47 @@ export OCMEMOG_STATE_DIR="${OCMEMOG_STATE_DIR:-${ROOT_DIR}/.ocmemog-state}"
 export PYTHONPATH="${ROOT_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
 mkdir -p "${OCMEMOG_STATE_DIR}" "${OCMEMOG_STATE_DIR}/logs"
 
+is_on_battery() {
+  if [[ "$(uname -s)" != "Darwin" ]]; then
+    return 1
+  fi
+  command -v pmset >/dev/null 2>&1 || return 1
+  pmset -g batt 2>/dev/null | grep -q "Battery Power"
+}
+
+LAPTOP_MODE="${OCMEMOG_LAPTOP_MODE:-auto}"
+if [[ "$LAPTOP_MODE" == "auto" ]]; then
+  if is_on_battery; then
+    LAPTOP_MODE="battery"
+  else
+    LAPTOP_MODE="ac"
+  fi
+fi
+export OCMEMOG_LAPTOP_MODE="$LAPTOP_MODE"
+
 # defaults for local ollama-backed inference/embeddings
 export OCMEMOG_USE_OLLAMA="${OCMEMOG_USE_OLLAMA:-true}"
 export OCMEMOG_OLLAMA_MODEL="${OCMEMOG_OLLAMA_MODEL:-phi3:latest}"
 export OCMEMOG_OLLAMA_EMBED_MODEL="${OCMEMOG_OLLAMA_EMBED_MODEL:-nomic-embed-text:latest}"
+export OCMEMOG_PONDER_MODEL="${OCMEMOG_PONDER_MODEL:-qwen2.5:7b}"
 export BRAIN_EMBED_MODEL_PROVIDER="${BRAIN_EMBED_MODEL_PROVIDER:-ollama}"
 export BRAIN_EMBED_MODEL_LOCAL="${BRAIN_EMBED_MODEL_LOCAL:-}"
 
-# always-on transcript watcher defaults
+# battery-aware transcript watcher defaults
 export OCMEMOG_TRANSCRIPT_WATCHER="${OCMEMOG_TRANSCRIPT_WATCHER:-true}"
 export OCMEMOG_SESSION_DIR="${OCMEMOG_SESSION_DIR:-$HOME/.openclaw/agents/main/sessions}"
-export OCMEMOG_TRANSCRIPT_POLL_SECONDS="${OCMEMOG_TRANSCRIPT_POLL_SECONDS:-30}"
-export OCMEMOG_INGEST_BATCH_SECONDS="${OCMEMOG_INGEST_BATCH_SECONDS:-30}"
-export OCMEMOG_INGEST_BATCH_MAX="${OCMEMOG_INGEST_BATCH_MAX:-25}"
+if [[ "$LAPTOP_MODE" == "battery" ]]; then
+  export OCMEMOG_TRANSCRIPT_POLL_SECONDS="${OCMEMOG_TRANSCRIPT_POLL_SECONDS:-120}"
+  export OCMEMOG_INGEST_BATCH_SECONDS="${OCMEMOG_INGEST_BATCH_SECONDS:-120}"
+  export OCMEMOG_INGEST_BATCH_MAX="${OCMEMOG_INGEST_BATCH_MAX:-10}"
+  export OCMEMOG_REINFORCE_SENTIMENT="${OCMEMOG_REINFORCE_SENTIMENT:-false}"
+else
+  export OCMEMOG_TRANSCRIPT_POLL_SECONDS="${OCMEMOG_TRANSCRIPT_POLL_SECONDS:-30}"
+  export OCMEMOG_INGEST_BATCH_SECONDS="${OCMEMOG_INGEST_BATCH_SECONDS:-30}"
+  export OCMEMOG_INGEST_BATCH_MAX="${OCMEMOG_INGEST_BATCH_MAX:-25}"
+  export OCMEMOG_REINFORCE_SENTIMENT="${OCMEMOG_REINFORCE_SENTIMENT:-true}"
+fi
 export OCMEMOG_INGEST_ENDPOINT="${OCMEMOG_INGEST_ENDPOINT:-http://127.0.0.1:17890/memory/ingest_async}"
-export OCMEMOG_REINFORCE_SENTIMENT="${OCMEMOG_REINFORCE_SENTIMENT:-true}"
 export OCMEMOG_INGEST_SOURCE="${OCMEMOG_INGEST_SOURCE:-transcript}"
 export OCMEMOG_INGEST_MEMORY_TYPE="${OCMEMOG_INGEST_MEMORY_TYPE:-reflections}"