@simbimbo/memory-ocmemog 0.1.13 → 0.1.15

package/ocmemog/runtime/memory/retrieval.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 from datetime import datetime, timezone
-from typing import Dict, List, Any, Iterable, Tuple
+from typing import Dict, List, Any, Iterable, Tuple, Optional
 
 import json
+import os
 
 from ocmemog.runtime import state_store
 from ocmemog.runtime.instrumentation import emit_event
@@ -87,12 +88,126 @@ def _governance_state(metadata: Dict[str, Any]) -> tuple[str, Dict[str, Any]]:
     return str(state["memory_status"] or "active"), state
 
 
+def _flatten_strings(value: Any) -> List[str]:
+    items: List[str] = []
+    if isinstance(value, str):
+        stripped = value.strip()
+        if stripped:
+            items.append(stripped)
+    elif isinstance(value, dict):
+        for child in value.values():
+            items.extend(_flatten_strings(child))
+    elif isinstance(value, (list, tuple, set)):
+        for child in value:
+            items.extend(_flatten_strings(child))
+    return items
+
+
+def _metadata_lookup(metadata: Dict[str, Any], dotted_key: str) -> Any:
+    current: Any = metadata
+    for part in (dotted_key or "").split("."):
+        if not isinstance(current, dict):
+            return None
+        current = current.get(part)
+    return current
+
+
+def _metadata_matches(metadata: Dict[str, Any], filters: Optional[Dict[str, Any]]) -> bool:
+    if not filters:
+        return True
+    for key, expected in filters.items():
+        actual = _metadata_lookup(metadata, key)
+        actual_values = {item.lower() for item in _flatten_strings(actual)}
+        expected_values = {item.lower() for item in _flatten_strings(expected)}
+        if expected_values:
+            if not actual_values.intersection(expected_values):
+                return False
+        else:
+            if actual not in (None, "", [], {}):
+                return False
+    return True
+
+
+def _load_lane_profiles() -> Dict[str, Dict[str, Any]]:
+    raw = os.getenv("OCMEMOG_MEMORY_LANES_JSON", "").strip()
+    if not raw:
+        return {}
+    try:
+        payload = json.loads(raw)
+    except Exception:
+        return {}
+    if not isinstance(payload, dict):
+        return {}
+    profiles: Dict[str, Dict[str, Any]] = {}
+    for lane_name, config in payload.items():
+        if not isinstance(config, dict):
+            continue
+        normalized_name = str(lane_name or "").strip().lower()
+        if not normalized_name:
+            continue
+        profiles[normalized_name] = {
+            "keywords": [item.lower() for item in _flatten_strings(config.get("keywords"))],
+            "metadata_filters": config.get("metadata_filters") if isinstance(config.get("metadata_filters"), dict) else {},
+        }
+    return profiles
+
+
+def infer_lane(prompt: str, explicit_lane: Optional[str] = None) -> Optional[str]:
+    lane = str(explicit_lane or "").strip().lower()
+    if lane:
+        return lane
+    profiles = _load_lane_profiles()
+    if not profiles:
+        return None
+    prompt_l = str(prompt or "").lower()
+    tokens = set(_tokenize(prompt))
+    best_lane: Optional[str] = None
+    best_score = 0
+    for lane_name, config in profiles.items():
+        keywords = {item for item in config.get("keywords", []) if item}
+        if not keywords:
+            continue
+        score = 0
+        for keyword in keywords:
+            keyword_tokens = set(_tokenize(keyword))
+            if not keyword_tokens:
+                continue
+            if len(keyword_tokens) == 1:
+                if next(iter(keyword_tokens)) in tokens:
+                    score += 1
+            elif keyword.lower() in prompt_l:
+                score += len(keyword_tokens)
+        if score > best_score:
+            best_score = score
+            best_lane = lane_name
+    return best_lane if best_score > 0 else None
+
+
+def _lane_bonus(metadata: Dict[str, Any], lane: Optional[str]) -> float:
+    lane_value = str(lane or "").strip().lower()
+    if not lane_value:
+        return 0.0
+    domain = str(_metadata_lookup(metadata, "domain") or "").strip().lower()
+    if domain == lane_value:
+        return 0.2
+    profile = _load_lane_profiles().get(lane_value) or {}
+    filters = profile.get("metadata_filters") if isinstance(profile.get("metadata_filters"), dict) else {}
+    if filters and _metadata_matches(metadata, filters):
+        return 0.16
+    source_labels = {item.lower() for item in _flatten_strings(_metadata_lookup(metadata, "source_labels"))}
+    if lane_value in source_labels:
+        return 0.08
+    return 0.0
+
+
 def retrieve(
     prompt: str,
     limit: int = 5,
     categories: Iterable[str] | None = None,
     *,
     skip_vector_provider: bool = False,
+    metadata_filters: Optional[Dict[str, Any]] = None,
+    lane: Optional[str] = None,
 ) -> Dict[str, List[Dict[str, Any]]]:
     emit_event(state_store.report_log_path(), "brain_memory_retrieval_start", status="ok")
     emit_event(state_store.report_log_path(), "brain_memory_retrieval_rank_start", status="ok")
@@ -100,6 +215,7 @@ def retrieve(
     conn = store.connect()
     results = _empty_results()
     selected_categories = tuple(dict.fromkeys(category for category in (categories or MEMORY_BUCKETS) if category in MEMORY_BUCKETS))
+    active_lane = infer_lane(prompt, explicit_lane=lane)
 
     reinf_rows = conn.execute("SELECT memory_reference, reward_score, confidence FROM experiences").fetchall()
     reinforcement: Dict[str, Dict[str, float]] = {}
@@ -129,20 +245,22 @@ def retrieve(
             if source_type in selected_categories and source_id:
                 semantic_scores[f"{source_type}:{source_id}"] = float(item.get("score") or 0.0)
 
-    def score_record(*, content: str, memory_ref: str, promo_conf: float, timestamp: str | None) -> tuple[float, Dict[str, float]]:
+    def score_record(*, content: str, memory_ref: str, promo_conf: float, timestamp: str | None, metadata_payload: Dict[str, Any]) -> tuple[float, Dict[str, float]]:
         keyword = _match_score(content, prompt)
         semantic = float(semantic_scores.get(memory_ref, 0.0))
         reinf = reinforcement.get(memory_ref, {})
         reinf_score = float(reinf.get("reward_score", 0.0)) * 0.35
         promo_score = float(promo_conf) * 0.2
         recency = _recency_score(timestamp)
-        score = round((keyword * 0.45) + (semantic * 0.35) + reinf_score + promo_score + recency, 3)
+        lane_bonus = _lane_bonus(metadata_payload, active_lane)
+        score = round((keyword * 0.45) + (semantic * 0.35) + reinf_score + promo_score + recency + lane_bonus, 3)
         return score, {
             "keyword": round(keyword, 3),
             "semantic": round(semantic, 3),
             "reinforcement": round(reinf_score, 3),
             "promotion": round(promo_score, 3),
             "recency": round(recency, 3),
+            "lane_bonus": round(lane_bonus, 3),
         }
 
     for table in selected_categories:
@@ -165,11 +283,13 @@ def retrieve(
             timestamp = row["timestamp"] if isinstance(row, dict) else row[1]
             raw_metadata = row["metadata_json"] if isinstance(row, dict) else row[4]
             metadata_payload = _parse_metadata(raw_metadata)
+            if not _metadata_matches(metadata_payload, metadata_filters):
+                continue
             memory_status, governance = _governance_state(metadata_payload)
             if memory_status in {"superseded", "duplicate"}:
                 continue
             metadata = provenance.fetch_reference(mem_ref)
-            score, signals = score_record(content=content, memory_ref=mem_ref, promo_conf=promo_conf, timestamp=timestamp)
+            score, signals = score_record(content=content, memory_ref=mem_ref, promo_conf=promo_conf, timestamp=timestamp, metadata_payload=metadata_payload)
             if memory_status == "contested":
                 score = round(max(0.0, score - 0.15), 3)
                 signals["contradiction_penalty"] = 0.15
@@ -200,6 +320,8 @@ def retrieve_for_queries(
     limit: int = 5,
     categories: Iterable[str] | None = None,
     skip_vector_provider: bool = False,
+    metadata_filters: Optional[Dict[str, Any]] = None,
+    lane: Optional[str] = None,
 ) -> Dict[str, List[Dict[str, Any]]]:
     merged = _empty_results()
     seen_refs = {bucket: set() for bucket in MEMORY_BUCKETS}
@@ -207,10 +329,17 @@ def retrieve_for_queries(
     normalized_queries = [query.strip() for query in queries if isinstance(query, str) and query.strip()]
 
     if not normalized_queries:
-        return retrieve("", limit=limit, categories=selected_categories)
+        return retrieve("", limit=limit, categories=selected_categories, metadata_filters=metadata_filters, lane=lane)
 
     for query in normalized_queries:
-        partial = retrieve(query, limit=limit, categories=selected_categories, skip_vector_provider=skip_vector_provider)
+        partial = retrieve(
+            query,
+            limit=limit,
+            categories=selected_categories,
+            skip_vector_provider=skip_vector_provider,
+            metadata_filters=metadata_filters,
+            lane=lane,
+        )
         for bucket in selected_categories:
             for item in partial.get(bucket, []):
                 ref = item.get("memory_reference")

package/ocmemog/sidecar/app.py

@@ -46,6 +46,28 @@ _BOOL_TRUE_VALUES = {"1", "true", "yes", "on", "y", "t"}
 _BOOL_FALSE_VALUES = {"0", "false", "no", "off", "n", "f"}
 
 
+def _default_openclaw_home() -> Path:
+    explicit = os.environ.get("OPENCLAW_HOME", "").strip() or os.environ.get("OCMEMOG_OPENCLAW_HOME", "").strip()
+    if explicit:
+        return Path(explicit).expanduser().resolve()
+    xdg = os.environ.get("XDG_DATA_HOME", "").strip()
+    if xdg:
+        return (Path(xdg).expanduser() / "openclaw").resolve()
+    if os.name == "nt":
+        appdata = os.environ.get("APPDATA", "").strip() or os.environ.get("LOCALAPPDATA", "").strip()
+        if appdata:
+            return (Path(appdata).expanduser() / "OpenClaw").resolve()
+    return (Path.home() / ".openclaw").resolve()
+
+
+def _default_transcript_root() -> Path:
+    home = _default_openclaw_home()
+    legacy = (Path.home() / ".openclaw" / "workspace" / "memory").resolve()
+    if home == legacy.parent.parent:
+        return legacy
+    return home / "workspace" / "memory"
+
+
 def _parse_bool_env_value(raw: Any | None, default: bool = False) -> tuple[bool, bool]:
     """Return ``(value, valid)``, where ``valid`` indicates parser confidence."""
     if raw is None:
@@ -138,6 +160,8 @@ _INGEST_WORKER_LOCK = threading.Lock()
 _WATCHER_STOP = threading.Event()
 _WATCHER_THREAD: threading.Thread | None = None
 _WATCHER_LOCK = threading.Lock()
+_HYDRATE_CACHE_LOCK = threading.Lock()
+_HYDRATE_CACHE: dict[tuple[str, str, str, int, int], tuple[float, dict[str, Any]]] = {}
 QUEUE_LOCK = threading.Lock()
 QUEUE_PROCESS_LOCK = threading.Lock()
 QUEUE_STATS = {
@@ -221,6 +245,17 @@ async def _auth_middleware(request: Request, call_next):
     return await call_next(request)
 
 
+def _watcher_direct_turn_ingest(payload: dict) -> bool:
+    try:
+        request = ConversationTurnRequest(**payload)
+        response = _ingest_conversation_turn(request)
+        return bool(response.get("ok"))
+    except Exception as exc:
+        print(f"[ocmemog][watcher] direct_turn_ingest_failed error={exc!r}", file=sys.stderr)
+        return False
+
+
+
 def _start_transcript_watcher() -> None:
     global _WATCHER_THREAD
     _load_queue_stats()
@@ -233,7 +268,7 @@ def _start_transcript_watcher() -> None:
         _WATCHER_STOP.clear()
         _WATCHER_THREAD = threading.Thread(
             target=watch_forever,
-            args=(_WATCHER_STOP,),
+            args=(_WATCHER_STOP, _watcher_direct_turn_ingest),
             daemon=True,
             name="ocmemog-transcript-watcher",
         )
@@ -325,15 +360,28 @@ def _enqueue_postprocess(reference: str, *, skip_embedding_provider: bool = True
 
 
 def _run_postprocess_payload(payload: Dict[str, Any]) -> None:
+    started = time.perf_counter()
     reference = str(payload.get("reference") or "").strip()
     if not reference:
         raise ValueError("missing_reference")
     skip_embedding_provider = bool(payload.get("skip_embedding_provider", True))
     result = api.postprocess_stored_memory(reference, skip_embedding_provider=skip_embedding_provider)
+    elapsed_ms = round((time.perf_counter() - started) * 1000, 3)
+    trace = _parse_bool_env("OCMEMOG_TRACE_INGEST_PIPELINE", default=False)
+    warn_ms = _parse_float_env("OCMEMOG_TRACE_INGEST_PIPELINE_WARN_MS", default=20.0, minimum=0.0)
+    if trace or elapsed_ms >= warn_ms:
+        print(f"[ocmemog][ingest] postprocess elapsed_ms={elapsed_ms:.3f} reference={reference}", file=sys.stderr)
     if not result.get("ok"):
         raise RuntimeError(str(result.get("error") or "postprocess_failed"))
 
 
+def _should_link_ingest_memory_to_turn(request: IngestRequest) -> bool:
+    source = str(request.source or "").strip().lower()
+    if source in {"session", "transcript"}:
+        return False
+    return True
+
+
 
 def _process_queue(limit: Optional[int] = None) -> Dict[str, Any]:
     processed = 0
@@ -530,6 +578,8 @@ class SearchRequest(BaseModel):
     query: str = Field(default="")
     limit: int = Field(default=5, ge=1, le=50)
     categories: Optional[List[str]] = None
+    metadata_filters: Optional[Dict[str, Any]] = None
+    lane: Optional[str] = Field(default=None, description="Optional retrieval lane/domain hint, e.g. 'tbc'")
 
 
 class DuplicateCandidatesRequest(BaseModel):
@@ -638,6 +688,7 @@ class IngestRequest(BaseModel):
     transcript_offset: Optional[int] = None
     transcript_end_offset: Optional[int] = None
     timestamp: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
 
 
 class ConversationTurnRequest(BaseModel):
@@ -661,6 +712,7 @@ class ConversationHydrateRequest(BaseModel):
     thread_id: Optional[str] = None
     turns_limit: int = Field(default=12, ge=1, le=100)
     memory_limit: int = Field(default=8, ge=1, le=50)
+    predictive_brief_limit: int = Field(default=5, ge=1, le=12)
 
 
 class ConversationCheckpointRequest(BaseModel):
@@ -770,16 +822,28 @@ def _retune_reflection_memory_type(content: str, memory_type: str) -> str:
     return memory_type
 
 
-def _fallback_search(query: str, limit: int, categories: List[str]) -> List[Dict[str, Any]]:
+def _fallback_search(
+    query: str,
+    limit: int,
+    categories: List[str],
+    *,
+    metadata_filters: Optional[Dict[str, Any]] = None,
+    lane: Optional[str] = None,
+) -> List[Dict[str, Any]]:
     conn = store.connect()
+    active_lane = retrieval.infer_lane(query, explicit_lane=lane)
     try:
         results: List[Dict[str, Any]] = []
         for table in categories:
             rows = conn.execute(
-                f"SELECT id, content, confidence FROM {table} WHERE content LIKE ? ORDER BY id DESC LIMIT ?",
-                (f"%{query}%", limit),
+                f"SELECT id, content, confidence, metadata_json FROM {table} WHERE content LIKE ? ORDER BY id DESC LIMIT ?",
+                (f"%{query}%", limit * 5),
             ).fetchall()
             for row in rows:
+                meta = json.loads(row["metadata_json"] or "{}") if row["metadata_json"] else {}
+                if not retrieval._metadata_matches(meta, metadata_filters):
+                    continue
+                lane_bonus = retrieval._lane_bonus(meta, active_lane)
                 results.append(
                     {
                         "bucket": table,
@@ -787,8 +851,9 @@ def _fallback_search(query: str, limit: int, categories: List[str]) -> List[Dict
                         "table": table,
                         "id": str(row["id"]),
                         "content": str(row["content"] or ""),
-                        "score": float(row["confidence"] or 0.0),
+                        "score": float(row["confidence"] or 0.0) + lane_bonus,
                         "links": [],
+                        "metadata": meta,
                     }
                 )
         results.sort(key=lambda item: item["score"], reverse=True)
@@ -797,6 +862,94 @@ def _fallback_search(query: str, limit: int, categories: List[str]) -> List[Dict
         conn.close()
 
 
+def _compact_text(value: Any, max_len: int = 180) -> str:
+    text = re.sub(r"\s+", " ", str(value or "")).strip()
+    if len(text) > max_len:
+        return f"{text[: max_len - 1].rstrip()}…"
+    return text
+
+
+def _build_predictive_brief(
+    *,
+    request: ConversationHydrateRequest,
+    turns: Sequence[Dict[str, Any]],
+    summary: Dict[str, Any],
+    linked_memories: Sequence[Dict[str, Any]],
+) -> Dict[str, Any]:
+    latest_user_ask = ((summary.get("latest_user_intent") or {}).get("effective_content") if isinstance(summary.get("latest_user_intent"), dict) else None) or ((summary.get("latest_user_ask") or {}).get("content") if isinstance(summary.get("latest_user_ask"), dict) else None) or ""
+    summary_text = str(summary.get("summary_text") or "").strip()
+    query = _compact_text(latest_user_ask or summary_text or "resume context", 260)
+    lane = retrieval.infer_lane(query)
+    profiles = retrieval._load_lane_profiles()
+    profile = profiles.get(lane or "") if lane else None
+    metadata_filters = profile.get("metadata_filters") if isinstance(profile, dict) else None
+    categories = ["knowledge", "runbooks", "tasks", "reflections", "directives"]
+    retrieved = retrieval.retrieve_for_queries(
+        [query],
+        limit=max(1, request.predictive_brief_limit),
+        categories=categories,
+        metadata_filters=metadata_filters,
+        lane=lane,
+        skip_vector_provider=True,
+    )
+    items: List[Dict[str, Any]] = []
+    seen: set[str] = set()
+    for bucket in categories:
+        for item in retrieved.get(bucket, []) or []:
+            ref = str(item.get("reference") or "")
+            if not ref or ref in seen:
+                continue
+            seen.add(ref)
+            items.append(
+                {
+                    "reference": ref,
+                    "category": bucket,
+                    "content": _compact_text(item.get("content") or "", 180),
+                    "selected_because": item.get("selected_because") or item.get("retrieval_signals") or "retrieval",
+                    "score": item.get("score"),
+                    "metadata": item.get("metadata") or {},
+                }
+            )
+            if len(items) >= request.predictive_brief_limit:
+                break
+        if len(items) >= request.predictive_brief_limit:
+            break
+
+    checkpoint = summary.get("latest_checkpoint") if isinstance(summary.get("latest_checkpoint"), dict) else None
+    open_loops = summary.get("open_loops") if isinstance(summary.get("open_loops"), list) else []
+    recent_linked = []
+    for item in linked_memories[:2]:
+        if not isinstance(item, dict):
+            continue
+        recent_linked.append({
+            "reference": item.get("reference"),
+            "summary": _compact_text(item.get("summary") or item.get("content") or item.get("reference") or "", 140),
+        })
+    return {
+        "lane": lane,
+        "query": query,
+        "metadata_filters": metadata_filters or {},
+        "checkpoint": {
+            "reference": checkpoint.get("reference") if checkpoint else None,
+            "summary": _compact_text(checkpoint.get("summary") if checkpoint else "", 180),
+        } if checkpoint else None,
+        "open_loops": [
+            {
+                "kind": item.get("kind"),
+                "summary": _compact_text(item.get("summary") or "", 120),
+                "reference": item.get("source_reference") or item.get("reference"),
+            }
+            for item in open_loops[:2]
+            if isinstance(item, dict) and str(item.get("summary") or "").strip()
+        ],
+        "memories": items,
+        "linked_memories": recent_linked,
+        "latest_user_ask": _compact_text(latest_user_ask, 180),
+        "summary_text": _compact_text(summary_text, 220),
+        "mode": "predictive-brief",
+    }
+
+
 _ALLOWED_MEMORY_REFERENCE_TYPES = {
     *store.MEMORY_TABLES,
     "conversation_turns",
@@ -895,7 +1048,7 @@ def _allowed_transcript_roots() -> list[Path]:
     if raw:
         roots = [Path(item).expanduser().resolve() for item in raw.split(",") if item.strip()]
     else:
-        roots = [Path.home() / ".openclaw" / "workspace" / "memory"]
+        roots = [_default_transcript_root()]
     return roots
 
 
@@ -963,13 +1116,15 @@ def memory_search(request: SearchRequest) -> dict[str, Any]:
             limit=request.limit,
             categories=categories,
             skip_vector_provider=skip_vector_provider,
+            metadata_filters=request.metadata_filters,
+            lane=request.lane,
         )
         flattened = flatten_results(results)
         if len(flattened) > request.limit:
             flattened = flattened[: request.limit]
         used_fallback = False
     except Exception as exc:
-        flattened = _fallback_search(request.query, request.limit, categories)
+        flattened = _fallback_search(request.query, request.limit, categories, metadata_filters=request.metadata_filters, lane=request.lane)
         used_fallback = True
         runtime["warnings"] = [*runtime["warnings"], f"search fallback enabled: {exc}"]
     elapsed_ms = round((time.perf_counter() - started) * 1000, 3)
@@ -1320,18 +1475,40 @@ def conversation_ingest_turn(request: ConversationTurnRequest) -> dict[str, Any]
 @app.post("/conversation/hydrate")
 def conversation_hydrate(request: ConversationHydrateRequest) -> dict[str, Any]:
     runtime = _runtime_payload()
+    route_started = time.perf_counter()
+    stage_marks: list[tuple[str, float]] = []
+
+    def _mark(stage: str) -> None:
+        stage_marks.append((stage, time.perf_counter()))
+
+    cache_ttl_ms = _parse_float_env("OCMEMOG_HYDRATE_CACHE_TTL_MS", default=350.0, minimum=0.0)
+    cache_key = (
+        str(request.conversation_id or ""),
+        str(request.session_id or ""),
+        str(request.thread_id or ""),
+        int(request.turns_limit),
+        int(request.memory_limit),
+    )
+    if cache_ttl_ms > 0:
+        with _HYDRATE_CACHE_LOCK:
+            cached = _HYDRATE_CACHE.get(cache_key)
+            now_ms = time.time() * 1000.0
+            if cached and (now_ms - cached[0]) <= cache_ttl_ms:
+                return {**cached[1], **runtime}
     turns = conversation_state.get_recent_turns(
         conversation_id=request.conversation_id,
         session_id=request.session_id,
         thread_id=request.thread_id,
         limit=request.turns_limit,
     )
+    _mark("get_recent_turns")
     linked_memories = conversation_state.get_linked_memories(
         conversation_id=request.conversation_id,
         session_id=request.session_id,
         thread_id=request.thread_id,
         limit=request.memory_limit,
     )
+    _mark("get_linked_memories")
     link_targets: List[Dict[str, Any]] = []
     if request.thread_id:
         link_targets.extend(memory_links.get_memory_links_for_thread(request.thread_id))
@@ -1339,6 +1516,11 @@ def conversation_hydrate(request: ConversationHydrateRequest) -> dict[str, Any]:
         link_targets.extend(memory_links.get_memory_links_for_session(request.session_id))
     if request.conversation_id:
         link_targets.extend(memory_links.get_memory_links_for_conversation(request.conversation_id))
+    conversation_state._self_heal_legacy_continuity_artifacts(
+        conversation_id=request.conversation_id,
+        session_id=request.session_id,
+        thread_id=request.thread_id,
+    )
     latest_checkpoint = conversation_state.get_latest_checkpoint(
         conversation_id=request.conversation_id,
         session_id=request.session_id,
@@ -1350,6 +1532,7 @@ def conversation_hydrate(request: ConversationHydrateRequest) -> dict[str, Any]:
         thread_id=request.thread_id,
         limit=10,
     )
+    _mark("list_relevant_unresolved_state")
     summary = conversation_state.infer_hydration_payload(
         turns,
         conversation_id=request.conversation_id,
@@ -1359,25 +1542,42 @@ def conversation_hydrate(request: ConversationHydrateRequest) -> dict[str, Any]:
         latest_checkpoint=latest_checkpoint,
         linked_memories=linked_memories,
     )
-    state_payload = conversation_state.refresh_state(
+    _mark("infer_hydration_payload")
+    state_payload = conversation_state.get_state(
         conversation_id=request.conversation_id,
         session_id=request.session_id,
         thread_id=request.thread_id,
-        tolerate_write_failure=True,
     )
+    _mark("get_state")
+    if not state_payload:
+        state_payload = conversation_state._state_from_payload(
+            summary,
+            conversation_id=request.conversation_id,
+            session_id=request.session_id,
+            thread_id=request.thread_id,
+        )
     state_meta = (state_payload or {}).get("metadata") if isinstance((state_payload or {}).get("metadata"), dict) else {}
     state_status = str(state_meta.get("state_status") or "")
+    runtime["warnings"] = [*runtime["warnings"], "hydrate returned state without inline state refresh"]
     if state_status == "stale_persisted":
         runtime["warnings"] = [*runtime["warnings"], "hydrate returned persisted state while state refresh was delayed"]
     elif state_status == "derived_not_persisted":
-        runtime["warnings"] = [*runtime["warnings"], "hydrate returned derived state while state refresh was delayed"]
-    return {
+        runtime["warnings"] = [*runtime["warnings"], "hydrate returned derived state without inline state refresh"]
+    predictive_brief = _build_predictive_brief(
+        request=request,
+        turns=turns,
+        summary=summary,
+        linked_memories=linked_memories,
+    )
+    _mark("build_predictive_brief")
+    response = {
         "ok": True,
         "conversation_id": request.conversation_id,
         "session_id": request.session_id,
         "thread_id": request.thread_id,
         "recent_turns": turns,
         "summary": summary,
+        "predictive_brief": predictive_brief,
         "turn_counts": conversation_state.get_turn_counts(
             conversation_id=request.conversation_id,
             session_id=request.session_id,
@@ -1390,6 +1590,33 @@ def conversation_hydrate(request: ConversationHydrateRequest) -> dict[str, Any]:
         "state": state_payload,
         **runtime,
     }
+    elapsed_ms = round((time.perf_counter() - route_started) * 1000, 3)
+    hydrate_trace_enabled = _parse_bool_env("OCMEMOG_TRACE_HYDRATE", default=False)
+    hydrate_warn_ms_raw = os.environ.get("OCMEMOG_TRACE_HYDRATE_WARN_MS", "25").strip()
+    try:
+        hydrate_warn_ms = max(0.0, float(hydrate_warn_ms_raw))
+    except Exception:
+        hydrate_warn_ms = 25.0
+    if cache_ttl_ms > 0:
+        with _HYDRATE_CACHE_LOCK:
+            _HYDRATE_CACHE[cache_key] = (time.time() * 1000.0, dict(response))
+            if len(_HYDRATE_CACHE) > 256:
+                oldest_key = min(_HYDRATE_CACHE.items(), key=lambda item: item[1][0])[0]
+                _HYDRATE_CACHE.pop(oldest_key, None)
+    if hydrate_trace_enabled or elapsed_ms >= hydrate_warn_ms:
+        stage_details: list[str] = []
+        previous = route_started
+        for name, mark in stage_marks:
+            stage_details.append(f"{name}={(mark - previous) * 1000.0:.3f}ms")
+            previous = mark
+        print(
+            "[ocmemog][route] conversation_hydrate "
+            f"elapsed_ms={elapsed_ms:.3f} turns={len(turns)} linked_memories={len(linked_memories)} "
+            f"unresolved_items={len(unresolved_items)} state_status={state_status or 'fresh'} "
+            f"stages={'|'.join(stage_details) or 'none'}",
+            file=sys.stderr,
+        )
+    return response
 
 
 @app.post("/conversation/checkpoint")
@@ -1495,6 +1722,7 @@ def memory_ponder_latest(limit: int = 5) -> dict[str, Any]:
 
 
 def _ingest_request(request: IngestRequest) -> dict[str, Any]:
+    ingest_started = time.perf_counter()
     runtime = _runtime_payload()
     content = request.content.strip() if isinstance(request.content, str) else ""
     if not content:
@@ -1508,6 +1736,7 @@ def _ingest_request(request: IngestRequest) -> dict[str, Any]:
             memory_type = "knowledge"
         memory_type = _retune_reflection_memory_type(content, memory_type)
         metadata = {
+            **(request.metadata or {}),
             "conversation_id": request.conversation_id,
             "session_id": request.session_id,
             "thread_id": request.thread_id,
@@ -1548,7 +1777,7 @@ def _ingest_request(request: IngestRequest) -> dict[str, Any]:
             else:
                 suffix = ""
             memory_links.add_memory_link(reference, "transcript", f"transcript:{request.transcript_path}{suffix}")
-        if request.role:
+        if request.role and _should_link_ingest_memory_to_turn(request):
             turn_response = _ingest_conversation_turn(
                 ConversationTurnRequest(
                     role=request.role,
@@ -1578,11 +1807,18 @@ def _ingest_request(request: IngestRequest) -> dict[str, Any]:
                     ]
                 },
             )
-        return {"ok": True, "kind": "memory", "memory_type": memory_type, "reference": reference, "turn": turn_response, **runtime}
+        response = {"ok": True, "kind": "memory", "memory_type": memory_type, "reference": reference, "turn": turn_response, **runtime}
+        elapsed_ms = round((time.perf_counter() - ingest_started) * 1000, 3)
+        trace = _parse_bool_env("OCMEMOG_TRACE_INGEST_PIPELINE", default=False)
+        warn_ms = _parse_float_env("OCMEMOG_TRACE_INGEST_PIPELINE_WARN_MS", default=20.0, minimum=0.0)
+        if trace or elapsed_ms >= warn_ms:
+            print(f"[ocmemog][ingest] ingest_request elapsed_ms={elapsed_ms:.3f} kind=memory source={request.source or ''} reference={reference}", file=sys.stderr)
+        return response
 
     # experience ingest
     experience_metadata = provenance.normalize_metadata(
         {
+            **(request.metadata or {}),
             "conversation_id": request.conversation_id,
             "session_id": request.session_id,
             "thread_id": request.thread_id,