loki-mode 6.81.1 → 6.83.0

package/memory/managed_memory/retrieve.py

@@ -0,0 +1,347 @@
+"""
+Loki Managed Agents Memory - Retrieve path (v6.83.0 Phase 1).
+
+Used by the REASON phase (retrieve_memory_context in autonomy/run.sh) and by
+the completion council (augment prompts with prior verdicts). Also provides
+hydrate_patterns() which pulls semantic patterns updated after a local mtime
+floor and merges them into .loki/memory/semantic/patterns.json via the
+existing MemoryStorage._atomic_write.
+
+CLI:
+    python3 -m memory.managed_memory.retrieve --query <str> [--top-k N] \\
+        [--store-id <id>]
+
+    python3 -m memory.managed_memory.retrieve --hydrate [--since-seconds N]
+
+Never raises: on any error the CLI prints nothing and exits 0. Bash callers
+apply a 5s hard timeout; this module keeps its own 10s SDK timeout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from . import ManagedDisabled, is_enabled
+from .events import emit_managed_event
+
+_LOG = logging.getLogger("loki.managed_memory.retrieve")
+
+_DEFAULT_STORE_NAME = "loki-rarv-c-learnings"
+
+
+def _store_name() -> str:
+    return os.environ.get("LOKI_MANAGED_STORE_NAME", _DEFAULT_STORE_NAME)
+
+
+def _warn_once(msg: str) -> None:
+    sys.stderr.write(f"WARN [managed_memory] {msg}\n")
+
+
+def _get_client():
+    from .client import get_client
+
+    return get_client()
+
+
+def _summarize(content: str, limit: int = 240) -> str:
+    """Return a short single-line summary for prompt injection."""
+    s = (content or "").strip().replace("\n", " ")
+    if len(s) > limit:
+        s = s[: limit - 3] + "..."
+    return s
+
+
+# ---------------------------------------------------------------------------
+# Retrieve related verdicts (used by REASON and completion council)
+# ---------------------------------------------------------------------------
+
+
+def retrieve_related_verdicts(
+    query: str,
+    top_k: int = 3,
+    store_id: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Return up to `top_k` recent verdicts most-related to `query`.
+
+    Phase 1 implementation: fetch memories under `verdicts/` and
+    prefix-match. This is naive but honest -- true similarity retrieval
+    ships in Phase 2 when the beta exposes a semantic search endpoint.
+    Returns [] on any error.
+    """
+    if not is_enabled():
+        return []
+
+    start = time.monotonic()
+    try:
+        client = _get_client()
+    except ManagedDisabled as e:
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "client_unavailable", "detail": str(e), "op": "retrieve"},
+        )
+        return []
+    except Exception as e:  # pragma: no cover - defensive
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "client_error", "detail": str(e), "op": "retrieve"},
+        )
+        return []
+
+    try:
+        if not store_id:
+            store = client.stores_get_or_create(
+                name=_store_name(),
+                description="Loki Mode RARV-C shadow-write store (v6.83.0)",
+                scope="project",
+            )
+            store_id = store.get("id") or store.get("store_id")
+        if not store_id:
+            return []
+        entries = client.memories_list(store_id=store_id, path_prefix="verdicts/")
+    except Exception as e:
+        _warn_once(f"memories_list failed ({e}); returning empty")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "list_error", "detail": str(e), "op": "retrieve"},
+        )
+        return []
+
+    # Naive ranking: keep entries whose content contains any query token.
+    tokens = [t.lower() for t in query.split() if len(t) > 2]
+    scored: List[tuple] = []
+    for e in entries:
+        content = e.get("content") or ""
+        content_lc = content.lower()
+        score = sum(1 for t in tokens if t in content_lc)
+        scored.append((score, e))
+    scored.sort(key=lambda pair: pair[0], reverse=True)
+
+    out: List[Dict[str, Any]] = []
+    for score, e in scored[:top_k]:
+        out.append(
+            {
+                "path": e.get("path"),
+                "content_summary": _summarize(e.get("content") or ""),
+                "version_id": e.get("version") or e.get("version_id"),
+                "score": score,
+            }
+        )
+
+    emit_managed_event(
+        "managed_memory_retrieve",
+        {
+            "query_tokens": len(tokens),
+            "hits": len(out),
+            "total_candidates": len(entries),
+            "elapsed_ms": int((time.monotonic() - start) * 1000),
+        },
+    )
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Hydrate semantic patterns
+# ---------------------------------------------------------------------------
+
+
+def hydrate_patterns(
+    local_mtime_floor: float,
+    target_dir: Optional[str] = None,
+) -> int:
+    """
+    Pull semantic patterns from the managed store and merge them into
+    .loki/memory/semantic/patterns.json. Returns the number of patterns
+    merged in. Returns 0 on disabled / error.
+
+    Only patterns whose remote version timestamp is newer than
+    `local_mtime_floor` are merged. The merge is additive: existing local
+    patterns with the same pattern_id are kept unchanged.
+    """
+    if not is_enabled():
+        return 0
+
+    try:
+        client = _get_client()
+    except ManagedDisabled as e:
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "client_unavailable", "detail": str(e), "op": "hydrate"},
+        )
+        return 0
+    except Exception as e:  # pragma: no cover
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "client_error", "detail": str(e), "op": "hydrate"},
+        )
+        return 0
+
+    try:
+        store = client.stores_get_or_create(
+            name=_store_name(),
+            description="Loki Mode RARV-C shadow-write store (v6.83.0)",
+            scope="project",
+        )
+        store_id = store.get("id") or store.get("store_id")
+        if not store_id:
+            return 0
+        entries = client.memories_list(store_id=store_id, path_prefix="patterns/")
+    except Exception as e:
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "list_error", "detail": str(e), "op": "hydrate"},
+        )
+        return 0
+
+    target_dir = target_dir or os.environ.get("LOKI_TARGET_DIR") or os.getcwd()
+    patterns_path = Path(target_dir) / ".loki" / "memory" / "semantic" / "patterns.json"
+
+    # Read existing local patterns file (may not exist).
+    existing: Dict[str, Any] = {"patterns": {}}
+    if patterns_path.exists():
+        try:
+            with open(patterns_path, "r", encoding="utf-8") as f:
+                existing = json.load(f)
+        except (OSError, json.JSONDecodeError):
+            existing = {"patterns": {}}
+    if "patterns" not in existing or not isinstance(existing["patterns"], dict):
+        existing["patterns"] = {}
+
+    merged = 0
+    for e in entries:
+        content = e.get("content")
+        if not content:
+            continue
+        try:
+            pat = json.loads(content)
+        except (TypeError, json.JSONDecodeError):
+            continue
+        pid = pat.get("pattern_id") or pat.get("id")
+        if not pid:
+            continue
+        if pid in existing["patterns"]:
+            # Local wins on duplicate ids (Phase 1 policy).
+            continue
+        # Optional mtime gate: skip if the remote entry has a timestamp and
+        # it predates the floor.
+        ts = pat.get("updated_at") or pat.get("created_at")
+        if ts and local_mtime_floor:
+            try:
+                # Very loose timestamp parse: accept both epoch and ISO.
+                if isinstance(ts, (int, float)) and float(ts) < local_mtime_floor:
+                    continue
+            except (TypeError, ValueError):
+                pass
+        existing["patterns"][pid] = pat
+        merged += 1
+
+    if merged == 0:
+        emit_managed_event(
+            "managed_memory_hydrate", {"merged": 0, "candidates": len(entries)}
+        )
+        return 0
+
+    # Use existing MemoryStorage._atomic_write where possible; fall back to
+    # tempfile+rename if MemoryStorage is not importable in this process.
+    try:
+        # Late import to avoid module-load cycles.
+        from memory.storage import MemoryStorage  # type: ignore
+
+        storage = MemoryStorage(str(patterns_path.parent.parent))
+        storage._atomic_write(patterns_path, existing)
+    except Exception:
+        # Fallback: inline atomic write.
+        import tempfile
+
+        patterns_path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(
+            dir=str(patterns_path.parent), prefix=".tmp_", suffix=".json"
+        )
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                json.dump(existing, f, indent=2, default=str)
+            os.replace(tmp, patterns_path)
+        except Exception as e:
+            if os.path.exists(tmp):
+                os.unlink(tmp)
+            emit_managed_event(
+                "managed_agents_fallback",
+                {"reason": "atomic_write_failed", "detail": str(e), "op": "hydrate"},
+            )
+            return 0
+
+    emit_managed_event(
+        "managed_memory_hydrate",
+        {"merged": merged, "candidates": len(entries)},
+    )
+    return merged
+
+
+# ---------------------------------------------------------------------------
+# Module CLI
+# ---------------------------------------------------------------------------
+
+
+def _main(argv: Optional[list] = None) -> int:
+    # Silent no-op if flags are off; bash callers rely on exit 0 + empty stdout.
+    if not is_enabled():
+        return 0
+
+    parser = argparse.ArgumentParser(
+        prog="python3 -m memory.managed_memory.retrieve",
+        description="Retrieve related verdicts / hydrate patterns from the managed store.",
+    )
+    parser.add_argument("--query", help="Retrieval query string")
+    parser.add_argument("--top-k", type=int, default=3)
+    parser.add_argument("--store-id", default=None)
+    parser.add_argument(
+        "--hydrate",
+        action="store_true",
+        help="Hydrate semantic patterns into .loki/memory/semantic/",
+    )
+    parser.add_argument(
+        "--since-seconds",
+        type=int,
+        default=0,
+        help="Hydrate: only merge patterns newer than NOW - since_seconds",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        if args.hydrate:
+            floor = 0.0
+            if args.since_seconds and args.since_seconds > 0:
+                floor = time.time() - args.since_seconds
+            hydrate_patterns(local_mtime_floor=floor)
+            return 0
+
+        query = args.query or ""
+        if not query:
+            return 0
+        results = retrieve_related_verdicts(
+            query=query, top_k=args.top_k, store_id=args.store_id
+        )
+        # Print one line per hit, suitable for pasting into a prompt section.
+        for r in results:
+            path = r.get("path", "")
+            summary = r.get("content_summary", "")
+            print(f"- [managed] {path}: {summary}")
+        return 0
+    except Exception as e:  # pragma: no cover - defensive
+        _warn_once(f"retrieve CLI unexpected error: {e}")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "cli_unexpected", "detail": str(e), "op": "retrieve"},
+        )
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(_main())

package/memory/managed_memory/shadow_write.py

@@ -0,0 +1,350 @@
+"""
+Loki Managed Agents Memory - Shadow-write path (v6.83.0 Phase 1).
+
+Writes a whitelisted subset of RARV-C artifacts to the managed memory store:
+
+    - Completion-council verdicts (one JSON file per iteration)
+    - High-importance semantic patterns (importance >= 0.6)
+
+Design rules enforced by this module:
+
+    1. Every call is gated on LOKI_MANAGED_AGENTS=true AND LOKI_MANAGED_MEMORY=true.
+    2. On API error, we emit ONE `managed_agents_fallback` event and return.
+       No retry storm.
+    3. On a 409 (sha256 precondition mismatch), we re-read the remote entry,
+       merge with the local content, and retry ONCE. After that, fall back.
+    4. Non-blocking from the caller: this module never raises to its caller.
+       Callers in bash can background with `&` for extra isolation.
+    5. This file must NOT import anthropic at module load time. Client
+       construction is deferred to first call.
+
+CLI:
+    python3 -m memory.managed_memory.shadow_write --verdict <path>
+    python3 -m memory.managed_memory.shadow_write --path <episode.json>
+    python3 -m memory.managed_memory.shadow_write --pattern-json <json_file>
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from . import ManagedDisabled, is_enabled
+from .events import emit_managed_event
+
+_LOG = logging.getLogger("loki.managed_memory.shadow_write")
+
+# Store name is stable across runs so different projects share lineage.
+# Callers can override via LOKI_MANAGED_STORE_NAME.
+_DEFAULT_STORE_NAME = "loki-rarv-c-learnings"
+
+
+def _store_name() -> str:
+    return os.environ.get("LOKI_MANAGED_STORE_NAME", _DEFAULT_STORE_NAME)
+
+
+def _warn_once(msg: str) -> None:
+    # Single-line WARN, no retry-storm. Kept trivially simple on purpose.
+    sys.stderr.write(f"WARN [managed_memory] {msg}\n")
+
+
+def _get_client():
+    """Import client lazily so importing this module stays SDK-free."""
+    from .client import get_client, compute_sha256  # noqa: F401
+
+    return get_client(), compute_sha256
+
+
+def _read_json(path: str) -> Optional[Dict[str, Any]]:
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Core write helpers
+# ---------------------------------------------------------------------------
+
+
+def _shadow_write_blob(
+    logical_path: str,
+    payload: Dict[str, Any],
+    kind: str,
+) -> bool:
+    """
+    Write `payload` (a JSON-serializable dict) at `logical_path` in the
+    managed store. Returns True on success, False on fallback.
+
+    This function is responsible for the single 409 retry cycle and for
+    emitting fallback/success events.
+    """
+    if not is_enabled():
+        return False
+
+    content = json.dumps(payload, sort_keys=True, default=str)
+    try:
+        client, compute_sha = _get_client()
+    except ManagedDisabled as e:
+        _warn_once(f"client unavailable ({e}); local path only")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "client_unavailable", "detail": str(e), "kind": kind},
+        )
+        return False
+    except Exception as e:  # pragma: no cover - defensive
+        _warn_once(f"client construction failed ({e}); local path only")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "client_error", "detail": str(e), "kind": kind},
+        )
+        return False
+
+    # Ensure the store exists. Failure here => fallback.
+    try:
+        store = client.stores_get_or_create(
+            name=_store_name(),
+            description="Loki Mode RARV-C shadow-write store (v6.83.0)",
+            scope="project",
+        )
+        store_id = store.get("id") or store.get("store_id")
+    except Exception as e:
+        _warn_once(f"stores_get_or_create failed ({e}); local path only")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "stores_error", "detail": str(e), "kind": kind},
+        )
+        return False
+
+    if not store_id:
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "missing_store_id", "kind": kind},
+        )
+        return False
+
+    sha = compute_sha(content)
+    start = time.monotonic()
+    try:
+        client.memory_create(
+            store_id=store_id,
+            path=logical_path,
+            content=content,
+            sha256_precondition=sha,
+        )
+        emit_managed_event(
+            "managed_memory_shadow_write",
+            {
+                "kind": kind,
+                "path": logical_path,
+                "sha256": sha,
+                "elapsed_ms": int((time.monotonic() - start) * 1000),
+            },
+        )
+        return True
+    except Exception as first_err:
+        # Detect 409 / precondition failure.
+        status = getattr(first_err, "status_code", None) or getattr(
+            first_err, "status", None
+        )
+        if status == 409:
+            # Re-read, merge, retry once.
+            try:
+                existing_list = client.memories_list(
+                    store_id=store_id, path_prefix=logical_path
+                )
+                existing_entry = next(
+                    (m for m in existing_list if m.get("path") == logical_path),
+                    None,
+                )
+                existing_sha = (
+                    existing_entry.get("sha") or existing_entry.get("content_sha256")
+                    if existing_entry
+                    else None
+                )
+                if existing_entry and existing_sha:
+                    # Simple merge: if existing content already contains a
+                    # "versions" list, append; else seed it. We do NOT attempt
+                    # semantic merging in Phase 1.
+                    try:
+                        existing_content_str = existing_entry.get("content", "{}")
+                        existing_doc = json.loads(existing_content_str)
+                    except (TypeError, json.JSONDecodeError):
+                        existing_doc = {}
+                    merged = {
+                        **existing_doc,
+                        **payload,
+                        "_merged_versions": (
+                            existing_doc.get("_merged_versions", [])
+                            + [existing_sha]
+                        ),
+                    }
+                    merged_content = json.dumps(merged, sort_keys=True, default=str)
+                    new_sha = compute_sha(merged_content)
+                    # Precondition is the sha of the CURRENT remote state that
+                    # we just read. The new sha is what the server will store.
+                    client.memory_create(
+                        store_id=store_id,
+                        path=logical_path,
+                        content=merged_content,
+                        sha256_precondition=existing_sha,
+                    )
+                    emit_managed_event(
+                        "managed_memory_shadow_write",
+                        {
+                            "kind": kind,
+                            "path": logical_path,
+                            "sha256": new_sha,
+                            "merged": True,
+                            "elapsed_ms": int(
+                                (time.monotonic() - start) * 1000
+                            ),
+                        },
+                    )
+                    return True
+            except Exception as retry_err:
+                _warn_once(
+                    f"409 retry for {logical_path} failed ({retry_err}); fallback"
+                )
+                emit_managed_event(
+                    "managed_agents_fallback",
+                    {
+                        "reason": "retry_failed",
+                        "detail": str(retry_err),
+                        "kind": kind,
+                        "path": logical_path,
+                    },
+                )
+                return False
+
+        # Non-409 error => immediate fallback, single WARN line.
+        _warn_once(f"memory_create failed ({first_err}); local path only")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {
+                "reason": "memory_create_error",
+                "detail": str(first_err),
+                "kind": kind,
+                "path": logical_path,
+            },
+        )
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Public entry points
+# ---------------------------------------------------------------------------
+
+
+def shadow_write_verdict(verdict_json_path: str) -> bool:
+    """
+    Read a council verdict JSON from disk and shadow-write it.
+
+    The logical remote path is `verdicts/<iteration>.json`. If the file is
+    missing or unreadable, the call is a no-op. Never raises.
+    """
+    if not is_enabled():
+        return False
+    payload = _read_json(verdict_json_path)
+    if not payload:
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "verdict_unreadable", "path": verdict_json_path},
+        )
+        return False
+    iteration = payload.get("iteration") or payload.get("round") or "unknown"
+    logical = f"verdicts/iteration-{iteration}.json"
+    return _shadow_write_blob(logical, payload, kind="verdict")
+
+
+def shadow_write_pattern(pattern: Dict[str, Any]) -> bool:
+    """
+    Shadow-write a SemanticPattern-shaped dict. Logical path is
+    `patterns/<pattern_id>.json`. Never raises.
+    """
+    if not is_enabled():
+        return False
+    if not isinstance(pattern, dict):
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "pattern_not_dict", "type": str(type(pattern))},
+        )
+        return False
+    pid = pattern.get("pattern_id") or pattern.get("id") or "unknown"
+    logical = f"patterns/{pid}.json"
+    return _shadow_write_blob(logical, pattern, kind="pattern")
+
+
+def shadow_write_episode(episode_path: str) -> bool:
+    """
+    Shadow-write a high-importance episode trace JSON. Logical path is
+    `episodes/<episode_id>.json`. Called from autonomy/run.sh's
+    auto_capture_episode ONLY when importance >= 0.6. Never raises.
+    """
+    if not is_enabled():
+        return False
+    payload = _read_json(episode_path)
+    if not payload:
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "episode_unreadable", "path": episode_path},
+        )
+        return False
+    eid = payload.get("id") or payload.get("task_id") or Path(episode_path).stem
+    logical = f"episodes/{eid}.json"
+    return _shadow_write_blob(logical, payload, kind="episode")
+
+
+# ---------------------------------------------------------------------------
+# Module CLI
+# ---------------------------------------------------------------------------
+
+
+def _main(argv: Optional[list] = None) -> int:
+    # Silent no-op if flags are off -- bash callers rely on exit 0.
+    if not is_enabled():
+        return 0
+
+    parser = argparse.ArgumentParser(
+        prog="python3 -m memory.managed_memory.shadow_write",
+        description="Shadow-write a RARV-C artifact to the managed memory store.",
+    )
+    parser.add_argument("--verdict", help="Path to a council verdict JSON")
+    parser.add_argument("--path", help="Path to an episode trace JSON")
+    parser.add_argument(
+        "--pattern-json", help="Path to a file containing a pattern JSON blob"
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        if args.verdict:
+            shadow_write_verdict(args.verdict)
+            return 0
+        if args.path:
+            shadow_write_episode(args.path)
+            return 0
+        if args.pattern_json:
+            pat = _read_json(args.pattern_json)
+            if pat:
+                shadow_write_pattern(pat)
+            return 0
+        parser.print_help(sys.stderr)
+        return 0
+    except Exception as e:  # pragma: no cover - defensive
+        _warn_once(f"shadow_write CLI unexpected error: {e}")
+        emit_managed_event(
+            "managed_agents_fallback",
+            {"reason": "cli_unexpected", "detail": str(e)},
+        )
+        return 0  # Bash callers must see exit 0.
+
+
+if __name__ == "__main__":
+    sys.exit(_main())