@simbimbo/memory-ocmemog 0.1.4

package/scripts/ocmemog-guard.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG="${OPENCLAW_CONFIG_PATH:-$HOME/.openclaw/openclaw.json}"
+EXPECTED="memory-ocmemog"
+
+CURRENT=$(jq -r '.plugins.slots.memory // ""' "$CONFIG" 2>/dev/null || echo "")
+if [[ "$CURRENT" != "$EXPECTED" ]]; then
+  osascript -e "display notification \"Memory slot switched to $CURRENT\" with title \"OpenClaw Memory Alert\""
+  echo "ALERT: memory slot is $CURRENT" >> /tmp/ocmemog-guard.alerts.log
+fi

package/scripts/ocmemog-install.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+LA_DIR="$HOME/Library/LaunchAgents"
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:${PATH:-}"
+
+mkdir -p "$LA_DIR"
+
+render_plist() {
+  local src="$1"
+  local dest="$2"
+  python3 - "$src" "$dest" "$ROOT_DIR" <<'PY'
+from pathlib import Path
+import sys
+
+src = Path(sys.argv[1])
+dest = Path(sys.argv[2])
+root = sys.argv[3]
+dest.write_text(src.read_text(encoding="utf-8").replace("__ROOT_DIR__", root), encoding="utf-8")
+PY
+}
+
+wait_for_label_unloaded() {
+  local label="$1"
+  local attempts="${2:-25}"
+  local sleep_s="${3:-0.2}"
+  for ((i=0; i<attempts; i++)); do
+    if ! launchctl print "gui/$UID/$label" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep "$sleep_s"
+  done
+  return 1
+}
+
+bootstrap_label() {
+  local label="$1"
+  local rendered="$2"
+  local last_output=""
+  for attempt in 1 2 3; do
+    if last_output=$(launchctl bootstrap "gui/$UID" "$rendered" 2>&1); then
+      return 0
+    fi
+    if [[ "$last_output" == *"Input/output error"* ]]; then
+      sleep 1
+      continue
+    fi
+    printf '%s\n' "$last_output" >&2
+    return 1
+  done
+  printf '%s\n' "$last_output" >&2
+  return 1
+}
+
+for plist in "$ROOT_DIR"/scripts/launchagents/com.openclaw.ocmemog.{sidecar,ponder,guard}.plist; do
+  rendered="$LA_DIR/$(basename "$plist")"
+  render_plist "$plist" "$rendered"
+  plutil -lint "$rendered" >/dev/null
+  label=$(basename "$plist" .plist)
+  launchctl bootout "gui/$UID/$label" 2>/dev/null || true
+  wait_for_label_unloaded "$label" || true
+  bootstrap_label "$label" "$rendered"
+  launchctl enable "gui/$UID/$label" 2>/dev/null || true
+  launchctl kickstart -kp "gui/$UID/$label"
+  echo "Loaded $label"
+done
+
+if ! command -v ollama >/dev/null 2>&1; then
+  echo "Ollama not found. Install from: https://ollama.com/download"
+  echo "Then run: ollama pull phi3 && ollama pull nomic-embed-text"
+  exit 0
+fi
+
+if ! ollama list | rg -q "phi3"; then
+  echo "Pulling phi3..."
+  ollama pull phi3
+fi
+
+if ! ollama list | rg -q "nomic-embed-text"; then
+  echo "Pulling nomic-embed-text..."
+  ollama pull nomic-embed-text
+fi
+
+if ! command -v ffmpeg >/dev/null 2>&1; then
+  echo "ffmpeg not found. Install with: brew install ffmpeg"
+fi
+
+if [ -z "${OPENAI_API_KEY:-}" ]; then
+  echo "OPENAI_API_KEY not set. Whisper transcription will be disabled."
+fi
+
+echo "ocmemog install complete."

package/scripts/ocmemog-load-test.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from urllib import request as urlrequest
+
+ENDPOINT = "http://127.0.0.1:17890"
+ASYNC_DEFAULT = os.environ.get("OCMEMOG_INGEST_ASYNC_DEFAULT", "true").lower() in {"1", "true", "yes"}
+INGEST_PATH = "/memory/ingest_async" if ASYNC_DEFAULT else "/memory/ingest"
+
+QUERIES = [
+    "ssh key policy",
+    "synology nas",
+    "openclaw status --deep",
+    "ollama embeddings",
+    "memory pipeline",
+    "calix arden",
+    "gateway bind loopback",
+]
+
+
+def post(path: str, payload: dict, token: str | None = None) -> None:
+    data = json.dumps(payload).encode("utf-8")
+    req = urlrequest.Request(f"{ENDPOINT}{path}", data=data, method="POST")
+    req.add_header("Content-Type", "application/json")
+    if token:
+        req.add_header("x-ocmemog-token", token)
+    with urlrequest.urlopen(req, timeout=30) as resp:
+        resp.read()
+
+
+def run_load(mode: str, duration: int, concurrency: int, token: str | None) -> dict:
+    start = time.time()
+    latencies = []
+    errors = 0
+    total = 0
+
+    stop_at = start + duration
+
+    def worker():
+        nonlocal errors, total
+        while time.time() < stop_at:
+            t0 = time.time()
+            try:
+                if mode == "search":
+                    query = random.choice(QUERIES)
+                    post("/memory/search", {"query": query, "limit": 5}, token)
+                elif mode == "ingest":
+                    content = f"load test {random.randint(1, 100000)}"
+                    post(INGEST_PATH, {"content": content, "kind": "memory", "memory_type": "knowledge"}, token)
+                else:
+                    if random.random() < 0.7:
+                        query = random.choice(QUERIES)
+                        post("/memory/search", {"query": query, "limit": 5}, token)
+                    else:
+                        content = f"load test {random.randint(1, 100000)}"
+                        post(INGEST_PATH, {"content": content, "kind": "memory", "memory_type": "knowledge"}, token)
+            except Exception:
+                errors += 1
+            else:
+                latencies.append(time.time() - t0)
+                total += 1
+
+    with ThreadPoolExecutor(max_workers=concurrency) as pool:
+        futures = [pool.submit(worker) for _ in range(concurrency)]
+        for f in as_completed(futures):
+            pass
+
+    if latencies:
+        lat_sorted = sorted(latencies)
+        p95 = lat_sorted[int(len(lat_sorted) * 0.95) - 1]
+        avg = sum(latencies) / len(latencies)
+    else:
+        avg = 0
+        p95 = 0
+
+    return {
+        "mode": mode,
+        "duration_s": duration,
+        "concurrency": concurrency,
+        "requests": total,
+        "errors": errors,
+        "avg_ms": round(avg * 1000, 2),
+        "p95_ms": round(p95 * 1000, 2),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--mode", default="mixed", choices=["search", "ingest", "mixed"])
+    parser.add_argument("--duration", type=int, default=20)
+    parser.add_argument("--concurrency", type=int, default=10)
+    parser.add_argument("--token", default="")
+    args = parser.parse_args()
+
+    result = run_load(args.mode, args.duration, args.concurrency, args.token or None)
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/ocmemog-ponder.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ENDPOINT="${OCMEMOG_ENDPOINT:-http://127.0.0.1:17890}"
+TOKEN="${OCMEMOG_API_TOKEN:-}"
+MAX_ITEMS="${OCMEMOG_PONDER_ITEMS:-5}"
+
+ARGS=("-H" "content-type: application/json")
+if [[ -n "${TOKEN}" ]]; then
+  ARGS+=("-H" "x-ocmemog-token: ${TOKEN}")
+fi
+
+QUEUE_DEPTH=$(python3 - <<'PY'
+import json, urllib.request, os
+endpoint = os.environ.get("OCMEMOG_ENDPOINT", "http://127.0.0.1:17890")
+req = urllib.request.Request(f"{endpoint}/memory/ingest_status")
+with urllib.request.urlopen(req, timeout=10) as resp:
+    data = json.loads(resp.read().decode("utf-8"))
+print(data.get("queueDepth", 0))
+PY
+)
+
+if [[ "${QUEUE_DEPTH}" != "0" ]]; then
+  echo "ponder skipped: queueDepth=${QUEUE_DEPTH}" >/tmp/ocmemog-ponder.last.json
+  exit 0
+fi
+
+curl -s "${ENDPOINT}/memory/ponder" \
+  "${ARGS[@]}" \
+  -d "{\"max_items\":${MAX_ITEMS}}" >/tmp/ocmemog-ponder.last.json

package/scripts/ocmemog-recall-test.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+from urllib import request as urlrequest
+
+ENDPOINT = "http://127.0.0.1:17890"
+QUERIES = [
+    "ssh key policy",
+    "synology nas",
+    "openclaw status --deep",
+    "ollama embeddings",
+    "memory pipeline",
+    "calix arden",
+]
+
+
+def post(path: str, payload: dict) -> dict:
+    data = json.dumps(payload).encode("utf-8")
+    req = urlrequest.Request(f"{ENDPOINT}{path}", data=data, method="POST")
+    req.add_header("Content-Type", "application/json")
+    with urlrequest.urlopen(req, timeout=20) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def run_search() -> dict:
+    results = {}
+    for q in QUERIES:
+        res = post("/memory/search", {"query": q, "limit": 5})
+        ids = [r.get("entry_id") for r in (res.get("results") or [])]
+        results[q] = ids
+    return results
+
+
+def overlap(a: list[str], b: list[str]) -> float:
+    if not a or not b:
+        return 0.0
+    return len(set(a) & set(b)) / max(1, len(set(a)))
+
+
+def main() -> None:
+    before = run_search()
+    import os
+    subprocess.run(["launchctl", "kickstart", "-k", f"gui/{os.getuid()}/com.openclaw.ocmemog.sidecar"], check=False)
+    time.sleep(2)
+    after = run_search()
+
+    rows = []
+    for q in QUERIES:
+        rows.append({"query": q, "overlap": round(overlap(before[q], after[q]), 3)})
+
+    print(json.dumps({"overlap": rows}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/ocmemog-reindex-vectors.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT))
+
+from brain.runtime.memory import vector_index
+
+if __name__ == "__main__":
+    count = vector_index.rebuild_vector_index()
+    print(f"reindexed: {count}")

package/scripts/ocmemog-reliability-soak.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Dict
+from urllib import error as urlerror
+from urllib import request as urlrequest
+
+
+def _post(base_url: str, path: str, payload: Dict[str, Any], *, token: str | None, timeout: float) -> Dict[str, Any]:
+    req = urlrequest.Request(
+        f"{base_url}{path}",
+        data=json.dumps(payload).encode("utf-8"),
+        method="POST",
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("x-ocmemog-token", token)
+    with urlrequest.urlopen(req, timeout=timeout) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def _run_sequence(base_url: str, *, token: str | None, timeout: float, worker_id: int, iteration: int) -> Dict[str, Any]:
+    session_id = f"soak-sess-{worker_id}"
+    thread_id = f"soak-thread-{worker_id}"
+    conversation_id = f"soak-conv-{worker_id}"
+    user_id = f"u-{worker_id}-{iteration}"
+    assistant_id = f"a-{worker_id}-{iteration}"
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+
+    ingest_user = _post(
+        base_url,
+        "/conversation/ingest_turn",
+        {
+            "role": "user",
+            "content": f"reliability soak iteration {iteration}: keep continuity hydrated",
+            "conversation_id": conversation_id,
+            "session_id": session_id,
+            "thread_id": thread_id,
+            "message_id": user_id,
+            "timestamp": stamp,
+        },
+        token=token,
+        timeout=timeout,
+    )
+    ingest_assistant = _post(
+        base_url,
+        "/conversation/ingest_turn",
+        {
+            "role": "assistant",
+            "content": f"I will checkpoint and ponder iteration {iteration}.",
+            "conversation_id": conversation_id,
+            "session_id": session_id,
+            "thread_id": thread_id,
+            "message_id": assistant_id,
+            "timestamp": stamp,
+            "metadata": {"reply_to_message_id": user_id},
+        },
+        token=token,
+        timeout=timeout,
+    )
+    hydrate = _post(
+        base_url,
+        "/conversation/hydrate",
+        {
+            "conversation_id": conversation_id,
+            "session_id": session_id,
+            "thread_id": thread_id,
+            "turns_limit": 12,
+            "memory_limit": 6,
+        },
+        token=token,
+        timeout=timeout,
+    )
+    checkpoint = _post(
+        base_url,
+        "/conversation/checkpoint",
+        {
+            "conversation_id": conversation_id,
+            "session_id": session_id,
+            "thread_id": thread_id,
+            "checkpoint_kind": "soak",
+            "turns_limit": 12,
+        },
+        token=token,
+        timeout=timeout,
+    )
+    checkpoint_expand = _post(
+        base_url,
+        "/conversation/checkpoint_expand",
+        {
+            "checkpoint_id": int((checkpoint.get("checkpoint") or {}).get("id") or 0),
+            "turns_limit": 24,
+            "radius_turns": 1,
+        },
+        token=token,
+        timeout=timeout,
+    )
+    ponder = _post(
+        base_url,
+        "/memory/ponder",
+        {"max_items": 4},
+        token=token,
+        timeout=timeout,
+    )
+    return {
+        "ok": all(
+            item.get("ok") is True
+            for item in (ingest_user, ingest_assistant, hydrate, checkpoint, checkpoint_expand, ponder)
+        ),
+        "hydrateWarnings": hydrate.get("warnings", []),
+        "checkpointId": (checkpoint.get("checkpoint") or {}).get("id"),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run a live reliability soak against the local ocmemog sidecar.")
+    parser.add_argument("--base-url", default="http://127.0.0.1:17890")
+    parser.add_argument("--iterations", type=int, default=8)
+    parser.add_argument("--workers", type=int, default=2)
+    parser.add_argument("--timeout", type=float, default=30.0)
+    parser.add_argument("--token", default="")
+    args = parser.parse_args()
+
+    started = time.monotonic()
+    failures = []
+    successes = 0
+
+    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
+        futures = [
+            pool.submit(
+                _run_sequence,
+                args.base_url,
+                token=args.token or None,
+                timeout=args.timeout,
+                worker_id=(idx % max(1, args.workers)) + 1,
+                iteration=idx + 1,
+            )
+            for idx in range(max(1, args.iterations))
+        ]
+        for future in as_completed(futures):
+            try:
+                result = future.result()
+            except Exception as exc:
+                failures.append({"error": str(exc)})
+                continue
+            if result.get("ok"):
+                successes += 1
+            else:
+                failures.append(result)
+
+    payload = {
+        "ok": not failures,
+        "iterations": max(1, args.iterations),
+        "workers": max(1, args.workers),
+        "successes": successes,
+        "failures": failures,
+        "elapsed_s": round(time.monotonic() - started, 3),
+    }
+    print(json.dumps(payload, indent=2))
+    if failures:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except urlerror.HTTPError as exc:
+        print(json.dumps({"ok": False, "error": f"http {exc.code}", "detail": exc.reason}, indent=2))
+        raise SystemExit(1)
+    except urlerror.URLError as exc:
+        print(json.dumps({"ok": False, "error": "unreachable", "detail": str(exc.reason)}, indent=2))
+        raise SystemExit(1)

package/scripts/ocmemog-sidecar.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+HOST="${OCMEMOG_HOST:-127.0.0.1}"
+PORT="${OCMEMOG_PORT:-17890}"
+PYTHON_BIN="${OCMEMOG_PYTHON_BIN:-}"
+
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:${PATH:-}"
+cd "${ROOT_DIR}"
+
+export OCMEMOG_STATE_DIR="${OCMEMOG_STATE_DIR:-${ROOT_DIR}/.ocmemog-state}"
+export PYTHONPATH="${ROOT_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
+mkdir -p "${OCMEMOG_STATE_DIR}" "${OCMEMOG_STATE_DIR}/logs"
+
+# defaults for local ollama-backed inference/embeddings
+export OCMEMOG_USE_OLLAMA="${OCMEMOG_USE_OLLAMA:-true}"
+export OCMEMOG_OLLAMA_MODEL="${OCMEMOG_OLLAMA_MODEL:-phi3:latest}"
+export OCMEMOG_OLLAMA_EMBED_MODEL="${OCMEMOG_OLLAMA_EMBED_MODEL:-nomic-embed-text:latest}"
+export BRAIN_EMBED_MODEL_PROVIDER="${BRAIN_EMBED_MODEL_PROVIDER:-ollama}"
+export BRAIN_EMBED_MODEL_LOCAL="${BRAIN_EMBED_MODEL_LOCAL:-}"
+
+# always-on transcript watcher defaults
+export OCMEMOG_TRANSCRIPT_WATCHER="${OCMEMOG_TRANSCRIPT_WATCHER:-true}"
+export OCMEMOG_SESSION_DIR="${OCMEMOG_SESSION_DIR:-$HOME/.openclaw/agents/main/sessions}"
+export OCMEMOG_TRANSCRIPT_POLL_SECONDS="${OCMEMOG_TRANSCRIPT_POLL_SECONDS:-30}"
+export OCMEMOG_INGEST_BATCH_SECONDS="${OCMEMOG_INGEST_BATCH_SECONDS:-30}"
+export OCMEMOG_INGEST_BATCH_MAX="${OCMEMOG_INGEST_BATCH_MAX:-25}"
+export OCMEMOG_INGEST_ENDPOINT="${OCMEMOG_INGEST_ENDPOINT:-http://127.0.0.1:17890/memory/ingest_async}"
+export OCMEMOG_REINFORCE_SENTIMENT="${OCMEMOG_REINFORCE_SENTIMENT:-true}"
+export OCMEMOG_INGEST_SOURCE="${OCMEMOG_INGEST_SOURCE:-transcript}"
+export OCMEMOG_INGEST_MEMORY_TYPE="${OCMEMOG_INGEST_MEMORY_TYPE:-reflections}"
+
+# promotion/demotion thresholds for stress testing
+export OCMEMOG_PROMOTION_THRESHOLD="${OCMEMOG_PROMOTION_THRESHOLD:-0.8}"
+export OCMEMOG_DEMOTION_THRESHOLD="${OCMEMOG_DEMOTION_THRESHOLD:-0.4}"
+
+if [[ -z "${PYTHON_BIN}" ]]; then
+  if [[ -x "${ROOT_DIR}/.venv/bin/python" ]]; then
+    PYTHON_BIN="${ROOT_DIR}/.venv/bin/python"
+  else
+    PYTHON_BIN="$(command -v python3)"
+  fi
+fi
+
+exec "${PYTHON_BIN}" -m uvicorn ocmemog.sidecar.app:app --host "${HOST}" --port "${PORT}"

package/scripts/ocmemog-soak-report.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in", dest="input", default=str(REPO_ROOT / "reports" / "load" / "soak-latest.jsonl"))
+    parser.add_argument("--out", default=str(REPO_ROOT / "reports" / "load" / "soak-report.html"))
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        raise SystemExit(f"missing {input_path}")
+
+    rows = []
+    for line in input_path.read_text(encoding="utf-8").splitlines():
+        try:
+            rows.append(json.loads(line))
+        except Exception:
+            continue
+
+    if not rows:
+        raise SystemExit("no data")
+
+    def row_html(r):
+        return (
+            f"<tr><td>{r.get('timestamp')}</td>"
+            f"<td>{r.get('mode')}</td>"
+            f"<td>{r.get('concurrency')}</td>"
+            f"<td>{r.get('requests')}</td>"
+            f"<td>{r.get('errors')}</td>"
+            f"<td>{r.get('avg_ms')}</td>"
+            f"<td>{r.get('p95_ms')}</td></tr>"
+        )
+
+    html = """<html><head><title>ocmemog soak report</title>
+    <style>table{border-collapse:collapse}td,th{border:1px solid #ccc;padding:4px}</style>
+    </head><body>
+    <h2>ocmemog soak report</h2>
+    <table>
+      <tr><th>time</th><th>mode</th><th>concurrency</th><th>requests</th><th>errors</th><th>avg_ms</th><th>p95_ms</th></tr>
+    """
+    html += "\n".join(row_html(r) for r in rows)
+    html += "</table></body></html>"
+
+    Path(args.out).write_text(html, encoding="utf-8")
+    print(args.out)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/ocmemog-soak-test.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+import subprocess
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--duration", type=int, default=3600)
+    parser.add_argument("--interval", type=int, default=60)
+    parser.add_argument("--concurrency", type=int, default=10)
+    parser.add_argument("--mode", default="mixed", choices=["search", "ingest", "mixed"])
+    parser.add_argument("--out", default=str(REPO_ROOT / "reports" / "load" / "soak-latest.jsonl"))
+    args = parser.parse_args()
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    start = time.time()
+    while time.time() - start < args.duration:
+        cmd = [
+            str(REPO_ROOT / "scripts" / "ocmemog-load-test.py"),
+            "--mode", args.mode,
+            "--duration", str(args.interval),
+            "--concurrency", str(args.concurrency),
+        ]
+        result = subprocess.check_output(cmd).decode("utf-8").strip()
+        data = json.loads(result)
+        data["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%S")
+        with out_path.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(data) + "\n")
+        time.sleep(1)
+
+
+if __name__ == "__main__":
+    main()