@techwavedev/agi-agent-kit 1.1.7 → 1.2.1

package/templates/base/execution/memory_manager.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+"""
+Script: memory_manager.py
+Purpose: Unified memory management wrapper for all qdrant-memory operations.
+         Provides a single entry point for agents across any AI environment.
+
+Usage:
+    # Auto-decide: check cache first, then retrieve context
+    python3 execution/memory_manager.py auto --query "How to set up auth middleware?"
+
+    # Explicit store (decision, code, error, technical, conversation)
+    python3 execution/memory_manager.py store --content "Chose PostgreSQL for relational model" --type decision --project myapp
+
+    # Retrieve context only
+    python3 execution/memory_manager.py retrieve --query "database architecture" --top-k 5
+
+    # Cache a response
+    python3 execution/memory_manager.py cache-store --query "How to X?" --response "Do Y..."
+
+    # Health check (Qdrant + Ollama)
+    python3 execution/memory_manager.py health
+
+Environment Variables:
+    EMBEDDING_PROVIDER  - "ollama" (default), "openai", or "bedrock"
+    OLLAMA_URL          - Ollama server URL (default: http://localhost:11434)
+    QDRANT_URL          - Qdrant server URL (default: http://localhost:6333)
+    MEMORY_COLLECTION   - Memory collection name (default: agent_memory)
+    CACHE_COLLECTION    - Cache collection name (default: semantic_cache)
+
+Exit Codes:
+    0 - Success
+    1 - No results / cache miss
+    2 - Connection error (Qdrant or embedding service down)
+    3 - Operation error
+"""
+
+import argparse
+import json
+import os
+import sys
+from urllib.request import Request, urlopen
+from urllib.error import URLError
+
+# Resolve path to qdrant-memory scripts
+SKILL_SCRIPTS_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "skills",
+    "qdrant-memory",
+    "scripts",
+)
+sys.path.insert(0, SKILL_SCRIPTS_DIR)
+
+from embedding_utils import check_embedding_service, get_embedding_dimension
+from semantic_cache import check_cache, store_response, clear_cache
+from memory_retrieval import retrieve_context, store_memory, list_memories, build_filter
+
+# Configuration
+QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
+
+
+def health_check() -> dict:
+    """Check Qdrant connectivity and embedding service status."""
+    result = {"qdrant": "unknown", "embeddings": "unknown", "collections": []}
+
+    # Check Qdrant
+    try:
+        req = Request(f"{QDRANT_URL}/collections", method="GET")
+        with urlopen(req, timeout=10) as response:
+            data = json.loads(response.read().decode())
+            collections = [
+                c["name"] for c in data.get("result", {}).get("collections", [])
+            ]
+            result["qdrant"] = "ok"
+            result["collections"] = collections
+    except Exception as e:
+        result["qdrant"] = f"error: {e}"
+
+    # Check embedding service
+    embed_status = check_embedding_service()
+    result["embeddings"] = embed_status
+
+    # Check expected collections
+    expected = ["agent_memory", "semantic_cache"]
+    result["missing_collections"] = [
+        c for c in expected if c not in result.get("collections", [])
+    ]
+
+    # Overall status
+    result["ready"] = (
+        result["qdrant"] == "ok"
+        and embed_status.get("status") == "ok"
+        and len(result["missing_collections"]) == 0
+    )
+
+    return result
+
+
+def auto_query(query: str, project: str = None, threshold: float = 0.92) -> dict:
+    """
+    Smart query: check cache first, then retrieve context.
+    This is the primary entry point for agents.
+
+    Flow:
+    1. Check semantic cache (exact match saves 100% tokens)
+    2. If miss, retrieve relevant memories (saves 80-95% tokens)
+    3. Return combined result with token savings estimate
+    """
+    result = {
+        "source": "none",
+        "cache_hit": False,
+        "context_chunks": [],
+        "tokens_saved_estimate": 0,
+    }
+
+    # Step 1: Semantic cache check
+    try:
+        cached = check_cache(query, threshold)
+        if cached and cached.get("cache_hit"):
+            result["source"] = "cache"
+            result["cache_hit"] = True
+            result["cached_response"] = cached.get("response", "")
+            result["cache_score"] = cached.get("score", 0)
+            result["tokens_saved_estimate"] = cached.get("tokens_saved", 0)
+            return result
+    except Exception:
+        pass  # Cache miss or error, continue to retrieval
+
+    # Step 2: Context retrieval
+    try:
+        filters = None
+        if project:
+            filters = {"must": [{"key": "project", "match": {"value": project}}]}
+
+        context = retrieve_context(query, filters=filters, top_k=5, score_threshold=0.7)
+        if context.get("total_chunks", 0) > 0:
+            result["source"] = "memory"
+            result["context_chunks"] = context.get("chunks", [])
+            result["total_chunks"] = context.get("total_chunks", 0)
+            result["tokens_saved_estimate"] = context.get("total_tokens_estimate", 0)
+    except Exception:
+        pass  # No context available
+
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Unified memory manager for AI agents",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Smart auto-query (cache + context retrieval)
+  python3 execution/memory_manager.py auto --query "How to handle JWT refresh?"
+
+  # Store a key decision
+  python3 execution/memory_manager.py store --content "Chose Supabase for auth" --type decision
+
+  # Health check
+  python3 execution/memory_manager.py health
+        """,
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # Auto command (primary entry point)
+    auto_parser = subparsers.add_parser(
+        "auto", help="Smart query: cache check + context retrieval"
+    )
+    auto_parser.add_argument("--query", required=True, help="Natural language query")
+    auto_parser.add_argument("--project", help="Filter by project name")
+    auto_parser.add_argument(
+        "--threshold", type=float, default=0.92, help="Cache similarity threshold"
+    )
+
+    # Store command
+    store_parser = subparsers.add_parser(
+        "store", help="Store memory (decision, code, error, technical, conversation)"
+    )
+    store_parser.add_argument("--content", required=True, help="Memory content")
+    store_parser.add_argument(
+        "--type",
+        required=True,
+        choices=["decision", "code", "error", "conversation", "technical"],
+        help="Memory type",
+    )
+    store_parser.add_argument("--project", help="Project name")
+    store_parser.add_argument("--tags", nargs="+", help="Tags for the memory")
+
+    # Retrieve command
+    retrieve_parser = subparsers.add_parser(
+        "retrieve", help="Retrieve relevant context"
+    )
+    retrieve_parser.add_argument("--query", required=True, help="Search query")
+    retrieve_parser.add_argument("--type", help="Filter by memory type")
+    retrieve_parser.add_argument("--project", help="Filter by project")
+    retrieve_parser.add_argument(
+        "--top-k", type=int, default=5, help="Number of results"
+    )
+    retrieve_parser.add_argument(
+        "--threshold", type=float, default=0.7, help="Score threshold"
+    )
+
+    # Cache store command
+    cache_parser = subparsers.add_parser(
+        "cache-store", help="Store query-response in semantic cache"
+    )
+    cache_parser.add_argument("--query", required=True, help="Original query")
+    cache_parser.add_argument("--response", required=True, help="LLM response to cache")
+    cache_parser.add_argument("--model", default="agent", help="Model identifier")
+    cache_parser.add_argument("--project", help="Project name")
+
+    # List command
+    list_parser = subparsers.add_parser("list", help="List stored memories")
+    list_parser.add_argument("--type", help="Filter by memory type")
+    list_parser.add_argument("--project", help="Filter by project")
+    list_parser.add_argument("--limit", type=int, default=20, help="Max results")
+
+    # Health command
+    subparsers.add_parser("health", help="Check Qdrant + embedding service health")
+
+    # Cache clear command
+    clear_parser = subparsers.add_parser("cache-clear", help="Clear old cache entries")
+    clear_parser.add_argument(
+        "--older-than", type=int, default=7, help="Delete entries older than N days"
+    )
+
+    args = parser.parse_args()
+
+    try:
+        if args.command == "auto":
+            result = auto_query(args.query, args.project, args.threshold)
+            print(json.dumps(result, indent=2))
+            sys.exit(0 if result["source"] != "none" else 1)
+
+        elif args.command == "store":
+            metadata = {}
+            if args.project:
+                metadata["project"] = args.project
+            if args.tags:
+                metadata["tags"] = args.tags
+            result = store_memory(args.content, args.type, metadata)
+            print(json.dumps(result, indent=2))
+            sys.exit(0)
+
+        elif args.command == "retrieve":
+            filters = build_filter(
+                type_filter=getattr(args, "type", None), project=args.project
+            )
+            result = retrieve_context(
+                args.query,
+                filters={"must": filters["must"]} if filters else None,
+                top_k=args.top_k,
+                score_threshold=args.threshold,
+            )
+            print(json.dumps(result, indent=2))
+            sys.exit(0 if result.get("total_chunks", 0) > 0 else 1)
+
+        elif args.command == "cache-store":
+            metadata = {"model": args.model}
+            if args.project:
+                metadata["project"] = args.project
+            result = store_response(args.query, args.response, metadata)
+            print(json.dumps(result, indent=2))
+            sys.exit(0)
+
+        elif args.command == "list":
+            filters = build_filter(
+                type_filter=getattr(args, "type", None), project=args.project
+            )
+            result = list_memories(
+                filters={"must": filters["must"]} if filters else None, limit=args.limit
+            )
+            print(json.dumps(result, indent=2))
+            sys.exit(0)
+
+        elif args.command == "health":
+            result = health_check()
+            print(json.dumps(result, indent=2))
+            sys.exit(0 if result["ready"] else 2)
+
+        elif args.command == "cache-clear":
+            result = clear_cache(args.older_than)
+            print(json.dumps(result, indent=2))
+            sys.exit(0)
+
+    except URLError as e:
+        print(
+            json.dumps(
+                {
+                    "status": "error",
+                    "type": "connection_error",
+                    "message": str(e),
+                    "hint": "Is Qdrant running? Try: docker run -p 6333:6333 qdrant/qdrant",
+                }
+            ),
+            file=sys.stderr,
+        )
+        sys.exit(2)
+    except Exception as e:
+        print(
+            json.dumps(
+                {"status": "error", "type": type(e).__name__, "message": str(e)}
+            ),
+            file=sys.stderr,
+        )
+        sys.exit(3)
+
+
+if __name__ == "__main__":
+    main()

package/templates/base/execution/session_boot.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""
+Script: session_boot.py
+Purpose: Single entry point for session initialization. Checks memory system,
+         initializes if needed, and returns a combined status report.
+
+         This is the FIRST script an agent should run at session start.
+         Combines: health check + session_init + platform detection.
+
+Usage:
+    python3 execution/session_boot.py
+    python3 execution/session_boot.py --json
+    python3 execution/session_boot.py --auto-fix
+
+Arguments:
+    --json       Output JSON only (for programmatic use)
+    --auto-fix   Automatically fix issues (pull model, create collections)
+
+Exit Codes:
+    0 - Memory system ready
+    1 - Memory available but degraded (missing model, empty collections)
+    2 - Memory unavailable (Qdrant or Ollama not running)
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from urllib.request import Request, urlopen
+from urllib.error import URLError, HTTPError
+
+# Configuration
+QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434")
+EMBEDDING_MODEL = "nomic-embed-text"
+PROJECT_DIR = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def check_qdrant() -> dict:
+    """Check Qdrant connectivity and collection status."""
+    result = {"status": "not_running", "collections": {}, "url": QDRANT_URL}
+    try:
+        req = Request(f"{QDRANT_URL}/collections", method="GET")
+        with urlopen(req, timeout=5) as response:
+            data = json.loads(response.read().decode())
+            names = [c["name"] for c in data.get("result", {}).get("collections", [])]
+            result["status"] = "ok"
+
+            for col_name in ["agent_memory", "semantic_cache"]:
+                if col_name in names:
+                    try:
+                        col_req = Request(f"{QDRANT_URL}/collections/{col_name}", method="GET")
+                        with urlopen(col_req, timeout=5) as col_resp:
+                            col_data = json.loads(col_resp.read().decode())
+                            points = col_data.get("result", {}).get("points_count", 0)
+                            result["collections"][col_name] = {"exists": True, "points": points}
+                    except Exception:
+                        result["collections"][col_name] = {"exists": True, "points": -1}
+                else:
+                    result["collections"][col_name] = {"exists": False, "points": 0}
+    except (URLError, HTTPError, Exception):
+        pass
+    return result
+
+
+def check_ollama() -> dict:
+    """Check Ollama connectivity and embedding model."""
+    result = {"status": "not_running", "has_model": False, "url": OLLAMA_URL}
+    try:
+        req = Request(f"{OLLAMA_URL}/api/tags", method="GET")
+        with urlopen(req, timeout=5) as response:
+            data = json.loads(response.read().decode())
+            models = [m["name"] for m in data.get("models", [])]
+            result["status"] = "ok"
+            result["models"] = models
+            result["has_model"] = any(EMBEDDING_MODEL in m for m in models)
+    except (URLError, HTTPError, Exception):
+        pass
+    return result
+
+
+def run_session_init() -> bool:
+    """Run session_init.py to create collections."""
+    init_script = PROJECT_DIR / "execution" / "session_init.py"
+    if not init_script.exists():
+        return False
+    try:
+        proc = subprocess.run(
+            ["python3", str(init_script)],
+            capture_output=True, text=True, timeout=30,
+            cwd=str(PROJECT_DIR),
+        )
+        return proc.returncode == 0
+    except Exception:
+        return False
+
+
+def pull_model() -> bool:
+    """Pull the embedding model via Ollama."""
+    try:
+        proc = subprocess.run(
+            ["ollama", "pull", EMBEDDING_MODEL],
+            capture_output=True, text=True, timeout=120,
+        )
+        return proc.returncode == 0
+    except Exception:
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Session boot: check + initialize memory system"
+    )
+    parser.add_argument("--json", action="store_true", dest="json_output",
+                        help="JSON output only")
+    parser.add_argument("--auto-fix", action="store_true",
+                        help="Auto-fix issues (pull model, create collections)")
+    args = parser.parse_args()
+
+    report = {
+        "qdrant": {},
+        "ollama": {},
+        "memory_ready": False,
+        "actions_taken": [],
+        "issues": [],
+    }
+
+    # Step 1: Check Qdrant
+    qdrant = check_qdrant()
+    report["qdrant"] = qdrant
+
+    if qdrant["status"] != "ok":
+        report["issues"].append("Qdrant not running. Start with: docker run -d -p 6333:6333 -v qdrant_storage:/qdrant/storage qdrant/qdrant")
+
+    # Step 2: Check Ollama
+    ollama = check_ollama()
+    report["ollama"] = ollama
+
+    if ollama["status"] != "ok":
+        report["issues"].append("Ollama not running. Start with: ollama serve")
+    elif not ollama["has_model"]:
+        if args.auto_fix:
+            if not args.json_output:
+                print(f"⏳ Pulling {EMBEDDING_MODEL}...")
+            if pull_model():
+                report["actions_taken"].append(f"Pulled {EMBEDDING_MODEL}")
+                ollama["has_model"] = True
+            else:
+                report["issues"].append(f"Failed to pull {EMBEDDING_MODEL}")
+        else:
+            report["issues"].append(f"Embedding model missing. Run: ollama pull {EMBEDDING_MODEL}")
+
+    # Step 3: Initialize collections if needed
+    if qdrant["status"] == "ok" and ollama["status"] == "ok" and ollama["has_model"]:
+        agent_mem = qdrant["collections"].get("agent_memory", {})
+        sem_cache = qdrant["collections"].get("semantic_cache", {})
+
+        if not agent_mem.get("exists") or not sem_cache.get("exists"):
+            if args.auto_fix:
+                if not args.json_output:
+                    print("⏳ Initializing collections...")
+                if run_session_init():
+                    report["actions_taken"].append("Created collections via session_init.py")
+                    # Re-check
+                    qdrant = check_qdrant()
+                    report["qdrant"] = qdrant
+                else:
+                    report["issues"].append("Failed to initialize collections")
+            else:
+                report["issues"].append("Collections missing. Run: python3 execution/session_init.py")
+
+    # Final readiness
+    qdrant = report["qdrant"]
+    ollama = report["ollama"]
+    agent_mem = qdrant.get("collections", {}).get("agent_memory", {})
+    sem_cache = qdrant.get("collections", {}).get("semantic_cache", {})
+
+    report["memory_ready"] = (
+        qdrant.get("status") == "ok"
+        and ollama.get("status") == "ok"
+        and ollama.get("has_model", False)
+        and agent_mem.get("exists", False)
+        and sem_cache.get("exists", False)
+    )
+
+    # Summary
+    total_points = (
+        agent_mem.get("points", 0) + sem_cache.get("points", 0)
+    )
+    report["summary"] = {
+        "ready": report["memory_ready"],
+        "total_memories": agent_mem.get("points", 0),
+        "total_cached": sem_cache.get("points", 0),
+    }
+
+    if args.json_output:
+        print(json.dumps(report, indent=2))
+    else:
+        if report["memory_ready"]:
+            mem_pts = agent_mem.get("points", 0)
+            cache_pts = sem_cache.get("points", 0)
+            print(f"✅ Memory system ready — {mem_pts} memories, {cache_pts} cached responses")
+        else:
+            print("❌ Memory system not ready:")
+            for issue in report["issues"]:
+                print(f"   • {issue}")
+
+        if report["actions_taken"]:
+            for action in report["actions_taken"]:
+                print(f"   ✅ {action}")
+
+    sys.exit(0 if report["memory_ready"] else (1 if qdrant.get("status") == "ok" else 2))
+
+
+if __name__ == "__main__":
+    main()