engine7 7.1.23 → 7.1.25

package/src/memory/everos/python/agentic_server.py

@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+"""
+Agentic Search Server — FastAPI wrapper around the 3-mode search pipeline.
+
+Modes:
+  hybrid          — hybrid + rerank (~2s)
+  hybrid_agentic  — hybrid + rerank + LLM sufficiency + optional multi_query (~10s)
+  agentic         — full cluster-scoped pipeline (~40s)
+
+Usage:
+  uvicorn agentic_server:app --host 127.0.0.1 --port 8101
+
+Or directly:
+  python agentic_server.py --port 8101
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from typing import Any, Literal
+
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+# Import all search logic from agentic_search.py
+from agentic_search import (
+    Episode,
+    AgenticResult,
+    hybrid_search,
+    rerank,
+    rrf_fusion,
+    DB,
+    cluster_scoped_search,
+    check_sufficiency,
+    generate_multi_queries,
+    generate_refined_query,
+    format_docs,
+    dedup_by_id,
+    # Hyperparams
+    ROUND1_TOP_N,
+    ROUND1_RERANK_TOP_N,
+    ROUND2_CAP,
+    CLUSTER_BASE_CANDIDATES,
+    MULTI_QUERY_COUNT,
+    HYBRID_RRF_K,
+    REFINEMENT_STRATEGY,
+)
+import asyncio
+
+app = FastAPI(title="Agentic Search Server", version="1.0.0")
+
+
+# ── Request/Response models ───────────────────────────────────────────
+class SearchRequest(BaseModel):
+    query: str
+    user_id: str = "xiaomei"
+    app_id: str | None = None
+    project_id: str | None = None
+    mode: Literal["hybrid", "hybrid_agentic", "agentic"] = "hybrid_agentic"
+    top_k: int = 5
+    strategy: Literal["multi_query", "refined_query"] = "multi_query"
+
+
+class EpisodeItem(BaseModel):
+    id: str
+    subject: str
+    summary: str
+    episode: str
+    timestamp: str
+    score: float
+
+
+class SearchResponse(BaseModel):
+    episodes: list[EpisodeItem]
+    mode: str
+    method: str
+    is_sufficient: bool = False
+    multi_queries: list[str] = []
+    reasoning: str = ""
+    timing: dict[str, float] = {}
+    total_time: float = 0.0
+
+
+# ── Helpers ───────────────────────────────────────────────────────────
+def episode_to_item(ep: Episode) -> EpisodeItem:
+    return EpisodeItem(
+        id=ep.id,
+        subject=ep.subject[:200],
+        summary=ep.summary[:500],
+        episode=ep.episode[:1000],
+        timestamp=ep.timestamp[:30],
+        score=ep.score,
+    )
+
+
+def result_to_response(result: AgenticResult) -> SearchResponse:
+    return SearchResponse(
+        episodes=[episode_to_item(ep) for ep in result.episodes],
+        mode=result.method,
+        method=result.method,
+        is_sufficient=result.is_sufficient,
+        multi_queries=result.multi_queries,
+        reasoning=result.reasoning,
+        timing=result.timing,
+        total_time=result.timing.get("total", 0.0),
+    )
+
+
+# ── Endpoints ─────────────────────────────────────────────────────────
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@app.post("/api/v1/search", response_model=SearchResponse)
+async def search(req: SearchRequest):
+    """Unified search endpoint — 3 modes."""
+    t_total = time.time()
+
+    uid = req.user_id
+    aid = req.app_id or uid
+    pid = req.project_id or "default"
+
+    if req.mode == "hybrid":
+        # Mode 1: hybrid + rerank only
+        eps, _ = hybrid_search(req.query, top_k=ROUND1_TOP_N, user_id=uid, app_id=aid, project_id=pid)
+        reranked, _ = rerank(req.query, eps)
+        reranked = reranked[: req.top_k]
+        result = AgenticResult(
+            episodes=reranked,
+            method="hybrid+rerank",
+            timing={"total": time.time() - t_total},
+        )
+        return result_to_response(result)
+
+    elif req.mode == "hybrid_agentic":
+        # Mode 2: hybrid + rerank + sufficiency + optional multi_query
+        result = await _hybrid_agentic(req, uid, aid, pid)
+        return result_to_response(result)
+
+    elif req.mode == "agentic":
+        # Mode 3: full cluster-scoped agentic
+        result = await _full_agentic(req, uid, aid, pid)
+        return result_to_response(result)
+
+
+# ── Mode 2: hybrid_agentic ────────────────────────────────────────────
+async def _hybrid_agentic(req: SearchRequest, uid: str, aid: str, pid: str) -> AgenticResult:
+    timing = {}
+    t_total = time.time()
+
+    # Round 1: hybrid + rerank
+    t0 = time.time()
+    eps, _ = hybrid_search(req.query, top_k=ROUND1_TOP_N, user_id=uid, app_id=aid, project_id=pid)
+    if not eps:
+        return AgenticResult(episodes=[], method="hybrid_agentic (empty)", timing={"total": time.time() - t_total})
+    reranked, _ = rerank(req.query, eps)
+    reranked = reranked[:ROUND1_RERANK_TOP_N]
+    timing["hybrid_rerank"] = time.time() - t0
+
+    # Sufficiency
+    t0 = time.time()
+    try:
+        sufficiency = check_sufficiency(req.query, reranked)
+    except Exception:
+        timing["sufficiency"] = time.time() - t0
+        return AgenticResult(
+            episodes=reranked[: req.top_k],
+            method="hybrid_agentic (sufficiency_error)",
+            timing={**timing, "total": time.time() - t_total},
+        )
+    timing["sufficiency"] = time.time() - t0
+
+    if sufficiency.get("is_sufficient", True):
+        return AgenticResult(
+            episodes=reranked[: req.top_k],
+            method="hybrid_agentic (sufficient_r1)",
+            is_sufficient=True,
+            reasoning=sufficiency.get("reasoning", ""),
+            timing={**timing, "total": time.time() - t_total},
+        )
+
+    # Round 2
+    missing = sufficiency.get("missing_information", [])
+    key_info = sufficiency.get("key_information_found", [])
+
+    if req.strategy == "refined_query":
+        t0 = time.time()
+        try:
+            refined = generate_refined_query(req.query, reranked, missing)
+        except Exception:
+            refined = req.query
+        timing["refined_query_gen"] = time.time() - t0
+
+        t0 = time.time()
+        r2_eps, _ = hybrid_search(refined, top_k=ROUND1_TOP_N, user_id=uid, app_id=aid, project_id=pid)
+        timing["round2_search"] = time.time() - t0
+
+        seen = {ep.id for ep in reranked}
+        r2_unique = [ep for ep in r2_eps if ep.id not in seen]
+        keep = max(0, ROUND2_CAP - len(reranked))
+        r2_unique = r2_unique[:keep]
+        merged = list(reranked) + r2_unique
+
+        t0 = time.time()
+        final, _ = rerank(req.query, merged)
+        final = final[: req.top_k]
+        timing["final_rerank"] = time.time() - t0
+
+        return AgenticResult(
+            episodes=final,
+            method="hybrid_agentic (refined_query)",
+            multi_queries=[refined],
+            reasoning=sufficiency.get("reasoning", ""),
+            timing={**timing, "total": time.time() - t_total},
+        )
+
+    # multi_query path
+    t0 = time.time()
+    try:
+        multi_queries = generate_multi_queries(req.query, reranked, missing, key_info)
+    except Exception:
+        multi_queries = []
+    timing["multi_query_gen"] = time.time() - t0
+
+    if not multi_queries:
+        return AgenticResult(
+            episodes=reranked[: req.top_k],
+            method="hybrid_agentic (multi_query_failed)",
+            reasoning=sufficiency.get("reasoning", ""),
+            timing={**timing, "total": time.time() - t_total},
+        )
+
+    t0 = time.time()
+    round2_lists = []
+    for mq in multi_queries:
+        r2_eps, _ = hybrid_search(mq, top_k=ROUND1_TOP_N, user_id=uid, app_id=aid, project_id=pid)
+        round2_lists.append(r2_eps)
+    timing["round2_search"] = time.time() - t0
+
+    if len(round2_lists) == 1:
+        fused = round2_lists[0]
+    else:
+        fused = rrf_fusion(*round2_lists, k=HYBRID_RRF_K)
+
+    seen = {ep.id for ep in reranked}
+    r2_unique = [ep for ep in fused if ep.id not in seen]
+    keep = max(0, ROUND2_CAP - len(reranked))
+    r2_unique = r2_unique[:keep]
+    merged = list(reranked) + r2_unique
+
+    t0 = time.time()
+    final, _ = rerank(req.query, merged)
+    final = final[: req.top_k]
+    timing["final_rerank"] = time.time() - t0
+
+    return AgenticResult(
+        episodes=final,
+        method="hybrid_agentic (multi_round)",
+        multi_queries=multi_queries,
+        reasoning=sufficiency.get("reasoning", ""),
+        timing={**timing, "total": time.time() - t_total},
+    )
+
+
+# ── Mode 3: full agentic (cluster-scoped) ─────────────────────────────
+async def _full_agentic(req: SearchRequest, uid: str, aid: str, pid: str) -> AgenticResult:
+    timing = {}
+    t_total = time.time()
+
+    db = DB()
+
+    t0 = time.time()
+    clusters = db.get_clusters()
+    all_episodes = db.get_all_episodes()
+    timing["load_corpus"] = time.time() - t0
+
+    t0 = time.time()
+    round1 = cluster_scoped_search(req.query, all_episodes, clusters)
+
+    if not round1:
+        fb_eps, _ = hybrid_search(req.query, top_k=req.top_k * 2, user_id=uid, app_id=aid, project_id=pid)
+        fb_reranked, rr_time = rerank(req.query, fb_eps)
+        fb_reranked = fb_reranked[: req.top_k]
+        timing["fallback_hybrid"] = time.time() - t0
+        return AgenticResult(
+            episodes=fb_reranked,
+            method="fallback_hybrid",
+            timing={**timing, "total": time.time() - t_total},
+        )
+
+    timing["cluster_scoped"] = time.time() - t0
+
+    t0 = time.time()
+    reranked, _ = rerank(req.query, round1)
+    reranked = reranked[:ROUND1_RERANK_TOP_N]
+    timing["round1_rerank"] = time.time() - t0
+
+    t0 = time.time()
+    try:
+        sufficiency = check_sufficiency(req.query, reranked)
+    except Exception:
+        timing["sufficiency"] = time.time() - t0
+        return AgenticResult(
+            episodes=reranked[: req.top_k],
+            method="agentic (sufficiency_error)",
+            timing={**timing, "total": time.time() - t_total},
+        )
+    timing["sufficiency"] = time.time() - t0
+
+    if sufficiency.get("is_sufficient", True):
+        return AgenticResult(
+            episodes=reranked[: req.top_k],
+            method="agentic (sufficient_r1)",
+            is_sufficient=True,
+            reasoning=sufficiency.get("reasoning", ""),
+            timing={**timing, "total": time.time() - t_total},
+        )
+
+    # Round 2 (same as hybrid_agentic but base_retrieve is hybrid_full)
+    missing = sufficiency.get("missing_information", [])
+    key_info = sufficiency.get("key_information_found", [])
+
+    t0 = time.time()
+    try:
+        multi_queries = generate_multi_queries(req.query, reranked, missing, key_info)
+    except Exception:
+        multi_queries = []
+    timing["multi_query_gen"] = time.time() - t0
+
+    if not multi_queries:
+        return AgenticResult(
+            episodes=reranked[: req.top_k],
+            method="agentic (multi_query_failed)",
+            reasoning=sufficiency.get("reasoning", ""),
+            timing={**timing, "total": time.time() - t_total},
+        )
+
+    t0 = time.time()
+    round2_lists = []
+    for mq in multi_queries:
+        r2_eps, _ = hybrid_search(mq, top_k=ROUND1_TOP_N, user_id=uid, app_id=aid, project_id=pid)
+        round2_lists.append(r2_eps)
+    timing["round2_search"] = time.time() - t0
+
+    if len(round2_lists) == 1:
+        fused = round2_lists[0]
+    else:
+        fused = rrf_fusion(*round2_lists, k=HYBRID_RRF_K)
+
+    seen = {ep.id for ep in reranked}
+    r2_unique = [ep for ep in fused if ep.id not in seen]
+    keep = max(0, ROUND2_CAP - len(reranked))
+    r2_unique = r2_unique[:keep]
+    merged = list(reranked) + r2_unique
+
+    t0 = time.time()
+    final, _ = rerank(req.query, merged)
+    final = final[: req.top_k]
+    timing["final_rerank"] = time.time() - t0
+
+    return AgenticResult(
+        episodes=final,
+        method="agentic (multi_round)",
+        multi_queries=multi_queries,
+        reasoning=sufficiency.get("reasoning", ""),
+        timing={**timing, "total": time.time() - t_total},
+    )
+
+
+# ── CLI ───────────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Agentic Search Server")
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8101)
+    args = parser.parse_args()
+
+    import uvicorn
+
+    print(f"Starting Agentic Search Server on {args.host}:{args.port}")
+    uvicorn.run(app, host=args.host, port=args.port, log_level="info")

package/src/memory/everos/python/fcntl_compat.py

@@ -0,0 +1,23 @@
+"""fcntl compatibility shim for Windows.
+
+Drop this into the EverOS venv's site-packages so `import fcntl`
+doesn't crash on Windows. The real fcntl module is POSIX-only.
+
+On macOS/Linux this file is never imported (Python finds the real fcntl first).
+"""
+
+# Constants (values don't matter — all functions are no-ops)
+LOCK_EX = 2
+LOCK_NB = 4
+LOCK_UN = 8
+LOCK_SH = 1
+
+
+def flock(fd, operation):
+    """No-op on Windows. Single-process mode doesn't need file locking."""
+    pass
+
+
+def lockf(fd, operation, length=0, start=0, whence=0):
+    """No-op on Windows."""
+    pass

package/src/memory/everos/python/requirements.txt

@@ -0,0 +1,7 @@
+everos==1.0.2
+everalgo-rank>=0.3.0
+lancedb>=0.30.0
+httpx>=0.27.0
+fastapi>=0.104.0
+uvicorn>=0.30.0
+pydantic>=2.0