@pentatonic-ai/ai-agent-sdk 0.6.0 → 0.7.0

package/packages/memory-engine/pme_memory/indexer.py

@@ -0,0 +1,200 @@
+"""
+pme_memory.indexer — Index life data into Milvus collections
+
+Scans workspace for chat transcripts (JSONL), email archives, people
+profiles, contacts, and memory files. Chunks text and embeds via Ollama.
+
+Collections:
+    chats:    JSONL chat transcripts + markdown chat summaries
+    emails:   Email archive markdown files
+    contacts: People profiles + contact records
+    memory:   Daily notes, project docs, research (excludes evolution run logs)
+"""
+
+import glob
+import hashlib
+import json
+import os
+from pathlib import Path
+
+from .embed import embed_texts, BATCH_SIZE
+from .store import CommsStore
+
+CHUNK_SIZE = 512
+CHUNK_OVERLAP = 64
+
+
+def chunk_text(text: str, chunk_size=CHUNK_SIZE, overlap=CHUNK_OVERLAP) -> list[str]:
+    """Split text into overlapping chunks."""
+    if len(text) <= chunk_size:
+        return [text] if text.strip() else []
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        chunk = text[start:end].strip()
+        if chunk:
+            chunks.append(chunk)
+        start = end - overlap
+    return chunks
+
+
+def text_id(text: str, source: str) -> str:
+    return hashlib.md5(f"{source}:{text[:200]}".encode()).hexdigest()
+
+
+def _upsert_batch(store, collection, batch):
+    """Embed and upsert a batch of documents."""
+    if not batch:
+        return 0
+    vectors = embed_texts([d["text"] for d in batch])
+    for d, v in zip(batch, vectors):
+        d["vector"] = v
+    store.upsert(collection, batch)
+    return len(batch)
+
+
+def index_chats(store: CommsStore, workspace: Path) -> int:
+    """Index JSONL chat transcripts and markdown chat summaries."""
+    total = 0
+    chats_dir = workspace / "chats"
+    if not chats_dir.exists():
+        return 0
+
+    # JSONL files
+    for f in chats_dir.rglob("*.jsonl"):
+        try:
+            lines = f.read_text(errors="replace").strip().split("\n")
+            batch = []
+            for line in lines:
+                try:
+                    msg = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                text = msg.get("text", "")
+                if not text or len(text) < 10:
+                    continue
+                source = str(f.relative_to(workspace))
+                for chunk in chunk_text(text):
+                    batch.append({
+                        "id": text_id(chunk, source),
+                        "text": chunk[:8000],
+                        "source": source[:500],
+                        "channel": str(msg.get("channel", "unknown"))[:60],
+                        "contact": str(msg.get("contact", msg.get("sender", "")))[:250],
+                        "timestamp": str(msg.get("timestamp", ""))[:30],
+                    })
+                    if len(batch) >= BATCH_SIZE:
+                        total += _upsert_batch(store, "chats", batch)
+                        batch = []
+            total += _upsert_batch(store, "chats", batch)
+        except Exception as e:
+            print(f"  Error indexing {f}: {e}")
+
+    # Markdown chat summaries
+    for channel in ["telegram", "whatsapp", "imessage", "slack", "unknown"]:
+        chat_dir = workspace / "memory" / "chats" / channel
+        if not chat_dir.exists():
+            continue
+        for f in chat_dir.glob("*.md"):
+            try:
+                text = f.read_text(errors="replace")
+                if len(text) < 20:
+                    continue
+                source = str(f.relative_to(workspace))
+                batch = [{"id": text_id(c, source), "text": c[:8000], "source": source[:500],
+                          "channel": channel, "contact": f.stem[:250], "timestamp": ""}
+                         for c in chunk_text(text)]
+                total += _upsert_batch(store, "chats", batch)
+            except Exception as e:
+                print(f"  Error: {e}")
+
+    return total
+
+
+def index_emails(store: CommsStore, workspace: Path) -> int:
+    """Index email archive markdown files."""
+    total = 0
+    emails_dir = workspace / "memory" / "chats" / "email"
+    if not emails_dir.exists():
+        return 0
+    for f in emails_dir.glob("*.md"):
+        try:
+            text = f.read_text(errors="replace")
+            if len(text) < 20:
+                continue
+            source = str(f.relative_to(workspace))
+            contact = f.stem.replace("_", " ")[:250]
+            batch = [{"id": text_id(c, source), "text": c[:8000], "source": source[:500],
+                      "channel": "email", "contact": contact, "timestamp": ""}
+                     for c in chunk_text(text)]
+            total += _upsert_batch(store, "emails", batch)
+        except Exception as e:
+            print(f"  Error: {e}")
+    return total
+
+
+def index_contacts(store: CommsStore, workspace: Path) -> int:
+    """Index people profiles and contact records."""
+    total = 0
+    for dir_path, channel in [(workspace / "memory" / "people", "profile"),
+                               (workspace / "memory" / "contacts", "contacts")]:
+        if not dir_path.exists():
+            continue
+        for f in dir_path.glob("*"):
+            if not f.is_file():
+                continue
+            try:
+                text = f.read_text(errors="replace")
+                if len(text) < 20:
+                    continue
+                source = str(f.relative_to(workspace))
+                batch = [{"id": text_id(c, source), "text": c[:8000], "source": source[:500],
+                          "channel": channel, "contact": f.stem[:250], "timestamp": ""}
+                         for c in chunk_text(text, chunk_size=1024 if channel == "contacts" else CHUNK_SIZE)]
+                total += _upsert_batch(store, "contacts", batch)
+            except Exception as e:
+                print(f"  Error: {e}")
+    return total
+
+
+def index_memory(store: CommsStore, workspace: Path) -> int:
+    """Index memory markdown files (excludes chats and evolution run logs)."""
+    total = 0
+    memory_dir = workspace / "memory"
+    skip_patterns = ["chats/", "evolution/loop-run-", "evolution/v3/runs/"]
+
+    for f in memory_dir.rglob("*.md"):
+        source = str(f.relative_to(workspace))
+        if any(p in source for p in skip_patterns):
+            continue
+        try:
+            text = f.read_text(errors="replace")
+            if len(text) < 30:
+                continue
+            batch = [{"id": text_id(c, source), "text": c[:8000], "source": source[:500],
+                      "channel": "memory", "contact": "", "timestamp": ""}
+                     for c in chunk_text(text)]
+            total += _upsert_batch(store, "memory", batch)
+        except Exception as e:
+            print(f"  Error: {e}")
+    return total
+
+
+def index_all(store: CommsStore, workspace: Path, targets=None) -> dict:
+    """Index specified targets (or all). Returns counts per collection."""
+    if targets is None:
+        targets = ["chats", "emails", "contacts", "memory"]
+    counts = {}
+    indexers = {
+        "chats": index_chats,
+        "emails": index_emails,
+        "contacts": index_contacts,
+        "memory": index_memory,
+    }
+    for target in targets:
+        if target in indexers:
+            print(f"Indexing {target}...")
+            counts[target] = indexers[target](store, workspace)
+            print(f"  {counts[target]:,} chunks")
+    return counts

package/packages/memory-engine/pme_memory/needs.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class NeedSignal:
+    need_type: str
+    produced_by_artifact: str
+    producer: str
+    priority_hint: str = "normal"
+    topic: str | None = None
+    created_at: str = field(default_factory=_utc_now)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "need_type": self.need_type,
+            "produced_by_artifact": self.produced_by_artifact,
+            "producer": self.producer,
+            "priority_hint": self.priority_hint,
+            "topic": self.topic,
+            "created_at": self.created_at,
+        }
+
+
+class NeedIndex:
+    """Append-only machine-readable unresolved needs index (JSONL)."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        if not self.path.exists():
+            self.path.touch()
+
+    def publish(self, signal: NeedSignal) -> None:
+        with self.path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(signal.to_dict(), sort_keys=True) + "\n")
+
+    def latest(self, limit: int = 50) -> List[Dict[str, Any]]:
+        lines = self.path.read_text(encoding="utf-8").splitlines()[-limit:]
+        out: List[Dict[str, Any]] = []
+        for line in lines:
+            try:
+                out.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+        return out

package/packages/memory-engine/pme_memory/provenance.py

@@ -0,0 +1,80 @@
+"""
+pme_memory.provenance — Provenance-first output rendering
+
+Renders artifact lineage chains for human-readable output.
+Given an artifact, walks parent pointers to build a full
+provenance trail with sources, tools, and timestamps.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def load_artifact_index(store_path: str | Path) -> Dict[str, Dict[str, Any]]:
+    """Load all artifacts into a dict keyed by artifact_id."""
+    store_path = Path(store_path)
+    index: Dict[str, Dict[str, Any]] = {}
+    if not store_path.exists():
+        return index
+    for line in store_path.read_text(encoding="utf-8").splitlines():
+        try:
+            art = json.loads(line)
+            index[art["artifact_id"]] = art
+        except (json.JSONDecodeError, KeyError):
+            continue
+    return index
+
+
+def trace_lineage(
+    artifact_id: str,
+    index: Dict[str, Dict[str, Any]],
+    max_depth: int = 20,
+) -> List[Dict[str, Any]]:
+    """Walk parent pointers and return lineage chain (newest first)."""
+    chain: List[Dict[str, Any]] = []
+    visited: set = set()
+    queue = [artifact_id]
+
+    while queue and len(chain) < max_depth:
+        aid = queue.pop(0)
+        if aid in visited:
+            continue
+        visited.add(aid)
+        art = index.get(aid)
+        if not art:
+            chain.append({"artifact_id": aid, "status": "missing"})
+            continue
+        chain.append(art)
+        for pid in art.get("parents", []):
+            if pid not in visited:
+                queue.append(pid)
+
+    return chain
+
+
+def render_lineage_text(
+    artifact_id: str,
+    index: Dict[str, Dict[str, Any]],
+    max_depth: int = 20,
+) -> str:
+    """Render a human-readable provenance chain."""
+    chain = trace_lineage(artifact_id, index, max_depth)
+    if not chain:
+        return f"No lineage found for {artifact_id}"
+
+    lines = [f"Provenance for {artifact_id[:12]}...\n"]
+    for i, art in enumerate(chain):
+        prefix = "  " * i + ("└─ " if i > 0 else "")
+        if art.get("status") == "missing":
+            lines.append(f"{prefix}[missing] {art['artifact_id'][:12]}...")
+        else:
+            tool = art.get("source_tool", "?")
+            atype = art.get("artifact_type", "?")
+            ts = art.get("created_at", "?")[:19]
+            aid = art["artifact_id"][:12]
+            lines.append(f"{prefix}{aid}... | {atype} | tool={tool} | {ts}")
+
+    return "\n".join(lines)

package/packages/memory-engine/pme_memory/scoring.py

@@ -0,0 +1,168 @@
+"""
+pme_memory.scoring — Pressure Scoring for Need Signals
+
+Ranks unresolved needs by four dimensions:
+  - recency:    how fresh the need is (exponential decay)
+  - novelty:    inverse frequency of this need_type in the index
+  - centrality: how many artifacts reference the producing artifact
+  - priority:   explicit priority_hint weight (critical > high > normal > low)
+
+Output: sorted list of needs with composite pressure score (0-1).
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections import Counter
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+# --- Weight configuration (tunable) ---
+WEIGHT_RECENCY = 0.30
+WEIGHT_NOVELTY = 0.25
+WEIGHT_CENTRALITY = 0.25
+WEIGHT_PRIORITY = 0.20
+
+PRIORITY_SCORES = {
+    "critical": 1.0,
+    "high": 0.75,
+    "normal": 0.5,
+    "low": 0.25,
+}
+
+# Recency half-life in hours (need loses half its recency score after this)
+RECENCY_HALF_LIFE_H = 12.0
+
+
+def _parse_iso(ts: str) -> datetime:
+    return datetime.fromisoformat(ts)
+
+
+def _recency_score(created_at: str, now: datetime) -> float:
+    """Exponential decay: score = 2^(-age_hours / half_life)."""
+    try:
+        age = (now - _parse_iso(created_at)).total_seconds() / 3600.0
+    except (ValueError, TypeError):
+        return 0.0
+    return math.pow(2, -age / RECENCY_HALF_LIFE_H)
+
+
+def _novelty_scores(needs: List[Dict[str, Any]]) -> Dict[str, float]:
+    """Inverse frequency: rarer need_types score higher."""
+    counts = Counter(n.get("need_type", "") for n in needs)
+    total = len(needs) or 1
+    return {
+        nt: 1.0 - (count / total)
+        for nt, count in counts.items()
+    }
+
+
+def _centrality_map(artifact_store_path: Path) -> Dict[str, int]:
+    """Count how many artifacts reference each artifact_id as a parent."""
+    refs: Dict[str, int] = {}
+    if not artifact_store_path.exists():
+        return refs
+    for line in artifact_store_path.read_text(encoding="utf-8").splitlines():
+        try:
+            art = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        for pid in art.get("parents", []):
+            refs[pid] = refs.get(pid, 0) + 1
+    return refs
+
+
+@dataclass
+class ScoredNeed:
+    need_type: str
+    produced_by_artifact: str
+    producer: str
+    topic: Optional[str]
+    created_at: str
+    priority_hint: str
+    recency: float
+    novelty: float
+    centrality: float
+    priority: float
+    pressure: float
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "need_type": self.need_type,
+            "produced_by_artifact": self.produced_by_artifact,
+            "producer": self.producer,
+            "topic": self.topic,
+            "created_at": self.created_at,
+            "priority_hint": self.priority_hint,
+            "scores": {
+                "recency": round(self.recency, 4),
+                "novelty": round(self.novelty, 4),
+                "centrality": round(self.centrality, 4),
+                "priority": round(self.priority, 4),
+            },
+            "pressure": round(self.pressure, 4),
+        }
+
+
+def rank_needs(
+    needs_path: str | Path,
+    artifact_store_path: str | Path,
+    limit: int = 50,
+) -> List[ScoredNeed]:
+    """Score and rank unresolved needs by composite pressure."""
+    needs_path = Path(needs_path)
+    artifact_store_path = Path(artifact_store_path)
+
+    if not needs_path.exists():
+        return []
+
+    raw = []
+    for line in needs_path.read_text(encoding="utf-8").splitlines():
+        try:
+            raw.append(json.loads(line))
+        except json.JSONDecodeError:
+            continue
+
+    if not raw:
+        return []
+
+    now = datetime.now(timezone.utc)
+    novelty_map = _novelty_scores(raw)
+    centrality_map = _centrality_map(artifact_store_path)
+    max_centrality = max(centrality_map.values()) if centrality_map else 1
+
+    scored: List[ScoredNeed] = []
+    for n in raw[-limit:]:
+        rec = _recency_score(n.get("created_at", ""), now)
+        nov = novelty_map.get(n.get("need_type", ""), 0.0)
+        art_id = n.get("produced_by_artifact", "")
+        cen = (centrality_map.get(art_id, 0) / max_centrality) if max_centrality else 0.0
+        pri = PRIORITY_SCORES.get(n.get("priority_hint", "normal"), 0.5)
+
+        pressure = (
+            WEIGHT_RECENCY * rec
+            + WEIGHT_NOVELTY * nov
+            + WEIGHT_CENTRALITY * cen
+            + WEIGHT_PRIORITY * pri
+        )
+
+        scored.append(ScoredNeed(
+            need_type=n.get("need_type", ""),
+            produced_by_artifact=art_id,
+            producer=n.get("producer", ""),
+            topic=n.get("topic"),
+            created_at=n.get("created_at", ""),
+            priority_hint=n.get("priority_hint", "normal"),
+            recency=rec,
+            novelty=nov,
+            centrality=cen,
+            priority=pri,
+            pressure=pressure,
+        ))
+
+    scored.sort(key=lambda s: s.pressure, reverse=True)
+    return scored

package/packages/memory-engine/pme_memory/search.py

@@ -0,0 +1,52 @@
+"""
+pme_memory.search — Semantic search across communications collections
+
+Searches across chats, emails, contacts, and memory using vector similarity.
+"""
+
+from .embed import embed_query
+from .store import CommsStore, COLLECTIONS
+
+
+def search(query: str, store: CommsStore = None, collection: str = None,
+           limit: int = 10) -> list[dict]:
+    """Search across all collections (or a specific one).
+
+    Returns list of dicts with: collection, score, text, source, channel, contact, timestamp
+    """
+    if store is None:
+        store = CommsStore()
+
+    vector = embed_query(query)
+    if all(v == 0.0 for v in vector):
+        return []
+
+    collections = [collection] if collection else COLLECTIONS
+    all_results = []
+
+    for coll in collections:
+        try:
+            results = store.search(coll, vector, limit=limit)
+            for hits in results:
+                for hit in hits:
+                    entity = hit.get("entity", {})
+                    all_results.append({
+                        "collection": coll,
+                        "score": round(hit.get("distance", 0), 4),
+                        "text": entity.get("text", ""),
+                        "source": entity.get("source", ""),
+                        "channel": entity.get("channel", ""),
+                        "contact": entity.get("contact", ""),
+                        "timestamp": entity.get("timestamp", ""),
+                    })
+        except Exception as e:
+            print(f"  Search error in {coll}: {e}")
+
+    all_results.sort(key=lambda x: x["score"], reverse=True)
+    return all_results[:limit]
+
+
+def search_collection(query: str, collection: str, store: CommsStore = None,
+                      limit: int = 10) -> list[dict]:
+    """Search a single collection."""
+    return search(query, store=store, collection=collection, limit=limit)

package/packages/memory-engine/pme_memory/store.py

@@ -0,0 +1,86 @@
+"""
+pme_memory.store — Milvus connection and collection management
+
+Supports Milvus Lite (local .db file) and full Milvus server.
+Collections: chats, emails, contacts, memory.
+"""
+
+import os
+from pathlib import Path
+from pymilvus import MilvusClient, DataType
+
+COLLECTIONS = ["chats", "emails", "contacts", "memory"]
+EMBED_DIM = int(os.environ.get("PME_EMBED_DIM", "4096"))
+
+
+def _default_db_path():
+    pme_dir = os.environ.get("PME_DIR", os.path.expanduser("~/pentatonic"))
+    db_dir = Path(pme_dir) / "memory" / "l5"
+    db_dir.mkdir(parents=True, exist_ok=True)
+    return str(db_dir / "comms.db")
+
+
+class CommsStore:
+    """Manages Milvus collections for the communications layer."""
+
+    def __init__(self, uri=None):
+        self.uri = uri or os.environ.get("MILVUS_URI", _default_db_path())
+        self._client = None
+
+    @property
+    def client(self):
+        if self._client is None:
+            self._client = MilvusClient(uri=self.uri)
+        return self._client
+
+    def ensure_collection(self, name: str):
+        """Create collection if it doesn't exist."""
+        if self.client.has_collection(name):
+            return
+        schema = self.client.create_schema(auto_id=False, enable_dynamic_field=True)
+        schema.add_field("id", DataType.VARCHAR, is_primary=True, max_length=64)
+        schema.add_field("vector", DataType.FLOAT_VECTOR, dim=EMBED_DIM)
+        schema.add_field("text", DataType.VARCHAR, max_length=8192)
+        schema.add_field("source", DataType.VARCHAR, max_length=512)
+        schema.add_field("channel", DataType.VARCHAR, max_length=64)
+        schema.add_field("contact", DataType.VARCHAR, max_length=256)
+        schema.add_field("timestamp", DataType.VARCHAR, max_length=32)
+
+        index_params = self.client.prepare_index_params()
+        index_params.add_index(field_name="vector", index_type="FLAT", metric_type="COSINE")
+        self.client.create_collection(collection_name=name, schema=schema, index_params=index_params)
+
+    def upsert(self, collection: str, data: list[dict]):
+        """Upsert documents into a collection."""
+        self.ensure_collection(collection)
+        self.client.upsert(collection_name=collection, data=data)
+
+    def search(self, collection: str, vector: list[float], limit: int = 10,
+               output_fields=None):
+        """Search a collection by vector similarity."""
+        if not self.client.has_collection(collection):
+            return []
+        if output_fields is None:
+            output_fields = ["text", "source", "channel", "contact", "timestamp"]
+        results = self.client.search(
+            collection_name=collection,
+            data=[vector],
+            limit=limit,
+            output_fields=output_fields,
+        )
+        return results
+
+    def collection_stats(self):
+        """Get stats for all collections."""
+        stats = {}
+        for name in COLLECTIONS:
+            if self.client.has_collection(name):
+                s = self.client.get_collection_stats(name)
+                stats[name] = {"exists": True, "count": s.get("row_count", 0)}
+            else:
+                stats[name] = {"exists": False, "count": 0}
+        return stats
+
+    def total_chunks(self):
+        stats = self.collection_stats()
+        return sum(c["count"] for c in stats.values())