@dinasor/mnemo-cli 0.0.1

package/scripts/memory/installer/templates/autonomy/ingest_pipeline.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""
+ingest_pipeline.py - Autonomous ingestion and chunking with typed metadata.
+
+Detects changed .md files in .mnemo/memory/ (with bridge fallback), chunks them with context-aware
+splitting, classifies memory type, and upserts into the DB as memory_units
+with full metadata (authority, time_scope, sensitivity, entity_tags).
+"""
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from autonomy.schema import get_db
+
+SKIP_NAMES = frozenset({"README.md", "index.md", "lessons-index.json",
+                        "journal-index.json", "journal-index.md"})
+SKIP_DIRS = frozenset({"legacy", "templates"})
+MAX_CHUNK_CHARS = 10000
+
+
+def _resolve_memory_root(repo_root: Path) -> Path:
+    override = os.getenv("MNEMO_MEMORY_ROOT", "").strip()
+    if override:
+        return Path(override).expanduser().resolve()
+
+    candidates = [
+        repo_root / ".mnemo" / "memory",
+        repo_root / ".cursor" / "memory",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return candidates[0]
+
+
+AUTHORITY_WEIGHTS: dict[str, float] = {
+    "core": 1.0,
+    "procedural": 0.9,
+    "semantic": 0.8,
+    "episodic": 0.7,
+    "resource": 0.5,
+    "vault": 0.0,
+}
+
+
+@dataclass
+class MemoryUnit:
+    unit_id: str
+    source_ref: str
+    memory_type: str
+    authority: float
+    time_scope: str
+    sensitivity: str
+    entity_tags: list[str]
+    content_hash: str
+    content: str
+    chunks: list[tuple[str, str]] = field(default_factory=list)  # (text, ref)
+    is_new: bool = True
+
+
+def _infer_memory_type(path_str: str) -> str:
+    p = path_str.lower().replace("\\", "/")
+    if "hot-rules" in p or "memo.md" in p:
+        return "core"
+    if "/lessons/" in p and re.search(r"/l-\d+", p):
+        return "procedural"
+    if "/journal/" in p or "active-context" in p:
+        return "episodic"
+    if "/digests/" in p:
+        return "semantic"
+    if "/vault/" in p:
+        return "vault"
+    if "/adr/" in p:
+        return "semantic"
+    return "semantic"
+
+
+def _infer_time_scope(memory_type: str) -> str:
+    if memory_type == "episodic":
+        return "recency-sensitive"
+    if memory_type in ("core", "procedural"):
+        return "atemporal"
+    return "time-bound"
+
+
+def _infer_sensitivity(path_str: str) -> str:
+    p = path_str.lower()
+    if "/vault/" in p or "secret" in p or ".secret." in p:
+        return "secret"
+    return "public"
+
+
+def _content_hash(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def _chunk_markdown(content: str, file_path: Path) -> list[tuple[str, str]]:
+    """Split markdown content into (text, ref_path) chunks."""
+    chunks: list[tuple[str, str]] = []
+    path_str = str(file_path).replace("\\", "/")
+
+    # Journal: split by date headings
+    if "journal/" in path_str.lower():
+        parts = re.split(r"^(##\s+\d{4}-\d{2}-\d{2})", content, flags=re.MULTILINE)
+        preamble = parts[0].strip()
+        if preamble:
+            chunks.append((preamble, f"@{path_str}"))
+        i = 1
+        while i < len(parts) - 1:
+            heading = parts[i].strip()
+            body = parts[i + 1].strip()
+            date_val = heading.replace("##", "").strip()
+            chunk_text = f"{heading}\n{body}".strip()
+            if chunk_text:
+                chunks.append((chunk_text[:MAX_CHUNK_CHARS], f"@{path_str}#{date_val}"))
+            i += 2
+        return chunks
+
+    # Lessons: single chunk per lesson file
+    if re.search(r"/lessons/l-\d+", path_str.lower()):
+        text = content.strip()
+        if text:
+            m = re.match(r"(L-\d{3})", file_path.name)
+            ref = f"@{path_str}#{m.group(1)}" if m else f"@{path_str}"
+            chunks.append((text[:MAX_CHUNK_CHARS], ref))
+        return chunks
+
+    # General: split by headers
+    parts = re.split(r"^(#{1,4}\s+.+)$", content, flags=re.MULTILINE)
+    preamble = parts[0].strip()
+    if preamble:
+        chunks.append((preamble[:MAX_CHUNK_CHARS], f"@{path_str}"))
+
+    i = 1
+    while i < len(parts) - 1:
+        heading_line = parts[i].strip()
+        body = parts[i + 1].strip()
+        heading_text = re.sub(r"^#{1,4}\s+", "", heading_line)
+        full = f"{heading_line}\n{body}".strip() if body else heading_line
+        if full:
+            chunks.append((full[:MAX_CHUNK_CHARS], f"@{path_str}#{heading_text}"))
+        i += 2
+
+    if not chunks and content.strip():
+        chunks.append((content.strip()[:MAX_CHUNK_CHARS], f"@{path_str}"))
+    return chunks
+
+
+class IngestPipeline:
+    def __init__(self, db: Optional[sqlite3.Connection] = None, repo_root: Optional[Path] = None):
+        self.db = db or get_db()
+        self.repo_root = repo_root or Path.cwd()
+        self.mem_root = _resolve_memory_root(self.repo_root)
+
+    def detect_changes(self) -> list[Path]:
+        """Return list of .md files that have changed hash."""
+        changed: list[Path] = []
+        for p in self.mem_root.glob("**/*.md"):
+            if p.name in SKIP_NAMES:
+                continue
+            if any(skip in p.parts for skip in SKIP_DIRS):
+                continue
+            try:
+                content = p.read_text(encoding="utf-8-sig")
+                h = _content_hash(content)
+                row = self.db.execute(
+                    "SELECT hash FROM file_meta WHERE path = ?", (str(p),)
+                ).fetchone()
+                if not row or row["hash"] != h:
+                    changed.append(p)
+            except OSError:
+                pass
+        return changed
+
+    def ingest_file(self, file_path: Path) -> list[MemoryUnit]:
+        """Ingest a single file, create/update memory units, return list."""
+        content = file_path.read_text(encoding="utf-8-sig")
+        h = _content_hash(content)
+        path_str = str(file_path)
+
+        mem_type = _infer_memory_type(path_str)
+        authority = AUTHORITY_WEIGHTS.get(mem_type, 0.5)
+        time_scope = _infer_time_scope(mem_type)
+        sensitivity = _infer_sensitivity(path_str)
+        chunks = _chunk_markdown(content, file_path)
+
+        existing_row = self.db.execute(
+            "SELECT unit_id FROM memory_units WHERE source_ref = ?", (path_str,)
+        ).fetchone()
+
+        if existing_row:
+            unit_id = existing_row["unit_id"]
+            is_new = False
+        else:
+            unit_id = str(uuid.uuid4())
+            is_new = True
+
+        unit = MemoryUnit(
+            unit_id=unit_id,
+            source_ref=path_str,
+            memory_type=mem_type,
+            authority=authority,
+            time_scope=time_scope,
+            sensitivity=sensitivity,
+            entity_tags=[],
+            content_hash=h,
+            content=content,
+            chunks=chunks,
+            is_new=is_new,
+        )
+
+        if is_new:
+            self.db.execute(
+                """
+                INSERT INTO memory_units
+                    (unit_id, source_ref, memory_type, authority, time_scope, sensitivity, entity_tags, content_hash)
+                VALUES (?, ?, ?, ?, ?, ?, '[]', ?)
+                """,
+                (unit_id, path_str, mem_type, authority, time_scope, sensitivity, h),
+            )
+        else:
+            self.db.execute(
+                """
+                UPDATE memory_units
+                SET memory_type=?, authority=?, time_scope=?, sensitivity=?,
+                    content_hash=?, updated_at=unixepoch('now')
+                WHERE unit_id=?
+                """,
+                (mem_type, authority, time_scope, sensitivity, h, unit_id),
+            )
+
+        self.db.execute(
+            "INSERT OR REPLACE INTO file_meta(path, hash, chunk_count, updated_at) VALUES (?,?,?,unixepoch('now'))",
+            (path_str, h, len(chunks)),
+        )
+        self.db.commit()
+        return [unit]
+
+    def update_entity_tags(self, unit: MemoryUnit, entity_ids: list[str]) -> None:
+        """Persist resolved entity tags back to the unit row."""
+        unit.entity_tags = entity_ids
+        self.db.execute(
+            "UPDATE memory_units SET entity_tags=?, updated_at=unixepoch('now') WHERE unit_id=?",
+            (json.dumps(entity_ids), unit.unit_id),
+        )
+        self.db.commit()

package/scripts/memory/installer/templates/autonomy/lifecycle_engine.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""
+lifecycle_engine.py - Autonomous fact lifecycle engine.
+
+Decides ADD / UPDATE / DEPRECATE / NOOP for each memory unit based on
+similarity to existing facts, freshness, and contradiction detection.
+All decisions are logged to lifecycle_events for full auditability.
+
+No human required: transitions happen automatically on every ingest cycle.
+"""
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from autonomy.schema import get_db
+from autonomy.ingest_pipeline import MemoryUnit
+
+DEPRECATION_SIMILARITY_THRESHOLD = 0.85
+PROMOTE_STABILITY_CYCLES = 3  # fact must appear N cycles before lesson promotion
+NOOP_HASH_MATCH = True  # if content_hash unchanged, always NOOP
+
+
+@dataclass
+class LifecycleDecision:
+    operation: str  # ADD | UPDATE | DEPRECATE | NOOP
+    unit_id: str
+    fact_id: Optional[str]
+    reason: str
+    confidence: float = 1.0
+
+
+def _extract_key_facts(content: str, memory_type: str) -> list[str]:
+    """
+    Heuristic extraction of canonical facts from content.
+    Returns list of short declarative sentences.
+    """
+    facts = []
+
+    # Hot rules / lessons: extract bullet points as facts
+    if memory_type in ("core", "procedural"):
+        for m in re.finditer(r"^[-*]\s+(.+)", content, re.MULTILINE):
+            fact = m.group(1).strip()
+            if len(fact) > 10:
+                facts.append(fact)
+
+    # Journal / active-context: extract decision lines
+    if memory_type in ("episodic",):
+        for m in re.finditer(r"(decided|confirmed|fixed|added|removed|changed):\s*(.+)", content, re.IGNORECASE):
+            facts.append(m.group(0).strip())
+
+    # Generic: extract first sentence of each section
+    for m in re.finditer(r"^#{1,4}\s+(.+)\n+(.*?)(?:\n|$)", content, re.MULTILINE):
+        heading = m.group(1).strip()
+        body = m.group(2).strip()
+        if body:
+            facts.append(f"{heading}: {body}")
+
+    return facts[:20]  # cap to prevent runaway
+
+
+def _simple_similarity(a: str, b: str) -> float:
+    """Token Jaccard similarity for contradiction detection (no embeddings needed)."""
+    ta = set(re.findall(r"\w+", a.lower()))
+    tb = set(re.findall(r"\w+", b.lower()))
+    if not ta or not tb:
+        return 0.0
+    return len(ta & tb) / len(ta | tb)
+
+
+def _resolve_lessons_dir(repo_root: Path) -> Path:
+    override = os.getenv("MNEMO_MEMORY_ROOT", "").strip()
+    if override:
+        return Path(override).expanduser().resolve() / "lessons"
+
+    candidates = [
+        repo_root / ".mnemo" / "memory" / "lessons",
+        repo_root / ".cursor" / "memory" / "lessons",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return candidates[0]
+
+
+class LifecycleEngine:
+    def __init__(self, db: Optional[sqlite3.Connection] = None):
+        self.db = db or get_db()
+
+    def process(self, unit: MemoryUnit) -> LifecycleDecision:
+        """
+        Process a memory unit through the lifecycle state machine.
+        Emits a lifecycle_event and returns the decision.
+        """
+        existing_hash = self.db.execute(
+            "SELECT content_hash FROM memory_units WHERE unit_id = ?",
+            (unit.unit_id,),
+        ).fetchone()
+
+        if existing_hash and NOOP_HASH_MATCH and existing_hash["content_hash"] == unit.content_hash:
+            decision = LifecycleDecision("NOOP", unit.unit_id, None, "content_hash_unchanged")
+            self._log_event(decision)
+            return decision
+
+        facts = _extract_key_facts(unit.content, unit.memory_type)
+        if not facts:
+            decision = LifecycleDecision("NOOP", unit.unit_id, None, "no_extractable_facts")
+            self._log_event(decision)
+            return decision
+
+        existing_facts = self.db.execute(
+            "SELECT fact_id, canonical_fact, status, confidence FROM facts WHERE source_ref = ?",
+            (unit.source_ref,),
+        ).fetchall()
+
+        # Check for contradiction / supersession in global facts
+        contradictions = self._detect_contradictions(facts)
+        for old_fact_id, old_fact_text in contradictions:
+            self.db.execute(
+                "UPDATE facts SET status='deprecated', updated_at=unixepoch('now') WHERE fact_id=?",
+                (old_fact_id,),
+            )
+            dep_decision = LifecycleDecision(
+                "DEPRECATE", unit.unit_id, old_fact_id,
+                reason=f"superseded_by_unit:{unit.unit_id}",
+                confidence=0.8,
+            )
+            self._log_event(dep_decision)
+
+        if existing_facts:
+            # UPDATE existing facts from this source
+            for ef in existing_facts:
+                self.db.execute(
+                    "UPDATE facts SET status='active', confidence=?, updated_at=unixepoch('now') WHERE fact_id=?",
+                    (min(ef["confidence"] + 0.05, 1.0), ef["fact_id"]),
+                )
+            decision = LifecycleDecision("UPDATE", unit.unit_id, existing_facts[0]["fact_id"], "source_file_changed")
+        else:
+            # ADD new facts
+            for fact_text in facts[:5]:  # cap facts per unit
+                fact_id = str(uuid.uuid4())
+                self.db.execute(
+                    "INSERT INTO facts(fact_id, canonical_fact, status, confidence, source_ref) VALUES (?,?,'active',1.0,?)",
+                    (fact_id, fact_text, unit.source_ref),
+                )
+            decision = LifecycleDecision("ADD", unit.unit_id, None, f"new_unit_{len(facts)}_facts_extracted")
+
+        self._log_event(decision)
+        self.db.commit()
+        return decision
+
+    def _detect_contradictions(self, new_facts: list[str]) -> list[tuple[str, str]]:
+        """
+        Find existing active facts that are semantically contradicted by new_facts.
+        Uses token Jaccard with high threshold — low false positive rate is more important than recall.
+        """
+        contradicted: list[tuple[str, str]] = []
+        existing = self.db.execute(
+            "SELECT fact_id, canonical_fact FROM facts WHERE status = 'active'"
+        ).fetchall()
+
+        contradiction_patterns = [
+            (r"\bdo\s+not\b", r"\bdo\b"),
+            (r"\bnever\b", r"\balways\b"),
+            (r"\bdisabled\b", r"\benabled\b"),
+        ]
+
+        for ef in existing:
+            ef_text = ef["canonical_fact"]
+            for new_fact in new_facts:
+                sim = _simple_similarity(new_fact, ef_text)
+                if sim >= DEPRECATION_SIMILARITY_THRESHOLD:
+                    for pat_a, pat_b in contradiction_patterns:
+                        a_in_new = bool(re.search(pat_a, new_fact, re.I))
+                        b_in_old = bool(re.search(pat_b, ef_text, re.I))
+                        if a_in_new and b_in_old:
+                            contradicted.append((ef["fact_id"], ef_text))
+                            break
+        return contradicted
+
+    def _log_event(self, decision: LifecycleDecision) -> None:
+        self.db.execute(
+            "INSERT INTO lifecycle_events(event_id, unit_id, operation, reason) VALUES (?,?,?,?)",
+            (str(uuid.uuid4()), decision.unit_id, decision.operation, decision.reason),
+        )
+
+    def promote_lessons(self, repo_root: Path) -> list[str]:
+        """
+        Auto-promote stable repeated signals into lesson files.
+        A fact qualifies when: status=active AND confidence >= 0.95
+        AND no lesson already covers the source_ref.
+        Returns list of created lesson paths.
+        """
+        candidates = self.db.execute(
+            """
+            SELECT f.fact_id, f.canonical_fact, f.source_ref, f.confidence
+            FROM facts f
+            WHERE f.status = 'active' AND f.confidence >= 0.95
+              AND f.source_ref NOT LIKE '%lessons/L-%'
+            ORDER BY f.confidence DESC
+            LIMIT 5
+            """
+        ).fetchall()
+
+        promoted = []
+        lessons_dir = _resolve_lessons_dir(repo_root)
+        lessons_dir.mkdir(parents=True, exist_ok=True)
+
+        existing = sorted(lessons_dir.glob("L-*.md"))
+        next_id = 1
+        if existing:
+            m = re.match(r"L-(\d+)", existing[-1].name)
+            if m:
+                next_id = int(m.group(1)) + 1
+
+        for row in candidates:
+            fact_text = row["canonical_fact"][:200]
+            lesson_id = f"L-{next_id:03d}"
+            slug = re.sub(r"[^a-z0-9]+", "-", fact_text.lower())[:40].strip("-")
+            lesson_file = lessons_dir / f"{lesson_id}-{slug}.md"
+
+            if lesson_file.exists():
+                continue
+
+            today = datetime.now().strftime("%Y-%m-%d")
+            content = (
+                f"---\nid: {lesson_id}\ntitle: {fact_text[:80]}\nstatus: Active\n"
+                f"tags: [Process]\nintroduced: {today}\napplies_to:\n  - \"**/*\"\n"
+                f"triggers:\n  - auto-promoted\nrule: {fact_text[:120]}\n---\n\n"
+                f"# {lesson_id} - Auto-Promoted Lesson\n\n"
+                f"**Source:** `{row['source_ref']}`\n\n"
+                f"**Canonical fact:** {fact_text}\n\n"
+                f"> This lesson was auto-promoted by the Mnemo autonomous runner.\n"
+                f"> Review and edit the rule to ensure accuracy.\n"
+            )
+            lesson_file.write_text(content, encoding="utf-8")
+            promoted.append(str(lesson_file))
+
+            # Mark fact as promoted
+            self.db.execute(
+                "UPDATE facts SET status='promoted', updated_at=unixepoch('now') WHERE fact_id=?",
+                (row["fact_id"],),
+            )
+            next_id += 1
+
+        if promoted:
+            self.db.commit()
+        return promoted

package/scripts/memory/installer/templates/autonomy/policies.yaml

@@ -0,0 +1,59 @@
+# Mnemo Memory System - Vault & Sensitivity Policy
+# Loaded by vault_policy.py at runtime.
+# Edit this file to customize sensitivity rules for your project.
+
+# Sensitivity classification rules (applied in order: secret > internal > public)
+sensitivity_paths:
+  secret:
+    - ".mnemo/memory/vault/"
+    - ".cursor/memory/vault/"
+    - ".env"
+    - "*.secret.*"
+    - "*credentials*"
+    - "*private-key*"
+  internal:
+    - ".mnemo/memory/active-context.md"
+    - ".cursor/memory/active-context.md"
+
+# Additional regex redaction patterns (applied on top of built-ins)
+# Built-ins already handle: API keys, bearer tokens, long random strings
+redaction_patterns:
+  # Add project-specific patterns here, e.g.:
+  # - "sk-[a-zA-Z0-9]{40,}"
+  # - "xoxb-[0-9]+-[0-9]+-[a-zA-Z0-9]+"
+
+# Roles allowed to see 'internal' sensitivity content
+allow_internal_for_roles:
+  - agent
+  - autonomous
+
+# Maximum sensitivity level allowed in outgoing context packs
+# Options: public | internal
+# Setting 'public' means only public content is included
+max_sensitivity_in_context: internal
+
+# Autonomous ingestion: re-run sensitivity classification every N cycles
+reclassify_every_n_cycles: 10
+
+# Lifecycle deprecation: minimum confidence before auto-deprecating a fact
+deprecation_confidence_threshold: 0.8
+
+# Entity resolution: minimum Jaccard similarity to merge an alias
+alias_merge_threshold: 0.85
+
+# Lesson promotion: minimum fact confidence to auto-promote to lesson
+lesson_promotion_threshold: 0.95
+
+# Retrieval safety: minimum final_score for results to appear in context packs
+min_retrieval_score: 0.2
+
+# Token budgets (in characters, ~4 chars per token)
+token_budget_default: 6000
+token_budget_extended: 12000
+
+# Benchmark thresholds (used by CI quality gates)
+benchmark:
+  min_hit_at_3: 0.7
+  min_ndcg_at_5: 0.65
+  max_p95_latency_ms: 2000
+  max_token_cost_per_query: 0.005