loki-mode 7.75.0 → 7.77.0

package/mcp/__init__.py

@@ -57,4 +57,4 @@ try:
 except ImportError:
     __all__ = ['mcp']
 
-__version__ = '7.75.0'
+__version__ = '7.77.0'

package/memory/consolidation.py

@@ -157,10 +157,15 @@ class ConsolidationPipeline:
             if lock_file is not None:
                 fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
                 lock_file.close()
-                try:
-                    lock_path.unlink()
-                except OSError:
-                    pass
+                # Do NOT unlink the lock inode here. Unlinking on release is a
+                # flock+unlink inode-replacement race: waiter B blocked on
+                # inode-1 acquires it after holder A unlinks inode-1, then a
+                # third consolidation C opens the path, finds it gone, creates
+                # inode-2 and flocks inode-2 -- entering _consolidate_locked
+                # while B is still inside, breaking the BUG-MEM-003
+                # single-consolidation guarantee. Same class fixed in
+                # storage._file_lock. Persistent lock files are the standard
+                # flock pattern; the file is reused across runs.
 
     def _consolidate_locked(self, since_hours: int) -> ConsolidationResult:
         """Run the consolidation pipeline under an exclusive lock."""
@@ -226,9 +231,9 @@ class ConsolidationPipeline:
                             # Re-read the target pattern fresh immediately before
                             # merging (BUG-MEM C1, lost-update). The whole-run
                             # snapshot at step 4 can be stale by now: a concurrent
-                            # engine.increment_pattern_usage() (load_pattern then
-                            # save_pattern) may have bumped usage_count/last_used
-                            # AFTER the snapshot. merge_with_existing() builds the
+                            # storage.increment_pattern_usage() (atomic read-mutate-
+                            # write under one exclusive lock) may have bumped
+                            # usage_count/last_used AFTER the snapshot. merge_with_existing() builds the
                             # merged record from best_match.usage_count/last_used,
                             # so merging from the stale snapshot clobbers that bump.
                             # Re-reading narrows the window to this single write.

package/memory/engine.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 import json
 import logging
 import os
+import re
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Union
@@ -304,6 +305,13 @@ class MemoryEngine:
         else:
             date_str = timestamp.strftime("%Y-%m-%d")
 
+        # Reject a junk date directory derived from a poisoned/round-tripped
+        # timestamp (mirrors storage.save_episode). Traversal is already
+        # contained by _resolve_path; this just stops non-date dirs being
+        # created. Only an exact YYYY-MM-DD string is allowed.
+        if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date_str):
+            date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
         episode_id = trace_dict.get("id", f"ep-{date_str}-{self._generate_id()}")
         trace_dict["id"] = episode_id
 
@@ -580,21 +588,13 @@ class MemoryEngine:
         Args:
             pattern_id: Pattern identifier
         """
-        # Load pattern via storage (which acquires read lock)
-        pattern_data = self.storage.load_pattern(pattern_id)
-        if pattern_data is None:
-            return
-
-        # Update fields. `or 0` guards against an explicit null usage_count
-        # (corrupt/hand-edited record) crashing the increment with a TypeError;
-        # a null and 0 are equivalent here so `or` is safe.
-        pattern_data["usage_count"] = (pattern_data.get("usage_count") or 0) + 1
-        pattern_data["last_used"] = datetime.now(timezone.utc).isoformat()
-
-        # Write back via save_pattern which holds an exclusive lock during
-        # the full read-modify-write (upsert) cycle
-        pattern_obj = self._dict_to_pattern(pattern_data)
-        self.storage.save_pattern(pattern_obj)
+        # Delegate the entire read-modify-write to storage, which performs it
+        # under a single exclusive lock. Doing the read here (load_pattern's
+        # shared lock is released immediately) and writing a detached snapshot
+        # back via save_pattern (wholesale entry replacement) loses updates
+        # when two agents increment concurrently. A no-op if the pattern is
+        # missing.
+        self.storage.increment_pattern_usage(pattern_id)
 
     # -------------------------------------------------------------------------
     # Skill Operations

package/memory/ingest.py

@@ -26,6 +26,7 @@ Safety:
 """
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import re
@@ -165,6 +166,52 @@ def _log_to_errors(memory_base: str, function_name: str, exc: BaseException) ->
         return
 
 
+def _deterministic_episode_id(stable_key: str) -> str:
+    """Derive a stable, idempotent episode id from a stable key.
+
+    Re-ingesting the same session/summary must NOT create a new episode.
+    EpisodeTrace.create() mints a random uuid every call, so without a
+    deterministic id the index dedup (which keys on episode_id) counts
+    the same session twice, inflating episode_count / total_cost_usd /
+    total_tokens. Deriving the id from the stable key (e.g. the session
+    or task id) makes re-ingest land on the same filename + the same
+    index dedup bucket. Date is intentionally omitted so the id does not
+    drift across day boundaries for the same source.
+    """
+    digest = hashlib.sha1(stable_key.encode("utf-8")).hexdigest()[:12]
+    return f"ep-{digest}"
+
+
+def _existing_episode_path(
+    storage: Any, memory_base: str, episode_id: str
+) -> Optional[str]:
+    """Return the on-disk path for an already-stored episode id, or None.
+
+    Used for the idempotency existence check before writing so a repeat
+    ingest of the same source is a no-op rather than an append-new.
+    """
+    try:
+        existing = storage.load_episode(episode_id)
+    except Exception:
+        return None
+    if not existing:
+        return None
+    # Mirror storage.save_episode's path layout; prefer the stored
+    # timestamp date, fall back to a directory scan if absent.
+    ts = existing.get("timestamp")
+    if isinstance(ts, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", ts[:10] or ""):
+        candidate = Path(memory_base) / "episodic" / ts[:10] / f"task-{episode_id}.json"
+        if candidate.is_file():
+            return str(candidate)
+    episodic_dir = Path(memory_base) / "episodic"
+    if episodic_dir.is_dir():
+        for date_dir in episodic_dir.iterdir():
+            candidate = date_dir / f"task-{episode_id}.json"
+            if candidate.is_file():
+                return str(candidate)
+    return None
+
+
 def _parse_transcript_line(line: str) -> Optional[Dict[str, Any]]:
     """Parse one JSONL line; return None on parse error."""
     try:
@@ -385,12 +432,21 @@ def ingest_from_claude_transcript(
         engine = MemoryEngine(storage=storage, base_path=memory_base)
         engine.initialize()
 
+        # The transcript task_id is always stable (sessionId or filename
+        # stem), so derive a deterministic episode id from it and skip the
+        # write if this session was already ingested (idempotent re-run).
+        episode_id = _deterministic_episode_id(task_id)
+        already = _existing_episode_path(storage, memory_base, episode_id)
+        if already is not None:
+            return already
+
         trace = EpisodeTrace.create(
             task_id=task_id,
             agent=agent,
             phase=phase,
             goal=goal,
         )
+        trace.id = episode_id
         trace.outcome = outcome
         trace.duration_seconds = duration
         # v7.7.18 council fix: apply BOTH scrubbers to file paths --
@@ -440,16 +496,30 @@ def ingest_from_summary(
         engine = MemoryEngine(storage=storage, base_path=memory_base)
         engine.initialize()
 
+        # Only a caller-supplied task_id is a stable key. The auto-generated
+        # timestamp fallback is unique per call by design, so it must NOT
+        # drive a deterministic id (that would alias unrelated captures).
+        caller_supplied_key = task_id is not None
         if task_id is None:
             ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S")
             task_id = f"mcp-capture-{ts}"
 
+        if caller_supplied_key:
+            episode_id = _deterministic_episode_id(task_id)
+            already = _existing_episode_path(storage, memory_base, episode_id)
+            if already is not None:
+                return already
+        else:
+            episode_id = None
+
         trace = EpisodeTrace.create(
             task_id=task_id,
             agent=agent,
             phase=phase,
             goal=_scrub(goal)[:500],
         )
+        if episode_id is not None:
+            trace.id = episode_id
         trace.outcome = outcome if outcome in ("success", "failure", "partial") else "success"
         trace.duration_seconds = max(0, int(duration_seconds))
         trace.files_read = [_scrub_path(_scrub(p)) for p in (files_read or [])]

package/memory/knowledge_graph.py

@@ -9,10 +9,20 @@ Storage: ~/.loki/knowledge/
 """
 
 import json
+import logging
 import os
 from pathlib import Path
 from datetime import datetime, timezone
 
+try:
+    import fcntl  # POSIX-only; absent on Windows.
+    _HAS_FCNTL = True
+except ImportError:  # pragma: no cover - non-POSIX fallback
+    fcntl = None
+    _HAS_FCNTL = False
+
+logger = logging.getLogger(__name__)
+
 
 class OrganizationKnowledgeGraph:
     """Aggregates patterns and knowledge across multiple projects."""
@@ -79,11 +89,30 @@ class OrganizationKnowledgeGraph:
         return unique
 
     def save_patterns(self, patterns):
-        """Save patterns to the knowledge store (appends to JSONL)."""
+        """Save patterns to the knowledge store (appends to JSONL).
+
+        The append is guarded by an exclusive flock and a single flushed
+        write so concurrent writers cannot interleave partial lines.
+        stdio buffering breaks O_APPEND atomicity and a record longer
+        than PIPE_BUF can split mid-line under concurrency, so we both
+        hold the lock and emit one contiguous buffer before unlocking.
+        """
         self.ensure_dir()
+        # Serialize first so a serialization error cannot leave a partial
+        # line on disk while the lock is held.
+        buffer = "".join(json.dumps(p) + "\n" for p in patterns)
+        if not buffer:
+            return
         with open(self.patterns_file, 'a') as f:
-            for p in patterns:
-                f.write(json.dumps(p) + '\n')
+            if _HAS_FCNTL:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+            try:
+                f.write(buffer)
+                f.flush()
+                os.fsync(f.fileno())
+            finally:
+                if _HAS_FCNTL:
+                    fcntl.flock(f.fileno(), fcntl.LOCK_UN)
 
     def load_patterns(self, limit=100):
         """Load patterns from the knowledge store."""
@@ -98,6 +127,12 @@ class OrganizationKnowledgeGraph:
                 try:
                     patterns.append(json.loads(line))
                 except json.JSONDecodeError:
+                    # Resilient: skip the bad row but surface it so silent
+                    # data loss (e.g. a torn legacy line) is observable.
+                    logger.warning(
+                        "knowledge_graph: dropped unparseable patterns.jsonl line: %.120r",
+                        line,
+                    )
                     continue
                 if len(patterns) >= limit:
                     break

package/memory/retrieval.py

@@ -297,6 +297,14 @@ class MemoryRetrieval:
 
     # Track which legacy entries we've already warned about to avoid log spam.
     _LEGACY_WARN_LIMIT = 5
+
+    # Cap on episode files read per keyword scan. The episodic store grows
+    # unbounded over a long-lived project, so an uncapped scan reads+parses
+    # every episode on every keyword query. Date dirs are walked newest-first,
+    # so the cap retains the most recent episodes (the natural relevance order)
+    # and only stops unbounded IO; it does not change scoring of retained
+    # candidates.
+    _KEYWORD_SCAN_MAX_EPISODES = 2000
     _legacy_warned_count: int = 0
 
     def _belongs_to_namespace(self, result: Dict[str, Any]) -> bool:
@@ -1509,7 +1517,14 @@ class MemoryRetrieval:
         if not date_dirs:
             return results
 
+        # Bound the scan so an unbounded episodic store does not force a
+        # read+parse of every episode on each query (see
+        # _KEYWORD_SCAN_MAX_EPISODES). Newest date dirs first keeps the most
+        # recent episodes.
+        scanned = 0
         for date_dir in sorted(date_dirs, reverse=True):
+            if scanned >= self._KEYWORD_SCAN_MAX_EPISODES:
+                break
             if not date_dir.is_dir():
                 continue
 
@@ -1519,7 +1534,10 @@ class MemoryRetrieval:
             for episode_file in episode_files:
                 if episode_file.name == "index.json":
                     continue
+                if scanned >= self._KEYWORD_SCAN_MAX_EPISODES:
+                    break
 
+                scanned += 1
                 data = self.storage.read_json(
                     f"episodic/{date_dir.name}/{episode_file.name}"
                 )

package/memory/storage.py

@@ -429,6 +429,21 @@ class MemoryStorage:
     # Episode Storage
     # -------------------------------------------------------------------------
 
+    @staticmethod
+    def _sanitize_episode_id(episode_id) -> str:
+        """
+        Sanitize an episode id for use in a filename.
+
+        Separators and "." segments cannot leak into the path (mirrors
+        save_skill). save_episode and load_episode must use the SAME transform
+        or a round-tripped id with ":", "/", or "." chars would write to one
+        file and read from another.
+        """
+        return "".join(
+            c if c.isalnum() or c in "-_" else "_"
+            for c in str(episode_id)
+        )
+
     def save_episode(self, episode: EpisodeTrace) -> str:
         """
         Save an episode trace to storage.
@@ -475,10 +490,7 @@ class MemoryStorage:
         if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date_str):
             date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
 
-        safe_episode_id = "".join(
-            c if c.isalnum() or c in "-_" else "_"
-            for c in str(episode_id)
-        )
+        safe_episode_id = self._sanitize_episode_id(episode_id)
 
         date_dir = self.base_path / "episodic" / date_str
         date_dir.mkdir(parents=True, exist_ok=True)
@@ -507,10 +519,14 @@ class MemoryStorage:
         if not episodic_dir.exists():
             return None
 
+        # Sanitize the same way save_episode does so an id carrying ":", "/",
+        # or "." chars resolves to the file it was actually written to.
+        safe_episode_id = self._sanitize_episode_id(episode_id)
+
         # Search all date directories
         for date_dir in episodic_dir.iterdir():
             if date_dir.is_dir():
-                file_path = date_dir / f"task-{episode_id}.json"
+                file_path = date_dir / f"task-{safe_episode_id}.json"
                 if file_path.exists():
                     data = self._load_json(file_path)
                     if data:
@@ -1519,6 +1535,53 @@ class MemoryStorage:
 
         return False
 
+    def increment_pattern_usage(self, pattern_id: str) -> bool:
+        """Atomically increment a semantic pattern's usage_count, keyed by id.
+
+        The entire read-mutate-write happens inside a single exclusive
+        _file_lock so concurrent increments cannot lose updates. Mirrors the
+        lock-spanning idiom of _persist_boost_semantic / save_pattern: fresh
+        read of patterns.json under the lock -> bump the matching entry ->
+        atomic write (which reuses the same reentrant lock).
+
+        Args:
+            pattern_id: Pattern identifier to increment.
+
+        Returns:
+            True if the pattern was found and incremented, False otherwise.
+        """
+        patterns_path = self.base_path / "semantic" / "patterns.json"
+        if not patterns_path.exists():
+            return False
+
+        with self._file_lock(patterns_path, exclusive=True):
+            if not patterns_path.exists():
+                return False
+            try:
+                with open(patterns_path, "r", encoding="utf-8") as f:
+                    patterns_file = json.load(f)
+            except (json.JSONDecodeError, OSError, UnicodeDecodeError):
+                return False
+            if not patterns_file:
+                return False
+
+            patterns = patterns_file.get("patterns", [])
+            for pattern in patterns:
+                if not isinstance(pattern, dict):
+                    continue
+                if pattern.get("id") == pattern_id:
+                    # `or 0` guards an explicit null usage_count (corrupt or
+                    # hand-edited record); null and 0 are equivalent here.
+                    pattern["usage_count"] = (pattern.get("usage_count") or 0) + 1
+                    pattern["last_used"] = datetime.now(timezone.utc).isoformat()
+                    patterns_file["last_updated"] = datetime.now(
+                        timezone.utc
+                    ).isoformat()
+                    self._atomic_write(patterns_path, patterns_file)
+                    return True
+
+        return False
+
     def batch_apply_decay(
         self,
         collection: str = "all",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "loki-mode",
   "mcpName": "io.github.asklokesh/loki-mode",
-  "version": "7.75.0",
+  "version": "7.77.0",
   "description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
   "keywords": [
     "agent",

package/plugins/loki-mode/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
   "name": "loki-mode",
   "displayName": "Loki Mode",
-  "version": "7.75.0",
+  "version": "7.77.0",
   "description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
   "author": {
     "name": "Autonomi",