loki-mode 7.95.0 → 7.97.0

package/docs/INSTALLATION.md

@@ -2,7 +2,7 @@
 
 The flagship product of [Autonomi](https://www.autonomi.dev/). Loki Mode is a spec-driven autonomous builder with a built-in trust layer that takes any spec to a deployed product and verifies completion with evidence (quality gates plus a completion council), not just a "done" claim. Complete installation instructions for all platforms and use cases.
 
-**Version:** v7.95.0
+**Version:** v7.97.0
 
 ---
 

package/events/bus.py

@@ -130,6 +130,16 @@ class LokiEvent:
         # dedup (_processed_ids + pending-file glob) and re-importing every flat
         # line on each call. Derive a DETERMINISTIC id from the record content
         # so repeated imports of the same line are idempotent.
+        #
+        # Width: the digest source is user-controllable (timestamp + type +
+        # payload). An 8-hex-char (32-bit) truncation collides at the birthday
+        # bound around ~77k distinct events (sqrt(2^32)); a collision makes two
+        # DISTINCT events share an id, so dedup (_processed_ids + pending glob)
+        # silently DROPS the second event. Widen to 16 hex chars (64 bits) so
+        # the collision bound moves to ~4 billion distinct events -- well past
+        # any realistic events.jsonl. This deterministic-derivation path is
+        # Python-only (bus.ts has no jsonl import and mints a random id when one
+        # is absent), so widening here needs no cross-language parity change.
         event_id = data.get('id', '')
         if not event_id:
             digest_src = '%s|%s|%s' % (
@@ -137,7 +147,7 @@ class LokiEvent:
                 raw_type,
                 json.dumps(payload, sort_keys=True),
             )
-            event_id = hashlib.sha1(digest_src.encode('utf-8')).hexdigest()[:8]
+            event_id = hashlib.sha1(digest_src.encode('utf-8')).hexdigest()[:16]
 
         return cls(
             id=event_id,

package/events/bus.ts

@@ -292,7 +292,13 @@ export class EventBus {
 
     if (archive) {
       try {
-        const files = fs.readdirSync(this.pendingDir).filter((f) => f.includes(event.id));
+        // Match the exact id at the filename boundary. Emitted files are named
+        // `${timestamp}_${id}.json`, so a substring match (f.includes(event.id))
+        // would also match a DIFFERENT event whose id contains this id as a
+        // substring, archiving/marking the wrong event file (event loss). The
+        // Python impl already uses the precise `*_${id}.json` glob; mirror it.
+        const suffix = `_${event.id}.json`;
+        const files = fs.readdirSync(this.pendingDir).filter((f) => f.endsWith(suffix));
         for (const file of files) {
           const src = path.join(this.pendingDir, file);
           const dst = path.join(this.archiveDir, file);

package/events/emit.sh

@@ -119,16 +119,27 @@ fi
 
 # JSON escape helper: handles \, ", and control characters including newlines
 #
-# The sed pass escapes the named short forms (\\ \" \t \r \b \f); the first awk
-# pass collapses embedded newlines to \n. Any OTHER C0 control byte
-# (0x01-0x07, 0x0B, 0x0E-0x1F) is invalid raw inside a JSON string and is
-# escaped as \uXXXX by the final awk pass -- otherwise json.loads / JSON.parse
-# reject the line and consumers (dashboard _read_events, learning aggregator)
-# silently drop it. The named short forms are already two-char ASCII by the
-# time the final pass runs, so they are never re-escaped.
+# The sed pass escapes backslash, double-quote, tab and carriage-return to
+# their named short forms (\\ \" \t \r); the first awk pass collapses embedded
+# newlines to \n. Every REMAINING C0 control byte (0x01-0x08, 0x0B-0x0C,
+# 0x0E-0x1F -- which includes backspace 0x08 and form-feed 0x0C) is invalid raw
+# inside a JSON string and is escaped as \uXXXX by the final awk pass (its map
+# covers all of i=1..31). Without that escaping json.loads / JSON.parse reject
+# the line and consumers (dashboard _read_events, learning aggregator) silently
+# drop it. The named short forms emitted by sed are already two-char ASCII by
+# the time the final pass runs, so they are never re-escaped.
+#
+# NOTE: backspace/form-feed are deliberately NOT escaped in the sed pass.
+# Doing so requires embedding the raw control bytes in the sed pattern, which
+# editors silently strip -- that left the two empty-pattern sed no-ops this
+# helper previously carried. An empty sed regex reuses the LAST applied regex,
+# so those entries were dead AND a latent corruption hazard. The final awk
+# pass already escapes every byte in i=1..31 (backspace and form-feed included)
+# to the six-char uXXXX unicode escape, producing valid JSON, so the
+# named-short-form variants are unnecessary.
 json_escape() {
     printf '%s' "$1" \
-        | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g; s/\r/\\r/g; s//\\b/g; s//\\f/g' \
+        | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g; s/\r/\\r/g' \
         | awk '{if(NR>1) printf "\\n"; printf "%s", $0}' \
         | awk 'BEGIN{for(i=1;i<=31;i++)m[sprintf("%c",i)]=sprintf("\\u%04x",i)}
                {s=""; n=length($0);
@@ -163,9 +174,23 @@ EVENT=$(cat <<EOF
 EOF
 )
 
-# Write event file
+# Write event file atomically.
+#
+# A bare `echo "$EVENT" > "$EVENT_FILE"` is NON-atomic: a reader (bus.py
+# get_pending_events / from_dict, dashboard) that globs `*.json` mid-write sees
+# a truncated/partial file and either fails json.loads (event silently dropped)
+# or, worse, reads a half-written record. Write to a sibling temp file in the
+# SAME directory (so rename(2) is an atomic same-filesystem operation) and then
+# rename into place; a `.tmp` suffix keeps the in-progress file out of the
+# `*.json` glob readers use. Clean up the temp on any failure so partials never
+# linger.
 EVENT_FILE="$EVENTS_DIR/${TIMESTAMP//:/-}_$EVENT_ID.json"
-echo "$EVENT" > "$EVENT_FILE"
+EVENT_TMP="$EVENT_FILE.tmp"
+if printf '%s\n' "$EVENT" > "$EVENT_TMP" && mv -f "$EVENT_TMP" "$EVENT_FILE"; then
+    :
+else
+    rm -f "$EVENT_TMP" 2>/dev/null || true
+fi
 
 # Rotate events.jsonl if it exceeds 50MB (keep 1 backup)
 EVENTS_LOG="$LOKI_DIR/events.jsonl"

package/loki-ts/dist/loki.js

@@ -1,5 +1,5 @@
 // @bun
-var z8=Object.defineProperty;var X8=($)=>$;function K8($,Q){this[$]=X8.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)z8($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:K8.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var J$=import.meta.require;var y1={};b(y1,{lokiDir:()=>P,homeLokiDir:()=>T$,findRepoRootForVersion:()=>e$,REPO_ROOT:()=>h});import{resolve as t,dirname as i$}from"path";import{fileURLToPath as q8}from"url";import{existsSync as E$}from"fs";import{homedir as J8}from"os";function V8(){let $=h1;for(let Q=0;Q<6;Q++){if(E$(t($,"VERSION"))&&E$(t($,"autonomy/run.sh")))return $;let Z=i$($);if(Z===$)break;$=Z}return t(h1,"..","..","..")}function e$($){let Q=$;for(let Z=0;Z<6;Z++){if(E$(t(Q,"VERSION"))&&E$(t(Q,"autonomy/run.sh")))return Q;let z=i$(Q);if(z===Q)break;Q=z}return t($,"..","..","..")}function P(){return process.env.LOKI_DIR??t(process.cwd(),".loki")}function T$(){return t(J8(),".loki")}var h1,h;var C=L(()=>{h1=i$(q8(import.meta.url));h=V8()});import{readFileSync as W8}from"fs";import{resolve as U8,dirname as H8}from"path";import{fileURLToPath as G8}from"url";function N$(){if(Q$!==null)return Q$;let $="7.95.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=H8(G8(import.meta.url)),Z=e$(Q);Q$=W8(U8(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var $1=L(()=>{C()});var u1={};b(u1,{runOrThrow:()=>x8,run:()=>F,readStreamCapped:()=>f1,commandVersion:()=>N8,commandExists:()=>f,ShellError:()=>Q1,MAX_STDOUT_BYTES:()=>m1});async function f1($,Q=m1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:W}=await Z.read();if(K)break;if(!W)continue;if(q+=W.byteLength,q>Q){let V=W.byteLength-(q-Q);X+=z.decode(W.subarray(0,V),{stream:!0});break}X+=z.decode(W,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,W]=await Promise.all([f1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:W}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function x8($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new Q1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=E8($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function E8($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function N8($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var m1=16777216,Q1;var d=L(()=>{Q1=class Q1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function r($){return S8?"":$}var S8,T,S,_,lZ,I,k,y,J;var c=L(()=>{S8=(process.env.NO_COLOR??"").length>0;T=r("\x1B[0;31m"),S=r("\x1B[0;32m"),_=r("\x1B[1;33m"),lZ=r("\x1B[0;34m"),I=r("\x1B[0;36m"),k=r("\x1B[1m"),y=r("\x1B[2m"),J=r("\x1B[0m")});import{existsSync as u8}from"fs";async function Z$(){if(A$!==void 0)return A$;let $="/opt/homebrew/bin/python3.12";if(u8($))return A$=$,$;let Q=await f("python3.12");if(Q)return A$=Q,Q;let Z=await f("python3");return A$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var A$;var V$=L(()=>{d()});var q0={};b(q0,{runStatus:()=>W3});import{existsSync as v,readFileSync as U$,readdirSync as i1,statSync as e1}from"fs";import{resolve as D,basename as $3}from"path";import{homedir as Q3}from"os";function $0($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function Q0($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*D$/Q);if(X>D$)X=D$;let q=D$-X,K=S;if(z>=80)K=T;else if(z>=50)K=_;let W="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),V=$0($),U=$0(Q);return`  ${k}${Z}${J} ${K}[${W}]${J} ${z}% (${V} / ${U})`}async function z3(){if(await f("jq"))return!0;return process.stdout.write(`${T}Error: jq is required but not installed.${J}
+var z8=Object.defineProperty;var X8=($)=>$;function K8($,Q){this[$]=X8.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)z8($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:K8.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var J$=import.meta.require;var y1={};b(y1,{lokiDir:()=>P,homeLokiDir:()=>T$,findRepoRootForVersion:()=>e$,REPO_ROOT:()=>h});import{resolve as t,dirname as i$}from"path";import{fileURLToPath as q8}from"url";import{existsSync as E$}from"fs";import{homedir as J8}from"os";function V8(){let $=h1;for(let Q=0;Q<6;Q++){if(E$(t($,"VERSION"))&&E$(t($,"autonomy/run.sh")))return $;let Z=i$($);if(Z===$)break;$=Z}return t(h1,"..","..","..")}function e$($){let Q=$;for(let Z=0;Z<6;Z++){if(E$(t(Q,"VERSION"))&&E$(t(Q,"autonomy/run.sh")))return Q;let z=i$(Q);if(z===Q)break;Q=z}return t($,"..","..","..")}function P(){return process.env.LOKI_DIR??t(process.cwd(),".loki")}function T$(){return t(J8(),".loki")}var h1,h;var C=L(()=>{h1=i$(q8(import.meta.url));h=V8()});import{readFileSync as W8}from"fs";import{resolve as U8,dirname as H8}from"path";import{fileURLToPath as G8}from"url";function N$(){if(Q$!==null)return Q$;let $="7.97.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=H8(G8(import.meta.url)),Z=e$(Q);Q$=W8(U8(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var $1=L(()=>{C()});var u1={};b(u1,{runOrThrow:()=>x8,run:()=>F,readStreamCapped:()=>f1,commandVersion:()=>N8,commandExists:()=>f,ShellError:()=>Q1,MAX_STDOUT_BYTES:()=>m1});async function f1($,Q=m1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:W}=await Z.read();if(K)break;if(!W)continue;if(q+=W.byteLength,q>Q){let V=W.byteLength-(q-Q);X+=z.decode(W.subarray(0,V),{stream:!0});break}X+=z.decode(W,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,W]=await Promise.all([f1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:W}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function x8($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new Q1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=E8($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function E8($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function N8($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var m1=16777216,Q1;var d=L(()=>{Q1=class Q1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function r($){return S8?"":$}var S8,T,S,_,lZ,I,k,y,J;var c=L(()=>{S8=(process.env.NO_COLOR??"").length>0;T=r("\x1B[0;31m"),S=r("\x1B[0;32m"),_=r("\x1B[1;33m"),lZ=r("\x1B[0;34m"),I=r("\x1B[0;36m"),k=r("\x1B[1m"),y=r("\x1B[2m"),J=r("\x1B[0m")});import{existsSync as u8}from"fs";async function Z$(){if(A$!==void 0)return A$;let $="/opt/homebrew/bin/python3.12";if(u8($))return A$=$,$;let Q=await f("python3.12");if(Q)return A$=Q,Q;let Z=await f("python3");return A$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var A$;var V$=L(()=>{d()});var q0={};b(q0,{runStatus:()=>W3});import{existsSync as v,readFileSync as U$,readdirSync as i1,statSync as e1}from"fs";import{resolve as D,basename as $3}from"path";import{homedir as Q3}from"os";function $0($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function Q0($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*D$/Q);if(X>D$)X=D$;let q=D$-X,K=S;if(z>=80)K=T;else if(z>=50)K=_;let W="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),V=$0($),U=$0(Q);return`  ${k}${Z}${J} ${K}[${W}]${J} ${z}% (${V} / ${U})`}async function z3(){if(await f("jq"))return!0;return process.stdout.write(`${T}Error: jq is required but not installed.${J}
 `),process.stdout.write(`Install with:
 `),process.stdout.write(`  brew install jq    (macOS)
 `),process.stdout.write(`  apt install jq     (Debian/Ubuntu)
@@ -802,4 +802,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
 `),2}default:return process.stderr.write(`Unknown command: ${Q}
 `),process.stderr.write(Z8),2}}r1();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var FZ=await jZ(Bun.argv.slice(2));process.exit(FZ);
 
-//# debugId=184A380A5462265664756E2164756E21
+//# debugId=ABBB64E42A394DE264756E2164756E21

package/mcp/__init__.py

@@ -57,4 +57,4 @@ try:
 except ImportError:
     __all__ = ['mcp']
 
-__version__ = '7.95.0'
+__version__ = '7.97.0'

package/mcp/learning_collector.py

@@ -512,6 +512,24 @@ def get_mcp_learning_collector(
         MCPLearningCollector instance
     """
     global _collector
+    # Recreate when the requested loki_dir differs from the cached collector's
+    # directory. The module-level singleton would otherwise return a collector
+    # bound to a PREVIOUS project's .loki dir on cwd change, contaminating one
+    # project's learnings into another in a multi-project session. Compare by
+    # realpath so symlinked/relative paths to the same dir do not churn.
+    if _collector is not None and loki_dir is not None:
+        try:
+            cached = os.path.realpath(str(getattr(_collector, "loki_dir", "")))
+            requested = os.path.realpath(str(loki_dir))
+            if cached != requested:
+                close = getattr(_collector, "close", None)
+                if callable(close):
+                    close()
+                _collector = None
+        except Exception:
+            # On any path-resolution error, fall through and reuse the cache
+            # rather than risk losing the collector.
+            pass
     if _collector is None:
         _collector = MCPLearningCollector(loki_dir=loki_dir)
     return _collector

package/mcp/server.py

@@ -64,13 +64,28 @@ _learning_collector = None
 
 
 def _get_learning_collector():
-    """Get or create the LearningCollector instance for MCP server."""
+    """Get or create the LearningCollector instance for MCP server.
+
+    Revalidates the cached collector against the current cwd-derived .loki dir
+    (mirrors the StateManager realpath-compare-recreate guard in
+    _get_mcp_state_manager). A multi-project MCP session can change cwd between
+    calls; without this, the cached collector stays bound to a PREVIOUS
+    project's .loki dir and contaminates one project's learnings into another.
+    """
     global _learning_collector
     if not LEARNING_COLLECTOR_AVAILABLE:
         return None
+    from pathlib import Path
+    loki_dir = Path(os.getcwd()) / '.loki'
+    if _learning_collector is not None:
+        existing_dir = getattr(_learning_collector, 'loki_dir', None)
+        if existing_dir and os.path.realpath(str(existing_dir)) != os.path.realpath(str(loki_dir)):
+            # Project directory changed, recreate
+            close = getattr(_learning_collector, 'close', None)
+            if callable(close):
+                close()
+            _learning_collector = None
     if _learning_collector is None:
-        from pathlib import Path
-        loki_dir = Path(os.getcwd()) / '.loki'
         _learning_collector = get_mcp_learning_collector(loki_dir=loki_dir)
     return _learning_collector
 

package/memory/consolidation.py

@@ -228,28 +228,34 @@ class ConsolidationPipeline:
                     merged = False
                     for idx, existing in enumerate(existing_patterns):
                         if self._patterns_similar(new_pattern, existing):
-                            # Re-read the target pattern fresh immediately before
-                            # merging (BUG-MEM C1, lost-update). The whole-run
-                            # snapshot at step 4 can be stale by now: a concurrent
-                            # storage.increment_pattern_usage() (atomic read-mutate-
-                            # write under one exclusive lock) may have bumped
-                            # usage_count/last_used AFTER the snapshot. merge_with_existing() builds the
-                            # merged record from best_match.usage_count/last_used,
-                            # so merging from the stale snapshot clobbers that bump.
-                            # Re-reading narrows the window to this single write.
-                            merge_base = self._reload_pattern(existing)
-                            merged_pattern = self.merge_with_existing(new_pattern, [merge_base])
-                            self.storage.update_pattern(merged_pattern)
-                            # Refresh the in-memory copy so a later new pattern in
-                            # this same run that also merges into this existing
-                            # pattern builds on the just-merged state. Without this,
-                            # the second merge reads the stale pre-merge base and its
-                            # update_pattern() overwrites storage, silently dropping
-                            # the first merge's conditions/source_episodes/confidence.
-                            existing_patterns[idx] = merged_pattern
-                            result.patterns_merged += 1
-                            merged = True
-                            break
+                            # Atomic read-merge-write (BUG-MEM C1, lost-update).
+                            # The whole-run snapshot at step 4 can be stale by now:
+                            # a concurrent storage.increment_pattern_usage()
+                            # (atomic read-mutate-write under one exclusive lock)
+                            # may have bumped usage_count/last_used AFTER the
+                            # snapshot, and merge_with_existing() builds the merged
+                            # record from the base's usage_count/last_used. The
+                            # merge now runs INSIDE update_pattern_with_merge's
+                            # single lock on patterns.json (the same path
+                            # increment_pattern_usage locks), so the merge base is
+                            # the live on-disk record and the bump cannot be lost.
+                            merged_holder = {}
+
+                            def _merge(current, _np=new_pattern, _holder=merged_holder):
+                                base = SemanticPattern.from_dict(current)
+                                m = self.merge_with_existing(_np, [base])
+                                _holder["pattern"] = m
+                                return m
+
+                            if self.storage.update_pattern_with_merge(existing.id, _merge):
+                                merged_pattern = merged_holder["pattern"]
+                                # Refresh the in-memory copy so a later new pattern
+                                # in this same run that also merges into this
+                                # existing pattern builds on the just-merged state.
+                                existing_patterns[idx] = merged_pattern
+                                result.patterns_merged += 1
+                                merged = True
+                                break
 
                     if not merged:
                         self.storage.save_pattern(new_pattern)
@@ -277,20 +283,27 @@ class ConsolidationPipeline:
             for idx, existing in enumerate(existing_patterns):
                 if (existing.incorrect_approach and
                     self._patterns_similar(anti_pattern, existing, threshold=0.6)):
-                    # Re-read fresh before merge (same C1 lost-update guard as the
-                    # cluster merge loop above): merge from current on-disk state,
-                    # not the potentially-stale step-4 snapshot.
-                    merge_base = self._reload_pattern(existing)
-                    merged_pattern = self.merge_with_existing(anti_pattern, [merge_base])
-                    self.storage.update_pattern(merged_pattern)
-                    # Refresh in-memory copy (same data-loss guard as the cluster
-                    # merge loop above): a later anti-pattern merging into this same
-                    # existing pattern must build on the just-merged state, not the
-                    # stale pre-merge base.
-                    existing_patterns[idx] = merged_pattern
-                    result.patterns_merged += 1
-                    merged = True
-                    break
+                    # Atomic read-merge-write (same C1 lost-update guard as the
+                    # cluster merge loop above): merge from the live on-disk record
+                    # inside update_pattern_with_merge's single lock so a concurrent
+                    # usage bump cannot be clobbered.
+                    merged_holder = {}
+
+                    def _merge(current, _ap=anti_pattern, _holder=merged_holder):
+                        base = SemanticPattern.from_dict(current)
+                        m = self.merge_with_existing(_ap, [base])
+                        _holder["pattern"] = m
+                        return m
+
+                    if self.storage.update_pattern_with_merge(existing.id, _merge):
+                        merged_pattern = merged_holder["pattern"]
+                        # Refresh in-memory copy: a later anti-pattern merging into
+                        # this same existing pattern must build on the just-merged
+                        # state, not the stale pre-merge base.
+                        existing_patterns[idx] = merged_pattern
+                        result.patterns_merged += 1
+                        merged = True
+                        break
 
             if not merged:
                 self.storage.save_pattern(anti_pattern)
@@ -324,34 +337,12 @@ class ConsolidationPipeline:
         result.duration_seconds = time.time() - start_time
         return result
 
-    def _reload_pattern(self, fallback: SemanticPattern) -> SemanticPattern:
-        """Re-read a pattern fresh from storage immediately before merging.
-
-        Used by the merge branches to avoid the C1 lost-update: the step-4
-        snapshot may be stale (a concurrent usage bump can land after it), and
-        merge_with_existing() copies usage_count/last_used from the base. Reading
-        the current on-disk record makes the merge build on live state.
-
-        Mirrors the snapshot's dict/object handling. If load_pattern returns
-        nothing (e.g. the record vanished), fall back to the in-memory copy so the
-        merge still proceeds rather than crashing.
-
-        Residual limitation (honest): load_pattern and update_pattern are SEPARATE
-        lock acquisitions, so a bump landing between this re-read and the write is
-        still lost. This narrows the race window from the whole run to a single
-        write; it is a mitigation, not cross-process atomicity. A full fix needs a
-        compare-and-set or merge-callback update in storage, which is out of scope
-        for this file (storage.py is frozen this batch).
-        """
-        try:
-            fresh = self.storage.load_pattern(fallback.id)
-        except Exception:
-            return fallback
-        if not fresh:
-            return fallback
-        if isinstance(fresh, dict):
-            return SemanticPattern.from_dict(fresh)
-        return fresh
+    # NOTE: The former _reload_pattern() helper (a partial C1 lost-update
+    # mitigation that re-read the pattern in a SEPARATE lock from the write) was
+    # removed once the merge moved fully inside storage.update_pattern_with_merge,
+    # which performs the read, merge, and write under ONE exclusive lock on
+    # patterns.json. That closes the lost-update race cross-process; no residual
+    # narrow window remains.
 
     # -------------------------------------------------------------------------
     # Clustering Methods

package/memory/engine.py

@@ -1034,6 +1034,39 @@ class MemoryEngine:
 
         return None
 
+    @staticmethod
+    def _parse_optional_datetime(
+        raw: Any, *, record_id: str = "<unknown>", field: str = "last_accessed"
+    ) -> Optional[datetime]:
+        """Tolerantly parse an optional ISO datetime field off a stored record.
+
+        A corrupt/non-ISO value on ONE record must not raise out of the
+        _dict_to_* converters and crash the whole batch list-comp in
+        get_recent_episodes / find_patterns / list_skills (which would drop EVERY
+        item in the scan, not just the bad one) on the RARV retrieval hot path.
+        An unparseable value falls back to None (treated as never-accessed) so
+        the record is still returned and retrievable.
+
+        Returns a tz-aware datetime (UTC assumed when naive), or None when the
+        value is missing/empty/unparseable.
+        """
+        if not raw:
+            return None
+        if isinstance(raw, datetime):
+            return raw.replace(tzinfo=timezone.utc) if raw.tzinfo is None else raw
+        if isinstance(raw, str):
+            value = raw[:-1] if raw.endswith("Z") else raw
+            try:
+                parsed = datetime.fromisoformat(value)
+            except ValueError:
+                logger.warning(
+                    "Record %s has unparseable %s %r; treating as never-accessed",
+                    record_id, field, raw,
+                )
+                return None
+            return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed
+        return None
+
     def _dict_to_episode(self, data: Dict[str, Any]) -> EpisodeTrace:
         """Convert dictionary to EpisodeTrace."""
         # Parse timestamp string to datetime
@@ -1081,20 +1114,13 @@ class MemoryEngine:
             for e in errors_raw
         ]
 
-        # Parse last_accessed datetime
-        last_accessed = None
-        last_accessed_raw = data.get("last_accessed")
-        if last_accessed_raw:
-            if isinstance(last_accessed_raw, str):
-                if last_accessed_raw.endswith("Z"):
-                    last_accessed_raw = last_accessed_raw[:-1]
-                last_accessed = datetime.fromisoformat(last_accessed_raw)
-                if last_accessed.tzinfo is None:
-                    last_accessed = last_accessed.replace(tzinfo=timezone.utc)
-            elif isinstance(last_accessed_raw, datetime):
-                last_accessed = last_accessed_raw
-                if last_accessed.tzinfo is None:
-                    last_accessed = last_accessed.replace(tzinfo=timezone.utc)
+        # Parse last_accessed datetime (tolerant: a corrupt value falls back to
+        # None so a single bad episode never crashes the retrieval batch).
+        last_accessed = self._parse_optional_datetime(
+            data.get("last_accessed"),
+            record_id=data.get("id", "<unknown>"),
+            field="last_accessed",
+        )
 
         return EpisodeTrace(
             id=data.get("id", ""),
@@ -1119,21 +1145,14 @@ class MemoryEngine:
 
     def _dict_to_pattern(self, data: Dict[str, Any]) -> SemanticPattern:
         """Convert dictionary to SemanticPattern."""
-        # Parse last_used string to datetime or None
-        last_used = None
-        last_used_raw = data.get("last_used")
-        if last_used_raw:
-            if isinstance(last_used_raw, str):
-                # Handle ISO format with Z suffix
-                if last_used_raw.endswith("Z"):
-                    last_used_raw = last_used_raw[:-1]
-                last_used = datetime.fromisoformat(last_used_raw)
-                if last_used.tzinfo is None:
-                    last_used = last_used.replace(tzinfo=timezone.utc)
-            elif isinstance(last_used_raw, datetime):
-                last_used = last_used_raw
-                if last_used.tzinfo is None:
-                    last_used = last_used.replace(tzinfo=timezone.utc)
+        # Parse last_used string to datetime or None (tolerant: a corrupt value
+        # on one pattern must not crash the find_patterns retrieval batch; it
+        # shares the same hot path and crash mechanism as last_accessed below).
+        last_used = self._parse_optional_datetime(
+            data.get("last_used"),
+            record_id=data.get("id", "<unknown>"),
+            field="last_used",
+        )
 
         # Convert links dicts to Link objects
         links_raw = data.get("links", [])
@@ -1142,20 +1161,12 @@ class MemoryEngine:
             for link in links_raw
         ]
 
-        # Parse last_accessed datetime
-        last_accessed = None
-        last_accessed_raw = data.get("last_accessed")
-        if last_accessed_raw:
-            if isinstance(last_accessed_raw, str):
-                if last_accessed_raw.endswith("Z"):
-                    last_accessed_raw = last_accessed_raw[:-1]
-                last_accessed = datetime.fromisoformat(last_accessed_raw)
-                if last_accessed.tzinfo is None:
-                    last_accessed = last_accessed.replace(tzinfo=timezone.utc)
-            elif isinstance(last_accessed_raw, datetime):
-                last_accessed = last_accessed_raw
-                if last_accessed.tzinfo is None:
-                    last_accessed = last_accessed.replace(tzinfo=timezone.utc)
+        # Parse last_accessed datetime (tolerant: see _dict_to_episode).
+        last_accessed = self._parse_optional_datetime(
+            data.get("last_accessed"),
+            record_id=data.get("id", "<unknown>"),
+            field="last_accessed",
+        )
 
         return SemanticPattern(
             id=data.get("id", ""),
@@ -1182,20 +1193,12 @@ class MemoryEngine:
             for e in raw_errors
         ]
 
-        # Parse last_accessed datetime
-        last_accessed = None
-        last_accessed_raw = data.get("last_accessed")
-        if last_accessed_raw:
-            if isinstance(last_accessed_raw, str):
-                if last_accessed_raw.endswith("Z"):
-                    last_accessed_raw = last_accessed_raw[:-1]
-                last_accessed = datetime.fromisoformat(last_accessed_raw)
-                if last_accessed.tzinfo is None:
-                    last_accessed = last_accessed.replace(tzinfo=timezone.utc)
-            elif isinstance(last_accessed_raw, datetime):
-                last_accessed = last_accessed_raw
-                if last_accessed.tzinfo is None:
-                    last_accessed = last_accessed.replace(tzinfo=timezone.utc)
+        # Parse last_accessed datetime (tolerant: see _dict_to_episode).
+        last_accessed = self._parse_optional_datetime(
+            data.get("last_accessed"),
+            record_id=data.get("id", "<unknown>"),
+            field="last_accessed",
+        )
 
         return ProceduralSkill(
             id=data.get("id", ""),

package/memory/rag_injector.py

@@ -9,10 +9,66 @@ for injection into the RARV prompt cycle.
 
 import argparse
 import json
+import re
 import sys
 from pathlib import Path
 
 
+# Per-field hard cap so a single oversized/garbage memory entry cannot dominate
+# the injected context or push real signal out of the token budget.
+_MAX_FIELD_CHARS = 600
+
+# Control characters (excluding ordinary whitespace handled separately) that
+# must never reach the prompt: they can corrupt rendering or smuggle hidden
+# directives. Covers C0 controls and DEL.
+_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+
+
+def _sanitize_field(value, max_chars=_MAX_FIELD_CHARS):
+    """Neutralize a single field from stored memory before it enters a prompt.
+
+    Memory entries are untrusted input: a malicious or garbage pattern could
+    contain prompt-injection payloads (fake instructions, fabricated section
+    headers, code-fence breakers, or directives that try to override the RARV
+    prompt). This collapses each field to a single safe inline line so it can
+    only ever read as data, never as structure or instructions.
+
+    Steps:
+    - Coerce non-strings to str (a dict/list value would otherwise break
+      formatting or interpolate unexpected structure).
+    - Drop control characters (keeps the visible text intact).
+    - Collapse ALL newlines/tabs/carriage-returns to single spaces so the
+      field cannot add new lines, fake "### " section headers, close the
+      knowledge block, or open/close a markdown code fence.
+    - Defang leading structural markdown markers (#, >, -, *, backticks) so the
+      field cannot pose as a heading/list/fence even after collapsing.
+    - Truncate to a bounded length.
+    """
+    if value is None:
+        return ''
+    if not isinstance(value, str):
+        value = str(value)
+    # Remove control characters first.
+    value = _CONTROL_CHARS_RE.sub('', value)
+    # Collapse any vertical whitespace and tabs into single spaces. This is the
+    # core injection defense: without newlines the field cannot introduce new
+    # markdown structure or standalone instruction lines.
+    value = re.sub(r'[\r\n\t]+', ' ', value)
+    # Collapse runs of spaces left behind.
+    value = re.sub(r' {2,}', ' ', value).strip()
+    # Defang leading structural markdown so the field cannot masquerade as a
+    # heading, blockquote, list item, or code fence.
+    value = re.sub(r'^[#>\-*`]+\s*', '', value)
+    # Defang markdown header markers anywhere in the (now single-line) value so
+    # a collapsed payload cannot reintroduce a "### " section marker inline.
+    # Backticks are neutralized too so a field cannot open/close a code fence.
+    value = re.sub(r'#{1,6}\s', '', value)
+    value = value.replace('`', '')
+    if len(value) > max_chars:
+        value = value[:max_chars].rstrip() + '...'
+    return value
+
+
 def build_rag_context(query, max_tokens=2000, knowledge_dir=None):
     """Build RAG context from the knowledge graph.
 
@@ -39,11 +95,20 @@ def build_rag_context(query, max_tokens=2000, knowledge_dir=None):
     total_chars = 0
 
     for p in patterns:
-        # Support both 'name'/'pattern' and 'description' fields
-        name = p.get('name', p.get('pattern', 'Unknown Pattern'))
-        desc = p.get('description', p.get('correct_approach', ''))
-        category = p.get('category', '')
-        source = p.get('_source_project', '')
+        # Support both 'name'/'pattern' and 'description' fields.
+        # Every field is sanitized: memory entries are untrusted input and must
+        # not be able to inject instructions or break the prompt structure.
+        name = _sanitize_field(p.get('name', p.get('pattern', 'Unknown Pattern')))
+        desc = _sanitize_field(p.get('description', p.get('correct_approach', '')))
+        category = _sanitize_field(p.get('category', ''))
+        source_raw = p.get('_source_project', '')
+        # Path().name strips any directory traversal; sanitize the basename too.
+        source = _sanitize_field(Path(str(source_raw)).name) if source_raw else ''
+
+        # name may be empty after sanitization (e.g. a field that was only
+        # markdown markers); fall back so the heading is never blank.
+        if not name:
+            name = 'Unknown Pattern'
 
         section = '### ' + name
         if category:
@@ -52,7 +117,7 @@ def build_rag_context(query, max_tokens=2000, knowledge_dir=None):
         if desc:
             section += desc + '\n'
         if source:
-            section += '_Source: ' + Path(source).name + '_\n'
+            section += '_Source: ' + source + '_\n'
 
         if total_chars + len(section) > max_chars:
             break