loki-mode 7.81.0 → 7.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/completion-council.sh +88 -27
- package/autonomy/lib/config-map.sh +61 -15
- package/autonomy/loki +172 -27
- package/autonomy/run.sh +34 -13
- package/completions/_loki +63 -3
- package/completions/loki.bash +6 -3
- package/dashboard/__init__.py +1 -1
- package/dashboard/audit.py +36 -10
- package/dashboard/registry.py +67 -24
- package/dashboard/server.py +131 -74
- package/docs/INSTALLATION.md +2 -2
- package/events/bus.py +17 -1
- package/events/emit.sh +15 -1
- package/loki-ts/dist/loki.js +2 -2
- package/lokistore/cloud.py +17 -1
- package/lokistore/factory.py +6 -1
- package/lokistore/local.py +11 -3
- package/mcp/__init__.py +1 -1
- package/mcp/server.py +16 -4
- package/memory/cross_project.py +74 -5
- package/memory/embeddings.py +19 -1
- package/memory/retrieval.py +81 -153
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/providers/codex.sh +9 -4
- package/memory/tree_index.py +0 -499
- package/memory/tree_search.py +0 -305
package/docs/INSTALLATION.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The flagship product of [Autonomi](https://www.autonomi.dev/). Loki Mode is a spec-driven autonomous builder with a built-in trust layer that takes any spec to a deployed product and verifies completion with evidence (quality gates plus a completion council), not just a "done" claim. Complete installation instructions for all platforms and use cases.
|
|
4
4
|
|
|
5
|
-
**Version:** v7.
|
|
5
|
+
**Version:** v7.82.0
|
|
6
6
|
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -395,7 +395,7 @@ provider works inside the container. Provide auth with your Anthropic API key:
|
|
|
395
395
|
# Run Loki Mode in Docker (Claude provider, API-key auth)
|
|
396
396
|
docker run --rm -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \
|
|
397
397
|
-v $(pwd):/workspace -w /workspace \
|
|
398
|
-
asklokesh/loki-mode:7.
|
|
398
|
+
asklokesh/loki-mode:7.82.0 start ./my-spec.md
|
|
399
399
|
```
|
|
400
400
|
|
|
401
401
|
##### docker compose + .env (no host install)
|
package/events/bus.py
CHANGED
|
@@ -5,6 +5,7 @@ Events are written to .loki/events/pending/ and processed by subscribers.
|
|
|
5
5
|
This enables CLI, API, VS Code, and MCP to communicate without shared memory.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
import hashlib
|
|
8
9
|
import json
|
|
9
10
|
import logging
|
|
10
11
|
import os
|
|
@@ -123,8 +124,23 @@ class LokiEvent:
|
|
|
123
124
|
except ValueError:
|
|
124
125
|
event_source = EventSource.CLI
|
|
125
126
|
|
|
127
|
+
# ID: prefer the canonical `id`. Flat events.jsonl lines (run.sh
|
|
128
|
+
# emit_event / emit_event_json) carry NO id, so __post_init__ would mint
|
|
129
|
+
# a fresh random uuid on every parse -- defeating import_from_jsonl()
|
|
130
|
+
# dedup (_processed_ids + pending-file glob) and re-importing every flat
|
|
131
|
+
# line on each call. Derive a DETERMINISTIC id from the record content
|
|
132
|
+
# so repeated imports of the same line are idempotent.
|
|
133
|
+
event_id = data.get('id', '')
|
|
134
|
+
if not event_id:
|
|
135
|
+
digest_src = '%s|%s|%s' % (
|
|
136
|
+
data.get('timestamp', ''),
|
|
137
|
+
raw_type,
|
|
138
|
+
json.dumps(payload, sort_keys=True),
|
|
139
|
+
)
|
|
140
|
+
event_id = hashlib.sha1(digest_src.encode('utf-8')).hexdigest()[:8]
|
|
141
|
+
|
|
126
142
|
return cls(
|
|
127
|
-
id=
|
|
143
|
+
id=event_id,
|
|
128
144
|
type=event_type,
|
|
129
145
|
source=event_source,
|
|
130
146
|
timestamp=data.get('timestamp', ''),
|
package/events/emit.sh
CHANGED
|
@@ -118,8 +118,22 @@ else
|
|
|
118
118
|
fi
|
|
119
119
|
|
|
120
120
|
# JSON escape helper: handles \, ", and control characters including newlines
|
|
121
|
+
#
|
|
122
|
+
# The sed pass escapes the named short forms (\\ \" \t \r \b \f); the first awk
|
|
123
|
+
# pass collapses embedded newlines to \n. Any OTHER C0 control byte
|
|
124
|
+
# (0x01-0x07, 0x0B, 0x0E-0x1F) is invalid raw inside a JSON string and is
|
|
125
|
+
# escaped as \uXXXX by the final awk pass -- otherwise json.loads / JSON.parse
|
|
126
|
+
# reject the line and consumers (dashboard _read_events, learning aggregator)
|
|
127
|
+
# silently drop it. The named short forms are already two-char ASCII by the
|
|
128
|
+
# time the final pass runs, so they are never re-escaped.
|
|
121
129
|
json_escape() {
|
|
122
|
-
printf '%s' "$1"
|
|
130
|
+
printf '%s' "$1" \
|
|
131
|
+
| sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g; s/\r/\\r/g; s//\\b/g; s//\\f/g' \
|
|
132
|
+
| awk '{if(NR>1) printf "\\n"; printf "%s", $0}' \
|
|
133
|
+
| awk 'BEGIN{for(i=1;i<=31;i++)m[sprintf("%c",i)]=sprintf("\\u%04x",i)}
|
|
134
|
+
{s=""; n=length($0);
|
|
135
|
+
for(i=1;i<=n;i++){c=substr($0,i,1); s=s (c in m ? m[c] : c)}
|
|
136
|
+
printf "%s", s}'
|
|
123
137
|
}
|
|
124
138
|
|
|
125
139
|
# Build payload JSON
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var QQ=Object.defineProperty;var ZQ=($)=>$;function zQ($,Q){this[$]=ZQ.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)QQ($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:zQ.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var q$=import.meta.require;var h1={};b(h1,{lokiDir:()=>P,homeLokiDir:()=>i$,findRepoRootForVersion:()=>t$,REPO_ROOT:()=>g});import{resolve as a,dirname as r$}from"path";import{fileURLToPath as XQ}from"url";import{existsSync as R$}from"fs";import{homedir as KQ}from"os";function qQ(){let $=b1;for(let Q=0;Q<6;Q++){if(R$(a($,"VERSION"))&&R$(a($,"autonomy/run.sh")))return $;let Z=r$($);if(Z===$)break;$=Z}return a(b1,"..","..","..")}function t$($){let Q=$;for(let Z=0;Z<6;Z++){if(R$(a(Q,"VERSION"))&&R$(a(Q,"autonomy/run.sh")))return Q;let z=r$(Q);if(z===Q)break;Q=z}return a($,"..","..","..")}function P(){return process.env.LOKI_DIR??a(process.cwd(),".loki")}function i$(){return a(KQ(),".loki")}var b1,g;var C=L(()=>{b1=r$(XQ(import.meta.url));g=qQ()});import{readFileSync as VQ}from"fs";import{resolve as JQ,dirname as UQ}from"path";import{fileURLToPath as WQ}from"url";function E$(){if(Q$!==null)return Q$;let $="7.
|
|
2
|
+
var QQ=Object.defineProperty;var ZQ=($)=>$;function zQ($,Q){this[$]=ZQ.bind(null,Q)}var b=($,Q)=>{for(var Z in Q)QQ($,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:zQ.bind(Q,Z)})};var L=($,Q)=>()=>($&&(Q=$($=0)),Q);var q$=import.meta.require;var h1={};b(h1,{lokiDir:()=>P,homeLokiDir:()=>i$,findRepoRootForVersion:()=>t$,REPO_ROOT:()=>g});import{resolve as a,dirname as r$}from"path";import{fileURLToPath as XQ}from"url";import{existsSync as R$}from"fs";import{homedir as KQ}from"os";function qQ(){let $=b1;for(let Q=0;Q<6;Q++){if(R$(a($,"VERSION"))&&R$(a($,"autonomy/run.sh")))return $;let Z=r$($);if(Z===$)break;$=Z}return a(b1,"..","..","..")}function t$($){let Q=$;for(let Z=0;Z<6;Z++){if(R$(a(Q,"VERSION"))&&R$(a(Q,"autonomy/run.sh")))return Q;let z=r$(Q);if(z===Q)break;Q=z}return a($,"..","..","..")}function P(){return process.env.LOKI_DIR??a(process.cwd(),".loki")}function i$(){return a(KQ(),".loki")}var b1,g;var C=L(()=>{b1=r$(XQ(import.meta.url));g=qQ()});import{readFileSync as VQ}from"fs";import{resolve as JQ,dirname as UQ}from"path";import{fileURLToPath as WQ}from"url";function E$(){if(Q$!==null)return Q$;let $="7.82.0";if(typeof $==="string"&&$.length>0)return Q$=$,Q$;try{let Q=UQ(WQ(import.meta.url)),Z=t$(Q);Q$=VQ(JQ(Z,"VERSION"),"utf-8").trim()}catch{Q$="unknown"}return Q$}var Q$=null;var e$=L(()=>{C()});var g1={};b(g1,{runOrThrow:()=>HQ,run:()=>F,readStreamCapped:()=>m1,commandVersion:()=>BQ,commandExists:()=>f,ShellError:()=>$1,MAX_STDOUT_BYTES:()=>v1});async function m1($,Q=v1){let Z=$.getReader(),z=new TextDecoder,X="",q=0;try{while(q<Q){let{done:K,value:U}=await Z.read();if(K)break;if(!U)continue;if(q+=U.byteLength,q>Q){let J=U.byteLength-(q-Q);X+=z.decode(U.subarray(0,J),{stream:!0});break}X+=z.decode(U,{stream:!0})}X+=z.decode()}finally{try{await Z.cancel()}catch{}Z.releaseLock()}return X}async function F($,Q={}){let Z=Bun.spawn({cmd:[...$],stdout:"pipe",stderr:"pipe",env:Q.env?{...process.env,...Q.env}:process.env,cwd:Q.cwd}),z,X;if(Q.timeoutMs&&Q.timeoutMs>0)z=setTimeout(()=>{try{Z.kill("SIGTERM")}catch{}X=setTimeout(()=>{try{Z.kill("SIGKILL")}catch{}},2000)},Q.timeoutMs);try{let[q,K,U]=await Promise.all([m1(Z.stdout),new Response(Z.stderr).text(),Z.exited]);return{stdout:q,stderr:K,exitCode:U}}finally{if(z)clearTimeout(z);if(X)clearTimeout(X)}}async function HQ($,Q={}){let Z=await F($,Q);if(Z.exitCode!==0)throw new $1(`command failed (${Z.exitCode}): ${$.join(" ")}`,Z.exitCode,Z.stdout,Z.stderr);return Z}async function f($){let Q=GQ($),Z=await F(["sh","-c",`command -v ${Q}`],{timeoutMs:5000});if(Z.exitCode===0)return Z.stdout.trim()||null;return null}function GQ($){if(!/^[A-Za-z0-9._/-]+$/.test($))throw Error(`refused to shell-escape suspect token: ${$}`);return $}async function BQ($,Q="--version"){if(!await f($))return null;let z=await F([$,Q],{timeoutMs:5000});if(z.exitCode!==0)return null;return((z.stdout||z.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var v1=16777216,$1;var d=L(()=>{$1=class $1 extends Error{message;exitCode;stdout;stderr;constructor($,Q,Z,z){super($);this.message=$;this.exitCode=Q;this.stdout=Z;this.stderr=z;this.name="ShellError"}}});function s($){return YQ?"":$}var YQ,O,S,_,_Z,I,k,h,V;var c=L(()=>{YQ=(process.env.NO_COLOR??"").length>0;O=s("\x1B[0;31m"),S=s("\x1B[0;32m"),_=s("\x1B[1;33m"),_Z=s("\x1B[0;34m"),I=s("\x1B[0;36m"),k=s("\x1B[1m"),h=s("\x1B[2m"),V=s("\x1B[0m")});import{existsSync as jQ}from"fs";async function Z$(){if(Y$!==void 0)return Y$;let $="/opt/homebrew/bin/python3.12";if(jQ($))return Y$=$,$;let Q=await f("python3.12");if(Q)return Y$=Q,Q;let Z=await f("python3");return Y$=Z,Z}async function z$($,Q={}){let Z=await Z$();if(!Z)return{stdout:"",stderr:"python3 not found",exitCode:127};return F([Z,"-c",$],Q)}var Y$;var V$=L(()=>{d()});var X0={};b(X0,{runStatus:()=>oQ});import{existsSync as y,readFileSync as U$,readdirSync as r1,statSync as t1}from"fs";import{resolve as D,basename as vQ}from"path";import{homedir as mQ}from"os";function i1($){let Q=Math.trunc($);if(Q>=1e6)return`${(Math.trunc(Q/1e6*10)/10).toFixed(1)}M`;if(Q>=1000)return`${(Math.trunc(Q/1000*10)/10).toFixed(1)}K`;return String(Q)}function e1($,Q,Z){if(Q===0)return null;let z=Math.trunc($*100/Q),X=Math.trunc($*N$/Q);if(X>N$)X=N$;let q=N$-X,K=S;if(z>=80)K=O;else if(z>=50)K=_;let U="=".repeat(Math.max(0,X))+" ".repeat(Math.max(0,q)),J=i1($),W=i1(Q);return` ${k}${Z}${V} ${K}[${U}]${V} ${z}% (${J} / ${W})`}async function fQ(){if(await f("jq"))return!0;return process.stdout.write(`${O}Error: jq is required but not installed.${V}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -793,4 +793,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
793
793
|
`),2}default:return process.stderr.write(`Unknown command: ${Q}
|
|
794
794
|
`),process.stderr.write($Q),2}}s1();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var qZ=await KZ(Bun.argv.slice(2));process.exit(qZ);
|
|
795
795
|
|
|
796
|
-
//# debugId=
|
|
796
|
+
//# debugId=3E8A7AD89D7DCCD164756E2164756E21
|
package/lokistore/cloud.py
CHANGED
|
@@ -63,6 +63,7 @@ class S3Store(LokiStore):
|
|
|
63
63
|
bucket: str,
|
|
64
64
|
prefix: Optional[str] = None,
|
|
65
65
|
region: Optional[str] = None,
|
|
66
|
+
endpoint: Optional[str] = None,
|
|
66
67
|
):
|
|
67
68
|
if not bucket:
|
|
68
69
|
raise ValueError("S3Store requires a bucket name")
|
|
@@ -78,7 +79,22 @@ class S3Store(LokiStore):
|
|
|
78
79
|
self._prefix = _normalize_prefix(prefix)
|
|
79
80
|
# boto3 picks up credentials from its default chain (env, shared
|
|
80
81
|
# config, instance/role metadata). region_name is optional.
|
|
81
|
-
|
|
82
|
+
# endpoint_url targets an S3-COMPATIBLE store (MinIO, Ceph, R2,
|
|
83
|
+
# Wasabi, ...) instead of real AWS S3; without it the docstring's
|
|
84
|
+
# "(or S3-compatible)" claim was false. Read from the arg or the
|
|
85
|
+
# AWS_ENDPOINT_URL env (boto3's own convention) as a fallback.
|
|
86
|
+
kwargs = {}
|
|
87
|
+
endpoint = endpoint or os.environ.get("AWS_ENDPOINT_URL")
|
|
88
|
+
if endpoint:
|
|
89
|
+
kwargs["endpoint_url"] = endpoint
|
|
90
|
+
if region:
|
|
91
|
+
kwargs["region_name"] = region
|
|
92
|
+
elif endpoint:
|
|
93
|
+
# A custom S3-compatible endpoint (MinIO/Ceph/...) still needs a
|
|
94
|
+
# region for SigV4 request signing even though the store ignores it.
|
|
95
|
+
# Default to us-east-1 (the conventional MinIO default) so a MinIO
|
|
96
|
+
# config without an explicit region does not fail signing.
|
|
97
|
+
kwargs["region_name"] = "us-east-1"
|
|
82
98
|
self._client = boto3.client("s3", **kwargs)
|
|
83
99
|
|
|
84
100
|
def _object_key(self, key: str) -> str:
|
package/lokistore/factory.py
CHANGED
|
@@ -60,6 +60,9 @@ def _config_from_env() -> Dict[str, Any]:
|
|
|
60
60
|
region = os.environ.get("LOKI_STORAGE_REGION")
|
|
61
61
|
if region:
|
|
62
62
|
cfg["region"] = region
|
|
63
|
+
endpoint = os.environ.get("LOKI_STORAGE_ENDPOINT")
|
|
64
|
+
if endpoint:
|
|
65
|
+
cfg["endpoint"] = endpoint
|
|
63
66
|
return cfg
|
|
64
67
|
|
|
65
68
|
|
|
@@ -75,6 +78,7 @@ def build_store(config: Optional[Dict[str, Any]] = None) -> LokiStore:
|
|
|
75
78
|
bucket : bucket/container name (cloud backends)
|
|
76
79
|
prefix : key prefix within the bucket (optional)
|
|
77
80
|
region : region (s3, optional)
|
|
81
|
+
endpoint: custom S3-compatible endpoint URL (s3 only; e.g. MinIO/Ceph/R2)
|
|
78
82
|
base_dir: local base directory (local backend only; overrides resolution)
|
|
79
83
|
"""
|
|
80
84
|
config = dict(config or {})
|
|
@@ -86,11 +90,12 @@ def build_store(config: Optional[Dict[str, Any]] = None) -> LokiStore:
|
|
|
86
90
|
bucket = config.get("bucket")
|
|
87
91
|
prefix = config.get("prefix")
|
|
88
92
|
region = config.get("region")
|
|
93
|
+
endpoint = config.get("endpoint")
|
|
89
94
|
|
|
90
95
|
if backend in ("s3", "aws", "aws-s3"):
|
|
91
96
|
from .cloud import S3Store
|
|
92
97
|
|
|
93
|
-
return S3Store(bucket=bucket, prefix=prefix, region=region)
|
|
98
|
+
return S3Store(bucket=bucket, prefix=prefix, region=region, endpoint=endpoint)
|
|
94
99
|
|
|
95
100
|
if backend in ("gcs", "gcp", "google", "google-cloud-storage"):
|
|
96
101
|
from .cloud import GCSStore
|
package/lokistore/local.py
CHANGED
|
@@ -178,16 +178,24 @@ class LocalStore(LokiStore):
|
|
|
178
178
|
def list(self, prefix: str = "") -> List[str]:
|
|
179
179
|
# Normalize the prefix to a path under the base. An empty prefix lists
|
|
180
180
|
# everything under the base.
|
|
181
|
+
#
|
|
182
|
+
# Walk the realpath of the base consistently with how relpath is taken
|
|
183
|
+
# below. If the base is reached through a symlink (macOS /tmp ->
|
|
184
|
+
# /private/tmp, k8s bind mounts, a symlinked LOKI_DIR), walking the
|
|
185
|
+
# symlink path while computing relpath against the realpath would emit
|
|
186
|
+
# "../../.." garbage keys that downstream sync then rejects as path
|
|
187
|
+
# traversal -- silently breaking the whole object-store sync.
|
|
188
|
+
real_base = os.path.realpath(self._base)
|
|
181
189
|
if prefix:
|
|
182
190
|
clean_prefix = normalize_key(prefix)
|
|
183
|
-
search_root =
|
|
191
|
+
search_root = Path(real_base) / clean_prefix
|
|
184
192
|
else:
|
|
185
|
-
search_root =
|
|
193
|
+
search_root = Path(real_base)
|
|
186
194
|
|
|
187
195
|
if not search_root.exists():
|
|
188
196
|
return []
|
|
189
197
|
|
|
190
|
-
base_str =
|
|
198
|
+
base_str = real_base
|
|
191
199
|
results: List[str] = []
|
|
192
200
|
|
|
193
201
|
if search_root.is_file():
|
package/mcp/__init__.py
CHANGED
package/mcp/server.py
CHANGED
|
@@ -1919,20 +1919,32 @@ async def mem_search(
|
|
|
1919
1919
|
context = {"goal": query, "task_type": "exploration"}
|
|
1920
1920
|
results = retriever.retrieve_task_aware(context, top_k=limit)
|
|
1921
1921
|
|
|
1922
|
-
#
|
|
1923
|
-
#
|
|
1924
|
-
#
|
|
1922
|
+
# Filter results by collection parameter when not "all".
|
|
1923
|
+
# retrieve_task_aware tags every item with _source in
|
|
1924
|
+
# {episodic, semantic, skills, anti_patterns}; it does NOT emit
|
|
1925
|
+
# _type/type. Map _source to the public collection vocabulary so the
|
|
1926
|
+
# filter actually matches (previously every result was dropped because
|
|
1927
|
+
# result_type defaulted to "unknown").
|
|
1925
1928
|
collection_type_map = {
|
|
1926
1929
|
"episodes": "episode",
|
|
1927
1930
|
"patterns": "pattern",
|
|
1928
1931
|
"skills": "skill",
|
|
1929
1932
|
}
|
|
1930
1933
|
filter_type = collection_type_map.get(collection)
|
|
1934
|
+
source_to_type = {
|
|
1935
|
+
"episodic": "episode",
|
|
1936
|
+
"semantic": "pattern",
|
|
1937
|
+
"skills": "skill",
|
|
1938
|
+
"anti_patterns": "pattern",
|
|
1939
|
+
}
|
|
1931
1940
|
|
|
1932
1941
|
# Compact results for token efficiency
|
|
1933
1942
|
compact = []
|
|
1934
1943
|
for r in results:
|
|
1935
|
-
result_type =
|
|
1944
|
+
result_type = source_to_type.get(
|
|
1945
|
+
r.get("_source"),
|
|
1946
|
+
r.get("_type", r.get("type", "unknown")),
|
|
1947
|
+
)
|
|
1936
1948
|
# Apply collection filter
|
|
1937
1949
|
if filter_type and result_type != filter_type:
|
|
1938
1950
|
continue
|
package/memory/cross_project.py
CHANGED
|
@@ -6,6 +6,7 @@ builds a unified index with memory statistics per project.
|
|
|
6
6
|
|
|
7
7
|
import json
|
|
8
8
|
import os
|
|
9
|
+
import tempfile
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
from datetime import datetime, timezone
|
|
11
12
|
|
|
@@ -49,6 +50,55 @@ class CrossProjectIndex:
|
|
|
49
50
|
})
|
|
50
51
|
return projects
|
|
51
52
|
|
|
53
|
+
@staticmethod
|
|
54
|
+
def _count_episodes(episodic_dir):
|
|
55
|
+
"""Count episode files in an episodic store.
|
|
56
|
+
|
|
57
|
+
Episodes are persisted by the storage layer under date subdirectories
|
|
58
|
+
(episodic/{YYYY-MM-DD}/task-*.json), not directly under episodic/. A
|
|
59
|
+
non-recursive episodic/*.json glob therefore reported 0 for every real
|
|
60
|
+
store. Walk recursively and skip the per-directory index.json sidecars.
|
|
61
|
+
"""
|
|
62
|
+
if not episodic_dir.exists():
|
|
63
|
+
return 0
|
|
64
|
+
return sum(
|
|
65
|
+
1
|
|
66
|
+
for f in episodic_dir.rglob('*.json')
|
|
67
|
+
if f.name != 'index.json'
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def _count_patterns(semantic_dir):
|
|
72
|
+
"""Count semantic patterns in a semantic store.
|
|
73
|
+
|
|
74
|
+
Production stores all patterns as a list inside a single
|
|
75
|
+
semantic/patterns.json, so a semantic/*.json file glob counted 1 (the
|
|
76
|
+
container) regardless of how many patterns it held. Count the entries
|
|
77
|
+
in patterns.json when present, and add any other *.json pattern files
|
|
78
|
+
(legacy one-file-per-pattern layout) so both shapes are counted.
|
|
79
|
+
"""
|
|
80
|
+
if not semantic_dir.exists():
|
|
81
|
+
return 0
|
|
82
|
+
count = 0
|
|
83
|
+
patterns_file = semantic_dir / 'patterns.json'
|
|
84
|
+
if patterns_file.exists():
|
|
85
|
+
try:
|
|
86
|
+
data = json.loads(patterns_file.read_text())
|
|
87
|
+
if isinstance(data, dict):
|
|
88
|
+
patterns = data.get('patterns', [])
|
|
89
|
+
if isinstance(patterns, list):
|
|
90
|
+
count += len(patterns)
|
|
91
|
+
elif isinstance(data, list):
|
|
92
|
+
count += len(data)
|
|
93
|
+
except (json.JSONDecodeError, OSError, ValueError):
|
|
94
|
+
pass
|
|
95
|
+
# Count legacy per-pattern files, excluding known container files.
|
|
96
|
+
for f in semantic_dir.glob('*.json'):
|
|
97
|
+
if f.name in ('patterns.json', 'anti-patterns.json'):
|
|
98
|
+
continue
|
|
99
|
+
count += 1
|
|
100
|
+
return count
|
|
101
|
+
|
|
52
102
|
def build_index(self):
|
|
53
103
|
"""Build a cross-project index with memory statistics.
|
|
54
104
|
|
|
@@ -70,8 +120,8 @@ class CrossProjectIndex:
|
|
|
70
120
|
semantic_dir = memory_dir / 'semantic'
|
|
71
121
|
skills_dir = memory_dir / 'skills'
|
|
72
122
|
|
|
73
|
-
episodic_count =
|
|
74
|
-
semantic_count =
|
|
123
|
+
episodic_count = self._count_episodes(episodic_dir)
|
|
124
|
+
semantic_count = self._count_patterns(semantic_dir)
|
|
75
125
|
skills_count = len(list(skills_dir.glob('*.json'))) if skills_dir.exists() else 0
|
|
76
126
|
|
|
77
127
|
project['episodic_count'] = episodic_count
|
|
@@ -86,12 +136,31 @@ class CrossProjectIndex:
|
|
|
86
136
|
return index
|
|
87
137
|
|
|
88
138
|
def save_index(self):
|
|
89
|
-
"""Save index to disk.
|
|
139
|
+
"""Save index to disk atomically.
|
|
140
|
+
|
|
141
|
+
Writes to a temp file in the destination directory then os.replace()s
|
|
142
|
+
it over the target, so a crash or a concurrent load_index() never sees
|
|
143
|
+
a truncated/torn file (the previous direct open('w') truncated in place
|
|
144
|
+
before the new bytes landed).
|
|
145
|
+
"""
|
|
90
146
|
if self._index is None:
|
|
91
147
|
return
|
|
92
148
|
self.index_file.parent.mkdir(parents=True, exist_ok=True)
|
|
93
|
-
|
|
94
|
-
|
|
149
|
+
tmp_fd, tmp_path = tempfile.mkstemp(
|
|
150
|
+
dir=str(self.index_file.parent), suffix='.tmp'
|
|
151
|
+
)
|
|
152
|
+
try:
|
|
153
|
+
with os.fdopen(tmp_fd, 'w') as f:
|
|
154
|
+
json.dump(self._index, f, indent=2)
|
|
155
|
+
f.flush()
|
|
156
|
+
os.fsync(f.fileno())
|
|
157
|
+
os.replace(tmp_path, str(self.index_file))
|
|
158
|
+
except BaseException:
|
|
159
|
+
try:
|
|
160
|
+
os.unlink(tmp_path)
|
|
161
|
+
except OSError:
|
|
162
|
+
pass
|
|
163
|
+
raise
|
|
95
164
|
|
|
96
165
|
def load_index(self):
|
|
97
166
|
"""Load index from disk."""
|
package/memory/embeddings.py
CHANGED
|
@@ -1008,8 +1008,26 @@ class EmbeddingEngine:
|
|
|
1008
1008
|
logger.warning("Primary provider failed: %s, trying fallback", e)
|
|
1009
1009
|
old_dimension = self.dimension
|
|
1010
1010
|
self._use_fallback()
|
|
1011
|
-
|
|
1011
|
+
# Run the fallback through the SAME chunk + weighted-average path as
|
|
1012
|
+
# the success branch so the vector is computed consistently (the old
|
|
1013
|
+
# code embedded the raw, un-chunked text, producing a different vector
|
|
1014
|
+
# for multi-chunk inputs).
|
|
1015
|
+
if len(chunks) == 1:
|
|
1016
|
+
embedding = self._primary_provider.embed(chunks[0])
|
|
1017
|
+
else:
|
|
1018
|
+
chunk_embeddings = self._primary_provider.embed_batch(chunks)
|
|
1019
|
+
weights = np.array([len(c) for c in chunks], dtype=np.float32)
|
|
1020
|
+
weights = weights / weights.sum()
|
|
1021
|
+
embedding = np.average(chunk_embeddings, axis=0, weights=weights)
|
|
1012
1022
|
embedding = self._normalize(embedding)
|
|
1023
|
+
# The cache key computed above used the pre-fallback provider name.
|
|
1024
|
+
# _use_fallback() switched the provider, so recompute the key to
|
|
1025
|
+
# reflect the provider that actually produced this vector. Without
|
|
1026
|
+
# this, the entry is stored under the old key while the next call
|
|
1027
|
+
# looks it up under the new provider key, causing a permanent cache
|
|
1028
|
+
# miss (and re-embedding) on every fallback request.
|
|
1029
|
+
if self.config.cache_enabled:
|
|
1030
|
+
cache_key = self._get_cache_key(text)
|
|
1013
1031
|
# If dimension changed after fallback, log a warning so callers
|
|
1014
1032
|
# know existing vector indices may be incompatible (BUG-MEM-006).
|
|
1015
1033
|
if self.dimension != old_dimension:
|