ltcai 6.3.0 → 6.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,28 +6,30 @@ import { ActionButton, DataPanel, EntityList, OperationResult, StructuredView, T
6
6
  import { Button } from "@/components/ui/button";
7
7
  import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
8
8
  import { Input } from "@/components/ui/input";
9
+ import { t, type Language } from "@/i18n";
9
10
  import { asArray } from "@/lib/utils";
11
+ import { useAppStore } from "@/store/appStore";
10
12
 
11
13
  type CaptureTab = "files" | "local" | "browser" | "pipeline";
12
14
 
13
- const tabs: Array<{ id: CaptureTab; label: string }> = [
14
- { id: "files", label: "Files" },
15
- { id: "local", label: "Folders" },
16
- { id: "browser", label: "Web" },
17
- { id: "pipeline", label: "Flow" },
18
- ];
19
-
20
15
  export function CapturePage({ initialTab }: { initialTab?: string }) {
16
+ const language = useAppStore((state) => state.language);
21
17
  const [tab, setTab] = React.useState<CaptureTab>((initialTab as CaptureTab) || "files");
18
+ const tabs: Array<{ id: CaptureTab; label: string }> = [
19
+ { id: "files", label: t(language, "capture.tab.files") },
20
+ { id: "local", label: t(language, "capture.tab.local") },
21
+ { id: "browser", label: t(language, "capture.tab.browser") },
22
+ { id: "pipeline", label: t(language, "capture.tab.pipeline") },
23
+ ];
22
24
  React.useEffect(() => {
23
25
  if (initialTab === "pipeline" || initialTab === "local" || initialTab === "files") setTab(initialTab);
24
26
  }, [initialTab]);
25
27
  return (
26
28
  <div className="space-y-5">
27
29
  <header className="page-hero">
28
- <div className="page-kicker"><Upload className="h-4 w-4" /> Add</div>
29
- <h1 className="page-title">Feed the brain what matters.</h1>
30
- <p className="page-copy">Drop in files, connect folders, or save a page. Lattice remembers the origin of every idea it learns.</p>
30
+ <div className="page-kicker"><Upload className="h-4 w-4" /> {t(language, "capture.kicker")}</div>
31
+ <h1 className="page-title">{t(language, "capture.title")}</h1>
32
+ <p className="page-copy">{t(language, "capture.body")}</p>
31
33
  </header>
32
34
  <Tabs tabs={tabs} value={tab} onChange={(id) => setTab(id as CaptureTab)} />
33
35
  {tab === "files" ? <FilesPanel /> : null}
@@ -39,6 +41,7 @@ export function CapturePage({ initialTab }: { initialTab?: string }) {
39
41
  }
40
42
 
41
43
  function FilesPanel() {
44
+ const language = useAppStore((state) => state.language);
42
45
  const qc = useQueryClient();
43
46
  const docs = useQuery({ queryKey: ["documents"], queryFn: () => latticeApi.documents(200) });
44
47
  const [queue, setQueue] = React.useState<UploadQueueItem[]>([]);
@@ -59,8 +62,8 @@ function FilesPanel() {
59
62
  <div className="grid gap-4 xl:grid-cols-[0.75fr_1.25fr]">
60
63
  <Card>
61
64
  <CardHeader>
62
- <CardTitle className="flex items-center gap-2"><Upload className="h-4 w-4" /> Add documents</CardTitle>
63
- <CardDescription>Choose files and Lattice will prepare them for search and memory.</CardDescription>
65
+ <CardTitle className="flex items-center gap-2"><Upload className="h-4 w-4" /> {t(language, "capture.files.title")}</CardTitle>
66
+ <CardDescription>{t(language, "capture.files.description")}</CardDescription>
64
67
  </CardHeader>
65
68
  <CardContent>
66
69
  <label
@@ -72,19 +75,19 @@ function FilesPanel() {
72
75
  }}
73
76
  >
74
77
  <Upload className="h-7 w-7 text-primary" />
75
- <span className="text-lg font-semibold">Drop files or choose documents</span>
76
- <span className="max-w-sm text-sm leading-6 text-muted-foreground">Each file is queued, parsed, written to Memory, and linked into the Brain graph with source metadata.</span>
78
+ <span className="text-lg font-semibold">{t(language, "capture.files.drop")}</span>
79
+ <span className="max-w-sm text-sm leading-6 text-muted-foreground">{t(language, "capture.files.dropDetail")}</span>
77
80
  <input type="file" multiple className="sr-only" onChange={(e) => e.target.files && beginUpload(e.target.files)} />
78
81
  </label>
79
82
  <DocumentUploadQueue queue={queue} onRetry={(file) => beginUpload([file])} />
80
83
  </CardContent>
81
84
  </Card>
82
- <DataPanel title="Uploaded documents" result={docs.data}>
85
+ <DataPanel title={t(language, "capture.files.uploaded")} result={docs.data}>
83
86
  {(data) => (
84
87
  <div className="space-y-3">
85
88
  <EntityList items={(data as Record<string, unknown>).documents || data} titleKey="filename" metaKey="ingest_state" limit={12} />
86
89
  <div className="rounded-md border border-border bg-background/55 p-3 text-sm text-muted-foreground">
87
- Completed uploads appear here after they enter Memory. Graph links may continue building briefly after parsing finishes.
90
+ {t(language, "capture.files.completed")}
88
91
  </div>
89
92
  </div>
90
93
  )}
@@ -122,11 +125,12 @@ async function uploadFiles(files: File[], setQueue: React.Dispatch<React.SetStat
122
125
  }
123
126
 
124
127
  function DocumentUploadQueue({ queue, onRetry }: { queue: UploadQueueItem[]; onRetry: (file: File) => void }) {
128
+ const language = useAppStore((state) => state.language);
125
129
  if (!queue.length) return null;
126
130
  return (
127
131
  <div className="mt-4 space-y-2">
128
132
  {queue.map((item) => {
129
- const detail = uploadResultDetail(item);
133
+ const detail = uploadResultDetail(item, language);
130
134
  return (
131
135
  <div key={item.id} className="rounded-md border border-border bg-background/55 p-3 text-sm">
132
136
  <div className="flex flex-wrap items-start justify-between gap-3">
@@ -141,11 +145,11 @@ function DocumentUploadQueue({ queue, onRetry }: { queue: UploadQueueItem[]; onR
141
145
  </div>
142
146
  {item.status === "failed" ? (
143
147
  <Button size="sm" variant="outline" onClick={() => onRetry(item.file)}>
144
- <RotateCcw className="h-3.5 w-3.5" /> Retry
148
+ <RotateCcw className="h-3.5 w-3.5" /> {t(language, "capture.retry")}
145
149
  </Button>
146
150
  ) : null}
147
151
  </div>
148
- {item.result ? <OperationResult result={item.result} successLabel="Entered Brain and graph queue" /> : null}
152
+ {item.result ? <OperationResult result={item.result} successLabel={t(language, "capture.files.success")} /> : null}
149
153
  </div>
150
154
  );
151
155
  })}
@@ -153,16 +157,17 @@ function DocumentUploadQueue({ queue, onRetry }: { queue: UploadQueueItem[]; onR
153
157
  );
154
158
  }
155
159
 
156
- function uploadResultDetail(item: UploadQueueItem) {
157
- if (item.status === "queued") return "Waiting in the ingest queue";
158
- if (item.status === "uploading") return "Parsing and sending to Memory";
159
- if (!item.result?.ok) return item.result?.error || "Ingest failed before it entered the Brain";
160
+ function uploadResultDetail(item: UploadQueueItem, language: Language) {
161
+ if (item.status === "queued") return t(language, "capture.files.queued");
162
+ if (item.status === "uploading") return t(language, "capture.files.uploading");
163
+ if (!item.result?.ok) return item.result?.error || t(language, "capture.files.failed");
160
164
  const data = item.result.data || {};
161
165
  const node = String(data.node_id || data.graph_node || data.provenance_id || "");
162
- return node ? `Captured with source metadata · ${node}` : "Captured with source metadata";
166
+ return node ? t(language, "capture.files.capturedWithNode", { node }) : t(language, "capture.files.captured");
163
167
  }
164
168
 
165
169
  function LocalPanel() {
170
+ const language = useAppStore((state) => state.language);
166
171
  const qc = useQueryClient();
167
172
  const [path, setPath] = React.useState("");
168
173
  const local = useQuery({ queryKey: ["localSources"], queryFn: latticeApi.localSources });
@@ -175,16 +180,16 @@ function LocalPanel() {
175
180
  <div className="grid gap-4 xl:grid-cols-[0.9fr_1.1fr]">
176
181
  <Card>
177
182
  <CardHeader>
178
- <CardTitle className="flex items-center gap-2"><FolderPlus className="h-4 w-4" /> Connect a folder</CardTitle>
179
- <CardDescription>Point Lattice at a folder you want it to remember.</CardDescription>
183
+ <CardTitle className="flex items-center gap-2"><FolderPlus className="h-4 w-4" /> {t(language, "capture.local.title")}</CardTitle>
184
+ <CardDescription>{t(language, "capture.local.description")}</CardDescription>
180
185
  </CardHeader>
181
186
  <CardContent className="space-y-3">
182
- <Input value={path} onChange={(e) => setPath(e.target.value)} placeholder="Folder path on this Mac" />
183
- <Button disabled={!path.trim() || connect.isPending} onClick={() => connect.mutate()}>Connect Folder</Button>
184
- {connect.data ? <OperationResult result={connect.data} successLabel="Folder connection requested" /> : null}
187
+ <Input value={path} onChange={(e) => setPath(e.target.value)} placeholder={t(language, "capture.local.placeholder")} />
188
+ <Button disabled={!path.trim() || connect.isPending} onClick={() => connect.mutate()}>{t(language, "capture.local.connect")}</Button>
189
+ {connect.data ? <OperationResult result={connect.data} successLabel={t(language, "capture.local.success")} /> : null}
185
190
  </CardContent>
186
191
  </Card>
187
- <DataPanel title="Connected sources" result={local.data}>
192
+ <DataPanel title={t(language, "capture.local.sources")} result={local.data}>
188
193
  {(data) => (
189
194
  <div className="space-y-3">
190
195
  <EntityList items={(data as Record<string, unknown>).sources} titleKey="path" metaKey="status" />
@@ -199,7 +204,7 @@ function LocalPanel() {
199
204
  </div>
200
205
  )}
201
206
  </DataPanel>
202
- <DataPanel title="Folder access" result={agent.data} className="xl:col-span-2">
207
+ <DataPanel title={t(language, "capture.local.access")} result={agent.data} className="xl:col-span-2">
203
208
  {(data) => <StructuredView value={data} />}
204
209
  </DataPanel>
205
210
  </div>
@@ -207,43 +212,45 @@ function LocalPanel() {
207
212
  }
208
213
 
209
214
  function BrowserPanel() {
215
+ const language = useAppStore((state) => state.language);
210
216
  const [url, setUrl] = React.useState("");
211
217
  const read = useMutation({ mutationFn: () => latticeApi.browserReadUrl(url) });
212
218
  return (
213
219
  <Card>
214
220
  <CardHeader>
215
- <CardTitle className="flex items-center gap-2"><Globe2 className="h-4 w-4" /> Save a web page</CardTitle>
216
- <CardDescription>Capture a page so Lattice can remember the useful parts.</CardDescription>
221
+ <CardTitle className="flex items-center gap-2"><Globe2 className="h-4 w-4" /> {t(language, "capture.browser.title")}</CardTitle>
222
+ <CardDescription>{t(language, "capture.browser.description")}</CardDescription>
217
223
  </CardHeader>
218
224
  <CardContent className="space-y-3">
219
225
  <div className="flex flex-col gap-2 sm:flex-row">
220
- <Input value={url} onChange={(e) => setUrl(e.target.value)} placeholder="https://example.com/article" />
221
- <Button disabled={!url.trim() || read.isPending} onClick={() => read.mutate()}>Capture URL</Button>
226
+ <Input value={url} onChange={(e) => setUrl(e.target.value)} placeholder={t(language, "capture.browser.placeholder")} />
227
+ <Button disabled={!url.trim() || read.isPending} onClick={() => read.mutate()}>{t(language, "capture.browser.capture")}</Button>
222
228
  </div>
223
- {read.data ? <OperationResult result={read.data} successLabel="URL capture requested" /> : null}
229
+ {read.data ? <OperationResult result={read.data} successLabel={t(language, "capture.browser.success")} /> : null}
224
230
  </CardContent>
225
231
  </Card>
226
232
  );
227
233
  }
228
234
 
229
235
  function PipelinePanel() {
236
+ const language = useAppStore((state) => state.language);
230
237
  const index = useQuery({ queryKey: ["index"], queryFn: latticeApi.indexStatus });
231
238
  const stats = useQuery({ queryKey: ["graphStats"], queryFn: latticeApi.graphStats });
232
239
  return (
233
240
  <div className="grid gap-4 xl:grid-cols-2">
234
- <DataPanel title="Processing status" result={index.data}>
241
+ <DataPanel title={t(language, "capture.pipeline.status")} result={index.data}>
235
242
  {(data) => <StructuredView value={data} />}
236
243
  </DataPanel>
237
- <DataPanel title="Brain growth" result={stats.data}>
244
+ <DataPanel title={t(language, "capture.pipeline.growth")} result={stats.data}>
238
245
  {(data) => <StructuredView value={data} />}
239
246
  </DataPanel>
240
247
  <Card className="xl:col-span-2">
241
248
  <CardHeader>
242
- <CardTitle className="flex items-center gap-2"><HardDrive className="h-4 w-4" /> Refresh memory</CardTitle>
243
- <CardDescription>Refresh search when you want Lattice to re-check captured material.</CardDescription>
249
+ <CardTitle className="flex items-center gap-2"><HardDrive className="h-4 w-4" /> {t(language, "capture.pipeline.refresh")}</CardTitle>
250
+ <CardDescription>{t(language, "capture.pipeline.description")}</CardDescription>
244
251
  </CardHeader>
245
252
  <CardContent>
246
- <ActionButton label="Rebuild retrieval index" action={() => latticeApi.rebuildIndex()} invalidate={["index"]} />
253
+ <ActionButton label={t(language, "capture.pipeline.rebuild")} action={() => latticeApi.rebuildIndex()} invalidate={["index"]} />
247
254
  </CardContent>
248
255
  </Card>
249
256
  </div>
@@ -26,7 +26,7 @@ from .storage import (
26
26
  storage_from_env,
27
27
  )
28
28
 
29
- __version__ = "6.3.0"
29
+ __version__ = "6.4.0"
30
30
 
31
31
  __all__ = [
32
32
  "AgentRuntime",
@@ -45,6 +45,7 @@ __all__ = [
45
45
  "IngestionPipeline",
46
46
  "KGPortabilityService",
47
47
  "KnowledgeGraphStore",
48
+ "LatticeBrainQuality",
48
49
  "MultiAgentOrchestrator",
49
50
  "PostgresConfig",
50
51
  "PostgresEngine",
@@ -65,6 +66,7 @@ _LAZY = {
65
66
  "ConversationStore": ("conversations", "ConversationStore"),
66
67
  "BrainMemory": ("memory", "BrainMemory"),
67
68
  "KnowledgeGraphStore": ("graph.store", "KnowledgeGraphStore"),
69
+ "LatticeBrainQuality": ("quality", "LatticeBrainQuality"),
68
70
  "IngestionItem": ("ingestion", "IngestionItem"),
69
71
  "IngestionPipeline": ("ingestion", "IngestionPipeline"),
70
72
  "KGPortabilityService": ("portability", "KGPortabilityService"),
@@ -0,0 +1,409 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ lattice_brain/quality.py
4
+ Pure Python Quality Layer for Lattice Brain (v6.4+ hardening)
5
+ - Does not modify any DB schema or existing APIs
6
+ - Self-contained, dependency-free core (stdlib only)
7
+ """
8
+
9
+ from __future__ import annotations
10
+ import math
11
+ import time
12
+ import hashlib
13
+ from dataclasses import dataclass, field
14
+ from typing import Any, Dict, List, Optional
15
+ from collections import defaultdict
16
+ import re
17
+
18
+ # -----------------------------
19
+ # 1. Embedding Fallback Labelling + Drift/Reindex Plan
20
+ # -----------------------------
21
+ @dataclass
22
+ class EmbeddingLabel:
23
+ vector_id: str
24
+ label: str
25
+ confidence: float
26
+ drift_score: float = 0.0
27
+ needs_reindex: bool = False
28
+
29
+ class EmbeddingFallbackLabeller:
30
+ """Fallback labelling when primary embedding unavailable + drift detection"""
31
+
32
+ def __init__(self, drift_threshold: float = 0.15):
33
+ self.drift_threshold = drift_threshold
34
+ self.label_cache: Dict[str, EmbeddingLabel] = {}
35
+ self.vector_cache: Dict[str, List[float]] = {}
36
+
37
+ def label(self, vector_id: str, embedding: Optional[List[float]] = None,
38
+ metadata: Optional[Dict] = None) -> EmbeddingLabel:
39
+ if embedding is None:
40
+ label = "unembedded_fallback"
41
+ conf = 0.3
42
+ else:
43
+ # Simple hash-based pseudo-label for determinism
44
+ h = hashlib.md5(str(embedding[:4]).encode()).hexdigest()[:8]
45
+ label = f"emb_cluster_{h}"
46
+ conf = 0.85
47
+
48
+ drift = self._compute_drift(vector_id, embedding)
49
+ needs_reindex = drift > self.drift_threshold
50
+
51
+ el = EmbeddingLabel(vector_id, label, conf, drift, needs_reindex)
52
+ self.label_cache[vector_id] = el
53
+ if embedding is not None:
54
+ self.vector_cache[vector_id] = [float(value) for value in embedding]
55
+ return el
56
+
57
+ def _compute_drift(self, vector_id: str, new_emb: Optional[List[float]]) -> float:
58
+ old = self.vector_cache.get(vector_id)
59
+ if old is None or new_emb is None:
60
+ return 0.0
61
+ return self._cosine_distance(old, new_emb)
62
+
63
+ def _cosine_distance(self, a: List[float], b: List[float]) -> float:
64
+ if not a or not b or len(a) != len(b):
65
+ return 1.0
66
+ dot = sum(x*y for x,y in zip(a,b))
67
+ na = math.sqrt(sum(x*x for x in a))
68
+ nb = math.sqrt(sum(x*x for x in b))
69
+ return 1.0 - (dot / (na*nb + 1e-9))
70
+
71
+ def generate_reindex_plan(self) -> List[str]:
72
+ return [vid for vid, lab in self.label_cache.items() if lab.needs_reindex]
73
+
74
+ # -----------------------------
75
+ # 2. BM25 Lexical Scoring + Hybrid Fusion + Reranker Interface
76
+ # -----------------------------
77
+ class BM25Scorer:
78
+ def __init__(self, k1: float = 1.5, b: float = 0.75):
79
+ self.k1 = k1
80
+ self.b = b
81
+ self.doc_freqs: Dict[str, int] = defaultdict(int)
82
+ self.doc_lens: Dict[str, int] = {}
83
+ self.corpus_size = 0
84
+ self.avgdl = 0.0
85
+
86
+ def fit(self, corpus: Dict[str, str]):
87
+ self.corpus_size = len(corpus)
88
+ total_len = 0
89
+ for did, text in corpus.items():
90
+ tokens = self._tokenize(text)
91
+ self.doc_lens[did] = len(tokens)
92
+ total_len += len(tokens)
93
+ for t in set(tokens):
94
+ self.doc_freqs[t] += 1
95
+ self.avgdl = total_len / max(1, self.corpus_size)
96
+
97
+ def score(self, query: str, doc_id: str, doc_text: str) -> float:
98
+ tokens = self._tokenize(query)
99
+ doc_tokens = self._tokenize(doc_text)
100
+ score = 0.0
101
+ for t in tokens:
102
+ if t not in doc_tokens:
103
+ continue
104
+ tf = doc_tokens.count(t)
105
+ df = self.doc_freqs.get(t, 0)
106
+ idf = math.log((self.corpus_size - df + 0.5) / (df + 0.5) + 1)
107
+ denom = tf + self.k1 * (1 - self.b + self.b * self.doc_lens.get(doc_id, 0) / self.avgdl)
108
+ score += idf * tf * (self.k1 + 1) / (denom + 1e-9)
109
+ return score
110
+
111
+ def _tokenize(self, text: str) -> List[str]:
112
+ return re.findall(r'\w+', text.lower())
113
+
114
+ class HybridFusion:
115
+ """Hybrid fusion of lexical (BM25) + vector scores"""
116
+ def __init__(self, alpha: float = 0.6):
117
+ self.alpha = alpha # weight for vector score
118
+ self.bm25 = BM25Scorer()
119
+
120
+ def fuse(self, query: str, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
121
+ # candidates: [{"id": , "text": , "vector_score": }]
122
+ corpus = {c["id"]: c.get("text", "") for c in candidates}
123
+ self.bm25.fit(corpus)
124
+ results = []
125
+ for c in candidates:
126
+ lex = self.bm25.score(query, c["id"], c.get("text", ""))
127
+ vec = c.get("vector_score", 0.5)
128
+ fused = self.alpha * vec + (1 - self.alpha) * (lex / 10.0) # normalize rough
129
+ c["fused_score"] = round(fused, 4)
130
+ results.append(c)
131
+ return sorted(results, key=lambda x: x["fused_score"], reverse=True)
132
+
133
+ class RerankerInterface:
134
+ """Pluggable reranker interface"""
135
+ def rerank(self, query: str, candidates: List[Dict], top_k: int = 5) -> List[Dict]:
136
+ # Default local-safe fallback: preserve fused ordering without making
137
+ # unearned cross-encoder claims.
138
+ for c in candidates:
139
+ c["rerank_score"] = c.get("fused_score", 0.0)
140
+ return sorted(candidates, key=lambda x: x.get("rerank_score", 0), reverse=True)[:top_k]
141
+
142
+ # -----------------------------
143
+ # 3. Memory Candidate Extraction / Scoring / Dedupe / Merge / Conflict / Retention
144
+ # -----------------------------
145
+ @dataclass
146
+ class MemoryCandidate:
147
+ id: str
148
+ content: str
149
+ score: float = 0.0
150
+ source: str = "unknown"
151
+ timestamp: float = field(default_factory=time.time)
152
+ conflicts: List[str] = field(default_factory=list)
153
+
154
+ class MemoryQualityManager:
155
+ def extract_candidates(self, memories: List[Dict]) -> List[MemoryCandidate]:
156
+ return [MemoryCandidate(m["id"], m["content"], m.get("score", 0.6), m.get("source", "mem")) for m in memories]
157
+
158
+ def score_candidates(self, cands: List[MemoryCandidate], query: str) -> List[MemoryCandidate]:
159
+ for c in cands:
160
+ # simple lexical overlap score
161
+ overlap = sum(1 for w in query.lower().split() if w in c.content.lower())
162
+ c.score = min(1.0, 0.4 + overlap * 0.15)
163
+ return sorted(cands, key=lambda x: x.score, reverse=True)
164
+
165
+ def dedupe(self, cands: List[MemoryCandidate], threshold: float = 0.85) -> List[MemoryCandidate]:
166
+ kept = []
167
+ seen = set()
168
+ for c in cands:
169
+ h = hashlib.md5(c.content.encode()).hexdigest()[:16]
170
+ if h not in seen:
171
+ seen.add(h)
172
+ kept.append(c)
173
+ return kept
174
+
175
+ def merge(self, cands: List[MemoryCandidate]) -> List[MemoryCandidate]:
176
+ # naive merge by content prefix
177
+ merged = {}
178
+ for c in cands:
179
+ key = c.content[:30]
180
+ if key not in merged or c.score > merged[key].score:
181
+ merged[key] = c
182
+ return list(merged.values())
183
+
184
+ def detect_conflicts(self, cands: List[MemoryCandidate]) -> List[MemoryCandidate]:
185
+ # Lightweight local heuristic; LLM conflict classifiers can replace it.
186
+ for i, c in enumerate(cands):
187
+ if "not" in c.content.lower() or "반대" in c.content:
188
+ c.conflicts.append("conflict:possible_negation")
189
+ return cands
190
+
191
+ def apply_retention(self, cands: List[MemoryCandidate], max_age_days: int = 90) -> List[MemoryCandidate]:
192
+ now = time.time()
193
+ return [c for c in cands if (now - c.timestamp) < max_age_days * 86400]
194
+
195
+ # -----------------------------
196
+ # 4. Graph Edge Validation / Confidence / Evidence / Duplicate Merge / Quality Metrics
197
+ # -----------------------------
198
+ @dataclass
199
+ class GraphEdgeQuality:
200
+ edge_id: str
201
+ confidence: float
202
+ evidence_count: int
203
+ is_duplicate: bool = False
204
+ quality_score: float = 0.0
205
+
206
+ class GraphEdgeQualityManager:
207
+ def validate_edge(self, edge: Dict[str, Any]) -> GraphEdgeQuality:
208
+ conf = edge.get("confidence", 0.7)
209
+ ev = len(edge.get("evidence", []))
210
+ q = min(1.0, conf * 0.75 + min(ev, 5) / 5 * 0.3)
211
+ return GraphEdgeQuality(edge.get("id", "e0"), conf, ev, quality_score=q)
212
+
213
+ def detect_duplicate_edges(self, edges: List[Dict]) -> List[str]:
214
+ seen = {}
215
+ dups = []
216
+ for e in edges:
217
+ key = (e.get("source"), e.get("target"), e.get("type"))
218
+ if key in seen:
219
+ dups.append(e.get("id"))
220
+ else:
221
+ seen[key] = e.get("id")
222
+ return dups
223
+
224
+ def merge_duplicate_edges(self, edges: List[Dict]) -> List[Dict]:
225
+ # keep highest confidence
226
+ best = {}
227
+ for e in edges:
228
+ key = (e.get("source"), e.get("target"), e.get("type"))
229
+ if key not in best or e.get("confidence", 0) > best[key].get("confidence", 0):
230
+ best[key] = e
231
+ return list(best.values())
232
+
233
+ def compute_quality_metrics(self, edges: List[Dict]) -> Dict[str, float]:
234
+ if not edges:
235
+ return {"avg_conf": 0.0, "avg_evidence": 0.0, "dup_rate": 0.0}
236
+ confs = [e.get("confidence", 0.5) for e in edges]
237
+ evs = [len(e.get("evidence", [])) for e in edges]
238
+ dups = len(self.detect_duplicate_edges(edges))
239
+ return {
240
+ "avg_conf": round(sum(confs)/len(confs), 3),
241
+ "avg_evidence": round(sum(evs)/len(evs), 2),
242
+ "dup_rate": round(dups / len(edges), 3)
243
+ }
244
+
245
+ # -----------------------------
246
+ # 5. Structured Context Assembly + Guardrails
247
+ # -----------------------------
248
+ @dataclass
249
+ class ContextGuardrails:
250
+ known: bool = True
251
+ inferred: bool = False
252
+ stale: bool = False
253
+ unknown: bool = False
254
+ confidence: float = 0.8
255
+ timestamp: float = field(default_factory=time.time)
256
+ attribution: str = "system"
257
+
258
+ class StructuredContextAssembler:
259
+ SECTIONS = ["Facts", "Decisions", "Preferences", "Relationships", "Projects", "Recent Events"]
260
+
261
+ def assemble(self, items: List[Dict[str, Any]]) -> Dict[str, List[Dict]]:
262
+ ctx: Dict[str, List[Dict]] = {s: [] for s in self.SECTIONS}
263
+ for item in items:
264
+ section = item.get("section", "Facts")
265
+ if section not in ctx:
266
+ section = "Facts"
267
+ guard = ContextGuardrails(
268
+ known=item.get("known", True),
269
+ inferred=item.get("inferred", False),
270
+ stale=item.get("stale", False),
271
+ unknown=item.get("unknown", False),
272
+ confidence=item.get("confidence", 0.75),
273
+ attribution=item.get("attribution", "user")
274
+ )
275
+ item["guardrails"] = guard.__dict__
276
+ ctx[section].append(item)
277
+ return ctx
278
+
279
+ def apply_guardrails(self, ctx: Dict[str, List[Dict]]) -> Dict[str, List[Dict]]:
280
+ cleaned = {}
281
+ for sec, items in ctx.items():
282
+ cleaned[sec] = [i for i in items if not i.get("guardrails", {}).get("unknown", False)]
283
+ return cleaned
284
+
285
+ # -----------------------------
286
+ # 6. Retrieval Benchmark Fixture Runner
287
+ # -----------------------------
288
+ class RetrievalBenchmarkRunner:
289
+ def __init__(self):
290
+ self.results: List[Dict] = []
291
+
292
+ def run_fixture(self, fixture_name: str, queries: List[Any], top_k: int = 5) -> Dict[str, Any]:
293
+ start = time.time()
294
+ judged = [q for q in queries if isinstance(q, dict)]
295
+ if judged:
296
+ recalls = []
297
+ precisions = []
298
+ ndcgs = []
299
+ for query in judged:
300
+ relevant = set(query.get("relevant") or [])
301
+ retrieved = list(query.get("retrieved") or [])[:top_k]
302
+ if not relevant:
303
+ continue
304
+ hits = [doc_id for doc_id in retrieved if doc_id in relevant]
305
+ recalls.append(len(hits) / len(relevant))
306
+ precisions.append(len(hits) / max(1, len(retrieved)))
307
+ dcg = sum(
308
+ 1.0 / math.log2(rank + 2)
309
+ for rank, doc_id in enumerate(retrieved)
310
+ if doc_id in relevant
311
+ )
312
+ ideal = sum(1.0 / math.log2(rank + 2) for rank in range(min(len(relevant), top_k)))
313
+ ndcgs.append(dcg / ideal if ideal else 0.0)
314
+ recall = sum(recalls) / len(recalls) if recalls else 0.0
315
+ precision = sum(precisions) / len(precisions) if precisions else 0.0
316
+ ndcg = sum(ndcgs) / len(ndcgs) if ndcgs else 0.0
317
+ else:
318
+ recall = precision = ndcg = 0.0
319
+ metrics = {
320
+ "fixture": fixture_name,
321
+ "queries": len(queries),
322
+ "avg_latency_ms": round((time.time() - start) * 1000 / max(1, len(queries)), 2),
323
+ "recall@5": round(recall, 4),
324
+ "precision@5": round(precision, 4),
325
+ "ndcg@5": round(ndcg, 4),
326
+ "top_k": top_k,
327
+ "judged": len(judged),
328
+ }
329
+ self.results.append(metrics)
330
+ return metrics
331
+
332
+ def summary(self) -> Dict[str, Any]:
333
+ if not self.results:
334
+ return {"status": "no runs"}
335
+ return {
336
+ "total_runs": len(self.results),
337
+ "avg_recall": round(sum(r["recall@5"] for r in self.results) / len(self.results), 3)
338
+ }
339
+
340
+ # -----------------------------
341
+ # Main Quality Layer Facade
342
+ # -----------------------------
343
+ class LatticeBrainQuality:
344
+ """Main entry point - pure Python quality hardening layer"""
345
+ def __init__(self):
346
+ self.embed_labeller = EmbeddingFallbackLabeller()
347
+ self.hybrid = HybridFusion()
348
+ self.reranker = RerankerInterface()
349
+ self.memory_mgr = MemoryQualityManager()
350
+ self.graph_mgr = GraphEdgeQualityManager()
351
+ self.context_assembler = StructuredContextAssembler()
352
+ self.benchmark = RetrievalBenchmarkRunner()
353
+
354
+ def full_quality_pass(self, payload: Dict[str, Any]) -> Dict[str, Any]:
355
+ """End-to-end quality pipeline (non-destructive)"""
356
+ result = {"status": "ok", "timestamp": time.time()}
357
+
358
+ # embedding
359
+ if "embeddings" in payload:
360
+ labels = [self.embed_labeller.label(e["id"], e.get("vector")) for e in payload["embeddings"]]
361
+ result["embedding_labels"] = [label.__dict__ for label in labels]
362
+ result["reindex_plan"] = self.embed_labeller.generate_reindex_plan()
363
+
364
+ # hybrid retrieval
365
+ if "retrieval" in payload:
366
+ fused = self.hybrid.fuse(payload["retrieval"]["query"], payload["retrieval"]["candidates"])
367
+ reranked = self.reranker.rerank(payload["retrieval"]["query"], fused)
368
+ result["retrieval"] = {"fused": fused, "reranked": reranked}
369
+
370
+ # memory
371
+ if "memories" in payload:
372
+ cands = self.memory_mgr.extract_candidates(payload["memories"])
373
+ cands = self.memory_mgr.score_candidates(cands, payload.get("query", ""))
374
+ cands = self.memory_mgr.dedupe(cands)
375
+ cands = self.memory_mgr.merge(cands)
376
+ cands = self.memory_mgr.detect_conflicts(cands)
377
+ cands = self.memory_mgr.apply_retention(cands)
378
+ result["memory_candidates"] = [c.__dict__ for c in cands]
379
+
380
+ # graph
381
+ if "graph_edges" in payload:
382
+ eqs = [self.graph_mgr.validate_edge(e) for e in payload["graph_edges"]]
383
+ result["graph_quality"] = [q.__dict__ for q in eqs]
384
+ result["graph_metrics"] = self.graph_mgr.compute_quality_metrics(payload["graph_edges"])
385
+
386
+ # context
387
+ if "context_items" in payload:
388
+ ctx = self.context_assembler.assemble(payload["context_items"])
389
+ ctx = self.context_assembler.apply_guardrails(ctx)
390
+ result["structured_context"] = ctx
391
+
392
+ return result
393
+
394
+
395
+ __all__ = [
396
+ "BM25Scorer",
397
+ "ContextGuardrails",
398
+ "EmbeddingFallbackLabeller",
399
+ "EmbeddingLabel",
400
+ "GraphEdgeQuality",
401
+ "GraphEdgeQualityManager",
402
+ "HybridFusion",
403
+ "LatticeBrainQuality",
404
+ "MemoryCandidate",
405
+ "MemoryQualityManager",
406
+ "RetrievalBenchmarkRunner",
407
+ "RerankerInterface",
408
+ "StructuredContextAssembler",
409
+ ]
@@ -19,7 +19,7 @@ from datetime import datetime
19
19
  from typing import Any, Callable, Dict, List, Optional
20
20
 
21
21
 
22
- MULTI_AGENT_VERSION = "6.3.0"
22
+ MULTI_AGENT_VERSION = "6.4.0"
23
23
 
24
24
  AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
25
25
  CORE_PIPELINE = ("planner", "executor", "reviewer")
@@ -1,3 +1,3 @@
1
1
  """Lattice AI - modular server package."""
2
2
 
3
- __version__ = "6.3.0"
3
+ __version__ = "6.4.0"