ltcai 9.7.0 → 9.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +98 -316
  2. package/docs/CHANGELOG.md +37 -0
  3. package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
  4. package/docs/DEVELOPMENT.md +1 -1
  5. package/docs/ONBOARDING.md +1 -1
  6. package/docs/TRUST_MODEL.md +1 -1
  7. package/docs/WHY_LATTICE.md +1 -1
  8. package/docs/kg-schema.md +1 -1
  9. package/lattice_brain/__init__.py +1 -1
  10. package/lattice_brain/graph/retrieval.py +93 -4
  11. package/lattice_brain/graph/retrieval_vector.py +43 -0
  12. package/lattice_brain/ingestion.py +399 -14
  13. package/lattice_brain/runtime/multi_agent.py +1 -1
  14. package/latticeai/__init__.py +1 -1
  15. package/latticeai/api/brain_intelligence.py +12 -0
  16. package/latticeai/api/chat.py +17 -0
  17. package/latticeai/api/chat_helpers.py +48 -0
  18. package/latticeai/api/chat_stream.py +9 -1
  19. package/latticeai/api/local_files.py +120 -2
  20. package/latticeai/core/agent_eval.py +123 -0
  21. package/latticeai/core/legacy_compatibility.py +1 -1
  22. package/latticeai/core/marketplace.py +1 -1
  23. package/latticeai/core/workspace_os.py +1 -1
  24. package/latticeai/services/architecture_readiness.py +1 -1
  25. package/latticeai/services/automation_intelligence.py +151 -14
  26. package/latticeai/services/brain_intelligence.py +72 -0
  27. package/latticeai/services/product_readiness.py +1 -1
  28. package/package.json +1 -1
  29. package/scripts/check_current_release_docs.mjs +1 -1
  30. package/src-tauri/Cargo.lock +1 -1
  31. package/src-tauri/Cargo.toml +1 -1
  32. package/src-tauri/tauri.conf.json +1 -1
  33. package/static/app/asset-manifest.json +11 -11
  34. package/static/app/assets/{Act-B6c39ays.js → Act-Dd3z8AzF.js} +2 -2
  35. package/static/app/assets/Brain-BMkgdWnI.js +321 -0
  36. package/static/app/assets/Capture-D2Aw9gkv.js +1 -0
  37. package/static/app/assets/Library-Yreq-KW5.js +1 -0
  38. package/static/app/assets/System-CXNmmtEo.js +1 -0
  39. package/static/app/assets/{index-85wQvEie.css → index-7gY9t9Sd.css} +1 -1
  40. package/static/app/assets/index-CndfILiF.js +18 -0
  41. package/static/app/assets/primitives-DxsIXb6G.js +1 -0
  42. package/static/app/assets/textarea-DH7ne8VI.js +1 -0
  43. package/static/app/index.html +2 -2
  44. package/static/sw.js +1 -1
  45. package/static/app/assets/Brain-D7Qg4k6M.js +0 -321
  46. package/static/app/assets/Capture-VF_di68r.js +0 -1
  47. package/static/app/assets/Library-D_Gis2PA.js +0 -1
  48. package/static/app/assets/System-C5s5H2ov.js +0 -1
  49. package/static/app/assets/index-DJC_2oub.js +0 -18
  50. package/static/app/assets/primitives-DL4Nip8C.js +0 -1
  51. package/static/app/assets/textarea-woZfCXHy.js +0 -1
@@ -42,9 +42,10 @@ from __future__ import annotations
42
42
  import fnmatch
43
43
  import hashlib
44
44
  import os
45
+ import threading
45
46
  from dataclasses import dataclass, field
46
47
  from pathlib import Path
47
- from typing import Any, Dict, Iterable, List, Optional, Tuple
48
+ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
48
49
 
49
50
  from .runtime.hooks import dispatch_tool
50
51
  from .utils import utc_now_iso
@@ -111,6 +112,138 @@ LATTICEIGNORE_FILENAME = ".latticeignore"
111
112
  # Opt-out escape hatch for the post-ingest incremental vector sync.
112
113
  AUTO_VECTOR_INDEX_ENV = "LATTICEAI_AUTO_VECTOR_INDEX"
113
114
 
115
+ # ── Extraction quality heuristics (v9.8.0 A1) ────────────────────────────────
116
+ # Pure heuristics over the extracted text — no model calls, no network. The
117
+ # score is *advisory*: it never blocks an ingest, it only annotates the result
118
+ # so capture surfaces (browser, folder scan) can surface low-quality warnings.
119
+ QUALITY_HIGH_THRESHOLD = 0.7
120
+ QUALITY_LOW_THRESHOLD = 0.4
121
+ QUALITY_LOW_WARNING = "추출 품질이 낮습니다 — 원문 확인을 권장합니다."
122
+ _WEB_SOURCE_TYPES = frozenset({"web_url", "browser_tab"})
123
+ # Standalone short lines that smell like leftover site chrome (nav/menu/footer).
124
+ _BOILERPLATE_LINE_MARKERS = frozenset(
125
+ {
126
+ "home", "menu", "nav", "navigation", "login", "log in", "sign in",
127
+ "sign up", "register", "subscribe", "search", "about", "about us",
128
+ "contact", "contact us", "privacy policy", "terms of service",
129
+ "cookie policy", "accept cookies", "accept all cookies", "share",
130
+ "skip to content", "copyright", "all rights reserved", "sitemap",
131
+ "back to top", "footer", "read more", "next", "previous",
132
+ }
133
+ )
134
+
135
+
136
+ def _quality_level(score: float) -> str:
137
+ if score >= QUALITY_HIGH_THRESHOLD:
138
+ return "high"
139
+ if score >= QUALITY_LOW_THRESHOLD:
140
+ return "medium"
141
+ return "low"
142
+
143
+
144
+ def assess_extraction_quality(
145
+ text: Optional[str],
146
+ *,
147
+ source_type: Optional[str] = None,
148
+ upstream_confidence: Optional[Any] = None,
149
+ ) -> Dict[str, Any]:
150
+ """Score extracted text 0..1 with reasons (pure heuristic, deterministic).
151
+
152
+ Signals: text length, whitespace ratio, character/word diversity
153
+ (repetition), sentence structure, and — for web sources — leftover
154
+ nav/menu boilerplate. When the upstream extractor supplies its own
155
+ confidence (``upstream_confidence``), that value wins verbatim: the
156
+ extractor saw the raw document, this function only sees its output.
157
+ """
158
+ if upstream_confidence is not None:
159
+ try:
160
+ score = max(0.0, min(1.0, float(upstream_confidence)))
161
+ except (TypeError, ValueError):
162
+ score = None
163
+ if score is not None:
164
+ return {
165
+ "score": round(score, 4),
166
+ "level": _quality_level(score),
167
+ "reasons": ["upstream_confidence"],
168
+ }
169
+
170
+ raw = str(text or "")
171
+ stripped = raw.strip()
172
+ if not stripped:
173
+ return {"score": 0.0, "level": "low", "reasons": ["empty_text"]}
174
+
175
+ reasons: List[str] = []
176
+ length = len(stripped)
177
+ sample = stripped[:4000]
178
+ lines = [ln.strip() for ln in stripped.splitlines() if ln.strip()]
179
+ words = stripped.split()
180
+
181
+ # 1) Length — very short extractions rarely carry recall value.
182
+ if length < 40:
183
+ length_factor = 0.35
184
+ reasons.append("very_short_text")
185
+ elif length < 120:
186
+ length_factor = 0.6
187
+ reasons.append("short_text")
188
+ elif length < 300:
189
+ length_factor = 0.85
190
+ else:
191
+ length_factor = 1.0
192
+
193
+ # 2) Sentence structure — prose has sentence-ending punctuation.
194
+ sentence_marks = sum(sample.count(mark) for mark in (".", "!", "?", "…", "。", "!", "?"))
195
+ if sentence_marks > 0:
196
+ structure_factor = 1.0
197
+ elif length < 200:
198
+ structure_factor = 0.75 # titles/snippets legitimately lack periods
199
+ else:
200
+ structure_factor = 0.45
201
+ reasons.append("no_sentence_structure")
202
+
203
+ # 3) Diversity — repeated characters/lines/words indicate extraction junk.
204
+ diversity_factor = 1.0
205
+ distinct_chars = len(set(sample.lower()))
206
+ if distinct_chars < 10:
207
+ diversity_factor *= 0.2
208
+ reasons.append("low_character_diversity")
209
+ elif distinct_chars < 20:
210
+ diversity_factor *= 0.7
211
+ if len(lines) >= 6:
212
+ top_count = max(lines.count(ln) for ln in set(lines))
213
+ if top_count >= max(3, len(lines) // 4):
214
+ diversity_factor *= 0.5
215
+ reasons.append("repetitive_lines")
216
+ if len(words) >= 30 and (len(set(w.lower() for w in words)) / len(words)) < 0.25:
217
+ diversity_factor *= 0.5
218
+ reasons.append("repetitive_words")
219
+
220
+ # 4) Cleanliness — whitespace floods, fragmented lines, site chrome.
221
+ cleanliness_factor = 1.0
222
+ whitespace_ratio = sum(1 for ch in raw if ch.isspace()) / max(1, len(raw))
223
+ if whitespace_ratio > 0.45:
224
+ cleanliness_factor *= 0.6
225
+ reasons.append("high_whitespace_ratio")
226
+ if len(lines) >= 8:
227
+ short_lines = sum(1 for ln in lines if len(ln.split()) <= 3)
228
+ if short_lines / len(lines) > 0.6:
229
+ cleanliness_factor *= 0.6
230
+ reasons.append("fragmented_lines")
231
+ boilerplate_hits = sum(
232
+ 1 for ln in lines if ln.lower().strip(" .:>|•·-–—*") in _BOILERPLATE_LINE_MARKERS
233
+ )
234
+ if lines and boilerplate_hits >= 3 and (boilerplate_hits / len(lines)) > 0.2:
235
+ cleanliness_factor *= 0.35
236
+ if str(source_type or "").lower() in _WEB_SOURCE_TYPES:
237
+ reasons.append("nav_menu_remnants")
238
+ else:
239
+ reasons.append("boilerplate_markers")
240
+
241
+ score = length_factor * structure_factor * diversity_factor * cleanliness_factor
242
+ score = max(0.0, min(1.0, score))
243
+ if not reasons:
244
+ reasons.append("clean_extraction")
245
+ return {"score": round(score, 4), "level": _quality_level(score), "reasons": reasons}
246
+
114
247
 
115
248
  def _load_latticeignore(root: Path) -> List[str]:
116
249
  """Parse ``root/.latticeignore`` → glob patterns (gitignore-like subset)."""
@@ -155,16 +288,58 @@ def _matches_ignore(
155
288
 
156
289
 
157
290
  # --- Large candidate 1 slice: incremental / background ingestion support ---
291
+ JOB_ERRORS_CAP = 50 # per-job error records kept (failed count keeps counting)
292
+
293
+
158
294
  @dataclass
159
295
  class BackgroundIngestionJob:
160
- """Job descriptor for background/incremental indexing (KG scale-up slice)."""
296
+ """Job descriptor + progress state for background/incremental indexing.
297
+
298
+ ``done_indices`` tracks per-item completion so an interrupted or partially
299
+ failed job can be *resumed* from the remaining items instead of restarting.
300
+ ``errors`` is capped at ``max_errors`` records; ``failed`` keeps counting.
301
+ """
161
302
  job_id: str
162
303
  items: List[IngestionItem]
163
- status: str = "pending" # pending | running | done | failed
304
+ status: str = "queued" # queued | running | completed | failed | partial
164
305
  created_at: str = field(default_factory=utc_now_iso)
306
+ updated_at: str = field(default_factory=utc_now_iso)
165
307
  processed: int = 0
308
+ failed: int = 0
166
309
  total: int = 0
167
- errors: List[str] = field(default_factory=list)
310
+ errors: List[Dict[str, Any]] = field(default_factory=list)
311
+ incremental: bool = True
312
+ user_email: Optional[str] = None
313
+ max_errors: int = JOB_ERRORS_CAP
314
+ done_indices: Set[int] = field(default_factory=set)
315
+
316
+ def touch(self) -> None:
317
+ self.updated_at = utc_now_iso()
318
+
319
+ def record_error(self, index: int, item: IngestionItem, detail: Any) -> None:
320
+ self.failed += 1
321
+ if len(self.errors) < self.max_errors:
322
+ self.errors.append({
323
+ "index": index,
324
+ "source": item.source_uri or item.path or item.title or item.source_type,
325
+ "detail": str(detail)[:500],
326
+ })
327
+
328
+ def remaining_indices(self) -> List[int]:
329
+ return [i for i in range(len(self.items)) if i not in self.done_indices]
330
+
331
+ def as_dict(self) -> Dict[str, Any]:
332
+ """Frozen job schema consumed by ``/api/ingestion/jobs*``."""
333
+ return {
334
+ "job_id": self.job_id,
335
+ "status": self.status,
336
+ "total": self.total,
337
+ "processed": self.processed,
338
+ "failed": self.failed,
339
+ "errors": list(self.errors),
340
+ "created_at": self.created_at,
341
+ "updated_at": self.updated_at,
342
+ }
168
343
 
169
344
 
170
345
  class BackgroundIngestionQueue:
@@ -177,27 +352,45 @@ class BackgroundIngestionQueue:
177
352
  def __init__(self) -> None:
178
353
  self._jobs: Dict[str, BackgroundIngestionJob] = {}
179
354
  self._counter = 0
355
+ self._lock = threading.Lock()
180
356
 
181
- def schedule(self, items: List[IngestionItem], *, incremental: bool = True) -> BackgroundIngestionJob:
182
- self._counter += 1
183
- job_id = f"bg_ingest_{self._counter:04d}"
357
+ def schedule(
358
+ self,
359
+ items: List[IngestionItem],
360
+ *,
361
+ incremental: bool = True,
362
+ user_email: Optional[str] = None,
363
+ ) -> BackgroundIngestionJob:
364
+ with self._lock:
365
+ self._counter += 1
366
+ job_id = f"bg_ingest_{self._counter:04d}"
184
367
  job = BackgroundIngestionJob(
185
368
  job_id=job_id,
186
369
  items=items,
187
370
  total=len(items),
371
+ incremental=incremental,
372
+ user_email=user_email,
188
373
  )
189
374
  # annotate items for downstream
190
375
  for it in job.items:
191
376
  # attach flag without breaking dataclass defaults (use metadata)
192
377
  it.metadata = {**it.metadata, "incremental": incremental, "bg_job": job_id}
193
- self._jobs[job_id] = job
378
+ with self._lock:
379
+ self._jobs[job_id] = job
194
380
  return job
195
381
 
196
382
  def get(self, job_id: str) -> Optional[BackgroundIngestionJob]:
197
383
  return self._jobs.get(job_id)
198
384
 
199
385
  def list_pending(self) -> List[BackgroundIngestionJob]:
200
- return [j for j in self._jobs.values() if j.status == "pending"]
386
+ return [j for j in self._jobs.values() if j.status == "queued"]
387
+
388
+ def list_recent(self, limit: int = 20) -> List[BackgroundIngestionJob]:
389
+ """Most recent jobs first (insertion order is schedule order)."""
390
+ limit = max(1, int(limit))
391
+ with self._lock:
392
+ jobs = list(self._jobs.values())
393
+ return list(reversed(jobs))[:limit]
201
394
 
202
395
 
203
396
  @dataclass
@@ -237,9 +430,13 @@ class IngestionResult:
237
430
  indexing_status: str = "pending" # indexed | skipped | failed | pending
238
431
  provenance_id: Optional[str] = None
239
432
  detail: Optional[str] = None
433
+ # v9.8.0 additive quality fields — advisory only, never gate behavior.
434
+ extraction_quality: Optional[Dict[str, Any]] = None
435
+ warnings: List[str] = field(default_factory=list)
436
+ quality_gate: Optional[Dict[str, Any]] = None
240
437
 
241
438
  def as_dict(self) -> Dict[str, Any]:
242
- return {
439
+ payload = {
243
440
  "status": self.status,
244
441
  "source_type": self.source_type,
245
442
  "node_id": self.node_id,
@@ -254,6 +451,14 @@ class IngestionResult:
254
451
  "provenance_id": self.provenance_id,
255
452
  "detail": self.detail,
256
453
  }
454
+ # Additive keys only when populated so pre-v9.8 payloads are unchanged.
455
+ if self.extraction_quality is not None:
456
+ payload["extraction_quality"] = self.extraction_quality
457
+ if self.warnings:
458
+ payload["warnings"] = list(self.warnings)
459
+ if self.quality_gate is not None:
460
+ payload["quality_gate"] = self.quality_gate
461
+ return payload
257
462
 
258
463
 
259
464
  class IngestionPipeline:
@@ -320,6 +525,14 @@ class IngestionPipeline:
320
525
  return self._ingest_file(item, source_type=source_type, owner=owner, captured_at=captured_at)
321
526
  return self._ingest_text(item, source_type=source_type, owner=owner, captured_at=captured_at)
322
527
 
528
+ # v9.8.0 observation-only quality gate: computed *before* the write so
529
+ # the search never matches the node we are about to create. It is
530
+ # recorded on the result and never skips an ingest (behavior unchanged).
531
+ quality_text = self._extractable_text(item)
532
+ quality_gate = self._observe_quality_gate(
533
+ item, source_type=source_type, text=quality_text,
534
+ )
535
+
323
536
  try:
324
537
  raw = dispatch_tool(
325
538
  self._hooks, tool_name, args, _run,
@@ -397,6 +610,13 @@ class IngestionPipeline:
397
610
  except Exception: # noqa: BLE001 — audit must never break ingestion
398
611
  pass
399
612
 
613
+ extraction_quality = self._assess_item_quality(
614
+ item, source_type=source_type, text=quality_text, chunk_ids=chunk_ids,
615
+ )
616
+ warnings: List[str] = []
617
+ if extraction_quality is not None and extraction_quality.get("level") == "low":
618
+ warnings.append(QUALITY_LOW_WARNING)
619
+
400
620
  details = [d for d in (provenance_detail, vector_detail) if d]
401
621
  return IngestionResult(
402
622
  status="ok",
@@ -412,8 +632,110 @@ class IngestionPipeline:
412
632
  indexing_status=indexing_status,
413
633
  provenance_id=prov.get("id"),
414
634
  detail="; ".join(details) if details else None,
635
+ extraction_quality=extraction_quality,
636
+ warnings=warnings,
637
+ quality_gate=quality_gate,
415
638
  )
416
639
 
640
+ # ── extraction quality (v9.8.0 A1 — advisory, never gates) ───────────────
641
+ @staticmethod
642
+ def _extractable_text(item: IngestionItem) -> Optional[str]:
643
+ """Best available extracted text for quality scoring/gating."""
644
+ if item.text is not None:
645
+ return item.text
646
+ extracted = (item.metadata or {}).get("extracted")
647
+ if isinstance(extracted, dict):
648
+ content = extracted.get("content") or extracted.get("text")
649
+ if content is not None:
650
+ return str(content)
651
+ return None
652
+
653
+ @staticmethod
654
+ def _upstream_confidence(item: IngestionItem) -> Optional[Any]:
655
+ """Upstream extractor confidence, if the capture surface supplied one."""
656
+ meta = item.metadata or {}
657
+ extracted = meta.get("extracted")
658
+ if isinstance(extracted, dict) and extracted.get("confidence") is not None:
659
+ return extracted.get("confidence")
660
+ if meta.get("extraction_confidence") is not None:
661
+ return meta.get("extraction_confidence")
662
+ return None
663
+
664
+ def _assess_item_quality(
665
+ self,
666
+ item: IngestionItem,
667
+ *,
668
+ source_type: str,
669
+ text: Optional[str],
670
+ chunk_ids: List[str],
671
+ ) -> Optional[Dict[str, Any]]:
672
+ """Quality annotation for document-like sources (not chat/memory)."""
673
+ if source_type in CHAT_SOURCE_TYPES or source_type in MEMORY_SOURCE_TYPES:
674
+ return None
675
+ confidence = self._upstream_confidence(item)
676
+ if text is not None or confidence is not None:
677
+ return assess_extraction_quality(
678
+ text, source_type=source_type, upstream_confidence=confidence,
679
+ )
680
+ # File door without inline extraction (e.g. PDF): the pipeline never saw
681
+ # the text, so score honestly from the chunk output instead of guessing.
682
+ if chunk_ids:
683
+ return {
684
+ "score": 0.5,
685
+ "level": "medium",
686
+ "reasons": ["content_extracted_upstream_not_scored"],
687
+ }
688
+ return {"score": 0.0, "level": "low", "reasons": ["no_extracted_text"]}
689
+
690
+ def _observe_quality_gate(
691
+ self,
692
+ item: IngestionItem,
693
+ *,
694
+ source_type: str,
695
+ text: Optional[str],
696
+ ) -> Optional[Dict[str, Any]]:
697
+ """Observation-mode ``gate_ingest_candidate`` wiring.
698
+
699
+ Records what the proactive gate *would* decide (ingest /
700
+ skip_duplicate / review) without ever acting on it. Any failure —
701
+ import, search, gate — yields ``None``; the ingest proceeds untouched.
702
+ """
703
+ if source_type in CHAT_SOURCE_TYPES or source_type in MEMORY_SOURCE_TYPES:
704
+ return None
705
+ body = str(text or "").strip()
706
+ if not body:
707
+ return None
708
+ try:
709
+ from .graph.proactive import gate_ingest_candidate
710
+ except Exception: # noqa: BLE001 — optional observation, never required
711
+ return None
712
+
713
+ def _search(query: str) -> Any:
714
+ snippet = str(query or "")[:400]
715
+ try:
716
+ if item.workspace_id:
717
+ return self._kg.search(
718
+ snippet, 20, allowed_workspaces={item.workspace_id},
719
+ )
720
+ return self._kg.search(snippet, 20)
721
+ except TypeError:
722
+ # Older store without workspace-scoped search.
723
+ return self._kg.search(snippet, 20)
724
+
725
+ try:
726
+ gate = gate_ingest_candidate(body, _search)
727
+ except Exception: # noqa: BLE001 — observation must never fail the ingest
728
+ return None
729
+ parts = [str(gate.get("reason") or "")]
730
+ if gate.get("similarity") is not None:
731
+ parts.append(f"similarity={gate.get('similarity')}")
732
+ if gate.get("match_id"):
733
+ parts.append(f"match={gate.get('match_id')}")
734
+ return {
735
+ "action": str(gate.get("action") or "review"),
736
+ "detail": "; ".join(p for p in parts if p),
737
+ }
738
+
417
739
  def _sync_vector_index(self, node_id: str) -> Tuple[str, Optional[str]]:
418
740
  """Best-effort incremental vector sync → (indexing_status, detail).
419
741
 
@@ -441,20 +763,81 @@ class IngestionPipeline:
441
763
  items: List[IngestionItem],
442
764
  *,
443
765
  incremental: bool = True,
766
+ user_email: Optional[str] = None,
444
767
  ) -> BackgroundIngestionJob:
445
768
  """Schedule items for background incremental indexing.
446
769
 
447
770
  Returns a job handle. Actual execution can be driven by caller
448
- (or future worker) calling pipeline.ingest on each. This seam enables
449
- large-corpus scale without blocking user requests.
771
+ (or future worker) calling pipeline.ingest on each or through
772
+ :meth:`run_background_job`. This seam enables large-corpus scale
773
+ without blocking user requests.
450
774
  """
451
- job = self._bg_queue.schedule(items, incremental=incremental)
775
+ job = self._bg_queue.schedule(items, incremental=incremental, user_email=user_email)
452
776
  # mark initial status on results concept (jobs track)
453
777
  return job
454
778
 
455
779
  def get_background_job(self, job_id: str) -> Optional[BackgroundIngestionJob]:
456
780
  return self._bg_queue.get(job_id)
457
781
 
782
+ def list_background_jobs(self, limit: int = 20) -> List[Dict[str, Any]]:
783
+ """Recent jobs (newest first) in the frozen ``/api/ingestion`` schema."""
784
+ return [job.as_dict() for job in self._bg_queue.list_recent(limit=limit)]
785
+
786
+ def run_background_job(
787
+ self, job_id: str, *, user_email: Optional[str] = None
788
+ ) -> Dict[str, Any]:
789
+ """Execute a queued/interrupted job's remaining items.
790
+
791
+ Per-item errors are recorded (capped) and never abort the job. The
792
+ final status is ``completed`` (all done), ``partial`` (some done),
793
+ or ``failed`` (nothing done). Already-completed items are skipped, so
794
+ the same method safely powers both first-run and resume.
795
+ """
796
+ job = self._bg_queue.get(job_id)
797
+ if job is None:
798
+ return {"status": "not_found", "job_id": job_id}
799
+ if job.status == "running":
800
+ return job.as_dict()
801
+ return self._execute_background_job(job, user_email=user_email)
802
+
803
+ def resume_background_job(
804
+ self, job_id: str, *, user_email: Optional[str] = None
805
+ ) -> Dict[str, Any]:
806
+ """Resume an interrupted/partial/failed job from its remaining items."""
807
+ return self.run_background_job(job_id, user_email=user_email)
808
+
809
+ def _execute_background_job(
810
+ self, job: BackgroundIngestionJob, *, user_email: Optional[str] = None
811
+ ) -> Dict[str, Any]:
812
+ job.status = "running"
813
+ # Retried items get a fresh verdict: reset failure state for this run.
814
+ job.failed = 0
815
+ job.errors = []
816
+ job.touch()
817
+ runner_email = user_email or job.user_email
818
+ for index in job.remaining_indices():
819
+ item = job.items[index]
820
+ try:
821
+ result = self.ingest(item, user_email=runner_email or item.owner)
822
+ status, detail = result.status, result.detail
823
+ except Exception as exc: # noqa: BLE001 — per-item isolation: keep going
824
+ status, detail = "failed", str(exc)
825
+ if status == "ok":
826
+ job.done_indices.add(index)
827
+ else:
828
+ job.record_error(index, item, detail or status)
829
+ job.processed = len(job.done_indices)
830
+ job.touch()
831
+ job.processed = len(job.done_indices)
832
+ if job.total == 0 or job.processed >= job.total:
833
+ job.status = "completed"
834
+ elif job.processed > 0:
835
+ job.status = "partial"
836
+ else:
837
+ job.status = "failed"
838
+ job.touch()
839
+ return job.as_dict()
840
+
458
841
  def ingest_web_page(
459
842
  self,
460
843
  url: str,
@@ -647,7 +1030,9 @@ class IngestionPipeline:
647
1030
 
648
1031
  summary["matched"] = len(items)
649
1032
  if background:
650
- job = self.schedule_background(items, incremental=True)
1033
+ job = self.schedule_background(
1034
+ items, incremental=True, user_email=user_email or owner,
1035
+ )
651
1036
  summary.update(status="scheduled", job_id=job.job_id, scheduled=len(items))
652
1037
  return summary
653
1038
 
@@ -48,7 +48,7 @@ from .contracts import multi_agent_contract
48
48
  from ..utils import now_iso as _now
49
49
 
50
50
 
51
- MULTI_AGENT_VERSION = "9.7.0"
51
+ MULTI_AGENT_VERSION = "9.8.0"
52
52
 
53
53
  AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
54
54
  CORE_PIPELINE = ("planner", "executor", "reviewer")
@@ -1,3 +1,3 @@
1
1
  """Lattice AI - modular server package."""
2
2
 
3
- __version__ = "9.7.0"
3
+ __version__ = "9.8.0"
@@ -57,6 +57,18 @@ def create_brain_intelligence_router(
57
57
  scope = gate_read(request)
58
58
  return service.contradictions(user_email=user, workspace_id=scope)
59
59
 
60
+ @router.get("/api/brain/vector-freshness")
61
+ async def brain_vector_freshness(request: Request):
62
+ """Vector index freshness summary (read-only, never raises).
63
+
64
+ Fixed contract consumed by the frontend:
65
+ ``{"status": "ready"|"pending"|"unavailable", "pending_items": int,
66
+ "total_items": int, "detail": str}``.
67
+ """
68
+ user = require_user(request)
69
+ scope = gate_read(request)
70
+ return service.vector_freshness(user_email=user, workspace_id=scope)
71
+
60
72
  @router.get("/api/brain/duplicates")
61
73
  async def brain_duplicates(request: Request):
62
74
  """Graph-layer duplicate node candidates (read-only)."""
@@ -28,6 +28,7 @@ from latticeai.api.chat_documents import (
28
28
  )
29
29
  from latticeai.api.chat_helpers import (
30
30
  _LANG_HINT,
31
+ build_context_quality,
31
32
  build_recent_chat_context,
32
33
  detect_language,
33
34
  file_action_target,
@@ -64,6 +65,7 @@ __all__ = [
64
65
  "AgentResumeRequest",
65
66
  "ChatRequest",
66
67
  "create_chat_router",
68
+ "build_context_quality",
67
69
  "build_recent_chat_context",
68
70
  "pair_user_history",
69
71
  "detect_language",
@@ -358,6 +360,19 @@ def create_chat_router(context: AppContext) -> APIRouter:
358
360
  if context_trace is not None and isinstance(trace_seed, dict):
359
361
  trace_seed["context_assembly"] = context_trace
360
362
 
363
+ # v9.8.0 honest RAG signal: how well the graph grounded this answer.
364
+ # Rides the same channel as sources/evidence (the answer trace) and is
365
+ # additionally exposed top-level on both response shapes below.
366
+ context_quality = build_context_quality(
367
+ req.message,
368
+ knowledge_graph=context.knowledge_graph
369
+ if (context.enable_graph and context.knowledge_graph)
370
+ else None,
371
+ allowed_workspaces={workspace_id} if workspace_id else None,
372
+ )
373
+ if isinstance(trace_seed, dict):
374
+ trace_seed["context_quality"] = context_quality
375
+
361
376
  history_message = (
362
377
  f"{req.message}\n[Image attached]" if req.image_data else req.message
363
378
  )
@@ -407,6 +422,7 @@ def create_chat_router(context: AppContext) -> APIRouter:
407
422
  history_meta=history_meta,
408
423
  model_id=selected_model_id,
409
424
  workspace_id=workspace_id,
425
+ context_quality=context_quality,
410
426
  ),
411
427
  media_type="text/event-stream",
412
428
  headers={"X-Model": selected_model_id},
@@ -462,6 +478,7 @@ def create_chat_router(context: AppContext) -> APIRouter:
462
478
  "response": response_text,
463
479
  "trace_id": trace_record["id"],
464
480
  "trace": trace_record,
481
+ "context_quality": context_quality,
465
482
  }
466
483
  )
467
484
 
@@ -197,6 +197,54 @@ def format_network_status(info: Dict) -> str:
197
197
  lines.extend(["", note])
198
198
  return "\n".join(lines)
199
199
 
200
+ def build_context_quality(
201
+ query: str,
202
+ *,
203
+ knowledge_graph=None,
204
+ allowed_workspaces=None,
205
+ limit: int = 6,
206
+ ) -> Dict[str, object]:
207
+ """Honest RAG context-quality signal for chat responses (v9.8.0, additive).
208
+
209
+ Returns ``{"mode": "hybrid"|"lexical_only"|"none", "nodes": int,
210
+ "limited": bool, "reason": str|None}``. ``limited`` is true when the
211
+ graph produced 0–1 matches, when vector retrieval fell back to
212
+ lexical-only, or when retrieval failed entirely — so the frontend can
213
+ tell the user the answer is weakly grounded instead of implying rich
214
+ graph context. Never raises.
215
+ """
216
+ from lattice_brain.graph.retrieval import context_quality_signal
217
+
218
+ query = str(query or "").strip()
219
+ if knowledge_graph is None or not query:
220
+ return context_quality_signal(
221
+ "none", 0, reason="지식 그래프 컨텍스트를 사용하지 않았습니다"
222
+ )
223
+ scope_kwargs = (
224
+ {"allowed_workspaces": allowed_workspaces}
225
+ if allowed_workspaces is not None
226
+ else {}
227
+ )
228
+ hybrid_fn = getattr(knowledge_graph, "hybrid_search", None)
229
+ if callable(hybrid_fn):
230
+ try:
231
+ result = hybrid_fn(query, top_k=limit, **scope_kwargs) or {}
232
+ except Exception: # noqa: BLE001 — signal building must never fail chat
233
+ return context_quality_signal("none", 0, reason="그래프 검색에 실패했습니다")
234
+ matches = result.get("matches") or []
235
+ return context_quality_signal(
236
+ str(result.get("mode") or "hybrid"), len(matches)
237
+ )
238
+ # Lexical-only stores without the hybrid mixin.
239
+ try:
240
+ matches = (
241
+ knowledge_graph.search(query, limit=limit, **scope_kwargs) or {}
242
+ ).get("matches") or []
243
+ except Exception: # noqa: BLE001 — signal building must never fail chat
244
+ return context_quality_signal("none", 0, reason="그래프 검색에 실패했습니다")
245
+ return context_quality_signal("lexical_only", len(matches))
246
+
247
+
200
248
  def workspace_scope_from_request(request: Request) -> Optional[str]:
201
249
  header = request.headers.get("X-Workspace-Id")
202
250
  if header and header.strip():