ltcai 9.9.1 → 9.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -39
- package/docs/CHANGELOG.md +80 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +1 -1
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/kg-schema.md +1 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/graph/curator.py +141 -0
- package/lattice_brain/graph/documents.py +111 -0
- package/lattice_brain/graph/fusion.py +179 -0
- package/lattice_brain/graph/projection.py +161 -0
- package/lattice_brain/graph/retrieval.py +19 -1
- package/lattice_brain/ingestion.py +66 -0
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/automation_intelligence.py +159 -0
- package/latticeai/api/browser.py +15 -4
- package/latticeai/api/chat.py +30 -2
- package/latticeai/api/chat_agent_http.py +193 -4
- package/latticeai/api/chat_contracts.py +7 -1
- package/latticeai/api/chat_helpers.py +147 -0
- package/latticeai/api/chat_intents.py +319 -8
- package/latticeai/api/chat_stream.py +17 -1
- package/latticeai/api/funnel_metrics.py +31 -0
- package/latticeai/api/knowledge_graph.py +30 -0
- package/latticeai/api/local_files.py +90 -0
- package/latticeai/api/setup.py +134 -1
- package/latticeai/api/tools.py +33 -3
- package/latticeai/app_factory.py +15 -0
- package/latticeai/core/agent.py +205 -11
- package/latticeai/core/agent_eval.py +107 -0
- package/latticeai/core/file_generation.py +337 -2
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/runtime/persistence_runtime.py +19 -1
- package/latticeai/runtime/router_registration.py +8 -1
- package/latticeai/services/app_context.py +6 -0
- package/latticeai/services/architecture_readiness.py +1 -1
- package/latticeai/services/automation_execution.py +266 -0
- package/latticeai/services/automation_intelligence.py +7 -0
- package/latticeai/services/command_center.py +20 -1
- package/latticeai/services/folder_watch.py +391 -0
- package/latticeai/services/funnel_metrics.py +202 -0
- package/latticeai/services/product_readiness.py +6 -1
- package/latticeai/services/search_service.py +17 -1
- package/latticeai/services/tool_dispatch.py +36 -0
- package/latticeai/setup/demo_corpus.py +98 -0
- package/latticeai/tools/filesystem.py +38 -1
- package/package.json +1 -1
- package/scripts/bench_models.py +234 -5
- package/scripts/check_current_release_docs.mjs +1 -1
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +28 -27
- package/static/app/assets/Act-DmdruVKV.js +1 -0
- package/static/app/assets/{Brain-CSCDZEHd.js → Brain-DEY9jLVt.js} +2 -2
- package/static/app/assets/BrainHome-CeNaxjP1.js +2 -0
- package/static/app/assets/BrainSignals-CStjIqYi.js +1 -0
- package/static/app/assets/{Capture-BiuerOio.js → Capture-CVZ09QXi.js} +1 -1
- package/static/app/assets/CommandPalette-DepwOQFv.js +1 -0
- package/static/app/assets/Library-Bp0n-HlW.js +1 -0
- package/static/app/assets/{LivingBrain-6FvA0H5T.js → LivingBrain-DxP4efJF.js} +1 -1
- package/static/app/assets/{ProductFlow-Cr8bkQPw.js → ProductFlow-DRbm7NEq.js} +1 -1
- package/static/app/assets/ReviewCard-C4HAO7A3.js +3 -0
- package/static/app/assets/{System-D6uJacqu.js → System-ByQcmJW-.js} +1 -1
- package/static/app/assets/{bot-DgC7xgUl.js → bot-BNDyZLR7.js} +1 -1
- package/static/app/assets/circle-play-BkhdcHgd.js +1 -0
- package/static/app/assets/{cpu-DDGb4XHl.js → cpu-C6jjYm6i.js} +1 -1
- package/static/app/assets/{folder-open-BbVuSO9M.js → folder-open-DjGIvDBQ.js} +1 -1
- package/static/app/assets/{hard-drive-yDZR6XYz.js → hard-drive-BlSbwSaT.js} +1 -1
- package/static/app/assets/index-Bge3DXW7.css +2 -0
- package/static/app/assets/{index-DrZSkwrE.js → index-CHu7cgj3.js} +3 -3
- package/static/app/assets/{input-DPIkTs9n.js → input-DVDI0YR3.js} +1 -1
- package/static/app/assets/{navigation-B1mkmyUe.js → navigation-BddhEWA0.js} +1 -1
- package/static/app/assets/{network-0XDI1Uf_.js → network-pYQt5oBu.js} +1 -1
- package/static/app/assets/{primitives-sIQYKyf5.js → primitives-D7gCdEvS.js} +1 -1
- package/static/app/assets/shield-alert-K9RKGQeg.js +1 -0
- package/static/app/assets/{textarea-DvnwpnqS.js → textarea-sqQmoBKL.js} +1 -1
- package/static/app/assets/useFocusTrap-7EV9dFP2.js +1 -0
- package/static/app/assets/utils-uQYKXNeq.js +7 -0
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/static/app/assets/Act-CcSWTQsx.js +0 -1
- package/static/app/assets/BrainHome-B5QpJ1Fx.js +0 -2
- package/static/app/assets/BrainSignals-CmFD8PWk.js +0 -1
- package/static/app/assets/CommandPalette-BrqpT7K2.js +0 -1
- package/static/app/assets/Library-BcaiPbum.js +0 -1
- package/static/app/assets/ReviewCard-D56TMypk.js +0 -1
- package/static/app/assets/download-C-Ei3GFa.js +0 -1
- package/static/app/assets/index-CFRhNBuH.css +0 -2
- package/static/app/assets/sunrise-DXMjueec.js +0 -1
- package/static/app/assets/utils-BrfFHAJl.js +0 -7
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
[](https://github.com/TaeSooPark-PTS/LatticeAI/actions/workflows/ci.yml)
|
|
12
12
|
[](LICENSE)
|
|
13
13
|
|
|
14
|
-

|
|
15
15
|
|
|
16
16
|
Chat, files, folders, notes, and web pages all flow into one durable knowledge
|
|
17
17
|
graph on your computer. Any model — local MLX or cloud — can speak with that
|
|
@@ -24,9 +24,9 @@ memory. Nothing leaves your machine without explicit consent.
|
|
|
24
24
|
|
|
25
25
|
| | |
|
|
26
26
|
| --- | --- |
|
|
27
|
-
| **Chat with a Brain that remembers** — every conversation grows durable, source-linked memory  | **See how knowledge connects** — a real relationship graph, not a file list  |
|
|
28
|
+
| **Capture anything** — files, whole folders, notes, screenshots, web pages  | **Automate with review** — agent changes become proposals you approve first  |
|
|
29
|
+
| **Pick a model in one click** — recommended local models for your hardware  | **Stay in control** — audit, roles, retention in a separate admin surface  |
|
|
30
30
|
|
|
31
31
|
## Why Lattice AI
|
|
32
32
|
|
|
@@ -57,49 +57,48 @@ First-run flow — wake the Brain, pick the owner, load a recommended model:
|
|
|
57
57
|
|
|
58
58
|
| | | |
|
|
59
59
|
| --- | --- | --- |
|
|
60
|
-
|  |  |  |
|
|
61
61
|
|
|
62
62
|
Screenshot index and capture notes:
|
|
63
|
-
[output/release/v9.9.
|
|
63
|
+
[output/release/v9.9.3/SCREENSHOT_INDEX.md](output/release/v9.9.3/SCREENSHOT_INDEX.md)
|
|
64
64
|
|
|
65
65
|
## Current Release
|
|
66
66
|
|
|
67
|
-
The current release is **9.9.
|
|
68
|
-
|
|
69
|
-
- **
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
- **
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
- **
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
retention policy.
|
|
67
|
+
The current release is **9.9.3 — Closed Loops**:
|
|
68
|
+
|
|
69
|
+
- **Multi-file projects, not single files.** "todo 앱 html+css+js" now infers
|
|
70
|
+
a project manifest, generates and validates every file through the same
|
|
71
|
+
sanitize pipeline, repairs cross-file references, verifies the bundle, and
|
|
72
|
+
offers a safe zip download — every file rides the `artifacts[]` contract.
|
|
73
|
+
- **Approval is a conversation, not a dead end.** Runs that need human
|
|
74
|
+
approval pause as `awaiting_approval` with a plan summary and a short-TTL
|
|
75
|
+
token; an inline card lets you approve, edit the plan, or cancel — the
|
|
76
|
+
fail-closed guarantee is unchanged.
|
|
77
|
+
- **First value in 30 seconds.** One click installs a 3-document demo corpus,
|
|
78
|
+
suggested questions prove recall with real sources, and answers carry an
|
|
79
|
+
honest grounding badge (근거 있음/근거 없음) bound to retrieved sources.
|
|
80
|
+
- **Retrieval that knows what you asked.** Hybrid search classifies queries
|
|
81
|
+
(fact/code/person/recency) and fuses channels with per-class weights, gated
|
|
82
|
+
by a benchmark-threshold CI test; a graph noise-curation job and an opt-in
|
|
83
|
+
folder watch keep the Brain fresh and clean.
|
|
84
|
+
- **Automation you can see and trust.** Per-automation "run now" with
|
|
85
|
+
dry-run-first, last execution surfaced on cards and the daily briefing,
|
|
86
|
+
failures routed to the Review queue — plus inline file previews, global
|
|
87
|
+
drag-and-drop capture, 409 conflict rebase, and a funnel-metrics endpoint.
|
|
88
|
+
- **A deeper harness.** 23 agent_eval scenarios including dirty-write filegen
|
|
89
|
+
paths, golden sanitize fixtures, a multi-model filegen benchmark, a
|
|
90
|
+
deterministic knowledge-pipeline E2E test, per-phase token budgets, and
|
|
91
|
+
`.tsx/.vue/.svelte` support with `ast.parse` Python validation.
|
|
93
92
|
|
|
94
93
|
Release notes: [RELEASE.md](RELEASE.md) · Full history: [docs/CHANGELOG.md](docs/CHANGELOG.md)
|
|
95
94
|
|
|
96
|
-
Expected artifacts for 9.9.
|
|
95
|
+
Expected artifacts for 9.9.3 release must use exact filenames:
|
|
97
96
|
|
|
98
|
-
- `dist/ltcai-9.9.
|
|
99
|
-
- `dist/ltcai-9.9.
|
|
100
|
-
- `ltcai-9.9.
|
|
101
|
-
- `dist/ltcai-9.9.
|
|
102
|
-
- `src-tauri/target/release/bundle/dmg/Lattice AI_9.9.
|
|
97
|
+
- `dist/ltcai-9.9.3-py3-none-any.whl`
|
|
98
|
+
- `dist/ltcai-9.9.3.tar.gz`
|
|
99
|
+
- `ltcai-9.9.3.tgz`
|
|
100
|
+
- `dist/ltcai-9.9.3.vsix`
|
|
101
|
+
- `src-tauri/target/release/bundle/dmg/Lattice AI_9.9.3_aarch64.dmg`
|
|
103
102
|
|
|
104
103
|
Do not use wildcard artifact uploads. Package registry publishing remains owner-run.
|
|
105
104
|
|
|
@@ -119,7 +118,7 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for details and
|
|
|
119
118
|
|
|
120
119
|
- External package registries are owner-published and can lag behind GitHub.
|
|
121
120
|
- PostgreSQL/pgvector is optional scale/migration tooling. SQLite remains the
|
|
122
|
-
live local Brain store in 9.9.
|
|
121
|
+
live local Brain store in 9.9.3.
|
|
123
122
|
- Docker, model downloads, cloud model calls, Telegram, Brain Network, and
|
|
124
123
|
update checks require explicit user action.
|
|
125
124
|
- Conversation does not fabricate answers when no model is loaded. Agent and
|
|
@@ -131,6 +130,8 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for details and
|
|
|
131
130
|
|
|
132
131
|
| Version | Theme |
|
|
133
132
|
| --- | --- |
|
|
133
|
+
| 9.9.3 | Closed Loops |
|
|
134
|
+
| 9.9.2 | Artifact Trust |
|
|
134
135
|
| 9.9.1 | Clean Foundations |
|
|
135
136
|
| 9.9.0 | Fail-Closed Trust |
|
|
136
137
|
| 9.8.0 | Honest Knowledge Pipeline |
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,86 @@ The top entry is either the current unreleased main-branch work or the current
|
|
|
4
4
|
release line. Older entries are historical and may describe behavior as it
|
|
5
5
|
existed at that release.
|
|
6
6
|
|
|
7
|
+
## [9.9.3] - 2026-07-22
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- Multi-file Artifact Loop: `infer_project_manifest` → per-file
|
|
11
|
+
generate/validate → cross-file reference repair → bundle validation →
|
|
12
|
+
`artifacts[]` + `project.zip_url`; safe `GET /tools/download_zip`.
|
|
13
|
+
- Interactive approval: governed plans pause as `awaiting_approval` with a
|
|
14
|
+
single-use 10-minute token; `POST /agent/resume` approves, edits, or
|
|
15
|
+
cancels. Inline approval card in chat.
|
|
16
|
+
- First Value Loop: `POST/GET/DELETE /api/setup/demo-corpus` (3 built-in demo
|
|
17
|
+
documents, `demo://` provenance, idempotent) + the "30초 체험" home track
|
|
18
|
+
with suggested-question chips.
|
|
19
|
+
- Answer grounding: `grounding {status, source_ids, overlap}` on chat
|
|
20
|
+
responses; 근거 있음/근거 없음 badge.
|
|
21
|
+
- Retrieval fusion: query-class detection (fact/code/person/recency) with
|
|
22
|
+
per-class channel weights and a benchmark-threshold CI gate
|
|
23
|
+
(`tests/unit/test_retrieval_fusion_gate.py`).
|
|
24
|
+
- Opt-in folder watch (`/api/ingestion/watch`, polling + persisted consent),
|
|
25
|
+
capture-quality CTA (`capture_quality` on browser captures), graph noise
|
|
26
|
+
curation job (`POST /knowledge-graph/curate/noise`, dry-run default).
|
|
27
|
+
- Automation visibility: `POST /api/automation/run-now` (dry-run-first),
|
|
28
|
+
`last_execution` on overview/briefing, failed runs → Review queue; Act-panel
|
|
29
|
+
"지금 한 번 실행" cards.
|
|
30
|
+
- UX: inline file preview (sandboxed iframe / modal), folder job report card,
|
|
31
|
+
global drag-and-drop capture, 409 proposal rebase flow, focus traps, graph
|
|
32
|
+
keyboard navigation, reduced-motion coverage, success pulse / inflow motes.
|
|
33
|
+
- Harness: agent_eval 23 scenarios (dirty-write filegen paths), golden
|
|
34
|
+
sanitize fixtures, `bench_models.py --filegen` (fail-open), deterministic
|
|
35
|
+
knowledge-pipeline E2E test, funnel metrics
|
|
36
|
+
(`GET /api/admin/funnel-metrics`), `PhaseBudgets` per-phase token caps,
|
|
37
|
+
`.ts/.tsx/.jsx/.vue/.svelte` filegen validation and Python `ast.parse`.
|
|
38
|
+
|
|
39
|
+
### Changed
|
|
40
|
+
- Approval-requiring agent runs return `awaiting_approval` instead of FAILED;
|
|
41
|
+
legacy `context_id` resume with `approved: true` now executes the gated
|
|
42
|
+
steps.
|
|
43
|
+
- Hybrid search responses expose `query_class`; chat responses expose
|
|
44
|
+
`grounding`; automation overview exposes `last_execution`.
|
|
45
|
+
|
|
46
|
+
### Fixed
|
|
47
|
+
- Fenced/chatty CSS is now sanitized before write (`.css` validation rejects
|
|
48
|
+
Markdown fences) — caught by the new golden fixtures.
|
|
49
|
+
- A legacy approved resume no longer re-fails at the approval gate.
|
|
50
|
+
|
|
51
|
+
## [9.9.2] - 2026-07-21
|
|
52
|
+
|
|
53
|
+
### Added
|
|
54
|
+
- ArtifactWritePipeline: `sanitize_write_content` in
|
|
55
|
+
`latticeai/core/file_generation.py` — a conservative validate-first →
|
|
56
|
+
extract → repair gate. The agent executor applies it to every `write_file`
|
|
57
|
+
`args.content` before dispatch, records a `content_sanitize` verdict on the
|
|
58
|
+
transcript, and tags `artifact_sanitize`/`artifact_repair` in the loop trace.
|
|
59
|
+
- Artifact-first chat contract: direct file-creation responses and agent runs
|
|
60
|
+
carry an `artifacts[]` array (`kind/path/filename/bytes/previewable/valid/
|
|
61
|
+
repaired`) via `collect_artifacts(transcript)`.
|
|
62
|
+
- Direct-path overwrite protection: an existing target is auto-suffixed
|
|
63
|
+
(`generated_page_2.html`) and announced, instead of silently overwritten.
|
|
64
|
+
- Generated files are optionally indexed into the Brain through the unified
|
|
65
|
+
`IngestionPipeline` (`workspace://` provenance, `origin: generated_file`);
|
|
66
|
+
disable with `LATTICEAI_INGEST_GENERATED=0`. The response reports an honest
|
|
67
|
+
`brain_ingest` status; ingest failures never fail the file creation.
|
|
68
|
+
- Plan schema enforcement (`normalize_plan`): non-empty goal, junk-step
|
|
69
|
+
filtering, `estimated_steps` clamping, and a deterministic single
|
|
70
|
+
`write_file` step for file-intent requests whose plan came back empty.
|
|
71
|
+
- Memory quality filter (`filter_learnings`): trivial and duplicate agent
|
|
72
|
+
learnings are dropped before entering the Brain.
|
|
73
|
+
- FG harness `tests/unit/test_artifact_write_scenarios.py` (FG-01..FG-08):
|
|
74
|
+
intent gating, filename inference, dirty-output extraction, truncated-HTML
|
|
75
|
+
repair, agent-path sanitization, how-to non-routing, scaffold validity.
|
|
76
|
+
- Frontend honesty surfaces: "Auto-repaired" badge on file cards
|
|
77
|
+
(`generation.repaired`), and a distinct warning strip for `NEEDS_REVIEW` /
|
|
78
|
+
`FAILED` terminal agent states (`role="alert"`, ko/en, dark/light).
|
|
79
|
+
|
|
80
|
+
### Changed
|
|
81
|
+
- HTML file validation now rejects documents wrapped in prose or Markdown
|
|
82
|
+
fences, so extraction gets a chance to slice out the real document instead
|
|
83
|
+
of the wrapper being saved as "valid".
|
|
84
|
+
- `product_readiness` `action-aware-chat` gate additionally proves the
|
|
85
|
+
ArtifactWritePipeline evidence (module, agent seam, FG harness test).
|
|
86
|
+
|
|
7
87
|
## [9.9.1] - 2026-07-21
|
|
8
88
|
|
|
9
89
|
### Removed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Community And Plugins
|
|
2
2
|
|
|
3
|
-
Current release: **9.9.
|
|
3
|
+
Current release: **9.9.3 — Closed Loops**.
|
|
4
4
|
|
|
5
5
|
LatticeAI defines the path from a strong local-first framework (8.4.0
|
|
6
6
|
action-aware baseline, 8.5.0 registry+DI hardening, 8.6.0 capture/navigation
|
package/docs/DEVELOPMENT.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
> **Status: canonical** — current contributor guidance, kept in sync with the
|
|
4
4
|
> current release.
|
|
5
5
|
|
|
6
|
-
Current release: **9.9.
|
|
6
|
+
Current release: **9.9.3 — Closed Loops**.
|
|
7
7
|
|
|
8
8
|
This document is for contributors working on the local-first Digital Brain
|
|
9
9
|
codebase. Product positioning and quick start stay in `README.md`; release
|
package/docs/ONBOARDING.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Lattice AI Onboarding
|
|
2
2
|
|
|
3
|
-
Current release: **9.9.
|
|
3
|
+
Current release: **9.9.3 — Closed Loops**.
|
|
4
4
|
|
|
5
5
|
The first-run goal is a five-minute path from "I opened the app" to "my Brain
|
|
6
6
|
has a source, a question, and proof." This page is the product contract behind
|
package/docs/OPERATIONS.md
CHANGED
package/docs/TRUST_MODEL.md
CHANGED
package/docs/WHY_LATTICE.md
CHANGED
package/docs/kg-schema.md
CHANGED
|
@@ -412,6 +412,142 @@ def curate_nodes(
|
|
|
412
412
|
return enriched
|
|
413
413
|
|
|
414
414
|
|
|
415
|
+
# ── Noise reduction (backlog #10, review §7.2 D) ─────────────────────────────
|
|
416
|
+
# Relation-verb normalization dictionary (ko/en). Keys are canonical verbs;
|
|
417
|
+
# values are the free-string labels observed in the legacy edge table. The
|
|
418
|
+
# canonical labels stay aligned with the v2 EdgeType legacy mapping so this
|
|
419
|
+
# never fights the schema normalization.
|
|
420
|
+
|
|
421
|
+
RELATION_VERB_GROUPS: Dict[str, List[str]] = {
|
|
422
|
+
"created": [
|
|
423
|
+
"creates", "create", "만들다", "만든", "만들었다", "만듦", "만들어냄",
|
|
424
|
+
"생성함", "생성", "생성했다", "작성함", "작성", "작성했다",
|
|
425
|
+
],
|
|
426
|
+
"mentions": ["mention", "언급함", "언급", "언급했다", "언급됨"],
|
|
427
|
+
"contains": ["contain", "포함함", "포함", "포함했다", "포함됨"],
|
|
428
|
+
"uses": ["use", "used", "사용함", "사용", "사용했다", "이용함", "이용"],
|
|
429
|
+
"related_to": ["related", "relates_to", "관련", "관련됨", "관련있음", "연관됨", "연관"],
|
|
430
|
+
"fixed": ["fixes", "fix", "수정함", "수정", "수정했다", "고침", "고쳤다"],
|
|
431
|
+
"decided": ["decides", "decide", "결정함", "결정", "결정했다"],
|
|
432
|
+
"uploaded": ["uploads", "upload", "업로드함", "업로드", "올림", "올렸다"],
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def build_relation_verb_index(
|
|
437
|
+
groups: Optional[Dict[str, List[str]]] = None,
|
|
438
|
+
) -> Dict[str, str]:
|
|
439
|
+
"""{alias(lower) → canonical} — canonical labels map to themselves."""
|
|
440
|
+
groups = groups or RELATION_VERB_GROUPS
|
|
441
|
+
index: Dict[str, str] = {}
|
|
442
|
+
for canonical, aliases in groups.items():
|
|
443
|
+
canon = str(canonical).strip().lower()
|
|
444
|
+
index[canon] = canon
|
|
445
|
+
for alias in aliases:
|
|
446
|
+
index[str(alias).strip().lower()] = canon
|
|
447
|
+
return index
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def normalize_relation_verb(
|
|
451
|
+
verb: str,
|
|
452
|
+
*,
|
|
453
|
+
index: Optional[Dict[str, str]] = None,
|
|
454
|
+
) -> str:
|
|
455
|
+
"""Map a free-string edge verb to its canonical form (identity if unknown).
|
|
456
|
+
|
|
457
|
+
Korean verbs are additionally tried with the trailing josa stripped so
|
|
458
|
+
"생성함을" style variants still normalize; unknown labels pass through
|
|
459
|
+
unchanged (lossless — this function is a rename map, not a filter).
|
|
460
|
+
"""
|
|
461
|
+
raw = str(verb or "").strip()
|
|
462
|
+
if not raw:
|
|
463
|
+
return raw
|
|
464
|
+
index = index if index is not None else build_relation_verb_index()
|
|
465
|
+
low = raw.lower()
|
|
466
|
+
if low in index:
|
|
467
|
+
return index[low]
|
|
468
|
+
stripped = _strip_josa(low)
|
|
469
|
+
if stripped in index:
|
|
470
|
+
return index[stripped]
|
|
471
|
+
return raw
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# v4 write-door enum labels are SCREAMING_SNAKE_CASE ASCII; those are already
|
|
475
|
+
# canonical schema taxonomy and must never be rewritten by the verb dictionary.
|
|
476
|
+
_V4_ENUM_LABEL_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def plan_relation_normalization(
|
|
480
|
+
edge_types: Iterable[str],
|
|
481
|
+
*,
|
|
482
|
+
index: Optional[Dict[str, str]] = None,
|
|
483
|
+
) -> Dict[str, str]:
|
|
484
|
+
"""{observed_type → canonical} for every type the dictionary changes.
|
|
485
|
+
|
|
486
|
+
Skips v4-canonical enum labels (``MENTIONS``, ``INDEXED_FROM``, …): this
|
|
487
|
+
plan targets the free-string verbs of pre-v4 rows, not the schema enum.
|
|
488
|
+
"""
|
|
489
|
+
index = index if index is not None else build_relation_verb_index()
|
|
490
|
+
plan: Dict[str, str] = {}
|
|
491
|
+
for edge_type in edge_types:
|
|
492
|
+
original = str(edge_type or "")
|
|
493
|
+
if _V4_ENUM_LABEL_RE.match(original):
|
|
494
|
+
continue
|
|
495
|
+
canonical = normalize_relation_verb(original, index=index)
|
|
496
|
+
if canonical and canonical != original:
|
|
497
|
+
plan[original] = canonical
|
|
498
|
+
return plan
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def plan_concept_noise_reduction(
|
|
502
|
+
concepts: Iterable[Dict[str, Any]],
|
|
503
|
+
total_docs: int,
|
|
504
|
+
*,
|
|
505
|
+
max_df_ratio: float = 0.8,
|
|
506
|
+
min_doc_frequency: int = 1,
|
|
507
|
+
min_corpus_docs: int = 5,
|
|
508
|
+
) -> Dict[str, List[Dict[str, Any]]]:
|
|
509
|
+
"""Decide which heuristic concept nodes are graph noise.
|
|
510
|
+
|
|
511
|
+
``concepts``: ``[{"id", "label", "df", "heuristic"}]`` where ``df`` is the
|
|
512
|
+
number of distinct content documents linking to the concept.
|
|
513
|
+
|
|
514
|
+
Rules (dry-run friendly — pure decision, no side effects):
|
|
515
|
+
|
|
516
|
+
* **never** flag ``heuristic=False`` nodes (explicit user-created nodes
|
|
517
|
+
are untouchable, whatever their stats);
|
|
518
|
+
* low IDF: with a corpus of at least ``min_corpus_docs`` documents, a
|
|
519
|
+
concept appearing in ``> max_df_ratio`` of them separates nothing —
|
|
520
|
+
remove;
|
|
521
|
+
* frequency floor: ``df < min_doc_frequency`` (orphaned/near-orphaned
|
|
522
|
+
auto concepts) — remove.
|
|
523
|
+
"""
|
|
524
|
+
total = max(0, int(total_docs))
|
|
525
|
+
remove: List[Dict[str, Any]] = []
|
|
526
|
+
keep: List[Dict[str, Any]] = []
|
|
527
|
+
for concept in concepts:
|
|
528
|
+
entry = {
|
|
529
|
+
"id": str(concept.get("id") or ""),
|
|
530
|
+
"label": concept.get("label"),
|
|
531
|
+
"df": int(concept.get("df") or 0),
|
|
532
|
+
"heuristic": bool(concept.get("heuristic")),
|
|
533
|
+
}
|
|
534
|
+
if not entry["heuristic"]:
|
|
535
|
+
keep.append({**entry, "reason": "user_created_protected"})
|
|
536
|
+
continue
|
|
537
|
+
df = entry["df"]
|
|
538
|
+
if df < int(min_doc_frequency):
|
|
539
|
+
df_ratio = (df / total) if total else 0.0
|
|
540
|
+
remove.append({**entry, "df_ratio": round(df_ratio, 4), "reason": "below_frequency_floor"})
|
|
541
|
+
continue
|
|
542
|
+
if total >= int(min_corpus_docs):
|
|
543
|
+
df_ratio = df / total
|
|
544
|
+
if df_ratio > float(max_df_ratio):
|
|
545
|
+
remove.append({**entry, "df_ratio": round(df_ratio, 4), "reason": "low_idf_ubiquitous"})
|
|
546
|
+
continue
|
|
547
|
+
keep.append({**entry, "reason": "signal"})
|
|
548
|
+
return {"remove": remove, "keep": keep}
|
|
549
|
+
|
|
550
|
+
|
|
415
551
|
# ── End-to-end helper ─────────────────────────────────────────────────────────
|
|
416
552
|
|
|
417
553
|
|
|
@@ -457,6 +593,11 @@ def auto_build_graph_overlay(
|
|
|
457
593
|
|
|
458
594
|
|
|
459
595
|
__all__ = [
|
|
596
|
+
"RELATION_VERB_GROUPS",
|
|
597
|
+
"build_relation_verb_index",
|
|
598
|
+
"normalize_relation_verb",
|
|
599
|
+
"plan_relation_normalization",
|
|
600
|
+
"plan_concept_noise_reduction",
|
|
460
601
|
"TopicCandidate",
|
|
461
602
|
"PromotionDecision",
|
|
462
603
|
"ThreadEdge",
|
|
@@ -6,6 +6,117 @@ from ._kg_common import * # noqa: F403,F401
|
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class KnowledgeGraphDocumentsMixin:
|
|
9
|
+
def find_documents_by_uri_prefix(self, prefix: str) -> List[Dict[str, Any]]:
|
|
10
|
+
"""Content nodes whose ``metadata.source_uri`` starts with ``prefix``.
|
|
11
|
+
|
|
12
|
+
Powers removable seeded corpora (e.g. the ``demo://`` First Value Loop
|
|
13
|
+
documents): callers stamp a URI scheme at ingest time and can later
|
|
14
|
+
enumerate exactly those nodes without guessing ids.
|
|
15
|
+
"""
|
|
16
|
+
prefix = str(prefix or "").strip()
|
|
17
|
+
if not prefix:
|
|
18
|
+
return []
|
|
19
|
+
nt, _ = self._read_tables()
|
|
20
|
+
documents: List[Dict[str, Any]] = []
|
|
21
|
+
with self._connect() as conn:
|
|
22
|
+
rows = conn.execute(
|
|
23
|
+
f"""
|
|
24
|
+
SELECT id, type, title, metadata_json, created_at, updated_at
|
|
25
|
+
FROM {nt}
|
|
26
|
+
WHERE json_extract(metadata_json, '$.source_uri') LIKE ? || '%'
|
|
27
|
+
AND type NOT IN ('Source', 'Chunk')
|
|
28
|
+
ORDER BY created_at ASC, id ASC
|
|
29
|
+
""",
|
|
30
|
+
(prefix,),
|
|
31
|
+
).fetchall()
|
|
32
|
+
for row in rows:
|
|
33
|
+
meta = _safe_loads(row["metadata_json"]) or {}
|
|
34
|
+
documents.append(
|
|
35
|
+
{
|
|
36
|
+
"id": row["id"],
|
|
37
|
+
"type": row["type"],
|
|
38
|
+
"title": row["title"],
|
|
39
|
+
"source_uri": meta.get("source_uri"),
|
|
40
|
+
"workspace_id": meta.get("workspace_id"),
|
|
41
|
+
"created_at": row["created_at"],
|
|
42
|
+
"updated_at": row["updated_at"],
|
|
43
|
+
}
|
|
44
|
+
)
|
|
45
|
+
return documents
|
|
46
|
+
|
|
47
|
+
def delete_document_tree(self, node_id: str) -> Dict[str, Any]:
|
|
48
|
+
"""Delete an ingested content node plus everything it owns.
|
|
49
|
+
|
|
50
|
+
Removes the node, its retrieval chunks (``chunks`` table + ``Chunk``
|
|
51
|
+
nodes), auto-extracted Task/Decision nodes whose ``source_node`` is
|
|
52
|
+
this document, every touching edge, the vector rows, and — when it
|
|
53
|
+
becomes orphaned — the linked ``Source`` node. Shared Concept nodes
|
|
54
|
+
are intentionally left alone (they may be cited by other content).
|
|
55
|
+
Mirrored into the v2 projection via ``_v2_delete_nodes``.
|
|
56
|
+
"""
|
|
57
|
+
node_id = str(node_id or "").strip()
|
|
58
|
+
if not node_id:
|
|
59
|
+
return {"status": "skipped", "removed_nodes": 0}
|
|
60
|
+
with self._connect() as conn:
|
|
61
|
+
if not conn.execute(
|
|
62
|
+
"SELECT 1 FROM nodes WHERE id=?", (node_id,)
|
|
63
|
+
).fetchone():
|
|
64
|
+
return {"status": "not_found", "node_id": node_id, "removed_nodes": 0}
|
|
65
|
+
|
|
66
|
+
remove_ids = {node_id}
|
|
67
|
+
# Owned children: chunks + auto-extracted semantic nodes that
|
|
68
|
+
# explicitly point back at this document via metadata.source_node.
|
|
69
|
+
for row in conn.execute(
|
|
70
|
+
"""
|
|
71
|
+
SELECT id FROM nodes
|
|
72
|
+
WHERE json_extract(metadata_json, '$.source_node') = ?
|
|
73
|
+
""",
|
|
74
|
+
(node_id,),
|
|
75
|
+
):
|
|
76
|
+
remove_ids.add(row["id"])
|
|
77
|
+
# The Source node this content was indexed from (if it ends up
|
|
78
|
+
# referenced by nothing else after the delete, drop it too).
|
|
79
|
+
source_ids = [
|
|
80
|
+
row["to_node"]
|
|
81
|
+
for row in conn.execute(
|
|
82
|
+
"SELECT to_node FROM edges WHERE from_node=? AND type IN ('indexed_from', 'INDEXED_FROM')",
|
|
83
|
+
(node_id,),
|
|
84
|
+
)
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
ids = list(remove_ids)
|
|
88
|
+
ph = ",".join("?" * len(ids))
|
|
89
|
+
conn.execute(f"DELETE FROM chunks WHERE source_node IN ({ph})", ids)
|
|
90
|
+
conn.execute(
|
|
91
|
+
f"DELETE FROM edges WHERE from_node IN ({ph}) OR to_node IN ({ph})",
|
|
92
|
+
ids * 2,
|
|
93
|
+
)
|
|
94
|
+
conn.execute(
|
|
95
|
+
f"DELETE FROM vector_embeddings WHERE item_id IN ({ph}) OR source_node IN ({ph})",
|
|
96
|
+
ids * 2,
|
|
97
|
+
)
|
|
98
|
+
conn.execute(f"DELETE FROM nodes WHERE id IN ({ph})", ids)
|
|
99
|
+
self._v2_delete_nodes(conn, ids)
|
|
100
|
+
|
|
101
|
+
removed_sources = 0
|
|
102
|
+
for source_id in source_ids:
|
|
103
|
+
still_linked = conn.execute(
|
|
104
|
+
"SELECT 1 FROM edges WHERE from_node=? OR to_node=? LIMIT 1",
|
|
105
|
+
(source_id, source_id),
|
|
106
|
+
).fetchone()
|
|
107
|
+
if still_linked:
|
|
108
|
+
continue
|
|
109
|
+
conn.execute("DELETE FROM vector_embeddings WHERE item_id=?", (source_id,))
|
|
110
|
+
conn.execute("DELETE FROM nodes WHERE id=?", (source_id,))
|
|
111
|
+
self._v2_delete_nodes(conn, [source_id])
|
|
112
|
+
removed_sources += 1
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
"status": "ok",
|
|
116
|
+
"node_id": node_id,
|
|
117
|
+
"removed_nodes": len(remove_ids) + removed_sources,
|
|
118
|
+
}
|
|
119
|
+
|
|
9
120
|
def _ingest_structure_nodes(
|
|
10
121
|
self,
|
|
11
122
|
conn: sqlite3.Connection,
|