ltcai 9.9.2 → 9.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -39
- package/docs/CHANGELOG.md +44 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +1 -1
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/kg-schema.md +1 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/graph/curator.py +141 -0
- package/lattice_brain/graph/documents.py +111 -0
- package/lattice_brain/graph/fusion.py +179 -0
- package/lattice_brain/graph/projection.py +161 -0
- package/lattice_brain/graph/retrieval.py +19 -1
- package/lattice_brain/ingestion.py +66 -0
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/automation_intelligence.py +159 -0
- package/latticeai/api/browser.py +15 -4
- package/latticeai/api/chat.py +27 -2
- package/latticeai/api/chat_agent_http.py +191 -3
- package/latticeai/api/chat_contracts.py +7 -1
- package/latticeai/api/chat_helpers.py +147 -0
- package/latticeai/api/chat_intents.py +189 -1
- package/latticeai/api/chat_stream.py +17 -1
- package/latticeai/api/funnel_metrics.py +31 -0
- package/latticeai/api/knowledge_graph.py +30 -0
- package/latticeai/api/local_files.py +90 -0
- package/latticeai/api/setup.py +134 -1
- package/latticeai/api/tools.py +33 -3
- package/latticeai/app_factory.py +14 -0
- package/latticeai/core/agent.py +91 -9
- package/latticeai/core/agent_eval.py +107 -0
- package/latticeai/core/file_generation.py +279 -2
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/runtime/persistence_runtime.py +19 -1
- package/latticeai/runtime/router_registration.py +8 -1
- package/latticeai/services/app_context.py +3 -0
- package/latticeai/services/architecture_readiness.py +1 -1
- package/latticeai/services/automation_execution.py +266 -0
- package/latticeai/services/automation_intelligence.py +7 -0
- package/latticeai/services/command_center.py +20 -1
- package/latticeai/services/folder_watch.py +391 -0
- package/latticeai/services/funnel_metrics.py +202 -0
- package/latticeai/services/product_readiness.py +1 -1
- package/latticeai/services/search_service.py +17 -1
- package/latticeai/setup/demo_corpus.py +98 -0
- package/latticeai/tools/filesystem.py +38 -1
- package/package.json +1 -1
- package/scripts/bench_models.py +234 -5
- package/scripts/check_current_release_docs.mjs +1 -1
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +28 -27
- package/static/app/assets/Act-DmdruVKV.js +1 -0
- package/static/app/assets/{Brain-BnGbytHa.js → Brain-DEY9jLVt.js} +2 -2
- package/static/app/assets/BrainHome-CeNaxjP1.js +2 -0
- package/static/app/assets/BrainSignals-CStjIqYi.js +1 -0
- package/static/app/assets/{Capture-CAKnLcb0.js → Capture-CVZ09QXi.js} +1 -1
- package/static/app/assets/CommandPalette-DepwOQFv.js +1 -0
- package/static/app/assets/Library-Bp0n-HlW.js +1 -0
- package/static/app/assets/{LivingBrain-C3VbogVo.js → LivingBrain-DxP4efJF.js} +1 -1
- package/static/app/assets/{ProductFlow-BLau7c7N.js → ProductFlow-DRbm7NEq.js} +1 -1
- package/static/app/assets/ReviewCard-C4HAO7A3.js +3 -0
- package/static/app/assets/{System-DRHc_6Su.js → System-ByQcmJW-.js} +1 -1
- package/static/app/assets/{bot-DZkDdh-A.js → bot-BNDyZLR7.js} +1 -1
- package/static/app/assets/circle-play-BkhdcHgd.js +1 -0
- package/static/app/assets/{cpu-DyvsDBbj.js → cpu-C6jjYm6i.js} +1 -1
- package/static/app/assets/{folder-open-CffoIr9t.js → folder-open-DjGIvDBQ.js} +1 -1
- package/static/app/assets/{hard-drive-C1oojrHb.js → hard-drive-BlSbwSaT.js} +1 -1
- package/static/app/assets/index-Bge3DXW7.css +2 -0
- package/static/app/assets/{index-BS_3aN1n.js → index-CHu7cgj3.js} +3 -3
- package/static/app/assets/{input-B_U7zvbR.js → input-DVDI0YR3.js} +1 -1
- package/static/app/assets/{navigation-xG-gyjbe.js → navigation-BddhEWA0.js} +1 -1
- package/static/app/assets/{network-BF13HJJF.js → network-pYQt5oBu.js} +1 -1
- package/static/app/assets/{primitives-DsuKh3M8.js → primitives-D7gCdEvS.js} +1 -1
- package/static/app/assets/shield-alert-K9RKGQeg.js +1 -0
- package/static/app/assets/{textarea-DsDL4ET8.js → textarea-sqQmoBKL.js} +1 -1
- package/static/app/assets/useFocusTrap-7EV9dFP2.js +1 -0
- package/static/app/assets/utils-uQYKXNeq.js +7 -0
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/static/app/assets/Act-Cx_gZNbE.js +0 -1
- package/static/app/assets/BrainHome-CRUMJL93.js +0 -2
- package/static/app/assets/BrainSignals-BP8-Qh2g.js +0 -1
- package/static/app/assets/CommandPalette-BoUTXBfm.js +0 -1
- package/static/app/assets/Library-PcfVUco6.js +0 -1
- package/static/app/assets/ReviewCard-CX5RqRqx.js +0 -1
- package/static/app/assets/download-C7yxYHvz.js +0 -1
- package/static/app/assets/index-DU78Kogz.css +0 -2
- package/static/app/assets/sunrise-CIP2ELJ1.js +0 -1
- package/static/app/assets/utils-Dh-Dwtkb.js +0 -7
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Built-in First Value Loop demo corpus (backlog #3, review §3.3 P0).
|
|
2
|
+
|
|
3
|
+
Three small Korean-friendly documents shipped in-repo so a brand-new user can
|
|
4
|
+
one-click seed the Brain and immediately experience a successful recall with
|
|
5
|
+
source highlighting — no folder connection, no model download, fully offline.
|
|
6
|
+
|
|
7
|
+
Contract:
|
|
8
|
+
|
|
9
|
+
* Documents are ingested through the normal :class:`IngestionPipeline`
|
|
10
|
+
(``source_type="note"``) with ``source_uri = demo://<id>`` and metadata
|
|
11
|
+
``{"demo_corpus": true}`` so they are identifiable and removable later.
|
|
12
|
+
* Ingestion is idempotent — the graph layer dedupes by content hash, so
|
|
13
|
+
re-POSTing never duplicates documents.
|
|
14
|
+
* :data:`SUGGESTED_QUESTIONS` are pre-filled questions whose answers exist in
|
|
15
|
+
the corpus (the UI renders them as "ask this" chips).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Any, Dict, List
|
|
21
|
+
|
|
22
|
+
DEMO_URI_PREFIX = "demo://"
|
|
23
|
+
DEMO_METADATA_FLAG = "demo_corpus"
|
|
24
|
+
|
|
25
|
+
DEMO_DOCUMENTS: List[Dict[str, str]] = [
|
|
26
|
+
{
|
|
27
|
+
"id": "meeting-note",
|
|
28
|
+
"title": "주간 회의록 — 사이드 프로젝트 킥오프",
|
|
29
|
+
"text": (
|
|
30
|
+
"2026-07-20 주간 회의록.\n"
|
|
31
|
+
"참석: 나, 김민준(백엔드), 박서연(디자인).\n"
|
|
32
|
+
"핵심 결정: 사이드 프로젝트 '새싹 가든'의 첫 공개 버전을 8월 15일에 "
|
|
33
|
+
"출시하기로 결정했다. 범위는 식물 기록과 물주기 알림 두 가지로 줄인다.\n"
|
|
34
|
+
"김민준이 알림 백엔드를 맡고, 박서연이 온보딩 화면을 맡는다.\n"
|
|
35
|
+
"다음 회의 전까지 각자 프로토타입을 준비하기로 했다."
|
|
36
|
+
),
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "project-doc",
|
|
40
|
+
"title": "프로젝트 개요 — 새싹 가든",
|
|
41
|
+
"text": (
|
|
42
|
+
"새싹 가든은 집에서 키우는 식물을 기록하는 작은 앱이다.\n"
|
|
43
|
+
"기술 스택: 프론트엔드는 React, 백엔드는 FastAPI, 데이터는 SQLite에 "
|
|
44
|
+
"로컬로 저장한다. 사진은 기기 밖으로 나가지 않는다.\n"
|
|
45
|
+
"첫 버전 목표: 식물 등록, 물주기 알림, 한 줄 관찰 일기.\n"
|
|
46
|
+
"수익화는 생각하지 않고, 주말에 만드는 것을 원칙으로 한다."
|
|
47
|
+
),
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "personal-note",
|
|
51
|
+
"title": "개인 노트 — 독서 메모: 아주 작은 습관의 힘",
|
|
52
|
+
"text": (
|
|
53
|
+
"『아주 작은 습관의 힘』을 읽고 남긴 메모.\n"
|
|
54
|
+
"가장 기억에 남는 문장: 습관은 목표가 아니라 시스템으로 만들어진다.\n"
|
|
55
|
+
"적용해 볼 것: 매일 아침 10분 스트레칭을 양치 직후에 붙여서 시작한다.\n"
|
|
56
|
+
"핵심은 2분 규칙 — 새 습관은 2분 안에 끝나는 크기로 시작하는 것이다."
|
|
57
|
+
),
|
|
58
|
+
},
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
# Pre-filled questions whose answers exist in the corpus above. ``source_uri``
|
|
62
|
+
# lets the UI verify the recall cited the right demo document.
|
|
63
|
+
SUGGESTED_QUESTIONS: List[Dict[str, str]] = [
|
|
64
|
+
{
|
|
65
|
+
"question": "회의에서 결정한 출시일이 언제야?",
|
|
66
|
+
"expected_source_uri": DEMO_URI_PREFIX + "meeting-note",
|
|
67
|
+
"expected_title": "주간 회의록 — 사이드 프로젝트 킥오프",
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"question": "새싹 가든의 기술 스택이 뭐야?",
|
|
71
|
+
"expected_source_uri": DEMO_URI_PREFIX + "project-doc",
|
|
72
|
+
"expected_title": "프로젝트 개요 — 새싹 가든",
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"question": "새 습관을 시작할 때 쓰는 2분 규칙이 뭐였지?",
|
|
76
|
+
"expected_source_uri": DEMO_URI_PREFIX + "personal-note",
|
|
77
|
+
"expected_title": "개인 노트 — 독서 메모: 아주 작은 습관의 힘",
|
|
78
|
+
},
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def demo_source_uri(doc_id: str) -> str:
|
|
83
|
+
return DEMO_URI_PREFIX + str(doc_id)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def suggested_questions() -> List[Dict[str, Any]]:
|
|
87
|
+
"""Copy of the suggestion chips (callers may annotate freely)."""
|
|
88
|
+
return [dict(item) for item in SUGGESTED_QUESTIONS]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
__all__ = [
|
|
92
|
+
"DEMO_DOCUMENTS",
|
|
93
|
+
"DEMO_METADATA_FLAG",
|
|
94
|
+
"DEMO_URI_PREFIX",
|
|
95
|
+
"SUGGESTED_QUESTIONS",
|
|
96
|
+
"demo_source_uri",
|
|
97
|
+
"suggested_questions",
|
|
98
|
+
]
|
|
@@ -6,11 +6,13 @@ Path resolution reads ``latticeai.tools.AGENT_ROOT`` so tests can redirect the s
|
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
+
import io
|
|
9
10
|
import json
|
|
10
11
|
import re
|
|
12
|
+
import zipfile
|
|
11
13
|
from html.parser import HTMLParser
|
|
12
14
|
from pathlib import Path
|
|
13
|
-
from typing import Any, Dict, List, Optional
|
|
15
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
14
16
|
|
|
15
17
|
import latticeai.tools as tools
|
|
16
18
|
from latticeai.tools import (
|
|
@@ -554,3 +556,38 @@ button { padding: 10px 14px; border-radius: 10px; border: 1px solid #d6d6d6; bac
|
|
|
554
556
|
"file_count": len(created),
|
|
555
557
|
"bytes": total_bytes,
|
|
556
558
|
}
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
# Generous cap for a generated-project archive — the workspace itself caps
|
|
562
|
+
# individual files at MAX_FILE_BYTES, this only bounds pathological trees.
|
|
563
|
+
MAX_ZIP_TOTAL_BYTES = 50_000_000
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def zip_workspace_dir(path: str, max_total_bytes: int = MAX_ZIP_TOTAL_BYTES) -> Tuple[bytes, str]:
|
|
567
|
+
"""Zip one workspace directory for download; confinement-safe by design.
|
|
568
|
+
|
|
569
|
+
Path resolution goes through the sandboxed ``_resolve_path`` (traversal
|
|
570
|
+
outside the agent workspace raises ``ToolError``), symlinks are skipped so
|
|
571
|
+
an in-workspace link can never leak files from elsewhere, and the archive
|
|
572
|
+
root is the directory name itself (``todo-app/index.html``).
|
|
573
|
+
"""
|
|
574
|
+
target = _resolve_path(path)
|
|
575
|
+
if target == tools.AGENT_ROOT:
|
|
576
|
+
raise ToolError("Cannot zip the entire workspace root — pick a project directory.")
|
|
577
|
+
if not target.exists() or not target.is_dir():
|
|
578
|
+
raise ToolError("Path is not a directory in the workspace.")
|
|
579
|
+
|
|
580
|
+
buffer = io.BytesIO()
|
|
581
|
+
total = 0
|
|
582
|
+
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
583
|
+
for child in sorted(target.rglob("*")):
|
|
584
|
+
if child.is_symlink() or not child.is_file():
|
|
585
|
+
continue
|
|
586
|
+
resolved = child.resolve()
|
|
587
|
+
if target not in resolved.parents:
|
|
588
|
+
continue # defense in depth — never archive escaped paths
|
|
589
|
+
total += child.stat().st_size
|
|
590
|
+
if total > max_total_bytes:
|
|
591
|
+
raise ToolError("Directory is too large to zip.")
|
|
592
|
+
archive.write(child, arcname=str(child.relative_to(target.parent)))
|
|
593
|
+
return buffer.getvalue(), f"{target.name}.zip"
|
package/package.json
CHANGED
package/scripts/bench_models.py
CHANGED
|
@@ -20,11 +20,11 @@ It measures **real code**: every output is fed through
|
|
|
20
20
|
``latticeai.core.agent.extract_action_details`` — the exact parser/repair
|
|
21
21
|
function the production loop uses (``agent.py`` calls it in plan/execute/verify).
|
|
22
22
|
An ``agent-loop`` reference row additionally runs the real
|
|
23
|
-
``latticeai.core.agent_eval.run_agent_eval`` state machine over its
|
|
24
|
-
scripted
|
|
23
|
+
``latticeai.core.agent_eval.run_agent_eval`` state machine over its full
|
|
24
|
+
scripted scenario suite.
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
Three modes
|
|
27
|
+
===========
|
|
28
28
|
1. **scripted** (default, always runnable, no model/network): a curated corpus
|
|
29
29
|
of model-realistic outputs per quality tier (frontier / mid-local /
|
|
30
30
|
weak-local). Proves the harness works and exposes the loop's repair boundary
|
|
@@ -34,6 +34,15 @@ Two modes
|
|
|
34
34
|
runs the *real* completions through the same parser, measuring true
|
|
35
35
|
end-to-end generation latency. Falls back to scripted with an honest message
|
|
36
36
|
if the endpoint is unreachable.
|
|
37
|
+
3. **filegen** (opt-in, ``--filegen``): the weekly multi-model file-generation
|
|
38
|
+
report. Discovers *installed* local gemma/qwen/llama MLX models via the
|
|
39
|
+
product's own model catalog + HF download checks, loads each with the real
|
|
40
|
+
``LLMRouter``, and drives the real ``generate_file_content`` pipeline
|
|
41
|
+
(prompt → extract → validate → retry → repair) for each canonical file type
|
|
42
|
+
(html/css/js/py/json/md). Reports a model × filetype success matrix.
|
|
43
|
+
**FAIL-OPEN by design**: no models installed (or a load failure) yields a
|
|
44
|
+
clear skip report and exit code 0 — this mode is a scheduled/manual report,
|
|
45
|
+
never a CI gate.
|
|
37
46
|
|
|
38
47
|
Honesty note on latency
|
|
39
48
|
========================
|
|
@@ -47,11 +56,14 @@ Usage
|
|
|
47
56
|
.venv/bin/python scripts/bench_models.py --json out.json # + machine output
|
|
48
57
|
.venv/bin/python scripts/bench_models.py \
|
|
49
58
|
--live-endpoint http://127.0.0.1:1234/v1 --model my-local-model
|
|
59
|
+
.venv/bin/python scripts/bench_models.py --filegen # weekly filegen report
|
|
60
|
+
.venv/bin/python scripts/bench_models.py --filegen --json filegen_report.json
|
|
50
61
|
"""
|
|
51
62
|
|
|
52
63
|
from __future__ import annotations
|
|
53
64
|
|
|
54
65
|
import argparse
|
|
66
|
+
import asyncio
|
|
55
67
|
import json
|
|
56
68
|
import statistics
|
|
57
69
|
import sys
|
|
@@ -59,12 +71,16 @@ import time
|
|
|
59
71
|
import urllib.error
|
|
60
72
|
import urllib.request
|
|
61
73
|
from pathlib import Path
|
|
62
|
-
from typing import Any, Dict, List, Tuple
|
|
74
|
+
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
|
|
63
75
|
|
|
64
76
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
65
77
|
|
|
66
78
|
from latticeai.core.agent import extract_action_details # noqa: E402
|
|
67
79
|
from latticeai.core.agent_eval import run_agent_eval # noqa: E402
|
|
80
|
+
from latticeai.core.file_generation import ( # noqa: E402
|
|
81
|
+
generate_file_content,
|
|
82
|
+
validate_file_content,
|
|
83
|
+
)
|
|
68
84
|
|
|
69
85
|
|
|
70
86
|
# ── Scripted corpora ─────────────────────────────────────────────────────
|
|
@@ -226,6 +242,204 @@ def _bench_live(base_url: str, model: str, *, timeout: float) -> Dict[str, Any]:
|
|
|
226
242
|
}
|
|
227
243
|
|
|
228
244
|
|
|
245
|
+
# ── Filegen benchmark (weekly multi-model report, fail-open) ─────────────
|
|
246
|
+
# One canonical request per supported file type. Requests are phrased the way
|
|
247
|
+
# users actually ask, so the report measures the real prompt+pipeline, not a
|
|
248
|
+
# synthetic best case.
|
|
249
|
+
FILEGEN_TARGETS: List[Tuple[str, str]] = [
|
|
250
|
+
("bench_page.html", "간단한 자기소개 웹 페이지를 만들어줘. 제목, 소개 문단, 연락처 목록을 포함해."),
|
|
251
|
+
("bench_styles.css", "Create a stylesheet with body typography, a header rule and a .card class."),
|
|
252
|
+
("bench_app.js", "Create a small browser script that renders a todo list and lets the user add items."),
|
|
253
|
+
("bench_tool.py", "Create a Python script that takes a CSV file path from argv and prints the row count."),
|
|
254
|
+
("bench_data.json", "Create a JSON document describing three sample books with title, author and year."),
|
|
255
|
+
("bench_notes.md", "Create a Markdown document with a title, two sections and a bullet list."),
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
_FILEGEN_FAMILIES = ("gemma", "qwen", "llama")
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def discover_filegen_models() -> List[Dict[str, str]]:
|
|
262
|
+
"""Installed local MLX models from the product catalog (gemma/qwen/llama).
|
|
263
|
+
|
|
264
|
+
Uses the same catalog + on-disk checks the runtime uses (``~/.ltcai``
|
|
265
|
+
model dir or the HF hub cache). Any import/probe failure is fail-open:
|
|
266
|
+
an empty list, never an exception.
|
|
267
|
+
"""
|
|
268
|
+
try:
|
|
269
|
+
from latticeai.models.router import (
|
|
270
|
+
_looks_like_hf_model_dir,
|
|
271
|
+
hf_cache_model_dir,
|
|
272
|
+
hf_model_dir,
|
|
273
|
+
)
|
|
274
|
+
from latticeai.services.model_catalog import ENGINE_MODEL_CATALOG
|
|
275
|
+
except Exception as exc: # noqa: BLE001 - discovery must never crash the report
|
|
276
|
+
print(f"filegen: model catalog unavailable ({exc}); no models discovered")
|
|
277
|
+
return []
|
|
278
|
+
found: List[Dict[str, str]] = []
|
|
279
|
+
for entry in ENGINE_MODEL_CATALOG.get("local_mlx", []):
|
|
280
|
+
model_id = str(entry.get("id") or "")
|
|
281
|
+
lowered = model_id.lower()
|
|
282
|
+
family = next((f for f in _FILEGEN_FAMILIES if f in lowered), None)
|
|
283
|
+
if not family:
|
|
284
|
+
continue
|
|
285
|
+
try:
|
|
286
|
+
downloaded = (
|
|
287
|
+
_looks_like_hf_model_dir(hf_model_dir(model_id))
|
|
288
|
+
or hf_cache_model_dir(model_id) is not None
|
|
289
|
+
)
|
|
290
|
+
except Exception: # noqa: BLE001
|
|
291
|
+
downloaded = False
|
|
292
|
+
if downloaded:
|
|
293
|
+
found.append({"id": model_id, "family": family})
|
|
294
|
+
return found
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
async def bench_filegen_model(
|
|
298
|
+
model_label: str,
|
|
299
|
+
generate_async: Callable[[str], Awaitable[str]],
|
|
300
|
+
targets: Optional[List[Tuple[str, str]]] = None,
|
|
301
|
+
) -> Dict[str, Any]:
|
|
302
|
+
"""Drive the real file-generation pipeline for every target file type.
|
|
303
|
+
|
|
304
|
+
``generate_async`` is the same ``context -> raw model text`` callable shape
|
|
305
|
+
the chat layer injects, so this measures exactly what production runs
|
|
306
|
+
(extraction, validation, corrective retry, deterministic repair included).
|
|
307
|
+
"""
|
|
308
|
+
rows: List[Dict[str, Any]] = []
|
|
309
|
+
for target_path, user_request in (targets or FILEGEN_TARGETS):
|
|
310
|
+
t0 = time.perf_counter()
|
|
311
|
+
content, meta = await generate_file_content(
|
|
312
|
+
generate_async, target_path=target_path, user_request=user_request,
|
|
313
|
+
)
|
|
314
|
+
elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 1)
|
|
315
|
+
valid, reason = validate_file_content(content, target_path)
|
|
316
|
+
attempts = meta.get("attempts") or []
|
|
317
|
+
rows.append({
|
|
318
|
+
"target": target_path,
|
|
319
|
+
"type": target_path.rsplit(".", 1)[-1],
|
|
320
|
+
"valid": valid, # valid after sanitize/repair
|
|
321
|
+
"reason": reason,
|
|
322
|
+
"clean_first_try": bool(attempts and attempts[0].get("valid")),
|
|
323
|
+
"repaired": bool(meta.get("repaired")),
|
|
324
|
+
"attempts": len(attempts),
|
|
325
|
+
"latency_ms": elapsed_ms,
|
|
326
|
+
"bytes": len(content.encode("utf-8", errors="replace")),
|
|
327
|
+
})
|
|
328
|
+
total = len(rows)
|
|
329
|
+
valid_count = sum(1 for r in rows if r["valid"])
|
|
330
|
+
clean_count = sum(1 for r in rows if r["valid"] and not r["repaired"])
|
|
331
|
+
return {
|
|
332
|
+
"model": model_label,
|
|
333
|
+
"targets": rows,
|
|
334
|
+
"total": total,
|
|
335
|
+
"success_rate": round(valid_count / total, 4) if total else 0.0,
|
|
336
|
+
# Of the valid files, how many did NOT need the deterministic repair
|
|
337
|
+
# fallback — the honest "the model itself produced usable output" rate.
|
|
338
|
+
"clean_rate": round(clean_count / total, 4) if total else 0.0,
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _make_router_generate(router: Any, model_id: str) -> Callable[[str], Awaitable[str]]:
|
|
343
|
+
"""Mirror the direct chat path's generation call (chat_intents)."""
|
|
344
|
+
|
|
345
|
+
async def _generate(context: str) -> str:
|
|
346
|
+
return str(
|
|
347
|
+
await router.generate_as(
|
|
348
|
+
model_id,
|
|
349
|
+
message="Return only the requested file content.",
|
|
350
|
+
context=context,
|
|
351
|
+
max_tokens=4096,
|
|
352
|
+
temperature=0.2,
|
|
353
|
+
)
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
return _generate
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
async def _filegen_run_models(models: List[Dict[str, str]]) -> List[Dict[str, Any]]:
|
|
360
|
+
from latticeai.models.router import LLMRouter
|
|
361
|
+
|
|
362
|
+
router = LLMRouter()
|
|
363
|
+
results: List[Dict[str, Any]] = []
|
|
364
|
+
for model in models:
|
|
365
|
+
model_id = model["id"]
|
|
366
|
+
try:
|
|
367
|
+
await router.load_model(model_id)
|
|
368
|
+
except Exception as exc: # noqa: BLE001 - fail-open per model
|
|
369
|
+
results.append({
|
|
370
|
+
"model": model_id, "family": model["family"],
|
|
371
|
+
"skipped": True, "reason": f"load failed: {exc}",
|
|
372
|
+
})
|
|
373
|
+
continue
|
|
374
|
+
result = await bench_filegen_model(model_id, _make_router_generate(router, model_id))
|
|
375
|
+
result["family"] = model["family"]
|
|
376
|
+
results.append(result)
|
|
377
|
+
try:
|
|
378
|
+
router.unload_model(model_id)
|
|
379
|
+
except Exception: # noqa: BLE001
|
|
380
|
+
pass
|
|
381
|
+
return results
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def run_filegen_benchmark(
|
|
385
|
+
models: Optional[List[Dict[str, str]]] = None,
|
|
386
|
+
) -> Dict[str, Any]:
|
|
387
|
+
"""Weekly model × filetype report. Fail-open: never raises, never gates."""
|
|
388
|
+
if models is None:
|
|
389
|
+
models = discover_filegen_models()
|
|
390
|
+
if not models:
|
|
391
|
+
return {
|
|
392
|
+
"mode": "filegen",
|
|
393
|
+
"skipped": True,
|
|
394
|
+
"reason": (
|
|
395
|
+
"no local gemma/qwen/llama models installed — install one via "
|
|
396
|
+
"the app's model picker, then re-run"
|
|
397
|
+
),
|
|
398
|
+
"models": [],
|
|
399
|
+
}
|
|
400
|
+
try:
|
|
401
|
+
results = asyncio.run(_filegen_run_models(models))
|
|
402
|
+
except Exception as exc: # noqa: BLE001 - fail-open at the run level too
|
|
403
|
+
return {
|
|
404
|
+
"mode": "filegen", "skipped": True,
|
|
405
|
+
"reason": f"benchmark run failed: {exc}", "models": [],
|
|
406
|
+
}
|
|
407
|
+
return {"mode": "filegen", "skipped": False, "models": results}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def format_filegen_report(report: Dict[str, Any]) -> str:
|
|
411
|
+
lines = [
|
|
412
|
+
"Weekly filegen benchmark (real pipeline: generate_file_content)",
|
|
413
|
+
"=" * 72,
|
|
414
|
+
]
|
|
415
|
+
if report.get("skipped"):
|
|
416
|
+
lines.append(f"SKIPPED (fail-open): {report.get('reason')}")
|
|
417
|
+
lines.append("=" * 72)
|
|
418
|
+
return "\n".join(lines)
|
|
419
|
+
for model in report.get("models", []):
|
|
420
|
+
if model.get("skipped"):
|
|
421
|
+
lines.append(f" {model['model']}: SKIPPED — {model.get('reason')}")
|
|
422
|
+
continue
|
|
423
|
+
lines.append(
|
|
424
|
+
f" {model['model']} "
|
|
425
|
+
f"(success={model['success_rate']}, clean={model['clean_rate']})"
|
|
426
|
+
)
|
|
427
|
+
for row in model.get("targets", []):
|
|
428
|
+
status = "ok" if row["valid"] else "FAIL"
|
|
429
|
+
if row["valid"] and row["repaired"]:
|
|
430
|
+
status = "ok(repaired)"
|
|
431
|
+
elif row["valid"] and not row["clean_first_try"]:
|
|
432
|
+
status = "ok(retry)"
|
|
433
|
+
lines.append(
|
|
434
|
+
f" {row['type']:<5} {status:<12} "
|
|
435
|
+
f"attempts={row['attempts']} latency_ms={row['latency_ms']} "
|
|
436
|
+
f"bytes={row['bytes']}"
|
|
437
|
+
)
|
|
438
|
+
lines.append("=" * 72)
|
|
439
|
+
lines.append("note: success = valid after sanitize/repair; clean = valid without repair fallback")
|
|
440
|
+
return "\n".join(lines)
|
|
441
|
+
|
|
442
|
+
|
|
229
443
|
# ── Reporting ────────────────────────────────────────────────────────────
|
|
230
444
|
def _fmt_row(name: str, r: Dict[str, Any]) -> str:
|
|
231
445
|
return (
|
|
@@ -243,8 +457,23 @@ def main(argv: List[str] | None = None) -> int:
|
|
|
243
457
|
parser.add_argument("--live-endpoint", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
|
|
244
458
|
parser.add_argument("--model", help="model id for the live endpoint")
|
|
245
459
|
parser.add_argument("--timeout", type=float, default=30.0, help="live request timeout seconds")
|
|
460
|
+
parser.add_argument(
|
|
461
|
+
"--filegen", action="store_true",
|
|
462
|
+
help="weekly multi-model file-generation report (fail-open, exit 0 even with no models)",
|
|
463
|
+
)
|
|
246
464
|
args = parser.parse_args(argv)
|
|
247
465
|
|
|
466
|
+
if args.filegen:
|
|
467
|
+
filegen_report = run_filegen_benchmark()
|
|
468
|
+
print(format_filegen_report(filegen_report))
|
|
469
|
+
if args.json_out:
|
|
470
|
+
Path(args.json_out).write_text(
|
|
471
|
+
json.dumps(filegen_report, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
472
|
+
)
|
|
473
|
+
print(f"wrote {args.json_out}")
|
|
474
|
+
# FAIL-OPEN: this is a scheduled report, never a CI gate.
|
|
475
|
+
return 0
|
|
476
|
+
|
|
248
477
|
report: Dict[str, Any] = {"mode": "scripted", "profiles": {}}
|
|
249
478
|
|
|
250
479
|
# Scripted matrix (always).
|
|
@@ -6,7 +6,7 @@ const root = process.cwd();
|
|
|
6
6
|
const pkg = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
|
|
7
7
|
const version = pkg.version;
|
|
8
8
|
const releaseDir = `output/release/v${version}`;
|
|
9
|
-
const releaseTheme = "
|
|
9
|
+
const releaseTheme = "Closed Loops";
|
|
10
10
|
const title = `${version} — ${releaseTheme}`;
|
|
11
11
|
const escapedVersion = version.replaceAll(".", "\\.");
|
|
12
12
|
|
package/src-tauri/Cargo.lock
CHANGED
package/src-tauri/Cargo.toml
CHANGED
|
@@ -1,35 +1,36 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "9.9.
|
|
2
|
+
"version": "9.9.3",
|
|
3
3
|
"generated_at": "vite",
|
|
4
4
|
"entrypoints": {
|
|
5
|
-
"app": "/static/app/assets/index-
|
|
5
|
+
"app": "/static/app/assets/index-CHu7cgj3.js"
|
|
6
6
|
},
|
|
7
7
|
"assets": {
|
|
8
8
|
"../node_modules/@tauri-apps/api/core.js": "/static/app/assets/core-CwxXejkd.js",
|
|
9
|
-
"_BrainSignals-
|
|
10
|
-
"_LivingBrain-
|
|
11
|
-
"_ReviewCard-
|
|
12
|
-
"_bot-
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"_folder-open-
|
|
16
|
-
"_hard-drive-
|
|
17
|
-
"_input-
|
|
18
|
-
"_navigation-
|
|
19
|
-
"_network-
|
|
20
|
-
"_primitives-
|
|
21
|
-
"
|
|
22
|
-
"_textarea-
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
"src/
|
|
28
|
-
"src/features/
|
|
29
|
-
"src/
|
|
30
|
-
"src/pages/
|
|
31
|
-
"src/pages/
|
|
32
|
-
"src/pages/
|
|
33
|
-
"src/pages/
|
|
9
|
+
"_BrainSignals-CStjIqYi.js": "/static/app/assets/BrainSignals-CStjIqYi.js",
|
|
10
|
+
"_LivingBrain-DxP4efJF.js": "/static/app/assets/LivingBrain-DxP4efJF.js",
|
|
11
|
+
"_ReviewCard-C4HAO7A3.js": "/static/app/assets/ReviewCard-C4HAO7A3.js",
|
|
12
|
+
"_bot-BNDyZLR7.js": "/static/app/assets/bot-BNDyZLR7.js",
|
|
13
|
+
"_circle-play-BkhdcHgd.js": "/static/app/assets/circle-play-BkhdcHgd.js",
|
|
14
|
+
"_cpu-C6jjYm6i.js": "/static/app/assets/cpu-C6jjYm6i.js",
|
|
15
|
+
"_folder-open-DjGIvDBQ.js": "/static/app/assets/folder-open-DjGIvDBQ.js",
|
|
16
|
+
"_hard-drive-BlSbwSaT.js": "/static/app/assets/hard-drive-BlSbwSaT.js",
|
|
17
|
+
"_input-DVDI0YR3.js": "/static/app/assets/input-DVDI0YR3.js",
|
|
18
|
+
"_navigation-BddhEWA0.js": "/static/app/assets/navigation-BddhEWA0.js",
|
|
19
|
+
"_network-pYQt5oBu.js": "/static/app/assets/network-pYQt5oBu.js",
|
|
20
|
+
"_primitives-D7gCdEvS.js": "/static/app/assets/primitives-D7gCdEvS.js",
|
|
21
|
+
"_shield-alert-K9RKGQeg.js": "/static/app/assets/shield-alert-K9RKGQeg.js",
|
|
22
|
+
"_textarea-sqQmoBKL.js": "/static/app/assets/textarea-sqQmoBKL.js",
|
|
23
|
+
"_useFocusTrap-7EV9dFP2.js": "/static/app/assets/useFocusTrap-7EV9dFP2.js",
|
|
24
|
+
"_utils-uQYKXNeq.js": "/static/app/assets/utils-uQYKXNeq.js",
|
|
25
|
+
"index.html": "/static/app/assets/index-CHu7cgj3.js",
|
|
26
|
+
"assets/index-Bge3DXW7.css": "/static/app/assets/index-Bge3DXW7.css",
|
|
27
|
+
"src/components/ProductFlow.tsx": "/static/app/assets/ProductFlow-DRbm7NEq.js",
|
|
28
|
+
"src/features/brain/BrainHome.tsx": "/static/app/assets/BrainHome-CeNaxjP1.js",
|
|
29
|
+
"src/features/command/CommandPalette.tsx": "/static/app/assets/CommandPalette-DepwOQFv.js",
|
|
30
|
+
"src/pages/Act.tsx": "/static/app/assets/Act-DmdruVKV.js",
|
|
31
|
+
"src/pages/Brain.tsx": "/static/app/assets/Brain-DEY9jLVt.js",
|
|
32
|
+
"src/pages/Capture.tsx": "/static/app/assets/Capture-CVZ09QXi.js",
|
|
33
|
+
"src/pages/Library.tsx": "/static/app/assets/Library-Bp0n-HlW.js",
|
|
34
|
+
"src/pages/System.tsx": "/static/app/assets/System-ByQcmJW-.js"
|
|
34
35
|
}
|
|
35
36
|
}
|