ltcai 9.9.2 → 9.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -39
- package/docs/CHANGELOG.md +109 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
- package/docs/DEVELOPMENT.md +1 -1
- package/docs/ONBOARDING.md +1 -1
- package/docs/OPERATIONS.md +1 -1
- package/docs/SURFACE_PARITY.md +47 -0
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/kg-schema.md +1 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/graph/_kg_common.py +335 -0
- package/lattice_brain/graph/curator.py +141 -0
- package/lattice_brain/graph/discovery_index.py +8 -1
- package/lattice_brain/graph/documents.py +111 -0
- package/lattice_brain/graph/fusion.py +179 -0
- package/lattice_brain/graph/ingest.py +43 -9
- package/lattice_brain/graph/projection.py +382 -37
- package/lattice_brain/graph/retrieval.py +71 -4
- package/lattice_brain/graph/retrieval_policy.py +174 -0
- package/lattice_brain/graph/retrieval_vector.py +122 -2
- package/lattice_brain/ingestion.py +66 -0
- package/lattice_brain/portability.py +41 -12
- package/lattice_brain/runtime/multi_agent.py +1 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/automation_intelligence.py +159 -0
- package/latticeai/api/browser.py +15 -4
- package/latticeai/api/chat.py +33 -2
- package/latticeai/api/chat_agent_http.py +363 -6
- package/latticeai/api/chat_contracts.py +7 -1
- package/latticeai/api/chat_helpers.py +147 -0
- package/latticeai/api/chat_intents.py +217 -10
- package/latticeai/api/chat_stream.py +93 -3
- package/latticeai/api/funnel_metrics.py +31 -0
- package/latticeai/api/knowledge_graph.py +65 -1
- package/latticeai/api/local_files.py +90 -0
- package/latticeai/api/setup.py +134 -1
- package/latticeai/api/tools.py +33 -3
- package/latticeai/app_factory.py +14 -0
- package/latticeai/core/agent.py +355 -18
- package/latticeai/core/agent_eval.py +107 -0
- package/latticeai/core/enterprise.py +5 -0
- package/latticeai/core/file_generation.py +404 -2
- package/latticeai/core/legacy_compatibility.py +1 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/run_store.py +243 -0
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/models/router.py +25 -12
- package/latticeai/runtime/persistence_runtime.py +19 -1
- package/latticeai/runtime/router_registration.py +8 -1
- package/latticeai/services/app_context.py +3 -0
- package/latticeai/services/architecture_readiness.py +1 -1
- package/latticeai/services/automation_execution.py +266 -0
- package/latticeai/services/automation_intelligence.py +7 -0
- package/latticeai/services/command_center.py +110 -2
- package/latticeai/services/folder_watch.py +396 -0
- package/latticeai/services/funnel_metrics.py +210 -0
- package/latticeai/services/product_readiness.py +1 -1
- package/latticeai/services/search_service.py +51 -5
- package/latticeai/setup/demo_corpus.py +98 -0
- package/latticeai/tools/filesystem.py +38 -1
- package/package.json +1 -1
- package/scripts/bench_agent_smoke.py +410 -0
- package/scripts/bench_models.py +234 -5
- package/scripts/check_current_release_docs.mjs +1 -1
- package/scripts/funnel_soft_gate.py +192 -0
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +29 -27
- package/static/app/assets/Act-BCmTU0E2.js +1 -0
- package/static/app/assets/{Brain-BnGbytHa.js → Brain-CTnjox7w.js} +2 -2
- package/static/app/assets/BrainHome-BJ3sFNX0.js +2 -0
- package/static/app/assets/BrainSignals-C52lwZVD.js +1 -0
- package/static/app/assets/{Capture-CAKnLcb0.js → Capture-B6vBhFa3.js} +1 -1
- package/static/app/assets/CommandPalette-90u9FWFH.js +1 -0
- package/static/app/assets/Library-pKCK0_tk.js +1 -0
- package/static/app/assets/{LivingBrain-C3VbogVo.js → LivingBrain-Jlf2wFqI.js} +1 -1
- package/static/app/assets/{ProductFlow-BLau7c7N.js → ProductFlow-yg1fKP1P.js} +1 -1
- package/static/app/assets/ReviewCard-DWvD7n9h.js +3 -0
- package/static/app/assets/{System-DRHc_6Su.js → System-BAEuHqNY.js} +1 -1
- package/static/app/assets/{bot-DZkDdh-A.js → bot-nB_buEZD.js} +1 -1
- package/static/app/assets/circle-pause-DLNw6Ucp.js +1 -0
- package/static/app/assets/circle-play-B-IsFL1y.js +1 -0
- package/static/app/assets/{cpu-DyvsDBbj.js → cpu-CEPBHaBl.js} +1 -1
- package/static/app/assets/{folder-open-CffoIr9t.js → folder-open-hmN0N9cX.js} +1 -1
- package/static/app/assets/{hard-drive-C1oojrHb.js → hard-drive-CBV_B_Yd.js} +1 -1
- package/static/app/assets/index-7FAfYm4v.css +2 -0
- package/static/app/assets/{index-BS_3aN1n.js → index-DrmOCySv.js} +3 -3
- package/static/app/assets/{input-B_U7zvbR.js → input-gtVCg-ll.js} +1 -1
- package/static/app/assets/{navigation-xG-gyjbe.js → navigation-Bot0hvuv.js} +1 -1
- package/static/app/assets/{network-BF13HJJF.js → network-jE42eKfT.js} +1 -1
- package/static/app/assets/{primitives-DsuKh3M8.js → primitives-CX2Komon.js} +1 -1
- package/static/app/assets/shield-alert-BftATuAA.js +1 -0
- package/static/app/assets/{textarea-DsDL4ET8.js → textarea-CiMJfOSI.js} +1 -1
- package/static/app/assets/useFocusTrap-B7RPGfFy.js +1 -0
- package/static/app/assets/utils-SJUNVOj5.js +7 -0
- package/static/app/index.html +4 -4
- package/static/sw.js +1 -1
- package/static/app/assets/Act-Cx_gZNbE.js +0 -1
- package/static/app/assets/BrainHome-CRUMJL93.js +0 -2
- package/static/app/assets/BrainSignals-BP8-Qh2g.js +0 -1
- package/static/app/assets/CommandPalette-BoUTXBfm.js +0 -1
- package/static/app/assets/Library-PcfVUco6.js +0 -1
- package/static/app/assets/ReviewCard-CX5RqRqx.js +0 -1
- package/static/app/assets/download-C7yxYHvz.js +0 -1
- package/static/app/assets/index-DU78Kogz.css +0 -2
- package/static/app/assets/sunrise-CIP2ELJ1.js +0 -1
- package/static/app/assets/utils-Dh-Dwtkb.js +0 -7
package/scripts/bench_models.py
CHANGED
|
@@ -20,11 +20,11 @@ It measures **real code**: every output is fed through
|
|
|
20
20
|
``latticeai.core.agent.extract_action_details`` — the exact parser/repair
|
|
21
21
|
function the production loop uses (``agent.py`` calls it in plan/execute/verify).
|
|
22
22
|
An ``agent-loop`` reference row additionally runs the real
|
|
23
|
-
``latticeai.core.agent_eval.run_agent_eval`` state machine over its
|
|
24
|
-
scripted
|
|
23
|
+
``latticeai.core.agent_eval.run_agent_eval`` state machine over its full
|
|
24
|
+
scripted scenario suite.
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
Three modes
|
|
27
|
+
===========
|
|
28
28
|
1. **scripted** (default, always runnable, no model/network): a curated corpus
|
|
29
29
|
of model-realistic outputs per quality tier (frontier / mid-local /
|
|
30
30
|
weak-local). Proves the harness works and exposes the loop's repair boundary
|
|
@@ -34,6 +34,15 @@ Two modes
|
|
|
34
34
|
runs the *real* completions through the same parser, measuring true
|
|
35
35
|
end-to-end generation latency. Falls back to scripted with an honest message
|
|
36
36
|
if the endpoint is unreachable.
|
|
37
|
+
3. **filegen** (opt-in, ``--filegen``): the weekly multi-model file-generation
|
|
38
|
+
report. Discovers *installed* local gemma/qwen/llama MLX models via the
|
|
39
|
+
product's own model catalog + HF download checks, loads each with the real
|
|
40
|
+
``LLMRouter``, and drives the real ``generate_file_content`` pipeline
|
|
41
|
+
(prompt → extract → validate → retry → repair) for each canonical file type
|
|
42
|
+
(html/css/js/py/json/md). Reports a model × filetype success matrix.
|
|
43
|
+
**FAIL-OPEN by design**: no models installed (or a load failure) yields a
|
|
44
|
+
clear skip report and exit code 0 — this mode is a scheduled/manual report,
|
|
45
|
+
never a CI gate.
|
|
37
46
|
|
|
38
47
|
Honesty note on latency
|
|
39
48
|
========================
|
|
@@ -47,11 +56,14 @@ Usage
|
|
|
47
56
|
.venv/bin/python scripts/bench_models.py --json out.json # + machine output
|
|
48
57
|
.venv/bin/python scripts/bench_models.py \
|
|
49
58
|
--live-endpoint http://127.0.0.1:1234/v1 --model my-local-model
|
|
59
|
+
.venv/bin/python scripts/bench_models.py --filegen # weekly filegen report
|
|
60
|
+
.venv/bin/python scripts/bench_models.py --filegen --json filegen_report.json
|
|
50
61
|
"""
|
|
51
62
|
|
|
52
63
|
from __future__ import annotations
|
|
53
64
|
|
|
54
65
|
import argparse
|
|
66
|
+
import asyncio
|
|
55
67
|
import json
|
|
56
68
|
import statistics
|
|
57
69
|
import sys
|
|
@@ -59,12 +71,16 @@ import time
|
|
|
59
71
|
import urllib.error
|
|
60
72
|
import urllib.request
|
|
61
73
|
from pathlib import Path
|
|
62
|
-
from typing import Any, Dict, List, Tuple
|
|
74
|
+
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
|
|
63
75
|
|
|
64
76
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
65
77
|
|
|
66
78
|
from latticeai.core.agent import extract_action_details # noqa: E402
|
|
67
79
|
from latticeai.core.agent_eval import run_agent_eval # noqa: E402
|
|
80
|
+
from latticeai.core.file_generation import ( # noqa: E402
|
|
81
|
+
generate_file_content,
|
|
82
|
+
validate_file_content,
|
|
83
|
+
)
|
|
68
84
|
|
|
69
85
|
|
|
70
86
|
# ── Scripted corpora ─────────────────────────────────────────────────────
|
|
@@ -226,6 +242,204 @@ def _bench_live(base_url: str, model: str, *, timeout: float) -> Dict[str, Any]:
|
|
|
226
242
|
}
|
|
227
243
|
|
|
228
244
|
|
|
245
|
+
# ── Filegen benchmark (weekly multi-model report, fail-open) ─────────────
|
|
246
|
+
# One canonical request per supported file type. Requests are phrased the way
|
|
247
|
+
# users actually ask, so the report measures the real prompt+pipeline, not a
|
|
248
|
+
# synthetic best case.
|
|
249
|
+
FILEGEN_TARGETS: List[Tuple[str, str]] = [
|
|
250
|
+
("bench_page.html", "간단한 자기소개 웹 페이지를 만들어줘. 제목, 소개 문단, 연락처 목록을 포함해."),
|
|
251
|
+
("bench_styles.css", "Create a stylesheet with body typography, a header rule and a .card class."),
|
|
252
|
+
("bench_app.js", "Create a small browser script that renders a todo list and lets the user add items."),
|
|
253
|
+
("bench_tool.py", "Create a Python script that takes a CSV file path from argv and prints the row count."),
|
|
254
|
+
("bench_data.json", "Create a JSON document describing three sample books with title, author and year."),
|
|
255
|
+
("bench_notes.md", "Create a Markdown document with a title, two sections and a bullet list."),
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
_FILEGEN_FAMILIES = ("gemma", "qwen", "llama")
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def discover_filegen_models() -> List[Dict[str, str]]:
|
|
262
|
+
"""Installed local MLX models from the product catalog (gemma/qwen/llama).
|
|
263
|
+
|
|
264
|
+
Uses the same catalog + on-disk checks the runtime uses (``~/.ltcai``
|
|
265
|
+
model dir or the HF hub cache). Any import/probe failure is fail-open:
|
|
266
|
+
an empty list, never an exception.
|
|
267
|
+
"""
|
|
268
|
+
try:
|
|
269
|
+
from latticeai.models.router import (
|
|
270
|
+
_looks_like_hf_model_dir,
|
|
271
|
+
hf_cache_model_dir,
|
|
272
|
+
hf_model_dir,
|
|
273
|
+
)
|
|
274
|
+
from latticeai.services.model_catalog import ENGINE_MODEL_CATALOG
|
|
275
|
+
except Exception as exc: # noqa: BLE001 - discovery must never crash the report
|
|
276
|
+
print(f"filegen: model catalog unavailable ({exc}); no models discovered")
|
|
277
|
+
return []
|
|
278
|
+
found: List[Dict[str, str]] = []
|
|
279
|
+
for entry in ENGINE_MODEL_CATALOG.get("local_mlx", []):
|
|
280
|
+
model_id = str(entry.get("id") or "")
|
|
281
|
+
lowered = model_id.lower()
|
|
282
|
+
family = next((f for f in _FILEGEN_FAMILIES if f in lowered), None)
|
|
283
|
+
if not family:
|
|
284
|
+
continue
|
|
285
|
+
try:
|
|
286
|
+
downloaded = (
|
|
287
|
+
_looks_like_hf_model_dir(hf_model_dir(model_id))
|
|
288
|
+
or hf_cache_model_dir(model_id) is not None
|
|
289
|
+
)
|
|
290
|
+
except Exception: # noqa: BLE001
|
|
291
|
+
downloaded = False
|
|
292
|
+
if downloaded:
|
|
293
|
+
found.append({"id": model_id, "family": family})
|
|
294
|
+
return found
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
async def bench_filegen_model(
|
|
298
|
+
model_label: str,
|
|
299
|
+
generate_async: Callable[[str], Awaitable[str]],
|
|
300
|
+
targets: Optional[List[Tuple[str, str]]] = None,
|
|
301
|
+
) -> Dict[str, Any]:
|
|
302
|
+
"""Drive the real file-generation pipeline for every target file type.
|
|
303
|
+
|
|
304
|
+
``generate_async`` is the same ``context -> raw model text`` callable shape
|
|
305
|
+
the chat layer injects, so this measures exactly what production runs
|
|
306
|
+
(extraction, validation, corrective retry, deterministic repair included).
|
|
307
|
+
"""
|
|
308
|
+
rows: List[Dict[str, Any]] = []
|
|
309
|
+
for target_path, user_request in (targets or FILEGEN_TARGETS):
|
|
310
|
+
t0 = time.perf_counter()
|
|
311
|
+
content, meta = await generate_file_content(
|
|
312
|
+
generate_async, target_path=target_path, user_request=user_request,
|
|
313
|
+
)
|
|
314
|
+
elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 1)
|
|
315
|
+
valid, reason = validate_file_content(content, target_path)
|
|
316
|
+
attempts = meta.get("attempts") or []
|
|
317
|
+
rows.append({
|
|
318
|
+
"target": target_path,
|
|
319
|
+
"type": target_path.rsplit(".", 1)[-1],
|
|
320
|
+
"valid": valid, # valid after sanitize/repair
|
|
321
|
+
"reason": reason,
|
|
322
|
+
"clean_first_try": bool(attempts and attempts[0].get("valid")),
|
|
323
|
+
"repaired": bool(meta.get("repaired")),
|
|
324
|
+
"attempts": len(attempts),
|
|
325
|
+
"latency_ms": elapsed_ms,
|
|
326
|
+
"bytes": len(content.encode("utf-8", errors="replace")),
|
|
327
|
+
})
|
|
328
|
+
total = len(rows)
|
|
329
|
+
valid_count = sum(1 for r in rows if r["valid"])
|
|
330
|
+
clean_count = sum(1 for r in rows if r["valid"] and not r["repaired"])
|
|
331
|
+
return {
|
|
332
|
+
"model": model_label,
|
|
333
|
+
"targets": rows,
|
|
334
|
+
"total": total,
|
|
335
|
+
"success_rate": round(valid_count / total, 4) if total else 0.0,
|
|
336
|
+
# Of the valid files, how many did NOT need the deterministic repair
|
|
337
|
+
# fallback — the honest "the model itself produced usable output" rate.
|
|
338
|
+
"clean_rate": round(clean_count / total, 4) if total else 0.0,
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _make_router_generate(router: Any, model_id: str) -> Callable[[str], Awaitable[str]]:
|
|
343
|
+
"""Mirror the direct chat path's generation call (chat_intents)."""
|
|
344
|
+
|
|
345
|
+
async def _generate(context: str) -> str:
|
|
346
|
+
return str(
|
|
347
|
+
await router.generate_as(
|
|
348
|
+
model_id,
|
|
349
|
+
message="Return only the requested file content.",
|
|
350
|
+
context=context,
|
|
351
|
+
max_tokens=4096,
|
|
352
|
+
temperature=0.2,
|
|
353
|
+
)
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
return _generate
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
async def _filegen_run_models(models: List[Dict[str, str]]) -> List[Dict[str, Any]]:
|
|
360
|
+
from latticeai.models.router import LLMRouter
|
|
361
|
+
|
|
362
|
+
router = LLMRouter()
|
|
363
|
+
results: List[Dict[str, Any]] = []
|
|
364
|
+
for model in models:
|
|
365
|
+
model_id = model["id"]
|
|
366
|
+
try:
|
|
367
|
+
await router.load_model(model_id)
|
|
368
|
+
except Exception as exc: # noqa: BLE001 - fail-open per model
|
|
369
|
+
results.append({
|
|
370
|
+
"model": model_id, "family": model["family"],
|
|
371
|
+
"skipped": True, "reason": f"load failed: {exc}",
|
|
372
|
+
})
|
|
373
|
+
continue
|
|
374
|
+
result = await bench_filegen_model(model_id, _make_router_generate(router, model_id))
|
|
375
|
+
result["family"] = model["family"]
|
|
376
|
+
results.append(result)
|
|
377
|
+
try:
|
|
378
|
+
router.unload_model(model_id)
|
|
379
|
+
except Exception: # noqa: BLE001
|
|
380
|
+
pass
|
|
381
|
+
return results
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def run_filegen_benchmark(
|
|
385
|
+
models: Optional[List[Dict[str, str]]] = None,
|
|
386
|
+
) -> Dict[str, Any]:
|
|
387
|
+
"""Weekly model × filetype report. Fail-open: never raises, never gates."""
|
|
388
|
+
if models is None:
|
|
389
|
+
models = discover_filegen_models()
|
|
390
|
+
if not models:
|
|
391
|
+
return {
|
|
392
|
+
"mode": "filegen",
|
|
393
|
+
"skipped": True,
|
|
394
|
+
"reason": (
|
|
395
|
+
"no local gemma/qwen/llama models installed — install one via "
|
|
396
|
+
"the app's model picker, then re-run"
|
|
397
|
+
),
|
|
398
|
+
"models": [],
|
|
399
|
+
}
|
|
400
|
+
try:
|
|
401
|
+
results = asyncio.run(_filegen_run_models(models))
|
|
402
|
+
except Exception as exc: # noqa: BLE001 - fail-open at the run level too
|
|
403
|
+
return {
|
|
404
|
+
"mode": "filegen", "skipped": True,
|
|
405
|
+
"reason": f"benchmark run failed: {exc}", "models": [],
|
|
406
|
+
}
|
|
407
|
+
return {"mode": "filegen", "skipped": False, "models": results}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def format_filegen_report(report: Dict[str, Any]) -> str:
|
|
411
|
+
lines = [
|
|
412
|
+
"Weekly filegen benchmark (real pipeline: generate_file_content)",
|
|
413
|
+
"=" * 72,
|
|
414
|
+
]
|
|
415
|
+
if report.get("skipped"):
|
|
416
|
+
lines.append(f"SKIPPED (fail-open): {report.get('reason')}")
|
|
417
|
+
lines.append("=" * 72)
|
|
418
|
+
return "\n".join(lines)
|
|
419
|
+
for model in report.get("models", []):
|
|
420
|
+
if model.get("skipped"):
|
|
421
|
+
lines.append(f" {model['model']}: SKIPPED — {model.get('reason')}")
|
|
422
|
+
continue
|
|
423
|
+
lines.append(
|
|
424
|
+
f" {model['model']} "
|
|
425
|
+
f"(success={model['success_rate']}, clean={model['clean_rate']})"
|
|
426
|
+
)
|
|
427
|
+
for row in model.get("targets", []):
|
|
428
|
+
status = "ok" if row["valid"] else "FAIL"
|
|
429
|
+
if row["valid"] and row["repaired"]:
|
|
430
|
+
status = "ok(repaired)"
|
|
431
|
+
elif row["valid"] and not row["clean_first_try"]:
|
|
432
|
+
status = "ok(retry)"
|
|
433
|
+
lines.append(
|
|
434
|
+
f" {row['type']:<5} {status:<12} "
|
|
435
|
+
f"attempts={row['attempts']} latency_ms={row['latency_ms']} "
|
|
436
|
+
f"bytes={row['bytes']}"
|
|
437
|
+
)
|
|
438
|
+
lines.append("=" * 72)
|
|
439
|
+
lines.append("note: success = valid after sanitize/repair; clean = valid without repair fallback")
|
|
440
|
+
return "\n".join(lines)
|
|
441
|
+
|
|
442
|
+
|
|
229
443
|
# ── Reporting ────────────────────────────────────────────────────────────
|
|
230
444
|
def _fmt_row(name: str, r: Dict[str, Any]) -> str:
|
|
231
445
|
return (
|
|
@@ -243,8 +457,23 @@ def main(argv: List[str] | None = None) -> int:
|
|
|
243
457
|
parser.add_argument("--live-endpoint", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
|
|
244
458
|
parser.add_argument("--model", help="model id for the live endpoint")
|
|
245
459
|
parser.add_argument("--timeout", type=float, default=30.0, help="live request timeout seconds")
|
|
460
|
+
parser.add_argument(
|
|
461
|
+
"--filegen", action="store_true",
|
|
462
|
+
help="weekly multi-model file-generation report (fail-open, exit 0 even with no models)",
|
|
463
|
+
)
|
|
246
464
|
args = parser.parse_args(argv)
|
|
247
465
|
|
|
466
|
+
if args.filegen:
|
|
467
|
+
filegen_report = run_filegen_benchmark()
|
|
468
|
+
print(format_filegen_report(filegen_report))
|
|
469
|
+
if args.json_out:
|
|
470
|
+
Path(args.json_out).write_text(
|
|
471
|
+
json.dumps(filegen_report, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
472
|
+
)
|
|
473
|
+
print(f"wrote {args.json_out}")
|
|
474
|
+
# FAIL-OPEN: this is a scheduled report, never a CI gate.
|
|
475
|
+
return 0
|
|
476
|
+
|
|
248
477
|
report: Dict[str, Any] = {"mode": "scripted", "profiles": {}}
|
|
249
478
|
|
|
250
479
|
# Scripted matrix (always).
|
|
@@ -6,7 +6,7 @@ const root = process.cwd();
|
|
|
6
6
|
const pkg = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
|
|
7
7
|
const version = pkg.version;
|
|
8
8
|
const releaseDir = `output/release/v${version}`;
|
|
9
|
-
const releaseTheme = "
|
|
9
|
+
const releaseTheme = "Durable Loops";
|
|
10
10
|
const title = `${version} — ${releaseTheme}`;
|
|
11
11
|
const escapedVersion = version.replaceAll(".", "\\.");
|
|
12
12
|
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Funnel soft gate (review Wave 3.3) — funnel metrics as an advisory verdict.
|
|
3
|
+
|
|
4
|
+
Reads the same ``funnel_metrics.json`` production writes and grades it through
|
|
5
|
+
``FunnelMetricsService.snapshot()`` — no rate logic is duplicated here. Two
|
|
6
|
+
funnel-health rates carry WARN thresholds:
|
|
7
|
+
|
|
8
|
+
* ``code_only_rate`` — WARN above 0.05 (goal: >95% of recognized file
|
|
9
|
+
requests deliver a real artifact, not a code/prose-only answer);
|
|
10
|
+
* ``needs_review_rate`` — WARN above 0.30 (verifier fail-closed is healthy,
|
|
11
|
+
but a third of runs ending NEEDS_REVIEW means verification is starving).
|
|
12
|
+
|
|
13
|
+
Rates whose denominator is still zero come back ``None`` from the snapshot;
|
|
14
|
+
they are reported as "no data" and never warn — a fresh install stays green.
|
|
15
|
+
``real_file_rate``, ``approval_resume_rate`` and TTFV are shown as
|
|
16
|
+
informational context only.
|
|
17
|
+
|
|
18
|
+
Exit codes
|
|
19
|
+
==========
|
|
20
|
+
* default — advisory: ALWAYS ``0``; warnings are printed, not enforced;
|
|
21
|
+
* ``--strict`` — CI opt-in: ``1`` when any WARN fired ("no data" never
|
|
22
|
+
fails, even under --strict);
|
|
23
|
+
* bad flags — argparse's usual ``2`` (the only real script error).
|
|
24
|
+
|
|
25
|
+
Usage
|
|
26
|
+
=====
|
|
27
|
+
.venv/bin/python scripts/funnel_soft_gate.py # advisory report
|
|
28
|
+
.venv/bin/python scripts/funnel_soft_gate.py --strict # CI gate opt-in
|
|
29
|
+
.venv/bin/python scripts/funnel_soft_gate.py --json # machine output
|
|
30
|
+
.venv/bin/python scripts/funnel_soft_gate.py --path /tmp/m.json
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import argparse
|
|
36
|
+
import json
|
|
37
|
+
import os
|
|
38
|
+
import sys
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import Any, Dict, List, Optional
|
|
41
|
+
|
|
42
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
43
|
+
|
|
44
|
+
from latticeai.services.funnel_metrics import FunnelMetricsService # noqa: E402
|
|
45
|
+
|
|
46
|
+
# Advisory thresholds (overridable per run via flags). WARN fires strictly
|
|
47
|
+
# ABOVE the threshold, so a rate sitting exactly on the goal boundary passes.
|
|
48
|
+
CODE_ONLY_WARN_THRESHOLD = 0.05
|
|
49
|
+
NEEDS_REVIEW_WARN_THRESHOLD = 0.30
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def default_metrics_path() -> Path:
|
|
53
|
+
"""The exact file production writes: ``Config.from_env`` data-dir pattern
|
|
54
|
+
(``LATTICEAI_DATA_DIR`` env or ``~/.ltcai``) + the ``funnel_metrics.json``
|
|
55
|
+
filename from ``latticeai/runtime/persistence_runtime.py``."""
|
|
56
|
+
data_dir = Path(os.environ.get("LATTICEAI_DATA_DIR") or (Path.home() / ".ltcai"))
|
|
57
|
+
return data_dir / "funnel_metrics.json"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def evaluate_snapshot(
|
|
61
|
+
snapshot: Dict[str, Any],
|
|
62
|
+
*,
|
|
63
|
+
code_only_warn: float = CODE_ONLY_WARN_THRESHOLD,
|
|
64
|
+
needs_review_warn: float = NEEDS_REVIEW_WARN_THRESHOLD,
|
|
65
|
+
) -> Dict[str, Any]:
|
|
66
|
+
"""Grade a ``FunnelMetricsService.snapshot()`` payload into verdicts."""
|
|
67
|
+
rates = snapshot.get("rates") or {}
|
|
68
|
+
checks: List[Dict[str, Any]] = []
|
|
69
|
+
|
|
70
|
+
def check(name: str, threshold: float, goal: str) -> None:
|
|
71
|
+
rate: Optional[float] = rates.get(name)
|
|
72
|
+
if rate is None:
|
|
73
|
+
verdict, detail = "no_data", "no denominator recorded yet"
|
|
74
|
+
elif rate > threshold:
|
|
75
|
+
verdict, detail = "warn", f"{rate} > {threshold} — {goal}"
|
|
76
|
+
else:
|
|
77
|
+
verdict, detail = "ok", f"{rate} <= {threshold}"
|
|
78
|
+
checks.append({
|
|
79
|
+
"name": name, "rate": rate, "threshold": threshold,
|
|
80
|
+
"verdict": verdict, "detail": detail,
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
check("code_only_rate", code_only_warn,
|
|
84
|
+
"goal: >95% of file requests deliver real files")
|
|
85
|
+
check("needs_review_rate", needs_review_warn,
|
|
86
|
+
"goal: <30% of agent runs end NEEDS_REVIEW")
|
|
87
|
+
|
|
88
|
+
warnings = [c["name"] for c in checks if c["verdict"] == "warn"]
|
|
89
|
+
if warnings:
|
|
90
|
+
status = "warn"
|
|
91
|
+
elif all(c["verdict"] == "no_data" for c in checks):
|
|
92
|
+
status = "no_data"
|
|
93
|
+
else:
|
|
94
|
+
status = "ok"
|
|
95
|
+
return {
|
|
96
|
+
"status": status,
|
|
97
|
+
"warnings": warnings,
|
|
98
|
+
"checks": checks,
|
|
99
|
+
# Informational context — never gated.
|
|
100
|
+
"info": {
|
|
101
|
+
"real_file_rate": rates.get("real_file_rate"),
|
|
102
|
+
"approval_resume_rate": rates.get("approval_resume_rate"),
|
|
103
|
+
"ttfv_seconds": snapshot.get("ttfv_seconds"),
|
|
104
|
+
"counters": snapshot.get("counters") or {},
|
|
105
|
+
},
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def format_report(payload: Dict[str, Any]) -> str:
|
|
110
|
+
lines = [
|
|
111
|
+
f"Funnel soft gate (advisory) — {payload['path']}",
|
|
112
|
+
"=" * 72,
|
|
113
|
+
]
|
|
114
|
+
for check in payload["checks"]:
|
|
115
|
+
rate = "no data" if check["rate"] is None else str(check["rate"])
|
|
116
|
+
verdict = {"warn": "WARN", "ok": "ok", "no_data": "-"}[check["verdict"]]
|
|
117
|
+
lines.append(
|
|
118
|
+
f" {check['name']:<20} {rate:<8} {verdict:<5} {check['detail']}"
|
|
119
|
+
)
|
|
120
|
+
info = payload["info"]
|
|
121
|
+
lines.append("-" * 72)
|
|
122
|
+
lines.append(
|
|
123
|
+
f" info: real_file_rate={info['real_file_rate']} "
|
|
124
|
+
f"approval_resume_rate={info['approval_resume_rate']} "
|
|
125
|
+
f"ttfv_seconds={info['ttfv_seconds']}"
|
|
126
|
+
)
|
|
127
|
+
lines.append("=" * 72)
|
|
128
|
+
if payload["status"] == "warn":
|
|
129
|
+
mode = (
|
|
130
|
+
"STRICT — exiting 1" if payload["strict"]
|
|
131
|
+
else "advisory — exit 0; opt into enforcement with --strict"
|
|
132
|
+
)
|
|
133
|
+
lines.append(f"verdict: WARN ({', '.join(payload['warnings'])}) [{mode}]")
|
|
134
|
+
elif payload["status"] == "no_data":
|
|
135
|
+
lines.append("verdict: no data yet — nothing to grade (exit 0)")
|
|
136
|
+
else:
|
|
137
|
+
lines.append("verdict: OK — funnel rates within goals")
|
|
138
|
+
return "\n".join(lines)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def main(argv: List[str] | None = None) -> int:
|
|
142
|
+
parser = argparse.ArgumentParser(
|
|
143
|
+
description="Advisory funnel-metrics gate (warn on funnel-health regressions)"
|
|
144
|
+
)
|
|
145
|
+
parser.add_argument(
|
|
146
|
+
"--path",
|
|
147
|
+
help="metrics JSON path (default: the production data-dir funnel_metrics.json)",
|
|
148
|
+
)
|
|
149
|
+
parser.add_argument(
|
|
150
|
+
"--strict", action="store_true",
|
|
151
|
+
help="exit 1 when any WARN fired (CI opt-in; 'no data' still exits 0)",
|
|
152
|
+
)
|
|
153
|
+
parser.add_argument(
|
|
154
|
+
"--json", dest="json_out", action="store_true",
|
|
155
|
+
help="print the machine-readable verdict JSON to stdout",
|
|
156
|
+
)
|
|
157
|
+
parser.add_argument(
|
|
158
|
+
"--code-only-warn", type=float, default=CODE_ONLY_WARN_THRESHOLD,
|
|
159
|
+
help=f"code_only_rate WARN threshold (default {CODE_ONLY_WARN_THRESHOLD})",
|
|
160
|
+
)
|
|
161
|
+
parser.add_argument(
|
|
162
|
+
"--needs-review-warn", type=float, default=NEEDS_REVIEW_WARN_THRESHOLD,
|
|
163
|
+
help=f"needs_review_rate WARN threshold (default {NEEDS_REVIEW_WARN_THRESHOLD})",
|
|
164
|
+
)
|
|
165
|
+
args = parser.parse_args(argv)
|
|
166
|
+
|
|
167
|
+
path = Path(args.path) if args.path else default_metrics_path()
|
|
168
|
+
# A missing or corrupt file yields the service's zeroed state → all rates
|
|
169
|
+
# None → "no data" (never a crash, never a failure).
|
|
170
|
+
snapshot = FunnelMetricsService(path).snapshot()
|
|
171
|
+
result = evaluate_snapshot(
|
|
172
|
+
snapshot,
|
|
173
|
+
code_only_warn=args.code_only_warn,
|
|
174
|
+
needs_review_warn=args.needs_review_warn,
|
|
175
|
+
)
|
|
176
|
+
exit_code = 1 if (args.strict and result["status"] == "warn") else 0
|
|
177
|
+
payload = {
|
|
178
|
+
"mode": "funnel-soft-gate",
|
|
179
|
+
"path": str(path),
|
|
180
|
+
"strict": bool(args.strict),
|
|
181
|
+
"exit_code": exit_code,
|
|
182
|
+
**result,
|
|
183
|
+
}
|
|
184
|
+
if args.json_out:
|
|
185
|
+
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
186
|
+
else:
|
|
187
|
+
print(format_report(payload))
|
|
188
|
+
return exit_code
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
if __name__ == "__main__":
|
|
192
|
+
raise SystemExit(main())
|
package/src-tauri/Cargo.lock
CHANGED
package/src-tauri/Cargo.toml
CHANGED
|
@@ -1,35 +1,37 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "9.9.
|
|
2
|
+
"version": "9.9.4",
|
|
3
3
|
"generated_at": "vite",
|
|
4
4
|
"entrypoints": {
|
|
5
|
-
"app": "/static/app/assets/index-
|
|
5
|
+
"app": "/static/app/assets/index-DrmOCySv.js"
|
|
6
6
|
},
|
|
7
7
|
"assets": {
|
|
8
8
|
"../node_modules/@tauri-apps/api/core.js": "/static/app/assets/core-CwxXejkd.js",
|
|
9
|
-
"_BrainSignals-
|
|
10
|
-
"_LivingBrain-
|
|
11
|
-
"_ReviewCard-
|
|
12
|
-
"_bot-
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
"
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
"
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"src/
|
|
29
|
-
"src/
|
|
30
|
-
"src/
|
|
31
|
-
"src/pages/
|
|
32
|
-
"src/pages/
|
|
33
|
-
"src/pages/
|
|
9
|
+
"_BrainSignals-C52lwZVD.js": "/static/app/assets/BrainSignals-C52lwZVD.js",
|
|
10
|
+
"_LivingBrain-Jlf2wFqI.js": "/static/app/assets/LivingBrain-Jlf2wFqI.js",
|
|
11
|
+
"_ReviewCard-DWvD7n9h.js": "/static/app/assets/ReviewCard-DWvD7n9h.js",
|
|
12
|
+
"_bot-nB_buEZD.js": "/static/app/assets/bot-nB_buEZD.js",
|
|
13
|
+
"_circle-pause-DLNw6Ucp.js": "/static/app/assets/circle-pause-DLNw6Ucp.js",
|
|
14
|
+
"_circle-play-B-IsFL1y.js": "/static/app/assets/circle-play-B-IsFL1y.js",
|
|
15
|
+
"_cpu-CEPBHaBl.js": "/static/app/assets/cpu-CEPBHaBl.js",
|
|
16
|
+
"_folder-open-hmN0N9cX.js": "/static/app/assets/folder-open-hmN0N9cX.js",
|
|
17
|
+
"_hard-drive-CBV_B_Yd.js": "/static/app/assets/hard-drive-CBV_B_Yd.js",
|
|
18
|
+
"_input-gtVCg-ll.js": "/static/app/assets/input-gtVCg-ll.js",
|
|
19
|
+
"_navigation-Bot0hvuv.js": "/static/app/assets/navigation-Bot0hvuv.js",
|
|
20
|
+
"_network-jE42eKfT.js": "/static/app/assets/network-jE42eKfT.js",
|
|
21
|
+
"_primitives-CX2Komon.js": "/static/app/assets/primitives-CX2Komon.js",
|
|
22
|
+
"_shield-alert-BftATuAA.js": "/static/app/assets/shield-alert-BftATuAA.js",
|
|
23
|
+
"_textarea-CiMJfOSI.js": "/static/app/assets/textarea-CiMJfOSI.js",
|
|
24
|
+
"_useFocusTrap-B7RPGfFy.js": "/static/app/assets/useFocusTrap-B7RPGfFy.js",
|
|
25
|
+
"_utils-SJUNVOj5.js": "/static/app/assets/utils-SJUNVOj5.js",
|
|
26
|
+
"index.html": "/static/app/assets/index-DrmOCySv.js",
|
|
27
|
+
"assets/index-7FAfYm4v.css": "/static/app/assets/index-7FAfYm4v.css",
|
|
28
|
+
"src/components/ProductFlow.tsx": "/static/app/assets/ProductFlow-yg1fKP1P.js",
|
|
29
|
+
"src/features/brain/BrainHome.tsx": "/static/app/assets/BrainHome-BJ3sFNX0.js",
|
|
30
|
+
"src/features/command/CommandPalette.tsx": "/static/app/assets/CommandPalette-90u9FWFH.js",
|
|
31
|
+
"src/pages/Act.tsx": "/static/app/assets/Act-BCmTU0E2.js",
|
|
32
|
+
"src/pages/Brain.tsx": "/static/app/assets/Brain-CTnjox7w.js",
|
|
33
|
+
"src/pages/Capture.tsx": "/static/app/assets/Capture-B6vBhFa3.js",
|
|
34
|
+
"src/pages/Library.tsx": "/static/app/assets/Library-pKCK0_tk.js",
|
|
35
|
+
"src/pages/System.tsx": "/static/app/assets/System-BAEuHqNY.js"
|
|
34
36
|
}
|
|
35
37
|
}
|