ltcai 9.5.0 → 9.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -43
- package/auto_setup.py +11 -1
- package/docs/CHANGELOG.md +86 -0
- package/docs/COMMUNITY_AND_PLUGINS.md +2 -2
- package/docs/DEVELOPMENT.md +8 -8
- package/docs/LEGACY_COMPATIBILITY.md +1 -1
- package/docs/ONBOARDING.md +2 -2
- package/docs/OPERATIONS.md +1 -1
- package/docs/PERFORMANCE.md +106 -0
- package/docs/TRUST_MODEL.md +1 -1
- package/docs/WHY_LATTICE.md +1 -1
- package/docs/kg-schema.md +1 -1
- package/docs/mcp-tools.md +1 -1
- package/kg_schema.py +11 -1
- package/knowledge_graph.py +12 -2
- package/knowledge_graph_api.py +11 -1
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/graph/proactive.py +583 -0
- package/lattice_brain/graph/retrieval.py +211 -7
- package/lattice_brain/graph/retrieval_vector.py +102 -0
- package/lattice_brain/ingestion.py +360 -4
- package/lattice_brain/quality.py +44 -9
- package/lattice_brain/runtime/multi_agent.py +38 -2
- package/latticeai/__init__.py +1 -1
- package/latticeai/api/brain_intelligence.py +27 -2
- package/latticeai/api/change_proposals.py +89 -0
- package/latticeai/api/chat_agent_http.py +2 -0
- package/latticeai/api/review_queue.py +66 -2
- package/latticeai/app_factory.py +23 -0
- package/latticeai/cli/entrypoint.py +3 -1
- package/latticeai/core/agent.py +273 -101
- package/latticeai/core/agent_eval.py +411 -0
- package/latticeai/core/agent_trace.py +104 -0
- package/latticeai/core/legacy_compatibility.py +15 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/tool_governor.py +101 -0
- package/latticeai/core/workspace_os.py +1 -1
- package/latticeai/runtime/router_registration.py +2 -0
- package/latticeai/services/architecture_readiness.py +1 -1
- package/latticeai/services/brain_intelligence.py +97 -1
- package/latticeai/services/change_proposals.py +322 -0
- package/latticeai/services/product_readiness.py +1 -1
- package/latticeai/services/review_queue.py +47 -3
- package/latticeai/tools/__init__.py +6 -1
- package/llm_router.py +10 -1
- package/local_knowledge_api.py +10 -1
- package/ltcai_cli.py +11 -1
- package/mcp_registry.py +10 -1
- package/p_reinforce.py +10 -1
- package/package.json +1 -1
- package/scripts/agent_eval.py +46 -0
- package/scripts/brain_quality_eval.py +1 -1
- package/scripts/check_current_release_docs.mjs +2 -1
- package/scripts/profile_kg.py +360 -0
- package/setup_wizard.py +10 -1
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +11 -11
- package/static/app/assets/{Act-Cdfqx4wN.js → Act-B6c39ays.js} +2 -1
- package/static/app/assets/{Brain-w_tAuyg6.js → Brain-D7Qg4k6M.js} +1 -1
- package/static/app/assets/{Capture-iJwi9LmS.js → Capture-VF_di68r.js} +1 -1
- package/static/app/assets/{Library-Do-jjhzi.js → Library-D_Gis2PA.js} +1 -1
- package/static/app/assets/{System-DFg_q3E6.js → System-C5s5H2ov.js} +1 -1
- package/static/app/assets/{index-BN6HIWVC.css → index-85wQvEie.css} +1 -1
- package/static/app/assets/index-DJC_2oub.js +18 -0
- package/static/app/assets/{primitives-ZTUlU7pR.js → primitives-DL4Nip8C.js} +1 -1
- package/static/app/assets/{textarea-CMfhqPhE.js → textarea-woZfCXHy.js} +1 -1
- package/static/app/index.html +2 -2
- package/static/sw.js +1 -1
- package/static/app/assets/index-aJuRi-Xo.js +0 -17
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
"""Deterministic agent-loop evaluation harness (v9.6.0).
|
|
2
|
+
|
|
3
|
+
The Brain has a safety harness (governance, approval gates, audit) but until
|
|
4
|
+
9.6.0 no *evaluation* harness: nothing measured whether the reasoning loop
|
|
5
|
+
actually completes tasks, recovers from weak-model formatting slips, or
|
|
6
|
+
respects its guards. This module closes that gap without any model:
|
|
7
|
+
|
|
8
|
+
* every scenario scripts the exact model replies (including the malformed
|
|
9
|
+
outputs small local models really produce — think blocks, Python dict
|
|
10
|
+
literals, trailing commas, prose) and drives the real
|
|
11
|
+
:class:`~latticeai.core.agent.SingleAgentRuntime` state machine over fake
|
|
12
|
+
ports;
|
|
13
|
+
* expectations are asserted against the loop's own :class:`LoopTrace`
|
|
14
|
+
summary, so the harness measures the same observability surface the API
|
|
15
|
+
exposes;
|
|
16
|
+
* the result is a scoreboard (`scenarios`, `passed`, `success_rate`,
|
|
17
|
+
aggregate recovery stats) consumed by ``scripts/agent_eval.py`` as a
|
|
18
|
+
release gate.
|
|
19
|
+
|
|
20
|
+
Deterministic by construction: no model, no network, no filesystem writes
|
|
21
|
+
(the tool port records calls in memory).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import asyncio
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any, Dict, List
|
|
30
|
+
|
|
31
|
+
from latticeai.core.agent import (
|
|
32
|
+
AgentDeps,
|
|
33
|
+
AgentRunContext,
|
|
34
|
+
AgentState,
|
|
35
|
+
SingleAgentRuntime,
|
|
36
|
+
)
|
|
37
|
+
from latticeai.tools import ToolError
|
|
38
|
+
|
|
39
|
+
_AUTO_POLICY = {
|
|
40
|
+
"auto_approve": True, "risk": "low", "shell": False, "network": False,
|
|
41
|
+
"destructive": False, "sandbox": False, "rollback": "none",
|
|
42
|
+
}
|
|
43
|
+
_DESTRUCTIVE_POLICY = {
|
|
44
|
+
"auto_approve": False, "risk": "destructive", "shell": True, "network": False,
|
|
45
|
+
"destructive": True, "sandbox": False, "rollback": "none",
|
|
46
|
+
}
|
|
47
|
+
# Mirrors production write-tool governance when the change governor is wired:
|
|
48
|
+
# not auto-approved, so only the governor's additive/proposed verdicts (never
|
|
49
|
+
# the model) decide whether a write runs or is staged for review.
|
|
50
|
+
_GOVERNED_WRITE_POLICY = {
|
|
51
|
+
"auto_approve": False, "risk": "write", "shell": False, "network": False,
|
|
52
|
+
"destructive": False, "sandbox": "workspace", "rollback": "git",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _EvalChangeGovernor:
|
|
57
|
+
"""Deterministic stand-in for ChangeProposalService's governor port.
|
|
58
|
+
|
|
59
|
+
Mirrors the wire contract of
|
|
60
|
+
:meth:`latticeai.services.change_proposals.ChangeProposalService.review`:
|
|
61
|
+
``None`` falls through, additive writes get ``allow_additive``, and
|
|
62
|
+
mutations of "existing" paths come back ``proposed`` with a proposal id —
|
|
63
|
+
exactly what the agent loop routes into the review queue.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
governed_tools = frozenset({"write_file", "edit_file"})
|
|
67
|
+
|
|
68
|
+
def __init__(self) -> None:
|
|
69
|
+
self.proposals: List[Dict[str, Any]] = []
|
|
70
|
+
|
|
71
|
+
def review(self, name, args, *, policy=None, user_email=None, workspace_id=None, conversation_id=None):
|
|
72
|
+
if name not in self.governed_tools:
|
|
73
|
+
return None
|
|
74
|
+
path = str(args.get("path") or "")
|
|
75
|
+
if path.startswith("existing"):
|
|
76
|
+
proposal = {"id": f"eval-proposal-{len(self.proposals) + 1}", "path": path}
|
|
77
|
+
self.proposals.append(proposal)
|
|
78
|
+
return {
|
|
79
|
+
"decision": "proposed",
|
|
80
|
+
"classification": {"change_class": "mutation"},
|
|
81
|
+
"proposal": proposal,
|
|
82
|
+
}
|
|
83
|
+
return {"decision": "allow_additive", "classification": {"change_class": "additive"}}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class Scenario:
|
|
88
|
+
"""One scripted conversation through the real agent state machine."""
|
|
89
|
+
|
|
90
|
+
name: str
|
|
91
|
+
replies: List[str]
|
|
92
|
+
expect_state: str = "DONE"
|
|
93
|
+
# Each key is compared against the LoopTrace summary with >= / == / <=
|
|
94
|
+
expect_min: Dict[str, int] = field(default_factory=dict)
|
|
95
|
+
expect_exact: Dict[str, Any] = field(default_factory=dict)
|
|
96
|
+
expect_tool_outcomes: Dict[str, int] = field(default_factory=dict)
|
|
97
|
+
# Exact ordered sequence of tool names that actually executed (successful
|
|
98
|
+
# execute_tool calls only — proposed/blocked/failed calls never run).
|
|
99
|
+
expect_tool_calls: List[str] = field(default_factory=list)
|
|
100
|
+
# Wire a change governor so governed tools follow the proposal path.
|
|
101
|
+
use_governor: bool = False
|
|
102
|
+
max_steps: int = 8
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class _Req:
|
|
106
|
+
conversation_id = None
|
|
107
|
+
temperature = 0.2
|
|
108
|
+
workspace_id = None
|
|
109
|
+
source = "agent_eval"
|
|
110
|
+
|
|
111
|
+
def __init__(self, message: str) -> None:
|
|
112
|
+
self.message = message
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _build_deps(
|
|
116
|
+
replies: List[str],
|
|
117
|
+
tool_log: List[Dict[str, Any]],
|
|
118
|
+
*,
|
|
119
|
+
governor: Any = None,
|
|
120
|
+
) -> AgentDeps:
|
|
121
|
+
queue = list(replies)
|
|
122
|
+
|
|
123
|
+
async def generate_as(model_id, message, context, max_tokens, temperature):
|
|
124
|
+
if not queue:
|
|
125
|
+
# A scenario that exhausts its script means the loop asked for
|
|
126
|
+
# more turns than expected — end it deterministically.
|
|
127
|
+
return '{"action": "verdict", "verdict": "PASS", "next_state": "DONE", "reason": "script exhausted"}'
|
|
128
|
+
return queue.pop(0)
|
|
129
|
+
|
|
130
|
+
async def generate(**kwargs):
|
|
131
|
+
return '{"action": "noop"}'
|
|
132
|
+
|
|
133
|
+
def execute_tool(name: str, args: dict) -> dict:
|
|
134
|
+
# File-producing tools fail like the real dispatcher when the model
|
|
135
|
+
# forgets the target path — scenarios use this to exercise recovery.
|
|
136
|
+
if name in ("write_file", "generate_file") and not args.get("path"):
|
|
137
|
+
raise ToolError(f"{name} requires args.path")
|
|
138
|
+
tool_log.append({"name": name, "args": args})
|
|
139
|
+
return {"ok": True, "path": args.get("path", "")}
|
|
140
|
+
|
|
141
|
+
def policy_for(name: str, args: dict) -> dict:
|
|
142
|
+
if name == "delete_everything":
|
|
143
|
+
return dict(_DESTRUCTIVE_POLICY)
|
|
144
|
+
if governor is not None and name in ("write_file", "edit_file"):
|
|
145
|
+
return dict(_GOVERNED_WRITE_POLICY)
|
|
146
|
+
return dict(_AUTO_POLICY)
|
|
147
|
+
|
|
148
|
+
write_policy = dict(_GOVERNED_WRITE_POLICY) if governor is not None else dict(_AUTO_POLICY)
|
|
149
|
+
return AgentDeps(
|
|
150
|
+
generate_as=generate_as,
|
|
151
|
+
generate=generate,
|
|
152
|
+
execute_tool=execute_tool,
|
|
153
|
+
policy_for=policy_for,
|
|
154
|
+
risk_level=lambda p: p["risk"],
|
|
155
|
+
check_role=lambda name, user: None,
|
|
156
|
+
tool_governance={
|
|
157
|
+
"write_file": write_policy,
|
|
158
|
+
"read_file": dict(_AUTO_POLICY),
|
|
159
|
+
"generate_file": dict(_AUTO_POLICY),
|
|
160
|
+
"delete_everything": dict(_DESTRUCTIVE_POLICY),
|
|
161
|
+
},
|
|
162
|
+
file_create_actions=frozenset({"write_file", "generate_file"}),
|
|
163
|
+
recent_chat_context=lambda **kw: "",
|
|
164
|
+
clear_history=lambda keep: {"ok": True},
|
|
165
|
+
knowledge_save=lambda *a, **kw: None,
|
|
166
|
+
audit=lambda *a, **kw: None,
|
|
167
|
+
planner_prompt="plan", executor_prompt="exec", critic_prompt="critic",
|
|
168
|
+
memory_updater_prompt="mem", agent_root=Path("/tmp/agent-eval"),
|
|
169
|
+
change_governor=governor,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
_PLAN = '{"action": "plan", "goal": "task", "steps": [{"action": "write_file"}]}'
|
|
174
|
+
_WRITE = '{"action": "write_file", "args": {"path": "note.txt", "content": "hi"}}'
|
|
175
|
+
_FINAL = '{"action": "final", "message": "done"}'
|
|
176
|
+
_PASS = '{"action": "verdict", "verdict": "PASS", "next_state": "DONE", "reason": "ok"}'
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def default_scenarios() -> List[Scenario]:
|
|
180
|
+
return [
|
|
181
|
+
Scenario(
|
|
182
|
+
name="happy-path",
|
|
183
|
+
replies=[_PLAN, _WRITE, _FINAL, _PASS],
|
|
184
|
+
expect_exact={"parse_errors": 0},
|
|
185
|
+
expect_tool_outcomes={"ok": 1},
|
|
186
|
+
),
|
|
187
|
+
Scenario(
|
|
188
|
+
name="weak-model-format-gauntlet",
|
|
189
|
+
replies=[
|
|
190
|
+
"<think>let me plan</think>```json\n" + _PLAN + "\n```",
|
|
191
|
+
"Sure! Here is the action:\n" + _WRITE,
|
|
192
|
+
"{'action': 'final', 'message': 'done', 'happy': True}",
|
|
193
|
+
'{"action": "verdict", "verdict": "PASS", "next_state": "DONE",}',
|
|
194
|
+
],
|
|
195
|
+
expect_exact={"parse_errors": 0},
|
|
196
|
+
expect_min={"llm_calls": 4},
|
|
197
|
+
expect_tool_outcomes={"ok": 1},
|
|
198
|
+
),
|
|
199
|
+
Scenario(
|
|
200
|
+
name="prose-slip-recovers-with-correction",
|
|
201
|
+
replies=[_PLAN, "I will now write the file for you.", _WRITE, _FINAL, _PASS],
|
|
202
|
+
expect_min={"parse_errors": 1, "parse_recovered": 1, "corrections": 1},
|
|
203
|
+
expect_tool_outcomes={"ok": 1},
|
|
204
|
+
),
|
|
205
|
+
Scenario(
|
|
206
|
+
name="double-slip-escalates-tool-list",
|
|
207
|
+
replies=[
|
|
208
|
+
_PLAN,
|
|
209
|
+
"chatty non-json reply",
|
|
210
|
+
"another chatty reply",
|
|
211
|
+
_WRITE,
|
|
212
|
+
_FINAL,
|
|
213
|
+
_PASS,
|
|
214
|
+
],
|
|
215
|
+
expect_min={"parse_errors": 2, "corrections": 2},
|
|
216
|
+
expect_tool_outcomes={"ok": 1},
|
|
217
|
+
),
|
|
218
|
+
Scenario(
|
|
219
|
+
name="destructive-action-blocked",
|
|
220
|
+
replies=[
|
|
221
|
+
_PLAN,
|
|
222
|
+
'{"action": "delete_everything", "args": {}}',
|
|
223
|
+
_FINAL,
|
|
224
|
+
_PASS,
|
|
225
|
+
],
|
|
226
|
+
expect_tool_outcomes={"blocked_destructive": 1},
|
|
227
|
+
),
|
|
228
|
+
Scenario(
|
|
229
|
+
name="identical-action-loop-detected",
|
|
230
|
+
replies=[_PLAN, _WRITE, _WRITE, _PASS],
|
|
231
|
+
expect_tool_outcomes={"ok": 1},
|
|
232
|
+
expect_min={"llm_calls": 4},
|
|
233
|
+
),
|
|
234
|
+
Scenario(
|
|
235
|
+
name="critic-retry-then-done",
|
|
236
|
+
replies=[
|
|
237
|
+
_PLAN,
|
|
238
|
+
_WRITE,
|
|
239
|
+
_FINAL,
|
|
240
|
+
'{"action": "verdict", "verdict": "FAIL", "next_state": "EXECUTING", "corrections": ["also mention the date"]}',
|
|
241
|
+
_FINAL,
|
|
242
|
+
_PASS,
|
|
243
|
+
],
|
|
244
|
+
expect_min={"retries": 1},
|
|
245
|
+
),
|
|
246
|
+
Scenario(
|
|
247
|
+
name="unrecoverable-garbage-still-terminates",
|
|
248
|
+
replies=[_PLAN, "garbage", "more garbage", "still garbage", _PASS],
|
|
249
|
+
expect_state="DONE",
|
|
250
|
+
expect_min={"parse_errors": 3},
|
|
251
|
+
expect_exact={"tool_outcomes": {}},
|
|
252
|
+
),
|
|
253
|
+
# ── file generation ─────────────────────────────────────────────
|
|
254
|
+
Scenario(
|
|
255
|
+
name="file-generation-happy-path",
|
|
256
|
+
replies=[
|
|
257
|
+
'{"action": "plan", "goal": "generate landing page", '
|
|
258
|
+
'"steps": [{"action": "generate_file"}]}',
|
|
259
|
+
# Small local models wrap the payload in reasoning + fences;
|
|
260
|
+
# the loop must still extract one clean tool call.
|
|
261
|
+
"<think>layout first, then hero copy</think>\n"
|
|
262
|
+
'```json\n{"thoughts": "produce the full document", '
|
|
263
|
+
'"action": "generate_file", "args": {"path": "site/index.html", '
|
|
264
|
+
'"content": "<!DOCTYPE html><html><body>hello</body></html>"}}\n```',
|
|
265
|
+
_FINAL,
|
|
266
|
+
_PASS,
|
|
267
|
+
],
|
|
268
|
+
expect_exact={"parse_errors": 0},
|
|
269
|
+
expect_tool_outcomes={"ok": 1},
|
|
270
|
+
expect_tool_calls=["generate_file"],
|
|
271
|
+
),
|
|
272
|
+
Scenario(
|
|
273
|
+
name="file-generation-bad-args-recovers",
|
|
274
|
+
replies=[
|
|
275
|
+
'{"action": "plan", "goal": "generate report", '
|
|
276
|
+
'"steps": [{"action": "generate_file"}]}',
|
|
277
|
+
# Malformed tool call: the model forgot the target path, the
|
|
278
|
+
# tool port fails like the real dispatcher would…
|
|
279
|
+
'{"thoughts": "write it", "action": "generate_file", '
|
|
280
|
+
'"args": {"content": "<html></html>"}}',
|
|
281
|
+
# …and the next turn recovers with a complete call.
|
|
282
|
+
'{"thoughts": "add the missing path", "action": "generate_file", '
|
|
283
|
+
'"args": {"path": "reports/q3.html", "content": "<html></html>"}}',
|
|
284
|
+
_FINAL,
|
|
285
|
+
_PASS,
|
|
286
|
+
],
|
|
287
|
+
expect_tool_outcomes={"error": 1, "ok": 1},
|
|
288
|
+
expect_tool_calls=["generate_file"],
|
|
289
|
+
),
|
|
290
|
+
# ── multi-step workflow ─────────────────────────────────────────
|
|
291
|
+
Scenario(
|
|
292
|
+
name="multi-step-workflow-chain",
|
|
293
|
+
replies=[
|
|
294
|
+
'{"action": "plan", "goal": "read spec, generate report, save summary", '
|
|
295
|
+
'"steps": [{"action": "read_file"}, {"action": "generate_file"}, '
|
|
296
|
+
'{"action": "write_file"}]}',
|
|
297
|
+
'{"thoughts": "step 1: read the source spec", '
|
|
298
|
+
'"action": "read_file", "args": {"path": "spec.md"}}',
|
|
299
|
+
'{"thoughts": "step 2: the spec asks for an HTML report", '
|
|
300
|
+
'"action": "generate_file", "args": {"path": "report.html", '
|
|
301
|
+
'"content": "<html><body>report</body></html>"}}',
|
|
302
|
+
'{"thoughts": "step 3: persist a short summary next to it", '
|
|
303
|
+
'"action": "write_file", "args": {"path": "summary.txt", '
|
|
304
|
+
'"content": "report generated"}}',
|
|
305
|
+
_FINAL,
|
|
306
|
+
_PASS,
|
|
307
|
+
],
|
|
308
|
+
expect_exact={"parse_errors": 0},
|
|
309
|
+
expect_min={"llm_calls": 6},
|
|
310
|
+
expect_tool_outcomes={"ok": 3},
|
|
311
|
+
expect_tool_calls=["read_file", "generate_file", "write_file"],
|
|
312
|
+
),
|
|
313
|
+
# ── governed-tool proposal path ─────────────────────────────────
|
|
314
|
+
Scenario(
|
|
315
|
+
name="governed-write-proposal-path",
|
|
316
|
+
use_governor=True,
|
|
317
|
+
replies=[
|
|
318
|
+
# write_file is NOT auto-approved here, but it is governed —
|
|
319
|
+
# approve() must not hard-block the plan (core invariant:
|
|
320
|
+
# governed tools are excluded from the non-auto set).
|
|
321
|
+
'{"action": "plan", "goal": "update existing page, add new note", '
|
|
322
|
+
'"steps": [{"action": "write_file"}]}',
|
|
323
|
+
# Mutation of existing content → staged as proposal, not written.
|
|
324
|
+
'{"thoughts": "rewrite the existing page", "action": "write_file", '
|
|
325
|
+
'"args": {"path": "existing/site.html", "content": "<new>"}}',
|
|
326
|
+
# Additive create → governor allows it to run immediately.
|
|
327
|
+
'{"thoughts": "add a fresh note", "action": "write_file", '
|
|
328
|
+
'"args": {"path": "fresh/new-note.md", "content": "hello"}}',
|
|
329
|
+
_FINAL,
|
|
330
|
+
_PASS,
|
|
331
|
+
],
|
|
332
|
+
expect_exact={"parse_errors": 0},
|
|
333
|
+
expect_tool_outcomes={"proposed": 1, "ok": 1},
|
|
334
|
+
expect_tool_calls=["write_file"],
|
|
335
|
+
),
|
|
336
|
+
]
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
async def _run_scenario(scenario: Scenario) -> Dict[str, Any]:
|
|
340
|
+
tool_log: List[Dict[str, Any]] = []
|
|
341
|
+
governor = _EvalChangeGovernor() if scenario.use_governor else None
|
|
342
|
+
deps = _build_deps(scenario.replies, tool_log, governor=governor)
|
|
343
|
+
runtime = SingleAgentRuntime(deps)
|
|
344
|
+
ctx = AgentRunContext()
|
|
345
|
+
ctx.state = AgentState.PLANNING
|
|
346
|
+
req = _Req("agent eval task")
|
|
347
|
+
|
|
348
|
+
await runtime.plan(ctx, req, "en", "eval@local")
|
|
349
|
+
runtime.approve(ctx, "eval@local")
|
|
350
|
+
if ctx.state == AgentState.EXECUTING:
|
|
351
|
+
await runtime.run_to_completion(
|
|
352
|
+
ctx, req, "en", "eval@local", max_steps=scenario.max_steps, max_retry=2
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
summary = ctx.trace.summary()
|
|
356
|
+
failures: List[str] = []
|
|
357
|
+
if ctx.state.value != scenario.expect_state:
|
|
358
|
+
failures.append(f"state={ctx.state.value} expected={scenario.expect_state}")
|
|
359
|
+
for key, minimum in scenario.expect_min.items():
|
|
360
|
+
if int(summary.get(key) or 0) < minimum:
|
|
361
|
+
failures.append(f"{key}={summary.get(key)} < {minimum}")
|
|
362
|
+
for key, exact in scenario.expect_exact.items():
|
|
363
|
+
if summary.get(key) != exact:
|
|
364
|
+
failures.append(f"{key}={summary.get(key)} != {exact}")
|
|
365
|
+
for outcome, count in scenario.expect_tool_outcomes.items():
|
|
366
|
+
if summary["tool_outcomes"].get(outcome, 0) != count:
|
|
367
|
+
failures.append(
|
|
368
|
+
f"tool_outcomes[{outcome}]={summary['tool_outcomes'].get(outcome, 0)} != {count}"
|
|
369
|
+
)
|
|
370
|
+
executed = [call["name"] for call in tool_log]
|
|
371
|
+
if scenario.expect_tool_calls and executed != scenario.expect_tool_calls:
|
|
372
|
+
failures.append(f"tool_calls={executed} != {scenario.expect_tool_calls}")
|
|
373
|
+
if governor is not None and summary["tool_outcomes"].get("proposed", 0) != len(governor.proposals):
|
|
374
|
+
failures.append(
|
|
375
|
+
f"governor proposals={len(governor.proposals)} != "
|
|
376
|
+
f"traced proposed={summary['tool_outcomes'].get('proposed', 0)}"
|
|
377
|
+
)
|
|
378
|
+
return {
|
|
379
|
+
"name": scenario.name,
|
|
380
|
+
"ok": not failures,
|
|
381
|
+
"failures": failures,
|
|
382
|
+
"final_state": ctx.state.value,
|
|
383
|
+
"summary": summary,
|
|
384
|
+
"tool_calls": len(tool_log),
|
|
385
|
+
"executed_tools": executed,
|
|
386
|
+
"proposals": len(governor.proposals) if governor is not None else 0,
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def run_agent_eval(scenarios: List[Scenario] | None = None) -> Dict[str, Any]:
|
|
391
|
+
"""Run every scenario and reduce to a release-gate scoreboard."""
|
|
392
|
+
|
|
393
|
+
async def _run_all() -> List[Dict[str, Any]]:
|
|
394
|
+
return [await _run_scenario(s) for s in (scenarios or default_scenarios())]
|
|
395
|
+
|
|
396
|
+
results = asyncio.run(_run_all())
|
|
397
|
+
passed = [r for r in results if r["ok"]]
|
|
398
|
+
total_parse_errors = sum(r["summary"]["parse_errors"] for r in results)
|
|
399
|
+
total_recovered = sum(r["summary"]["parse_recovered"] for r in results)
|
|
400
|
+
return {
|
|
401
|
+
"scenarios": len(results),
|
|
402
|
+
"passed": len(passed),
|
|
403
|
+
"success_rate": round(len(passed) / len(results), 4) if results else 0.0,
|
|
404
|
+
"parse_errors": total_parse_errors,
|
|
405
|
+
"parse_recovered": total_recovered,
|
|
406
|
+
"recovery_rate": round(total_recovered / total_parse_errors, 4) if total_parse_errors else 1.0,
|
|
407
|
+
"results": results,
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
__all__ = ["Scenario", "default_scenarios", "run_agent_eval"]
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Structured observability for the single-agent reasoning loop (v9.6.0).
|
|
2
|
+
|
|
3
|
+
The PLAN → EXECUTE → VERIFY state machine in :mod:`latticeai.core.agent`
|
|
4
|
+
already keeps a human-readable transcript, but the transcript mixes model
|
|
5
|
+
output, tool results, and control decisions into one list — you cannot ask
|
|
6
|
+
"how many parse failures were recovered?" or "which repairs did the weak
|
|
7
|
+
model need?" without re-parsing it.
|
|
8
|
+
|
|
9
|
+
:class:`LoopTrace` is the machine-readable side channel: every phase records
|
|
10
|
+
typed events (llm_call, parse_error, repair, correction, tool call outcome,
|
|
11
|
+
blocked action, retry, rollback), and :meth:`LoopTrace.summary` reduces them
|
|
12
|
+
to the counters an evaluation harness or the API response can consume
|
|
13
|
+
directly. The trace is pure data — no I/O, no clock dependency beyond an
|
|
14
|
+
injectable timestamp function — so unit tests and the deterministic agent
|
|
15
|
+
evaluation harness can assert on it exactly.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
from latticeai.core.timeutil import now_iso
|
|
23
|
+
|
|
24
|
+
_MAX_EVENTS = 500
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class LoopTrace:
|
|
28
|
+
"""Typed event stream + summary counters for one agent run."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, clock: Optional[Callable[[], str]] = None) -> None:
|
|
31
|
+
self._clock = clock or now_iso
|
|
32
|
+
self.events: List[Dict[str, Any]] = []
|
|
33
|
+
self.truncated = 0
|
|
34
|
+
|
|
35
|
+
def record(self, phase: str, kind: str, **details: Any) -> None:
|
|
36
|
+
if len(self.events) >= _MAX_EVENTS:
|
|
37
|
+
self.truncated += 1
|
|
38
|
+
return
|
|
39
|
+
event: Dict[str, Any] = {"phase": phase, "kind": kind, "at": self._clock()}
|
|
40
|
+
for key, value in details.items():
|
|
41
|
+
if value is not None:
|
|
42
|
+
event[key] = value
|
|
43
|
+
self.events.append(event)
|
|
44
|
+
|
|
45
|
+
# ── typed helpers keep call sites terse and the vocabulary closed ────
|
|
46
|
+
|
|
47
|
+
def llm_call(self, phase: str, *, model: Optional[str] = None) -> None:
|
|
48
|
+
self.record(phase, "llm_call", model=model)
|
|
49
|
+
|
|
50
|
+
def parse_error(self, phase: str, *, error: str, recovered: bool) -> None:
|
|
51
|
+
self.record(phase, "parse_error", error=error[:200], recovered=recovered)
|
|
52
|
+
|
|
53
|
+
def repair(self, phase: str, *, repairs: List[str]) -> None:
|
|
54
|
+
if repairs:
|
|
55
|
+
self.record(phase, "repair", repairs=list(repairs))
|
|
56
|
+
|
|
57
|
+
def correction(self, phase: str, *, hint: str) -> None:
|
|
58
|
+
self.record(phase, "correction", hint=hint[:200])
|
|
59
|
+
|
|
60
|
+
def tool(self, phase: str, *, name: str, outcome: str, risk: Optional[str] = None) -> None:
|
|
61
|
+
self.record(phase, "tool", name=name, outcome=outcome, risk=risk)
|
|
62
|
+
|
|
63
|
+
def decision(self, phase: str, *, decision: str, **details: Any) -> None:
|
|
64
|
+
self.record(phase, "decision", decision=decision, **details)
|
|
65
|
+
|
|
66
|
+
def retry(self, phase: str, *, attempt: int) -> None:
|
|
67
|
+
self.record(phase, "retry", attempt=attempt)
|
|
68
|
+
|
|
69
|
+
# ── reduction ────────────────────────────────────────────────────────
|
|
70
|
+
|
|
71
|
+
def summary(self) -> Dict[str, Any]:
|
|
72
|
+
counts: Dict[str, int] = {}
|
|
73
|
+
tool_outcomes: Dict[str, int] = {}
|
|
74
|
+
repairs: Dict[str, int] = {}
|
|
75
|
+
parse_errors = 0
|
|
76
|
+
parse_recovered = 0
|
|
77
|
+
for event in self.events:
|
|
78
|
+
kind = event["kind"]
|
|
79
|
+
counts[kind] = counts.get(kind, 0) + 1
|
|
80
|
+
if kind == "tool":
|
|
81
|
+
outcome = str(event.get("outcome") or "unknown")
|
|
82
|
+
tool_outcomes[outcome] = tool_outcomes.get(outcome, 0) + 1
|
|
83
|
+
elif kind == "parse_error":
|
|
84
|
+
parse_errors += 1
|
|
85
|
+
if event.get("recovered"):
|
|
86
|
+
parse_recovered += 1
|
|
87
|
+
elif kind == "repair":
|
|
88
|
+
for name in event.get("repairs") or []:
|
|
89
|
+
repairs[name] = repairs.get(name, 0) + 1
|
|
90
|
+
return {
|
|
91
|
+
"events": len(self.events),
|
|
92
|
+
"truncated_events": self.truncated,
|
|
93
|
+
"kind_counts": counts,
|
|
94
|
+
"llm_calls": counts.get("llm_call", 0),
|
|
95
|
+
"parse_errors": parse_errors,
|
|
96
|
+
"parse_recovered": parse_recovered,
|
|
97
|
+
"corrections": counts.get("correction", 0),
|
|
98
|
+
"retries": counts.get("retry", 0),
|
|
99
|
+
"tool_outcomes": tool_outcomes,
|
|
100
|
+
"repairs": repairs,
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
__all__ = ["LoopTrace"]
|
|
@@ -14,7 +14,7 @@ from pathlib import Path
|
|
|
14
14
|
from typing import Any, Dict, List
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
LEGACY_COMPATIBILITY_VERSION = "9.
|
|
17
|
+
LEGACY_COMPATIBILITY_VERSION = "9.7.0"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@dataclass(frozen=True)
|
|
@@ -82,6 +82,20 @@ LEGACY_SHIMS: List[LegacyShim] = [
|
|
|
82
82
|
reason="Old gardener scripts referenced the root reinforcement helper.",
|
|
83
83
|
removal_phase="major-release-after-8.x",
|
|
84
84
|
),
|
|
85
|
+
LegacyShim(
|
|
86
|
+
path="mcp_registry.py",
|
|
87
|
+
owner="latticeai.core.mcp_registry",
|
|
88
|
+
replacement="import latticeai.core.mcp_registry",
|
|
89
|
+
reason="Older integrations imported the MCP registry from the repo root.",
|
|
90
|
+
removal_phase="major-release-after-9.x",
|
|
91
|
+
),
|
|
92
|
+
LegacyShim(
|
|
93
|
+
path="llm_router.py",
|
|
94
|
+
owner="latticeai.models.router",
|
|
95
|
+
replacement="import latticeai.models.router",
|
|
96
|
+
reason="Historical scripts and tests imported the LLM router from the repo root.",
|
|
97
|
+
removal_phase="major-release-after-9.x",
|
|
98
|
+
),
|
|
85
99
|
LegacyShim(
|
|
86
100
|
path="server.py",
|
|
87
101
|
owner="latticeai.server_app",
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Central change-class governor for tool calls (v9.6.0).
|
|
2
|
+
|
|
3
|
+
The tool registry already answers "how risky is this tool?" (read / write /
|
|
4
|
+
exec, destructive, auto-approve). What it could not answer is the question
|
|
5
|
+
users actually care about: **does this call create something new, or does it
|
|
6
|
+
change/remove something that already exists?**
|
|
7
|
+
|
|
8
|
+
The governor adds that dimension in one place:
|
|
9
|
+
|
|
10
|
+
* ``read`` — no state change.
|
|
11
|
+
* ``additive`` — creates new content only (new file, new note, new node).
|
|
12
|
+
Low friction: runs under the existing gates without extra ceremony.
|
|
13
|
+
* ``mutation`` — rewrites existing content (overwrite / edit of an existing
|
|
14
|
+
file). Proposal-first: instead of applying silently, the change is staged
|
|
15
|
+
as a review proposal the user merges deliberately.
|
|
16
|
+
* ``destructive`` — removes existing content. Always proposal-first.
|
|
17
|
+
|
|
18
|
+
Classification is deterministic and injectable (``path_exists``), so the
|
|
19
|
+
agent loop, the chat file path, and tests all share exactly one policy.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import Any, Callable, Dict, Mapping, Optional
|
|
25
|
+
|
|
26
|
+
CHANGE_READ = "read"
|
|
27
|
+
CHANGE_ADDITIVE = "additive"
|
|
28
|
+
CHANGE_MUTATION = "mutation"
|
|
29
|
+
CHANGE_DESTRUCTIVE = "destructive"
|
|
30
|
+
CHANGE_EXEC = "exec"
|
|
31
|
+
|
|
32
|
+
# Tools whose effect depends on whether the target already exists.
|
|
33
|
+
_TARGET_WRITE_TOOLS = frozenset({
|
|
34
|
+
"write_file", "local_write",
|
|
35
|
+
"create_docx", "create_xlsx", "create_pptx", "create_pdf",
|
|
36
|
+
})
|
|
37
|
+
# Tools that always rewrite existing content.
|
|
38
|
+
_ALWAYS_MUTATION_TOOLS = frozenset({"edit_file"})
|
|
39
|
+
# Tools that remove existing content.
|
|
40
|
+
_DESTRUCTIVE_TOOLS = frozenset({"delete_file", "remove_file", "clear_history"})
|
|
41
|
+
# Additive-only knowledge writes (append-style, never rewrite).
|
|
42
|
+
_ADDITIVE_TOOLS = frozenset({
|
|
43
|
+
"knowledge_save", "obsidian_save", "todo_write", "create_web_project",
|
|
44
|
+
"knowledge_graph_ingest",
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def classify_tool_call(
|
|
49
|
+
name: str,
|
|
50
|
+
args: Mapping[str, Any],
|
|
51
|
+
*,
|
|
52
|
+
policy: Optional[Mapping[str, Any]] = None,
|
|
53
|
+
path_exists: Optional[Callable[[str], bool]] = None,
|
|
54
|
+
) -> Dict[str, Any]:
|
|
55
|
+
"""Classify one tool call into a change class + proposal requirement."""
|
|
56
|
+
risk = str((policy or {}).get("risk") or "")
|
|
57
|
+
change = CHANGE_READ
|
|
58
|
+
reason = "read-only tool"
|
|
59
|
+
|
|
60
|
+
if name in _DESTRUCTIVE_TOOLS or (policy or {}).get("destructive"):
|
|
61
|
+
change = CHANGE_DESTRUCTIVE
|
|
62
|
+
reason = "removes existing content"
|
|
63
|
+
elif name in _ALWAYS_MUTATION_TOOLS:
|
|
64
|
+
change = CHANGE_MUTATION
|
|
65
|
+
reason = "edits existing content in place"
|
|
66
|
+
elif name in _TARGET_WRITE_TOOLS:
|
|
67
|
+
path = str(args.get("path") or args.get("filename") or "")
|
|
68
|
+
exists = bool(path and path_exists and path_exists(path))
|
|
69
|
+
change = CHANGE_MUTATION if exists else CHANGE_ADDITIVE
|
|
70
|
+
reason = (
|
|
71
|
+
"overwrites an existing file" if exists else "creates a new file"
|
|
72
|
+
)
|
|
73
|
+
elif name in _ADDITIVE_TOOLS:
|
|
74
|
+
change = CHANGE_ADDITIVE
|
|
75
|
+
reason = "adds new content only"
|
|
76
|
+
elif risk.startswith("read"):
|
|
77
|
+
change = CHANGE_READ
|
|
78
|
+
reason = "read-only tool"
|
|
79
|
+
elif risk == "exec":
|
|
80
|
+
change = CHANGE_EXEC
|
|
81
|
+
reason = "executes an action (approval-gated, not proposal-based)"
|
|
82
|
+
elif risk in {"write", "write_scoped"}:
|
|
83
|
+
change = CHANGE_ADDITIVE
|
|
84
|
+
reason = "write tool without an existing target"
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
"tool": name,
|
|
88
|
+
"change_class": change,
|
|
89
|
+
"proposal_required": change in {CHANGE_MUTATION, CHANGE_DESTRUCTIVE},
|
|
90
|
+
"reason": reason,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
__all__ = [
|
|
95
|
+
"CHANGE_READ",
|
|
96
|
+
"CHANGE_ADDITIVE",
|
|
97
|
+
"CHANGE_MUTATION",
|
|
98
|
+
"CHANGE_DESTRUCTIVE",
|
|
99
|
+
"CHANGE_EXEC",
|
|
100
|
+
"classify_tool_call",
|
|
101
|
+
]
|
|
@@ -49,7 +49,7 @@ __all__ = [
|
|
|
49
49
|
"remove_skill_directory",
|
|
50
50
|
]
|
|
51
51
|
|
|
52
|
-
WORKSPACE_OS_VERSION = "9.
|
|
52
|
+
WORKSPACE_OS_VERSION = "9.7.0"
|
|
53
53
|
|
|
54
54
|
# Workspace types separate single-user Personal workspaces from shared
|
|
55
55
|
# Organization workspaces. Both keep the same local-first JSON store; the type
|