ltcai 7.2.0 → 7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -20
- package/docs/CHANGELOG.md +45 -0
- package/docs/ROADMAP_RECOMMENDATIONS.md +67 -0
- package/lattice_brain/__init__.py +1 -1
- package/lattice_brain/quality.py +31 -4
- package/lattice_brain/retrieval_benchmark_fixtures.py +56 -0
- package/lattice_brain/runtime/agent_runtime.py +11 -1
- package/lattice_brain/runtime/contracts.py +342 -0
- package/lattice_brain/runtime/multi_agent.py +9 -2
- package/lattice_brain/workflow.py +11 -1
- package/latticeai/__init__.py +1 -1
- package/latticeai/core/agent.py +5 -0
- package/latticeai/core/audit.py +15 -1
- package/latticeai/core/marketplace.py +1 -1
- package/latticeai/core/realtime.py +3 -0
- package/latticeai/core/workspace_os.py +34 -1
- package/package.json +1 -1
- package/scripts/brain_quality_eval.py +73 -1
- package/src-tauri/Cargo.lock +1 -1
- package/src-tauri/Cargo.toml +1 -1
- package/src-tauri/tauri.conf.json +1 -1
- package/static/app/asset-manifest.json +1 -1
package/README.md
CHANGED
|
@@ -200,29 +200,34 @@ See [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) for developer workflow details.
|
|
|
200
200
|
|
|
201
201
|
## Current Release
|
|
202
202
|
|
|
203
|
-
The current release is **7.
|
|
204
|
-
|
|
205
|
-
- Agent runs
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
-
|
|
210
|
-
|
|
211
|
-
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
- `ltcai-7.
|
|
221
|
-
- `dist/ltcai-7.
|
|
222
|
-
- `
|
|
203
|
+
The current release is **7.4.0 Runtime Contract Convergence & Corpus Retrieval**:
|
|
204
|
+
|
|
205
|
+
- Agent runs, workflow runs, audit events, and realtime events now carry the
|
|
206
|
+
`agent-run-contract/v1` family envelope, so replay, admin review, and live
|
|
207
|
+
feeds can inspect run identity, runtime, status, timeline, and artifacts
|
|
208
|
+
through one contract shape.
|
|
209
|
+
- Persisted queued/running/terminal agent and workflow rows refresh their
|
|
210
|
+
contract on every state transition, including cancellation and interruption.
|
|
211
|
+
- Audit rows keep their existing admin fields while adding redacted contract
|
|
212
|
+
metadata; realtime SSE events keep `area`, `event_type`, `workspace_id`, and
|
|
213
|
+
`payload` while adding the same contract family.
|
|
214
|
+
- Brain quality evaluation now runs a real KnowledgeGraphStore + SearchService
|
|
215
|
+
corpus fixture with judged queries, recall, precision, NDCG, and must-include
|
|
216
|
+
hit-rate thresholds.
|
|
217
|
+
|
|
218
|
+
Expected artifacts for 7.4.0 release must use exact filenames:
|
|
219
|
+
|
|
220
|
+
- `dist/ltcai-7.4.0-py3-none-any.whl`
|
|
221
|
+
- `dist/ltcai-7.4.0.tar.gz`
|
|
222
|
+
- `ltcai-7.4.0.tgz`
|
|
223
|
+
- `dist/ltcai-7.4.0.vsix`
|
|
224
|
+
- `src-tauri/target/release/bundle/dmg/Lattice AI_7.4.0_aarch64.dmg`
|
|
223
225
|
|
|
224
226
|
Do not upload `dist/*`. Package registry publishing remains owner-run.
|
|
225
227
|
|
|
228
|
+
See [docs/ROADMAP_RECOMMENDATIONS.md](docs/ROADMAP_RECOMMENDATIONS.md) for the
|
|
229
|
+
strategic roadmap slices applied through 7.4.0 and the follow-up tracks.
|
|
230
|
+
|
|
226
231
|
## Known Limitations
|
|
227
232
|
|
|
228
233
|
- External package registries are owner-published and can lag behind GitHub.
|
|
@@ -238,6 +243,8 @@ Do not upload `dist/*`. Package registry publishing remains owner-run.
|
|
|
238
243
|
|
|
239
244
|
| Version | Theme |
|
|
240
245
|
| --- | --- |
|
|
246
|
+
| 7.4.0 | Runtime Contract Convergence & Corpus Retrieval: agent/workflow/audit/realtime records share the agent-run-contract/v1 family, and retrieval quality gates run against a real corpus-scale fixture |
|
|
247
|
+
| 7.3.0 | Runtime Contract & Retrieval Quality: shared agent-run contract across runtimes plus deterministic hybrid recall/ranking regression gates |
|
|
241
248
|
| 7.2.0 | Runtime Trust Baseline: agent run preview/readiness, simulation-mode guardrails, live ToolRegistry manifest/diagnostics, and tests for dispatch/governance/catalog drift |
|
|
242
249
|
| 7.1.0 | Brain Usability Completion: clearer first-run onboarding, ingestion progress/emergence, richer graph controls, inline answer proof, workspace/profile/admin discovery, empty/error/consent feedback, and VS Code sync status |
|
|
243
250
|
| 7.0.0 | Brain Productization Loop: first-screen ingestion for files/folders/notes/web, answer-level memory proof and source citations, model-continuity demo flow, five-minute first-run loop, and recall/KG quality eval in CI |
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,51 @@ The top entry is either the current unreleased main-branch work or the current
|
|
|
4
4
|
release line. Older entries are historical and may describe behavior as it
|
|
5
5
|
existed at that release.
|
|
6
6
|
|
|
7
|
+
## [7.4.0] - 2026-06-20
|
|
8
|
+
|
|
9
|
+
> Runtime Contract Convergence & Corpus Retrieval. Completes the
|
|
10
|
+
> agent-run-contract/v1 family across run storage, workflow execution, audit
|
|
11
|
+
> events, realtime events, and a real corpus-scale retrieval quality gate.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- Persisted agent and workflow run rows now carry refreshed contract metadata
|
|
15
|
+
for queued, running, terminal, cancelled, and interrupted states.
|
|
16
|
+
- Workflow engine results, replay payloads, audit log events, and realtime SSE
|
|
17
|
+
feed events now expose the same `agent-run-contract/v1` family envelope while
|
|
18
|
+
preserving existing top-level compatibility fields.
|
|
19
|
+
- Corpus-scale retrieval fixture with 30+ documents, judged queries,
|
|
20
|
+
must-include expectations, and thresholds for recall, precision, NDCG, and
|
|
21
|
+
hit rate.
|
|
22
|
+
- `scripts/brain_quality_eval.py` now exercises the real local
|
|
23
|
+
`KnowledgeGraphStore` + `SearchService` hybrid retrieval path before scoring.
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
- `RetrievalBenchmarkRunner` reports dynamic metric aliases for the selected
|
|
27
|
+
`top_k` and a `must_include_hit_rate`.
|
|
28
|
+
- Package/runtime/static metadata is synchronized to 7.4.0; package publish and
|
|
29
|
+
deployment remain owner-run only.
|
|
30
|
+
|
|
31
|
+
## [7.3.0] - 2026-06-20
|
|
32
|
+
|
|
33
|
+
> Runtime Contract & Retrieval Quality. Turns the next AgentRuntime extraction
|
|
34
|
+
> step and the uploaded roadmap's hybrid-search quality goals into a small,
|
|
35
|
+
> tested release: shared run contracts and deterministic recall regression.
|
|
36
|
+
|
|
37
|
+
### Added
|
|
38
|
+
- `lattice_brain.runtime.contracts.AgentRunContract`, a serializable
|
|
39
|
+
`agent-run-contract/v1` payload shared by single-agent and multi-agent
|
|
40
|
+
execution paths.
|
|
41
|
+
- Multi-agent API result/run patches now include the shared contract with
|
|
42
|
+
runtime, mode, status, roles, retries, timeline, and terminal-state data.
|
|
43
|
+
- Single-agent runtime exposes the same contract helper for UI/API/storage
|
|
44
|
+
convergence in the next extraction pass.
|
|
45
|
+
- `scripts/brain_quality_eval.py` now runs deterministic hybrid recall/ranking
|
|
46
|
+
regression checks with recall and precision thresholds.
|
|
47
|
+
|
|
48
|
+
### Changed
|
|
49
|
+
- Package/runtime/static metadata is synchronized to 7.3.0; package publish and
|
|
50
|
+
deployment remain owner-run only.
|
|
51
|
+
|
|
7
52
|
## [7.2.0] - 2026-06-20
|
|
8
53
|
|
|
9
54
|
> Runtime Trust Baseline. Adds execution preview and registry diagnostics so
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Strategic Roadmap Recommendations
|
|
2
|
+
|
|
3
|
+
This roadmap captures the 2026-06-20 product direction input and maps it to
|
|
4
|
+
small release-sized work. The operating principle stays unchanged:
|
|
5
|
+
|
|
6
|
+
> Models are temporary. Knowledge is durable. The Brain is the product.
|
|
7
|
+
|
|
8
|
+
## 7.3.0 Applied Slice
|
|
9
|
+
|
|
10
|
+
7.3.0 implements the first narrow slice of the roadmap:
|
|
11
|
+
|
|
12
|
+
- Runtime evolution: single-agent and multi-agent execution share
|
|
13
|
+
`agent-run-contract/v1`, making mode/status/timeline evidence inspectable.
|
|
14
|
+
- Hybrid search optimization: the Brain quality gate now includes deterministic
|
|
15
|
+
recall and ranking regression thresholds.
|
|
16
|
+
- Security and trust: run contracts distinguish runtime type and execution mode
|
|
17
|
+
so simulated output is not presented as real execution.
|
|
18
|
+
|
|
19
|
+
## 7.4.0 Applied Slice
|
|
20
|
+
|
|
21
|
+
7.4.0 completes the next roadmap slice without deferring it:
|
|
22
|
+
|
|
23
|
+
- Runtime convergence: agent runs, workflow runs, audit events, and realtime
|
|
24
|
+
events all expose the `agent-run-contract/v1` family envelope while keeping
|
|
25
|
+
legacy top-level fields for compatibility.
|
|
26
|
+
- Trust and operations: persisted run rows refresh their contract through
|
|
27
|
+
queued, running, terminal, cancelled, and interrupted states; audit events are
|
|
28
|
+
contracted only after secret redaction.
|
|
29
|
+
- Retrieval quality and scale: the CI quality gate now seeds a real local
|
|
30
|
+
Knowledge Graph corpus and scores SearchService hybrid retrieval with judged
|
|
31
|
+
queries, recall, precision, NDCG, and must-include hit-rate thresholds.
|
|
32
|
+
|
|
33
|
+
## Near-Term Tracks
|
|
34
|
+
|
|
35
|
+
1. Retrieval quality and scale
|
|
36
|
+
- Add latency budgets for larger corpora.
|
|
37
|
+
- Track per-channel keyword/vector/graph diagnostics in CI.
|
|
38
|
+
- Tune semantic/graph/keyword fusion by query class.
|
|
39
|
+
|
|
40
|
+
2. Incremental ingestion
|
|
41
|
+
- Add background indexing jobs.
|
|
42
|
+
- Detect duplicates and merge memory candidates.
|
|
43
|
+
- Surface conflict resolution for contradictory memories.
|
|
44
|
+
|
|
45
|
+
3. Runtime convergence
|
|
46
|
+
- Drive UI run/replay/admin views from the shared contract family.
|
|
47
|
+
- Keep simulation mode explicit and never record it as product success.
|
|
48
|
+
- Route tools through explicit registry/governance contracts.
|
|
49
|
+
|
|
50
|
+
4. Brain SDK boundary
|
|
51
|
+
- Continue extracting `lattice_brain` as a reusable Brain Core package.
|
|
52
|
+
- Keep compatibility shims until downstream imports are migrated.
|
|
53
|
+
- Preserve migration rollback paths for graph and memory storage.
|
|
54
|
+
|
|
55
|
+
5. Trust and operations
|
|
56
|
+
- Extend audit logging around tool execution and retrieval injection.
|
|
57
|
+
- Add dependency vulnerability monitoring.
|
|
58
|
+
- Add Tauri update/rollback planning before enabling auto-update.
|
|
59
|
+
|
|
60
|
+
## Longer-Term Tracks
|
|
61
|
+
|
|
62
|
+
- Multi-modal ingestion and retrieval for images, audio, and video.
|
|
63
|
+
- Proactive Brain synthesis, contradiction detection, and recommendations.
|
|
64
|
+
- Temporal reasoning over historical graph states.
|
|
65
|
+
- Interoperability with Obsidian, Notion, Email, Calendar, Git, Slack, and Teams.
|
|
66
|
+
- Encrypted Brain Network sharing.
|
|
67
|
+
- Plugin marketplace and public benchmarks.
|
package/lattice_brain/quality.py
CHANGED
|
@@ -285,6 +285,18 @@ class StructuredContextAssembler:
|
|
|
285
285
|
# -----------------------------
|
|
286
286
|
# 6. Retrieval Benchmark Fixture Runner
|
|
287
287
|
# -----------------------------
|
|
288
|
+
def _relevance_grades(relevant: Any) -> Dict[str, float]:
|
|
289
|
+
"""Normalize a ``relevant`` spec into a ``{doc_id: grade}`` map.
|
|
290
|
+
|
|
291
|
+
Accepts a graded dict (``{"doc": 3}``) or a binary list/set/tuple (grade 1
|
|
292
|
+
for every id), so judged fixtures can express graded relevance for nDCG
|
|
293
|
+
while older binary fixtures keep working unchanged.
|
|
294
|
+
"""
|
|
295
|
+
if isinstance(relevant, dict):
|
|
296
|
+
return {str(k): float(v) for k, v in relevant.items()}
|
|
297
|
+
return {str(doc_id): 1.0 for doc_id in (relevant or [])}
|
|
298
|
+
|
|
299
|
+
|
|
288
300
|
class RetrievalBenchmarkRunner:
|
|
289
301
|
def __init__(self):
|
|
290
302
|
self.results: List[Dict] = []
|
|
@@ -297,25 +309,36 @@ class RetrievalBenchmarkRunner:
|
|
|
297
309
|
precisions = []
|
|
298
310
|
ndcgs = []
|
|
299
311
|
for query in judged:
|
|
300
|
-
|
|
312
|
+
grades = _relevance_grades(query.get("relevant"))
|
|
313
|
+
relevant = set(grades)
|
|
301
314
|
retrieved = list(query.get("retrieved") or [])[:top_k]
|
|
302
315
|
if not relevant:
|
|
303
316
|
continue
|
|
304
317
|
hits = [doc_id for doc_id in retrieved if doc_id in relevant]
|
|
305
318
|
recalls.append(len(hits) / len(relevant))
|
|
306
319
|
precisions.append(len(hits) / max(1, len(retrieved)))
|
|
320
|
+
# Graded DCG: each retrieved doc contributes its relevance grade
|
|
321
|
+
# discounted by log2(rank + 2). IDCG ranks the highest grades
|
|
322
|
+
# first, so nDCG rewards putting the *most* relevant docs on top.
|
|
323
|
+
# Binary fixtures fall back to grade 1 via _relevance_grades.
|
|
307
324
|
dcg = sum(
|
|
308
|
-
|
|
325
|
+
grades.get(doc_id, 0.0) / math.log2(rank + 2)
|
|
309
326
|
for rank, doc_id in enumerate(retrieved)
|
|
310
|
-
if doc_id in relevant
|
|
311
327
|
)
|
|
312
|
-
|
|
328
|
+
ideal_grades = sorted(grades.values(), reverse=True)[:top_k]
|
|
329
|
+
ideal = sum(g / math.log2(rank + 2) for rank, g in enumerate(ideal_grades))
|
|
313
330
|
ndcgs.append(dcg / ideal if ideal else 0.0)
|
|
314
331
|
recall = sum(recalls) / len(recalls) if recalls else 0.0
|
|
315
332
|
precision = sum(precisions) / len(precisions) if precisions else 0.0
|
|
316
333
|
ndcg = sum(ndcgs) / len(ndcgs) if ndcgs else 0.0
|
|
317
334
|
else:
|
|
318
335
|
recall = precision = ndcg = 0.0
|
|
336
|
+
must_include = [
|
|
337
|
+
bool(set(query.get("must_include") or []).intersection(set(list(query.get("retrieved") or [])[:top_k])))
|
|
338
|
+
for query in judged
|
|
339
|
+
if query.get("must_include")
|
|
340
|
+
]
|
|
341
|
+
must_include_hit_rate = sum(1 for hit in must_include if hit) / len(must_include) if must_include else 1.0
|
|
319
342
|
metrics = {
|
|
320
343
|
"fixture": fixture_name,
|
|
321
344
|
"queries": len(queries),
|
|
@@ -323,6 +346,10 @@ class RetrievalBenchmarkRunner:
|
|
|
323
346
|
"recall@5": round(recall, 4),
|
|
324
347
|
"precision@5": round(precision, 4),
|
|
325
348
|
"ndcg@5": round(ndcg, 4),
|
|
349
|
+
f"recall@{top_k}": round(recall, 4),
|
|
350
|
+
f"precision@{top_k}": round(precision, 4),
|
|
351
|
+
f"ndcg@{top_k}": round(ndcg, 4),
|
|
352
|
+
"must_include_hit_rate": round(must_include_hit_rate, 4),
|
|
326
353
|
"top_k": top_k,
|
|
327
354
|
"judged": len(judged),
|
|
328
355
|
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Corpus-scale retrieval fixture for the 7.4.0 Brain quality gate."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
FIXTURE_NAME = "7.4.0-corpus-scale-hybrid-retrieval"
|
|
6
|
+
TOP_K = 5
|
|
7
|
+
|
|
8
|
+
DOCUMENTS = [
|
|
9
|
+
{"id": "doc:agent-runtime-contract", "type": "Decision", "title": "Agent runtime contract", "content": "Workflow run, agent run, audit event, and realtime event records must carry the agent-run-contract/v1 family envelope with run id, runtime, status, timeline, and artifacts."},
|
|
10
|
+
{"id": "doc:workflow-observability", "type": "Document", "title": "Workflow observability", "content": "Workflow Designer persists queued, running, awaiting approval, cancelled, interrupted, partial, failed, and ok runs with replayable timeline events."},
|
|
11
|
+
{"id": "doc:audit-trust", "type": "Security", "title": "Audit trust", "content": "Audit events are append-only, redacted before persistence, and linked to execution contracts for admin review without leaking secrets."},
|
|
12
|
+
{"id": "doc:realtime-feed", "type": "Document", "title": "Realtime execution feed", "content": "The realtime SSE bus publishes workspace, agent, workflow, and execution events with workspace scope, event type, payload, sequence, and contract metadata."},
|
|
13
|
+
{"id": "doc:retrieval-benchmark", "type": "Benchmark", "title": "Retrieval benchmark strategy", "content": "Hybrid retrieval quality must be checked with corpus-scale fixtures, judged queries, recall, precision, ndcg, and must-include hit rate."},
|
|
14
|
+
{"id": "doc:hybrid-search", "type": "Architecture", "title": "Hybrid search", "content": "SearchService fuses keyword, vector, and graph retrieval channels with weighted scores and scoped workspace filtering."},
|
|
15
|
+
{"id": "doc:knowledge-graph", "type": "Architecture", "title": "Knowledge graph stabilization", "content": "The Knowledge Graph preserves legacy read compatibility, non-destructive migrations, vector indexing, relationship traversal, and provenance."},
|
|
16
|
+
{"id": "doc:local-first", "type": "Strategy", "title": "Local-first product wedge", "content": "Lattice AI is local-first: durable knowledge, private workspace memory, local LLM routing, and model-independent context are the product wedge."},
|
|
17
|
+
{"id": "doc:brain-proof", "type": "UX", "title": "Brain proof and citations", "content": "Brain proof shows durable recall, source citation, graph counts, vector counts, and model continuity evidence in chat."},
|
|
18
|
+
{"id": "doc:onboarding", "type": "UX", "title": "Five minute onboarding loop", "content": "The first loop asks users to add a source, ask a question, see Brain proof, and understand expected installation time."},
|
|
19
|
+
{"id": "doc:vscode-sync", "type": "Integration", "title": "VS Code extension state", "content": "VS Code extension integration shows app connection state, indexing status, command routing, and extension-to-app sync health."},
|
|
20
|
+
{"id": "doc:workspace-admin", "type": "Workspace", "title": "Workspace profile admin discovery", "content": "Personal workspace, organization workspace, profile, admin, members, roles, permissions, and security surfaces must be discoverable."},
|
|
21
|
+
{"id": "doc:empty-error-consent", "type": "UX", "title": "Empty error consent states", "content": "Empty states, error states, consent prompts, retries, and privacy feedback should be explicit across ingestion, graph, chat, and workspace flows."},
|
|
22
|
+
{"id": "doc:tool-registry", "type": "Architecture", "title": "Tool registry separation", "content": "ToolRegistry separates tool definitions, permissions, dispatch, audit, and UI discovery from AgentRuntime orchestration."},
|
|
23
|
+
{"id": "doc:config-centralization", "type": "Architecture", "title": "Config centralization", "content": "Configuration should be centralized into explicit objects instead of hidden globals so runtime, tests, and packaging are predictable."},
|
|
24
|
+
{"id": "doc:server-decomposition", "type": "Architecture", "title": "Server decomposition", "content": "Server decomposition moves routers, services, runtime context, and persistence seams into focused modules with dependency injection."},
|
|
25
|
+
{"id": "doc:incremental-ingestion", "type": "Pipeline", "title": "Incremental ingestion", "content": "Background indexing needs duplicate detection, conflict resolution, incremental updates, and merge behavior for large personal corpora."},
|
|
26
|
+
{"id": "doc:vector-index-evolution", "type": "Pipeline", "title": "Vector index evolution", "content": "sqlite-vec migration, HNSW evaluation, query caching, embedding caching, and retrieval caching support larger corpora."},
|
|
27
|
+
{"id": "doc:security-zero-trust", "type": "Security", "title": "Zero trust defaults", "content": "Zero-trust defaults include audit logging, secret redaction, dependency vulnerability monitoring, and automated security scanning."},
|
|
28
|
+
{"id": "doc:agentic-hitl", "type": "Workflow", "title": "Agentic human review", "content": "Agentic workflows need human-in-the-loop review, approval gates, rollback paths, review queues, and auditable decisions."},
|
|
29
|
+
{"id": "doc:temporal-reasoning", "type": "Brain", "title": "Temporal reasoning", "content": "Temporal graph states, historical memory, contradiction detection, synthesis, and recommendations make the Brain proactive."},
|
|
30
|
+
{"id": "doc:multimodal-memory", "type": "Brain", "title": "Multimodal memory", "content": "Images, audio, video, documents, and chat should ingest into one durable Brain with retrieval and citation proof."},
|
|
31
|
+
{"id": "doc:developer-api", "type": "API", "title": "Developer API", "content": "A secure developer API exposes third-party access to workspace knowledge, tools, runs, contracts, and audit trails."},
|
|
32
|
+
{"id": "doc:team-brains", "type": "Business", "title": "Team Brains", "content": "Team Brains, encrypted sharing, premium cloud sync, hosted models, plugin marketplace, and public benchmarks support monetization."},
|
|
33
|
+
{"id": "doc:docker-tauri", "type": "Deployment", "title": "Deployment operations", "content": "Docker Compose deployment, one-click installation, Tauri auto-updates, rollback support, and exact release artifacts improve operations."},
|
|
34
|
+
{"id": "doc:model-routing", "type": "Runtime", "title": "Model orchestration", "content": "Ollama, LM Studio, MLX, and cloud models need automatic fallback, routing, and replaceable model boundaries."},
|
|
35
|
+
{"id": "doc:strict-quality", "type": "Quality", "title": "Strict code quality", "content": "Async-first architecture, dependency injection, centralized exceptions, mypy or pyright, TypeScript strict mode, E2E tests, and property tests reduce regressions."},
|
|
36
|
+
{"id": "distractor:theme-editor", "type": "Distractor", "title": "Theme editor", "content": "Color palettes, typography, and sidebar icon sizing are useful but unrelated to retrieval contract benchmarks."},
|
|
37
|
+
{"id": "distractor:calendar", "type": "Distractor", "title": "Calendar reminders", "content": "Calendar reminders and meeting agenda formatting are integrations but not the runtime event contract path."},
|
|
38
|
+
{"id": "distractor:billing", "type": "Distractor", "title": "Billing invoices", "content": "Invoices, payment receipts, and subscription emails can be stored but are not the Brain quality gate."},
|
|
39
|
+
{"id": "distractor:mobile-shell", "type": "Distractor", "title": "Mobile shell", "content": "A mobile companion can improve access but should not replace local-first desktop retrieval and run observability."},
|
|
40
|
+
{"id": "distractor:marketing", "type": "Distractor", "title": "Marketing landing page", "content": "Landing pages and hero copy do not prove durable knowledge or agent runtime correctness."},
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
QUERIES = [
|
|
44
|
+
{"query": "agent workflow audit realtime contract run status timeline", "relevant": {"doc:agent-runtime-contract": 3, "doc:workflow-observability": 2, "doc:audit-trust": 2, "doc:realtime-feed": 2}, "must_include": ["doc:agent-runtime-contract"]},
|
|
45
|
+
{"query": "workflow queued running cancelled interrupted replay approval", "relevant": {"doc:workflow-observability": 3, "doc:agentic-hitl": 2}, "must_include": ["doc:workflow-observability"]},
|
|
46
|
+
{"query": "audit log redacted secrets admin review contract", "relevant": {"doc:audit-trust": 3, "doc:security-zero-trust": 2}, "must_include": ["doc:audit-trust"]},
|
|
47
|
+
{"query": "SSE realtime workspace event payload sequence contract", "relevant": {"doc:realtime-feed": 3, "doc:agent-runtime-contract": 2}, "must_include": ["doc:realtime-feed"]},
|
|
48
|
+
{"query": "hybrid retrieval keyword vector graph weighted scoped search", "relevant": {"doc:hybrid-search": 3, "doc:knowledge-graph": 2, "doc:retrieval-benchmark": 2}, "must_include": ["doc:hybrid-search"]},
|
|
49
|
+
{"query": "corpus scale recall precision ndcg judged queries", "relevant": {"doc:retrieval-benchmark": 3, "doc:hybrid-search": 2}, "must_include": ["doc:retrieval-benchmark"]},
|
|
50
|
+
{"query": "local first model independent durable memory brain proof citation", "relevant": {"doc:local-first": 3, "doc:brain-proof": 2}, "must_include": ["doc:local-first"]},
|
|
51
|
+
{"query": "incremental indexing duplicate conflict merge background ingestion", "relevant": {"doc:incremental-ingestion": 3, "doc:vector-index-evolution": 2}, "must_include": ["doc:incremental-ingestion"]},
|
|
52
|
+
{"query": "workspace admin profile organization members roles permissions", "relevant": {"doc:workspace-admin": 3, "doc:security-zero-trust": 2}, "must_include": ["doc:workspace-admin"]},
|
|
53
|
+
{"query": "tool registry separation dependency injection runtime context", "relevant": {"doc:tool-registry": 3, "doc:server-decomposition": 2, "doc:config-centralization": 2}, "must_include": ["doc:tool-registry"]},
|
|
54
|
+
{"query": "human in the loop approval review queue rollback", "relevant": {"doc:agentic-hitl": 3, "doc:workflow-observability": 2}, "must_include": ["doc:agentic-hitl"]},
|
|
55
|
+
{"query": "model routing ollama lm studio mlx cloud fallback replaceable", "relevant": {"doc:model-routing": 3, "doc:local-first": 2}, "must_include": ["doc:model-routing"]},
|
|
56
|
+
]
|
|
@@ -37,6 +37,7 @@ from .multi_agent import (
|
|
|
37
37
|
MULTI_AGENT_VERSION,
|
|
38
38
|
ROLE_AGENT_IDS,
|
|
39
39
|
)
|
|
40
|
+
from .contracts import multi_agent_contract
|
|
40
41
|
|
|
41
42
|
ROLE_DESCRIPTIONS = {
|
|
42
43
|
"researcher": "Gathers workspace context and memory for the goal.",
|
|
@@ -344,6 +345,7 @@ class AgentRuntime:
|
|
|
344
345
|
"retry_history": result.retry_history,
|
|
345
346
|
"memory_snapshots": result.memory_snapshots,
|
|
346
347
|
"mode": getattr(result, "mode", "simulation"),
|
|
348
|
+
"contract": multi_agent_contract(result=result, goal=goal),
|
|
347
349
|
"current_role": None,
|
|
348
350
|
}
|
|
349
351
|
|
|
@@ -509,6 +511,12 @@ class AgentRuntime:
|
|
|
509
511
|
status=updated.get("status") or result.status,
|
|
510
512
|
)
|
|
511
513
|
result_payload = result.as_dict()
|
|
514
|
+
result_payload["contract"] = multi_agent_contract(
|
|
515
|
+
result=result,
|
|
516
|
+
goal=goal,
|
|
517
|
+
run_id=run_id,
|
|
518
|
+
current_role=updated.get("current_role") if isinstance(updated, dict) else None,
|
|
519
|
+
)
|
|
512
520
|
if updated.get("status") == "cancelled":
|
|
513
521
|
result_payload = {"status": "cancelled", "reason": updated.get("cancel_reason"), "completed_result": result_payload}
|
|
514
522
|
payload = {"run": updated, "result": result_payload}
|
|
@@ -601,7 +609,9 @@ class AgentRuntime:
|
|
|
601
609
|
user_email=user_email, workspace_id=scope,
|
|
602
610
|
)
|
|
603
611
|
|
|
604
|
-
|
|
612
|
+
result_payload = result.as_dict()
|
|
613
|
+
result_payload["contract"] = multi_agent_contract(result=result, goal=goal, run_id=run.get("id") if isinstance(run, dict) else None)
|
|
614
|
+
payload = {"run": run, "result": result_payload}
|
|
605
615
|
if pre_dispatch is not None:
|
|
606
616
|
payload["pre_run_hooks"] = pre_dispatch
|
|
607
617
|
if post_dispatch is not None:
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Shared runtime contracts for agent execution surfaces.
|
|
2
|
+
|
|
3
|
+
These contracts are deliberately small and serializable. The single-agent
|
|
4
|
+
state machine and the multi-agent facade can evolve independently, but callers
|
|
5
|
+
should see the same run identity, mode, status, role, timeline and artifact
|
|
6
|
+
shape across both paths.
|
|
7
|
+
|
|
8
|
+
Contract family (7.4.0)
|
|
9
|
+
-----------------------
|
|
10
|
+
Four observability surfaces — agent runs, workflow runs, audit events, and
|
|
11
|
+
realtime events — now share one **contract family**, ``agent-run-contract/v1``.
|
|
12
|
+
Every record emitted by any of these surfaces carries the same envelope keys so
|
|
13
|
+
a single consumer (timeline UI, exporter, replay tooling) can treat them
|
|
14
|
+
uniformly:
|
|
15
|
+
|
|
16
|
+
* ``family`` — always :data:`CONTRACT_FAMILY` (``agent-run-contract/v1``).
|
|
17
|
+
* ``schema_version`` — the per-surface schema string (see ``*_SCHEMA`` below).
|
|
18
|
+
* ``kind`` — one of :data:`CONTRACT_KINDS`.
|
|
19
|
+
* ``id`` — the record identity (run id / event id), may be ``None``.
|
|
20
|
+
* ``status`` — a normalized lifecycle status string.
|
|
21
|
+
* ``timestamp`` — ISO-8601 second-precision emit/observe time.
|
|
22
|
+
|
|
23
|
+
Each surface keeps its own rich, surface-specific keys *in addition* to the
|
|
24
|
+
envelope — the envelope is purely additive, so existing consumers are never
|
|
25
|
+
broken. :func:`stamp_contract` is the single place that writes the envelope;
|
|
26
|
+
:func:`is_contract_member` validates membership for tests and importers.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
from datetime import datetime
|
|
33
|
+
from typing import Any, Dict, List, Optional
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def runtime_timestamp() -> str:
|
|
37
|
+
return datetime.now().isoformat(timespec="seconds")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ── Contract family identity ────────────────────────────────────────────────
|
|
41
|
+
# A single family string ties every observability surface together. Per-surface
|
|
42
|
+
# schema versions live *under* the family so each surface can rev independently
|
|
43
|
+
# without forking the family contract consumers depend on.
|
|
44
|
+
CONTRACT_FAMILY = "agent-run-contract/v1"
|
|
45
|
+
|
|
46
|
+
AGENT_RUN_SCHEMA = "agent-run-contract/v1"
|
|
47
|
+
WORKFLOW_RUN_SCHEMA = "workflow-run-contract/v1"
|
|
48
|
+
AUDIT_EVENT_SCHEMA = "audit-event-contract/v1"
|
|
49
|
+
REALTIME_EVENT_SCHEMA = "realtime-event-contract/v1"
|
|
50
|
+
|
|
51
|
+
CONTRACT_KINDS = ("agent_run", "workflow_run", "audit_event", "realtime_event")
|
|
52
|
+
|
|
53
|
+
# The envelope keys every family member is guaranteed to expose.
|
|
54
|
+
CONTRACT_ENVELOPE_KEYS = ("family", "schema_version", "kind", "id", "status", "timestamp")
|
|
55
|
+
|
|
56
|
+
_SCHEMA_FOR_KIND = {
|
|
57
|
+
"agent_run": AGENT_RUN_SCHEMA,
|
|
58
|
+
"workflow_run": WORKFLOW_RUN_SCHEMA,
|
|
59
|
+
"audit_event": AUDIT_EVENT_SCHEMA,
|
|
60
|
+
"realtime_event": REALTIME_EVENT_SCHEMA,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def stamp_contract(
|
|
65
|
+
body: Dict[str, Any],
|
|
66
|
+
*,
|
|
67
|
+
kind: str,
|
|
68
|
+
identity: Optional[str],
|
|
69
|
+
status: str,
|
|
70
|
+
timestamp: Optional[str] = None,
|
|
71
|
+
) -> Dict[str, Any]:
|
|
72
|
+
"""Return ``body`` with the ``agent-run-contract/v1`` envelope merged in.
|
|
73
|
+
|
|
74
|
+
Additive and idempotent: existing keys in ``body`` win for everything except
|
|
75
|
+
the six envelope keys, which are authoritative. ``kind`` must be one of
|
|
76
|
+
:data:`CONTRACT_KINDS`; the schema version is derived from it so callers
|
|
77
|
+
cannot mismatch ``kind`` and ``schema_version``.
|
|
78
|
+
"""
|
|
79
|
+
if kind not in _SCHEMA_FOR_KIND:
|
|
80
|
+
raise ValueError(f"unknown contract kind: {kind!r}")
|
|
81
|
+
enveloped = dict(body)
|
|
82
|
+
enveloped["family"] = CONTRACT_FAMILY
|
|
83
|
+
enveloped["schema_version"] = _SCHEMA_FOR_KIND[kind]
|
|
84
|
+
enveloped["kind"] = kind
|
|
85
|
+
enveloped["id"] = identity
|
|
86
|
+
enveloped["status"] = status
|
|
87
|
+
enveloped["timestamp"] = timestamp or body.get("timestamp") or runtime_timestamp()
|
|
88
|
+
return enveloped
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def is_contract_member(record: Any) -> bool:
|
|
92
|
+
"""True if ``record`` carries a valid ``agent-run-contract/v1`` envelope."""
|
|
93
|
+
if not isinstance(record, dict):
|
|
94
|
+
return False
|
|
95
|
+
if record.get("family") != CONTRACT_FAMILY:
|
|
96
|
+
return False
|
|
97
|
+
kind = record.get("kind")
|
|
98
|
+
if kind not in _SCHEMA_FOR_KIND:
|
|
99
|
+
return False
|
|
100
|
+
if record.get("schema_version") != _SCHEMA_FOR_KIND[kind]:
|
|
101
|
+
return False
|
|
102
|
+
return all(key in record for key in CONTRACT_ENVELOPE_KEYS)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass(frozen=True)
|
|
106
|
+
class AgentRunContract:
|
|
107
|
+
run_id: Optional[str]
|
|
108
|
+
agent_id: str
|
|
109
|
+
runtime: str
|
|
110
|
+
mode: str
|
|
111
|
+
status: str
|
|
112
|
+
goal: str
|
|
113
|
+
roles: List[str] = field(default_factory=list)
|
|
114
|
+
current_role: Optional[str] = None
|
|
115
|
+
retries: int = 0
|
|
116
|
+
timeline: List[Dict[str, Any]] = field(default_factory=list)
|
|
117
|
+
artifacts: List[Dict[str, Any]] = field(default_factory=list)
|
|
118
|
+
blocking_reasons: List[str] = field(default_factory=list)
|
|
119
|
+
schema_version: str = "agent-run-contract/v1"
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def is_terminal(self) -> bool:
|
|
123
|
+
return self.status in {"ok", "retried_ok", "failed", "rejected", "cancelled", "interrupted", "partial", "done"}
|
|
124
|
+
|
|
125
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
126
|
+
body = {
|
|
127
|
+
"run_id": self.run_id,
|
|
128
|
+
"agent_id": self.agent_id,
|
|
129
|
+
"runtime": self.runtime,
|
|
130
|
+
"mode": self.mode,
|
|
131
|
+
"goal": self.goal,
|
|
132
|
+
"roles": list(self.roles),
|
|
133
|
+
"current_role": self.current_role,
|
|
134
|
+
"retries": self.retries,
|
|
135
|
+
"timeline": list(self.timeline),
|
|
136
|
+
"artifacts": list(self.artifacts),
|
|
137
|
+
"blocking_reasons": list(self.blocking_reasons),
|
|
138
|
+
"is_terminal": self.is_terminal,
|
|
139
|
+
}
|
|
140
|
+
return stamp_contract(
|
|
141
|
+
body,
|
|
142
|
+
kind="agent_run",
|
|
143
|
+
identity=self.run_id,
|
|
144
|
+
status=self.status,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def multi_agent_contract(
|
|
149
|
+
*,
|
|
150
|
+
result: Any,
|
|
151
|
+
goal: str,
|
|
152
|
+
run_id: Optional[str] = None,
|
|
153
|
+
current_role: Optional[str] = None,
|
|
154
|
+
blocking_reasons: Optional[List[str]] = None,
|
|
155
|
+
) -> Dict[str, Any]:
|
|
156
|
+
return AgentRunContract(
|
|
157
|
+
run_id=run_id,
|
|
158
|
+
agent_id=getattr(result, "agent_id", "agent:executor"),
|
|
159
|
+
runtime="multi_agent",
|
|
160
|
+
mode=getattr(result, "mode", "simulation"),
|
|
161
|
+
status=getattr(result, "status", "unknown"),
|
|
162
|
+
goal=goal,
|
|
163
|
+
roles=list(getattr(result, "roles_run", []) or []),
|
|
164
|
+
current_role=current_role,
|
|
165
|
+
retries=int(getattr(result, "retries", 0) or 0),
|
|
166
|
+
timeline=list(getattr(result, "timeline", []) or []),
|
|
167
|
+
artifacts=[],
|
|
168
|
+
blocking_reasons=list(blocking_reasons or []),
|
|
169
|
+
).as_dict()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def run_record_contract(run: Dict[str, Any], *, runtime: str = "multi_agent") -> Dict[str, Any]:
|
|
173
|
+
"""Build the family contract from a persisted agent run row."""
|
|
174
|
+
run = dict(run or {})
|
|
175
|
+
timeline = list(run.get("timeline") or [])
|
|
176
|
+
retry_history = list(run.get("retry_history") or [])
|
|
177
|
+
roles = list(run.get("roles_run") or run.get("requested_roles") or [])
|
|
178
|
+
if not roles:
|
|
179
|
+
roles = [
|
|
180
|
+
str(item.get("role"))
|
|
181
|
+
for item in timeline
|
|
182
|
+
if isinstance(item, dict) and item.get("role")
|
|
183
|
+
]
|
|
184
|
+
return AgentRunContract(
|
|
185
|
+
run_id=run.get("id") or run.get("run_id"),
|
|
186
|
+
agent_id=str(run.get("agent_id") or "agent:executor"),
|
|
187
|
+
runtime=runtime,
|
|
188
|
+
mode=str(run.get("mode") or "simulation"),
|
|
189
|
+
status=str(run.get("status") or "unknown"),
|
|
190
|
+
goal=str(run.get("input") or run.get("goal") or ""),
|
|
191
|
+
roles=roles,
|
|
192
|
+
current_role=run.get("current_role"),
|
|
193
|
+
retries=len(retry_history),
|
|
194
|
+
timeline=timeline,
|
|
195
|
+
artifacts=[
|
|
196
|
+
{
|
|
197
|
+
"type": "run_record",
|
|
198
|
+
"workspace_id": run.get("workspace_id"),
|
|
199
|
+
"graph_node_id": run.get("graph_node_id"),
|
|
200
|
+
"execution_mode": run.get("execution_mode"),
|
|
201
|
+
}
|
|
202
|
+
],
|
|
203
|
+
blocking_reasons=[str(run.get("error"))] if run.get("error") else [],
|
|
204
|
+
).as_dict()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def single_agent_contract(
|
|
208
|
+
*,
|
|
209
|
+
ctx: Any,
|
|
210
|
+
goal: str,
|
|
211
|
+
run_id: Optional[str] = None,
|
|
212
|
+
agent_id: str = "agent:single",
|
|
213
|
+
mode: str = "llm",
|
|
214
|
+
blocking_reasons: Optional[List[str]] = None,
|
|
215
|
+
) -> Dict[str, Any]:
|
|
216
|
+
state = getattr(getattr(ctx, "state", None), "value", getattr(ctx, "state", "unknown"))
|
|
217
|
+
status = "done" if state == "DONE" else "failed" if state == "FAILED" else str(state or "unknown").lower()
|
|
218
|
+
return AgentRunContract(
|
|
219
|
+
run_id=run_id,
|
|
220
|
+
agent_id=agent_id,
|
|
221
|
+
runtime="single_agent",
|
|
222
|
+
mode=mode,
|
|
223
|
+
status=status,
|
|
224
|
+
goal=goal,
|
|
225
|
+
roles=["planner", "executor", "critic"],
|
|
226
|
+
current_role=str(state or ""),
|
|
227
|
+
retries=int(getattr(ctx, "retry_count", 0) or 0),
|
|
228
|
+
timeline=list(getattr(ctx, "transcript", []) or []),
|
|
229
|
+
artifacts=[],
|
|
230
|
+
blocking_reasons=list(blocking_reasons or []),
|
|
231
|
+
).as_dict()
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
# ── Sibling-surface contract builders ───────────────────────────────────────
|
|
235
|
+
# These wrap the other three observability surfaces into the same family so a
|
|
236
|
+
# consumer can iterate agent runs, workflow runs, audit events and realtime
|
|
237
|
+
# events through one envelope.
|
|
238
|
+
|
|
239
|
+
def workflow_run_contract(run: Any) -> Dict[str, Any]:
|
|
240
|
+
"""Wrap a workflow run dict (``WorkflowRun.as_dict()``) in the family envelope.
|
|
241
|
+
|
|
242
|
+
Accepts either a ``WorkflowRun`` instance or its serialized dict so the
|
|
243
|
+
engine can stamp without importing the dataclass here (avoids a cycle).
|
|
244
|
+
"""
|
|
245
|
+
body = run.as_dict() if hasattr(run, "as_dict") else dict(run or {})
|
|
246
|
+
run_id = body.get("id") or body.get("run_id") or body.get("workflow_id")
|
|
247
|
+
workflow_id = body.get("workflow_id")
|
|
248
|
+
contract_body = {
|
|
249
|
+
"run_id": run_id,
|
|
250
|
+
"agent_id": f"workflow:{workflow_id or body.get('name') or 'workflow'}",
|
|
251
|
+
"runtime": "workflow",
|
|
252
|
+
"mode": body.get("mode") or "live",
|
|
253
|
+
"goal": body.get("name") or "workflow",
|
|
254
|
+
"roles": ["workflow"],
|
|
255
|
+
"current_role": body.get("current_node") or body.get("paused_node"),
|
|
256
|
+
"retries": 0,
|
|
257
|
+
"timeline": list(body.get("timeline") or []),
|
|
258
|
+
"artifacts": [
|
|
259
|
+
{
|
|
260
|
+
"type": "workflow_outputs",
|
|
261
|
+
"workflow_id": workflow_id,
|
|
262
|
+
"outputs": body.get("outputs") or {},
|
|
263
|
+
"pause": body.get("pause") or body.get("pending_approval"),
|
|
264
|
+
}
|
|
265
|
+
],
|
|
266
|
+
"blocking_reasons": [str((body.get("outputs") or {}).get("error"))] if (body.get("outputs") or {}).get("error") else [],
|
|
267
|
+
"is_terminal": str(body.get("status") or "") in {"ok", "failed", "cancelled", "interrupted", "partial", "rejected"},
|
|
268
|
+
}
|
|
269
|
+
return stamp_contract(
|
|
270
|
+
contract_body,
|
|
271
|
+
kind="workflow_run",
|
|
272
|
+
identity=run_id,
|
|
273
|
+
status=str(body.get("status") or "unknown"),
|
|
274
|
+
timestamp=body.get("started_at") or body.get("created_at"),
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def audit_event_contract(event: Dict[str, Any]) -> Dict[str, Any]:
|
|
279
|
+
"""Wrap a single audit-log event in the family envelope.
|
|
280
|
+
|
|
281
|
+
Audit events have no lifecycle ``status``; we expose the event type as the
|
|
282
|
+
status so the envelope stays uniform and filterable. Identity is a stable
|
|
283
|
+
``event_type@timestamp`` key (audit events are append-only, not addressable).
|
|
284
|
+
"""
|
|
285
|
+
body = dict(event or {})
|
|
286
|
+
event_type = str(body.get("event_type") or "event")
|
|
287
|
+
ts = body.get("timestamp")
|
|
288
|
+
identity = body.get("event_id") or (f"{event_type}@{ts}" if ts else event_type)
|
|
289
|
+
contract_body = {
|
|
290
|
+
"run_id": body.get("run_id"),
|
|
291
|
+
"agent_id": str(body.get("agent_id") or body.get("workflow_id") or f"audit:{event_type}"),
|
|
292
|
+
"runtime": "audit",
|
|
293
|
+
"mode": "event",
|
|
294
|
+
"goal": event_type,
|
|
295
|
+
"roles": [],
|
|
296
|
+
"current_role": None,
|
|
297
|
+
"retries": int(body.get("retries") or 0),
|
|
298
|
+
"timeline": [{"event": event_type, "timestamp": ts, "status": body.get("status") or event_type}],
|
|
299
|
+
"artifacts": [{"type": "audit_payload", "payload": body}],
|
|
300
|
+
"blocking_reasons": [],
|
|
301
|
+
"is_terminal": True,
|
|
302
|
+
}
|
|
303
|
+
return stamp_contract(
|
|
304
|
+
contract_body,
|
|
305
|
+
kind="audit_event",
|
|
306
|
+
identity=identity,
|
|
307
|
+
status=event_type,
|
|
308
|
+
timestamp=ts,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def realtime_event_contract(event: Dict[str, Any]) -> Dict[str, Any]:
|
|
313
|
+
"""Wrap a realtime bus event in the family envelope.
|
|
314
|
+
|
|
315
|
+
The bus already assigns a monotonic ``seq`` and ``received_at``; we reuse
|
|
316
|
+
those as identity and timestamp so the envelope is consistent with the
|
|
317
|
+
feed's own ordering guarantees.
|
|
318
|
+
"""
|
|
319
|
+
body = dict(event or {})
|
|
320
|
+
seq = body.get("seq")
|
|
321
|
+
payload = body.get("payload") if isinstance(body.get("payload"), dict) else {}
|
|
322
|
+
contract_body = {
|
|
323
|
+
"run_id": payload.get("run_id") or body.get("run_id"),
|
|
324
|
+
"agent_id": str(payload.get("agent_id") or payload.get("workflow_id") or f"realtime:{body.get('area') or 'workspace'}"),
|
|
325
|
+
"runtime": "realtime",
|
|
326
|
+
"mode": "event",
|
|
327
|
+
"goal": str(body.get("event_type") or "event"),
|
|
328
|
+
"roles": [],
|
|
329
|
+
"current_role": payload.get("current_role"),
|
|
330
|
+
"retries": int(payload.get("retries") or 0),
|
|
331
|
+
"timeline": [{"event": body.get("event_type") or "event", "timestamp": body.get("received_at"), "payload": payload}],
|
|
332
|
+
"artifacts": [{"type": "realtime_payload", "payload": payload}],
|
|
333
|
+
"blocking_reasons": [str(payload.get("error"))] if payload.get("error") else [],
|
|
334
|
+
"is_terminal": str(payload.get("status") or "") in {"ok", "failed", "cancelled", "interrupted", "partial", "rejected"},
|
|
335
|
+
}
|
|
336
|
+
return stamp_contract(
|
|
337
|
+
contract_body,
|
|
338
|
+
kind="realtime_event",
|
|
339
|
+
identity=f"rt:{seq}" if seq is not None else None,
|
|
340
|
+
status=str(body.get("event_type") or "event"),
|
|
341
|
+
timestamp=body.get("received_at"),
|
|
342
|
+
)
|
|
@@ -18,8 +18,10 @@ from dataclasses import dataclass, field
|
|
|
18
18
|
from datetime import datetime
|
|
19
19
|
from typing import Any, Callable, Dict, List, Optional
|
|
20
20
|
|
|
21
|
+
from .contracts import multi_agent_contract
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
|
|
24
|
+
MULTI_AGENT_VERSION = "7.4.0"
|
|
23
25
|
|
|
24
26
|
AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
|
|
25
27
|
CORE_PIPELINE = ("planner", "executor", "reviewer")
|
|
@@ -284,11 +286,12 @@ class AgentRunResult:
|
|
|
284
286
|
retry_history: List[Dict[str, Any]] = field(default_factory=list)
|
|
285
287
|
plan_review: Dict[str, Any] = field(default_factory=dict)
|
|
286
288
|
memory_snapshots: List[Dict[str, Any]] = field(default_factory=list)
|
|
289
|
+
goal: str = ""
|
|
287
290
|
# "simulation" = deterministic LLM-free runner; "llm" = model-driven (v4 runtime).
|
|
288
291
|
mode: str = "simulation"
|
|
289
292
|
|
|
290
293
|
def as_dict(self) -> Dict[str, Any]:
|
|
291
|
-
|
|
294
|
+
payload = {
|
|
292
295
|
"agent_id": self.agent_id,
|
|
293
296
|
"mode": self.mode,
|
|
294
297
|
"status": self.status,
|
|
@@ -304,7 +307,10 @@ class AgentRunResult:
|
|
|
304
307
|
"retry_history": self.retry_history,
|
|
305
308
|
"plan_review": self.plan_review,
|
|
306
309
|
"memory_snapshots": self.memory_snapshots,
|
|
310
|
+
"goal": self.goal,
|
|
307
311
|
}
|
|
312
|
+
payload["contract"] = multi_agent_contract(result=self, goal=self.goal or self.output)
|
|
313
|
+
return payload
|
|
308
314
|
|
|
309
315
|
|
|
310
316
|
def default_role_runner(
|
|
@@ -796,5 +802,6 @@ class MultiAgentOrchestrator:
|
|
|
796
802
|
retry_history=ctx.retry_history,
|
|
797
803
|
plan_review=ctx.plan_review,
|
|
798
804
|
memory_snapshots=ctx.memory_snapshots,
|
|
805
|
+
goal=ctx.goal,
|
|
799
806
|
mode=self.mode,
|
|
800
807
|
)
|
|
@@ -27,6 +27,8 @@ from dataclasses import dataclass, field
|
|
|
27
27
|
from datetime import datetime
|
|
28
28
|
from typing import Any, Callable, Dict, List, Optional
|
|
29
29
|
|
|
30
|
+
from lattice_brain.runtime.contracts import workflow_run_contract
|
|
31
|
+
|
|
30
32
|
|
|
31
33
|
WORKFLOW_ENGINE_VERSION = "2.2.0"
|
|
32
34
|
|
|
@@ -198,7 +200,13 @@ class WorkflowRun:
|
|
|
198
200
|
paused_context: Optional[Dict[str, Any]] = None
|
|
199
201
|
|
|
200
202
|
def as_dict(self) -> Dict[str, Any]:
|
|
201
|
-
|
|
203
|
+
# Native workflow-run shape is preserved unchanged for existing readers.
|
|
204
|
+
# A normalized ``agent-run-contract/v1`` projection is attached under
|
|
205
|
+
# ``contract`` so workflow runs sit in the same observability family as
|
|
206
|
+
# agent runs, audit events and realtime events (mirrors the audit-log
|
|
207
|
+
# and realtime-bus convention). The projection is built from the native
|
|
208
|
+
# dict — never from ``self`` — so it cannot recurse back into as_dict().
|
|
209
|
+
body = {
|
|
202
210
|
"workflow_id": self.workflow_id,
|
|
203
211
|
"name": self.name,
|
|
204
212
|
"status": self.status,
|
|
@@ -211,6 +219,8 @@ class WorkflowRun:
|
|
|
211
219
|
"pending_approval": self.pending_approval,
|
|
212
220
|
"paused_context": self.paused_context,
|
|
213
221
|
}
|
|
222
|
+
body["contract"] = workflow_run_contract(body)
|
|
223
|
+
return body
|
|
214
224
|
|
|
215
225
|
|
|
216
226
|
class ApprovalRequired(Exception):
|
package/latticeai/__init__.py
CHANGED
package/latticeai/core/agent.py
CHANGED
|
@@ -30,6 +30,7 @@ from pathlib import Path
|
|
|
30
30
|
from typing import Any, Awaitable, Callable, Dict, FrozenSet, List, Optional
|
|
31
31
|
|
|
32
32
|
from lattice_brain.runtime.hooks import dispatch_tool
|
|
33
|
+
from lattice_brain.runtime.contracts import single_agent_contract
|
|
33
34
|
from tools import ToolError
|
|
34
35
|
|
|
35
36
|
|
|
@@ -142,6 +143,10 @@ class AgentRuntime:
|
|
|
142
143
|
def __init__(self, deps: AgentDeps) -> None:
|
|
143
144
|
self.deps = deps
|
|
144
145
|
|
|
146
|
+
def contract(self, ctx: AgentRunContext, req: Any, *, run_id: Optional[str] = None) -> Dict[str, Any]:
|
|
147
|
+
"""Expose the shared agent-run contract for the single-agent loop."""
|
|
148
|
+
return single_agent_contract(ctx=ctx, goal=getattr(req, "message", ""), run_id=run_id)
|
|
149
|
+
|
|
145
150
|
# ── PLAN ─────────────────────────────────────────────────────────
|
|
146
151
|
async def plan(
|
|
147
152
|
self, ctx: AgentRunContext, req: Any, lang_hint: str, current_user: str,
|
package/latticeai/core/audit.py
CHANGED
|
@@ -5,9 +5,12 @@ import logging
|
|
|
5
5
|
import os
|
|
6
6
|
import re
|
|
7
7
|
import threading
|
|
8
|
+
import hashlib
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from typing import Any, Callable, Dict, List, Optional
|
|
10
11
|
|
|
12
|
+
from lattice_brain.runtime.contracts import audit_event_contract
|
|
13
|
+
|
|
11
14
|
from . import timezones
|
|
12
15
|
from .security import redact_secrets
|
|
13
16
|
|
|
@@ -42,12 +45,21 @@ def get_audit_log(audit_file: Path) -> List[Dict]:
|
|
|
42
45
|
def append_audit_event(audit_file: Path, event_type: str, **payload) -> None:
|
|
43
46
|
try:
|
|
44
47
|
safe_payload = redact_secrets(payload)
|
|
48
|
+
timestamp = timezones.now_iso()
|
|
49
|
+
event_hash = hashlib.sha256(
|
|
50
|
+
json.dumps([event_type, timestamp, safe_payload], ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
|
|
51
|
+
).hexdigest()[:24]
|
|
45
52
|
event = {
|
|
53
|
+
"event_id": f"audit-{event_hash}",
|
|
46
54
|
"event_type": event_type,
|
|
47
55
|
# item 7: 대시보드 "오늘" 계산과 동일한 시간대 기준으로 기록한다.
|
|
48
|
-
"timestamp":
|
|
56
|
+
"timestamp": timestamp,
|
|
49
57
|
**safe_payload,
|
|
50
58
|
}
|
|
59
|
+
# Stamp the agent-run-contract/v1 family envelope so audit rows are
|
|
60
|
+
# uniform with agent/workflow/realtime records. Additive — existing
|
|
61
|
+
# readers ignore the extra keys; new consumers can filter by family.
|
|
62
|
+
event["contract"] = audit_event_contract(event)
|
|
51
63
|
with _history_lock:
|
|
52
64
|
events = get_audit_log(audit_file)
|
|
53
65
|
events.append(event)
|
|
@@ -236,6 +248,8 @@ def build_admin_audit_report(
|
|
|
236
248
|
|
|
237
249
|
def _public_audit_event(event: Dict) -> Dict:
|
|
238
250
|
allowed = {
|
|
251
|
+
# Stable event id + nested agent-run-contract/v1 family projection.
|
|
252
|
+
"event_id", "contract",
|
|
239
253
|
"event_type", "timestamp", "role", "user_email", "user_nickname", "source",
|
|
240
254
|
"conversation_id", "workspace_id", "command", "scope", "target_email", "filename", "mime_type",
|
|
241
255
|
"ext", "bytes", "extracted_chars", "graph_node", "keep_last", "removed", "kept",
|
|
@@ -32,6 +32,8 @@ import threading
|
|
|
32
32
|
from datetime import datetime
|
|
33
33
|
from typing import Any, AsyncIterator, Dict, List, Optional, Set
|
|
34
34
|
|
|
35
|
+
from lattice_brain.runtime.contracts import realtime_event_contract
|
|
36
|
+
|
|
35
37
|
|
|
36
38
|
REALTIME_VERSION = "2.2.0"
|
|
37
39
|
_FEED_LIMIT = 200
|
|
@@ -99,6 +101,7 @@ class RealtimeBus:
|
|
|
99
101
|
"payload": event.get("payload", {}),
|
|
100
102
|
**{k: v for k, v in event.items() if k not in {"area", "event_type", "workspace_id", "payload"}},
|
|
101
103
|
}
|
|
104
|
+
enriched["contract"] = realtime_event_contract(enriched)
|
|
102
105
|
self._feed.append(enriched)
|
|
103
106
|
if len(self._feed) > _FEED_LIMIT:
|
|
104
107
|
self._feed = self._feed[-_FEED_LIMIT:]
|
|
@@ -18,8 +18,10 @@ from datetime import datetime
|
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
|
20
20
|
|
|
21
|
+
from lattice_brain.runtime.contracts import realtime_event_contract, run_record_contract, workflow_run_contract
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
|
|
24
|
+
WORKSPACE_OS_VERSION = "7.4.0"
|
|
23
25
|
|
|
24
26
|
# Workspace types separate single-user Personal workspaces from shared
|
|
25
27
|
# Organization workspaces. Both keep the same local-first JSON store; the type
|
|
@@ -526,6 +528,7 @@ class WorkspaceOSStore:
|
|
|
526
528
|
"workspace_id": self._resolve_scope(workspace_id, state),
|
|
527
529
|
"payload": payload,
|
|
528
530
|
}
|
|
531
|
+
event["contract"] = realtime_event_contract({"seq": event["id"], "received_at": event["timestamp"], **event})
|
|
529
532
|
state.setdefault("timeline", []).append(event)
|
|
530
533
|
self.save_state(state)
|
|
531
534
|
if self.event_sink is not None:
|
|
@@ -1565,6 +1568,13 @@ class WorkspaceOSStore:
|
|
|
1565
1568
|
if status == "failed":
|
|
1566
1569
|
self._emit_execution_event(area="agent", event_type="execution_failed", payload={"run_id": run["id"], "agent_id": agent_id, "status": status}, workspace_id=resolved_workspace)
|
|
1567
1570
|
self.record_timeline_event("agent", "agent_run", {"run_id": run["id"], "agent_id": agent_id, "status": status}, workspace_id=resolved_workspace)
|
|
1571
|
+
run["contract"] = run_record_contract(run)
|
|
1572
|
+
state = self.load_state()
|
|
1573
|
+
for item in _listify(state.get("agent_runs")):
|
|
1574
|
+
if item.get("id") == run["id"]:
|
|
1575
|
+
item["contract"] = run["contract"]
|
|
1576
|
+
break
|
|
1577
|
+
self.save_state(state)
|
|
1568
1578
|
return run
|
|
1569
1579
|
|
|
1570
1580
|
def update_agent_run(
|
|
@@ -1636,6 +1646,13 @@ class WorkspaceOSStore:
|
|
|
1636
1646
|
run["graph_error"] = str(exc)
|
|
1637
1647
|
|
|
1638
1648
|
self.save_state(state)
|
|
1649
|
+
run["contract"] = run_record_contract(run)
|
|
1650
|
+
state = self.load_state()
|
|
1651
|
+
for item in _listify(state.get("agent_runs")):
|
|
1652
|
+
if item.get("id") == run_id:
|
|
1653
|
+
item["contract"] = run["contract"]
|
|
1654
|
+
break
|
|
1655
|
+
self.save_state(state)
|
|
1639
1656
|
|
|
1640
1657
|
timeline = run.get("timeline") or []
|
|
1641
1658
|
if len(timeline) > old_timeline_len:
|
|
@@ -1774,6 +1791,13 @@ class WorkspaceOSStore:
|
|
|
1774
1791
|
elif status in {"ok", "partial"}:
|
|
1775
1792
|
self._emit_execution_event(area="workflow", event_type="workflow_completed", payload={"run_id": run["id"], "workflow_id": workflow_id, "status": status}, workspace_id=resolved_workspace)
|
|
1776
1793
|
self.record_timeline_event("workflow", "workflow_run", {"run_id": run["id"], "workflow_id": workflow_id, "status": status}, workspace_id=resolved_workspace)
|
|
1794
|
+
run["contract"] = workflow_run_contract(run)
|
|
1795
|
+
state = self.load_state()
|
|
1796
|
+
for item in _listify(state.get("workflow_runs")):
|
|
1797
|
+
if item.get("id") == run["id"]:
|
|
1798
|
+
item["contract"] = run["contract"]
|
|
1799
|
+
break
|
|
1800
|
+
self.save_state(state)
|
|
1777
1801
|
return run
|
|
1778
1802
|
|
|
1779
1803
|
def update_workflow_run(
|
|
@@ -1835,6 +1859,13 @@ class WorkspaceOSStore:
|
|
|
1835
1859
|
run["graph_error"] = str(exc)
|
|
1836
1860
|
|
|
1837
1861
|
self.save_state(state)
|
|
1862
|
+
run["contract"] = workflow_run_contract(run)
|
|
1863
|
+
state = self.load_state()
|
|
1864
|
+
for item in _listify(state.get("workflow_runs")):
|
|
1865
|
+
if item.get("id") == run_id:
|
|
1866
|
+
item["contract"] = run["contract"]
|
|
1867
|
+
break
|
|
1868
|
+
self.save_state(state)
|
|
1838
1869
|
|
|
1839
1870
|
timeline = run.get("timeline") or []
|
|
1840
1871
|
if len(timeline) > old_timeline_len:
|
|
@@ -2053,6 +2084,7 @@ class WorkspaceOSStore:
|
|
|
2053
2084
|
"run_id": run_id,
|
|
2054
2085
|
"status": run.get("status"),
|
|
2055
2086
|
"workspace_id": self._record_workspace(run),
|
|
2087
|
+
"contract": run.get("contract") or run_record_contract(run),
|
|
2056
2088
|
"replayable": True,
|
|
2057
2089
|
"frames": self._replay_frames(run, kind="agent"),
|
|
2058
2090
|
"handoffs": run.get("handoffs") or [],
|
|
@@ -2068,6 +2100,7 @@ class WorkspaceOSStore:
|
|
|
2068
2100
|
"run_id": run_id,
|
|
2069
2101
|
"status": run.get("status"),
|
|
2070
2102
|
"workspace_id": self._record_workspace(run),
|
|
2103
|
+
"contract": run.get("contract") or workflow_run_contract(run),
|
|
2071
2104
|
"replayable": True,
|
|
2072
2105
|
"frames": self._replay_frames(run, kind="workflow"),
|
|
2073
2106
|
"outputs": run.get("outputs") or {},
|
package/package.json
CHANGED
|
@@ -12,7 +12,15 @@ import sys
|
|
|
12
12
|
import tempfile
|
|
13
13
|
from pathlib import Path
|
|
14
14
|
|
|
15
|
+
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
16
|
+
if str(REPO_ROOT) not in sys.path:
|
|
17
|
+
sys.path.insert(0, str(REPO_ROOT))
|
|
18
|
+
|
|
19
|
+
from knowledge_graph import KnowledgeGraphStore
|
|
20
|
+
from lattice_brain.quality import RetrievalBenchmarkRunner
|
|
21
|
+
from latticeai.services.search_service import SearchService
|
|
15
22
|
from latticeai.services.memory_service import MemoryService
|
|
23
|
+
from lattice_brain.retrieval_benchmark_fixtures import DOCUMENTS, FIXTURE_NAME, QUERIES, TOP_K
|
|
16
24
|
|
|
17
25
|
|
|
18
26
|
class _EvalStore:
|
|
@@ -80,6 +88,37 @@ def _fail(message: str) -> int:
|
|
|
80
88
|
return 1
|
|
81
89
|
|
|
82
90
|
|
|
91
|
+
def _corpus_scale_retrieval_metrics(tmp_path: Path) -> dict:
|
|
92
|
+
graph = KnowledgeGraphStore(tmp_path / "kg.sqlite", tmp_path / "blobs")
|
|
93
|
+
id_map = {}
|
|
94
|
+
for doc in DOCUMENTS:
|
|
95
|
+
result = graph.ingest_event(
|
|
96
|
+
doc["type"],
|
|
97
|
+
f"{doc['title']} {doc['content']}",
|
|
98
|
+
source="retrieval-benchmark-v740",
|
|
99
|
+
conversation_id="retrieval-benchmark-v740",
|
|
100
|
+
metadata={
|
|
101
|
+
"fixture_id": doc["id"],
|
|
102
|
+
"title": doc["title"],
|
|
103
|
+
"content": doc["content"],
|
|
104
|
+
"workspace_id": "personal",
|
|
105
|
+
},
|
|
106
|
+
)
|
|
107
|
+
id_map[doc["id"]] = result["node_id"]
|
|
108
|
+
service = SearchService(graph)
|
|
109
|
+
judged = []
|
|
110
|
+
for query in QUERIES:
|
|
111
|
+
payload = service.hybrid_search(query["query"], limit=TOP_K, keyword_limit=TOP_K, vector_limit=TOP_K, graph_limit=TOP_K)
|
|
112
|
+
retrieved = [item.get("node_id") or item.get("id") for item in payload.get("matches", [])]
|
|
113
|
+
judged.append({
|
|
114
|
+
"query": query["query"],
|
|
115
|
+
"relevant": [id_map[item] for item in query["relevant"]],
|
|
116
|
+
"must_include": [id_map[item] for item in query.get("must_include", [])],
|
|
117
|
+
"retrieved": retrieved,
|
|
118
|
+
})
|
|
119
|
+
return RetrievalBenchmarkRunner().run_fixture(FIXTURE_NAME, judged, top_k=TOP_K)
|
|
120
|
+
|
|
121
|
+
|
|
83
122
|
def main() -> int:
|
|
84
123
|
with tempfile.TemporaryDirectory() as tmp:
|
|
85
124
|
service = MemoryService(
|
|
@@ -110,7 +149,40 @@ def main() -> int:
|
|
|
110
149
|
if proof.get("proofs", {}).get("vector_items", 0) < 1:
|
|
111
150
|
return _fail("vector items must be counted in Brain proof")
|
|
112
151
|
|
|
113
|
-
|
|
152
|
+
benchmark = RetrievalBenchmarkRunner()
|
|
153
|
+
retrieval_metrics = benchmark.run_fixture(
|
|
154
|
+
"7.3.0-hybrid-recall-regression",
|
|
155
|
+
[
|
|
156
|
+
{
|
|
157
|
+
"query": "first five minute Brain proof",
|
|
158
|
+
"relevant": ["note:first-loop", "doc:first-loop"],
|
|
159
|
+
"retrieved": [item.get("id") for item in recall_items],
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
"query": "local-first memory product wedge",
|
|
163
|
+
"relevant": ["decision:pricing"],
|
|
164
|
+
"retrieved": ["decision:pricing", "note:first-loop"],
|
|
165
|
+
},
|
|
166
|
+
],
|
|
167
|
+
top_k=4,
|
|
168
|
+
)
|
|
169
|
+
if retrieval_metrics.get("recall@5", 0.0) < 0.75:
|
|
170
|
+
return _fail(f"hybrid recall regression below threshold: {retrieval_metrics}")
|
|
171
|
+
if retrieval_metrics.get("precision@5", 0.0) < 0.4:
|
|
172
|
+
return _fail(f"hybrid precision regression below threshold: {retrieval_metrics}")
|
|
173
|
+
|
|
174
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
175
|
+
corpus_metrics = _corpus_scale_retrieval_metrics(Path(tmp))
|
|
176
|
+
if corpus_metrics.get("recall@5", 0.0) < 0.80:
|
|
177
|
+
return _fail(f"corpus hybrid recall below threshold: {corpus_metrics}")
|
|
178
|
+
if corpus_metrics.get("precision@5", 0.0) < 0.25:
|
|
179
|
+
return _fail(f"corpus hybrid precision below threshold: {corpus_metrics}")
|
|
180
|
+
if corpus_metrics.get("ndcg@5", 0.0) < 0.70:
|
|
181
|
+
return _fail(f"corpus hybrid ndcg below threshold: {corpus_metrics}")
|
|
182
|
+
if corpus_metrics.get("must_include_hit_rate", 0.0) < 0.90:
|
|
183
|
+
return _fail(f"corpus must-include hit rate below threshold: {corpus_metrics}")
|
|
184
|
+
|
|
185
|
+
print(f"brain-quality-eval: OK small={retrieval_metrics} corpus={corpus_metrics}")
|
|
114
186
|
return 0
|
|
115
187
|
|
|
116
188
|
|
package/src-tauri/Cargo.lock
CHANGED
package/src-tauri/Cargo.toml
CHANGED