@jaguilar87/gaia 5.1.0 → 5.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +11 -0
- package/agents/gaia-orchestrator.md +4 -10
- package/bin/README.md +1 -1
- package/bin/cli/approvals.py +0 -144
- package/bin/cli/cleanup.py +1 -30
- package/bin/cli/context.py +379 -6
- package/bin/cli/doctor.py +1 -1
- package/bin/cli/history.py +18 -6
- package/bin/cli/install.py +4 -2
- package/bin/cli/memory.py +86 -10
- package/bin/cli/metrics.py +1216 -275
- package/bin/cli/query.py +89 -0
- package/bin/cli/scan.py +68 -2
- package/bin/cli/uninstall.py +9 -5
- package/bin/pre-publish-validate.js +7 -4
- package/gaia/project.py +49 -20
- package/gaia/store/reader.py +175 -11
- package/gaia/store/schema.sql +180 -1
- package/gaia/store/writer.py +1146 -165
- package/hooks/README.md +2 -2
- package/hooks/adapters/claude_code.py +6 -12
- package/hooks/elicitation_result.py +4 -9
- package/hooks/modules/agents/contract_validator.py +2 -1
- package/hooks/modules/agents/handoff_persister.py +5 -220
- package/hooks/modules/agents/task_info_builder.py +9 -2
- package/hooks/modules/agents/transcript_reader.py +41 -21
- package/hooks/modules/audit/logger.py +8 -3
- package/hooks/modules/context/context_injector.py +10 -3
- package/hooks/modules/core/__init__.py +5 -2
- package/hooks/modules/core/logging_setup.py +60 -0
- package/hooks/modules/core/paths.py +0 -12
- package/hooks/modules/security/approval_grants.py +24 -92
- package/hooks/modules/security/gaia_db_write_guard.py +7 -2
- package/hooks/modules/security/mutative_verbs.py +194 -11
- package/hooks/modules/session/pending_scanner.py +24 -222
- package/hooks/modules/session/session_manifest.py +13 -288
- package/hooks/post_compact.py +4 -9
- package/hooks/post_tool_use.py +4 -9
- package/hooks/pre_compact.py +5 -10
- package/hooks/pre_tool_use.py +4 -11
- package/hooks/session_end_hook.py +4 -9
- package/hooks/session_start.py +18 -21
- package/hooks/stop_hook.py +5 -10
- package/hooks/subagent_start.py +5 -10
- package/hooks/subagent_stop.py +9 -23
- package/hooks/task_completed.py +5 -10
- package/hooks/user_prompt_submit.py +4 -9
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/scripts/migrations/schema.checksum +2 -2
- package/scripts/migrations/v21_to_v22.sql +80 -0
- package/scripts/migrations/v22_to_v23.sql +20 -0
- package/scripts/migrations/v23_to_v24.sql +30 -0
- package/scripts/migrations/v24_to_v25.sql +77 -0
- package/scripts/migrations/v25_to_v26.sql +116 -0
- package/skills/README.md +8 -0
- package/skills/agent-approval-protocol/SKILL.md +27 -27
- package/skills/agent-approval-protocol/reference.md +20 -12
- package/skills/agent-protocol/SKILL.md +1 -1
- package/skills/agent-protocol/examples.md +17 -9
- package/skills/agent-response/SKILL.md +1 -1
- package/skills/diagram-builder/GLOSSARY.md +77 -0
- package/skills/diagram-builder/SKILL.md +156 -0
- package/skills/diagram-builder/assets/README.md +45 -0
- package/skills/diagram-builder/assets/data/data.generated.js +77 -0
- package/skills/diagram-builder/assets/data/document.yaml +12 -0
- package/skills/diagram-builder/assets/data/pages/overview.yaml +47 -0
- package/skills/diagram-builder/assets/engine/build-data.mjs +51 -0
- package/skills/diagram-builder/assets/engine/engine.js +863 -0
- package/skills/diagram-builder/assets/index.html +638 -0
- package/skills/diagram-builder/assets/package.json +15 -0
- package/skills/diagram-builder/assets/tools/verify.mjs +128 -0
- package/skills/diagram-builder/reference.md +444 -0
- package/skills/execution/SKILL.md +3 -3
- package/skills/gaia-patterns/reference.md +2 -2
- package/skills/memory/SKILL.md +49 -0
- package/skills/orchestrator-present-approval/SKILL.md +90 -79
- package/skills/orchestrator-present-approval/reference.md +59 -42
- package/skills/orchestrator-present-approval/template.md +16 -14
- package/skills/pending-approvals/SKILL.md +80 -29
- package/skills/pending-approvals/reference.md +48 -60
- package/skills/security-tiers/SKILL.md +1 -1
- package/skills/subagent-request-approval/SKILL.md +65 -68
- package/skills/subagent-request-approval/reference.md +63 -63
- package/skills/visual-verify/SKILL.md +107 -0
- package/skills/visual-verify/scripts/screenshot.js +210 -0
- package/tools/scan/classify.py +584 -15
- package/tools/scan/migrate_workspace.py +9 -4
- package/tools/scan/store_populator.py +231 -2
- package/tools/scan/tests/conftest.py +31 -0
package/bin/cli/metrics.py
CHANGED
|
@@ -2,30 +2,57 @@
|
|
|
2
2
|
gaia metrics -- Usage analytics: tier classification, agent invocations,
|
|
3
3
|
anomaly counters.
|
|
4
4
|
|
|
5
|
+
Dashboard v3: cmd_metrics() computes one MetricsSnapshot (see the
|
|
6
|
+
MetricsSnapshot dataclass below) per invocation and feeds it to both
|
|
7
|
+
render_console() and json.dumps() -- the data is calculated exactly once,
|
|
8
|
+
under one canonical set of keys, versioned via schema_version.
|
|
9
|
+
|
|
10
|
+
New in v3: a single TimeWindow governs every data read. --range=today|3d|
|
|
11
|
+
7d|30d|all (default 30d) or an explicit --since/--until (mutually exclusive
|
|
12
|
+
with --range) selects the window; gaia.store.reader.parse_when() does all
|
|
13
|
+
duration/date parsing (no new parser here). Episodes/episode_anomalies are
|
|
14
|
+
filtered in SQL to the requested bound exactly; the three audit-log sections
|
|
15
|
+
(tier usage, command breakdown, top commands) are additionally capped at the
|
|
16
|
+
audit log's ~30d retention floor, and the window records that cap so the
|
|
17
|
+
affected boxes can declare it.
|
|
18
|
+
|
|
5
19
|
Displays system metrics dashboard:
|
|
6
|
-
- Security tier usage distribution
|
|
20
|
+
- Security tier usage distribution (T3-today folded in here, not repeated)
|
|
21
|
+
- Runtime Skill Snapshots (Gaia specialists only, busiest profile first)
|
|
7
22
|
- Command type breakdown
|
|
8
|
-
- Top commands by frequency
|
|
9
|
-
- Agent invocations
|
|
10
|
-
-
|
|
11
|
-
|
|
12
|
-
-
|
|
13
|
-
-
|
|
23
|
+
- Top commands by frequency, with average duration
|
|
24
|
+
- Agent invocations (Gaia specialists only)
|
|
25
|
+
- Native agent activity (Explore/Plan/claude-code-guide/general-purpose,
|
|
26
|
+
segregated so harness noise doesn't drown out Gaia specialist signal)
|
|
27
|
+
- Agent outcomes, translated to plain language
|
|
28
|
+
- Token usage (real, transcript-parsed when available; approx/chars-4
|
|
29
|
+
fallback labeled as such -- never guaranteed to be "real")
|
|
30
|
+
- Context snapshot / context update summaries
|
|
31
|
+
- Anomaly summary, severity-sorted with a one-line human gloss per type
|
|
32
|
+
- A compact "Today (UTC)" strip under the header (not a boxed section)
|
|
14
33
|
|
|
15
34
|
With --agent NAME shows a detail view for that agent.
|
|
16
35
|
|
|
17
36
|
Data sources:
|
|
18
|
-
~/.gaia/gaia.db (substrate SQLite
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
.claude/
|
|
24
|
-
|
|
37
|
+
~/.gaia/gaia.db (substrate SQLite)
|
|
38
|
+
- episodes table -> agent invocations, outcomes, token usage,
|
|
39
|
+
runtime skills, context snapshots/updates
|
|
40
|
+
(context_metrics JSON column)
|
|
41
|
+
- episode_anomalies table -> anomaly summary
|
|
42
|
+
.claude/logs/audit-*.jsonl (security tier events -> tier usage, command
|
|
43
|
+
breakdown, top commands; ~30d retention,
|
|
44
|
+
not filtered by workspace)
|
|
25
45
|
|
|
26
46
|
Flags:
|
|
27
|
-
--agent NAME
|
|
28
|
-
--
|
|
47
|
+
--agent NAME Show detail view for a specific agent
|
|
48
|
+
--workspace NAME Workspace identity override (default: gaia.project.current())
|
|
49
|
+
--range VALUE today|3d|7d|30d|all (default: 30d). Mutually exclusive
|
|
50
|
+
with --since/--until.
|
|
51
|
+
--since VALUE Lower bound -- duration ('24h', '7d') or ISO date.
|
|
52
|
+
Mutually exclusive with --range.
|
|
53
|
+
--until VALUE Upper bound, same format as --since.
|
|
54
|
+
--json Machine-readable output (MetricsSnapshot.to_dict(), versioned
|
|
55
|
+
via schema_version)
|
|
29
56
|
"""
|
|
30
57
|
|
|
31
58
|
import fnmatch
|
|
@@ -33,8 +60,150 @@ import json
|
|
|
33
60
|
import os
|
|
34
61
|
import re
|
|
35
62
|
import sys
|
|
63
|
+
from dataclasses import dataclass, asdict
|
|
36
64
|
from datetime import datetime, timezone, timedelta
|
|
37
65
|
from pathlib import Path
|
|
66
|
+
from typing import Optional
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# Schema version for the JSON contract emitted by MetricsSnapshot.to_dict().
|
|
70
|
+
# Bump this whenever a section's shape changes in a way a machine consumer
|
|
71
|
+
# (future web/API surface) would need to branch on.
|
|
72
|
+
# v2: adds 'window' (label/since_iso/until_iso/capped_by_retention) and
|
|
73
|
+
# 'window_support' (which sections are episodes- vs audit-log-backed).
|
|
74
|
+
SCHEMA_VERSION = "2"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ---------------------------------------------------------------------------
|
|
78
|
+
# Repo-root sys.path helper (shared by every gaia.* import below)
|
|
79
|
+
# ---------------------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def _ensure_repo_on_path() -> None:
|
|
82
|
+
"""Put the gaia repo root on sys.path so `hooks.*` / `gaia.*` import.
|
|
83
|
+
|
|
84
|
+
metrics.py can be invoked with bin/ NOT on sys.path (e.g. standalone),
|
|
85
|
+
so every module-level import of a repo-internal package goes through
|
|
86
|
+
this first.
|
|
87
|
+
"""
|
|
88
|
+
_bin_dir = Path(__file__).resolve().parent.parent
|
|
89
|
+
_repo_root = _bin_dir.parent
|
|
90
|
+
if str(_repo_root) not in sys.path:
|
|
91
|
+
sys.path.insert(0, str(_repo_root))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# ---------------------------------------------------------------------------
|
|
95
|
+
# Native agent segregation
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
_FALLBACK_NATIVE_AGENTS = frozenset({"Explore", "Plan", "general-purpose", "claude-code-guide"})
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _native_agent_names() -> frozenset:
|
|
102
|
+
"""Harness-native Claude Code agent names.
|
|
103
|
+
|
|
104
|
+
Reuses the canonical ``NATIVE_AGENTS`` list from
|
|
105
|
+
hooks/modules/tools/task_validator.py -- the same list the Task-tool
|
|
106
|
+
validator uses to recognize non-Gaia dispatch targets -- instead of
|
|
107
|
+
re-declaring it here. Falls back to a mirrored literal if the hooks
|
|
108
|
+
package cannot be imported (e.g. metrics.py invoked standalone).
|
|
109
|
+
"""
|
|
110
|
+
_ensure_repo_on_path()
|
|
111
|
+
try:
|
|
112
|
+
from hooks.modules.tools.task_validator import NATIVE_AGENTS
|
|
113
|
+
return frozenset(NATIVE_AGENTS)
|
|
114
|
+
except Exception:
|
|
115
|
+
return _FALLBACK_NATIVE_AGENTS
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
NATIVE_AGENT_NAMES = _native_agent_names()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# Time window (dashboard v3) -- one window governs every data read
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
_RANGE_CHOICES = ("today", "3d", "7d", "30d", "all")
|
|
126
|
+
|
|
127
|
+
# Fallback window used when a caller (or an older test) builds a
|
|
128
|
+
# MetricsSnapshot without specifying one -- equivalent to no filtering.
|
|
129
|
+
_DEFAULT_WINDOW_LABEL = "all"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass
|
|
133
|
+
class TimeWindow:
|
|
134
|
+
"""The single time bound governing one `gaia metrics` invocation.
|
|
135
|
+
|
|
136
|
+
Built once in cmd_metrics() and threaded into every reader and into
|
|
137
|
+
MetricsSnapshot.build(). ``since_iso`` / ``until_iso`` are the bound as
|
|
138
|
+
requested (used verbatim for the SQL-backed episodes/episode_anomalies
|
|
139
|
+
reads). ``capped_by_retention`` is set by cmd_metrics when the
|
|
140
|
+
audit-log-backed sections (tier usage, command breakdown, top commands)
|
|
141
|
+
had to clamp the lower bound to the ~30d audit-log retention floor --
|
|
142
|
+
the episodes-backed sections are never capped, since gaia.db keeps full
|
|
143
|
+
history.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
label: str
|
|
147
|
+
since_iso: Optional[str] = None
|
|
148
|
+
until_iso: Optional[str] = None
|
|
149
|
+
capped_by_retention: bool = False
|
|
150
|
+
|
|
151
|
+
def to_dict(self) -> dict:
|
|
152
|
+
return {
|
|
153
|
+
"label": self.label,
|
|
154
|
+
"since_iso": self.since_iso,
|
|
155
|
+
"until_iso": self.until_iso,
|
|
156
|
+
"capped_by_retention": self.capped_by_retention,
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
_DEFAULT_WINDOW = TimeWindow(label=_DEFAULT_WINDOW_LABEL)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _reader_parse_when():
|
|
164
|
+
"""Return gaia.store.reader.parse_when -- the ONE duration/date parser.
|
|
165
|
+
|
|
166
|
+
No new parser is written here; --range/--since/--until all funnel
|
|
167
|
+
through the same normalizer `gaia query` uses (see bin/cli/query.py).
|
|
168
|
+
"""
|
|
169
|
+
_ensure_repo_on_path()
|
|
170
|
+
from gaia.store.reader import parse_when
|
|
171
|
+
return parse_when
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _resolve_time_window(
|
|
175
|
+
range_value: Optional[str],
|
|
176
|
+
since_value: Optional[str],
|
|
177
|
+
until_value: Optional[str],
|
|
178
|
+
) -> TimeWindow:
|
|
179
|
+
"""Build the TimeWindow for this invocation from --range or --since/--until.
|
|
180
|
+
|
|
181
|
+
Mutual exclusivity between --range and --since/--until is validated by
|
|
182
|
+
the caller (cmd_metrics) before this runs. ``range_value`` defaults to
|
|
183
|
+
"30d" when none of the three flags were passed.
|
|
184
|
+
|
|
185
|
+
Raises:
|
|
186
|
+
ValueError: propagated from parse_when() on an unparseable
|
|
187
|
+
--since/--until value.
|
|
188
|
+
"""
|
|
189
|
+
if since_value or until_value:
|
|
190
|
+
parse_when = _reader_parse_when()
|
|
191
|
+
since_iso = parse_when(since_value) if since_value else None
|
|
192
|
+
until_iso = parse_when(until_value) if until_value else None
|
|
193
|
+
label = f"{since_value or '-'}..{until_value or 'now'}"
|
|
194
|
+
return TimeWindow(label=label, since_iso=since_iso, until_iso=until_iso)
|
|
195
|
+
|
|
196
|
+
range_value = range_value or "30d"
|
|
197
|
+
if range_value == "all":
|
|
198
|
+
return TimeWindow(label="all", since_iso=None, until_iso=None)
|
|
199
|
+
if range_value == "today":
|
|
200
|
+
since_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z")
|
|
201
|
+
return TimeWindow(label="today", since_iso=since_iso, until_iso=None)
|
|
202
|
+
|
|
203
|
+
# "3d" / "7d" / "30d" -- durations parse_when already understands.
|
|
204
|
+
parse_when = _reader_parse_when()
|
|
205
|
+
since_iso = parse_when(range_value)
|
|
206
|
+
return TimeWindow(label=range_value, since_iso=since_iso, until_iso=None)
|
|
38
207
|
|
|
39
208
|
|
|
40
209
|
# ---------------------------------------------------------------------------
|
|
@@ -80,7 +249,15 @@ def _read_jsonl(path: Path) -> list:
|
|
|
80
249
|
return entries
|
|
81
250
|
|
|
82
251
|
|
|
83
|
-
def _read_audit_logs(root: Path) -> list:
|
|
252
|
+
def _read_audit_logs(root: Path, since_iso: str = None, until_iso: str = None) -> list:
|
|
253
|
+
"""Read every audit-*.jsonl entry, optionally bounded to [since_iso, until_iso].
|
|
254
|
+
|
|
255
|
+
``since_iso`` / ``until_iso`` are ISO8601 strings (as produced by
|
|
256
|
+
parse_when()); when both are None (the default), behaves exactly as
|
|
257
|
+
before -- no filtering. Callers pass the retention-capped bound here
|
|
258
|
+
(see cmd_metrics), never the raw requested window, since these files
|
|
259
|
+
only retain ~30d.
|
|
260
|
+
"""
|
|
84
261
|
logs_dir = root / ".claude" / "logs"
|
|
85
262
|
if not logs_dir.exists():
|
|
86
263
|
return []
|
|
@@ -91,41 +268,298 @@ def _read_audit_logs(root: Path) -> list:
|
|
|
91
268
|
all_entries.extend(_read_jsonl(f))
|
|
92
269
|
except OSError:
|
|
93
270
|
pass
|
|
271
|
+
|
|
272
|
+
if since_iso or until_iso:
|
|
273
|
+
all_entries = [
|
|
274
|
+
e for e in all_entries
|
|
275
|
+
if (not since_iso or (e.get("timestamp") or "") >= since_iso)
|
|
276
|
+
and (not until_iso or (e.get("timestamp") or "") <= until_iso)
|
|
277
|
+
]
|
|
94
278
|
return all_entries
|
|
95
279
|
|
|
96
280
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
281
|
+
# ---------------------------------------------------------------------------
|
|
282
|
+
# DB-backed readers (T6 episodic-workflow-to-db migration)
|
|
283
|
+
#
|
|
284
|
+
# T4 migrated the workflow episodic writers from JSONL/JSON files to the
|
|
285
|
+
# gaia.db ``episodes`` + ``episode_anomalies`` tables. These readers were the
|
|
286
|
+
# missing half of that migration (T6): they now query gaia.db via
|
|
287
|
+
# gaia.store.reader instead of the dead .claude/project-context/*.jsonl files.
|
|
288
|
+
# The connection setup mirrors bin/cli/history.py's own T6 migration exactly:
|
|
289
|
+
# resolve the workspace via gaia.project.current, open the store connection,
|
|
290
|
+
# and fall back to [] on any import/connection failure. The three audit-log
|
|
291
|
+
# sections (tier usage, command breakdown, top commands) still read
|
|
292
|
+
# .claude/logs/audit-*.jsonl and are untouched.
|
|
293
|
+
#
|
|
294
|
+
# Dashboard v3: all three gain an optional since_iso/until_iso SQL bound
|
|
295
|
+
# (the TimeWindow built once in cmd_metrics), since gaia.db keeps full
|
|
296
|
+
# history and can answer any requested range exactly.
|
|
297
|
+
# ---------------------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
def _open_store(root: Path, workspace_override: str = None):
|
|
300
|
+
"""Resolve (connection, workspace) against gaia.db, or (None, None).
|
|
301
|
+
|
|
302
|
+
``workspace_override`` -- explicit ``--workspace`` value from the CLI.
|
|
303
|
+
When set, it wins over ``gaia.project.current()`` resolution. Mirrors the
|
|
304
|
+
connection setup in bin/cli/history.py._read_workflow_metrics.
|
|
305
|
+
"""
|
|
306
|
+
_ensure_repo_on_path()
|
|
307
|
+
|
|
308
|
+
try:
|
|
309
|
+
from gaia.store.reader import _connect
|
|
310
|
+
from gaia.project import current as _project_current
|
|
311
|
+
except ImportError:
|
|
312
|
+
return None, None
|
|
313
|
+
|
|
314
|
+
if workspace_override:
|
|
315
|
+
ws = workspace_override
|
|
316
|
+
else:
|
|
317
|
+
try:
|
|
318
|
+
ws = _project_current(cwd=root)
|
|
319
|
+
except Exception:
|
|
320
|
+
ws = None
|
|
321
|
+
|
|
322
|
+
try:
|
|
323
|
+
con = _connect()
|
|
324
|
+
except Exception:
|
|
325
|
+
return None, None
|
|
326
|
+
return con, ws
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _episode_time_filters(
|
|
330
|
+
ws: Optional[str],
|
|
331
|
+
since_iso: Optional[str],
|
|
332
|
+
until_iso: Optional[str],
|
|
333
|
+
extra: list = None,
|
|
334
|
+
ws_col: str = "workspace",
|
|
335
|
+
ts_col: str = "timestamp",
|
|
336
|
+
) -> tuple:
|
|
337
|
+
"""Build the shared (WHERE clauses, params) for the three DB readers.
|
|
338
|
+
|
|
339
|
+
``ws_col`` / ``ts_col`` let the anomaly reader (which joins two tables)
|
|
340
|
+
qualify the columns (``ea.workspace``, ``ea.timestamp``).
|
|
341
|
+
"""
|
|
342
|
+
clauses = list(extra or [])
|
|
343
|
+
params = []
|
|
344
|
+
if ws:
|
|
345
|
+
clauses.append(f"{ws_col} = ?")
|
|
346
|
+
params.append(ws)
|
|
347
|
+
if since_iso:
|
|
348
|
+
clauses.append(f"{ts_col} >= ?")
|
|
349
|
+
params.append(since_iso)
|
|
350
|
+
if until_iso:
|
|
351
|
+
clauses.append(f"{ts_col} <= ?")
|
|
352
|
+
params.append(until_iso)
|
|
353
|
+
return clauses, params
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _extract_real_token_fields(raw_context_metrics) -> dict:
|
|
357
|
+
"""Pull real (transcript-parsed) token counts out of an episode's
|
|
358
|
+
``context_metrics`` JSON blob.
|
|
359
|
+
|
|
360
|
+
Mirrors the ``metrics`` dict shape written by
|
|
361
|
+
hooks/modules/audit/workflow_recorder.py: alongside the always-present
|
|
362
|
+
``output_tokens_approx`` (chars/4 heuristic), a transcript-backed run also
|
|
363
|
+
carries ``output_tokens_real`` / ``input_tokens`` / ``cache_creation_tokens``
|
|
364
|
+
/ ``cache_read_tokens``. Returns all four keys, each ``None`` when absent,
|
|
365
|
+
so ``_calculate_token_usage`` can tell "real" from "approx" per entry.
|
|
366
|
+
"""
|
|
367
|
+
fields = {
|
|
368
|
+
"output_tokens_real": None,
|
|
369
|
+
"input_tokens": None,
|
|
370
|
+
"cache_creation_tokens": None,
|
|
371
|
+
"cache_read_tokens": None,
|
|
372
|
+
}
|
|
373
|
+
if not raw_context_metrics:
|
|
374
|
+
return fields
|
|
375
|
+
try:
|
|
376
|
+
blob = json.loads(raw_context_metrics)
|
|
377
|
+
except (json.JSONDecodeError, TypeError):
|
|
378
|
+
return fields
|
|
379
|
+
if isinstance(blob, dict) and isinstance(blob.get("metrics"), dict):
|
|
380
|
+
metrics = blob["metrics"]
|
|
381
|
+
else:
|
|
382
|
+
metrics = blob
|
|
383
|
+
if not isinstance(metrics, dict):
|
|
384
|
+
return fields
|
|
385
|
+
for key in fields:
|
|
386
|
+
if key in metrics:
|
|
387
|
+
fields[key] = metrics[key]
|
|
388
|
+
return fields
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _read_workflow_metrics(
|
|
392
|
+
root: Path,
|
|
393
|
+
workspace_override: str = None,
|
|
394
|
+
since_iso: str = None,
|
|
395
|
+
until_iso: str = None,
|
|
396
|
+
) -> list:
|
|
397
|
+
"""Agent-session rows from the gaia.db ``episodes`` table (T6 migration).
|
|
398
|
+
|
|
399
|
+
Returns episode dicts carrying agent/timestamp/plan_status/exit_code/
|
|
400
|
+
output_length/output_tokens_approx -- the fields the agent-invocation,
|
|
401
|
+
agent-outcome, and token-usage calculators consume. Replaces the dead
|
|
402
|
+
episodic-memory/index.json + workflow-episodic-memory/metrics.jsonl reads.
|
|
403
|
+
|
|
404
|
+
``since_iso`` / ``until_iso`` (dashboard v3) bound the SQL query to the
|
|
405
|
+
caller's TimeWindow -- gaia.db has full history, so this filter is exact
|
|
406
|
+
(unlike the audit-log readers, which are additionally capped to ~30d).
|
|
407
|
+
|
|
408
|
+
Each row also carries the real-token fields extracted from
|
|
409
|
+
``context_metrics`` (output_tokens_real / input_tokens /
|
|
410
|
+
cache_creation_tokens / cache_read_tokens, all ``None`` when the episode
|
|
411
|
+
has no transcript-backed metrics) so token-usage reporting can prefer real
|
|
412
|
+
counts over the chars/4 approximation. The raw ``context_metrics`` blob
|
|
413
|
+
itself is not retained on the row -- only its extracted token fields.
|
|
414
|
+
"""
|
|
415
|
+
con, ws = _open_store(root, workspace_override)
|
|
416
|
+
if con is None:
|
|
417
|
+
return []
|
|
418
|
+
try:
|
|
101
419
|
try:
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
420
|
+
clauses, params = _episode_time_filters(ws, since_iso, until_iso, extra=["agent IS NOT NULL"])
|
|
421
|
+
where = " AND ".join(clauses)
|
|
422
|
+
rows = con.execute(
|
|
423
|
+
"SELECT episode_id, workspace, timestamp, session_id, task_id, "
|
|
424
|
+
"agent, type, title, plan_status, outcome, exit_code, "
|
|
425
|
+
"duration_seconds, output_length, output_tokens_approx, tier, "
|
|
426
|
+
"context_metrics "
|
|
427
|
+
f"FROM episodes WHERE {where} "
|
|
428
|
+
"ORDER BY timestamp DESC",
|
|
429
|
+
params,
|
|
430
|
+
).fetchall()
|
|
431
|
+
result = []
|
|
432
|
+
for r in rows:
|
|
433
|
+
row = dict(r)
|
|
434
|
+
raw_cm = row.pop("context_metrics", None)
|
|
435
|
+
row.update(_extract_real_token_fields(raw_cm))
|
|
436
|
+
result.append(row)
|
|
437
|
+
return result
|
|
438
|
+
finally:
|
|
439
|
+
con.close()
|
|
440
|
+
except Exception:
|
|
441
|
+
return []
|
|
108
442
|
|
|
109
|
-
metrics_path = root / ".claude" / "project-context" / "workflow-episodic-memory" / "metrics.jsonl"
|
|
110
|
-
return [e for e in _read_jsonl(metrics_path) if e.get("agent")]
|
|
111
443
|
|
|
444
|
+
def _read_run_snapshots(
|
|
445
|
+
root: Path,
|
|
446
|
+
workspace_override: str = None,
|
|
447
|
+
since_iso: str = None,
|
|
448
|
+
until_iso: str = None,
|
|
449
|
+
) -> list:
|
|
450
|
+
"""Per-episode workflow-metrics blobs from ``episodes.context_metrics``.
|
|
451
|
+
|
|
452
|
+
T4 folded the old run-snapshots.jsonl signals (context_snapshot,
|
|
453
|
+
context_updated / *_sections, default_skills_snapshot, model, skills) into
|
|
454
|
+
the ``episodes.context_metrics`` JSON column under the ``metrics`` key.
|
|
455
|
+
This reader parses that blob per episode so the context-snapshot,
|
|
456
|
+
context-update, and runtime-skill calculators keep working. Older migrated
|
|
457
|
+
rows that stored the metrics dict at the top level are handled too.
|
|
458
|
+
|
|
459
|
+
``since_iso`` / ``until_iso`` (dashboard v3) bound the query the same way
|
|
460
|
+
as ``_read_workflow_metrics``.
|
|
461
|
+
"""
|
|
462
|
+
con, ws = _open_store(root, workspace_override)
|
|
463
|
+
if con is None:
|
|
464
|
+
return []
|
|
465
|
+
try:
|
|
466
|
+
try:
|
|
467
|
+
clauses, params = _episode_time_filters(
|
|
468
|
+
ws, since_iso, until_iso, extra=["context_metrics IS NOT NULL"]
|
|
469
|
+
)
|
|
470
|
+
where = " AND ".join(clauses)
|
|
471
|
+
rows = con.execute(
|
|
472
|
+
f"SELECT context_metrics FROM episodes WHERE {where} ORDER BY timestamp DESC",
|
|
473
|
+
params,
|
|
474
|
+
).fetchall()
|
|
475
|
+
finally:
|
|
476
|
+
con.close()
|
|
477
|
+
except Exception:
|
|
478
|
+
return []
|
|
112
479
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
480
|
+
snapshots = []
|
|
481
|
+
for r in rows:
|
|
482
|
+
raw = r["context_metrics"]
|
|
483
|
+
if not raw:
|
|
484
|
+
continue
|
|
485
|
+
try:
|
|
486
|
+
blob = json.loads(raw)
|
|
487
|
+
except (json.JSONDecodeError, TypeError):
|
|
488
|
+
continue
|
|
489
|
+
if isinstance(blob, dict) and isinstance(blob.get("metrics"), dict):
|
|
490
|
+
snap = blob["metrics"]
|
|
491
|
+
else:
|
|
492
|
+
snap = blob
|
|
493
|
+
if isinstance(snap, dict):
|
|
494
|
+
snapshots.append(snap)
|
|
495
|
+
return snapshots
|
|
117
496
|
|
|
118
497
|
|
|
119
498
|
def _read_agent_skill_snapshots(root: Path) -> list:
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
)
|
|
123
|
-
|
|
499
|
+
"""Explicit per-agent skill snapshots.
|
|
500
|
+
|
|
501
|
+
The legacy agent-skills.jsonl (explicit snapshots) has no gaia.db
|
|
502
|
+
equivalent -- the runtime-skill summary now derives every profile from
|
|
503
|
+
each episode's ``default_skills_snapshot`` (supplied by
|
|
504
|
+
_read_run_snapshots). Returns [] so _calculate_runtime_skill_summary falls
|
|
505
|
+
back to those run-default profiles.
|
|
506
|
+
"""
|
|
507
|
+
return []
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _read_anomaly_entries(
|
|
511
|
+
root: Path,
|
|
512
|
+
workspace_override: str = None,
|
|
513
|
+
since_iso: str = None,
|
|
514
|
+
until_iso: str = None,
|
|
515
|
+
) -> list:
|
|
516
|
+
"""Anomaly entries grouped per episode from the ``episode_anomalies`` table.
|
|
517
|
+
|
|
518
|
+
T4 migrated anomalies from workflow-episodic-memory/anomalies.jsonl into
|
|
519
|
+
the ``episode_anomalies`` child table (one row per anomaly). This reader
|
|
520
|
+
regroups them into the per-session shape the anomaly-summary calculator
|
|
521
|
+
expects: ``{timestamp, anomalies: [{type}, ...], metrics: {agent}}``.
|
|
522
|
+
|
|
523
|
+
``since_iso`` / ``until_iso`` (dashboard v3) bound ``ea.timestamp`` --
|
|
524
|
+
the calculator no longer applies its own age cutoff (see
|
|
525
|
+
_calculate_anomaly_summary), so this is the sole point of time filtering.
|
|
526
|
+
"""
|
|
527
|
+
con, ws = _open_store(root, workspace_override)
|
|
528
|
+
if con is None:
|
|
529
|
+
return []
|
|
530
|
+
try:
|
|
531
|
+
try:
|
|
532
|
+
clauses, params = _episode_time_filters(
|
|
533
|
+
ws, since_iso, until_iso, ws_col="ea.workspace", ts_col="ea.timestamp"
|
|
534
|
+
)
|
|
535
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
536
|
+
rows = con.execute(
|
|
537
|
+
"SELECT ea.episode_id AS episode_id, ea.timestamp AS timestamp, "
|
|
538
|
+
"ea.type AS type, ea.severity AS severity, e.agent AS agent "
|
|
539
|
+
"FROM episode_anomalies ea "
|
|
540
|
+
"LEFT JOIN episodes e ON e.episode_id = ea.episode_id "
|
|
541
|
+
f"{where} "
|
|
542
|
+
"ORDER BY ea.timestamp DESC",
|
|
543
|
+
params,
|
|
544
|
+
).fetchall()
|
|
545
|
+
finally:
|
|
546
|
+
con.close()
|
|
547
|
+
except Exception:
|
|
548
|
+
return []
|
|
124
549
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
550
|
+
grouped = {}
|
|
551
|
+
order = []
|
|
552
|
+
for r in rows:
|
|
553
|
+
ep = r["episode_id"]
|
|
554
|
+
if ep not in grouped:
|
|
555
|
+
grouped[ep] = {
|
|
556
|
+
"timestamp": r["timestamp"],
|
|
557
|
+
"anomalies": [],
|
|
558
|
+
"metrics": {"agent": r["agent"] or "unknown"},
|
|
559
|
+
}
|
|
560
|
+
order.append(ep)
|
|
561
|
+
grouped[ep]["anomalies"].append({"type": r["type"], "severity": r["severity"]})
|
|
562
|
+
return [grouped[ep] for ep in order]
|
|
129
563
|
|
|
130
564
|
|
|
131
565
|
def _read_agent_definition(root: Path, agent_name: str) -> dict:
|
|
@@ -233,9 +667,30 @@ def _format_chars(n) -> str:
|
|
|
233
667
|
return str(n)
|
|
234
668
|
|
|
235
669
|
|
|
236
|
-
def
|
|
237
|
-
|
|
238
|
-
|
|
670
|
+
def _format_duration_ms(ms) -> str:
|
|
671
|
+
"""Human duration from a millisecond value; 'n/a' when unavailable.
|
|
672
|
+
|
|
673
|
+
Backs the Top Commands avg-duration column (P1 #5) -- ``duration_ms`` is
|
|
674
|
+
recorded per audit entry by hooks/modules/audit/logger.py but was unused
|
|
675
|
+
by metrics.py until now.
|
|
676
|
+
"""
|
|
677
|
+
if ms is None:
|
|
678
|
+
return "n/a"
|
|
679
|
+
if ms >= 1000:
|
|
680
|
+
return f"{ms / 1000:.1f}s"
|
|
681
|
+
return f"{ms:.0f}ms"
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _make_bar(percentage: float, width: int = 24) -> str:
|
|
685
|
+
"""Fixed-width Unicode bar: filled cells (█) + empty cells (░).
|
|
686
|
+
|
|
687
|
+
Unlike the old ``#``-only bar (which returned only the filled prefix and
|
|
688
|
+
relied on the caller's ``:<N`` string format for padding), this returns
|
|
689
|
+
the full ``width``-length string so callers can drop straight into a
|
|
690
|
+
fixed-column box row without separate padding.
|
|
691
|
+
"""
|
|
692
|
+
filled = max(0, min(width, round((percentage / 100) * width)))
|
|
693
|
+
return "█" * filled + "░" * (width - filled)
|
|
239
694
|
|
|
240
695
|
|
|
241
696
|
def _count_values(values: list) -> dict:
|
|
@@ -272,6 +727,66 @@ def _format_skills(skills: list, limit: int = 4) -> str:
|
|
|
272
727
|
return ", ".join(skills[:limit]) + f", +{len(skills) - limit} more"
|
|
273
728
|
|
|
274
729
|
|
|
730
|
+
# ---------------------------------------------------------------------------
|
|
731
|
+
# Human-readable glossaries (dashboard v3, P1/P3 legibility pass)
|
|
732
|
+
# ---------------------------------------------------------------------------
|
|
733
|
+
|
|
734
|
+
# One-line plain-language meaning per workflow-auditor / subagent_stop
|
|
735
|
+
# anomaly type. Keys are the real ``type`` values emitted by
|
|
736
|
+
# hooks/modules/audit/workflow_auditor.py (audit_workflow) and
|
|
737
|
+
# hooks/subagent_stop.py (response_contract_violation).
|
|
738
|
+
_ANOMALY_TYPE_GLOSSARY = {
|
|
739
|
+
"response_contract_violation": "agent's response envelope was missing or invalid",
|
|
740
|
+
"execution_failure": "agent exited with a non-zero code",
|
|
741
|
+
"investigation_skip": "agent acted without investigating first",
|
|
742
|
+
"context_ignored": "agent ignored the project context it was given",
|
|
743
|
+
"context_update_missing": "agent didn't write back an expected context update",
|
|
744
|
+
"missing_evidence": "evidence_report was empty or missing required fields",
|
|
745
|
+
"empty_evidence": "an evidence sub-section (e.g. files_checked) came back empty",
|
|
746
|
+
"skipped_verification": "declared COMPLETE without a verification record",
|
|
747
|
+
"scope_escalation": "agent touched resources outside its declared scope",
|
|
748
|
+
"pipe_retroactive": "a pipe was detected in the executed command (post-hoc)",
|
|
749
|
+
"excessive_tool_calls": "tool-call count far above the normal range",
|
|
750
|
+
"token_budget": "token usage approaching or exceeding the expected budget",
|
|
751
|
+
"token_explosion": "unusually large token output for a single turn",
|
|
752
|
+
"model_mismatch": "model actually used differs from the agent's declared model",
|
|
753
|
+
"skill_order": "skills loaded in an unexpected order",
|
|
754
|
+
"duplicate_tools": "the same tool declared more than once",
|
|
755
|
+
"cache_efficiency": "low prompt-cache hit rate for the turn",
|
|
756
|
+
"bash_permission_gate": "a bash command was gated/blocked by the permission layer",
|
|
757
|
+
"duplicate_write_storm": "many duplicate writes to the same resource in one turn",
|
|
758
|
+
"duration_outlier": "turn duration was a statistical outlier vs. this agent's history",
|
|
759
|
+
"tool_call_velocity": "tool calls fired at an abnormally high rate",
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
# Plain-language meaning per plan_status (agent_contract_handoff enum).
|
|
763
|
+
_PLAN_STATUS_GLOSS = {
|
|
764
|
+
"COMPLETE": "finished successfully",
|
|
765
|
+
"BLOCKED": "got stuck and need help",
|
|
766
|
+
"NEEDS_INPUT": "waiting on a decision from you",
|
|
767
|
+
"APPROVAL_REQUEST": "waiting on your approval",
|
|
768
|
+
"IN_PROGRESS": "still working",
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
# window_support (schema v2): which snapshot sections are backed by gaia.db
|
|
772
|
+
# (episodes/episode_anomalies -- full history, exact window) vs. the
|
|
773
|
+
# audit-*.jsonl files (~30d retention, capped_by_retention may apply).
|
|
774
|
+
_WINDOW_SUPPORT = {
|
|
775
|
+
"episodes_backed": [
|
|
776
|
+
"agent_invocations",
|
|
777
|
+
"native_agent_activity",
|
|
778
|
+
"agent_outcomes",
|
|
779
|
+
"token_usage",
|
|
780
|
+
"anomaly_summary",
|
|
781
|
+
"runtime_skills",
|
|
782
|
+
"context_snapshots",
|
|
783
|
+
"context_updates",
|
|
784
|
+
],
|
|
785
|
+
"audit_log_backed": ["security_tiers", "cmd_types", "top_cmds"],
|
|
786
|
+
"cap_days": 30,
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
|
|
275
790
|
# ---------------------------------------------------------------------------
|
|
276
791
|
# Metric calculators
|
|
277
792
|
# ---------------------------------------------------------------------------
|
|
@@ -334,27 +849,37 @@ def _calculate_command_type_breakdown(audit_logs: list) -> dict:
|
|
|
334
849
|
|
|
335
850
|
|
|
336
851
|
def _calculate_top_commands(audit_logs: list) -> list:
|
|
337
|
-
|
|
338
|
-
label_map = {}
|
|
852
|
+
"""Top command labels by frequency, with share-of-total and avg duration.
|
|
339
853
|
|
|
854
|
+
Dashboard v3 (P1 #5): the tier/⚠ column is gone (it duplicated Security
|
|
855
|
+
Tier Usage); a percentage bar and an average ``duration_ms`` (from
|
|
856
|
+
hooks/modules/audit/logger.py's audit record, previously unused here)
|
|
857
|
+
replace it.
|
|
858
|
+
"""
|
|
859
|
+
label_map = {}
|
|
340
860
|
for e in audit_logs:
|
|
341
861
|
if not e.get("command"):
|
|
342
862
|
continue
|
|
343
863
|
label = _extract_command_label(e["command"])
|
|
344
|
-
tier = e.get("tier") or "unknown"
|
|
345
|
-
|
|
346
864
|
if label not in label_map:
|
|
347
|
-
label_map[label] = {"count": 0, "
|
|
865
|
+
label_map[label] = {"count": 0, "duration_total": 0.0, "duration_count": 0}
|
|
348
866
|
label_map[label]["count"] += 1
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
label_map[label]["
|
|
867
|
+
dur = e.get("duration_ms")
|
|
868
|
+
if isinstance(dur, (int, float)):
|
|
869
|
+
label_map[label]["duration_total"] += dur
|
|
870
|
+
label_map[label]["duration_count"] += 1
|
|
353
871
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
872
|
+
total = sum(v["count"] for v in label_map.values())
|
|
873
|
+
result = [
|
|
874
|
+
{
|
|
875
|
+
"label": l,
|
|
876
|
+
"count": v["count"],
|
|
877
|
+
"percentage": v["count"] / total * 100 if total else 0,
|
|
878
|
+
"avg_duration_ms": (v["duration_total"] / v["duration_count"]) if v["duration_count"] else None,
|
|
879
|
+
}
|
|
880
|
+
for l, v in label_map.items()
|
|
881
|
+
]
|
|
882
|
+
return sorted(result, key=lambda x: -x["count"])[:10]
|
|
358
883
|
|
|
359
884
|
|
|
360
885
|
def _calculate_error_rate(audit_logs: list) -> dict:
|
|
@@ -370,6 +895,25 @@ def _calculate_error_rate(audit_logs: list) -> dict:
|
|
|
370
895
|
}
|
|
371
896
|
|
|
372
897
|
|
|
898
|
+
def _split_native_agents(entries: list) -> tuple:
|
|
899
|
+
"""Separate harness-native Claude Code agents from Gaia domain specialists.
|
|
900
|
+
|
|
901
|
+
Native agents (Explore, Plan, claude-code-guide, general-purpose -- see
|
|
902
|
+
``NATIVE_AGENT_NAMES``) are utility subagents built into the harness, not
|
|
903
|
+
Gaia specialists. Explore alone can dwarf every Gaia specialist's
|
|
904
|
+
invocation count, which drowns out the invocation/outcome/token/
|
|
905
|
+
runtime-skill reads that exist to gauge Gaia specialist usage. Works on
|
|
906
|
+
any list of dicts carrying an ``agent`` key -- both ``workflow_metrics``
|
|
907
|
+
rows and ``run_snapshots`` rows qualify. Returns
|
|
908
|
+
``(gaia_entries, native_entries)``.
|
|
909
|
+
"""
|
|
910
|
+
gaia, native = [], []
|
|
911
|
+
for r in entries:
|
|
912
|
+
target = native if (r.get("agent") or "") in NATIVE_AGENT_NAMES else gaia
|
|
913
|
+
target.append(r)
|
|
914
|
+
return gaia, native
|
|
915
|
+
|
|
916
|
+
|
|
373
917
|
def _calculate_agent_invocations(workflow_metrics: list) -> dict:
|
|
374
918
|
today = datetime.now(timezone.utc).date().isoformat()
|
|
375
919
|
today_count = sum(1 for r in workflow_metrics if (r.get("timestamp") or "").startswith(today))
|
|
@@ -420,19 +964,57 @@ def _calculate_agent_outcomes(workflow_metrics: list):
|
|
|
420
964
|
|
|
421
965
|
|
|
422
966
|
def _calculate_token_usage(workflow_metrics: list):
|
|
423
|
-
|
|
967
|
+
"""Aggregate output-token usage per agent, preferring real counts.
|
|
968
|
+
|
|
969
|
+
Each entry may carry ``output_tokens_real`` (transcript-parsed
|
|
970
|
+
usage.output_tokens, set by workflow_recorder.py when a transcript was
|
|
971
|
+
available) alongside the always-present ``output_tokens_approx``
|
|
972
|
+
(chars/4 heuristic). The "effective" total per entry is the real count
|
|
973
|
+
when present, degrading to the approximation otherwise -- both the
|
|
974
|
+
per-entry and the aggregate output are labeled so a caller can tell which
|
|
975
|
+
source backed the number. When any entry carries input/cache token data,
|
|
976
|
+
the totals for those are surfaced too (None when no entry has them, so a
|
|
977
|
+
workspace with no transcript-backed episodes doesn't show a false zero).
|
|
978
|
+
"""
|
|
979
|
+
with_tokens = [
|
|
980
|
+
r
|
|
981
|
+
for r in workflow_metrics
|
|
982
|
+
if isinstance(r.get("output_tokens_real"), (int, float))
|
|
983
|
+
or isinstance(r.get("output_tokens_approx"), (int, float))
|
|
984
|
+
]
|
|
424
985
|
if not with_tokens:
|
|
425
986
|
return None
|
|
426
987
|
|
|
427
988
|
agent_map = {}
|
|
989
|
+
grand_total = 0
|
|
990
|
+
real_count = 0
|
|
991
|
+
total_input = 0
|
|
992
|
+
total_cache_creation = 0
|
|
993
|
+
total_cache_read = 0
|
|
994
|
+
has_input_data = False
|
|
995
|
+
|
|
428
996
|
for e in with_tokens:
|
|
429
997
|
name = e.get("agent") or "unknown"
|
|
998
|
+
real = e.get("output_tokens_real")
|
|
999
|
+
is_real = isinstance(real, (int, float))
|
|
1000
|
+
effective = real if is_real else (e.get("output_tokens_approx") or 0)
|
|
1001
|
+
if is_real:
|
|
1002
|
+
real_count += 1
|
|
1003
|
+
grand_total += effective
|
|
1004
|
+
|
|
430
1005
|
if name not in agent_map:
|
|
431
|
-
agent_map[name] = {"total": 0, "count": 0}
|
|
432
|
-
agent_map[name]["total"] +=
|
|
1006
|
+
agent_map[name] = {"total": 0, "count": 0, "real_count": 0}
|
|
1007
|
+
agent_map[name]["total"] += effective
|
|
433
1008
|
agent_map[name]["count"] += 1
|
|
1009
|
+
if is_real:
|
|
1010
|
+
agent_map[name]["real_count"] += 1
|
|
1011
|
+
|
|
1012
|
+
if isinstance(e.get("input_tokens"), (int, float)):
|
|
1013
|
+
has_input_data = True
|
|
1014
|
+
total_input += e["input_tokens"]
|
|
1015
|
+
total_cache_creation += e.get("cache_creation_tokens") or 0
|
|
1016
|
+
total_cache_read += e.get("cache_read_tokens") or 0
|
|
434
1017
|
|
|
435
|
-
grand_total = sum(e["output_tokens_approx"] for e in with_tokens)
|
|
436
1018
|
agents = sorted(
|
|
437
1019
|
[
|
|
438
1020
|
{
|
|
@@ -440,34 +1022,82 @@ def _calculate_token_usage(workflow_metrics: list):
|
|
|
440
1022
|
"total": v["total"],
|
|
441
1023
|
"avg": round(v["total"] / v["count"]) if v["count"] else 0,
|
|
442
1024
|
"count": v["count"],
|
|
1025
|
+
"source": "real" if v["real_count"] == v["count"] else ("mixed" if v["real_count"] else "approx"),
|
|
443
1026
|
}
|
|
444
1027
|
for n, v in agent_map.items()
|
|
445
1028
|
],
|
|
446
1029
|
key=lambda x: -x["total"],
|
|
447
1030
|
)
|
|
448
|
-
return {
|
|
1031
|
+
return {
|
|
1032
|
+
"agents": agents,
|
|
1033
|
+
"grand_total": grand_total,
|
|
1034
|
+
"entry_count": len(with_tokens),
|
|
1035
|
+
"real_count": real_count,
|
|
1036
|
+
"approx_count": len(with_tokens) - real_count,
|
|
1037
|
+
"input_tokens": total_input if has_input_data else None,
|
|
1038
|
+
"cache_creation_tokens": total_cache_creation if has_input_data else None,
|
|
1039
|
+
"cache_read_tokens": total_cache_read if has_input_data else None,
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
_SEVERITY_RANK = {"critical": 3, "error": 2, "warning": 1, "info": 0, "unknown": -1}
|
|
449
1044
|
|
|
450
1045
|
|
|
451
1046
|
def _calculate_anomaly_summary(anomaly_entries: list):
|
|
452
|
-
|
|
453
|
-
|
|
1047
|
+
"""Anomaly summary, sorted so severity beats volume.
|
|
1048
|
+
|
|
1049
|
+
A single high-volume, low-severity type (e.g. ``pipe_retroactive``, a
|
|
1050
|
+
warning fired on every pipe) can otherwise dominate a plain count-sorted
|
|
1051
|
+
list and bury a rare but critical entry (e.g.
|
|
1052
|
+
``response_contract_violation``). ``by_type`` sorts by severity rank
|
|
1053
|
+
first, count second, so critical/error entries always surface above
|
|
1054
|
+
warning/info noise regardless of how often the noisy type fires.
|
|
1055
|
+
``by_severity`` gives the aggregate breakdown for a one-line read.
|
|
1056
|
+
|
|
1057
|
+
Dashboard v3: no longer applies its own 30-day age cutoff. Time
|
|
1058
|
+
filtering is the caller's TimeWindow, enforced once in SQL by
|
|
1059
|
+
``_read_anomaly_entries`` -- filtering again here would silently
|
|
1060
|
+
re-impose a fixed 30-day window even when the user asked for
|
|
1061
|
+
``--range=all`` or ``--range=today``.
|
|
1062
|
+
"""
|
|
1063
|
+
entries = [e for e in anomaly_entries if e]
|
|
454
1064
|
if not entries:
|
|
455
1065
|
return None
|
|
456
1066
|
|
|
457
1067
|
type_counts = {}
|
|
1068
|
+
type_severity = {}
|
|
1069
|
+
severity_counts = {}
|
|
458
1070
|
agent_counts = {}
|
|
459
1071
|
for e in entries:
|
|
460
1072
|
agent = (e.get("metrics") or {}).get("agent", "unknown")
|
|
461
1073
|
for anomaly in e.get("anomalies") or []:
|
|
462
1074
|
t = anomaly.get("type", "unknown")
|
|
1075
|
+
sev = anomaly.get("severity") or "unknown"
|
|
463
1076
|
type_counts[t] = type_counts.get(t, 0) + 1
|
|
1077
|
+
severity_counts[sev] = severity_counts.get(sev, 0) + 1
|
|
464
1078
|
agent_counts[agent] = agent_counts.get(agent, 0) + 1
|
|
1079
|
+
if _SEVERITY_RANK.get(sev, -1) > _SEVERITY_RANK.get(type_severity.get(t, "unknown"), -1):
|
|
1080
|
+
type_severity[t] = sev
|
|
465
1081
|
|
|
466
1082
|
total = sum(type_counts.values())
|
|
1083
|
+
if total == 0:
|
|
1084
|
+
return None
|
|
467
1085
|
by_type = sorted(
|
|
468
|
-
[
|
|
469
|
-
|
|
470
|
-
|
|
1086
|
+
[
|
|
1087
|
+
{
|
|
1088
|
+
"type": t,
|
|
1089
|
+
"count": c,
|
|
1090
|
+
"percentage": c / total * 100 if total else 0,
|
|
1091
|
+
"severity": type_severity.get(t, "unknown"),
|
|
1092
|
+
}
|
|
1093
|
+
for t, c in type_counts.items()
|
|
1094
|
+
],
|
|
1095
|
+
key=lambda x: (-_SEVERITY_RANK.get(x["severity"], -1), -x["count"]),
|
|
1096
|
+
)
|
|
1097
|
+
by_severity = sorted(
|
|
1098
|
+
[{"severity": s, "count": c, "percentage": c / total * 100 if total else 0}
|
|
1099
|
+
for s, c in severity_counts.items()],
|
|
1100
|
+
key=lambda x: -_SEVERITY_RANK.get(x["severity"], -1),
|
|
471
1101
|
)
|
|
472
1102
|
by_agent = _sorted_counts(agent_counts)[:5]
|
|
473
1103
|
|
|
@@ -475,11 +1105,19 @@ def _calculate_anomaly_summary(anomaly_entries: list):
|
|
|
475
1105
|
"total": total,
|
|
476
1106
|
"session_count": len(entries),
|
|
477
1107
|
"by_type": by_type,
|
|
1108
|
+
"by_severity": by_severity,
|
|
478
1109
|
"by_agent": by_agent,
|
|
479
1110
|
}
|
|
480
1111
|
|
|
481
1112
|
|
|
482
1113
|
def _calculate_runtime_skill_summary(skill_snapshots: list, run_snapshots: list) -> dict:
|
|
1114
|
+
"""Latest model/tools/skills profile per Gaia specialist agent.
|
|
1115
|
+
|
|
1116
|
+
Callers must pass native-agent-filtered lists (see MetricsSnapshot.build,
|
|
1117
|
+
P0 fix): unlike invocations/outcomes/tokens, this calculator historically
|
|
1118
|
+
received the raw, unfiltered run_snapshots, so Explore's default profile
|
|
1119
|
+
could appear alongside Gaia specialists.
|
|
1120
|
+
"""
|
|
483
1121
|
explicit = [e for e in skill_snapshots if e and e.get("agent")]
|
|
484
1122
|
run_defaults = [
|
|
485
1123
|
{
|
|
@@ -511,7 +1149,8 @@ def _calculate_runtime_skill_summary(skill_snapshots: list, run_snapshots: list)
|
|
|
511
1149
|
"source": snap.get("source", "explicit"),
|
|
512
1150
|
}
|
|
513
1151
|
|
|
514
|
-
|
|
1152
|
+
# P3 #10: busiest profile first (was alphabetical by agent name).
|
|
1153
|
+
profiles = sorted(latest_by_agent.values(), key=lambda x: (-x["skills_count"], x["agent"]))
|
|
515
1154
|
all_skills = [s for p in profiles for s in p["skills"]]
|
|
516
1155
|
top_skills = _top_counts(all_skills, 6)
|
|
517
1156
|
|
|
@@ -573,158 +1212,451 @@ def _calculate_context_update_summary(run_snapshots: list):
|
|
|
573
1212
|
|
|
574
1213
|
|
|
575
1214
|
# ---------------------------------------------------------------------------
|
|
576
|
-
#
|
|
1215
|
+
# MetricsSnapshot -- unified data model (dashboard v2, extended in v3)
|
|
1216
|
+
#
|
|
1217
|
+
# Before this model, cmd_metrics() ran the 11 calculators TWICE -- once for
|
|
1218
|
+
# --json, once for the console display -- and the two branches carried
|
|
1219
|
+
# divergent key names for the same data (the JSON branch's "security_tiers"
|
|
1220
|
+
# vs. the display branch's "tiers"). MetricsSnapshot is computed exactly
|
|
1221
|
+
# once per invocation and fed to both json.dumps(snapshot.to_dict()) and
|
|
1222
|
+
# render_console(snapshot), so there is one source of truth for both data
|
|
1223
|
+
# and canonical key names. schema_version makes the JSON shape a contract a
|
|
1224
|
+
# future web/API consumer can version against.
|
|
1225
|
+
#
|
|
1226
|
+
# v3 adds the ``window`` field (the TimeWindow governing this read) and
|
|
1227
|
+
# filters native agents out of the runtime-skills input, matching the
|
|
1228
|
+
# segregation invocations/outcomes/tokens already had (P0 fix).
|
|
577
1229
|
# ---------------------------------------------------------------------------
|
|
578
1230
|
|
|
579
|
-
|
|
580
|
-
|
|
1231
|
+
@dataclass
|
|
1232
|
+
class MetricsSnapshot:
|
|
1233
|
+
schema_version: str
|
|
1234
|
+
generated_at: str
|
|
1235
|
+
workspace: Optional[str]
|
|
1236
|
+
window: dict
|
|
1237
|
+
audit_entry_count: int
|
|
1238
|
+
security_tiers: dict
|
|
1239
|
+
cmd_types: dict
|
|
1240
|
+
top_cmds: list
|
|
1241
|
+
agent_invocations: dict
|
|
1242
|
+
native_agent_activity: dict
|
|
1243
|
+
error_stats: dict
|
|
1244
|
+
agent_outcomes: Optional[dict]
|
|
1245
|
+
token_usage: Optional[dict]
|
|
1246
|
+
anomaly_summary: Optional[dict]
|
|
1247
|
+
runtime_skills: dict
|
|
1248
|
+
context_snapshots: Optional[dict]
|
|
1249
|
+
context_updates: Optional[dict]
|
|
1250
|
+
agent_filter: Optional[str] = None
|
|
1251
|
+
|
|
1252
|
+
def to_dict(self) -> dict:
|
|
1253
|
+
d = asdict(self)
|
|
1254
|
+
d["window_support"] = dict(_WINDOW_SUPPORT)
|
|
1255
|
+
return d
|
|
1256
|
+
|
|
1257
|
+
@classmethod
|
|
1258
|
+
def build(
|
|
1259
|
+
cls,
|
|
1260
|
+
*,
|
|
1261
|
+
workspace: Optional[str],
|
|
1262
|
+
audit_logs: list,
|
|
1263
|
+
workflow_metrics: list,
|
|
1264
|
+
run_snapshots: list,
|
|
1265
|
+
skill_snapshots: list,
|
|
1266
|
+
anomaly_entries: list,
|
|
1267
|
+
agent_filter: Optional[str] = None,
|
|
1268
|
+
window: Optional[TimeWindow] = None,
|
|
1269
|
+
) -> "MetricsSnapshot":
|
|
1270
|
+
win = window or _DEFAULT_WINDOW
|
|
1271
|
+
gaia_metrics, native_metrics = _split_native_agents(workflow_metrics)
|
|
1272
|
+
# P0 fix: runtime skills previously computed over run_snapshots
|
|
1273
|
+
# WITHOUT native-agent filtering, unlike invocations/outcomes/tokens
|
|
1274
|
+
# (which already used gaia_metrics). Segregate the same way here so
|
|
1275
|
+
# Explore/Plan/etc. don't show up as "Gaia specialist" skill profiles.
|
|
1276
|
+
gaia_run_snapshots, _ = _split_native_agents(run_snapshots)
|
|
1277
|
+
gaia_skill_snapshots, _ = _split_native_agents(skill_snapshots)
|
|
1278
|
+
return cls(
|
|
1279
|
+
schema_version=SCHEMA_VERSION,
|
|
1280
|
+
generated_at=datetime.now(timezone.utc).isoformat(),
|
|
1281
|
+
workspace=workspace,
|
|
1282
|
+
window=win.to_dict(),
|
|
1283
|
+
audit_entry_count=len(audit_logs),
|
|
1284
|
+
security_tiers=_calculate_tier_usage(audit_logs),
|
|
1285
|
+
cmd_types=_calculate_command_type_breakdown(audit_logs),
|
|
1286
|
+
top_cmds=_calculate_top_commands(audit_logs),
|
|
1287
|
+
agent_invocations=_calculate_agent_invocations(gaia_metrics),
|
|
1288
|
+
native_agent_activity=_calculate_agent_invocations(native_metrics),
|
|
1289
|
+
error_stats=_calculate_error_rate(audit_logs),
|
|
1290
|
+
agent_outcomes=_calculate_agent_outcomes(gaia_metrics),
|
|
1291
|
+
token_usage=_calculate_token_usage(gaia_metrics),
|
|
1292
|
+
anomaly_summary=_calculate_anomaly_summary(anomaly_entries),
|
|
1293
|
+
runtime_skills=_calculate_runtime_skill_summary(gaia_skill_snapshots, gaia_run_snapshots),
|
|
1294
|
+
context_snapshots=_calculate_context_snapshot_summary(run_snapshots),
|
|
1295
|
+
context_updates=_calculate_context_update_summary(run_snapshots),
|
|
1296
|
+
agent_filter=agent_filter,
|
|
1297
|
+
)
|
|
581
1298
|
|
|
582
|
-
|
|
583
|
-
|
|
1299
|
+
@classmethod
|
|
1300
|
+
def empty(cls, workspace: Optional[str] = None, window: Optional[TimeWindow] = None) -> "MetricsSnapshot":
|
|
1301
|
+
empty_inv = {"agents": [], "total": 0, "today_count": 0}
|
|
1302
|
+
win = window or _DEFAULT_WINDOW
|
|
1303
|
+
return cls(
|
|
1304
|
+
schema_version=SCHEMA_VERSION,
|
|
1305
|
+
generated_at=datetime.now(timezone.utc).isoformat(),
|
|
1306
|
+
workspace=workspace,
|
|
1307
|
+
window=win.to_dict(),
|
|
1308
|
+
audit_entry_count=0,
|
|
1309
|
+
security_tiers={"total": 0, "distribution": [], "today_count": 0, "today_t3": 0, "peak_hour": None, "peak_count": 0},
|
|
1310
|
+
cmd_types={"total": 0, "breakdown": []},
|
|
1311
|
+
top_cmds=[],
|
|
1312
|
+
agent_invocations=dict(empty_inv),
|
|
1313
|
+
native_agent_activity=dict(empty_inv),
|
|
1314
|
+
error_stats={"total": 0, "errors": 0, "error_rate": 0, "limited_by_api": False},
|
|
1315
|
+
agent_outcomes=None,
|
|
1316
|
+
token_usage=None,
|
|
1317
|
+
anomaly_summary=None,
|
|
1318
|
+
runtime_skills={"explicit_count": 0, "run_default_count": 0, "agent_count": 0, "latest_profiles": [], "top_skills": []},
|
|
1319
|
+
context_snapshots=None,
|
|
1320
|
+
context_updates=None,
|
|
1321
|
+
)
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
# ---------------------------------------------------------------------------
|
|
1325
|
+
# Render -- hand-rolled Unicode box-drawing (zero new dependencies)
|
|
1326
|
+
# ---------------------------------------------------------------------------
|
|
1327
|
+
|
|
1328
|
+
_BOX_W = 74 # interior width, excluding the two border characters
|
|
1329
|
+
|
|
1330
|
+
_TIER_LABELS = {"T0": "read-only", "T1": "validation", "T2": "dry-run", "T3": "mutating"}
|
|
1331
|
+
_SEVERITY_LABELS = {"critical": "CRIT", "error": "ERR ", "warning": "WARN", "info": "INFO", "unknown": "? "}
|
|
584
1332
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
1333
|
+
|
|
1334
|
+
def _box_top(title: str, right_label: str = "") -> str:
|
|
1335
|
+
left = f"─ {title} "
|
|
1336
|
+
right = f" {right_label} ─" if right_label else "─"
|
|
1337
|
+
fill = max(0, _BOX_W - len(left) - len(right))
|
|
1338
|
+
return "┌" + left + "─" * fill + right + "┐"
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def _box_row(text: str = "") -> str:
|
|
1342
|
+
content = f" {text}" if text else ""
|
|
1343
|
+
if len(content) > _BOX_W:
|
|
1344
|
+
content = content[: _BOX_W - 1] + "…"
|
|
1345
|
+
return "│" + content.ljust(_BOX_W) + "│"
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
def _box_divider(label: str = "") -> str:
|
|
1349
|
+
left = f"── {label} " if label else ""
|
|
1350
|
+
fill = max(0, _BOX_W - len(left))
|
|
1351
|
+
return "├" + left + "─" * fill + "┤"
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
def _box_bottom() -> str:
|
|
1355
|
+
return "└" + "─" * _BOX_W + "┘"
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def _print_box(title: str, rows: list, legend: list = None, right_label: str = "") -> None:
|
|
1359
|
+
print(_box_top(title, right_label))
|
|
1360
|
+
for r in rows:
|
|
1361
|
+
print(_box_row(r))
|
|
1362
|
+
if legend:
|
|
1363
|
+
print(_box_divider("legend"))
|
|
1364
|
+
for l in legend:
|
|
1365
|
+
print(_box_row(l))
|
|
1366
|
+
print(_box_bottom())
|
|
1367
|
+
print()
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
def render_console(snapshot: MetricsSnapshot) -> None:
|
|
1371
|
+
"""Render the dashboard from a single, already-computed MetricsSnapshot.
|
|
1372
|
+
|
|
1373
|
+
Every section is a box: a title bar (with an at-a-glance right-aligned
|
|
1374
|
+
total), the data rows, and an embedded legend with a "WHAT:" line (what
|
|
1375
|
+
the metric IS, in plain language) and one or more "NOTE:" lines
|
|
1376
|
+
(caveats, classification quirks, window scope).
|
|
1377
|
+
|
|
1378
|
+
Dashboard v3 order: Security Tier Usage and Runtime Skill Snapshots lead
|
|
1379
|
+
(the two most-referenced boxes); Activity Today is a compact strip under
|
|
1380
|
+
the header rather than its own box, and its T3-today count lives solely
|
|
1381
|
+
in Security Tier Usage (not repeated).
|
|
1382
|
+
"""
|
|
1383
|
+
window = snapshot.window
|
|
1384
|
+
print("\nGaia System Metrics (dashboard v3)")
|
|
1385
|
+
ws_label = snapshot.workspace or "unfiltered (all workspaces)"
|
|
1386
|
+
print(f"Generated {snapshot.generated_at} | workspace: {ws_label} | range: {window['label']}")
|
|
1387
|
+
|
|
1388
|
+
# Compact "Today (UTC)" strip -- not a box. T3-today lives in Security
|
|
1389
|
+
# Tier Usage below; this strip does not repeat it.
|
|
1390
|
+
tiers = snapshot.security_tiers
|
|
1391
|
+
err = snapshot.error_stats
|
|
1392
|
+
today_line = f"Today (UTC): {tiers['today_count']} calls"
|
|
1393
|
+
if tiers["peak_hour"] is not None:
|
|
1394
|
+
today_line += f" | peak {tiers['peak_hour']}:00-{tiers['peak_hour']}:59 UTC ({tiers['peak_count']})"
|
|
1395
|
+
else:
|
|
1396
|
+
today_line += " | no peak-hour data"
|
|
1397
|
+
if err["limited_by_api"]:
|
|
1398
|
+
today_line += " | error rate n/a (hook API always reports exit_code=0)"
|
|
1399
|
+
elif err["total"] == 0:
|
|
1400
|
+
today_line += " | error rate: no exit_code data"
|
|
1401
|
+
else:
|
|
1402
|
+
today_line += f" | error rate {err['errors']}/{err['total']} ({err['error_rate']:.1f}%)"
|
|
1403
|
+
print(today_line)
|
|
1404
|
+
print()
|
|
1405
|
+
|
|
1406
|
+
window_note = f"NOTE: window: {window['label']}"
|
|
1407
|
+
if window["capped_by_retention"]:
|
|
1408
|
+
window_note += " (capped to the last 30d -- audit-log retention)"
|
|
1409
|
+
window_note += ", not filtered by workspace."
|
|
597
1410
|
|
|
598
1411
|
# Security Tier Usage
|
|
599
|
-
|
|
600
|
-
tier_label = {"T0": "read-only", "T1": "validation", "T2": "simulation", "T3": "realization"}
|
|
1412
|
+
rows = []
|
|
601
1413
|
if tiers["total"] == 0:
|
|
602
|
-
|
|
1414
|
+
rows.append("no tier data")
|
|
603
1415
|
else:
|
|
604
1416
|
for item in tiers["distribution"]:
|
|
605
1417
|
tier = item["tier"]
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
1418
|
+
bar = _make_bar(item["percentage"], 24)
|
|
1419
|
+
label = _TIER_LABELS.get(tier, tier)
|
|
1420
|
+
warn = " ⚠ " if tier == "T3" else " "
|
|
1421
|
+
rows.append(f"{tier:<3}{warn}{label:<11}{item['count']:>4} {bar} {item['percentage']:>5.1f}%")
|
|
1422
|
+
rows.append(f"T3 today: {tiers['today_t3']}" + (" ⚠" if tiers["today_t3"] > 0 else ""))
|
|
1423
|
+
legend = [
|
|
1424
|
+
"WHAT: how many commands ran at each risk tier in the selected window.",
|
|
1425
|
+
"NOTE: T0 read-only / T1 local validation / T2 dry-run need no approval;",
|
|
1426
|
+
" T3 mutates state and REQUIRES approval before it runs.",
|
|
1427
|
+
"NOTE: How commands get classified: curl is T0 unless -X POST/--data;",
|
|
1428
|
+
" a bare `python3 script.py` defaults to T3 when its body can't",
|
|
1429
|
+
" be resolved (conservative).",
|
|
1430
|
+
window_note,
|
|
1431
|
+
]
|
|
1432
|
+
_print_box("SECURITY TIER USAGE", rows, legend, right_label=f"{tiers['total']} ops")
|
|
1433
|
+
|
|
1434
|
+
# Runtime Skill Snapshots (moved up -- P2 #7)
|
|
1435
|
+
rs = snapshot.runtime_skills
|
|
1436
|
+
if rs["agent_count"] > 0:
|
|
1437
|
+
rows = []
|
|
1438
|
+
for profile in rs["latest_profiles"][:6]:
|
|
1439
|
+
model = profile.get("model") or "default"
|
|
1440
|
+
rows.append(
|
|
1441
|
+
f"{profile['agent']:<22}model {model:<8}skills {profile['skills_count']:>2} "
|
|
1442
|
+
f"tools {len(profile['tools']):>2} {_format_skills(profile['skills'], 3)}"
|
|
1443
|
+
)
|
|
1444
|
+
if len(rs["latest_profiles"]) > 6:
|
|
1445
|
+
rows.append(f"... {len(rs['latest_profiles']) - 6} more agents with captured snapshots")
|
|
1446
|
+
rows.append(f"Common skills: {_format_count_summary(rs['top_skills'])}")
|
|
1447
|
+
legend = [
|
|
1448
|
+
"WHAT: which model/tools/skills the harness actually loaded for each",
|
|
1449
|
+
" agent's latest dispatch -- not its .md declaration.",
|
|
1450
|
+
"NOTE: Gaia specialists only -- harness-native agents (Explore, Plan,",
|
|
1451
|
+
" claude-code-guide, general-purpose) are excluded.",
|
|
1452
|
+
"NOTE: sorted by skill count (busiest profile first), not alphabetically.",
|
|
1453
|
+
]
|
|
1454
|
+
_print_box("RUNTIME SKILL SNAPSHOTS", rows, legend, right_label=f"{rs['agent_count']} agents")
|
|
612
1455
|
|
|
613
1456
|
# Command Type Breakdown
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
1457
|
+
ct = snapshot.cmd_types
|
|
1458
|
+
rows = []
|
|
1459
|
+
if not ct["breakdown"]:
|
|
1460
|
+
rows.append("no command data")
|
|
617
1461
|
else:
|
|
618
|
-
for item in
|
|
619
|
-
bar = _make_bar(item["percentage"],
|
|
620
|
-
|
|
1462
|
+
for item in ct["breakdown"]:
|
|
1463
|
+
bar = _make_bar(item["percentage"], 20)
|
|
1464
|
+
rows.append(f"{item['type']:<12}{item['count']:>4} {bar} {item['percentage']:>5.1f}%")
|
|
1465
|
+
legend = [
|
|
1466
|
+
"WHAT: groups the same commands by domain (terraform, kubernetes, git,",
|
|
1467
|
+
" gcp, docker, dev, general) instead of by risk tier.",
|
|
1468
|
+
"NOTE: classified from Bash tool_name entries in audit-*.jsonl.",
|
|
1469
|
+
window_note,
|
|
1470
|
+
]
|
|
1471
|
+
_print_box("COMMAND TYPE BREAKDOWN", rows, legend, right_label=f"{snapshot.audit_entry_count} entries")
|
|
621
1472
|
|
|
622
1473
|
# Top Commands
|
|
623
|
-
|
|
624
|
-
if not top_cmds:
|
|
625
|
-
|
|
1474
|
+
rows = []
|
|
1475
|
+
if not snapshot.top_cmds:
|
|
1476
|
+
rows.append("no command data")
|
|
626
1477
|
else:
|
|
627
|
-
for item in top_cmds:
|
|
628
|
-
|
|
629
|
-
|
|
1478
|
+
for item in snapshot.top_cmds:
|
|
1479
|
+
bar = _make_bar(item["percentage"], 16)
|
|
1480
|
+
dur = _format_duration_ms(item.get("avg_duration_ms"))
|
|
1481
|
+
rows.append(
|
|
1482
|
+
f"{item['label']:<26}{item['count']:>4} {bar} {item['percentage']:>5.1f}% avg {dur:>6}"
|
|
1483
|
+
)
|
|
1484
|
+
legend = [
|
|
1485
|
+
"WHAT: the most frequent command labels, with each one's share of all",
|
|
1486
|
+
" logged commands and its average wall-clock duration.",
|
|
1487
|
+
"NOTE: duration is averaged from duration_ms recorded per audit entry",
|
|
1488
|
+
" (n/a when no timed entries exist for that label).",
|
|
1489
|
+
window_note,
|
|
1490
|
+
]
|
|
1491
|
+
_print_box("TOP COMMANDS", rows, legend)
|
|
630
1492
|
|
|
631
|
-
# Agent Invocations
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
print(f"\nAgent Invocations {agent_header}")
|
|
637
|
-
if not agent_inv["agents"]:
|
|
638
|
-
print(" no invocation data")
|
|
1493
|
+
# Agent Invocations (Gaia specialists only)
|
|
1494
|
+
inv = snapshot.agent_invocations
|
|
1495
|
+
rows = []
|
|
1496
|
+
if not inv["agents"]:
|
|
1497
|
+
rows.append("no invocation data")
|
|
639
1498
|
else:
|
|
640
|
-
for item in
|
|
641
|
-
bar = _make_bar(item["percentage"],
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
1499
|
+
for item in inv["agents"]:
|
|
1500
|
+
bar = _make_bar(item["percentage"], 14)
|
|
1501
|
+
rows.append(
|
|
1502
|
+
f"{item['name']:<22}{item['count']:>3} {bar} "
|
|
1503
|
+
f"avg {_format_chars(item['avg_output']):>6} chars {item['success_rate']:>3.0f}% ok"
|
|
1504
|
+
)
|
|
1505
|
+
rows.append("")
|
|
1506
|
+
rows.append("tip: gaia metrics --agent <name> for detail view")
|
|
1507
|
+
legend = [
|
|
1508
|
+
"WHAT: how many times each Gaia specialist was dispatched, with average",
|
|
1509
|
+
" output size and exit-code success rate.",
|
|
1510
|
+
"NOTE: Gaia domain specialists only -- harness-native agents (Explore,",
|
|
1511
|
+
" Plan, claude-code-guide, general-purpose) are in Native Agent",
|
|
1512
|
+
" Activity below.",
|
|
1513
|
+
f"NOTE: header splits today (UTC) vs the selected window ({window['label']});",
|
|
1514
|
+
" the rows above cover the full window.",
|
|
1515
|
+
]
|
|
1516
|
+
_print_box(
|
|
1517
|
+
"AGENT INVOCATIONS",
|
|
1518
|
+
rows,
|
|
1519
|
+
legend,
|
|
1520
|
+
right_label=f"{inv['today_count']} today · {inv['total']} in {window['label']}",
|
|
1521
|
+
)
|
|
1522
|
+
|
|
1523
|
+
# Native Agent Activity (segregated -- P1 fix: Explore et al. are harness
|
|
1524
|
+
# noise, not Gaia specialist signal)
|
|
1525
|
+
native = snapshot.native_agent_activity
|
|
1526
|
+
if native["total"] > 0:
|
|
1527
|
+
rows = []
|
|
1528
|
+
for item in native["agents"]:
|
|
1529
|
+
bar = _make_bar(item["percentage"], 14)
|
|
1530
|
+
rows.append(f"{item['name']:<22}{item['count']:>3} {bar} {item['percentage']:>5.1f}%")
|
|
1531
|
+
legend = [
|
|
1532
|
+
"WHAT: dispatch counts for harness-native utility agents -- not Gaia",
|
|
1533
|
+
" domain specialists.",
|
|
1534
|
+
"NOTE: excluded from Agent Invocations/Outcomes/Token Usage above so",
|
|
1535
|
+
" those reads measure Gaia specialists, not harness plumbing.",
|
|
1536
|
+
]
|
|
1537
|
+
_print_box(
|
|
1538
|
+
"NATIVE AGENT ACTIVITY (not Gaia specialists)",
|
|
1539
|
+
rows,
|
|
1540
|
+
legend,
|
|
1541
|
+
right_label=f"{native['today_count']} today · {native['total']} in {window['label']}",
|
|
1542
|
+
)
|
|
647
1543
|
|
|
648
1544
|
# Agent Outcomes
|
|
649
|
-
if agent_outcomes:
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
1545
|
+
if snapshot.agent_outcomes:
|
|
1546
|
+
ao = snapshot.agent_outcomes
|
|
1547
|
+
rows = []
|
|
1548
|
+
for item in ao["distribution"]:
|
|
1549
|
+
bar = _make_bar(item["percentage"], 12)
|
|
1550
|
+
gloss = _PLAN_STATUS_GLOSS.get(item["status"], "")
|
|
1551
|
+
rows.append(f"{item['status']:<16}{item['count']:>3} {bar} {item['percentage']:>5.1f}% {gloss}")
|
|
1552
|
+
legend = [
|
|
1553
|
+
"WHAT: the final plan_status Gaia specialists reported, in plain words.",
|
|
1554
|
+
"NOTE: Gaia specialists only -- same segregation as Agent Invocations.",
|
|
1555
|
+
]
|
|
1556
|
+
_print_box(
|
|
1557
|
+
"AGENT OUTCOMES",
|
|
1558
|
+
rows,
|
|
1559
|
+
legend,
|
|
1560
|
+
right_label=f"{ao['total']} invocations with status",
|
|
1561
|
+
)
|
|
654
1562
|
|
|
655
1563
|
# Token Usage
|
|
656
|
-
if token_usage:
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
f"
|
|
662
|
-
f"
|
|
1564
|
+
if snapshot.token_usage:
|
|
1565
|
+
tu = snapshot.token_usage
|
|
1566
|
+
rows = [f"{'AGENT':<22}{'INV':>4} {'TOTAL':>8} {'AVG/INV':>8} SOURCE"]
|
|
1567
|
+
for item in tu["agents"]:
|
|
1568
|
+
rows.append(
|
|
1569
|
+
f"{item['name']:<22}{item['count']:>4} {_format_tokens(item['total']):>8} "
|
|
1570
|
+
f"{_format_tokens(item['avg']):>8} {item['source']}"
|
|
663
1571
|
)
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
f"
|
|
676
|
-
f"
|
|
677
|
-
f"{_format_skills(profile['skills'], 3)}"
|
|
1572
|
+
legend = [
|
|
1573
|
+
"WHAT: output tokens per agent -- INV = invocations, TOTAL = summed",
|
|
1574
|
+
" output tokens, AVG/INV = TOTAL / INV.",
|
|
1575
|
+
f"NOTE: real = transcript-parsed usage.output_tokens ({tu['real_count']} entries);",
|
|
1576
|
+
f" approx = chars/4 heuristic ({tu['approx_count']} entries).",
|
|
1577
|
+
"NOTE: 'real' is NOT guaranteed -- it silently falls back to 'approx'",
|
|
1578
|
+
" per entry when transcript parsing fails (transcript_analyzer.analyze).",
|
|
1579
|
+
]
|
|
1580
|
+
if tu["input_tokens"] is not None:
|
|
1581
|
+
legend.append(
|
|
1582
|
+
f"NOTE: input {_format_tokens(tu['input_tokens'])} "
|
|
1583
|
+
f"cache-write {_format_tokens(tu['cache_creation_tokens'])} "
|
|
1584
|
+
f"cache-read {_format_tokens(tu['cache_read_tokens'])}"
|
|
678
1585
|
)
|
|
679
|
-
|
|
680
|
-
print(f" ... {len(rs['latest_profiles']) - 6} more agents with captured snapshots")
|
|
681
|
-
print(f" Common skills: {_format_count_summary(rs['top_skills'])}")
|
|
1586
|
+
_print_box("TOKEN USAGE", rows, legend, right_label=f"~{_format_tokens(tu['grand_total'])} total")
|
|
682
1587
|
|
|
683
1588
|
# Context Snapshot Summary
|
|
684
|
-
if
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
1589
|
+
if snapshot.context_snapshots:
|
|
1590
|
+
cs = snapshot.context_snapshots
|
|
1591
|
+
rows = [
|
|
1592
|
+
f"Primary surfaces: {_format_count_summary(cs['primary_surfaces'])}",
|
|
1593
|
+
f"Multi-surface: {cs['multi_surface_count']}/{cs['total']} invocations",
|
|
1594
|
+
f"Contract sections: {_format_count_summary(cs['contract_sections'])}",
|
|
1595
|
+
]
|
|
1596
|
+
if cs["writable_sections"]:
|
|
1597
|
+
rows.append(f"Writable scope: {_format_count_summary(cs['writable_sections'])}")
|
|
1598
|
+
legend = [
|
|
1599
|
+
"WHAT: which surface/contract sections the orchestrator injected into",
|
|
1600
|
+
" agents' project context per invocation.",
|
|
1601
|
+
"NOTE: project_identity = repo/workspace facts; application_services =",
|
|
1602
|
+
" per-app config; stack = scanner-detected languages/frameworks.",
|
|
1603
|
+
]
|
|
1604
|
+
_print_box("CONTEXT SNAPSHOT SUMMARY", rows, legend, right_label=f"{cs['total']} invocations")
|
|
691
1605
|
|
|
692
1606
|
# Context Updates
|
|
693
|
-
if
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
1607
|
+
if snapshot.context_updates:
|
|
1608
|
+
cu = snapshot.context_updates
|
|
1609
|
+
rows = [
|
|
1610
|
+
f"Updated sections: {_format_count_summary(cu['updated_sections'])}",
|
|
1611
|
+
f"Rejected writes: {cu['rejected_runs']} invocations",
|
|
1612
|
+
]
|
|
1613
|
+
if cu["rejected_sections"]:
|
|
1614
|
+
rows.append(f"Rejected sections: {_format_count_summary(cu['rejected_sections'])}")
|
|
1615
|
+
legend = [
|
|
1616
|
+
"WHAT: how often agents wrote updates back into project context, and",
|
|
1617
|
+
" which sections they touched or had rejected.",
|
|
1618
|
+
f"NOTE: reflects invocations within the selected window ({window['label']})",
|
|
1619
|
+
" -- not an all-time total.",
|
|
1620
|
+
]
|
|
1621
|
+
_print_box(
|
|
1622
|
+
"CONTEXT UPDATES",
|
|
1623
|
+
rows,
|
|
1624
|
+
legend,
|
|
1625
|
+
right_label=f"{cu['updated_runs']}/{cu['total_runs']} updated",
|
|
1626
|
+
)
|
|
699
1627
|
|
|
700
1628
|
# Anomaly Summary
|
|
701
|
-
if anomaly_summary and anomaly_summary["total"] > 0:
|
|
702
|
-
a = anomaly_summary
|
|
703
|
-
|
|
1629
|
+
if snapshot.anomaly_summary and snapshot.anomaly_summary["total"] > 0:
|
|
1630
|
+
a = snapshot.anomaly_summary
|
|
1631
|
+
rows = []
|
|
704
1632
|
for item in a["by_type"]:
|
|
705
|
-
|
|
706
|
-
|
|
1633
|
+
sev = _SEVERITY_LABELS.get(item.get("severity", "unknown"), "? ")
|
|
1634
|
+
bar = _make_bar(item["percentage"], 14)
|
|
1635
|
+
rows.append(f"[{sev}] {item['type']:<28}{item['count']:>3} {bar} {item['percentage']:>5.1f}%")
|
|
1636
|
+
gloss = _ANOMALY_TYPE_GLOSSARY.get(item["type"], "no description available")
|
|
1637
|
+
rows.append(f" -> {gloss}")
|
|
707
1638
|
if a["by_agent"]:
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
else:
|
|
724
|
-
print(f" Error rate: {error_stats['errors']}/{error_stats['total']} ({error_stats['error_rate']:.1f}%)")
|
|
1639
|
+
rows.append("")
|
|
1640
|
+
rows.append(f"Agents: {_format_count_summary(a['by_agent'])}")
|
|
1641
|
+
legend = [
|
|
1642
|
+
"WHAT: workflow-auditor / contract-validator findings, one row per",
|
|
1643
|
+
" type with a plain-language meaning underneath.",
|
|
1644
|
+
"NOTE: sorted by severity first, count second -- a high-volume WARN",
|
|
1645
|
+
" (e.g. pipe_retroactive) can never bury a rarer CRIT/ERR entry.",
|
|
1646
|
+
f"NOTE: window: {window['label']}.",
|
|
1647
|
+
]
|
|
1648
|
+
_print_box(
|
|
1649
|
+
f"ANOMALY SUMMARY ({window['label']})",
|
|
1650
|
+
rows,
|
|
1651
|
+
legend,
|
|
1652
|
+
right_label=f"{a['total']} across {a['session_count']} invocations",
|
|
1653
|
+
)
|
|
725
1654
|
|
|
726
|
-
print(
|
|
727
|
-
|
|
1655
|
+
print(
|
|
1656
|
+
f"schema_version={snapshot.schema_version} | "
|
|
1657
|
+
"source: ~/.gaia/gaia.db (episodes, episode_anomalies) | "
|
|
1658
|
+
".claude/logs/audit-*.jsonl\n"
|
|
1659
|
+
)
|
|
728
1660
|
|
|
729
1661
|
|
|
730
1662
|
def _display_agent_detail(root: Path, agent_name: str, data: dict):
|
|
@@ -804,7 +1736,7 @@ def _display_agent_detail(root: Path, agent_name: str, data: dict):
|
|
|
804
1736
|
|
|
805
1737
|
print("\nInvocation History (last 7 days)")
|
|
806
1738
|
if not agent_sessions:
|
|
807
|
-
print(" no invocations found in
|
|
1739
|
+
print(" no invocations found in gaia.db episodes")
|
|
808
1740
|
else:
|
|
809
1741
|
print(
|
|
810
1742
|
f" Total: {len(agent_sessions)} invocations | "
|
|
@@ -850,12 +1782,12 @@ def _display_agent_detail(root: Path, agent_name: str, data: dict):
|
|
|
850
1782
|
print(" no context update or anomaly data")
|
|
851
1783
|
else:
|
|
852
1784
|
if agent_ctx_updates:
|
|
853
|
-
print(f" Context updated: {agent_ctx_updates['updated_runs']}/{agent_ctx_updates['total_runs']}
|
|
1785
|
+
print(f" Context updated: {agent_ctx_updates['updated_runs']}/{agent_ctx_updates['total_runs']} invocations")
|
|
854
1786
|
print(f" Updated sections: {_format_count_summary(agent_ctx_updates['updated_sections'])}")
|
|
855
1787
|
if agent_ctx_updates["rejected_sections"]:
|
|
856
1788
|
print(f" Rejected sections: {_format_count_summary(agent_ctx_updates['rejected_sections'])}")
|
|
857
1789
|
if agent_anomaly_total:
|
|
858
|
-
print(f" Anomalies: {agent_anomaly_total} across {len(agent_anomalies_entries)}
|
|
1790
|
+
print(f" Anomalies: {agent_anomaly_total} across {len(agent_anomalies_entries)} invocations")
|
|
859
1791
|
print(f" Types: {_format_count_summary(agent_anomaly_by_type)}")
|
|
860
1792
|
|
|
861
1793
|
# Top Commands (correlated from audit log -- approximate)
|
|
@@ -935,10 +1867,8 @@ def register(subparsers):
|
|
|
935
1867
|
"Display Gaia system metrics dashboard.\n"
|
|
936
1868
|
"\n"
|
|
937
1869
|
"Data sources:\n"
|
|
938
|
-
" ~/.gaia/gaia.db (
|
|
1870
|
+
" ~/.gaia/gaia.db (episodes + episode_anomalies tables)\n"
|
|
939
1871
|
" .claude/logs/audit-*.jsonl (security tier events)\n"
|
|
940
|
-
" .claude/project-context/episodic-memory/index.json (legacy fallback)\n"
|
|
941
|
-
" .claude/project-context/workflow-episodic-memory/ (legacy fallback)\n"
|
|
942
1872
|
),
|
|
943
1873
|
)
|
|
944
1874
|
p.add_argument(
|
|
@@ -947,6 +1877,26 @@ def register(subparsers):
|
|
|
947
1877
|
default=None,
|
|
948
1878
|
help="Show detail view for a specific agent",
|
|
949
1879
|
)
|
|
1880
|
+
p.add_argument(
|
|
1881
|
+
"--workspace", default=None,
|
|
1882
|
+
help="Workspace identity override. Default: gaia.project.current().",
|
|
1883
|
+
)
|
|
1884
|
+
p.add_argument(
|
|
1885
|
+
"--range",
|
|
1886
|
+
choices=_RANGE_CHOICES,
|
|
1887
|
+
default=None,
|
|
1888
|
+
help="Time range for episode/anomaly data: today|3d|7d|30d|all "
|
|
1889
|
+
"(default: 30d). Mutually exclusive with --since/--until.",
|
|
1890
|
+
)
|
|
1891
|
+
p.add_argument(
|
|
1892
|
+
"--since", default=None, metavar="DUR_OR_DATE",
|
|
1893
|
+
help="Lower bound -- duration ('24h', '7d') or ISO date. "
|
|
1894
|
+
"Mutually exclusive with --range.",
|
|
1895
|
+
)
|
|
1896
|
+
p.add_argument(
|
|
1897
|
+
"--until", default=None, metavar="DUR_OR_DATE",
|
|
1898
|
+
help="Upper bound, same format as --since. Mutually exclusive with --range.",
|
|
1899
|
+
)
|
|
950
1900
|
p.add_argument(
|
|
951
1901
|
"--json",
|
|
952
1902
|
action="store_true",
|
|
@@ -962,6 +1912,10 @@ def cmd_metrics(args) -> int:
|
|
|
962
1912
|
claude_dir = root / ".claude"
|
|
963
1913
|
agent_name = getattr(args, "agent", None)
|
|
964
1914
|
as_json = getattr(args, "json", False)
|
|
1915
|
+
workspace_override = getattr(args, "workspace", None)
|
|
1916
|
+
range_arg = getattr(args, "range", None)
|
|
1917
|
+
since_arg = getattr(args, "since", None)
|
|
1918
|
+
until_arg = getattr(args, "until", None)
|
|
965
1919
|
|
|
966
1920
|
if not claude_dir.exists():
|
|
967
1921
|
if as_json:
|
|
@@ -971,101 +1925,88 @@ def cmd_metrics(args) -> int:
|
|
|
971
1925
|
print("Run: gaia scan\n")
|
|
972
1926
|
return 1
|
|
973
1927
|
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
1928
|
+
if range_arg and (since_arg or until_arg):
|
|
1929
|
+
msg = "--range is mutually exclusive with --since/--until"
|
|
1930
|
+
if as_json:
|
|
1931
|
+
print(json.dumps({"error": msg}))
|
|
1932
|
+
else:
|
|
1933
|
+
print(f"\nError: {msg}\n", file=sys.stderr)
|
|
1934
|
+
return 2
|
|
1935
|
+
|
|
1936
|
+
try:
|
|
1937
|
+
window = _resolve_time_window(range_arg, since_arg, until_arg)
|
|
1938
|
+
except ValueError as exc:
|
|
1939
|
+
msg = f"invalid --since/--until value: {exc}"
|
|
1940
|
+
if as_json:
|
|
1941
|
+
print(json.dumps({"error": msg}))
|
|
1942
|
+
else:
|
|
1943
|
+
print(f"\nError: {msg}\n", file=sys.stderr)
|
|
1944
|
+
return 2
|
|
1945
|
+
|
|
1946
|
+
# Audit-log sections (tier usage, command breakdown, top commands) read
|
|
1947
|
+
# rotated .claude/logs/audit-*.jsonl files with ~30d retention. Cap the
|
|
1948
|
+
# audit-facing lower bound at that floor and flag it on the shared
|
|
1949
|
+
# window so those boxes can declare the cap in their own NOTE line.
|
|
1950
|
+
# Episodes/episode_anomalies (SQL-backed, full history) use the
|
|
1951
|
+
# uncapped, as-requested bound below.
|
|
1952
|
+
retention_floor_iso = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
1953
|
+
audit_since = window.since_iso
|
|
1954
|
+
if window.since_iso is None or window.since_iso < retention_floor_iso:
|
|
1955
|
+
audit_since = retention_floor_iso
|
|
1956
|
+
window.capped_by_retention = True
|
|
1957
|
+
audit_until = window.until_iso
|
|
1958
|
+
|
|
1959
|
+
audit_logs = _read_audit_logs(root, since_iso=audit_since, until_iso=audit_until)
|
|
1960
|
+
workflow_metrics = _read_workflow_metrics(
|
|
1961
|
+
root, workspace_override, since_iso=window.since_iso, until_iso=window.until_iso
|
|
1962
|
+
)
|
|
1963
|
+
run_snapshots = _read_run_snapshots(
|
|
1964
|
+
root, workspace_override, since_iso=window.since_iso, until_iso=window.until_iso
|
|
1965
|
+
)
|
|
977
1966
|
skill_snapshots = _read_agent_skill_snapshots(root)
|
|
978
|
-
anomaly_entries = _read_anomaly_entries(
|
|
1967
|
+
anomaly_entries = _read_anomaly_entries(
|
|
1968
|
+
root, workspace_override, since_iso=window.since_iso, until_iso=window.until_iso
|
|
1969
|
+
)
|
|
979
1970
|
|
|
980
1971
|
if not audit_logs and not workflow_metrics and not run_snapshots and not skill_snapshots and not anomaly_entries:
|
|
1972
|
+
snapshot = MetricsSnapshot.empty(workspace_override, window=window)
|
|
981
1973
|
if as_json:
|
|
982
|
-
|
|
983
|
-
"security_tiers": {"total": 0, "distribution": [], "today_count": 0, "today_t3": 0, "peak_hour": None, "peak_count": 0},
|
|
984
|
-
"cmd_types": {"total": 0, "breakdown": []},
|
|
985
|
-
"top_cmds": [],
|
|
986
|
-
"agent_invocations": {"agents": [], "total": 0, "today_count": 0},
|
|
987
|
-
"error_stats": {"total": 0, "errors": 0, "error_rate": 0, "limited_by_api": False},
|
|
988
|
-
"agent_outcomes": None,
|
|
989
|
-
"token_usage": None,
|
|
990
|
-
"anomaly_summary": None,
|
|
991
|
-
"runtime_skills": {"explicit_count": 0, "run_default_count": 0, "agent_count": 0, "latest_profiles": [], "top_skills": []},
|
|
992
|
-
"context_snapshots": None,
|
|
993
|
-
"context_updates": None,
|
|
994
|
-
}
|
|
995
|
-
print(json.dumps(empty_output))
|
|
1974
|
+
print(json.dumps(snapshot.to_dict(), indent=2))
|
|
996
1975
|
else:
|
|
997
1976
|
print("\nNo metrics data available yet")
|
|
998
1977
|
print("Metrics will be generated as you use the system\n")
|
|
999
1978
|
return 0
|
|
1000
1979
|
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
runtime_skills = _calculate_runtime_skill_summary(skill_snapshots, run_snapshots)
|
|
1012
|
-
ctx_snapshots = _calculate_context_snapshot_summary(run_snapshots)
|
|
1013
|
-
ctx_updates = _calculate_context_update_summary(run_snapshots)
|
|
1014
|
-
|
|
1015
|
-
output = {
|
|
1016
|
-
"security_tiers": tiers,
|
|
1017
|
-
"cmd_types": cmd_types,
|
|
1018
|
-
"top_cmds": top_cmds,
|
|
1019
|
-
"agent_invocations": agent_inv,
|
|
1020
|
-
"error_stats": error_stats,
|
|
1021
|
-
"agent_outcomes": agent_outcomes,
|
|
1022
|
-
"token_usage": token_usage,
|
|
1023
|
-
"anomaly_summary": anomaly_summary,
|
|
1024
|
-
"runtime_skills": runtime_skills,
|
|
1025
|
-
"context_snapshots": ctx_snapshots,
|
|
1026
|
-
"context_updates": ctx_updates,
|
|
1980
|
+
# The --agent detail view (non-JSON) stays a single-pass path of its own:
|
|
1981
|
+
# it renders raw episode/audit rows for one agent rather than the
|
|
1982
|
+
# aggregate dashboard, so there is no double-computation to unify here.
|
|
1983
|
+
if agent_name and not as_json:
|
|
1984
|
+
data = {
|
|
1985
|
+
"workflow_metrics": workflow_metrics,
|
|
1986
|
+
"audit_logs": audit_logs,
|
|
1987
|
+
"run_snapshots": run_snapshots,
|
|
1988
|
+
"skill_snapshots": skill_snapshots,
|
|
1989
|
+
"anomaly_entries": anomaly_entries,
|
|
1027
1990
|
}
|
|
1028
|
-
|
|
1029
|
-
output["agent_filter"] = agent_name
|
|
1030
|
-
print(json.dumps(output, indent=2))
|
|
1991
|
+
_display_agent_detail(root, agent_name, data)
|
|
1031
1992
|
return 0
|
|
1032
1993
|
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1994
|
+
# Build the unified snapshot ONCE -- both --json and the console render
|
|
1995
|
+
# read from this single computation (dashboard v2, dim #18).
|
|
1996
|
+
snapshot = MetricsSnapshot.build(
|
|
1997
|
+
workspace=workspace_override,
|
|
1998
|
+
audit_logs=audit_logs,
|
|
1999
|
+
workflow_metrics=workflow_metrics,
|
|
2000
|
+
run_snapshots=run_snapshots,
|
|
2001
|
+
skill_snapshots=skill_snapshots,
|
|
2002
|
+
anomaly_entries=anomaly_entries,
|
|
2003
|
+
agent_filter=agent_name,
|
|
2004
|
+
window=window,
|
|
2005
|
+
)
|
|
1040
2006
|
|
|
1041
|
-
if
|
|
1042
|
-
|
|
2007
|
+
if as_json:
|
|
2008
|
+
print(json.dumps(snapshot.to_dict(), indent=2))
|
|
1043
2009
|
else:
|
|
1044
|
-
|
|
1045
|
-
cmd_types = _calculate_command_type_breakdown(audit_logs)
|
|
1046
|
-
top_cmds = _calculate_top_commands(audit_logs)
|
|
1047
|
-
agent_inv = _calculate_agent_invocations(workflow_metrics)
|
|
1048
|
-
error_stats = _calculate_error_rate(audit_logs)
|
|
1049
|
-
agent_outcomes = _calculate_agent_outcomes(workflow_metrics)
|
|
1050
|
-
token_usage = _calculate_token_usage(workflow_metrics)
|
|
1051
|
-
anomaly_summary = _calculate_anomaly_summary(anomaly_entries)
|
|
1052
|
-
runtime_skills = _calculate_runtime_skill_summary(skill_snapshots, run_snapshots)
|
|
1053
|
-
ctx_snapshots = _calculate_context_snapshot_summary(run_snapshots)
|
|
1054
|
-
ctx_updates = _calculate_context_update_summary(run_snapshots)
|
|
1055
|
-
|
|
1056
|
-
_display_metrics({
|
|
1057
|
-
"tiers": tiers,
|
|
1058
|
-
"cmd_types": cmd_types,
|
|
1059
|
-
"top_cmds": top_cmds,
|
|
1060
|
-
"agent_invocations": agent_inv,
|
|
1061
|
-
"error_stats": error_stats,
|
|
1062
|
-
"audit_total": len(audit_logs),
|
|
1063
|
-
"agent_outcomes": agent_outcomes,
|
|
1064
|
-
"token_usage": token_usage,
|
|
1065
|
-
"anomaly_summary": anomaly_summary,
|
|
1066
|
-
"runtime_skills": runtime_skills,
|
|
1067
|
-
"context_snapshots": ctx_snapshots,
|
|
1068
|
-
"context_updates": ctx_updates,
|
|
1069
|
-
})
|
|
2010
|
+
render_console(snapshot)
|
|
1070
2011
|
|
|
1071
2012
|
return 0
|