arkaos 4.5.0 → 4.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/VERSION +1 -1
- package/arka/SKILL.md +2 -1
- package/core/hooks/__pycache__/post_tool_use.cpython-313.pyc +0 -0
- package/core/hooks/post_tool_use.py +10 -0
- package/core/runtime/__pycache__/mcp_telemetry.cpython-313.pyc +0 -0
- package/core/runtime/__pycache__/mcp_telemetry_cli.cpython-313.pyc +0 -0
- package/core/runtime/mcp_telemetry.py +184 -0
- package/core/runtime/mcp_telemetry_cli.py +62 -0
- package/dashboard/package.json +1 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.
|
|
1
|
+
4.6.0
|
package/arka/SKILL.md
CHANGED
|
@@ -179,9 +179,10 @@ violation (squad-routing, arka-supremacy, spec-driven, mandatory-qa).
|
|
|
179
179
|
|
|
180
180
|
| Command | Description |
|
|
181
181
|
|---------|-------------|
|
|
182
|
-
| `/arka status` | System status (version, departments, agents, active projects). Includes **LLM costs (24h)** section: top-line cost + cache hit rate + call count from `core.runtime.llm_cost_telemetry.summarise(period="today")`. Also includes **Enforcement (24h)** section: total calls, block rate, top blocked tools/reasons from `core.governance.enforcement_telemetry.summarise(period="today")` (PR19 v2.41.0). Plus **Reorganization (today)** section: today's proposal path + artifact count from `core.cognition.reorganizer_scheduler.status_summary()` (PR24 v2.46.0). Plus **Model routing** section: gateway live/off + resolved per-slot routes + served counts from `core.runtime.model_routing_check.status_summary()`. |
|
|
182
|
+
| `/arka status` | System status (version, departments, agents, active projects). Includes **LLM costs (24h)** section: top-line cost + cache hit rate + call count from `core.runtime.llm_cost_telemetry.summarise(period="today")`. Also includes **Enforcement (24h)** section: total calls, block rate, top blocked tools/reasons from `core.governance.enforcement_telemetry.summarise(period="today")` (PR19 v2.41.0). Plus **Reorganization (today)** section: today's proposal path + artifact count from `core.cognition.reorganizer_scheduler.status_summary()` (PR24 v2.46.0). Plus **Model routing** section: gateway live/off + resolved per-slot routes + served counts from `core.runtime.model_routing_check.status_summary()`. Plus **MCP usage (24h)** section: total calls + servers in use + top servers from `core.runtime.mcp_telemetry.summarise(period="today")`. |
|
|
183
183
|
| `/arka costs [period]` | LLM cost visibility — aggregates telemetry by day/week/month/all, with top expensive sessions. See `arka/skills/costs/SKILL.md`. Shells out to `~/.arkaos/bin/arka-py -m core.runtime.llm_cost_telemetry_cli <period>`. |
|
|
184
184
|
| `/arka enforcement [period]` | Enforcement compliance — aggregates flow-marker enforcement telemetry by day/week/month/all. Shows block rate, top blocked tools, top reasons. Shells out to `~/.arkaos/bin/arka-py -m core.governance.enforcement_telemetry_cli <period>`. |
|
|
185
|
+
| `/arka mcps [period]` | MCP usage — aggregates the PostToolUse MCP telemetry (`~/.arkaos/telemetry/mcp-usage.jsonl`) by day/week/month/all. Shows total calls, servers in use, top servers and top tools. Shells out to `~/.arkaos/bin/arka-py -m core.runtime.mcp_telemetry_cli <period>`. |
|
|
185
186
|
| `/arka compliance [period]` | Behavior compliance summary (PR29 v2.48.0) — aggregates stop-hook telemetry by day/week/month/all. Shows rates for the four contracts: closing marker, `[arka:meta]` tag, KB citation pass, sycophancy clean. Shells out to `~/.arkaos/bin/arka-py -m core.governance.compliance_telemetry_cli <period>`. |
|
|
186
187
|
| `/arka reorganize [--since-days N]` | Dreaming → Agent reorganizer. Aggregates recent KB pattern/anti-pattern/lesson artifacts (default last 7 days) into a markdown proposal at `~/.arkaos/reorganize-proposals/<date>.md`. **Propose-only** — never modifies agent YAMLs. Sanitizes client identifiers from titles and body excerpts; drops `tags:` field entirely to prevent project-name leaks. **Auto-fires on session start when today's proposal is missing** (PR24 v2.46.0 stale-aware trigger, 30s timeout, background). Shells out to `~/.arkaos/bin/arka-py -m core.cognition.reorganizer_cli [--since-days N] [--dry-run]`. |
|
|
187
188
|
| `/arka standup` | Daily standup (projects, priorities, blockers, updates) |
|
|
Binary file
|
|
@@ -632,6 +632,16 @@ def main(stdin_json: dict | None = None) -> int:
|
|
|
632
632
|
# and confirm persistent authorization when a marker is present.
|
|
633
633
|
_confirm_flow_authorization(session_id, transcript_path)
|
|
634
634
|
|
|
635
|
+
# MCP usage telemetry — must run BEFORE the error-signal early exit
|
|
636
|
+
# below (MCP calls normally succeed). record() is a no-op for
|
|
637
|
+
# non-MCP tools and never raises on I/O; the guard covers import
|
|
638
|
+
# failures on stripped installs.
|
|
639
|
+
try:
|
|
640
|
+
from core.runtime.mcp_telemetry import record as _record_mcp_usage
|
|
641
|
+
_record_mcp_usage(tool_name, session_id=session_id)
|
|
642
|
+
except Exception: # noqa: BLE001 — telemetry must never break the hook
|
|
643
|
+
pass
|
|
644
|
+
|
|
635
645
|
if tool_name in ("Task", "Agent"):
|
|
636
646
|
subagent_type = get_str(stdin_json, "tool_input", "subagent_type")
|
|
637
647
|
if subagent_type == "cqo":
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""MCP usage telemetry (audit E2E v4.3.6, P1).
|
|
2
|
+
|
|
3
|
+
Writer + summarizer for ``~/.arkaos/telemetry/mcp-usage.jsonl``. The
|
|
4
|
+
PostToolUse hook calls :func:`record` on every tool event; MCP tool
|
|
5
|
+
names (``mcp__<server>__<tool>``) are parsed and appended as one JSONL
|
|
6
|
+
line, everything else is ignored. Before this module the only usage
|
|
7
|
+
signal was grepping session transcripts.
|
|
8
|
+
|
|
9
|
+
Mirrors ``core.governance.enforcement_telemetry`` so periods,
|
|
10
|
+
malformed-line tolerance, and zero-division safety behave the same way
|
|
11
|
+
across telemetry surfaces. :func:`summarise` is read-only.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from collections import Counter
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from datetime import datetime, timedelta, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Iterable
|
|
22
|
+
|
|
23
|
+
DEFAULT_PATH: Path = Path.home() / ".arkaos" / "telemetry" / "mcp-usage.jsonl"
|
|
24
|
+
_VALID_PERIODS: frozenset[str] = frozenset({"today", "week", "month", "all"})
|
|
25
|
+
_TOP_SERVERS: int = 10
|
|
26
|
+
_TOP_TOOLS: int = 5
|
|
27
|
+
_MCP_PREFIX = "mcp__"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class McpUsageSummary:
|
|
32
|
+
"""Aggregated MCP usage over a time slice."""
|
|
33
|
+
period: str
|
|
34
|
+
total_calls: int
|
|
35
|
+
unique_servers: int
|
|
36
|
+
top_servers: list[tuple[str, int]] = field(default_factory=list)
|
|
37
|
+
top_tools: list[tuple[str, int]] = field(default_factory=list)
|
|
38
|
+
corrupt_line_count: int = 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def parse_mcp_tool(tool_name: str) -> tuple[str, str] | None:
|
|
42
|
+
"""Split ``mcp__<server>__<tool>`` into (server, tool).
|
|
43
|
+
|
|
44
|
+
Returns None for non-MCP tool names. Handles servers whose slug
|
|
45
|
+
contains single underscores (``claude_ai_Canva``, plugin servers
|
|
46
|
+
like ``plugin_claude-mem_mcp-search``) by splitting on the FIRST
|
|
47
|
+
double underscore after the prefix.
|
|
48
|
+
"""
|
|
49
|
+
if not tool_name.startswith(_MCP_PREFIX):
|
|
50
|
+
return None
|
|
51
|
+
rest = tool_name[len(_MCP_PREFIX):]
|
|
52
|
+
server, sep, tool = rest.partition("__")
|
|
53
|
+
if not sep or not server or not tool:
|
|
54
|
+
return None
|
|
55
|
+
return server, tool
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def record(
|
|
59
|
+
tool_name: str,
|
|
60
|
+
*,
|
|
61
|
+
session_id: str = "",
|
|
62
|
+
path: Path | None = None,
|
|
63
|
+
) -> bool:
|
|
64
|
+
"""Append one usage line when tool_name is an MCP tool.
|
|
65
|
+
|
|
66
|
+
Never raises — the caller is a hook that must not block on
|
|
67
|
+
telemetry. Returns True when a line was written.
|
|
68
|
+
"""
|
|
69
|
+
parsed = parse_mcp_tool(tool_name)
|
|
70
|
+
if parsed is None:
|
|
71
|
+
return False
|
|
72
|
+
server, tool = parsed
|
|
73
|
+
dest = path or DEFAULT_PATH
|
|
74
|
+
entry = {
|
|
75
|
+
"ts": datetime.now(timezone.utc).isoformat(),
|
|
76
|
+
"server": server,
|
|
77
|
+
"tool": tool,
|
|
78
|
+
"session": session_id,
|
|
79
|
+
}
|
|
80
|
+
try:
|
|
81
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
with dest.open("a", encoding="utf-8") as fh:
|
|
83
|
+
fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
84
|
+
except OSError:
|
|
85
|
+
return False
|
|
86
|
+
return True
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def summarise(period: str, *, path: Path | None = None) -> McpUsageSummary:
|
|
90
|
+
"""Return a McpUsageSummary for the requested period.
|
|
91
|
+
|
|
92
|
+
period: one of "today", "week", "month", "all".
|
|
93
|
+
path: override telemetry source (used by tests; defaults to DEFAULT_PATH).
|
|
94
|
+
"""
|
|
95
|
+
if period not in _VALID_PERIODS:
|
|
96
|
+
raise ValueError(f"invalid period: {period!r}")
|
|
97
|
+
src = path or DEFAULT_PATH
|
|
98
|
+
cutoff = _period_cutoff(period)
|
|
99
|
+
entries, corrupt = _read_jsonl(src, cutoff)
|
|
100
|
+
return _build_summary(period, entries, corrupt)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _period_cutoff(period: str, now: datetime | None = None) -> datetime | None:
|
|
104
|
+
ref = now or datetime.now(timezone.utc)
|
|
105
|
+
if period == "today":
|
|
106
|
+
return ref.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
107
|
+
if period == "week":
|
|
108
|
+
return ref - timedelta(days=7)
|
|
109
|
+
if period == "month":
|
|
110
|
+
return ref - timedelta(days=30)
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _read_jsonl(src: Path, cutoff: datetime | None) -> tuple[list[dict[str, Any]], int]:
|
|
115
|
+
if not src.exists():
|
|
116
|
+
return [], 0
|
|
117
|
+
entries: list[dict[str, Any]] = []
|
|
118
|
+
corrupt = 0
|
|
119
|
+
# Line-stream to keep memory O(1) — this file grows on every MCP call.
|
|
120
|
+
try:
|
|
121
|
+
with src.open("r", encoding="utf-8", errors="replace") as fh:
|
|
122
|
+
for line in fh:
|
|
123
|
+
if not line.strip():
|
|
124
|
+
continue
|
|
125
|
+
try:
|
|
126
|
+
entry = json.loads(line)
|
|
127
|
+
except json.JSONDecodeError:
|
|
128
|
+
corrupt += 1
|
|
129
|
+
continue
|
|
130
|
+
if not isinstance(entry, dict):
|
|
131
|
+
corrupt += 1
|
|
132
|
+
continue
|
|
133
|
+
if cutoff is not None and not _within_cutoff(entry, cutoff):
|
|
134
|
+
continue
|
|
135
|
+
entries.append(entry)
|
|
136
|
+
except OSError:
|
|
137
|
+
return entries, corrupt
|
|
138
|
+
return entries, corrupt
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _within_cutoff(entry: dict[str, Any], cutoff: datetime) -> bool:
|
|
142
|
+
ts = _parse_ts(entry.get("ts"))
|
|
143
|
+
if ts is None:
|
|
144
|
+
return False
|
|
145
|
+
return ts >= cutoff
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _parse_ts(raw: Any) -> datetime | None:
|
|
149
|
+
if not isinstance(raw, str) or not raw:
|
|
150
|
+
return None
|
|
151
|
+
try:
|
|
152
|
+
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
153
|
+
except ValueError:
|
|
154
|
+
return None
|
|
155
|
+
if parsed.tzinfo is None:
|
|
156
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
157
|
+
return parsed
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _build_summary(
|
|
161
|
+
period: str,
|
|
162
|
+
entries: Iterable[dict[str, Any]],
|
|
163
|
+
corrupt: int,
|
|
164
|
+
) -> McpUsageSummary:
|
|
165
|
+
total = 0
|
|
166
|
+
servers: Counter[str] = Counter()
|
|
167
|
+
tools: Counter[str] = Counter()
|
|
168
|
+
for entry in entries:
|
|
169
|
+
server = str(entry.get("server", ""))
|
|
170
|
+
tool = str(entry.get("tool", ""))
|
|
171
|
+
if not server:
|
|
172
|
+
continue
|
|
173
|
+
total += 1
|
|
174
|
+
servers[server] += 1
|
|
175
|
+
if tool:
|
|
176
|
+
tools[f"{server}/{tool}"] += 1
|
|
177
|
+
return McpUsageSummary(
|
|
178
|
+
period=period,
|
|
179
|
+
total_calls=total,
|
|
180
|
+
unique_servers=len(servers),
|
|
181
|
+
top_servers=servers.most_common(_TOP_SERVERS),
|
|
182
|
+
top_tools=tools.most_common(_TOP_TOOLS),
|
|
183
|
+
corrupt_line_count=corrupt,
|
|
184
|
+
)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""CLI entry for the MCP usage telemetry summarizer.
|
|
2
|
+
|
|
3
|
+
Invoked as ``python -m core.runtime.mcp_telemetry_cli <period>`` where
|
|
4
|
+
``<period>`` is one of: today, week, month, all (default: today).
|
|
5
|
+
|
|
6
|
+
Output is plain markdown so it renders cleanly in both the terminal and
|
|
7
|
+
the ``/arka status`` skill which concatenates it into its report.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
from core.runtime.mcp_telemetry import McpUsageSummary, summarise
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _sanitize_md(value: str) -> str:
|
|
18
|
+
"""Strip markdown-breaking characters before rendering JSONL strings.
|
|
19
|
+
|
|
20
|
+
Telemetry is local-writer-only, but a malformed server/tool value
|
|
21
|
+
(newline, backtick) can still distort the markdown if /arka status
|
|
22
|
+
pipes the output to a UI.
|
|
23
|
+
"""
|
|
24
|
+
return value.replace("\n", " ").replace("\r", " ").replace("`", "").strip()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _render(summary: McpUsageSummary) -> str:
|
|
28
|
+
lines = [
|
|
29
|
+
f"# MCP usage — {summary.period}",
|
|
30
|
+
"",
|
|
31
|
+
f"- Calls: **{summary.total_calls}**",
|
|
32
|
+
f"- Servers in use: **{summary.unique_servers}**",
|
|
33
|
+
]
|
|
34
|
+
if summary.top_servers:
|
|
35
|
+
lines += ["", "## Top servers"]
|
|
36
|
+
for server, count in summary.top_servers:
|
|
37
|
+
lines.append(f"- `{_sanitize_md(server)}` — {count}")
|
|
38
|
+
if summary.top_tools:
|
|
39
|
+
lines += ["", "## Top tools"]
|
|
40
|
+
for tool, count in summary.top_tools:
|
|
41
|
+
lines.append(f"- `{_sanitize_md(tool)}` — {count}")
|
|
42
|
+
if summary.corrupt_line_count:
|
|
43
|
+
lines += ["", f"_Skipped {summary.corrupt_line_count} corrupt line(s)._"]
|
|
44
|
+
if summary.total_calls == 0:
|
|
45
|
+
lines += ["", "_No MCP calls recorded for this period._"]
|
|
46
|
+
return "\n".join(lines)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main(argv: list[str] | None = None) -> int:
|
|
50
|
+
args = argv if argv is not None else sys.argv[1:]
|
|
51
|
+
period = args[0] if args else "today"
|
|
52
|
+
try:
|
|
53
|
+
summary = summarise(period)
|
|
54
|
+
except ValueError as exc:
|
|
55
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
56
|
+
return 2
|
|
57
|
+
print(_render(summary))
|
|
58
|
+
return 0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
if __name__ == "__main__":
|
|
62
|
+
raise SystemExit(main())
|
package/dashboard/package.json
CHANGED
package/package.json
CHANGED