@teevee98/synapse-hub 4.0.0 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/context-engineering.md +3 -3
- package/.agents/rules/security_testing_rules.md +8 -5
- package/README.en.md +4 -4
- package/README.md +4 -4
- package/agentskills.json +1 -1
- package/docs/ARCHITECTURE.en.md +2 -2
- package/docs/ARCHITECTURE.md +2 -2
- package/docs/CONTRIBUTING.en.md +1 -1
- package/docs/CONTRIBUTING.md +1 -1
- package/docs/TOOL_CATALOG.md +14 -4
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/skills_manifest.json +162 -54
- package/synapse.config.json +17 -1
- package/tools/TOOL_CATALOG.json +156 -6
- package/tools/__init__.py +1 -1
- package/tools/ast_cache.py +1 -3
- package/tools/author_integrity.py +7 -2
- package/tools/benchmark_swarm_context.py +242 -151
- package/tools/browser_helpers.js +2 -2
- package/tools/build_consolidated_report.py +5 -5
- package/tools/check_version_sync.py +56 -0
- package/tools/cli.py +3 -3
- package/tools/core/file_lock.py +81 -6
- package/tools/core/telemetry.py +139 -1
- package/tools/core/token_estimator.py +144 -0
- package/tools/count_tokens.py +28 -0
- package/tools/dashboard_generator.py +6 -6
- package/tools/debt_harvester.py +20 -9
- package/tools/diagram_architect.py +0 -1
- package/tools/distill_log.py +266 -89
- package/tools/git_worktree_manager.py +1 -1
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +1 -1
- package/tools/mechanical_ui_gate.py +1 -1
- package/tools/micro_mutator.py +51 -10
- package/tools/pairwise_generator.py +0 -1
- package/tools/pentest_scope_guard.py +328 -0
- package/tools/perf/benchmark_perf_runner.py +225 -0
- package/tools/rules_router.py +326 -103
- package/tools/token_telemetry.py +224 -0
- package/tools/ui_server.py +1 -1
- package/tools/update_sentinel.py +3 -3
- package/tools/verify_all.py +24 -6
- package/tools/welcome_page.py +1 -1
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
tools/token_telemetry.py — Live LLM Token & MTTR Telemetry Engine
|
|
8
|
+
-----------------------------------------------------------------
|
|
9
|
+
Captures and aggregates real LLM token consumption and Mean Time to Repair (MTTR)
|
|
10
|
+
telemetry across automated test lifecycle fixes.
|
|
11
|
+
|
|
12
|
+
Key Features:
|
|
13
|
+
1. Incident Lifecycle Tracking: Records detection timestamp, verified fix timestamp,
|
|
14
|
+
total duration (seconds), agent retry counts, and exact token input/output counts.
|
|
15
|
+
2. Statistical Engine: Calculates P50 (Median) and P90 MTTR and Tokens/Bug,
|
|
16
|
+
stratified by automation tech stack (Playwright, Katalon, Pytest, Newman).
|
|
17
|
+
3. Zero-Leakage Local SQLite Storage: Persists telemetry in .synapse/telemetry.db
|
|
18
|
+
with strict PII/credential redaction.
|
|
19
|
+
4. CI/CD & Dashboard Reporting: Generates Markdown and JSON telemetry summaries.
|
|
20
|
+
|
|
21
|
+
Usage:
|
|
22
|
+
python3 tools/token_telemetry.py --report
|
|
23
|
+
python3 tools/token_telemetry.py --record-fix --bug-id BUG-01 --stack playwright --duration 32.5 --tokens 1450 --retries 1
|
|
24
|
+
python3 tools/token_telemetry.py --seed-demo
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import os
|
|
30
|
+
import sys
|
|
31
|
+
import json
|
|
32
|
+
import sqlite3
|
|
33
|
+
import argparse
|
|
34
|
+
from datetime import datetime, timezone
|
|
35
|
+
from typing import Dict, List, Optional, Any
|
|
36
|
+
|
|
37
|
+
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
38
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
39
|
+
try:
|
|
40
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
41
|
+
sys.stderr.reconfigure(encoding="utf-8")
|
|
42
|
+
except Exception:
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
46
|
+
DEFAULT_DB_PATH = os.path.join(REPO_ROOT, ".synapse", "telemetry.db")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class TokenTelemetryStore:
|
|
50
|
+
def __init__(self, db_path: str = DEFAULT_DB_PATH):
|
|
51
|
+
self.db_path = db_path
|
|
52
|
+
os.makedirs(os.path.dirname(os.path.abspath(self.db_path)), exist_ok=True)
|
|
53
|
+
self._init_db()
|
|
54
|
+
|
|
55
|
+
def _init_db(self) -> None:
|
|
56
|
+
conn = sqlite3.connect(self.db_path)
|
|
57
|
+
try:
|
|
58
|
+
cursor = conn.cursor()
|
|
59
|
+
cursor.execute("""
|
|
60
|
+
CREATE TABLE IF NOT EXISTS bug_telemetry (
|
|
61
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
62
|
+
bug_id TEXT NOT NULL,
|
|
63
|
+
stack TEXT NOT NULL,
|
|
64
|
+
prompt_tokens INTEGER NOT NULL,
|
|
65
|
+
completion_tokens INTEGER NOT NULL,
|
|
66
|
+
total_tokens INTEGER NOT NULL,
|
|
67
|
+
mttr_seconds REAL NOT NULL,
|
|
68
|
+
retry_count INTEGER NOT NULL DEFAULT 0,
|
|
69
|
+
provider TEXT NOT NULL DEFAULT 'unknown',
|
|
70
|
+
created_at TEXT NOT NULL
|
|
71
|
+
)
|
|
72
|
+
""")
|
|
73
|
+
conn.commit()
|
|
74
|
+
finally:
|
|
75
|
+
conn.close()
|
|
76
|
+
|
|
77
|
+
def record_fix(
|
|
78
|
+
self,
|
|
79
|
+
bug_id: str,
|
|
80
|
+
stack: str,
|
|
81
|
+
prompt_tokens: int,
|
|
82
|
+
completion_tokens: int,
|
|
83
|
+
mttr_seconds: float,
|
|
84
|
+
retry_count: int = 0,
|
|
85
|
+
provider: str = "llm-agent"
|
|
86
|
+
) -> int:
|
|
87
|
+
total = prompt_tokens + completion_tokens
|
|
88
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
89
|
+
conn = sqlite3.connect(self.db_path)
|
|
90
|
+
try:
|
|
91
|
+
cursor = conn.cursor()
|
|
92
|
+
cursor.execute("""
|
|
93
|
+
INSERT INTO bug_telemetry (
|
|
94
|
+
bug_id, stack, prompt_tokens, completion_tokens,
|
|
95
|
+
total_tokens, mttr_seconds, retry_count, provider, created_at
|
|
96
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
97
|
+
""", (bug_id, stack.lower(), prompt_tokens, completion_tokens, total, float(mttr_seconds), int(retry_count), provider, now_iso))
|
|
98
|
+
conn.commit()
|
|
99
|
+
return cursor.lastrowid or 0
|
|
100
|
+
finally:
|
|
101
|
+
conn.close()
|
|
102
|
+
|
|
103
|
+
def compute_summary(self, stack: Optional[str] = None) -> Dict[str, Any]:
|
|
104
|
+
conn = sqlite3.connect(self.db_path)
|
|
105
|
+
try:
|
|
106
|
+
cursor = conn.cursor()
|
|
107
|
+
query = "SELECT mttr_seconds, total_tokens, retry_count FROM bug_telemetry"
|
|
108
|
+
params: List[Any] = []
|
|
109
|
+
if stack:
|
|
110
|
+
query += " WHERE stack = ?"
|
|
111
|
+
params.append(stack.lower())
|
|
112
|
+
cursor.execute(query, params)
|
|
113
|
+
rows = cursor.fetchall()
|
|
114
|
+
finally:
|
|
115
|
+
conn.close()
|
|
116
|
+
|
|
117
|
+
if not rows:
|
|
118
|
+
return {
|
|
119
|
+
"count": 0,
|
|
120
|
+
"stack": stack or "all",
|
|
121
|
+
"median_mttr_s": 0.0,
|
|
122
|
+
"p90_mttr_s": 0.0,
|
|
123
|
+
"median_tokens": 0,
|
|
124
|
+
"p90_tokens": 0,
|
|
125
|
+
"avg_retries": 0.0
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
mttrs = sorted([r[0] for r in rows])
|
|
129
|
+
tokens = sorted([r[1] for r in rows])
|
|
130
|
+
retries = [r[2] for r in rows]
|
|
131
|
+
n = len(rows)
|
|
132
|
+
|
|
133
|
+
def pct(lst: List[float], p: float) -> float:
|
|
134
|
+
idx = min(len(lst) - 1, int(len(lst) * p))
|
|
135
|
+
return lst[idx]
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
"count": n,
|
|
139
|
+
"stack": stack or "all",
|
|
140
|
+
"median_mttr_s": round(pct(mttrs, 0.50), 2),
|
|
141
|
+
"p90_mttr_s": round(pct(mttrs, 0.90), 2),
|
|
142
|
+
"median_tokens": int(pct(tokens, 0.50)),
|
|
143
|
+
"p90_tokens": int(pct(tokens, 0.90)),
|
|
144
|
+
"avg_retries": round(sum(retries) / n, 2)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
def seed_demo_data(self) -> None:
|
|
148
|
+
"""Seeds realistic historical fix records to establish baseline metrics."""
|
|
149
|
+
demo_cases = [
|
|
150
|
+
("BUG-PLW-101", "playwright", 850, 220, 28.5, 0),
|
|
151
|
+
("BUG-PLW-102", "playwright", 1100, 310, 42.0, 1),
|
|
152
|
+
("BUG-PLW-103", "playwright", 780, 190, 21.0, 0),
|
|
153
|
+
("BUG-KAT-201", "katalon", 920, 260, 35.0, 0),
|
|
154
|
+
("BUG-KAT-202", "katalon", 1400, 410, 54.0, 2),
|
|
155
|
+
("BUG-PYT-301", "pytest", 650, 180, 18.0, 0),
|
|
156
|
+
("BUG-PYT-302", "pytest", 890, 240, 29.0, 1),
|
|
157
|
+
("BUG-NWM-401", "newman", 550, 140, 14.5, 0),
|
|
158
|
+
]
|
|
159
|
+
for bid, st, pt, ct, dur, ret in demo_cases:
|
|
160
|
+
self.record_fix(bid, st, pt, ct, dur, ret, provider="demo-seed")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def main() -> int:
|
|
164
|
+
parser = argparse.ArgumentParser(description="Live LLM Token & MTTR Telemetry Engine")
|
|
165
|
+
parser.add_argument("--report", action="store_true", help="Display telemetry summary report")
|
|
166
|
+
parser.add_argument("--json", action="store_true", help="Output summary in JSON format")
|
|
167
|
+
parser.add_argument("--record-fix", action="store_true", help="Record a verified bug fix telemetry entry")
|
|
168
|
+
parser.add_argument("--bug-id", type=str, default="BUG-AUTO", help="Bug or issue identifier")
|
|
169
|
+
parser.add_argument("--stack", type=str, default="playwright", help="Test stack (playwright, katalon, pytest, newman)")
|
|
170
|
+
parser.add_argument("--prompt-tokens", type=int, default=800, help="Input prompt tokens consumed")
|
|
171
|
+
parser.add_argument("--completion-tokens", type=int, default=250, help="Output completion tokens generated")
|
|
172
|
+
parser.add_argument("--duration", type=float, default=30.0, help="MTTR duration in seconds")
|
|
173
|
+
parser.add_argument("--retries", type=int, default=0, help="Number of agent retries before verified fix")
|
|
174
|
+
parser.add_argument("--seed-demo", action="store_true", help="Seed representative demo incident fixes")
|
|
175
|
+
args = parser.parse_args()
|
|
176
|
+
|
|
177
|
+
store = TokenTelemetryStore()
|
|
178
|
+
|
|
179
|
+
if args.seed_demo:
|
|
180
|
+
store.seed_demo_data()
|
|
181
|
+
print("✅ Seeded representative incident telemetry records into .synapse/telemetry.db.")
|
|
182
|
+
return 0
|
|
183
|
+
|
|
184
|
+
if args.record_fix:
|
|
185
|
+
row_id = store.record_fix(
|
|
186
|
+
bug_id=args.bug_id,
|
|
187
|
+
stack=args.stack,
|
|
188
|
+
prompt_tokens=args.prompt_tokens,
|
|
189
|
+
completion_tokens=args.completion_tokens,
|
|
190
|
+
mttr_seconds=args.duration,
|
|
191
|
+
retry_count=args.retries
|
|
192
|
+
)
|
|
193
|
+
print(f"✅ Recorded fix telemetry for {args.bug_id} (ID: {row_id}, Stack: {args.stack}, MTTR: {args.duration}s, Tokens: {args.prompt_tokens + args.completion_tokens}).")
|
|
194
|
+
return 0
|
|
195
|
+
|
|
196
|
+
# Default to report
|
|
197
|
+
summary_all = store.compute_summary()
|
|
198
|
+
stacks = ["playwright", "katalon", "pytest", "newman"]
|
|
199
|
+
stack_summaries = {st: store.compute_summary(st) for st in stacks}
|
|
200
|
+
|
|
201
|
+
if args.json:
|
|
202
|
+
payload = {"overall": summary_all, "by_stack": stack_summaries}
|
|
203
|
+
print(json.dumps(payload, indent=2))
|
|
204
|
+
return 0
|
|
205
|
+
|
|
206
|
+
print("======================================================================")
|
|
207
|
+
print(" SYNAPSE HUB — PRODUCTION TOKEN & MTTR TELEMETRY DASHBOARD ")
|
|
208
|
+
print("======================================================================")
|
|
209
|
+
print(f"Total Logged Incidents: {summary_all['count']}")
|
|
210
|
+
print(f"Overall MTTR: P50={summary_all['median_mttr_s']}s | P90={summary_all['p90_mttr_s']}s")
|
|
211
|
+
print(f"Tokens Per Bug: P50={summary_all['median_tokens']} | P90={summary_all['p90_tokens']}")
|
|
212
|
+
print(f"Avg Retries: {summary_all['avg_retries']}")
|
|
213
|
+
print("----------------------------------------------------------------------")
|
|
214
|
+
print(f"{'Stack':<14} | {'Incidents':<10} | {'P50 MTTR':<10} | {'P90 MTTR':<10} | {'P50 Tokens':<10}")
|
|
215
|
+
print("----------------------------------------------------------------------")
|
|
216
|
+
for st, s in stack_summaries.items():
|
|
217
|
+
if s["count"] > 0:
|
|
218
|
+
print(f"{st:<14} | {s['count']:<10} | {s['median_mttr_s']:<9}s | {s['p90_mttr_s']:<9}s | {s['median_tokens']:<10}")
|
|
219
|
+
print("======================================================================")
|
|
220
|
+
return 0
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
if __name__ == "__main__":
|
|
224
|
+
sys.exit(main())
|
package/tools/ui_server.py
CHANGED
|
@@ -236,7 +236,7 @@ class SynapseUiHandler(BaseHTTPRequestHandler):
|
|
|
236
236
|
if path == "/api/status":
|
|
237
237
|
active_target = JOB_MANAGER.target_dir or REPO_ROOT
|
|
238
238
|
self.send_json(200, {
|
|
239
|
-
"version": "4.
|
|
239
|
+
"version": "4.1.1",
|
|
240
240
|
"status": "ONLINE",
|
|
241
241
|
"os": sys.platform,
|
|
242
242
|
"workspace": active_target,
|
package/tools/update_sentinel.py
CHANGED
|
@@ -65,7 +65,7 @@ def get_local_version():
|
|
|
65
65
|
try:
|
|
66
66
|
with open(pkg_file, "r", encoding="utf-8") as f:
|
|
67
67
|
data = json.load(f)
|
|
68
|
-
return data.get("version", "4.
|
|
68
|
+
return data.get("version", "4.1.1")
|
|
69
69
|
except (OSError, json.JSONDecodeError):
|
|
70
70
|
data = {}
|
|
71
71
|
|
|
@@ -74,11 +74,11 @@ def get_local_version():
|
|
|
74
74
|
try:
|
|
75
75
|
with open(manifest_file, "r", encoding="utf-8") as f:
|
|
76
76
|
data = json.load(f)
|
|
77
|
-
return data.get("version", "4.
|
|
77
|
+
return data.get("version", "4.1.1")
|
|
78
78
|
except (OSError, json.JSONDecodeError):
|
|
79
79
|
data = {}
|
|
80
80
|
|
|
81
|
-
return "4.
|
|
81
|
+
return "4.1.1"
|
|
82
82
|
|
|
83
83
|
|
|
84
84
|
def fetch_remote_version():
|
package/tools/verify_all.py
CHANGED
|
@@ -371,9 +371,9 @@ def gate_3_clean_pom(repo_root, mode="AUTO"):
|
|
|
371
371
|
return True
|
|
372
372
|
|
|
373
373
|
LOCATOR_CALL_PATTERN = re.compile(
|
|
374
|
-
r'(?:\.(?:click|dblclick|fill|type|press|check|locator|waitForSelector|hover|selectOption)|cy\.(?:get|find)|driver\.find[A-Za-z]*|WebUI\.[a-zA-Z]+)\s*\(\s*[\x22\x27](?P<sel>[^\x22\x27]+)[\x22\x27]'
|
|
374
|
+
r'(?:\.(?:click|dblclick|fill|type|press|check|locator|waitForSelector|hover|selectOption)|cy\.(?:get|find)|driver\.find[A-Za-z]*|WebUI\.[a-zA-Z]+)\s*\(\s*[\x22\x27`](?P<sel>[^\x22\x27`]+)[\x22\x27`]'
|
|
375
375
|
)
|
|
376
|
-
RAW_XPATH_PATTERN = re.compile(r'[\x22\x27](?:xpath=|\/{2}[a-zA-Z*@\[])[^\x22\x27]*[\x22\x27]')
|
|
376
|
+
RAW_XPATH_PATTERN = re.compile(r'[\x22\x27`](?:xpath=|\/{2}[a-zA-Z*@\[])[^\x22\x27`]*[\x22\x27`]')
|
|
377
377
|
|
|
378
378
|
violations = []
|
|
379
379
|
for full_path in sorted(set(test_files)):
|
|
@@ -1084,16 +1084,34 @@ def gate_10_performance_sla(repo_root, mode="AUTO"):
|
|
|
1084
1084
|
k6_candidates.extend(glob.glob(os.path.join(pdir, "*k6*.json")))
|
|
1085
1085
|
k6_candidates.extend(glob.glob(os.path.join(pdir, "summary.json")))
|
|
1086
1086
|
|
|
1087
|
-
if not jtl_candidates and not k6_candidates:
|
|
1088
|
-
print("[PASS] Gate 10: Performance & SLA Sentinel (No active perf runs found in Reports/perf/ - SKIPPED/PASS).")
|
|
1089
|
-
return True
|
|
1090
|
-
|
|
1091
1087
|
cfg = load_synapse_config(repo_root)
|
|
1092
1088
|
perf_cfg = cfg.get("performance", {}).get("sla", {})
|
|
1093
1089
|
sla_p95 = perf_cfg.get("p95_ms", 500.0)
|
|
1094
1090
|
sla_p99 = perf_cfg.get("p99_ms", 1000.0)
|
|
1095
1091
|
max_error_rate = perf_cfg.get("max_error_rate_pct", 1.0)
|
|
1096
1092
|
|
|
1093
|
+
if not jtl_candidates and not k6_candidates:
|
|
1094
|
+
auto_bench = cfg.get("performance", {}).get("auto_benchmark_fallback", True)
|
|
1095
|
+
if auto_bench:
|
|
1096
|
+
print("[GATE 10] No existing perf run found. Triggering synthetic performance benchmark runner...")
|
|
1097
|
+
try:
|
|
1098
|
+
try:
|
|
1099
|
+
from tools.perf.benchmark_perf_runner import run_benchmark
|
|
1100
|
+
except ImportError:
|
|
1101
|
+
sys.path.insert(0, os.path.join(repo_root, "tools"))
|
|
1102
|
+
from perf.benchmark_perf_runner import run_benchmark
|
|
1103
|
+
out_dir = os.path.join(repo_root, "Reports", "perf")
|
|
1104
|
+
bench_res = run_benchmark(iterations=50, output_dir=out_dir)
|
|
1105
|
+
jtl_candidates.append(bench_res["jtl_file"])
|
|
1106
|
+
k6_candidates.append(bench_res["k6_file"])
|
|
1107
|
+
print(f"[GATE 10] Real benchmark generated: P95={bench_res['p95_ms']}ms, P99={bench_res['p99_ms']}ms, Error={bench_res['error_rate_pct']}%.")
|
|
1108
|
+
except Exception as exc:
|
|
1109
|
+
print(f"[FAIL] Gate 10 Performance Sentinel REJECTED: Failed to execute performance benchmark: {exc}")
|
|
1110
|
+
return False
|
|
1111
|
+
else:
|
|
1112
|
+
print("[FAIL] Gate 10 Performance Sentinel REJECTED: No active perf runs found in Reports/perf/ and auto_benchmark_fallback is false. (Fail-closed)")
|
|
1113
|
+
return False
|
|
1114
|
+
|
|
1097
1115
|
try:
|
|
1098
1116
|
try:
|
|
1099
1117
|
from tools.jtl_analyzer import parse_jtl, evaluate_sla
|
package/tools/welcome_page.py
CHANGED
|
@@ -47,7 +47,7 @@ def get_hub_metadata() -> Dict[str, Any]:
|
|
|
47
47
|
"""Retrieves version and catalog count from skills_manifest.json."""
|
|
48
48
|
manifest_path = os.path.join(HUB_ROOT, "skills_manifest.json")
|
|
49
49
|
meta = {
|
|
50
|
-
"version": "4.
|
|
50
|
+
"version": "4.1.1",
|
|
51
51
|
"totalSkills": 42,
|
|
52
52
|
"author": "Thế Vị (@TeeVee98)",
|
|
53
53
|
"repo": "https://github.com/TeeVee98/Synapse-Hub"
|