@teevee98/synapse-hub 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.agents/rules/context-engineering.md +3 -3
  2. package/.agents/rules/security_testing_rules.md +8 -5
  3. package/README.en.md +2 -2
  4. package/README.md +2 -2
  5. package/agentskills.json +1 -1
  6. package/docs/ARCHITECTURE.en.md +1 -1
  7. package/docs/ARCHITECTURE.md +1 -1
  8. package/package.json +1 -1
  9. package/plugin.json +1 -1
  10. package/skills_manifest.json +162 -54
  11. package/synapse.config.json +17 -1
  12. package/tools/__init__.py +1 -1
  13. package/tools/ast_cache.py +1 -3
  14. package/tools/author_integrity.py +7 -2
  15. package/tools/benchmark_swarm_context.py +242 -151
  16. package/tools/browser_helpers.js +2 -2
  17. package/tools/build_consolidated_report.py +5 -5
  18. package/tools/check_version_sync.py +56 -0
  19. package/tools/cli.py +3 -3
  20. package/tools/core/file_lock.py +81 -6
  21. package/tools/core/telemetry.py +139 -1
  22. package/tools/core/token_estimator.py +145 -0
  23. package/tools/count_tokens.py +28 -0
  24. package/tools/dashboard_generator.py +6 -6
  25. package/tools/debt_harvester.py +20 -9
  26. package/tools/diagram_architect.py +0 -1
  27. package/tools/distill_log.py +266 -89
  28. package/tools/git_worktree_manager.py +1 -1
  29. package/tools/mcp_docx_server.py +1 -1
  30. package/tools/mcp_excel_server.py +1 -1
  31. package/tools/mcp_server.py +1 -1
  32. package/tools/mechanical_ui_gate.py +1 -1
  33. package/tools/micro_mutator.py +51 -10
  34. package/tools/pairwise_generator.py +0 -1
  35. package/tools/pentest_scope_guard.py +328 -0
  36. package/tools/perf/benchmark_perf_runner.py +226 -0
  37. package/tools/rules_router.py +326 -103
  38. package/tools/token_telemetry.py +224 -0
  39. package/tools/ui_server.py +1 -1
  40. package/tools/update_sentinel.py +3 -3
  41. package/tools/verify_all.py +24 -6
  42. package/tools/welcome_page.py +1 -1
@@ -0,0 +1,224 @@
1
+ #!/usr/bin/env python3
2
+ # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
3
+ # SPDX-License-Identifier: MIT
4
+ # Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
5
+
6
+ """
7
+ tools/token_telemetry.py — Live LLM Token & MTTR Telemetry Engine
8
+ -----------------------------------------------------------------
9
+ Captures and aggregates real LLM token consumption and Mean Time to Repair (MTTR)
10
+ telemetry across automated test lifecycle fixes.
11
+
12
+ Key Features:
13
+ 1. Incident Lifecycle Tracking: Records detection timestamp, verified fix timestamp,
14
+ total duration (seconds), agent retry counts, and exact token input/output counts.
15
+ 2. Statistical Engine: Calculates P50 (Median) and P90 MTTR and Tokens/Bug,
16
+ stratified by automation tech stack (Playwright, Katalon, Pytest, Newman).
17
+ 3. Zero-Leakage Local SQLite Storage: Persists telemetry in .synapse/telemetry.db
18
+ with strict PII/credential redaction.
19
+ 4. CI/CD & Dashboard Reporting: Generates Markdown and JSON telemetry summaries.
20
+
21
+ Usage:
22
+ python3 tools/token_telemetry.py --report
23
+ python3 tools/token_telemetry.py --record-fix --bug-id BUG-01 --stack playwright --duration 32.5 --tokens 1450 --retries 1
24
+ python3 tools/token_telemetry.py --seed-demo
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import os
30
+ import sys
31
+ import json
32
+ import sqlite3
33
+ import argparse
34
+ from datetime import datetime, timezone
35
+ from typing import Dict, List, Optional, Any, Tuple
36
+
37
+ # Enforce UTF-8 stdout encoding for Windows console compatibility
38
+ if hasattr(sys.stdout, "reconfigure"):
39
+ try:
40
+ sys.stdout.reconfigure(encoding="utf-8")
41
+ sys.stderr.reconfigure(encoding="utf-8")
42
+ except Exception:
43
+ pass
44
+
45
+ REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
46
+ DEFAULT_DB_PATH = os.path.join(REPO_ROOT, ".synapse", "telemetry.db")
47
+
48
+
49
+ class TokenTelemetryStore:
50
+ def __init__(self, db_path: str = DEFAULT_DB_PATH):
51
+ self.db_path = db_path
52
+ os.makedirs(os.path.dirname(os.path.abspath(self.db_path)), exist_ok=True)
53
+ self._init_db()
54
+
55
+ def _init_db(self) -> None:
56
+ conn = sqlite3.connect(self.db_path)
57
+ try:
58
+ cursor = conn.cursor()
59
+ cursor.execute("""
60
+ CREATE TABLE IF NOT EXISTS bug_telemetry (
61
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
62
+ bug_id TEXT NOT NULL,
63
+ stack TEXT NOT NULL,
64
+ prompt_tokens INTEGER NOT NULL,
65
+ completion_tokens INTEGER NOT NULL,
66
+ total_tokens INTEGER NOT NULL,
67
+ mttr_seconds REAL NOT NULL,
68
+ retry_count INTEGER NOT NULL DEFAULT 0,
69
+ provider TEXT NOT NULL DEFAULT 'unknown',
70
+ created_at TEXT NOT NULL
71
+ )
72
+ """)
73
+ conn.commit()
74
+ finally:
75
+ conn.close()
76
+
77
+ def record_fix(
78
+ self,
79
+ bug_id: str,
80
+ stack: str,
81
+ prompt_tokens: int,
82
+ completion_tokens: int,
83
+ mttr_seconds: float,
84
+ retry_count: int = 0,
85
+ provider: str = "llm-agent"
86
+ ) -> int:
87
+ total = prompt_tokens + completion_tokens
88
+ now_iso = datetime.now(timezone.utc).isoformat()
89
+ conn = sqlite3.connect(self.db_path)
90
+ try:
91
+ cursor = conn.cursor()
92
+ cursor.execute("""
93
+ INSERT INTO bug_telemetry (
94
+ bug_id, stack, prompt_tokens, completion_tokens,
95
+ total_tokens, mttr_seconds, retry_count, provider, created_at
96
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
97
+ """, (bug_id, stack.lower(), prompt_tokens, completion_tokens, total, float(mttr_seconds), int(retry_count), provider, now_iso))
98
+ conn.commit()
99
+ return cursor.lastrowid or 0
100
+ finally:
101
+ conn.close()
102
+
103
+ def compute_summary(self, stack: Optional[str] = None) -> Dict[str, Any]:
104
+ conn = sqlite3.connect(self.db_path)
105
+ try:
106
+ cursor = conn.cursor()
107
+ query = "SELECT mttr_seconds, total_tokens, retry_count FROM bug_telemetry"
108
+ params: List[Any] = []
109
+ if stack:
110
+ query += " WHERE stack = ?"
111
+ params.append(stack.lower())
112
+ cursor.execute(query, params)
113
+ rows = cursor.fetchall()
114
+ finally:
115
+ conn.close()
116
+
117
+ if not rows:
118
+ return {
119
+ "count": 0,
120
+ "stack": stack or "all",
121
+ "median_mttr_s": 0.0,
122
+ "p90_mttr_s": 0.0,
123
+ "median_tokens": 0,
124
+ "p90_tokens": 0,
125
+ "avg_retries": 0.0
126
+ }
127
+
128
+ mttrs = sorted([r[0] for r in rows])
129
+ tokens = sorted([r[1] for r in rows])
130
+ retries = [r[2] for r in rows]
131
+ n = len(rows)
132
+
133
+ def pct(lst: List[float], p: float) -> float:
134
+ idx = min(len(lst) - 1, int(len(lst) * p))
135
+ return lst[idx]
136
+
137
+ return {
138
+ "count": n,
139
+ "stack": stack or "all",
140
+ "median_mttr_s": round(pct(mttrs, 0.50), 2),
141
+ "p90_mttr_s": round(pct(mttrs, 0.90), 2),
142
+ "median_tokens": int(pct(tokens, 0.50)),
143
+ "p90_tokens": int(pct(tokens, 0.90)),
144
+ "avg_retries": round(sum(retries) / n, 2)
145
+ }
146
+
147
+ def seed_demo_data(self) -> None:
148
+ """Seeds realistic historical fix records to establish baseline metrics."""
149
+ demo_cases = [
150
+ ("BUG-PLW-101", "playwright", 850, 220, 28.5, 0),
151
+ ("BUG-PLW-102", "playwright", 1100, 310, 42.0, 1),
152
+ ("BUG-PLW-103", "playwright", 780, 190, 21.0, 0),
153
+ ("BUG-KAT-201", "katalon", 920, 260, 35.0, 0),
154
+ ("BUG-KAT-202", "katalon", 1400, 410, 54.0, 2),
155
+ ("BUG-PYT-301", "pytest", 650, 180, 18.0, 0),
156
+ ("BUG-PYT-302", "pytest", 890, 240, 29.0, 1),
157
+ ("BUG-NWM-401", "newman", 550, 140, 14.5, 0),
158
+ ]
159
+ for bid, st, pt, ct, dur, ret in demo_cases:
160
+ self.record_fix(bid, st, pt, ct, dur, ret, provider="demo-seed")
161
+
162
+
163
+ def main() -> int:
164
+ parser = argparse.ArgumentParser(description="Live LLM Token & MTTR Telemetry Engine")
165
+ parser.add_argument("--report", action="store_true", help="Display telemetry summary report")
166
+ parser.add_argument("--json", action="store_true", help="Output summary in JSON format")
167
+ parser.add_argument("--record-fix", action="store_true", help="Record a verified bug fix telemetry entry")
168
+ parser.add_argument("--bug-id", type=str, default="BUG-AUTO", help="Bug or issue identifier")
169
+ parser.add_argument("--stack", type=str, default="playwright", help="Test stack (playwright, katalon, pytest, newman)")
170
+ parser.add_argument("--prompt-tokens", type=int, default=800, help="Input prompt tokens consumed")
171
+ parser.add_argument("--completion-tokens", type=int, default=250, help="Output completion tokens generated")
172
+ parser.add_argument("--duration", type=float, default=30.0, help="MTTR duration in seconds")
173
+ parser.add_argument("--retries", type=int, default=0, help="Number of agent retries before verified fix")
174
+ parser.add_argument("--seed-demo", action="store_true", help="Seed representative demo incident fixes")
175
+ args = parser.parse_args()
176
+
177
+ store = TokenTelemetryStore()
178
+
179
+ if args.seed_demo:
180
+ store.seed_demo_data()
181
+ print("✅ Seeded representative incident telemetry records into .synapse/telemetry.db.")
182
+ return 0
183
+
184
+ if args.record_fix:
185
+ row_id = store.record_fix(
186
+ bug_id=args.bug_id,
187
+ stack=args.stack,
188
+ prompt_tokens=args.prompt_tokens,
189
+ completion_tokens=args.completion_tokens,
190
+ mttr_seconds=args.duration,
191
+ retry_count=args.retries
192
+ )
193
+ print(f"✅ Recorded fix telemetry for {args.bug_id} (ID: {row_id}, Stack: {args.stack}, MTTR: {args.duration}s, Tokens: {args.prompt_tokens + args.completion_tokens}).")
194
+ return 0
195
+
196
+ # Default to report
197
+ summary_all = store.compute_summary()
198
+ stacks = ["playwright", "katalon", "pytest", "newman"]
199
+ stack_summaries = {st: store.compute_summary(st) for st in stacks}
200
+
201
+ if args.json:
202
+ payload = {"overall": summary_all, "by_stack": stack_summaries}
203
+ print(json.dumps(payload, indent=2))
204
+ return 0
205
+
206
+ print("======================================================================")
207
+ print(" SYNAPSE HUB — PRODUCTION TOKEN & MTTR TELEMETRY DASHBOARD ")
208
+ print("======================================================================")
209
+ print(f"Total Logged Incidents: {summary_all['count']}")
210
+ print(f"Overall MTTR: P50={summary_all['median_mttr_s']}s | P90={summary_all['p90_mttr_s']}s")
211
+ print(f"Tokens Per Bug: P50={summary_all['median_tokens']} | P90={summary_all['p90_tokens']}")
212
+ print(f"Avg Retries: {summary_all['avg_retries']}")
213
+ print("----------------------------------------------------------------------")
214
+ print(f"{'Stack':<14} | {'Incidents':<10} | {'P50 MTTR':<10} | {'P90 MTTR':<10} | {'P50 Tokens':<10}")
215
+ print("----------------------------------------------------------------------")
216
+ for st, s in stack_summaries.items():
217
+ if s["count"] > 0:
218
+ print(f"{st:<14} | {s['count']:<10} | {s['median_mttr_s']:<9}s | {s['p90_mttr_s']:<9}s | {s['median_tokens']:<10}")
219
+ print("======================================================================")
220
+ return 0
221
+
222
+
223
+ if __name__ == "__main__":
224
+ sys.exit(main())
@@ -236,7 +236,7 @@ class SynapseUiHandler(BaseHTTPRequestHandler):
236
236
  if path == "/api/status":
237
237
  active_target = JOB_MANAGER.target_dir or REPO_ROOT
238
238
  self.send_json(200, {
239
- "version": "4.0.0",
239
+ "version": "4.1.0",
240
240
  "status": "ONLINE",
241
241
  "os": sys.platform,
242
242
  "workspace": active_target,
@@ -65,7 +65,7 @@ def get_local_version():
65
65
  try:
66
66
  with open(pkg_file, "r", encoding="utf-8") as f:
67
67
  data = json.load(f)
68
- return data.get("version", "4.0.0")
68
+ return data.get("version", "4.1.0")
69
69
  except (OSError, json.JSONDecodeError):
70
70
  data = {}
71
71
 
@@ -74,11 +74,11 @@ def get_local_version():
74
74
  try:
75
75
  with open(manifest_file, "r", encoding="utf-8") as f:
76
76
  data = json.load(f)
77
- return data.get("version", "4.0.0")
77
+ return data.get("version", "4.1.0")
78
78
  except (OSError, json.JSONDecodeError):
79
79
  data = {}
80
80
 
81
- return "4.0.0"
81
+ return "4.1.0"
82
82
 
83
83
 
84
84
  def fetch_remote_version():
@@ -371,9 +371,9 @@ def gate_3_clean_pom(repo_root, mode="AUTO"):
371
371
  return True
372
372
 
373
373
  LOCATOR_CALL_PATTERN = re.compile(
374
- r'(?:\.(?:click|dblclick|fill|type|press|check|locator|waitForSelector|hover|selectOption)|cy\.(?:get|find)|driver\.find[A-Za-z]*|WebUI\.[a-zA-Z]+)\s*\(\s*[\x22\x27](?P<sel>[^\x22\x27]+)[\x22\x27]'
374
+ r'(?:\.(?:click|dblclick|fill|type|press|check|locator|waitForSelector|hover|selectOption)|cy\.(?:get|find)|driver\.find[A-Za-z]*|WebUI\.[a-zA-Z]+)\s*\(\s*[\x22\x27`](?P<sel>[^\x22\x27`]+)[\x22\x27`]'
375
375
  )
376
- RAW_XPATH_PATTERN = re.compile(r'[\x22\x27](?:xpath=|\/{2}[a-zA-Z*@\[])[^\x22\x27]*[\x22\x27]')
376
+ RAW_XPATH_PATTERN = re.compile(r'[\x22\x27`](?:xpath=|\/{2}[a-zA-Z*@\[])[^\x22\x27`]*[\x22\x27`]')
377
377
 
378
378
  violations = []
379
379
  for full_path in sorted(set(test_files)):
@@ -1084,16 +1084,34 @@ def gate_10_performance_sla(repo_root, mode="AUTO"):
1084
1084
  k6_candidates.extend(glob.glob(os.path.join(pdir, "*k6*.json")))
1085
1085
  k6_candidates.extend(glob.glob(os.path.join(pdir, "summary.json")))
1086
1086
 
1087
- if not jtl_candidates and not k6_candidates:
1088
- print("[PASS] Gate 10: Performance & SLA Sentinel (No active perf runs found in Reports/perf/ - SKIPPED/PASS).")
1089
- return True
1090
-
1091
1087
  cfg = load_synapse_config(repo_root)
1092
1088
  perf_cfg = cfg.get("performance", {}).get("sla", {})
1093
1089
  sla_p95 = perf_cfg.get("p95_ms", 500.0)
1094
1090
  sla_p99 = perf_cfg.get("p99_ms", 1000.0)
1095
1091
  max_error_rate = perf_cfg.get("max_error_rate_pct", 1.0)
1096
1092
 
1093
+ if not jtl_candidates and not k6_candidates:
1094
+ auto_bench = cfg.get("performance", {}).get("auto_benchmark_fallback", True)
1095
+ if auto_bench:
1096
+ print("[GATE 10] No existing perf run found. Triggering synthetic performance benchmark runner...")
1097
+ try:
1098
+ try:
1099
+ from tools.perf.benchmark_perf_runner import run_benchmark
1100
+ except ImportError:
1101
+ sys.path.insert(0, os.path.join(repo_root, "tools"))
1102
+ from perf.benchmark_perf_runner import run_benchmark
1103
+ out_dir = os.path.join(repo_root, "Reports", "perf")
1104
+ bench_res = run_benchmark(iterations=50, output_dir=out_dir)
1105
+ jtl_candidates.append(bench_res["jtl_file"])
1106
+ k6_candidates.append(bench_res["k6_file"])
1107
+ print(f"[GATE 10] Real benchmark generated: P95={bench_res['p95_ms']}ms, P99={bench_res['p99_ms']}ms, Error={bench_res['error_rate_pct']}%.")
1108
+ except Exception as exc:
1109
+ print(f"[FAIL] Gate 10 Performance Sentinel REJECTED: Failed to execute performance benchmark: {exc}")
1110
+ return False
1111
+ else:
1112
+ print("[FAIL] Gate 10 Performance Sentinel REJECTED: No active perf runs found in Reports/perf/ and auto_benchmark_fallback is false. (Fail-closed)")
1113
+ return False
1114
+
1097
1115
  try:
1098
1116
  try:
1099
1117
  from tools.jtl_analyzer import parse_jtl, evaluate_sla
@@ -47,7 +47,7 @@ def get_hub_metadata() -> Dict[str, Any]:
47
47
  """Retrieves version and catalog count from skills_manifest.json."""
48
48
  manifest_path = os.path.join(HUB_ROOT, "skills_manifest.json")
49
49
  meta = {
50
- "version": "4.0.0",
50
+ "version": "4.1.0",
51
51
  "totalSkills": 42,
52
52
  "author": "Thế Vị (@TeeVee98)",
53
53
  "repo": "https://github.com/TeeVee98/Synapse-Hub"