@teevee98/synapse-hub 4.1.4 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/.agents/rules/git-push-protocol.md +2 -2
  2. package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +6 -1
  3. package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +6 -1
  4. package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +8 -3
  5. package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +6 -1
  6. package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +6 -1
  7. package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +6 -1
  8. package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +6 -1
  9. package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
  10. package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
  11. package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
  12. package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +7 -2
  13. package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +6 -1
  14. package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +6 -1
  15. package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +6 -1
  16. package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +6 -1
  17. package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +6 -1
  18. package/.agents/skills/02-test-design/review-spec/SKILL.md +4 -1
  19. package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +6 -1
  20. package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +6 -1
  21. package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +6 -1
  22. package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +7 -2
  23. package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +7 -2
  24. package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +4 -1
  25. package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +6 -1
  26. package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +6 -1
  27. package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +6 -1
  28. package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +6 -1
  29. package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +6 -1
  30. package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +6 -1
  31. package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +7 -2
  32. package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +6 -1
  33. package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +6 -1
  34. package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +6 -1
  35. package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +6 -1
  36. package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +6 -1
  37. package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +6 -1
  38. package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +6 -1
  39. package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +6 -1
  40. package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +6 -1
  41. package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +6 -1
  42. package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +6 -1
  43. package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +6 -1
  44. package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +4 -1
  45. package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +4 -1
  46. package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +6 -1
  47. package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +4 -1
  48. package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +6 -1
  49. package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +6 -1
  50. package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +6 -1
  51. package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +6 -1
  52. package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +6 -1
  53. package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +6 -1
  54. package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +6 -1
  55. package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +6 -1
  56. package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +6 -1
  57. package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +6 -1
  58. package/AGENTS.md +7 -1
  59. package/LICENSE +5 -5
  60. package/NOTICE +1 -1
  61. package/README.en.md +4 -4
  62. package/README.md +4 -4
  63. package/agentskills.json +1 -1
  64. package/docs/ARCHITECTURE.en.md +2 -2
  65. package/docs/ARCHITECTURE.md +11 -2
  66. package/docs/CLI_REFERENCE.en.md +6 -4
  67. package/docs/CLI_REFERENCE.md +1 -1
  68. package/docs/CONTRIBUTING.en.md +1 -1
  69. package/docs/CONTRIBUTING.md +2 -2
  70. package/docs/TOOL_CATALOG.md +6 -2
  71. package/docs/research/agent_skill_training_analysis.md +1 -1
  72. package/package.json +1 -1
  73. package/plugin.json +1 -1
  74. package/skills_manifest.json +320 -55
  75. package/synapse.config.json +1 -1
  76. package/templates/ui/index.html +2 -2
  77. package/tools/TOOL_CATALOG.json +63 -3
  78. package/tools/__init__.py +1 -1
  79. package/tools/api_fuzzer.py +179 -118
  80. package/tools/author_integrity.py +121 -11
  81. package/tools/benchmark_swarm_context.py +119 -18
  82. package/tools/break_ui_generator.py +233 -0
  83. package/tools/browser_helpers.js +2 -2
  84. package/tools/check_version_sync.py +18 -50
  85. package/tools/cli.py +109 -18
  86. package/tools/core/bm25_retriever.py +218 -0
  87. package/tools/core/i18n.py +4 -4
  88. package/tools/core/provider_usage_adapter.py +317 -0
  89. package/tools/core/token_estimator.py +42 -14
  90. package/tools/count_tokens.py +5 -0
  91. package/tools/coverage_reporter.py +245 -0
  92. package/tools/dashboard_generator.py +6 -6
  93. package/tools/db_sandbox.py +146 -4
  94. package/tools/distill_skill.py +191 -0
  95. package/tools/git_pre_commit_guard.py +12 -0
  96. package/tools/humanize_linter.py +3 -0
  97. package/tools/mcp_docx_server.py +1 -1
  98. package/tools/mcp_excel_server.py +1 -1
  99. package/tools/mcp_server.py +75 -15
  100. package/tools/micro_mutator.py +84 -14
  101. package/tools/pentest_scope_guard.py +399 -56
  102. package/tools/perf/benchmark_perf_runner.py +42 -0
  103. package/tools/rules_router.py +401 -41
  104. package/tools/security_scanner.py +80 -0
  105. package/tools/senior_code_review.py +14 -0
  106. package/tools/skill_canonical_digest.py +244 -0
  107. package/tools/skill_profiles.py +77 -7
  108. package/tools/slide_deck_exporter.py +477 -0
  109. package/tools/sync_skills_manifest.py +166 -0
  110. package/tools/task_memory_manager.py +260 -0
  111. package/tools/ui_server.py +1 -1
  112. package/tools/update_sentinel.py +3 -3
  113. package/tools/validate_host_interop.py +485 -0
  114. package/tools/validate_skills_schema.py +282 -0
  115. package/tools/verify_all.py +572 -42
  116. package/tools/welcome_page.py +2 -2
@@ -0,0 +1,317 @@
1
+ #!/usr/bin/env python3
2
+ # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
3
+ # SPDX-License-Identifier: MIT
4
+ # Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
5
+
6
+ """
7
+ tools/core/provider_usage_adapter.py — Standardized Provider Usage Ingestion & Context Splitter
8
+ ------------------------------------------------------------------------------------------------
9
+ Provides strict boundary enforcement between:
10
+ 1. Real LLM provider-reported tokens (OpenAI, Anthropic, Gemini, Antigravity Transcripts).
11
+ 2. Static documentation / rule estimates (cl100k_base / heuristic polyglot).
12
+ 3. Synthetic micro-workload benchmark tokens.
13
+ 4. Avoided filesystem read bytes (I/O footprint, strictly non-token).
14
+
15
+ Invariants:
16
+ - Zero Guesswork / Zero Interpolation: When live provider usage response is not present,
17
+ provider fields MUST remain None (null).
18
+ - Explicit Provenance: Every record declares measurement_kind, tokenizer/model, version, commit,
19
+ and timestamp.
20
+ - Context Decomposition: Statically estimated skill payloads are subtracted from total provider
21
+ input tokens to accurately isolate the external system prompt, conversation history, and tool schemas.
22
+ - Paired Run Verification: Live reduction claims are strictly guarded and rejected unless BOTH
23
+ baseline and optimized runs originate from real provider-observed responses.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ from dataclasses import asdict, dataclass
29
+ from datetime import datetime, timezone
30
+ import json
31
+ import logging
32
+ import os
33
+ import subprocess
34
+ from typing import Any, Dict, Optional, Tuple
35
+
36
+ _logger = logging.getLogger("provider_usage_adapter")
37
+ _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
38
+
39
+
40
+ def get_repo_version_and_commit(repo_root: Optional[str] = None) -> Tuple[str, str]:
41
+ """Retrieves current hub version from skills_manifest.json and git commit hash."""
42
+ root = repo_root or _REPO_ROOT
43
+ version = "1.0.0"
44
+ manifest_path = os.path.join(root, "skills_manifest.json")
45
+ if os.path.isfile(manifest_path):
46
+ try:
47
+ with open(manifest_path, "r", encoding="utf-8") as f:
48
+ data = json.load(f)
49
+ version = data.get("version", "1.0.0")
50
+ except (OSError, json.JSONDecodeError, KeyError) as err:
51
+ _logger.debug("Failed to read version from manifest: %s", err)
52
+
53
+ commit = "unversioned"
54
+ try:
55
+ out = subprocess.check_output(
56
+ ["git", "rev-parse", "HEAD"],
57
+ cwd=root,
58
+ stderr=subprocess.DEVNULL,
59
+ )
60
+ commit = out.decode("utf-8").strip()
61
+ except (OSError, subprocess.SubprocessError) as err:
62
+ _logger.debug("Failed to get git commit: %s", err)
63
+ commit = "unversioned"
64
+
65
+ return version, commit
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class ProviderUsageRecord:
70
+ """Immutable, audited record of actual live LLM provider usage."""
71
+ provider: str
72
+ model: str
73
+ measurement_kind: str # Always "provider_observed"
74
+ provider_reported_input_tokens: int
75
+ provider_reported_output_tokens: int
76
+ provider_cached_tokens: Optional[int]
77
+ total_provider_tokens: int
78
+ timestamp: str
79
+ version: str
80
+ commit: str
81
+ raw_usage: Dict[str, Any]
82
+
83
+ def to_dict(self) -> Dict[str, Any]:
84
+ """Converts to dictionary representation."""
85
+ return asdict(self)
86
+
87
+
88
+ def create_unobserved_provider_record(
89
+ measurement_kind: str = "static_estimate",
90
+ note: str = "No live LLM provider usage observed; fields remain null.",
91
+ ) -> Dict[str, Any]:
92
+ """
93
+ Returns a standardized dictionary for records where live provider data was not captured.
94
+ Guarantees that provider fields remain strictly None without interpolation.
95
+ """
96
+ version, commit = get_repo_version_and_commit()
97
+ return {
98
+ "measurement_kind": measurement_kind,
99
+ "provider": None,
100
+ "model": None,
101
+ "provider_observed": False,
102
+ "provider_reported_input_tokens": None,
103
+ "provider_reported_output_tokens": None,
104
+ "provider_cached_tokens": None,
105
+ "total_provider_tokens": None,
106
+ "timestamp": datetime.now(timezone.utc).isoformat(),
107
+ "version": version,
108
+ "commit": commit,
109
+ "note": note,
110
+ }
111
+
112
+
113
+ def parse_provider_usage(
114
+ response: Dict[str, Any],
115
+ provider_hint: Optional[str] = None,
116
+ model_hint: Optional[str] = None,
117
+ repo_root: Optional[str] = None,
118
+ ) -> ProviderUsageRecord:
119
+ """
120
+ Parses live provider usage from standard LLM response payloads.
121
+ Supported payloads:
122
+ 1. OpenAI / Compatible: {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "prompt_tokens_details": {"cached_tokens": ...}}}
123
+ 2. Anthropic: {"usage": {"input_tokens": ..., "output_tokens": ..., "cache_read_input_tokens": ...}}
124
+ 3. Google GenAI: {"usage_metadata": {"prompt_token_count": ..., "candidates_token_count": ..., "cached_content_token_count": ...}}
125
+ 4. Antigravity transcript step: {"input_tokens": ..., "output_tokens": ..., "cache_read_tokens": ...}
126
+ 5. Mock / Test payload: {"provider": ..., "model": ..., "usage": {"input_tokens": ..., ...}}
127
+
128
+ Raises:
129
+ ValueError: When no valid provider usage payload is present (refuses to guess).
130
+ """
131
+ if not isinstance(response, dict):
132
+ raise ValueError(f"Expected dict response payload, got {type(response).__name__}")
133
+
134
+ version, commit = get_repo_version_and_commit(repo_root)
135
+ now_iso = datetime.now(timezone.utc).isoformat()
136
+
137
+ # 1. Google GenAI (Gemini SDK) format: usage_metadata
138
+ if "usage_metadata" in response and isinstance(response["usage_metadata"], dict):
139
+ um = response["usage_metadata"]
140
+ in_tok = um.get("prompt_token_count")
141
+ out_tok = um.get("candidates_token_count", 0)
142
+ cache_tok = um.get("cached_content_token_count")
143
+ model = response.get("model") or model_hint or "gemini"
144
+ provider = provider_hint or "google"
145
+ if in_tok is not None:
146
+ total = in_tok + out_tok
147
+ return ProviderUsageRecord(
148
+ provider=provider,
149
+ model=model,
150
+ measurement_kind="provider_observed",
151
+ provider_reported_input_tokens=int(in_tok),
152
+ provider_reported_output_tokens=int(out_tok),
153
+ provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
154
+ total_provider_tokens=total,
155
+ timestamp=now_iso,
156
+ version=version,
157
+ commit=commit,
158
+ raw_usage=um,
159
+ )
160
+
161
+ # 2. OpenAI / Anthropic format: usage key
162
+ if "usage" in response and isinstance(response["usage"], dict):
163
+ u = response["usage"]
164
+ model = response.get("model") or model_hint or "unknown_model"
165
+
166
+ # OpenAI style: prompt_tokens, completion_tokens
167
+ if "prompt_tokens" in u:
168
+ in_tok = u["prompt_tokens"]
169
+ out_tok = u.get("completion_tokens", 0)
170
+ cache_tok = None
171
+ if "prompt_tokens_details" in u and isinstance(u["prompt_tokens_details"], dict):
172
+ cache_tok = u["prompt_tokens_details"].get("cached_tokens")
173
+ provider = provider_hint or ("openai" if "gpt" in model.lower() else "llm_provider")
174
+ total = u.get("total_tokens", in_tok + out_tok)
175
+ return ProviderUsageRecord(
176
+ provider=provider,
177
+ model=model,
178
+ measurement_kind="provider_observed",
179
+ provider_reported_input_tokens=int(in_tok),
180
+ provider_reported_output_tokens=int(out_tok),
181
+ provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
182
+ total_provider_tokens=int(total),
183
+ timestamp=now_iso,
184
+ version=version,
185
+ commit=commit,
186
+ raw_usage=u,
187
+ )
188
+
189
+ # Anthropic style: input_tokens, output_tokens
190
+ if "input_tokens" in u:
191
+ in_tok = u["input_tokens"]
192
+ out_tok = u.get("output_tokens", 0)
193
+ cache_tok = u.get("cache_read_input_tokens")
194
+ provider = provider_hint or ("anthropic" if "claude" in model.lower() else "llm_provider")
195
+ total = in_tok + out_tok
196
+ return ProviderUsageRecord(
197
+ provider=provider,
198
+ model=model,
199
+ measurement_kind="provider_observed",
200
+ provider_reported_input_tokens=int(in_tok),
201
+ provider_reported_output_tokens=int(out_tok),
202
+ provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
203
+ total_provider_tokens=total,
204
+ timestamp=now_iso,
205
+ version=version,
206
+ commit=commit,
207
+ raw_usage=u,
208
+ )
209
+
210
+ # 3. Direct / Antigravity transcript step format: input_tokens, output_tokens directly in root
211
+ if "input_tokens" in response:
212
+ in_tok = response["input_tokens"]
213
+ out_tok = response.get("output_tokens", 0)
214
+ cache_tok = response.get("cache_read_tokens")
215
+ provider = response.get("provider") or provider_hint or "antigravity"
216
+ model = response.get("model") or model_hint or "gemini"
217
+ total = in_tok + out_tok + (cache_tok or 0)
218
+ return ProviderUsageRecord(
219
+ provider=provider,
220
+ model=model,
221
+ measurement_kind="provider_observed",
222
+ provider_reported_input_tokens=int(in_tok),
223
+ provider_reported_output_tokens=int(out_tok),
224
+ provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
225
+ total_provider_tokens=int(total),
226
+ timestamp=now_iso,
227
+ version=version,
228
+ commit=commit,
229
+ raw_usage={k: response[k] for k in ("input_tokens", "output_tokens", "cache_read_tokens") if k in response},
230
+ )
231
+
232
+ raise ValueError(
233
+ "No valid LLM provider usage metadata found in response. "
234
+ "Synapse Hub refuses to fabricate synthetic numbers as live provider data."
235
+ )
236
+
237
+
238
+ def decompose_prompt_context(
239
+ provider_record: ProviderUsageRecord,
240
+ skill_payload_tokens: int,
241
+ rule_payload_tokens: int = 0,
242
+ ) -> Dict[str, Any]:
243
+ """
244
+ Separates the statically estimated skill/rule documentation payload from the
245
+ overall provider input tokens to isolate external host system instructions,
246
+ conversation history turns, and provider tool calling schemas.
247
+ """
248
+ if not isinstance(provider_record, ProviderUsageRecord):
249
+ raise TypeError(f"Expected ProviderUsageRecord, got {type(provider_record).__name__}")
250
+
251
+ total_in = provider_record.provider_reported_input_tokens
252
+ managed_tokens = skill_payload_tokens + rule_payload_tokens
253
+ external_wrapper = max(0, total_in - managed_tokens)
254
+ ratio = round((managed_tokens / max(1, total_in)) * 100.0, 2)
255
+
256
+ return {
257
+ "measurement_kind": "provider_observed_with_decomposition",
258
+ "provider": provider_record.provider,
259
+ "model": provider_record.model,
260
+ "provider_reported_input_tokens": total_in,
261
+ "provider_reported_output_tokens": provider_record.provider_reported_output_tokens,
262
+ "provider_cached_tokens": provider_record.provider_cached_tokens,
263
+ "skill_payload_static_estimated_tokens": skill_payload_tokens,
264
+ "rule_payload_static_estimated_tokens": rule_payload_tokens,
265
+ "total_managed_static_tokens": managed_tokens,
266
+ "external_system_prompt_and_wrapper_tokens": external_wrapper,
267
+ "managed_payload_ratio_pct": ratio,
268
+ "timestamp": provider_record.timestamp,
269
+ "version": provider_record.version,
270
+ "commit": provider_record.commit,
271
+ "disclaimer": (
272
+ "Provider reported input tokens reflect total live API billing usage. "
273
+ "Skill and rule payloads are statically estimated via offline tokenizer and subtracted "
274
+ "to isolate unmanaged host system prompt, user prompt, and provider tool schemas."
275
+ ),
276
+ }
277
+
278
+
279
+ def compare_paired_provider_runs(
280
+ baseline_run: ProviderUsageRecord,
281
+ optimized_run: ProviderUsageRecord,
282
+ ) -> Dict[str, Any]:
283
+ """
284
+ Computes verified token reduction between two live provider runs.
285
+ Enforces scientific honesty: strictly refuses to claim live provider reduction unless
286
+ BOTH baseline and optimized runs originate from real provider_observed responses.
287
+ """
288
+ if not isinstance(baseline_run, ProviderUsageRecord) or not isinstance(optimized_run, ProviderUsageRecord):
289
+ raise TypeError("Both baseline_run and optimized_run must be verified ProviderUsageRecord instances.")
290
+
291
+ if baseline_run.measurement_kind != "provider_observed" or optimized_run.measurement_kind != "provider_observed":
292
+ raise ValueError("Both runs must have measurement_kind == 'provider_observed' to claim live provider reduction.")
293
+
294
+ in_saved = baseline_run.provider_reported_input_tokens - optimized_run.provider_reported_input_tokens
295
+ out_saved = baseline_run.provider_reported_output_tokens - optimized_run.provider_reported_output_tokens
296
+ total_saved = baseline_run.total_provider_tokens - optimized_run.total_provider_tokens
297
+ reduction_pct = round((total_saved / max(1, baseline_run.total_provider_tokens)) * 100.0, 2)
298
+
299
+ return {
300
+ "verified_provider_savings": True,
301
+ "measurement_kind": "paired_provider_observed",
302
+ "baseline_provider": baseline_run.provider,
303
+ "baseline_model": baseline_run.model,
304
+ "baseline_input_tokens": baseline_run.provider_reported_input_tokens,
305
+ "baseline_output_tokens": baseline_run.provider_reported_output_tokens,
306
+ "baseline_total_tokens": baseline_run.total_provider_tokens,
307
+ "optimized_provider": optimized_run.provider,
308
+ "optimized_model": optimized_run.model,
309
+ "optimized_input_tokens": optimized_run.provider_reported_input_tokens,
310
+ "optimized_output_tokens": optimized_run.provider_reported_output_tokens,
311
+ "optimized_total_tokens": optimized_run.total_provider_tokens,
312
+ "input_tokens_saved": in_saved,
313
+ "output_tokens_saved": out_saved,
314
+ "net_tokens_saved": total_saved,
315
+ "reduction_pct": reduction_pct,
316
+ "timestamp": datetime.now(timezone.utc).isoformat(),
317
+ }
@@ -22,6 +22,7 @@ Key Capabilities:
22
22
  from __future__ import annotations
23
23
 
24
24
  import glob
25
+ import hashlib
25
26
  import json
26
27
  import os
27
28
  import re
@@ -119,8 +120,8 @@ def _get_skills_manifest(repo_root: str) -> Dict[str, Dict[str, Any]]:
119
120
  """
120
121
  Loads and caches skills_manifest.json mapping skill_id to metadata.
121
122
  Cache is keyed by the canonical realpath of the manifest file and validated
122
- against file modification time (mtime) to support multi-repo isolation,
123
- runtime manifest mutations, and deletions.
123
+ against file mtime_ns, size, and content SHA-256 digest to guarantee that
124
+ mutations are detected even if file modification time remains identical.
124
125
  """
125
126
  global _SKILLS_MANIFEST_CACHE
126
127
  manifest_path = os.path.realpath(os.path.join(repo_root, "skills_manifest.json"))
@@ -130,26 +131,41 @@ def _get_skills_manifest(repo_root: str) -> Dict[str, Dict[str, Any]]:
130
131
  return {}
131
132
 
132
133
  try:
133
- current_mtime = os.path.getmtime(manifest_path)
134
+ st = os.stat(manifest_path)
135
+ current_mtime_ns = st.st_mtime_ns
136
+ current_size = st.st_size
137
+ except OSError:
138
+ _SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
139
+ return {}
140
+
141
+ try:
142
+ with open(manifest_path, "rb") as f:
143
+ raw_bytes = f.read()
144
+ current_digest = hashlib.sha256(raw_bytes).digest()
134
145
  except OSError:
135
146
  _SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
136
147
  return {}
137
148
 
138
149
  if manifest_path in _SKILLS_MANIFEST_CACHE:
139
- cached_mtime, cached_data = _SKILLS_MANIFEST_CACHE[manifest_path]
140
- if cached_mtime == current_mtime:
141
- return cached_data
150
+ cached_entry = _SKILLS_MANIFEST_CACHE[manifest_path]
151
+ if len(cached_entry) == 4:
152
+ cached_mtime_ns, cached_size, cached_digest, cached_data = cached_entry
153
+ if cached_digest == current_digest:
154
+ return cached_data
155
+ elif len(cached_entry) == 2:
156
+ cached_mtime, cached_data = cached_entry
157
+ if cached_mtime == st.st_mtime:
158
+ return cached_data
142
159
 
143
160
  manifest_data: Dict[str, Dict[str, Any]] = {}
144
161
  try:
145
- with open(manifest_path, "r", encoding="utf-8") as f:
146
- data = json.load(f)
147
- for s in data.get("skills", []):
148
- sid = s.get("id")
149
- if sid:
150
- manifest_data[sid] = s
151
- _SKILLS_MANIFEST_CACHE[manifest_path] = (current_mtime, manifest_data)
152
- except (OSError, json.JSONDecodeError) as err:
162
+ data = json.loads(raw_bytes.decode("utf-8"))
163
+ for s in data.get("skills", []):
164
+ sid = s.get("id")
165
+ if sid:
166
+ manifest_data[sid] = s
167
+ _SKILLS_MANIFEST_CACHE[manifest_path] = (current_mtime_ns, current_size, current_digest, manifest_data)
168
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as err:
153
169
  import sys
154
170
  sys.stderr.write(f"Warning: Failed to load manifest at {manifest_path}: {err}\n")
155
171
  _SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
@@ -216,6 +232,12 @@ def calculate_hydration_payload(
216
232
  catalog_skill_tokens = count_tokens("\n".join(catalog_lines)) if catalog_lines else 0
217
233
 
218
234
  return {
235
+ "measurement_kind": "static_estimate",
236
+ "static_estimated_tokens": rule_tokens + full_skill_tokens,
237
+ "provider_reported_input_tokens": None,
238
+ "provider_reported_output_tokens": None,
239
+ "provider_cached_tokens": None,
240
+ "avoided_disk_bytes": 0,
219
241
  "rule_tokens": rule_tokens,
220
242
  "rule_bytes": rule_bytes,
221
243
  "skill_catalog_tokens": catalog_skill_tokens,
@@ -270,6 +292,12 @@ def get_full_unrouted_baseline(repo_root: str) -> Dict[str, Any]:
270
292
  total_cat_tokens = count_tokens("\n".join(catalog_lines)) if catalog_lines else 0
271
293
 
272
294
  return {
295
+ "measurement_kind": "static_estimate",
296
+ "static_estimated_tokens": total_rule_tokens + total_skill_tokens,
297
+ "provider_reported_input_tokens": None,
298
+ "provider_reported_output_tokens": None,
299
+ "provider_cached_tokens": None,
300
+ "avoided_disk_bytes": 0,
273
301
  "rule_count": len(all_rule_files),
274
302
  "rule_tokens": total_rule_tokens,
275
303
  "rule_bytes": total_rule_bytes,
@@ -775,6 +775,10 @@ def calculate_hub_savings(input_tokens: int, cache_tokens: int, output_tokens: i
775
775
 
776
776
  return {
777
777
  "model_name": pricing["name"],
778
+ "measurement_kind": "counterfactual_model_estimate",
779
+ "provider_reported_input_tokens": input_tokens,
780
+ "provider_reported_output_tokens": output_tokens,
781
+ "provider_cached_tokens": cache_tokens,
778
782
  "non_hub_input": estimated_non_hub_input,
779
783
  "non_hub_cache": estimated_non_hub_cache,
780
784
  "non_hub_total": estimated_non_hub_total,
@@ -788,6 +792,7 @@ def calculate_hub_savings(input_tokens: int, cache_tokens: int, output_tokens: i
788
792
  "savings_pct": round(savings_pct, 1),
789
793
  "cost_saved_usd": round(cost_saved_usd, 4),
790
794
  "cost_savings_pct": round(cost_savings_pct, 1),
795
+ "note": "Actual hub tokens are parsed from session transcripts. Non-hub baseline is an estimated counterfactual.",
791
796
  }
792
797
 
793
798
 
@@ -0,0 +1,245 @@
1
+ #!/usr/bin/env python3
2
+ # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
3
+ # SPDX-License-Identifier: MIT
4
+ # Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
5
+
6
+ """
7
+ tools/coverage_reporter.py — Standardized Coverage Metrics & Roadmap Evaluator
8
+ -------------------------------------------------------------------------------
9
+ Evaluates and distinguishes:
10
+ 1. Statement / Line Coverage (fail_under: 60.00% calibrated floor, roadmap: 70% -> 80%).
11
+ 2. Branch Coverage (floor: 50.00% baseline floor, roadmap: 60% -> 70%).
12
+ 3. Combined Coverage (statements + branches hit / total).
13
+
14
+ Maintains strict separation between line coverage, branch coverage, static heuristic,
15
+ and dynamic mutation kill scores.
16
+
17
+ Outputs:
18
+ - reports/coverage_summary.json (Standardized multi-metric artifact)
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import argparse
24
+ import json
25
+ import logging
26
+ import os
27
+ import subprocess
28
+ import sys
29
+ from datetime import datetime, timezone
30
+ from typing import Any, Dict
31
+
32
+ _logger = logging.getLogger("coverage_reporter")
33
+ REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
34
+ DEFAULT_COVERAGE_JSON = os.path.join(REPO_ROOT, "reports", "coverage.json")
35
+ DEFAULT_COVERAGE_XML = os.path.join(REPO_ROOT, "reports", "coverage.xml")
36
+ DEFAULT_SUMMARY_JSON = os.path.join(REPO_ROOT, "reports", "coverage_summary.json")
37
+
38
+ # Calibrated floors and roadmap milestones
39
+ LINE_COVERAGE_FLOOR_PCT = 60.00
40
+ BRANCH_COVERAGE_FLOOR_PCT = 50.00
41
+
42
+ ROADMAP_THRESHOLDS = {
43
+ "current_calibrated": {"line_pct": 60.00, "branch_pct": 50.00},
44
+ "phase_2_consolidation": {"line_pct": 70.00, "branch_pct": 60.00},
45
+ "target_enterprise": {"line_pct": 80.00, "branch_pct": 70.00},
46
+ }
47
+
48
+
49
+ def get_git_commit(repo_root: str = REPO_ROOT) -> str:
50
+ try:
51
+ out = subprocess.check_output(
52
+ ["git", "rev-parse", "HEAD"],
53
+ cwd=repo_root,
54
+ stderr=subprocess.DEVNULL,
55
+ )
56
+ return out.decode("utf-8").strip()
57
+ except (OSError, subprocess.SubprocessError) as err:
58
+ _logger.debug("Could not determine git commit: %s", err)
59
+ return "unversioned"
60
+
61
+
62
+ def parse_coverage_metrics(
63
+ json_path: str = DEFAULT_COVERAGE_JSON,
64
+ ) -> Dict[str, Any]:
65
+ """Parses coverage metrics from coverage.py json report."""
66
+ if not os.path.exists(json_path):
67
+ raise FileNotFoundError(f"Coverage JSON report not found: {json_path}")
68
+
69
+ with open(json_path, "r", encoding="utf-8") as f:
70
+ data = json.load(f)
71
+
72
+ totals = data.get("totals", {})
73
+ num_statements = totals.get("num_statements", 0)
74
+ covered_lines = totals.get("covered_lines", 0)
75
+ missing_lines = totals.get("missing_lines", 0)
76
+
77
+ # Line / Statement coverage
78
+ line_cov_pct = float(
79
+ totals.get(
80
+ "percent_statements_covered",
81
+ (covered_lines / num_statements * 100.0) if num_statements > 0 else 0.0,
82
+ )
83
+ )
84
+
85
+ # Branch coverage
86
+ num_branches = totals.get("num_branches", 0)
87
+ covered_branches = totals.get("covered_branches", 0)
88
+ missing_branches = totals.get("missing_branches", 0)
89
+ branch_cov_pct = float(
90
+ totals.get(
91
+ "percent_branches_covered",
92
+ (covered_branches / num_branches * 100.0) if num_branches > 0 else 0.0,
93
+ )
94
+ )
95
+
96
+ # Combined coverage
97
+ combined_cov_pct = float(totals.get("percent_covered", 0.0))
98
+
99
+ return {
100
+ "num_statements": num_statements,
101
+ "covered_lines": covered_lines,
102
+ "missing_lines": missing_lines,
103
+ "line_coverage_pct": round(line_cov_pct, 2),
104
+ "num_branches": num_branches,
105
+ "covered_branches": covered_branches,
106
+ "missing_branches": missing_branches,
107
+ "branch_coverage_pct": round(branch_cov_pct, 2),
108
+ "combined_coverage_pct": round(combined_cov_pct, 2),
109
+ }
110
+
111
+
112
+ def evaluate_coverage(
113
+ coverage_data: Dict[str, Any],
114
+ line_floor: float = LINE_COVERAGE_FLOOR_PCT,
115
+ branch_floor: float = BRANCH_COVERAGE_FLOOR_PCT,
116
+ enforce_branch: bool = True,
117
+ ) -> Dict[str, Any]:
118
+ """Evaluates metrics against calibrated floor and roadmap milestones."""
119
+ line_pct = coverage_data["line_coverage_pct"]
120
+ branch_pct = coverage_data["branch_coverage_pct"]
121
+
122
+ line_pass = line_pct >= line_floor
123
+ branch_pass = branch_pct >= branch_floor if enforce_branch and coverage_data["num_branches"] > 0 else True
124
+ all_pass = line_pass and branch_pass
125
+
126
+ return {
127
+ "line_pass": line_pass,
128
+ "branch_pass": branch_pass,
129
+ "all_pass": all_pass,
130
+ "line_floor": line_floor,
131
+ "branch_floor": branch_floor,
132
+ "enforce_branch": enforce_branch,
133
+ }
134
+
135
+
136
+ def generate_coverage_summary(
137
+ coverage_json: str = DEFAULT_COVERAGE_JSON,
138
+ coverage_xml: str = DEFAULT_COVERAGE_XML,
139
+ output_summary_json: str = DEFAULT_SUMMARY_JSON,
140
+ line_floor: float = LINE_COVERAGE_FLOOR_PCT,
141
+ branch_floor: float = BRANCH_COVERAGE_FLOOR_PCT,
142
+ enforce_branch: bool = True,
143
+ repo_root: str = REPO_ROOT,
144
+ ) -> Dict[str, Any]:
145
+ """Generates standardized coverage summary artifact and writes to disk."""
146
+ metrics = parse_coverage_metrics(coverage_json)
147
+ eval_res = evaluate_coverage(
148
+ metrics,
149
+ line_floor=line_floor,
150
+ branch_floor=branch_floor,
151
+ enforce_branch=enforce_branch,
152
+ )
153
+
154
+ commit_sha = get_git_commit(repo_root)
155
+ now_iso = datetime.now(timezone.utc).isoformat()
156
+
157
+ summary_doc = {
158
+ "generated_at": now_iso,
159
+ "commit": commit_sha,
160
+ "metrics": {
161
+ "line_coverage_pct": metrics["line_coverage_pct"],
162
+ "branch_coverage_pct": metrics["branch_coverage_pct"],
163
+ "combined_coverage_pct": metrics["combined_coverage_pct"],
164
+ "statements_total": metrics["num_statements"],
165
+ "statements_covered": metrics["covered_lines"],
166
+ "statements_missing": metrics["missing_lines"],
167
+ "branches_total": metrics["num_branches"],
168
+ "branches_covered": metrics["covered_branches"],
169
+ "branches_missing": metrics["missing_branches"],
170
+ },
171
+ "thresholds": {
172
+ "line_floor_pct": line_floor,
173
+ "branch_floor_pct": branch_floor,
174
+ "roadmap": ROADMAP_THRESHOLDS,
175
+ },
176
+ "evaluation": {
177
+ "line_passed": eval_res["line_pass"],
178
+ "branch_passed": eval_res["branch_pass"],
179
+ "all_passed": eval_res["all_pass"],
180
+ },
181
+ "artifacts": {
182
+ "coverage_json": os.path.relpath(coverage_json, repo_root) if os.path.exists(coverage_json) else None,
183
+ "coverage_xml": os.path.relpath(coverage_xml, repo_root) if os.path.exists(coverage_xml) else None,
184
+ "summary_json": os.path.relpath(output_summary_json, repo_root),
185
+ },
186
+ }
187
+
188
+ out_dir = os.path.dirname(output_summary_json)
189
+ if out_dir:
190
+ os.makedirs(out_dir, exist_ok=True)
191
+ with open(output_summary_json, "w", encoding="utf-8") as f:
192
+ json.dump(summary_doc, f, indent=2)
193
+
194
+ return summary_doc
195
+
196
+
197
+ def main() -> int:
198
+ parser = argparse.ArgumentParser(description="Synapse Hub Standardized Coverage Reporter")
199
+ parser.add_argument("--json-report", default=DEFAULT_COVERAGE_JSON, help="Path to coverage.json")
200
+ parser.add_argument("--xml-report", default=DEFAULT_COVERAGE_XML, help="Path to coverage.xml")
201
+ parser.add_argument("--summary-out", default=DEFAULT_SUMMARY_JSON, help="Path to write coverage_summary.json")
202
+ parser.add_argument("--line-floor", type=float, default=LINE_COVERAGE_FLOOR_PCT, help="Statement/line coverage floor")
203
+ parser.add_argument("--branch-floor", type=float, default=BRANCH_COVERAGE_FLOOR_PCT, help="Branch coverage floor")
204
+ parser.add_argument("--no-branch", action="store_true", help="Do not enforce branch coverage threshold")
205
+ parser.add_argument("--check", action="store_true", help="Fail with non-zero exit code if coverage is below threshold")
206
+ parser.add_argument("--json", action="store_true", help="Print summary JSON to stdout")
207
+ args = parser.parse_args()
208
+
209
+ if not os.path.exists(args.json_report):
210
+ sys.stderr.write(f"Error: Coverage JSON report not found at {args.json_report}.\n")
211
+ return 1
212
+
213
+ summary = generate_coverage_summary(
214
+ coverage_json=args.json_report,
215
+ coverage_xml=args.xml_report,
216
+ output_summary_json=args.summary_out,
217
+ line_floor=args.line_floor,
218
+ branch_floor=args.branch_floor,
219
+ enforce_branch=not args.no_branch,
220
+ )
221
+
222
+ if args.json:
223
+ print(json.dumps(summary, indent=2))
224
+ else:
225
+ m = summary["metrics"]
226
+ e = summary["evaluation"]
227
+ print("======================================================================")
228
+ print(" SYNAPSE HUB STANDARDIZED TEST COVERAGE REPORT ")
229
+ print("======================================================================")
230
+ print(f" Line Coverage (Statements) : {m['line_coverage_pct']}% (Floor: >= {args.line_floor}%) - {'✅ PASS' if e['line_passed'] else '❌ FAIL'}")
231
+ print(f" Branch Coverage : {m['branch_coverage_pct']}% (Floor: >= {args.branch_floor}%) - {'✅ PASS' if e['branch_passed'] else '❌ FAIL'}")
232
+ print(f" Combined (Stmt + Branch) : {m['combined_coverage_pct']}%")
233
+ print(f" Total Statements : {m['statements_total']:,} (Covered: {m['statements_covered']:,})")
234
+ print(f" Total Branches : {m['branches_total']:,} (Covered: {m['branches_covered']:,})")
235
+ print(f" Coverage Summary Artifact : {summary['artifacts']['summary_json']}")
236
+ print(f" Coverage XML Artifact : {summary['artifacts']['coverage_xml']}")
237
+ print("======================================================================")
238
+
239
+ if args.check and not summary["evaluation"]["all_passed"]:
240
+ return 1
241
+ return 0
242
+
243
+
244
+ if __name__ == "__main__":
245
+ sys.exit(main())