@teevee98/synapse-hub 4.1.4 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/git-push-protocol.md +2 -2
- package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +8 -3
- package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
- package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +7 -2
- package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +6 -1
- package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +6 -1
- package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +6 -1
- package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +6 -1
- package/.agents/skills/02-test-design/review-spec/SKILL.md +4 -1
- package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +4 -1
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +7 -2
- package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +4 -1
- package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +6 -1
- package/AGENTS.md +7 -1
- package/LICENSE +5 -5
- package/NOTICE +1 -1
- package/README.en.md +4 -4
- package/README.md +4 -4
- package/agentskills.json +1 -1
- package/docs/ARCHITECTURE.en.md +2 -2
- package/docs/ARCHITECTURE.md +11 -2
- package/docs/CLI_REFERENCE.en.md +6 -4
- package/docs/CLI_REFERENCE.md +1 -1
- package/docs/CONTRIBUTING.en.md +1 -1
- package/docs/CONTRIBUTING.md +2 -2
- package/docs/TOOL_CATALOG.md +6 -2
- package/docs/research/agent_skill_training_analysis.md +1 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/skills_manifest.json +320 -55
- package/synapse.config.json +1 -1
- package/templates/ui/index.html +2 -2
- package/tools/TOOL_CATALOG.json +63 -3
- package/tools/__init__.py +1 -1
- package/tools/api_fuzzer.py +179 -118
- package/tools/author_integrity.py +121 -11
- package/tools/benchmark_swarm_context.py +119 -18
- package/tools/break_ui_generator.py +233 -0
- package/tools/browser_helpers.js +2 -2
- package/tools/check_version_sync.py +18 -50
- package/tools/cli.py +109 -18
- package/tools/core/bm25_retriever.py +218 -0
- package/tools/core/i18n.py +4 -4
- package/tools/core/provider_usage_adapter.py +317 -0
- package/tools/core/token_estimator.py +42 -14
- package/tools/count_tokens.py +5 -0
- package/tools/coverage_reporter.py +245 -0
- package/tools/dashboard_generator.py +6 -6
- package/tools/db_sandbox.py +146 -4
- package/tools/distill_skill.py +191 -0
- package/tools/git_pre_commit_guard.py +12 -0
- package/tools/humanize_linter.py +3 -0
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +75 -15
- package/tools/micro_mutator.py +84 -14
- package/tools/pentest_scope_guard.py +399 -56
- package/tools/perf/benchmark_perf_runner.py +42 -0
- package/tools/rules_router.py +401 -41
- package/tools/security_scanner.py +80 -0
- package/tools/senior_code_review.py +14 -0
- package/tools/skill_canonical_digest.py +244 -0
- package/tools/skill_profiles.py +77 -7
- package/tools/slide_deck_exporter.py +477 -0
- package/tools/sync_skills_manifest.py +166 -0
- package/tools/task_memory_manager.py +260 -0
- package/tools/ui_server.py +1 -1
- package/tools/update_sentinel.py +3 -3
- package/tools/validate_host_interop.py +485 -0
- package/tools/validate_skills_schema.py +282 -0
- package/tools/verify_all.py +572 -42
- package/tools/welcome_page.py +2 -2
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
tools/core/provider_usage_adapter.py — Standardized Provider Usage Ingestion & Context Splitter
|
|
8
|
+
------------------------------------------------------------------------------------------------
|
|
9
|
+
Provides strict boundary enforcement between:
|
|
10
|
+
1. Real LLM provider-reported tokens (OpenAI, Anthropic, Gemini, Antigravity Transcripts).
|
|
11
|
+
2. Static documentation / rule estimates (cl100k_base / heuristic polyglot).
|
|
12
|
+
3. Synthetic micro-workload benchmark tokens.
|
|
13
|
+
4. Avoided filesystem read bytes (I/O footprint, strictly non-token).
|
|
14
|
+
|
|
15
|
+
Invariants:
|
|
16
|
+
- Zero Guesswork / Zero Interpolation: When live provider usage response is not present,
|
|
17
|
+
provider fields MUST remain None (null).
|
|
18
|
+
- Explicit Provenance: Every record declares measurement_kind, tokenizer/model, version, commit,
|
|
19
|
+
and timestamp.
|
|
20
|
+
- Context Decomposition: Statically estimated skill payloads are subtracted from total provider
|
|
21
|
+
input tokens to accurately isolate the external system prompt, conversation history, and tool schemas.
|
|
22
|
+
- Paired Run Verification: Live reduction claims are strictly guarded and rejected unless BOTH
|
|
23
|
+
baseline and optimized runs originate from real provider-observed responses.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from dataclasses import asdict, dataclass
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
import json
|
|
31
|
+
import logging
|
|
32
|
+
import os
|
|
33
|
+
import subprocess
|
|
34
|
+
from typing import Any, Dict, Optional, Tuple
|
|
35
|
+
|
|
36
|
+
_logger = logging.getLogger("provider_usage_adapter")
|
|
37
|
+
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_repo_version_and_commit(repo_root: Optional[str] = None) -> Tuple[str, str]:
|
|
41
|
+
"""Retrieves current hub version from skills_manifest.json and git commit hash."""
|
|
42
|
+
root = repo_root or _REPO_ROOT
|
|
43
|
+
version = "1.0.0"
|
|
44
|
+
manifest_path = os.path.join(root, "skills_manifest.json")
|
|
45
|
+
if os.path.isfile(manifest_path):
|
|
46
|
+
try:
|
|
47
|
+
with open(manifest_path, "r", encoding="utf-8") as f:
|
|
48
|
+
data = json.load(f)
|
|
49
|
+
version = data.get("version", "1.0.0")
|
|
50
|
+
except (OSError, json.JSONDecodeError, KeyError) as err:
|
|
51
|
+
_logger.debug("Failed to read version from manifest: %s", err)
|
|
52
|
+
|
|
53
|
+
commit = "unversioned"
|
|
54
|
+
try:
|
|
55
|
+
out = subprocess.check_output(
|
|
56
|
+
["git", "rev-parse", "HEAD"],
|
|
57
|
+
cwd=root,
|
|
58
|
+
stderr=subprocess.DEVNULL,
|
|
59
|
+
)
|
|
60
|
+
commit = out.decode("utf-8").strip()
|
|
61
|
+
except (OSError, subprocess.SubprocessError) as err:
|
|
62
|
+
_logger.debug("Failed to get git commit: %s", err)
|
|
63
|
+
commit = "unversioned"
|
|
64
|
+
|
|
65
|
+
return version, commit
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class ProviderUsageRecord:
|
|
70
|
+
"""Immutable, audited record of actual live LLM provider usage."""
|
|
71
|
+
provider: str
|
|
72
|
+
model: str
|
|
73
|
+
measurement_kind: str # Always "provider_observed"
|
|
74
|
+
provider_reported_input_tokens: int
|
|
75
|
+
provider_reported_output_tokens: int
|
|
76
|
+
provider_cached_tokens: Optional[int]
|
|
77
|
+
total_provider_tokens: int
|
|
78
|
+
timestamp: str
|
|
79
|
+
version: str
|
|
80
|
+
commit: str
|
|
81
|
+
raw_usage: Dict[str, Any]
|
|
82
|
+
|
|
83
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
84
|
+
"""Converts to dictionary representation."""
|
|
85
|
+
return asdict(self)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def create_unobserved_provider_record(
|
|
89
|
+
measurement_kind: str = "static_estimate",
|
|
90
|
+
note: str = "No live LLM provider usage observed; fields remain null.",
|
|
91
|
+
) -> Dict[str, Any]:
|
|
92
|
+
"""
|
|
93
|
+
Returns a standardized dictionary for records where live provider data was not captured.
|
|
94
|
+
Guarantees that provider fields remain strictly None without interpolation.
|
|
95
|
+
"""
|
|
96
|
+
version, commit = get_repo_version_and_commit()
|
|
97
|
+
return {
|
|
98
|
+
"measurement_kind": measurement_kind,
|
|
99
|
+
"provider": None,
|
|
100
|
+
"model": None,
|
|
101
|
+
"provider_observed": False,
|
|
102
|
+
"provider_reported_input_tokens": None,
|
|
103
|
+
"provider_reported_output_tokens": None,
|
|
104
|
+
"provider_cached_tokens": None,
|
|
105
|
+
"total_provider_tokens": None,
|
|
106
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
107
|
+
"version": version,
|
|
108
|
+
"commit": commit,
|
|
109
|
+
"note": note,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def parse_provider_usage(
|
|
114
|
+
response: Dict[str, Any],
|
|
115
|
+
provider_hint: Optional[str] = None,
|
|
116
|
+
model_hint: Optional[str] = None,
|
|
117
|
+
repo_root: Optional[str] = None,
|
|
118
|
+
) -> ProviderUsageRecord:
|
|
119
|
+
"""
|
|
120
|
+
Parses live provider usage from standard LLM response payloads.
|
|
121
|
+
Supported payloads:
|
|
122
|
+
1. OpenAI / Compatible: {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "prompt_tokens_details": {"cached_tokens": ...}}}
|
|
123
|
+
2. Anthropic: {"usage": {"input_tokens": ..., "output_tokens": ..., "cache_read_input_tokens": ...}}
|
|
124
|
+
3. Google GenAI: {"usage_metadata": {"prompt_token_count": ..., "candidates_token_count": ..., "cached_content_token_count": ...}}
|
|
125
|
+
4. Antigravity transcript step: {"input_tokens": ..., "output_tokens": ..., "cache_read_tokens": ...}
|
|
126
|
+
5. Mock / Test payload: {"provider": ..., "model": ..., "usage": {"input_tokens": ..., ...}}
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
ValueError: When no valid provider usage payload is present (refuses to guess).
|
|
130
|
+
"""
|
|
131
|
+
if not isinstance(response, dict):
|
|
132
|
+
raise ValueError(f"Expected dict response payload, got {type(response).__name__}")
|
|
133
|
+
|
|
134
|
+
version, commit = get_repo_version_and_commit(repo_root)
|
|
135
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
136
|
+
|
|
137
|
+
# 1. Google GenAI (Gemini SDK) format: usage_metadata
|
|
138
|
+
if "usage_metadata" in response and isinstance(response["usage_metadata"], dict):
|
|
139
|
+
um = response["usage_metadata"]
|
|
140
|
+
in_tok = um.get("prompt_token_count")
|
|
141
|
+
out_tok = um.get("candidates_token_count", 0)
|
|
142
|
+
cache_tok = um.get("cached_content_token_count")
|
|
143
|
+
model = response.get("model") or model_hint or "gemini"
|
|
144
|
+
provider = provider_hint or "google"
|
|
145
|
+
if in_tok is not None:
|
|
146
|
+
total = in_tok + out_tok
|
|
147
|
+
return ProviderUsageRecord(
|
|
148
|
+
provider=provider,
|
|
149
|
+
model=model,
|
|
150
|
+
measurement_kind="provider_observed",
|
|
151
|
+
provider_reported_input_tokens=int(in_tok),
|
|
152
|
+
provider_reported_output_tokens=int(out_tok),
|
|
153
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
154
|
+
total_provider_tokens=total,
|
|
155
|
+
timestamp=now_iso,
|
|
156
|
+
version=version,
|
|
157
|
+
commit=commit,
|
|
158
|
+
raw_usage=um,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# 2. OpenAI / Anthropic format: usage key
|
|
162
|
+
if "usage" in response and isinstance(response["usage"], dict):
|
|
163
|
+
u = response["usage"]
|
|
164
|
+
model = response.get("model") or model_hint or "unknown_model"
|
|
165
|
+
|
|
166
|
+
# OpenAI style: prompt_tokens, completion_tokens
|
|
167
|
+
if "prompt_tokens" in u:
|
|
168
|
+
in_tok = u["prompt_tokens"]
|
|
169
|
+
out_tok = u.get("completion_tokens", 0)
|
|
170
|
+
cache_tok = None
|
|
171
|
+
if "prompt_tokens_details" in u and isinstance(u["prompt_tokens_details"], dict):
|
|
172
|
+
cache_tok = u["prompt_tokens_details"].get("cached_tokens")
|
|
173
|
+
provider = provider_hint or ("openai" if "gpt" in model.lower() else "llm_provider")
|
|
174
|
+
total = u.get("total_tokens", in_tok + out_tok)
|
|
175
|
+
return ProviderUsageRecord(
|
|
176
|
+
provider=provider,
|
|
177
|
+
model=model,
|
|
178
|
+
measurement_kind="provider_observed",
|
|
179
|
+
provider_reported_input_tokens=int(in_tok),
|
|
180
|
+
provider_reported_output_tokens=int(out_tok),
|
|
181
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
182
|
+
total_provider_tokens=int(total),
|
|
183
|
+
timestamp=now_iso,
|
|
184
|
+
version=version,
|
|
185
|
+
commit=commit,
|
|
186
|
+
raw_usage=u,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# Anthropic style: input_tokens, output_tokens
|
|
190
|
+
if "input_tokens" in u:
|
|
191
|
+
in_tok = u["input_tokens"]
|
|
192
|
+
out_tok = u.get("output_tokens", 0)
|
|
193
|
+
cache_tok = u.get("cache_read_input_tokens")
|
|
194
|
+
provider = provider_hint or ("anthropic" if "claude" in model.lower() else "llm_provider")
|
|
195
|
+
total = in_tok + out_tok
|
|
196
|
+
return ProviderUsageRecord(
|
|
197
|
+
provider=provider,
|
|
198
|
+
model=model,
|
|
199
|
+
measurement_kind="provider_observed",
|
|
200
|
+
provider_reported_input_tokens=int(in_tok),
|
|
201
|
+
provider_reported_output_tokens=int(out_tok),
|
|
202
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
203
|
+
total_provider_tokens=total,
|
|
204
|
+
timestamp=now_iso,
|
|
205
|
+
version=version,
|
|
206
|
+
commit=commit,
|
|
207
|
+
raw_usage=u,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# 3. Direct / Antigravity transcript step format: input_tokens, output_tokens directly in root
|
|
211
|
+
if "input_tokens" in response:
|
|
212
|
+
in_tok = response["input_tokens"]
|
|
213
|
+
out_tok = response.get("output_tokens", 0)
|
|
214
|
+
cache_tok = response.get("cache_read_tokens")
|
|
215
|
+
provider = response.get("provider") or provider_hint or "antigravity"
|
|
216
|
+
model = response.get("model") or model_hint or "gemini"
|
|
217
|
+
total = in_tok + out_tok + (cache_tok or 0)
|
|
218
|
+
return ProviderUsageRecord(
|
|
219
|
+
provider=provider,
|
|
220
|
+
model=model,
|
|
221
|
+
measurement_kind="provider_observed",
|
|
222
|
+
provider_reported_input_tokens=int(in_tok),
|
|
223
|
+
provider_reported_output_tokens=int(out_tok),
|
|
224
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
225
|
+
total_provider_tokens=int(total),
|
|
226
|
+
timestamp=now_iso,
|
|
227
|
+
version=version,
|
|
228
|
+
commit=commit,
|
|
229
|
+
raw_usage={k: response[k] for k in ("input_tokens", "output_tokens", "cache_read_tokens") if k in response},
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
raise ValueError(
|
|
233
|
+
"No valid LLM provider usage metadata found in response. "
|
|
234
|
+
"Synapse Hub refuses to fabricate synthetic numbers as live provider data."
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def decompose_prompt_context(
|
|
239
|
+
provider_record: ProviderUsageRecord,
|
|
240
|
+
skill_payload_tokens: int,
|
|
241
|
+
rule_payload_tokens: int = 0,
|
|
242
|
+
) -> Dict[str, Any]:
|
|
243
|
+
"""
|
|
244
|
+
Separates the statically estimated skill/rule documentation payload from the
|
|
245
|
+
overall provider input tokens to isolate external host system instructions,
|
|
246
|
+
conversation history turns, and provider tool calling schemas.
|
|
247
|
+
"""
|
|
248
|
+
if not isinstance(provider_record, ProviderUsageRecord):
|
|
249
|
+
raise TypeError(f"Expected ProviderUsageRecord, got {type(provider_record).__name__}")
|
|
250
|
+
|
|
251
|
+
total_in = provider_record.provider_reported_input_tokens
|
|
252
|
+
managed_tokens = skill_payload_tokens + rule_payload_tokens
|
|
253
|
+
external_wrapper = max(0, total_in - managed_tokens)
|
|
254
|
+
ratio = round((managed_tokens / max(1, total_in)) * 100.0, 2)
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
"measurement_kind": "provider_observed_with_decomposition",
|
|
258
|
+
"provider": provider_record.provider,
|
|
259
|
+
"model": provider_record.model,
|
|
260
|
+
"provider_reported_input_tokens": total_in,
|
|
261
|
+
"provider_reported_output_tokens": provider_record.provider_reported_output_tokens,
|
|
262
|
+
"provider_cached_tokens": provider_record.provider_cached_tokens,
|
|
263
|
+
"skill_payload_static_estimated_tokens": skill_payload_tokens,
|
|
264
|
+
"rule_payload_static_estimated_tokens": rule_payload_tokens,
|
|
265
|
+
"total_managed_static_tokens": managed_tokens,
|
|
266
|
+
"external_system_prompt_and_wrapper_tokens": external_wrapper,
|
|
267
|
+
"managed_payload_ratio_pct": ratio,
|
|
268
|
+
"timestamp": provider_record.timestamp,
|
|
269
|
+
"version": provider_record.version,
|
|
270
|
+
"commit": provider_record.commit,
|
|
271
|
+
"disclaimer": (
|
|
272
|
+
"Provider reported input tokens reflect total live API billing usage. "
|
|
273
|
+
"Skill and rule payloads are statically estimated via offline tokenizer and subtracted "
|
|
274
|
+
"to isolate unmanaged host system prompt, user prompt, and provider tool schemas."
|
|
275
|
+
),
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def compare_paired_provider_runs(
|
|
280
|
+
baseline_run: ProviderUsageRecord,
|
|
281
|
+
optimized_run: ProviderUsageRecord,
|
|
282
|
+
) -> Dict[str, Any]:
|
|
283
|
+
"""
|
|
284
|
+
Computes verified token reduction between two live provider runs.
|
|
285
|
+
Enforces scientific honesty: strictly refuses to claim live provider reduction unless
|
|
286
|
+
BOTH baseline and optimized runs originate from real provider_observed responses.
|
|
287
|
+
"""
|
|
288
|
+
if not isinstance(baseline_run, ProviderUsageRecord) or not isinstance(optimized_run, ProviderUsageRecord):
|
|
289
|
+
raise TypeError("Both baseline_run and optimized_run must be verified ProviderUsageRecord instances.")
|
|
290
|
+
|
|
291
|
+
if baseline_run.measurement_kind != "provider_observed" or optimized_run.measurement_kind != "provider_observed":
|
|
292
|
+
raise ValueError("Both runs must have measurement_kind == 'provider_observed' to claim live provider reduction.")
|
|
293
|
+
|
|
294
|
+
in_saved = baseline_run.provider_reported_input_tokens - optimized_run.provider_reported_input_tokens
|
|
295
|
+
out_saved = baseline_run.provider_reported_output_tokens - optimized_run.provider_reported_output_tokens
|
|
296
|
+
total_saved = baseline_run.total_provider_tokens - optimized_run.total_provider_tokens
|
|
297
|
+
reduction_pct = round((total_saved / max(1, baseline_run.total_provider_tokens)) * 100.0, 2)
|
|
298
|
+
|
|
299
|
+
return {
|
|
300
|
+
"verified_provider_savings": True,
|
|
301
|
+
"measurement_kind": "paired_provider_observed",
|
|
302
|
+
"baseline_provider": baseline_run.provider,
|
|
303
|
+
"baseline_model": baseline_run.model,
|
|
304
|
+
"baseline_input_tokens": baseline_run.provider_reported_input_tokens,
|
|
305
|
+
"baseline_output_tokens": baseline_run.provider_reported_output_tokens,
|
|
306
|
+
"baseline_total_tokens": baseline_run.total_provider_tokens,
|
|
307
|
+
"optimized_provider": optimized_run.provider,
|
|
308
|
+
"optimized_model": optimized_run.model,
|
|
309
|
+
"optimized_input_tokens": optimized_run.provider_reported_input_tokens,
|
|
310
|
+
"optimized_output_tokens": optimized_run.provider_reported_output_tokens,
|
|
311
|
+
"optimized_total_tokens": optimized_run.total_provider_tokens,
|
|
312
|
+
"input_tokens_saved": in_saved,
|
|
313
|
+
"output_tokens_saved": out_saved,
|
|
314
|
+
"net_tokens_saved": total_saved,
|
|
315
|
+
"reduction_pct": reduction_pct,
|
|
316
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
317
|
+
}
|
|
@@ -22,6 +22,7 @@ Key Capabilities:
|
|
|
22
22
|
from __future__ import annotations
|
|
23
23
|
|
|
24
24
|
import glob
|
|
25
|
+
import hashlib
|
|
25
26
|
import json
|
|
26
27
|
import os
|
|
27
28
|
import re
|
|
@@ -119,8 +120,8 @@ def _get_skills_manifest(repo_root: str) -> Dict[str, Dict[str, Any]]:
|
|
|
119
120
|
"""
|
|
120
121
|
Loads and caches skills_manifest.json mapping skill_id to metadata.
|
|
121
122
|
Cache is keyed by the canonical realpath of the manifest file and validated
|
|
122
|
-
against file
|
|
123
|
-
|
|
123
|
+
against file mtime_ns, size, and content SHA-256 digest to guarantee that
|
|
124
|
+
mutations are detected even if file modification time remains identical.
|
|
124
125
|
"""
|
|
125
126
|
global _SKILLS_MANIFEST_CACHE
|
|
126
127
|
manifest_path = os.path.realpath(os.path.join(repo_root, "skills_manifest.json"))
|
|
@@ -130,26 +131,41 @@ def _get_skills_manifest(repo_root: str) -> Dict[str, Dict[str, Any]]:
|
|
|
130
131
|
return {}
|
|
131
132
|
|
|
132
133
|
try:
|
|
133
|
-
|
|
134
|
+
st = os.stat(manifest_path)
|
|
135
|
+
current_mtime_ns = st.st_mtime_ns
|
|
136
|
+
current_size = st.st_size
|
|
137
|
+
except OSError:
|
|
138
|
+
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
139
|
+
return {}
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
with open(manifest_path, "rb") as f:
|
|
143
|
+
raw_bytes = f.read()
|
|
144
|
+
current_digest = hashlib.sha256(raw_bytes).digest()
|
|
134
145
|
except OSError:
|
|
135
146
|
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
136
147
|
return {}
|
|
137
148
|
|
|
138
149
|
if manifest_path in _SKILLS_MANIFEST_CACHE:
|
|
139
|
-
|
|
140
|
-
if
|
|
141
|
-
|
|
150
|
+
cached_entry = _SKILLS_MANIFEST_CACHE[manifest_path]
|
|
151
|
+
if len(cached_entry) == 4:
|
|
152
|
+
cached_mtime_ns, cached_size, cached_digest, cached_data = cached_entry
|
|
153
|
+
if cached_digest == current_digest:
|
|
154
|
+
return cached_data
|
|
155
|
+
elif len(cached_entry) == 2:
|
|
156
|
+
cached_mtime, cached_data = cached_entry
|
|
157
|
+
if cached_mtime == st.st_mtime:
|
|
158
|
+
return cached_data
|
|
142
159
|
|
|
143
160
|
manifest_data: Dict[str, Dict[str, Any]] = {}
|
|
144
161
|
try:
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
except (OSError, json.JSONDecodeError) as err:
|
|
162
|
+
data = json.loads(raw_bytes.decode("utf-8"))
|
|
163
|
+
for s in data.get("skills", []):
|
|
164
|
+
sid = s.get("id")
|
|
165
|
+
if sid:
|
|
166
|
+
manifest_data[sid] = s
|
|
167
|
+
_SKILLS_MANIFEST_CACHE[manifest_path] = (current_mtime_ns, current_size, current_digest, manifest_data)
|
|
168
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as err:
|
|
153
169
|
import sys
|
|
154
170
|
sys.stderr.write(f"Warning: Failed to load manifest at {manifest_path}: {err}\n")
|
|
155
171
|
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
@@ -216,6 +232,12 @@ def calculate_hydration_payload(
|
|
|
216
232
|
catalog_skill_tokens = count_tokens("\n".join(catalog_lines)) if catalog_lines else 0
|
|
217
233
|
|
|
218
234
|
return {
|
|
235
|
+
"measurement_kind": "static_estimate",
|
|
236
|
+
"static_estimated_tokens": rule_tokens + full_skill_tokens,
|
|
237
|
+
"provider_reported_input_tokens": None,
|
|
238
|
+
"provider_reported_output_tokens": None,
|
|
239
|
+
"provider_cached_tokens": None,
|
|
240
|
+
"avoided_disk_bytes": 0,
|
|
219
241
|
"rule_tokens": rule_tokens,
|
|
220
242
|
"rule_bytes": rule_bytes,
|
|
221
243
|
"skill_catalog_tokens": catalog_skill_tokens,
|
|
@@ -270,6 +292,12 @@ def get_full_unrouted_baseline(repo_root: str) -> Dict[str, Any]:
|
|
|
270
292
|
total_cat_tokens = count_tokens("\n".join(catalog_lines)) if catalog_lines else 0
|
|
271
293
|
|
|
272
294
|
return {
|
|
295
|
+
"measurement_kind": "static_estimate",
|
|
296
|
+
"static_estimated_tokens": total_rule_tokens + total_skill_tokens,
|
|
297
|
+
"provider_reported_input_tokens": None,
|
|
298
|
+
"provider_reported_output_tokens": None,
|
|
299
|
+
"provider_cached_tokens": None,
|
|
300
|
+
"avoided_disk_bytes": 0,
|
|
273
301
|
"rule_count": len(all_rule_files),
|
|
274
302
|
"rule_tokens": total_rule_tokens,
|
|
275
303
|
"rule_bytes": total_rule_bytes,
|
package/tools/count_tokens.py
CHANGED
|
@@ -775,6 +775,10 @@ def calculate_hub_savings(input_tokens: int, cache_tokens: int, output_tokens: i
|
|
|
775
775
|
|
|
776
776
|
return {
|
|
777
777
|
"model_name": pricing["name"],
|
|
778
|
+
"measurement_kind": "counterfactual_model_estimate",
|
|
779
|
+
"provider_reported_input_tokens": input_tokens,
|
|
780
|
+
"provider_reported_output_tokens": output_tokens,
|
|
781
|
+
"provider_cached_tokens": cache_tokens,
|
|
778
782
|
"non_hub_input": estimated_non_hub_input,
|
|
779
783
|
"non_hub_cache": estimated_non_hub_cache,
|
|
780
784
|
"non_hub_total": estimated_non_hub_total,
|
|
@@ -788,6 +792,7 @@ def calculate_hub_savings(input_tokens: int, cache_tokens: int, output_tokens: i
|
|
|
788
792
|
"savings_pct": round(savings_pct, 1),
|
|
789
793
|
"cost_saved_usd": round(cost_saved_usd, 4),
|
|
790
794
|
"cost_savings_pct": round(cost_savings_pct, 1),
|
|
795
|
+
"note": "Actual hub tokens are parsed from session transcripts. Non-hub baseline is an estimated counterfactual.",
|
|
791
796
|
}
|
|
792
797
|
|
|
793
798
|
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
tools/coverage_reporter.py — Standardized Coverage Metrics & Roadmap Evaluator
|
|
8
|
+
-------------------------------------------------------------------------------
|
|
9
|
+
Evaluates and distinguishes:
|
|
10
|
+
1. Statement / Line Coverage (fail_under: 60.00% calibrated floor, roadmap: 70% -> 80%).
|
|
11
|
+
2. Branch Coverage (floor: 50.00% baseline floor, roadmap: 60% -> 70%).
|
|
12
|
+
3. Combined Coverage (statements + branches hit / total).
|
|
13
|
+
|
|
14
|
+
Maintains strict separation between line coverage, branch coverage, static heuristic,
|
|
15
|
+
and dynamic mutation kill scores.
|
|
16
|
+
|
|
17
|
+
Outputs:
|
|
18
|
+
- reports/coverage_summary.json (Standardized multi-metric artifact)
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import os
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
from typing import Any, Dict
|
|
31
|
+
|
|
32
|
+
_logger = logging.getLogger("coverage_reporter")
|
|
33
|
+
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
34
|
+
DEFAULT_COVERAGE_JSON = os.path.join(REPO_ROOT, "reports", "coverage.json")
|
|
35
|
+
DEFAULT_COVERAGE_XML = os.path.join(REPO_ROOT, "reports", "coverage.xml")
|
|
36
|
+
DEFAULT_SUMMARY_JSON = os.path.join(REPO_ROOT, "reports", "coverage_summary.json")
|
|
37
|
+
|
|
38
|
+
# Calibrated floors and roadmap milestones
|
|
39
|
+
LINE_COVERAGE_FLOOR_PCT = 60.00
|
|
40
|
+
BRANCH_COVERAGE_FLOOR_PCT = 50.00
|
|
41
|
+
|
|
42
|
+
ROADMAP_THRESHOLDS = {
|
|
43
|
+
"current_calibrated": {"line_pct": 60.00, "branch_pct": 50.00},
|
|
44
|
+
"phase_2_consolidation": {"line_pct": 70.00, "branch_pct": 60.00},
|
|
45
|
+
"target_enterprise": {"line_pct": 80.00, "branch_pct": 70.00},
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def get_git_commit(repo_root: str = REPO_ROOT) -> str:
|
|
50
|
+
try:
|
|
51
|
+
out = subprocess.check_output(
|
|
52
|
+
["git", "rev-parse", "HEAD"],
|
|
53
|
+
cwd=repo_root,
|
|
54
|
+
stderr=subprocess.DEVNULL,
|
|
55
|
+
)
|
|
56
|
+
return out.decode("utf-8").strip()
|
|
57
|
+
except (OSError, subprocess.SubprocessError) as err:
|
|
58
|
+
_logger.debug("Could not determine git commit: %s", err)
|
|
59
|
+
return "unversioned"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def parse_coverage_metrics(
|
|
63
|
+
json_path: str = DEFAULT_COVERAGE_JSON,
|
|
64
|
+
) -> Dict[str, Any]:
|
|
65
|
+
"""Parses coverage metrics from coverage.py json report."""
|
|
66
|
+
if not os.path.exists(json_path):
|
|
67
|
+
raise FileNotFoundError(f"Coverage JSON report not found: {json_path}")
|
|
68
|
+
|
|
69
|
+
with open(json_path, "r", encoding="utf-8") as f:
|
|
70
|
+
data = json.load(f)
|
|
71
|
+
|
|
72
|
+
totals = data.get("totals", {})
|
|
73
|
+
num_statements = totals.get("num_statements", 0)
|
|
74
|
+
covered_lines = totals.get("covered_lines", 0)
|
|
75
|
+
missing_lines = totals.get("missing_lines", 0)
|
|
76
|
+
|
|
77
|
+
# Line / Statement coverage
|
|
78
|
+
line_cov_pct = float(
|
|
79
|
+
totals.get(
|
|
80
|
+
"percent_statements_covered",
|
|
81
|
+
(covered_lines / num_statements * 100.0) if num_statements > 0 else 0.0,
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# Branch coverage
|
|
86
|
+
num_branches = totals.get("num_branches", 0)
|
|
87
|
+
covered_branches = totals.get("covered_branches", 0)
|
|
88
|
+
missing_branches = totals.get("missing_branches", 0)
|
|
89
|
+
branch_cov_pct = float(
|
|
90
|
+
totals.get(
|
|
91
|
+
"percent_branches_covered",
|
|
92
|
+
(covered_branches / num_branches * 100.0) if num_branches > 0 else 0.0,
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# Combined coverage
|
|
97
|
+
combined_cov_pct = float(totals.get("percent_covered", 0.0))
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
"num_statements": num_statements,
|
|
101
|
+
"covered_lines": covered_lines,
|
|
102
|
+
"missing_lines": missing_lines,
|
|
103
|
+
"line_coverage_pct": round(line_cov_pct, 2),
|
|
104
|
+
"num_branches": num_branches,
|
|
105
|
+
"covered_branches": covered_branches,
|
|
106
|
+
"missing_branches": missing_branches,
|
|
107
|
+
"branch_coverage_pct": round(branch_cov_pct, 2),
|
|
108
|
+
"combined_coverage_pct": round(combined_cov_pct, 2),
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def evaluate_coverage(
|
|
113
|
+
coverage_data: Dict[str, Any],
|
|
114
|
+
line_floor: float = LINE_COVERAGE_FLOOR_PCT,
|
|
115
|
+
branch_floor: float = BRANCH_COVERAGE_FLOOR_PCT,
|
|
116
|
+
enforce_branch: bool = True,
|
|
117
|
+
) -> Dict[str, Any]:
|
|
118
|
+
"""Evaluates metrics against calibrated floor and roadmap milestones."""
|
|
119
|
+
line_pct = coverage_data["line_coverage_pct"]
|
|
120
|
+
branch_pct = coverage_data["branch_coverage_pct"]
|
|
121
|
+
|
|
122
|
+
line_pass = line_pct >= line_floor
|
|
123
|
+
branch_pass = branch_pct >= branch_floor if enforce_branch and coverage_data["num_branches"] > 0 else True
|
|
124
|
+
all_pass = line_pass and branch_pass
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
"line_pass": line_pass,
|
|
128
|
+
"branch_pass": branch_pass,
|
|
129
|
+
"all_pass": all_pass,
|
|
130
|
+
"line_floor": line_floor,
|
|
131
|
+
"branch_floor": branch_floor,
|
|
132
|
+
"enforce_branch": enforce_branch,
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def generate_coverage_summary(
|
|
137
|
+
coverage_json: str = DEFAULT_COVERAGE_JSON,
|
|
138
|
+
coverage_xml: str = DEFAULT_COVERAGE_XML,
|
|
139
|
+
output_summary_json: str = DEFAULT_SUMMARY_JSON,
|
|
140
|
+
line_floor: float = LINE_COVERAGE_FLOOR_PCT,
|
|
141
|
+
branch_floor: float = BRANCH_COVERAGE_FLOOR_PCT,
|
|
142
|
+
enforce_branch: bool = True,
|
|
143
|
+
repo_root: str = REPO_ROOT,
|
|
144
|
+
) -> Dict[str, Any]:
|
|
145
|
+
"""Generates standardized coverage summary artifact and writes to disk."""
|
|
146
|
+
metrics = parse_coverage_metrics(coverage_json)
|
|
147
|
+
eval_res = evaluate_coverage(
|
|
148
|
+
metrics,
|
|
149
|
+
line_floor=line_floor,
|
|
150
|
+
branch_floor=branch_floor,
|
|
151
|
+
enforce_branch=enforce_branch,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
commit_sha = get_git_commit(repo_root)
|
|
155
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
156
|
+
|
|
157
|
+
summary_doc = {
|
|
158
|
+
"generated_at": now_iso,
|
|
159
|
+
"commit": commit_sha,
|
|
160
|
+
"metrics": {
|
|
161
|
+
"line_coverage_pct": metrics["line_coverage_pct"],
|
|
162
|
+
"branch_coverage_pct": metrics["branch_coverage_pct"],
|
|
163
|
+
"combined_coverage_pct": metrics["combined_coverage_pct"],
|
|
164
|
+
"statements_total": metrics["num_statements"],
|
|
165
|
+
"statements_covered": metrics["covered_lines"],
|
|
166
|
+
"statements_missing": metrics["missing_lines"],
|
|
167
|
+
"branches_total": metrics["num_branches"],
|
|
168
|
+
"branches_covered": metrics["covered_branches"],
|
|
169
|
+
"branches_missing": metrics["missing_branches"],
|
|
170
|
+
},
|
|
171
|
+
"thresholds": {
|
|
172
|
+
"line_floor_pct": line_floor,
|
|
173
|
+
"branch_floor_pct": branch_floor,
|
|
174
|
+
"roadmap": ROADMAP_THRESHOLDS,
|
|
175
|
+
},
|
|
176
|
+
"evaluation": {
|
|
177
|
+
"line_passed": eval_res["line_pass"],
|
|
178
|
+
"branch_passed": eval_res["branch_pass"],
|
|
179
|
+
"all_passed": eval_res["all_pass"],
|
|
180
|
+
},
|
|
181
|
+
"artifacts": {
|
|
182
|
+
"coverage_json": os.path.relpath(coverage_json, repo_root) if os.path.exists(coverage_json) else None,
|
|
183
|
+
"coverage_xml": os.path.relpath(coverage_xml, repo_root) if os.path.exists(coverage_xml) else None,
|
|
184
|
+
"summary_json": os.path.relpath(output_summary_json, repo_root),
|
|
185
|
+
},
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
out_dir = os.path.dirname(output_summary_json)
|
|
189
|
+
if out_dir:
|
|
190
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
191
|
+
with open(output_summary_json, "w", encoding="utf-8") as f:
|
|
192
|
+
json.dump(summary_doc, f, indent=2)
|
|
193
|
+
|
|
194
|
+
return summary_doc
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def main() -> int:
|
|
198
|
+
parser = argparse.ArgumentParser(description="Synapse Hub Standardized Coverage Reporter")
|
|
199
|
+
parser.add_argument("--json-report", default=DEFAULT_COVERAGE_JSON, help="Path to coverage.json")
|
|
200
|
+
parser.add_argument("--xml-report", default=DEFAULT_COVERAGE_XML, help="Path to coverage.xml")
|
|
201
|
+
parser.add_argument("--summary-out", default=DEFAULT_SUMMARY_JSON, help="Path to write coverage_summary.json")
|
|
202
|
+
parser.add_argument("--line-floor", type=float, default=LINE_COVERAGE_FLOOR_PCT, help="Statement/line coverage floor")
|
|
203
|
+
parser.add_argument("--branch-floor", type=float, default=BRANCH_COVERAGE_FLOOR_PCT, help="Branch coverage floor")
|
|
204
|
+
parser.add_argument("--no-branch", action="store_true", help="Do not enforce branch coverage threshold")
|
|
205
|
+
parser.add_argument("--check", action="store_true", help="Fail with non-zero exit code if coverage is below threshold")
|
|
206
|
+
parser.add_argument("--json", action="store_true", help="Print summary JSON to stdout")
|
|
207
|
+
args = parser.parse_args()
|
|
208
|
+
|
|
209
|
+
if not os.path.exists(args.json_report):
|
|
210
|
+
sys.stderr.write(f"Error: Coverage JSON report not found at {args.json_report}.\n")
|
|
211
|
+
return 1
|
|
212
|
+
|
|
213
|
+
summary = generate_coverage_summary(
|
|
214
|
+
coverage_json=args.json_report,
|
|
215
|
+
coverage_xml=args.xml_report,
|
|
216
|
+
output_summary_json=args.summary_out,
|
|
217
|
+
line_floor=args.line_floor,
|
|
218
|
+
branch_floor=args.branch_floor,
|
|
219
|
+
enforce_branch=not args.no_branch,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
if args.json:
|
|
223
|
+
print(json.dumps(summary, indent=2))
|
|
224
|
+
else:
|
|
225
|
+
m = summary["metrics"]
|
|
226
|
+
e = summary["evaluation"]
|
|
227
|
+
print("======================================================================")
|
|
228
|
+
print(" SYNAPSE HUB STANDARDIZED TEST COVERAGE REPORT ")
|
|
229
|
+
print("======================================================================")
|
|
230
|
+
print(f" Line Coverage (Statements) : {m['line_coverage_pct']}% (Floor: >= {args.line_floor}%) - {'✅ PASS' if e['line_passed'] else '❌ FAIL'}")
|
|
231
|
+
print(f" Branch Coverage : {m['branch_coverage_pct']}% (Floor: >= {args.branch_floor}%) - {'✅ PASS' if e['branch_passed'] else '❌ FAIL'}")
|
|
232
|
+
print(f" Combined (Stmt + Branch) : {m['combined_coverage_pct']}%")
|
|
233
|
+
print(f" Total Statements : {m['statements_total']:,} (Covered: {m['statements_covered']:,})")
|
|
234
|
+
print(f" Total Branches : {m['branches_total']:,} (Covered: {m['branches_covered']:,})")
|
|
235
|
+
print(f" Coverage Summary Artifact : {summary['artifacts']['summary_json']}")
|
|
236
|
+
print(f" Coverage XML Artifact : {summary['artifacts']['coverage_xml']}")
|
|
237
|
+
print("======================================================================")
|
|
238
|
+
|
|
239
|
+
if args.check and not summary["evaluation"]["all_passed"]:
|
|
240
|
+
return 1
|
|
241
|
+
return 0
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
if __name__ == "__main__":
|
|
245
|
+
sys.exit(main())
|