@teevee98/synapse-hub 4.1.3 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/git-push-protocol.md +2 -2
- package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +8 -3
- package/.agents/skills/01-web-automation/clean-pom-crafter/examples/standard_test_case.groovy +3 -0
- package/.agents/skills/01-web-automation/clean-pom-crafter/references/pom_patterns.groovy +3 -0
- package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/e2e-testing-patterns/examples/playwright_sharded_config.ts +3 -0
- package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
- package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +7 -2
- package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/playwright-webapp-testing/examples/server_lifecycle_playwright.py +3 -0
- package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +6 -1
- package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +6 -1
- package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +6 -1
- package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +6 -1
- package/.agents/skills/02-test-design/review-spec/SKILL.md +4 -1
- package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +4 -1
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +6 -1
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/examples/playwright_cwv_profiler_fixture.ts +3 -0
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/references/dast_recipes.ts +3 -0
- package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +7 -2
- package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/examples/refactored_page_sample.groovy +3 -0
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/examples/refactored_playwright_pom.ts +3 -0
- package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +4 -1
- package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/examples/surgical_code_window_walkthrough.md +1 -1
- package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +6 -1
- package/AGENTS.md +7 -1
- package/LICENSE +5 -5
- package/NOTICE +1 -1
- package/README.en.md +4 -4
- package/README.md +4 -4
- package/agentskills.json +1 -1
- package/docs/ARCHITECTURE.en.md +2 -2
- package/docs/ARCHITECTURE.md +11 -2
- package/docs/CLI_REFERENCE.en.md +6 -4
- package/docs/CLI_REFERENCE.md +1 -1
- package/docs/CONTRIBUTING.en.md +1 -1
- package/docs/CONTRIBUTING.md +2 -2
- package/docs/PERFORMANCE_AUDIT_REPORT.md +2 -2
- package/docs/TOOL_CATALOG.md +6 -2
- package/docs/research/agent_skill_training_analysis.md +1 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/scripts/generate_skill_examples.py +3 -0
- package/scripts/package_dist.py +3 -0
- package/skills_manifest.json +320 -55
- package/synapse.config.json +1 -1
- package/templates/ui/index.html +2 -2
- package/tools/.infer_baseline.json +3 -796
- package/tools/TOOL_CATALOG.json +63 -3
- package/tools/__init__.py +4 -1
- package/tools/agent_canary.py +3 -0
- package/tools/analyze_project.py +3 -0
- package/tools/api_drift_sentinel.py +3 -0
- package/tools/api_fuzzer.py +182 -118
- package/tools/ast_cache.py +4 -2
- package/tools/author_integrity.py +174 -12
- package/tools/auto_heal.py +7 -2
- package/tools/autogen_qa_broker.py +14 -15
- package/tools/benchmark_swarm_context.py +119 -18
- package/tools/break_ui_generator.py +233 -0
- package/tools/browser_helpers.js +2 -2
- package/tools/bug_reporter.py +3 -0
- package/tools/build_consolidated_report.py +3 -0
- package/tools/bump_version.py +4 -1
- package/tools/cache_resolver.py +7 -3
- package/tools/check_env.py +3 -0
- package/tools/check_version_sync.py +21 -50
- package/tools/ci_healing_hook.py +3 -0
- package/tools/cli.py +116 -22
- package/tools/config_decoupler.py +3 -0
- package/tools/core/__init__.py +3 -0
- package/tools/core/bm25_retriever.py +218 -0
- package/tools/core/file_lock.py +23 -14
- package/tools/core/i18n.py +13 -11
- package/tools/core/memory_bus.py +6 -3
- package/tools/core/process_supervisor.py +28 -18
- package/tools/core/provider_usage_adapter.py +317 -0
- package/tools/core/token_estimator.py +83 -18
- package/tools/count_tokens.py +16 -7
- package/tools/coverage_reporter.py +245 -0
- package/tools/create_mock_docx.py +3 -0
- package/tools/dashboard_generator.py +17 -15
- package/tools/db_sandbox.py +150 -5
- package/tools/debt_harvester.py +2 -2
- package/tools/diagram_architect.py +3 -0
- package/tools/distill_log.py +22 -2
- package/tools/distill_skill.py +191 -0
- package/tools/e2e_flow_prober.py +3 -0
- package/tools/export_postman_collection.py +3 -0
- package/tools/flaky_detector.py +3 -0
- package/tools/fsm_enforcer.py +2 -2
- package/tools/fuzz_payloads.py +3 -0
- package/tools/generate_allure_report.py +3 -0
- package/tools/generate_api_matrix.py +3 -0
- package/tools/generate_jmx_plan.py +3 -0
- package/tools/generate_tc_from_story.py +3 -0
- package/tools/generate_test_reports.py +3 -0
- package/tools/git_pre_commit_guard.py +25 -3
- package/tools/git_worktree_manager.py +3 -0
- package/tools/heal/__init__.py +3 -0
- package/tools/heal_pr_automator.py +3 -0
- package/tools/heuristic_locator_matcher.py +3 -0
- package/tools/hub_publisher.py +3 -3
- package/tools/humanize_linter.py +3 -0
- package/tools/infer_linter.py +19 -9
- package/tools/init_config.py +6 -9
- package/tools/jtl_analyzer.py +3 -0
- package/tools/lint/__init__.py +3 -0
- package/tools/lint_ai_docs.py +3 -0
- package/tools/lookup_tc.py +3 -0
- package/tools/lookup_xpath.py +3 -0
- package/tools/matrix_sharder.py +3 -0
- package/tools/mcp/__init__.py +3 -0
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +4 -1
- package/tools/mcp_server.py +78 -15
- package/tools/md_to_xlsx.py +3 -0
- package/tools/mechanical_ui_gate.py +3 -0
- package/tools/micro_mutator.py +93 -19
- package/tools/network_mock_shield.py +3 -0
- package/tools/newman_analyzer.py +3 -0
- package/tools/omni_emitter.py +3 -0
- package/tools/omni_parser.py +3 -0
- package/tools/omni_transpiler.py +3 -0
- package/tools/output_formatter.py +3 -0
- package/tools/package_release.py +6 -6
- package/tools/pairwise_generator.py +3 -0
- package/tools/pentest_scope_guard.py +401 -58
- package/tools/perf/__init__.py +3 -0
- package/tools/perf/benchmark_perf_runner.py +42 -0
- package/tools/pinpoint_failure.py +3 -0
- package/tools/project_state.py +3 -0
- package/tools/repo_map.py +3 -0
- package/tools/report/__init__.py +3 -0
- package/tools/rules_router.py +426 -34
- package/tools/runaway_stopper.py +3 -0
- package/tools/safety_firewall.py +3 -0
- package/tools/security_scanner.py +80 -0
- package/tools/senior_code_review.py +14 -0
- package/tools/setup_project.py +9 -4
- package/tools/skill_canonical_digest.py +244 -0
- package/tools/skill_collision_guard.py +2 -2
- package/tools/skill_profiles.py +77 -7
- package/tools/skillspector.py +2 -1
- package/tools/slide_deck_exporter.py +477 -0
- package/tools/smart_visual_masker.py +3 -0
- package/tools/sync_skills_manifest.py +166 -0
- package/tools/task_memory_manager.py +260 -0
- package/tools/telemetry_correlator.py +3 -0
- package/tools/template_excel_exporter.py +2 -2
- package/tools/test_impact.py +3 -0
- package/tools/trace_data_flow.py +3 -0
- package/tools/traffic_session_converter.py +3 -0
- package/tools/ui_server.py +2 -2
- package/tools/universal_ir.py +3 -0
- package/tools/update_sentinel.py +15 -9
- package/tools/validate_groovy_scripts.py +3 -0
- package/tools/validate_host_interop.py +485 -0
- package/tools/validate_skills_schema.py +282 -0
- package/tools/verify_all.py +600 -63
- package/tools/visual_diff_canvas.py +3 -0
- package/tools/visual_som_helper.py +3 -0
- package/tools/watch_mode.py +4 -4
- package/tools/wcag_audit.py +3 -0
- package/tools/welcome_page.py +2 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
3
|
# SPDX-License-Identifier: MIT
|
|
4
|
-
# Project: Synapse-Hub (https://github.com/
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
5
|
|
|
6
6
|
"""
|
|
7
7
|
tools/author_integrity.py — Author Attribution & Repository Provenance Guard
|
|
@@ -17,7 +17,7 @@ Note on Legal Scope:
|
|
|
17
17
|
for external legal SBOM scans or formal trademark/copyright enforcement.
|
|
18
18
|
|
|
19
19
|
Checks:
|
|
20
|
-
1. LICENSE file exists with original copyright notice (Thế Vị @
|
|
20
|
+
1. LICENSE file exists with original copyright notice (Thế Vị @TeeVee98).
|
|
21
21
|
2. package.json maintains the original author attribution.
|
|
22
22
|
3. README.md retains the original creator and repository provenance.
|
|
23
23
|
4. pyproject.toml preserves the author declaration.
|
|
@@ -65,6 +65,106 @@ def verify_author_fingerprint() -> bool:
|
|
|
65
65
|
return computed == CANONICAL_AUTHOR_SIG or computed in CANONICAL_AUTHOR_SIGS
|
|
66
66
|
|
|
67
67
|
|
|
68
|
+
def _resolve_license_content(root: str, allow_global_metadata: bool = False) -> Tuple[Optional[str], Optional[str]]:
|
|
69
|
+
"""
|
|
70
|
+
Locates LICENSE content and its origin path across repository root, tools/,
|
|
71
|
+
dist-info directories, or distribution metadata (importlib.metadata).
|
|
72
|
+
Returns (license_text, origin_description).
|
|
73
|
+
"""
|
|
74
|
+
# 1. Direct LICENSE in root
|
|
75
|
+
license_path = os.path.join(root, "LICENSE")
|
|
76
|
+
if os.path.isfile(license_path):
|
|
77
|
+
try:
|
|
78
|
+
with open(license_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
79
|
+
return f.read(), license_path
|
|
80
|
+
except Exception as e:
|
|
81
|
+
return None, f"Could not read LICENSE file: {e}"
|
|
82
|
+
|
|
83
|
+
# 2. Check dist-info directories specifically inside the given root
|
|
84
|
+
if os.path.isdir(root):
|
|
85
|
+
try:
|
|
86
|
+
for entry in os.listdir(root):
|
|
87
|
+
if entry.endswith(".dist-info") and ("synapse" in entry.lower()):
|
|
88
|
+
dist_info_dir = os.path.join(root, entry)
|
|
89
|
+
for sub in (
|
|
90
|
+
os.path.join("licenses", "LICENSE"),
|
|
91
|
+
"LICENSE",
|
|
92
|
+
"LICENSE.txt",
|
|
93
|
+
):
|
|
94
|
+
lic_candidate = os.path.join(dist_info_dir, sub)
|
|
95
|
+
if os.path.isfile(lic_candidate):
|
|
96
|
+
try:
|
|
97
|
+
with open(lic_candidate, "r", encoding="utf-8", errors="ignore") as f:
|
|
98
|
+
return f.read(), lic_candidate
|
|
99
|
+
except (OSError, UnicodeDecodeError) as read_err:
|
|
100
|
+
lic_read_err = str(read_err)
|
|
101
|
+
continue
|
|
102
|
+
except OSError as list_err:
|
|
103
|
+
lic_list_err = str(list_err)
|
|
104
|
+
|
|
105
|
+
# 3. Check tools/LICENSE inside root
|
|
106
|
+
root_tools_lic = os.path.join(root, "tools", "LICENSE")
|
|
107
|
+
if os.path.isfile(root_tools_lic):
|
|
108
|
+
try:
|
|
109
|
+
with open(root_tools_lic, "r", encoding="utf-8", errors="ignore") as f:
|
|
110
|
+
return f.read(), root_tools_lic
|
|
111
|
+
except (OSError, UnicodeDecodeError) as tools_err:
|
|
112
|
+
lic_tools_err = str(tools_err)
|
|
113
|
+
|
|
114
|
+
# 4. Only if allowed (running in installed runtime / default root), check global metadata and SCRIPT_DIR
|
|
115
|
+
if allow_global_metadata:
|
|
116
|
+
tools_lic = os.path.join(SCRIPT_DIR, "LICENSE")
|
|
117
|
+
if os.path.isfile(tools_lic):
|
|
118
|
+
try:
|
|
119
|
+
with open(tools_lic, "r", encoding="utf-8", errors="ignore") as f:
|
|
120
|
+
return f.read(), tools_lic
|
|
121
|
+
except (OSError, UnicodeDecodeError) as s_err:
|
|
122
|
+
lic_s_err = str(s_err)
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
import importlib.metadata as meta
|
|
126
|
+
for pkg in ("synapse-hub", "synapse_hub"):
|
|
127
|
+
try:
|
|
128
|
+
pkg_files = meta.files(pkg)
|
|
129
|
+
if pkg_files:
|
|
130
|
+
for pf in pkg_files:
|
|
131
|
+
p_str = str(pf).replace("\\", "/")
|
|
132
|
+
if p_str.endswith("LICENSE") or "licenses/LICENSE" in p_str:
|
|
133
|
+
try:
|
|
134
|
+
return pf.read_text(encoding="utf-8"), f"metadata:{p_str}"
|
|
135
|
+
except (OSError, UnicodeDecodeError) as meta_read_err:
|
|
136
|
+
lic_meta_err = str(meta_read_err)
|
|
137
|
+
continue
|
|
138
|
+
except meta.PackageNotFoundError:
|
|
139
|
+
continue
|
|
140
|
+
except Exception as meta_err:
|
|
141
|
+
lic_import_err = str(meta_err)
|
|
142
|
+
|
|
143
|
+
parent_script = os.path.dirname(SCRIPT_DIR)
|
|
144
|
+
if os.path.isdir(parent_script) and parent_script != root:
|
|
145
|
+
try:
|
|
146
|
+
for entry in os.listdir(parent_script):
|
|
147
|
+
if entry.endswith(".dist-info") and ("synapse" in entry.lower()):
|
|
148
|
+
dist_info_dir = os.path.join(parent_script, entry)
|
|
149
|
+
for sub in (
|
|
150
|
+
os.path.join("licenses", "LICENSE"),
|
|
151
|
+
"LICENSE",
|
|
152
|
+
"LICENSE.txt",
|
|
153
|
+
):
|
|
154
|
+
lic_candidate = os.path.join(dist_info_dir, sub)
|
|
155
|
+
if os.path.isfile(lic_candidate):
|
|
156
|
+
try:
|
|
157
|
+
with open(lic_candidate, "r", encoding="utf-8", errors="ignore") as f:
|
|
158
|
+
return f.read(), lic_candidate
|
|
159
|
+
except (OSError, UnicodeDecodeError) as p_err:
|
|
160
|
+
lic_p_err = str(p_err)
|
|
161
|
+
continue
|
|
162
|
+
except OSError as p_list_err:
|
|
163
|
+
lic_p_list_err = str(p_list_err)
|
|
164
|
+
|
|
165
|
+
return None, None
|
|
166
|
+
|
|
167
|
+
|
|
68
168
|
def assert_integrity_or_raise(repo_root: Optional[str] = None) -> bool:
|
|
69
169
|
"""
|
|
70
170
|
Enforces author attribution and MIT License provenance in accordance with open-source licensing.
|
|
@@ -74,15 +174,23 @@ def assert_integrity_or_raise(repo_root: Optional[str] = None) -> bool:
|
|
|
74
174
|
if not verify_author_fingerprint():
|
|
75
175
|
raise RuntimeError(
|
|
76
176
|
"🛑 CRITICAL INTEGRITY CORRUPTED: Canonical author fingerprint mismatch. "
|
|
77
|
-
"Synapse Hub requires original author attribution ('Thế Vị @
|
|
177
|
+
"Synapse Hub requires original author attribution ('Thế Vị @TeeVee98') to execute."
|
|
78
178
|
)
|
|
79
179
|
valid, violations = verify_author_attribution(repo_root)
|
|
80
180
|
license_violations = [v for v in violations if "LICENSE" in v]
|
|
181
|
+
if license_violations:
|
|
182
|
+
# Fallback inspection of distribution metadata before declaring critical violation
|
|
183
|
+
root = repo_root or DEFAULT_REPO_ROOT
|
|
184
|
+
lic_content, _ = _resolve_license_content(root, allow_global_metadata=True)
|
|
185
|
+
if lic_content and ORIGINAL_AUTHOR_NAME in lic_content and ("@TeeVee98" in lic_content or "@TeeVee998" in lic_content) and "Synapse-Hub" in lic_content:
|
|
186
|
+
license_violations = []
|
|
187
|
+
violations = [v for v in violations if "LICENSE" not in v]
|
|
188
|
+
|
|
81
189
|
if license_violations:
|
|
82
190
|
err_msg = "🛑 CRITICAL INTEGRITY VIOLATION: Mandatory MIT copyright notice missing or altered.\n"
|
|
83
191
|
for v in license_violations:
|
|
84
192
|
err_msg += f" • {v}\n"
|
|
85
|
-
err_msg += "Under the MIT License, original author attribution to 'Thế Vị (@
|
|
193
|
+
err_msg += "Under the MIT License, original author attribution to 'Thế Vị (@TeeVee98)' must be preserved in LICENSE."
|
|
86
194
|
raise RuntimeError(err_msg)
|
|
87
195
|
elif not valid:
|
|
88
196
|
# Non-fatal warning for peripheral files (README/package.json) to allow flexible downstream fork usage
|
|
@@ -97,16 +205,18 @@ def verify_author_attribution(repo_root: Optional[str] = None) -> Tuple[bool, Li
|
|
|
97
205
|
Returns (is_valid, list_of_violations).
|
|
98
206
|
"""
|
|
99
207
|
root = repo_root or DEFAULT_REPO_ROOT
|
|
208
|
+
allow_global = (repo_root is None or repo_root == DEFAULT_REPO_ROOT)
|
|
100
209
|
violations: List[str] = []
|
|
101
210
|
|
|
102
|
-
# 1. Check LICENSE file
|
|
103
|
-
|
|
104
|
-
if
|
|
105
|
-
|
|
211
|
+
# 1. Check LICENSE file or distribution metadata
|
|
212
|
+
lic_content, lic_source = _resolve_license_content(root, allow_global_metadata=allow_global)
|
|
213
|
+
if lic_content is None:
|
|
214
|
+
if lic_source and lic_source.startswith("Could not read"):
|
|
215
|
+
violations.append(lic_source)
|
|
216
|
+
else:
|
|
217
|
+
violations.append("Missing LICENSE file. MIT License with author copyright notice is mandatory.")
|
|
106
218
|
else:
|
|
107
219
|
try:
|
|
108
|
-
with open(license_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
109
|
-
lic_content = f.read()
|
|
110
220
|
if ORIGINAL_AUTHOR_NAME not in lic_content or ("@TeeVee98" not in lic_content and "@TeeVee998" not in lic_content):
|
|
111
221
|
violations.append("LICENSE file modified: original copyright notice for 'Thế Vị (@TeeVee98)' was removed or altered.")
|
|
112
222
|
if "Synapse-Hub" not in lic_content:
|
|
@@ -186,10 +296,62 @@ def enforce_author_attribution(repo_root: Optional[str] = None, strict: bool = T
|
|
|
186
296
|
return True
|
|
187
297
|
|
|
188
298
|
|
|
299
|
+
def verify_source_headers(
|
|
300
|
+
repo_root: Optional[str] = None,
|
|
301
|
+
extensions: Tuple[str, ...] = (".py", ".ts", ".groovy"),
|
|
302
|
+
scan_dirs: Optional[List[str]] = None,
|
|
303
|
+
) -> Tuple[bool, List[str]]:
|
|
304
|
+
"""
|
|
305
|
+
Validates that internal source files preserve the mandatory author attribution header:
|
|
306
|
+
SPDX-FileCopyrightText: © 2026 Thế Vị (@TeeVee98) <https://github.com/TeeVee98/Synapse-Hub>
|
|
307
|
+
SPDX-License-Identifier: MIT
|
|
308
|
+
Returns (is_valid, list_of_missing_files).
|
|
309
|
+
"""
|
|
310
|
+
root = repo_root or DEFAULT_REPO_ROOT
|
|
311
|
+
ignored_dirs = {
|
|
312
|
+
".git", ".gradle", "node_modules", "allure-results", "graphify-out",
|
|
313
|
+
"__pycache__", ".pytest_cache", ".ruff_cache", ".mypy_cache",
|
|
314
|
+
"build", "dist", "claude-ecosystem-repos", "external_repos",
|
|
315
|
+
"teamwork", ".synapse", ".sandbox_snapshots"
|
|
316
|
+
}
|
|
317
|
+
missing: List[str] = []
|
|
318
|
+
dirs_to_scan = scan_dirs or ["tools", "tests", "examples", "scripts", "benchmarks", ".agents"]
|
|
319
|
+
|
|
320
|
+
for d in dirs_to_scan:
|
|
321
|
+
full_d = os.path.join(root, d)
|
|
322
|
+
if not os.path.isdir(full_d):
|
|
323
|
+
continue
|
|
324
|
+
for cur_root, dirs, files in os.walk(full_d):
|
|
325
|
+
dirs[:] = [sub for sub in dirs if sub not in ignored_dirs]
|
|
326
|
+
for f in sorted(files):
|
|
327
|
+
if any(f.endswith(ext) for ext in extensions):
|
|
328
|
+
file_path = os.path.join(cur_root, f)
|
|
329
|
+
try:
|
|
330
|
+
with open(file_path, "r", encoding="utf-8", errors="ignore") as fp:
|
|
331
|
+
header = fp.read(600)
|
|
332
|
+
has_author = (
|
|
333
|
+
("Thế Vị" in header or "TeeVee98" in header or "TeeVee998" in header)
|
|
334
|
+
and "SPDX-FileCopyrightText" in header
|
|
335
|
+
)
|
|
336
|
+
has_license = "SPDX-License-Identifier" in header
|
|
337
|
+
if not (has_author and has_license):
|
|
338
|
+
missing.append(os.path.relpath(file_path, root).replace(os.sep, "/"))
|
|
339
|
+
except OSError as err:
|
|
340
|
+
missing.append(f"{os.path.relpath(file_path, root)}: {err}")
|
|
341
|
+
|
|
342
|
+
return len(missing) == 0, missing
|
|
343
|
+
|
|
344
|
+
|
|
189
345
|
if __name__ == "__main__":
|
|
190
346
|
valid, viols = verify_author_attribution()
|
|
191
|
-
|
|
347
|
+
headers_valid, missing_headers = verify_source_headers()
|
|
348
|
+
if valid and headers_valid:
|
|
192
349
|
print(f"✅ Author attribution is 100% verified intact: {get_canonical_watermark()}")
|
|
350
|
+
print("✅ Source file SPDX headers verified intact (0 missing across source tree).")
|
|
193
351
|
sys.exit(0)
|
|
194
352
|
else:
|
|
353
|
+
if not headers_valid:
|
|
354
|
+
print(f"[WARN] Missing SPDX headers in {len(missing_headers)} files:", file=sys.stderr)
|
|
355
|
+
for mf in missing_headers[:10]:
|
|
356
|
+
print(f" • {mf}", file=sys.stderr)
|
|
195
357
|
enforce_author_attribution(strict=True)
|
package/tools/auto_heal.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: © 2026 Thế Vị (@TeeVee98) <https://github.com/TeeVee98/Synapse-Hub>
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
2
5
|
"""
|
|
3
6
|
Enterprise Closed-Loop Autonomous Test Healer Engine
|
|
4
7
|
-----------------------------------------------------
|
|
@@ -55,7 +58,8 @@ def capture_live_aom_snapshot(url: str, timeout: float = 10.0) -> Optional[Dict[
|
|
|
55
58
|
browser.close()
|
|
56
59
|
return snapshot
|
|
57
60
|
except Exception as _err:
|
|
58
|
-
|
|
61
|
+
import logging
|
|
62
|
+
logging.getLogger("auto_heal").debug("Playwright snapshot capture failed: %s", _err)
|
|
59
63
|
return None
|
|
60
64
|
|
|
61
65
|
|
|
@@ -227,7 +231,8 @@ def main():
|
|
|
227
231
|
with open(args.aom, "r", encoding="utf-8") as af:
|
|
228
232
|
aom_snap = json.load(af)
|
|
229
233
|
except Exception as _err:
|
|
230
|
-
|
|
234
|
+
import logging
|
|
235
|
+
logging.getLogger("auto_heal").debug("Failed to read AOM snapshot: %s", _err)
|
|
231
236
|
|
|
232
237
|
if args.record_lesson:
|
|
233
238
|
ok = record_heal_lesson(
|
|
@@ -36,8 +36,8 @@ if hasattr(sys.stdout, 'reconfigure'):
|
|
|
36
36
|
try:
|
|
37
37
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
38
38
|
sys.stderr.reconfigure(encoding='utf-8')
|
|
39
|
-
except Exception
|
|
40
|
-
|
|
39
|
+
except Exception:
|
|
40
|
+
pass
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
class StorageBackend(ABC):
|
|
@@ -190,7 +190,8 @@ class SQLiteStorageBackend(StorageBackend):
|
|
|
190
190
|
try:
|
|
191
191
|
payload = json.loads(row["payload"])
|
|
192
192
|
except Exception as err:
|
|
193
|
-
|
|
193
|
+
import logging
|
|
194
|
+
logging.getLogger("autogen_qa_broker").debug("Failed to decode message payload: %s", err)
|
|
194
195
|
payload = row["payload"]
|
|
195
196
|
|
|
196
197
|
messages.append({
|
|
@@ -253,7 +254,8 @@ class SQLiteStorageBackend(StorageBackend):
|
|
|
253
254
|
try:
|
|
254
255
|
payload = json.loads(row["payload"])
|
|
255
256
|
except Exception as err:
|
|
256
|
-
|
|
257
|
+
import logging
|
|
258
|
+
logging.getLogger("autogen_qa_broker").debug("Failed to decode message payload: %s", err)
|
|
257
259
|
payload = row["payload"]
|
|
258
260
|
mailboxes[to_act].append({
|
|
259
261
|
"id": row["id"],
|
|
@@ -368,7 +370,6 @@ class RedisStorageBackend(StorageBackend):
|
|
|
368
370
|
try:
|
|
369
371
|
self.client.ping()
|
|
370
372
|
except Exception as err:
|
|
371
|
-
_ = err
|
|
372
373
|
sys.stderr.write(f"Notice: Redis backend ping failed ({err}).\n")
|
|
373
374
|
|
|
374
375
|
def save_message(
|
|
@@ -411,8 +412,7 @@ class RedisStorageBackend(StorageBackend):
|
|
|
411
412
|
continue
|
|
412
413
|
try:
|
|
413
414
|
data = json.loads(raw)
|
|
414
|
-
except Exception
|
|
415
|
-
_ = err
|
|
415
|
+
except Exception:
|
|
416
416
|
continue
|
|
417
417
|
|
|
418
418
|
if topic and data.get("topic") != topic:
|
|
@@ -421,7 +421,8 @@ class RedisStorageBackend(StorageBackend):
|
|
|
421
421
|
try:
|
|
422
422
|
payload = json.loads(data["payload"])
|
|
423
423
|
except Exception as err:
|
|
424
|
-
|
|
424
|
+
import logging
|
|
425
|
+
logging.getLogger("autogen_qa_broker").debug("Failed to decode data payload: %s", err)
|
|
425
426
|
payload = data.get("payload")
|
|
426
427
|
|
|
427
428
|
was_read = bool(data.get("read", 0))
|
|
@@ -477,7 +478,8 @@ class RedisStorageBackend(StorageBackend):
|
|
|
477
478
|
try:
|
|
478
479
|
val = json.loads(val_str)
|
|
479
480
|
except Exception as err:
|
|
480
|
-
|
|
481
|
+
import logging
|
|
482
|
+
logging.getLogger("autogen_qa_broker").debug("Failed to decode subscription: %s", err)
|
|
481
483
|
val = {"handler": "default_handler", "subscribed_at": ""}
|
|
482
484
|
subscriptions[ev_topic].append({
|
|
483
485
|
"actor": act,
|
|
@@ -507,8 +509,7 @@ class RedisStorageBackend(StorageBackend):
|
|
|
507
509
|
continue
|
|
508
510
|
try:
|
|
509
511
|
data = json.loads(raw)
|
|
510
|
-
except Exception
|
|
511
|
-
_ = err
|
|
512
|
+
except Exception:
|
|
512
513
|
continue
|
|
513
514
|
|
|
514
515
|
if not data.get("read", 0):
|
|
@@ -793,8 +794,7 @@ def main():
|
|
|
793
794
|
if args.subcommand == "send-message":
|
|
794
795
|
try:
|
|
795
796
|
payload = json.loads(args.payload)
|
|
796
|
-
except Exception
|
|
797
|
-
_ = err
|
|
797
|
+
except Exception:
|
|
798
798
|
payload = args.payload
|
|
799
799
|
res = broker.send_message(args.from_actor, args.to_actor, args.topic, payload)
|
|
800
800
|
print(json.dumps(res, indent=2))
|
|
@@ -813,8 +813,7 @@ def main():
|
|
|
813
813
|
elif args.subcommand == "publish":
|
|
814
814
|
try:
|
|
815
815
|
payload = json.loads(args.payload)
|
|
816
|
-
except Exception
|
|
817
|
-
_ = err
|
|
816
|
+
except Exception:
|
|
818
817
|
payload = args.payload
|
|
819
818
|
res = broker.publish(args.event, payload)
|
|
820
819
|
print(json.dumps(res, indent=2))
|
|
@@ -18,6 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import argparse
|
|
20
20
|
from datetime import datetime, timezone
|
|
21
|
+
import hashlib
|
|
21
22
|
import json
|
|
22
23
|
import os
|
|
23
24
|
import re
|
|
@@ -36,7 +37,7 @@ REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
|
36
37
|
if REPO_ROOT not in sys.path:
|
|
37
38
|
sys.path.insert(0, REPO_ROOT)
|
|
38
39
|
|
|
39
|
-
# Import core token estimator
|
|
40
|
+
# Import core token estimator and provider usage adapter
|
|
40
41
|
try:
|
|
41
42
|
from tools.core.token_estimator import count_tokens, get_estimator_info
|
|
42
43
|
except ImportError:
|
|
@@ -53,6 +54,15 @@ except ImportError:
|
|
|
53
54
|
def get_estimator_info() -> Dict[str, Any]:
|
|
54
55
|
return {"engine": "fallback_words", "is_exact": False, "encoding": "cl100k_base"}
|
|
55
56
|
|
|
57
|
+
try:
|
|
58
|
+
from tools.core.provider_usage_adapter import get_repo_version_and_commit
|
|
59
|
+
except ImportError:
|
|
60
|
+
try:
|
|
61
|
+
from core.provider_usage_adapter import get_repo_version_and_commit
|
|
62
|
+
except ImportError:
|
|
63
|
+
def get_repo_version_and_commit(repo_root=None):
|
|
64
|
+
return "1.0.0", "unversioned"
|
|
65
|
+
|
|
56
66
|
|
|
57
67
|
def estimate_tokens(text: str) -> int:
|
|
58
68
|
"""Wrapper for backward compatibility."""
|
|
@@ -287,6 +297,9 @@ def run_synthetic_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]
|
|
|
287
297
|
results = []
|
|
288
298
|
total_baseline_tokens = 0
|
|
289
299
|
total_synapse_tokens = 0
|
|
300
|
+
info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
|
|
301
|
+
version, commit = get_repo_version_and_commit(REPO_ROOT)
|
|
302
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
290
303
|
|
|
291
304
|
for scn in scenarios:
|
|
292
305
|
baseline_tokens = count_tokens(scn["baseline_payload"])
|
|
@@ -294,10 +307,26 @@ def run_synthetic_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]
|
|
|
294
307
|
saved_tokens = baseline_tokens - synapse_tokens
|
|
295
308
|
savings_pct = round((saved_tokens / max(1, baseline_tokens)) * 100.0, 2)
|
|
296
309
|
|
|
310
|
+
# Fixture hash for reproducibility
|
|
311
|
+
fixture_bytes = (scn.get("baseline_payload", "") + "::" + scn.get("synapse_payload", "")).encode("utf-8")
|
|
312
|
+
fixture_hash = hashlib.sha256(fixture_bytes).hexdigest()[:16]
|
|
313
|
+
|
|
297
314
|
results.append({
|
|
298
315
|
"id": scn["id"],
|
|
299
316
|
"name": scn["name"],
|
|
300
317
|
"description": scn["description"],
|
|
318
|
+
"measurement_kind": "synthetic",
|
|
319
|
+
"measurement_source": "deterministic_micro_workload",
|
|
320
|
+
"fixture_hash": fixture_hash,
|
|
321
|
+
"synthetic_tokens": synapse_tokens,
|
|
322
|
+
"baseline_synthetic_tokens": baseline_tokens,
|
|
323
|
+
"saved_synthetic_tokens": saved_tokens,
|
|
324
|
+
"reduction_pct": savings_pct,
|
|
325
|
+
"provider_reported_input_tokens": None,
|
|
326
|
+
"provider_reported_output_tokens": None,
|
|
327
|
+
"provider_cached_tokens": None,
|
|
328
|
+
"avoided_disk_bytes": 0,
|
|
329
|
+
# Backward compatibility fields
|
|
301
330
|
"baseline_tokens": baseline_tokens,
|
|
302
331
|
"synapse_tokens": synapse_tokens,
|
|
303
332
|
"saved_tokens": saved_tokens,
|
|
@@ -311,12 +340,29 @@ def run_synthetic_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]
|
|
|
311
340
|
net_savings_pct = round((net_savings_tokens / max(1, total_baseline_tokens)) * 100.0, 2)
|
|
312
341
|
|
|
313
342
|
return {
|
|
314
|
-
"timestamp":
|
|
343
|
+
"timestamp": now_iso,
|
|
344
|
+
"measurement_kind": "synthetic",
|
|
345
|
+
"measurement_source": "synthetic_micro_workloads_suite",
|
|
346
|
+
"tokenizer_engine": info.get("engine", "heuristic_polyglot"),
|
|
347
|
+
"encoding": info.get("encoding", "cl100k_base"),
|
|
348
|
+
"is_exact": info.get("is_exact", False),
|
|
349
|
+
"version": version,
|
|
350
|
+
"commit": commit,
|
|
351
|
+
"total_baseline_synthetic_tokens": total_baseline_tokens,
|
|
352
|
+
"total_synapse_synthetic_tokens": total_synapse_tokens,
|
|
353
|
+
"net_saved_synthetic_tokens": net_savings_tokens,
|
|
354
|
+
"net_savings_pct": net_savings_pct,
|
|
355
|
+
"provider_reported_input_tokens": None,
|
|
356
|
+
"provider_reported_output_tokens": None,
|
|
357
|
+
"provider_cached_tokens": None,
|
|
358
|
+
"provider_observed": False,
|
|
359
|
+
"avoided_disk_bytes": 0,
|
|
360
|
+
"verified_claim_ge_40_pct": (net_savings_pct >= 40.0),
|
|
361
|
+
"note": "Synthetic micro-workload measurements evaluated via offline tokenizer; does not represent live LLM provider billing usage.",
|
|
362
|
+
# Backward compatibility fields
|
|
315
363
|
"total_baseline_tokens": total_baseline_tokens,
|
|
316
364
|
"total_synapse_tokens": total_synapse_tokens,
|
|
317
365
|
"net_saved_tokens": net_savings_tokens,
|
|
318
|
-
"net_savings_pct": net_savings_pct,
|
|
319
|
-
"verified_claim_ge_40_pct": (net_savings_pct >= 40.0),
|
|
320
366
|
"scenarios": results,
|
|
321
367
|
}
|
|
322
368
|
|
|
@@ -335,6 +381,10 @@ def run_empirical_projects_benchmark() -> Dict[str, Any]:
|
|
|
335
381
|
("Selenium Suite", os.path.join(REPO_ROOT, "examples", "selenium-suite")),
|
|
336
382
|
]
|
|
337
383
|
|
|
384
|
+
info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
|
|
385
|
+
version, commit = get_repo_version_and_commit(REPO_ROOT)
|
|
386
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
387
|
+
|
|
338
388
|
project_results = []
|
|
339
389
|
for name, p_path in project_folders:
|
|
340
390
|
if not os.path.isdir(p_path):
|
|
@@ -346,21 +396,45 @@ def run_empirical_projects_benchmark() -> Dict[str, Any]:
|
|
|
346
396
|
|
|
347
397
|
project_results.append({
|
|
348
398
|
"project_name": name,
|
|
399
|
+
"measurement_kind": "static_estimate",
|
|
400
|
+
"measurement_source": "repository_reference_suites",
|
|
349
401
|
"detected_stacks": routed["detected_stacks"],
|
|
350
402
|
"confidence_score": routed["confidence_score"],
|
|
351
|
-
"
|
|
352
|
-
"
|
|
403
|
+
"static_estimated_tokens": hyd["routed_tokens"],
|
|
404
|
+
"catalog_summary_tokens": cat["routed_tokens"],
|
|
405
|
+
"catalog_baseline_tokens": cat["baseline_tokens"],
|
|
353
406
|
"catalog_saved_tokens": cat["tokens_saved"],
|
|
354
407
|
"catalog_reduction_pct": cat["reduction_pct"],
|
|
355
|
-
"
|
|
356
|
-
"
|
|
408
|
+
"hydrated_baseline_tokens": hyd["baseline_tokens"],
|
|
409
|
+
"hydrated_routed_tokens": hyd["routed_tokens"],
|
|
357
410
|
"hydrated_saved_tokens": hyd["tokens_saved"],
|
|
358
411
|
"hydrated_reduction_pct": hyd["reduction_pct"],
|
|
359
412
|
"avoided_disk_bytes": rf.get("avoided_bytes", 0),
|
|
413
|
+
"provider_reported_input_tokens": None,
|
|
414
|
+
"provider_reported_output_tokens": None,
|
|
415
|
+
"provider_cached_tokens": None,
|
|
416
|
+
"provider_observed": False,
|
|
417
|
+
# Backward compatibility fields
|
|
418
|
+
"catalog_baseline": cat["baseline_tokens"],
|
|
419
|
+
"catalog_routed": cat["routed_tokens"],
|
|
420
|
+
"hydrated_baseline": hyd["baseline_tokens"],
|
|
421
|
+
"hydrated_routed": hyd["routed_tokens"],
|
|
360
422
|
})
|
|
361
423
|
|
|
362
424
|
return {
|
|
363
|
-
"timestamp":
|
|
425
|
+
"timestamp": now_iso,
|
|
426
|
+
"measurement_kind": "static_estimate",
|
|
427
|
+
"measurement_source": "disk_workspace_inspection",
|
|
428
|
+
"tokenizer_engine": info.get("engine", "heuristic_polyglot"),
|
|
429
|
+
"encoding": info.get("encoding", "cl100k_base"),
|
|
430
|
+
"is_exact": info.get("is_exact", False),
|
|
431
|
+
"version": version,
|
|
432
|
+
"commit": commit,
|
|
433
|
+
"provider_observed": False,
|
|
434
|
+
"provider_reported_input_tokens": None,
|
|
435
|
+
"provider_reported_output_tokens": None,
|
|
436
|
+
"provider_cached_tokens": None,
|
|
437
|
+
"note": "Real project filesystem workspace inspections evaluated via static rules_router hydration; avoided bytes represent disk I/O not context tokens.",
|
|
364
438
|
"projects_count": len(project_results),
|
|
365
439
|
"results": project_results,
|
|
366
440
|
}
|
|
@@ -370,8 +444,22 @@ def run_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]:
|
|
|
370
444
|
"""Runs combined dual-track benchmark."""
|
|
371
445
|
synthetic = run_synthetic_benchmark(dataset_dir)
|
|
372
446
|
empirical = run_empirical_projects_benchmark()
|
|
447
|
+
info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
|
|
448
|
+
version, commit = get_repo_version_and_commit(REPO_ROOT)
|
|
449
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
450
|
+
|
|
373
451
|
return {
|
|
374
|
-
"timestamp":
|
|
452
|
+
"timestamp": now_iso,
|
|
453
|
+
"measurement_kind": "dual_track_benchmark",
|
|
454
|
+
"tokenizer_engine": info.get("engine", "heuristic_polyglot"),
|
|
455
|
+
"encoding": info.get("encoding", "cl100k_base"),
|
|
456
|
+
"is_exact": info.get("is_exact", False),
|
|
457
|
+
"version": version,
|
|
458
|
+
"commit": commit,
|
|
459
|
+
"provider_observed": False,
|
|
460
|
+
"provider_reported_input_tokens": None,
|
|
461
|
+
"provider_reported_output_tokens": None,
|
|
462
|
+
"provider_cached_tokens": None,
|
|
375
463
|
"synthetic_track": synthetic,
|
|
376
464
|
"empirical_track": empirical,
|
|
377
465
|
# Backward compatibility fields
|
|
@@ -388,16 +476,17 @@ def format_markdown_report(bench: Dict[str, Any]) -> str:
|
|
|
388
476
|
"""Formats benchmark results in a clean, scientifically honest markdown report."""
|
|
389
477
|
syn = bench.get("synthetic_track", bench)
|
|
390
478
|
emp = bench.get("empirical_track", {})
|
|
391
|
-
info = get_estimator_info() if "get_estimator_info"
|
|
479
|
+
info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
|
|
392
480
|
|
|
393
481
|
md = [
|
|
394
482
|
"# Empirical Context & Token Reduction Benchmark (Dual-Track)",
|
|
395
483
|
"",
|
|
396
484
|
"> **Scientific Disclosure:** This benchmark distinguishes between:",
|
|
397
485
|
"> 1. **Track 1 (Synthetic Micro-Workloads):** Deterministic micro-workloads evaluating isolated context patterns.",
|
|
398
|
-
"> 2. **Track 2 (Empirical Reference Suites):** Live
|
|
399
|
-
f"> - **Tokenizer Engine:** `{info.get('engine', '
|
|
400
|
-
f"> - **Encoding Scope:** `{info.get('encoding', 'cl100k_base')}` (Standard OpenAI BPE encoding for LLMs
|
|
486
|
+
"> 2. **Track 2 (Empirical Reference Suites):** Live static file inspections on real example workspaces within the repository.",
|
|
487
|
+
f"> - **Tokenizer Engine:** `{info.get('engine', 'heuristic_polyglot')}` (Exact: `{info.get('is_exact', False)}`)",
|
|
488
|
+
f"> - **Encoding Scope:** `{info.get('encoding', 'cl100k_base')}` (Standard OpenAI BPE encoding for LLMs).",
|
|
489
|
+
"> - **Billing Notice:** Offline estimates do not represent live LLM provider billing usage. Live provider usage reduction is only verified when real provider response metrics are paired.",
|
|
401
490
|
"",
|
|
402
491
|
"## Track 1: Deterministic Micro-Workload Benchmark (Synthetic)",
|
|
403
492
|
"",
|
|
@@ -420,10 +509,10 @@ def format_markdown_report(bench: Dict[str, Any]) -> str:
|
|
|
420
509
|
if emp and emp.get("results"):
|
|
421
510
|
md.extend([
|
|
422
511
|
"",
|
|
423
|
-
"## Track 2: Empirical Reference Suites Benchmark (
|
|
512
|
+
"## Track 2: Empirical Reference Suites Benchmark (Static Workspace Inspection)",
|
|
424
513
|
"",
|
|
425
514
|
"> *Methodology: Dynamically enumerates all 13 rules and 53 skills in repository without static per-skill multiplier assumptions.*",
|
|
426
|
-
"> *Disclosure: 'Avoided Disk Read' measures
|
|
515
|
+
"> *Disclosure: 'Avoided Disk Read (I/O)' measures filesystem file size bytes not read from storage; it is strictly distinguished from context tokens and is not billed token savings.*",
|
|
427
516
|
"",
|
|
428
517
|
"| Reference Project | Detected Stack | Confidence | Catalog Summary Saved | Full Hydration Saved | Avoided Disk Read (I/O) |",
|
|
429
518
|
"|---|---|:---:|:---:|:---:|:---:|",
|
|
@@ -480,9 +569,21 @@ def main() -> int:
|
|
|
480
569
|
json.dump(bench, f, indent=2)
|
|
481
570
|
|
|
482
571
|
if args.json:
|
|
572
|
+
# Guarantee single parseable JSON object on stdout
|
|
483
573
|
print(json.dumps(bench, indent=2))
|
|
484
|
-
|
|
485
|
-
|
|
574
|
+
if bench["net_savings_pct"] < args.verify_min_savings:
|
|
575
|
+
print(
|
|
576
|
+
f"\n[FAIL] Token savings {bench['net_savings_pct']}% is below required threshold of {args.verify_min_savings}%.",
|
|
577
|
+
file=sys.stderr,
|
|
578
|
+
)
|
|
579
|
+
return 1
|
|
580
|
+
print(
|
|
581
|
+
f"\n[PASS] Empirical token reduction verified: {bench['net_savings_pct']}% >= {args.verify_min_savings}% threshold.",
|
|
582
|
+
file=sys.stderr,
|
|
583
|
+
)
|
|
584
|
+
return 0
|
|
585
|
+
|
|
586
|
+
print(format_markdown_report(bench))
|
|
486
587
|
|
|
487
588
|
if bench["net_savings_pct"] < args.verify_min_savings:
|
|
488
589
|
print(f"\n[FAIL] Token savings {bench['net_savings_pct']}% is below required threshold of {args.verify_min_savings}%.")
|