@teevee98/synapse-hub 4.1.4 → 4.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/.agents/rules/git-push-protocol.md +2 -2
  2. package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +52 -3
  3. package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +55 -5
  4. package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +62 -67
  5. package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +54 -4
  6. package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +62 -12
  7. package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +55 -8
  8. package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +60 -12
  9. package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
  10. package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
  11. package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
  12. package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +54 -91
  13. package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +59 -11
  14. package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +50 -4
  15. package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +55 -9
  16. package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +55 -9
  17. package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +55 -8
  18. package/.agents/skills/02-test-design/review-spec/SKILL.md +50 -6
  19. package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +50 -4
  20. package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +56 -5
  21. package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +53 -7
  22. package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +86 -36
  23. package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +54 -6
  24. package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +50 -6
  25. package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +50 -3
  26. package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +63 -14
  27. package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +51 -5
  28. package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +54 -4
  29. package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +51 -5
  30. package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +53 -6
  31. package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +54 -8
  32. package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +60 -24
  33. package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +52 -6
  34. package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +56 -10
  35. package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +58 -12
  36. package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +58 -12
  37. package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +53 -5
  38. package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +56 -5
  39. package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +53 -4
  40. package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +58 -5
  41. package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +57 -11
  42. package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +83 -34
  43. package/.agents/skills/08-code-quality/graphify-impact-analyzer/examples/dependency_blast_radius.json +3 -4
  44. package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +55 -10
  45. package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +49 -6
  46. package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +52 -7
  47. package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +55 -10
  48. package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +51 -5
  49. package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +54 -17
  50. package/.agents/skills/09-ci-and-devops/github-actions-templates/examples/matrix_test_pipeline.yml +2 -1
  51. package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +50 -4
  52. package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +53 -7
  53. package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +50 -4
  54. package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +58 -13
  55. package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +51 -5
  56. package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +57 -10
  57. package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +50 -4
  58. package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +50 -4
  59. package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +55 -9
  60. package/.cursor/rules/synapse-qa.mdc +20 -7
  61. package/.cursorrules +19 -5
  62. package/.windsurfrules +18 -3
  63. package/AGENTS.md +51 -59
  64. package/CLAUDE.md +38 -24
  65. package/LICENSE +5 -5
  66. package/NOTICE +1 -1
  67. package/README.en.md +4 -4
  68. package/README.md +4 -4
  69. package/agentskills.json +1 -1
  70. package/docs/ARCHITECTURE.en.md +2 -2
  71. package/docs/ARCHITECTURE.md +11 -2
  72. package/docs/CLIENT_SUPPORT_MATRIX.md +44 -0
  73. package/docs/CLI_REFERENCE.en.md +6 -4
  74. package/docs/CLI_REFERENCE.md +1 -1
  75. package/docs/CONTRIBUTING.en.md +1 -1
  76. package/docs/CONTRIBUTING.md +2 -2
  77. package/docs/SIDE_EFFECTING_TOOLS_INVENTORY.md +233 -0
  78. package/docs/TOOL_CATALOG.md +6 -2
  79. package/docs/research/agent_skill_training_analysis.md +1 -1
  80. package/package.json +1 -1
  81. package/plugin.json +1 -1
  82. package/skills_manifest.json +320 -55
  83. package/synapse.config.json +1 -1
  84. package/templates/ai_bridges/CLAUDE.md +29 -26
  85. package/templates/ai_bridges/copilot-instructions.md +14 -5
  86. package/templates/ai_bridges/cursor_synapse_qa.mdc +16 -7
  87. package/templates/ai_bridges/windsurfrules +14 -3
  88. package/templates/ui/index.html +2 -2
  89. package/tools/TOOLS_RUNBOOK.md +1 -1
  90. package/tools/TOOL_CATALOG.json +63 -3
  91. package/tools/__init__.py +1 -1
  92. package/tools/api_fuzzer.py +447 -132
  93. package/tools/author_integrity.py +121 -11
  94. package/tools/benchmark_swarm_context.py +130 -22
  95. package/tools/break_ui_generator.py +233 -0
  96. package/tools/browser_helpers.js +2 -2
  97. package/tools/check_version_sync.py +18 -50
  98. package/tools/cli.py +110 -19
  99. package/tools/core/bm25_retriever.py +218 -0
  100. package/tools/core/i18n.py +5 -5
  101. package/tools/core/provider_usage_adapter.py +383 -0
  102. package/tools/core/token_estimator.py +81 -15
  103. package/tools/count_tokens.py +7 -2
  104. package/tools/coverage_reporter.py +245 -0
  105. package/tools/dashboard_generator.py +8 -8
  106. package/tools/db_sandbox.py +290 -9
  107. package/tools/distill_log.py +1 -1
  108. package/tools/distill_skill.py +206 -0
  109. package/tools/git_pre_commit_guard.py +202 -1
  110. package/tools/humanize_linter.py +3 -0
  111. package/tools/mcp_docx_server.py +1 -1
  112. package/tools/mcp_excel_server.py +1 -1
  113. package/tools/mcp_server.py +104 -16
  114. package/tools/micro_mutator.py +114 -15
  115. package/tools/pentest_scope_guard.py +674 -58
  116. package/tools/perf/benchmark_perf_runner.py +42 -0
  117. package/tools/project_state.py +29 -5
  118. package/tools/reversible_sandbox.py +58 -13
  119. package/tools/rules_router.py +473 -60
  120. package/tools/security_scanner.py +80 -0
  121. package/tools/senior_code_review.py +14 -0
  122. package/tools/skill_canonical_digest.py +302 -0
  123. package/tools/skill_profiles.py +77 -7
  124. package/tools/slide_deck_exporter.py +477 -0
  125. package/tools/sync_agent_rules.py +187 -61
  126. package/tools/sync_skills_manifest.py +166 -0
  127. package/tools/task_memory_manager.py +275 -0
  128. package/tools/ui_server.py +3 -1
  129. package/tools/update_sentinel.py +3 -3
  130. package/tools/validate_host_interop.py +485 -0
  131. package/tools/validate_skills_schema.py +575 -0
  132. package/tools/verify_all.py +668 -44
  133. package/tools/welcome_page.py +2 -2
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env python3
2
- # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee998). All rights reserved.
2
+ # SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
3
3
  # SPDX-License-Identifier: MIT
4
- # Project: Synapse-Hub (https://github.com/TeeVee998/Synapse-Hub)
4
+ # Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
5
5
 
6
6
  """
7
7
  tools/author_integrity.py — Author Attribution & Repository Provenance Guard
@@ -17,7 +17,7 @@ Note on Legal Scope:
17
17
  for external legal SBOM scans or formal trademark/copyright enforcement.
18
18
 
19
19
  Checks:
20
- 1. LICENSE file exists with original copyright notice (Thế Vị @TeeVee998).
20
+ 1. LICENSE file exists with original copyright notice (Thế Vị @TeeVee98).
21
21
  2. package.json maintains the original author attribution.
22
22
  3. README.md retains the original creator and repository provenance.
23
23
  4. pyproject.toml preserves the author declaration.
@@ -65,6 +65,106 @@ def verify_author_fingerprint() -> bool:
65
65
  return computed == CANONICAL_AUTHOR_SIG or computed in CANONICAL_AUTHOR_SIGS
66
66
 
67
67
 
68
+ def _resolve_license_content(root: str, allow_global_metadata: bool = False) -> Tuple[Optional[str], Optional[str]]:
69
+ """
70
+ Locates LICENSE content and its origin path across repository root, tools/,
71
+ dist-info directories, or distribution metadata (importlib.metadata).
72
+ Returns (license_text, origin_description).
73
+ """
74
+ # 1. Direct LICENSE in root
75
+ license_path = os.path.join(root, "LICENSE")
76
+ if os.path.isfile(license_path):
77
+ try:
78
+ with open(license_path, "r", encoding="utf-8", errors="ignore") as f:
79
+ return f.read(), license_path
80
+ except Exception as e:
81
+ return None, f"Could not read LICENSE file: {e}"
82
+
83
+ # 2. Check dist-info directories specifically inside the given root
84
+ if os.path.isdir(root):
85
+ try:
86
+ for entry in os.listdir(root):
87
+ if entry.endswith(".dist-info") and ("synapse" in entry.lower()):
88
+ dist_info_dir = os.path.join(root, entry)
89
+ for sub in (
90
+ os.path.join("licenses", "LICENSE"),
91
+ "LICENSE",
92
+ "LICENSE.txt",
93
+ ):
94
+ lic_candidate = os.path.join(dist_info_dir, sub)
95
+ if os.path.isfile(lic_candidate):
96
+ try:
97
+ with open(lic_candidate, "r", encoding="utf-8", errors="ignore") as f:
98
+ return f.read(), lic_candidate
99
+ except (OSError, UnicodeDecodeError) as read_err:
100
+ lic_read_err = str(read_err)
101
+ continue
102
+ except OSError as list_err:
103
+ lic_list_err = str(list_err)
104
+
105
+ # 3. Check tools/LICENSE inside root
106
+ root_tools_lic = os.path.join(root, "tools", "LICENSE")
107
+ if os.path.isfile(root_tools_lic):
108
+ try:
109
+ with open(root_tools_lic, "r", encoding="utf-8", errors="ignore") as f:
110
+ return f.read(), root_tools_lic
111
+ except (OSError, UnicodeDecodeError) as tools_err:
112
+ lic_tools_err = str(tools_err)
113
+
114
+ # 4. Only if allowed (running in installed runtime / default root), check global metadata and SCRIPT_DIR
115
+ if allow_global_metadata:
116
+ tools_lic = os.path.join(SCRIPT_DIR, "LICENSE")
117
+ if os.path.isfile(tools_lic):
118
+ try:
119
+ with open(tools_lic, "r", encoding="utf-8", errors="ignore") as f:
120
+ return f.read(), tools_lic
121
+ except (OSError, UnicodeDecodeError) as s_err:
122
+ lic_s_err = str(s_err)
123
+
124
+ try:
125
+ import importlib.metadata as meta
126
+ for pkg in ("synapse-hub", "synapse_hub"):
127
+ try:
128
+ pkg_files = meta.files(pkg)
129
+ if pkg_files:
130
+ for pf in pkg_files:
131
+ p_str = str(pf).replace("\\", "/")
132
+ if p_str.endswith("LICENSE") or "licenses/LICENSE" in p_str:
133
+ try:
134
+ return pf.read_text(encoding="utf-8"), f"metadata:{p_str}"
135
+ except (OSError, UnicodeDecodeError) as meta_read_err:
136
+ lic_meta_err = str(meta_read_err)
137
+ continue
138
+ except meta.PackageNotFoundError:
139
+ continue
140
+ except Exception as meta_err:
141
+ lic_import_err = str(meta_err)
142
+
143
+ parent_script = os.path.dirname(SCRIPT_DIR)
144
+ if os.path.isdir(parent_script) and parent_script != root:
145
+ try:
146
+ for entry in os.listdir(parent_script):
147
+ if entry.endswith(".dist-info") and ("synapse" in entry.lower()):
148
+ dist_info_dir = os.path.join(parent_script, entry)
149
+ for sub in (
150
+ os.path.join("licenses", "LICENSE"),
151
+ "LICENSE",
152
+ "LICENSE.txt",
153
+ ):
154
+ lic_candidate = os.path.join(dist_info_dir, sub)
155
+ if os.path.isfile(lic_candidate):
156
+ try:
157
+ with open(lic_candidate, "r", encoding="utf-8", errors="ignore") as f:
158
+ return f.read(), lic_candidate
159
+ except (OSError, UnicodeDecodeError) as p_err:
160
+ lic_p_err = str(p_err)
161
+ continue
162
+ except OSError as p_list_err:
163
+ lic_p_list_err = str(p_list_err)
164
+
165
+ return None, None
166
+
167
+
68
168
  def assert_integrity_or_raise(repo_root: Optional[str] = None) -> bool:
69
169
  """
70
170
  Enforces author attribution and MIT License provenance in accordance with open-source licensing.
@@ -74,15 +174,23 @@ def assert_integrity_or_raise(repo_root: Optional[str] = None) -> bool:
74
174
  if not verify_author_fingerprint():
75
175
  raise RuntimeError(
76
176
  "🛑 CRITICAL INTEGRITY CORRUPTED: Canonical author fingerprint mismatch. "
77
- "Synapse Hub requires original author attribution ('Thế Vị @TeeVee998') to execute."
177
+ "Synapse Hub requires original author attribution ('Thế Vị @TeeVee98') to execute."
78
178
  )
79
179
  valid, violations = verify_author_attribution(repo_root)
80
180
  license_violations = [v for v in violations if "LICENSE" in v]
181
+ if license_violations:
182
+ # Fallback inspection of distribution metadata before declaring critical violation
183
+ root = repo_root or DEFAULT_REPO_ROOT
184
+ lic_content, _ = _resolve_license_content(root, allow_global_metadata=True)
185
+ if lic_content and ORIGINAL_AUTHOR_NAME in lic_content and ("@TeeVee98" in lic_content or "@TeeVee998" in lic_content) and "Synapse-Hub" in lic_content:
186
+ license_violations = []
187
+ violations = [v for v in violations if "LICENSE" not in v]
188
+
81
189
  if license_violations:
82
190
  err_msg = "🛑 CRITICAL INTEGRITY VIOLATION: Mandatory MIT copyright notice missing or altered.\n"
83
191
  for v in license_violations:
84
192
  err_msg += f" • {v}\n"
85
- err_msg += "Under the MIT License, original author attribution to 'Thế Vị (@TeeVee998)' must be preserved in LICENSE."
193
+ err_msg += "Under the MIT License, original author attribution to 'Thế Vị (@TeeVee98)' must be preserved in LICENSE."
86
194
  raise RuntimeError(err_msg)
87
195
  elif not valid:
88
196
  # Non-fatal warning for peripheral files (README/package.json) to allow flexible downstream fork usage
@@ -97,16 +205,18 @@ def verify_author_attribution(repo_root: Optional[str] = None) -> Tuple[bool, Li
97
205
  Returns (is_valid, list_of_violations).
98
206
  """
99
207
  root = repo_root or DEFAULT_REPO_ROOT
208
+ allow_global = (repo_root is None or repo_root == DEFAULT_REPO_ROOT)
100
209
  violations: List[str] = []
101
210
 
102
- # 1. Check LICENSE file
103
- license_path = os.path.join(root, "LICENSE")
104
- if not os.path.exists(license_path):
105
- violations.append("Missing LICENSE file. MIT License with author copyright notice is mandatory.")
211
+ # 1. Check LICENSE file or distribution metadata
212
+ lic_content, lic_source = _resolve_license_content(root, allow_global_metadata=allow_global)
213
+ if lic_content is None:
214
+ if lic_source and lic_source.startswith("Could not read"):
215
+ violations.append(lic_source)
216
+ else:
217
+ violations.append("Missing LICENSE file. MIT License with author copyright notice is mandatory.")
106
218
  else:
107
219
  try:
108
- with open(license_path, "r", encoding="utf-8", errors="ignore") as f:
109
- lic_content = f.read()
110
220
  if ORIGINAL_AUTHOR_NAME not in lic_content or ("@TeeVee98" not in lic_content and "@TeeVee998" not in lic_content):
111
221
  violations.append("LICENSE file modified: original copyright notice for 'Thế Vị (@TeeVee98)' was removed or altered.")
112
222
  if "Synapse-Hub" not in lic_content:
@@ -18,6 +18,7 @@ from __future__ import annotations
18
18
 
19
19
  import argparse
20
20
  from datetime import datetime, timezone
21
+ import hashlib
21
22
  import json
22
23
  import os
23
24
  import re
@@ -36,7 +37,7 @@ REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
36
37
  if REPO_ROOT not in sys.path:
37
38
  sys.path.insert(0, REPO_ROOT)
38
39
 
39
- # Import core token estimator
40
+ # Import core token estimator and provider usage adapter
40
41
  try:
41
42
  from tools.core.token_estimator import count_tokens, get_estimator_info
42
43
  except ImportError:
@@ -53,6 +54,15 @@ except ImportError:
53
54
  def get_estimator_info() -> Dict[str, Any]:
54
55
  return {"engine": "fallback_words", "is_exact": False, "encoding": "cl100k_base"}
55
56
 
57
+ try:
58
+ from tools.core.provider_usage_adapter import get_repo_version_and_commit
59
+ except ImportError:
60
+ try:
61
+ from core.provider_usage_adapter import get_repo_version_and_commit
62
+ except ImportError:
63
+ def get_repo_version_and_commit(repo_root=None):
64
+ return "1.0.0", "unversioned"
65
+
56
66
 
57
67
  def estimate_tokens(text: str) -> int:
58
68
  """Wrapper for backward compatibility."""
@@ -287,6 +297,9 @@ def run_synthetic_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]
287
297
  results = []
288
298
  total_baseline_tokens = 0
289
299
  total_synapse_tokens = 0
300
+ info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
301
+ version, commit = get_repo_version_and_commit(REPO_ROOT)
302
+ now_iso = datetime.now(timezone.utc).isoformat()
290
303
 
291
304
  for scn in scenarios:
292
305
  baseline_tokens = count_tokens(scn["baseline_payload"])
@@ -294,10 +307,26 @@ def run_synthetic_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]
294
307
  saved_tokens = baseline_tokens - synapse_tokens
295
308
  savings_pct = round((saved_tokens / max(1, baseline_tokens)) * 100.0, 2)
296
309
 
310
+ # Fixture hash for reproducibility
311
+ fixture_bytes = (scn.get("baseline_payload", "") + "::" + scn.get("synapse_payload", "")).encode("utf-8")
312
+ fixture_hash = hashlib.sha256(fixture_bytes).hexdigest()[:16]
313
+
297
314
  results.append({
298
315
  "id": scn["id"],
299
316
  "name": scn["name"],
300
317
  "description": scn["description"],
318
+ "measurement_kind": "synthetic",
319
+ "measurement_source": "deterministic_micro_workload",
320
+ "fixture_hash": fixture_hash,
321
+ "synthetic_tokens": synapse_tokens,
322
+ "baseline_synthetic_tokens": baseline_tokens,
323
+ "saved_synthetic_tokens": saved_tokens,
324
+ "reduction_pct": savings_pct,
325
+ "provider_reported_input_tokens": None,
326
+ "provider_reported_output_tokens": None,
327
+ "provider_cached_tokens": None,
328
+ "avoided_disk_bytes": 0,
329
+ # Backward compatibility fields
301
330
  "baseline_tokens": baseline_tokens,
302
331
  "synapse_tokens": synapse_tokens,
303
332
  "saved_tokens": saved_tokens,
@@ -311,12 +340,29 @@ def run_synthetic_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]
311
340
  net_savings_pct = round((net_savings_tokens / max(1, total_baseline_tokens)) * 100.0, 2)
312
341
 
313
342
  return {
314
- "timestamp": datetime.now(timezone.utc).isoformat(),
343
+ "timestamp": now_iso,
344
+ "measurement_kind": "synthetic",
345
+ "measurement_source": "synthetic_micro_workloads_suite",
346
+ "tokenizer_engine": info.get("engine", "heuristic_polyglot"),
347
+ "encoding": info.get("encoding", "cl100k_base"),
348
+ "is_exact": info.get("is_exact", False),
349
+ "version": version,
350
+ "commit": commit,
351
+ "total_baseline_synthetic_tokens": total_baseline_tokens,
352
+ "total_synapse_synthetic_tokens": total_synapse_tokens,
353
+ "net_saved_synthetic_tokens": net_savings_tokens,
354
+ "net_savings_pct": net_savings_pct,
355
+ "provider_reported_input_tokens": None,
356
+ "provider_reported_output_tokens": None,
357
+ "provider_cached_tokens": None,
358
+ "provider_observed": False,
359
+ "avoided_disk_bytes": 0,
360
+ "verified_claim_ge_40_pct": (net_savings_pct >= 40.0),
361
+ "note": "Synthetic micro-workload measurements evaluated via offline tokenizer; does not represent live LLM provider billing usage.",
362
+ # Backward compatibility fields
315
363
  "total_baseline_tokens": total_baseline_tokens,
316
364
  "total_synapse_tokens": total_synapse_tokens,
317
365
  "net_saved_tokens": net_savings_tokens,
318
- "net_savings_pct": net_savings_pct,
319
- "verified_claim_ge_40_pct": (net_savings_pct >= 40.0),
320
366
  "scenarios": results,
321
367
  }
322
368
 
@@ -335,6 +381,10 @@ def run_empirical_projects_benchmark() -> Dict[str, Any]:
335
381
  ("Selenium Suite", os.path.join(REPO_ROOT, "examples", "selenium-suite")),
336
382
  ]
337
383
 
384
+ info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
385
+ version, commit = get_repo_version_and_commit(REPO_ROOT)
386
+ now_iso = datetime.now(timezone.utc).isoformat()
387
+
338
388
  project_results = []
339
389
  for name, p_path in project_folders:
340
390
  if not os.path.isdir(p_path):
@@ -346,21 +396,45 @@ def run_empirical_projects_benchmark() -> Dict[str, Any]:
346
396
 
347
397
  project_results.append({
348
398
  "project_name": name,
399
+ "measurement_kind": "static_estimate",
400
+ "measurement_source": "repository_reference_suites",
349
401
  "detected_stacks": routed["detected_stacks"],
350
402
  "confidence_score": routed["confidence_score"],
351
- "catalog_baseline": cat["baseline_tokens"],
352
- "catalog_routed": cat["routed_tokens"],
403
+ "static_estimated_tokens": hyd["routed_tokens"],
404
+ "catalog_summary_tokens": cat["routed_tokens"],
405
+ "catalog_baseline_tokens": cat["baseline_tokens"],
353
406
  "catalog_saved_tokens": cat["tokens_saved"],
354
407
  "catalog_reduction_pct": cat["reduction_pct"],
355
- "hydrated_baseline": hyd["baseline_tokens"],
356
- "hydrated_routed": hyd["routed_tokens"],
408
+ "hydrated_baseline_tokens": hyd["baseline_tokens"],
409
+ "hydrated_routed_tokens": hyd["routed_tokens"],
357
410
  "hydrated_saved_tokens": hyd["tokens_saved"],
358
411
  "hydrated_reduction_pct": hyd["reduction_pct"],
359
412
  "avoided_disk_bytes": rf.get("avoided_bytes", 0),
413
+ "provider_reported_input_tokens": None,
414
+ "provider_reported_output_tokens": None,
415
+ "provider_cached_tokens": None,
416
+ "provider_observed": False,
417
+ # Backward compatibility fields
418
+ "catalog_baseline": cat["baseline_tokens"],
419
+ "catalog_routed": cat["routed_tokens"],
420
+ "hydrated_baseline": hyd["baseline_tokens"],
421
+ "hydrated_routed": hyd["routed_tokens"],
360
422
  })
361
423
 
362
424
  return {
363
- "timestamp": datetime.now(timezone.utc).isoformat(),
425
+ "timestamp": now_iso,
426
+ "measurement_kind": "static_estimate",
427
+ "measurement_source": "disk_workspace_inspection",
428
+ "tokenizer_engine": info.get("engine", "heuristic_polyglot"),
429
+ "encoding": info.get("encoding", "cl100k_base"),
430
+ "is_exact": info.get("is_exact", False),
431
+ "version": version,
432
+ "commit": commit,
433
+ "provider_observed": False,
434
+ "provider_reported_input_tokens": None,
435
+ "provider_reported_output_tokens": None,
436
+ "provider_cached_tokens": None,
437
+ "note": "Real project filesystem workspace inspections evaluated via static rules_router hydration; avoided bytes represent disk I/O not context tokens.",
364
438
  "projects_count": len(project_results),
365
439
  "results": project_results,
366
440
  }
@@ -370,8 +444,22 @@ def run_benchmark(dataset_dir: Optional[str] = None) -> Dict[str, Any]:
370
444
  """Runs combined dual-track benchmark."""
371
445
  synthetic = run_synthetic_benchmark(dataset_dir)
372
446
  empirical = run_empirical_projects_benchmark()
447
+ info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
448
+ version, commit = get_repo_version_and_commit(REPO_ROOT)
449
+ now_iso = datetime.now(timezone.utc).isoformat()
450
+
373
451
  return {
374
- "timestamp": datetime.now(timezone.utc).isoformat(),
452
+ "timestamp": now_iso,
453
+ "measurement_kind": "dual_track_benchmark",
454
+ "tokenizer_engine": info.get("engine", "heuristic_polyglot"),
455
+ "encoding": info.get("encoding", "cl100k_base"),
456
+ "is_exact": info.get("is_exact", False),
457
+ "version": version,
458
+ "commit": commit,
459
+ "provider_observed": False,
460
+ "provider_reported_input_tokens": None,
461
+ "provider_reported_output_tokens": None,
462
+ "provider_cached_tokens": None,
375
463
  "synthetic_track": synthetic,
376
464
  "empirical_track": empirical,
377
465
  # Backward compatibility fields
@@ -388,16 +476,17 @@ def format_markdown_report(bench: Dict[str, Any]) -> str:
388
476
  """Formats benchmark results in a clean, scientifically honest markdown report."""
389
477
  syn = bench.get("synthetic_track", bench)
390
478
  emp = bench.get("empirical_track", {})
391
- info = get_estimator_info() if "get_estimator_info" in globals() else {"engine": "cl100k_base", "is_exact": True, "encoding": "cl100k_base"}
479
+ info = get_estimator_info() if callable(globals().get("get_estimator_info")) else {"engine": "heuristic_polyglot", "is_exact": False, "encoding": "cl100k_base"}
392
480
 
393
481
  md = [
394
482
  "# Empirical Context & Token Reduction Benchmark (Dual-Track)",
395
483
  "",
396
484
  "> **Scientific Disclosure:** This benchmark distinguishes between:",
397
485
  "> 1. **Track 1 (Synthetic Micro-Workloads):** Deterministic micro-workloads evaluating isolated context patterns.",
398
- "> 2. **Track 2 (Empirical Reference Suites):** Live measurements on real example workspaces within the repository.",
399
- f"> - **Tokenizer Engine:** `{info.get('engine', 'cl100k_base')}` (Exact: `{info.get('is_exact', True)}`)",
400
- f"> - **Encoding Scope:** `{info.get('encoding', 'cl100k_base')}` (Standard OpenAI BPE encoding for LLMs; benchmark artifact specifies exact tokenizer scope).",
486
+ "> 2. **Track 2 (Empirical Reference Suites):** Live static file inspections on real example workspaces within the repository.",
487
+ f"> - **Tokenizer Engine:** `{info.get('engine', 'heuristic_polyglot')}` (Exact: `{info.get('is_exact', False)}`)",
488
+ f"> - **Encoding Scope:** `{info.get('encoding', 'cl100k_base')}` (Standard OpenAI BPE encoding for LLMs).",
489
+ "> - **Billing Notice:** Offline estimates do not represent live LLM provider billing usage. Live provider usage reduction is only verified when real provider response metrics are paired.",
401
490
  "",
402
491
  "## Track 1: Deterministic Micro-Workload Benchmark (Synthetic)",
403
492
  "",
@@ -420,10 +509,10 @@ def format_markdown_report(bench: Dict[str, Any]) -> str:
420
509
  if emp and emp.get("results"):
421
510
  md.extend([
422
511
  "",
423
- "## Track 2: Empirical Reference Suites Benchmark (Real Projects)",
512
+ "## Track 2: Empirical Reference Suites Benchmark (Static Workspace Inspection)",
424
513
  "",
425
514
  "> *Methodology: Dynamically enumerates all 13 rules and 53 skills in repository without static per-skill multiplier assumptions.*",
426
- "> *Disclosure: 'Avoided Disk Read' measures disk file bytes not read from storage, distinguished from prompt context tokens.*",
515
+ "> *Disclosure: 'Avoided Disk Read (I/O)' measures filesystem file size bytes not read from storage; it is strictly distinguished from context tokens and is not billed token savings.*",
427
516
  "",
428
517
  "| Reference Project | Detected Stack | Confidence | Catalog Summary Saved | Full Hydration Saved | Avoided Disk Read (I/O) |",
429
518
  "|---|---|:---:|:---:|:---:|:---:|",
@@ -459,15 +548,22 @@ def main() -> int:
459
548
  parser.add_argument("--dataset-dir", default=None,
460
549
  help="Path to external standardized benchmark dataset directory (containing scenarios.json)")
461
550
  parser.add_argument("--json", action="store_true", help="Output results in JSON format")
462
- parser.add_argument("--save-report", action="store_true", default=True, help="Persist markdown report to benchmarks/CONTEXT_TOKEN_BENCHMARK.md")
551
+ parser.add_argument("--save-report", action="store_true", default=False, help="Persist markdown report to benchmarks/CONTEXT_TOKEN_BENCHMARK.md (default: False)")
552
+ parser.add_argument("--no-write", action="store_true", default=False, help="Completely suppress writing reports or benchmarks to disk")
553
+ parser.add_argument("--output-dir", default=None, help="Directory to write reports instead of default benchmarks/ and reports/")
463
554
  args = parser.parse_args()
464
555
 
465
556
  bench = run_benchmark(dataset_dir=args.dataset_dir)
466
557
 
467
558
  # Save to disk
468
- if args.save_report:
469
- bench_dir = os.path.join(REPO_ROOT, "benchmarks")
470
- reports_dir = os.path.join(REPO_ROOT, "reports")
559
+ should_save = (args.save_report or args.output_dir is not None) and not args.no_write
560
+ if should_save:
561
+ if args.output_dir:
562
+ bench_dir = os.path.abspath(args.output_dir)
563
+ reports_dir = os.path.abspath(args.output_dir)
564
+ else:
565
+ bench_dir = os.path.join(REPO_ROOT, "benchmarks")
566
+ reports_dir = os.path.join(REPO_ROOT, "reports")
471
567
  os.makedirs(bench_dir, exist_ok=True)
472
568
  os.makedirs(reports_dir, exist_ok=True)
473
569
 
@@ -480,9 +576,21 @@ def main() -> int:
480
576
  json.dump(bench, f, indent=2)
481
577
 
482
578
  if args.json:
579
+ # Guarantee single parseable JSON object on stdout
483
580
  print(json.dumps(bench, indent=2))
484
- else:
485
- print(format_markdown_report(bench))
581
+ if bench["net_savings_pct"] < args.verify_min_savings:
582
+ print(
583
+ f"\n[FAIL] Token savings {bench['net_savings_pct']}% is below required threshold of {args.verify_min_savings}%.",
584
+ file=sys.stderr,
585
+ )
586
+ return 1
587
+ print(
588
+ f"\n[PASS] Empirical token reduction verified: {bench['net_savings_pct']}% >= {args.verify_min_savings}% threshold.",
589
+ file=sys.stderr,
590
+ )
591
+ return 0
592
+
593
+ print(format_markdown_report(bench))
486
594
 
487
595
  if bench["net_savings_pct"] < args.verify_min_savings:
488
596
  print(f"\n[FAIL] Token savings {bench['net_savings_pct']}% is below required threshold of {args.verify_min_savings}%.")