@teevee98/synapse-hub 4.1.3 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/git-push-protocol.md +2 -2
- package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +8 -3
- package/.agents/skills/01-web-automation/clean-pom-crafter/examples/standard_test_case.groovy +3 -0
- package/.agents/skills/01-web-automation/clean-pom-crafter/references/pom_patterns.groovy +3 -0
- package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/e2e-testing-patterns/examples/playwright_sharded_config.ts +3 -0
- package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
- package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +7 -2
- package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/playwright-webapp-testing/examples/server_lifecycle_playwright.py +3 -0
- package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +6 -1
- package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +6 -1
- package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +6 -1
- package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +6 -1
- package/.agents/skills/02-test-design/review-spec/SKILL.md +4 -1
- package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +4 -1
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +6 -1
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/examples/playwright_cwv_profiler_fixture.ts +3 -0
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/references/dast_recipes.ts +3 -0
- package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +7 -2
- package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/examples/refactored_page_sample.groovy +3 -0
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/examples/refactored_playwright_pom.ts +3 -0
- package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +4 -1
- package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/examples/surgical_code_window_walkthrough.md +1 -1
- package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +6 -1
- package/AGENTS.md +7 -1
- package/LICENSE +5 -5
- package/NOTICE +1 -1
- package/README.en.md +4 -4
- package/README.md +4 -4
- package/agentskills.json +1 -1
- package/docs/ARCHITECTURE.en.md +2 -2
- package/docs/ARCHITECTURE.md +11 -2
- package/docs/CLI_REFERENCE.en.md +6 -4
- package/docs/CLI_REFERENCE.md +1 -1
- package/docs/CONTRIBUTING.en.md +1 -1
- package/docs/CONTRIBUTING.md +2 -2
- package/docs/PERFORMANCE_AUDIT_REPORT.md +2 -2
- package/docs/TOOL_CATALOG.md +6 -2
- package/docs/research/agent_skill_training_analysis.md +1 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/scripts/generate_skill_examples.py +3 -0
- package/scripts/package_dist.py +3 -0
- package/skills_manifest.json +320 -55
- package/synapse.config.json +1 -1
- package/templates/ui/index.html +2 -2
- package/tools/.infer_baseline.json +3 -796
- package/tools/TOOL_CATALOG.json +63 -3
- package/tools/__init__.py +4 -1
- package/tools/agent_canary.py +3 -0
- package/tools/analyze_project.py +3 -0
- package/tools/api_drift_sentinel.py +3 -0
- package/tools/api_fuzzer.py +182 -118
- package/tools/ast_cache.py +4 -2
- package/tools/author_integrity.py +174 -12
- package/tools/auto_heal.py +7 -2
- package/tools/autogen_qa_broker.py +14 -15
- package/tools/benchmark_swarm_context.py +119 -18
- package/tools/break_ui_generator.py +233 -0
- package/tools/browser_helpers.js +2 -2
- package/tools/bug_reporter.py +3 -0
- package/tools/build_consolidated_report.py +3 -0
- package/tools/bump_version.py +4 -1
- package/tools/cache_resolver.py +7 -3
- package/tools/check_env.py +3 -0
- package/tools/check_version_sync.py +21 -50
- package/tools/ci_healing_hook.py +3 -0
- package/tools/cli.py +116 -22
- package/tools/config_decoupler.py +3 -0
- package/tools/core/__init__.py +3 -0
- package/tools/core/bm25_retriever.py +218 -0
- package/tools/core/file_lock.py +23 -14
- package/tools/core/i18n.py +13 -11
- package/tools/core/memory_bus.py +6 -3
- package/tools/core/process_supervisor.py +28 -18
- package/tools/core/provider_usage_adapter.py +317 -0
- package/tools/core/token_estimator.py +83 -18
- package/tools/count_tokens.py +16 -7
- package/tools/coverage_reporter.py +245 -0
- package/tools/create_mock_docx.py +3 -0
- package/tools/dashboard_generator.py +17 -15
- package/tools/db_sandbox.py +150 -5
- package/tools/debt_harvester.py +2 -2
- package/tools/diagram_architect.py +3 -0
- package/tools/distill_log.py +22 -2
- package/tools/distill_skill.py +191 -0
- package/tools/e2e_flow_prober.py +3 -0
- package/tools/export_postman_collection.py +3 -0
- package/tools/flaky_detector.py +3 -0
- package/tools/fsm_enforcer.py +2 -2
- package/tools/fuzz_payloads.py +3 -0
- package/tools/generate_allure_report.py +3 -0
- package/tools/generate_api_matrix.py +3 -0
- package/tools/generate_jmx_plan.py +3 -0
- package/tools/generate_tc_from_story.py +3 -0
- package/tools/generate_test_reports.py +3 -0
- package/tools/git_pre_commit_guard.py +25 -3
- package/tools/git_worktree_manager.py +3 -0
- package/tools/heal/__init__.py +3 -0
- package/tools/heal_pr_automator.py +3 -0
- package/tools/heuristic_locator_matcher.py +3 -0
- package/tools/hub_publisher.py +3 -3
- package/tools/humanize_linter.py +3 -0
- package/tools/infer_linter.py +19 -9
- package/tools/init_config.py +6 -9
- package/tools/jtl_analyzer.py +3 -0
- package/tools/lint/__init__.py +3 -0
- package/tools/lint_ai_docs.py +3 -0
- package/tools/lookup_tc.py +3 -0
- package/tools/lookup_xpath.py +3 -0
- package/tools/matrix_sharder.py +3 -0
- package/tools/mcp/__init__.py +3 -0
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +4 -1
- package/tools/mcp_server.py +78 -15
- package/tools/md_to_xlsx.py +3 -0
- package/tools/mechanical_ui_gate.py +3 -0
- package/tools/micro_mutator.py +93 -19
- package/tools/network_mock_shield.py +3 -0
- package/tools/newman_analyzer.py +3 -0
- package/tools/omni_emitter.py +3 -0
- package/tools/omni_parser.py +3 -0
- package/tools/omni_transpiler.py +3 -0
- package/tools/output_formatter.py +3 -0
- package/tools/package_release.py +6 -6
- package/tools/pairwise_generator.py +3 -0
- package/tools/pentest_scope_guard.py +401 -58
- package/tools/perf/__init__.py +3 -0
- package/tools/perf/benchmark_perf_runner.py +42 -0
- package/tools/pinpoint_failure.py +3 -0
- package/tools/project_state.py +3 -0
- package/tools/repo_map.py +3 -0
- package/tools/report/__init__.py +3 -0
- package/tools/rules_router.py +426 -34
- package/tools/runaway_stopper.py +3 -0
- package/tools/safety_firewall.py +3 -0
- package/tools/security_scanner.py +80 -0
- package/tools/senior_code_review.py +14 -0
- package/tools/setup_project.py +9 -4
- package/tools/skill_canonical_digest.py +244 -0
- package/tools/skill_collision_guard.py +2 -2
- package/tools/skill_profiles.py +77 -7
- package/tools/skillspector.py +2 -1
- package/tools/slide_deck_exporter.py +477 -0
- package/tools/smart_visual_masker.py +3 -0
- package/tools/sync_skills_manifest.py +166 -0
- package/tools/task_memory_manager.py +260 -0
- package/tools/telemetry_correlator.py +3 -0
- package/tools/template_excel_exporter.py +2 -2
- package/tools/test_impact.py +3 -0
- package/tools/trace_data_flow.py +3 -0
- package/tools/traffic_session_converter.py +3 -0
- package/tools/ui_server.py +2 -2
- package/tools/universal_ir.py +3 -0
- package/tools/update_sentinel.py +15 -9
- package/tools/validate_groovy_scripts.py +3 -0
- package/tools/validate_host_interop.py +485 -0
- package/tools/validate_skills_schema.py +282 -0
- package/tools/verify_all.py +600 -63
- package/tools/visual_diff_canvas.py +3 -0
- package/tools/visual_som_helper.py +3 -0
- package/tools/watch_mode.py +4 -4
- package/tools/wcag_audit.py +3 -0
- package/tools/welcome_page.py +2 -2
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
tools/core/provider_usage_adapter.py — Standardized Provider Usage Ingestion & Context Splitter
|
|
8
|
+
------------------------------------------------------------------------------------------------
|
|
9
|
+
Provides strict boundary enforcement between:
|
|
10
|
+
1. Real LLM provider-reported tokens (OpenAI, Anthropic, Gemini, Antigravity Transcripts).
|
|
11
|
+
2. Static documentation / rule estimates (cl100k_base / heuristic polyglot).
|
|
12
|
+
3. Synthetic micro-workload benchmark tokens.
|
|
13
|
+
4. Avoided filesystem read bytes (I/O footprint, strictly non-token).
|
|
14
|
+
|
|
15
|
+
Invariants:
|
|
16
|
+
- Zero Guesswork / Zero Interpolation: When live provider usage response is not present,
|
|
17
|
+
provider fields MUST remain None (null).
|
|
18
|
+
- Explicit Provenance: Every record declares measurement_kind, tokenizer/model, version, commit,
|
|
19
|
+
and timestamp.
|
|
20
|
+
- Context Decomposition: Statically estimated skill payloads are subtracted from total provider
|
|
21
|
+
input tokens to accurately isolate the external system prompt, conversation history, and tool schemas.
|
|
22
|
+
- Paired Run Verification: Live reduction claims are strictly guarded and rejected unless BOTH
|
|
23
|
+
baseline and optimized runs originate from real provider-observed responses.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from dataclasses import asdict, dataclass
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
import json
|
|
31
|
+
import logging
|
|
32
|
+
import os
|
|
33
|
+
import subprocess
|
|
34
|
+
from typing import Any, Dict, Optional, Tuple
|
|
35
|
+
|
|
36
|
+
_logger = logging.getLogger("provider_usage_adapter")
|
|
37
|
+
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_repo_version_and_commit(repo_root: Optional[str] = None) -> Tuple[str, str]:
|
|
41
|
+
"""Retrieves current hub version from skills_manifest.json and git commit hash."""
|
|
42
|
+
root = repo_root or _REPO_ROOT
|
|
43
|
+
version = "1.0.0"
|
|
44
|
+
manifest_path = os.path.join(root, "skills_manifest.json")
|
|
45
|
+
if os.path.isfile(manifest_path):
|
|
46
|
+
try:
|
|
47
|
+
with open(manifest_path, "r", encoding="utf-8") as f:
|
|
48
|
+
data = json.load(f)
|
|
49
|
+
version = data.get("version", "1.0.0")
|
|
50
|
+
except (OSError, json.JSONDecodeError, KeyError) as err:
|
|
51
|
+
_logger.debug("Failed to read version from manifest: %s", err)
|
|
52
|
+
|
|
53
|
+
commit = "unversioned"
|
|
54
|
+
try:
|
|
55
|
+
out = subprocess.check_output(
|
|
56
|
+
["git", "rev-parse", "HEAD"],
|
|
57
|
+
cwd=root,
|
|
58
|
+
stderr=subprocess.DEVNULL,
|
|
59
|
+
)
|
|
60
|
+
commit = out.decode("utf-8").strip()
|
|
61
|
+
except (OSError, subprocess.SubprocessError) as err:
|
|
62
|
+
_logger.debug("Failed to get git commit: %s", err)
|
|
63
|
+
commit = "unversioned"
|
|
64
|
+
|
|
65
|
+
return version, commit
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class ProviderUsageRecord:
|
|
70
|
+
"""Immutable, audited record of actual live LLM provider usage."""
|
|
71
|
+
provider: str
|
|
72
|
+
model: str
|
|
73
|
+
measurement_kind: str # Always "provider_observed"
|
|
74
|
+
provider_reported_input_tokens: int
|
|
75
|
+
provider_reported_output_tokens: int
|
|
76
|
+
provider_cached_tokens: Optional[int]
|
|
77
|
+
total_provider_tokens: int
|
|
78
|
+
timestamp: str
|
|
79
|
+
version: str
|
|
80
|
+
commit: str
|
|
81
|
+
raw_usage: Dict[str, Any]
|
|
82
|
+
|
|
83
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
84
|
+
"""Converts to dictionary representation."""
|
|
85
|
+
return asdict(self)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def create_unobserved_provider_record(
|
|
89
|
+
measurement_kind: str = "static_estimate",
|
|
90
|
+
note: str = "No live LLM provider usage observed; fields remain null.",
|
|
91
|
+
) -> Dict[str, Any]:
|
|
92
|
+
"""
|
|
93
|
+
Returns a standardized dictionary for records where live provider data was not captured.
|
|
94
|
+
Guarantees that provider fields remain strictly None without interpolation.
|
|
95
|
+
"""
|
|
96
|
+
version, commit = get_repo_version_and_commit()
|
|
97
|
+
return {
|
|
98
|
+
"measurement_kind": measurement_kind,
|
|
99
|
+
"provider": None,
|
|
100
|
+
"model": None,
|
|
101
|
+
"provider_observed": False,
|
|
102
|
+
"provider_reported_input_tokens": None,
|
|
103
|
+
"provider_reported_output_tokens": None,
|
|
104
|
+
"provider_cached_tokens": None,
|
|
105
|
+
"total_provider_tokens": None,
|
|
106
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
107
|
+
"version": version,
|
|
108
|
+
"commit": commit,
|
|
109
|
+
"note": note,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def parse_provider_usage(
|
|
114
|
+
response: Dict[str, Any],
|
|
115
|
+
provider_hint: Optional[str] = None,
|
|
116
|
+
model_hint: Optional[str] = None,
|
|
117
|
+
repo_root: Optional[str] = None,
|
|
118
|
+
) -> ProviderUsageRecord:
|
|
119
|
+
"""
|
|
120
|
+
Parses live provider usage from standard LLM response payloads.
|
|
121
|
+
Supported payloads:
|
|
122
|
+
1. OpenAI / Compatible: {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "prompt_tokens_details": {"cached_tokens": ...}}}
|
|
123
|
+
2. Anthropic: {"usage": {"input_tokens": ..., "output_tokens": ..., "cache_read_input_tokens": ...}}
|
|
124
|
+
3. Google GenAI: {"usage_metadata": {"prompt_token_count": ..., "candidates_token_count": ..., "cached_content_token_count": ...}}
|
|
125
|
+
4. Antigravity transcript step: {"input_tokens": ..., "output_tokens": ..., "cache_read_tokens": ...}
|
|
126
|
+
5. Mock / Test payload: {"provider": ..., "model": ..., "usage": {"input_tokens": ..., ...}}
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
ValueError: When no valid provider usage payload is present (refuses to guess).
|
|
130
|
+
"""
|
|
131
|
+
if not isinstance(response, dict):
|
|
132
|
+
raise ValueError(f"Expected dict response payload, got {type(response).__name__}")
|
|
133
|
+
|
|
134
|
+
version, commit = get_repo_version_and_commit(repo_root)
|
|
135
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
136
|
+
|
|
137
|
+
# 1. Google GenAI (Gemini SDK) format: usage_metadata
|
|
138
|
+
if "usage_metadata" in response and isinstance(response["usage_metadata"], dict):
|
|
139
|
+
um = response["usage_metadata"]
|
|
140
|
+
in_tok = um.get("prompt_token_count")
|
|
141
|
+
out_tok = um.get("candidates_token_count", 0)
|
|
142
|
+
cache_tok = um.get("cached_content_token_count")
|
|
143
|
+
model = response.get("model") or model_hint or "gemini"
|
|
144
|
+
provider = provider_hint or "google"
|
|
145
|
+
if in_tok is not None:
|
|
146
|
+
total = in_tok + out_tok
|
|
147
|
+
return ProviderUsageRecord(
|
|
148
|
+
provider=provider,
|
|
149
|
+
model=model,
|
|
150
|
+
measurement_kind="provider_observed",
|
|
151
|
+
provider_reported_input_tokens=int(in_tok),
|
|
152
|
+
provider_reported_output_tokens=int(out_tok),
|
|
153
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
154
|
+
total_provider_tokens=total,
|
|
155
|
+
timestamp=now_iso,
|
|
156
|
+
version=version,
|
|
157
|
+
commit=commit,
|
|
158
|
+
raw_usage=um,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# 2. OpenAI / Anthropic format: usage key
|
|
162
|
+
if "usage" in response and isinstance(response["usage"], dict):
|
|
163
|
+
u = response["usage"]
|
|
164
|
+
model = response.get("model") or model_hint or "unknown_model"
|
|
165
|
+
|
|
166
|
+
# OpenAI style: prompt_tokens, completion_tokens
|
|
167
|
+
if "prompt_tokens" in u:
|
|
168
|
+
in_tok = u["prompt_tokens"]
|
|
169
|
+
out_tok = u.get("completion_tokens", 0)
|
|
170
|
+
cache_tok = None
|
|
171
|
+
if "prompt_tokens_details" in u and isinstance(u["prompt_tokens_details"], dict):
|
|
172
|
+
cache_tok = u["prompt_tokens_details"].get("cached_tokens")
|
|
173
|
+
provider = provider_hint or ("openai" if "gpt" in model.lower() else "llm_provider")
|
|
174
|
+
total = u.get("total_tokens", in_tok + out_tok)
|
|
175
|
+
return ProviderUsageRecord(
|
|
176
|
+
provider=provider,
|
|
177
|
+
model=model,
|
|
178
|
+
measurement_kind="provider_observed",
|
|
179
|
+
provider_reported_input_tokens=int(in_tok),
|
|
180
|
+
provider_reported_output_tokens=int(out_tok),
|
|
181
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
182
|
+
total_provider_tokens=int(total),
|
|
183
|
+
timestamp=now_iso,
|
|
184
|
+
version=version,
|
|
185
|
+
commit=commit,
|
|
186
|
+
raw_usage=u,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# Anthropic style: input_tokens, output_tokens
|
|
190
|
+
if "input_tokens" in u:
|
|
191
|
+
in_tok = u["input_tokens"]
|
|
192
|
+
out_tok = u.get("output_tokens", 0)
|
|
193
|
+
cache_tok = u.get("cache_read_input_tokens")
|
|
194
|
+
provider = provider_hint or ("anthropic" if "claude" in model.lower() else "llm_provider")
|
|
195
|
+
total = in_tok + out_tok
|
|
196
|
+
return ProviderUsageRecord(
|
|
197
|
+
provider=provider,
|
|
198
|
+
model=model,
|
|
199
|
+
measurement_kind="provider_observed",
|
|
200
|
+
provider_reported_input_tokens=int(in_tok),
|
|
201
|
+
provider_reported_output_tokens=int(out_tok),
|
|
202
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
203
|
+
total_provider_tokens=total,
|
|
204
|
+
timestamp=now_iso,
|
|
205
|
+
version=version,
|
|
206
|
+
commit=commit,
|
|
207
|
+
raw_usage=u,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# 3. Direct / Antigravity transcript step format: input_tokens, output_tokens directly in root
|
|
211
|
+
if "input_tokens" in response:
|
|
212
|
+
in_tok = response["input_tokens"]
|
|
213
|
+
out_tok = response.get("output_tokens", 0)
|
|
214
|
+
cache_tok = response.get("cache_read_tokens")
|
|
215
|
+
provider = response.get("provider") or provider_hint or "antigravity"
|
|
216
|
+
model = response.get("model") or model_hint or "gemini"
|
|
217
|
+
total = in_tok + out_tok + (cache_tok or 0)
|
|
218
|
+
return ProviderUsageRecord(
|
|
219
|
+
provider=provider,
|
|
220
|
+
model=model,
|
|
221
|
+
measurement_kind="provider_observed",
|
|
222
|
+
provider_reported_input_tokens=int(in_tok),
|
|
223
|
+
provider_reported_output_tokens=int(out_tok),
|
|
224
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
225
|
+
total_provider_tokens=int(total),
|
|
226
|
+
timestamp=now_iso,
|
|
227
|
+
version=version,
|
|
228
|
+
commit=commit,
|
|
229
|
+
raw_usage={k: response[k] for k in ("input_tokens", "output_tokens", "cache_read_tokens") if k in response},
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
raise ValueError(
|
|
233
|
+
"No valid LLM provider usage metadata found in response. "
|
|
234
|
+
"Synapse Hub refuses to fabricate synthetic numbers as live provider data."
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def decompose_prompt_context(
|
|
239
|
+
provider_record: ProviderUsageRecord,
|
|
240
|
+
skill_payload_tokens: int,
|
|
241
|
+
rule_payload_tokens: int = 0,
|
|
242
|
+
) -> Dict[str, Any]:
|
|
243
|
+
"""
|
|
244
|
+
Separates the statically estimated skill/rule documentation payload from the
|
|
245
|
+
overall provider input tokens to isolate external host system instructions,
|
|
246
|
+
conversation history turns, and provider tool calling schemas.
|
|
247
|
+
"""
|
|
248
|
+
if not isinstance(provider_record, ProviderUsageRecord):
|
|
249
|
+
raise TypeError(f"Expected ProviderUsageRecord, got {type(provider_record).__name__}")
|
|
250
|
+
|
|
251
|
+
total_in = provider_record.provider_reported_input_tokens
|
|
252
|
+
managed_tokens = skill_payload_tokens + rule_payload_tokens
|
|
253
|
+
external_wrapper = max(0, total_in - managed_tokens)
|
|
254
|
+
ratio = round((managed_tokens / max(1, total_in)) * 100.0, 2)
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
"measurement_kind": "provider_observed_with_decomposition",
|
|
258
|
+
"provider": provider_record.provider,
|
|
259
|
+
"model": provider_record.model,
|
|
260
|
+
"provider_reported_input_tokens": total_in,
|
|
261
|
+
"provider_reported_output_tokens": provider_record.provider_reported_output_tokens,
|
|
262
|
+
"provider_cached_tokens": provider_record.provider_cached_tokens,
|
|
263
|
+
"skill_payload_static_estimated_tokens": skill_payload_tokens,
|
|
264
|
+
"rule_payload_static_estimated_tokens": rule_payload_tokens,
|
|
265
|
+
"total_managed_static_tokens": managed_tokens,
|
|
266
|
+
"external_system_prompt_and_wrapper_tokens": external_wrapper,
|
|
267
|
+
"managed_payload_ratio_pct": ratio,
|
|
268
|
+
"timestamp": provider_record.timestamp,
|
|
269
|
+
"version": provider_record.version,
|
|
270
|
+
"commit": provider_record.commit,
|
|
271
|
+
"disclaimer": (
|
|
272
|
+
"Provider reported input tokens reflect total live API billing usage. "
|
|
273
|
+
"Skill and rule payloads are statically estimated via offline tokenizer and subtracted "
|
|
274
|
+
"to isolate unmanaged host system prompt, user prompt, and provider tool schemas."
|
|
275
|
+
),
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def compare_paired_provider_runs(
|
|
280
|
+
baseline_run: ProviderUsageRecord,
|
|
281
|
+
optimized_run: ProviderUsageRecord,
|
|
282
|
+
) -> Dict[str, Any]:
|
|
283
|
+
"""
|
|
284
|
+
Computes verified token reduction between two live provider runs.
|
|
285
|
+
Enforces scientific honesty: strictly refuses to claim live provider reduction unless
|
|
286
|
+
BOTH baseline and optimized runs originate from real provider_observed responses.
|
|
287
|
+
"""
|
|
288
|
+
if not isinstance(baseline_run, ProviderUsageRecord) or not isinstance(optimized_run, ProviderUsageRecord):
|
|
289
|
+
raise TypeError("Both baseline_run and optimized_run must be verified ProviderUsageRecord instances.")
|
|
290
|
+
|
|
291
|
+
if baseline_run.measurement_kind != "provider_observed" or optimized_run.measurement_kind != "provider_observed":
|
|
292
|
+
raise ValueError("Both runs must have measurement_kind == 'provider_observed' to claim live provider reduction.")
|
|
293
|
+
|
|
294
|
+
in_saved = baseline_run.provider_reported_input_tokens - optimized_run.provider_reported_input_tokens
|
|
295
|
+
out_saved = baseline_run.provider_reported_output_tokens - optimized_run.provider_reported_output_tokens
|
|
296
|
+
total_saved = baseline_run.total_provider_tokens - optimized_run.total_provider_tokens
|
|
297
|
+
reduction_pct = round((total_saved / max(1, baseline_run.total_provider_tokens)) * 100.0, 2)
|
|
298
|
+
|
|
299
|
+
return {
|
|
300
|
+
"verified_provider_savings": True,
|
|
301
|
+
"measurement_kind": "paired_provider_observed",
|
|
302
|
+
"baseline_provider": baseline_run.provider,
|
|
303
|
+
"baseline_model": baseline_run.model,
|
|
304
|
+
"baseline_input_tokens": baseline_run.provider_reported_input_tokens,
|
|
305
|
+
"baseline_output_tokens": baseline_run.provider_reported_output_tokens,
|
|
306
|
+
"baseline_total_tokens": baseline_run.total_provider_tokens,
|
|
307
|
+
"optimized_provider": optimized_run.provider,
|
|
308
|
+
"optimized_model": optimized_run.model,
|
|
309
|
+
"optimized_input_tokens": optimized_run.provider_reported_input_tokens,
|
|
310
|
+
"optimized_output_tokens": optimized_run.provider_reported_output_tokens,
|
|
311
|
+
"optimized_total_tokens": optimized_run.total_provider_tokens,
|
|
312
|
+
"input_tokens_saved": in_saved,
|
|
313
|
+
"output_tokens_saved": out_saved,
|
|
314
|
+
"net_tokens_saved": total_saved,
|
|
315
|
+
"reduction_pct": reduction_pct,
|
|
316
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
317
|
+
}
|
|
@@ -22,6 +22,7 @@ Key Capabilities:
|
|
|
22
22
|
from __future__ import annotations
|
|
23
23
|
|
|
24
24
|
import glob
|
|
25
|
+
import hashlib
|
|
25
26
|
import json
|
|
26
27
|
import os
|
|
27
28
|
import re
|
|
@@ -39,15 +40,30 @@ try:
|
|
|
39
40
|
except ImportError:
|
|
40
41
|
_TIKTOKEN_ENCODER = None
|
|
41
42
|
|
|
42
|
-
|
|
43
|
+
# Cache mapping canonical manifest_path -> (mtime, parsed_manifest)
|
|
44
|
+
_SKILLS_MANIFEST_CACHE: Dict[str, Any] = {}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def clear_skills_manifest_cache() -> None:
|
|
48
|
+
"""Clears the in-memory skills manifest cache."""
|
|
49
|
+
global _SKILLS_MANIFEST_CACHE
|
|
50
|
+
_SKILLS_MANIFEST_CACHE.clear()
|
|
43
51
|
|
|
44
52
|
|
|
45
53
|
def get_estimator_info() -> Dict[str, Any]:
|
|
46
54
|
"""Returns active tokenizer engine metadata, exactness status, and model hint."""
|
|
55
|
+
pkg_version = None
|
|
56
|
+
if _TIKTOKEN_ENCODER is not None:
|
|
57
|
+
try:
|
|
58
|
+
pkg_version = getattr(tiktoken, "__version__", None)
|
|
59
|
+
except Exception:
|
|
60
|
+
pkg_version = None
|
|
61
|
+
|
|
47
62
|
return {
|
|
48
63
|
"engine": "cl100k_base" if _TIKTOKEN_ENCODER is not None else "heuristic_polyglot",
|
|
49
64
|
"is_exact": _TIKTOKEN_ENCODER is not None,
|
|
50
65
|
"encoding": "cl100k_base",
|
|
66
|
+
"package_version": pkg_version,
|
|
51
67
|
"calibrated_ratios": {
|
|
52
68
|
"ascii_chars_per_token": 3.85,
|
|
53
69
|
"non_ascii_chars_per_token": 1.80,
|
|
@@ -70,7 +86,8 @@ def count_tokens(text: str, model_hint: str = "cl100k") -> int:
|
|
|
70
86
|
try:
|
|
71
87
|
return len(_TIKTOKEN_ENCODER.encode(text, disallowed_special=()))
|
|
72
88
|
except Exception as err:
|
|
73
|
-
|
|
89
|
+
import sys
|
|
90
|
+
sys.stderr.write(f"Warning: tiktoken encoding error, fallback to heuristic: {err}\n")
|
|
74
91
|
|
|
75
92
|
# Heuristic calibrated against standard LLM subword tokenizers
|
|
76
93
|
non_ascii_chars = len([c for c in text if ord(c) > 127])
|
|
@@ -100,24 +117,60 @@ def count_file_tokens(file_path: str) -> int:
|
|
|
100
117
|
|
|
101
118
|
|
|
102
119
|
def _get_skills_manifest(repo_root: str) -> Dict[str, Dict[str, Any]]:
|
|
103
|
-
"""
|
|
120
|
+
"""
|
|
121
|
+
Loads and caches skills_manifest.json mapping skill_id to metadata.
|
|
122
|
+
Cache is keyed by the canonical realpath of the manifest file and validated
|
|
123
|
+
against file mtime_ns, size, and content SHA-256 digest to guarantee that
|
|
124
|
+
mutations are detected even if file modification time remains identical.
|
|
125
|
+
"""
|
|
104
126
|
global _SKILLS_MANIFEST_CACHE
|
|
105
|
-
|
|
106
|
-
return _SKILLS_MANIFEST_CACHE
|
|
127
|
+
manifest_path = os.path.realpath(os.path.join(repo_root, "skills_manifest.json"))
|
|
107
128
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
129
|
+
if not os.path.isfile(manifest_path):
|
|
130
|
+
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
131
|
+
return {}
|
|
132
|
+
|
|
133
|
+
try:
|
|
134
|
+
st = os.stat(manifest_path)
|
|
135
|
+
current_mtime_ns = st.st_mtime_ns
|
|
136
|
+
current_size = st.st_size
|
|
137
|
+
except OSError:
|
|
138
|
+
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
139
|
+
return {}
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
with open(manifest_path, "rb") as f:
|
|
143
|
+
raw_bytes = f.read()
|
|
144
|
+
current_digest = hashlib.sha256(raw_bytes).digest()
|
|
145
|
+
except OSError:
|
|
146
|
+
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
147
|
+
return {}
|
|
148
|
+
|
|
149
|
+
if manifest_path in _SKILLS_MANIFEST_CACHE:
|
|
150
|
+
cached_entry = _SKILLS_MANIFEST_CACHE[manifest_path]
|
|
151
|
+
if len(cached_entry) == 4:
|
|
152
|
+
cached_mtime_ns, cached_size, cached_digest, cached_data = cached_entry
|
|
153
|
+
if cached_digest == current_digest:
|
|
154
|
+
return cached_data
|
|
155
|
+
elif len(cached_entry) == 2:
|
|
156
|
+
cached_mtime, cached_data = cached_entry
|
|
157
|
+
if cached_mtime == st.st_mtime:
|
|
158
|
+
return cached_data
|
|
159
|
+
|
|
160
|
+
manifest_data: Dict[str, Dict[str, Any]] = {}
|
|
161
|
+
try:
|
|
162
|
+
data = json.loads(raw_bytes.decode("utf-8"))
|
|
163
|
+
for s in data.get("skills", []):
|
|
164
|
+
sid = s.get("id")
|
|
165
|
+
if sid:
|
|
166
|
+
manifest_data[sid] = s
|
|
167
|
+
_SKILLS_MANIFEST_CACHE[manifest_path] = (current_mtime_ns, current_size, current_digest, manifest_data)
|
|
168
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as err:
|
|
169
|
+
import sys
|
|
170
|
+
sys.stderr.write(f"Warning: Failed to load manifest at {manifest_path}: {err}\n")
|
|
171
|
+
_SKILLS_MANIFEST_CACHE.pop(manifest_path, None)
|
|
172
|
+
|
|
173
|
+
return manifest_data
|
|
121
174
|
|
|
122
175
|
|
|
123
176
|
def serialize_skill_summary(skill_name: str, meta: Optional[Dict[str, Any]] = None) -> str:
|
|
@@ -179,6 +232,12 @@ def calculate_hydration_payload(
|
|
|
179
232
|
catalog_skill_tokens = count_tokens("\n".join(catalog_lines)) if catalog_lines else 0
|
|
180
233
|
|
|
181
234
|
return {
|
|
235
|
+
"measurement_kind": "static_estimate",
|
|
236
|
+
"static_estimated_tokens": rule_tokens + full_skill_tokens,
|
|
237
|
+
"provider_reported_input_tokens": None,
|
|
238
|
+
"provider_reported_output_tokens": None,
|
|
239
|
+
"provider_cached_tokens": None,
|
|
240
|
+
"avoided_disk_bytes": 0,
|
|
182
241
|
"rule_tokens": rule_tokens,
|
|
183
242
|
"rule_bytes": rule_bytes,
|
|
184
243
|
"skill_catalog_tokens": catalog_skill_tokens,
|
|
@@ -233,6 +292,12 @@ def get_full_unrouted_baseline(repo_root: str) -> Dict[str, Any]:
|
|
|
233
292
|
total_cat_tokens = count_tokens("\n".join(catalog_lines)) if catalog_lines else 0
|
|
234
293
|
|
|
235
294
|
return {
|
|
295
|
+
"measurement_kind": "static_estimate",
|
|
296
|
+
"static_estimated_tokens": total_rule_tokens + total_skill_tokens,
|
|
297
|
+
"provider_reported_input_tokens": None,
|
|
298
|
+
"provider_reported_output_tokens": None,
|
|
299
|
+
"provider_cached_tokens": None,
|
|
300
|
+
"avoided_disk_bytes": 0,
|
|
236
301
|
"rule_count": len(all_rule_files),
|
|
237
302
|
"rule_tokens": total_rule_tokens,
|
|
238
303
|
"rule_bytes": total_rule_bytes,
|
package/tools/count_tokens.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: © 2026 Thế Vị (@TeeVee98) <https://github.com/TeeVee98/Synapse-Hub>
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
2
5
|
"""
|
|
3
6
|
Cross-Platform Token & Cost Auditor (Synapse-Hub)
|
|
4
7
|
Author: Thế Vị (@TeeVee98) <https://github.com/TeeVee98/Synapse-Hub>
|
|
@@ -147,8 +150,9 @@ def get_budget_config() -> Dict[str, int]:
|
|
|
147
150
|
"budget_5h": data.get("budget_5h", DEFAULT_5H_BUDGET),
|
|
148
151
|
"budget_7d": data.get("budget_7d", DEFAULT_7D_BUDGET),
|
|
149
152
|
}
|
|
150
|
-
except (OSError, json.JSONDecodeError):
|
|
151
|
-
|
|
153
|
+
except (OSError, json.JSONDecodeError) as err:
|
|
154
|
+
import logging
|
|
155
|
+
logging.getLogger("count_tokens").debug("Failed to read budget config: %s", err)
|
|
152
156
|
return {
|
|
153
157
|
"budget_5h": DEFAULT_5H_BUDGET,
|
|
154
158
|
"budget_7d": DEFAULT_7D_BUDGET,
|
|
@@ -221,8 +225,9 @@ class AntigravityAdapter:
|
|
|
221
225
|
}
|
|
222
226
|
finally:
|
|
223
227
|
conn.close()
|
|
224
|
-
except (sqlite3.Error, OSError):
|
|
225
|
-
|
|
228
|
+
except (sqlite3.Error, OSError) as err:
|
|
229
|
+
import logging
|
|
230
|
+
logging.getLogger("count_tokens").debug("Failed to query database: %s", err)
|
|
226
231
|
|
|
227
232
|
conversations = []
|
|
228
233
|
for entry in os.listdir(brain_dir):
|
|
@@ -402,7 +407,7 @@ class AntigravityAdapter:
|
|
|
402
407
|
usage_hours["cache"] += cache_tok
|
|
403
408
|
usage_hours["output"] += out_tok
|
|
404
409
|
except (OSError, UnicodeDecodeError):
|
|
405
|
-
|
|
410
|
+
continue
|
|
406
411
|
|
|
407
412
|
budgets = get_budget_config()
|
|
408
413
|
b_5h = budgets["budget_5h"]
|
|
@@ -679,8 +684,7 @@ def detect_platform_from_file(file_path: str) -> str:
|
|
|
679
684
|
if "claude" in line.lower():
|
|
680
685
|
return "claude"
|
|
681
686
|
except (OSError, UnicodeDecodeError):
|
|
682
|
-
|
|
683
|
-
return "antigravity"
|
|
687
|
+
return "antigravity"
|
|
684
688
|
|
|
685
689
|
|
|
686
690
|
def get_adapter(platform: str):
|
|
@@ -771,6 +775,10 @@ def calculate_hub_savings(input_tokens: int, cache_tokens: int, output_tokens: i
|
|
|
771
775
|
|
|
772
776
|
return {
|
|
773
777
|
"model_name": pricing["name"],
|
|
778
|
+
"measurement_kind": "counterfactual_model_estimate",
|
|
779
|
+
"provider_reported_input_tokens": input_tokens,
|
|
780
|
+
"provider_reported_output_tokens": output_tokens,
|
|
781
|
+
"provider_cached_tokens": cache_tokens,
|
|
774
782
|
"non_hub_input": estimated_non_hub_input,
|
|
775
783
|
"non_hub_cache": estimated_non_hub_cache,
|
|
776
784
|
"non_hub_total": estimated_non_hub_total,
|
|
@@ -784,6 +792,7 @@ def calculate_hub_savings(input_tokens: int, cache_tokens: int, output_tokens: i
|
|
|
784
792
|
"savings_pct": round(savings_pct, 1),
|
|
785
793
|
"cost_saved_usd": round(cost_saved_usd, 4),
|
|
786
794
|
"cost_savings_pct": round(cost_savings_pct, 1),
|
|
795
|
+
"note": "Actual hub tokens are parsed from session transcripts. Non-hub baseline is an estimated counterfactual.",
|
|
787
796
|
}
|
|
788
797
|
|
|
789
798
|
|