@teevee98/synapse-hub 4.1.4 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/git-push-protocol.md +2 -2
- package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +52 -3
- package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +55 -5
- package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +62 -67
- package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +54 -4
- package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +62 -12
- package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +55 -8
- package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +60 -12
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
- package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +54 -91
- package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +59 -11
- package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +50 -4
- package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +55 -9
- package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +55 -9
- package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +55 -8
- package/.agents/skills/02-test-design/review-spec/SKILL.md +50 -6
- package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +50 -4
- package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +56 -5
- package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +53 -7
- package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +86 -36
- package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +54 -6
- package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +50 -6
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +50 -3
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +63 -14
- package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +51 -5
- package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +54 -4
- package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +51 -5
- package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +53 -6
- package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +54 -8
- package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +60 -24
- package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +52 -6
- package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +56 -10
- package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +58 -12
- package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +58 -12
- package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +53 -5
- package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +56 -5
- package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +53 -4
- package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +58 -5
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +57 -11
- package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +83 -34
- package/.agents/skills/08-code-quality/graphify-impact-analyzer/examples/dependency_blast_radius.json +3 -4
- package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +55 -10
- package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +49 -6
- package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +52 -7
- package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +55 -10
- package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +51 -5
- package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +54 -17
- package/.agents/skills/09-ci-and-devops/github-actions-templates/examples/matrix_test_pipeline.yml +2 -1
- package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +50 -4
- package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +53 -7
- package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +50 -4
- package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +58 -13
- package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +51 -5
- package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +57 -10
- package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +50 -4
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +50 -4
- package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +55 -9
- package/.cursor/rules/synapse-qa.mdc +20 -7
- package/.cursorrules +19 -5
- package/.windsurfrules +18 -3
- package/AGENTS.md +51 -59
- package/CLAUDE.md +38 -24
- package/LICENSE +5 -5
- package/NOTICE +1 -1
- package/README.en.md +4 -4
- package/README.md +4 -4
- package/agentskills.json +1 -1
- package/docs/ARCHITECTURE.en.md +2 -2
- package/docs/ARCHITECTURE.md +11 -2
- package/docs/CLIENT_SUPPORT_MATRIX.md +44 -0
- package/docs/CLI_REFERENCE.en.md +6 -4
- package/docs/CLI_REFERENCE.md +1 -1
- package/docs/CONTRIBUTING.en.md +1 -1
- package/docs/CONTRIBUTING.md +2 -2
- package/docs/SIDE_EFFECTING_TOOLS_INVENTORY.md +233 -0
- package/docs/TOOL_CATALOG.md +6 -2
- package/docs/research/agent_skill_training_analysis.md +1 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/skills_manifest.json +320 -55
- package/synapse.config.json +1 -1
- package/templates/ai_bridges/CLAUDE.md +29 -26
- package/templates/ai_bridges/copilot-instructions.md +14 -5
- package/templates/ai_bridges/cursor_synapse_qa.mdc +16 -7
- package/templates/ai_bridges/windsurfrules +14 -3
- package/templates/ui/index.html +2 -2
- package/tools/TOOLS_RUNBOOK.md +1 -1
- package/tools/TOOL_CATALOG.json +63 -3
- package/tools/__init__.py +1 -1
- package/tools/api_fuzzer.py +447 -132
- package/tools/author_integrity.py +121 -11
- package/tools/benchmark_swarm_context.py +130 -22
- package/tools/break_ui_generator.py +233 -0
- package/tools/browser_helpers.js +2 -2
- package/tools/check_version_sync.py +18 -50
- package/tools/cli.py +110 -19
- package/tools/core/bm25_retriever.py +218 -0
- package/tools/core/i18n.py +5 -5
- package/tools/core/provider_usage_adapter.py +383 -0
- package/tools/core/token_estimator.py +81 -15
- package/tools/count_tokens.py +7 -2
- package/tools/coverage_reporter.py +245 -0
- package/tools/dashboard_generator.py +8 -8
- package/tools/db_sandbox.py +290 -9
- package/tools/distill_log.py +1 -1
- package/tools/distill_skill.py +206 -0
- package/tools/git_pre_commit_guard.py +202 -1
- package/tools/humanize_linter.py +3 -0
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +104 -16
- package/tools/micro_mutator.py +114 -15
- package/tools/pentest_scope_guard.py +674 -58
- package/tools/perf/benchmark_perf_runner.py +42 -0
- package/tools/project_state.py +29 -5
- package/tools/reversible_sandbox.py +58 -13
- package/tools/rules_router.py +473 -60
- package/tools/security_scanner.py +80 -0
- package/tools/senior_code_review.py +14 -0
- package/tools/skill_canonical_digest.py +302 -0
- package/tools/skill_profiles.py +77 -7
- package/tools/slide_deck_exporter.py +477 -0
- package/tools/sync_agent_rules.py +187 -61
- package/tools/sync_skills_manifest.py +166 -0
- package/tools/task_memory_manager.py +275 -0
- package/tools/ui_server.py +3 -1
- package/tools/update_sentinel.py +3 -3
- package/tools/validate_host_interop.py +485 -0
- package/tools/validate_skills_schema.py +575 -0
- package/tools/verify_all.py +668 -44
- package/tools/welcome_page.py +2 -2
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: © 2026 Thế Vị (@TeeVee98) <https://github.com/TeeVee98/Synapse-Hub>
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
# -*- coding: utf-8 -*-
|
|
6
|
+
"""
|
|
7
|
+
tools/core/bm25_retriever.py — Zero-Dependency Pure Python BM25 Knowledge Retriever
|
|
8
|
+
Inspired by nextlevelbuilder/ui-ux-pro-max-skill core.py algorithm.
|
|
9
|
+
Enables instant sub-millisecond retrieval of guidelines, rules, and QA patterns
|
|
10
|
+
with zero external dependencies and guaranteed < 300 character truncation (targeted context reduction).
|
|
11
|
+
100% UTF-8 No BOM.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import os
|
|
20
|
+
import re
|
|
21
|
+
import sys
|
|
22
|
+
from collections import Counter
|
|
23
|
+
from typing import Any, Dict, List, Tuple
|
|
24
|
+
|
|
25
|
+
# Enforce UTF-8 stdout encoding
|
|
26
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
27
|
+
try:
|
|
28
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
29
|
+
sys.stderr.reconfigure(encoding="utf-8")
|
|
30
|
+
except Exception:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
TRUNCATE_AT: int = 300
|
|
34
|
+
|
|
35
|
+
# Common English / QA stopwords
|
|
36
|
+
STOPWORDS: set[str] = {
|
|
37
|
+
"a", "an", "the", "and", "or", "in", "on", "at", "to", "for", "with",
|
|
38
|
+
"by", "of", "from", "is", "are", "was", "were", "be", "been", "being",
|
|
39
|
+
"have", "has", "had", "do", "does", "did", "this", "that", "these",
|
|
40
|
+
"those", "it", "its", "as", "if", "each", "which", "into", "than", "then"
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# Domain acronyms & synonym expansion mapping
|
|
44
|
+
SYNONYMS: Dict[str, List[str]] = {
|
|
45
|
+
"pom": ["page", "object", "model", "locator"],
|
|
46
|
+
"a11y": ["accessibility", "wcag", "aria", "contrast"],
|
|
47
|
+
"perf": ["performance", "latency", "p95", "throughput", "jmeter", "k6"],
|
|
48
|
+
"sec": ["security", "credential", "sarif", "cwe", "vuln"],
|
|
49
|
+
"fsm": ["finite", "state", "machine", "stage", "lifecycle"],
|
|
50
|
+
"dod": ["definition", "done", "gate", "verify", "quality"],
|
|
51
|
+
"ast": ["abstract", "syntax", "tree", "mutation", "linter"],
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def tokenize(text: str) -> List[str]:
|
|
56
|
+
"""Tokenizes text into lowercase alphanumeric tokens, filtering stopwords."""
|
|
57
|
+
if not text:
|
|
58
|
+
return []
|
|
59
|
+
words = re.findall(r"[a-zA-Z0-9_]+", text.lower())
|
|
60
|
+
tokens: List[str] = []
|
|
61
|
+
for w in words:
|
|
62
|
+
if len(w) > 1 and w not in STOPWORDS:
|
|
63
|
+
tokens.append(w)
|
|
64
|
+
if w in SYNONYMS:
|
|
65
|
+
tokens.extend(SYNONYMS[w])
|
|
66
|
+
return tokens
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class BM25Document:
|
|
70
|
+
"""Represents an indexed document with precomputed tokens."""
|
|
71
|
+
|
|
72
|
+
def __init__(self, doc_id: str, title: str, content: str, source: str = ""):
|
|
73
|
+
self.doc_id = doc_id
|
|
74
|
+
self.title = title
|
|
75
|
+
self.content = content
|
|
76
|
+
self.source = source
|
|
77
|
+
self.tokens = tokenize(f"{title} {content}")
|
|
78
|
+
self.length = len(self.tokens)
|
|
79
|
+
self.term_freqs = Counter(self.tokens)
|
|
80
|
+
|
|
81
|
+
def snippet(self, max_len: int = TRUNCATE_AT) -> str:
|
|
82
|
+
"""Returns a trimmed snippet bounded by max_len."""
|
|
83
|
+
cleaned = re.sub(r"\s+", " ", self.content).strip()
|
|
84
|
+
if len(cleaned) <= max_len:
|
|
85
|
+
return cleaned
|
|
86
|
+
return cleaned[:max_len].rstrip() + "..."
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class BM25Index:
|
|
90
|
+
"""
|
|
91
|
+
Rank BM25 search index using the Okapi BM25 formula:
|
|
92
|
+
IDF(q) = ln((N - n(q) + 0.5) / (n(q) + 0.5) + 1)
|
|
93
|
+
Score = sum(IDF(q) * (f(q, D) * (k1 + 1)) / (f(q, D) + k1 * (1 - b + b * (|D| / avgdl))))
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
def __init__(self, k1: float = 1.5, b: float = 0.75):
|
|
97
|
+
self.k1 = k1
|
|
98
|
+
self.b = b
|
|
99
|
+
self.documents: List[BM25Document] = []
|
|
100
|
+
self.doc_count: int = 0
|
|
101
|
+
self.avg_doc_len: float = 0.0
|
|
102
|
+
self.doc_freqs: Counter[str] = Counter()
|
|
103
|
+
self.idf_cache: Dict[str, float] = {}
|
|
104
|
+
|
|
105
|
+
def add_document(self, doc_id: str, title: str, content: str, source: str = "") -> None:
|
|
106
|
+
"""Adds a single document to the index."""
|
|
107
|
+
doc = BM25Document(doc_id, title, content, source)
|
|
108
|
+
self.documents.append(doc)
|
|
109
|
+
self.doc_count = len(self.documents)
|
|
110
|
+
|
|
111
|
+
def build(self) -> None:
|
|
112
|
+
"""Finalizes the index, computes average document length and IDF scores."""
|
|
113
|
+
self.doc_freqs.clear()
|
|
114
|
+
total_len = 0
|
|
115
|
+
for doc in self.documents:
|
|
116
|
+
total_len += doc.length
|
|
117
|
+
for token in set(doc.tokens):
|
|
118
|
+
self.doc_freqs[token] += 1
|
|
119
|
+
|
|
120
|
+
self.avg_doc_len = (total_len / self.doc_count) if self.doc_count > 0 else 0.0
|
|
121
|
+
|
|
122
|
+
# Precompute IDF
|
|
123
|
+
self.idf_cache.clear()
|
|
124
|
+
for token, df in self.doc_freqs.items():
|
|
125
|
+
# Robertson-Spärck Jones IDF formula with +1 smoothing
|
|
126
|
+
idf = math.log(((self.doc_count - df + 0.5) / (df + 0.5)) + 1.0)
|
|
127
|
+
self.idf_cache[token] = max(idf, 0.0)
|
|
128
|
+
|
|
129
|
+
def search(
|
|
130
|
+
self, query: str, top_k: int = 3, truncate_at: int = TRUNCATE_AT
|
|
131
|
+
) -> List[Dict[str, Any]]:
|
|
132
|
+
"""Searches documents matching the query and returns top_k ranked results."""
|
|
133
|
+
if not self.documents:
|
|
134
|
+
return []
|
|
135
|
+
|
|
136
|
+
query_tokens = tokenize(query)
|
|
137
|
+
if not query_tokens:
|
|
138
|
+
return []
|
|
139
|
+
|
|
140
|
+
scores: List[Tuple[float, BM25Document]] = []
|
|
141
|
+
for doc in self.documents:
|
|
142
|
+
if doc.length == 0:
|
|
143
|
+
continue
|
|
144
|
+
score = 0.0
|
|
145
|
+
doc_len_ratio = doc.length / self.avg_doc_len if self.avg_doc_len > 0 else 1.0
|
|
146
|
+
len_norm = 1.0 - self.b + self.b * doc_len_ratio
|
|
147
|
+
|
|
148
|
+
for q in query_tokens:
|
|
149
|
+
if q in doc.term_freqs:
|
|
150
|
+
freq = doc.term_freqs[q]
|
|
151
|
+
idf = self.idf_cache.get(q, 0.0)
|
|
152
|
+
numerator = freq * (self.k1 + 1.0)
|
|
153
|
+
denominator = freq + self.k1 * len_norm
|
|
154
|
+
score += idf * (numerator / denominator)
|
|
155
|
+
|
|
156
|
+
if score > 0.0:
|
|
157
|
+
scores.append((score, doc))
|
|
158
|
+
|
|
159
|
+
scores.sort(key=lambda x: x[0], reverse=True)
|
|
160
|
+
|
|
161
|
+
results: List[Dict[str, Any]] = []
|
|
162
|
+
for rank, (score, doc) in enumerate(scores[:top_k], start=1):
|
|
163
|
+
results.append({
|
|
164
|
+
"rank": rank,
|
|
165
|
+
"score": round(score, 4),
|
|
166
|
+
"doc_id": doc.doc_id,
|
|
167
|
+
"title": doc.title,
|
|
168
|
+
"snippet": doc.snippet(truncate_at),
|
|
169
|
+
"source": doc.source,
|
|
170
|
+
})
|
|
171
|
+
return results
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def index_directory(dir_path: str, extensions: Tuple[str, ...] = (".md", ".txt")) -> BM25Index:
|
|
175
|
+
"""Indexes all files with matching extensions under dir_path."""
|
|
176
|
+
index = BM25Index()
|
|
177
|
+
if not os.path.exists(dir_path):
|
|
178
|
+
return index
|
|
179
|
+
|
|
180
|
+
for root, _, files in os.walk(dir_path):
|
|
181
|
+
for f in files:
|
|
182
|
+
if any(f.endswith(ext) for ext in extensions):
|
|
183
|
+
full_path = os.path.join(root, f)
|
|
184
|
+
try:
|
|
185
|
+
with open(full_path, "r", encoding="utf-8", errors="replace") as fp:
|
|
186
|
+
content = fp.read()
|
|
187
|
+
rel_path = os.path.relpath(full_path, dir_path)
|
|
188
|
+
title = f.replace(".md", "").replace("_", " ").title()
|
|
189
|
+
index.add_document(doc_id=rel_path, title=title, content=content, source=full_path)
|
|
190
|
+
except OSError:
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
index.build()
|
|
194
|
+
return index
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def main() -> None:
|
|
198
|
+
parser = argparse.ArgumentParser(description="BM25 Knowledge Retriever for Synapse Hub")
|
|
199
|
+
parser.add_argument("--query", "-q", type=str, required=True, help="Search query string")
|
|
200
|
+
parser.add_argument("--dir", "-d", type=str, default=".agents/rules", help="Directory to search (default: .agents/rules)")
|
|
201
|
+
parser.add_argument("--top-k", "-k", type=int, default=3, help="Number of results to return (default: 3)")
|
|
202
|
+
parser.add_argument("--json", action="store_true", help="Output results in JSON format")
|
|
203
|
+
args = parser.parse_args()
|
|
204
|
+
|
|
205
|
+
index = index_directory(args.dir)
|
|
206
|
+
results = index.search(args.query, top_k=args.top_k)
|
|
207
|
+
|
|
208
|
+
if args.json:
|
|
209
|
+
print(json.dumps(results, indent=2, ensure_ascii=False))
|
|
210
|
+
else:
|
|
211
|
+
print(f"🔍 BM25 Search Results for: '{args.query}' in '{args.dir}' (Found: {len(results)})")
|
|
212
|
+
for res in results:
|
|
213
|
+
print(f"\n[#{res['rank']}] {res['title']} (Score: {res['score']}) -> {res['doc_id']}")
|
|
214
|
+
print(f" {res['snippet']}")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
if __name__ == "__main__":
|
|
218
|
+
main()
|
package/tools/core/i18n.py
CHANGED
|
@@ -26,13 +26,13 @@ MESSAGES: Dict[str, Dict[str, str]] = {
|
|
|
26
26
|
"core_commands_title": "🌟 CORE & DAILY WORKFLOWS:",
|
|
27
27
|
"advanced_highlights_title": "🎯 ADVANCED AI/QA CAPABILITIES:",
|
|
28
28
|
"general_options_title": "GENERAL OPTIONS:",
|
|
29
|
-
"opt_all": "Display all
|
|
29
|
+
"opt_all": "Display all registered advanced and infrastructure commands",
|
|
30
30
|
"opt_version": "Show current version and author provenance",
|
|
31
31
|
"opt_offline": "Force 100% offline air-gapped mode (zero network / LLM calls)",
|
|
32
32
|
"opt_quiet": "Suppress banner, show only verdict and critical output",
|
|
33
33
|
"opt_lang": "Set CLI interface language (en: English, vi: Vietnamese)",
|
|
34
34
|
"tips_header": "💡 Quick Navigation Tips:",
|
|
35
|
-
"tip_all": "👉 Run 'agent-qa --help --all' to inspect all
|
|
35
|
+
"tip_all": "👉 Run 'agent-qa --help --all' to inspect all registered commands.",
|
|
36
36
|
"tip_cmd": "👉 Run 'agent-qa <command> --help' for detailed command options.",
|
|
37
37
|
"tip_welcome": "👉 Run 'agent-qa welcome' to launch the interactive visual landing page.",
|
|
38
38
|
"tip_lang_switch": "💡 Switch language: agent-qa --lang vi (Vietnamese) | agent-qa --lang en (English)",
|
|
@@ -122,13 +122,13 @@ MESSAGES: Dict[str, Dict[str, str]] = {
|
|
|
122
122
|
"core_commands_title": "🌟 LỆNH CỐT LÕI & THƯỜNG DÙNG:",
|
|
123
123
|
"advanced_highlights_title": "🎯 CÁC KHÁI NIỆM AI/QA NÂNG CAO TIÊU BIỂU:",
|
|
124
124
|
"general_options_title": "TÙY CHỌN CHUNG:",
|
|
125
|
-
"opt_all": "Hiển thị toàn bộ
|
|
125
|
+
"opt_all": "Hiển thị toàn bộ các lệnh nâng cao & hạ tầng đã đăng ký",
|
|
126
126
|
"opt_version": "Hiển thị phiên bản hiện tại và thông tin tác quyền MIT",
|
|
127
127
|
"opt_offline": "Ép chế độ 100% offline air-gapped (zero network / LLM calls)",
|
|
128
128
|
"opt_quiet": "Ẩn banner thông tin, chỉ xuất kết quả phán quyết quan trọng",
|
|
129
129
|
"opt_lang": "Chọn ngôn ngữ hiển thị giao diện CLI (en: Tiếng Anh, vi: Tiếng Việt)",
|
|
130
130
|
"tips_header": "💡 Mẹo Tra Cứu:",
|
|
131
|
-
"tip_all": "👉 Chạy 'agent-qa --help --all' để xem toàn bộ
|
|
131
|
+
"tip_all": "👉 Chạy 'agent-qa --help --all' để xem toàn bộ các lệnh đã đăng ký.",
|
|
132
132
|
"tip_cmd": "👉 Chạy 'agent-qa <lệnh> --help' để xem hướng dẫn chi tiết cho từng lệnh cụ thể.",
|
|
133
133
|
"tip_welcome": "👉 Chạy 'agent-qa welcome' để mở giao diện Landing Page tương tác trực quan.",
|
|
134
134
|
"tip_lang_switch": "💡 Chuyển đổi ngôn ngữ: agent-qa --lang vi (Tiếng Việt) | agent-qa --lang en (Tiếng Anh)",
|
|
@@ -178,7 +178,7 @@ MESSAGES: Dict[str, Dict[str, str]] = {
|
|
|
178
178
|
"cmd_review_spec": "Đánh giá chất lượng tài liệu đặc tả & user story theo mô hình Elephant-Goldfish",
|
|
179
179
|
"cmd_drift_check": "Giám sát độ lệch giữa đặc tả và mã kiểm thử thực tế (Deficit & Scope Drift)",
|
|
180
180
|
"cmd_trace_data_flow": "Theo dõi vòng đời dữ liệu test để phát hiện rò rỉ DB và thiếu sót teardown",
|
|
181
|
-
"cmd_distill": "Cô đọng và tinh lọc log dài dòng giảm
|
|
181
|
+
"cmd_distill": "Cô đọng và tinh lọc log dài dòng giảm nhiễu",
|
|
182
182
|
"cmd_gherkin": "Biên dịch file Gherkin .feature thành ma trận kiểm thử, Universal IR và báo cáo thiếu phủ (0-token)",
|
|
183
183
|
"cmd_omni_scan": "Quét toàn bộ dự án và thống kê danh mục kiểm thử đa ngôn ngữ (0-token)",
|
|
184
184
|
"cmd_report_bug": "Tự động tạo phiếu báo lỗi Jira/Markdown chuyên nghiệp kèm phân tích nguyên nhân gốc",
|
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee98). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee98/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
tools/core/provider_usage_adapter.py — Standardized Provider Usage Ingestion & Context Splitter
|
|
8
|
+
------------------------------------------------------------------------------------------------
|
|
9
|
+
Provides strict boundary enforcement between:
|
|
10
|
+
1. Real LLM provider-reported tokens (OpenAI, Anthropic, Gemini, Antigravity Transcripts).
|
|
11
|
+
2. Static documentation / rule estimates (cl100k_base / heuristic polyglot).
|
|
12
|
+
3. Synthetic micro-workload benchmark tokens.
|
|
13
|
+
4. Avoided filesystem read bytes (I/O footprint, strictly non-token).
|
|
14
|
+
|
|
15
|
+
Invariants:
|
|
16
|
+
- Zero Guesswork / Zero Interpolation: When live provider usage response is not present,
|
|
17
|
+
provider fields MUST remain None (null).
|
|
18
|
+
- Explicit Provenance: Every record declares measurement_kind, tokenizer/model, version, commit,
|
|
19
|
+
and timestamp.
|
|
20
|
+
- Context Decomposition: Statically estimated skill payloads are subtracted from total provider
|
|
21
|
+
input tokens to accurately isolate the external system prompt, conversation history, and tool schemas.
|
|
22
|
+
- Paired Run Verification: Live reduction claims are strictly guarded and rejected unless BOTH
|
|
23
|
+
baseline and optimized runs originate from real provider-observed responses.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from dataclasses import asdict, dataclass
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
import json
|
|
31
|
+
import logging
|
|
32
|
+
import os
|
|
33
|
+
import subprocess
|
|
34
|
+
from typing import Any, Dict, Optional, Tuple
|
|
35
|
+
|
|
36
|
+
_logger = logging.getLogger("provider_usage_adapter")
|
|
37
|
+
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_repo_version_and_commit(repo_root: Optional[str] = None) -> Tuple[str, str]:
|
|
41
|
+
"""Retrieves current hub version from skills_manifest.json and git commit hash."""
|
|
42
|
+
root = repo_root or _REPO_ROOT
|
|
43
|
+
version = "1.0.0"
|
|
44
|
+
manifest_path = os.path.join(root, "skills_manifest.json")
|
|
45
|
+
if os.path.isfile(manifest_path):
|
|
46
|
+
try:
|
|
47
|
+
with open(manifest_path, "r", encoding="utf-8") as f:
|
|
48
|
+
data = json.load(f)
|
|
49
|
+
version = data.get("version", "1.0.0")
|
|
50
|
+
except (OSError, json.JSONDecodeError, KeyError) as err:
|
|
51
|
+
_logger.debug("Failed to read version from manifest: %s", err)
|
|
52
|
+
|
|
53
|
+
commit = "unversioned"
|
|
54
|
+
try:
|
|
55
|
+
out = subprocess.check_output(
|
|
56
|
+
["git", "rev-parse", "HEAD"],
|
|
57
|
+
cwd=root,
|
|
58
|
+
stderr=subprocess.DEVNULL,
|
|
59
|
+
)
|
|
60
|
+
commit = out.decode("utf-8").strip()
|
|
61
|
+
except (OSError, subprocess.SubprocessError) as err:
|
|
62
|
+
_logger.debug("Failed to get git commit: %s", err)
|
|
63
|
+
commit = "unversioned"
|
|
64
|
+
|
|
65
|
+
return version, commit
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class ProviderUsageRecord:
|
|
70
|
+
"""Immutable, audited record of actual live LLM provider usage."""
|
|
71
|
+
provider: str
|
|
72
|
+
model: str
|
|
73
|
+
measurement_kind: str # Always "provider_observed"
|
|
74
|
+
provider_reported_input_tokens: int
|
|
75
|
+
provider_reported_output_tokens: int
|
|
76
|
+
provider_cached_tokens: Optional[int]
|
|
77
|
+
total_provider_tokens: int
|
|
78
|
+
timestamp: str
|
|
79
|
+
version: str
|
|
80
|
+
commit: str
|
|
81
|
+
raw_usage: Dict[str, Any]
|
|
82
|
+
|
|
83
|
+
def __post_init__(self) -> None:
|
|
84
|
+
if self.measurement_kind != "provider_observed":
|
|
85
|
+
raise ValueError(
|
|
86
|
+
f"ProviderUsageRecord requires measurement_kind == 'provider_observed', got {self.measurement_kind!r}"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
90
|
+
"""Converts to dictionary representation."""
|
|
91
|
+
return asdict(self)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass
|
|
95
|
+
class TokenMeasurementRecord:
|
|
96
|
+
"""Standardized record of token measurements with full provenance metadata."""
|
|
97
|
+
measurement_kind: str # "provider_observed" | "synthetic_estimate"
|
|
98
|
+
token_count: int
|
|
99
|
+
model_name: str
|
|
100
|
+
encoding: str
|
|
101
|
+
is_fallback: bool
|
|
102
|
+
commit_sha: str
|
|
103
|
+
input_hash: str
|
|
104
|
+
timestamp: float
|
|
105
|
+
|
|
106
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
107
|
+
"""Converts to dictionary representation."""
|
|
108
|
+
return asdict(self)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def serialize_provider_usage(record: Any) -> Dict[str, Any]:
|
|
112
|
+
"""
|
|
113
|
+
Serializes a TokenMeasurementRecord or ProviderUsageRecord into provider usage fields.
|
|
114
|
+
Strictly enforces that synthetic estimates or non-provider-observed measurements
|
|
115
|
+
cannot be serialized into provider billing fields.
|
|
116
|
+
|
|
117
|
+
Raises:
|
|
118
|
+
ValueError: If record.measurement_kind != 'provider_observed'.
|
|
119
|
+
"""
|
|
120
|
+
kind = getattr(record, "measurement_kind", None)
|
|
121
|
+
if isinstance(record, dict):
|
|
122
|
+
kind = record.get("measurement_kind")
|
|
123
|
+
|
|
124
|
+
if kind != "provider_observed":
|
|
125
|
+
raise ValueError(
|
|
126
|
+
f"Cannot serialize non-provider-observed record into provider usage fields: "
|
|
127
|
+
f"measurement_kind={kind!r}"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
if isinstance(record, ProviderUsageRecord):
|
|
131
|
+
return record.to_dict()
|
|
132
|
+
|
|
133
|
+
if isinstance(record, TokenMeasurementRecord):
|
|
134
|
+
return {
|
|
135
|
+
"measurement_kind": "provider_observed",
|
|
136
|
+
"provider_reported_input_tokens": record.token_count,
|
|
137
|
+
"provider_reported_output_tokens": 0,
|
|
138
|
+
"provider_cached_tokens": None,
|
|
139
|
+
"total_provider_tokens": record.token_count,
|
|
140
|
+
"model": record.model_name,
|
|
141
|
+
"encoding": record.encoding,
|
|
142
|
+
"is_fallback": record.is_fallback,
|
|
143
|
+
"commit_sha": record.commit_sha,
|
|
144
|
+
"input_hash": record.input_hash,
|
|
145
|
+
"timestamp": record.timestamp,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if isinstance(record, dict):
|
|
149
|
+
return dict(record)
|
|
150
|
+
|
|
151
|
+
raise ValueError(f"Unsupported record type: {type(record).__name__}")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def create_unobserved_provider_record(
|
|
155
|
+
measurement_kind: str = "static_estimate",
|
|
156
|
+
note: str = "No live LLM provider usage observed; fields remain null.",
|
|
157
|
+
) -> Dict[str, Any]:
|
|
158
|
+
"""
|
|
159
|
+
Returns a standardized dictionary for records where live provider data was not captured.
|
|
160
|
+
Guarantees that provider fields remain strictly None without interpolation.
|
|
161
|
+
"""
|
|
162
|
+
version, commit = get_repo_version_and_commit()
|
|
163
|
+
return {
|
|
164
|
+
"measurement_kind": measurement_kind,
|
|
165
|
+
"provider": None,
|
|
166
|
+
"model": None,
|
|
167
|
+
"provider_observed": False,
|
|
168
|
+
"provider_reported_input_tokens": None,
|
|
169
|
+
"provider_reported_output_tokens": None,
|
|
170
|
+
"provider_cached_tokens": None,
|
|
171
|
+
"total_provider_tokens": None,
|
|
172
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
173
|
+
"version": version,
|
|
174
|
+
"commit": commit,
|
|
175
|
+
"note": note,
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def parse_provider_usage(
|
|
180
|
+
response: Dict[str, Any],
|
|
181
|
+
provider_hint: Optional[str] = None,
|
|
182
|
+
model_hint: Optional[str] = None,
|
|
183
|
+
repo_root: Optional[str] = None,
|
|
184
|
+
) -> ProviderUsageRecord:
|
|
185
|
+
"""
|
|
186
|
+
Parses live provider usage from standard LLM response payloads.
|
|
187
|
+
Supported payloads:
|
|
188
|
+
1. OpenAI / Compatible: {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "prompt_tokens_details": {"cached_tokens": ...}}}
|
|
189
|
+
2. Anthropic: {"usage": {"input_tokens": ..., "output_tokens": ..., "cache_read_input_tokens": ...}}
|
|
190
|
+
3. Google GenAI: {"usage_metadata": {"prompt_token_count": ..., "candidates_token_count": ..., "cached_content_token_count": ...}}
|
|
191
|
+
4. Antigravity transcript step: {"input_tokens": ..., "output_tokens": ..., "cache_read_tokens": ...}
|
|
192
|
+
5. Mock / Test payload: {"provider": ..., "model": ..., "usage": {"input_tokens": ..., ...}}
|
|
193
|
+
|
|
194
|
+
Raises:
|
|
195
|
+
ValueError: When no valid provider usage payload is present (refuses to guess).
|
|
196
|
+
"""
|
|
197
|
+
if not isinstance(response, dict):
|
|
198
|
+
raise ValueError(f"Expected dict response payload, got {type(response).__name__}")
|
|
199
|
+
|
|
200
|
+
version, commit = get_repo_version_and_commit(repo_root)
|
|
201
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
202
|
+
|
|
203
|
+
# 1. Google GenAI (Gemini SDK) format: usage_metadata
|
|
204
|
+
if "usage_metadata" in response and isinstance(response["usage_metadata"], dict):
|
|
205
|
+
um = response["usage_metadata"]
|
|
206
|
+
in_tok = um.get("prompt_token_count")
|
|
207
|
+
out_tok = um.get("candidates_token_count", 0)
|
|
208
|
+
cache_tok = um.get("cached_content_token_count")
|
|
209
|
+
model = response.get("model") or model_hint or "gemini"
|
|
210
|
+
provider = provider_hint or "google"
|
|
211
|
+
if in_tok is not None:
|
|
212
|
+
total = in_tok + out_tok
|
|
213
|
+
return ProviderUsageRecord(
|
|
214
|
+
provider=provider,
|
|
215
|
+
model=model,
|
|
216
|
+
measurement_kind="provider_observed",
|
|
217
|
+
provider_reported_input_tokens=int(in_tok),
|
|
218
|
+
provider_reported_output_tokens=int(out_tok),
|
|
219
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
220
|
+
total_provider_tokens=total,
|
|
221
|
+
timestamp=now_iso,
|
|
222
|
+
version=version,
|
|
223
|
+
commit=commit,
|
|
224
|
+
raw_usage=um,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# 2. OpenAI / Anthropic format: usage key
|
|
228
|
+
if "usage" in response and isinstance(response["usage"], dict):
|
|
229
|
+
u = response["usage"]
|
|
230
|
+
model = response.get("model") or model_hint or "unknown_model"
|
|
231
|
+
|
|
232
|
+
# OpenAI style: prompt_tokens, completion_tokens
|
|
233
|
+
if "prompt_tokens" in u:
|
|
234
|
+
in_tok = u["prompt_tokens"]
|
|
235
|
+
out_tok = u.get("completion_tokens", 0)
|
|
236
|
+
cache_tok = None
|
|
237
|
+
if "prompt_tokens_details" in u and isinstance(u["prompt_tokens_details"], dict):
|
|
238
|
+
cache_tok = u["prompt_tokens_details"].get("cached_tokens")
|
|
239
|
+
provider = provider_hint or ("openai" if "gpt" in model.lower() else "llm_provider")
|
|
240
|
+
total = u.get("total_tokens", in_tok + out_tok)
|
|
241
|
+
return ProviderUsageRecord(
|
|
242
|
+
provider=provider,
|
|
243
|
+
model=model,
|
|
244
|
+
measurement_kind="provider_observed",
|
|
245
|
+
provider_reported_input_tokens=int(in_tok),
|
|
246
|
+
provider_reported_output_tokens=int(out_tok),
|
|
247
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
248
|
+
total_provider_tokens=int(total),
|
|
249
|
+
timestamp=now_iso,
|
|
250
|
+
version=version,
|
|
251
|
+
commit=commit,
|
|
252
|
+
raw_usage=u,
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
# Anthropic style: input_tokens, output_tokens
|
|
256
|
+
if "input_tokens" in u:
|
|
257
|
+
in_tok = u["input_tokens"]
|
|
258
|
+
out_tok = u.get("output_tokens", 0)
|
|
259
|
+
cache_tok = u.get("cache_read_input_tokens")
|
|
260
|
+
provider = provider_hint or ("anthropic" if "claude" in model.lower() else "llm_provider")
|
|
261
|
+
total = in_tok + out_tok
|
|
262
|
+
return ProviderUsageRecord(
|
|
263
|
+
provider=provider,
|
|
264
|
+
model=model,
|
|
265
|
+
measurement_kind="provider_observed",
|
|
266
|
+
provider_reported_input_tokens=int(in_tok),
|
|
267
|
+
provider_reported_output_tokens=int(out_tok),
|
|
268
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
269
|
+
total_provider_tokens=total,
|
|
270
|
+
timestamp=now_iso,
|
|
271
|
+
version=version,
|
|
272
|
+
commit=commit,
|
|
273
|
+
raw_usage=u,
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
# 3. Direct / Antigravity transcript step format: input_tokens, output_tokens directly in root
|
|
277
|
+
if "input_tokens" in response:
|
|
278
|
+
in_tok = response["input_tokens"]
|
|
279
|
+
out_tok = response.get("output_tokens", 0)
|
|
280
|
+
cache_tok = response.get("cache_read_tokens")
|
|
281
|
+
provider = response.get("provider") or provider_hint or "antigravity"
|
|
282
|
+
model = response.get("model") or model_hint or "gemini"
|
|
283
|
+
total = in_tok + out_tok + (cache_tok or 0)
|
|
284
|
+
return ProviderUsageRecord(
|
|
285
|
+
provider=provider,
|
|
286
|
+
model=model,
|
|
287
|
+
measurement_kind="provider_observed",
|
|
288
|
+
provider_reported_input_tokens=int(in_tok),
|
|
289
|
+
provider_reported_output_tokens=int(out_tok),
|
|
290
|
+
provider_cached_tokens=int(cache_tok) if cache_tok is not None else None,
|
|
291
|
+
total_provider_tokens=int(total),
|
|
292
|
+
timestamp=now_iso,
|
|
293
|
+
version=version,
|
|
294
|
+
commit=commit,
|
|
295
|
+
raw_usage={k: response[k] for k in ("input_tokens", "output_tokens", "cache_read_tokens") if k in response},
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
raise ValueError(
|
|
299
|
+
"No valid LLM provider usage metadata found in response. "
|
|
300
|
+
"Synapse Hub refuses to fabricate synthetic numbers as live provider data."
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def decompose_prompt_context(
|
|
305
|
+
provider_record: ProviderUsageRecord,
|
|
306
|
+
skill_payload_tokens: int,
|
|
307
|
+
rule_payload_tokens: int = 0,
|
|
308
|
+
) -> Dict[str, Any]:
|
|
309
|
+
"""
|
|
310
|
+
Separates the statically estimated skill/rule documentation payload from the
|
|
311
|
+
overall provider input tokens to isolate external host system instructions,
|
|
312
|
+
conversation history turns, and provider tool calling schemas.
|
|
313
|
+
"""
|
|
314
|
+
if not isinstance(provider_record, ProviderUsageRecord):
|
|
315
|
+
raise TypeError(f"Expected ProviderUsageRecord, got {type(provider_record).__name__}")
|
|
316
|
+
|
|
317
|
+
total_in = provider_record.provider_reported_input_tokens
|
|
318
|
+
managed_tokens = skill_payload_tokens + rule_payload_tokens
|
|
319
|
+
external_wrapper = max(0, total_in - managed_tokens)
|
|
320
|
+
ratio = round((managed_tokens / max(1, total_in)) * 100.0, 2)
|
|
321
|
+
|
|
322
|
+
return {
|
|
323
|
+
"measurement_kind": "provider_observed_with_decomposition",
|
|
324
|
+
"provider": provider_record.provider,
|
|
325
|
+
"model": provider_record.model,
|
|
326
|
+
"provider_reported_input_tokens": total_in,
|
|
327
|
+
"provider_reported_output_tokens": provider_record.provider_reported_output_tokens,
|
|
328
|
+
"provider_cached_tokens": provider_record.provider_cached_tokens,
|
|
329
|
+
"skill_payload_static_estimated_tokens": skill_payload_tokens,
|
|
330
|
+
"rule_payload_static_estimated_tokens": rule_payload_tokens,
|
|
331
|
+
"total_managed_static_tokens": managed_tokens,
|
|
332
|
+
"external_system_prompt_and_wrapper_tokens": external_wrapper,
|
|
333
|
+
"managed_payload_ratio_pct": ratio,
|
|
334
|
+
"timestamp": provider_record.timestamp,
|
|
335
|
+
"version": provider_record.version,
|
|
336
|
+
"commit": provider_record.commit,
|
|
337
|
+
"disclaimer": (
|
|
338
|
+
"Provider reported input tokens reflect total live API billing usage. "
|
|
339
|
+
"Skill and rule payloads are statically estimated via offline tokenizer and subtracted "
|
|
340
|
+
"to isolate unmanaged host system prompt, user prompt, and provider tool schemas."
|
|
341
|
+
),
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def compare_paired_provider_runs(
|
|
346
|
+
baseline_run: ProviderUsageRecord,
|
|
347
|
+
optimized_run: ProviderUsageRecord,
|
|
348
|
+
) -> Dict[str, Any]:
|
|
349
|
+
"""
|
|
350
|
+
Computes verified token reduction between two live provider runs.
|
|
351
|
+
Enforces scientific honesty: strictly refuses to claim live provider reduction unless
|
|
352
|
+
BOTH baseline and optimized runs originate from real provider_observed responses.
|
|
353
|
+
"""
|
|
354
|
+
if not isinstance(baseline_run, ProviderUsageRecord) or not isinstance(optimized_run, ProviderUsageRecord):
|
|
355
|
+
raise TypeError("Both baseline_run and optimized_run must be verified ProviderUsageRecord instances.")
|
|
356
|
+
|
|
357
|
+
if baseline_run.measurement_kind != "provider_observed" or optimized_run.measurement_kind != "provider_observed":
|
|
358
|
+
raise ValueError("Both runs must have measurement_kind == 'provider_observed' to claim live provider reduction.")
|
|
359
|
+
|
|
360
|
+
in_saved = baseline_run.provider_reported_input_tokens - optimized_run.provider_reported_input_tokens
|
|
361
|
+
out_saved = baseline_run.provider_reported_output_tokens - optimized_run.provider_reported_output_tokens
|
|
362
|
+
total_saved = baseline_run.total_provider_tokens - optimized_run.total_provider_tokens
|
|
363
|
+
reduction_pct = round((total_saved / max(1, baseline_run.total_provider_tokens)) * 100.0, 2)
|
|
364
|
+
|
|
365
|
+
return {
|
|
366
|
+
"verified_provider_savings": True,
|
|
367
|
+
"measurement_kind": "paired_provider_observed",
|
|
368
|
+
"baseline_provider": baseline_run.provider,
|
|
369
|
+
"baseline_model": baseline_run.model,
|
|
370
|
+
"baseline_input_tokens": baseline_run.provider_reported_input_tokens,
|
|
371
|
+
"baseline_output_tokens": baseline_run.provider_reported_output_tokens,
|
|
372
|
+
"baseline_total_tokens": baseline_run.total_provider_tokens,
|
|
373
|
+
"optimized_provider": optimized_run.provider,
|
|
374
|
+
"optimized_model": optimized_run.model,
|
|
375
|
+
"optimized_input_tokens": optimized_run.provider_reported_input_tokens,
|
|
376
|
+
"optimized_output_tokens": optimized_run.provider_reported_output_tokens,
|
|
377
|
+
"optimized_total_tokens": optimized_run.total_provider_tokens,
|
|
378
|
+
"input_tokens_saved": in_saved,
|
|
379
|
+
"output_tokens_saved": out_saved,
|
|
380
|
+
"net_tokens_saved": total_saved,
|
|
381
|
+
"reduction_pct": reduction_pct,
|
|
382
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
383
|
+
}
|