mcp-efficiency-engine 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/autodocs-sync.yml +2 -15
- package/.github/workflows/release-npm.yml +111 -0
- package/FINAL_USAGE_GUIDE.md +56 -0
- package/README.md +133 -0
- package/autodocs/README.md +6 -0
- package/observability/metrics.md +33 -0
- package/observability/scoring-rubric.md +20 -0
- package/orchestrator/wiki/graph_consolidator.py +60 -1
- package/orchestrator/wiki/incremental_engine.py +23 -1
- package/orchestrator/wiki/plugin_manager.py +87 -22
- package/package.json +8 -2
- package/requirements.txt +1 -0
- package/scripts/intake/resolve-routing.py +124 -87
- package/scripts/intake/validate-database-routing.py +13 -4
- package/scripts/intake/validate-rag-routing.py +4 -3
- package/scripts/learning/learning-loop-report.py +28 -10
- package/scripts/ops/llm-token-probe.py +103 -39
- package/scripts/ops/telemetry-benchmark.py +184 -0
- package/scripts/wiki/compiler_main.py +108 -76
- package/scripts/wiki/providers/repo_content_provider.py +51 -0
- package/scripts/wiki/validator.py +40 -15
- package/telemetry/__init__.py +4 -0
- package/telemetry/bootstrap.py +47 -0
- package/telemetry/collector/engine.py +261 -0
- package/telemetry/config.json +20 -0
- package/telemetry/config.py +131 -0
- package/telemetry/context/propagation.py +85 -0
- package/telemetry/events/catalog.py +24 -0
- package/telemetry/events/factory.py +42 -0
- package/telemetry/exporters/base.py +16 -0
- package/telemetry/exporters/console/exporter.py +30 -0
- package/telemetry/exporters/json/exporter.py +34 -0
- package/telemetry/exporters/langsmith/exporter.py +415 -0
- package/telemetry/metrics/aggregator.py +60 -0
- package/telemetry/scoring/efficiency.py +68 -0
- package/telemetry/tracing/span.py +43 -0
- package/observability/evals/chat-token-usage-report.json +0 -64
- package/observability/evals/chat-token-usage-report.md +0 -33
- package/observability/evals/iteration-value-report.json +0 -76
- package/observability/evals/iteration-value-report.md +0 -39
- package/observability/evals/learning-loop-report.json +0 -268
- package/observability/evals/learning-loop-report.md +0 -37
- package/observability/evals/repo-intake-eval-cases.md +0 -23
- package/observability/evals/routing-eval-cases.json +0 -68
- package/observability/evals/routing-eval-cases.md +0 -25
- package/observability/evals/routing-eval-report.json +0 -72
- package/observability/logs/copilot-usage-ingest-report.json +0 -13
- package/observability/logs/iteration-metrics.jsonl +0 -75
- package/observability/logs/learning-feedback.jsonl +0 -152
- package/observability/logs/routing-decisions.jsonl +0 -333
- package/observability/logs/session/agent-pipeline-preflight-20260701-004146.log +0 -8
- package/observability/logs/session/agent-pipeline-preflight-20260701-005554.log +0 -8
- package/observability/logs/session/agent-pipeline-preflight-20260701-010351.log +0 -8
- package/observability/logs/session/agent-pipeline-preflight-20260704-145829.log +0 -8
- package/observability/logs/session/agent-pipeline-preflight-20260704-150021.log +0 -8
- package/observability/logs/session/agent-pipeline-preflight-20260704-154328.log +0 -8
- package/observability/logs/session/bye-20260629-221513.json +0 -39
- package/observability/logs/session/bye-20260629-221545.json +0 -59
- package/observability/logs/session/bye-20260629-222007.json +0 -52
- package/observability/logs/session/bye-20260629-222133.json +0 -59
- package/observability/logs/session/bye-20260629-222412.json +0 -45
- package/observability/logs/session/bye-20260629-223224.json +0 -52
- package/observability/logs/session/bye-20260629-224050.json +0 -59
- package/observability/logs/session/bye-20260630-105732.json +0 -24
- package/observability/logs/session/bye-20260630-211536.json +0 -87
- package/observability/logs/session/bye-20260630-223423.json +0 -87
- package/observability/logs/session/bye-20260630-232534.json +0 -87
- package/observability/logs/session/bye-20260701-003638.json +0 -126
- package/observability/logs/session/bye-20260701-003849.json +0 -126
- package/observability/logs/session/bye-20260701-004315.json +0 -138
- package/observability/logs/session/bye-20260701-004559.json +0 -76
- package/observability/logs/session/bye-20260701-005913.json +0 -138
- package/observability/logs/session/bye-20260701-010753.json +0 -138
- package/observability/logs/session/bye-20260703-194106.json +0 -24
- package/observability/logs/session/bye-20260703-233443.json +0 -76
- package/observability/logs/session/bye-20260703-233653.json +0 -76
- package/observability/logs/session/bye-20260704-150317.json +0 -152
- package/observability/logs/session/bye-20260704-150929.json +0 -75
- package/observability/logs/session/bye-20260704-151050.json +0 -75
- package/observability/logs/session/bye-20260705-115458.json +0 -165
- package/observability/logs/session/codegraph-status-20260701-004228.log +0 -39
- package/observability/logs/session/codegraph-status-20260701-004310.log +0 -39
- package/observability/logs/session/codegraph-status-20260701-005637.log +0 -39
- package/observability/logs/session/codegraph-status-20260701-005908.log +0 -39
- package/observability/logs/session/codegraph-status-20260701-010444.log +0 -39
- package/observability/logs/session/codegraph-status-20260701-010747.log +0 -39
- package/observability/logs/session/codegraph-status-20260704-150137.log +0 -26
- package/observability/logs/session/codegraph-status-20260704-150313.log +0 -28
- package/observability/logs/session/codegraph-status-20260704-154445.log +0 -26
- package/observability/logs/session/codegraph-status-20260705-115454.log +0 -27
- package/observability/logs/session/codegraph-sync-20260701-004211.log +0 -88
- package/observability/logs/session/codegraph-sync-20260701-004241.log +0 -82
- package/observability/logs/session/codegraph-sync-20260701-005620.log +0 -89
- package/observability/logs/session/codegraph-sync-20260701-005840.log +0 -88
- package/observability/logs/session/codegraph-sync-20260701-010419.log +0 -93
- package/observability/logs/session/codegraph-sync-20260701-010609.log +0 -90
- package/observability/logs/session/codegraph-sync-20260704-145856.log +0 -70
- package/observability/logs/session/codegraph-sync-20260704-150047.log +0 -53
- package/observability/logs/session/codegraph-sync-20260704-150204.log +0 -44
- package/observability/logs/session/codegraph-sync-20260704-154353.log +0 -26
- package/observability/logs/session/codegraph-sync-20260705-115337.log +0 -32
- package/observability/logs/session/hi-20260629-221459.json +0 -38
- package/observability/logs/session/hi-20260629-221945.json +0 -59
- package/observability/logs/session/hi-20260629-222108.json +0 -66
- package/observability/logs/session/hi-20260629-222352.json +0 -45
- package/observability/logs/session/hi-20260629-225719.json +0 -59
- package/observability/logs/session/hi-20260629-233722.json +0 -52
- package/observability/logs/session/hi-20260629-233913.json +0 -198
- package/observability/logs/session/hi-20260630-110156.json +0 -177
- package/observability/logs/session/hi-20260630-191243.json +0 -184
- package/observability/logs/session/hi-20260630-230050.json +0 -234
- package/observability/logs/session/hi-20260630-230523.json +0 -227
- package/observability/logs/session/hi-20260630-230855.json +0 -227
- package/observability/logs/session/hi-20260630-231638.json +0 -227
- package/observability/logs/session/hi-20260630-231756.json +0 -219
- package/observability/logs/session/hi-20260701-002842.json +0 -234
- package/observability/logs/session/hi-20260701-003458.json +0 -282
- package/observability/logs/session/hi-20260701-004233.json +0 -290
- package/observability/logs/session/hi-20260701-004554.json +0 -236
- package/observability/logs/session/hi-20260701-005641.json +0 -290
- package/observability/logs/session/hi-20260701-010450.json +0 -290
- package/observability/logs/session/hi-20260703-233854.json +0 -312
- package/observability/logs/session/hi-20260703-234448.json +0 -318
- package/observability/logs/session/hi-20260704-145907.json +0 -353
- package/observability/logs/session/hi-20260704-150141.json +0 -392
- package/observability/logs/session/hi-20260704-154448.json +0 -392
- package/observability/logs/session/iteration-value-refresh-20260701-004309.log +0 -3
- package/observability/logs/session/iteration-value-refresh-20260701-005907.log +0 -3
- package/observability/logs/session/iteration-value-refresh-20260701-010747.log +0 -3
- package/observability/logs/session/iteration-value-refresh-20260704-150312.log +0 -3
- package/observability/logs/session/iteration-value-refresh-20260705-115453.log +0 -3
- package/observability/logs/session/learning-loop-refresh-20260701-004309.log +0 -3
- package/observability/logs/session/learning-loop-refresh-20260701-005906.log +0 -3
- package/observability/logs/session/learning-loop-refresh-20260701-010745.log +0 -3
- package/observability/logs/session/learning-loop-refresh-20260704-150309.log +0 -3
- package/observability/logs/session/learning-loop-refresh-20260705-115450.log +0 -3
- package/observability/logs/session/project-notes-refresh-20260701-004227.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260701-004308.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260701-005636.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260701-005906.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260701-010443.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260701-010744.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260704-150133.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260704-150309.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260704-154442.log +0 -1
- package/observability/logs/session/project-notes-refresh-20260705-115449.log +0 -1
- package/observability/logs/session/repo-intake-20260701-004218.log +0 -6
- package/observability/logs/session/repo-intake-20260701-005626.log +0 -6
- package/observability/logs/session/repo-intake-20260701-010433.log +0 -6
- package/observability/logs/session/repo-intake-20260704-150053.log +0 -6
- package/observability/logs/session/repo-intake-20260704-154404.log +0 -6
- package/observability/logs/session/repo-intake-refresh-20260701-004247.log +0 -6
- package/observability/logs/session/repo-intake-refresh-20260701-005847.log +0 -6
- package/observability/logs/session/repo-intake-refresh-20260701-010615.log +0 -6
- package/observability/logs/session/repo-intake-refresh-20260704-150209.log +0 -6
- package/observability/logs/session/repo-intake-refresh-20260705-115342.log +0 -6
- package/observability/logs/session/repo-registry-validation-20260701-004234.log +0 -2
- package/observability/logs/session/repo-registry-validation-20260701-004556.log +0 -2
- package/observability/logs/session/repo-registry-validation-20260701-005833.log +0 -2
- package/observability/logs/session/repo-registry-validation-20260701-010601.log +0 -2
- package/observability/logs/session/repo-registry-validation-20260704-150155.log +0 -2
- package/observability/logs/session/repo-registry-validation-20260705-115323.log +0 -2
- package/observability/logs/session/repomix-refresh-20260701-003823.log +0 -4236
- package/observability/logs/session/repomix-refresh-20260701-004251.log +0 -4275
- package/observability/logs/session/repomix-refresh-20260701-005850.log +0 -4248
- package/observability/logs/session/repomix-refresh-20260701-010619.log +0 -6531
- package/observability/logs/session/repomix-refresh-20260704-150239.log +0 -652
- package/observability/logs/session/repomix-refresh-20260705-115419.log +0 -178
- package/observability/logs/session/routing-evals-20260701-004221.log +0 -2
- package/observability/logs/session/routing-evals-20260701-004235.log +0 -2
- package/observability/logs/session/routing-evals-20260701-005629.log +0 -2
- package/observability/logs/session/routing-evals-20260701-005834.log +0 -2
- package/observability/logs/session/routing-evals-20260701-010437.log +0 -2
- package/observability/logs/session/routing-evals-20260701-010603.log +0 -2
- package/observability/logs/session/routing-evals-20260704-150125.log +0 -2
- package/observability/logs/session/routing-evals-20260704-150157.log +0 -2
- package/observability/logs/session/routing-evals-20260704-154434.log +0 -2
- package/observability/logs/session/routing-evals-20260705-115327.log +0 -2
- package/observability/logs/session/sibling-discovery-refresh-20260701-004250.log +0 -3
- package/observability/logs/session/sibling-discovery-refresh-20260701-005850.log +0 -3
- package/observability/logs/session/sibling-discovery-refresh-20260701-010618.log +0 -3
- package/observability/logs/session/sibling-discovery-refresh-20260704-150238.log +0 -3
- package/observability/logs/session/sibling-discovery-refresh-20260705-115418.log +0 -3
|
@@ -17,9 +17,16 @@ from __future__ import annotations
|
|
|
17
17
|
|
|
18
18
|
import argparse
|
|
19
19
|
import os
|
|
20
|
+
import sys
|
|
20
21
|
from pathlib import Path
|
|
21
22
|
from typing import List
|
|
22
23
|
|
|
24
|
+
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
25
|
+
if str(_REPO_ROOT) not in sys.path:
|
|
26
|
+
sys.path.insert(0, str(_REPO_ROOT))
|
|
27
|
+
|
|
28
|
+
from telemetry import build_telemetry_collector
|
|
29
|
+
|
|
23
30
|
try:
|
|
24
31
|
from openai import AzureOpenAI
|
|
25
32
|
except Exception:
|
|
@@ -81,7 +88,10 @@ def main() -> int:
|
|
|
81
88
|
args = parser.parse_args()
|
|
82
89
|
|
|
83
90
|
repo_root = Path(args.repo_root)
|
|
91
|
+
collector = build_telemetry_collector(_REPO_ROOT)
|
|
84
92
|
if not repo_root.exists():
|
|
93
|
+
collector.record_event("WarningGenerated", {"warning": "repo_root_missing", "repo_root": str(repo_root)}, level="WARNING")
|
|
94
|
+
collector.shutdown()
|
|
85
95
|
print("prompt_tokens=0")
|
|
86
96
|
print("completion_tokens=0")
|
|
87
97
|
print("total_tokens=0")
|
|
@@ -99,6 +109,17 @@ def main() -> int:
|
|
|
99
109
|
model = os.getenv("OPENAI_CHAT_MODEL", "gpt-4o-mini")
|
|
100
110
|
|
|
101
111
|
if AzureOpenAI is None or not api_key or not endpoint:
|
|
112
|
+
collector.record_event(
|
|
113
|
+
"WarningGenerated",
|
|
114
|
+
{
|
|
115
|
+
"warning": "missing_credentials_or_openai_pkg",
|
|
116
|
+
"has_openai_pkg": AzureOpenAI is not None,
|
|
117
|
+
"has_api_key": bool(api_key),
|
|
118
|
+
"has_endpoint": bool(endpoint),
|
|
119
|
+
},
|
|
120
|
+
level="WARNING",
|
|
121
|
+
)
|
|
122
|
+
collector.shutdown()
|
|
102
123
|
print("prompt_tokens=0")
|
|
103
124
|
print("completion_tokens=0")
|
|
104
125
|
print("total_tokens=0")
|
|
@@ -119,45 +140,88 @@ def main() -> int:
|
|
|
119
140
|
"Devuelve un resumen tecnico corto con riesgos y evidencia."
|
|
120
141
|
)
|
|
121
142
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
143
|
+
with collector.start_execution(
|
|
144
|
+
operation="llm-token-probe",
|
|
145
|
+
session_id="ops",
|
|
146
|
+
provider="azure-openai",
|
|
147
|
+
model=model,
|
|
148
|
+
):
|
|
149
|
+
with collector.start_span(name="provider.azure_openai_call", kind="CLIENT", attributes={"profile": args.profile}):
|
|
150
|
+
collector.record_event(
|
|
151
|
+
"LLMRequest",
|
|
152
|
+
{
|
|
153
|
+
"provider": "azure-openai",
|
|
154
|
+
"model": model,
|
|
155
|
+
"profile": args.profile,
|
|
156
|
+
"selected_files": len(files),
|
|
157
|
+
},
|
|
158
|
+
)
|
|
159
|
+
try:
|
|
160
|
+
response = client.chat.completions.create(
|
|
161
|
+
model=model,
|
|
162
|
+
messages=[
|
|
163
|
+
{"role": "system", "content": system_prompt},
|
|
164
|
+
{"role": "user", "content": user_prompt},
|
|
165
|
+
],
|
|
166
|
+
temperature=0,
|
|
167
|
+
max_completion_tokens=400,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
usage = response.usage
|
|
171
|
+
prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
|
|
172
|
+
completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
|
|
173
|
+
total_tokens = int(getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0)
|
|
174
|
+
|
|
175
|
+
price_in = to_float(os.getenv("LLM_PRICE_INPUT_USD_PER_1K"), 0.0)
|
|
176
|
+
price_out = to_float(os.getenv("LLM_PRICE_OUTPUT_USD_PER_1K"), 0.0)
|
|
177
|
+
if price_in > 0 or price_out > 0:
|
|
178
|
+
cost = (prompt_tokens / 1000.0) * price_in + (completion_tokens / 1000.0) * price_out
|
|
179
|
+
else:
|
|
180
|
+
# fallback simple when split pricing is unknown
|
|
181
|
+
flat = to_float(os.getenv("LLM_PRICE_USD_PER_1K_TOKENS"), 0.0)
|
|
182
|
+
cost = (total_tokens / 1000.0) * flat if flat > 0 else 0.0
|
|
183
|
+
|
|
184
|
+
collector.record_usage(
|
|
185
|
+
input_tokens=prompt_tokens,
|
|
186
|
+
output_tokens=completion_tokens,
|
|
187
|
+
estimated_cost_usd=cost,
|
|
188
|
+
)
|
|
189
|
+
collector.record_event(
|
|
190
|
+
"LLMResponse",
|
|
191
|
+
{
|
|
192
|
+
"provider": "azure-openai",
|
|
193
|
+
"model": model,
|
|
194
|
+
"prompt_tokens": prompt_tokens,
|
|
195
|
+
"completion_tokens": completion_tokens,
|
|
196
|
+
"total_tokens": total_tokens,
|
|
197
|
+
"estimated_cost_usd": round(cost, 6),
|
|
198
|
+
},
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
print(f"prompt_tokens={prompt_tokens}")
|
|
202
|
+
print(f"completion_tokens={completion_tokens}")
|
|
203
|
+
print(f"total_tokens={total_tokens}")
|
|
204
|
+
print(f"total_cost_usd={cost:.6f}")
|
|
205
|
+
print("probe_status=ok")
|
|
206
|
+
collector.shutdown()
|
|
207
|
+
return 0
|
|
208
|
+
|
|
209
|
+
except Exception as ex:
|
|
210
|
+
collector.record_event(
|
|
211
|
+
"ExceptionThrown",
|
|
212
|
+
{
|
|
213
|
+
"provider": "azure-openai",
|
|
214
|
+
"error": str(ex),
|
|
215
|
+
},
|
|
216
|
+
level="ERROR",
|
|
217
|
+
)
|
|
218
|
+
print("prompt_tokens=0")
|
|
219
|
+
print("completion_tokens=0")
|
|
220
|
+
print("total_tokens=0")
|
|
221
|
+
print("total_cost_usd=0")
|
|
222
|
+
print(f"probe_status=error:{str(ex).replace(os.linesep, ' ')[:200]}")
|
|
223
|
+
collector.shutdown()
|
|
224
|
+
return 0
|
|
161
225
|
|
|
162
226
|
|
|
163
227
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import statistics
|
|
7
|
+
import subprocess
|
|
8
|
+
import time
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def utc_now() -> str:
|
|
15
|
+
return datetime.now(timezone.utc).isoformat()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def build_command(python_exe: str) -> list[str]:
|
|
19
|
+
return [
|
|
20
|
+
python_exe,
|
|
21
|
+
"scripts/intake/resolve-routing.py",
|
|
22
|
+
"--input",
|
|
23
|
+
"telemetry benchmark",
|
|
24
|
+
"--intent",
|
|
25
|
+
"feature",
|
|
26
|
+
"--domain",
|
|
27
|
+
"backend",
|
|
28
|
+
"--source-type",
|
|
29
|
+
"code",
|
|
30
|
+
"--input-tokens",
|
|
31
|
+
"120",
|
|
32
|
+
"--output-tokens",
|
|
33
|
+
"45",
|
|
34
|
+
"--estimated-cost-usd",
|
|
35
|
+
"0.0032",
|
|
36
|
+
"--model",
|
|
37
|
+
"GPT-5.3-Codex",
|
|
38
|
+
"--local-tools",
|
|
39
|
+
"2",
|
|
40
|
+
"--remote-tools",
|
|
41
|
+
"1",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def run_variant(*, repo_root: Path, iterations: int, telemetry_enabled: bool, python_exe: str) -> dict[str, Any]:
|
|
46
|
+
durations_ms: list[float] = []
|
|
47
|
+
failures = 0
|
|
48
|
+
for _ in range(iterations):
|
|
49
|
+
env = os.environ.copy()
|
|
50
|
+
env["TELEMETRY_ENABLED"] = "true" if telemetry_enabled else "false"
|
|
51
|
+
env["TELEMETRY_EXPORTERS"] = "console,json"
|
|
52
|
+
|
|
53
|
+
start = time.perf_counter()
|
|
54
|
+
proc = subprocess.run(
|
|
55
|
+
build_command(python_exe),
|
|
56
|
+
cwd=str(repo_root),
|
|
57
|
+
env=env,
|
|
58
|
+
stdout=subprocess.DEVNULL,
|
|
59
|
+
stderr=subprocess.DEVNULL,
|
|
60
|
+
check=False,
|
|
61
|
+
)
|
|
62
|
+
elapsed_ms = (time.perf_counter() - start) * 1000.0
|
|
63
|
+
durations_ms.append(elapsed_ms)
|
|
64
|
+
if proc.returncode != 0:
|
|
65
|
+
failures += 1
|
|
66
|
+
|
|
67
|
+
avg_ms = statistics.mean(durations_ms) if durations_ms else 0.0
|
|
68
|
+
p95_ms = statistics.quantiles(durations_ms, n=20)[18] if len(durations_ms) >= 20 else max(durations_ms, default=0.0)
|
|
69
|
+
|
|
70
|
+
return {
|
|
71
|
+
"telemetry_enabled": telemetry_enabled,
|
|
72
|
+
"iterations": iterations,
|
|
73
|
+
"failures": failures,
|
|
74
|
+
"avg_ms": round(avg_ms, 3),
|
|
75
|
+
"min_ms": round(min(durations_ms) if durations_ms else 0.0, 3),
|
|
76
|
+
"max_ms": round(max(durations_ms) if durations_ms else 0.0, 3),
|
|
77
|
+
"p95_ms": round(p95_ms, 3),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def main() -> int:
|
|
82
|
+
parser = argparse.ArgumentParser(description="Benchmark telemetry overhead on/off using resolve-routing as workload.")
|
|
83
|
+
parser.add_argument("--iterations", type=int, default=10)
|
|
84
|
+
parser.add_argument("--python", default="py")
|
|
85
|
+
parser.add_argument("--python-arg", default="-3")
|
|
86
|
+
parser.add_argument("--out", default="observability/evals/telemetry-overhead-benchmark.json")
|
|
87
|
+
args = parser.parse_args()
|
|
88
|
+
|
|
89
|
+
repo_root = Path(__file__).resolve().parents[2]
|
|
90
|
+
iterations = max(2, int(args.iterations))
|
|
91
|
+
python_exe = args.python
|
|
92
|
+
|
|
93
|
+
# Windows default launcher pattern: py -3
|
|
94
|
+
if args.python == "py" and args.python_arg:
|
|
95
|
+
python_exe = f"{args.python} {args.python_arg}" # handled below
|
|
96
|
+
|
|
97
|
+
if " " in python_exe:
|
|
98
|
+
tokens = python_exe.split()
|
|
99
|
+
base = tokens[0]
|
|
100
|
+
extra = tokens[1:]
|
|
101
|
+
else:
|
|
102
|
+
base = python_exe
|
|
103
|
+
extra = []
|
|
104
|
+
|
|
105
|
+
def build_python() -> str:
|
|
106
|
+
if not extra:
|
|
107
|
+
return base
|
|
108
|
+
return base
|
|
109
|
+
|
|
110
|
+
# Probe python command in current environment.
|
|
111
|
+
probe = subprocess.run([base, *extra, "--version"], cwd=str(repo_root), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
|
112
|
+
if probe.returncode != 0:
|
|
113
|
+
print("Unable to resolve python interpreter for benchmark")
|
|
114
|
+
return 1
|
|
115
|
+
|
|
116
|
+
def command_python() -> str:
|
|
117
|
+
return base
|
|
118
|
+
|
|
119
|
+
# Wrap variant runner to include optional extra interpreter args.
|
|
120
|
+
def run_with_variant(telemetry_enabled: bool) -> dict[str, Any]:
|
|
121
|
+
durations_ms: list[float] = []
|
|
122
|
+
failures = 0
|
|
123
|
+
for _ in range(iterations):
|
|
124
|
+
env = os.environ.copy()
|
|
125
|
+
env["TELEMETRY_ENABLED"] = "true" if telemetry_enabled else "false"
|
|
126
|
+
env["TELEMETRY_EXPORTERS"] = "console,json"
|
|
127
|
+
cmd = [base, *extra, *build_command(command_python())[1:]]
|
|
128
|
+
|
|
129
|
+
start = time.perf_counter()
|
|
130
|
+
proc = subprocess.run(
|
|
131
|
+
cmd,
|
|
132
|
+
cwd=str(repo_root),
|
|
133
|
+
env=env,
|
|
134
|
+
stdout=subprocess.DEVNULL,
|
|
135
|
+
stderr=subprocess.DEVNULL,
|
|
136
|
+
check=False,
|
|
137
|
+
)
|
|
138
|
+
elapsed_ms = (time.perf_counter() - start) * 1000.0
|
|
139
|
+
durations_ms.append(elapsed_ms)
|
|
140
|
+
if proc.returncode != 0:
|
|
141
|
+
failures += 1
|
|
142
|
+
|
|
143
|
+
avg_ms = statistics.mean(durations_ms) if durations_ms else 0.0
|
|
144
|
+
p95_ms = statistics.quantiles(durations_ms, n=20)[18] if len(durations_ms) >= 20 else max(durations_ms, default=0.0)
|
|
145
|
+
|
|
146
|
+
return {
|
|
147
|
+
"telemetry_enabled": telemetry_enabled,
|
|
148
|
+
"iterations": iterations,
|
|
149
|
+
"failures": failures,
|
|
150
|
+
"avg_ms": round(avg_ms, 3),
|
|
151
|
+
"min_ms": round(min(durations_ms) if durations_ms else 0.0, 3),
|
|
152
|
+
"max_ms": round(max(durations_ms) if durations_ms else 0.0, 3),
|
|
153
|
+
"p95_ms": round(p95_ms, 3),
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
enabled = run_with_variant(True)
|
|
157
|
+
disabled = run_with_variant(False)
|
|
158
|
+
|
|
159
|
+
overhead_ms = enabled["avg_ms"] - disabled["avg_ms"]
|
|
160
|
+
overhead_pct = ((overhead_ms / disabled["avg_ms"]) * 100.0) if disabled["avg_ms"] > 0 else 0.0
|
|
161
|
+
|
|
162
|
+
report = {
|
|
163
|
+
"timestamp": utc_now(),
|
|
164
|
+
"workload": "scripts/intake/resolve-routing.py",
|
|
165
|
+
"iterations": iterations,
|
|
166
|
+
"with_telemetry": enabled,
|
|
167
|
+
"without_telemetry": disabled,
|
|
168
|
+
"overhead": {
|
|
169
|
+
"avg_ms": round(overhead_ms, 3),
|
|
170
|
+
"avg_pct": round(overhead_pct, 3),
|
|
171
|
+
},
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
out_path = (repo_root / args.out).resolve()
|
|
175
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
176
|
+
out_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
177
|
+
|
|
178
|
+
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
179
|
+
print(f"Benchmark report: {out_path}")
|
|
180
|
+
return 0
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__":
|
|
184
|
+
raise SystemExit(main())
|
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
|
+
import sys
|
|
3
4
|
from datetime import datetime, timezone
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from typing import Any, Dict, List
|
|
6
7
|
|
|
8
|
+
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
9
|
+
if str(_REPO_ROOT) not in sys.path:
|
|
10
|
+
sys.path.insert(0, str(_REPO_ROOT))
|
|
11
|
+
|
|
12
|
+
from telemetry import build_telemetry_collector
|
|
13
|
+
|
|
7
14
|
from orchestrator.wiki.graph_consolidator import GraphConsolidator
|
|
8
15
|
from orchestrator.wiki.incremental_engine import IncrementalEngine
|
|
9
16
|
from orchestrator.wiki.plugin_manager import PluginManager
|
|
@@ -106,82 +113,107 @@ def _write_routing_event(
|
|
|
106
113
|
def main() -> int:
|
|
107
114
|
start = datetime.now(timezone.utc)
|
|
108
115
|
env_snapshot = _read_session_env()
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
"
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
"
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
116
|
+
collector = build_telemetry_collector(REPO_ROOT)
|
|
117
|
+
|
|
118
|
+
with collector.start_execution(operation="autodocs-compiler", session_id="autodocs"):
|
|
119
|
+
collector.record_event("ExecutionStarted", {"pipeline": "autodocs", "env_keys": sorted(env_snapshot.keys())})
|
|
120
|
+
|
|
121
|
+
manager = PluginManager(telemetry_collector=collector)
|
|
122
|
+
include_external = str(os.getenv("AUTODOCS_INCLUDE_EXTERNAL_PROVIDERS", "")).strip().lower() in {
|
|
123
|
+
"1",
|
|
124
|
+
"true",
|
|
125
|
+
"yes",
|
|
126
|
+
"on",
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
# Keep CI output deterministic: external providers depend on local intake artifacts that
|
|
130
|
+
# may not exist on clean runners.
|
|
131
|
+
if include_external:
|
|
132
|
+
manager.register_provider(CodeGraphProvider(REPO_ROOT / "repo-intake" / "generated"))
|
|
133
|
+
manager.register_provider(GraphifyProvider(REPO_ROOT / "repo-intake" / "generated"))
|
|
134
|
+
|
|
135
|
+
manager.register_provider(RepoContentProvider(REPO_ROOT))
|
|
136
|
+
|
|
137
|
+
with collector.start_span(name="autodocs.providers", kind="INTERNAL"):
|
|
138
|
+
provider_result = manager.gather_all()
|
|
139
|
+
contracts = provider_result.get("contracts", [])
|
|
140
|
+
provider_errors = provider_result.get("errors", [])
|
|
141
|
+
collector.record_metric("provider_calls", float(len(contracts) + len(provider_errors)), unit="count")
|
|
142
|
+
|
|
143
|
+
consolidator = GraphConsolidator(
|
|
144
|
+
output_graph_path=UNIFIED_GRAPH_PATH,
|
|
145
|
+
markdown_output_dir=MARKDOWN_OUTPUT_DIR,
|
|
146
|
+
telemetry_collector=collector,
|
|
147
|
+
)
|
|
148
|
+
with collector.start_span(name="autodocs.consolidate", kind="INTERNAL"):
|
|
149
|
+
unified_graph = consolidator.consolidate(contracts)
|
|
150
|
+
wiki_nodes = unified_graph.get("wiki_nodes", {})
|
|
151
|
+
|
|
152
|
+
incremental_engine = IncrementalEngine(UNIFIED_GRAPH_PATH, telemetry_collector=collector)
|
|
153
|
+
with collector.start_span(name="autodocs.incremental", kind="INTERNAL"):
|
|
154
|
+
diff = incremental_engine.diff(wiki_nodes)
|
|
155
|
+
dirty_nodes = diff.get("dirty_nodes", {})
|
|
156
|
+
deleted_nodes = diff.get("deleted_nodes", [])
|
|
157
|
+
cached_nodes = diff.get("cached_graph", {}).get("nodes", {})
|
|
158
|
+
|
|
159
|
+
validator = WikiValidator(SCHEMA_PATH, telemetry_collector=collector)
|
|
160
|
+
with collector.start_span(name="autodocs.validate", kind="INTERNAL"):
|
|
161
|
+
validation_report = validator.validate(wiki_nodes)
|
|
162
|
+
validator.write_reports(
|
|
163
|
+
json_path=VALIDATION_REPORT_JSON,
|
|
164
|
+
markdown_path=VALIDATION_REPORT_MD,
|
|
165
|
+
report=validation_report,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
with collector.start_span(name="autodocs.project", kind="INTERNAL"):
|
|
169
|
+
project_stats = consolidator.project_dirty_nodes(
|
|
170
|
+
dirty_nodes=dirty_nodes,
|
|
171
|
+
deleted_nodes=deleted_nodes,
|
|
172
|
+
all_nodes=wiki_nodes,
|
|
173
|
+
cached_nodes=cached_nodes,
|
|
174
|
+
)
|
|
175
|
+
consolidator.persist_graph(unified_graph)
|
|
176
|
+
consolidator.persist_manifests(unified_graph)
|
|
177
|
+
|
|
178
|
+
elapsed_ms = int((datetime.now(timezone.utc) - start).total_seconds() * 1000)
|
|
179
|
+
iteration_event: Dict[str, Any] = {
|
|
180
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
181
|
+
"source_type": "technical-docs",
|
|
182
|
+
"module": "autodocs",
|
|
183
|
+
"total_nodes": len(wiki_nodes),
|
|
184
|
+
"dirty_nodes": len(dirty_nodes),
|
|
185
|
+
"deleted_nodes": len(deleted_nodes),
|
|
186
|
+
"rendered_markdown": int(project_stats.get("rendered", 0)),
|
|
187
|
+
"deleted_markdown": int(project_stats.get("deleted", 0)),
|
|
188
|
+
"rendered_indexes": int(project_stats.get("indexes", 0)),
|
|
189
|
+
"rendered_manifests": int(project_stats.get("manifests", 0)),
|
|
190
|
+
"provider_errors": len(provider_errors),
|
|
191
|
+
"validation_errors": int(validation_report.get("summary", {}).get("error_count", 0)),
|
|
192
|
+
"validation_warnings": int(validation_report.get("summary", {}).get("warning_count", 0)),
|
|
193
|
+
"elapsed_ms": elapsed_ms,
|
|
194
|
+
}
|
|
195
|
+
_append_jsonl(ITERATION_LOG, iteration_event)
|
|
196
|
+
_write_routing_event(
|
|
197
|
+
env_snapshot=env_snapshot,
|
|
198
|
+
dirty_count=len(dirty_nodes),
|
|
199
|
+
total_nodes=len(wiki_nodes),
|
|
200
|
+
errors=provider_errors,
|
|
201
|
+
elapsed_ms=elapsed_ms,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
collector.record_metric("execution_time_ms", float(elapsed_ms), unit="ms")
|
|
205
|
+
collector.record_metric("warning_count", float(len(provider_errors)), unit="count")
|
|
206
|
+
collector.record_event(
|
|
207
|
+
"KnowledgeGenerated",
|
|
208
|
+
{
|
|
209
|
+
"total_nodes": len(wiki_nodes),
|
|
210
|
+
"dirty_nodes": len(dirty_nodes),
|
|
211
|
+
"deleted_nodes": len(deleted_nodes),
|
|
212
|
+
"elapsed_ms": elapsed_ms,
|
|
213
|
+
},
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
collector.shutdown()
|
|
185
217
|
|
|
186
218
|
if int(validation_report.get("summary", {}).get("error_count", 0)) > 0:
|
|
187
219
|
return 1
|
|
@@ -14,6 +14,7 @@ class RepoContentProvider(BaseWikiProvider):
|
|
|
14
14
|
|
|
15
15
|
def gather_knowledge(self) -> Dict[str, Any]:
|
|
16
16
|
entities: List[Dict[str, Any]] = []
|
|
17
|
+
entities.extend(self._collect_core_docs())
|
|
17
18
|
entities.extend(self._collect_agents())
|
|
18
19
|
entities.extend(self._collect_skills())
|
|
19
20
|
entities.extend(self._collect_policies())
|
|
@@ -25,6 +26,42 @@ class RepoContentProvider(BaseWikiProvider):
|
|
|
25
26
|
self._enrich_relations(entities)
|
|
26
27
|
return {"provider_id": self.provider_id, "entities": entities}
|
|
27
28
|
|
|
29
|
+
def _collect_core_docs(self) -> List[Dict[str, Any]]:
|
|
30
|
+
targets: List[Tuple[Path, str]] = [
|
|
31
|
+
(self.repo_root / "README.md", "Guia principal del engine y sus flujos operativos."),
|
|
32
|
+
(self.repo_root / "README_WIKI.md", "Guia de uso y alcance de la wiki AutoDocs del repositorio."),
|
|
33
|
+
(self.repo_root / "FINAL_USAGE_GUIDE.md", "Guia de uso final con pasos operativos y buenas practicas."),
|
|
34
|
+
(self.repo_root / "FILE_INDEX.md", "Indice del contenido y estructura del repositorio."),
|
|
35
|
+
(self.repo_root / "autodocs" / "README.md", "Referencia de ejecucion y artefactos de AutoDocs."),
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
entities: List[Dict[str, Any]] = []
|
|
39
|
+
for path, fallback_summary in targets:
|
|
40
|
+
if not path.exists():
|
|
41
|
+
continue
|
|
42
|
+
entities.append(
|
|
43
|
+
self._entity_from_markdown(
|
|
44
|
+
path,
|
|
45
|
+
kind="report",
|
|
46
|
+
section="reports",
|
|
47
|
+
domain="documentation",
|
|
48
|
+
fallback_summary=fallback_summary,
|
|
49
|
+
)
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
package_json = self.repo_root / "package.json"
|
|
53
|
+
if package_json.exists():
|
|
54
|
+
entities.append(
|
|
55
|
+
self._entity_from_json(
|
|
56
|
+
package_json,
|
|
57
|
+
kind="report",
|
|
58
|
+
section="reports",
|
|
59
|
+
domain="packaging",
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
return entities
|
|
64
|
+
|
|
28
65
|
def _collect_agents(self) -> List[Dict[str, Any]]:
|
|
29
66
|
agents_dir = self.repo_root / ".github" / "agents"
|
|
30
67
|
if not agents_dir.exists():
|
|
@@ -292,6 +329,18 @@ class RepoContentProvider(BaseWikiProvider):
|
|
|
292
329
|
for target_id in ["AGENTS.md", "orchestrator/router.md"]:
|
|
293
330
|
if target_id in entities_by_id:
|
|
294
331
|
self._add_relation(entity, target_id, "explains")
|
|
332
|
+
if source_ref == "README.md":
|
|
333
|
+
for target_id in ["AGENTS.md", "ARCHITECTURE.md", "FINAL_USAGE_GUIDE.md", "package.json"]:
|
|
334
|
+
if target_id in entities_by_id:
|
|
335
|
+
self._add_relation(entity, target_id, "documents")
|
|
336
|
+
if source_ref == "FINAL_USAGE_GUIDE.md":
|
|
337
|
+
for target_id in ["README.md", "scripts/README.md"]:
|
|
338
|
+
if target_id in entities_by_id:
|
|
339
|
+
self._add_relation(entity, target_id, "extends")
|
|
340
|
+
if source_ref == "autodocs/README.md":
|
|
341
|
+
for target_id in ["README_WIKI.md", "scripts/wiki/wiki_compiler.py"]:
|
|
342
|
+
if target_id in entities_by_id:
|
|
343
|
+
self._add_relation(entity, target_id, "documents")
|
|
295
344
|
if source_ref == "autodocs/analysis_mcpee/OPENWIKI_INTERNAL_BLUEPRINT.md":
|
|
296
345
|
for target_id in ["autodocs/README.md", "README_WIKI.md"]:
|
|
297
346
|
if target_id in entities_by_id:
|
|
@@ -364,6 +413,8 @@ class RepoContentProvider(BaseWikiProvider):
|
|
|
364
413
|
stem = path.name
|
|
365
414
|
if path.name.lower() == "skill.md" and path.parent.name:
|
|
366
415
|
stem = path.parent.name
|
|
416
|
+
elif path.stem.lower() == "readme" and path.parent != self.repo_root:
|
|
417
|
+
stem = f"{path.parent.name}-{path.name}"
|
|
367
418
|
tokens = [kind, stem]
|
|
368
419
|
if kind == "report" and "analysis_mcpee" in relative:
|
|
369
420
|
tokens.insert(1, "autodocs")
|