mcp-efficiency-engine 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/.github/workflows/autodocs-sync.yml +2 -15
  2. package/.github/workflows/release-npm.yml +111 -0
  3. package/FINAL_USAGE_GUIDE.md +56 -0
  4. package/README.md +133 -0
  5. package/autodocs/README.md +6 -0
  6. package/observability/metrics.md +33 -0
  7. package/observability/scoring-rubric.md +20 -0
  8. package/orchestrator/wiki/graph_consolidator.py +60 -1
  9. package/orchestrator/wiki/incremental_engine.py +23 -1
  10. package/orchestrator/wiki/plugin_manager.py +87 -22
  11. package/package.json +8 -2
  12. package/requirements.txt +1 -0
  13. package/scripts/intake/resolve-routing.py +124 -87
  14. package/scripts/intake/validate-database-routing.py +13 -4
  15. package/scripts/intake/validate-rag-routing.py +4 -3
  16. package/scripts/learning/learning-loop-report.py +28 -10
  17. package/scripts/ops/llm-token-probe.py +103 -39
  18. package/scripts/ops/telemetry-benchmark.py +184 -0
  19. package/scripts/wiki/compiler_main.py +108 -76
  20. package/scripts/wiki/providers/repo_content_provider.py +51 -0
  21. package/scripts/wiki/validator.py +40 -15
  22. package/telemetry/__init__.py +4 -0
  23. package/telemetry/bootstrap.py +47 -0
  24. package/telemetry/collector/engine.py +261 -0
  25. package/telemetry/config.json +20 -0
  26. package/telemetry/config.py +131 -0
  27. package/telemetry/context/propagation.py +85 -0
  28. package/telemetry/events/catalog.py +24 -0
  29. package/telemetry/events/factory.py +42 -0
  30. package/telemetry/exporters/base.py +16 -0
  31. package/telemetry/exporters/console/exporter.py +30 -0
  32. package/telemetry/exporters/json/exporter.py +34 -0
  33. package/telemetry/exporters/langsmith/exporter.py +415 -0
  34. package/telemetry/metrics/aggregator.py +60 -0
  35. package/telemetry/scoring/efficiency.py +68 -0
  36. package/telemetry/tracing/span.py +43 -0
  37. package/observability/evals/chat-token-usage-report.json +0 -64
  38. package/observability/evals/chat-token-usage-report.md +0 -33
  39. package/observability/evals/iteration-value-report.json +0 -76
  40. package/observability/evals/iteration-value-report.md +0 -39
  41. package/observability/evals/learning-loop-report.json +0 -268
  42. package/observability/evals/learning-loop-report.md +0 -37
  43. package/observability/evals/repo-intake-eval-cases.md +0 -23
  44. package/observability/evals/routing-eval-cases.json +0 -68
  45. package/observability/evals/routing-eval-cases.md +0 -25
  46. package/observability/evals/routing-eval-report.json +0 -72
  47. package/observability/logs/copilot-usage-ingest-report.json +0 -13
  48. package/observability/logs/iteration-metrics.jsonl +0 -75
  49. package/observability/logs/learning-feedback.jsonl +0 -152
  50. package/observability/logs/routing-decisions.jsonl +0 -333
  51. package/observability/logs/session/agent-pipeline-preflight-20260701-004146.log +0 -8
  52. package/observability/logs/session/agent-pipeline-preflight-20260701-005554.log +0 -8
  53. package/observability/logs/session/agent-pipeline-preflight-20260701-010351.log +0 -8
  54. package/observability/logs/session/agent-pipeline-preflight-20260704-145829.log +0 -8
  55. package/observability/logs/session/agent-pipeline-preflight-20260704-150021.log +0 -8
  56. package/observability/logs/session/agent-pipeline-preflight-20260704-154328.log +0 -8
  57. package/observability/logs/session/bye-20260629-221513.json +0 -39
  58. package/observability/logs/session/bye-20260629-221545.json +0 -59
  59. package/observability/logs/session/bye-20260629-222007.json +0 -52
  60. package/observability/logs/session/bye-20260629-222133.json +0 -59
  61. package/observability/logs/session/bye-20260629-222412.json +0 -45
  62. package/observability/logs/session/bye-20260629-223224.json +0 -52
  63. package/observability/logs/session/bye-20260629-224050.json +0 -59
  64. package/observability/logs/session/bye-20260630-105732.json +0 -24
  65. package/observability/logs/session/bye-20260630-211536.json +0 -87
  66. package/observability/logs/session/bye-20260630-223423.json +0 -87
  67. package/observability/logs/session/bye-20260630-232534.json +0 -87
  68. package/observability/logs/session/bye-20260701-003638.json +0 -126
  69. package/observability/logs/session/bye-20260701-003849.json +0 -126
  70. package/observability/logs/session/bye-20260701-004315.json +0 -138
  71. package/observability/logs/session/bye-20260701-004559.json +0 -76
  72. package/observability/logs/session/bye-20260701-005913.json +0 -138
  73. package/observability/logs/session/bye-20260701-010753.json +0 -138
  74. package/observability/logs/session/bye-20260703-194106.json +0 -24
  75. package/observability/logs/session/bye-20260703-233443.json +0 -76
  76. package/observability/logs/session/bye-20260703-233653.json +0 -76
  77. package/observability/logs/session/bye-20260704-150317.json +0 -152
  78. package/observability/logs/session/bye-20260704-150929.json +0 -75
  79. package/observability/logs/session/bye-20260704-151050.json +0 -75
  80. package/observability/logs/session/bye-20260705-115458.json +0 -165
  81. package/observability/logs/session/codegraph-status-20260701-004228.log +0 -39
  82. package/observability/logs/session/codegraph-status-20260701-004310.log +0 -39
  83. package/observability/logs/session/codegraph-status-20260701-005637.log +0 -39
  84. package/observability/logs/session/codegraph-status-20260701-005908.log +0 -39
  85. package/observability/logs/session/codegraph-status-20260701-010444.log +0 -39
  86. package/observability/logs/session/codegraph-status-20260701-010747.log +0 -39
  87. package/observability/logs/session/codegraph-status-20260704-150137.log +0 -26
  88. package/observability/logs/session/codegraph-status-20260704-150313.log +0 -28
  89. package/observability/logs/session/codegraph-status-20260704-154445.log +0 -26
  90. package/observability/logs/session/codegraph-status-20260705-115454.log +0 -27
  91. package/observability/logs/session/codegraph-sync-20260701-004211.log +0 -88
  92. package/observability/logs/session/codegraph-sync-20260701-004241.log +0 -82
  93. package/observability/logs/session/codegraph-sync-20260701-005620.log +0 -89
  94. package/observability/logs/session/codegraph-sync-20260701-005840.log +0 -88
  95. package/observability/logs/session/codegraph-sync-20260701-010419.log +0 -93
  96. package/observability/logs/session/codegraph-sync-20260701-010609.log +0 -90
  97. package/observability/logs/session/codegraph-sync-20260704-145856.log +0 -70
  98. package/observability/logs/session/codegraph-sync-20260704-150047.log +0 -53
  99. package/observability/logs/session/codegraph-sync-20260704-150204.log +0 -44
  100. package/observability/logs/session/codegraph-sync-20260704-154353.log +0 -26
  101. package/observability/logs/session/codegraph-sync-20260705-115337.log +0 -32
  102. package/observability/logs/session/hi-20260629-221459.json +0 -38
  103. package/observability/logs/session/hi-20260629-221945.json +0 -59
  104. package/observability/logs/session/hi-20260629-222108.json +0 -66
  105. package/observability/logs/session/hi-20260629-222352.json +0 -45
  106. package/observability/logs/session/hi-20260629-225719.json +0 -59
  107. package/observability/logs/session/hi-20260629-233722.json +0 -52
  108. package/observability/logs/session/hi-20260629-233913.json +0 -198
  109. package/observability/logs/session/hi-20260630-110156.json +0 -177
  110. package/observability/logs/session/hi-20260630-191243.json +0 -184
  111. package/observability/logs/session/hi-20260630-230050.json +0 -234
  112. package/observability/logs/session/hi-20260630-230523.json +0 -227
  113. package/observability/logs/session/hi-20260630-230855.json +0 -227
  114. package/observability/logs/session/hi-20260630-231638.json +0 -227
  115. package/observability/logs/session/hi-20260630-231756.json +0 -219
  116. package/observability/logs/session/hi-20260701-002842.json +0 -234
  117. package/observability/logs/session/hi-20260701-003458.json +0 -282
  118. package/observability/logs/session/hi-20260701-004233.json +0 -290
  119. package/observability/logs/session/hi-20260701-004554.json +0 -236
  120. package/observability/logs/session/hi-20260701-005641.json +0 -290
  121. package/observability/logs/session/hi-20260701-010450.json +0 -290
  122. package/observability/logs/session/hi-20260703-233854.json +0 -312
  123. package/observability/logs/session/hi-20260703-234448.json +0 -318
  124. package/observability/logs/session/hi-20260704-145907.json +0 -353
  125. package/observability/logs/session/hi-20260704-150141.json +0 -392
  126. package/observability/logs/session/hi-20260704-154448.json +0 -392
  127. package/observability/logs/session/iteration-value-refresh-20260701-004309.log +0 -3
  128. package/observability/logs/session/iteration-value-refresh-20260701-005907.log +0 -3
  129. package/observability/logs/session/iteration-value-refresh-20260701-010747.log +0 -3
  130. package/observability/logs/session/iteration-value-refresh-20260704-150312.log +0 -3
  131. package/observability/logs/session/iteration-value-refresh-20260705-115453.log +0 -3
  132. package/observability/logs/session/learning-loop-refresh-20260701-004309.log +0 -3
  133. package/observability/logs/session/learning-loop-refresh-20260701-005906.log +0 -3
  134. package/observability/logs/session/learning-loop-refresh-20260701-010745.log +0 -3
  135. package/observability/logs/session/learning-loop-refresh-20260704-150309.log +0 -3
  136. package/observability/logs/session/learning-loop-refresh-20260705-115450.log +0 -3
  137. package/observability/logs/session/project-notes-refresh-20260701-004227.log +0 -1
  138. package/observability/logs/session/project-notes-refresh-20260701-004308.log +0 -1
  139. package/observability/logs/session/project-notes-refresh-20260701-005636.log +0 -1
  140. package/observability/logs/session/project-notes-refresh-20260701-005906.log +0 -1
  141. package/observability/logs/session/project-notes-refresh-20260701-010443.log +0 -1
  142. package/observability/logs/session/project-notes-refresh-20260701-010744.log +0 -1
  143. package/observability/logs/session/project-notes-refresh-20260704-150133.log +0 -1
  144. package/observability/logs/session/project-notes-refresh-20260704-150309.log +0 -1
  145. package/observability/logs/session/project-notes-refresh-20260704-154442.log +0 -1
  146. package/observability/logs/session/project-notes-refresh-20260705-115449.log +0 -1
  147. package/observability/logs/session/repo-intake-20260701-004218.log +0 -6
  148. package/observability/logs/session/repo-intake-20260701-005626.log +0 -6
  149. package/observability/logs/session/repo-intake-20260701-010433.log +0 -6
  150. package/observability/logs/session/repo-intake-20260704-150053.log +0 -6
  151. package/observability/logs/session/repo-intake-20260704-154404.log +0 -6
  152. package/observability/logs/session/repo-intake-refresh-20260701-004247.log +0 -6
  153. package/observability/logs/session/repo-intake-refresh-20260701-005847.log +0 -6
  154. package/observability/logs/session/repo-intake-refresh-20260701-010615.log +0 -6
  155. package/observability/logs/session/repo-intake-refresh-20260704-150209.log +0 -6
  156. package/observability/logs/session/repo-intake-refresh-20260705-115342.log +0 -6
  157. package/observability/logs/session/repo-registry-validation-20260701-004234.log +0 -2
  158. package/observability/logs/session/repo-registry-validation-20260701-004556.log +0 -2
  159. package/observability/logs/session/repo-registry-validation-20260701-005833.log +0 -2
  160. package/observability/logs/session/repo-registry-validation-20260701-010601.log +0 -2
  161. package/observability/logs/session/repo-registry-validation-20260704-150155.log +0 -2
  162. package/observability/logs/session/repo-registry-validation-20260705-115323.log +0 -2
  163. package/observability/logs/session/repomix-refresh-20260701-003823.log +0 -4236
  164. package/observability/logs/session/repomix-refresh-20260701-004251.log +0 -4275
  165. package/observability/logs/session/repomix-refresh-20260701-005850.log +0 -4248
  166. package/observability/logs/session/repomix-refresh-20260701-010619.log +0 -6531
  167. package/observability/logs/session/repomix-refresh-20260704-150239.log +0 -652
  168. package/observability/logs/session/repomix-refresh-20260705-115419.log +0 -178
  169. package/observability/logs/session/routing-evals-20260701-004221.log +0 -2
  170. package/observability/logs/session/routing-evals-20260701-004235.log +0 -2
  171. package/observability/logs/session/routing-evals-20260701-005629.log +0 -2
  172. package/observability/logs/session/routing-evals-20260701-005834.log +0 -2
  173. package/observability/logs/session/routing-evals-20260701-010437.log +0 -2
  174. package/observability/logs/session/routing-evals-20260701-010603.log +0 -2
  175. package/observability/logs/session/routing-evals-20260704-150125.log +0 -2
  176. package/observability/logs/session/routing-evals-20260704-150157.log +0 -2
  177. package/observability/logs/session/routing-evals-20260704-154434.log +0 -2
  178. package/observability/logs/session/routing-evals-20260705-115327.log +0 -2
  179. package/observability/logs/session/sibling-discovery-refresh-20260701-004250.log +0 -3
  180. package/observability/logs/session/sibling-discovery-refresh-20260701-005850.log +0 -3
  181. package/observability/logs/session/sibling-discovery-refresh-20260701-010618.log +0 -3
  182. package/observability/logs/session/sibling-discovery-refresh-20260704-150238.log +0 -3
  183. package/observability/logs/session/sibling-discovery-refresh-20260705-115418.log +0 -3
@@ -17,9 +17,16 @@ from __future__ import annotations
17
17
 
18
18
  import argparse
19
19
  import os
20
+ import sys
20
21
  from pathlib import Path
21
22
  from typing import List
22
23
 
24
+ _REPO_ROOT = Path(__file__).resolve().parents[2]
25
+ if str(_REPO_ROOT) not in sys.path:
26
+ sys.path.insert(0, str(_REPO_ROOT))
27
+
28
+ from telemetry import build_telemetry_collector
29
+
23
30
  try:
24
31
  from openai import AzureOpenAI
25
32
  except Exception:
@@ -81,7 +88,10 @@ def main() -> int:
81
88
  args = parser.parse_args()
82
89
 
83
90
  repo_root = Path(args.repo_root)
91
+ collector = build_telemetry_collector(_REPO_ROOT)
84
92
  if not repo_root.exists():
93
+ collector.record_event("WarningGenerated", {"warning": "repo_root_missing", "repo_root": str(repo_root)}, level="WARNING")
94
+ collector.shutdown()
85
95
  print("prompt_tokens=0")
86
96
  print("completion_tokens=0")
87
97
  print("total_tokens=0")
@@ -99,6 +109,17 @@ def main() -> int:
99
109
  model = os.getenv("OPENAI_CHAT_MODEL", "gpt-4o-mini")
100
110
 
101
111
  if AzureOpenAI is None or not api_key or not endpoint:
112
+ collector.record_event(
113
+ "WarningGenerated",
114
+ {
115
+ "warning": "missing_credentials_or_openai_pkg",
116
+ "has_openai_pkg": AzureOpenAI is not None,
117
+ "has_api_key": bool(api_key),
118
+ "has_endpoint": bool(endpoint),
119
+ },
120
+ level="WARNING",
121
+ )
122
+ collector.shutdown()
102
123
  print("prompt_tokens=0")
103
124
  print("completion_tokens=0")
104
125
  print("total_tokens=0")
@@ -119,45 +140,88 @@ def main() -> int:
119
140
  "Devuelve un resumen tecnico corto con riesgos y evidencia."
120
141
  )
121
142
 
122
- try:
123
- response = client.chat.completions.create(
124
- model=model,
125
- messages=[
126
- {"role": "system", "content": system_prompt},
127
- {"role": "user", "content": user_prompt},
128
- ],
129
- temperature=0,
130
- max_completion_tokens=400,
131
- )
132
-
133
- usage = response.usage
134
- prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
135
- completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
136
- total_tokens = int(getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0)
137
-
138
- price_in = to_float(os.getenv("LLM_PRICE_INPUT_USD_PER_1K"), 0.0)
139
- price_out = to_float(os.getenv("LLM_PRICE_OUTPUT_USD_PER_1K"), 0.0)
140
- if price_in > 0 or price_out > 0:
141
- cost = (prompt_tokens / 1000.0) * price_in + (completion_tokens / 1000.0) * price_out
142
- else:
143
- # fallback simple when split pricing is unknown
144
- flat = to_float(os.getenv("LLM_PRICE_USD_PER_1K_TOKENS"), 0.0)
145
- cost = (total_tokens / 1000.0) * flat if flat > 0 else 0.0
146
-
147
- print(f"prompt_tokens={prompt_tokens}")
148
- print(f"completion_tokens={completion_tokens}")
149
- print(f"total_tokens={total_tokens}")
150
- print(f"total_cost_usd={cost:.6f}")
151
- print("probe_status=ok")
152
- return 0
153
-
154
- except Exception as ex:
155
- print("prompt_tokens=0")
156
- print("completion_tokens=0")
157
- print("total_tokens=0")
158
- print("total_cost_usd=0")
159
- print(f"probe_status=error:{str(ex).replace(os.linesep, ' ')[:200]}")
160
- return 0
143
+ with collector.start_execution(
144
+ operation="llm-token-probe",
145
+ session_id="ops",
146
+ provider="azure-openai",
147
+ model=model,
148
+ ):
149
+ with collector.start_span(name="provider.azure_openai_call", kind="CLIENT", attributes={"profile": args.profile}):
150
+ collector.record_event(
151
+ "LLMRequest",
152
+ {
153
+ "provider": "azure-openai",
154
+ "model": model,
155
+ "profile": args.profile,
156
+ "selected_files": len(files),
157
+ },
158
+ )
159
+ try:
160
+ response = client.chat.completions.create(
161
+ model=model,
162
+ messages=[
163
+ {"role": "system", "content": system_prompt},
164
+ {"role": "user", "content": user_prompt},
165
+ ],
166
+ temperature=0,
167
+ max_completion_tokens=400,
168
+ )
169
+
170
+ usage = response.usage
171
+ prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
172
+ completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
173
+ total_tokens = int(getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0)
174
+
175
+ price_in = to_float(os.getenv("LLM_PRICE_INPUT_USD_PER_1K"), 0.0)
176
+ price_out = to_float(os.getenv("LLM_PRICE_OUTPUT_USD_PER_1K"), 0.0)
177
+ if price_in > 0 or price_out > 0:
178
+ cost = (prompt_tokens / 1000.0) * price_in + (completion_tokens / 1000.0) * price_out
179
+ else:
180
+ # fallback simple when split pricing is unknown
181
+ flat = to_float(os.getenv("LLM_PRICE_USD_PER_1K_TOKENS"), 0.0)
182
+ cost = (total_tokens / 1000.0) * flat if flat > 0 else 0.0
183
+
184
+ collector.record_usage(
185
+ input_tokens=prompt_tokens,
186
+ output_tokens=completion_tokens,
187
+ estimated_cost_usd=cost,
188
+ )
189
+ collector.record_event(
190
+ "LLMResponse",
191
+ {
192
+ "provider": "azure-openai",
193
+ "model": model,
194
+ "prompt_tokens": prompt_tokens,
195
+ "completion_tokens": completion_tokens,
196
+ "total_tokens": total_tokens,
197
+ "estimated_cost_usd": round(cost, 6),
198
+ },
199
+ )
200
+
201
+ print(f"prompt_tokens={prompt_tokens}")
202
+ print(f"completion_tokens={completion_tokens}")
203
+ print(f"total_tokens={total_tokens}")
204
+ print(f"total_cost_usd={cost:.6f}")
205
+ print("probe_status=ok")
206
+ collector.shutdown()
207
+ return 0
208
+
209
+ except Exception as ex:
210
+ collector.record_event(
211
+ "ExceptionThrown",
212
+ {
213
+ "provider": "azure-openai",
214
+ "error": str(ex),
215
+ },
216
+ level="ERROR",
217
+ )
218
+ print("prompt_tokens=0")
219
+ print("completion_tokens=0")
220
+ print("total_tokens=0")
221
+ print("total_cost_usd=0")
222
+ print(f"probe_status=error:{str(ex).replace(os.linesep, ' ')[:200]}")
223
+ collector.shutdown()
224
+ return 0
161
225
 
162
226
 
163
227
  if __name__ == "__main__":
@@ -0,0 +1,184 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ import os
6
+ import statistics
7
+ import subprocess
8
+ import time
9
+ from datetime import datetime, timezone
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+
14
+ def utc_now() -> str:
15
+ return datetime.now(timezone.utc).isoformat()
16
+
17
+
18
+ def build_command(python_exe: str) -> list[str]:
19
+ return [
20
+ python_exe,
21
+ "scripts/intake/resolve-routing.py",
22
+ "--input",
23
+ "telemetry benchmark",
24
+ "--intent",
25
+ "feature",
26
+ "--domain",
27
+ "backend",
28
+ "--source-type",
29
+ "code",
30
+ "--input-tokens",
31
+ "120",
32
+ "--output-tokens",
33
+ "45",
34
+ "--estimated-cost-usd",
35
+ "0.0032",
36
+ "--model",
37
+ "GPT-5.3-Codex",
38
+ "--local-tools",
39
+ "2",
40
+ "--remote-tools",
41
+ "1",
42
+ ]
43
+
44
+
45
+ def run_variant(*, repo_root: Path, iterations: int, telemetry_enabled: bool, python_exe: str) -> dict[str, Any]:
46
+ durations_ms: list[float] = []
47
+ failures = 0
48
+ for _ in range(iterations):
49
+ env = os.environ.copy()
50
+ env["TELEMETRY_ENABLED"] = "true" if telemetry_enabled else "false"
51
+ env["TELEMETRY_EXPORTERS"] = "console,json"
52
+
53
+ start = time.perf_counter()
54
+ proc = subprocess.run(
55
+ build_command(python_exe),
56
+ cwd=str(repo_root),
57
+ env=env,
58
+ stdout=subprocess.DEVNULL,
59
+ stderr=subprocess.DEVNULL,
60
+ check=False,
61
+ )
62
+ elapsed_ms = (time.perf_counter() - start) * 1000.0
63
+ durations_ms.append(elapsed_ms)
64
+ if proc.returncode != 0:
65
+ failures += 1
66
+
67
+ avg_ms = statistics.mean(durations_ms) if durations_ms else 0.0
68
+ p95_ms = statistics.quantiles(durations_ms, n=20)[18] if len(durations_ms) >= 20 else max(durations_ms, default=0.0)
69
+
70
+ return {
71
+ "telemetry_enabled": telemetry_enabled,
72
+ "iterations": iterations,
73
+ "failures": failures,
74
+ "avg_ms": round(avg_ms, 3),
75
+ "min_ms": round(min(durations_ms) if durations_ms else 0.0, 3),
76
+ "max_ms": round(max(durations_ms) if durations_ms else 0.0, 3),
77
+ "p95_ms": round(p95_ms, 3),
78
+ }
79
+
80
+
81
+ def main() -> int:
82
+ parser = argparse.ArgumentParser(description="Benchmark telemetry overhead on/off using resolve-routing as workload.")
83
+ parser.add_argument("--iterations", type=int, default=10)
84
+ parser.add_argument("--python", default="py")
85
+ parser.add_argument("--python-arg", default="-3")
86
+ parser.add_argument("--out", default="observability/evals/telemetry-overhead-benchmark.json")
87
+ args = parser.parse_args()
88
+
89
+ repo_root = Path(__file__).resolve().parents[2]
90
+ iterations = max(2, int(args.iterations))
91
+ python_exe = args.python
92
+
93
+ # Windows default launcher pattern: py -3
94
+ if args.python == "py" and args.python_arg:
95
+ python_exe = f"{args.python} {args.python_arg}" # handled below
96
+
97
+ if " " in python_exe:
98
+ tokens = python_exe.split()
99
+ base = tokens[0]
100
+ extra = tokens[1:]
101
+ else:
102
+ base = python_exe
103
+ extra = []
104
+
105
+ def build_python() -> str:
106
+ if not extra:
107
+ return base
108
+ return base
109
+
110
+ # Probe python command in current environment.
111
+ probe = subprocess.run([base, *extra, "--version"], cwd=str(repo_root), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
112
+ if probe.returncode != 0:
113
+ print("Unable to resolve python interpreter for benchmark")
114
+ return 1
115
+
116
+ def command_python() -> str:
117
+ return base
118
+
119
+ # Wrap variant runner to include optional extra interpreter args.
120
+ def run_with_variant(telemetry_enabled: bool) -> dict[str, Any]:
121
+ durations_ms: list[float] = []
122
+ failures = 0
123
+ for _ in range(iterations):
124
+ env = os.environ.copy()
125
+ env["TELEMETRY_ENABLED"] = "true" if telemetry_enabled else "false"
126
+ env["TELEMETRY_EXPORTERS"] = "console,json"
127
+ cmd = [base, *extra, *build_command(command_python())[1:]]
128
+
129
+ start = time.perf_counter()
130
+ proc = subprocess.run(
131
+ cmd,
132
+ cwd=str(repo_root),
133
+ env=env,
134
+ stdout=subprocess.DEVNULL,
135
+ stderr=subprocess.DEVNULL,
136
+ check=False,
137
+ )
138
+ elapsed_ms = (time.perf_counter() - start) * 1000.0
139
+ durations_ms.append(elapsed_ms)
140
+ if proc.returncode != 0:
141
+ failures += 1
142
+
143
+ avg_ms = statistics.mean(durations_ms) if durations_ms else 0.0
144
+ p95_ms = statistics.quantiles(durations_ms, n=20)[18] if len(durations_ms) >= 20 else max(durations_ms, default=0.0)
145
+
146
+ return {
147
+ "telemetry_enabled": telemetry_enabled,
148
+ "iterations": iterations,
149
+ "failures": failures,
150
+ "avg_ms": round(avg_ms, 3),
151
+ "min_ms": round(min(durations_ms) if durations_ms else 0.0, 3),
152
+ "max_ms": round(max(durations_ms) if durations_ms else 0.0, 3),
153
+ "p95_ms": round(p95_ms, 3),
154
+ }
155
+
156
+ enabled = run_with_variant(True)
157
+ disabled = run_with_variant(False)
158
+
159
+ overhead_ms = enabled["avg_ms"] - disabled["avg_ms"]
160
+ overhead_pct = ((overhead_ms / disabled["avg_ms"]) * 100.0) if disabled["avg_ms"] > 0 else 0.0
161
+
162
+ report = {
163
+ "timestamp": utc_now(),
164
+ "workload": "scripts/intake/resolve-routing.py",
165
+ "iterations": iterations,
166
+ "with_telemetry": enabled,
167
+ "without_telemetry": disabled,
168
+ "overhead": {
169
+ "avg_ms": round(overhead_ms, 3),
170
+ "avg_pct": round(overhead_pct, 3),
171
+ },
172
+ }
173
+
174
+ out_path = (repo_root / args.out).resolve()
175
+ out_path.parent.mkdir(parents=True, exist_ok=True)
176
+ out_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
177
+
178
+ print(json.dumps(report, indent=2, ensure_ascii=False))
179
+ print(f"Benchmark report: {out_path}")
180
+ return 0
181
+
182
+
183
+ if __name__ == "__main__":
184
+ raise SystemExit(main())
@@ -1,9 +1,16 @@
1
1
  import json
2
2
  import os
3
+ import sys
3
4
  from datetime import datetime, timezone
4
5
  from pathlib import Path
5
6
  from typing import Any, Dict, List
6
7
 
8
+ _REPO_ROOT = Path(__file__).resolve().parents[2]
9
+ if str(_REPO_ROOT) not in sys.path:
10
+ sys.path.insert(0, str(_REPO_ROOT))
11
+
12
+ from telemetry import build_telemetry_collector
13
+
7
14
  from orchestrator.wiki.graph_consolidator import GraphConsolidator
8
15
  from orchestrator.wiki.incremental_engine import IncrementalEngine
9
16
  from orchestrator.wiki.plugin_manager import PluginManager
@@ -106,82 +113,107 @@ def _write_routing_event(
106
113
  def main() -> int:
107
114
  start = datetime.now(timezone.utc)
108
115
  env_snapshot = _read_session_env()
109
-
110
- manager = PluginManager()
111
- include_external = str(os.getenv("AUTODOCS_INCLUDE_EXTERNAL_PROVIDERS", "")).strip().lower() in {
112
- "1",
113
- "true",
114
- "yes",
115
- "on",
116
- }
117
-
118
- # Keep CI output deterministic: external providers depend on local intake artifacts that
119
- # may not exist on clean runners.
120
- if include_external:
121
- manager.register_provider(CodeGraphProvider(REPO_ROOT / "repo-intake" / "generated"))
122
- manager.register_provider(GraphifyProvider(REPO_ROOT / "repo-intake" / "generated"))
123
-
124
- manager.register_provider(RepoContentProvider(REPO_ROOT))
125
-
126
- provider_result = manager.gather_all()
127
- contracts = provider_result.get("contracts", [])
128
- provider_errors = provider_result.get("errors", [])
129
-
130
- consolidator = GraphConsolidator(
131
- output_graph_path=UNIFIED_GRAPH_PATH,
132
- markdown_output_dir=MARKDOWN_OUTPUT_DIR,
133
- )
134
- unified_graph = consolidator.consolidate(contracts)
135
- wiki_nodes = unified_graph.get("wiki_nodes", {})
136
-
137
- incremental_engine = IncrementalEngine(UNIFIED_GRAPH_PATH)
138
- diff = incremental_engine.diff(wiki_nodes)
139
- dirty_nodes = diff.get("dirty_nodes", {})
140
- deleted_nodes = diff.get("deleted_nodes", [])
141
- cached_nodes = diff.get("cached_graph", {}).get("nodes", {})
142
-
143
- validator = WikiValidator(SCHEMA_PATH)
144
- validation_report = validator.validate(wiki_nodes)
145
- validator.write_reports(
146
- json_path=VALIDATION_REPORT_JSON,
147
- markdown_path=VALIDATION_REPORT_MD,
148
- report=validation_report,
149
- )
150
-
151
- project_stats = consolidator.project_dirty_nodes(
152
- dirty_nodes=dirty_nodes,
153
- deleted_nodes=deleted_nodes,
154
- all_nodes=wiki_nodes,
155
- cached_nodes=cached_nodes,
156
- )
157
- consolidator.persist_graph(unified_graph)
158
- consolidator.persist_manifests(unified_graph)
159
-
160
- elapsed_ms = int((datetime.now(timezone.utc) - start).total_seconds() * 1000)
161
- iteration_event: Dict[str, Any] = {
162
- "timestamp": datetime.now(timezone.utc).isoformat(),
163
- "source_type": "technical-docs",
164
- "module": "autodocs",
165
- "total_nodes": len(wiki_nodes),
166
- "dirty_nodes": len(dirty_nodes),
167
- "deleted_nodes": len(deleted_nodes),
168
- "rendered_markdown": int(project_stats.get("rendered", 0)),
169
- "deleted_markdown": int(project_stats.get("deleted", 0)),
170
- "rendered_indexes": int(project_stats.get("indexes", 0)),
171
- "rendered_manifests": int(project_stats.get("manifests", 0)),
172
- "provider_errors": len(provider_errors),
173
- "validation_errors": int(validation_report.get("summary", {}).get("error_count", 0)),
174
- "validation_warnings": int(validation_report.get("summary", {}).get("warning_count", 0)),
175
- "elapsed_ms": elapsed_ms,
176
- }
177
- _append_jsonl(ITERATION_LOG, iteration_event)
178
- _write_routing_event(
179
- env_snapshot=env_snapshot,
180
- dirty_count=len(dirty_nodes),
181
- total_nodes=len(wiki_nodes),
182
- errors=provider_errors,
183
- elapsed_ms=elapsed_ms,
184
- )
116
+ collector = build_telemetry_collector(REPO_ROOT)
117
+
118
+ with collector.start_execution(operation="autodocs-compiler", session_id="autodocs"):
119
+ collector.record_event("ExecutionStarted", {"pipeline": "autodocs", "env_keys": sorted(env_snapshot.keys())})
120
+
121
+ manager = PluginManager(telemetry_collector=collector)
122
+ include_external = str(os.getenv("AUTODOCS_INCLUDE_EXTERNAL_PROVIDERS", "")).strip().lower() in {
123
+ "1",
124
+ "true",
125
+ "yes",
126
+ "on",
127
+ }
128
+
129
+ # Keep CI output deterministic: external providers depend on local intake artifacts that
130
+ # may not exist on clean runners.
131
+ if include_external:
132
+ manager.register_provider(CodeGraphProvider(REPO_ROOT / "repo-intake" / "generated"))
133
+ manager.register_provider(GraphifyProvider(REPO_ROOT / "repo-intake" / "generated"))
134
+
135
+ manager.register_provider(RepoContentProvider(REPO_ROOT))
136
+
137
+ with collector.start_span(name="autodocs.providers", kind="INTERNAL"):
138
+ provider_result = manager.gather_all()
139
+ contracts = provider_result.get("contracts", [])
140
+ provider_errors = provider_result.get("errors", [])
141
+ collector.record_metric("provider_calls", float(len(contracts) + len(provider_errors)), unit="count")
142
+
143
+ consolidator = GraphConsolidator(
144
+ output_graph_path=UNIFIED_GRAPH_PATH,
145
+ markdown_output_dir=MARKDOWN_OUTPUT_DIR,
146
+ telemetry_collector=collector,
147
+ )
148
+ with collector.start_span(name="autodocs.consolidate", kind="INTERNAL"):
149
+ unified_graph = consolidator.consolidate(contracts)
150
+ wiki_nodes = unified_graph.get("wiki_nodes", {})
151
+
152
+ incremental_engine = IncrementalEngine(UNIFIED_GRAPH_PATH, telemetry_collector=collector)
153
+ with collector.start_span(name="autodocs.incremental", kind="INTERNAL"):
154
+ diff = incremental_engine.diff(wiki_nodes)
155
+ dirty_nodes = diff.get("dirty_nodes", {})
156
+ deleted_nodes = diff.get("deleted_nodes", [])
157
+ cached_nodes = diff.get("cached_graph", {}).get("nodes", {})
158
+
159
+ validator = WikiValidator(SCHEMA_PATH, telemetry_collector=collector)
160
+ with collector.start_span(name="autodocs.validate", kind="INTERNAL"):
161
+ validation_report = validator.validate(wiki_nodes)
162
+ validator.write_reports(
163
+ json_path=VALIDATION_REPORT_JSON,
164
+ markdown_path=VALIDATION_REPORT_MD,
165
+ report=validation_report,
166
+ )
167
+
168
+ with collector.start_span(name="autodocs.project", kind="INTERNAL"):
169
+ project_stats = consolidator.project_dirty_nodes(
170
+ dirty_nodes=dirty_nodes,
171
+ deleted_nodes=deleted_nodes,
172
+ all_nodes=wiki_nodes,
173
+ cached_nodes=cached_nodes,
174
+ )
175
+ consolidator.persist_graph(unified_graph)
176
+ consolidator.persist_manifests(unified_graph)
177
+
178
+ elapsed_ms = int((datetime.now(timezone.utc) - start).total_seconds() * 1000)
179
+ iteration_event: Dict[str, Any] = {
180
+ "timestamp": datetime.now(timezone.utc).isoformat(),
181
+ "source_type": "technical-docs",
182
+ "module": "autodocs",
183
+ "total_nodes": len(wiki_nodes),
184
+ "dirty_nodes": len(dirty_nodes),
185
+ "deleted_nodes": len(deleted_nodes),
186
+ "rendered_markdown": int(project_stats.get("rendered", 0)),
187
+ "deleted_markdown": int(project_stats.get("deleted", 0)),
188
+ "rendered_indexes": int(project_stats.get("indexes", 0)),
189
+ "rendered_manifests": int(project_stats.get("manifests", 0)),
190
+ "provider_errors": len(provider_errors),
191
+ "validation_errors": int(validation_report.get("summary", {}).get("error_count", 0)),
192
+ "validation_warnings": int(validation_report.get("summary", {}).get("warning_count", 0)),
193
+ "elapsed_ms": elapsed_ms,
194
+ }
195
+ _append_jsonl(ITERATION_LOG, iteration_event)
196
+ _write_routing_event(
197
+ env_snapshot=env_snapshot,
198
+ dirty_count=len(dirty_nodes),
199
+ total_nodes=len(wiki_nodes),
200
+ errors=provider_errors,
201
+ elapsed_ms=elapsed_ms,
202
+ )
203
+
204
+ collector.record_metric("execution_time_ms", float(elapsed_ms), unit="ms")
205
+ collector.record_metric("warning_count", float(len(provider_errors)), unit="count")
206
+ collector.record_event(
207
+ "KnowledgeGenerated",
208
+ {
209
+ "total_nodes": len(wiki_nodes),
210
+ "dirty_nodes": len(dirty_nodes),
211
+ "deleted_nodes": len(deleted_nodes),
212
+ "elapsed_ms": elapsed_ms,
213
+ },
214
+ )
215
+
216
+ collector.shutdown()
185
217
 
186
218
  if int(validation_report.get("summary", {}).get("error_count", 0)) > 0:
187
219
  return 1
@@ -14,6 +14,7 @@ class RepoContentProvider(BaseWikiProvider):
14
14
 
15
15
  def gather_knowledge(self) -> Dict[str, Any]:
16
16
  entities: List[Dict[str, Any]] = []
17
+ entities.extend(self._collect_core_docs())
17
18
  entities.extend(self._collect_agents())
18
19
  entities.extend(self._collect_skills())
19
20
  entities.extend(self._collect_policies())
@@ -25,6 +26,42 @@ class RepoContentProvider(BaseWikiProvider):
25
26
  self._enrich_relations(entities)
26
27
  return {"provider_id": self.provider_id, "entities": entities}
27
28
 
29
+ def _collect_core_docs(self) -> List[Dict[str, Any]]:
30
+ targets: List[Tuple[Path, str]] = [
31
+ (self.repo_root / "README.md", "Guia principal del engine y sus flujos operativos."),
32
+ (self.repo_root / "README_WIKI.md", "Guia de uso y alcance de la wiki AutoDocs del repositorio."),
33
+ (self.repo_root / "FINAL_USAGE_GUIDE.md", "Guia de uso final con pasos operativos y buenas practicas."),
34
+ (self.repo_root / "FILE_INDEX.md", "Indice del contenido y estructura del repositorio."),
35
+ (self.repo_root / "autodocs" / "README.md", "Referencia de ejecucion y artefactos de AutoDocs."),
36
+ ]
37
+
38
+ entities: List[Dict[str, Any]] = []
39
+ for path, fallback_summary in targets:
40
+ if not path.exists():
41
+ continue
42
+ entities.append(
43
+ self._entity_from_markdown(
44
+ path,
45
+ kind="report",
46
+ section="reports",
47
+ domain="documentation",
48
+ fallback_summary=fallback_summary,
49
+ )
50
+ )
51
+
52
+ package_json = self.repo_root / "package.json"
53
+ if package_json.exists():
54
+ entities.append(
55
+ self._entity_from_json(
56
+ package_json,
57
+ kind="report",
58
+ section="reports",
59
+ domain="packaging",
60
+ )
61
+ )
62
+
63
+ return entities
64
+
28
65
  def _collect_agents(self) -> List[Dict[str, Any]]:
29
66
  agents_dir = self.repo_root / ".github" / "agents"
30
67
  if not agents_dir.exists():
@@ -292,6 +329,18 @@ class RepoContentProvider(BaseWikiProvider):
292
329
  for target_id in ["AGENTS.md", "orchestrator/router.md"]:
293
330
  if target_id in entities_by_id:
294
331
  self._add_relation(entity, target_id, "explains")
332
+ if source_ref == "README.md":
333
+ for target_id in ["AGENTS.md", "ARCHITECTURE.md", "FINAL_USAGE_GUIDE.md", "package.json"]:
334
+ if target_id in entities_by_id:
335
+ self._add_relation(entity, target_id, "documents")
336
+ if source_ref == "FINAL_USAGE_GUIDE.md":
337
+ for target_id in ["README.md", "scripts/README.md"]:
338
+ if target_id in entities_by_id:
339
+ self._add_relation(entity, target_id, "extends")
340
+ if source_ref == "autodocs/README.md":
341
+ for target_id in ["README_WIKI.md", "scripts/wiki/wiki_compiler.py"]:
342
+ if target_id in entities_by_id:
343
+ self._add_relation(entity, target_id, "documents")
295
344
  if source_ref == "autodocs/analysis_mcpee/OPENWIKI_INTERNAL_BLUEPRINT.md":
296
345
  for target_id in ["autodocs/README.md", "README_WIKI.md"]:
297
346
  if target_id in entities_by_id:
@@ -364,6 +413,8 @@ class RepoContentProvider(BaseWikiProvider):
364
413
  stem = path.name
365
414
  if path.name.lower() == "skill.md" and path.parent.name:
366
415
  stem = path.parent.name
416
+ elif path.stem.lower() == "readme" and path.parent != self.repo_root:
417
+ stem = f"{path.parent.name}-{path.name}"
367
418
  tokens = [kind, stem]
368
419
  if kind == "report" and "analysis_mcpee" in relative:
369
420
  tokens.insert(1, "autodocs")